From 7bcc46246bb977aa206f23860b6aefc0d0101e23 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:31:56 +0000 Subject: [PATCH 001/189] feat(cost-map): rolling registry update (claude-mythos-5-2, Azure Mythos 5.2, Friendli GLM-5.1 date, GovCloud Fable 5.2 token counting) (#50777) * fix(cost-map): drop unsupported Mantle token_count_model from the Bedrock GovCloud Claude Fable 5.2 row * feat(cost-map): add claude-mythos-5-2, absorb Azure Mythos 5.2 rows and Friendli GLM-5.1 deprecation Co-authored-by: Lee-Si-Yoon <76679207+Lee-Si-Yoon@users.noreply.github.com> Co-authored-by: Emerson Gomes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Lee-Si-Yoon <76679207+Lee-Si-Yoon@users.noreply.github.com> Co-authored-by: Emerson Gomes --- ...odel_prices_and_context_window_backup.json | 163 +++++++++++++++--- model_prices_and_context_window.json | 163 +++++++++++++++--- ...azure_ai_foundry_catalog_model_metadata.py | 5 + 3 files changed, 283 insertions(+), 48 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 37a5c5a2437..1bf985cd134 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29393,7 +29393,8 @@ "supports_video_input": false, "mode": "chat", "comment": "Strong GLM coding model for agentic engineering, terminals, and repository generation", - "source": "https://api.friendli.ai/serverless/v1/models" + "source": "https://api.friendli.ai/serverless/v1/models", + "deprecation_date": "2026-11-20" }, "ft:babbage-002": { "deprecation_date": "2026-09-28", @@ -65004,6 +65005,53 @@ "supports_native_structured_output": true, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-mythos-5-2": { + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 1e-05, + "cache_creation_input_token_cost_above_1hr": 1.6e-05, + "cache_creation_input_token_cost_batches": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "input_cost_per_token": 8e-06, + "input_cost_per_token_batches": 4e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-05, + "output_cost_per_token_batches": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "fast": 2, + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "supports_native_structured_output": true, + "source": "https://platform.claude.com/docs/en/about-claude/pricing", + "supports_fast_mode": true, + "supports_speed": true + }, "claude-mythos-preview": { "deprecation_date": "2026-06-09", "deprecation_source": "https://platform.claude.com/docs/en/about-claude/model-deprecations", @@ -70011,6 +70059,7 @@ } }, "bedrock/us-gov-west-1/anthropic.claude-fable-5-2": { + "bedrock_output_config_effort_ceiling": "xhigh", "cache_creation_input_token_cost": 1.2e-05, "cache_creation_input_token_cost_above_1hr": 1.92e-05, "cache_read_input_token_cost": 2.4e-07, @@ -70020,13 +70069,19 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.8e-05, + "prompt_cache_min_tokens": 512.0, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, - "thinking_always_on": true, - "supports_mid_conversation_system": true, "supports_assistant_prefill": false, + "supports_bedrock_native_structured_output": false, "supports_computer_use": true, "supports_forced_tool_use": false, "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -70035,18 +70090,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, - "supports_bedrock_native_structured_output": false, - "supports_max_reasoning_effort": true, - "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512, - "token_count_model": { - "model": "anthropic.claude-fable-5-2", - "litellm_provider": "bedrock_mantle", - "region": "us-gov-west-1" - } + "thinking_always_on": true }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -70357,6 +70401,7 @@ "prompt_cache_min_tokens": 512 }, "bedrock/us-gov-east-1/anthropic.claude-fable-5-2": { + "bedrock_output_config_effort_ceiling": "xhigh", "cache_creation_input_token_cost": 1.2e-05, "cache_creation_input_token_cost_above_1hr": 1.92e-05, "cache_read_input_token_cost": 2.4e-07, @@ -70366,13 +70411,19 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.8e-05, + "prompt_cache_min_tokens": 512.0, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, - "thinking_always_on": true, - "supports_mid_conversation_system": true, "supports_assistant_prefill": false, + "supports_bedrock_native_structured_output": false, "supports_computer_use": true, "supports_forced_tool_use": false, "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -70381,13 +70432,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, - "supports_bedrock_native_structured_output": false, - "supports_max_reasoning_effort": true, - "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "thinking_always_on": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": { "litellm_provider": "bedrock_mantle", @@ -87967,6 +88012,76 @@ "supports_xhigh_reasoning_effort": true, "thinking_always_on": true }, + "azure_ai/claude-mythos-5-2": { + "cache_creation_input_token_cost": 1e-05, + "cache_creation_input_token_cost_above_1hr": 1.6e-05, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 8e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-05, + "prompt_cache_min_tokens": 512.0, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://platform.claude.com/docs/en/about-claude/pricing", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_max_reasoning_effort": false, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true + }, + "azure_ai/claude-mythos-5-2-fast": { + "cache_creation_input_token_cost": 2e-05, + "cache_creation_input_token_cost_above_1hr": 3.2e-05, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_token": 1.6e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 8e-05, + "prompt_cache_min_tokens": 512.0, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://platform.claude.com/docs/en/about-claude/pricing", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_max_reasoning_effort": false, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true + }, "azure_ai/claude-fable-5-2": { "cache_creation_input_token_cost": 1e-05, "cache_creation_input_token_cost_above_1hr": 1.6e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 37a5c5a2437..1bf985cd134 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29393,7 +29393,8 @@ "supports_video_input": false, "mode": "chat", "comment": "Strong GLM coding model for agentic engineering, terminals, and repository generation", - "source": "https://api.friendli.ai/serverless/v1/models" + "source": "https://api.friendli.ai/serverless/v1/models", + "deprecation_date": "2026-11-20" }, "ft:babbage-002": { "deprecation_date": "2026-09-28", @@ -65004,6 +65005,53 @@ "supports_native_structured_output": true, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-mythos-5-2": { + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 1e-05, + "cache_creation_input_token_cost_above_1hr": 1.6e-05, + "cache_creation_input_token_cost_batches": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "input_cost_per_token": 8e-06, + "input_cost_per_token_batches": 4e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-05, + "output_cost_per_token_batches": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "fast": 2, + "us": 1.1 + }, + "supports_output_config": true, + "prompt_cache_min_tokens": 512, + "supports_native_structured_output": true, + "source": "https://platform.claude.com/docs/en/about-claude/pricing", + "supports_fast_mode": true, + "supports_speed": true + }, "claude-mythos-preview": { "deprecation_date": "2026-06-09", "deprecation_source": "https://platform.claude.com/docs/en/about-claude/model-deprecations", @@ -70011,6 +70059,7 @@ } }, "bedrock/us-gov-west-1/anthropic.claude-fable-5-2": { + "bedrock_output_config_effort_ceiling": "xhigh", "cache_creation_input_token_cost": 1.2e-05, "cache_creation_input_token_cost_above_1hr": 1.92e-05, "cache_read_input_token_cost": 2.4e-07, @@ -70020,13 +70069,19 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.8e-05, + "prompt_cache_min_tokens": 512.0, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, - "thinking_always_on": true, - "supports_mid_conversation_system": true, "supports_assistant_prefill": false, + "supports_bedrock_native_structured_output": false, "supports_computer_use": true, "supports_forced_tool_use": false, "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -70035,18 +70090,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, - "supports_bedrock_native_structured_output": false, - "supports_max_reasoning_effort": true, - "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512, - "token_count_model": { - "model": "anthropic.claude-fable-5-2", - "litellm_provider": "bedrock_mantle", - "region": "us-gov-west-1" - } + "thinking_always_on": true }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -70357,6 +70401,7 @@ "prompt_cache_min_tokens": 512 }, "bedrock/us-gov-east-1/anthropic.claude-fable-5-2": { + "bedrock_output_config_effort_ceiling": "xhigh", "cache_creation_input_token_cost": 1.2e-05, "cache_creation_input_token_cost_above_1hr": 1.92e-05, "cache_read_input_token_cost": 2.4e-07, @@ -70366,13 +70411,19 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.8e-05, + "prompt_cache_min_tokens": 512.0, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, - "thinking_always_on": true, - "supports_mid_conversation_system": true, "supports_assistant_prefill": false, + "supports_bedrock_native_structured_output": false, "supports_computer_use": true, "supports_forced_tool_use": false, "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -70381,13 +70432,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, - "supports_bedrock_native_structured_output": false, - "supports_max_reasoning_effort": true, - "supports_output_config": true, - "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "thinking_always_on": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": { "litellm_provider": "bedrock_mantle", @@ -87967,6 +88012,76 @@ "supports_xhigh_reasoning_effort": true, "thinking_always_on": true }, + "azure_ai/claude-mythos-5-2": { + "cache_creation_input_token_cost": 1e-05, + "cache_creation_input_token_cost_above_1hr": 1.6e-05, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 8e-06, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-05, + "prompt_cache_min_tokens": 512.0, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://platform.claude.com/docs/en/about-claude/pricing", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_max_reasoning_effort": false, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true + }, + "azure_ai/claude-mythos-5-2-fast": { + "cache_creation_input_token_cost": 2e-05, + "cache_creation_input_token_cost_above_1hr": 3.2e-05, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_token": 1.6e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 8e-05, + "prompt_cache_min_tokens": 512.0, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://platform.claude.com/docs/en/about-claude/pricing", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_max_reasoning_effort": false, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true + }, "azure_ai/claude-fable-5-2": { "cache_creation_input_token_cost": 1e-05, "cache_creation_input_token_cost_above_1hr": 1.6e-05, diff --git a/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index a5b66efe7cd..404b3240399 100644 --- a/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -23,6 +23,11 @@ TOKEN_PRICED_NAMES: Final = ( "grok-4-20-reasoning", "grok-4-20-non-reasoning", "claude-mythos-5-1", + "claude-mythos-5-2", + "claude-mythos-5-2-fast", + "claude-fable-5-2", + "claude-fable-5-2-fast", + "claude-opus-5-5-fast", ) GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning") CATALOG_NAMES: Final = TOKEN_PRICED_NAMES + ("whisper",) From 1c68ec429aea522dde41bd9f6485ded369ad3fd8 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:33:25 -0800 Subject: [PATCH 002/189] fix(proxy): bill an OpenAI pass-through background response once across repeated reads (#50841) * fix(proxy): bill an OpenAI pass-through background response once across repeated reads A finished background response now bills only on the first pass-through call that sees it finished with usage. The claim is an insert on the new LiteLLM_BackgroundResponseSettlement table, keyed by response id, so it holds across workers and replicas. Every later retrieve, cancel, streamed create, or resumed stream logs like an unbilled read. With no database or a failed insert the call still bills. * fix(proxy): keep pass-through background billing fail-open without prisma and skip foreground stream re-parse * fix(proxy): claim background responses on failed streams and bound the claim A resumed stream that fails after response.completed billed its usage through the partial-usage path without claiming the response id, so a later read billed it again. That path now claims the id and logs $0 when an earlier read paid. The claim uses create_many(skip_duplicates=True) so a repeat read no longer raises a unique violation in Postgres, and it waits at most 2 s before billing, since the claim now runs before the client gets its response. * fix(pass_through): keep the background response claim and its spend write together under cancellation and managed ids * test(pass_through): cover a failed stream whose failure logging cannot be scheduled * fix(proxy): start the background response settlement migration with a lock timeout * fix(passthrough): leave a failed background stream's charge to its next read A stream that failed after response.completed claimed the response id while its failure row billed $0, so no later read paid. The failure path is back to main's behavior, so the next finished read claims and bills once --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../migration.sql | 9 + .../litellm_proxy_extras/schema.prisma | 5 + litellm/constants.py | 1 + litellm/integrations/otel/model/spans.py | 1 + .../llms/openai/responses/transformation.py | 2 +- .../background_response_settlement.py | 52 +++ .../openai_passthrough_logging_handler.py | 54 ++- .../streaming_handler.py | 34 ++ .../pass_through_endpoints/success_handler.py | 86 +++- litellm/proxy/schema.prisma | 5 + litellm/repositories/table_repositories.py | 6 + schema.prisma | 5 + ..._passthrough_stream_pricing_paths_spend.py | 60 ++- ...test_openai_passthrough_logging_handler.py | 422 +++++++++++++++++- 14 files changed, 714 insertions(+), 28 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000000_add_background_response_settlement/migration.sql create mode 100644 litellm/proxy/pass_through_endpoints/background_response_settlement.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000000_add_background_response_settlement/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000000_add_background_response_settlement/migration.sql new file mode 100644 index 00000000000..4965ed3cfc4 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000000_add_background_response_settlement/migration.sql @@ -0,0 +1,9 @@ +SET LOCAL lock_timeout = '10s'; + +-- CreateTable +CREATE TABLE IF NOT EXISTS "LiteLLM_BackgroundResponseSettlement" ( + "response_id" TEXT NOT NULL, + "settled_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_BackgroundResponseSettlement_pkey" PRIMARY KEY ("response_id") +); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index e63580da721..37783275d9b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2055,6 +2055,11 @@ model LiteLLM_BackgroundInteractionSettlement { @@index([settled_at], map: "idx_background_interaction_settlement_settled_at") } +model LiteLLM_BackgroundResponseSettlement { + response_id String @id + settled_at DateTime @default(now()) +} + model LiteLLM_Lens { reviews LiteLLM_LensReview[] id String @id diff --git a/litellm/constants.py b/litellm/constants.py index 8df92c46630..b29c22810e7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1969,6 +1969,7 @@ BATCH_LINE_ITEMS_MAX_PART_BYTES: Final = get_env_int_in_range( MANAGED_OBJECT_STALENESS_CUTOFF_DAYS: Final = max(1, get_env_int("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7)) PENDING_RESPONSE_STATUSES: Final = ("queued", "in_progress") TERMINAL_RESPONSE_STATUSES: Final = frozenset({"completed", "failed", "cancelled", "incomplete"}) +BACKGROUND_RESPONSE_SETTLEMENT_TIMEOUT_SECONDS: Final = 2.0 BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS: Final = 10.0 BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS: Final = 60 STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, get_env_int("STALE_OBJECT_CLEANUP_BATCH_SIZE", 1000)) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 0f98a8fce0e..8f9e2c8c6db 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -292,6 +292,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_SpendLogs", "LiteLLM_BudgetWindowSpend", "LiteLLM_BackgroundInteractionSettlement", + "LiteLLM_BackgroundResponseSettlement", "LiteLLM_ErrorLogs", "LiteLLM_UserNotifications", "LiteLLM_UserProviderCredentials", diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 314a1d22508..199f9525484 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1136,7 +1136,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return None @staticmethod - def parse_terminal_response_from_stream_chunks(all_chunks: list[str]) -> ResponsesAPIResponse | None: + def parse_terminal_response_from_stream_chunks(all_chunks: Sequence[str]) -> ResponsesAPIResponse | None: terminal_event: Final = OpenAIResponsesAPIConfig.parse_terminal_event_from_stream_chunks(all_chunks) return None if terminal_event is None else terminal_event.response diff --git a/litellm/proxy/pass_through_endpoints/background_response_settlement.py b/litellm/proxy/pass_through_endpoints/background_response_settlement.py new file mode 100644 index 00000000000..c5497e51fa4 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/background_response_settlement.py @@ -0,0 +1,52 @@ +import asyncio +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from typing import Final, Protocol + +from typing_extensions import ReadOnly, TypedDict + +from litellm._logging import verbose_proxy_logger +from litellm.constants import BACKGROUND_RESPONSE_SETTLEMENT_TIMEOUT_SECONDS +from litellm.repositories.table_repositories import BackgroundResponseSettlementRepository + + +class _NewSettlementRow(TypedDict): + response_id: ReadOnly[str] + + +class BackgroundResponseSettlementTable(Protocol): + def create_many(self, data: Sequence[_NewSettlementRow], *, skip_duplicates: bool) -> Awaitable[int]: ... + + +class BackgroundResponseSettlementStore(Protocol): + async def claim(self, response_id: str) -> bool: ... + + +def _proxy_settlement_table() -> BackgroundResponseSettlementTable | None: + from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # runtime global, set after load + + if prisma_client is None: + return None + return BackgroundResponseSettlementRepository(prisma_client).table + + +@dataclass(frozen=True, slots=True) +class PrismaBackgroundResponseSettlementStore: + table: Callable[[], BackgroundResponseSettlementTable | None] = _proxy_settlement_table + timeout_seconds: float = BACKGROUND_RESPONSE_SETTLEMENT_TIMEOUT_SECONDS + + async def claim(self, response_id: str) -> bool: + try: + table: Final = self.table() + if table is None: + return True + inserted: Final = await asyncio.wait_for( + table.create_many([_NewSettlementRow(response_id=response_id)], skip_duplicates=True), + timeout=self.timeout_seconds, + ) + except Exception as e: # noqa: BLE001 # any DB failure must bill as before rather than drop the charge + verbose_proxy_logger.warning( + "Could not record the settlement of background response %s, billing it anyway: %s", response_id, e + ) + return True + return inserted > 0 diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index e4e3fec855d..0b97820fb0d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -124,6 +124,10 @@ def _is_finished_background_response(response_body: Mapping[str, object]) -> boo return is_background_response(response_body) and response_body.get("usage") is not None +def _settles_background_response(terminal_response: ResponsesAPIResponse) -> bool: + return is_background_response(terminal_response) and terminal_response.usage is not None + + def _responses_sub_route(path: str) -> tuple[str, ...] | None: segments: Final = path.rstrip("/").split("/") anchor: Final = max((index for index, segment in enumerate(segments) if segment == "responses"), default=None) @@ -263,7 +267,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): @staticmethod def openai_responses_call_billing( - method: str, url_route: str, response_body: Mapping[str, object] + method: str, url_route: str, response_body: Mapping[str, object], settled_before: bool = False ) -> ResponsesCallBilling: if not OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route): return "unbilled" @@ -271,10 +275,35 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): case ("POST", ("input_tokens",)) | ("GET", (_, "input_items")) | ("DELETE", (_,)): return "unbilled" case ("GET", (_,)) | ("POST", (_, "cancel")): - return "background_settlement" if _is_finished_background_response(response_body) else "unbilled" + settles: Final = not settled_before and _is_finished_background_response(response_body) + return "background_settlement" if settles else "unbilled" case _: return "billed" + @staticmethod + def background_response_settled_by_call( + method: str, url_route: str, response_body: Mapping[str, object] + ) -> str | None: + response_id: Final = response_body.get("id") + billing: Final = OpenAIPassthroughLoggingHandler.openai_responses_call_billing(method, url_route, response_body) + return response_id if billing == "background_settlement" and isinstance(response_id, str) else None + + @staticmethod + def stream_may_settle_background_response(url_route: str, request_body: Mapping[str, object]) -> bool: + sub_route: Final = _responses_sub_route(urlparse(url_route).path) + creates_in_background: Final = sub_route == () and request_body.get("background") is True + resumes_a_response: Final = sub_route is not None and len(sub_route) == 1 + return OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and ( + creates_in_background or resumes_a_response + ) + + @staticmethod + def background_response_settled_by_stream(all_chunks: Sequence[str]) -> str | None: + terminal_response: Final = OpenAIResponsesAPIConfig.parse_terminal_response_from_stream_chunks(all_chunks) + if terminal_response is None or not _settles_background_response(terminal_response): + return None + return terminal_response.id + @staticmethod def is_openai_agents_route(url_route: str) -> bool: return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path) @@ -483,6 +512,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): end_time: datetime, cache_hit: bool, request_body: dict, + background_response_settled_before: bool = False, **kwargs, ) -> PassThroughEndpointLoggingTypedDict: """ @@ -494,7 +524,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): is_image_generation: Final = OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route) is_image_editing: Final = OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route) responses_billing: Final = OpenAIPassthroughLoggingHandler.openai_responses_call_billing( - httpx_response.request.method, url_route, response_body + httpx_response.request.method, url_route, response_body, background_response_settled_before ) is_responses: Final = responses_billing != "unbilled" is_decisions: Final = ( @@ -801,6 +831,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): start_time: datetime, all_chunks: list[str], end_time: datetime, + background_response_settled_before: bool = False, ) -> PassThroughEndpointLoggingTypedDict: """ Handle logging for collected OpenAI streaming chunks with cost tracking. @@ -811,11 +842,16 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): ) try: is_responses: Final = OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) - terminal_response: Final = ( + parsed_terminal_response: Final = ( OpenAIResponsesAPIConfig.parse_terminal_response_from_stream_chunks(all_chunks=all_chunks) if is_responses else None ) + terminal_response: Final = ( + parsed_terminal_response.model_copy(update={"usage": None}) + if background_response_settled_before and parsed_terminal_response is not None + else parsed_terminal_response + ) requested_model: Final[object] = request_body.get("model") model: Final = ( requested_model @@ -846,7 +882,9 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): custom_llm_provider: Final = litellm_logging_obj.model_call_details.get("custom_llm_provider", "openai") # Calculate cost using LiteLLM's cost calculator response_cost: Final = ( - litellm.completion_cost( + 0.0 + if background_response_settled_before + else litellm.completion_cost( completion_response=complete_response, model=model, custom_llm_provider=custom_llm_provider, @@ -862,9 +900,9 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): settles_background_response: Final = ( BackgroundResponseSettlement.SETTLES - if terminal_response is not None - and is_background_response(terminal_response) - and terminal_response.usage is not None + if not background_response_settled_before + and terminal_response is not None + and _settles_background_response(terminal_response) else None ) diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index ef8ffdf640e..345e17c435d 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -5,6 +5,7 @@ from datetime import datetime, timezone from typing import Final, Protocol import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.asyncify import asyncify @@ -42,6 +43,8 @@ from .llm_provider_handlers.vertex_passthrough_logging_handler import ( ) from .success_handler import PassThroughEndpointLogging +_REQUEST_FIELDS: Final = TypeAdapter(dict[str, object]) + class RouteStreamingLogging(Protocol): def __call__( @@ -395,6 +398,15 @@ class PassThroughStreamingHandler: **tinyfish_payload["kwargs"], ) return + background_response_settled_before: Final = ( + endpoint_type == EndpointType.OPENAI + and await PassThroughStreamingHandler._stream_background_response_settled_before( + passthrough_success_handler_obj=passthrough_success_handler_obj, + url_route=url_route, + request_body=_REQUEST_FIELDS.validate_python(request_body), + raw_bytes=raw_bytes, + ) + ) ( standard_logging_response_object, kwargs, @@ -408,6 +420,7 @@ class PassThroughStreamingHandler: raw_bytes=raw_bytes, end_time=end_time, model=model, + background_response_settled_before=background_response_settled_before, ) # Always reached from an async context (anthropic_messages, # google_genai, and proxy pass-through stream tasks). prefer_async_handlers @@ -426,6 +439,25 @@ class PassThroughStreamingHandler: _route_streaming_logging_to_handler = route_streaming_logging_to_handler + @staticmethod + async def _stream_background_response_settled_before( + passthrough_success_handler_obj: PassThroughEndpointLogging, + url_route: str, + request_body: Mapping[str, object], + raw_bytes: Sequence[bytes], + ) -> bool: + return OpenAIPassthroughLoggingHandler.stream_may_settle_background_response( + url_route, request_body + ) and await passthrough_success_handler_obj.background_response_settled_before( + await asyncify(PassThroughStreamingHandler._background_response_settled_by_stream)(raw_bytes) + ) + + @staticmethod + def _background_response_settled_by_stream(raw_bytes: Sequence[bytes]) -> str | None: + return OpenAIPassthroughLoggingHandler.background_response_settled_by_stream( + PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes) + ) + @staticmethod def _build_passthrough_logging_result( litellm_logging_obj: LiteLLMLoggingObj, @@ -437,6 +469,7 @@ class PassThroughStreamingHandler: raw_bytes: Sequence[bytes], end_time: datetime, model: str | None, + background_response_settled_before: bool = False, ) -> tuple[PassThroughEndpointLoggingResultValues, dict]: """ Synchronous, CPU-bound reconstruction of the standard logging payload @@ -510,6 +543,7 @@ class PassThroughStreamingHandler: start_time=start_time, all_chunks=all_chunks, end_time=end_time, + background_response_settled_before=background_response_settled_before, ) ) standard_logging_response_object = openai_passthrough_logging_handler_result["result"] diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 8f55906498b..4552ac98624 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -1,14 +1,16 @@ import asyncio +import functools import json -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Coroutine, Mapping, Sequence from contextlib import suppress from datetime import datetime from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, ParamSpec from urllib.parse import urlparse import anyio import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.constants import AZURE_SPEECH_CUSTOM_LLM_PROVIDER @@ -20,6 +22,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( ) from litellm.types.utils import StandardPassThroughResponseObject +from .background_response_settlement import BackgroundResponseSettlementStore, PrismaBackgroundResponseSettlementStore from .llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, ) @@ -54,9 +57,43 @@ from .llm_provider_handlers.transcribe_passthrough_logging_handler import ( from .llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) +from .managed_id_codec import decode as decode_managed_id from .upstream_usage_headers import has_upstream_reported_usage cohere_passthrough_logging_handler: Final = CoherePassthroughLoggingHandler() +_RESPONSE_BODY: Final = TypeAdapter(dict[str, object]) + + +async def complete_despite_cancellation(work: Coroutine[object, object, None]) -> None: + pending: Final = asyncio.create_task(work) + try: + await asyncio.shield(pending) + except asyncio.CancelledError: + with anyio.CancelScope(shield=True): + while not pending.done(): + with suppress(asyncio.CancelledError): + await asyncio.shield(pending) + pending.result() + raise + + +_HandlerParams: Final = ParamSpec("_HandlerParams") + + +def _finishes_despite_cancellation_when_finalizing( + handler: Callable[_HandlerParams, Coroutine[object, object, None]], +) -> Callable[_HandlerParams, Coroutine[object, object, None]]: + @functools.wraps(handler) + async def handle( + *args: _HandlerParams.args, + **kwargs: _HandlerParams.kwargs, # kwargs-ok: ParamSpec forwards the wrapped handler's exact signature + ) -> None: + if kwargs.get("finalize_before_dispatch") is True: + await complete_despite_cancellation(handler(*args, **kwargs)) + return + await handler(*args, **kwargs) + + return handle def _safe_response_text(httpx_response: httpx.Response) -> str: @@ -84,11 +121,17 @@ class PassThroughEndpointLogging: self, transcribe_handler: TranscribePassthroughLoggingHandler | None = None, log_dispatch: PassThroughLogDispatch | None = None, + background_response_settlement: BackgroundResponseSettlementStore | None = None, ): self.transcribe_passthrough_logging_handler: Final = ( transcribe_handler if transcribe_handler is not None else TranscribePassthroughLoggingHandler() ) self._injected_log_dispatch: Final = log_dispatch + self.background_response_settlement: Final[BackgroundResponseSettlementStore] = ( + background_response_settlement + if background_response_settlement is not None + else PrismaBackgroundResponseSettlementStore() + ) self.TRACKED_VERTEX_METHOD_ROUTES = ( "generateContent", "streamGenerateContent", @@ -174,16 +217,7 @@ class PassThroughEndpointLogging: ) bind_budget_reservation_to_callbacks(logging_obj.litellm_params) - pending: Final = asyncio.create_task(finalize_and_enqueue()) - try: - await asyncio.shield(pending) - except asyncio.CancelledError: - with anyio.CancelScope(shield=True): - while not pending.done(): - with suppress(asyncio.CancelledError): - await asyncio.shield(pending) - pending.result() - raise + await complete_despite_cancellation(finalize_and_enqueue()) return await logging_obj.dispatch_success_handlers( @@ -197,6 +231,13 @@ class PassThroughEndpointLogging: _handle_logging = handle_logging + async def background_response_settled_before(self, settled_response_id: str | None) -> bool: + if settled_response_id is None: + return False + managed_id: Final = decode_managed_id(settled_response_id) + upstream_response_id: Final = settled_response_id if managed_id is None else managed_id.raw_provider_id + return not await self.background_response_settlement.claim(upstream_response_id) + def normalize_llm_passthrough_logging_payload( self, httpx_response: httpx.Response, @@ -209,6 +250,7 @@ class PassThroughEndpointLogging: end_time: datetime, cache_hit: bool, custom_llm_provider: str | None = None, + background_response_settled_before: bool = False, **kwargs, ): return_dict: Final = { @@ -294,6 +336,7 @@ class PassThroughEndpointLogging: end_time=end_time, cache_hit=cache_hit, request_body=request_body, + background_response_settled_before=background_response_settled_before, **kwargs, ) standard_logging_response_object = openai_passthrough_logging_handler_result["result"] @@ -489,6 +532,7 @@ class PassThroughEndpointLogging: ) return None + @_finishes_despite_cancellation_when_finalizing async def pass_through_async_success_handler( self, httpx_response: httpx.Response, @@ -504,7 +548,7 @@ class PassThroughEndpointLogging: custom_llm_provider: str | None = None, finalize_before_dispatch: bool = False, **kwargs, - ): + ) -> None: standard_logging_response_object: PassThroughEndpointLoggingResultValues | None = None logging_obj.model_call_details["passthrough_logging_payload"] = passthrough_logging_payload if self.is_tinyfish_route(url_route, custom_llm_provider): @@ -558,7 +602,22 @@ class PassThroughEndpointLogging: ) return else: + from .llm_provider_handlers.openai_passthrough_logging_handler import ( + OpenAIPassthroughLoggingHandler, + ) + try: + settled_response_id: Final = ( + OpenAIPassthroughLoggingHandler.background_response_settled_by_call( + httpx_response.request.method, url_route, _RESPONSE_BODY.validate_python(response_body) + ) + if isinstance(response_body, dict) + and OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) + else None + ) + background_response_settled_before: Final = await self.background_response_settled_before( + settled_response_id + ) normalized_llm_passthrough_logging_payload: Final = self.normalize_llm_passthrough_logging_payload( httpx_response=httpx_response, response_body=response_body, @@ -570,6 +629,7 @@ class PassThroughEndpointLogging: end_time=end_time, cache_hit=cache_hit, custom_llm_provider=custom_llm_provider, + background_response_settled_before=background_response_settled_before, **kwargs, ) standard_logging_response_object = normalized_llm_passthrough_logging_payload[ diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index e63580da721..37783275d9b 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2055,6 +2055,11 @@ model LiteLLM_BackgroundInteractionSettlement { @@index([settled_at], map: "idx_background_interaction_settlement_settled_at") } +model LiteLLM_BackgroundResponseSettlement { + response_id String @id + settled_at DateTime @default(now()) +} + model LiteLLM_Lens { reviews LiteLLM_LensReview[] id String @id diff --git a/litellm/repositories/table_repositories.py b/litellm/repositories/table_repositories.py index fbf9e5d07f0..fbef73a6114 100644 --- a/litellm/repositories/table_repositories.py +++ b/litellm/repositories/table_repositories.py @@ -408,6 +408,12 @@ class BackgroundInteractionSettlementRepository( table_name = "litellm_backgroundinteractionsettlement" +class BackgroundResponseSettlementRepository( + PrismaTableRepository["prisma_models.LiteLLM_BackgroundResponseSettlement"] +): + table_name = "litellm_backgroundresponsesettlement" + + class RetiredAgentRepository(PrismaTableRepository["prisma_models.LiteLLM_RetiredAgent"]): table_name = "litellm_retiredagent" diff --git a/schema.prisma b/schema.prisma index e63580da721..37783275d9b 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2055,6 +2055,11 @@ model LiteLLM_BackgroundInteractionSettlement { @@index([settled_at], map: "idx_background_interaction_settlement_settled_at") } +model LiteLLM_BackgroundResponseSettlement { + response_id String @id + settled_at DateTime @default(now()) +} + model LiteLLM_Lens { reviews LiteLLM_LensReview[] id String @id diff --git a/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py b/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py index 0ead544a2ee..508ceab38f9 100644 --- a/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py +++ b/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py @@ -11,7 +11,7 @@ from urllib.parse import urlsplit import httpx import pytest import yaml -from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.client import Gateway, eventually, gateway_from_environment, held from integration._support.forward_proxy import tunnelling_forward_proxy from integration._support.process import graceful_stop_seconds, owned_proxy from integration._support.tls import server_context, write_self_signed_cert @@ -50,6 +50,7 @@ _GEMINI_USAGE: Final = { _DEPLOYMENT_LOCATIONS: Final = MappingProxyType({"audit-vertex-global": "global", "audit-vertex-us": "us"}) _GEMINI_ACTIONS: Final = ("generateContent", "streamGenerateContent?alt=sse") _CONFIGURED_GATEWAY_PATH: Final = "/google-egress" +_SPEND_HOLD_SECONDS: Final = 10 @dataclass(frozen=True, slots=True) @@ -157,9 +158,40 @@ def _openai_reply(request: Request) -> Reply: return Reply(chunks=(*encoded, b"data: [DONE]\n\n"), content_type="text/event-stream") +def _finished_background_response(request: Request) -> Reply: + return Reply( + body=json.dumps( + { + "id": urlsplit(request.target).path.rsplit("/", 1)[-1], + "object": "response", + "created_at": 1, + "status": "completed", + "background": True, + "model": _OPENAI_MODEL, + "output": [ + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "scripted", "annotations": []}], + } + ], + "usage": { + "input_tokens": _PROMPT_TOKENS, + "output_tokens": _COMPLETION_TOKENS, + "total_tokens": _PROMPT_TOKENS + _COMPLETION_TOKENS, + }, + } + ).encode() + ) + + def _upstream(request: Request) -> Reply: if request.target.startswith("/v1/chat/completions"): return _openai_reply(request) + if request.target.startswith("/v1/responses/"): + return _finished_background_response(request) if "/publishers/anthropic/models/" in request.target: assert f"/v1/projects/{_PROJECT}/locations/" in request.target, request.target return _claude_reply(request) @@ -300,6 +332,15 @@ def _spend_row(gateway: Gateway, key: str) -> dict[str, JsonValue]: return rows[0] +def _key_spend(gateway: Gateway, key: str) -> float: + rows: Final = gateway.rows( + 'SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (sha256(key.encode()).hexdigest(),) + ) + spend: Final = rows[0]["spend"] + assert isinstance(spend, float), rows + return spend + + def _spend_for_key(gateway: Gateway, key: str) -> float: spend: Final = _spend_row(gateway, key)["spend"] assert isinstance(spend, float), key @@ -448,6 +489,23 @@ def test_openai_passthrough_stream_cut_off_after_its_usage_bills_what_the_finish assert spend == pytest.approx({"finished": list_price, "cut off": list_price}) +def test_an_openai_passthrough_background_response_read_three_times_bills_one_read(rig: _Rig) -> None: + response_path: Final = f"/openai_passthrough/v1/responses/resp_{uuid.uuid4().hex}" + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + statuses: Final = [ + rig.proxy.client.get(response_path, headers={"Authorization": f"Bearer {key}"}).status_code + for _ in range(3) + ] + assert statuses == [200, 200, 200] + held( + lambda: _key_spend(rig.proxy, key), + lambda spend: spend == pytest.approx(_list_price(_OPENAI_MODEL)), + holding=_SPEND_HOLD_SECONDS, + seconds=70, + ) + + def _caller_headers(route: str, key: str) -> Mapping[str, str]: if route == "vertex_ai": return {"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"} diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index bb4b41fd09d..2ed75548702 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -2,17 +2,20 @@ from dataclasses import dataclass import asyncio import json import logging -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime +from types import SimpleNamespace from typing import Any, Dict, Final, List from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from prisma.errors import UniqueViolationError from pydantic import BaseModel, ConfigDict import litellm +from litellm.proxy import proxy_server from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound @@ -21,10 +24,15 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrou count_relayed_prompt_tokens, ) from litellm.proxy._types import SpendLogsPayload +from litellm.proxy.pass_through_endpoints.background_response_settlement import ( + PrismaBackgroundResponseSettlementStore, +) +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.proxy.pass_through_endpoints.managed_id_codec import new_managed_id +from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamContext, PassThroughStreamingHandler from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) -from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, @@ -2174,6 +2182,60 @@ class _SpendRowRecorder(CustomLogger): ) +class _SettlementTable: + def __init__(self) -> None: + self.settled_response_ids: Final[set[str]] = set() + + async def create_many(self, data: Sequence[Mapping[str, str]], *, skip_duplicates: bool) -> int: + new_ids: Final = {row["response_id"] for row in data} - self.settled_response_ids + if not skip_duplicates and len(new_ids) < len(data): + raise UniqueViolationError({"user_facing_error": {"error_code": "P2002"}}) + self.settled_response_ids.update(new_ids) + return len(new_ids) + + +class _UnreachableSettlementTable: + async def create_many(self, data: Sequence[Mapping[str, str]], *, skip_duplicates: bool) -> int: + raise ConnectionResetError("server closed the connection unexpectedly") + + +@dataclass(frozen=True, slots=True) +class _StalledSettlementTable: + table: _SettlementTable + + async def create_many(self, data: Sequence[Mapping[str, str]], *, skip_duplicates: bool) -> int: + await asyncio.sleep(3) + return await self.table.create_many(data, skip_duplicates=skip_duplicates) + + +@dataclass(frozen=True, slots=True) +class _SlowAckSettlementTable: + table: _SettlementTable + inserted: asyncio.Event + + async def create_many(self, data: Sequence[Mapping[str, str]], *, skip_duplicates: bool) -> int: + inserted_count: Final = await self.table.create_many(data, skip_duplicates=skip_duplicates) + self.inserted.set() + await asyncio.sleep(0.3) + return inserted_count + + +def _settling_passthrough_logging( + table: _SettlementTable | _UnreachableSettlementTable | _StalledSettlementTable | _SlowAckSettlementTable | None, + timeout_seconds: float = 0.05, +) -> PassThroughEndpointLogging: + return PassThroughEndpointLogging( + background_response_settlement=PrismaBackgroundResponseSettlementStore( + table=lambda: table, timeout_seconds=timeout_seconds + ) + ) + + +def _completed_stream_bytes(response: Mapping[str, object]) -> list[bytes]: + completed: Final = {"type": "response.completed", "sequence_number": 9, "response": response} + return [f"event: response.completed\ndata: {json.dumps(completed)}\n\n".encode()] + + class TestOpenAIPassthroughResponsesSubRouteBilling: def _dispatch( self, @@ -2362,12 +2424,31 @@ class TestOpenAIPassthroughResponsesSubRouteBilling: response_body: dict[str, object], litellm_call_id: str, base_url: str = RESPONSES_URL, + *, + passthrough_logging: PassThroughEndpointLogging | None = None, ) -> _LoggedCall: - url_route: Final = f"{base_url}{sub_route}" recorder: Final = _SpendRowRecorder() - logging_obj: Final = self._recorded_logging_obj(recorder, litellm_call_id, stream=False) + await self._handled_call( + method, + f"{base_url}{sub_route}", + request_body, + response_body, + self._recorded_logging_obj(recorder, litellm_call_id, stream=False), + passthrough_logging or PassThroughEndpointLogging(), + ) + return await asyncio.wait_for(recorder.call, timeout=5) - await PassThroughEndpointLogging().pass_through_async_success_handler( + @staticmethod + async def _handled_call( + method: str, + url_route: str, + request_body: dict[str, object], + response_body: dict[str, object], + logging_obj: LiteLLMLoggingObj, + passthrough_logging: PassThroughEndpointLogging, + finalize_before_dispatch: bool = False, + ) -> None: + await passthrough_logging.pass_through_async_success_handler( httpx_response=httpx.Response(200, json=response_body, request=httpx.Request(method, url_route)), response_body=response_body, logging_obj=logging_obj, @@ -2381,9 +2462,340 @@ class TestOpenAIPassthroughResponsesSubRouteBilling: url=url_route, request_body=request_body, request_method=method ), custom_llm_provider="openai", + finalize_before_dispatch=finalize_before_dispatch, + ) + + async def _streamed_call( + self, + sub_route: str, + request_body: dict[str, object], + raw_bytes: list[bytes], + litellm_call_id: str, + passthrough_logging: PassThroughEndpointLogging, + ) -> _LoggedCall: + recorder: Final = _SpendRowRecorder() + logging_obj: Final = self._recorded_logging_obj(recorder, litellm_call_id, stream=True) + logging_obj.model_call_details["custom_llm_provider"] = "openai" + + await PassThroughStreamingHandler.route_streaming_logging_to_handler( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=passthrough_logging, + url_route=f"{RESPONSES_URL}{sub_route}", + request_body=request_body, + endpoint_type=EndpointType.OPENAI, + start_time=DISPATCH_TIME, + raw_bytes=raw_bytes, + end_time=DISPATCH_TIME, ) return await asyncio.wait_for(recorder.call, timeout=5) + def _failed_stream_logging_obj(self, litellm_call_id: str) -> LiteLLMLoggingObj: + logging_obj: Final = self._recorded_logging_obj(_SpendRowRecorder(), litellm_call_id, stream=True) + logging_obj.model_call_details["custom_llm_provider"] = "openai" + return logging_obj + + @staticmethod + async def _failed_stream( + logging_obj: LiteLLMLoggingObj, + sub_route: str, + raw_bytes: list[bytes], + passthrough_logging: PassThroughEndpointLogging, + ) -> None: + await PassThroughStreamingHandler.schedule_stream_failure_logging( + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.OPENAI, + request_body={}, + raw_bytes=raw_bytes, + exception=httpx.RemoteProtocolError("peer closed connection without sending complete message body"), + stream_context=PassThroughStreamContext(passthrough_logging, f"{RESPONSES_URL}{sub_route}", DISPATCH_TIME), + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("method", "sub_route", "response_body"), + [ + pytest.param( + "GET", + f"/{STORED_RESPONSE_ID}", + _stored_response("completed", True, RESPONSES_USAGE), + id="retrieve_then_retrieve", + ), + pytest.param( + "POST", + f"/{STORED_RESPONSE_ID}/cancel", + _stored_response("cancelled", True, RESPONSES_USAGE), + id="retrieve_then_cancel", + ), + ], + ) + async def test_a_background_response_read_twice_bills_only_the_first_read( + self, local_model_cost_map, method: str, sub_route: str, response_body: dict[str, object] + ): + passthrough_logging: Final = _settling_passthrough_logging(_SettlementTable()) + later_call_id: Final = "e4a1c7b2-3d58-4f9e-8b06-91c2d7f3a5e8" + + first: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + _stored_response("completed", True, RESPONSES_USAGE), + "2f9d6b13-8c4e-4a71-b5d0-3e7a1c9f2b64", + passthrough_logging=passthrough_logging, + ) + later: Final = await self._logged_call( + method, sub_route, {}, response_body, later_call_id, passthrough_logging=passthrough_logging + ) + + assert first.spend_row["spend"] > 0 + assert first.spend_row["total_tokens"] > 0 + assert first.spend_row["request_id"].startswith(STORED_RESPONSE_ID) + assert not later.spend_row["spend"] + assert not later.spend_row["total_tokens"] + assert later.spend_row["request_id"] == later_call_id + + @pytest.mark.asyncio + async def test_the_proxy_database_keeps_a_background_response_billed_once(self, local_model_cost_map, monkeypatch): + settlement_table: Final = _SettlementTable() + monkeypatch.setattr( + proxy_server, + "prisma_client", + SimpleNamespace(db=SimpleNamespace(litellm_backgroundresponsesettlement=settlement_table)), + ) + passthrough_logging: Final = PassThroughEndpointLogging() + finished: Final = _stored_response("completed", True, RESPONSES_USAGE) + + first: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + finished, + "5a9c2e71-3f4b-4d86-a1e7-c8b6d3f05e92", + passthrough_logging=passthrough_logging, + ) + second: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + finished, + "b2d7f4a8-6e1c-4a39-8f5b-0c9e3a7d1b64", + passthrough_logging=passthrough_logging, + ) + + assert first.spend_row["spend"] > 0 + assert not second.spend_row["spend"] + assert settlement_table.settled_response_ids == {STORED_RESPONSE_ID} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "retrieved_id", + [ + pytest.param(STORED_RESPONSE_ID, id="raw_id"), + pytest.param(new_managed_id("openai", STORED_RESPONSE_ID), id="managed_id"), + ], + ) + async def test_a_streamed_background_create_bills_once_and_its_later_retrieve_does_not( + self, local_model_cost_map, retrieved_id: str + ): + passthrough_logging: Final = _settling_passthrough_logging(_SettlementTable()) + finished: Final = _stored_response("completed", True, RESPONSES_USAGE) + + streamed_create: Final = await self._streamed_call( + "", + {**CREATE_REQUEST, "background": True, "stream": True}, + _completed_stream_bytes(finished), + "8d2f4a61-7b3c-4e95-a0d8-c5e1f9b7234a", + passthrough_logging, + ) + retrieve: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + {**finished, "id": retrieved_id}, + "4b7e9c25-1a6d-4f38-9e02-d8c3a5f1b760", + passthrough_logging=passthrough_logging, + ) + + assert streamed_create.spend_row["spend"] > 0 + assert not retrieve.spend_row["spend"] + + @pytest.mark.asyncio + async def test_a_finished_read_cancelled_while_it_settles_still_bills_and_its_later_read_does_not( + self, local_model_cost_map + ): + settled: Final = asyncio.Event() + passthrough_logging: Final = _settling_passthrough_logging( + _SlowAckSettlementTable(_SettlementTable(), settled), timeout_seconds=2 + ) + finished: Final = _stored_response("completed", True, RESPONSES_USAGE) + recorder: Final = _SpendRowRecorder() + cancelled_read: Final = asyncio.create_task( + self._handled_call( + "GET", + f"{RESPONSES_URL}/{STORED_RESPONSE_ID}", + {}, + finished, + self._recorded_logging_obj(recorder, "6c3e8a14-2f7b-4d95-b1e0-a8d4c6f2e735", stream=False), + passthrough_logging, + finalize_before_dispatch=True, + ) + ) + await settled.wait() + cancelled_read.cancel() + with pytest.raises(asyncio.CancelledError): + await cancelled_read + first: Final = await asyncio.wait_for(recorder.call, timeout=5) + later: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + finished, + "d1a7f3c9-5e2b-4806-9c4d-3b8e6f0a2d17", + passthrough_logging=passthrough_logging, + ) + + assert first.spend_row["spend"] > 0 + assert not later.spend_row["spend"] + + @pytest.mark.asyncio + async def test_a_background_create_stream_cut_before_it_finishes_leaves_the_bill_to_the_finished_retrieve( + self, local_model_cost_map + ): + passthrough_logging: Final = _settling_passthrough_logging(_SettlementTable()) + created: Final = { + "type": "response.created", + "sequence_number": 0, + "response": _stored_response("in_progress", True, None), + } + + await self._streamed_call( + "", + {**CREATE_REQUEST, "background": True, "stream": True}, + [f"event: response.created\ndata: {json.dumps(created)}\n\n".encode()], + "d4b8e2f6-9a1c-4573-8e6d-1f7c3a5b9e20", + passthrough_logging, + ) + retrieve: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + _stored_response("completed", True, RESPONSES_USAGE), + "f1a6c3e8-4d2b-4b97-a5e0-7c9d2f4b6a13", + passthrough_logging=passthrough_logging, + ) + + assert retrieve.spend_row["spend"] > 0 + + @pytest.mark.asyncio + async def test_a_streamed_foreground_create_bills_without_recording_a_settlement(self, local_model_cost_map): + settlement_table: Final = _SettlementTable() + + streamed_create: Final = await self._streamed_call( + "", + {**CREATE_REQUEST, "stream": True}, + _completed_stream_bytes(_stored_response("completed", False, RESPONSES_USAGE)), + "3e8b1f57-6c2a-4d94-b7e0-9a5c4f2d816b", + _settling_passthrough_logging(settlement_table), + ) + + assert streamed_create.spend_row["spend"] > 0 + assert not settlement_table.settled_response_ids + + @pytest.mark.asyncio + async def test_a_resumed_stream_after_a_billed_retrieve_does_not_bill_again(self, local_model_cost_map): + passthrough_logging: Final = _settling_passthrough_logging(_SettlementTable()) + finished: Final = _stored_response("completed", True, RESPONSES_USAGE) + + retrieve: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + finished, + "6c3a8e14-5f2b-4d97-b1e6-a7d4c2f9e853", + passthrough_logging=passthrough_logging, + ) + resumed_stream: Final = await self._streamed_call( + f"/{STORED_RESPONSE_ID}?stream=true", + {}, + _completed_stream_bytes(finished), + "a9e5d3c7-2b8f-4146-8d0a-f3b6e1c4d295", + passthrough_logging, + ) + + assert retrieve.spend_row["spend"] > 0 + assert retrieve.spend_row["total_tokens"] > 0 + assert not resumed_stream.spend_row["spend"] + assert not resumed_stream.spend_row["total_tokens"] + assert resumed_stream.spend_row["request_id"] != retrieve.spend_row["request_id"] + + @pytest.mark.asyncio + async def test_a_resumed_stream_that_fails_after_it_finished_leaves_the_charge_to_its_next_retrieve( + self, local_model_cost_map + ): + passthrough_logging: Final = _settling_passthrough_logging(_SettlementTable()) + finished: Final = _stored_response("completed", True, RESPONSES_USAGE) + + await self._failed_stream( + self._failed_stream_logging_obj("5d9a3f72-1e8b-4c46-a7d0-b3f6e2c9a581"), + f"/{STORED_RESPONSE_ID}?stream=true", + _completed_stream_bytes(finished), + passthrough_logging, + ) + await GLOBAL_LOGGING_WORKER.flush() + first_retrieve: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + finished, + "c1e7b4a9-6f2d-4835-9b0e-a8d5f3c2e176", + passthrough_logging=passthrough_logging, + ) + later_retrieve: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + finished, + "7a3e9c15-4b8d-4f62-a0e7-d5c2b9f1e384", + passthrough_logging=passthrough_logging, + ) + + assert first_retrieve.spend_row["spend"] > 0 + assert not later_retrieve.spend_row["spend"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "table", + [ + pytest.param(None, id="no_database"), + pytest.param(_UnreachableSettlementTable(), id="database_error"), + pytest.param(_StalledSettlementTable(_SettlementTable()), id="database_stalled"), + ], + ) + async def test_finished_reads_bill_as_before_when_their_settlement_cannot_be_recorded( + self, local_model_cost_map, table: _UnreachableSettlementTable | _StalledSettlementTable | None + ): + passthrough_logging: Final = _settling_passthrough_logging(table) + finished: Final = _stored_response("completed", True, RESPONSES_USAGE) + + first: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + finished, + "1d6b4f82-9e3a-4c75-a2d8-b5f7e0c3a916", + passthrough_logging=passthrough_logging, + ) + second: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + finished, + "7f2c9a35-4e1d-4b68-9c07-e6a3d8b5f142", + passthrough_logging=passthrough_logging, + ) + + assert first.spend_row["spend"] > 0 + assert second.spend_row["spend"] == pytest.approx(first.spend_row["spend"]) + @pytest.mark.asyncio async def test_unbilled_retrieve_spend_log_keeps_the_response_model(self, local_model_cost_map): response_body: Final = _stored_response("completed", False, RESPONSES_USAGE) From a6b61a2c3b1f08c8ef712d1e5b757496c7b66db2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:06:25 +0000 Subject: [PATCH 003/189] feat(cost-map): add gemini/gemini-4-carbon from the Gemini API pricing page (#50931) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 57 +++++++++++++++++++ model_prices_and_context_window.json | 57 +++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1bf985cd134..b2428aa4c94 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -88246,5 +88246,62 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_vision": true + }, + "gemini/gemini-4-carbon": { + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, + "cache_read_input_token_cost_flex": 5e-08, + "cache_read_input_token_cost_priority": 1.8e-07, + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 3.6e-06, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_reasoning_token": 1e-05, + "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 1.8e-05, + "prompt_cache_min_tokens": 4096.0, + "rpm": 2000.0, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "source": "https://aistudio.google.com/docs/pricing#gemini-4-carbon", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_computer_use": true, + "supports_continuation": true, + "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000.0, + "web_search_billing_unit": "per_query" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1bf985cd134..b2428aa4c94 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -88246,5 +88246,62 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_vision": true + }, + "gemini/gemini-4-carbon": { + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, + "cache_read_input_token_cost_flex": 5e-08, + "cache_read_input_token_cost_priority": 1.8e-07, + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 3.6e-06, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_reasoning_token": 1e-05, + "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 1.8e-05, + "prompt_cache_min_tokens": 4096.0, + "rpm": 2000.0, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "source": "https://aistudio.google.com/docs/pricing#gemini-4-carbon", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_computer_use": true, + "supports_continuation": true, + "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000.0, + "web_search_billing_unit": "per_query" } } From f8933ecae4d610f07be2ecc3edf78537e5b1480f Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 10 Nov 2026 10:14:12 -0600 Subject: [PATCH 004/189] fix(redis): free each request's Redis batch scope once the request ends (#50751) * fix(spend): respect use_redis_transaction_buffer when selecting the SpendLogs ledger * revert: keep the SpendLogs ledger independent of use_redis_transaction_buffer Measured on a live proxy, ledger on and ledger off grow RSS at the same rate per request, so gating the ledger on the Redis buffer moved deployments off it without fixing the growth. * fix(redis): free each request's Redis batch scope once the request ends The native request batch keeps a copy of the context it is built in. It was built inside the request scope, so that context pointed back at the scope, and the cycle collector cannot see through the native object. Every request's batches, spend counter batches and native Redis state stayed alive, growing worker RSS about 40 KiB per request and slowing each request as the heap grew. Build the native batch in the context the scope was opened from instead. --- litellm/caching/redis_batch.py | 11 ++++++++--- tests/unit/caching/test_redis_batch.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/litellm/caching/redis_batch.py b/litellm/caching/redis_batch.py index b3e6d2eb291..d621d09b439 100644 --- a/litellm/caching/redis_batch.py +++ b/litellm/caching/redis_batch.py @@ -16,7 +16,7 @@ import logging import time import weakref from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence -from contextvars import ContextVar, Token +from contextvars import ContextVar, Token, copy_context from dataclasses import dataclass, field from datetime import timedelta from enum import Enum, auto @@ -528,6 +528,7 @@ class RequestRedisBatches: "_deadline", "_deadline_flush", "_native", + "_native_context", "_native_factory", "_native_flush_hooks", "_native_misses", @@ -551,6 +552,7 @@ class RequestRedisBatches: self._native_factory: Final = native_factory self._native_send_factory: Final = native_send_factory self._native: NativeRequestRedis | None = None + self._native_context: Final = copy_context() self._native_flush_hooks: Final[list[Callable[[], None]]] = [] # mutable-ok: append-only registry self._native_misses: frozenset[str] = frozenset() self._deadline: asyncio.TimerHandle | None = None @@ -580,9 +582,12 @@ class RequestRedisBatches: return batch def native(self) -> NativeRequestRedis: - """The request's native batches, built on first use so requests that touch no native store pay nothing.""" + """The request's native batches, built on first use so requests that touch no native store pay nothing. + + They keep a copy of the context they are built in, and the cycle collector cannot see through native + objects, so they are built in the context this scope was opened from, which cannot reach the scope.""" if self._native is None: - self._native = self._native_factory(self.post_call_deadline) + self._native = self._native_context.run(self._native_factory, self.post_call_deadline) return self._native def native_post_call(self) -> NativeRequestRedis: diff --git a/tests/unit/caching/test_redis_batch.py b/tests/unit/caching/test_redis_batch.py index 536d510f6d6..158b9754309 100644 --- a/tests/unit/caching/test_redis_batch.py +++ b/tests/unit/caching/test_redis_batch.py @@ -3,9 +3,11 @@ from __future__ import annotations import asyncio +import gc import hashlib import json import uuid +import weakref from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass, field from datetime import timedelta @@ -19,6 +21,7 @@ from litellm._service_logger import ServiceLogging from litellm.caching.redis_batch import ( MIXED_PIPELINE_TARGET, RedisBatch, + RequestRedisBatches, active_request_redis_batch, request_redis_batch_scope, ) @@ -418,6 +421,17 @@ def test_request_scope_hands_out_one_batch_per_backend_and_nests() -> None: assert active_request_redis_batch(cache_a) is None +def test_a_request_scope_that_built_its_native_batches_is_freed_once_it_closes() -> None: + def serve_request() -> weakref.ref[RequestRedisBatches]: + with request_redis_batch_scope() as batches: + batches.native() + return weakref.ref(batches) + + closed: Final = serve_request() + gc.collect() + assert closed() is None + + @pytest.mark.asyncio async def test_a_key_an_mget_read_as_absent_stays_known_missing_until_something_sets_it() -> None: cache, client = make() From c90b5e057f5511a94d6bd325a82ef3648165ca3f Mon Sep 17 00:00:00 2001 From: norkelbyte <348963847+norkelbyte@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:37:00 -0800 Subject: [PATCH 005/189] fix(openai): download PDF URLs off the event loop and transform async chat messages once (#49176) * fix(openai): transform async chat messages once, whatever configs were built before * fix(openai): keep tool-result PDF downloads off the event loop and stop Mistral rewriting retried messages * fix(openai): pre-download PDF URLs into the sync transform so async requests match sync Async chat requests now download every PDF URL in user and tool messages at once, then run the sync transform_request on a worker thread that reads those downloads. The request, filename, logged messages, and download count match a sync request, and a failed download raises the sync error with no second attempt. DeepSeek joins the configs that flatten content and skip the up-front download. * fix(openai): skip up-front PDF downloads past a file part the transform rejects * fix(openai): keep system-message PDF downloads off the event loop and stop downloads after a dead PDF URL * fix(openai): let PDF downloads after a dead URL finish and retry rate-limited ones one at a time Cancelling in-flight downloads after a dead PDF URL could abort a proxy worker on uvloop, so the downloads now run to the end in the background. A URL whose up-front download got a 429 downloads again inside the transform one at a time, as on main. A content part whose type is not a string is no longer read as a PDF part, so it reaches the provider instead of raising a TypeError. * fix(openai): download a request's PDF URLs one at a time and stop at the first dead one * fix(openai): prefetch the PDF URLs the sync transform downloads, after role mapping and up to the first unreadable part --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../prompt_templates/image_handling.py | 60 +- litellm/llms/deepseek/chat/transformation.py | 3 + .../chat/transformation.py | 3 + litellm/llms/heroku/chat/transformation.py | 3 + litellm/llms/mistral/chat/transformation.py | 2 - .../llms/openai/chat/gpt_transformation.py | 76 ++- .../openai/chat/o_series_transformation.py | 17 +- litellm/llms/openai/openai.py | 2 +- litellm/llms/openai_like/dynamic_config.py | 5 +- litellm/llms/sambanova/chat.py | 3 + .../chat/test_openai_gpt_transformation.py | 624 +++++++++++++++++- 11 files changed, 747 insertions(+), 51 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index fa6f1c1ed0a..a5e4cb396b7 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -5,7 +5,7 @@ Helper functions to handle images passed in messages import asyncio import base64 import time -from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from functools import partial @@ -531,6 +531,29 @@ async def async_convert_urls_to_base64( ) +async def _download_outcome(url: str, timeout: float | httpx.Timeout | None) -> "str | litellm.ImageFetchError": + try: + return await async_convert_url_to_base64(url, timeout=timeout) + except litellm.ImageFetchError as error: + return error + + +async def _downloads_through_first_failure( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None +) -> "AsyncIterator[str | litellm.ImageFetchError]": + for url in remote_urls: + yield (outcome := await _download_outcome(url, timeout)) + if isinstance(outcome, litellm.ImageFetchError): + return + + +async def async_fetch_urls_to_base64( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None +) -> "Mapping[str, str | litellm.ImageFetchError]": + outcomes: Final = tuple([outcome async for outcome in _downloads_through_first_failure(remote_urls, timeout)]) + return MappingProxyType(dict(zip(remote_urls[: len(outcomes)], outcomes, strict=True))) + + def _remote_urls_to_inline( messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool] ) -> tuple[str, ...]: @@ -539,6 +562,41 @@ def _remote_urls_to_inline( return tuple(dict.fromkeys(remote.url for remote in remotes if should_inline(_remote_media(remote)))) +def _message_parts(messages: Iterable[AllMessageValues]) -> Iterator[object]: + return chain.from_iterable(_content_parts(message) for message in messages) + + +def _file_of_file_part(part: object) -> Mapping[str, object] | None: + fields: Final = _as_mapping(part) + return _as_mapping(fields.get("file")) if fields is not None and fields.get("type") == "file" else None + + +def _file_id_url(part: object, url_starts: tuple[str, ...]) -> str | None: + file: Final = _file_of_file_part(part) + file_id: Final = file.get("file_id") if file is not None else None + return file_id if isinstance(file_id, str) and file_id.startswith(url_starts) else None + + +def _is_readable_file_part_or_other_part(part: object) -> bool: + fields: Final = _as_mapping(part) + if fields is None or fields.get("type") != "file": + return fields is not None + file: Final = _file_of_file_part(part) + file_id: Final = file.get("file_id") if file is not None else None + return file is not None and (not file_id or isinstance(file_id, str)) + + +def has_file_id_urls(messages: Iterable[AllMessageValues], url_starts: tuple[str, ...]) -> bool: + return any(_file_id_url(part, url_starts) is not None for part in _message_parts(messages)) + + +def file_id_urls_before_an_unreadable_part( + messages: Iterable[AllMessageValues], url_starts: tuple[str, ...] +) -> tuple[str, ...]: + readable_parts: Final = takewhile(_is_readable_file_part_or_other_part, _message_parts(messages)) + return tuple(dict.fromkeys(url for part in readable_parts if (url := _file_id_url(part, url_starts)) is not None)) + + def inline_remote_media( messages: list[AllMessageValues], # mutable-ok: every transform_request takes list[AllMessageValues] should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url, diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index 9c222f03c8c..872e153c91a 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -34,6 +34,9 @@ def _warn_on_dropped_budget(thinking: object, model: str) -> None: class DeepSeekChatConfig(OpenAIGPTConfig): + def _flattens_content_lists(self) -> bool: + return True + def get_supported_openai_params(self, model: str) -> list[str]: """ DeepSeek reasoner models support thinking parameter. diff --git a/litellm/llms/docker_model_runner/chat/transformation.py b/litellm/llms/docker_model_runner/chat/transformation.py index 68b63812248..bd8e335b617 100644 --- a/litellm/llms/docker_model_runner/chat/transformation.py +++ b/litellm/llms/docker_model_runner/chat/transformation.py @@ -25,6 +25,9 @@ class DockerModelRunnerChatConfig(OpenAIGPTConfig): The engine name (e.g., "llama.cpp") is part of the API endpoint path. """ + def _flattens_content_lists(self) -> bool: + return True + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] diff --git a/litellm/llms/heroku/chat/transformation.py b/litellm/llms/heroku/chat/transformation.py index 002078dd04b..e9cfb5e09cf 100644 --- a/litellm/llms/heroku/chat/transformation.py +++ b/litellm/llms/heroku/chat/transformation.py @@ -22,6 +22,9 @@ class HerokuError(Exception): class HerokuChatConfig(OpenAIGPTConfig): + def _flattens_content_lists(self) -> bool: + return True + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index d13e46658ae..1dfb98fc65a 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -142,8 +142,6 @@ class MistralConfig(OpenAIGPTConfig): - `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message. """ - _is_base_class = False - temperature: int | None = None top_p: int | None = None max_tokens: int | None = None diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 044c0131699..f7bd46536a1 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -2,9 +2,11 @@ Support for gpt model family """ +import asyncio import json import os from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from contextvars import ContextVar from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload from urllib.parse import urlparse @@ -28,7 +30,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, + async_fetch_urls_to_base64, convert_url_to_base64, + file_id_urls_before_an_unreadable_part, + has_file_id_urls, ) from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( drop_non_python_regex_patterns, @@ -85,6 +90,20 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) +_ROLES_WHOSE_FILES_ARE_DOWNLOADED: Final = ("user", "tool") +_PDF_URL_STARTS: Final = ("https://", "http://", "www.") +_NOTHING_PREFETCHED: Final[Mapping[str, "str | litellm.ImageFetchError"]] = MappingProxyType({}) +_prefetched_pdf_urls: Final[ContextVar[Mapping[str, "str | litellm.ImageFetchError"]]] = ContextVar( + "prefetched_pdf_urls" +) + + +def _pdf_data_url(url: str) -> str: + prefetched: Final = _prefetched_pdf_urls.get(_NOTHING_PREFETCHED).get(url) + if isinstance(prefetched, litellm.ImageFetchError): + raise prefetched + return prefetched if prefetched is not None else convert_url_to_base64(url) + def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: if custom_llm_provider != "openai": @@ -191,9 +210,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. """ - # Add a class variable to track if this is the base class - _is_base_class = True - frequency_penalty: int | None = None function_call: str | dict | None = None functions: list | None = None @@ -225,8 +241,6 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) - self.__class__._is_base_class = False - @classmethod def get_config(cls): return super().get_config() @@ -322,9 +336,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) def contains_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> bool: - potential_pdf_url_starts: Final = ["https://", "http://", "www."] file_id: Final = content_item.get("file_id") - if file_id and any(file_id.startswith(start) for start in potential_pdf_url_starts): + if file_id and any(file_id.startswith(start) for start in _PDF_URL_STARTS): return True return False @@ -332,7 +345,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): content_copy: Final = content_item.copy() file_id: Final = content_copy.get("file_id") if file_id is not None: - base64_data: Final = convert_url_to_base64(file_id) + base64_data: Final = _pdf_data_url(file_id) content_copy["file_data"] = base64_data content_copy["filename"] = "my_file.pdf" content_copy.pop("file_id") @@ -688,6 +701,12 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): **self._sanitized_tools_update_for_openai(optional_params, litellm_params), } + def _flattens_content_lists(self) -> bool: + return False + + def _with_provider_roles(self, messages: Sequence[AllMessageValues], model: str) -> Sequence[AllMessageValues]: + return messages + async def async_transform_request( self, model: str, @@ -696,30 +715,25 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - transformed_messages, tools = self._messages_and_tools_in_cache_mark_dialect( - model=model, - messages=await self._transform_messages( - messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True - ), - tools=optional_params.get("tools", []), - prompt_cache_options=_prompt_cache_options(optional_params), - litellm_params=litellm_params, - request_params=optional_params, - ) - if tools is not None and len(tools) > 0: - optional_params["tools"] = tools - if self.__class__._is_base_class: - if not optional_params.get("tools") and not optional_params.get("functions"): - optional_params.pop("tool_choice", None) - return { - "model": model, - "messages": transformed_messages, - **optional_params, - **self._sanitized_tools_update_for_openai(optional_params, litellm_params), - } - else: - ## allow for any object specific behaviour to be handled + if not has_file_id_urls(messages, _PDF_URL_STARTS): return self.transform_request(model, messages, optional_params, litellm_params, headers) + sent_messages: Final = self._with_provider_roles( + self._prompt_cache_ordered_messages(messages, litellm_params), model + ) + pdf_urls: Final = file_id_urls_before_an_unreadable_part( + (message for message in sent_messages if message.get("role") in _ROLES_WHOSE_FILES_ARE_DOWNLOADED), + _PDF_URL_STARTS, + ) + prefetched: Final = ( + _NOTHING_PREFETCHED if self._flattens_content_lists() else await async_fetch_urls_to_base64(pdf_urls) + ) + reset_token: Final = _prefetched_pdf_urls.set(prefetched) + try: + return await asyncio.to_thread( + self.transform_request, model, messages, optional_params, litellm_params, headers + ) + finally: + _prefetched_pdf_urls.reset(reset_token) def _passed_in_tools(self, optional_params: dict) -> bool: return optional_params.get("tools", None) is not None diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index 4d56a3b38c5..06257af7075 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -122,6 +122,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): return super()._map_openai_params(non_default_params, optional_params, model, drop_params) + def _with_provider_roles(self, messages: Sequence[AllMessageValues], model: str) -> Sequence[AllMessageValues]: + keeps_system_messages: Final = supports_system_messages(model, "openai") + return tuple( + message + if keeps_system_messages or message["role"] != "system" + else ChatCompletionUserMessage(content=message["content"], role="user") + for message in messages + ) + def is_model_o_series_model(self, model: str) -> bool: model = model.split("/")[-1] # could be "openai/o3" or "o3" return ( @@ -152,13 +161,7 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): - modalities: image => drop param (if user opts in to dropping param) - role: system ==> translate to role 'user' """ - _supports_system_messages: Final = supports_system_messages(model, "openai") - transformed_messages: Final = list(messages) - for i, message in enumerate(transformed_messages): - if message["role"] == "system" and not _supports_system_messages: - new_message = ChatCompletionUserMessage(content=message["content"], role="user") - transformed_messages[i] = new_message # Replace the old message with the new one - + transformed_messages: Final = list(self._with_provider_roles(messages, model)) if is_async: return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True)) else: diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index f330d7fc6d4..495ebf594d8 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1166,7 +1166,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): shared_session: Optional["ClientSession"] = None, ): response = None - data = provider_config.transform_request( + data = await provider_config.async_transform_request( model=model, messages=messages, optional_params=optional_params, diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 66b41967ca7..05cf90d7abd 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -50,6 +50,9 @@ def create_config_class(provider: SimpleProviderConfig): base_class: Final[type] = OpenAIGPTConfig if provider.base_class == "openai_gpt" else OpenAILikeChatConfig class JSONProviderConfig(base_class): + def _flattens_content_lists(self) -> bool: + return bool(provider.special_handling.get("convert_content_list_to_string")) + @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] @@ -69,7 +72,7 @@ def create_config_class(provider: SimpleProviderConfig): """Transform messages based on special_handling config""" # Handle content list to string conversion if configured - if provider.special_handling.get("convert_content_list_to_string"): + if self._flattens_content_lists(): messages = handle_messages_with_content_list_to_str_conversion(messages) if is_async: diff --git a/litellm/llms/sambanova/chat.py b/litellm/llms/sambanova/chat.py index 0350950f342..8247ec2770d 100644 --- a/litellm/llms/sambanova/chat.py +++ b/litellm/llms/sambanova/chat.py @@ -32,6 +32,9 @@ class SambanovaConfig(OpenAIGPTConfig): response_format: dict | None = None tools: list | None = None + def _flattens_content_lists(self) -> bool: + return True + def __init__( self, max_tokens: int | None = None, diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 263c2f7618d..964b39551a2 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1,15 +1,18 @@ """ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ +import asyncio import base64 import copy import json +import threading from collections.abc import Callable from typing import Final +import httpx import pytest import respx -from httpx import Response +from httpx import Request, Response import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -17,11 +20,21 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_MEDIA_BOUNDARY, ) from litellm.litellm_core_utils.prompt_templates.image_handling import in_memory_cache +from litellm.llms.bedrock.chat.chat_completions.transformation import AmazonBedrockRuntimeChatCompletionsConfig +from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig +from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig +from litellm.llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig +from litellm.llms.heroku.chat.transformation import HerokuChatConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from litellm.llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig +from litellm.llms.openai_like.dynamic_config import create_config_class +from litellm.llms.openai_like.json_loader import SimpleProviderConfig +from litellm.llms.sambanova.chat import SambanovaConfig +from litellm.llms.vertex_ai.vertex_gemma_models.transformation import VertexGemmaConfig class TestOpenAIGPTConfig: @@ -1348,6 +1361,607 @@ async def test_openai_media_download_preserves_supported_chat_fields( +_PDF_BYTES: Final = b"%PDF-1.7\nmock-pdf" +_DOWNLOADED_PDF_PART: Final = { + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{base64.b64encode(_PDF_BYTES).decode('utf-8')}", + "filename": "my_file.pdf", + }, +} + + +def _pdf_url_part(url: str) -> dict[str, object]: + return {"type": "file", "file": {"file_id": url}} + + +async def _serve_pdf_to_async_clients_only(request: Request) -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=_PDF_BYTES) + + +def _tool_result_messages(*content: object) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "read the report"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "fetch_report", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": list(content)}, + ] + + +@pytest.fixture +def pdf_urls_reachable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +_SUMMARIZE: Final = {"type": "text", "text": "Summarize the report."} +_UPLOADED_FILE_PART: Final = {"type": "file", "file": {"file_id": "file-uploaded"}} +_ONCE_URL: Final = "https://files.example.test/download-once.pdf" + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("messages", "expected_messages"), + [ + pytest.param( + [{"role": "user", "content": [_SUMMARIZE, _UPLOADED_FILE_PART, _pdf_url_part(_ONCE_URL)]}], + [{"role": "user", "content": [_SUMMARIZE, _UPLOADED_FILE_PART, _DOWNLOADED_PDF_PART]}], + id="user_message", + ), + pytest.param( + _tool_result_messages(_SUMMARIZE, _pdf_url_part(_ONCE_URL)), + [ + *_tool_result_messages(_SUMMARIZE)[:2], + {"role": "tool", "tool_call_id": "call_1", "content": [_SUMMARIZE]}, + {"role": "user", "content": [{"type": "text", "text": TOOL_RESULT_MEDIA_BOUNDARY}, _DOWNLOADED_PDF_PART]}, + ], + id="tool_result", + ), + ], +) +async def test_async_transform_request_downloads_a_pdf_url_once_with_the_async_client( + respx_mock: respx.MockRouter, messages: list[dict[str, object]], expected_messages: list[dict[str, object]] +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + download: Final = respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=copy.deepcopy(messages), + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert download.call_count == 1 + assert request["messages"] == expected_messages + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_pdf_too_large_for_the_url_cache_once( + respx_mock: respx.MockRouter, +) -> None: + pdf_url: Final = "https://files.example.test/too-large-to-cache.pdf" + large_pdf: Final = _PDF_BYTES + b"0" * in_memory_cache.max_size_per_item * 1024 + + async def serve_large_pdf_to_async_clients_only(request: Request) -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=large_pdf) + + download: Final = respx_mock.get(pdf_url).mock(side_effect=serve_large_pdf_to_async_clients_only) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert download.call_count == 1 + assert request["messages"][0]["content"][1]["file"] == { + "file_data": f"data:application/pdf;base64,{base64.b64encode(large_pdf).decode('utf-8')}", + "filename": "my_file.pdf", + } + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_pdf_urls_one_at_a_time_for_a_host_that_refuses_bursts( + respx_mock: respx.MockRouter, +) -> None: + pdf_urls: Final = tuple(f"https://files.example.test/in-turn-{index}.pdf" for index in range(3)) + for pdf_url in pdf_urls: + in_memory_cache.delete_cache(pdf_url) + download_running: Final = asyncio.Lock() + + async def refuse_a_second_download_at_once(request: Request) -> Response: + if download_running.locked(): + return Response(429) + async with download_running: + await asyncio.sleep(0.05) + return await _serve_pdf_to_async_clients_only(request) + + downloads: Final = respx_mock.route(method="GET", host="files.example.test").mock( + side_effect=refuse_a_second_download_at_once + ) + + request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, *map(_pdf_url_part, pdf_urls)]}], + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, *[_DOWNLOADED_PDF_PART] * 3]}] + assert downloads.call_count == len(pdf_urls) + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_streamed_completion_downloads_its_pdf_url_with_the_async_client(respx_mock: respx.MockRouter) -> None: + pdf_url: Final = "https://files.example.test/streamed.pdf" + in_memory_cache.delete_cache(pdf_url) + download: Final = respx_mock.get(pdf_url).mock(side_effect=_serve_pdf_to_async_clients_only) + chunk: Final = { + "id": "chatcmpl-stream", + "object": "chat.completion.chunk", + "created": 0, + "model": "gpt-4.1", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "seen"}, "finish_reason": "stop"}], + } + chat: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=Response( + 200, + headers={"content-type": "text/event-stream"}, + content=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode(), + ) + ) + + stream: Final = await litellm.acompletion( + model="gpt-4.1", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}], + api_key="sk-openai-test", + stream=True, + ) + text: Final = "".join([part.choices[0].delta.content or "" async for part in stream]) + + assert text == "seen" + assert download.call_count == 1 + assert json.loads(chat.calls[0].request.content)["messages"] == [ + {"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART]} + ] + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("unreadable_part", "error"), + [ + pytest.param( + {"type": "file", "file": None}, + "Content block has type='file' but is missing the required 'file' field", + id="file_part_without_its_file", + ), + pytest.param( + {"type": "file", "file": "file-uploaded"}, "has no attribute 'items'", id="file_that_is_not_an_object" + ), + pytest.param( + {"type": "file", "file": {"file_id": 123}}, "has no attribute 'startswith'", id="file_id_not_a_string" + ), + pytest.param("Summarize the report.", "has no attribute 'get'", id="part_that_is_not_an_object"), + ], +) +async def test_async_transform_request_rejects_an_unreadable_part_like_the_sync_transform_before_a_later_pdf_url( + respx_mock: respx.MockRouter, unreadable_part: object, error: str +) -> None: + pdf_url: Final = "https://files.example.test/after-a-broken-part.pdf" + in_memory_cache.delete_cache(pdf_url) + download: Final = respx_mock.get(pdf_url).mock(return_value=Response(200, content=b"%PDF-1.4")) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, unreadable_part, _pdf_url_part(pdf_url)]}] + + with pytest.raises((litellm.BadRequestError, AttributeError), match=error) as sync_error: + OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + with pytest.raises(type(sync_error.value), match=error): + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", + messages=copy.deepcopy(messages), + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert not download.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("config", [OpenAIGPTConfig(), DeepInfraConfig()], ids=["openai", "deepinfra"]) +async def test_async_transform_request_sends_a_plain_string_tool_result_part_like_the_sync_transform( + config: OpenAIGPTConfig, +) -> None: + messages: Final = _tool_result_messages("plain string result") + + request: Final = await config.async_transform_request( + model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert request == config.transform_request( + model="test-model", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + +def _json_provider_config(special_handling: dict[str, bool]) -> OpenAIGPTConfig: + provider: Final = SimpleProviderConfig( + "test-json-provider", + { + "base_url": "https://json-provider.example.test/v1", + "api_key_env": "TEST_JSON_PROVIDER_API_KEY", + "special_handling": special_handling, + }, + ) + return create_config_class(provider)() + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "config", + [ + pytest.param(SambanovaConfig(), id="sambanova"), + pytest.param(HerokuChatConfig(), id="heroku"), + pytest.param(DockerModelRunnerChatConfig(), id="docker_model_runner"), + pytest.param(_json_provider_config({"convert_content_list_to_string": True}), id="json_provider"), + pytest.param(DeepSeekChatConfig(), id="deepseek"), + ], +) +async def test_async_transform_request_does_not_download_a_pdf_url_for_a_provider_that_sends_only_text( + respx_mock: respx.MockRouter, config: OpenAIGPTConfig +) -> None: + pdf_url: Final = "https://files.example.test/dropped.pdf" + in_memory_cache.delete_cache(pdf_url) + download: Final = respx_mock.get(pdf_url).mock(side_effect=_serve_pdf_to_async_clients_only) + + request: Final = await config.async_transform_request( + model="test-model", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(pdf_url)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert download.call_count == 0 + assert request["messages"] == [{"role": "user", "content": "Summarize the report."}] + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("config", "messages", "expected_messages"), + [ + pytest.param( + HerokuChatConfig(), + [{"role": "user", "content": [_pdf_url_part(_ONCE_URL)]}], + [{"role": "user", "content": [_DOWNLOADED_PDF_PART]}], + id="pdf_only_message_for_a_provider_that_sends_only_text", + ), + ], +) +async def test_async_transform_request_downloads_a_pdf_the_sync_transform_reaches_off_the_event_loop( + respx_mock: respx.MockRouter, + config: OpenAIGPTConfig, + messages: list[dict[str, object]], + expected_messages: list[dict[str, object]], +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + event_loop_thread: Final = threading.get_ident() + + def serve_pdf_off_the_event_loop_only(request: Request) -> Response: + if threading.get_ident() == event_loop_thread: + return Response(503) + return Response(200, headers={"content-type": "application/pdf"}, content=_PDF_BYTES) + + respx_mock.get(_ONCE_URL).mock(side_effect=serve_pdf_off_the_event_loop_only) + + request: Final = await config.async_transform_request( + model="test-model-without-system-messages", + messages=copy.deepcopy(messages), + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == expected_messages + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_system_pdf_with_the_async_client_for_a_model_without_system_messages( + respx_mock: respx.MockRouter, +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + download: Final = respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + request: Final = await OpenAIOSeriesConfig().async_transform_request( + model="test-model-without-system-messages", + messages=[{"role": "system", "content": [_pdf_url_part(_ONCE_URL)]}, {"role": "user", "content": [_SUMMARIZE]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert download.call_count == 1 + assert request["messages"] == [ + {"role": "user", "content": [_DOWNLOADED_PDF_PART]}, + {"role": "user", "content": [_SUMMARIZE]}, + ] + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_pdf_url_with_the_async_client_for_a_json_provider_that_keeps_lists( + respx_mock: respx.MockRouter, +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + respx_mock.get(_ONCE_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + request: Final = await _json_provider_config({}).async_transform_request( + model="test-model", + messages=[{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_ONCE_URL)]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART]}] + + +def _serve_pdf(request: Request) -> Response: + return Response(200, headers={"content-type": "application/pdf"}, content=_PDF_BYTES) + + +def _answer_not_found(request: Request) -> Response: + return Response(404) + + +def _answer_too_many_requests(request: Request) -> Response: + return Response(429) + + +def _refuse_connection(request: Request) -> Response: + raise httpx.ConnectError("connection refused", request=request) + + +_PARITY_URL: Final = "https://files.example.test/parity.pdf" + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "messages", + [ + pytest.param([{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_PARITY_URL)]}], id="user_message"), + pytest.param( + [ + { + "role": "user", + "content": [_SUMMARIZE, {"type": "file", "file": {"file_id": _PARITY_URL, "filename": "report.pdf"}}], + } + ], + id="named_file", + ), + pytest.param(_tool_result_messages(_SUMMARIZE, _pdf_url_part(_PARITY_URL)), id="tool_result"), + pytest.param( + [ + {"role": "system", "content": [_pdf_url_part(_PARITY_URL)]}, + {"role": "user", "content": [_SUMMARIZE]}, + {"role": "assistant", "content": [_pdf_url_part(_PARITY_URL)]}, + {"role": "user", "content": "Go on."}, + ], + id="system_and_assistant_messages", + ), + ], +) +@pytest.mark.parametrize( + ("config", "model"), + [ + pytest.param(OpenAIGPTConfig(), "gpt-4.1", id="openai"), + pytest.param(AmazonBedrockRuntimeChatCompletionsConfig(), "us.xai.grok-4.6", id="bedrock_chat_completions"), + pytest.param(VertexGemmaConfig(), "gemma-3-27b-it", id="vertex_gemma"), + ], +) +async def test_async_transform_request_matches_the_sync_request_downloads_and_callers_messages( + respx_mock: respx.MockRouter, messages: list[dict[str, object]], config: OpenAIGPTConfig, model: str +) -> None: + download: Final = respx_mock.get(_PARITY_URL).mock(side_effect=_serve_pdf) + sync_messages: Final = copy.deepcopy(messages) + async_messages: Final = copy.deepcopy(messages) + + in_memory_cache.delete_cache(_PARITY_URL) + sync_request: Final = config.transform_request( + model=model, messages=sync_messages, optional_params={}, litellm_params={}, headers={} + ) + sync_downloads: Final = download.call_count + in_memory_cache.delete_cache(_PARITY_URL) + async_request: Final = await config.async_transform_request( + model=model, messages=async_messages, optional_params={}, litellm_params={}, headers={} + ) + + assert async_request == sync_request + assert async_messages == sync_messages + assert download.call_count - sync_downloads == sync_downloads + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dead_url_answer", + [ + pytest.param(_answer_not_found, id="not_found"), + pytest.param(_answer_too_many_requests, id="rate_limited"), + pytest.param(_refuse_connection, id="refused"), + ], +) +async def test_async_transform_request_fails_a_dead_pdf_url_with_the_sync_error_and_download_count( + respx_mock: respx.MockRouter, dead_url_answer: Callable[[Request], Response] +) -> None: + healthy_url: Final = "https://files.example.test/healthy.pdf" + dead_url: Final = "https://files.example.test/dead.pdf" + healthy: Final = respx_mock.get(healthy_url).mock(side_effect=_serve_pdf) + dead: Final = respx_mock.get(dead_url).mock(side_effect=dead_url_answer) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(healthy_url), _pdf_url_part(dead_url)]}] + + in_memory_cache.delete_cache(healthy_url) + with pytest.raises(litellm.ImageFetchError) as sync_error: + OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + sync_downloads: Final = healthy.call_count + dead.call_count + in_memory_cache.delete_cache(healthy_url) + with pytest.raises(litellm.ImageFetchError) as async_error: + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert str(async_error.value) == str(sync_error.value) + assert healthy.call_count + dead.call_count - sync_downloads == sync_downloads + + +_DEAD_URL: Final = "https://files.example.test/dead-first.pdf" +_AFTER_DEAD_URL: Final = "https://files.example.test/after-dead.pdf" +_WWW_FILE_ID: Final = "www.files.example.test/no-scheme.pdf" + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("config", "model", "messages"), + [ + pytest.param( + OpenAIGPTConfig(), + "gpt-4.1", + [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_DEAD_URL), _pdf_url_part(_AFTER_DEAD_URL)]}], + id="dead_url", + ), + pytest.param( + OpenAIGPTConfig(), + "gpt-4.1", + [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_WWW_FILE_ID), _pdf_url_part(_AFTER_DEAD_URL)]}], + id="www_file_id", + ), + pytest.param( + OpenAIOSeriesConfig(), + "test-model-without-system-messages", + [ + {"role": "system", "content": [_pdf_url_part(_DEAD_URL)]}, + {"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_AFTER_DEAD_URL)]}, + ], + id="system_message_for_a_model_without_system_messages", + ), + ], +) +async def test_async_transform_request_fails_a_dead_pdf_url_without_downloading_the_pdf_urls_after_it( + respx_mock: respx.MockRouter, config: OpenAIGPTConfig, model: str, messages: list[dict[str, object]] +) -> None: + in_memory_cache.delete_cache(_AFTER_DEAD_URL) + respx_mock.get(_DEAD_URL).mock(return_value=Response(404)) + later: Final = respx_mock.get(_AFTER_DEAD_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + with pytest.raises(litellm.ImageFetchError): + await config.async_transform_request( + model=model, messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + await asyncio.wait_for(asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})), timeout=5) + + assert later.call_count == 0 + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.asyncio +async def test_async_transform_request_downloads_a_moved_system_pdf_first_when_system_messages_go_first( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "openai_system_messages_first", True) + in_memory_cache.delete_cache(_AFTER_DEAD_URL) + respx_mock.get(_DEAD_URL).mock(return_value=Response(404)) + later: Final = respx_mock.get(_AFTER_DEAD_URL).mock(side_effect=_serve_pdf_to_async_clients_only) + + with pytest.raises(litellm.ImageFetchError): + await OpenAIOSeriesConfig().async_transform_request( + model="test-model-without-system-messages", + messages=[ + {"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_AFTER_DEAD_URL)]}, + {"role": "system", "content": [_pdf_url_part(_DEAD_URL)]}, + ], + optional_params={}, + litellm_params={"custom_llm_provider": "openai"}, + headers={}, + ) + + assert later.call_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role", ["system", "assistant", "developer"]) +@pytest.mark.parametrize("part_type", [pytest.param(["text"], id="list"), pytest.param({"kind": "text"}, id="dict")]) +async def test_async_transform_request_passes_a_part_with_a_non_string_type_through_like_the_sync_request( + role: str, part_type: object +) -> None: + messages: Final = [ + {"role": role, "content": [{"type": part_type, "text": "Answer briefly."}]}, + {"role": "user", "content": "hi"}, + ] + + sync_request: Final = OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + async_request: Final = await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert async_request == sync_request + + +@pytest.mark.usefixtures("pdf_urls_reachable") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.asyncio +async def test_a_pdf_url_that_failed_in_an_async_request_downloads_again_in_a_later_request_of_the_same_task( + respx_mock: respx.MockRouter, +) -> None: + in_memory_cache.delete_cache(_ONCE_URL) + respx_mock.get(_ONCE_URL).mock(side_effect=[Response(404), _serve_pdf(Request("GET", _ONCE_URL))]) + messages: Final = [{"role": "user", "content": [_SUMMARIZE, _pdf_url_part(_ONCE_URL)]}] + + with pytest.raises(litellm.ImageFetchError): + await OpenAIGPTConfig().async_transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + request: Final = OpenAIGPTConfig().transform_request( + model="gpt-4.1", messages=copy.deepcopy(messages), optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"] == [{"role": "user", "content": [_SUMMARIZE, _DOWNLOADED_PDF_PART]}] + + @pytest.mark.respx(assert_all_called=True) def test_vision_with_custom_model_preserves_base64_image_in_request(respx_mock: respx.MockRouter) -> None: image_uri: Final = f"data:image/png;base64,{base64.b64encode(b'test-image').decode('utf-8')}" @@ -1560,14 +2174,8 @@ class TestSystemMessagesFirst: @pytest.mark.asyncio async def test_async_transform_request_moves_system_messages_first(self, monkeypatch): - class UninstantiatedOpenAIGPTConfig(OpenAIGPTConfig): - _is_base_class = True - - def __init__(self) -> None: - pass - monkeypatch.setattr(litellm, "openai_system_messages_first", True) - request = await UninstantiatedOpenAIGPTConfig().async_transform_request( + request = await self.config.async_transform_request( model="gpt-4.1", messages=self._messages(), optional_params={}, From bece938ccc88a64aca57a18d382a52c4be182556 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:39:08 +0000 Subject: [PATCH 006/189] feat(cost-map): add vertex_ai/gemini-4-carbon and gemini-4-carbon-cyber-permissive from the Vertex pricing page (#50942) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 112 ++++++++++++++++++ model_prices_and_context_window.json | 112 ++++++++++++++++++ 2 files changed, 224 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b2428aa4c94..6046e3f863f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -88303,5 +88303,117 @@ "supports_web_search": true, "tpm": 800000.0, "web_search_billing_unit": "per_query" + }, + "vertex_ai/gemini-4-carbon": { + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, + "cache_read_input_token_cost_priority": 1.8e-07, + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token": 2e-06, + "input_cost_per_audio_token_batches": 1e-06, + "input_cost_per_audio_token_priority": 3.6e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_priority": 3.6e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_reasoning_token": 1e-05, + "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 1.8e-05, + "prompt_cache_min_tokens": 4096.0, + "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_computer_use": true, + "supports_continuation": true, + "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "vertex_ai/gemini-4-carbon-cyber-permissive": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_priority": 3.6e-07, + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token": 4e-06, + "input_cost_per_audio_token_batches": 2e-06, + "input_cost_per_audio_token_priority": 7.2e-06, + "input_cost_per_token": 4e-06, + "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_priority": 7.2e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_reasoning_token": 2e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_priority": 3.6e-05, + "prompt_cache_min_tokens": 4096.0, + "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_computer_use": false, + "supports_continuation": true, + "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b2428aa4c94..6046e3f863f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -88303,5 +88303,117 @@ "supports_web_search": true, "tpm": 800000.0, "web_search_billing_unit": "per_query" + }, + "vertex_ai/gemini-4-carbon": { + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, + "cache_read_input_token_cost_priority": 1.8e-07, + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token": 2e-06, + "input_cost_per_audio_token_batches": 1e-06, + "input_cost_per_audio_token_priority": 3.6e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_priority": 3.6e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_reasoning_token": 1e-05, + "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 1.8e-05, + "prompt_cache_min_tokens": 4096.0, + "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_computer_use": true, + "supports_continuation": true, + "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "vertex_ai/gemini-4-carbon-cyber-permissive": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_priority": 3.6e-07, + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token": 4e-06, + "input_cost_per_audio_token_batches": 2e-06, + "input_cost_per_audio_token_priority": 7.2e-06, + "input_cost_per_token": 4e-06, + "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_priority": 7.2e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_reasoning_token": 2e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_priority": 3.6e-05, + "prompt_cache_min_tokens": 4096.0, + "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_computer_use": false, + "supports_continuation": true, + "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" } } From 82ef50bacba00f14687751be5b72a23879338395 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:55:08 +0000 Subject: [PATCH 007/189] refactor: read include_cost_in_streaming_usage directly, type MaskedHTTPStatusError init, drop unused encrypted reasoning check (#50617) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/core_helpers.py | 2 +- .../prompt_templates/reasoning_utils.py | 8 -------- litellm/llms/custom_httpx/http_handler.py | 4 +++- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 800973573c9..8dec2941e02 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -123,7 +123,7 @@ def proxy_request_headers(container: Mapping[str, object] | None) -> Mapping[str def stream_cost_injection_requested(request_headers: Mapping[str, object] | None) -> bool: """The global include_cost_in_streaming_usage flag or the per-request header asks for usage.cost on streams.""" - if bool(getattr(litellm, "include_cost_in_streaming_usage", False)): + if litellm.include_cost_in_streaming_usage: return True if not request_headers: return False diff --git a/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py b/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py index ff1f2e017e3..7773c189b53 100644 --- a/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py @@ -282,14 +282,6 @@ def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: ) -def _content_carries_encrypted_reasoning(content: object) -> bool: - return any(is_encrypted_reasoning_block(block) for block in _object_sequence(content)) - - -def messages_carry_encrypted_reasoning(messages: Sequence[object]) -> bool: - return any(_content_carries_encrypted_reasoning(content) for content in anthropic_content_lists(messages)) - - def _strip_encrypted_reasoning_from_blocks( content: object, *, diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 0cfdbc74227..307d67bab21 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -663,7 +663,9 @@ class HTTPResponseLimitError(ValueError): class MaskedHTTPStatusError(httpx.HTTPStatusError): - def __init__(self, original_error: httpx.HTTPStatusError, message: str | None = None, text: str | None = None): + def __init__( + self, original_error: httpx.HTTPStatusError, message: str | None = None, text: str | None = None + ) -> None: # Create a new error with the masked URL masked_url: Final = mask_sensitive_info(str(original_error.request.url)) # Mask the original exception message too (it contains the full URL) From d70855ca826d5aa1eeab7adfef56d8524ca40bb7 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:03:21 -0800 Subject: [PATCH 008/189] fix(proxy): open the Logs drawer by request_id without scanning for litellm_call_id (#50789) * fix(proxy): open the Logs drawer by request_id without scanning for litellm_call_id The detail lookup matched request_id OR litellm_call_id, which Postgres runs as a full table scan when the litellm_call_id index is missing or invalid. Read by request_id, the primary key, first and by litellm_call_id only on a miss, both under the spend log statement timeout. * test(proxy): type the call ID index test helpers * fix(proxy): limit the Logs drawer call ID lookup to start_date and answer 400 for a malformed date * fix(proxy): let spend log reads wait for a busy connection like a plain query * test(proxy): let the session pages read double take max_wait --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../spend_management_endpoints.py | 171 +++++++++++---- litellm/proxy/utils.py | 9 +- .../spend/test_session_grouped_logs_pages.py | 4 +- .../spend/test_spend_log_read_scope.py | 126 ++++++++++- .../test_spend_management_endpoints.py | 199 ++++++++++++++++-- 5 files changed, 445 insertions(+), 64 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index f3dc672a072..7d9a0dcc971 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -132,11 +132,6 @@ class _SupportsModelDump(Protocol): def model_dump(self) -> Mapping[str, object]: ... -class _SpendLogOwnerRow(TypedDict): - user: ReadOnly[str | None] - team_id: ReadOnly[str | None] - - class _ActivityRow(TypedDict): date: str api_requests: int @@ -273,6 +268,60 @@ async def _query_spend_logs(prisma_client: PrismaClient, sql_query: str, *args: ) +_SpendLogIdColumn: TypeAlias = Literal["request_id", "litellm_call_id"] + + +@dataclass(frozen=True, slots=True) +class _SpendLogWindow: + start: datetime | None = None + end: datetime | None = None + + +_UNBOUNDED_SPEND_LOG_WINDOW: Final = _SpendLogWindow() + + +def _spend_log_id_filter( + id_column: _SpendLogIdColumn, lookup_id: str, window: _SpendLogWindow +) -> tuple[str, tuple[object, ...]]: + match id_column: + case "request_id": + return "request_id = $1", (lookup_id,) + case "litellm_call_id": + bounds: Final = tuple( + (operator, bound.isoformat()) + for operator, bound in ((">=", window.start), ("<=", window.end)) + if bound is not None + ) + bound_sql: Final = "".join( + f""" AND "startTime" {operator} (${index}::timestamptz AT TIME ZONE 'UTC')""" + for index, (operator, _) in enumerate(bounds, start=2) + ) + return f"litellm_call_id = $1{bound_sql}", (lookup_id, *(bound for _, bound in bounds)) + + +def _spend_log_lookup_timeout_hint(id_column: _SpendLogIdColumn) -> str: + match id_column: + case "request_id": + return "Try again" + case "litellm_call_id": + return ( + "Pass a narrower start_date and end_date, or ask an admin to check that the " + "LiteLLM_SpendLogs_litellm_call_id_idx index exists and is valid" + ) + + +async def _query_spend_logs_by_id( + prisma_client: PrismaClient, id_column: _SpendLogIdColumn, sql_query: str, *args: object +) -> Sequence[Mapping[str, object]]: + return await query_raw_within_statement_timeout( + prisma_client, + sql_query, + *args, + what=f"Spend log lookup by {id_column}", + hint=_spend_log_lookup_timeout_hint(id_column), + ) + + @dataclass(frozen=True, slots=True) class _SpendLogsReader: prisma_client: PrismaClient @@ -399,9 +448,11 @@ def _request_id_or_call_id_clause(request_id: str) -> tuple[_RequestIdEquals, _L return (request_id_clause, call_id_clause) -async def _find_spend_log_owners(prisma_client: PrismaClient, request_id: str) -> Sequence[_SpendLogOwnerRow]: +async def _find_spend_log_owners( + prisma_client: PrismaClient, id_column: _SpendLogIdColumn, lookup_id: str, window: _SpendLogWindow +) -> frozenset[tuple[str | None, str | None]]: """Read the distinct ``(user, team_id)`` owner pairs across every spend log row - identified by ``request_id`` or ``litellm_call_id``. + whose ``id_column`` is ``lookup_id``. ``litellm_call_id`` is populated from the client-settable ``x-litellm-call-id`` request header, so it is not guaranteed unique to one tenant: any number of rows @@ -409,13 +460,14 @@ async def _find_spend_log_owners(prisma_client: PrismaClient, request_id: str) - carrying the caller's id could otherwise push the caller's own owner pair past a row-sample cap and lock them out of their own lookup. """ - sql_query: Final = """ - SELECT DISTINCT "user", team_id - FROM "LiteLLM_SpendLogs" - WHERE request_id = $1 OR litellm_call_id = $1 - """ - owners: Final[Sequence[_SpendLogOwnerRow] | None] = await _query_raw_or_none(prisma_client, sql_query, request_id) - return owners if owners is not None else () + id_filter, id_params = _spend_log_id_filter(id_column, lookup_id, window) + rows: Final = await _query_spend_logs_by_id( + prisma_client, + id_column, + f'SELECT DISTINCT "user", team_id FROM "LiteLLM_SpendLogs" WHERE {id_filter}', + *id_params, + ) + return frozenset(_fetched_row_owner(row) for row in rows) async def _count_spend_logs(prisma_client: PrismaClient, where: Mapping[str, object]) -> int: @@ -3292,6 +3344,18 @@ async def _resolve_request_response_payload( ) +def _parse_spend_log_ui_date(value: str | None) -> datetime | None: + if value is None: + return None + try: + return datetime.strptime(value, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) + except ValueError as error: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": f"Invalid date {value!r}, expected YYYY-MM-DD HH:MM:SS in UTC"}, + ) from error + + @router.get( "/spend/logs/ui/{request_id}", tags=["Budget & Spend Tracking"], @@ -3319,6 +3383,7 @@ async def ui_view_request_response_for_request_id( """ from litellm.proxy.proxy_server import prisma_client + window: Final = _SpendLogWindow(start=_parse_spend_log_ui_date(start_date), end=_parse_spend_log_ui_date(end_date)) caller_is_admin: Final = is_admin_view_safe(user_api_key_dict=user_api_key_dict) if not caller_is_admin: if prisma_client is None: @@ -3335,16 +3400,10 @@ async def ui_view_request_response_for_request_id( prisma_client=prisma_client, user_api_key_dict=user_api_key_dict, request_id=request_id, + window=window, ) custom_loggers: Final = litellm.logging_callback_manager.get_active_additional_logging_utils_from_custom_logger() - start_date_obj: datetime | None = None - end_date_obj: datetime | None = None - if start_date is not None: - start_date_obj = datetime.strptime(start_date, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) - if end_date is not None: - end_date_obj = datetime.strptime(end_date, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) - spend_log_row: Final = ( None if prisma_client is None @@ -3354,6 +3413,7 @@ async def ui_view_request_response_for_request_id( request_id=request_id, caller_is_admin=caller_is_admin, log_team_lookup=log_team_lookup, + window=window, ) ) stored_request_id: Final = _stored_request_id(spend_log_row, request_id) @@ -3361,8 +3421,8 @@ async def ui_view_request_response_for_request_id( for custom_logger in custom_loggers: payload = await custom_logger.get_request_response_payload( request_id=stored_request_id, - start_time_utc=start_date_obj, - end_time_utc=end_date_obj, + start_time_utc=window.start, + end_time_utc=window.end, ) if payload is not None: if not caller_is_admin and prisma_client is not None: @@ -4877,6 +4937,7 @@ async def _assert_user_can_view_request_id( prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, request_id: str, + window: _SpendLogWindow = _UNBOUNDED_SPEND_LOG_WINDOW, ) -> None: """ Verify the requesting non-admin user is allowed to view at least one spend-log @@ -4889,11 +4950,24 @@ async def _assert_user_can_view_request_id( so a missing row can't be used to read a payload out of cold storage via the detail endpoint. """ - owners: Final = await _find_spend_log_owners(prisma_client, request_id) - for owner in owners: - if await _user_can_view_spend_log_owner(prisma_client, user_api_key_dict, owner["user"], owner["team_id"]): - return - raise _spend_log_forbidden(request_id) + if not ( + await _user_can_view_spend_log_by_id(prisma_client, user_api_key_dict, "request_id", request_id, window) + or await _user_can_view_spend_log_by_id(prisma_client, user_api_key_dict, "litellm_call_id", request_id, window) + ): + raise _spend_log_forbidden(request_id) + + +async def _user_can_view_spend_log_by_id( + prisma_client: PrismaClient, + user_api_key_dict: UserAPIKeyAuth, + id_column: _SpendLogIdColumn, + lookup_id: str, + window: _SpendLogWindow, +) -> bool: + for user, team_id in await _find_spend_log_owners(prisma_client, id_column, lookup_id, window): + if await _user_can_view_spend_log_owner(prisma_client, user_api_key_dict, user, team_id): + return True + return False async def _spend_log_read_scope(user_api_key_dict: UserAPIKeyAuth, log_team_lookup: LogTeamLookup) -> OwnedRows: @@ -4927,32 +5001,54 @@ def _read_scope_where(scope: ReadScope) -> Mapping[str, object]: return grants[0] if len(grants) == 1 else {"OR": list(grants)} -def _spend_log_payload_query(request_id: str, scope: ReadScope) -> tuple[str, tuple[object, ...]]: +def _spend_log_payload_query( + id_column: _SpendLogIdColumn, lookup_id: str, scope: ReadScope, window: _SpendLogWindow +) -> tuple[str, tuple[object, ...]]: """ - Fetch the one row an id lookup resolves to, preferring the exact ``request_id`` - match over rows that merely carry the id as their client-set ``litellm_call_id``. - A non-admin viewer only ever gets rows they own or rows of a team they may view. + Fetch one row whose ``id_column`` is ``lookup_id``. A non-admin viewer only ever + gets rows they own or rows of a team they may view. """ - scope_clause, scope_params = read_scope_sql(scope, 2) + id_filter, id_params = _spend_log_id_filter(id_column, lookup_id, window) + scope_clause, scope_params = read_scope_sql(scope, len(id_params) + 1) scope_sql: Final = f" AND {scope_clause}" if scope_clause else "" return ( f""" SELECT request_id, messages, response, proxy_server_request, metadata, "user", team_id FROM "LiteLLM_SpendLogs" - WHERE (request_id = $1 OR litellm_call_id = $1){scope_sql} - ORDER BY (request_id = $1) DESC + WHERE {id_filter}{scope_sql} LIMIT 1 """, - (request_id, *scope_params), + (*id_params, *scope_params), ) +async def _query_spend_log_payload( + prisma_client: PrismaClient, + id_column: _SpendLogIdColumn, + lookup_id: str, + scope: ReadScope, + window: _SpendLogWindow, +) -> Sequence[Mapping[str, object]]: + sql_query, sql_params = _spend_log_payload_query(id_column, lookup_id, scope, window) + return await _query_spend_logs_by_id(prisma_client, id_column, sql_query, *sql_params) + + +async def _fetch_spend_log_payload_rows( + prisma_client: PrismaClient, lookup_id: str, scope: ReadScope, window: _SpendLogWindow +) -> Sequence[Mapping[str, object]]: + by_request_id: Final = await _query_spend_log_payload(prisma_client, "request_id", lookup_id, scope, window) + if by_request_id: + return by_request_id + return await _query_spend_log_payload(prisma_client, "litellm_call_id", lookup_id, scope, window) + + async def _resolve_spend_log_payload_row( prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, request_id: str, caller_is_admin: bool, log_team_lookup: LogTeamLookup, + window: _SpendLogWindow, ) -> Mapping[str, object] | None: """ Resolve an id lookup to the caller's own spend-log row before any payload @@ -4962,8 +5058,7 @@ async def _resolve_spend_log_payload_row( ``request_id`` is the key that names the caller's own request. """ scope: Final = AllRows() if caller_is_admin else await _spend_log_read_scope(user_api_key_dict, log_team_lookup) - sql_query, sql_params = _spend_log_payload_query(request_id, scope) - rows: Final[Sequence[Mapping[str, object]] | None] = await _query_raw_or_none(prisma_client, sql_query, *sql_params) + rows: Final = await _fetch_spend_log_payload_rows(prisma_client, request_id, scope, window) if not rows: return None if not caller_is_admin: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 97c327cbe4e..f78e68220b1 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -5367,16 +5367,19 @@ async def query_raw_within_statement_timeout( sql: str, *params: object, what: str = "Spend logs", + hint: str = "Pick a shorter date range", timeout: timedelta = _SPEND_LOG_READ_STATEMENT_TIMEOUT, ) -> Sequence[Mapping[str, object]]: """One raw read on the read target that Postgres cancels at ``timeout``. Prisma's HTTP client gives up on a statement after 30s anyway, so without this the Admin UI's error is the same but Postgres keeps running the scan. - A cancelled read surfaces as a 503 asking for a narrower date range. + A cancelled read surfaces as a 503 carrying ``hint``. """ try: - async with prisma_client.read_db.tx(timeout=timeout + _STATEMENT_TIMEOUT_TRANSACTION_GRACE) as transaction: + async with prisma_client.read_db.tx( + max_wait=timeout, timeout=timeout + _STATEMENT_TIMEOUT_TRANSACTION_GRACE + ) as transaction: await transaction.execute_raw(f"SET LOCAL statement_timeout = {int(timeout / timedelta(milliseconds=1))}") return await transaction.query_raw(sql, *params) except Exception as e: @@ -5384,7 +5387,7 @@ async def query_raw_within_statement_timeout( raise raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail={"error": f"{what} took longer than {int(timeout.total_seconds())}s. Pick a shorter date range"}, + detail={"error": f"{what} took longer than {int(timeout.total_seconds())}s. {hint}"}, ) from e diff --git a/tests/integration/spend/test_session_grouped_logs_pages.py b/tests/integration/spend/test_session_grouped_logs_pages.py index 7989cb6a7bc..9d457c63157 100644 --- a/tests/integration/spend/test_session_grouped_logs_pages.py +++ b/tests/integration/spend/test_session_grouped_logs_pages.py @@ -320,8 +320,8 @@ class _ExplainedReads: self.rows_scanned: tuple[int, ...] = () @asynccontextmanager - async def tx(self, *, timeout: timedelta) -> AsyncIterator["_ExplainedReads"]: - async with self.database.tx(timeout=timeout) as transaction: + async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncIterator["_ExplainedReads"]: + async with self.database.tx(max_wait=max_wait, timeout=timeout) as transaction: reads: Final = _ExplainedReads(transaction) try: yield reads diff --git a/tests/integration/spend/test_spend_log_read_scope.py b/tests/integration/spend/test_spend_log_read_scope.py index 8a208d61aed..b6226d0664e 100644 --- a/tests/integration/spend/test_spend_log_read_scope.py +++ b/tests/integration/spend/test_spend_log_read_scope.py @@ -1,10 +1,12 @@ +import asyncio +import json import os import uuid from collections.abc import AsyncIterator from dataclasses import dataclass from datetime import datetime, timedelta, timezone from hashlib import sha256 -from typing import Final +from typing import Final, Literal from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit import psycopg @@ -18,7 +20,13 @@ from psycopg.types.json import Jsonb from pydantic import TypeAdapter from litellm.proxy.auth.authorization import AllRows, OwnedRows, ReadScope -from litellm.proxy.spend_tracking.spend_management_endpoints import _spend_log_payload_query, read_scope_sql +from litellm.proxy.db.prisma_client import PrismaWrapper +from litellm.proxy.spend_tracking.spend_management_endpoints import ( + _spend_log_payload_query, + _SpendLogWindow, + read_scope_sql, +) +from litellm.proxy.utils import query_raw_within_statement_timeout @dataclass(frozen=True, slots=True) @@ -146,17 +154,121 @@ async def test_ownership_sql_selects_allowed_rows_and_intersects_filters( @pytest.mark.asyncio @pytest.mark.parametrize( - ("scope", "expected"), - [(AllRows(), ("foreign",)), (OwnedRows("caller"), ("own",)), (OwnedRows(None), ())], + ("id_column", "scope", "expected"), + [ + ("request_id", AllRows(), ("foreign",)), + ("request_id", OwnedRows("caller"), ()), + ("litellm_call_id", AllRows(), ("own",)), + ("litellm_call_id", OwnedRows("caller"), ("own",)), + ("litellm_call_id", OwnedRows(None), ()), + ], ) -async def test_payload_sql_filters_foreign_collisions_and_prefers_exact_ids_for_admins( - spend_database: Prisma, scope: ReadScope, expected: tuple[str, ...] +async def test_payload_sql_filters_foreign_collisions( + spend_database: Prisma, + id_column: Literal["request_id", "litellm_call_id"], + scope: ReadScope, + expected: tuple[str, ...], ) -> None: - query, params = _spend_log_payload_query("foreign", scope) + query, params = _spend_log_payload_query(id_column, "foreign", scope, _SpendLogWindow()) result: Final = await spend_database.query_raw(query, *params) assert tuple(row.request_id for row in REQUEST_IDS.validate_python(result)) == expected +SEEDED_AT: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope", [AllRows(), OwnedRows("caller", ("first",))]) +@pytest.mark.parametrize( + ("window", "expected"), + [ + (_SpendLogWindow(start=SEEDED_AT - timedelta(seconds=1)), ("own",)), + (_SpendLogWindow(start=SEEDED_AT, end=SEEDED_AT), ("own",)), + (_SpendLogWindow(start=SEEDED_AT + timedelta(seconds=1)), ()), + (_SpendLogWindow(end=SEEDED_AT - timedelta(seconds=1)), ()), + ], +) +async def test_payload_sql_by_call_id_keeps_to_the_window( + spend_database: Prisma, scope: ReadScope, window: _SpendLogWindow, expected: tuple[str, ...] +) -> None: + query, params = _spend_log_payload_query("litellm_call_id", "foreign", scope, window) + result: Final = await spend_database.query_raw(query, *params) + assert tuple(row.request_id for row in REQUEST_IDS.validate_python(result)) == expected + + +async def _drop_call_id_indexes(spend_database: Prisma) -> None: + await spend_database.execute_raw( + """ + DO $$ + DECLARE index_name text; + BEGIN + FOR index_name IN + SELECT indexname FROM pg_indexes + WHERE schemaname = current_schema() + AND tablename = 'LiteLLM_SpendLogs' + AND indexdef LIKE '%litellm_call_id%' + LOOP + EXECUTE format('DROP INDEX %I', index_name); + END LOOP; + END $$ + """ + ) + + +async def _plan_without_seq_scans(spend_database: Prisma, query: str, params: tuple[object, ...]) -> str: + async with spend_database.tx() as transaction: + await transaction.execute_raw("SET LOCAL enable_seqscan = off") + return json.dumps(await transaction.query_raw(f"EXPLAIN (FORMAT JSON) {query}", *params)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope", [AllRows(), OwnedRows("caller", ("first",))]) +@pytest.mark.parametrize( + ("id_column", "lookup_id", "window"), + [ + ("request_id", "own", _SpendLogWindow()), + ("litellm_call_id", "foreign", _SpendLogWindow(start=SEEDED_AT)), + ], +) +async def test_payload_sql_reads_through_an_index_without_the_call_id_index( + spend_database: Prisma, + scope: ReadScope, + id_column: Literal["request_id", "litellm_call_id"], + lookup_id: str, + window: _SpendLogWindow, +) -> None: + await _drop_call_id_indexes(spend_database) + query, params = _spend_log_payload_query(id_column, lookup_id, scope, window) + plan: Final = await _plan_without_seq_scans(spend_database, query, params) + assert "Seq Scan" not in plan, plan + assert "Index Cond" in plan, plan + + +@dataclass(frozen=True, slots=True) +class _ReadTarget: + read_db: PrismaWrapper + + +@pytest.mark.asyncio +async def test_statement_timeout_read_waits_for_a_busy_connection() -> None: + parsed: Final = urlsplit(os.environ["DATABASE_URL"]) + one_connection_url: Final = urlunsplit( + parsed._replace(query=urlencode({**dict(parse_qsl(parsed.query)), "connection_limit": 1, "pool_timeout": 60})) + ) + database: Final = Prisma(datasource={"url": one_connection_url}) + await database.connect() + try: + holder: Final = asyncio.create_task(database.query_raw("SELECT pg_sleep(3)::text AS slept")) + await asyncio.sleep(0.5) + rows: Final = await query_raw_within_statement_timeout( + _ReadTarget(PrismaWrapper(original_prisma=database, iam_token_db_auth=False)), "SELECT 1 AS ready" + ) + await holder + finally: + await database.disconnect() + assert rows == [{"ready": 1}] + + def _delete_session(session_id: str) -> None: with psycopg.connect(os.environ["DATABASE_URL"]) as connection: connection.execute('DELETE FROM "LiteLLM_SpendLogs" WHERE session_id = %s', (session_id,)) diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 27a5734eb45..4a418d87f22 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -6,10 +6,11 @@ import hashlib import json import re import sqlite3 -from collections.abc import AsyncIterator, Mapping, Sequence +from collections.abc import AsyncIterator, Awaitable, Mapping, Sequence from datetime import timezone +from functools import partial from types import SimpleNamespace -from typing import Final, Literal +from typing import Final, Literal, Protocol, TypeAlias from unittest.mock import AsyncMock, MagicMock, patch import prisma @@ -469,13 +470,15 @@ def test_can_user_view_spend_log_false_for_other_roles(): def _emulate_spend_log_owner_lookup(rows, sql_query, params): """Emulate the ownership lookup SQL over an in-memory spend-log corpus, + matching only the id columns its WHERE clause compares to the lookup id and honoring DISTINCT and any literal LIMIT the query carries so a capped or non-distinct query produces the truncated result it would in Postgres.""" lookup_id = params[0] + id_columns = [column for column in ("request_id", "litellm_call_id") if f"{column} = $1" in sql_query] matches = [ {"user": row.get("user"), "team_id": row.get("team_id")} for row in rows - if lookup_id in (row.get("request_id"), row.get("litellm_call_id")) + if any(row.get(column) == lookup_id for column in id_columns) ] if "DISTINCT" in sql_query: deduped = [] @@ -2808,15 +2811,17 @@ async def test_ui_view_request_response_custom_logger_is_keyed_by_callers_own_re that id as its request_id. The custom logger is asked for the caller's own stored request_id, so the caller gets their payload rather than a 403 from the foreign payload's owner check, and the foreign payload is never fetched.""" - prisma = MagicMock( - db=MagicMock( - query_raw=AsyncMock( - side_effect=[ - [{"user": "other_user", "team_id": None}, {"user": "caller_user", "team_id": None}], - [_payload_row("caller-req", "shared-id", "caller_user", "caller prompt")], - ] - ) - ) + prisma = MagicMock() + statement_timeout_reads( + prisma, + AsyncMock( + side_effect=[ + [{"user": "other_user", "team_id": None}], + [{"user": "caller_user", "team_id": None}], + [], + [_payload_row("caller-req", "shared-id", "caller_user", "caller prompt")], + ] + ), ) cold_storage = { "shared-id": { @@ -6649,7 +6654,8 @@ def test_ui_view_request_response_reads_from_cold_storage(client, monkeypatch): async def _query_raw(_sql, *_args): return [placeholder_row] - fake_prisma = SimpleNamespace(db=SimpleNamespace(query_raw=_query_raw)) + fake_db = SimpleNamespace(query_raw=_query_raw) + fake_prisma = SimpleNamespace(db=fake_db, read_db=ReadTargetOf(fake_db)) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", fake_prisma) cold_logger = _FakeColdStorageLogger( @@ -7736,6 +7742,8 @@ def _fake_prisma_with_owned_spend_log(owner_user_id, messages_json, response_jso ] class _Prisma: + read_db = property(lambda self: ReadTargetOf(self.db)) + def __init__(self): self.db = _DB() @@ -7842,7 +7850,8 @@ def test_ui_view_request_response_internal_user_missing_row_forbidden(client, mo from types import SimpleNamespace - fake_prisma = SimpleNamespace(db=_DB()) + fake_db = _DB() + fake_prisma = SimpleNamespace(db=fake_db, read_db=ReadTargetOf(fake_db)) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", fake_prisma) custom_logger = _RecordingAdditionalLoggingUtils( @@ -7871,6 +7880,168 @@ def test_ui_view_request_response_internal_user_missing_row_forbidden(client, mo app.dependency_overrides.pop(ps.user_api_key_auth, None) +_FakeSpendLogRows: TypeAlias = Sequence[Mapping[str, object]] + + +class _FakeRawQuery(Protocol): + def __call__(self, statement_timeout_set: bool, sql_query: str, /, *params: object) -> Awaitable[_FakeSpendLogRows]: ... + + +def _started_at_or_after(row: Mapping[str, object], start: object) -> bool: + started: Final = row.get("startTime") + return ( + isinstance(started, datetime.datetime) + and isinstance(start, str) + and started >= datetime.datetime.fromisoformat(start) + ) + + +class _SpendLogsWithoutCallIdIndex: + """A spend-log table missing LiteLLM_SpendLogs_litellm_call_id_idx. A read filtering on + litellm_call_id scans the whole table, so it fails the test unless it runs under a + statement timeout, which Postgres cancels it at. One that also bounds "startTime" from + below reads only the rows of that window, through the startTime index.""" + + def __init__(self, rows: _FakeSpendLogRows) -> None: + self.rows: Final = rows + self.db: Final = SimpleNamespace(query_raw=partial(self._query_raw, False)) + + @property + def read_db(self) -> SimpleNamespace: + return SimpleNamespace(tx=lambda **_kwargs: _StatementTimeoutTransaction(self._query_raw)) + + async def _query_raw(self, statement_timeout_set: bool, sql_query: str, *params: object) -> _FakeSpendLogRows: + where: Final = sql_query.split("WHERE", 1)[1] + by_call_id: Final = "litellm_call_id" in where + if by_call_id and not statement_timeout_set: + raise AssertionError("unbounded full scan of LiteLLM_SpendLogs") + if by_call_id and '"startTime"' not in where: + raise statement_timeout_error() + matches: Final = ( + [row for row in self.rows if row["litellm_call_id"] == params[0] and _started_at_or_after(row, params[1])] + if by_call_id + else [row for row in self.rows if row["request_id"] == params[0]] + ) + if 'SELECT DISTINCT "user", team_id' in sql_query: + return [{"user": row["user"], "team_id": row["team_id"]} for row in matches] + return matches + + +class _StatementTimeoutTransaction: + def __init__(self, query_raw: _FakeRawQuery) -> None: + self._query_raw: Final = query_raw + self._statement_timeout_set = False + + async def execute_raw(self, sql: str) -> int: + self._statement_timeout_set = self._statement_timeout_set or sql.startswith("SET LOCAL statement_timeout") + return 0 + + async def query_raw(self, sql_query: str, *params: object) -> _FakeSpendLogRows: + return await self._query_raw(self._statement_timeout_set, sql_query, *params) + + async def __aenter__(self) -> "_StatementTimeoutTransaction": + return self + + async def __aexit__(self, *exc_info: object) -> None: + pass + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER]) +def test_ui_view_request_response_opens_request_id_without_call_id_index(client, monkeypatch, user_role): + """ + Regression: the Logs drawer opens a row by its request_id. Matching litellm_call_id in + the same read made Postgres scan the whole table when its index was missing, which on a + large table ran until the load balancer gave up, so the row must resolve by request_id. + """ + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + _SpendLogsWithoutCallIdIndex([_payload_row("req-1", "call-1", "user_a", "drawer prompt")]), + ) + monkeypatch.setattr( + litellm.logging_callback_manager, "get_active_additional_logging_utils_from_custom_logger", lambda: [] + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=user_role, user_id="user_a") + try: + response = client.get("/spend/logs/ui/req-1", headers={"Authorization": "Bearer sk-test"}) + assert response.status_code == 200, response.text + assert response.json()["messages"] == [{"role": "user", "content": "drawer prompt"}] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER]) +def test_ui_view_request_response_call_id_scan_times_out_with_503(client, monkeypatch, user_role): + """An id only a litellm_call_id carries still needs the scan, which Postgres cancels at + the read statement timeout, so the caller gets a 503 naming the index, not a hang.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + _SpendLogsWithoutCallIdIndex([_payload_row("req-1", "call-1", "user_a", "drawer prompt")]), + ) + monkeypatch.setattr( + litellm.logging_callback_manager, "get_active_additional_logging_utils_from_custom_logger", lambda: [] + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=user_role, user_id="user_a") + try: + response = client.get("/spend/logs/ui/call-1", headers={"Authorization": "Bearer sk-test"}) + assert response.status_code == 503, response.text + assert "LiteLLM_SpendLogs_litellm_call_id_idx" in response.json()["detail"]["error"] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.parametrize( + ("user_role", "start_date", "expected_status", "expected_messages"), + [ + (LitellmUserRoles.PROXY_ADMIN, "2026-11-09 11:59:59", 200, [{"role": "user", "content": "drawer prompt"}]), + (LitellmUserRoles.INTERNAL_USER, "2026-11-09 11:59:59", 200, [{"role": "user", "content": "drawer prompt"}]), + (LitellmUserRoles.PROXY_ADMIN, "2026-11-09 12:00:01", 200, None), + (LitellmUserRoles.INTERNAL_USER, "2026-11-09 12:00:01", 403, None), + ], +) +def test_ui_view_request_response_reads_call_id_within_start_date_without_call_id_index( + client, monkeypatch, user_role, start_date, expected_status, expected_messages +): + """ + Regression: the drawer passes start_date, so a call ID lookup reads only the rows from + then on instead of scanning the whole table, and a row from before it is not found. + """ + row: Final = { + **_payload_row("req-1", "call-1", "user_a", "drawer prompt"), + "startTime": datetime.datetime(2026, 11, 9, 12, tzinfo=timezone.utc), + } + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _SpendLogsWithoutCallIdIndex([row])) + monkeypatch.setattr( + litellm.logging_callback_manager, "get_active_additional_logging_utils_from_custom_logger", lambda: [] + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=user_role, user_id="user_a") + try: + response = client.get( + "/spend/logs/ui/call-1", params={"start_date": start_date}, headers={"Authorization": "Bearer sk-test"} + ) + assert response.status_code == expected_status, response.text + if expected_status == 200: + assert (response.json() or {}).get("messages") == expected_messages, response.text + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_ui_view_request_response_rejects_malformed_start_date_with_400(client, monkeypatch): + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + _SpendLogsWithoutCallIdIndex([_payload_row("req-1", "call-1", "user_a", "drawer prompt")]), + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="user_a" + ) + try: + response = client.get( + "/spend/logs/ui/req-1", params={"start_date": "2026-11-09"}, headers={"Authorization": "Bearer sk-test"} + ) + assert response.status_code == 400, response.text + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + @pytest.mark.parametrize( ("parent_status", "child_status", "expected"), [("failure", "success", "failure"), ("success", "failure", "success")], From 20be20bf4571facb64f64576c5f6dcb0c0b9319c Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 09:09:09 -0800 Subject: [PATCH 009/189] fix(cost-map): give the Gemini 4 Carbon rows the shared Gemini 3 search free tier (#50951) Gemini API and Vertex both include 5,000 grounding requests a month shared across Gemini 3 and newer, so gemini/gemini-4-carbon, vertex_ai/gemini-4-carbon and vertex_ai/gemini-4-carbon-cyber-permissive now carry the same search_free_requests allowance as the Gemini 3.x grounding rows in both catalog copies. Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 3 +++ model_prices_and_context_window.json | 3 +++ .../test_model_prices_and_context_window.py | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6046e3f863f..959d2e46afe 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -88269,6 +88269,7 @@ "prompt_cache_min_tokens": 4096.0, "rpm": 2000.0, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://aistudio.google.com/docs/pricing#gemini-4-carbon", "supported_endpoints": [ "/v1/chat/completions", @@ -88326,6 +88327,7 @@ "prompt_cache_min_tokens": 4096.0, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -88382,6 +88384,7 @@ "prompt_cache_min_tokens": 4096.0, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6046e3f863f..959d2e46afe 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -88269,6 +88269,7 @@ "prompt_cache_min_tokens": 4096.0, "rpm": 2000.0, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://aistudio.google.com/docs/pricing#gemini-4-carbon", "supported_endpoints": [ "/v1/chat/completions", @@ -88326,6 +88327,7 @@ "prompt_cache_min_tokens": 4096.0, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -88382,6 +88384,7 @@ "prompt_cache_min_tokens": 4096.0, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index e195b970fd5..62c1fab244c 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -958,6 +958,27 @@ def test_gemini_4_argon_google_rows_carry_supports_continuation_in_both_cost_map assert _supports_continuation_rows(BACKUP_PRICES_PATH) == main_rows +def test_gemini_4_carbon_rows_share_the_gemini_3_search_allowance_in_both_cost_map_copies() -> None: + main_rows: Final = _search_free_requests_rows(PRICES_PATH) + # 5,000 free grounding requests per month shared across Gemini 3 and newer: + # https://ai.google.dev/gemini-api/docs/pricing#gemini-4-carbon and + # https://cloud.google.com/vertex-ai/generative-ai/pricing (2026-11-10) + carbon_rows: Final = { + "gemini/gemini-4-carbon", + "vertex_ai/gemini-4-carbon", + "vertex_ai/gemini-4-carbon-cyber-permissive", + } + assert carbon_rows <= main_rows.keys(), sorted(main_rows) + gemini_3_allowances: Final = { + json.dumps(main_rows[key]["search_free_requests"], sort_keys=True) + for key in ("gemini/gemini-3.8-flash", "vertex_ai/gemini-3.8-flash") + } + assert { + json.dumps(main_rows[key]["search_free_requests"], sort_keys=True) for key in carbon_rows + } == gemini_3_allowances + assert _search_free_requests_rows(BACKUP_PRICES_PATH) == main_rows + + @pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) def test_search_free_requests_rows_are_grounding_rows_with_one_allowance_per_group(path: Path) -> None: rows = _search_free_requests_rows(path) From 97b4de224a115af62e258ca1b0699301f75bb2f8 Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:09:18 -0800 Subject: [PATCH 010/189] perf(streaming): reuse the upstream choice for plain litellm HTTP-handler stream chunks (#50786) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../litellm_core_utils/streaming_handler.py | 48 +++++--- .../test_streaming_handler.py | 115 ++++++++++++++++++ 2 files changed, 145 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index f7bd86f3a0c..361e576f3e1 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -82,6 +82,8 @@ FUNCTION_CALL_ATTRIBUTE: Final = "function_call" _SYNC_ITER_EXHAUSTED: Final = object() +_STREAMING_CHOICE_KEYS: Final = frozenset({"finish_reason", "index", "delta", "logprobs", "message"}) + _GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__) _CompleteStreamingResponse = ModelResponse | TextCompletionResponse | ModelResponseStream | None _USAGE_COST_HEADER_PROVIDERS: Final[frozenset[str]] = frozenset( @@ -139,11 +141,18 @@ def _entry_len(entries: Mapping[str, object], key: str) -> int: def _prepend_upstream_choice_extras( - choice: StreamingChoices, upstream_choice: ChunkChoice, extra_keys: frozenset[str] + choice: StreamingChoices, upstream_choice: ChunkChoice | StreamingChoices, extra_keys: frozenset[str] ) -> None: if not extra_keys: return - upstream_extras: Final[Mapping[str, object]] = upstream_choice.model_dump(include=set(extra_keys)) + stored_extras: Final = { + key: value for key, value in (upstream_choice.__pydantic_extra__ or {}).items() if key in extra_keys + } + upstream_extras: Final[Mapping[str, object]] = ( + stored_extras + if _holds_only_scalars(stored_extras.items()) + else upstream_choice.model_dump(include=set(extra_keys)) + ) choice.__pydantic_extra__ = {**upstream_extras, **(choice.__pydantic_extra__ or {})} # rebind-ok: reuse choice choice.__pydantic_fields_set__.update(upstream_extras) @@ -439,7 +448,7 @@ class CustomStreamWrapper: self.completion_stream = completion_stream self.sent_first_chunk = False self.sent_last_chunk = False - self._openai_handler_receives_chat_chunks = False + self._openai_handler_receives_chunk_objects = False self._failure_logging: asyncio.Task[None] | None = None self._closed = False self._stream_created_time: float = time.time() @@ -1162,13 +1171,13 @@ class CustomStreamWrapper: def _prebuilt_plain_delta(self, chunk: object) -> Delta | None: if ( - not self._openai_handler_receives_chat_chunks - or not isinstance(chunk, ChatCompletionChunk) + not self._openai_handler_receives_chunk_objects + or not isinstance(chunk, (ChatCompletionChunk, ModelResponseStream)) or not chunk.choices ): return None upstream_delta: Final = cast( # cast-ok: [LIT006] runtime delta may be absent - ChoiceDelta | None, + ChoiceDelta | Delta | None, chunk.choices[0].delta, ) if ( @@ -1296,22 +1305,23 @@ class CustomStreamWrapper: upstream_choice: Final = ( original_chunk_for_reuse.choices[0] if plain_delta_created - and isinstance(original_chunk_for_reuse, ChatCompletionChunk) + and isinstance(original_chunk_for_reuse, (ChatCompletionChunk, ModelResponseStream)) and len(original_chunk_for_reuse.choices) == 1 else None ) upstream_delta: Final = cast( # cast-ok: [LIT006] runtime delta may be absent - ChoiceDelta | None, + ChoiceDelta | Delta | None, getattr(upstream_choice, "delta", None), ) upstream_choice_extra_keys: Final = frozenset( (upstream_choice.__pydantic_extra__ or {}) if upstream_choice is not None else () ) + upstream_logprobs: Final[object] = getattr(upstream_choice, "logprobs", None) can_reuse_choice: Final = ( upstream_choice is not None and upstream_delta is not None and "enhancements" not in upstream_choice_extra_keys - and upstream_choice.logprobs is None + and upstream_logprobs is None and _holds_only_scalars(upstream_delta) ) if can_reuse_choice and upstream_choice is not None: @@ -1320,7 +1330,7 @@ class CustomStreamWrapper: _prepend_upstream_choice_extras( model_response.choices[0], upstream_choice, - upstream_choice_extra_keys - {"message"}, + upstream_choice_extra_keys - _STREAMING_CHOICE_KEYS, ) else: choices: Final = [] @@ -1737,14 +1747,16 @@ class CustomStreamWrapper: self._record_finish_choice_fields(cached_chunk) else: # openai / azure chat model self.uses_openai_chat_stream = True - if isinstance(chunk, ChatCompletionChunk): - self._openai_handler_receives_chat_chunks = True - if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()): - raise litellm.InternalServerError( - message="Upstream generation aborted (finish_reason=abort)", - model=chunk.model, - llm_provider=self.custom_llm_provider, - ) + if isinstance(chunk, (ChatCompletionChunk, ModelResponseStream)): + self._openai_handler_receives_chunk_objects = True + if isinstance(chunk, ChatCompletionChunk) and any( + str(choice.finish_reason) == "abort" for choice in chunk.choices or () + ): + raise litellm.InternalServerError( + message="Upstream generation aborted (finish_reason=abort)", + model=chunk.model, + llm_provider=self.custom_llm_provider, + ) if self.custom_llm_provider in [ LlmProviders.AZURE.value, LlmProviders.AZURE_AI.value, diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index df50ae9dc44..3b798b35096 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -7427,6 +7427,121 @@ def test_azure_chunk_creator_streams_choice_content_filter_results_on_later_chun assert ["content_filter_results" in output.choices[0].model_fields_set for output in outputs] == [True] +def _parsed_openai_like_chunk( + chunk_id: str, delta: Mapping[str, object], finish_reason: str | None = None, **choice_extras: object +) -> ModelResponseStream: + parser: Final = OpenAIChatCompletionStreamingHandler(streaming_response=None, sync_stream=True) + return parser.chunk_parser( + { + "id": chunk_id, + "object": "chat.completion.chunk", + "created": 1, + "model": "served-model", + "choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish_reason, **choice_extras}], + } + ) + + +def _parsed_stream_wrapper(custom_llm_provider: str, logging_obj: Logging) -> CustomStreamWrapper: + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="served-model", + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + wrapper.chunk_creator(chunk=_parsed_openai_like_chunk("chunk-0", {"role": "assistant", "content": ""})) + return wrapper + + +@pytest.mark.parametrize("custom_llm_provider", ["hosted_vllm", "deepseek", "groq"]) +def test_parsed_openai_like_chunk_creator_builds_one_delta_and_choice_and_no_dump_per_later_chunk( + logging_obj: Logging, custom_llm_provider: str +) -> None: + wrapper: Final = _parsed_stream_wrapper(custom_llm_provider, logging_obj) + later_chunks: Final = ( + _parsed_openai_like_chunk("chunk-1", {"content": "hello"}, stop_reason=None), + _parsed_openai_like_chunk("chunk-2", {"reasoning": "thought"}, stop_reason=None), + _parsed_openai_like_chunk("chunk-3", {"content": " world", "reasoning_content": None}, logprobs=None), + ) + delta_init: Final = Delta.__init__.__code__ + streaming_choices_init: Final = StreamingChoices.__init__.__code__ + model_dump: Final = BaseModel.model_dump.__code__ + + outputs, calls = _chunk_creator_calls( + wrapper, later_chunks, frozenset({delta_init, streaming_choices_init, model_dump}) + ) + + assert sum(call.code is delta_init for call in calls) == len(later_chunks) + assert sum(call.code is streaming_choices_init for call in calls) == len(later_chunks) + assert sum(call.code is model_dump for call in calls) == 0 + assert [_wire_dump(output)["choices"] for output in outputs if output is not None] == [ + [{"index": 0, "delta": {"content": "hello"}}], + [{"index": 0, "delta": {"reasoning_content": "thought"}}], + [{"index": 0, "delta": {"content": " world"}}], + ] + + +def test_parsed_openai_like_chunk_creator_keeps_choice_extras_ahead_of_choice_keys(logging_obj: Logging) -> None: + wrapper: Final = _parsed_stream_wrapper("hosted_vllm", logging_obj) + later_chunks: Final = ( + _parsed_openai_like_chunk("chunk-1", {"content": "hi"}, stop_reason=None, token_ids=[7, 8]), + _parsed_openai_like_chunk("chunk-2", {"content": "!"}, stop_reason=None, matched_stop=2), + ) + + outputs: Final = tuple(wrapper.chunk_creator(chunk=chunk) for chunk in later_chunks) + + choice_dumps: Final = [output.choices[0].model_dump() for output in outputs if output is not None] + assert [list(choice_dump) for choice_dump in choice_dumps] == [ + ["stop_reason", "token_ids", "finish_reason", "index", "delta", "logprobs"], + ["stop_reason", "matched_stop", "finish_reason", "index", "delta", "logprobs"], + ] + assert [choice_dump["token_ids"] for choice_dump in choice_dumps[:1]] == [[7, 8]] + assert [choice_dump["matched_stop"] for choice_dump in choice_dumps[1:]] == [2] + assert [choice_dump["delta"]["content"] for choice_dump in choice_dumps] == ["hi", "!"] + + +def test_parsed_openai_like_stream_still_emits_tool_calls_and_finish_after_reused_chunks( + logging_obj: Logging, +) -> None: + wrapper: Final = _parsed_stream_wrapper("deepseek", logging_obj) + later_chunks: Final = ( + _parsed_openai_like_chunk("chunk-1", {"content": "checking"}), + _parsed_openai_like_chunk( + "chunk-2", + { + "tool_calls": [ + {"index": 0, "id": "call_1", "type": "function", "function": {"name": "f", "arguments": ""}} + ] + }, + ), + _parsed_openai_like_chunk("chunk-3", {"tool_calls": [{"index": 0, "function": {"arguments": "{}"}}]}), + _parsed_openai_like_chunk("chunk-4", {}, finish_reason="tool_calls"), + ) + + outputs: Final = tuple(wrapper.chunk_creator(chunk=chunk) for chunk in later_chunks) + + assert [_wire_dump(output)["choices"] for output in outputs if output is not None] == [ + [{"index": 0, "delta": {"content": "checking"}}], + [ + { + "index": 0, + "delta": { + "tool_calls": [ + {"id": "call_1", "function": {"arguments": "", "name": "f"}, "type": "function", "index": 0} + ] + }, + } + ], + [ + { + "index": 0, + "delta": {"tool_calls": [{"function": {"arguments": "{}"}, "type": "function", "index": 0}]}, + } + ], + [{"finish_reason": "tool_calls", "index": 0, "delta": {}}], + ] + + @pytest.mark.parametrize( "special_delta", [ From 901f914ba262a3611a137f7520cc2b1ead9505e7 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:13:40 -0800 Subject: [PATCH 011/189] fix(responses): replay Claude tool search on follow-up turns (#50821) * fix(responses): replay Claude tool search on follow-up turns Claude tool search on /v1/responses now returns a tool_search_output after each tool_search_call, streamed and not, listing the tools the search loaded as the caller sent them A stateless follow-up sends the search back to Anthropic as the server_tool_use and tool_search_tool_result Claude produced, and the Anthropic request drops a replayed search whose loaded tools are no longer sent, so a changed tool list no longer answers 400 The guardrail write-back for SDK output items reads part text through a typed helper, so the new output item type adds no attribute errors * fix(responses): emit Claude's tool search output in stream order The streamed tool_search_output item was announced at the end of the stream with an output index reserved earlier, so output_item.added events arrived as 0, 2, 1 and the OpenAI SDK's stream helper raised IndexError. The item is now emitted when Claude's search result arrives. * fix(anthropic): keep the caller's tool names in tool search results A tool name Anthropic cannot take gets rewritten on the request, so Claude's tool_search_tool_result named the rewritten tool. The response now maps those tool references back to the caller's names, streamed and not, and a replayed search result maps them forward again on the next turn. * fix(responses): replay Claude's tool search only on routes that run it A follow-up sent to a route that cannot run tool search, a Claude model without it included, now leaves out the replayed search the way it did before, instead of sending Claude search blocks that model cannot take. The stateless gate and the stored-turn filter now use the same check that already decides whether the tool_search tool itself is kept A streamed tool_search_call now gets its output_item.done when Claude's search result arrives, so it finishes before its tool_search_output. Clients that build history from done events, like Codex, kept the output ahead of its call * fix(responses): keep a replayed search when the caller sent Anthropic's own search tool A Claude model the cost map does not flag for tool search still runs it when the request carries Anthropic's own tool_search_tool_regex or bm25 tool, which is how turn 1 already decides to keep it. Both replay gates now follow that rule, so a stored follow-up on such a model keeps Claude's search as it did before, and a stateless one replays it * fix(responses): replay a BM25 tool search under its own name * fix(responses): keep a replayed tool search visible to guardrails and prune only missing tools A replayed Claude tool search now reaches pre-call guardrails and the Purview DLP scan with its search arguments, a stale search drops only the tool references the follow-up no longer sends, and the stored previous_response_id path keeps the base gate for routes the cost map does not flag * fix(responses): replay Claude's search on a stateless follow-up only when it still sends a search tool A store false follow-up that sent no search tool still got the search replayed, and Claude then said it called a tool it no longer had * fix(responses): replay a stored Claude search only when the follow-up still sends a search tool A previous_response_id follow-up that sent no search tool still got the stored search, the same gap a store false follow-up had. The stale search pruning now logs the tools it leaves out at debug level. handler.py reads the reported speed from the usage chunk, and the guardrail write-back skips tool search outputs, so the branch adds no type errors. * fix(responses): leave out a stored Claude search kept in server_tool_calls Main stores Claude's server-side tool search on the assistant message in provider_specific_fields.server_tool_calls and tool_results, not in tool_calls. The previous_response_id path only looked at tool_calls, so a follow-up that sent no search tool still replayed the stored search. The session handler now finds the search in either place and drops it from both server_tool_calls and tool_results when the follow-up cannot run a search. * test(responses): give the replayed search guardrail fixture its own call id The guardrail test reused toolu_01Weather, an id that streaming and bridge tests already put in the process-wide TOOL_CALLS_CACHE. When one of those ran first in the same worker, the guardrail write-back took its rebuild path and the test failed for a reason unrelated to the replayed search. * fix(anthropic): keep Claude's replayed tool search on renamed tools and stored follow-ups A search-only assistant turn now gets its search result renamed to the sanitized tool names too, so the stale-tool trim no longer drops it. Stored follow-ups on Claude routes replay the search as base did, trimmed to the tools still sent, and Bedrock invoke sends the tool search beta whenever the history holds a search * test(responses): pin that guardrails scan a replayed Claude search on a follow-up without a search tool * fix(anthropic): keep Anthropic's tool names in chat tool search results A chat history whose search result used the request's tool names (weather.get) got Anthropic's 400 when replayed on a proxy still on the old code, during a rolling deploy or a rollback. Chat completions now returns Claude's search result as main does, and the Responses bridge maps a rewritten name back to the request's tool when it builds tool_search_output. That also fills tool_search_output on Bedrock invoke streams, which carry no name map --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../mid_conversation_system.py | 12 + litellm/llms/anthropic/chat/handler.py | 5 +- litellm/llms/anthropic/chat/transformation.py | 46 ++- litellm/llms/anthropic/common_utils.py | 110 +++++ .../anthropic_claude_transformation.py | 7 +- .../guardrail_translation/handler.py | 64 ++- litellm/proxy/_lazy_openapi_snapshot.json | 62 +++ .../microsoft_purview/purview_dlp.py | 1 + .../mcp_server/openapi_to_mcp_generator.py | 2 +- .../custom_tools.py | 2 +- .../session_handler.py | 2 + .../streaming_iterator.py | 55 ++- .../tool_search.py | 16 + .../tool_search_results.py | 299 ++++++++++++++ .../transformation.py | 139 +++++-- litellm/types/llms/anthropic.py | 31 ++ litellm/types/llms/openai.py | 2 + litellm/types/responses/main.py | 11 + ...nthropic_tool_search_result_matrix_wire.py | 56 ++- .../test_responses_bridge_tool_search_wire.py | 174 ++++++-- .../test_anthropic_chat_transformation.py | 269 ++++++++++++- .../anthropic/test_anthropic_common_utils.py | 80 +++- ...mations_anthropic_claude_transformation.py | 86 ++++ ...test_openai_responses_guardrail_handler.py | 125 ++++++ .../guardrail_hooks/test_microsoft_purview.py | 40 ++ .../test_anthropic_responses_bridge.py | 380 +++++++++++++++++- .../test_litellm_completion_responses.py | 268 +++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 33 +- 28 files changed, 2254 insertions(+), 123 deletions(-) create mode 100644 litellm/responses/litellm_completion_transformation/tool_search_results.py diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index ec6de4e206d..4a0fd6e26f0 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -35,11 +35,14 @@ from operator import itemgetter from types import MappingProxyType from typing import Final, Literal, TypeAlias, TypeVar +from pydantic import ValidationError + from litellm.types.llms.anthropic import ( ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, ANTHROPIC_TOOL_SEARCH_TOOL_NAMES, AnthropicMessagesSystemMessageParam, AnthropicSystemMessageContent, + AnthropicToolSearchToolResultBlock, ) from litellm.types.llms.openai import ( AllMessageValues, @@ -299,6 +302,15 @@ def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool: return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES +def anthropic_tool_search_result(block: object) -> AnthropicToolSearchToolResultBlock | None: + if message_field(block, "type") != "tool_search_tool_result": + return None + try: + return AnthropicToolSearchToolResultBlock.model_validate(block) + except ValidationError: + return None + + def find_anthropic_server_tool_result( tool_id: str, web_search_results: Sequence[object] | None, diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 66c560013fb..00047e52f15 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -812,8 +812,9 @@ class ModelResponseIterator: reasoning_content=reasoning_content, speed=self.speed, ) - if usage.speed is not None: - self.speed = usage.speed + reported_speed: Final = self._cumulative_usage.get("speed") + if isinstance(reported_speed, str): + self.speed = reported_speed return usage def _content_block_delta_helper( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 48bccc7c001..c777f1f6ac9 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -31,6 +31,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.asyncify import is_event_loop_running from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, coerce_token_limit, drops_unsupported_params, map_finish_reason, @@ -151,8 +152,10 @@ from ..common_utils import ( requires_native_compaction_beta, stop_reason_provider_specific_fields, strip_advisor_blocks_from_messages, + tool_search_result_renamed, tools_marked_for_a_tool_config_point, validated_object_mapping, + without_stale_tool_search_results, ) if TYPE_CHECKING: @@ -411,6 +414,17 @@ class _AnthropicToolResultBlock(TypedDict, total=False): content: ReadOnly[object] +def _provider_fields_with_tool_names(message: object, forward_map: Mapping[str, str]) -> Mapping[str, object] | None: + provider_fields: Final = as_str_mapping(message_field(message, "provider_specific_fields")) + replayed_results: Final = parts_of(provider_fields.get("tool_results")) if provider_fields is not None else () + if provider_fields is None or not replayed_results: + return None + return { + **provider_fields, + "tool_results": [tool_search_result_renamed(result, forward_map) for result in replayed_results], + } + + _ENUM_TYPE_CHECKS: Final[Mapping[object, Callable[[object], bool]]] = MappingProxyType( { "null": lambda v: v is None, @@ -457,7 +471,7 @@ def _basic_sanitize_anthropic_tool_name(name: str) -> str: return _ANTHROPIC_TOOL_NAME_INVALID_CHARS.sub("_", name)[:_ANTHROPIC_TOOL_NAME_MAX_LEN] -def _build_anthropic_tool_name_maps( +def build_anthropic_tool_name_maps( original_names: list[str], ) -> tuple[dict[str, str], dict[str, str]]: """Build (forward, reverse) tool-name maps for a single request. @@ -1458,8 +1472,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): messages: list[AllMessageValues], name_forward_map: dict[str, str], ) -> list[AllMessageValues]: - """Return a copy of `messages` with tool_call/function_call names - rewritten using the per-request forward map. + """Return a copy of `messages` with tool_call/function_call names, and + the tools a replayed tool search found, rewritten using the per-request + forward map. Only mutates messages whose tool_call/function_call name is *in* the forward map. Names absent from the map (already valid, no collision) @@ -1475,7 +1490,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): continue tool_calls = msg.get("tool_calls") function_call = msg.get("function_call") - if not tool_calls and not function_call: + renamed_provider_fields = _provider_fields_with_tool_names(msg, name_forward_map) + if not tool_calls and not function_call and renamed_provider_fields is None: new_messages.append(msg) continue new_msg = dict(msg) @@ -1501,6 +1517,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): new_fc = dict(function_call) new_fc["name"] = name_forward_map[fc_name] new_msg["function_call"] = new_fc + if renamed_provider_fields is not None: + new_msg["provider_specific_fields"] = renamed_provider_fields new_messages.append(cast(AllMessageValues, new_msg)) return new_messages @@ -1517,7 +1535,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): callers that need to compute the maps from the raw OpenAI shape -- e.g. test setup or future pre-mapping consumers. - See _build_anthropic_tool_name_maps for the collision rules. Pulls + See build_anthropic_tool_name_maps for the collision rules. Pulls the original name out of either ``{"function": {"name": ...}}`` (legacy OpenAI shape) or ``{"name": ...}`` (rare top-level shape). """ @@ -1530,7 +1548,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): original = tool.get("name") if isinstance(original, str) and original: original_names.append(original) - return _build_anthropic_tool_name_maps(original_names) + return build_anthropic_tool_name_maps(original_names) @staticmethod def _sanitize_tool_names_in_request( @@ -1558,7 +1576,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # 1. Collect originals from the Anthropic-shaped custom-tool entries. # Order matters: the first occurrence wins the canonical slot; # later collisions get numeric suffixes (see - # ``_build_anthropic_tool_name_maps``). + # ``build_anthropic_tool_name_maps``). original_names: Final[list[str]] = [] for t in tools: if not isinstance(t, dict): @@ -1572,7 +1590,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not original_names: return {}, {} - forward, reverse = _build_anthropic_tool_name_maps(original_names) + forward, reverse = build_anthropic_tool_name_maps(original_names) if not forward: # Every name was already valid -- nothing to do. return forward, reverse @@ -2544,7 +2562,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # The reverse map only contains entries for names that were actually # rewritten -- so a tool legitimately named ``foo_bar`` is never # incorrectly retyped to ``foo/bar`` on the response side. - # See _build_anthropic_tool_name_maps for the collision-handling + # See build_anthropic_tool_name_maps for the collision-handling # rules and rationale. _name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request( optional_params=optional_params, @@ -2600,8 +2618,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. _all_tools: Final = optional_params.get("tools") or [] _has_advisor = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _all_tools) - if not _has_advisor: - anthropic_messages = strip_advisor_blocks_from_messages(anthropic_messages) + anthropic_messages = list( + without_stale_tool_search_results( + anthropic_messages if _has_advisor else strip_advisor_blocks_from_messages(anthropic_messages), + _TOOLS_ADAPTER.validate_python(_all_tools), + ) + ) ## Add code_execution tool if container_upload is in messages _tools: Final = ( @@ -3239,7 +3261,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Reverse-map rewritten tool names back to caller's originals so a # downstream OpenAI-style dispatcher can match on the registered name. - # See _build_anthropic_tool_name_maps for why this is keyed on the + # See build_anthropic_tool_name_maps for why this is keyed on the # per-request reverse map (so a tool legitimately named `foo_bar` is # never incorrectly retyped to `foo/bar`). No-op when the map is # empty (the common case). diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1c42dd15853..937afeae402 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -34,6 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( get_file_ids_from_messages, ) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( + anthropic_tool_search_result, find_anthropic_server_tool_result, message_field, parts_of, @@ -64,10 +65,12 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS, + AllAnthropicPassThroughMessageValues, AllAnthropicToolsValues, AnthropicMcpServerTool, AnthropicMessagesToolChoice, AnthropicThinkingParam, + AnthropicToolSearchToolResultBlock, ) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( @@ -1918,6 +1921,113 @@ def strip_advisor_blocks_from_messages(messages: list[_MessageT], replace_with_t return messages +def _assistant_blocks(message: AllAnthropicPassThroughMessageValues) -> tuple[object, ...]: + return parts_of(message["content"]) if message["role"] == "assistant" else () + + +def _results_with_unloaded_tools( + messages: Sequence[AllAnthropicPassThroughMessageValues], tool_names: frozenset[str] +) -> tuple[AnthropicToolSearchToolResultBlock, ...]: + blocks: Final = chain.from_iterable(map(_assistant_blocks, messages)) + results: Final = (result for result in map(anthropic_tool_search_result, blocks) if result is not None) + return tuple(result for result in results if not tool_names.issuperset(result.tool_names)) + + +def _is_stale_tool_search_block(block: object, stale_ids: frozenset[str]) -> bool: + match message_field(block, "type"): + case "server_tool_use": + return message_field(block, "id") in stale_ids + case "tool_search_tool_result": + return message_field(block, "tool_use_id") in stale_ids + case _: + return False + + +def _with_loaded_tool_references_only(block: object, tool_names: frozenset[str]) -> object: + result: Final = anthropic_tool_search_result(block) + fields: Final = as_str_mapping(block) + if result is None or fields is None or tool_names.issuperset(result.tool_names): + return block + content: Final = as_str_mapping(fields.get("content")) or MappingProxyType({}) + references: Final = parts_of(content.get("tool_references")) + loaded: Final = [reference for reference in references if message_field(reference, "tool_name") in tool_names] + return {**fields, "content": {**content, "tool_references": loaded}} + + +def _with_loaded_tool_searches_only( + message: AllAnthropicPassThroughMessageValues, stale_ids: frozenset[str], tool_names: frozenset[str] +) -> AllAnthropicPassThroughMessageValues | None: + if message["role"] != "assistant": + return message + content: Final = message["content"] + if isinstance(content, str): + return message + blocks: Final = list(content) + kept: Final = [ + _with_loaded_tool_references_only(block, tool_names) + for block in blocks + if not _is_stale_tool_search_block(block, stale_ids) + ] + if kept == blocks: + return message + return {**message, "content": kept} if kept else None + + +def _tool_reference_renamed(reference: object, name_map: Mapping[str, str]) -> object: + fields: Final = as_str_mapping(reference) + name: Final = fields.get("tool_name") if fields is not None else None + if fields is None or not isinstance(name, str) or name not in name_map: + return reference + return {**fields, "tool_name": name_map[name]} + + +def tool_search_content_renamed(content: object, name_map: Mapping[str, str]) -> object: + fields: Final = as_str_mapping(content) + if fields is None or "tool_references" not in fields: + return content + references: Final = parts_of(fields["tool_references"]) + return {**fields, "tool_references": [_tool_reference_renamed(reference, name_map) for reference in references]} + + +def tool_search_result_renamed(block: object, name_map: Mapping[str, str]) -> object: + fields: Final = as_str_mapping(block) + if fields is None or fields.get("type") != "tool_search_tool_result": + return block + return {**fields, "content": tool_search_content_renamed(fields.get("content"), name_map)} + + +def replays_tool_search(messages: object) -> bool: + contents: Final = (message_field(message, "content") for message in parts_of(messages)) + blocks: Final = chain.from_iterable(map(parts_of, contents)) + return any(anthropic_tool_search_result(block) is not None for block in blocks) + + +def without_stale_tool_search_results( + messages: Sequence[AllAnthropicPassThroughMessageValues], tools: Sequence[object] +) -> tuple[AllAnthropicPassThroughMessageValues, ...]: + """ + Leave out the tools a replayed tool search found that are no longer in ``tools``, + and the whole search, its ``server_tool_use`` and its ``tool_search_tool_result``, + when none of them are left. Anthropic rejects such a history with "Tool reference + '' not found in available tools", for example after a follow-up turn removes + a tool or sends none. + """ + tool_names: Final = frozenset(name for tool in tools if isinstance(name := message_field(tool, "name"), str)) + unloaded: Final = _results_with_unloaded_tools(messages, tool_names) + if not unloaded: + return tuple(messages) + litellm.verbose_logger.debug( + "Leaving out the tools a replayed tool search found that this request no longer sends: %s", + sorted(frozenset(chain.from_iterable(result.tool_names for result in unloaded)) - tool_names), + ) + stale_ids: Final = frozenset(result.tool_use_id for result in unloaded if tool_names.isdisjoint(result.tool_names)) + return tuple( + kept + for message in messages + if (kept := _with_loaded_tool_searches_only(message, stale_ids, tool_names)) is not None + ) + + def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool: """ Detect Anthropic 400 errors caused by invalid thinking blocks in replayed diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index 1c38642e062..6e0cdbcce08 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -23,6 +23,7 @@ from litellm.llms.anthropic.common_utils import ( OBJECT_MAPPING_ADAPTER, AnthropicModelInfo, carried_thinking_keys, + replays_tool_search, ) from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, @@ -265,6 +266,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): optional_params=optional_params, headers=headers, thinking=_anthropic_request.get("thinking"), + history_holds_tool_search=replays_tool_search( + OBJECT_MAPPING_ADAPTER.validate_python(_anthropic_request).get("messages") + ), ) if beta_list: _anthropic_request["anthropic_beta"] = beta_list @@ -397,9 +401,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): optional_params: dict, headers: dict, thinking: AnthropicThinkingParam | None, + history_holds_tool_search: bool, ) -> list[str]: tools: Final = optional_params.get("tools") - tool_search_used: Final = self.is_tool_search_used(tools) + tool_search_used: Final = self.is_tool_search_used(tools) or history_holds_tool_search programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools) input_examples_used: Final = self.is_input_examples_used(tools) is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index f81957401ff..5b3b8e88834 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -119,6 +119,7 @@ from litellm.types.llms.openai import ( from litellm.types.responses.main import ( GenericResponseOutputItem, OutputText, + OutputToolSearchOutput, ) from litellm.types.utils import GenericGuardrailAPIInputs @@ -276,6 +277,7 @@ _ITEM_ID_KEYED_TOOL_CALL_TYPES: Final = frozenset({_TOOL_SEARCH_CALL_TYPE, _WEB_ _OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE} _SCANNED_TOOL_SEARCH_NAME: Final = "tool_search" _TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_RAW_ITEMS_ADAPTER: Final = TypeAdapter(tuple[object, ...]) _STREAM_EVENT_TYPE_ADAPTER: Final = TypeAdapter(str) _TOOL_CALL_PAYLOAD_FIELDS: Final[Mapping[str, str]] = MappingProxyType( {"function_call": "arguments", "custom_tool_call": "input"} @@ -343,6 +345,18 @@ def _is_tool_call_item(item: object) -> bool: return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES +def _is_tool_search_item(item: object) -> bool: + match item: + case {"type": "tool_search_call" | "tool_search_output"}: + return True + case _: + return False + + +def _joins_assistant_turn(item: object) -> bool: + return _is_tool_call_item(item) or _is_tool_search_item(item) + + def _claude_web_search_tool_call(item: object) -> ChatCompletionAssistantToolCall | None: search: Final = web_search_call(item) return None if search is None else claude_web_search_call_tool_call(search) @@ -490,14 +504,17 @@ def _provenance_unit_bounds( ) -> tuple[tuple[int, int], ...]: trailing_roles: Final = tuple( accumulate( - (_last_message_role(messages) for messages in solo_conversions), + ( + "assistant" if _is_tool_search_item(item) else _last_message_role(messages) + for item, messages in zip(raw_input, solo_conversions) + ), lambda previous, current: current if current is not None else previous, ) ) start_indexes: Final = tuple( index for index in range(len(raw_input)) - if index == 0 or not (_is_tool_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant") + if index == 0 or not (_joins_assistant_turn(raw_input[index]) and trailing_roles[index - 1] == "assistant") ) return tuple(zip(start_indexes, (*start_indexes[1:], len(raw_input)))) @@ -512,6 +529,7 @@ def _input_item_provenance( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=cast("ResponseInputParam", [item]), # cast-ok: items checked as Mappings above responses_api_request=_EMPTY_RESPONSES_REQUEST, + replay_tool_search=True, ) for item in raw_input ) @@ -519,6 +537,7 @@ def _input_item_provenance( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=cast("ResponseInputParam", list(raw_input)), # cast-ok: items checked as Mappings above responses_api_request=_EMPTY_RESPONSES_REQUEST, + replay_tool_search=True, ) ) if full_conversion != tuple(expected_messages): @@ -531,6 +550,7 @@ def _input_item_provenance( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=cast("ResponseInputParam", list(raw_input[start:end])), # cast-ok: checked as Mappings above responses_api_request=_EMPTY_RESPONSES_REQUEST, + replay_tool_search=True, ) ) for start, end in units @@ -744,7 +764,42 @@ def _patch_or_convert_request_fields( input_items, converted_instructions = ( LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(structured_messages) ) - return _RequestFields(input=tuple(input_items), instructions=converted_instructions) + return _RequestFields( + input=_with_tool_searches_restored(input_items, raw_input), instructions=converted_instructions + ) + + +def _tool_search_items_by_call_id(raw_input: Sequence[object], item_type: str) -> Mapping[str, object]: + return MappingProxyType( + { + call_id: item + for item in raw_input + if stream_item_field(item, "type") == item_type + and isinstance(call_id := stream_item_field(item, "call_id"), str) + } + ) + + +def _restored_tool_search( + item: object, calls: Mapping[str, object], outputs: Mapping[str, object] +) -> tuple[object, ...]: + call_id: Final = stream_item_field(item, "call_id") + if stream_item_field(item, "type") != "function_call" or not isinstance(call_id, str) or call_id not in calls: + return (item,) + arguments: Final = stream_item_field(item, "arguments") + restored_call: Final = _rewritten_tool_search_call( + calls[call_id], arguments if isinstance(arguments, str) else None + ) + return (restored_call, outputs[call_id]) if call_id in outputs else (restored_call,) + + +def _with_tool_searches_restored(input_items: Sequence[object], raw_input: object) -> tuple[object, ...]: + raw_items: Final = _RAW_ITEMS_ADAPTER.validate_python(raw_input) if isinstance(raw_input, list) else () + if not any(map(_is_tool_search_item, raw_items)): + return tuple(input_items) + calls: Final = _tool_search_items_by_call_id(raw_items, _TOOL_SEARCH_CALL_TYPE) + outputs: Final = _tool_search_items_by_call_id(raw_items, "tool_search_output") + return tuple(chain.from_iterable(_restored_tool_search(item, calls, outputs) for item in input_items)) def _next_stream_sequence_number(responses_so_far: Sequence[object] | None) -> int: @@ -791,6 +846,7 @@ class OpenAIResponsesHandler(BaseTranslation): messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=input_data, responses_api_request=data, + replay_tool_search=True, ) return cast(list[AllMessageValues], messages) if messages else None @@ -1953,7 +2009,7 @@ class OpenAIResponsesHandler(BaseTranslation): if isinstance(output_item, GenericResponseOutputItem): if output_item.content and content_idx < len(output_item.content): self._write_content_part_text(output_item.content[content_idx], guardrail_response) - elif isinstance(output_item, BaseModel): + elif isinstance(output_item, BaseModel) and not isinstance(output_item, OutputToolSearchOutput): # Handle other Pydantic models by converting to GenericResponseOutputItem try: generic_item = GenericResponseOutputItem.model_validate(output_item.model_dump()) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index f57d092d552..2cadfd6c4a3 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -27308,6 +27308,65 @@ "title": "OutputToolSearchCall", "type": "object" }, + "OutputToolSearchOutput": { + "additionalProperties": true, + "properties": { + "call_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Call Id" + }, + "execution": { + "enum": [ + "server", + "client" + ], + "title": "Execution", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "status": { + "enum": [ + "in_progress", + "completed", + "incomplete" + ], + "title": "Status", + "type": "string" + }, + "tools": { + "default": [], + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Tools", + "type": "array" + }, + "type": { + "const": "tool_search_output", + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "execution", + "status", + "type" + ], + "title": "OutputToolSearchOutput", + "type": "object" + }, "PendingSafetyCheck": { "additionalProperties": true, "description": "A pending safety check for the computer call.", @@ -29678,6 +29737,9 @@ { "$ref": "#/components/schemas/OutputToolSearchCall" }, + { + "$ref": "#/components/schemas/OutputToolSearchOutput" + }, { "$ref": "#/components/schemas/ResponseCompactionItem" } diff --git a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/purview_dlp.py b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/purview_dlp.py index 0b3d56a016b..dff5f5a7e6e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/purview_dlp.py +++ b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/purview_dlp.py @@ -325,6 +325,7 @@ class MicrosoftPurviewDLPGuardrail(PurviewGuardrailBase, CustomGuardrail): messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=input_data if input_data is not None else "", responses_api_request=data, + replay_tool_search=True, ) return self.get_prompt_text_for_dlp(cast(list["AllMessageValues"], messages)) except Exception: diff --git a/litellm/proxy/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/mcp_server/openapi_to_mcp_generator.py index 7282e59d746..fc88d8da1a3 100644 --- a/litellm/proxy/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/mcp_server/openapi_to_mcp_generator.py @@ -760,7 +760,7 @@ def register_tools_from_openapi(spec: _OpenAPISpec, base_url: str) -> None: # by sanitized characters (e.g. "foo/list" and "foo.list") # would both become "foo_list". Append _2, _3, … to keep # every tool reachable, mirroring the Anthropic-side logic - # in _build_anthropic_tool_name_maps. + # in build_anthropic_tool_name_maps. unique = tool_name n = 1 while unique in used_names: diff --git a/litellm/responses/litellm_completion_transformation/custom_tools.py b/litellm/responses/litellm_completion_transformation/custom_tools.py index 98e1984550c..2a2cea77efe 100644 --- a/litellm/responses/litellm_completion_transformation/custom_tools.py +++ b/litellm/responses/litellm_completion_transformation/custom_tools.py @@ -34,7 +34,7 @@ _MAX_ARGUMENTS_LEN: Final = 1_000_000 _DECODED_JSON: Final = TypeAdapter(object) TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE: Final = MappingProxyType( - {"function_call": "fc", "custom_tool_call": "ctc", "tool_search_call": "tsc"} + {"function_call": "fc", "custom_tool_call": "ctc", "tool_search_call": "tsc", "tool_search_output": "tso"} ) diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 900cfa3d597..6dbd7d69d11 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -468,6 +468,7 @@ class ResponsesSessionHandler: item_cache_control=item_cache_control, tool_call_provider_fields=tool_call_provider_fields, only_since_compaction=True, + replay_tool_search=True, ) ) @@ -484,6 +485,7 @@ class ResponsesSessionHandler: item_cache_control=item_cache_control, tool_call_provider_fields=tool_call_provider_fields, only_since_compaction=True, + replay_tool_search=True, ) ) elif isinstance(_response_output, dict) and _response_output and _response_output != {}: diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 8b6c53a272d..040e2af2a08 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -5,10 +5,13 @@ from itertools import filterfalse from types import MappingProxyType from typing import Any, Final, Protocol, cast +from pydantic import TypeAdapter + import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items, require_attributes from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import parts_of from litellm.llms.anthropic.common_utils import anthropic_server_call_fields, anthropic_server_tool_calls from litellm.llms.anthropic.messages.native.utils import openai_chat_refusal_text from litellm.main import stream_chunk_builder @@ -25,6 +28,10 @@ from litellm.responses.litellm_completion_transformation.tool_search import ( build_tool_search_call, tool_search_routing, ) +from litellm.responses.litellm_completion_transformation.tool_search_results import ( + build_tool_search_output, + tool_search_results_by_call_id, +) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -76,6 +83,8 @@ from litellm.types.utils import ( TextCompletionResponse, ) +_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object]) + def _index_of_output_item_type(items: Sequence[object], item_type: str) -> int | None: return next( @@ -237,6 +246,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._queued_web_search_call_ids: set[str] = set() # mutable-ok: emitted call ids self._tool_search = tool_search_routing(request_tools) self._tool_search_executions: Mapping[str, ToolSearchExecution] = MappingProxyType({}) + self._tool_search_output_index_by_call_id: Mapping[str, int] = MappingProxyType({}) self._done_tool_search_call_ids: frozenset[str] = frozenset() async def aclose(self) -> None: @@ -275,12 +285,43 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _queue_tool_search_call_done( self, call_id: str, output_index: int, final_args: str, execution: ToolSearchExecution ) -> None: - if call_id in self._done_tool_search_call_ids: + if call_id not in self._done_tool_search_call_ids: + self._done_tool_search_call_ids = self._done_tool_search_call_ids | {call_id} + self._queue_tool_search_call_added(call_id, output_index, "", execution) + item: Final = build_tool_search_call( + call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution + ) + self._pending_tool_events.append( + OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + output_index=output_index, + item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()), + ) + ) + self._queue_tool_search_output(call_id) + + def _queue_arrived_tool_search_outputs(self) -> None: + if "tool_results" not in self._accumulated_provider_specific_fields: return - self._done_tool_search_call_ids = self._done_tool_search_call_ids | {call_id} - self._queue_tool_search_call_added(call_id, output_index, "", execution) - item: Final = build_tool_search_call( - call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution + for call_id in tool_search_results_by_call_id((self._accumulated_provider_specific_fields,)): + if self._tool_search_executions.get(call_id) == "server": + self._queue_tool_search_call_done(call_id, self._get_or_assign_tool_output_index(call_id), "", "server") + + def _queue_tool_search_output(self, call_id: str) -> None: + if call_id in self._tool_search_output_index_by_call_id: + return + result: Final = tool_search_results_by_call_id((self._accumulated_provider_specific_fields,)).get(call_id) + if result is None: + return + output_index: Final = self._allocate_output_index() + self._tool_search_output_index_by_call_id = MappingProxyType( + {**self._tool_search_output_index_by_call_id, call_id: output_index} + ) + request_tools: Final = parts_of(_REQUEST_FIELDS.validate_python(self.responses_api_request).get("tools")) + item: Final = build_tool_search_output(result, request_tools) + self._queue_output_item_added( + output_index, + BaseLiteLLMOpenAIResponseObject(**item.model_copy(update={"status": "in_progress"}).model_dump()), ) self._pending_tool_events.append( OutputItemDoneEvent( @@ -333,6 +374,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return self._reasoning_output_index case "compaction": return self._compaction_output_index + case "tool_search_output": + return self._tool_search_output_index_by_call_id.get(str(getattr(item, "call_id", None))) case _: call_id: Final = getattr(item, "call_id", None) or str(getattr(item, "id", "")).removeprefix("ws_") return self._tool_output_index_by_call_id.get(str(call_id)) @@ -1331,6 +1374,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if src and isinstance(src, dict): self._merge_provider_specific_fields(src) self._reserve_web_search_indexes(src) + self._queue_arrived_tool_search_outputs() self._ensure_output_item_for_chunk(chunk) # Proceed to transformation self.collected_chat_completion_chunks.append( @@ -1394,6 +1438,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if src and isinstance(src, dict): self._merge_provider_specific_fields(src) self._reserve_web_search_indexes(src) + self._queue_arrived_tool_search_outputs() self._ensure_output_item_for_chunk(chunk) # Always snapshot before returning any pending events so that # finish_reason (e.g. content_filter) is captured even when diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index 5a1b27f9092..08b683f40a2 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -41,6 +41,8 @@ __all__ = [ "is_server_tool_search_tool", "provider_builds_anthropic_request", "provider_chat_config", + "request_runs_tool_search", + "route_runs_tool_search", "runs_tool_search_on_the_server", "tool_search_call_ids", "tool_search_routing", @@ -217,6 +219,20 @@ def provider_builds_anthropic_request(model: str, custom_llm_provider: str | Non return chat_config_builds_anthropic_request(provider_chat_config(model, custom_llm_provider)) +def route_runs_tool_search(model: str, custom_llm_provider: str | None) -> bool: + from litellm.utils import supports_tool_search + + return supports_tool_search( + model=model, custom_llm_provider=custom_llm_provider + ) and provider_builds_anthropic_request(model, custom_llm_provider) + + +def request_runs_tool_search(tools: Sequence[object], model: str, custom_llm_provider: str | None) -> bool: + if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): + return provider_builds_anthropic_request(model, custom_llm_provider) + return has_server_tool_search(tools) and route_runs_tool_search(model, custom_llm_provider) + + def chat_config_builds_anthropic_request(chat_config: object) -> bool: from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig diff --git a/litellm/responses/litellm_completion_transformation/tool_search_results.py b/litellm/responses/litellm_completion_transformation/tool_search_results.py new file mode 100644 index 00000000000..dd14124e5a6 --- /dev/null +++ b/litellm/responses/litellm_completion_transformation/tool_search_results.py @@ -0,0 +1,299 @@ +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from types import MappingProxyType +from typing import Final, Literal, TypeVar, cast # noqa: TID251 # validating a message union drops its untyped keys + +from openai.types.responses.response_function_tool_call_param import ResponseFunctionToolCallParam +from openai.types.responses.response_input_param import ResponseInputItemParam +from pydantic import ConfigDict, TypeAdapter, ValidationError + +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( + anthropic_tool_search_result, + is_anthropic_tool_search_use, + message_field, + parts_of, +) +from litellm.types.llms.anthropic import ( + ANTHROPIC_TOOL_SEARCH_RESULT_CONTENT_TYPE, + AnthropicToolReferenceBlock, + AnthropicToolSearchResultContent, + AnthropicToolSearchToolResultBlock, +) +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.openai import ResponseInputParam +from litellm.types.responses.main import OutputToolSearchCall, OutputToolSearchOutput + +from .custom_tools import openai_shaped_tool_call_item_id, serialize_tool_call_arguments +from .tool_search import ANTHROPIC_TOOL_SEARCH_TOOL_NAME + +_ItemT: Final = TypeVar("_ItemT") +_MessageT: Final = TypeVar("_MessageT") + +_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_TOOL_SEARCH_ITEM_TYPES: Final = frozenset({"tool_search_call", "tool_search_output"}) +_BM25_TOOL_SEARCH_TOOL_NAME: Final = "tool_search_tool_bm25" + + +class _ProviderToolResults(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + tool_results: tuple[object, ...] = () + + +def _provider_tool_results(provider_fields: object) -> tuple[object, ...]: + try: + return _ProviderToolResults.model_validate(provider_fields).tool_results + except ValidationError: + return () + + +def tool_search_results_by_call_id( + provider_fields: Iterable[object], +) -> Mapping[str, AnthropicToolSearchToolResultBlock]: + results: Final = map( + anthropic_tool_search_result, chain.from_iterable(map(_provider_tool_results, provider_fields)) + ) + return MappingProxyType({result.tool_use_id: result for result in results if result is not None}) + + +class _ToolFields(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: str = "" + name: str = "" + tools: tuple[object, ...] | None = None + + +@dataclass(frozen=True, slots=True) +class _LoadableTool: + chat_name: str + position: int + tool: Mapping[str, object] + member: Mapping[str, object] | None + + +def _tool_entry(tool: object) -> tuple[Mapping[str, object], _ToolFields] | None: + try: + mapping: Final = _MAPPING_ADAPTER.validate_python(tool) + return mapping, _ToolFields.model_validate(mapping) + except ValidationError: + return None + + +def _member_chat_name(namespace: str, member: _ToolFields) -> str | None: + match member.type: + case "function": + return f"{namespace}__{member.name}" + case "custom": + return member.name + case _: + return None + + +def _namespace_members( + position: int, namespace: Mapping[str, object], fields: _ToolFields +) -> tuple[_LoadableTool, ...]: + entries: Final = tuple(entry for raw in fields.tools or () if (entry := _tool_entry(raw)) is not None) + return tuple( + _LoadableTool(chat_name, position, namespace, member) + for member, member_fields in entries + if (chat_name := _member_chat_name(fields.name, member_fields)) is not None + ) + + +def _loadable_tools(position: int, tool: object) -> tuple[_LoadableTool, ...]: + entry: Final = _tool_entry(tool) + if entry is None: + return () + mapping, fields = entry + if fields.type == "namespace" and fields.tools is not None: + return _namespace_members(position, mapping, fields) + return (_LoadableTool(fields.name, position, mapping, None),) if fields.name else () + + +def _loaded_tool(hits: Sequence[_LoadableTool]) -> Mapping[str, object]: + members: Final = [hit.member for hit in hits if hit.member is not None] + return {**hits[0].tool, "tools": members} if members else hits[0].tool + + +def _catalog_entries( + loadable: _LoadableTool, anthropic_names: Mapping[str, str] +) -> tuple[tuple[str, _LoadableTool], ...]: + anthropic_name: Final = anthropic_names.get(loadable.chat_name) + names: Final = (loadable.chat_name,) if anthropic_name is None else (loadable.chat_name, anthropic_name) + return tuple((name, loadable) for name in names) + + +def loaded_tools(tool_names: Iterable[str], request_tools: Sequence[object] | None) -> tuple[Mapping[str, object], ...]: + from litellm.llms.anthropic.chat.transformation import build_anthropic_tool_name_maps + + loadables: Final = tuple( + chain.from_iterable(_loadable_tools(position, tool) for position, tool in enumerate(request_tools or ())) + ) + anthropic_names, _ = build_anthropic_tool_name_maps([loadable.chat_name for loadable in loadables]) + catalog: Final = MappingProxyType( + dict(chain.from_iterable(_catalog_entries(loadable, anthropic_names) for loadable in loadables)) + ) + hits: Final = tuple(catalog[name] for name in dict.fromkeys(tool_names) if name in catalog) + positions: Final = dict.fromkeys(hit.position for hit in hits) + return tuple(_loaded_tool(tuple(hit for hit in hits if hit.position == position)) for position in positions) + + +def build_tool_search_output( + result: AnthropicToolSearchToolResultBlock, request_tools: Sequence[object] | None +) -> OutputToolSearchOutput: + return OutputToolSearchOutput( + id=openai_shaped_tool_call_item_id("tool_search_output", result.tool_use_id), + call_id=result.tool_use_id, + execution="server", + status="completed" if result.found_tools else "incomplete", + tools=loaded_tools(result.tool_names, request_tools) if result.found_tools else (), + type="tool_search_output", + ) + + +def _with_tool_search_output( + item: _ItemT, + results: Mapping[str, AnthropicToolSearchToolResultBlock], + request_tools: Sequence[object] | None, +) -> tuple[_ItemT | OutputToolSearchOutput, ...]: + if not isinstance(item, OutputToolSearchCall) or item.call_id is None: + return (item,) + result: Final = results.get(item.call_id) + return (item,) if result is None else (item, build_tool_search_output(result, request_tools)) + + +def with_tool_search_outputs( + items: Sequence[_ItemT], + results: Mapping[str, AnthropicToolSearchToolResultBlock], + request_tools: Sequence[object] | None, +) -> tuple[_ItemT | OutputToolSearchOutput, ...]: + return tuple(chain.from_iterable(_with_tool_search_output(item, results, request_tools) for item in items)) + + +class _ToolSearchCallItem(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + type: Literal["tool_search_call"] + call_id: str + arguments: object = None + + +class _ToolSearchOutputItem(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + type: Literal["tool_search_output"] + call_id: str + status: Literal["in_progress", "completed", "incomplete"] | None = None + tools: tuple[object, ...] = () + + +def _tool_search_call(item: object) -> _ToolSearchCallItem | None: + if message_field(item, "type") != "tool_search_call": + return None + try: + return _ToolSearchCallItem.model_validate(item) + except ValidationError: + return None + + +def _tool_search_output(item: object) -> _ToolSearchOutputItem | None: + if message_field(item, "type") != "tool_search_output": + return None + try: + return _ToolSearchOutputItem.model_validate(item) + except ValidationError: + return None + + +def holds_tool_search_items(input: str | ResponseInputParam) -> bool: + return not isinstance(input, str) and any(message_field(item, "type") in _TOOL_SEARCH_ITEM_TYPES for item in input) + + +def _anthropic_tool_search_result(output: _ToolSearchOutputItem) -> Mapping[str, object]: + loaded: Final = chain.from_iterable(_loadable_tools(0, tool) for tool in output.tools) + result: Final = AnthropicToolSearchToolResultBlock( + tool_use_id=output.call_id, + content=AnthropicToolSearchResultContent( + type=ANTHROPIC_TOOL_SEARCH_RESULT_CONTENT_TYPE, + tool_references=tuple(AnthropicToolReferenceBlock(tool_name=tool.chat_name) for tool in loaded), + ), + ) + return result.model_dump(mode="json") + + +def _search_arguments(arguments: object) -> Mapping[str, object]: + try: + if isinstance(arguments, str): + return _MAPPING_ADAPTER.validate_json(arguments) + return _MAPPING_ADAPTER.validate_python(arguments) + except ValidationError: + return MappingProxyType({}) + + +def _search_tool_name(arguments: object) -> str: + return _BM25_TOOL_SEARCH_TOOL_NAME if "query" in _search_arguments(arguments) else ANTHROPIC_TOOL_SEARCH_TOOL_NAME + + +def _replayed_item(item: ResponseInputItemParam, replayed_ids: frozenset[str]) -> ResponseInputItemParam | None: + call: Final = _tool_search_call(item) + if call is not None and call.call_id in replayed_ids: + return ResponseFunctionToolCallParam( + type="function_call", + call_id=call.call_id, + name=_search_tool_name(call.arguments), + arguments=serialize_tool_call_arguments(call.arguments, "{}"), + ) + output: Final = _tool_search_output(item) + return None if output is not None and output.call_id in replayed_ids else item + + +def _with_tool_search_results(message: _MessageT, results: Mapping[str, Mapping[str, object]]) -> _MessageT: + tool_call_ids: Final = ( + message_field(tool_call, "id") for tool_call in parts_of(message_field(message, "tool_calls")) + ) + found: Final = tuple( + results[tool_call_id] + for tool_call_id in tool_call_ids + if isinstance(tool_call_id, str) and tool_call_id in results + ) + if not found: + return message + fields: Final = _MAPPING_ADAPTER.validate_python(message) + provider_fields: Final = _MAPPING_ADAPTER.validate_python(fields.get("provider_specific_fields") or {}) + tool_results: Final = (*parts_of(provider_fields.get("tool_results")), *found) + with_results: Final = { + **fields, + "provider_specific_fields": {**provider_fields, "tool_results": list(tool_results)}, + } + return cast(_MessageT, with_results) # cast-ok: a copy of the same message dict with provider fields added + + +def converted_with_tool_searches_replayed( + input: str | ResponseInputParam, + convert: Callable[[str | ResponseInputParam], Sequence[_MessageT]], +) -> Sequence[_MessageT]: + if isinstance(input, str) or not holds_tool_search_items(input): + return convert(input) + outputs: Final = MappingProxyType( + { + output.call_id: output + for output in map(_tool_search_output, input) + if output is not None and output.status in (None, "completed") + } + ) + replayed_ids: Final = frozenset( + call.call_id + for call in map(_tool_search_call, input) + if call is not None + and call.call_id in outputs + and is_anthropic_tool_search_use(call.call_id, ANTHROPIC_TOOL_SEARCH_TOOL_NAME) + ) + if not replayed_ids: + return convert(input) + replayed_input: Final = [kept for item in input if (kept := _replayed_item(item, replayed_ids)) is not None] + results: Final = MappingProxyType( + {call_id: _anthropic_tool_search_result(outputs[call_id]) for call_id in replayed_ids} + ) + return tuple(_with_tool_search_results(message, results) for message in convert(replayed_input)) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 05379ffde3e..3c1abbfe492 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -71,7 +71,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( is_present_non_string, is_unhashable_value_behind, ) -from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( + is_anthropic_tool_search_use, + message_field, + parts_of, +) from litellm.llms.anthropic.common_utils import ( anthropic_server_tool_call_params, anthropic_server_tool_calls, @@ -126,7 +130,6 @@ from litellm.types.llms.openai import ( ResponsesAPIStatus, ToolChoice, ToolMessageContentPart, - ValidChatCompletionMessageContentTypes, ValidChatCompletionMessageContentTypesLiteral, ) from litellm.types.responses.main import ( @@ -138,6 +141,7 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, OutputText, OutputToolSearchCall, + OutputToolSearchOutput, ) from litellm.types.utils import ( ChatCompletionAnnotation, @@ -186,11 +190,19 @@ from .tool_search import ( is_anthropic_tool_search_chat_tool, is_server_tool_search_tool, provider_builds_anthropic_request, + request_runs_tool_search, + route_runs_tool_search, tool_search_call_ids, tool_search_routing, without_tool_search_calls, without_tool_search_uses, ) +from .tool_search_results import ( + converted_with_tool_searches_replayed, + holds_tool_search_items, + tool_search_results_by_call_id, + with_tool_search_outputs, +) NamespaceNameMap: TypeAlias = Mapping[str, tuple[str, str]] NamespaceTool: TypeAlias = Mapping[str, object] @@ -279,6 +291,20 @@ TOOL_CALLS_CACHE: Final = InMemoryCache() _ANY_KEY_DICT_ADAPTER: Final = TypeAdapter(dict[object, object]) _STR_KEY_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) + + +_CONTENT_TYPE_ADAPTER: Final[TypeAdapter[ValidChatCompletionMessageContentTypesLiteral]] = TypeAdapter( + ValidChatCompletionMessageContentTypesLiteral +) + + +def _valid_content_type(content_type: str) -> ValidChatCompletionMessageContentTypesLiteral | None: + try: + return _CONTENT_TYPE_ADAPTER.validate_python(content_type) + except ValidationError: + return None + + _AUDIO_PART_ADAPTER: Final = TypeAdapter(ChatCompletionAudioObject) _IMAGE_URL_ADAPTER: Final = TypeAdapter(ChatCompletionImageUrlObject) _FILE_PART_ADAPTER: Final = TypeAdapter(_ResponsesFilePart) @@ -595,16 +621,58 @@ def _replayed_assistant_without_tool_search_uses(message: Mapping[str, object]) ) -def _without_stored_tool_search_uses(message: SessionMessage, search_call_ids: frozenset[str]) -> SessionMessage | None: +def _server_tool_search_ids(message: object) -> frozenset[str]: + return frozenset( + tool_id + for call in anthropic_server_tool_call_params(message) + if is_anthropic_tool_search_use(tool_id := call["id"] or "", call["function"].get("name") or "") + ) + + +def _holds_stored_tool_search(message: object) -> bool: + return holds_tool_search_use(message) or bool(_server_tool_search_ids(message)) + + +def _provider_fields_without_tool_searches(message: object, search_ids: frozenset[str]) -> Mapping[object, object]: + fields: Final = validated_object_mapping(message_field(message, "provider_specific_fields")) or {} + return { + **fields, + "server_tool_calls": [ + call for call in anthropic_server_tool_call_params(message) if (call["id"] or "") not in search_ids + ], + "tool_results": [ + result + for result in parts_of(fields.get("tool_results")) + if message_field(result, "tool_use_id") not in search_ids + ], + } + + +def _without_server_tool_searches(message: SessionMessage) -> SessionMessage: + search_ids: Final = _server_tool_search_ids(message) + if not search_ids: + return message + fields: Final = _provider_fields_without_tool_searches(message, search_ids) if isinstance(message, Message): - return without_tool_search_uses(message) - if answers_tool_search(message, search_call_ids): - return None - if isinstance(message, Mapping) and holds_tool_search_use(message): - return _replayed_assistant_without_tool_search_uses(message) + return message.model_copy(update={"provider_specific_fields": fields}) + if isinstance(message, Mapping): + return cast( # cast-ok: the stored assistant message minus its search keeps every other field as stored + SessionMessage, {**message, "provider_specific_fields": fields} + ) return message +def _without_stored_tool_search_uses(message: SessionMessage, search_call_ids: frozenset[str]) -> SessionMessage | None: + kept: Final = _without_server_tool_searches(message) + if isinstance(kept, Message): + return without_tool_search_uses(kept) + if answers_tool_search(kept, search_call_ids): + return None + if isinstance(kept, Mapping) and holds_tool_search_use(kept): + return _replayed_assistant_without_tool_search_uses(kept) + return kept + + def _instruction_messages(messages: Sequence[SessionMessage]) -> tuple[SessionMessage, ...]: return tuple( message @@ -1094,16 +1162,11 @@ class LiteLLMCompletionResponsesConfig: def _without_unsupported_tool_search( tools: ResponseTools, model: str, custom_llm_provider: str | None ) -> ResponseTools: - from litellm.utils import supports_tool_search - if not has_server_tool_search(tools): return tools if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools or ()): return [tool for tool in tools or () if not is_server_tool_search_tool(tool)] - route_runs_tool_search: Final = supports_tool_search( - model=model, custom_llm_provider=custom_llm_provider - ) and provider_builds_anthropic_request(model, custom_llm_provider) - if route_runs_tool_search: + if route_runs_tool_search(model, custom_llm_provider): return tools verbose_logger.warning( "Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') cannot run tool search " @@ -1281,6 +1344,8 @@ class LiteLLMCompletionResponsesConfig: tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs), native_compaction=native_compaction, only_since_compaction=not native_compaction, + replay_tool_search=holds_tool_search_items(request_input) + and request_runs_tool_search(request_tools or (), model, custom_llm_provider), caller_input=numbered_input, ) malformed_field_error: Final = ( @@ -1366,6 +1431,7 @@ class LiteLLMCompletionResponsesConfig: tool_call_provider_fields: bool = False, native_compaction: bool = False, only_since_compaction: bool = False, + replay_tool_search: bool = False, caller_input: str | ResponseInputParam | None = None, ) -> list[ AllMessageValues @@ -1403,6 +1469,11 @@ class LiteLLMCompletionResponsesConfig: messages, the way Anthropic ignores content before a compaction block. Guardrail scans leave it off because they write rewrites back by input item position. + + ``replay_tool_search`` belongs to callers building history for a target + that builds an Anthropic request: a Claude ``tool_search_call`` and its + ``tool_search_output`` then go back as the search Claude ran. Everyone + else drops both items. """ messages: list[ AllMessageValues @@ -1429,7 +1500,11 @@ class LiteLLMCompletionResponsesConfig: native_compaction=native_compaction, ) try: - converted: Final = convert_input(sent_input) + converted: Final = ( + converted_with_tool_searches_replayed(sent_input, convert_input) + if replay_tool_search + else convert_input(sent_input) + ) except TypeError as error: call_id_error: Final = _unhashable_tool_output_call_id_error( dumped_input if caller_input is None else caller_input, @@ -1515,7 +1590,7 @@ class LiteLLMCompletionResponsesConfig: ) replayed_messages: Final = ( LiteLLMCompletionResponsesConfig._without_tool_search_uses_off_anthropic(stored_messages, route) - if any(holds_tool_search_use(message) for message in stored_messages) + if any(_holds_stored_tool_search(message) for message in stored_messages) else stored_messages ) session_messages: Final = ( @@ -2831,8 +2906,8 @@ class LiteLLMCompletionResponsesConfig: if content_type.startswith("input_"): stripped: Final = content_type[len("input_") :] # Validate stripped type is valid, otherwise default to "text" - if stripped in ValidChatCompletionMessageContentTypes: - return stripped + if valid_stripped := _valid_content_type(stripped): + return valid_stripped # Handle input_audio -> input_audio (it's already valid) if stripped == "audio": return "input_audio" @@ -2843,10 +2918,7 @@ class LiteLLMCompletionResponsesConfig: return "text" # Return as-is if it's a valid type, otherwise default to "text" - if content_type in ValidChatCompletionMessageContentTypes: - return content_type - - return "text" + return _valid_content_type(content_type) or "text" @staticmethod def transform_instructions_to_system_message( @@ -3120,7 +3192,13 @@ class LiteLLMCompletionResponsesConfig: def transform_chat_completion_tools_to_responses_tools( chat_completion_response: ModelResponse, responses_api_request: ResponsesAPIOptionalRequestParams | None = None, - ) -> list[ResponseFunctionToolCall | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall]: + ) -> list[ + ResponseFunctionToolCall + | ResponseFunctionWebSearch + | CustomToolCallOutputItem + | OutputToolSearchCall + | OutputToolSearchOutput + ]: """ Transform a Chat Completion tools into a Responses API tools. @@ -3128,7 +3206,8 @@ class LiteLLMCompletionResponsesConfig: with ``type="custom_tool_call"``. For regular function tools, returns ``ResponseFunctionToolCall`` with ``type="function_call"``. A provider-side tool search call made for a request's ``tool_search`` tool returns - ``OutputToolSearchCall`` with ``type="tool_search_call"``. + ``OutputToolSearchCall`` with ``type="tool_search_call"``, followed by + an ``OutputToolSearchOutput`` listing the tools it loaded. """ all_chat_completion_tools: Final[list[ChatCompletionMessageToolCall]] = [] for choice in chat_completion_response.choices: @@ -3217,7 +3296,15 @@ class LiteLLMCompletionResponsesConfig: ) responses_tools.append(output_tool_call) - return responses_tools + return list( + with_tool_search_outputs( + responses_tools, + tool_search_results_by_call_id( + choice.message.provider_specific_fields for choice in chat_completion_response.choices + ), + request_tools, + ) + ) @staticmethod def _server_tool_call_item( @@ -3522,6 +3609,7 @@ class LiteLLMCompletionResponsesConfig: | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall + | OutputToolSearchOutput | ResponseCompactionItem ]: responses_output: list[ @@ -3533,6 +3621,7 @@ class LiteLLMCompletionResponsesConfig: | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall + | OutputToolSearchOutput | ResponseCompactionItem ] = [] diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index c322eb25b3e..3abe64df3c6 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -156,6 +156,36 @@ class ToolReference(TypedDict, total=False): tool_name: Required[str] +class AnthropicToolReferenceBlock(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["tool_reference"] = "tool_reference" + tool_name: str + + +class AnthropicToolSearchResultContent(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: str + tool_references: tuple[AnthropicToolReferenceBlock, ...] = () + + +class AnthropicToolSearchToolResultBlock(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["tool_search_tool_result"] = "tool_search_tool_result" + tool_use_id: str + content: AnthropicToolSearchResultContent + + @property + def found_tools(self) -> bool: + return self.content.type == ANTHROPIC_TOOL_SEARCH_RESULT_CONTENT_TYPE + + @property + def tool_names(self) -> tuple[str, ...]: + return tuple(reference.tool_name for reference in self.content.tool_references) + + class DirectToolCaller(TypedDict, total=False): """Indicates a tool was called directly by Claude.""" @@ -863,6 +893,7 @@ ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: Final = frozenset( ) ANTHROPIC_TOOL_SEARCH_TOOL_NAMES: Final = frozenset({"tool_search_tool_regex", "tool_search_tool_bm25"}) ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX: Final = "srvtoolu_" +ANTHROPIC_TOOL_SEARCH_RESULT_CONTENT_TYPE: Final = "tool_search_tool_search_result" def claude_hosted_web_tool_call(search: ResponseFunctionWebSearch) -> tuple[str, str, str] | None: diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 378899b667d..268bce1cf6f 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -95,6 +95,7 @@ from litellm.types.responses.main import ( OutputFunctionToolCall, OutputImageGenerationCall, OutputToolSearchCall, + OutputToolSearchOutput, ) from .base import CachedTokensDetails @@ -1595,6 +1596,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall + | OutputToolSearchOutput | ResponseCompactionItem ] ) diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 7a2f58e9c16..61cd91be7ec 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -225,6 +225,17 @@ class OutputToolSearchCall(BaseLiteLLMOpenAIResponseObject): type: Literal["tool_search_call"] +class OutputToolSearchOutput(BaseLiteLLMOpenAIResponseObject): + model_config = ConfigDict(extra="allow", frozen=True, protected_namespaces=()) + + id: str + call_id: str | None = None + execution: Literal["server", "client"] + status: Literal["in_progress", "completed", "incomplete"] + tools: tuple[Mapping[str, object], ...] = () + type: Literal["tool_search_output"] + + class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): """ Generic response API output item diff --git a/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py b/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py index adce80d3c75..eb5220b9b41 100644 --- a/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py +++ b/tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py @@ -424,37 +424,35 @@ def test_odd_string_tool_answer_ids_keep_the_search_rebuilt(gateway: Gateway, to assert SEARCH_ID not in forwarded, received[0].body -_ODD_RESULTS: Final = ( - pytest.param( - { - "type": "tool_search_tool_result", - "tool_use_id": SEARCH_ID, - "content": {"type": "tool_search_tool_result_error", "error_code": "unavailable"}, - }, - id="error-content", - ), - pytest.param( - { - "type": "tool_search_tool_result", - "tool_use_id": SEARCH_ID, - "content": { - "type": "tool_search_tool_search_result", - "tool_references": [{"type": "tool_reference", "tool_name": "w" * 5120}], - }, - }, - id="5kb-reference", - ), -) - - -@pytest.mark.parametrize("odd", _ODD_RESULTS) @_STREAM_MODES -def test_unusual_search_results_from_the_upstream_are_kept_and_replayed_verbatim( - gateway: Gateway, odd: Event, stream: bool +def test_an_errored_search_result_from_the_upstream_is_kept_and_replayed_verbatim( + gateway: Gateway, stream: bool ) -> None: - assistant, _, _, received, _ = _round_trip(gateway, "anthropic", stream, (SEARCH_USE, odd, WEATHER_USE)) - assert kept_results(assistant) == [odd], assistant - assert wire_assistant(received[1]) == [SEARCH_USE, odd, WEATHER_USE], received[1].body + errored: Final[Event] = { + "type": "tool_search_tool_result", + "tool_use_id": SEARCH_ID, + "content": {"type": "tool_search_tool_result_error", "error_code": "unavailable"}, + } + assistant, _, _, received, _ = _round_trip(gateway, "anthropic", stream, (SEARCH_USE, errored, WEATHER_USE)) + assert kept_results(assistant) == [errored], assistant + assert wire_assistant(received[1]) == [SEARCH_USE, errored, WEATHER_USE], received[1].body + + +@_STREAM_MODES +def test_a_search_result_naming_a_tool_the_request_never_sent_is_kept_but_left_out_of_the_replay( + gateway: Gateway, stream: bool +) -> None: + unknown_tool: Final[Event] = { + "type": "tool_search_tool_result", + "tool_use_id": SEARCH_ID, + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [{"type": "tool_reference", "tool_name": "w" * 5120}], + }, + } + assistant, _, _, received, _ = _round_trip(gateway, "anthropic", stream, (SEARCH_USE, unknown_tool, WEATHER_USE)) + assert kept_results(assistant) == [unknown_tool], assistant + assert wire_assistant(received[1]) == [WEATHER_USE], received[1].body @_STREAM_MODES diff --git a/tests/integration/providers/test_responses_bridge_tool_search_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_wire.py index 95c05610fba..da7d5fa3508 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_wire.py @@ -153,18 +153,18 @@ def _prompt(marker: str) -> str: return f"Weather in Paris and the ACME price marker-{marker}" -def _search_result(marker: str) -> dict[str, JsonValue]: +def _search_result(marker: str, found_tool: str = "get_weather") -> dict[str, JsonValue]: return { "type": "tool_search_tool_result", "tool_use_id": f"srvtoolu_{marker}", "content": { "type": "tool_search_tool_search_result", - "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}], + "tool_references": [{"type": "tool_reference", "tool_name": found_tool}], }, } -def _search_blocks(marker: str) -> tuple[dict[str, JsonValue], ...]: +def _search_blocks(marker: str, found_tool: str = "get_weather") -> tuple[dict[str, JsonValue], ...]: return ( { "type": "server_tool_use", @@ -172,12 +172,12 @@ def _search_blocks(marker: str) -> tuple[dict[str, JsonValue], ...]: "name": _REGEX_TOOL_NAME, "input": json.loads(_SEARCH_ARGUMENTS), }, - _search_result(marker), + _search_result(marker, found_tool), ) -def _message(marker: str, searched: bool, tool_use_name: str) -> dict[str, JsonValue]: - search_blocks: Final = _search_blocks(marker) if searched else () +def _message(marker: str, searched: bool, tool_use_name: str, found_tool: str = "get_weather") -> dict[str, JsonValue]: + search_blocks: Final = _search_blocks(marker, found_tool) if searched else () return { "id": f"msg_{marker}", "type": "message", @@ -194,7 +194,7 @@ def _message(marker: str, searched: bool, tool_use_name: str) -> dict[str, JsonV } -def _search_events(marker: str, search_arguments: str) -> tuple[Event, ...]: +def _search_events(marker: str, search_arguments: str, found_tool: str) -> tuple[Event, ...]: half: Final = len(search_arguments) // 2 opened: Final[Event] = { "type": "server_tool_use", @@ -215,14 +215,16 @@ def _search_events(marker: str, search_arguments: str) -> tuple[Event, ...]: "delta": {"type": "input_json_delta", "partial_json": search_arguments[half:]}, }, {"type": "content_block_stop", "index": 1}, - {"type": "content_block_start", "index": 2, "content_block": _search_result(marker)}, + {"type": "content_block_start", "index": 2, "content_block": _search_result(marker, found_tool)}, {"type": "content_block_stop", "index": 2}, ) -def _events(marker: str, search_arguments: str, searched: bool, tool_use_name: str) -> tuple[Event, ...]: +def _events( + marker: str, search_arguments: str, searched: bool, tool_use_name: str, found_tool: str = "get_weather" +) -> tuple[Event, ...]: opened: Final = {**_message(marker, searched, tool_use_name), "content": [], "stop_reason": None} - search_events: Final = _search_events(marker, search_arguments) if searched else () + search_events: Final = _search_events(marker, search_arguments, found_tool) if searched else () tool_index: Final = 3 if searched else 1 tool_use: Final[Event] = {"type": "tool_use", "id": f"toolu_{marker}", "name": tool_use_name, "input": {}} return ( @@ -266,6 +268,7 @@ class _Vendor: pause: float = 0 searched: bool | None = None tool_use_name: str = "get_weather" + found_tool: str = "get_weather" error: tuple[int, str] | None = None def respond(self, request: Request) -> Reply: @@ -284,15 +287,17 @@ class _Vendor: return _anthropic_error(400, f"tools: {_ALL_DEFERRED_ERROR} marker-{marker}") searched: Final = bool(regex_tools) if self.searched is None else self.searched if request.target.endswith("/invoke-with-response-stream"): - events: Final = _events(marker, self.search_arguments, searched, self.tool_use_name) + events: Final = _events(marker, self.search_arguments, searched, self.tool_use_name, self.found_tool) return Reply(content_type=EVENT_STREAM, chunks=aws_chunks(events), pause_between_chunks=self.pause) if _streams(request, body): return Reply( content_type="text/event-stream", - chunks=sse_chunks(_events(marker, self.search_arguments, searched, self.tool_use_name)), + chunks=sse_chunks( + _events(marker, self.search_arguments, searched, self.tool_use_name, self.found_tool) + ), pause_between_chunks=self.pause, ) - return Reply(body=json.dumps(_message(marker, searched, self.tool_use_name)).encode()) + return Reply(body=json.dumps(_message(marker, searched, self.tool_use_name, self.found_tool)).encode()) def _base_url(gateway: Gateway) -> str: @@ -413,8 +418,8 @@ def _search_call_fields(item: Mapping[str, JsonValue]) -> dict[str, JsonValue]: def _assert_search_call_output(payload: Mapping[str, JsonValue], marker: str) -> None: items: Final = _tool_items(payload) - assert len(items) == 2, payload - search, function = items + assert len(items) == 3, payload + search, search_output, function = items assert search["type"] == "tool_search_call", payload assert _search_call_fields(search) == { "id": f"tsc_srvtoolu_{marker}", @@ -423,6 +428,14 @@ def _assert_search_call_output(payload: Mapping[str, JsonValue], marker: str) -> "execution": "server", "status": "completed", }, search + assert {key: search_output[key] for key in ("type", "id", "call_id", "execution", "status")} == { + "type": "tool_search_output", + "id": f"tso_srvtoolu_{marker}", + "call_id": f"srvtoolu_{marker}", + "execution": "server", + "status": "completed", + }, search_output + assert [tool["name"] for tool in _ITEMS.validate_python(search_output["tools"])] == ["get_weather"], search_output assert (function["type"], function["name"], function["call_id"]) == ( "function_call", "get_weather", @@ -445,6 +458,8 @@ def _item_events(events: Sequence[Mapping[str, JsonValue]], item_type: str) -> l def _assert_search_call_events(events: Sequence[Mapping[str, JsonValue]], marker: str, final_status: str) -> None: + added_indexes: Final = [event["output_index"] for event in events if event["type"] == "response.output_item.added"] + assert added_indexes == list(range(len(added_indexes))), [event["type"] for event in events] search_events: Final = _item_events(events, "tool_search_call") assert [event["type"] for event in search_events] == [ "response.output_item.added", @@ -593,6 +608,24 @@ def test_tool_search_reaches_claude_with_deferred_tools_and_returns_a_tool_searc _assert_spend_rows(model, (str(answer.payload["id"]),)) +def test_the_openai_sdk_stream_helper_reads_a_streamed_claude_tool_search(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url, gateway.upstream_url) + body: Final = _responses_body(model, marker, (_tool_search(), _function_tool(), _namespace_tool()), True) + with _sdk(gateway).responses.stream( + model=model, + input=str(body["input"]), + tools=_ITEMS.validate_python(body["tools"]), + extra_body=dict(_CACHE_BUST), + ) as stream: + events: Final = [event.model_dump() for event in stream] + final: Final = stream.get_final_response() + _assert_search_call_output(final.model_dump(), marker) + _assert_search_call_events(events, marker, "completed") + _only_request(wire) + + @pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"]) @pytest.mark.parametrize("provider", ["bedrock_invoke", "vertex_ai"]) def test_bedrock_invoke_and_vertex_claude_keep_the_tool_search_tool( @@ -665,8 +698,10 @@ def test_search_arguments_that_never_parse_stream_an_incomplete_tool_search_call tools: Final = (_tool_search(), _function_tool()) answer: Final = _raw_answer(gateway, _responses_body(model, marker, tools, True)) _assert_search_call_events(answer.events, marker, "incomplete") - search, function = _tool_items(answer.payload) + search, search_output, function = _tool_items(answer.payload) assert (search["type"], search["status"], search["arguments"]) == ("tool_search_call", "incomplete", {}), search + assert search_output["type"] == "tool_search_output", search_output + assert search_output["call_id"] == search["call_id"], (search, search_output) assert (function["type"], function["name"]) == ("function_call", "get_weather"), function request, body = _only_request(wire) _assert_anthropic_wire(request, body, ("get_weather",)) @@ -678,7 +713,11 @@ def test_tool_search_alone_reaches_claude_as_the_only_tool(gateway: Gateway) -> with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, "anthropic", wire.url, gateway.upstream_url) answer: Final = _raw_answer(gateway, _responses_body(model, marker, (_tool_search(),), False)) - assert [item["type"] for item in _tool_items(answer.payload)] == ["tool_search_call", "function_call"], answer + assert [item["type"] for item in _tool_items(answer.payload)] == [ + "tool_search_call", + "tool_search_output", + "function_call", + ], answer request, body = _only_request(wire) _assert_anthropic_wire(request, body, ()) @@ -891,9 +930,18 @@ def _assert_only_the_function_call_is_replayed(messages: Sequence[Mapping[str, J assert [block["tool_use_id"] for block in _tool_results(messages)] == [f"toolu_{marker}"], messages -def test_tool_search_items_are_not_replayed_to_claude(gateway: Gateway) -> None: - marker: Final = uuid.uuid4().hex - history: Final[list[JsonValue]] = [ +def _assert_the_search_is_replayed_before_the_function_call( + messages: Sequence[Mapping[str, JsonValue]], marker: str +) -> None: + (assistant,) = [turn for turn in messages if turn["role"] == "assistant"] + blocks: Final = _ITEMS.validate_python(assistant["content"]) + assert blocks[:2] == list(_search_blocks(marker)), blocks + assert [(block["type"], block.get("id")) for block in blocks[2:]] == [("tool_use", f"toolu_{marker}")], blocks + assert [block["tool_use_id"] for block in _tool_results(messages)] == [f"toolu_{marker}"], messages + + +def _replayed_search_history(marker: str) -> list[JsonValue]: + return [ {"role": "user", "content": _prompt(marker)}, { "type": "tool_search_call", @@ -921,11 +969,16 @@ def test_tool_search_items_are_not_replayed_to_claude(gateway: Gateway) -> None: {"type": "function_call_output", "call_id": f"toolu_{marker}", "output": "22C"}, {"role": "user", "content": f"And tomorrow? marker-{marker}"}, ] + + +@pytest.mark.parametrize("provider", ["anthropic", "bedrock_invoke", "vertex_ai"]) +def test_a_replayed_tool_search_reaches_claude_as_its_search_and_result(gateway: Gateway, provider: Provider) -> None: + marker: Final = uuid.uuid4().hex with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: - model: Final = _deployment(scenario, "anthropic", wire.url, gateway.upstream_url) + model: Final = _deployment(scenario, provider, wire.url, gateway.upstream_url) body: Final[dict[str, JsonValue]] = { "model": model, - "input": history, + "input": _replayed_search_history(marker), "tools": [_tool_search(), _function_tool()], **_CACHE_BUST, } @@ -933,10 +986,85 @@ def test_tool_search_items_are_not_replayed_to_claude(gateway: Gateway) -> None: assert response.status_code == 200, response.text _assert_search_call_output(_JSON_OBJECT.validate_json(response.content), marker) request, sent = _only_request(wire) - _assert_anthropic_wire(request, sent, ("get_weather",)) + _assert_claude_wire(provider, False, request, sent, ("get_weather",)) + _assert_the_search_is_replayed_before_the_function_call(_ITEMS.validate_python(sent["messages"]), marker) + + +@pytest.mark.parametrize("provider", ["anthropic", "bedrock_invoke", "vertex_ai"]) +def test_a_replayed_tool_search_whose_tool_is_no_longer_sent_is_left_out(gateway: Gateway, provider: Provider) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, provider, wire.url, gateway.upstream_url) + body: Final[dict[str, JsonValue]] = { + "model": model, + "input": _replayed_search_history(marker), + "tools": [_tool_search(), _namespace_tool()], + **_CACHE_BUST, + } + response: Final = _raw(gateway, "/v1/responses", body) + assert response.status_code == 200, response.text + request, sent = _only_request(wire) _assert_only_the_function_call_is_replayed(_ITEMS.validate_python(sent["messages"]), marker) +@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"]) +def test_a_tool_name_anthropic_rewrites_comes_back_and_is_replayed_under_the_callers_name( + gateway: Gateway, stream: bool +) -> None: + marker: Final = uuid.uuid4().hex + vendor: Final = _Vendor(tool_use_name="weather_get", found_tool="weather_get") + tools: Final = (_tool_search(), {**_function_tool(), "name": "weather.get"}) + with wire_server(vendor.respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url, gateway.upstream_url) + items: Final = _tool_items(_raw_answer(gateway, _responses_body(model, marker, tools, stream)).payload) + _only_request(wire) + assert [(item["type"], item.get("name")) for item in items] == [ + ("tool_search_call", None), + ("tool_search_output", None), + ("function_call", "weather.get"), + ], items + assert [tool["name"] for tool in _ITEMS.validate_python(items[1]["tools"])] == ["weather.get"], items[1] + replay: Final = _raw( + gateway, + "/v1/responses", + { + "model": model, + "store": False, + "input": [ + {"role": "user", "content": _prompt(marker)}, + *items, + {"type": "function_call_output", "call_id": f"toolu_{marker}", "output": "22C"}, + {"role": "user", "content": f"And tomorrow? marker-{marker}"}, + ], + "tools": [dict(tool) for tool in tools], + **_CACHE_BUST, + }, + ) + assert replay.status_code == 200, replay.text + _, sent = _only_request(wire) + (assistant,) = [turn for turn in _ITEMS.validate_python(sent["messages"]) if turn["role"] == "assistant"] + blocks: Final = _ITEMS.validate_python(assistant["content"]) + assert [block for block in blocks if block["type"] != "text"] == [ + *_search_blocks(marker, "weather_get"), + {"type": "tool_use", "id": f"toolu_{marker}", "name": "weather_get", "input": {"city": "Paris"}}, + ], blocks + + +@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"]) +@pytest.mark.parametrize("provider", ["anthropic", "bedrock_invoke"]) +def test_the_tool_search_output_lists_the_callers_tool_when_anthropic_rewrites_its_name( + gateway: Gateway, provider: Provider, stream: bool +) -> None: + marker: Final = uuid.uuid4().hex + vendor: Final = _Vendor(tool_use_name="weather_get", found_tool="weather_get") + tools: Final = (_tool_search(), {**_function_tool(), "name": "weather.get"}) + with wire_server(vendor.respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, provider, wire.url, gateway.upstream_url) + items: Final = _tool_items(_raw_answer(gateway, _responses_body(model, marker, tools, stream)).payload) + (search_output,) = [item for item in items if item["type"] == "tool_search_output"] + assert [tool["name"] for tool in _ITEMS.validate_python(search_output["tools"])] == ["weather.get"], items + + def test_chat_completions_drop_a_replayed_search_call_that_has_no_result(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index bf093f1a0d8..54f55aba46a 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3794,7 +3794,9 @@ def _tool_search_turn(tool_references: list[str], last_block: dict[str, object]) ] -def _parsed_choice(content: list[dict[str, object]], stop_reason: str) -> Choices: +def _parsed_choice( + content: list[dict[str, object]], stop_reason: str, tool_name_reverse_map: dict[str, str] | None = None +) -> Choices: anthropic_response: Final = { "id": "msg_01XYZ", "type": "message", @@ -3817,6 +3819,7 @@ def _parsed_choice(content: list[dict[str, object]], stop_reason: str) -> Choice model_response=ModelResponse(), json_mode=False, prefix_prompt=None, + tool_name_reverse_map=tool_name_reverse_map, ) .choices[0] ) @@ -3824,8 +3827,10 @@ def _parsed_choice(content: list[dict[str, object]], stop_reason: str) -> Choice return choice -def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str) -> Message: - return _parsed_choice(content, stop_reason).message +def _parsed_assistant_message( + content: list[dict[str, object]], stop_reason: str, tool_name_reverse_map: dict[str, str] | None = None +) -> Message: + return _parsed_choice(content, stop_reason, tool_name_reverse_map).message _WEATHER_TOOL_USE: Final = { @@ -4092,6 +4097,110 @@ def test_replayed_search_that_found_nothing_answers_every_tool_use_and_ends_on_a assert _block_ids(replayed[1], "server_tool_use", "id") == server_tool_use_ids +def _function_tool(name: str) -> dict[str, object]: + return { + "type": "function", + "function": {"name": name, "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}, + "defer_loading": True, + } + + +def _anthropic_request_after( + assistant_content: list[dict[str, object]], stop_reason: str, tool_names: tuple[str, ...] | None +) -> dict[str, object]: + config: Final = AnthropicConfig() + message: Final = _parsed_assistant_message(assistant_content, stop_reason) + answers: Final = [ + {"role": "tool", "tool_call_id": block["id"], "content": "18C"} + for block in assistant_content + if block["type"] == "tool_use" + ] + optional_params: Final = ( + {} + if tool_names is None + else config.map_openai_params( + non_default_params={ + "tools": [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + *map(_function_tool, tool_names), + ] + }, + optional_params={}, + model="claude-opus-5-5", + drop_params=False, + ) + ) + return config.transform_request( + model="claude-opus-5-5", + messages=[ + {"role": "user", "content": "What is the weather in Paris?"}, + message.model_dump(), + *answers, + {"role": "user", "content": "And in Osaka?"}, + ], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize( + ("tool_names", "search_replayed"), + [ + pytest.param(("get_weather", "get_local_time"), True, id="every_loaded_tool_is_still_sent"), + pytest.param(("get_local_time",), False, id="a_loaded_tool_is_no_longer_sent"), + pytest.param(None, False, id="no_tools_so_litellm_sends_a_dummy_tool"), + ], +) +def test_replayed_tool_search_is_sent_only_while_every_tool_it_loaded_is_still_sent( + tool_names: tuple[str, ...] | None, search_replayed: bool +): + search_turn: Final = _tool_search_turn(["get_weather"], _WEATHER_TOOL_USE) + + request: Final = _anthropic_request_after(search_turn, "tool_use", tool_names) + + assert request["messages"][1]["content"] == (search_turn if search_replayed else [_WEATHER_TOOL_USE]) + assert [block["type"] for block in request["messages"][2]["content"]] == ["tool_result", "text"] + + +def test_replayed_tool_search_turn_that_held_only_the_search_is_dropped_once_its_tool_is_no_longer_sent(): + request: Final = _anthropic_request_after(_tool_search_turn(["get_weather"], _WEATHER_TOOL_USE)[:2], "end_turn", ()) + + assert [message["role"] for message in request["messages"]] == ["user", "user"] + assert "srvtoolu_01Search" not in json.dumps(request["messages"]) + + +def test_replayed_tool_search_result_litellm_cannot_read_is_sent_as_the_client_replayed_it(): + unreadable_result: Final = { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_01Search", + "content": {"tool_references": "get_weather"}, + } + message: Final = _parsed_assistant_message(_tool_search_turn(["get_weather"], _WEATHER_TOOL_USE), "tool_use") + provider_fields: Final = {**(message.provider_specific_fields or {}), "tool_results": [unreadable_result]} + replayed: Final = message.model_copy(update={"provider_specific_fields": provider_fields}) + config: Final = AnthropicConfig() + + request: Final = config.transform_request( + model="claude-opus-5-5", + messages=[ + {"role": "user", "content": "What is the weather in Paris?"}, + replayed.model_dump(), + {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"}, + ], + optional_params=config.map_openai_params( + non_default_params={"tools": [_function_tool("get_local_time")]}, + optional_params={}, + model="claude-opus-5-5", + drop_params=False, + ), + litellm_params={}, + headers={}, + ) + + assert unreadable_result in request["messages"][1]["content"] + + def test_replayed_search_that_found_nothing_keeps_its_result_when_the_call_never_reached_the_client( monkeypatch: pytest.MonkeyPatch, ): @@ -5656,10 +5765,10 @@ def test_basic_sanitize_anthropic_tool_name_replaces_invalid_chars(): def test_build_anthropic_tool_name_maps_no_collisions(): """Names that need rewriting go in the maps; valid names stay out.""" from litellm.llms.anthropic.chat.transformation import ( - _build_anthropic_tool_name_maps, + build_anthropic_tool_name_maps, ) - forward, reverse = _build_anthropic_tool_name_maps( + forward, reverse = build_anthropic_tool_name_maps( [ "fine_name", "actions/download-job-logs-for-workflow-run", @@ -5681,10 +5790,10 @@ def test_build_anthropic_tool_name_maps_disambiguates_collision_with_existing_va the rewritten one must get a unique suffix and only THAT one shows up in the reverse map. The legitimately-named `foo_bar` round-trips identically.""" from litellm.llms.anthropic.chat.transformation import ( - _build_anthropic_tool_name_maps, + build_anthropic_tool_name_maps, ) - forward, reverse = _build_anthropic_tool_name_maps(["foo_bar", "foo/bar"]) + forward, reverse = build_anthropic_tool_name_maps(["foo_bar", "foo/bar"]) # The original valid name keeps its slot. assert "foo_bar" not in forward # untouched # The rewritten one gets a disambiguating suffix. @@ -5700,10 +5809,10 @@ def test_build_anthropic_tool_name_maps_disambiguates_two_rewrites_to_same_targe """Two different invalid names that collapse to the same candidate must both end up with unique sanitized forms.""" from litellm.llms.anthropic.chat.transformation import ( - _build_anthropic_tool_name_maps, + build_anthropic_tool_name_maps, ) - forward, reverse = _build_anthropic_tool_name_maps(["foo/bar", "foo.bar"]) + forward, reverse = build_anthropic_tool_name_maps(["foo/bar", "foo.bar"]) # First wins the canonical slot, second gets a suffix. assert forward["foo/bar"] == "foo_bar" assert forward["foo.bar"] == "foo_bar_2" @@ -5715,10 +5824,10 @@ def test_build_anthropic_tool_name_maps_disambiguates_two_rewrites_to_same_targe def test_build_anthropic_tool_name_maps_three_way_collision(): """`foo/bar`, `foo.bar`, and an existing `foo_bar` must all coexist.""" from litellm.llms.anthropic.chat.transformation import ( - _build_anthropic_tool_name_maps, + build_anthropic_tool_name_maps, ) - forward, reverse = _build_anthropic_tool_name_maps(["foo_bar", "foo/bar", "foo.bar"]) + forward, reverse = build_anthropic_tool_name_maps(["foo_bar", "foo/bar", "foo.bar"]) assert "foo_bar" not in forward # untouched assert forward["foo/bar"] == "foo_bar_2" assert forward["foo.bar"] == "foo_bar_3" @@ -5733,10 +5842,10 @@ def test_build_anthropic_tool_name_maps_reverse_order_collision(): its sanitized form collides with, both must still end up with distinct names on the wire.""" from litellm.llms.anthropic.chat.transformation import ( - _build_anthropic_tool_name_maps, + build_anthropic_tool_name_maps, ) - forward, reverse = _build_anthropic_tool_name_maps(["foo/bar", "foo_bar"]) + forward, reverse = build_anthropic_tool_name_maps(["foo/bar", "foo_bar"]) # The valid name keeps its slot untouched. assert "foo_bar" not in forward # The rewritten one gets a disambiguating suffix. @@ -5756,10 +5865,10 @@ def test_build_anthropic_tool_name_maps_duplicate_originals(): on duplicates. """ from litellm.llms.anthropic.chat.transformation import ( - _build_anthropic_tool_name_maps, + build_anthropic_tool_name_maps, ) - forward, reverse = _build_anthropic_tool_name_maps(["foo/bar", "foo/bar"]) + forward, reverse = build_anthropic_tool_name_maps(["foo/bar", "foo/bar"]) # Same original sanitizes to the same target — no spurious suffix. assert forward == {"foo/bar": "foo_bar"} assert reverse == {"foo_bar": "foo/bar"} @@ -10707,6 +10816,136 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert litellm_params["metadata"] == {} +def test_a_renamed_tool_leaves_tool_results_without_tool_references_as_claude_sent_them(): + errored_search: Final = { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_01Search", + "content": {"type": "tool_search_tool_result_error", "error_code": "unavailable"}, + } + results: Final = [_CODE_EXECUTION_BLOCKS[1], errored_search] + message: Final = _parsed_assistant_message( + [ + *_CODE_EXECUTION_BLOCKS, + _tool_search_turn([], _WEATHER_TOOL_USE)[0], + errored_search, + {**_WEATHER_TOOL_USE, "name": "weather_get"}, + ], + "tool_use", + {"weather_get": "weather.get"}, + ) + config: Final = AnthropicConfig() + + request: Final = config.transform_request( + model="claude-opus-5-5", + messages=[ + {"role": "user", "content": "What is the weather in Paris?"}, + message.model_dump(), + {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"}, + ], + optional_params=config.map_openai_params( + non_default_params={ + "tools": [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + _function_tool("weather.get"), + ] + }, + optional_params={}, + model="claude-opus-5-5", + drop_params=False, + ), + litellm_params={}, + headers={}, + ) + + assert message.provider_specific_fields["tool_results"] == results + assert message.tool_calls[-1].function.name == "weather.get" + sent: Final = request["messages"][1]["content"] + assert [block for block in sent if block["type"].endswith("_tool_result")] == results + assert [block["name"] for block in sent if block["type"] == "tool_use"] == ["weather_get"] + + +@pytest.mark.parametrize( + ("last_block", "stop_reason"), + [ + pytest.param(dict(_PARIS_ANSWER), "end_turn", id="search_then_answer"), + pytest.param({**_WEATHER_TOOL_USE, "name": "weather_get"}, "tool_use", id="search_then_tool"), + ], +) +def test_a_renamed_tool_keeps_the_replayed_search_that_loaded_it(last_block: dict[str, object], stop_reason: str): + search_turn: Final = _tool_search_turn(["weather_get"], last_block) + message: Final = _parsed_assistant_message(search_turn, stop_reason, {"weather_get": "weather.get"}) + answers: Final = ( + [{"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"}] if stop_reason == "tool_use" else [] + ) + config: Final = AnthropicConfig() + + request: Final = config.transform_request( + model="claude-opus-5-5", + messages=[ + {"role": "user", "content": "What is the weather in Paris?"}, + message.model_dump(), + *answers, + {"role": "user", "content": "And in Osaka?"}, + ], + optional_params=config.map_openai_params( + non_default_params={ + "tools": [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + _function_tool("weather.get"), + ] + }, + optional_params={}, + model="claude-opus-5-5", + drop_params=False, + ), + litellm_params={}, + headers={}, + ) + + sent: Final = request["messages"][1]["content"] + assert [block for block in sent if block["type"] == "tool_search_tool_result"] == [search_turn[1]] + assert [block["id"] for block in sent if block["type"] == "server_tool_use"] == ["srvtoolu_01Search"] + + +def _parsed_tool_search_fields( + content: list[dict[str, object]], tool_name_reverse_map: dict[str, str] +) -> tuple[object, list[str]]: + message: Final = _parsed_assistant_message(content, "tool_use", tool_name_reverse_map) + return message.provider_specific_fields["tool_results"], [call.function.name for call in message.tool_calls] + + +def _streamed_tool_search_fields( + content: list[dict[str, object]], tool_name_reverse_map: dict[str, str] +) -> tuple[object, list[str]]: + iterator: Final = ModelResponseIterator(None, sync_stream=True, tool_name_reverse_map=tool_name_reverse_map) + events: Final = chain.from_iterable( + ( + {"type": "content_block_start", "index": index, "content_block": block}, + {"type": "content_block_stop", "index": index}, + ) + for index, block in enumerate(content) + ) + deltas: Final = [iterator.chunk_parser(event).choices[0].delta for event in events] + fields: Final = [delta.provider_specific_fields for delta in deltas if delta.provider_specific_fields] + calls: Final = chain.from_iterable(delta.tool_calls or () for delta in deltas) + return fields[-1]["tool_results"], [call.function.name for call in calls] + + +@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"]) +def test_a_renamed_tool_search_result_keeps_claudes_names_so_an_older_proxy_accepts_the_replay(stream: bool): + search_turn: Final = _tool_search_turn(["weather_get"], {**_WEATHER_TOOL_USE, "name": "weather_get", "input": {}}) + reverse_map: Final = {"weather_get": "weather.get"} + + tool_results, tool_names = ( + _streamed_tool_search_fields(search_turn, reverse_map) + if stream + else _parsed_tool_search_fields(search_turn, reverse_map) + ) + + assert tool_results == [search_turn[1]] + assert tool_names == ["weather.get"] + + _INPUT_TRANSFORMATIONS: Final = [{"type": "thinking_block_binding", "count": 2}] diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 14cabb683f1..4e8dc69fd01 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -28,7 +28,12 @@ import pytest import respx import litellm -from litellm.llms.anthropic.common_utils import ToolsWithoutStrict, is_claude_tool_target, tools_without_strict +from litellm.llms.anthropic.common_utils import ( + ToolsWithoutStrict, + is_claude_tool_target, + tools_without_strict, + without_stale_tool_search_results, +) from litellm.llms.custom_httpx.http_handler import HTTPHandler sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))) @@ -4709,6 +4714,79 @@ def test_is_claude_tool_target(custom_llm_provider: str, model: str, expected: b assert is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model) is expected +def test_dropping_a_stale_tool_search_keeps_every_other_turn_as_it_was_sent() -> None: + search_use: Final = { + "type": "server_tool_use", + "id": "srvtoolu_01Search", + "name": "tool_search_tool_regex", + "input": {"pattern": "weather"}, + } + search_result: Final = { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_01Search", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}], + }, + } + weather_call: Final = { + "type": "tool_use", + "id": "toolu_01Weather", + "name": "get_weather", + "input": {"city": "Paris"}, + } + messages: Final = [ + {"role": "user", "content": "What is the weather in Paris?"}, + {"role": "assistant", "content": [search_use, search_result, weather_call]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_01Weather", "content": "18C"}]}, + {"role": "assistant", "content": "It is 18C in Paris."}, + {"role": "user", "content": "Thanks"}, + {"role": "assistant", "content": [{"type": "text", "text": "Anything else?"}]}, + {"role": "user", "content": "And in Osaka?"}, + ] + + kept: Final = without_stale_tool_search_results(messages, [{"name": "get_local_time"}]) + + assert kept == (messages[0], {"role": "assistant", "content": [weather_call]}, *messages[2:]) + + +def test_a_replayed_tool_search_keeps_the_tools_it_found_that_the_follow_up_still_sends() -> None: + def reference(name: str) -> dict[str, str]: + return {"type": "tool_reference", "tool_name": name} + + search_use: Final = { + "type": "server_tool_use", + "id": "srvtoolu_01Search", + "name": "tool_search_tool_regex", + "input": {"pattern": "weather|time"}, + } + + def search_result(*names: str) -> dict[str, object]: + return { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_01Search", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [reference(name) for name in names], + }, + } + + weather_call: Final = {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}} + messages: Final = [ + {"role": "user", "content": "What is the weather and the time in Paris?"}, + {"role": "assistant", "content": [search_use, search_result("get_weather", "get_local_time"), weather_call]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_01Weather", "content": "18C"}]}, + ] + + kept: Final = without_stale_tool_search_results(messages, [{"name": "get_weather"}, {"name": "get_news"}]) + + assert kept == ( + messages[0], + {"role": "assistant", "content": [search_use, search_result("get_weather"), weather_call]}, + messages[2], + ) + + @pytest.mark.parametrize( "tools", [ diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 4f17f8b10e1..5e056e494a1 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -1164,6 +1164,92 @@ def test_bedrock_chat_invoke_tool_search_beta_follows_model_map( assert result.get("anthropic_beta") == expected_betas +_REPLAYED_WEATHER_SEARCH: Final = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "toolu_01Weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'}, + } + ], + "tool_results": [ + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_01Search", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}], + }, + } + ], + }, +} +_PLAIN_WEATHER_CALL: Final = { + key: value for key, value in _REPLAYED_WEATHER_SEARCH.items() if key != "provider_specific_fields" +} + + +@pytest.mark.parametrize( + ("model", "assistant_turn", "expected_betas"), + [ + pytest.param( + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + _REPLAYED_WEATHER_SEARCH, + ["tool-search-tool-2025-10-19"], + id="replayed_search", + ), + pytest.param("us.anthropic.claude-sonnet-4-5-20250929-v1:0", _PLAIN_WEATHER_CALL, None, id="no_search"), + pytest.param( + "us.anthropic.claude-opus-4-1-20250805-v1:0", _REPLAYED_WEATHER_SEARCH, None, id="opus_4_1_unsupported" + ), + ], +) +def test_bedrock_chat_invoke_sends_the_tool_search_beta_for_a_search_replayed_without_the_search_tool( + local_model_cost_map, local_beta_headers_config, model, assistant_turn, expected_betas +): + config: Final = AmazonAnthropicClaudeConfig() + result: Final = config.transform_request( + model=model, + messages=[ + {"role": "user", "content": "What is the weather in Paris?"}, + assistant_turn, + {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"}, + {"role": "user", "content": "And in Osaka?"}, + ], + optional_params=config.map_openai_params( + non_default_params={ + "max_tokens": 64, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + } + ], + }, + optional_params={}, + model=model, + drop_params=False, + ), + litellm_params={}, + headers={}, + ) + + assert result.get("anthropic_beta") == expected_betas + + def test_bedrock_chat_invoke_adds_thinking_display_updates_beta( local_model_cost_map, local_beta_headers_config ) -> None: diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 7daf773d55b..8902cc59f0f 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -9,6 +9,7 @@ import copy import json import uuid from collections.abc import Callable +from itertools import chain from typing import Any, Final, List, Literal, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch @@ -5245,6 +5246,130 @@ class TestCompactionOutputSummaryReachesOutputGuardrails: assert ended_key(self.SUMMARY) != ended_key("The user lives elsewhere.") +@pytest.mark.asyncio +async def test_output_guardrail_rewrite_reaches_the_text_of_an_openai_sdk_message_item() -> None: + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + response: Final = ResponsesAPIResponse( + id="resp_123", + created_at=1234567890, + model="gpt-4", + object="response", + status="completed", + output=[ + ResponseOutputMessage( + type="message", + id="msg_123", + status="completed", + role="assistant", + content=[ResponseOutputText(type="output_text", text="My SSN is 123-45-6789", annotations=[])], + ) + ], + ) + + await OpenAIResponsesHandler().process_output_response( + response, TextsReplacingGuardrail(guardrail_name="masker", texts=("My SSN is [MASKED]",)) + ) + + assert [part.text for part in response.output[0].content] == ["My SSN is [MASKED]"] + + +class _WordMaskingGuardrail(CustomGuardrail): + def __init__(self, word: str) -> None: + super().__init__(guardrail_name="word-masker") + self.word = word + self.seen_tool_call_arguments: list[str] = [] + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + messages: Final = inputs.get("structured_messages") or [] + tool_calls: Final = chain.from_iterable(message.get("tool_calls") or [] for message in messages) + self.seen_tool_call_arguments.extend(tool_call["function"]["arguments"] for tool_call in tool_calls) + return {**inputs, "structured_messages": json.loads(json.dumps(messages).replace(self.word, "[MASKED]"))} + + +_REPLAYED_CLAUDE_SEARCH_INPUT: Final = ( + {"role": "user", "content": "What is the weather in Paris?"}, + { + "type": "tool_search_call", + "id": "tsc_srvtoolu_01Search", + "call_id": "srvtoolu_01Search", + "execution": "server", + "status": "completed", + "arguments": {"pattern": "Paris weather"}, + }, + { + "type": "tool_search_output", + "id": "tso_srvtoolu_01Search", + "call_id": "srvtoolu_01Search", + "execution": "server", + "status": "completed", + "tools": [{"type": "function", "name": "get_weather", "parameters": {"type": "object"}, "defer_loading": True}], + }, + {"type": "function_call", "call_id": "toolu_01GuardrailWeather", "name": "get_weather", "arguments": '{"city": "Paris"}'}, + {"type": "function_call_output", "call_id": "toolu_01GuardrailWeather", "output": "14C"}, + {"role": "user", "content": "Thanks"}, +) + + +@pytest.mark.asyncio +async def test_a_guardrail_masks_the_arguments_of_a_replayed_claude_tool_search_and_keeps_the_search() -> None: + guardrail: Final = _WordMaskingGuardrail("Paris") + data: Final = { + "model": "anthropic/claude-sonnet-5-5", + "input": copy.deepcopy(list(_REPLAYED_CLAUDE_SEARCH_INPUT)), + "tools": [{"type": "tool_search"}], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(data, guardrail) + + assert guardrail.seen_tool_call_arguments == ['{"pattern": "Paris weather"}', '{"city": "Paris"}'] + assert result["input"][1:4] == [ + {**_REPLAYED_CLAUDE_SEARCH_INPUT[1], "arguments": {"pattern": "[MASKED] weather"}}, + _REPLAYED_CLAUDE_SEARCH_INPUT[2], + {**_REPLAYED_CLAUDE_SEARCH_INPUT[3], "arguments": '{"city": "[MASKED]"}'}, + ] + assert "Paris" not in json.dumps(result["input"]) + + +@pytest.mark.asyncio +async def test_a_follow_up_without_a_search_tool_still_scans_the_claude_search_and_keeps_it_through_a_rewrite() -> None: + guardrail: Final = _WordMaskingGuardrail("Paris") + sent_input: Final = [{"role": "user", "content": "What is the weather?"}, *_REPLAYED_CLAUDE_SEARCH_INPUT[1:]] + data: Final = { + "model": "anthropic/claude-sonnet-5-5", + "input": copy.deepcopy(sent_input), + "tools": [{"type": "function", "name": "get_weather", "parameters": {"type": "object"}}], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(data, guardrail) + + assert guardrail.seen_tool_call_arguments == ['{"pattern": "Paris weather"}', '{"city": "Paris"}'] + assert result["input"][1:4] == [ + {**sent_input[1], "arguments": {"pattern": "[MASKED] weather"}}, + sent_input[2], + {**sent_input[3], "arguments": '{"city": "[MASKED]"}'}, + ] + + +@pytest.mark.asyncio +async def test_a_guardrail_rewriting_another_turn_leaves_a_replayed_claude_tool_search_as_sent() -> None: + data: Final = { + "model": "anthropic/claude-sonnet-5-5", + "input": copy.deepcopy(list(_REPLAYED_CLAUDE_SEARCH_INPUT)), + "tools": [{"type": "tool_search"}], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(data, _WordMaskingGuardrail("Thanks")) + + assert result["input"] == [*_REPLAYED_CLAUDE_SEARCH_INPUT[:5], {"role": "user", "content": "[MASKED]"}] + + _REFUSAL: Final = "I can't share where Jane lives." _MASKED_REFUSAL: Final = "I can't share where [NAME] lives." diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py index ea6892379c2..8500a1083df 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py @@ -2805,6 +2805,46 @@ class TestRegistration: ) +def test_responses_input_scan_includes_the_arguments_of_a_replayed_claude_tool_search(): + guardrail = _make_guardrail() + data = { + "model": "claude-sonnet-5-5", + "input": [ + {"role": "user", "content": "What is the weather?"}, + { + "type": "tool_search_call", + "id": "tsc_srvtoolu_01Search", + "call_id": "srvtoolu_01Search", + "arguments": {"pattern": "ssn 123-45-6789"}, + "execution": "server", + "status": "completed", + }, + { + "type": "tool_search_output", + "id": "tso_01Search", + "call_id": "srvtoolu_01Search", + "tools": [{"type": "function", "name": "get_weather"}], + "status": "completed", + }, + { + "type": "function_call", + "id": "fc_01Weather", + "call_id": "toolu_01Weather", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "status": "completed", + }, + {"type": "function_call_output", "call_id": "toolu_01Weather", "output": "14C"}, + ], + } + + scanned = guardrail._responses_api_input_to_str(data) + + assert scanned is not None + assert "123-45-6789" in scanned + assert "Paris" in scanned + + @pytest.mark.parametrize("odata_type", [None, 7], ids=["null", "number"]) def test_should_block_rejects_a_policy_action_whose_odata_type_is_not_text(odata_type: object) -> None: response: Final = {"policyActions": [{"@odata.type": odata_type, "restrictionAction": "block"}]} diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index 5ae985ea7e5..cbd26618312 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -18,6 +18,7 @@ from litellm.responses.litellm_completion_transformation.handler import ( LiteLLMCompletionTransformationHandler, ) from litellm.responses.litellm_completion_transformation.session_handler import StoredTurns +from litellm.responses.litellm_completion_transformation.tool_search import anthropic_tool_search_chat_tool from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -372,6 +373,9 @@ def _search_turn(tool_calls: Sequence[Mapping[str, object]]) -> Mapping[str, obj } +_ANTHROPIC_SEARCH_TOOL: Final = anthropic_tool_search_chat_tool() + + async def _follow_up_messages( tool_calls: Sequence[Mapping[str, object]], model: str, custom_llm_provider: str ) -> Sequence[object]: @@ -379,7 +383,10 @@ async def _follow_up_messages( async def _follow_up_messages_after( - spend_log: Mapping[str, object], model: str, custom_llm_provider: str + spend_log: Mapping[str, object], + model: str, + custom_llm_provider: str, + tools: Sequence[Mapping[str, object]] = (_ANTHROPIC_SEARCH_TOOL,), ) -> Sequence[object]: with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([spend_log])): request: Final = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( @@ -388,6 +395,7 @@ async def _follow_up_messages_after( "model": model, "custom_llm_provider": custom_llm_provider, "messages": [{"role": "user", "content": "And in Osaka?"}], + **({"tools": [dict(tool) for tool in tools]} if tools else {}), }, instructions=None, ) @@ -492,7 +500,12 @@ def _history_tool_ids(messages: Sequence[object]) -> tuple[Sequence[str], Sequen @pytest.mark.parametrize("search_answered", [False, True]) @pytest.mark.parametrize( ("model", "custom_llm_provider", "expect_search"), - [("mistral/mistral-small-latest", "mistral", False), (_MODEL, "anthropic", True)], + [ + ("mistral/mistral-small-latest", "mistral", False), + ("anthropic/claude-3-7-sonnet-20250219", "anthropic", True), + ("azure_ai/claude-deployment", "azure_ai", True), + (_MODEL, "anthropic", True), + ], ) async def test_previous_response_id_drops_a_client_replayed_search_call_only_off_anthropic( model: str, custom_llm_provider: str, expect_search: bool, search_answered: bool @@ -1598,6 +1611,369 @@ async def test_previous_response_id_keeps_the_tools_an_earlier_codex_tool_search assert prisma_client.db.queries == 1 +_WEATHER_TOOL: Final = { + "type": "function", + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + "defer_loading": True, +} +_SEARCHABLE_TOOLS: Final = ({"type": "tool_search"}, _WEATHER_TOOL) +_WEATHER_QUESTION: Final = ({"role": "user", "content": "What is the weather in Paris?"},) +_SEARCH_USE: Final = { + "type": "server_tool_use", + "id": "srvtoolu_01Search", + "name": "tool_search_tool_regex", + "input": {"pattern": "weather"}, +} +_SEARCH_RESULT: Final = { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_01Search", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}], + }, +} +_WEATHER_USE: Final = {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}} +_WEATHER_ANSWER: Final = {"type": "function_call_output", "call_id": "toolu_01Weather", "output": "18C and sunny"} + + +_TOOL_NAMES_ON_EACH_SIDE: Final = pytest.mark.parametrize( + ("caller_name", "claude_name"), + [("get_weather", "get_weather"), ("weather.get", "weather_get")], + ids=["valid_name", "name_anthropic_rewrites"], +) + + +def _search_result_finding(tool_name: str) -> Mapping[str, object]: + return { + **_SEARCH_RESULT, + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [{"type": "tool_reference", "tool_name": tool_name}], + }, + } + + +@_TOOL_NAMES_ON_EACH_SIDE +def test_responses_over_anthropic_return_the_tool_search_output_and_send_the_search_back_without_a_stored_turn( + respx_mock: respx.MockRouter, caller_name: str, claude_name: str +) -> None: + tools: Final = [{"type": "tool_search"}, {**_WEATHER_TOOL, "name": caller_name}] + search_result: Final = _search_result_finding(claude_name) + weather_use: Final = {**_WEATHER_USE, "name": claude_name} + route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + return_value=_anthropic_message(_SEARCH_USE, search_result, weather_use) + ) + + first: Final = litellm.responses(model=_MODEL, input=list(_WEATHER_QUESTION), tools=tools, api_key="mock_api_key") + litellm.responses( + model=_MODEL, + input=[*_WEATHER_QUESTION, *(item.model_dump(exclude_none=True) for item in first.output), _WEATHER_ANSWER], + tools=tools, + api_key="mock_api_key", + ) + + assert [item.type for item in first.output] == ["tool_search_call", "tool_search_output", "function_call"] + search_output: Final = first.output[1] + assert (search_output.id, search_output.call_id) == ("tso_srvtoolu_01Search", "srvtoolu_01Search") + assert (search_output.execution, search_output.status) == ("server", "completed") + assert [dict(tool) for tool in search_output.tools] == tools[1:] + assert first.output[2].name == caller_name + assert _sent_assistant_content(route) == [_SEARCH_USE, search_result, weather_use] + + + +@pytest.mark.parametrize( + ("search_tools", "search_use"), + [ + pytest.param((), _SEARCH_USE, id="regex"), + pytest.param( + ({"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"},), + {**_SEARCH_USE, "name": "tool_search_tool_bm25", "input": {"query": "weather"}}, + id="bm25", + ), + ], +) +def test_responses_over_anthropic_send_a_search_back_under_the_anthropic_search_tool_that_ran_it( + respx_mock: respx.MockRouter, search_tools: tuple[Mapping[str, object], ...], search_use: Mapping[str, object] +) -> None: + tools: Final = [{"type": "tool_search"}, *search_tools, _WEATHER_TOOL] + route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + return_value=_anthropic_message(search_use, _SEARCH_RESULT, _WEATHER_USE) + ) + + first: Final = litellm.responses(model=_MODEL, input=list(_WEATHER_QUESTION), tools=tools, api_key="mock_api_key") + litellm.responses( + model=_MODEL, + input=[*_WEATHER_QUESTION, *(item.model_dump(exclude_none=True) for item in first.output), _WEATHER_ANSWER], + tools=tools, + api_key="mock_api_key", + ) + + assert _sent_assistant_content(route) == [search_use, _SEARCH_RESULT, _WEATHER_USE] + +def _tool_search_stream(*events: Mapping[str, object]) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=_anthropic_sse( + { + "type": "message_start", + "message": { + "id": "msg_tool_search_stream", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [], + "usage": {"input_tokens": 9, "output_tokens": 1}, + }, + }, + *events, + {"type": "message_delta", "delta": {"stop_reason": "tool_use"}, "usage": {"output_tokens": 4}}, + {"type": "message_stop"}, + ), + ) + + +def _streamed_tool_use(index: int, block: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return ( + {"type": "content_block_start", "index": index, "content_block": {**block, "input": {}}}, + { + "type": "content_block_delta", + "index": index, + "delta": {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}, + }, + {"type": "content_block_stop", "index": index}, + ) + + +@_TOOL_NAMES_ON_EACH_SIDE +def test_streamed_responses_over_anthropic_emit_the_tool_search_output_in_order_at_its_completed_position( + respx_mock: respx.MockRouter, caller_name: str, claude_name: str +) -> None: + tools: Final = [{"type": "tool_search"}, {**_WEATHER_TOOL, "name": caller_name}] + respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + return_value=_tool_search_stream( + *_streamed_tool_use(0, _SEARCH_USE), + {"type": "content_block_start", "index": 1, "content_block": _search_result_finding(claude_name)}, + {"type": "content_block_stop", "index": 1}, + *_streamed_tool_use(2, {**_WEATHER_USE, "name": claude_name}), + ) + ) + + events: Final = tuple( + litellm.responses(model=_MODEL, input=list(_WEATHER_QUESTION), tools=tools, stream=True, api_key="mock") + ) + + completed: Final = events[-1].response.output + added: Final = { + event.item.id: event.output_index + for event in events + if event.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + } + done: Final = [event for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE] + assert [item.type for item in completed] == ["tool_search_call", "tool_search_output", "function_call"] + assert [event.output_index for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] == [ + 0, + 1, + 2, + ] + assert [(event.output_index, event.item.id) for event in done] == [ + (index, item.id) for index, item in enumerate(completed) + ] + assert {event.item.id: event.output_index for event in done} == added + search_output: Final = next(event.item for event in done if event.item.type == "tool_search_output") + assert search_output.model_dump() == completed[1].model_dump() + assert (search_output.call_id, search_output.status) == ("srvtoolu_01Search", "completed") + assert [dict(tool) for tool in search_output.tools] == tools[1:] + assert completed[2].name == caller_name + + +_STATELESS_SEARCH_TURN: Final = { + **_search_turn([]), + "proxy_server_request": { + "model": _MODEL, + "input": [ + *_WEATHER_QUESTION, + { + "type": "tool_search_call", + "id": "tsc_srvtoolu_01Search", + "call_id": "srvtoolu_01Search", + "arguments": {"pattern": "weather"}, + "execution": "server", + "status": "completed", + }, + { + "type": "tool_search_output", + "id": "tso_srvtoolu_01Search", + "call_id": "srvtoolu_01Search", + "execution": "server", + "status": "completed", + "tools": [_WEATHER_TOOL], + }, + { + "type": "function_call", + "id": "fc_toolu_01Weather", + "call_id": "toolu_01Weather", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, + _WEATHER_ANSWER, + ], + }, + "response": { + "id": "chatcmpl-search-turn", + "object": "chat.completion", + "created": 0, + "model": _MODEL, + "choices": [ + {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "It is 18C in Paris."}} + ], + }, +} + + +@pytest.mark.asyncio +async def test_previous_response_id_sends_back_a_search_the_stored_turn_got_as_input_items() -> None: + anthropic: Final = _RecordingRawAnthropicMessages() + + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_STATELESS_SEARCH_TURN])): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-search-turn", + input="And in Osaka?", + tools=list(_SEARCHABLE_TOOLS), + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + messages: Final = json.loads(anthropic.body)["messages"] + assert next(message["content"] for message in messages if message["role"] == "assistant") == [ + _SEARCH_USE, + _SEARCH_RESULT, + _WEATHER_USE, + ] + + +@pytest.mark.asyncio +async def test_previous_response_id_keeps_a_stored_search_when_the_follow_up_sends_no_search_tool() -> None: + anthropic: Final = _RecordingRawAnthropicMessages() + loaded_weather_tool: Final = {key: value for key, value in _WEATHER_TOOL.items() if key != "defer_loading"} + + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_STATELESS_SEARCH_TURN])): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-search-turn", + input="And in Osaka?", + tools=[loaded_weather_tool], + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + messages: Final = json.loads(anthropic.body)["messages"] + assert next(message["content"] for message in messages if message["role"] == "assistant") == [ + _SEARCH_USE, + _SEARCH_RESULT, + _WEATHER_USE, + ] + + +_SERVER_HELD_SEARCH_TURN: Final = { + **_search_turn([]), + "proxy_server_request": {"model": _MODEL, "input": list(_WEATHER_QUESTION)}, + "response": { + "id": "chatcmpl-search-turn", + "object": "chat.completion", + "created": 0, + "model": _MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "toolu_01Weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'}, + } + ], + "tool_results": [_SEARCH_RESULT], + }, + }, + } + ], + }, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tools", + [ + list(_SEARCHABLE_TOOLS), + [{key: value for key, value in _WEATHER_TOOL.items() if key != "defer_loading"}], + ], + ids=["follow_up_sends_the_search_tool", "follow_up_sends_no_search_tool"], +) +async def test_previous_response_id_replays_a_search_claude_ran_whatever_search_tool_the_follow_up_sends( + tools: list[Mapping[str, object]], +) -> None: + anthropic: Final = _RecordingRawAnthropicMessages() + + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_SERVER_HELD_SEARCH_TURN])): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-search-turn", + input=[_WEATHER_ANSWER, {"role": "user", "content": "And in Osaka?"}], + tools=tools, + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + messages: Final = json.loads(anthropic.body)["messages"] + assert next(message["content"] for message in messages if message["role"] == "assistant") == [ + _SEARCH_USE, + _SEARCH_RESULT, + _WEATHER_USE, + ] + + +@pytest.mark.asyncio +async def test_previous_response_id_drops_a_search_the_stored_turn_got_as_input_items_off_anthropic() -> None: + messages: Final = await _follow_up_messages_after(_STATELESS_SEARCH_TURN, "mistral/mistral-small-latest", "mistral") + + assert _history_tool_ids(messages) == (["toolu_01Weather"], ["toolu_01Weather"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [ + ("anthropic/claude-3-7-sonnet-20250219", "anthropic"), + ("azure_ai/claude-deployment", "azure_ai"), + ("bedrock/invoke/us.anthropic.claude-sonnet-5-5", "bedrock"), + ], +) +async def test_previous_response_id_keeps_a_stored_search_on_claude_routes_the_cost_map_does_not_flag( + model: str, custom_llm_provider: str +) -> None: + messages: Final = await _follow_up_messages_after(_STATELESS_SEARCH_TURN, model, custom_llm_provider) + + assert _history_tool_ids(messages)[0] == ["srvtoolu_01Search", "toolu_01Weather"] + + @pytest.mark.asyncio async def test_a_codex_additional_tools_item_sends_claude_its_tools_but_not_its_text() -> None: anthropic: Final = _RecordingRawAnthropicMessages() diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 31ab4c6ac8c..8fb066d437f 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -35,7 +35,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( ) from litellm.types.llms.base import HiddenParams from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse -from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call +from litellm.types.responses.main import OutputToolSearchCall, OutputToolSearchOutput, build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, Choices, @@ -1696,6 +1696,77 @@ def _chat_response_with_tool_search_call( ) +_STOCK_PRICE_TOOL: Final = {"type": "function", "name": "get_stock_price", "defer_loading": True} +_FX_RATE_TOOL: Final = {"type": "function", "name": "get_fx_rate", "defer_loading": True} +_CONVERT_TOOL: Final = {"type": "custom", "name": "convert_currency", "defer_loading": True} +_FINANCE_NAMESPACE: Final = { + "type": "namespace", + "name": "finance", + "description": "Market data", + "tools": [_STOCK_PRICE_TOOL, _FX_RATE_TOOL, _CONVERT_TOOL, {"type": "web_search"}], +} +_UNNAMED_TOOL: Final = {"type": "function", "name": None} + + +def _chat_response_with_tool_search_result(result_content: Mapping[str, object]) -> ModelResponse: + response: Final = _chat_response_with_tool_search_call('{"pattern": "weather"}') + message: Final = response.choices[0].message + return response.model_copy( + update={ + "choices": [ + response.choices[0].model_copy( + update={ + "message": Message( + content=None, + tool_calls=message.tool_calls, + provider_specific_fields={ + "tool_results": [ + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_01Search", + "content": result_content, + } + ] + }, + ) + } + ) + ] + } + ) + + +def _stateless_tool_search_input( + call_id: str | None, output_status: Literal["completed", "incomplete"] +) -> Sequence[Mapping[str, object]]: + return [ + {"type": "message", "role": "user", "content": "What is the weather in Paris right now?"}, + { + "type": "tool_search_call", + "id": "tsc_01", + "call_id": call_id, + "arguments": {"pattern": "weather"}, + "execution": "server", + "status": "completed", + }, + { + "type": "tool_search_output", + "id": "tso_01", + "call_id": call_id, + "execution": "server", + "status": output_status, + "tools": [_deferred_tools_with_server_tool_search()[1]], + }, + { + "type": "function_call", + "call_id": "toolu_01Weather", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, + {"type": "function_call_output", "call_id": "toolu_01Weather", "output": "18C and sunny"}, + ] + + class TestToolTransformation: """Test cases for tool transformation from Responses API to Chat Completion format""" @@ -2250,6 +2321,201 @@ class TestToolTransformation: assert [call["id"] for call in messages[1]["tool_calls"]] == ["toolu_01Weather"] assert "tool_search" not in json.dumps(messages) + @pytest.mark.parametrize( + ("result_content", "status", "loaded_tools"), + [ + pytest.param( + { + "type": "tool_search_tool_search_result", + "tool_references": [ + {"type": "tool_reference", "tool_name": name} + for name in ("get_weather", "finance__get_stock_price", "convert_currency", "not_a_tool") + ], + }, + "completed", + [ + _deferred_tools_with_server_tool_search()[1], + {**_FINANCE_NAMESPACE, "tools": [_STOCK_PRICE_TOOL, _CONVERT_TOOL]}, + ], + id="found_tools", + ), + pytest.param( + {"type": "tool_search_tool_result_error", "error_code": "unavailable"}, + "incomplete", + [], + id="search_failed", + ), + ], + ) + def test_anthropic_tool_search_result_becomes_a_tool_search_output_of_the_tools_as_sent( + self, + result_content: Mapping[str, object], + status: str, + loaded_tools: list[Mapping[str, object]], + ) -> None: + items: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( + chat_completion_response=_chat_response_with_tool_search_result(result_content), + responses_api_request={ + "tools": [*_deferred_tools_with_server_tool_search()[:2], _FINANCE_NAMESPACE, _UNNAMED_TOOL], + }, + ) + + assert [item.type for item in items] == ["tool_search_call", "tool_search_output", "function_call"] + search_output: Final = items[1] + assert isinstance(search_output, OutputToolSearchOutput) + assert (search_output.call_id, search_output.execution, search_output.status) == ( + "srvtoolu_01Search", + "server", + status, + ) + assert [dict(tool) for tool in search_output.tools] == loaded_tools + + @pytest.mark.parametrize( + ("model", "custom_llm_provider", "call_id", "output_status", "replayed_call_ids"), + [ + pytest.param( + "anthropic/claude-sonnet-5-5", + "anthropic", + "srvtoolu_01Search", + "completed", + ["srvtoolu_01Search", "toolu_01Weather"], + id="claude_search_on_anthropic", + ), + pytest.param( + "anthropic/claude-sonnet-5-5", + "anthropic", + "srvtoolu_01Search", + "incomplete", + ["toolu_01Weather"], + id="failed_claude_search_on_anthropic", + ), + pytest.param( + "anthropic/claude-sonnet-5-5", "anthropic", None, "completed", ["toolu_01Weather"], id="openai_search" + ), + pytest.param( + "mistral/mistral-small-latest", + "mistral", + "srvtoolu_01Search", + "completed", + ["toolu_01Weather"], + id="claude_search_off_anthropic", + ), + pytest.param( + "anthropic/claude-3-7-sonnet-20250219", + "anthropic", + "srvtoolu_01Search", + "completed", + ["toolu_01Weather"], + id="claude_search_sent_to_a_claude_without_tool_search", + ), + ], + ) + def test_stateless_tool_search_items_are_replayed_only_for_a_completed_claude_search_on_anthropic( + self, + model: str, + custom_llm_provider: str, + call_id: str | None, + output_status: Literal["completed", "incomplete"], + replayed_call_ids: Sequence[str], + ) -> None: + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input=cast( # cast-ok: tool search items are not in the SDK's input TypedDicts + ResponseInputParam, list(_stateless_tool_search_input(call_id, output_status)) + ), + responses_api_request={"tools": _deferred_tools_with_server_tool_search()}, + custom_llm_provider=custom_llm_provider, + ) + + assert [message["role"] for message in request["messages"]] == ["user", "assistant", "tool"] + assert [call["id"] for call in request["messages"][1]["tool_calls"]] == replayed_call_ids + + @pytest.mark.parametrize( + ("model", "custom_llm_provider", "replayed_call_ids"), + [ + pytest.param( + "anthropic/claude-3-7-sonnet-20250219", + "anthropic", + ["srvtoolu_01Search", "toolu_01Weather"], + id="on_anthropic", + ), + pytest.param("mistral/mistral-small-latest", "mistral", ["toolu_01Weather"], id="off_anthropic"), + ], + ) + def test_stateless_tool_search_items_follow_anthropics_own_search_tool_on_a_claude_the_cost_map_does_not_flag( + self, model: str, custom_llm_provider: str, replayed_call_ids: Sequence[str] + ) -> None: + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input=cast( # cast-ok: tool search items are not in the SDK's input TypedDicts + ResponseInputParam, list(_stateless_tool_search_input("srvtoolu_01Search", "completed")) + ), + responses_api_request={ + "tools": [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + *_deferred_tools_with_server_tool_search()[1:], + ] + }, + custom_llm_provider=custom_llm_provider, + ) + + assert [call["id"] for call in request["messages"][1]["tool_calls"]] == replayed_call_ids + + @pytest.mark.parametrize( + "tools", + [ + pytest.param(_deferred_tools_with_server_tool_search()[1:], id="only_the_tools_it_found"), + pytest.param( + [{"type": "tool_search", "execution": "client"}, *_deferred_tools_with_server_tool_search()[1:]], + id="a_search_the_client_runs", + ), + pytest.param([], id="no_tools"), + ], + ) + def test_stateless_claude_search_is_dropped_when_the_follow_up_sends_no_search_tool_the_route_runs( + self, tools: list[dict[str, object]] + ) -> None: + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input=cast( # cast-ok: tool search items are not in the SDK's input TypedDicts + ResponseInputParam, list(_stateless_tool_search_input("srvtoolu_01Search", "completed")) + ), + responses_api_request={"tools": tools}, + custom_llm_provider="anthropic", + ) + + assert [call["id"] for call in request["messages"][1]["tool_calls"]] == ["toolu_01Weather"] + + @pytest.mark.parametrize( + ("arguments", "search_tool_name"), + [ + pytest.param({"pattern": "weather"}, "tool_search_tool_regex", id="regex_object"), + pytest.param({"query": "weather"}, "tool_search_tool_bm25", id="bm25_object"), + pytest.param('{"query": "weather"}', "tool_search_tool_bm25", id="bm25_json"), + pytest.param("weather", "tool_search_tool_regex", id="not_json"), + ], + ) + def test_stateless_claude_search_is_replayed_under_the_search_tool_its_arguments_belong_to( + self, arguments: object, search_tool_name: str + ) -> None: + items: Final = ( + {**item, "arguments": arguments} if item["type"] == "tool_search_call" else item + for item in _stateless_tool_search_input("srvtoolu_01Search", "completed") + ) + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input=cast( # cast-ok: tool search items are not in the SDK's input TypedDicts + ResponseInputParam, list(items) + ), + responses_api_request={"tools": _deferred_tools_with_server_tool_search()}, + custom_llm_provider="anthropic", + ) + + assert [(call["id"], call["function"]["name"]) for call in request["messages"][1]["tool_calls"]] == [ + ("srvtoolu_01Search", search_tool_name), + ("toolu_01Weather", "get_weather"), + ] + def test_transform_local_shell_tools_dropped(self) -> None: ( result_tools, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 29cc8782aa5..b229461e981 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -41761,6 +41761,37 @@ export interface components { } & { [key: string]: unknown; }; + /** OutputToolSearchOutput */ + OutputToolSearchOutput: { + /** Call Id */ + call_id?: string | null; + /** + * Execution + * @enum {string} + */ + execution: "server" | "client"; + /** Id */ + id: string; + /** + * Status + * @enum {string} + */ + status: "in_progress" | "completed" | "incomplete"; + /** + * Tools + * @default [] + */ + tools: { + [key: string]: unknown; + }[]; + /** + * Type + * @constant + */ + type: "tool_search_output"; + } & { + [key: string]: unknown; + }; /** * PageLinks * @description Hypermedia for a paginated list. No `first`/`last`: without a total count the last page is unknown. @@ -45853,7 +45884,7 @@ export interface components { /** Output */ output: (components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpCall"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["ResponseCustomToolCall"] | components["schemas"]["ResponseCustomToolCallOutputItem"] | { [key: string]: unknown; - })[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["OutputToolSearchCall"] | components["schemas"]["ResponseCompactionItem"])[]; + })[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["OutputToolSearchCall"] | components["schemas"]["OutputToolSearchOutput"] | components["schemas"]["ResponseCompactionItem"])[]; /** Parallel Tool Calls */ parallel_tool_calls?: boolean | null; /** Previous Response Id */ From 4714aac42bff39fa21430d308bb47c9fabd6ec3f Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 09:22:18 -0800 Subject: [PATCH 012/189] feat(rate-limit): enforce per-customer per-model tpm and rpm limits in the v3 limiter (#50857) * feat(rate-limit): enforce per-customer per-model tpm and rpm limits in the v3 limiter Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(rate-limit): build model scope settlement targets from one owner tuple Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../hooks/parallel_request_limiter_v3.py | 48 ++++++++++------ .../management/test_customer_model_limits.py | 55 +++++++++++++++++++ .../hooks/test_parallel_request_limiter_v3.py | 32 ++++++++++- 3 files changed, 116 insertions(+), 19 deletions(-) create mode 100644 tests/integration/management/test_customer_model_limits.py diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 9fb026a64d9..ae9fa548cc9 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -2507,6 +2507,26 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): }, ) + def _end_user_model_rate_limit_descriptor( + self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None + ) -> RateLimitDescriptor | None: + model: Final = self._rate_limited_model(requested_model) + if model is None or not user_api_key_dict.end_user_id: + return None + tpm_limit: Final = model.limit_in(user_api_key_dict.end_user_model_tpm_limit) + rpm_limit: Final = model.limit_in(user_api_key_dict.end_user_model_rpm_limit) + if tpm_limit is None and rpm_limit is None: + return None + return RateLimitDescriptor( + key="model_per_end_user", + value=f"{user_api_key_dict.end_user_id}:{model.group}", + rate_limit={ + "requests_per_unit": rpm_limit, + "tokens_per_unit": tpm_limit, + "window_size": self.window_size, + }, + ) + def _add_model_per_key_rate_limit_descriptor( self, user_api_key_dict: UserAPIKeyAuth, @@ -2999,6 +3019,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model=candidate, descriptors=descriptors, ) + if (end_user_model := self._end_user_model_rate_limit_descriptor(user_api_key_dict, candidate)) is not None: + descriptors.append(end_user_model) # Per-request-tag rate limits scoped to this key self._add_tag_per_key_rate_limit_descriptor( @@ -4525,26 +4547,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): targets.append(("end_user", user_api_key_end_user_id)) if user_api_key_organization_id: targets.append(("organization", user_api_key_organization_id)) + model_scope_owners: Final = ( + ("model_per_key", user_api_key), + ("model_per_team", user_api_key_team_id), + ("model_per_end_user", f"{user_api_key_end_user_id}" if user_api_key_end_user_id else None), + ("model_per_organization", user_api_key_organization_id), + ("model_per_project", user_api_key_project_id), + ) if model_group: for group in dict.fromkeys(model_groups or (model_group,)): - if user_api_key: - targets.append(("model_per_key", f"{user_api_key}:{group}")) - if user_api_key_team_id: - targets.append(("model_per_team", f"{user_api_key_team_id}:{group}")) - if user_api_key_organization_id: - targets.append( - ( - "model_per_organization", - f"{user_api_key_organization_id}:{group}", - ) - ) - if user_api_key_project_id: - targets.append( - ( - "model_per_project", - f"{user_api_key_project_id}:{group}", - ) - ) + targets.extend((scope, f"{owner}:{group}") for scope, owner in model_scope_owners if owner) if agent_id: targets.append(("agent", agent_id)) if session_id: diff --git a/tests/integration/management/test_customer_model_limits.py b/tests/integration/management/test_customer_model_limits.py new file mode 100644 index 00000000000..f1c207661d2 --- /dev/null +++ b/tests/integration/management/test_customer_model_limits.py @@ -0,0 +1,55 @@ +import uuid +from typing import Final + +import httpx +import pytest + +from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, object_value, string_value + +pytestmark: Final = pytest.mark.shares_worker_state + + +def _customer(scenario: Scenario, **fields: object) -> str: + customer: Final = f"integration-customer-{uuid.uuid4().hex}" + created: Final = scenario.gateway.post("/customer/new", {"user_id": customer, **fields}) # pyright: ignore[reportArgumentType] # JsonValue kwargs + scenario.cleanups.callback(scenario.gateway.post, "/customer/delete", {"user_ids": [customer]}) + budget_id: Final = created.get("budget_id") + if isinstance(budget_id, str): + scenario.cleanups.callback(scenario.delete_budget, budget_id) + return customer + + +def _chat(gateway: Gateway, key: str, model: str, customer: str) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"customer quota probe {uuid.uuid4().hex}"}], + "user": customer, + }, + key=key, + ) + + +def _assert_customer_model_limit_error(response: httpx.Response, customer: str, model: str) -> None: + assert response.status_code == 429, response.text + message: Final = string_value(object_value(JSON_OBJECT.validate_json(response.content)["error"])["message"]) + assert message.startswith( + f"Rate limit exceeded for model_per_end_user: {customer}:{model}. Limit type: requests. Current limit: 1," + ), message + + +def test_customer_per_model_rpm_limit_is_shared_across_keys_and_isolated_per_customer(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + capped: Final = scenario.model() + open_model: Final = scenario.model() + acme: Final = _customer(scenario, model_rpm_limit={capped: 1}) + globex: Final = _customer(scenario, model_rpm_limit={capped: 1}) + key_one: Final = scenario.key() + key_two: Final = scenario.key() + + assert _chat(gateway, key_one, capped, acme).status_code == 200 + _assert_customer_model_limit_error(_chat(gateway, key_two, capped, acme), acme, capped) + assert _chat(gateway, key_one, capped, globex).status_code == 200 + assert _chat(gateway, key_one, open_model, acme).status_code == 200 diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 741b6cb74b5..a39d02a2788 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -8247,6 +8247,11 @@ async def test_batch_increment_refunds_pipelined_groups_declared_after_the_one_t {}, "model_per_organization", ), + ( + {"end_user_id": "customer", "end_user_model_rpm_limit": {"test-model": 1}}, + {}, + "model_per_end_user", + ), ], ) @pytest.mark.parametrize("request_kind", ["count", "generation"]) @@ -8695,8 +8700,9 @@ async def test_model_group_alias_shares_deployment_default_rpm_bucket_with_its_t ), ({"org_id": "o", "organization_metadata": {"model_rpm_limit": {"target": 1}}}, "model_per_organization"), ({"project_id": "p", "project_metadata": {"model_rpm_limit": {"target": 1}}}, "model_per_project"), + ({"end_user_id": "customer", "end_user_model_rpm_limit": {"target": 1}}, "model_per_end_user"), ], - ids=["key_metadata", "team_metadata", "organization_metadata", "project_metadata"], + ids=["key_metadata", "team_metadata", "organization_metadata", "project_metadata", "end_user_budget"], ) async def test_model_group_alias_shares_metadata_model_rpm_bucket_with_its_target( limits: dict[str, object], counter_scope: str, first_name: str, second_name: str @@ -9911,3 +9917,27 @@ def test_media_models_resolve_to_each_deployment_base_model(): assert _resolve_media_models(None, "whisper", None) == ("whisper",) assert _resolve_media_models(router, "whisper", None) == ("azure/whisper-1",) assert _resolve_media_models(router, "unknown", None) == ("unknown",) + + +@pytest.mark.asyncio +async def test_end_user_model_rpm_limit_is_shared_across_keys_and_isolated_per_customer() -> None: + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + acme_via_key_one: Final = UserAPIKeyAuth( + api_key=hash_token("sk-acme-1"), end_user_id="acme", end_user_model_rpm_limit={"target": 1} + ) + acme_via_key_two: Final = UserAPIKeyAuth( + api_key=hash_token("sk-acme-2"), end_user_id="acme", end_user_model_rpm_limit={"target": 1} + ) + globex: Final = UserAPIKeyAuth( + api_key=hash_token("sk-globex"), end_user_id="globex", end_user_model_rpm_limit={"target": 1} + ) + + await _rpm_request(handler, cache, acme_via_key_one, "target") + with pytest.raises(HTTPException) as exc: + await _rpm_request(handler, cache, acme_via_key_two, "target") + assert exc.value.status_code == 429 + assert "model_per_end_user: acme:target" in str(exc.value.detail), exc.value.detail + + await _rpm_request(handler, cache, globex, "target") + await _rpm_request(handler, cache, acme_via_key_one, "other-model") From c2d7a99ecd9373eced536d4e72b8fcca6fb6c2e0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:24:50 -0800 Subject: [PATCH 013/189] test(cost): sync vertex 3.1 pro price fields and bedrock converse usage shapes (#50956) Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cases/bedrock_converse.py | 51 ++++++++----------- .../integration/cost_calculation/cost_map.py | 1 - 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/tests/integration/cost_calculation/cases/bedrock_converse.py b/tests/integration/cost_calculation/cases/bedrock_converse.py index 96033a8b49a..298c19b3e4b 100644 --- a/tests/integration/cost_calculation/cases/bedrock_converse.py +++ b/tests/integration/cost_calculation/cases/bedrock_converse.py @@ -3098,13 +3098,10 @@ US_ANTHROPIC_CLAUDE_OPUS_5_5_INPUT_TEXT: Final = CostTrackingTestCase( "output": {"message": {"role": "assistant", "content": [{"text": "Hello."}]}}, "stopReason": "end_turn", "usage": { - "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, "cacheWriteInputTokens": 0, "inputTokens": 1840, "outputTokens": 412, - "serverToolUsage": {}, "totalTokens": 2252, }, "metrics": {"latencyMs": 42}, @@ -3410,13 +3407,10 @@ US_ANTHROPIC_CLAUDE_SONNET_5_5_INPUT_TEXT: Final = CostTrackingTestCase( "output": {"message": {"content": [{"text": "scripted answer 1a50e720f40a"}], "role": "assistant"}}, "stopReason": "end_turn", "usage": { - "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, "cacheWriteInputTokens": 0, "inputTokens": 1840, "outputTokens": 412, - "serverToolUsage": {}, "totalTokens": 2252, }, }, @@ -3470,7 +3464,13 @@ US_ANTHROPIC_CLAUDE_SONNET_5_5_STREAM: Final = CostTrackingTestCase( event_type="metadata", payload={ "metrics": {"latencyMs": 883}, - "usage": {"inputTokens": 1840, "outputTokens": 412, "serverToolUsage": {}, "totalTokens": 2252}, + "usage": { + "cacheReadInputTokens": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252, + }, }, ), ), @@ -3517,13 +3517,10 @@ US_ANTHROPIC_CLAUDE_SONNET_5_5_CACHE_READ: Final = CostTrackingTestCase( "output": {"message": {"content": [{"text": "scripted answer 5eff7f685f14"}], "role": "assistant"}}, "stopReason": "end_turn", "usage": { - "cacheReadInputTokenCount": 12288, "cacheReadInputTokens": 12288, - "cacheWriteInputTokenCount": 0, "cacheWriteInputTokens": 0, "inputTokens": 640, "outputTokens": 380, - "serverToolUsage": {}, "totalTokens": 13308, }, }, @@ -3937,7 +3934,13 @@ US_ANTHROPIC_CLAUDE_OPUS_5_5_STREAM: Final = CostTrackingTestCase( event_type="metadata", payload={ "metrics": {"latencyMs": 883}, - "usage": {"inputTokens": 1840, "outputTokens": 412, "serverToolUsage": {}, "totalTokens": 2252}, + "usage": { + "cacheReadInputTokens": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252, + }, }, ), ), @@ -3984,13 +3987,10 @@ US_ANTHROPIC_CLAUDE_OPUS_5_5_CACHE_READ: Final = CostTrackingTestCase( "output": {"message": {"content": [{"text": "scripted answer bbd4da2e326b"}], "role": "assistant"}}, "stopReason": "end_turn", "usage": { - "cacheReadInputTokenCount": 12288, "cacheReadInputTokens": 12288, - "cacheWriteInputTokenCount": 0, "cacheWriteInputTokens": 0, "inputTokens": 640, "outputTokens": 380, - "serverToolUsage": {}, "totalTokens": 13308, }, }, @@ -4138,13 +4138,10 @@ US_ANTHROPIC_CLAUDE_HAIKU_5_5_INPUT_TEXT: Final = CostTrackingTestCase( "output": {"message": {"content": [{"text": "scripted answer 8ff4d0ccb5a5"}], "role": "assistant"}}, "stopReason": "end_turn", "usage": { - "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, "cacheWriteInputTokens": 0, "inputTokens": 1840, "outputTokens": 412, - "serverToolUsage": {}, "totalTokens": 2252, }, }, @@ -4198,7 +4195,13 @@ US_ANTHROPIC_CLAUDE_HAIKU_5_5_STREAM: Final = CostTrackingTestCase( event_type="metadata", payload={ "metrics": {"latencyMs": 883}, - "usage": {"inputTokens": 1840, "outputTokens": 412, "serverToolUsage": {}, "totalTokens": 2252}, + "usage": { + "cacheReadInputTokens": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252, + }, }, ), ), @@ -4245,13 +4248,10 @@ US_ANTHROPIC_CLAUDE_HAIKU_5_5_CACHE_READ: Final = CostTrackingTestCase( "output": {"message": {"content": [{"text": "scripted answer cf757e3e24df"}], "role": "assistant"}}, "stopReason": "end_turn", "usage": { - "cacheReadInputTokenCount": 12288, "cacheReadInputTokens": 12288, - "cacheWriteInputTokenCount": 0, "cacheWriteInputTokens": 0, "inputTokens": 640, "outputTokens": 380, - "serverToolUsage": {}, "totalTokens": 13308, }, }, @@ -5411,13 +5411,10 @@ US_ANTHROPIC_CLAUDE_SONNET_5_5_THINKING: Final = CostTrackingTestCase( "output": {"message": {"role": "assistant", "content": [{"text": "scripted answer fe9c9714d04d"}]}}, "stopReason": "end_turn", "usage": { - "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, "cacheWriteInputTokens": 0, "inputTokens": 1240, "outputTokens": 412, - "serverToolUsage": {}, "totalTokens": 1652, }, "metrics": {"latencyMs": 2071}, @@ -5708,13 +5705,10 @@ US_ANTHROPIC_CLAUDE_OPUS_5_5_THINKING: Final = CostTrackingTestCase( }, "stopReason": "end_turn", "usage": { - "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, "cacheWriteInputTokens": 0, "inputTokens": 1240, "outputTokens": 4040, - "serverToolUsage": {}, "totalTokens": 5280, }, "metrics": {"latencyMs": 2071}, @@ -5843,13 +5837,10 @@ US_ANTHROPIC_CLAUDE_HAIKU_5_5_THINKING: Final = CostTrackingTestCase( "output": {"message": {"role": "assistant", "content": [{"text": "scripted answer 90a6bb1996a5"}]}}, "stopReason": "end_turn", "usage": { - "cacheReadInputTokenCount": 0, "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 0, "cacheWriteInputTokens": 0, "inputTokens": 1240, "outputTokens": 412, - "serverToolUsage": {}, "totalTokens": 1652, }, "metrics": {"latencyMs": 2071}, diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 4f87caf7854..67a4fe86274 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -549,7 +549,6 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "web_search_billing_unit": "per_query", }, "vertex_ai/gemini-3.1-pro-preview": { - "cache_creation_input_token_cost_above_200k_tokens": 2.75e-07, "cache_read_input_token_cost": 2.2e-07, "cache_read_input_token_cost_above_200k_tokens": 4.4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.92e-07, From 1035971a0536d0d91933df1b053e831b75d5fa3f Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:26:46 -0800 Subject: [PATCH 014/189] fix(decisions): keep box questions on a deployment when a same-model sibling turns them off (#50724) * fix(decisions): keep a same-model deployment's false box-question flag off the shared cost-map row * fix(router): take a withdrawn supports_box_questions true off the shared row on update and delete A deployment's true reached the shared backend row, but a later false no longer overwrites it, so updating or deleting that deployment left box questions on for its unflagged siblings until restart. The row now gets the catalog's value back unless another served deployment of the model still sets true. * fix(router): count every live router and set_model_list when withdrawing a shared box-questions true * fix(router): match box-questions siblings by their cost-map row and withdraw on retire A sibling spelled with its provider registers onto the same litellm.model_cost row, so the withdrawal now compares rows instead of backend keys. Retiring a router takes back the true its deployments put on rows no other live router still sets. Withdrawals are deduped per backend so a large set_model_list stays cheap. * fix(router): take back a box-questions true an aborted set_model_list registered A set_model_list that fails partway rolls back the deployment list, but a true the aborted build already wrote to the shared cost-map row stayed there, so unflagged same-model deployments kept sending box questions. The rollback now withdraws the attempted entries too A withdrawal now only rewrites rows that still hold true, so an entry the failed build never reached gets no row written for it, even on a model the catalog does not list The delete wire test now uses a decisions-mode catalog backend whose catalog lists no box questions, since an unflagged deployment only reads the shared row for a decisions-mode model * fix(router): take back an env-named box-questions true a failed set_model_list registered The rollback took flags back from the raw model_list entries, so a deployment whose model came from os.environ/ was looked up under the reference instead of the model it registered onto, and its true stayed on the shared row It now takes them back from the entries the failed build reached, whose os.environ/ values are already read --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/router_rust/core.py | 1 + litellm/router_rust/surface/registry.py | 146 ++++++++- litellm/router_rust/surface/replay.py | 4 + litellm/types/utils.py | 15 +- litellm/utils.py | 6 + ...test_decisions_box_deployment_flag_wire.py | 67 +++++ .../unit/test_router_model_cost_isolation.py | 284 ++++++++++++++++++ 7 files changed, 511 insertions(+), 12 deletions(-) diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index 8ff0798cafa..5015795657f 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -2009,6 +2009,7 @@ class RustCore( def retire(self) -> None: """``_core_discard``: only the router's global registrations go; it keeps serving calls.""" untrack_live_registry(self._hosted) + self._hosted.retire_shared_capabilities() self._retire_callbacks() if self._hosted.strategies is not None: self._hosted.strategies.retire() diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py index 0fca3e781d9..e48569c4c9d 100644 --- a/litellm/router_rust/surface/registry.py +++ b/litellm/router_rust/surface/registry.py @@ -42,6 +42,7 @@ router and every ``registry()`` query dispatches the way it does on the Python c import copy import dataclasses +import itertools import json import os import re @@ -102,6 +103,7 @@ from litellm.router_rust.invoke.shaping import ROUTER_CLIENTS_TARGET from litellm.router_rust.surface.indexes import HostedIndexes from litellm.router_rust.surface.model_info import HostModelInfo from litellm.router_rust.surface.model_list import HostedModelList +from litellm.router_rust.surface.replay import live_registries from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model from litellm.router_utils.deployment_numbers import refuse_unservable_numbers, refuse_wide_numbers from litellm.router_utils.endpoint_eligibility import warn_on_unenforced_endpoint_flag @@ -121,6 +123,7 @@ from litellm.types.router import ( ) from litellm.types.utils import ( DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS, + SHARED_ONLY_WHEN_TRUE_CAPABILITY_FIELDS, CustomPricingLiteLLMParams, LlmProviders, LlmProvidersSet, @@ -133,6 +136,7 @@ from litellm.utils import ( get_potential_model_names, get_secret, lookup_model_info_for_registration, + model_cost_registration_key, model_info_with_pricing_overrides, ) @@ -222,6 +226,17 @@ class _Build: # step is answered as invalid and the build ends, then this is raised. failure: Exception | None = None + def reached_entries(self) -> tuple[UntypedEntry, ...]: + return tuple( + {"model_name": entry.model_name, "litellm_params": entry.litellm_params, "model_info": entry.model_info} + for entry in self.entries.values() + ) + + +def _creating(model_list: Sequence[object]) -> _Build: + raw: Final[list[UntypedEntry]] = copy.deepcopy(list(model_list)) # pyright: ignore[reportAssignmentType] # model_list entries # mutable-ok: a deep copy the build consumes + return _Build(path="create", raw=raw) + def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool: """The environment-gating rule (``model_info_is_active_for_environment``): a deployment whose @@ -551,6 +566,95 @@ def register_deployment_pricing(host: "PythonHost", deployment: Deployment) -> N ) +def _entry_section(entry: Mapping[str, object], section: str) -> Mapping[str, object]: + value: Final = entry.get(section) + return _MODEL_INFO_ADAPTER.validate_python(value) if isinstance(value, Mapping) else {} + + +def _entry_backend(entry: Mapping[str, object]) -> tuple[str, str | None] | None: + params: Final = _entry_section(entry, "litellm_params") + model: Final = params.get("model") + provider: Final = params.get("custom_llm_provider") + return (model, provider if isinstance(provider, str) else None) if isinstance(model, str) else None + + +def _entry_backend_keys(entry: Mapping[str, object]) -> frozenset[str]: + backend: Final = _entry_backend(entry) + return frozenset() if backend is None else frozenset(_backend_cost_map_keys(*backend)) + + +def _entry_row_keys(entry: Mapping[str, object]) -> frozenset[str]: + return frozenset(model_cost_registration_key(key) for key in _entry_backend_keys(entry)) + + +def _shared_capabilities_set_true(entry: Mapping[str, object]) -> frozenset[str]: + model_info: Final = _entry_section(entry, "model_info") + return frozenset(field for field in SHARED_ONLY_WHEN_TRUE_CAPABILITY_FIELDS if model_info.get(field) is True) + + +def _distinct_withdrawals(entries: Iterable[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple( + {(_entry_backend_keys(entry), _shared_capabilities_set_true(entry)): entry for entry in entries}.values() + ) + + +def _row_sets_true(host: "PythonHost", key: str, fields: frozenset[str]) -> bool: + row: Final[object] = host.dynamic().model_cost.get(model_cost_registration_key(key)) + return isinstance(row, dict) and any( + _MODEL_INFO_ADAPTER.validate_python(row).get(field) is True for field in fields + ) + + +def _catalog_sets_true(catalog: Mapping[str, Mapping[str, object]], catalog_keys: tuple[str, ...], field: str) -> bool: + entry: Final = next((catalog[key] for key in catalog_keys if key in catalog), None) + return entry is not None and entry.get(field) is True + + +def withdraw_shared_capabilities( + host: "PythonHost", + removed: Mapping[str, object], + replacement: Mapping[str, object] | None, + served: Iterable[Mapping[str, object]], +) -> None: + """For each ``SHARED_ONLY_WHEN_TRUE_CAPABILITY_FIELDS`` flag ``removed`` set ``True``, set the shared + ``litellm.model_cost`` row(s) ``removed`` registers onto that still hold it ``True`` to whether ``replacement``, + a ``served`` deployment registering onto one of them, or the catalog still sets it ``True``.""" + backend: Final = _entry_backend(removed) + set_true: Final = _shared_capabilities_set_true(removed) + if backend is None or not set_true: + return + row_keys: Final = _entry_row_keys(removed) + kept: Final = ( + _shared_capabilities_set_true(replacement) + if replacement is not None and _entry_row_keys(replacement) == row_keys + else frozenset[str]() + ) + withdrawn: Final = set_true - kept + held: Final = tuple(key for key in _entry_backend_keys(removed) if _row_sets_true(host, key, withdrawn)) + if not held: + return + candidates: Final = tuple( + entry for entry in served if not withdrawn.isdisjoint(_shared_capabilities_set_true(entry)) + ) + rows_by_backend: Final = { + keys: frozenset(model_cost_registration_key(key) for key in keys) + for keys in frozenset(_entry_backend_keys(entry) for entry in candidates) + } + siblings_set_true: Final = frozenset( + itertools.chain.from_iterable( + _shared_capabilities_set_true(entry) & withdrawn + for entry in candidates + if not rows_by_backend[_entry_backend_keys(entry)].isdisjoint(row_keys) + ) + ) + catalog: Final = catalog_with_runtime_registrations() + catalog_keys: Final = _catalog_backend_keys(*backend, _backend_cost_map_keys(*backend)) + restored: Final = { + field: field in siblings_set_true or _catalog_sets_true(catalog, catalog_keys, field) for field in withdrawn + } + host.invoke("register_model")(model_cost={key: restored for key in held}, persist_across_reloads=False) + + # -- the registry -------------------------------------------------------------------------------- @@ -850,15 +954,24 @@ class HostRegistry(HostModelInfo): """Replace every deployment (``set_model_list``). All or nothing: when an entry raises, the deployments served before the call are put back and the error is raised.""" previous: Final = [copy.deepcopy(dict(entry)) for entry in self._core_model_list] + attempt: Final = _creating(model_list) try: - self._replace_model_list(model_list) + self._replace_model_list(attempt) except Exception: - self._replace_model_list(previous) + self._replace_model_list(_creating(previous)) + self._withdraw_shared_capabilities(attempt.reached_entries(), self._served_by_live_routers()) raise + self._withdraw_shared_capabilities(previous, self._served_by_live_routers()) - def _replace_model_list(self, model_list: Sequence[object]) -> None: + def _withdraw_shared_capabilities( + self, entries: Iterable[Mapping[str, object]], served: Sequence[Mapping[str, object]] + ) -> None: + for entry in _distinct_withdrawals(entries): + withdraw_shared_capabilities(self._host, entry, None, served) + + def _replace_model_list(self, build: _Build) -> None: self._discovered_model_info_cache.flush_cache() - original_model_list: Final[list[UntypedEntry]] = copy.deepcopy(list(model_list)) # pyright: ignore[reportAssignmentType] # model_list entries # mutable-ok: a deep copy the build consumes + original_model_list: Final = build.raw if self.strategies is not None: self.strategies.reset() self._provider_unresolved_deployments = () @@ -879,12 +992,7 @@ class HostRegistry(HostModelInfo): # router-core walks the list (one deployment per organization, secrets resolved first, # ids assigned), asking this registry about each deployment in turn; it starts the list # and its id, name and team indexes over, and takes the given names as `model_names`. - self._run_build( - _Build(path="create", raw=original_model_list), - original_model_list, - resolve_secrets=True, - install="replace", - ) + self._run_build(build, original_model_list, resolve_secrets=True, install="replace") if self.strategies is not None: self.strategies.finalize() self._on_change() @@ -1312,8 +1420,11 @@ class HostRegistry(HostModelInfo): with self._deployment_index_lock: removal_idx: Final = self.model_id_to_deployment_index_map.get(model_id) if removal_idx is not None: - _ = self._core_model_list[removal_idx] # pyright: ignore[reportArgumentType] # raises IndexError for a position past the list, as the pop does + removed: Final = self._core_model_list[removal_idx] # pyright: ignore[reportArgumentType] # raises IndexError for a position past the list, as the pop does self._remove(model_id) + withdraw_shared_capabilities( + self._host, removed, deployment.to_json(exclude_none=True), self._served_by_live_routers() + ) if self.strategies is not None: self.strategies.release(on_router) self._on_change() @@ -1490,6 +1601,18 @@ class HostRegistry(HostModelInfo): self._registry_version += 1 self._on_change() + def _served_by_other_routers(self) -> tuple[UntypedEntry, ...]: + others: Final = (tuple(registry._core_model_list) for registry in live_registries() if registry is not self) + return tuple(itertools.chain.from_iterable(others)) + + def _served_by_live_routers(self) -> tuple[UntypedEntry, ...]: + return (*self._core_model_list, *self._served_by_other_routers()) + + def retire_shared_capabilities(self) -> None: + """Take back the ``SHARED_ONLY_WHEN_TRUE_CAPABILITY_FIELDS`` flags this router's deployments put on shared + ``litellm.model_cost`` rows that no other live router's deployment still sets (``retire``).""" + self._withdraw_shared_capabilities(self._core_model_list, self._served_by_other_routers()) + def _core_delete_deployment(self, id: str) -> UntypedEntry | None: """Remove one deployment and return its stored entry (``_delete_deployment``).""" try: @@ -1499,6 +1622,7 @@ class HostRegistry(HostModelInfo): return None item: Final = self._core_model_list[deployment_idx] # pyright: ignore[reportArgumentType] # raises IndexError for a position past the list, as the pop does self._remove(id) + withdraw_shared_capabilities(self._host, item, None, self._served_by_live_routers()) if self.checks is not None: self.checks.unregister_deployment_budget(id) if self.strategies is not None: diff --git a/litellm/router_rust/surface/replay.py b/litellm/router_rust/surface/replay.py index 6c770e07883..cd78b1ec76e 100644 --- a/litellm/router_rust/surface/replay.py +++ b/litellm/router_rust/surface/replay.py @@ -22,6 +22,10 @@ def untrack_live_registry(registry: HostRegistry) -> None: _live_registries.discard(registry) +def live_registries() -> tuple[HostRegistry, ...]: + return tuple(_live_registries) + + def _replay_live_registries() -> None: for registry in tuple(_live_registries): registry.replay_model_cost_registrations() diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3ddeb118c33..ffc9e915ea4 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4344,6 +4344,14 @@ SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( - DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS ) +SHARED_ONLY_WHEN_TRUE_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset({"supports_box_questions"}) + + +def _belongs_on_shared_backend_key(field: str, value: object) -> bool: + if field in SHARED_ONLY_WHEN_TRUE_CAPABILITY_FIELDS: + return value is True + return field in SHARED_BACKEND_MODEL_INFO_FIELDS + def shared_backend_model_info(model_info: dict[str, Any]) -> dict[str, Any]: """Return only the fields safe to register under a shared ``{provider}/{model}`` @@ -4352,8 +4360,13 @@ def shared_backend_model_info(model_info: dict[str, Any]) -> dict[str, Any]: ``off_peak_pricing``. Per-deployment metadata (``id``, ``access_via_team_ids``, arbitrary custom keys) never belongs on the shared key; it stays under the deployment's unique model id. + + A ``SHARED_ONLY_WHEN_TRUE_CAPABILITY_FIELDS`` flag reaches the shared key only as + ``True``. Its reader checks the deployment's own flag first, so a ``False`` or + non-boolean value already applies to its deployment, and on the shared key it would + only take the capability away from same-model deployments that set no flag. """ - return {k: v for k, v in model_info.items() if k in SHARED_BACKEND_MODEL_INFO_FIELDS} + return {k: v for k, v in model_info.items() if _belongs_on_shared_backend_key(k, v)} ABOVE_THRESHOLD_COST_KEY_PATTERN: Final = re.compile(r"_above_\d+k?_tokens(?:_batches)?$") diff --git a/litellm/utils.py b/litellm/utils.py index 7f7c6ffbc4d..ca916e049d1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4147,6 +4147,12 @@ def model_cost_registration_row(key: str, provider: object) -> Mapping[str, obje return _builtin_cost_map().get(base.key) or base.base +def model_cost_registration_key(key: str) -> str: + """The ``litellm.model_cost`` row a ``register_model`` write of ``key`` lands on (``typesafe/typesafe/jev-1.13.0`` + lands on ``typesafe/jev-1.13.0``).""" + return registration_base(key, {}, custom_llm_provider=None).key + + def registration_base(key: str, value: Mapping[str, object], custom_llm_provider: str | None) -> Landing: provider: Final = str(value.get("litellm_provider") or "") if _skips_builtin_registration_lookup(key, provider): diff --git a/tests/integration/providers/test_decisions_box_deployment_flag_wire.py b/tests/integration/providers/test_decisions_box_deployment_flag_wire.py index 70be286569a..6dc2d3c2809 100644 --- a/tests/integration/providers/test_decisions_box_deployment_flag_wire.py +++ b/tests/integration/providers/test_decisions_box_deployment_flag_wire.py @@ -354,6 +354,73 @@ async def test_a_router_deployment_flagged_false_refuses_boxes_even_when_its_bac ] * 2 +def _catalog_member(box_questions: bool) -> _Member: + backend: Final = next( + key + for key in tuple(litellm.model_cost) + if key.startswith("typesafe/") and litellm.supports_box_questions(model=key) is box_questions + ) + return _Member( + "typesafe", + backend, + "/v1/systemone", + backend.removeprefix("typesafe/"), + headers=MappingProxyType({_IMAGES_HEADER: "true"}), + ) + + +@pytest.mark.parametrize("sibling_flag", (False, "true"), ids=("false", "string")) +def test_a_later_same_model_deployment_flag_never_takes_catalog_box_questions_from_an_unflagged_one( + gateway: Gateway, sibling_flag: JsonValue +) -> None: + member: Final = _catalog_member(box_questions=True) + with wire_server(member.reply) as wire, gateway.scenario() as scenario: + unflagged: Final = member.deployment(scenario, wire.url, None) + member.deployment(scenario, wire.url, {"supports_box_questions": sibling_flag}) + answered: Final = _decide(gateway, unflagged) + assert answered.status_code == 200, answered.text + assert answered.json()["answers"] == _expected_answers(_QUESTIONS), answered.text + (call,) = wire.drain() + assert _JSON_OBJECT.validate_json(call.body) == member.upstream_body(_QUESTIONS) + + +def test_an_unflagged_deployment_stops_sending_box_questions_once_its_last_true_sibling_is_deleted( + gateway: Gateway, +) -> None: + member: Final = _catalog_member(box_questions=False) + with wire_server(member.reply) as wire, gateway.scenario() as scenario: + unflagged: Final = member.deployment(scenario, wire.url, None) + with gateway.scenario() as sibling: + member.deployment(sibling, wire.url, {"supports_box_questions": True}) + borrowed: Final = _decide(gateway, unflagged, (_BOX_QUESTION,)) + assert borrowed.status_code == 200, borrowed.text + assert borrowed.json()["answers"] == [_BOX_ANSWER], borrowed.text + refused: Final = eventually( + lambda: tuple(_decide(gateway, unflagged, (_BOX_QUESTION,)) for _ in range(8)), + lambda responses: all(response.status_code == 400 for response in responses), + seconds=3 * PROXY_CONFIG_RELOAD_INTERVAL_SECONDS, + ) + for response in refused: + _assert_refused_box(response) + received: Final = wire.drain() + assert received, "the borrowed box question never reached the upstream" + assert all(_JSON_OBJECT.validate_json(call.body) == member.upstream_body((_BOX_QUESTION,)) for call in received) + + +@pytest.mark.parametrize("value", ("true", 1), ids=("string", "int")) +def test_a_non_boolean_deployment_flag_keeps_the_catalog_box_questions_of_its_backend( + gateway: Gateway, value: JsonValue +) -> None: + member: Final = _catalog_member(box_questions=True) + with wire_server(member.reply) as wire, gateway.scenario() as scenario: + model: Final = member.deployment(scenario, wire.url, {"supports_box_questions": value}) + answered: Final = _decide(gateway, model) + assert answered.status_code == 200, answered.text + assert answered.json()["answers"] == _expected_answers(_QUESTIONS), answered.text + (call,) = wire.drain() + assert _JSON_OBJECT.validate_json(call.body) == member.upstream_body(_QUESTIONS) + + @pytest.mark.parametrize( "value", ("true", 1, [True], "", "x" * 5000, None), ids=("string", "int", "list", "empty", "5kb", "null") ) diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index f5c71668e63..8d15ba02a38 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -8,6 +8,7 @@ should still use the built-in pricing. """ import asyncio +import contextlib import copy import itertools import json @@ -18,6 +19,7 @@ import queue import re import sys import threading +from collections.abc import Iterator from pathlib import Path from types import MappingProxyType from typing import Final, cast @@ -32,8 +34,10 @@ from litellm import Router from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE from litellm.litellm_core_utils.ptu_pricing import ptu_config_error from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy.utils import create_model_info_response +from litellm.router_api import ChangeSet from litellm.types.router import ( Deployment, DeploymentTypedDict, @@ -634,6 +638,286 @@ def test_pdf_opt_out_on_one_deployment_does_not_block_its_sibling(respx_mock: re _restore_model_cost_entries(model_keys) +@pytest.mark.parametrize( + ("catalog_flag", "sibling_flag", "shared_flag"), + ((True, False, True), (True, "true", True), (None, True, True)), + ids=("false-keeps-catalog-true", "string-keeps-catalog-true", "true-reaches-shared"), +) +def test_a_sibling_box_questions_flag_reaches_the_shared_key_only_as_true( + catalog_flag: bool | None, sibling_flag: bool | str, shared_flag: bool +) -> None: + backend_model: Final = "perplexity/box-isolation-check" + catalog_row: Final = { + "litellm_provider": "perplexity", + "mode": "evaluation", + **({} if catalog_flag is None else {"supports_box_questions": catalog_flag}), + } + model_keys: Final = { + key: copy.deepcopy(litellm.model_cost.get(key)) for key in ("unflagged-id", "sibling-id", backend_model) + } + litellm.model_cost[backend_model] = catalog_row + publish(litellm.model_cost) + try: + Router( + model_list=[ + { + "model_name": "box-isolation", + "litellm_params": {"model": backend_model}, + "model_info": {"id": "unflagged-id"}, + }, + { + "model_name": "box-isolation", + "litellm_params": {"model": backend_model}, + "model_info": {"id": "sibling-id", "supports_box_questions": sibling_flag}, + }, + ], + ) + + assert litellm.supports_box_questions(model=backend_model) is shared_flag + assert litellm.model_cost["sibling-id"]["supports_box_questions"] == sibling_flag + finally: + _restore_model_cost_entries(model_keys) + + +def _box_flag_deployment( + backend_model: str, model_id: str, flag: bool | None, custom_llm_provider: str | None = None +) -> DeploymentTypedDict: + return { + "model_name": "box-withdraw", + "litellm_params": { + "model": backend_model, + **({} if custom_llm_provider is None else {"custom_llm_provider": custom_llm_provider}), + }, + "model_info": {"id": model_id, **({} if flag is None else {"supports_box_questions": flag})}, + } + + +@contextlib.contextmanager +def _live_box_router(backend_model: str, deployments: tuple[tuple[str, bool | None], ...]) -> Iterator[Router]: + router: Final = Router( + model_list=[_box_flag_deployment(backend_model, model_id, flag) for model_id, flag in deployments] + ) + try: + yield router + finally: + router.retire() + + +@pytest.mark.parametrize( + ("catalog_box", "true_sibling_stays", "change", "shared_flag"), + ( + (False, False, "update", False), + (False, False, "delete", False), + (False, False, "replace", False), + (False, True, "update", True), + (True, False, "update", True), + ), + ids=( + "update-to-false-clears", + "delete-clears", + "replace-clears", + "another-true-sibling-keeps", + "catalog-true-keeps", + ), +) +def test_a_withdrawn_box_questions_true_leaves_the_shared_key_once_no_sibling_sets_it( + catalog_box: bool, true_sibling_stays: bool, change: str, shared_flag: bool +) -> None: + backend_model: Final = ( + next( + key + for key, entry in GetModelCostMap.loaded_model_cost_map().items() + if entry.get("supports_box_questions") is True + ) + if catalog_box + else "perplexity/box-withdraw-check" + ) + deployments: Final = ( + ("unflagged-id", None), + ("sibling-id", True), + *((("other-true-id", True),) if true_sibling_stays else ()), + ) + model_keys: Final = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in (*(model_id for model_id, _ in deployments), backend_model) + } + try: + with _live_box_router(backend_model, deployments) as router: + assert litellm.supports_box_questions(model=backend_model) is True + if change == "replace": + router.set_model_list( + [ + _box_flag_deployment(backend_model, model_id, flag) + for model_id, flag in deployments + if model_id != "sibling-id" + ] + ) + else: + _ = router.apply( + ChangeSet(upsert=[Deployment(**_box_flag_deployment(backend_model, "sibling-id", False))]) + if change == "update" + else ChangeSet(delete=["sibling-id"]) + ) + + assert litellm.supports_box_questions(model=backend_model) is shared_flag + finally: + _restore_model_cost_entries(model_keys) + + +def test_a_failed_set_model_list_takes_back_the_box_questions_true_its_aborted_build_registered() -> None: + backend_model: Final = "perplexity/box-withdraw-on-rollback" + model_keys: Final = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in ("unflagged-id", "false-id", "aborted-true-id", "unservable-id", backend_model) + } + unservable: Final = { + "model_name": "box-withdraw", + "litellm_params": {"model": backend_model, "input_cost_per_token": float("inf")}, + "model_info": {"id": "unservable-id"}, + } + try: + with _live_box_router(backend_model, (("unflagged-id", None), ("false-id", False))) as router: + with pytest.raises(ValueError, match="cannot be served"): + router.set_model_list( + [ + _box_flag_deployment(backend_model, "unflagged-id", None), + _box_flag_deployment(backend_model, "aborted-true-id", True), + unservable, + ] + ) + + assert sorted( + deployment["model_info"]["id"] for deployment in router.registry().deployments("box-withdraw") + ) == ["false-id", "unflagged-id"] + assert litellm.supports_box_questions(model=backend_model) is False + finally: + _restore_model_cost_entries(model_keys) + + +def test_a_failed_set_model_list_writes_no_row_for_a_box_questions_true_it_never_built() -> None: + backend_model: Final = "perplexity/box-withdraw-on-rollback-served" + never_built_model: Final = "perplexity/box-withdraw-never-built" + model_keys: Final = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in ("unflagged-id", "unservable-id", "never-built-id", backend_model, never_built_model) + } + unservable: Final = { + "model_name": "box-withdraw", + "litellm_params": {"model": backend_model, "input_cost_per_token": float("inf")}, + "model_info": {"id": "unservable-id"}, + } + try: + with _live_box_router(backend_model, (("unflagged-id", None),)) as router: + with pytest.raises(ValueError, match="cannot be served"): + router.set_model_list([unservable, _box_flag_deployment(never_built_model, "never-built-id", True)]) + + assert never_built_model not in litellm.model_cost + finally: + _restore_model_cost_entries(model_keys) + + +def test_a_failed_set_model_list_takes_back_a_box_questions_true_whose_model_comes_from_the_environment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + backend_model: Final = "perplexity/box-withdraw-on-rollback-env" + monkeypatch.setenv("BOX_WITHDRAW_ROLLBACK_MODEL", backend_model) + model_keys: Final = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in ("unflagged-id", "env-true-id", "unservable-id", backend_model) + } + unservable: Final = { + "model_name": "box-withdraw", + "litellm_params": {"model": backend_model, "input_cost_per_token": float("inf")}, + "model_info": {"id": "unservable-id"}, + } + try: + with _live_box_router(backend_model, (("unflagged-id", None),)) as router: + with pytest.raises(ValueError, match="cannot be served"): + router.set_model_list( + [ + _box_flag_deployment(backend_model, "unflagged-id", None), + _box_flag_deployment("os.environ/BOX_WITHDRAW_ROLLBACK_MODEL", "env-true-id", True), + unservable, + ] + ) + + assert litellm.supports_box_questions(model=backend_model) is False + finally: + _restore_model_cost_entries(model_keys) + + +def test_a_box_questions_true_another_live_router_serves_stays_on_the_shared_key() -> None: + backend_model: Final = "perplexity/box-withdraw-across-routers" + model_keys: Final = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in ("unflagged-id", "sibling-id", "other-router-id", backend_model) + } + try: + with ( + _live_box_router(backend_model, (("unflagged-id", None), ("sibling-id", True))) as router, + _live_box_router(backend_model, (("other-router-id", True),)) as other_router, + ): + _ = router.apply(ChangeSet(delete=["sibling-id"])) + assert litellm.supports_box_questions(model=backend_model) is True + + _ = other_router.apply(ChangeSet(delete=["other-router-id"])) + assert litellm.supports_box_questions(model=backend_model) is False + finally: + _restore_model_cost_entries(model_keys) + + +@pytest.mark.parametrize("deleted_first", ("ui-true-id", "config-true-id")) +def test_a_box_questions_true_stays_while_a_sibling_spelled_with_its_provider_still_sets_it(deleted_first: str) -> None: + backend_model: Final = next( + key + for key, entry in GetModelCostMap.loaded_model_cost_map().items() + if entry.get("litellm_provider") == "typesafe" and entry.get("supports_box_questions") is not True + ) + model_keys: Final = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in ("unflagged-id", "config-true-id", "ui-true-id", backend_model) + } + try: + with _live_box_router(backend_model, (("unflagged-id", None), ("config-true-id", True))) as router: + _ = router.apply( + ChangeSet( + upsert=[ + Deployment( + **_box_flag_deployment(backend_model, "ui-true-id", True, custom_llm_provider="typesafe") + ) + ] + ) + ) + + _ = router.apply(ChangeSet(delete=[deleted_first])) + assert litellm.supports_box_questions(model=backend_model) is True + + _ = router.apply(ChangeSet(delete=[({"ui-true-id", "config-true-id"} - {deleted_first}).pop()])) + assert litellm.supports_box_questions(model=backend_model) is False + finally: + _restore_model_cost_entries(model_keys) + + +def test_a_retired_router_takes_back_a_box_questions_true_no_live_router_still_sets() -> None: + backend_model: Final = "perplexity/box-withdraw-on-retire" + model_keys: Final = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in ("unflagged-id", "first-true-id", "second-true-id", backend_model) + } + try: + with _live_box_router(backend_model, (("unflagged-id", None),)): + first: Final = Router(model_list=[_box_flag_deployment(backend_model, "first-true-id", True)]) + second: Final = Router(model_list=[_box_flag_deployment(backend_model, "second-true-id", True)]) + + first.retire() + assert litellm.supports_box_questions(model=backend_model) is True + + second.retire() + assert litellm.supports_box_questions(model=backend_model) is False + finally: + _restore_model_cost_entries(model_keys) + + def test_token_count_settings_stay_on_the_deployment_that_set_them() -> None: """A deployment's ``token_count_model`` and framing settings must not land on the shared ``{provider}/{model}`` key, or a sibling deployment of that model would count with them.""" From 0b070d25d336a41bbac76fb8b0d1638509d61b6d Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 09:35:37 -0800 Subject: [PATCH 015/189] feat: add claude mythos 5.2 to vertex ai cost map (#50940) * Add Claude Mythos 5.2 to Vertex AI cost map Copies the vertex_ai/claude-fable-5-2 and @default rows as vertex_ai/claude-mythos-5-2 and @default. Anthropic's Mythos 5.2 model page lists Google Cloud as a platform and says it has the same specifications and pricing as Fable 5.2, and Google publishes no Mythos row yet, so the new rows cite Anthropic's page as their source. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Bill Mythos 5.2 on Vertex through the cost path in tests Adds cost_per_token checks with cache reads, 5m and 1h cache writes and the regional uplift so the Vertex Mythos 5.2 rows are proven to charge what Fable 5.2 charges, and annotates the new test locals with Final. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Read the regional_endpoint_uplift_multiplier key in the Vertex Mythos uplift test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 82 +++++++++++++++++ model_prices_and_context_window.json | 82 +++++++++++++++++ tests/unit/test_claude_fable_5_config.py | 89 +++++++++++++++++++ 3 files changed, 253 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 959d2e46afe..3196b3c0c08 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -53236,6 +53236,47 @@ "output_cost_per_token_batches": 2e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-mythos-5-2": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 1e-05, + "cache_creation_input_token_cost_above_1hr": 1.6e-05, + "cache_creation_input_token_cost_batches": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "input_cost_per_token": 8e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512, + "deprecation_date": "2027-11-09", + "input_cost_per_token_batches": 4e-06, + "output_cost_per_token_batches": 2e-05, + "source": "https://platform.claude.com/docs/en/models/mythos-5-2/overview" + }, "vertex_ai/claude-fable-latest": { "alias_of": "vertex_ai/claude-fable-5-2" }, @@ -53360,6 +53401,47 @@ "output_cost_per_token_batches": 2e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-mythos-5-2@default": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 1e-05, + "cache_creation_input_token_cost_above_1hr": 1.6e-05, + "cache_creation_input_token_cost_batches": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "input_cost_per_token": 8e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512, + "deprecation_date": "2027-11-09", + "input_cost_per_token_batches": 4e-06, + "output_cost_per_token_batches": 2e-05, + "source": "https://platform.claude.com/docs/en/models/mythos-5-2/overview" + }, "vertex_ai/claude-opus-5": { "deprecation_date": "2027-01-24", "regional_endpoint_uplift_multiplier": 1.1, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 959d2e46afe..3196b3c0c08 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -53236,6 +53236,47 @@ "output_cost_per_token_batches": 2e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-mythos-5-2": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 1e-05, + "cache_creation_input_token_cost_above_1hr": 1.6e-05, + "cache_creation_input_token_cost_batches": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "input_cost_per_token": 8e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512, + "deprecation_date": "2027-11-09", + "input_cost_per_token_batches": 4e-06, + "output_cost_per_token_batches": 2e-05, + "source": "https://platform.claude.com/docs/en/models/mythos-5-2/overview" + }, "vertex_ai/claude-fable-latest": { "alias_of": "vertex_ai/claude-fable-5-2" }, @@ -53360,6 +53401,47 @@ "output_cost_per_token_batches": 2e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-mythos-5-2@default": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 1e-05, + "cache_creation_input_token_cost_above_1hr": 1.6e-05, + "cache_creation_input_token_cost_batches": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, + "input_cost_per_token": 8e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512, + "deprecation_date": "2027-11-09", + "input_cost_per_token_batches": 4e-06, + "output_cost_per_token_batches": 2e-05, + "source": "https://platform.claude.com/docs/en/models/mythos-5-2/overview" + }, "vertex_ai/claude-opus-5": { "deprecation_date": "2027-01-24", "regional_endpoint_uplift_multiplier": 1.1, diff --git a/tests/unit/test_claude_fable_5_config.py b/tests/unit/test_claude_fable_5_config.py index 2157364efb3..25f62e37c04 100644 --- a/tests/unit/test_claude_fable_5_config.py +++ b/tests/unit/test_claude_fable_5_config.py @@ -18,6 +18,7 @@ import pytest import litellm from litellm.constants import BEDROCK_CONVERSE_MODELS from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.types.utils import CacheCreationTokenDetails, PromptTokensDetailsWrapper, Usage REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") @@ -172,3 +173,91 @@ def test_fable_5_2_accepts_the_speed_param_on_the_anthropic_api(local_model_cost assert AnthropicModelInfo.supports_fast_mode("claude-fable-5-2", "anthropic") is True assert AnthropicConfig._model_supports_speed_param("claude-fable-5-2", "anthropic") is True + + +MYTHOS_5_2_VERTEX_ROWS: Final = ( + ("vertex_ai/claude-mythos-5-2", "vertex_ai/claude-fable-5-2"), + ("vertex_ai/claude-mythos-5-2@default", "vertex_ai/claude-fable-5-2@default"), +) + + +def _without_source(row: dict[str, object]) -> dict[str, object]: + return {k: v for k, v in row.items() if k != "source"} + + +def test_mythos_5_2_vertex_rows_present_in_bundled_backup() -> None: + backup: Final = GetModelCostMap.load_local_model_cost_map() + root: Final = _load_root_cost_map() + for model_name, _ in MYTHOS_5_2_VERTEX_ROWS: + assert model_name in backup, f"Missing from backup cost map: {model_name}" + assert backup[model_name] == root[model_name], model_name + + +@pytest.mark.parametrize(("mythos_row", "fable_row"), MYTHOS_5_2_VERTEX_ROWS) +def test_mythos_5_2_vertex_rows_match_fable_5_2(mythos_row: str, fable_row: str) -> None: + """Anthropic ships Mythos 5.2 as a verified-access twin of Fable 5.2 with "the same + specifications and pricing" (platform.claude.com/docs/en/models/mythos-5-2/overview, + read 2026-11-10), so the Vertex rows must stay identical apart from their source.""" + root: Final = _load_root_cost_map() + assert _without_source(root[mythos_row]) == _without_source(root[fable_row]) + assert root[mythos_row]["source"] == "https://platform.claude.com/docs/en/models/mythos-5-2/overview" + + +@pytest.mark.parametrize("model", [row for row, _ in MYTHOS_5_2_VERTEX_ROWS]) +def test_adaptive_thinking_detected_for_mythos_5_2_on_vertex(local_model_cost_map, model: str) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + assert AnthropicModelInfo.is_adaptive_thinking_model(model, "anthropic") is True + + +def _vertex_claude_cached_usage() -> Usage: + return Usage( + prompt_tokens=1600, + completion_tokens=50, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300, cache_creation_tokens=300), + cache_read_input_tokens=300, + cache_creation_input_tokens=300, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_5m_input_tokens=200, ephemeral_1h_input_tokens=100 + ), + ) + + +@pytest.mark.parametrize(("mythos_row", "fable_row"), MYTHOS_5_2_VERTEX_ROWS) +def test_mythos_5_2_on_vertex_bills_like_fable_5_2(local_model_cost_map, mythos_row: str, fable_row: str) -> None: + """Billing through the Vertex cost path, with cache reads, 5m and 1h cache writes and the + regional-endpoint uplift, must charge Mythos 5.2 exactly what Fable 5.2 is charged.""" + for location in ("global", "us-east5"): + mythos_cost: Final = litellm.cost_per_token( + model=mythos_row, + custom_llm_provider="vertex_ai", + usage_object=_vertex_claude_cached_usage(), + vertex_location=location, + ) + fable_cost: Final = litellm.cost_per_token( + model=fable_row, + custom_llm_provider="vertex_ai", + usage_object=_vertex_claude_cached_usage(), + vertex_location=location, + ) + assert mythos_cost == fable_cost, location + assert mythos_cost[0] > 0 and mythos_cost[1] > 0, location + + +@pytest.mark.parametrize("model", [row for row, _ in MYTHOS_5_2_VERTEX_ROWS]) +def test_mythos_5_2_on_vertex_applies_regional_uplift(local_model_cost_map, model: str) -> None: + uplift: Final = _load_root_cost_map()[model]["regional_endpoint_uplift_multiplier"] + global_cost: Final = litellm.cost_per_token( + model=model, + custom_llm_provider="vertex_ai", + usage_object=_vertex_claude_cached_usage(), + vertex_location="global", + ) + regional_cost: Final = litellm.cost_per_token( + model=model, + custom_llm_provider="vertex_ai", + usage_object=_vertex_claude_cached_usage(), + vertex_location="us-east5", + ) + assert regional_cost[0] == pytest.approx(global_cost[0] * uplift) + assert regional_cost[1] == pytest.approx(global_cost[1] * uplift) From 461d3407bf669cb2751502ae50215c4e2994961b Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Tue, 10 Nov 2026 09:58:46 -0800 Subject: [PATCH 016/189] fix(router): hold the registry lock across router-core writes and mirror reads (#50933) * fix(router): hold the registry lock across router-core writes and the mirror reads at their positions A registry lookup returns positions in router-core's list and the host reads its mirror at them, while an upsert wrote router-core and read the mirror back unlocked. A read on another thread in between got an IndexError past the mirror, or after a removal the neighbouring model's entries * test(router): cover the registry lock's read and write paths Move the concurrent-read regression into the registry writes characterization, drop the explanatory lock comments and the unused litellm-model lookup --- litellm/router_rust/core.py | 15 +- litellm/router_rust/surface/model_list.py | 63 ++-- litellm/router_rust/surface/registry.py | 278 ++++++++++-------- .../test_registry_characterization_lookups.py | 15 + .../test_registry_characterization_writes.py | 81 +++++ 5 files changed, 293 insertions(+), 159 deletions(-) diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index 5015795657f..b7e67fffd57 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -1815,13 +1815,14 @@ class RustCore( def _rebuild(self) -> None: """A new Rust core for changed static settings, over the same deployments and the providers the old one's builds resolved for them.""" - old: Final = self._handle - model_list: Final = list(self._hosted._core_model_list) # pyright: ignore[reportPrivateUsage] # see __init__ - self._handle = self._new_handle(model_list) - self._handle.adopt_providers(old) - old.retire() - # The live list and indexes read the new handle's (the same entries, kept as the same objects). - self._hosted.refresh() + with self._hosted._deployment_index_lock: # pyright: ignore[reportPrivateUsage] # the registry's lock, held across the swap and the read back + old: Final = self._handle + model_list: Final = list(self._hosted._core_model_list) # pyright: ignore[reportPrivateUsage] # see __init__ + self._handle = self._new_handle(model_list) + self._handle.adopt_providers(old) + old.retire() + # The live list and indexes read the new handle's (the same entries, kept as the same objects). + self._hosted.refresh() # The new handle builds its indexes from the deployments: written ones are handed again. self._index_sent = None diff --git a/litellm/router_rust/surface/model_list.py b/litellm/router_rust/surface/model_list.py index bb7fb53acea..32ab8f1cb76 100644 --- a/litellm/router_rust/surface/model_list.py +++ b/litellm/router_rust/surface/model_list.py @@ -26,7 +26,9 @@ model info caches) mean what they did: the same object until a write changes tha import copy import datetime +import threading from collections.abc import Callable, Iterable, Mapping, Sequence +from contextlib import AbstractContextManager from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -189,9 +191,12 @@ class HostedModelList(list[Entry]): apply: Callable[[Mapping[str, object]], None], live: Callable[[object], Mapping[str, object]] = lambda _: {}, live_info: Callable[[object], Mapping[str, object]] = lambda _: {}, + *, + lock: AbstractContextManager[object] | None = None, ) -> None: super().__init__() self._handle: Final = handle + self._lock: Final[AbstractContextManager[object]] = threading.RLock() if lock is None else lock self._apply: Final = apply # A deployment's litellm_params values its record carries as text, by id (``live_params``). self._live: Final = live @@ -209,6 +214,10 @@ class HostedModelList(list[Entry]): def refresh(self) -> None: """Read router-core's deployments again when its registry moved since the last read.""" + with self._lock: + self._refresh() + + def _refresh(self) -> None: handle: Final = self._handle() if handle is None or self._building is not None: # While a build runs, the list shows what it accepted until router-core serves it. @@ -260,28 +269,31 @@ class HostedModelList(list[Entry]): """A build starts: router-core serves what it adds once it ends, so until then the list shows each deployment it accepts here (``built``), after the entries it keeps, or alone when it replaces them, as the Python router's list grows while it adds them.""" - self.refresh() - if replacing: - self._show([]) - self._id_counts = {} - self._building = (len(self), self._id_counts, self._read_at) + with self._lock: + self._refresh() + if replacing: + self._show([]) + self._id_counts = {} + self._building = (len(self), self._id_counts, self._read_at) def built(self, entry: Mapping[str, object]) -> None: """A deployment the build accepted, shown until the build ends.""" - if self._building is not None: - super().extend(self._kept([entry], appended=True)) + with self._lock: + if self._building is not None: + super().extend(self._kept([entry], appended=True)) def end_build(self) -> None: """The build ended (or failed): router-core serves what it added, read back in place of the entries shown meanwhile, each kept as the object it was shown as while unchanged.""" - if self._building is None: - return - start, counts, read_at = self._building - self._building = None - super().__delitem__(slice(start, None)) - self._id_counts = counts - self._read_at = read_at - self.refresh() + with self._lock: + if self._building is None: + return + start, counts, read_at = self._building + self._building = None + super().__delitem__(slice(start, None)) + self._id_counts = counts + self._read_at = read_at + self._refresh() def _seed(self, entries: Sequence[object]) -> None: """Keep ``entries``, the list router-core now holds, as the objects its records for them @@ -311,19 +323,21 @@ class HostedModelList(list[Entry]): def _replace(self, entries: list[Entry]) -> None: # mutable-ok: the list router-core takes # Seeded first: router-core's records for them read back as these objects however soon # the list is read again. - self._seed(entries) - self._apply({"replace": entries}) - self.refresh() + with self._lock: + self._seed(entries) + self._apply({"replace": entries}) + self._refresh() def replace(self, entries: Iterable[object]) -> None: """Every deployment at once (``router.model_list = [...]``).""" self._replace(cast(list[Entry], list(entries))) # cast-ok: the entries as assigned, which router-core validates def append(self, value: Entry) -> None: - self.refresh() - self._seed_appended([value]) - self._apply({"append": [value]}) - self.refresh() + with self._lock: + self._refresh() + self._seed_appended([value]) + self._apply({"append": [value]}) + self._refresh() def extend(self, values: Iterable[Entry]) -> None: for value in tuple(values): @@ -362,8 +376,9 @@ class HostedModelList(list[Entry]): # A delete removes every deployment with the id; the list removes only this one. self._replace(changed) return - self._apply({"delete": [removed_id]}) - self.refresh() + with self._lock: + self._apply({"delete": [removed_id]}) + self._refresh() def pop(self, index: SupportsIndex = -1) -> Entry: entry: Final = self[index] diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py index e48569c4c9d..83008bdbb94 100644 --- a/litellm/router_rust/surface/registry.py +++ b/litellm/router_rust/surface/registry.py @@ -743,7 +743,7 @@ class HostRegistry(HostModelInfo): self._core_model_group_alias: dict[str, str | RouterModelGroupAliasItem] = model_group_alias # mutable-ok: the live alias map # fmt: skip # The live model_list: router-core's deployments read back, every write a changeset. self._core_model_list: Final[HostedModelList] = HostedModelList( - self._live_handle, self._hand_changes, self.live_params, self.live_info + self._live_handle, self._hand_changes, self.live_params, self.live_info, lock=self._deployment_index_lock ) # The litellm_params values a deployment's JSON cannot carry (an Exception or model object as # ``mock_response``, a callable ``azure_ad_token_provider``), by deployment id: an attempt is called with them, as Python's list held them. @@ -1472,9 +1472,10 @@ class HostRegistry(HostModelInfo): def _hand_changes(self, changes: Mapping[str, object]) -> None: if self.changes_runner is None: raise RuntimeError("the host registry writes deployments through its Rust core") - self.changes_runner(changes) - self._core_model_list.refresh() - self._drop_removed_live_values() + with self._deployment_index_lock: + self.changes_runner(changes) + self._core_model_list.refresh() + self._drop_removed_live_values() def _drop_removed_live_values(self) -> None: """Forget the live values of ids no deployment carries any more, so a router that rotates @@ -1678,10 +1679,11 @@ class HostRegistry(HostModelInfo): return handle is not None and key is not None and handle.has_model_id(key) def _core_get_model_info(self, id: str) -> UntypedEntry | None: - handle: Final = self._live_handle() - key: Final = _index_key(id) - idx: Final[object] = None if handle is None or key is None else handle.id_position(key) - return None if idx is None else self._core_model_list[idx] # pyright: ignore[reportArgumentType, reportCallIssue] # a position, or what a caller wrote, read as Python reads it + with self._deployment_index_lock: + handle: Final = self._live_handle() + key: Final = _index_key(id) + idx: Final[object] = None if handle is None or key is None else handle.id_position(key) + return None if idx is None else self._core_model_list[idx] # pyright: ignore[reportArgumentType, reportCallIssue] # a position, or what a caller wrote, read as Python reads it def live_params(self, model_id: object) -> Mapping[str, object]: """The litellm_params values of deployment ``model_id`` that its JSON record cannot carry.""" @@ -1696,21 +1698,23 @@ class HostRegistry(HostModelInfo): return None if entry is None else Deployment(**entry) # pyright: ignore[reportArgumentType] # a stored entry def _core_get_deployment_by_model_group_name(self, model_group_name: str) -> Deployment | None: - handle: Final = self._live_handle() - key: Final = _index_key(model_group_name) - indices: Final[object] = None if handle is None or key is None else handle.named_positions(key) - model: Final = self._core_model_list[indices[0]] if indices else None # pyright: ignore[reportIndexIssue, reportArgumentType, reportCallIssue] # positions, or what a caller wrote, read as Python reads them - return None if model is None else Deployment(**model) # pyright: ignore[reportArgumentType] # a stored entry + with self._deployment_index_lock: + handle: Final = self._live_handle() + key: Final = _index_key(model_group_name) + indices: Final[object] = None if handle is None or key is None else handle.named_positions(key) + model: Final = self._core_model_list[indices[0]] if indices else None # pyright: ignore[reportIndexIssue, reportArgumentType, reportCallIssue] # positions, or what a caller wrote, read as Python reads them + return None if model is None else Deployment(**model) # pyright: ignore[reportArgumentType] # a stored entry def _at(self, indices: object) -> tuple[UntypedEntry, ...]: """The entries at ``indices`` (what an index holds), read as Python's list reads them.""" return tuple(self._core_model_list[idx] for idx in indices) # pyright: ignore[reportGeneralTypeIssues, reportArgumentType, reportCallIssue] # positions, or what a caller wrote def _core_deployments_named(self, model_name: str) -> tuple[UntypedEntry, ...] | None: - handle: Final = self._live_handle() - key: Final = _index_key(model_name) - indices: Final[object] = None if handle is None or key is None else handle.named_positions(key) - return None if indices is None else self._at(indices) + with self._deployment_index_lock: + handle: Final = self._live_handle() + key: Final = _index_key(model_name) + indices: Final[object] = None if handle is None or key is None else handle.named_positions(key) + return None if indices is None else self._at(indices) def team_public_model_names_of(self, team_id: object) -> frozenset[str]: handle: Final = self._live_handle() @@ -1719,13 +1723,14 @@ class HostRegistry(HostModelInfo): return frozenset(handle.team_public_names_of(team_id)) def _team_deployments_named(self, team_id: str, public_model_name: str) -> tuple[UntypedEntry, ...] | None: - handle: Final = self._live_handle() - team: Final = _index_key(team_id) - public: Final = _index_key(public_model_name) - indices: Final[object] = ( - None if handle is None or team is None or public is None else handle.team_positions(team, public) - ) - return None if indices is None else self._at(indices) + with self._deployment_index_lock: + handle: Final = self._live_handle() + team: Final = _index_key(team_id) + public: Final = _index_key(public_model_name) + indices: Final[object] = ( + None if handle is None or team is None or public is None else handle.team_positions(team, public) + ) + return None if indices is None else self._at(indices) # -- lookups (router-core `lookup.rs`): rows read at the stored entries -------------------- @@ -1841,24 +1846,26 @@ class HostRegistry(HostModelInfo): def _team_deployments_across_teams(self, model: str) -> list[UntypedEntry]: # mutable-ok: a fresh list """Every team's deployments under public name ``model``, for a proxy admin calling without a team.""" - rows, teams = self._lookup().lookup_across_teams(model) - team_deployments: Final = self._rows(rows) - if teams > 1: - raise litellm_exceptions.BadRequestError( - message=( - f"Model name '{model}' matches deployments from multiple teams. " - "Specify the deployment ID directly to disambiguate." - ), - model=model, - llm_provider="", - ) - return team_deployments + with self._deployment_index_lock: + rows, teams = self._lookup().lookup_across_teams(model) + team_deployments: Final = self._rows(rows) + if teams > 1: + raise litellm_exceptions.BadRequestError( + message=( + f"Model name '{model}' matches deployments from multiple teams. " + "Specify the deployment ID directly to disambiguate." + ), + model=model, + llm_provider="", + ) + return team_deployments def _core_get_all_deployments( self, model_name: str, model_alias: str | None = None, team_id: str | None = None ) -> list[UntypedEntry]: # mutable-ok: a fresh list, as the core returns it - key: Final = _index_key(model_name) - return [] if key is None else self._rows(self._lookup().lookup_all(key, team_id, model_alias)) + with self._deployment_index_lock: + key: Final = _index_key(model_name) + return [] if key is None else self._rows(self._lookup().lookup_all(key, team_id, model_alias)) def get_routing_group(self, model_name: str) -> RoutingGroup | None: """The routing group callable as ``model_name``: a deployment model_name or an alias of the @@ -1877,61 +1884,70 @@ class HostRegistry(HostModelInfo): def _core_routing_group_has_alternatives(self, model_group: str | None) -> bool: """A callable routing group whose member union spans more than one deployment.""" - if model_group is None: - return False - resolved: Final = self._core_get_model_from_alias(model=model_group) or model_group - group: Final = self.get_routing_group(resolved) - if group is None: - return False - handle: Final = self._lookup() + with self._deployment_index_lock: + if model_group is None: + return False + resolved: Final = self._core_get_model_from_alias(model=model_group) or model_group + group: Final = self.get_routing_group(resolved) + if group is None: + return False + handle: Final = self._lookup() - def held(member: str) -> Sequence[object]: - return cast(Sequence[object], handle.named_positions(member) or ()) # cast-ok: positions, or junk + def held(member: str) -> Sequence[object]: + return cast(Sequence[object], handle.named_positions(member) or ()) # cast-ok: positions, or junk - return sum(len(held(member)) for member in group.models) > 1 + return sum(len(held(member)) for member in group.models) > 1 def _core_get_model_list_from_model_alias( self, model_name: str | None = None ) -> list[UntypedEntry]: # mutable-ok: a fresh list - return self._rows( - self._lookup().lookup_model_list(model_name, None, self._list_scope(model_name, only="aliases")) - ) + with self._deployment_index_lock: + return self._rows( + self._lookup().lookup_model_list(model_name, None, self._list_scope(model_name, only="aliases")) + ) def _core_get_model_list_from_routing_groups(self, model_name: str | None = None) -> Sequence[UntypedEntry]: """Callable routing groups as model-list rows: each member deployment under the group's name.""" - if model_name is None and self._routing_group_rows is not None: - return self._routing_group_rows - rows: Final = tuple( - self._rows(self._lookup().lookup_model_list(model_name, None, self._list_scope(model_name, only="groups"))) - ) - if model_name is None: - self._routing_group_rows = rows - return rows + with self._deployment_index_lock: + if model_name is None and self._routing_group_rows is not None: + return self._routing_group_rows + rows: Final = tuple( + self._rows( + self._lookup().lookup_model_list(model_name, None, self._list_scope(model_name, only="groups")) + ) + ) + if model_name is None: + self._routing_group_rows = rows + return rows def _core_get_model_list( self, model_name: str | None = None, team_id: str | None = None ) -> list[UntypedEntry] | None: # mutable-ok: a fresh list, as get_model_list returns - if model_name is not None and _index_key(model_name) is None: - # No index holds it, so `_get_wildcard_deployments` reads it: what the pattern routers raise. - team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None - return [ - *self._core_pattern_router.get_deployments_by_pattern(model=model_name), - *(team_router.get_deployments_by_pattern(model=model_name) if team_router is not None else ()), - ] - return self._rows(self._lookup().lookup_model_list(model_name, team_id, self._list_scope(model_name))) + with self._deployment_index_lock: + if model_name is not None and _index_key(model_name) is None: + # No index holds it, so `_get_wildcard_deployments` reads it: what the pattern routers raise. + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + return [ + *self._core_pattern_router.get_deployments_by_pattern(model=model_name), + *(team_router.get_deployments_by_pattern(model=model_name) if team_router is not None else ()), + ] + return self._rows(self._lookup().lookup_model_list(model_name, team_id, self._list_scope(model_name))) def _core_wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: - from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + with self._deployment_index_lock: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider - if not isinstance(model, str): # pyright: ignore[reportUnnecessaryIsInstance] # a caller's non-string model reads the base routers - team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None - team_owned: Final = team_router.get_deployments_by_pattern(model=model) if team_router is not None else () - return team_owned or self._core_pattern_router.get_deployments_by_pattern(model=model) - rows: Final = self._lookup().lookup_wildcard_route( - model, team_id, [model, f"{inferred_provider(model)}/{model}"] - ) - routed: Final = cast(Sequence[Mapping[str, object]], rows) # cast-ok: router-core's wildcard rows - return [cast(Mapping[str, object], row["pattern"]) for row in routed] # cast-ok: each carries its routed copy + if not isinstance(model, str): # pyright: ignore[reportUnnecessaryIsInstance] # a caller's non-string model reads the base routers + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None + team_owned: Final = ( + team_router.get_deployments_by_pattern(model=model) if team_router is not None else () + ) + return team_owned or self._core_pattern_router.get_deployments_by_pattern(model=model) + rows: Final = self._lookup().lookup_wildcard_route( + model, team_id, [model, f"{inferred_provider(model)}/{model}"] + ) + routed: Final = cast(Sequence[Mapping[str, object]], rows) # cast-ok: router-core's wildcard rows + return [cast(Mapping[str, object], row["pattern"]) for row in routed] # cast-ok: its routed copy def _core_wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None @@ -1942,23 +1958,27 @@ class HostRegistry(HostModelInfo): def _core_config_deployments(self) -> Iterator[Mapping[str, object]]: """A generator, as the Python core's: the rows are read when iteration starts.""" - yield from self._rows(self._lookup().lookup_config()) + with self._deployment_index_lock: + rows: Final = self._rows(self._lookup().lookup_config()) + yield from rows # -- names ------------------------------------------------------------------------------------ def _core_get_model_names(self, team_id: str | None = None) -> list[str]: # mutable-ok: a fresh list - names, raw = self._lookup().lookup_model_names(team_id, self._list_scope(None)) - self._raise_at(raw) - return names + with self._deployment_index_lock: + names, raw = self._lookup().lookup_model_names(team_id, self._list_scope(None)) + self._raise_at(raw) + return names def _core_get_model_ids( self, model_name: str | None = None, exclude_team_models: bool = False ) -> list[str]: # mutable-ok: a fresh list - if model_name is not None and _index_key(model_name) is None: - return [] - ids, raw = self._lookup().lookup_model_ids(model_name, exclude_team_models) - self._raise_at(raw) - return ids + with self._deployment_index_lock: + if model_name is not None and _index_key(model_name) is None: + return [] + ids, raw = self._lookup().lookup_model_ids(model_name, exclude_team_models) + self._raise_at(raw) + return ids # -- resolving a name ------------------------------------------------------------------------- @@ -2017,12 +2037,6 @@ class HostRegistry(HostModelInfo): return provider_prefixed return None - def _get_deployment_by_litellm_model(self, model: object) -> list[UntypedEntry]: # mutable-ok: a fresh list - handle: Final = self._live_handle() - if handle is None or not isinstance(model, str): - return [] - return list(self._at(handle.litellm_model_positions(model))) - # -- credentials ------------------------------------------------------------------------------ @staticmethod @@ -2041,18 +2055,20 @@ class HostRegistry(HostModelInfo): def _get_model_group_deployment_usable_by_team( self, model_group_name: str, team_id: str | None ) -> Deployment | None: - if _index_key(model_group_name) is None: - return None - position, raw = self._lookup().lookup_group_usable_by_team(model_group_name, team_id) - self._raise_at(raw) - return None if position is None else Deployment(**self._core_model_list[position]) # pyright: ignore[reportArgumentType] # a stored entry + with self._deployment_index_lock: + if _index_key(model_group_name) is None: + return None + position, raw = self._lookup().lookup_group_usable_by_team(model_group_name, team_id) + self._raise_at(raw) + return None if position is None else Deployment(**self._core_model_list[position]) # pyright: ignore[reportArgumentType] # a stored entry def _get_team_public_name_deployment(self, model_id: str, team_id: str | None) -> Deployment | None: - if team_id is None or _index_key(model_id) is None: - return None - position, raw = self._lookup().lookup_team_public_name(model_id, team_id) - self._raise_at(raw) - return None if position is None else Deployment(**self._core_model_list[position]) # pyright: ignore[reportArgumentType] # a stored entry + with self._deployment_index_lock: + if team_id is None or _index_key(model_id) is None: + return None + position, raw = self._lookup().lookup_team_public_name(model_id, team_id) + self._raise_at(raw) + return None if position is None else Deployment(**self._core_model_list[position]) # pyright: ignore[reportArgumentType] # a stored entry def _get_wildcard_deployment_usable_by_team(self, model_id: str, team_id: str | None) -> Deployment | None: team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None @@ -2105,7 +2121,10 @@ class HostRegistry(HostModelInfo): # -- routable names --------------------------------------------------------------------------- def _routable_deployments(self, model_name: str, team_id: str | None) -> tuple[UntypedEntry, ...]: - return tuple(self._rows(self._lookup().lookup_routable(self._core_routable_model_group(model_name), team_id))) + with self._deployment_index_lock: + return tuple( + self._rows(self._lookup().lookup_routable(self._core_routable_model_group(model_name), team_id)) + ) def _core_team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None": """The per-minute ceiling ``team_id``'s PTU shares set on the group serving ``model_group``, @@ -2169,24 +2188,26 @@ class HostRegistry(HostModelInfo): """The deployment ids that could serve ``model`` for ``team_id``: the alias target, then a routing group, then a team public name or wildcard route for a name that is no model group, then the model group (``get_candidate_model_ids_for_route``). Read-only.""" - from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + with self._deployment_index_lock: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider - resolved: Final = self._core_get_model_from_alias(model=model) or model - group: Final = self.get_routing_group(resolved) if self._core_routing_groups else None - ids, raw = self._lookup().lookup_candidate_ids( - resolved, - team_id, - None if group is None else list(group.models), - [resolved, f"{inferred_provider(resolved)}/{resolved}"], - ) - self._raise_at(raw) - return frozenset(ids) + resolved: Final = self._core_get_model_from_alias(model=model) or model + group: Final = self.get_routing_group(resolved) if self._core_routing_groups else None + ids, raw = self._lookup().lookup_candidate_ids( + resolved, + team_id, + None if group is None else list(group.models), + [resolved, f"{inferred_provider(resolved)}/{resolved}"], + ) + self._raise_at(raw) + return frozenset(ids) def _core_team_model_has_alternatives(self, deployment_id: str) -> bool: """Whether a team deployment's public name has another unpaused deployment of that team.""" - found, raw = self._lookup().lookup_team_model_has_alternatives(deployment_id) - self._raise_at(raw) - return found + with self._deployment_index_lock: + found, raw = self._lookup().lookup_team_model_has_alternatives(deployment_id) + self._raise_at(raw) + return found def _core_resolve_served_model_name(self, model_name: str, served_model: str) -> str: """``served_model`` when ``model_name`` routes to it (the alias target, a routing group @@ -2315,17 +2336,18 @@ class HostRegistry(HostModelInfo): def _core_get_model_access_groups( self, model_name: str | None = None, model_access_group: str | None = None, team_id: str | None = None ) -> Mapping[str, Sequence[str]]: - use_cache: Final = model_name is None and model_access_group is None and team_id is None - if use_cache and self._access_groups_cache is not None: - return self._access_groups_cache - pairs, raw = self._lookup().lookup_access_groups( - model_name, model_access_group, team_id, self._list_scope(model_name) - ) - self._raise_at(raw) - groups: Final[dict[str, list[str]]] = {g: list(n) for g, n in pairs} # mutable-ok: as a defaultdict - if use_cache: - self._access_groups_cache = groups - return groups + with self._deployment_index_lock: + use_cache: Final = model_name is None and model_access_group is None and team_id is None + if use_cache and self._access_groups_cache is not None: + return self._access_groups_cache + pairs, raw = self._lookup().lookup_access_groups( + model_name, model_access_group, team_id, self._list_scope(model_name) + ) + self._raise_at(raw) + groups: Final[dict[str, list[str]]] = {g: list(n) for g, n in pairs} # mutable-ok: as a defaultdict + if use_cache: + self._access_groups_cache = groups + return groups def _core_is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: groups: Final = self._core_get_model_access_groups(model_access_group=model_access_group) diff --git a/tests/unit/test_router/core/test_registry_characterization_lookups.py b/tests/unit/test_router/core/test_registry_characterization_lookups.py index 19d58ff10d4..94c88de73fe 100644 --- a/tests/unit/test_router/core/test_registry_characterization_lookups.py +++ b/tests/unit/test_router/core/test_registry_characterization_lookups.py @@ -4,6 +4,7 @@ access groups; blocked and unhealthy groups; credentials lookups. """ import time +from typing import Final import pytest @@ -680,3 +681,17 @@ def test_assigning_model_list_swaps_the_list_without_reindexing(make_core: MakeC found = core.registry().deployment("a") assert found is not None assert found.model_name == "n" + + +def test_the_routing_groups_source_reads_its_rows_once(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], + routing_groups=[{"group_name": "grp", "models": ["g"], "routing_strategy": "simple-shuffle"}], + ) + first: Final = core.registry().deployments(source="routing_groups") + assert ids(first) == ["a", "b"] + assert core.registry().deployments(source="routing_groups") is first + + +def test_the_credential_deployment_of_a_non_string_id_is_none(make_core: MakeCore) -> None: + assert _core(make_core).registry().credentials(123, form="deployment") is None # pyright: ignore[reportArgumentType] # a caller's non-string id diff --git a/tests/unit/test_router/core/test_registry_characterization_writes.py b/tests/unit/test_router/core/test_registry_characterization_writes.py index 0558a16fe9d..23fa2a8e0e9 100644 --- a/tests/unit/test_router/core/test_registry_characterization_writes.py +++ b/tests/unit/test_router/core/test_registry_characterization_writes.py @@ -5,7 +5,11 @@ what ``add_deployment``, ``set_model_list``, upserts and deletes leave behind in and ``registry()`` views. """ +import threading import warnings +from collections.abc import Callable, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from typing import Final import pytest @@ -514,3 +518,80 @@ async def test_the_deprecated_arefresh_model_info_warns(make_core: MakeCore) -> await core.arefresh_model_info() # pyright: ignore[reportAttributeAccessIssue] # deprecated, still public assert any("arefresh_model_info is deprecated" in str(w.message) for w in caught) + + +# -- reads racing writes ---------------------------------------------------------------------------- + +_GROUPS: Final = 4 + + +def _two_per_group(make_core: MakeCore) -> RouterCore: + return make_core(model_list=[deployment(f"m{index % _GROUPS}", f"d{index}") for index in range(2 * _GROUPS)]) + + +def _group_ids(core: RouterCore) -> Mapping[int, Sequence[str]]: + return {group: ids(core.registry().deployments(f"m{group}", source="all")) for group in range(_GROUPS)} + + +@pytest.mark.parametrize("upserted", [0, 3, _GROUPS + 1]) +def test_a_read_on_another_thread_waits_for_an_upsert_to_read_the_list_back(make_core: MakeCore, upserted: int) -> None: + """A lookup's positions are read against the mirror of the same registry version: a read that + starts once router-core holds a write waits until the mirror is read back, so it never reads + past the list nor, after a removal, the neighbouring group's entries.""" + core: Final = _two_per_group(make_core) + hosted: Final = core._hosted # pyright: ignore[reportAttributeAccessIssue] # the registry under test + write: Final[Callable[[Mapping[str, object]], None]] = hosted.changes_runner + started: Final = threading.Event() + pool: Final = ThreadPoolExecutor(max_workers=1) + reads: Final[list[object]] = [] # mutable-ok: one per write + + def read() -> Mapping[int, Sequence[str]]: + started.set() + return _group_ids(core) + + def write_with_a_read_started(changes: Mapping[str, object]) -> None: + write(changes) + started.clear() + reads.append(pool.submit(read)) + assert started.wait(timeout=5) + # The reader runs while router-core holds the write and the mirror does not yet. + _ = threading.Event().wait(0.02) + + hosted.changes_runner = write_with_a_read_started + try: + _ = core.apply(ChangeSet(upsert=[typed(f"m{upserted % _GROUPS}", f"d{upserted}", timeout=11)])) + finally: + hosted.changes_runner = write + + assert reads + for future in reads: + seen = future.result(timeout=5) # pyright: ignore[reportAttributeAccessIssue] # a Future + for group in range(_GROUPS): + # Between the upsert's removal and its add, its group is served by the other deployment. + assert seen[group], (group, seen) + assert set(seen[group]) <= {f"d{group}", f"d{group + _GROUPS}"}, (group, seen) + pool.shutdown() + assert sorted(_group_ids(core)[upserted % _GROUPS]) == sorted( + [f"d{upserted % _GROUPS}", f"d{upserted % _GROUPS + _GROUPS}"] + ) + + +def test_appending_to_the_live_model_list_reads_the_entry_back(make_core: MakeCore) -> None: + core: Final = _two_per_group(make_core) + live: Final = core._hosted._core_model_list # pyright: ignore[reportAttributeAccessIssue] # the live list + live.append(deployment("m0", "d8")) + assert ids(live) == [*(f"d{index}" for index in range(2 * _GROUPS)), "d8"] + + +def test_deleting_a_live_model_list_entry_reads_the_list_back(make_core: MakeCore) -> None: + core: Final = _two_per_group(make_core) + live: Final = core._hosted._core_model_list # pyright: ignore[reportAttributeAccessIssue] # the live list + del live[0] + assert ids(live) == [f"d{index}" for index in range(1, 2 * _GROUPS)] + + +def test_ending_a_build_that_never_began_leaves_the_list(make_core: MakeCore) -> None: + core: Final = _two_per_group(make_core) + live: Final = core._hosted._core_model_list # pyright: ignore[reportAttributeAccessIssue] # the live list + live.end_build() + assert ids(live) == [f"d{index}" for index in range(2 * _GROUPS)] From 3e3842046d8b2b4b70fefe7e2819333c770aaa9c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:59:48 +0000 Subject: [PATCH 017/189] test(budgets): expect per-model tpm and rpm limits in budget info (#50967) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/management/test_budget_updates.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/management/test_budget_updates.py b/tests/integration/management/test_budget_updates.py index 6939b447866..aa351078ff9 100644 --- a/tests/integration/management/test_budget_updates.py +++ b/tests/integration/management/test_budget_updates.py @@ -36,7 +36,8 @@ def test_shortening_budget_duration_moves_reset_at_onto_the_new_schedule(gateway BUDGET_COLUMNS: Final = ( "budget_id, max_budget, soft_budget, max_parallel_requests, tpm_limit, rpm_limit, tpd_limit, model_max_budget, " - "budget_duration, allowed_models, temp_budget_increase, temp_budget_expiry, created_by, updated_by" + "model_tpm_limit, model_rpm_limit, budget_duration, allowed_models, temp_budget_increase, temp_budget_expiry, " + "created_by, updated_by" ) VOLATILE_INFO_FIELDS: Final = frozenset({"created_at", "updated_at", "budget_reset_at"}) UNLOADED_RELATIONS: Final[dict[str, JsonValue]] = dict.fromkeys( @@ -128,6 +129,8 @@ def test_partial_update_keeps_sibling_budget_fields(gateway: Gateway) -> None: "rpm_limit": 60, "tpd_limit": 100000, "model_max_budget": {"openai/gpt-4o-mini": {"max_budget": 1.0, "budget_duration": "1d"}}, + "model_tpm_limit": None, + "model_rpm_limit": None, "budget_duration": "30d", "allowed_models": [], "temp_budget_increase": None, From c229851924c4187718d9b102ca32e4a804319a94 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 10:00:20 -0800 Subject: [PATCH 018/189] feat(ui): customer portal (beta) surfacing the existing customer endpoints and customer usage (#50807) * feat(ui): Customer Portal (Beta) surfacing the existing customer endpoints and customer usage Adds a Customer Portal mode to the dashboard (product switcher entry, Customers and Chargebacks nav) built only on endpoints and data the proxy already serves. Customers: list ranked by this month's spend from the customer daily-activity aggregate, status (active, over budget, blocked), search, Create Customer through /customer/new (id, alias, monthly max_budget, tpm_limit, rpm_limit), Edit alias and budget through /customer/update, Block and Unblock. Customer detail shows the keys that sent traffic as that customer this month from the api_key_breakdown of the same aggregate, read only. Chargebacks: per month, grouped by customer, customer and key, or customer and model, with CSV export. All strings go through i18n. Pure helpers (customerForms, customerPortalData, chargebackRows) carry unit tests, and the ViewSwitcher integration test covers entering and leaving the portal. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): refresh the customers list after editing or blocking from customer detail Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): name the expected objects in the customer portal tests so the inline-object lint budget holds Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): mock next/navigation in the navbar test now that ViewSwitcher routes with useRouter Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): format customer portal money and limits through the format keys instead of inline formatters Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): note that customer blocking needs blocked_user_check and neutralise formula prefixes in the chargeback CSV Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chargebacks/chargebackRows.test.ts | 106 +++++++++++ .../(dashboard)/chargebacks/chargebackRows.ts | 87 +++++++++ .../src/app/(dashboard)/chargebacks/page.tsx | 175 ++++++++++++++++++ .../_components/CreateCustomerDialog.tsx | 122 ++++++++++++ .../customers/_components/CustomerDetail.tsx | 156 ++++++++++++++++ .../_components/EditCustomerDialog.tsx | 96 ++++++++++ .../customers/_components/StatusBadge.tsx | 15 ++ .../_components/customerForms.test.ts | 49 +++++ .../customers/_components/customerForms.ts | 46 +++++ .../_components/customerPortalData.test.ts | 110 +++++++++++ .../_components/customerPortalData.ts | 89 +++++++++ .../src/app/(dashboard)/customers/page.tsx | 153 +++++++++++++++ .../src/app/(dashboard)/layout.tsx | 10 +- .../src/app/(dashboard)/legacyPageRoutes.ts | 2 + .../Navbar/ViewSwitcher.integration.test.tsx | 28 ++- .../src/components/Navbar/ViewSwitcher.tsx | 40 +++- .../src/components/leftnav.tsx | 32 +++- .../src/components/navbar.test.tsx | 2 + .../src/contexts/PluginModeContext.tsx | 6 +- ui/litellm-dashboard/src/i18n/locales/en.json | 118 +++++++++++- 20 files changed, 1429 insertions(+), 13 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/chargebacks/chargebackRows.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/chargebacks/chargebackRows.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/chargebacks/page.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerDetail.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/customers/_components/EditCustomerDialog.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/customers/_components/StatusBadge.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerForms.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerForms.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerPortalData.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerPortalData.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/customers/page.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/chargebacks/chargebackRows.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/chargebacks/chargebackRows.test.ts new file mode 100644 index 00000000000..041d683803c --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/chargebacks/chargebackRows.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "vitest"; +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import type { CustomerUsage } from "../customers/_components/customerPortalData"; +import { rowsByCustomer, rowsByCustomerAndKey, rowsByCustomerAndModel, sumRows, toCsv } from "./chargebackRows"; + +const totals = (spend: number, requests = 1) => ({ + spend, + requests, + inputTokens: requests * 10, + outputTokens: requests * 5, +}); + +const usage: ReadonlyMap = new Map([ + [ + "acme", + { + ...totals(6, 3), + keys: [ + { hash: "sk-hash-aaaaaaaaaaaa", alias: "prod", ...totals(5, 2) }, + { hash: "sk-hash-bbbbbbbbbbbb", alias: null, ...totals(1, 1) }, + ], + }, + ], + ["idle", { ...totals(0, 0), keys: [] }], + ["globex", { ...totals(9, 1), keys: [{ hash: "h3", alias: "main", ...totals(9, 1) }] }], +]); + +describe("rowsByCustomer", () => { + test("one row per customer with traffic, highest spend first", () => { + expect(rowsByCustomer(usage).map((r) => [r.customer, r.dimension, r.spend])).toEqual([ + ["globex", null, 9], + ["acme", null, 6], + ]); + }); +}); + +describe("rowsByCustomerAndKey", () => { + test("one row per customer and key, labelled by alias or hash prefix", () => { + expect(rowsByCustomerAndKey(usage).map((r) => [r.customer, r.dimension, r.spend])).toEqual([ + ["globex", "main", 9], + ["acme", "prod", 5], + ["acme", "sk-hash-bbbb", 1], + ]); + }); +}); + +describe("rowsByCustomerAndModel", () => { + const metrics = (spend: number): SpendMetrics => ({ + spend, + prompt_tokens: 1, + completion_tokens: 1, + total_tokens: 2, + api_requests: 1, + successful_requests: 1, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }); + const day = (models: Record): DailyData => ({ + date: "2026-11-01", + metrics: metrics(0), + breakdown: { + models: Object.fromEntries( + Object.entries(models).map(([m, s]) => [m, { metrics: metrics(s), metadata: {}, api_key_breakdown: {} }]), + ), + model_groups: {}, + mcp_servers: {}, + providers: {}, + api_keys: {}, + entities: {}, + }, + }); + test("sums each customer's models across days", () => { + const rows = rowsByCustomerAndModel(new Map([["acme", [day({ "gpt-5": 1, claude: 2 }), day({ "gpt-5": 4 })]]])); + expect(rows.map((r) => [r.customer, r.dimension, r.spend, r.requests])).toEqual([ + ["acme", "gpt-5", 5, 2], + ["acme", "claude", 2, 1], + ]); + }); +}); + +describe("sumRows and toCsv", () => { + test("total row matches the sum of the rows", () => { + const expected = { spend: 15, requests: 4, inputTokens: 40, outputTokens: 20 }; + expect(sumRows(rowsByCustomer(usage))).toEqual(expected); + }); + test("csv carries alias, optional dimension column and escaped quotes", () => { + const rows = rowsByCustomerAndKey(usage).slice(0, 1); + const csv = toCsv(rows, new Map([["globex", 'Globex "Inc"']]), "key"); + expect(csv.split("\n")).toEqual([ + '"customer_id","customer","key","requests","input_tokens","output_tokens","spend_usd"', + '"globex","Globex ""Inc""","main","1","10","5","9.000000"', + ]); + }); + test("csv neutralises ids and aliases that a spreadsheet would run as a formula", () => { + const rows = rowsByCustomer(new Map([['=HYPERLINK("x")', usage.get("acme")!]])); + const line = toCsv(rows, new Map([['=HYPERLINK("x")', "-1+1"]]), null).split("\n")[1]; + expect(line.startsWith(`"'=HYPERLINK(""x"")","'-1+1",`)).toBe(true); + expect(line.endsWith('"3","30","15","6.000000"')).toBe(true); + }); + test("csv without a dimension has no dimension column", () => { + expect(toCsv(rowsByCustomer(usage), new Map(), null).split("\n")[0]).toBe( + '"customer_id","customer","requests","input_tokens","output_tokens","spend_usd"', + ); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/chargebacks/chargebackRows.ts b/ui/litellm-dashboard/src/app/(dashboard)/chargebacks/chargebackRows.ts new file mode 100644 index 00000000000..7f86b7cf6d6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/chargebacks/chargebackRows.ts @@ -0,0 +1,87 @@ +import type { DailyData } from "@/components/UsagePage/types"; +import { addMetrics, type CustomerUsage, type Totals } from "../customers/_components/customerPortalData"; + +export type GroupBy = "customer" | "key" | "model"; + +export interface ChargebackRow extends Totals { + customer: string; + dimension: string | null; +} + +const ZERO: Totals = { spend: 0, requests: 0, inputTokens: 0, outputTokens: 0 }; + +const hasTraffic = (r: Totals) => r.requests > 0 || r.spend > 0; +const bySpend = (a: Totals, b: Totals) => b.spend - a.spend; + +export const rowsByCustomer = (usage: ReadonlyMap): ChargebackRow[] => + [...usage.entries()] + .map(([customer, u]) => ({ + customer, + dimension: null, + spend: u.spend, + requests: u.requests, + inputTokens: u.inputTokens, + outputTokens: u.outputTokens, + })) + .filter(hasTraffic) + .sort(bySpend); + +export const rowsByCustomerAndKey = (usage: ReadonlyMap): ChargebackRow[] => + [...usage.entries()] + .flatMap(([customer, u]) => u.keys.map((k) => ({ ...k, customer, dimension: k.alias ?? k.hash.slice(0, 12) }))) + .filter(hasTraffic) + .sort(bySpend); + +export const rowsByCustomerAndModel = (perCustomer: ReadonlyMap): ChargebackRow[] => + [...perCustomer.entries()] + .flatMap(([customer, days]) => { + const byModel = new Map(); + for (const day of days) { + for (const [model, m] of Object.entries(day.breakdown.models)) + byModel.set(model, addMetrics(byModel.get(model) ?? ZERO, m.metrics)); + } + return [...byModel.entries()].map(([model, t]) => ({ ...t, customer, dimension: model })); + }) + .filter(hasTraffic) + .sort(bySpend); + +export const sumRows = (rows: readonly Totals[]): Totals => + rows.reduce( + (acc, r) => ({ + spend: acc.spend + r.spend, + requests: acc.requests + r.requests, + inputTokens: acc.inputTokens + r.inputTokens, + outputTokens: acc.outputTokens + r.outputTokens, + }), + ZERO, + ); + +const FORMULA_PREFIX = /^[=+\-@\t\r]/; +const neutralise = (v: string) => (FORMULA_PREFIX.test(v) ? `'${v}` : v); +const csvCell = (v: string | number) => `"${(typeof v === "string" ? neutralise(v) : String(v)).replace(/"/g, '""')}"`; + +export const toCsv = ( + rows: readonly ChargebackRow[], + aliasOf: ReadonlyMap, + dimension: string | null, +): string => { + const header = [ + "customer_id", + "customer", + ...(dimension ? [dimension] : []), + "requests", + "input_tokens", + "output_tokens", + "spend_usd", + ]; + const lines = rows.map((r) => [ + r.customer, + aliasOf.get(r.customer) ?? r.customer, + ...(dimension ? [r.dimension ?? ""] : []), + r.requests, + r.inputTokens, + r.outputTokens, + r.spend.toFixed(6), + ]); + return [header, ...lines].map((l) => l.map(csvCell).join(",")).join("\n"); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/chargebacks/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/chargebacks/page.tsx new file mode 100644 index 00000000000..262b918bde5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/chargebacks/page.tsx @@ -0,0 +1,175 @@ +"use client"; + +import { useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { useQuery } from "@tanstack/react-query"; +import { Download } from "lucide-react"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useCustomers } from "@/app/(dashboard)/hooks/customers/useCustomers"; +import { dailyActivityAggregatedCall } from "@/components/networking"; +import { toDailyData } from "@/components/UsagePage/dailyActivityApi"; +import { Button } from "@/components/ui/button"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { monthRange, useCustomerUsage } from "../customers/_components/customerPortalData"; +import { + rowsByCustomer, + rowsByCustomerAndKey, + rowsByCustomerAndModel, + sumRows, + toCsv, + type GroupBy, +} from "./chargebackRows"; + +const GROUPS: readonly GroupBy[] = ["customer", "key", "model"]; +const MONTHS = [0, 1, 2] as const; + +const downloadCsv = (csv: string, filename: string) => { + const url = URL.createObjectURL(new Blob([csv], { type: "text/csv" })); + Object.assign(document.createElement("a"), { href: url, download: filename }).click(); + URL.revokeObjectURL(url); +}; + +export default function ChargebacksPage() { + const { t, i18n } = useTranslation(); + const { accessToken } = useAuthorized(); + const [monthsBack, setMonthsBack] = useState(0); + const [groupBy, setGroupBy] = useState("customer"); + const customersQuery = useCustomers(100); + const customers = useMemo(() => customersQuery.data?.pages.flat() ?? [], [customersQuery.data]); + const aliasOf = useMemo(() => new Map(customers.map((c) => [c.user_id, c.alias || c.user_id])), [customers]); + const usageQuery = useCustomerUsage(accessToken, monthsBack); + const customerIds = useMemo(() => [...(usageQuery.data?.keys() ?? [])].sort(), [usageQuery.data]); + + const perModel = useQuery({ + queryKey: ["customer-portal", "usage-by-model", monthsBack, customerIds], + enabled: Boolean(accessToken) && groupBy === "model" && usageQuery.isSuccess, + queryFn: async () => { + const { start, end } = monthRange(monthsBack); + const entries = await Promise.all( + customerIds.map(async (id) => { + const request = { accessToken: accessToken!, startTime: start, endTime: end, entityIds: [id] }; + const res = await dailyActivityAggregatedCall("customer", request); + return [id, toDailyData(res)] as const; + }), + ); + return new Map(entries); + }, + }); + + const rows = useMemo(() => { + if (groupBy === "model") return perModel.data ? rowsByCustomerAndModel(perModel.data) : []; + if (!usageQuery.data) return []; + return groupBy === "key" ? rowsByCustomerAndKey(usageQuery.data) : rowsByCustomer(usageQuery.data); + }, [groupBy, usageQuery.data, perModel.data]); + const total = sumRows(rows); + const isLoading = usageQuery.isLoading || (groupBy === "model" && perModel.isLoading); + const dimensionLabel = groupBy === "customer" ? null : t(`customerPortal.chargebacks.dimension.${groupBy}`); + const monthLabel = (back: number) => + new Intl.DateTimeFormat(i18n.language, { month: "long", year: "numeric", timeZone: "UTC" }).format( + monthRange(back).start, + ); + const exportCsv = () => + downloadCsv( + toCsv(rows, aliasOf, dimensionLabel?.toLowerCase() ?? null), + `chargebacks-${monthRange(monthsBack).start.toISOString().slice(0, 7)}-${groupBy}.csv`, + ); + + return ( +
+
+
+

{t("customerPortal.chargebacks.title")}

+

{t("customerPortal.chargebacks.subtitle")}

+
+
+ + + +
+
+
+ + + + {t("customerPortal.chargebacks.columns.customer")} + {dimensionLabel && {dimensionLabel}} + {t("customerPortal.chargebacks.columns.requests")} + {t("customerPortal.chargebacks.columns.inputTokens")} + {t("customerPortal.chargebacks.columns.outputTokens")} + {t("customerPortal.chargebacks.columns.spend")} + + + + {isLoading && ( + + + {t("customerPortal.common.loading")} + + + )} + {!isLoading && rows.length === 0 && ( + + + {t("customerPortal.chargebacks.empty", { month: monthLabel(monthsBack) })} + + + )} + {rows.map((r) => ( + + +

{aliasOf.get(r.customer) ?? r.customer}

+

{r.customer}

+ + {dimensionLabel && {r.dimension}} + {t("format.number", { val: r.requests })} + {t("format.number", { val: r.inputTokens })} + {t("format.number", { val: r.outputTokens })} + {t("format.usd", { val: r.spend })} + + ))} + {rows.length > 0 && ( + + + {t("customerPortal.common.total")} + + {t("format.number", { val: total.requests })} + + {t("format.number", { val: total.inputTokens })} + + + {t("format.number", { val: total.outputTokens })} + + {t("format.usd", { val: total.spend })} + + )} + +
+
+
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.tsx b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.tsx new file mode 100644 index 00000000000..c645c180e53 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.tsx @@ -0,0 +1,122 @@ +"use client"; + +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { toast } from "sonner"; +import { $api } from "@/lib/http/api"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { buildCreateCustomerBody, digitsOnly, decimalOnly } from "./customerForms"; + +interface Props { + open: boolean; + onOpenChange: (o: boolean) => void; + onCreated: (userId: string) => void; +} + +const errorText = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +export default function CreateCustomerDialog({ open, onOpenChange, onCreated }: Props) { + const { t } = useTranslation(); + const [userId, setUserId] = useState(""); + const [alias, setAlias] = useState(""); + const [budget, setBudget] = useState(""); + const [tpm, setTpm] = useState(""); + const [rpm, setRpm] = useState(""); + const create = $api.useMutation("post", "/customer/new", { + onSuccess: () => { + toast.success(t("customerPortal.toasts.created", { name: alias.trim() || userId.trim() })); + onCreated(userId.trim()); + onOpenChange(false); + }, + onError: (e: unknown) => toast.error(t("customerPortal.toasts.createFailed", { error: errorText(e) })), + }); + const submit = (e: React.FormEvent) => { + e.preventDefault(); + const fields = { userId, alias, budget, tpm, rpm }; + const body = buildCreateCustomerBody(fields); + if (body) create.mutate({ body }); + }; + return ( + + +
+ + {t("customerPortal.createDialog.title")} + {t("customerPortal.createDialog.description")} + +
+ + setUserId(e.target.value)} + placeholder={t("customerPortal.createDialog.customerIdPlaceholder")} + autoFocus + required + /> +
+
+ + setAlias(e.target.value)} + placeholder={t("customerPortal.createDialog.aliasPlaceholder")} + /> +
+
+ + setBudget(decimalOnly(e.target.value))} + placeholder={t("customerPortal.createDialog.maxBudgetPlaceholder")} + /> +
+
+
+ + setTpm(digitsOnly(e.target.value))} + placeholder={t("customerPortal.createDialog.noLimit")} + /> +
+
+ + setRpm(digitsOnly(e.target.value))} + placeholder={t("customerPortal.createDialog.noLimit")} + /> +
+
+

{t("customerPortal.createDialog.limitsHelp")}

+ + + + +
+ + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerDetail.tsx b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerDetail.tsx new file mode 100644 index 00000000000..6e9e94dedf2 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerDetail.tsx @@ -0,0 +1,156 @@ +"use client"; + +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { useQueryClient } from "@tanstack/react-query"; +import { Ban, Pencil } from "lucide-react"; +import { toast } from "sonner"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { $api } from "@/lib/http/api"; +import type { EndUser } from "@/app/(dashboard)/hooks/customers/useCustomers"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { customerStatus, useCustomerUsage } from "./customerPortalData"; +import EditCustomerDialog from "./EditCustomerDialog"; +import StatusBadge from "./StatusBadge"; + +function UsageSummary({ spend, budget }: { spend: number; budget: EndUser["litellm_budget_table"] | undefined }) { + const { t } = useTranslation(); + const max = budget?.max_budget ?? null; + const tpmLimit = budget?.tpm_limit ?? null; + const rpmLimit = budget?.rpm_limit ?? null; + return ( +

+ + {t("customerPortal.detail.spendThisMonth")}{" "} + {t("format.usd", { val: spend })} + + + {max === null + ? t("customerPortal.detail.noBudget") + : t("customerPortal.detail.ofBudget", { max: t("format.usd", { val: max }) })} + + {tpmLimit !== null && ( + + {t("customerPortal.detail.tpm", { val: t("format.number", { val: tpmLimit }) })} + + )} + {rpmLimit !== null && ( + + {t("customerPortal.detail.rpm", { val: t("format.number", { val: rpmLimit }) })} + + )} +

+ ); +} + +export default function CustomerDetail({ customerId, onBack }: { customerId: string; onBack: () => void }) { + const { t } = useTranslation(); + const { accessToken } = useAuthorized(); + const info = $api.useQuery( + "get", + "/customer/info", + { params: { query: { end_user_id: customerId } } }, + { enabled: Boolean(accessToken) }, + ); + const usageQuery = useCustomerUsage(accessToken); + const [editOpen, setEditOpen] = useState(false); + const queryClient = useQueryClient(); + const refresh = () => { + info.refetch(); + queryClient.invalidateQueries({ queryKey: ["get", "/customer/list"] }); + }; + const block = $api.useMutation("post", "/customer/block", { + onSuccess: () => { + toast.success(t("customerPortal.toasts.blocked")); + refresh(); + }, + }); + const unblock = $api.useMutation("post", "/customer/update", { + onSuccess: () => { + toast.success(t("customerPortal.toasts.unblocked")); + refresh(); + }, + }); + + const c = info.data; + const usage = usageQuery.data?.get(customerId); + const spend = usage?.spend ?? 0; + const keys = usage?.keys ?? []; + const label = c?.alias || customerId; + + return ( +
+
+
+
+

{label}

+ {c && } + {customerId} +
+ +
+
+ + {c?.blocked ? ( + + ) : ( + + )} +
+
+

{t("customerPortal.common.blockHint")}

+ +
+
+

{t("customerPortal.detail.keysTitle")}

+

{t("customerPortal.detail.keysSubtitle")}

+
+ {keys.length === 0 ? ( +

{t("customerPortal.detail.keysEmpty")}

+ ) : ( + + + + {t("customerPortal.detail.columns.key")} + {t("customerPortal.detail.columns.requests")} + {t("customerPortal.detail.columns.tokens")} + {t("customerPortal.detail.columns.spend")} + + + + {keys.map((k) => ( + + +

{k.alias ?? t("customerPortal.detail.unnamedKey")}

+

{k.hash.slice(0, 12)}

+ + {t("format.number", { val: k.requests })} + + {t("format.number", { val: k.inputTokens + k.outputTokens })} + + {t("format.usd", { val: k.spend })} + + ))} + +
+ )} +
+ + {c && } +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/EditCustomerDialog.tsx b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/EditCustomerDialog.tsx new file mode 100644 index 00000000000..aebc8d60053 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/EditCustomerDialog.tsx @@ -0,0 +1,96 @@ +"use client"; + +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { toast } from "sonner"; +import { $api } from "@/lib/http/api"; +import type { EndUser } from "@/app/(dashboard)/hooks/customers/useCustomers"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { buildUpdateCustomerBody, decimalOnly } from "./customerForms"; + +interface Props { + open: boolean; + onOpenChange: (o: boolean) => void; + customer: EndUser; + onSaved: () => void; +} + +const asText = (n: number | null | undefined) => (n === null || n === undefined ? "" : String(n)); + +function EditCustomerForm({ onOpenChange, customer, onSaved }: Omit) { + const { t } = useTranslation(); + const [alias, setAlias] = useState(customer.alias ?? ""); + const [budget, setBudget] = useState(asText(customer.litellm_budget_table?.max_budget)); + const budgetId = customer.litellm_budget_table?.budget_id ?? null; + const update = $api.useMutation("post", "/customer/update", { + onSuccess: () => { + toast.success(t("customerPortal.toasts.updated")); + onSaved(); + onOpenChange(false); + }, + onError: (e: unknown) => + toast.error(t("customerPortal.toasts.updateFailed", { error: e instanceof Error ? e.message : String(e) })), + }); + const submit = (e: React.FormEvent) => { + e.preventDefault(); + const fields = { userId: customer.user_id, alias, budget, blocked: Boolean(customer.blocked) }; + update.mutate({ body: buildUpdateCustomerBody(fields) }); + }; + return ( + +
+ + {t("customerPortal.editDialog.title")} + + {customer.user_id} + + +
+ + setAlias(e.target.value)} autoFocus /> +
+
+ + setBudget(decimalOnly(e.target.value))} + placeholder={t("customerPortal.editDialog.maxBudgetPlaceholder")} + /> +
+ {budgetId && ( +

+ {t("customerPortal.editDialog.sharedBudgetWarning", { budgetId })} +

+ )} + + + + +
+ + ); +} + +export default function EditCustomerDialog({ open, ...form }: Props) { + return ( + + {open && } + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/StatusBadge.tsx b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/StatusBadge.tsx new file mode 100644 index 00000000000..005171abc53 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/StatusBadge.tsx @@ -0,0 +1,15 @@ +import { useTranslation } from "react-i18next"; +import { Badge } from "@/components/ui/badge"; +import type { CustomerStatus } from "./customerPortalData"; + +const VARIANT = { active: "secondary", over_budget: "destructive", blocked: "outline" } as const; +const LABEL_KEY = { + active: "customerPortal.status.active", + over_budget: "customerPortal.status.overBudget", + blocked: "customerPortal.status.blocked", +} as const; + +export default function StatusBadge({ status }: { status: CustomerStatus }) { + const { t } = useTranslation(); + return {t(LABEL_KEY[status])}; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerForms.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerForms.test.ts new file mode 100644 index 00000000000..3c7ecc97484 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerForms.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from "vitest"; +import { buildCreateCustomerBody, buildUpdateCustomerBody } from "./customerForms"; + +describe("buildCreateCustomerBody", () => { + const fields = { userId: " acme ", alias: " Acme Corp ", budget: "25", tpm: "1000", rpm: "60" }; + const emptyFields = { userId: "acme", alias: "", budget: "", tpm: "", rpm: "" }; + + test("trims ids and aliases and sends numeric limits with a monthly budget reset", () => { + const expected = { + user_id: "acme", + alias: "Acme Corp", + blocked: false, + max_budget: 25, + budget_duration: "1mo", + tpm_limit: 1000, + rpm_limit: 60, + }; + expect(buildCreateCustomerBody(fields)).toEqual(expected); + }); + + test("empty optional fields become null and no budget means no budget_duration", () => { + const expected = { + user_id: "acme", + alias: null, + blocked: false, + max_budget: null, + budget_duration: null, + tpm_limit: null, + rpm_limit: null, + }; + expect(buildCreateCustomerBody(emptyFields)).toEqual(expected); + }); + + test("a blank customer id produces no request", () => { + expect(buildCreateCustomerBody({ ...fields, userId: " " })).toBeNull(); + }); +}); + +describe("buildUpdateCustomerBody", () => { + test("preserves the blocked flag and clears the budget when emptied", () => { + const fields = { userId: "acme", alias: "", budget: "", blocked: true }; + const expected = { user_id: "acme", alias: null, blocked: true, max_budget: null }; + expect(buildUpdateCustomerBody(fields)).toEqual(expected); + }); + test("parses a decimal budget", () => { + const fields = { userId: "acme", alias: "Acme", budget: "12.50", blocked: false }; + expect(buildUpdateCustomerBody(fields).max_budget).toBe(12.5); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerForms.ts b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerForms.ts new file mode 100644 index 00000000000..f1fafc0c718 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerForms.ts @@ -0,0 +1,46 @@ +import type { components } from "@/lib/http/schema"; + +export type NewCustomerBody = components["schemas"]["NewCustomerRequest"]; +export type UpdateCustomerBody = components["schemas"]["UpdateCustomerRequest"]; + +export const digitsOnly = (s: string) => s.replace(/[^0-9]/g, ""); +export const decimalOnly = (s: string) => s.replace(/[^0-9.]/g, ""); + +const numberOrNull = (s: string): number | null => (s.trim() === "" ? null : Number(s)); + +export interface CreateCustomerFields { + userId: string; + alias: string; + budget: string; + tpm: string; + rpm: string; +} + +export const buildCreateCustomerBody = (f: CreateCustomerFields): NewCustomerBody | null => { + const userId = f.userId.trim(); + if (!userId) return null; + const maxBudget = numberOrNull(f.budget); + return { + user_id: userId, + alias: f.alias.trim() || null, + blocked: false, + max_budget: maxBudget, + budget_duration: maxBudget === null ? null : "1mo", + tpm_limit: numberOrNull(f.tpm), + rpm_limit: numberOrNull(f.rpm), + }; +}; + +export interface EditCustomerFields { + userId: string; + alias: string; + budget: string; + blocked: boolean; +} + +export const buildUpdateCustomerBody = (f: EditCustomerFields): UpdateCustomerBody => ({ + user_id: f.userId, + alias: f.alias.trim() || null, + blocked: f.blocked, + max_budget: numberOrNull(f.budget), +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerPortalData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerPortalData.test.ts new file mode 100644 index 00000000000..cf48c81db5b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerPortalData.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from "vitest"; +import type { DailyData, KeyMetricWithMetadata, MetricWithMetadata, SpendMetrics } from "@/components/UsagePage/types"; +import { customerStatus, monthRange, rankBySpend, usageByCustomer } from "./customerPortalData"; + +const metrics = (spend: number, requests = 1, prompt = 10, completion = 5): SpendMetrics => ({ + spend, + prompt_tokens: prompt, + completion_tokens: completion, + total_tokens: prompt + completion, + api_requests: requests, + successful_requests: requests, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}); + +const key = (alias: string | null, spend: number, requests = 1): KeyMetricWithMetadata => ({ + metrics: metrics(spend, requests), + metadata: { key_alias: alias, team_id: null }, +}); + +const entity = (spend: number, keys: Record, requests = 1): MetricWithMetadata => ({ + metrics: metrics(spend, requests), + metadata: {}, + api_key_breakdown: keys, +}); + +const day = (date: string, entities: Record): DailyData => ({ + date, + metrics: metrics(0, 0, 0, 0), + breakdown: { models: {}, model_groups: {}, mcp_servers: {}, providers: {}, api_keys: {}, entities }, +}); + +describe("usageByCustomer", () => { + test("sums a customer's spend, requests and tokens across days", () => { + const usage = usageByCustomer([ + day("2026-11-01", { acme: entity(1.5, {}, 2) }), + day("2026-11-02", { acme: entity(2.25, {}, 3) }), + ]); + const expected = { spend: 3.75, requests: 5, inputTokens: 20, outputTokens: 10 }; + expect(usage.get("acme")).toMatchObject(expected); + }); + + test("keeps customers separate", () => { + const usage = usageByCustomer([day("2026-11-01", { acme: entity(1, {}), globex: entity(4, {}) })]); + expect(usage.get("acme")?.spend).toBe(1); + expect(usage.get("globex")?.spend).toBe(4); + }); + + test("merges the same key across days and sorts keys by spend", () => { + const usage = usageByCustomer([ + day("2026-11-01", { acme: entity(3, { h1: key("prod", 1), h2: key("staging", 2) }) }), + day("2026-11-02", { acme: entity(5, { h1: key("prod", 5) }) }), + ]); + const prod = { hash: "h1", alias: "prod", spend: 6, requests: 2 }; + const staging = { hash: "h2", alias: "staging", spend: 2, requests: 1 }; + expect(usage.get("acme")?.keys).toEqual([expect.objectContaining(prod), expect.objectContaining(staging)]); + }); + + test("keeps the first non-null alias seen for a key", () => { + const usage = usageByCustomer([ + day("2026-11-01", { acme: entity(1, { h1: key(null, 1) }) }), + day("2026-11-02", { acme: entity(1, { h1: key("prod", 1) }) }), + ]); + expect(usage.get("acme")?.keys[0]?.alias).toBe("prod"); + }); +}); + +describe("customerStatus", () => { + test("blocked wins over budget", () => { + expect(customerStatus({ blocked: true, litellm_budget_table: { max_budget: 1 } }, 5)).toBe("blocked"); + }); + test("over budget once spend reaches max_budget", () => { + expect(customerStatus({ blocked: false, litellm_budget_table: { max_budget: 10 } }, 10)).toBe("over_budget"); + expect(customerStatus({ blocked: false, litellm_budget_table: { max_budget: 10 } }, 9.99)).toBe("active"); + }); + test("no budget is active regardless of spend", () => { + expect(customerStatus({ blocked: false, litellm_budget_table: null }, 1000)).toBe("active"); + }); +}); + +describe("rankBySpend", () => { + const customer = (user_id: string) => ({ user_id, blocked: false, spend: 0, litellm_budget_table: null }); + test("orders customers by this month's spend, highest first, zero for customers without traffic", () => { + const usage = usageByCustomer([day("2026-11-01", { b: entity(2, {}), c: entity(9, { h1: key("k", 9) }) })]); + const rows = rankBySpend([customer("a"), customer("b"), customer("c")], usage); + expect(rows.map((r) => [r.c.user_id, r.spend, r.keys])).toEqual([ + ["c", 9, 1], + ["b", 2, 0], + ["a", 0, 0], + ]); + }); +}); + +describe("monthRange", () => { + test("covers the whole UTC month, monthsBack months ago", () => { + const now = new Date(Date.UTC(2026, 10, 9, 12)); + expect(monthRange(0, now)).toEqual({ + start: new Date(Date.UTC(2026, 10, 1)), + end: new Date(Date.UTC(2026, 10, 30, 23, 59, 59)), + }); + expect(monthRange(1, now)).toEqual({ + start: new Date(Date.UTC(2026, 9, 1)), + end: new Date(Date.UTC(2026, 9, 31, 23, 59, 59)), + }); + }); + test("crosses a year boundary", () => { + expect(monthRange(2, new Date(Date.UTC(2026, 0, 15))).start).toEqual(new Date(Date.UTC(2025, 10, 1))); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerPortalData.ts b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerPortalData.ts new file mode 100644 index 00000000000..5bc1c2ec437 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerPortalData.ts @@ -0,0 +1,89 @@ +import { useQuery } from "@tanstack/react-query"; +import { dailyActivityAggregatedCall } from "@/components/networking"; +import { toDailyData } from "@/components/UsagePage/dailyActivityApi"; +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import type { EndUser } from "@/app/(dashboard)/hooks/customers/useCustomers"; + +export type CustomerStatus = "active" | "over_budget" | "blocked"; + +export const monthRange = (monthsBack = 0, now = new Date()): { start: Date; end: Date } => ({ + start: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - monthsBack, 1)), + end: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - monthsBack + 1, 0, 23, 59, 59)), +}); + +export interface Totals { + spend: number; + requests: number; + inputTokens: number; + outputTokens: number; +} + +export interface CustomerKeyUsage extends Totals { + hash: string; + alias: string | null; +} + +export interface CustomerUsage extends Totals { + keys: readonly CustomerKeyUsage[]; +} + +const ZERO: Totals = { spend: 0, requests: 0, inputTokens: 0, outputTokens: 0 }; + +export const addMetrics = (t: Totals, m: Partial | undefined): Totals => ({ + spend: t.spend + (m?.spend ?? 0), + requests: t.requests + (m?.api_requests ?? 0), + inputTokens: t.inputTokens + (m?.prompt_tokens ?? 0), + outputTokens: t.outputTokens + (m?.completion_tokens ?? 0), +}); + +export const usageByCustomer = (days: readonly DailyData[]): Map => { + const totals = new Map(); + const keys = new Map>(); + for (const day of days) { + for (const [id, entity] of Object.entries(day.breakdown.entities)) { + totals.set(id, addMetrics(totals.get(id) ?? ZERO, entity.metrics)); + const byKey = keys.get(id) ?? new Map(); + for (const [hash, key] of Object.entries(entity.api_key_breakdown)) { + const prev = byKey.get(hash) ?? { ...ZERO, hash, alias: key.metadata.key_alias }; + byKey.set(hash, { ...prev, ...addMetrics(prev, key.metrics), alias: prev.alias ?? key.metadata.key_alias }); + } + keys.set(id, byKey); + } + } + return new Map( + [...totals.entries()].map(([id, t]) => [ + id, + { ...t, keys: [...(keys.get(id)?.values() ?? [])].sort((a, b) => b.spend - a.spend) }, + ]), + ); +}; + +export const useCustomerUsage = (accessToken: string | null, monthsBack = 0) => + useQuery({ + queryKey: ["customer-portal", "usage", monthsBack], + enabled: Boolean(accessToken), + queryFn: async () => { + const { start, end } = monthRange(monthsBack); + const res = await dailyActivityAggregatedCall("customer", { + accessToken: accessToken!, + startTime: start, + endTime: end, + }); + return usageByCustomer(toDailyData(res)); + }, + }); + +export const customerStatus = (c: Pick, spend: number): CustomerStatus => { + if (c.blocked) return "blocked"; + const max = c.litellm_budget_table?.max_budget; + return max !== null && max !== undefined && spend >= max ? "over_budget" : "active"; +}; + +export const rankBySpend = (customers: readonly EndUser[], usage: ReadonlyMap | undefined) => + customers + .map((c) => { + const u = usage?.get(c.user_id); + const spend = u?.spend ?? 0; + return { c, spend, keys: u?.keys.length ?? 0, status: customerStatus(c, spend) }; + }) + .sort((a, b) => b.spend - a.spend); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/customers/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/customers/page.tsx new file mode 100644 index 00000000000..f80d15f5acf --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/customers/page.tsx @@ -0,0 +1,153 @@ +"use client"; + +import { useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { parseAsString, useQueryState } from "nuqs"; +import { Plus } from "lucide-react"; +import { toast } from "sonner"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useCustomers, type EndUser } from "@/app/(dashboard)/hooks/customers/useCustomers"; +import { $api } from "@/lib/http/api"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import CustomerDetail from "./_components/CustomerDetail"; +import CreateCustomerDialog from "./_components/CreateCustomerDialog"; +import StatusBadge from "./_components/StatusBadge"; +import { rankBySpend, useCustomerUsage } from "./_components/customerPortalData"; + +export default function CustomersPage() { + const { t } = useTranslation(); + const { accessToken } = useAuthorized(); + const [selectedId, setSelectedId] = useQueryState("id", parseAsString); + const [search, setSearch] = useState(""); + const [createOpen, setCreateOpen] = useState(false); + const customersQuery = useCustomers(100, search || undefined); + const usageQuery = useCustomerUsage(accessToken); + const customers: EndUser[] = useMemo(() => customersQuery.data?.pages.flat() ?? [], [customersQuery.data]); + const rows = useMemo(() => rankBySpend(customers, usageQuery.data), [customers, usageQuery.data]); + const block = $api.useMutation("post", "/customer/block", { + onSuccess: () => { + toast.success(t("customerPortal.toasts.blocked")); + customersQuery.refetch(); + }, + }); + const unblock = $api.useMutation("post", "/customer/update", { + onSuccess: () => { + toast.success(t("customerPortal.toasts.unblocked")); + customersQuery.refetch(); + }, + }); + + if (selectedId) return setSelectedId(null)} />; + + return ( +
+
+
+

{t("customerPortal.list.title")}

+

{t("customerPortal.list.subtitle")}

+
+ +
+

{t("customerPortal.common.blockHint")}

+
+
+ setSearch(e.target.value)} + /> +
+ + + + {t("customerPortal.list.columns.customer")} + {t("customerPortal.list.columns.spend")} + {t("customerPortal.list.columns.budget")} + {t("customerPortal.list.columns.keys")} + {t("customerPortal.list.columns.status")} + + + + + {customersQuery.isLoading && ( + + + {t("customerPortal.list.loading")} + + + )} + {!customersQuery.isLoading && rows.length === 0 && ( + + + {t("customerPortal.list.empty")} + + + )} + {rows.map(({ c, spend, keys, status }) => { + const max = c.litellm_budget_table?.max_budget ?? null; + return ( + setSelectedId(c.user_id, { history: "push" })} + > + +

{c.alias || c.user_id}

+

{c.user_id}

+ + + {t("format.usd", { val: spend })} + + + {max === null + ? t("customerPortal.common.noBudget") + : t("customerPortal.list.budgetUsage", { + spend: t("format.usd", { val: spend }), + max: t("format.usd", { val: max }), + })} + + {keys} + + + + e.stopPropagation()}> + {c.blocked ? ( + + ) : ( + + )} + + + ); + })} + +
+
+ { + customersQuery.refetch(); + setSelectedId(id, { history: "push" }); + }} + /> +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index 5c93d214812..d898f1b0373 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -116,13 +116,19 @@ export function AgentControlPlaneView() { } const FULL_BLEED_SEGMENTS = new Set(["logs"]); +const CUSTOMER_PORTAL_SEGMENTS = new Set(["customers", "chargebacks"]); function DashboardShell({ children }: { children: React.ReactNode }) { const { t } = useTranslation(); const { accessToken } = useAuth(); - const { mode } = usePluginMode(); + const { mode, setMode } = usePluginMode(); const pathname = usePathname(); const routeSegment = routeSegmentForPathname(pathname); + const isPortalRoute = CUSTOMER_PORTAL_SEGMENTS.has(routeSegment); + useEffect(() => { + if (isPortalRoute && mode !== "customer-portal") setMode("customer-portal"); + if (!isPortalRoute && mode === "customer-portal") setMode("ai-gateway"); + }, [isPortalRoute, mode, setMode]); useEffect(() => { void recordUiEvent({ page: routeSegment || "home", action: "view" }); }, [routeSegment]); @@ -142,7 +148,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) { const sidebarCollapsed = sidebarOverride?.segment === routeSegment ? sidebarOverride.collapsed : isFullBleed; const toggleSidebar = () => setSidebarOverride({ segment: routeSegment, collapsed: !sidebarCollapsed }); - const isGateway = mode === "ai-gateway"; + const isGateway = mode === "ai-gateway" || mode === "customer-portal"; // Non-gateway (agent control plane) mode keeps the original full-width Navbar, // which carries the account menu; the redesigned sidebar + header shell is diff --git a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts index 19dac805948..90b49e56c5a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts @@ -12,6 +12,8 @@ const LEGACY_PAGE_ROUTES: ReadonlyMap = new Map( chat: "chat", "access-groups": "access-groups", budgets: "budgets", + customers: "customers", + chargebacks: "chargebacks", workflows: "workflows", "guardrails-monitor": "guardrails-monitor", "mcp-servers": "mcp-servers", diff --git a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.integration.test.tsx b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.integration.test.tsx index 64bb8f15585..a0f6e59d029 100644 --- a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.integration.test.tsx @@ -8,13 +8,14 @@ import { setServerRootPath } from "@/lib/serverRootPath"; import ViewSwitcher from "./ViewSwitcher"; -const { state, getUiSettingsMock, getUiConfigMock } = vi.hoisted(() => ({ +const { state, getUiSettingsMock, getUiConfigMock, pushMock } = vi.hoisted(() => ({ state: { pathname: "/ui/", plugins: [] as Plugin[] }, getUiSettingsMock: vi.fn(), getUiConfigMock: vi.fn(), + pushMock: vi.fn(), })); -vi.mock("next/navigation", () => ({ usePathname: () => state.pathname })); +vi.mock("next/navigation", () => ({ usePathname: () => state.pathname, useRouter: () => ({ push: pushMock }) })); vi.mock("@/components/networking", () => ({ getProxyBaseUrl: () => "", getUiSettings: getUiSettingsMock, @@ -56,6 +57,8 @@ describe("ViewSwitcher", () => { }); afterEach(() => { + pushMock.mockClear(); + assignSpy.mockClear(); Object.defineProperty(window, "location", { configurable: true, value: originalLocation }); testQueryClient.clear(); localStorage.clear(); @@ -103,6 +106,27 @@ describe("ViewSwitcher", () => { expect(assignSpy).not.toHaveBeenCalled(); }); + it("should open the Customer Portal at the customers page and come back to the gateway", async () => { + const user = userEvent.setup(); + renderSwitcher(); + + await user.click(screen.getByRole("button", { name: "Switch workspace, current: AI Gateway" })); + await user.click(await screen.findByRole("menuitem", { name: /Customer Portal/ })); + expect(pushMock).toHaveBeenCalledWith("/ui/customers"); + expect(assignSpy).not.toHaveBeenCalled(); + expect(localStorage.getItem("litellm_plugin_mode")).toBeNull(); + }); + + it("should route back to the gateway from Customer Portal mode instead of only flipping the mode", async () => { + localStorage.setItem("litellm_plugin_mode", "customer-portal"); + const user = userEvent.setup(); + renderSwitcher(); + + await user.click(await screen.findByRole("button", { name: "Switch workspace, current: Customer Portal" })); + await user.click(await screen.findByRole("menuitem", { name: /^AI Gateway/ })); + expect(pushMock).toHaveBeenCalledWith("/ui/api-keys"); + }); + it("should navigate from the gateway to enabled Chat without changing plugin mode", async () => { getUiSettingsMock.mockResolvedValue({ values: { enable_chat_ui: true } }); const user = userEvent.setup(); diff --git a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx index 5c8d4a7a850..38b051515fb 100644 --- a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx @@ -1,7 +1,7 @@ import { useTerminology } from "@/contexts/TerminologyContext"; import React from "react"; import { useTranslation } from "react-i18next"; -import { usePathname } from "next/navigation"; +import { usePathname, useRouter } from "next/navigation"; import { Button } from "@/components/ui/button"; import { DropdownMenu, @@ -11,6 +11,7 @@ import { } from "@/components/ui/dropdown-menu"; import { Check, ChevronDown, ChevronsUpDown } from "lucide-react"; import { useSidebar } from "@/components/shared/Sidebar"; +import BetaBadge from "@/components/BetaBadge"; import { usePluginMode } from "@/contexts/PluginModeContext"; import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; import { uiHref } from "@/utils/uiHref"; @@ -19,6 +20,7 @@ import i18n from "@/i18n/i18n"; import { productNames } from "@/i18n/untranslated/productNames"; const GATEWAY = "ai-gateway"; +const CUSTOMER_PORTAL = "customer-portal"; const CHAT = "chat"; const MOYAI = "moyai"; @@ -51,6 +53,9 @@ interface ViewSwitcherProps { contentClassName?: string; } +const pluginLabel = (mode: string, plugins: readonly { name: string; display_name: string }[], portalLabel: string) => + mode === CUSTOMER_PORTAL ? portalLabel : plugins.find((p) => p.name === mode)?.display_name; + export default function ViewSwitcher({ className, contentClassName }: ViewSwitcherProps) { const { t } = useTranslation(); const term = useTerminology(); @@ -58,6 +63,7 @@ export default function ViewSwitcher({ className, contentClassName }: ViewSwitch const { mode, setMode, plugins } = usePluginMode(); const { data: uiSettings } = useUISettings(); const pathname = usePathname(); + const router = useRouter(); const chatEnabled = Boolean(uiSettings?.values?.enable_chat_ui); const moyaiUrl = (uiSettings?.values?.moyai_url as string | undefined) ?? null; @@ -71,7 +77,7 @@ export default function ViewSwitcher({ className, contentClassName }: ViewSwitch const activeLabel = activeWorkspaceLabel( isChatRoute, isMoyaiRoute, - plugins.find((p) => p.name === mode)?.display_name, + pluginLabel(mode, plugins, t("customerPortal.switcher.label")), gatewayLabel, ); @@ -81,6 +87,14 @@ export default function ViewSwitcher({ className, contentClassName }: ViewSwitch ]; const selectMode = (key: string) => { + if (key === CUSTOMER_PORTAL) { + router.push(uiHref("customers")); + return; + } + if (mode === CUSTOMER_PORTAL && key === GATEWAY) { + router.push(uiHref("api-keys")); + return; + } setMode(key); if (isStandaloneRoute) { window.location.assign(uiHref("")); @@ -107,6 +121,21 @@ export default function ViewSwitcher({ className, contentClassName }: ViewSwitch onClick: chatEnabled ? () => window.location.assign(uiHref(CHAT)) : undefined, }; + const customerPortalItem: ViewSwitcherItem = { + key: CUSTOMER_PORTAL, + label: ( +
+ + + {t("customerPortal.switcher.label")} + {" "} + {t("customerPortal.switcher.description")} + + {!isStandaloneRoute && mode === CUSTOMER_PORTAL && } +
+ ), + onClick: () => selectMode(CUSTOMER_PORTAL), + }; const items: ViewSwitcherItem[] = [ ...modeEntries.map((e) => ({ key: e.key, @@ -118,6 +147,7 @@ export default function ViewSwitcher({ className, contentClassName }: ViewSwitch ), onClick: () => selectMode(e.key), })), + customerPortalItem, chatItem, { key: MOYAI, @@ -161,6 +191,12 @@ export default function ViewSwitcher({ className, contentClassName }: ViewSwitch <> {activeLabel} + {!isStandaloneRoute && mode === CUSTOMER_PORTAL && ( + <> + {" "} + + + )} diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index f48271a872c..6040da32f6c 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -55,12 +55,14 @@ import { Trophy, User, Users, + Receipt, Wallet, Wrench, Workflow, } from "lucide-react"; import Link from "next/link"; import ViewSwitcher from "@/components/Navbar/ViewSwitcher"; +import { usePluginMode } from "@/contexts/PluginModeContext"; import { AriaSidebarLink } from "@/components/liteadmin/LiteAdmin"; import { usePathname } from "next/navigation"; import { useMemo, useState } from "react"; @@ -197,6 +199,28 @@ export const visibleMenuGroups = (groups: readonly MenuGroup[], context: MenuVis // Menu groups organized by category - defined outside component for export. // Shape (key/page/label/roles/children) is consumed by page_utils.ts; only the // icons changed to lucide as part of the sidebar redesign. +const customerPortalGroups: MenuGroup[] = [ + { + groupLabel: "CUSTOMER PORTAL", + items: [ + { + key: "customers", + page: "customers", + label: "shell.nav.items.customers", + icon: , + roles: all_admin_roles, + }, + { + key: "chargebacks", + page: "chargebacks", + label: "shell.nav.items.chargebacks", + icon: , + roles: all_admin_roles, + }, + ], + }, +]; + const menuGroups: MenuGroup[] = [ { groupLabel: "AI GATEWAY", @@ -542,6 +566,7 @@ const findMenuItemKey = (route: string): string => { const GROUP_LABELS: Record = { "AI GATEWAY": "shell.nav.groups.aiGateway", + "CUSTOMER PORTAL": "shell.nav.groups.customerPortal", OBSERVABILITY: "shell.nav.groups.observability", "ACCESS CONTROL": "shell.nav.groups.accessControl", "DEVELOPER TOOLS": "shell.nav.groups.developerTools", @@ -553,6 +578,7 @@ export const groupText = (groupLabel: string): string => const SECTION_DISPLAY: Record = { "AI GATEWAY": "shell.nav.sections.aiGateway", + "CUSTOMER PORTAL": "shell.nav.sections.customerPortal", OBSERVABILITY: "shell.nav.sections.observability", "ACCESS CONTROL": "shell.nav.sections.accessControl", "DEVELOPER TOOLS": "shell.nav.sections.developerTools", @@ -575,7 +601,7 @@ export const labelText = (item: MenuItem): string => export const getBreadcrumb = (pathname: string): { section: string | null; title: string } => { const route = routeForPathname(pathname); if (route === "aria") return { section: null, title: "Aria Agent" }; - for (const group of menuGroups) { + for (const group of [...menuGroups, ...customerPortalGroups]) { for (const item of group.items) { const section = sectionText(group.groupLabel); if (routeOf(item) === route) return { section, title: labelText(item) }; @@ -597,6 +623,7 @@ const Sidebar_: React.FC = ({ allowVectorStoresForTeamAdmins, }) => { const { userId, accessToken, userRole, isViewOnly } = useAuthorized(); + const { mode: pluginMode } = usePluginMode(); const { t } = useTranslation(); const term = useTerminology(); const labelOf = (item: MenuItem) => (typeof item.label === "string" ? term(labelText(item)) : item.label); @@ -639,8 +666,9 @@ const Sidebar_: React.FC = ({ disableVectorStoresForInternalUsers, allowVectorStoresForTeamAdmins, }; - return visibleMenuGroups(menuGroups, context); + return visibleMenuGroups(pluginMode === "customer-portal" ? customerPortalGroups : menuGroups, context); }, [ + pluginMode, allowAgentsForTeamAdmins, allowVectorStoresForTeamAdmins, disableAgentsForInternalUsers, diff --git a/ui/litellm-dashboard/src/components/navbar.test.tsx b/ui/litellm-dashboard/src/components/navbar.test.tsx index cc90e0b8e99..38b539c51b9 100644 --- a/ui/litellm-dashboard/src/components/navbar.test.tsx +++ b/ui/litellm-dashboard/src/components/navbar.test.tsx @@ -4,6 +4,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, screen, waitFor, within } from "../../tests/test-utils"; import Navbar from "./navbar"; +vi.mock("next/navigation", () => ({ usePathname: () => "/api-keys", useRouter: () => ({ push: vi.fn() }) })); + // Mock the hooks and utilities vi.mock("@/components/networking", () => ({ getProxyBaseUrl: vi.fn(() => "http://localhost:4000"), diff --git a/ui/litellm-dashboard/src/contexts/PluginModeContext.tsx b/ui/litellm-dashboard/src/contexts/PluginModeContext.tsx index 380a9479829..247cae44235 100644 --- a/ui/litellm-dashboard/src/contexts/PluginModeContext.tsx +++ b/ui/litellm-dashboard/src/contexts/PluginModeContext.tsx @@ -4,7 +4,8 @@ import React, { createContext, useContext, useState, useEffect } from "react"; import { createApiClient } from "@/lib/http/client"; import { getProxyBaseUrl } from "@/components/networking"; -export type PluginMode = "ai-gateway" | string; // "ai-gateway" or a registered plugin name +export type PluginMode = "ai-gateway" | "customer-portal" | string; // built-in modes or a registered plugin name +export const BUILT_IN_MODES: ReadonlySet = new Set(["ai-gateway", "customer-portal"]); export interface PluginNavItem { key: string; @@ -84,7 +85,8 @@ export function PluginModeProvider({ children, accessToken }: PluginModeProvider // Once plugins have loaded, fall back to ai-gateway if the persisted mode is // no longer registered — including when the list came back empty (all plugins // removed). Derived rather than setState-in-effect to avoid cascading renders. - const effectiveMode = mode !== "ai-gateway" && loaded && !plugins.some((p) => p.name === mode) ? "ai-gateway" : mode; + const effectiveMode = + !BUILT_IN_MODES.has(mode) && loaded && !plugins.some((p) => p.name === mode) ? "ai-gateway" : mode; const setMode = (m: PluginMode) => { setModeState(m); diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index d4ba9c14998..113c77e1916 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -7994,14 +7994,16 @@ "observability": "OBSERVABILITY", "accessControl": "ACCESS CONTROL", "developerTools": "DEVELOPER TOOLS", - "settings": "SETTINGS" + "settings": "SETTINGS", + "customerPortal": "CUSTOMER PORTAL" }, "sections": { "aiGateway": "AI Gateway", "observability": "Observability", "accessControl": "Access Control", "developerTools": "Developer Tools", - "settings": "Settings" + "settings": "Settings", + "customerPortal": "Customer Portal" }, "items": { "home": "Home", @@ -8047,7 +8049,9 @@ "adminSettings": "Admin Settings", "costTracking": "Cost Tracking", "uiTheme": "UI Theme", - "telemetry": "Telemetry" + "telemetry": "Telemetry", + "customers": "Customers", + "chargebacks": "Chargebacks" }, "homeLink": "LiteLLM home" }, @@ -20493,5 +20497,113 @@ "removeGroup": "Remove weights for {{group}}", "summary": "Deployment Weights:", "loadFailed": "Could not load this model group's deployments. Saved weights are kept." + }, + "customerPortal": { + "switcher": { + "label": "Customer Portal", + "description": "End customers, budgets and chargebacks" + }, + "status": { + "active": "Active", + "overBudget": "Over budget", + "blocked": "Blocked" + }, + "common": { + "noBudget": "No budget", + "cancel": "Cancel", + "save": "Save", + "block": "Block", + "blockHint": "Block and Unblock change the customer record. The proxy only rejects a blocked customer's requests when the blocked_user_check callback is enabled in litellm_settings.callbacks.", + "unblock": "Unblock", + "edit": "Edit", + "total": "Total", + "loading": "Loading..." + }, + "toasts": { + "blocked": "Customer blocked", + "unblocked": "Customer unblocked", + "created": "Created {{name}}", + "updated": "Customer updated", + "createFailed": "Failed to create customer: {{error}}", + "updateFailed": "Failed to update customer: {{error}}" + }, + "list": { + "title": "Customers", + "subtitle": "End users of your product, identified by customer id on every request.", + "create": "Create Customer", + "searchPlaceholder": "Search by customer id or alias", + "columns": { + "customer": "Customer", + "spend": "Spend this month", + "budget": "Budget", + "keys": "Keys", + "status": "Status" + }, + "loading": "Loading customers...", + "empty": "No customers yet. Create one to start tracking spend.", + "budgetUsage": "{{spend}} / {{max}}" + }, + "detail": { + "spendThisMonth": "Spend this month", + "ofBudget": "of {{max}} budget", + "noBudget": "no budget", + "tpm": "{{val}} TPM", + "rpm": "{{val}} RPM", + "keysTitle": "Keys", + "keysSubtitle": "Keys that sent requests as this customer this month.", + "keysEmpty": "No traffic from this customer this month.", + "columns": { + "key": "Key", + "requests": "Requests", + "tokens": "Tokens", + "spend": "Spend" + }, + "unnamedKey": "Unnamed key", + "back": "Back to customers" + }, + "createDialog": { + "title": "Create customer", + "description": "A customer is an end user of your product. Pass its id as `user` on every request to track spend and apply its limits.", + "customerId": "Customer id", + "customerIdPlaceholder": "e.g. acme-corp", + "alias": "Alias", + "aliasPlaceholder": "Optional display name", + "maxBudget": "Monthly budget (USD)", + "maxBudgetPlaceholder": "Leave empty for no budget", + "tpm": "TPM limit", + "rpm": "RPM limit", + "noLimit": "No limit", + "limitsHelp": "Budget and limits apply to this customer across every key it uses.", + "submit": "Create" + }, + "editDialog": { + "title": "Edit customer", + "alias": "Alias", + "maxBudget": "Max budget (USD)", + "maxBudgetPlaceholder": "Leave empty for no budget", + "sharedBudgetWarning": "This customer is on budget {{budgetId}}. Changing the budget here changes it for every customer on that budget." + }, + "chargebacks": { + "title": "Chargebacks", + "subtitle": "Spend per customer for a month, from the daily spend rollup.", + "groupBy": { + "customer": "Group by customer", + "key": "Customer and key", + "model": "Customer and model" + }, + "dimension": { + "key": "Key", + "model": "Model" + }, + "exportCsv": "Export CSV", + "columns": { + "customer": "Customer", + "requests": "Requests", + "inputTokens": "Input tokens", + "outputTokens": "Output tokens", + "spend": "Spend" + }, + "empty": "No customer traffic in {{month}}." + } } } From c1abd490b26758df7ea8b426d2a1ee4efa55c264 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 10:08:21 -0800 Subject: [PATCH 019/189] feat(customers): reject unknown fields on /customer/new and /customer/update instead of dropping them (#50965) * feat(customers): reject unknown fields on customer create and update instead of dropping them Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(customers): derive the budget hint from the budget request fields and fix the create example Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 25 +++++++++ .../customer_endpoints.py | 3 +- .../test_customer_unknown_fields.py | 55 +++++++++++++++++++ .../test_customer_budget.py | 3 - tests/unit/proxy/test__types.py | 50 +++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 +- 6 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 tests/integration/management/test_customer_unknown_fields.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index abf61a909b2..228e80da4bd 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1860,6 +1860,21 @@ class CustomerBase(LiteLLMPydanticObjectBase): blocked: bool = False +def reject_unknown_customer_fields(request: type[BaseModel], values: object) -> object: + if not isinstance(values, dict): + return values + unknown: Final = tuple(sorted(str(key) for key in values if key not in request.model_fields)) # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # raw request body + if not unknown: + return values + on_budget: Final = tuple(field for field in unknown if field in BudgetNewRequest.model_fields) + hint: Final = ( + f" {', '.join(on_budget)}: these live on the customer's budget object, update it with /budget/update." + if on_budget + else "" + ) + raise ValueError(f"Unknown fields: {', '.join(unknown)}.{hint}") + + class NewCustomerRequest(BudgetNewRequest): """ Create a new customer, allocate a budget to them @@ -1877,6 +1892,11 @@ class NewCustomerRequest(BudgetNewRequest): models: list[str] | None = None object_permission: LiteLLM_ObjectPermissionBase | None = None + @model_validator(mode="before") + @classmethod + def reject_unknown_fields(cls, values: object) -> object: + return reject_unknown_customer_fields(cls, values) + @model_validator(mode="before") @classmethod def check_user_info(cls, values): @@ -1904,6 +1924,11 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase): models: list[str] | None = None object_permission: LiteLLM_ObjectPermissionBase | None = None + @model_validator(mode="before") + @classmethod + def reject_unknown_fields(cls, values: object) -> object: + return reject_unknown_customer_fields(cls, values) + class DeleteCustomerRequest(LiteLLMPydanticObjectBase): """ diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index aec5e6840a5..bd202b85f73 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -353,7 +353,6 @@ async def new_end_user( - allowed_model_region: Optional[Union[Literal["eu"], Literal["us"]]] - Require all user requests to use models in this specific region. - default_model: Optional[str] - If no equivalent model in the allowed region, default all requests to this model. - models: Optional[list[str]] - Restrict this customer's access to the listed models. - - metadata: Optional[dict] = Metadata for customer, store information for customer. Example metadata = {"data_training_opt_out": True} - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). - tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute) - rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute) @@ -387,7 +386,7 @@ async def new_end_user( --header 'Content-Type: application/json' \ --data '{ "user_id" : "ishaan-jaff-3", - "allowed_region": "eu", + "allowed_model_region": "eu", "budget_id": "free_tier", "models": ["gpt-4o-mini"], "default_model": "azure/gpt-3.5-turbo-eu" diff --git a/tests/integration/management/test_customer_unknown_fields.py b/tests/integration/management/test_customer_unknown_fields.py new file mode 100644 index 00000000000..235d5745791 --- /dev/null +++ b/tests/integration/management/test_customer_unknown_fields.py @@ -0,0 +1,55 @@ +import uuid +from typing import Final + +import httpx +import pytest + +from tests.integration._support.client import Gateway, Scenario + +pytestmark: Final = pytest.mark.shares_worker_state + + +def _customer(scenario: Scenario, **fields: object) -> str: + customer: Final = f"integration-customer-{uuid.uuid4().hex}" + scenario.gateway.post("/customer/new", {"user_id": customer, **fields}) # pyright: ignore[reportArgumentType] # JsonValue kwargs + scenario.cleanups.callback(scenario.gateway.post, "/customer/delete", {"user_ids": [customer]}) + return customer + + +def _post(gateway: Gateway, route: str, body: dict[str, object]) -> httpx.Response: + return gateway.request("POST", route, body) # pyright: ignore[reportArgumentType] # JsonValue body + + +def _info(gateway: Gateway, customer: str) -> dict[str, object]: + return gateway.get("/customer/info", {"end_user_id": customer}) + + +@pytest.mark.parametrize( + ("body", "unknown_field"), + [ + pytest.param({"tpm_limit": 5}, "tpm_limit", id="budget_field_on_update"), + pytest.param({"quota": {"opus": 1}}, "quota", id="made_up_field"), + ], +) +def test_customer_update_rejects_unknown_fields_and_leaves_the_customer_unchanged( + gateway: Gateway, body: dict[str, object], unknown_field: str +) -> None: + with gateway.scenario() as scenario: + customer: Final = _customer(scenario, alias="before") + rejected: Final = _post(gateway, "/customer/update", {"user_id": customer, **body}) + assert rejected.status_code == 422, rejected.text + assert unknown_field in rejected.text, rejected.text + assert _info(gateway, customer)["alias"] == "before" + + accepted: Final = _post(gateway, "/customer/update", {"user_id": customer, "alias": "after"}) + assert accepted.status_code == 200, accepted.text + assert _info(gateway, customer)["alias"] == "after" + + +def test_customer_new_rejects_a_misspelled_field_and_creates_nothing(gateway: Gateway) -> None: + customer: Final = f"integration-customer-{uuid.uuid4().hex}" + rejected: Final = _post(gateway, "/customer/new", {"user_id": customer, "tpm_limti": 5}) + assert rejected.status_code == 422, rejected.text + assert "tpm_limti" in rejected.text, rejected.text + missing: Final = gateway.request("GET", "/customer/info", params={"end_user_id": customer}) + assert missing.status_code == 404, missing.text diff --git a/tests/unit/proxy/management_endpoints/test_customer_budget.py b/tests/unit/proxy/management_endpoints/test_customer_budget.py index 0beca0c15e8..ac172ac2ffb 100644 --- a/tests/unit/proxy/management_endpoints/test_customer_budget.py +++ b/tests/unit/proxy/management_endpoints/test_customer_budget.py @@ -144,7 +144,6 @@ async def test_update_customer_creates_budget_with_proper_relations( update_request = UpdateCustomerRequest( user_id="test-user", max_budget=100.0, # This triggers budget creation - rpm_limit=200, # Use valid budget field ) # Act @@ -267,7 +266,6 @@ async def test_update_customer_budget_creation_with_fallback_admin( update_request = UpdateCustomerRequest( user_id="test-user", max_budget=150.0, - tpm_limit=1000, # Add another budget field ) # Act @@ -325,7 +323,6 @@ async def test_update_customer_with_budget_id_and_creation_fields( user_id="test-user", budget_id="existing-budget-link", # For linking to existing budget max_budget=300.0, # This should trigger new budget creation - rpm_limit=500, # Use valid budget field ) # Act diff --git a/tests/unit/proxy/test__types.py b/tests/unit/proxy/test__types.py index 40e66f1da65..1c97e9054ae 100644 --- a/tests/unit/proxy/test__types.py +++ b/tests/unit/proxy/test__types.py @@ -26,6 +26,7 @@ from litellm.proxy._types import ( OrganizationMemberUpdateRequest, RegenerateKeyRequest, ResetSpendRequest, + UpdateCustomerRequest, UpdateKeyRequest, UpdateMCPServerRequest, UpdateTeamRequest, @@ -634,3 +635,52 @@ def test_batch_file_caps_accept_only_positive_limits(setting): with pytest.raises(ValidationError): ConfigGeneralSettings.model_validate({setting: invalid}) assert getattr(ConfigGeneralSettings.model_validate({setting: 3}), setting) == 3 + + +@pytest.mark.parametrize( + ("request_type", "body", "message"), + [ + pytest.param( + UpdateCustomerRequest, + {"user_id": "acme", "tpm_limit": 5, "quota": {"opus": 1}}, + "Unknown fields: quota, tpm_limit. tpm_limit: these live on the customer's budget object, " + "update it with /budget/update.", + id="update_with_a_budget_field_and_a_made_up_one", + ), + pytest.param( + UpdateCustomerRequest, + {"user_id": "acme", "alias": "Acme", "quota": 1}, + "Unknown fields: quota.", + id="update_with_a_made_up_field_has_no_budget_hint", + ), + pytest.param( + NewCustomerRequest, + {"user_id": "acme", "tpm_limti": 5}, + "Unknown fields: tpm_limti.", + id="create_with_a_misspelled_budget_field", + ), + ], +) +def test_a_customer_request_with_unknown_fields_is_refused_naming_them( + request_type: type[BaseModel], body: dict[str, object], message: str +) -> None: + with pytest.raises(ValidationError) as exc: + request_type.model_validate(body) + assert [error["msg"] for error in exc.value.errors()] == [f"Value error, {message}"] + + +@pytest.mark.parametrize( + ("request_type", "body"), + [ + pytest.param(UpdateCustomerRequest, {"user_id": "acme", "alias": "Acme", "blocked": True}, id="update"), + pytest.param( + NewCustomerRequest, + {"user_id": "acme", "tpm_limit": 5, "model_tpm_limit": {"opus": 60}}, + id="create_with_inline_budget_fields", + ), + ], +) +def test_a_customer_request_with_only_known_fields_is_accepted( + request_type: type[BaseModel], body: dict[str, object] +) -> None: + assert request_type.model_validate(body).model_dump(exclude_unset=True) == body diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b229461e981..e1b0104e089 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4549,7 +4549,6 @@ export interface paths { * - allowed_model_region: Optional[Union[Literal["eu"], Literal["us"]]] - Require all user requests to use models in this specific region. * - default_model: Optional[str] - If no equivalent model in the allowed region, default all requests to this model. * - models: Optional[list[str]] - Restrict this customer's access to the listed models. - * - metadata: Optional[dict] = Metadata for customer, store information for customer. Example metadata = {"data_training_opt_out": True} * - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). * - tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute) * - rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute) @@ -4580,7 +4579,7 @@ export interface paths { * ``` * curl --location 'http://0.0.0.0:4000/customer/new' --header "Authorization: Bearer $LITELLM_MASTER_KEY" --header 'Content-Type: application/json' --data '{ * "user_id" : "ishaan-jaff-3", - * "allowed_region": "eu", + * "allowed_model_region": "eu", * "budget_id": "free_tier", * "models": ["gpt-4o-mini"], * "default_model": "azure/gpt-3.5-turbo-eu" @@ -5229,7 +5228,6 @@ export interface paths { * - allowed_model_region: Optional[Union[Literal["eu"], Literal["us"]]] - Require all user requests to use models in this specific region. * - default_model: Optional[str] - If no equivalent model in the allowed region, default all requests to this model. * - models: Optional[list[str]] - Restrict this customer's access to the listed models. - * - metadata: Optional[dict] = Metadata for customer, store information for customer. Example metadata = {"data_training_opt_out": True} * - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). * - tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute) * - rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute) @@ -5260,7 +5258,7 @@ export interface paths { * ``` * curl --location 'http://0.0.0.0:4000/customer/new' --header "Authorization: Bearer $LITELLM_MASTER_KEY" --header 'Content-Type: application/json' --data '{ * "user_id" : "ishaan-jaff-3", - * "allowed_region": "eu", + * "allowed_model_region": "eu", * "budget_id": "free_tier", * "models": ["gpt-4o-mini"], * "default_model": "azure/gpt-3.5-turbo-eu" From 90f37e113ca80b593e5c25ef2550dac9ce48af0a Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:09:19 -0800 Subject: [PATCH 020/189] fix(proxy): send a prompt-only call_endpoint token count to every provider count API (#50783) * fix(proxy): count a prompt-only call_endpoint request with the provider on every counter POST /utils/token_counter?call_endpoint=true with only a prompt sent the prompt to the provider count API on Gemini and Vertex only. Anthropic, OpenAI, Bedrock and Azure AI (Claude) deployments answered a local estimate instead. Every counter now gets the prompt as one user message, so a provider error surfaces the same as the messages form, and the framed stand-in after a failed provider count frames the prompt as one user turn, matching the messages form * fix(proxy): count a call_endpoint prompt like a user message on providers without a count API * fix(proxy): count a call_endpoint prompt as a turn only on chat deployments A prompt-only /utils/token_counter request with call_endpoint=true now goes to the count API as one user message only when the deployment serves chat (mode chat, responses, or unknown). Embedding rows and models with no deployment keep the raw local count. The Anthropic, Azure AI, and OpenAI count calls now honor the deployment's timeout, and a count error with an empty body names its status code. * fix(proxy): fall back to router_settings.timeout for count calls and answer 408 on a count timeout Count calls to Anthropic, Azure AI, and OpenAI now pick their timeout the way a chat call to the same deployment does: the deployment's timeout, else router_settings.timeout, else request_timeout. A count call that times out answers 408 instead of a 500 processing error. * fix(proxy): count a prompt as a turn only where a count API exists and give count calls chat's timeout order * fix(proxy): read the count call timeout through the router settings snapshot The token counter now asks llm_router.settings().timeout_for() for the chat timeout order instead of a new Router member, so the router surface does not grow * fix(router): answer timeout_for on every router settings snapshot The proxy reads the count call timeout through llm_router.settings().timeout_for(), but only the Rust core's snapshot had that method, so a plain RouterSettings (the unit tests' router stub) raised AttributeError. timeout_for now lives on RouterSettings and reads the core's _core_deployment_call_timeout, like the snapshot's other members --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llms/anthropic/count_tokens/handler.py | 2 + .../anthropic/count_tokens/token_counter.py | 3 +- .../anthropic/count_tokens/handler.py | 2 + .../anthropic/count_tokens/token_counter.py | 3 +- litellm/llms/base_llm/base_utils.py | 15 +- .../openai/responses/count_tokens/handler.py | 2 + .../responses/count_tokens/token_counter.py | 13 +- litellm/proxy/proxy_server.py | 40 ++- litellm/router_api.py | 3 + litellm/router_rust/core.py | 5 +- ...anthropic_count_tokens_system_lift_wire.py | 16 + .../test_bedrock_mantle_count_tokens_wire.py | 26 ++ tests/unit/proxy/router_entry_point_stubs.py | 1 + tests/unit/proxy/test_proxy_token_counter.py | 332 ++++++++++++++++-- 14 files changed, 427 insertions(+), 36 deletions(-) diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index 7a7d1d776d3..46c90295198 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -125,6 +125,8 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): status_code=e.response.status_code, message=e.response.text, ) + except litellm.Timeout as e: + raise AnthropicError(status_code=e.status_code, message=str(e)) from e except Exception as e: verbose_logger.error("Error in CountTokens handler: %s", e) raise AnthropicError( diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 99b1cd7f241..755ccd129d8 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -18,7 +18,7 @@ from litellm.llms.anthropic.count_tokens.transformation import ( ChatCountTranslationError, anthropic_count_fields, ) -from litellm.llms.base_llm.base_utils import BaseTokenCounter +from litellm.llms.base_llm.base_utils import BaseTokenCounter, deployment_timeout from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse # Global handler instance - reuse across all token counting requests @@ -134,6 +134,7 @@ class AnthropicTokenCounter(BaseTokenCounter): optional_params=count_options, context_management=context_management.spec if context_management is not None else None, drop_params=drops_unsupported_params(litellm_params.get("drop_params")), + timeout=deployment_timeout(litellm_params), ) if result is not None: diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 5fcb84593d2..443a3a5edad 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -147,6 +147,8 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): status_code=e.response.status_code, message=e.response.text, ) + except litellm.Timeout as e: + raise AnthropicError(status_code=e.status_code, message=str(e)) from e except Exception as e: verbose_logger.error("Error in CountTokens handler: %s", e) raise AnthropicError( diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index 8657fcddab4..5031b69f83b 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -20,7 +20,7 @@ from litellm.llms.anthropic.count_tokens.transformation import ( from litellm.llms.azure_ai.anthropic.count_tokens.handler import ( AzureAIAnthropicCountTokensHandler, ) -from litellm.llms.base_llm.base_utils import BaseTokenCounter +from litellm.llms.base_llm.base_utils import BaseTokenCounter, deployment_timeout from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse # Global handler instance - reuse across all token counting requests @@ -137,6 +137,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): system=system, optional_params=count_options, context_management=None if context_management is None else context_management.spec, + timeout=deployment_timeout(litellm_params), ) if result is not None: diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index 45f623730a8..8b1ccda3a61 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -15,8 +15,9 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Any, Final, TypeAlias +import httpx from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals -from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, JsonValue, PositiveFloat, TypeAdapter, ValidationError from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal from litellm._logging import verbose_logger @@ -38,6 +39,18 @@ from litellm.types.utils import ( TokenCountResponse, ) +_SECONDS: Final[TypeAdapter[float]] = TypeAdapter(PositiveFloat) + + +def deployment_timeout(litellm_params: Mapping[str, object]) -> float | httpx.Timeout | None: + timeout: Final = litellm_params.get("timeout") + if timeout is None or isinstance(timeout, httpx.Timeout): + return timeout + try: + return _SECONDS.validate_python(timeout) + except ValidationError: + return None + class BaseTokenCounter(ABC): @abstractmethod diff --git a/litellm/llms/openai/responses/count_tokens/handler.py b/litellm/llms/openai/responses/count_tokens/handler.py index 2cf48a75524..ce9ed6c0940 100644 --- a/litellm/llms/openai/responses/count_tokens/handler.py +++ b/litellm/llms/openai/responses/count_tokens/handler.py @@ -111,6 +111,8 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): status_code=e.response.status_code, message=e.response.text, ) + except litellm.Timeout as e: + raise OpenAIError(status_code=e.status_code, message=str(e)) from e except (httpx.RequestError, json.JSONDecodeError, ValueError) as e: verbose_logger.error("Error in CountTokens handler: %s", e) raise OpenAIError( diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index c82a2982a54..cd60bb1327f 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Any, Final +import httpx import openai from pydantic import JsonValue from typing_extensions import assert_never @@ -21,7 +22,7 @@ from litellm.llms.anthropic.messages.responses_adapters.handler import count_vie from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) -from litellm.llms.base_llm.base_utils import BaseTokenCounter +from litellm.llms.base_llm.base_utils import BaseTokenCounter, deployment_timeout from litellm.llms.openai.common_utils import OpenAIError, decoded_json_object, get_openai_credentials from litellm.llms.openai.responses.count_tokens.handler import ( OpenAICountTokensHandler, @@ -61,6 +62,7 @@ def _workload_identity_config( class _CountCredential: api_base: str | None credential: str | OpenAIWorkloadIdentityConfig + timeout: float | httpx.Timeout | None def _count_credential(deployment: Mapping[str, object] | None, model_label: str) -> _CountCredential | OpenAIError: @@ -82,7 +84,9 @@ def _count_credential(deployment: Mapping[str, object] | None, model_label: str) "workload identity on the deployment, or OPENAI_API_KEY in the proxy environment." ), ) - return _CountCredential(api_base=credentials.api_base, credential=credential) + return _CountCredential( + api_base=credentials.api_base, credential=credential, timeout=deployment_timeout(litellm_params) + ) def _count_error(model_to_use: str, request_model: str, message: str, status_code: int) -> TokenCountResponse: @@ -245,6 +249,7 @@ class OpenAITokenCounter(BaseTokenCounter): tools=counted_tools, instructions=instructions, text=text, + timeout=resolved.timeout, ), model_to_use, request_model, @@ -264,7 +269,9 @@ class OpenAITokenCounter(BaseTokenCounter): body: Final = native_count_request(request, model_to_use, policy) return await self._counted( resolved, - lambda api_key, api_base: openai_count_tokens_handler.post_count_request(body, api_key, api_base), + lambda api_key, api_base: openai_count_tokens_handler.post_count_request( + body, api_key, api_base, resolved.timeout + ), model_to_use, request_model, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e6447754436..3fddca6f4d8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15325,7 +15325,12 @@ def _provider_count_outcome( if result is not None and result.error is True: return ProviderCountFailure( status_code=result.status_code or 500, - message=result.error_message or "Token counting failed", + message=result.error_message + or ( + "Token counting failed" + if result.status_code is None + else f"The provider's count API answered {result.status_code} with no error message" + ), rejected_before_provider_call=result.rejected_before_provider_call, ) return result @@ -15794,6 +15799,14 @@ def _token_count_settings_model(deployment: _TokenCountDeployment, request_model _PROVIDERS_COUNTING_WITHOUT_MESSAGES: Final = frozenset( {litellm.LlmProviders.GEMINI.value, litellm.LlmProviders.VERTEX_AI.value} ) +_CHAT_MODES: Final = frozenset({"chat", "responses"}) + + +def _serves_chat(deployment: _TokenCountDeployment, cost_map_row: Mapping[str, object]) -> bool: + mode: Final = _non_empty_str(deployment.get("model_info", {}).get("mode")) or _non_empty_str( + cost_map_row.get("mode") + ) + return mode is None or mode in _CHAT_MODES _ESTIMABLE_GOOGLE_PART_KEYS: Final = frozenset({"text", "functionCall", "functionResponse", "thoughtSignature"}) @@ -16104,6 +16117,12 @@ async def _routed_count_deployment( return None +def _with_chat_timeout(litellm_params: Mapping[str, object]) -> Mapping[str, object]: + if llm_router is None: + return litellm_params + return {**litellm_params, "timeout": llm_router.settings().timeout_for(litellm_params)} + + async def _count_request_tokens( request: TokenCountRequest, routed: Mapping[str, object] | None, @@ -16128,8 +16147,8 @@ async def _count_request_tokens( litellm_model_name = deployment.get("litellm_params", {}).get("model") deployment = { # rebind-ok: merge the named credential into a copy **deployment, - "litellm_params": with_named_credential( - _LITELLM_PARAMS_COPY.validate_python(deployment.get("litellm_params", {})) + "litellm_params": _with_chat_timeout( + with_named_credential(_LITELLM_PARAMS_COPY.validate_python(deployment.get("litellm_params", {}))) ), } # remove the custom_llm_provider_prefix in the litellm_model_name @@ -16142,9 +16161,10 @@ async def _count_request_tokens( deployment_view: Final = _TOKEN_COUNT_DEPLOYMENT.validate_python( deployment or _model_group_deployment(llm_router, request.model) or {} ) + cost_map_row: Final = await asyncify(cost_map_entry)(*_token_count_settings_model(deployment_view, request.model)) settings: Final = await asyncify(resolve_token_count_settings)( configured=deployment_view.get("model_info", {}), - default=await asyncify(cost_map_entry)(*_token_count_settings_model(deployment_view, request.model)), + default=cost_map_row, deployment_region=partial(_deployment_aws_region, deployment_view), ) @@ -16169,6 +16189,15 @@ async def _count_request_tokens( code=400, ) prompt_only: Final = bool(prompt) and conversation_empty + prompt_as_turn: Final = ( + prompt_only + and provider_counter is not None + and provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider) + and _serves_chat(deployment_view, cost_map_row) + ) + prompt_as_message: Final = prompt_as_turn or ( + prompt_only and custom_llm_provider in _PROVIDERS_COUNTING_WITHOUT_MESSAGES + ) request_messages: Final = _counted_request_messages(messages, messages_format, count_deployment) local_count_error: Final = malformed_tool_result_count_error(request_messages, messages_format, count_deployment) malformed_messages: Final = ( @@ -16205,7 +16234,6 @@ async def _count_request_tokens( ) inference_system: Final = system if edited_request is None else edited_request.system contents_estimable: Final = not contents or (bool(messages) and _estimable_contents(contents)) - prompt_as_message: Final = prompt_only and custom_llm_provider in _PROVIDERS_COUNTING_WITHOUT_MESSAGES routed_deployment: Final = _ROUTED_DEPLOYMENT.validate_python(deployment) routed_request: Final = ( None if responses_request is None else with_provider_file_ids(responses_request, routed_deployment) @@ -16306,7 +16334,7 @@ async def _count_request_tokens( _with_get(model_info).get("custom_tokenizer", None), ) system_message: Final = _system_message(local_system) - prompt_turn: Final = prompt_only and not (system_message is None and not tools) + prompt_turn: Final = prompt_as_turn or (prompt_only and not (system_message is None and not tools)) typed_messages: Final = cast( # cast-ok: request messages are raw chat-shaped dicts that token_counter normalizes Sequence[AllMessageValues] | None, [{"role": "user", "content": prompt}] if prompt_turn else (None if prompt_only else local_messages), diff --git a/litellm/router_api.py b/litellm/router_api.py index d07b135e9a5..c85a764151f 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -1308,6 +1308,9 @@ class RouterSettings: """The guardrail configured under this name (was ``get_available_guardrail``).""" return self._owner("_core_get_available_guardrail")._core_get_available_guardrail(name) + def timeout_for(self, litellm_params: Mapping[str, object]) -> object: + return self._owner("_core_deployment_call_timeout")._core_deployment_call_timeout(litellm_params) + class RouterHealth: """Cooldowns, blocked deployments, health checks and usage (``health()``).""" diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index b7e67fffd57..682aa557007 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -113,7 +113,7 @@ from litellm.router_rust.invoke.endpoints import ( stamp_error, unrouted_fields, ) -from litellm.router_rust.invoke.shaping import Shaping +from litellm.router_rust.invoke.shaping import Shaping, non_stream_timeout from litellm.router_rust.invoke.slots import enter_deployment_slot, max_parallel_requests_limit from litellm.router_rust.invoke.streaming import ( StreamAttempt, @@ -1592,6 +1592,9 @@ class RustCore( def _shaping(self) -> Shaping: return self._endpoints.shaping(self._resolved) + def _core_deployment_call_timeout(self, litellm_params: Mapping[str, object]) -> object: + return non_stream_timeout(self._shaping, MappingProxyType({}), litellm_params) + # -- building -------------------------------------------------------------------------------- def _new_handle(self, model_list: Sequence[object]) -> "RouterCoreHandle": diff --git a/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py b/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py index 380881a6796..86a8cb3df4f 100644 --- a/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py +++ b/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py @@ -332,6 +332,22 @@ def test_utils_token_counter_local_mode_never_calls_the_peer_for_a_leading_syste assert peer.drain() == () +def test_utils_token_counter_call_endpoint_counts_a_prompt_as_one_user_message( + deployment: _Deployment, gateway: Gateway +) -> None: + with _listening(deployment) as peer: + response: Final = gateway.request( + "POST", + "/utils/token_counter", + {"model": deployment.model, "prompt": USER_TEXT}, + params={"call_endpoint": "true"}, + ) + payload: Final = _payload(response) + assert (payload["total_tokens"], payload["count_source"]) == (_COUNT, "provider"), response.text + assert payload["tokenizer_type"] == deployment.provider.tokenizer, response.text + assert _bodies(peer, deployment.provider) == ({"model": _MODEL, "messages": [USER]},) + + def test_responses_input_tokens_lifts_instructions(deployment: _Deployment, gateway: Gateway) -> None: with _listening(deployment) as peer: response: Final = gateway.request( diff --git a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py index 2a0508402ae..07394c03f0c 100644 --- a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py @@ -550,6 +550,32 @@ def test_runtime_404_and_mantle_404_surface_provider_error(counting_proxy: Owned assert _mantle_bodies(mantle) == (_mantle_body(system=system, tools=_TOOLS),) +def test_utils_token_counter_prompt_answers_the_count_model_404_like_the_messages_form( + counting_proxy: OwnedProxy, mantle_port: int +) -> None: + gateway: Final = counting_proxy.gateway + with ( + wire_server(_runtime) as runtime, + wire_server(_mantle(_rejecting(404)), port=mantle_port) as mantle, + gateway.scenario() as scenario, + ): + model: Final = _metadata_deployment(scenario, runtime.url) + from_prompt: Final = gateway.request( + "POST", + "/utils/token_counter", + {"model": model, "prompt": "Count this message"}, + params={"call_endpoint": "true"}, + ) + from_messages: Final = gateway.request( + "POST", "/utils/token_counter", _bare(model), params={"call_endpoint": "true"} + ) + assert from_prompt.status_code == from_messages.status_code == 404, from_prompt.text + assert _REJECTION in from_prompt.text, from_prompt.text + assert _JSON_OBJECT.validate_json(from_prompt.content) == _JSON_OBJECT.validate_json(from_messages.content) + assert runtime.drain() == () + assert _mantle_bodies(mantle) == (_MANTLE_BARE, _MANTLE_BARE) + + def test_runtime_404_on_nova_does_not_call_mantle(counting_proxy: OwnedProxy, mantle_port: int) -> None: gateway: Final = counting_proxy.gateway with ( diff --git a/tests/unit/proxy/router_entry_point_stubs.py b/tests/unit/proxy/router_entry_point_stubs.py index 306740489d7..10176accf9f 100644 --- a/tests/unit/proxy/router_entry_point_stubs.py +++ b/tests/unit/proxy/router_entry_point_stubs.py @@ -184,6 +184,7 @@ def route_entry_points_to_stubs(router: _RouterDouble) -> _RouterDouble: _core_get_settings=lambda: router.get_settings(), _core_get_available_guardrail=lambda name: router.get_available_guardrail(guardrail_name=name), _core_get_allowed_fails_from_policy=lambda exception: router.get_allowed_fails_from_policy(exception=exception), + _core_deployment_call_timeout=lambda litellm_params: router.deployment_call_timeout(litellm_params), ) router.settings.side_effect = lambda: RouterSettings( **{name: getattr(router, name) for name in SETTING_FIELDS}, _router=settings_backend diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 694b257b1f9..34868606012 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -8,10 +8,12 @@ import hashlib import itertools import json import logging +import socket import threading import traceback import uuid -from collections.abc import Callable, Coroutine, Mapping +from collections.abc import Callable, Coroutine, Iterator, Mapping +from contextlib import contextmanager from dataclasses import replace from functools import partial from pathlib import Path @@ -44,8 +46,11 @@ import litellm from litellm import Router from litellm._logging import JsonFormatter, verbose_proxy_logger from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler +from litellm.llms.azure_ai.anthropic.count_tokens import handler as azure_ai_count_tokens_http from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.bedrock.count_tokens import handler as bedrock_count_tokens_http +from litellm.llms.bedrock.count_tokens import mantle_handler as bedrock_mantle_count_tokens_http from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler from litellm.llms.bedrock.count_tokens.kimi_k3 import ( @@ -4833,36 +4838,317 @@ async def test_local_count_of_a_prompt_with_system_or_tools_counts_them_like_the assert from_prompt.total_tokens == from_message.total_tokens > prompt_alone.total_tokens +_PROVIDER_COUNTED_DEPLOYMENTS: Final = ( + pytest.param( + {"model": "anthropic/claude-sonnet-5-5", "api_key": "test-key", "api_base": "https://count-tokens.test"}, + id="anthropic", + ), + pytest.param( + {"model": "azure_ai/claude-sonnet-5-5", "api_key": "test-key", "api_base": "https://count-tokens.test"}, + id="azure_ai", + ), + pytest.param( + {"model": "openai/gpt-5.6", "api_key": "test-key", "api_base": "https://count-tokens.test/v1"}, id="openai" + ), + pytest.param( + { + "model": "bedrock/us.anthropic.claude-sonnet-5-5", + "aws_access_key_id": "AKIATESTACCESSKEY", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-west-2", + }, + id="bedrock", + ), +) + + +@contextmanager +def _count_apis_answering(peer: Callable[[httpx.Request], httpx.Response]) -> Iterator[None]: + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(peer)) + with ( + patch.object(anthropic_count_tokens_handler, "get_async_httpx_client", return_value=client), + patch.object(azure_ai_count_tokens_http, "get_async_httpx_client", return_value=client), + patch.object(bedrock_count_tokens_http, "get_async_httpx_client", return_value=client), + patch.object(bedrock_mantle_count_tokens_http, "get_async_httpx_client", return_value=client), + patch.object(openai_count_tokens_http, "get_async_httpx_client", return_value=client), + ): + yield + + +def _route_to_one_deployment(litellm_params: dict[str, JsonValue], monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router(model_list=[{"model_name": "provider-counted", "litellm_params": litellm_params}]), + ) + + +_FOX_AS_PROMPT: Final = TokenCountRequest(model="provider-counted", prompt=_FOX) +_FOX_AS_MESSAGE: Final = TokenCountRequest(model="provider-counted", messages=[{"role": "user", "content": _FOX}]) + + @pytest.mark.asyncio -async def test_call_endpoint_prompt_keeps_the_local_count_on_a_counter_that_needs_messages( +@pytest.mark.parametrize("litellm_params", _PROVIDER_COUNTED_DEPLOYMENTS) +async def test_call_endpoint_sends_a_prompt_to_the_provider_as_one_user_message( + litellm_params: dict[str, JsonValue], monkeypatch: pytest.MonkeyPatch +) -> None: + _route_to_one_deployment(litellm_params, monkeypatch) + sent_bodies: Final[list[JsonValue]] = [] # mutable-ok: collects what the mock peer receives + + def count_api(request: httpx.Request) -> httpx.Response: + sent_bodies.append(json.loads(request.content)) + return httpx.Response( + 200, json={"object": "response.input_tokens", "input_tokens": 41, "inputTokens": 41}, request=request + ) + + with _count_apis_answering(count_api): + from_prompt: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + from_message: Final = await token_counter(request=_FOX_AS_MESSAGE, call_endpoint=True) + + assert (from_prompt.total_tokens, from_prompt.error) == (41, False) + assert from_prompt.count_source != "local_estimate" + assert from_prompt == from_message + assert len(sent_bodies) == 2 and sent_bodies[0] == sent_bodies[1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [404, 503]) +@pytest.mark.parametrize("litellm_params", _PROVIDER_COUNTED_DEPLOYMENTS) +async def test_call_endpoint_answers_a_provider_error_on_a_prompt_like_on_the_same_text_sent_as_a_user_message( + litellm_params: dict[str, JsonValue], status: int, monkeypatch: pytest.MonkeyPatch +) -> None: + _route_to_one_deployment(litellm_params, monkeypatch) + + def count_api(request: httpx.Request) -> httpx.Response: + return httpx.Response(status, json={"error": {"message": f"scripted {status}"}}, request=request) + + with _count_apis_answering(count_api): + with pytest.raises(ProxyException) as from_prompt: + await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + with pytest.raises(ProxyException) as from_message: + await token_counter(request=_FOX_AS_MESSAGE, call_endpoint=True) + + assert from_prompt.value.code == str(status) + assert f"scripted {status}" in from_prompt.value.message + assert (from_prompt.value.code, from_prompt.value.type, from_prompt.value.message) == ( + from_message.value.code, + from_message.value.type, + from_message.value.message, + ) + + +@pytest.mark.asyncio +async def test_framed_stand_in_counts_a_prompt_like_the_same_text_sent_as_a_user_message( monkeypatch: pytest.MonkeyPatch, ) -> None: - router: Final = Router( - model_list=[ - { - "model_name": "anthropic-counted", - "litellm_params": { - "model": "anthropic/claude-sonnet-5-5", - "api_key": "test-key", - "api_base": "https://count-tokens.test", - }, - } - ] - ) - monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) - client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_unavailable_peer)) - request: Final = TokenCountRequest(model="anthropic-counted", prompt="The quick brown fox jumps over the lazy dog.") + _framed_kimi(monkeypatch) - with patch.object(anthropic_count_tokens_handler, "get_async_httpx_client", return_value=client): - with_endpoint: Final = await token_counter(request=request, call_endpoint=True) + from_prompt: Final = await token_counter(request=TokenCountRequest(model="kimi", prompt=_FOX), call_endpoint=True) + from_message: Final = await token_counter( + request=TokenCountRequest(model="kimi", messages=[{"role": "user", "content": _FOX}]), call_endpoint=True + ) + local_prompt: Final = await token_counter(request=TokenCountRequest(model="kimi", prompt=_FOX)) + + assert (from_prompt.count_source, from_prompt.error) == ("local_estimate", False) + assert from_prompt.total_tokens == from_message.total_tokens > local_prompt.total_tokens + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model", ["fireworks_ai/accounts/fireworks/models/glm-5", "deepseek/deepseek-v4-flash"], ids=["plain", "framed"] +) +async def test_call_endpoint_counts_a_prompt_on_a_provider_without_a_count_api_as_raw_text( + model: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _route_to_one_deployment({"model": model, "api_key": "test-key"}, monkeypatch) + sent: Final[list[httpx.Request]] = [] # mutable-ok: collects what the mock peer receives + + def count_api(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response(200, json={"input_tokens": 41}, request=request) + + with _count_apis_answering(count_api): + from_prompt: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + from_message: Final = await token_counter(request=_FOX_AS_MESSAGE, call_endpoint=True) + local_prompt: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=False) + + assert sent == [] + assert (from_prompt.count_source, from_prompt.error) == ("local_estimate", False) + assert from_prompt.total_tokens == local_prompt.total_tokens < from_message.total_tokens + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "deployment", + [ + pytest.param( + {"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "test-key"}}, id="cost map mode" + ), + pytest.param( + { + "litellm_params": {"model": "openai/in-house-embedder", "api_key": "test-key"}, + "model_info": {"mode": "embedding"}, + }, + id="configured mode", + ), + ], +) +async def test_call_endpoint_counts_a_prompt_on_a_non_chat_deployment_as_raw_text_without_the_provider( + deployment: dict[str, JsonValue], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + litellm.proxy.proxy_server, "llm_router", Router(model_list=[{"model_name": "provider-counted", **deployment}]) + ) + sent: Final[list[httpx.Request]] = [] # mutable-ok: collects what the mock peer receives + + def count_api(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response(200, json={"input_tokens": 41}, request=request) + + with _count_apis_answering(count_api): + from_endpoint: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + local: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=False) + + assert sent == [] + assert (from_endpoint.total_tokens, from_endpoint.error) == (local.total_tokens, False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["chat", "responses"]) +async def test_call_endpoint_sends_a_prompt_on_a_chat_mode_deployment_to_the_provider( + mode: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "provider-counted", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "test-key", + "api_base": "https://count-tokens.test/v1", + }, + "model_info": {"mode": mode}, + } + ] + ), + ) + sent: Final[list[httpx.Request]] = [] # mutable-ok: collects what the mock peer receives + + def count_api(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response(200, json={"object": "response.input_tokens", "input_tokens": 41}, request=request) + + with _count_apis_answering(count_api): + from_prompt: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + + assert (from_prompt.total_tokens, from_prompt.error) == (41, False) + assert len(sent) == 1 + + +@pytest.mark.asyncio +async def test_call_endpoint_counts_a_prompt_for_a_model_without_a_deployment_as_raw_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router(model_list=[{"model_name": "other", "litellm_params": {"model": "openai/gpt-5.6"}}]), + ) + request: Final = TokenCountRequest(model="deepseek/deepseek-v4-flash", prompt=_FOX) + + from_endpoint: Final = await token_counter(request=request, call_endpoint=True) local: Final = await token_counter(request=request, call_endpoint=False) - assert (with_endpoint.error, with_endpoint.total_tokens, with_endpoint.tokenizer_type) == ( - False, - local.total_tokens, - local.tokenizer_type, + assert (from_endpoint.total_tokens, from_endpoint.error) == (local.total_tokens, False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "path"), + [("anthropic/claude-sonnet-5-5", ""), ("azure_ai/claude-sonnet-5-5", ""), ("openai/gpt-5.6", "/v1")], + ids=["anthropic", "azure_ai", "openai"], +) +@pytest.mark.parametrize( + ("row_timeout", "router_timeout"), + [({"timeout": 0.5}, None), ({"request_timeout": 0.5}, 30.0), ({}, 0.5)], + ids=["deployment timeout", "deployment request_timeout", "router_settings timeout"], +) +async def test_call_endpoint_answers_a_timeout_when_a_stalled_count_api_outlasts_the_chat_timeout( + model: str, + path: str, + row_timeout: dict[str, JsonValue], + router_timeout: float | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + with socket.create_server(("127.0.0.1", 0)) as never_answers: + port: Final = never_answers.getsockname()[1] + litellm_params: Final[dict[str, JsonValue]] = { + "model": model, + "api_key": "test-key", + "api_base": f"http://127.0.0.1:{port}{path}", + } + deployment: Final[dict[str, JsonValue]] = { + "model_name": "provider-counted", + "litellm_params": {**litellm_params, **row_timeout}, + } + monkeypatch.setattr( + litellm.proxy.proxy_server, "llm_router", Router(model_list=[deployment], timeout=router_timeout) + ) + + with pytest.raises(ProxyException) as error: + async with asyncio.timeout(10): + await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + + assert error.value.code == "408" + + +@pytest.mark.asyncio +async def test_responses_count_answers_a_timeout_when_a_stalled_count_api_outlasts_the_deployment_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with socket.create_server(("127.0.0.1", 0)) as never_answers: + port: Final = never_answers.getsockname()[1] + _route_to_one_deployment( + { + "model": "openai/gpt-5.6", + "api_key": "test-key", + "api_base": f"http://127.0.0.1:{port}/v1", + "timeout": 0.5, + }, + monkeypatch, + ) + + with pytest.raises(ProxyException) as error: + async with asyncio.timeout(10): + await count_request_tokens( + request=_FOX_AS_MESSAGE, + call_endpoint=True, + responses_request={"model": "provider-counted", "input": _FOX}, + ) + + assert error.value.code == "408" + + +@pytest.mark.asyncio +async def test_call_endpoint_names_the_status_of_a_count_api_error_with_an_empty_body( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _route_to_one_deployment( + {"model": "openai/gpt-5.6", "api_key": "test-key", "api_base": "https://count-tokens.test/v1"}, monkeypatch ) + with _count_apis_answering(lambda request: httpx.Response(404, request=request)): + with pytest.raises(ProxyException) as error: + await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + + assert error.value.code == "404" + assert "404" in error.value.message + _PINNED_COUNTS: Final = MappingProxyType({"deployment-a.test": 11, "deployment-b.test": 22}) From 3539c0afc999c47eec8e07b027c746f49ed2a38d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 10 Nov 2026 10:11:16 -0800 Subject: [PATCH 021/189] test(integration): cap the OAuth metadata lookup in the mixed capabilities subscription case (#50867) The second server in this case is a plain MCP SDK server registered as oauth_delegate. The gateway's first catalog listing waits for that server's OAuth metadata lookup, which never resolves and runs for the full MCP_METADATA_TIMEOUT of 10 seconds. The MCP SDK client also gives server/discover 10 seconds, so the case passed or failed on about a tenth of a second of head start. Setting LITELLM_MCP_METADATA_TIMEOUT to 5 for this case keeps the lookup well inside the client limit. The case still asserts that subscription changes are delivered and tools are refreshed across both servers. --- tests/integration/mcp/test_interactions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/mcp/test_interactions.py b/tests/integration/mcp/test_interactions.py index 5577cf16067..b77deb5236a 100644 --- a/tests/integration/mcp/test_interactions.py +++ b/tests/integration/mcp/test_interactions.py @@ -1227,6 +1227,7 @@ def test_modern_subscription_delivers_changes_between_calls_and_refreshes_tools( "PYTHONPATH": os.pathsep.join((str(source), str(source / "tests"))), "LITELLM_SALT_KEY": "subscription-pagination-key", **({"LITELLM_MCP_TOOL_LISTING_TIMEOUT": "2"} if access == "probe-timeout" else {}), + **({"LITELLM_MCP_METADATA_TIMEOUT": "5"} if access == "mixed-capabilities" else {}), "STORE_MODEL_IN_DB": "False", "DISABLE_SCHEMA_UPDATE": "true", "LITELLM_LOCAL_MODEL_COST_MAP": "True", From e6db6206d19de2c8d33aeaccdd00049fdb296603 Mon Sep 17 00:00:00 2001 From: shrey Date: Tue, 10 Nov 2026 10:23:50 -0800 Subject: [PATCH 022/189] fix(mcp): accept comma-separated vendor scope grants on the connect check (#50717) --- .../mcp_server/discoverable_endpoints.py | 12 +- litellm/proxy/mcp_server/oauth_utils.py | 31 ++++- .../mcp_server/test_discoverable_endpoints.py | 110 +++++++++++++++++- 3 files changed, 150 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/mcp_server/discoverable_endpoints.py b/litellm/proxy/mcp_server/discoverable_endpoints.py index 07a267d6caa..ecc40b6e98f 100644 --- a/litellm/proxy/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/mcp_server/discoverable_endpoints.py @@ -105,9 +105,11 @@ from litellm.proxy.mcp_server.oauth_utils import ( get_cimd_client_id, get_cimd_document_url, get_request_base_url, + grant_covers_scopes, needs_cimd_discovery, oauth_client_registration_matches, resolve_upstream_resource, + server_scope_names, validate_trusted_redirect_uri, well_known_root_suffix, ) @@ -2537,6 +2539,9 @@ async def _vendor_credential_state( get_user_oauth_credential, oauth_grant_state, ) + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # circular import at module load + global_mcp_server_manager, + ) from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # circular import at module load if prisma_client is None: @@ -2548,7 +2553,12 @@ async def _vendor_credential_state( if oauth_grant_state(credential) == "absent": return "absent" granted_scopes: Final = credential.get("scopes") if credential else None - return "present" if set(requested_scopes).issubset(granted_scopes or ()) else "absent" + covered: Final = grant_covers_scopes( + tuple(granted_scopes or ()), + requested_scopes, + lambda: server_scope_names(global_mcp_server_manager.get_mcp_server_by_id(server_id)), + ) + return "present" if covered else "absent" @router.get("/authorize/flow") diff --git a/litellm/proxy/mcp_server/oauth_utils.py b/litellm/proxy/mcp_server/oauth_utils.py index 76343f77e32..3171c476ede 100644 --- a/litellm/proxy/mcp_server/oauth_utils.py +++ b/litellm/proxy/mcp_server/oauth_utils.py @@ -3,7 +3,7 @@ import os import re -from collections.abc import Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass from ipaddress import ip_address from types import MappingProxyType @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Final, NoReturn from urllib.parse import ParseResult, parse_qsl, urlencode, urlparse, urlsplit, urlunparse, urlunsplit from fastapi import HTTPException, Request +from pydantic import TypeAdapter from starlette.types import Scope from litellm._logging import verbose_logger @@ -131,6 +132,34 @@ def build_upstream_authorize_url(authorization_url: str, params: Mapping[str, st return urlunparse(parsed._replace(query=urlencode(merged))) +def grant_covers_scopes( + granted: tuple[str, ...], requested: tuple[str, ...], server_scopes: Callable[[], frozenset[str]] +) -> bool: + """Whether a stored grant holds every requested scope. A grant stored as one comma-joined token, the form + Slack and GitHub reply with, counts as its parts only when every part is a requested or server scope.""" + if all(scope in granted for scope in requested): + return True + if len(granted) != 1 or "," not in granted[0]: + return False + parts: Final = frozenset(granted[0].split(",")) + known: Final = frozenset(requested) | server_scopes() + return parts <= known and all(scope in granted or scope in parts for scope in requested) + + +_PUBLISHED_SCOPES: Final = TypeAdapter[tuple[str, ...] | None](tuple[str, ...] | None) + + +def server_scope_names(server: "MCPServer | None") -> frozenset[str]: + if server is None: + return frozenset() + published: Final = ( + _PUBLISHED_SCOPES.validate_python(server.authorization_server_metadata.get("scopes_supported")) + if server.authorization_server_metadata is not None + else None + ) + return frozenset((*(server.scopes or ()), *(published or ()))) + + def proxy_base_url_from_env() -> str | None: """The gateway's fixed public URL from ``PROXY_BASE_URL``, ``None`` when unset or malformed.""" return _resolve_proxy_base_url_env() diff --git a/tests/unit/proxy/mcp_server/test_discoverable_endpoints.py b/tests/unit/proxy/mcp_server/test_discoverable_endpoints.py index c17f4b392cd..47d24a9626f 100644 --- a/tests/unit/proxy/mcp_server/test_discoverable_endpoints.py +++ b/tests/unit/proxy/mcp_server/test_discoverable_endpoints.py @@ -16,7 +16,7 @@ from litellm.proxy._types import LiteLLM_MCPServerTable from litellm.types.mcp import MCPAuth if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Iterator, Mapping import httpx from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey @@ -15278,6 +15278,9 @@ async def test_managed_resource_metadata_excludes_refresh_permission(monkeypatch (["tools.read", "tools.write"], ("tools.write",), "present"), (None, ("tools.write",), "absent"), (["tools.read"], (), "present"), + (["channels:read,chat:write"], ("channels:read", "chat:write"), "present"), + (["channels:read,chat:write"], ("channels:read", "files:write"), "absent"), + (["channels:read"], ("channels:read", "chat:write"), "absent"), ], ) async def test_connect_requires_the_requested_vendor_grant(monkeypatch, granted, required, expected): @@ -15293,6 +15296,111 @@ async def test_connect_requires_the_requested_vendor_grant(monkeypatch, granted, read.assert_awaited_once_with(prisma, "owner", "server") +_SLACK_SCOPES: Final = frozenset(("channels:read", "chat:write", "files:write")) +_COMMA_NAMED_SCOPES: Final = frozenset(("files:read", "files:read,write")) + + +@pytest.mark.parametrize( + ("granted", "requested", "known", "expected"), + [ + (("channels:read,chat:write",), ("channels:read", "chat:write"), _SLACK_SCOPES, True), + (("channels:read,chat:write",), ("channels:read", "files:write"), _SLACK_SCOPES, False), + (("channels:read,chat:write,files:read",), ("channels:read",), _SLACK_SCOPES, False), + (("x:unknown,chat:write",), ("chat:write",), _SLACK_SCOPES, False), + (("files:read,write",), ("files:read",), _COMMA_NAMED_SCOPES, False), + (("files:read,write",), ("files:read,write",), _COMMA_NAMED_SCOPES, True), + (("channels:read,chat:write",), ("read",), _SLACK_SCOPES | {"read"}, False), + (("read", "write,admin"), ("admin",), frozenset(("read", "write", "admin")), False), + (("write,admin", "read"), ("admin",), frozenset(("read", "write", "admin")), False), + (("read", "write"), ("read", "write"), frozenset[str](), True), + ((), ("read",), frozenset(("read",)), False), + ((), (), frozenset[str](), True), + ], +) +def test_grant_covers_scopes_splits_a_lone_token_only_into_known_scopes( + granted: tuple[str, ...], requested: tuple[str, ...], known: frozenset[str], expected: bool +) -> None: + from litellm.proxy.mcp_server.oauth_utils import grant_covers_scopes + + assert grant_covers_scopes(granted, requested, lambda: known) is expected + + +@pytest.mark.parametrize( + ("granted", "requested", "expected"), + [ + (("channels:read",), ("channels:read",), True), + (("read", "write"), ("read", "write"), True), + (("read", "write"), ("admin",), False), + (("read",), ("admin",), False), + ((), (), True), + ], +) +def test_grant_covers_scopes_answers_like_a_subset_check_without_reading_server_scopes( + granted: tuple[str, ...], requested: tuple[str, ...], expected: bool +) -> None: + from litellm.proxy.mcp_server.oauth_utils import grant_covers_scopes + + def unreachable_server_scopes() -> frozenset[str]: + raise AssertionError("server scopes read for a grant with no comma token") + + assert grant_covers_scopes(granted, requested, unreachable_server_scopes) is expected + + +def test_server_scope_names_joins_configured_and_published_scopes() -> None: + from litellm.proxy.mcp_server.oauth_utils import server_scope_names + + server: Final = _cimd_oauth_server().model_copy( + update={ + "scopes": ["channels:read"], + "authorization_server_metadata": { + "issuer": "https://idp.example.com", + "authorization_endpoint": "https://idp.example.com/authorize", + "token_endpoint": "https://idp.example.com/token", + "response_types_supported": ["code"], + "scopes_supported": ["chat:write", "files:write"], + }, + } + ) + assert server_scope_names(server) == frozenset(("channels:read", "chat:write", "files:write")) + assert server_scope_names(_cimd_oauth_server()) == frozenset[str]() + assert server_scope_names(None) == frozenset[str]() + + +_PUBLISHED_SLACK_METADATA: Final = { + "issuer": "https://idp.example.com", + "authorization_endpoint": "https://idp.example.com/authorize", + "token_endpoint": "https://idp.example.com/token", + "response_types_supported": ["code"], + "scopes_supported": ["channels:read", "chat:write", "files:read"], +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "server_update", + [ + {"scopes": ["channels:read", "chat:write", "files:read"]}, + {"authorization_server_metadata": _PUBLISHED_SLACK_METADATA}, + ], +) +async def test_connect_reads_a_comma_grant_against_the_server_scopes( + monkeypatch: pytest.MonkeyPatch, server_update: "Mapping[str, object]" +) -> None: + from litellm.proxy.mcp_server import db + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + + server: Final = _cimd_oauth_server().model_copy(update=server_update) + credential: Final = {**_stored_grant(), "scopes": ["channels:read,chat:write,files:read"]} + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + monkeypatch.setattr(db, "get_user_oauth_credential", AsyncMock(return_value=credential)) + assert await endpoints._vendor_credential_state("owner", server.server_id, ("channels:read",)) == "absent" + monkeypatch.setitem(global_mcp_server_manager.registry, server.server_id, server) + assert await endpoints._vendor_credential_state("owner", server.server_id, ("channels:read",)) == "present" + assert await endpoints._vendor_credential_state("owner", server.server_id, ("files:write",)) == "absent" + assert await endpoints._vendor_credential_state("owner", server.server_id, ()) == "present" + + @pytest.mark.parametrize( ("header", "expected"), [ From 43a4b465e94a849c64816e1bfaceb1eed6e7a200 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 10:30:00 -0800 Subject: [PATCH 023/189] fix(cost-map): give the Gemini 4 Argon rows the shared Gemini 3 search free tier (#50959) --- litellm/model_prices_and_context_window_backup.json | 3 +++ model_prices_and_context_window.json | 3 +++ tests/unit/test_model_prices_and_context_window.py | 11 +++++++---- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3196b3c0c08..3145ca4a181 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -78910,6 +78910,7 @@ "prompt_cache_min_tokens": 4096.0, "rpm": 2000.0, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -78970,6 +78971,7 @@ "prompt_cache_min_tokens": 6144.0, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -79029,6 +79031,7 @@ "prompt_cache_min_tokens": 6144.0, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3196b3c0c08..3145ca4a181 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -78910,6 +78910,7 @@ "prompt_cache_min_tokens": 4096.0, "rpm": 2000.0, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -78970,6 +78971,7 @@ "prompt_cache_min_tokens": 6144.0, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -79029,6 +79031,7 @@ "prompt_cache_min_tokens": 6144.0, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 62c1fab244c..3408a95935d 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -958,23 +958,26 @@ def test_gemini_4_argon_google_rows_carry_supports_continuation_in_both_cost_map assert _supports_continuation_rows(BACKUP_PRICES_PATH) == main_rows -def test_gemini_4_carbon_rows_share_the_gemini_3_search_allowance_in_both_cost_map_copies() -> None: +def test_gemini_4_rows_share_the_gemini_3_search_allowance_in_both_cost_map_copies() -> None: main_rows: Final = _search_free_requests_rows(PRICES_PATH) # 5,000 free grounding requests per month shared across Gemini 3 and newer: # https://ai.google.dev/gemini-api/docs/pricing#gemini-4-carbon and # https://cloud.google.com/vertex-ai/generative-ai/pricing (2026-11-10) - carbon_rows: Final = { + gemini_4_rows: Final = { + "gemini/gemini-4-argon", + "vertex_ai/gemini-4-argon", + "vertex_ai/gemini-4-argon-cyber-permissive", "gemini/gemini-4-carbon", "vertex_ai/gemini-4-carbon", "vertex_ai/gemini-4-carbon-cyber-permissive", } - assert carbon_rows <= main_rows.keys(), sorted(main_rows) + assert gemini_4_rows <= main_rows.keys(), sorted(main_rows) gemini_3_allowances: Final = { json.dumps(main_rows[key]["search_free_requests"], sort_keys=True) for key in ("gemini/gemini-3.8-flash", "vertex_ai/gemini-3.8-flash") } assert { - json.dumps(main_rows[key]["search_free_requests"], sort_keys=True) for key in carbon_rows + json.dumps(main_rows[key]["search_free_requests"], sort_keys=True) for key in gemini_4_rows } == gemini_3_allowances assert _search_free_requests_rows(BACKUP_PRICES_PATH) == main_rows From 52999149640659344155cbfe6ded6634c3511e9e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:31:57 +0000 Subject: [PATCH 024/189] test(http2): drain client traffic after sending GOAWAY (#50972) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/sdk/test_http2_wire.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/sdk/test_http2_wire.py b/tests/integration/sdk/test_http2_wire.py index a263472a45f..30a0635f3b8 100644 --- a/tests/integration/sdk/test_http2_wire.py +++ b/tests/integration/sdk/test_http2_wire.py @@ -531,6 +531,9 @@ def _serve_h2(context: ssl.SSLContext, raw: socket.socket, drain: bool) -> None: tls.sendall(connection.data_to_send()) while data := tls.recv(65535): if _reply_to_ended_streams(connection, tls, data, drain) and drain: + tls.shutdown(socket.SHUT_WR) + while tls.recv(65535): + pass return From d47c03175bba41a39c4c3f8a947e28938708eef2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:33:27 +0000 Subject: [PATCH 025/189] fix(responses): keep unset-effort gpt-5.4/5.5 tool calls on chat completions (#50968) * fix(responses): keep unset-effort gpt-5.4/5.5 tool calls on chat completions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): explicitly route bridge fixtures to responses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kenneth Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 4 + litellm/responses/bridge_check.py | 43 ++---- model_prices_and_context_window.json | 4 + .../sdk/test_unset_effort_tool_bridge_wire.py | 146 ++++++++++++++++++ ...hropic_messages_adapters_transformation.py | 26 ++-- .../test_deferred_guardrail_logging.py | 3 +- .../test_responses_api_bridge_flag.py | 65 +++++++- tests/unit/test_main.py | 23 +-- 8 files changed, 249 insertions(+), 65 deletions(-) create mode 100644 tests/integration/sdk/test_unset_effort_tool_bridge_wire.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3145ca4a181..01a32ee7cdb 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -72516,6 +72516,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.4-mini": { + "default_reasoning_effort": "none", "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 8.25e-08, @@ -72537,6 +72538,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.4-nano": { + "default_reasoning_effort": "none", "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.2e-08, @@ -73140,6 +73142,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.4-mini": { + "default_reasoning_effort": "none", "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 8.25e-08, @@ -73161,6 +73164,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.4-nano": { + "default_reasoning_effort": "none", "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.2e-08, diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 683a51fc6b6..6a55302027d 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -192,35 +192,11 @@ def responses_api_bridge_check( model_info["mode"] = "responses" model = model.replace("responses/", "") - # OpenAI/Azure GPT-5 chat-completions that need Responses-only fields (e.g. - # ``reasoningSummary`` in ``extra_body``) must be bridged; Chat Completions rejects - # those keys. - # - # - gpt-5.4+: FUNCTION tools with reasoning active must be bridged. OpenAI enables - # reasoning by default for these models (unset reasoning_effort means medium - # server-side), and Chat Completions rejects function tools whenever reasoning is - # on ("Function tools with reasoning_effort are not supported ... use - # /v1/responses or set reasoning_effort to 'none'"), so only an explicit - # ``"none"`` the chat gate forwards keeps the request chat-servable: a ``none`` - # the model's map row refuses is dropped (or refused) before the call, which - # leaves the provider's default effort in force. Custom (grammar) tools are served - # natively by Chat Completions with reasoning on, so custom-only requests stay on - # chat and keep their native custom tool_call response shape. - # - The UNSET-effort arm only fires against endpoints known to enforce that - # constraint (any api.openai.com host, or Azure OpenAI where api_base is - # always set): chat-only OpenAI-compatible backends registered under the openai - # provider with a custom api_base and gpt-5.4+ model names serve tools without - # reasoning fine and have no /responses route, so they keep pre-existing - # behavior (bridge only on an explicit reasoning_effort). - # - Azure AI Foundry's OpenAI v1 hosts (azure_ai provider) enforce it later in the series: - # an explicit effort with function tools is rejected from gpt-5.6 on, and the unset - # effort only from gpt-6 on (gpt-5.6 serves tools with reasoning silently off), so the - # azure_ai gate keys on those measured boundaries instead of gpt-5.4+. - # - Bedrock Mantle serves the same API and enforces the gpt-5.4+ rule even with the effort - # unset (measured on gpt-5.6-terra), so a Mantle row that also serves Responses follows - # these rules, which keeps a deployment pinned to ``mode: chat`` from a 400 on them. - # - Older GPT-5 names (e.g. ``gpt-5``, ``gpt-5.1``): bridge only when a reasoning - # summary alias is present with ``reasoning_effort`` (tools alone stay on chat). + # Function tools with explicit active reasoning bridge on GPT-5.4+. With effort unset, + # OpenAI/Azure and Responses-capable Mantle endpoints bridge only GPT-5.6+ whose map + # default is not "none". Custom OpenAI-compatible endpoints keep unset effort on chat. + # Foundry uses its separate measured boundaries. Custom-only tools remain chat-native; + # reasoning summaries and tool_search retain their independent Responses requirements. has_function_tool: Final = any( ( tool.get("type") == "function" and (isinstance(tool.get("function"), dict) or "name" in tool) @@ -274,7 +250,14 @@ def responses_api_bridge_check( if on_foundry_openai_endpoint else ( OpenAIGPT5Config.is_model_gpt_5_4_plus_model(gpt_series_name) - and (reasoning_effort is not None or on_constraint_enforcing_endpoint) + and ( + reasoning_effort is not None + or ( + on_constraint_enforcing_endpoint + and OpenAIGPT5Config.is_model_gpt_5_6_plus_model(gpt_series_name) + and model_info.get("default_reasoning_effort") != "none" + ) + ) ) ) ) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3145ca4a181..01a32ee7cdb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -72516,6 +72516,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.4-mini": { + "default_reasoning_effort": "none", "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 8.25e-08, @@ -72537,6 +72538,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.4-nano": { + "default_reasoning_effort": "none", "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.2e-08, @@ -73140,6 +73142,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.4-mini": { + "default_reasoning_effort": "none", "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 8.25e-08, @@ -73161,6 +73164,7 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.4-nano": { + "default_reasoning_effort": "none", "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "cache_read_input_token_cost": 2.2e-08, diff --git a/tests/integration/sdk/test_unset_effort_tool_bridge_wire.py b/tests/integration/sdk/test_unset_effort_tool_bridge_wire.py new file mode 100644 index 00000000000..11740f1d163 --- /dev/null +++ b/tests/integration/sdk/test_unset_effort_tool_bridge_wire.py @@ -0,0 +1,146 @@ +import json +from typing import Final + +import pytest +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm import Router +from tests.integration._support.wire import Reply, Request, wire_server + +_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_TOOL: Final = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} +_ARGUMENTS: Final = '{"city":"Paris"}' + + +def _tool_reply(request: Request) -> Reply: + body: Final = _OBJECT.validate_json(request.body) + if "/responses" in request.target: + return Reply( + body=json.dumps( + { + "id": "resp_weather", + "object": "response", + "created_at": 1789788253, + "model": "gpt-5.6", + "status": "completed", + "output": [ + { + "type": "function_call", + "id": "fc_weather", + "call_id": "call_weather", + "name": "get_weather", + "arguments": _ARGUMENTS, + "status": "completed", + } + ], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + ).encode() + ) + return Reply( + body=json.dumps( + { + "id": "chatcmpl-weather", + "object": "chat.completion", + "created": 1789788253, + "model": body.get("model"), + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": _ARGUMENTS}, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + ).encode() + ) + + +@pytest.mark.parametrize( + ("model", "placement", "endpoint"), + ( + ("gpt-5.4-mini", "unset", "chat/completions"), + ("gpt-5.5", "unset", "chat/completions"), + ("gpt-5.6", "unset", "responses"), + ("gpt-5.4-mini", "extra-body", "responses"), + ("gpt-5.4-mini", "none", "chat/completions"), + ("gpt-5.4-mini", "deployment", "responses"), + ), +) +def test_function_tools_reach_the_chat_servable_endpoint_and_keep_tool_call_shape( + model: str, placement: str, endpoint: str +) -> None: + with wire_server(_tool_reply) as wire: + parameters: Final = { + "model": f"azure/{model}", + "api_base": wire.url, + "api_key": "synthetic-azure-key", + "api_version": "2025-04-01-preview", + } + fields: Final = ( + {"extra_body": {"reasoning_effort": "medium"}} + if placement == "extra-body" + else {"reasoning_effort": "none"} + if placement == "none" + else {} + ) + messages: Final = [{"role": "user", "content": "What's the weather in Paris? Use the tool."}] + if placement == "deployment": + router: Final = Router( + model_list=[ + { + "model_name": "weather", + "litellm_params": {**parameters, "reasoning_effort": "medium"}, + } + ], + num_retries=0, + ) + response: Final = router.completion(model="weather", messages=messages, tools=[_TOOL]) + else: + response_direct: Final = litellm.completion( + **parameters, + **fields, + messages=messages, + tools=[_TOOL], + num_retries=0, + ) + answer: Final = response if placement == "deployment" else response_direct + (sent,) = tuple(request for request in wire.drain() if request.method == "POST") + expected_path: Final = ( + "/openai/responses" if endpoint == "responses" else f"/openai/deployments/{model}/chat/completions" + ) + assert sent.target.split("?")[0] == expected_path, sent.target + assert answer.choices[0].finish_reason == "tool_calls" + assert answer.choices[0].message.tool_calls[0].function.name == "get_weather" + assert json.loads(answer.choices[0].message.tool_calls[0].function.arguments) == {"city": "Paris"} + outbound: Final = _OBJECT.validate_json(sent.body) + if endpoint == "responses": + assert outbound["tools"] == [ + { + "type": "function", + "name": "get_weather", + "parameters": _TOOL["function"]["parameters"], + "description": None, + "strict": None, + } + ] + else: + assert outbound["tools"] == [_TOOL] diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index bad7b3f1750..a76d5e82a37 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -759,7 +759,7 @@ def test_translate_openai_reasoning_items_are_not_duplicated_when_thinking_block def _replayed_reasoning_request(tools: list[dict[str, object]]) -> AnthropicMessagesRequest: return AnthropicMessagesRequest( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", max_tokens=64, messages=[ AnthropicMessagesUserMessageParam(role="user", content="Which city is colder?"), @@ -5820,7 +5820,7 @@ _RETRY_REPLY: Final = { "object": "response", "created_at": 1700000000, "status": "completed", - "model": "gpt-5.4-nano", + "model": "gpt-5.6", "output": [ { "type": "message", @@ -5868,7 +5868,7 @@ async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_de with caplog.at_level(logging.WARNING, logger="LiteLLM"): response: Final = await litellm.anthropic.messages.acreate( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", api_base=_RETRY_API_BASE, api_key="fake-key", api_version="2025-04-01-preview", @@ -5890,7 +5890,7 @@ async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_de assert retry_warnings == ( "Backend rejected replayed encrypted reasoning, retrying once without it. " "Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. " - "provider=azure model=azure/gpt-5.4-nano status=400 code=invalid_encrypted_content", + "provider=azure model=azure/gpt-5.6 status=400 code=invalid_encrypted_content", ) assert all( "gAAA_foreign" not in message and "litellm_encrypted_reasoning:" not in message for message in warnings @@ -5906,7 +5906,7 @@ def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( ) response: Final = litellm.anthropic.messages.create( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", api_base=_RETRY_API_BASE, api_key="fake-key", api_version="2025-04-01-preview", @@ -5930,7 +5930,7 @@ async def test_async_messages_with_num_retries_send_the_rejected_reasoning_only_ ) await litellm.anthropic.messages.acreate( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", api_base=_RETRY_API_BASE, api_key="fake-key", api_version="2025-04-01-preview", @@ -5953,7 +5953,7 @@ def test_sync_messages_with_global_num_retries_send_the_rejected_reasoning_once_ ) litellm.anthropic.messages.create( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", api_base=_RETRY_API_BASE, api_key="fake-key", api_version="2025-04-01-preview", @@ -5981,7 +5981,7 @@ async def test_async_messages_retry_a_server_error_with_the_reasoning_and_drop_i ) response: Final = await litellm.anthropic.messages.acreate( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", api_base=_RETRY_API_BASE, api_key="fake-key", api_version="2025-04-01-preview", @@ -6004,7 +6004,7 @@ def test_sync_messages_retry_a_server_error_with_the_reasoning_and_drop_it_only_ ) response: Final = litellm.anthropic.messages.create( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", api_base=_RETRY_API_BASE, api_key="fake-key", api_version="2025-04-01-preview", @@ -6063,7 +6063,7 @@ async def test_async_messages_log_only_the_outcome_of_the_turn_sent_again( with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext(): await litellm.anthropic.messages.acreate( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", api_base=_RETRY_API_BASE, api_key="fake-key", api_version="2025-04-01-preview", @@ -6089,7 +6089,7 @@ def test_sync_messages_log_only_the_outcome_of_the_turn_sent_again( with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext(): litellm.anthropic.messages.create( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", api_base=_RETRY_API_BASE, api_key="fake-key", api_version="2025-04-01-preview", @@ -6124,7 +6124,7 @@ async def test_messages_do_not_retry_other_rejections( with pytest.raises(litellm.BadRequestError): await litellm.anthropic.messages.acreate( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", api_base=_RETRY_API_BASE, api_key="fake-key", api_version="2025-04-01-preview", @@ -6277,7 +6277,7 @@ async def test_streamed_reasoning_summaries_stay_with_their_own_encrypted_item( ) stream: Final = await litellm.anthropic.messages.acreate( - model="azure/gpt-5.4-nano", + model="azure/gpt-5.6", api_base=_RETRY_API_BASE, api_key="fake-key", api_version="2025-04-01-preview", diff --git a/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py b/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py index d2161fddeb0..bfbf7318d52 100644 --- a/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py +++ b/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py @@ -330,7 +330,7 @@ async def test_deferred_slot_keeps_the_innermost_wrapper_result(): async def test_deferred_anthropic_messages_bridged_to_the_responses_api_logs_the_provider_usage( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ): - """/v1/messages on an Azure gpt-5.4+ deployment with function tools runs three nested + """/v1/messages on an Azure deployment with function tools and explicit reasoning runs three nested wrappers: anthropic_messages, the chat adapter's acompletion, and the Responses bridge acompletion hands the call to, which retags the call as ``responses``. With logging deferred for a post-call guardrail the stored closure must carry the innermost provider @@ -382,6 +382,7 @@ async def test_deferred_anthropic_messages_bridged_to_the_responses_api_logs_the response: Final = await litellm.anthropic_messages( model="azure/gpt-5.4-nano", + reasoning_effort="medium", messages=[{"role": "user", "content": "hi"}], max_tokens=16, tools=[ diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index 210488f0888..f6084529ed6 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -786,17 +786,74 @@ _FUNCTION_TOOL: Final[dict[str, JsonValue]] = { @pytest.mark.parametrize("custom_llm_provider", ["openai", "azure"]) -def test_a_chat_pin_cannot_pull_gpt_5_4_function_tool_calls_off_the_responses_api(custom_llm_provider: str) -> None: - """Chat Completions rejects function tools while reasoning is on for gpt-5.4+, so the capability rule - must still bridge the request even when the deployment pins ``mode: chat``.""" +@pytest.mark.parametrize(("effort", "mode"), ((None, "chat"), ("medium", "responses"))) +def test_a_chat_pin_routes_gpt_5_4_function_tools_by_explicit_effort( + custom_llm_provider: str, effort: str | None, mode: str +) -> None: info, model = responses_api_bridge_check( model="gpt-5.4", custom_llm_provider=custom_llm_provider, tools=[_FUNCTION_TOOL], + reasoning_effort=effort, api_base="https://example.openai.azure.com" if custom_llm_provider == "azure" else None, deployment_model_info=_CHAT, ) - assert (info["mode"], model) == ("responses", "gpt-5.4") + assert (info["mode"], model) == (mode, "gpt-5.4") + + +@pytest.mark.parametrize("provider", ("openai", "azure")) +@pytest.mark.parametrize( + ("model", "effort", "extra_body", "expected"), + ( + ("gpt-5.4-mini", None, None, "chat"), + ("gpt-5.4-nano", None, None, "chat"), + ("gpt-5.5", None, None, "chat"), + ("gpt-5.6", None, None, "responses"), + ("gpt-6.1-sol", None, None, "responses"), + ("gpt-5.4-mini", "none", None, "chat"), + ("gpt-5.4-mini", "medium", None, "responses"), + ("gpt-5.4-mini", {"effort": "medium"}, None, "responses"), + ("gpt-5.4-mini", None, {"reasoning_effort": "medium"}, "responses"), + ), +) +def test_function_tools_bridge_only_when_chat_rejects_the_effective_effort( + provider: str, + model: str, + effort: str | Mapping[str, object] | None, + extra_body: Mapping[str, object] | None, + expected: str, +) -> None: + info, _ = responses_api_bridge_check( + model=model, + custom_llm_provider=provider, + tools=[_FUNCTION_TOOL], + reasoning_effort=bridge_reasoning_effort(effort, extra_body), + api_base="https://api.openai.com/v1", + ) + assert info.get("mode") == expected + + +@pytest.mark.parametrize("region", ("us", "eu")) +@pytest.mark.parametrize("model", ("gpt-5.4-mini", "gpt-5.4-nano")) +def test_regional_azure_default_none_function_tools_stay_on_chat(region: str, model: str) -> None: + info, _ = responses_api_bridge_check( + model=f"{region}/{model}", custom_llm_provider="azure", tools=[_FUNCTION_TOOL] + ) + assert info.get("mode") == "chat" + assert info.get("default_reasoning_effort") == litellm.model_cost[f"azure/{model}"]["default_reasoning_effort"] + + +def test_registered_gpt_5_6_default_none_function_tools_stay_on_chat(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "gpt-5.6-test-default-none" + monkeypatch.setitem( + litellm.model_cost, + model, + {**litellm.model_cost["gpt-5.6"], "default_reasoning_effort": "none"}, + ) + info, _ = responses_api_bridge_check( + model=model, custom_llm_provider="openai", tools=[_FUNCTION_TOOL], api_base="https://api.openai.com/v1" + ) + assert info.get("mode") == "chat" @pytest.mark.parametrize("model", ["openai.gpt-5.6-terra", "us-east-1/openai.gpt-5.6-terra"]) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index dbd9ba8a32e..d2035b6a682 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -1418,12 +1418,7 @@ def test_responses_api_bridge_check_azure_gpt_5_4_tools_plus_reasoning_routes_to assert model_info.get("mode") == "responses" -def test_responses_api_bridge_check_azure_gpt_5_4_tools_with_default_reasoning_routes_to_responses(): - """ - Azure gpt-5.4 with tools and UNSET reasoning_effort must bridge: OpenAI enables - reasoning by default for gpt-5.4+, and Chat Completions rejects function tools - whenever reasoning is on. - """ +def test_responses_api_bridge_check_azure_gpt_5_4_tools_with_default_reasoning_stays_chat(): from litellm.main import responses_api_bridge_check with patch("litellm.responses.bridge_check.get_model_info_helper") as mock_get_model_info: @@ -1436,15 +1431,10 @@ def test_responses_api_bridge_check_azure_gpt_5_4_tools_with_default_reasoning_r ) assert model == "gpt-5.4" - assert model_info.get("mode") == "responses" + assert model_info.get("mode") != "responses" -def test_responses_api_bridge_check_gpt_5_4_tools_with_default_reasoning_routes_to_responses(): - """ - gpt-5.4 with tools and UNSET reasoning_effort must bridge: OpenAI enables reasoning - by default for gpt-5.4+, and Chat Completions rejects function tools whenever - reasoning is on ("use /v1/responses or set reasoning_effort to 'none'"). - """ +def test_responses_api_bridge_check_gpt_5_4_tools_with_default_reasoning_stays_chat(): from litellm.main import responses_api_bridge_check with patch("litellm.responses.bridge_check.get_model_info_helper") as mock_get_model_info: @@ -1457,7 +1447,7 @@ def test_responses_api_bridge_check_gpt_5_4_tools_with_default_reasoning_routes_ ) assert model == "gpt-5.4" - assert model_info.get("mode") == "responses" + assert model_info.get("mode") != "responses" @pytest.mark.parametrize("region", ("us", "eu")) @@ -1972,8 +1962,7 @@ def test_responses_api_bridge_check_custom_api_base_with_explicit_effort_still_r assert model_info.get("mode") == "responses" -def test_responses_api_bridge_check_azure_with_api_base_and_unset_effort_routes(): - """Azure OpenAI always sets api_base and does enforce the constraint; keep bridging.""" +def test_responses_api_bridge_check_azure_gpt_5_4_with_api_base_and_unset_effort_stays_chat(): from litellm.main import responses_api_bridge_check with patch("litellm.responses.bridge_check.get_model_info_helper") as mock_get_model_info: @@ -1987,7 +1976,7 @@ def test_responses_api_bridge_check_azure_with_api_base_and_unset_effort_routes( ) assert model == "gpt-5.4" - assert model_info.get("mode") == "responses" + assert model_info.get("mode") != "responses" _FOUNDRY_API_BASE: Final = "https://myproject.services.ai.azure.com" From 2624f7caead93e7e84e85a282a22650bb3d98ae9 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 10:42:31 -0800 Subject: [PATCH 026/189] fix(cost-map): mark the azure_ai Claude rows that bill web search as supporting it (#50960) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 15 +++++++++++++++ model_prices_and_context_window.json | 15 +++++++++++++++ .../unit/test_model_prices_and_context_window.py | 13 +++++++++++++ 3 files changed, 43 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 01a32ee7cdb..a4c16fbeaa3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3965,6 +3965,7 @@ "supports_vision": true, "supports_output_config": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-7": { @@ -3997,6 +3998,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 2048 }, "azure_ai/claude-fable-5": { @@ -4031,6 +4033,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": false, + "supports_web_search": true, "prompt_cache_min_tokens": 512 }, "azure_ai/claude-fable-5-1": { @@ -4065,6 +4068,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": false, + "supports_web_search": true, "prompt_cache_min_tokens": 512, "deprecation_date": "2027-12-05", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" @@ -4136,6 +4140,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 512 }, "azure_ai/claude-opus-5-5": { @@ -4171,6 +4176,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" }, @@ -4208,6 +4214,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-5": { @@ -4267,6 +4274,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-6": { @@ -76637,6 +76645,7 @@ "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, "supports_forced_tool_use": false, + "supports_web_search": true, "thinking_always_on": true, "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" @@ -78208,6 +78217,7 @@ "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, "supports_forced_tool_use": true, + "supports_web_search": true, "thinking_always_on": false, "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide" @@ -79277,6 +79287,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, + "supports_web_search": true, "thinking_always_on": true }, "baseten/meta-models/muse-spark-open": { @@ -88134,6 +88145,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, + "supports_web_search": true, "thinking_always_on": true }, "azure_ai/claude-mythos-5-2-fast": { @@ -88169,6 +88181,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, + "supports_web_search": true, "thinking_always_on": true }, "azure_ai/claude-fable-5-2": { @@ -88200,6 +88213,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, + "supports_web_search": true, "thinking_always_on": true }, "azure_ai/claude-fable-5-2-fast": { @@ -88231,6 +88245,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, + "supports_web_search": true, "thinking_always_on": true }, "bedrock/us-gov-east-1/anthropic.claude-mythos-5-2": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 01a32ee7cdb..a4c16fbeaa3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3965,6 +3965,7 @@ "supports_vision": true, "supports_output_config": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 4096 }, "azure_ai/claude-opus-4-7": { @@ -3997,6 +3998,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 2048 }, "azure_ai/claude-fable-5": { @@ -4031,6 +4033,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": false, + "supports_web_search": true, "prompt_cache_min_tokens": 512 }, "azure_ai/claude-fable-5-1": { @@ -4065,6 +4068,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": false, + "supports_web_search": true, "prompt_cache_min_tokens": 512, "deprecation_date": "2027-12-05", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" @@ -4136,6 +4140,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 512 }, "azure_ai/claude-opus-5-5": { @@ -4171,6 +4176,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" }, @@ -4208,6 +4214,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-5": { @@ -4267,6 +4274,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, + "supports_web_search": true, "prompt_cache_min_tokens": 1024 }, "azure_ai/claude-sonnet-4-6": { @@ -76637,6 +76645,7 @@ "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, "supports_forced_tool_use": false, + "supports_web_search": true, "thinking_always_on": true, "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" @@ -78208,6 +78217,7 @@ "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, "supports_forced_tool_use": true, + "supports_web_search": true, "thinking_always_on": false, "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide" @@ -79277,6 +79287,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, + "supports_web_search": true, "thinking_always_on": true }, "baseten/meta-models/muse-spark-open": { @@ -88134,6 +88145,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, + "supports_web_search": true, "thinking_always_on": true }, "azure_ai/claude-mythos-5-2-fast": { @@ -88169,6 +88181,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, + "supports_web_search": true, "thinking_always_on": true }, "azure_ai/claude-fable-5-2": { @@ -88200,6 +88213,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, + "supports_web_search": true, "thinking_always_on": true }, "azure_ai/claude-fable-5-2-fast": { @@ -88231,6 +88245,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, + "supports_web_search": true, "thinking_always_on": true }, "bedrock/us-gov-east-1/anthropic.claude-mythos-5-2": { diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 3408a95935d..b270d385693 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -1048,3 +1048,16 @@ def test_bedrock_gpt_oss_rows_declare_vision_support_like_their_safeguard_siblin def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> None: assert litellm.utils.is_vision_explicitly_disabled("bedrock/openai.gpt-oss-120b-1:0") is True assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_azure_ai_claude_rows_priced_for_web_search_say_they_support_it(path: Path) -> None: + prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) + priced_rows: Final = { + key: row + for key, row in prices.items() + if key.startswith("azure_ai/claude") and isinstance(row, dict) and row.get("search_context_cost_per_query") + } + assert priced_rows + unflagged: Final = tuple(key for key, row in priced_rows.items() if row.get("supports_web_search") is not True) + assert unflagged == (), unflagged From 906578df394423e65f8de0ad190464290ff42d54 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 10 Nov 2026 10:52:27 -0800 Subject: [PATCH 027/189] test(integration): recognize the model list's lock in the held write helper (#50974) The registry lock fix (#50933) makes add_deployment and set_model_list wait for a held delete on the same lock, taken inside HostedModelList as self._lock. The helper only knew lines spelled with self._deployment_index_lock, so it never saw those two writes reach the lock and timed out after 10 seconds. --- ...est_router_concurrent_deployment_delete_sdk.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py b/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py index 9f8aa966335..4cc6aca5a03 100644 --- a/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py +++ b/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py @@ -39,15 +39,16 @@ def _router_source_files() -> frozenset[str]: return frozenset({_HAND_CHANGES.co_filename, *(path for path in facade_files if path is not None)}) -def _lock_lines(path: str) -> frozenset[int]: - return frozenset( - number - for number, line in enumerate(linecache.getlines(path), start=1) - if "with self._deployment_index_lock" in line - ) +def _lock_lines(path: str, statement: str) -> frozenset[int]: + return frozenset(number for number, line in enumerate(linecache.getlines(path), start=1) if statement in line) -_INDEX_LOCK_LINES: Final = MappingProxyType({path: _lock_lines(path) for path in _router_source_files()}) +_INDEX_LOCK_LINES: Final = MappingProxyType( + { + **{path: _lock_lines(path, "with self._deployment_index_lock") for path in _router_source_files()}, + _LIST_REFRESH.co_filename: _lock_lines(_LIST_REFRESH.co_filename, "with self._lock"), + } +) T = TypeVar("T") From 59a3e75368a6923076bd728bb13ee68b22f9b29f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:53:04 +0000 Subject: [PATCH 028/189] test(anthropic): scope structured output captures to inference routes (#50983) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../providers/test_vertex_anthropic_structured_output_wire.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index edecdf9e785..c0964478da6 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -605,6 +605,7 @@ def _delivered(wire: Wire) -> tuple[_Delivery, ...]: return tuple( _Delivery(request.target, request.headers.get("anthropic-beta"), _JSON_OBJECT.validate_json(request.body)) for request in wire.drain() + if _scripted_target(request) ) From 9ba551b871691bb7f9aa3eb59310ac5eb60b5ac0 Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Tue, 10 Nov 2026 11:06:17 -0800 Subject: [PATCH 029/189] feat(proxy): let teams use their own allowed S3 buckets for batch files (#50571) * feat(proxy): let teams send allowed S3 buckets for batch files and read them back A team's metadata can now list allowed_s3_buckets. Keys on that team may send one of those buckets as s3_bucket_name on POST /v1/files and as s3_output_bucket_name on POST /v1/batches, and the upload and batch-create handlers check the bucket against the team list plus general_settings.allowed_client_side_s3_buckets. The early auth body check still rejects bucket fields on every other route and method Reads of managed files now trust the bucket LiteLLM recorded in the stored s3:// uri through an internal stored_s3_bucket_name param, so content, retrieve, delete and the batch cost job can read files written to a request-chosen bucket. Callers can never send that param, and files in the model bucket resolve exactly as before * test(proxy): assert outcomes in team bucket tests and type the managed content read Gives the admit-only tests a real assertion, has the refresh test check what the caller gets back, and narrows the provider file id before reading its bucket so the type gate stays at its base count * fix(proxy): keep team S3 bucket lists proxy-admin only and read prefixed request buckets back A team admin allowed to edit team metadata could add any bucket to the team's allowed_s3_buckets, so the key now sits with the other proxy-controlled team metadata keys. The stored bucket for a managed S3 file now keeps any prefix that comes before LiteLLM's own path, so a file uploaded to a request bucket like team-bucket/tenant reads, refreshes and deletes from that prefix * fix(proxy): keep the request's approved S3 bucket when merging deployment credentials Model-routed uploads and batch creates merge the deployment's credentials into the request, and that merge let a deployment's s3_bucket_name or s3_output_bucket_name replace the bucket the request had already been approved for, so Bedrock wrote to the deployment bucket. The merge now keeps any bucket the request carries and fills it from the deployment only when the request has none * fix(proxy): keep the request bucket only on upload and batch create, not on reads The previous commit let a request's S3 bucket win over the deployment's on every credential merge, which undid #47496 on reads, where the deployment's bucket bounds which buckets a model-routed file id may point at. Deployment credentials win again by default, and only model-routed and provider-only upload and batch create opt in to keep the bucket the request was approved for * fix(proxy): only proxy admins can set a team's allowed_s3_buckets Org admins and users allowed to create teams could put any bucket on a team's allowed_s3_buckets through /team/new or a team update, which grants that team the proxy's AWS access to the bucket. The key now goes through the same proxy-admin-only metadata check as the batch limits, so only a proxy admin can add, change or remove it, and resending the stored value still passes --- .../proxy/common_utils/check_batch_cost.py | 3 +- .../proxy/hooks/managed_files.py | 22 +- litellm/batches/batch_utils.py | 3 +- .../cloud_storage_security.py | 22 ++ .../litellm_core_utils/get_litellm_params.py | 1 + litellm/llms/bedrock/files/transformation.py | 13 +- litellm/proxy/auth/auth_utils.py | 115 ++++++++--- litellm/proxy/batches_endpoints/endpoints.py | 13 ++ .../team_admin_field_permissions.py | 2 + .../openai_files_endpoints/common_utils.py | 24 ++- .../openai_files_endpoints/files_endpoints.py | 12 ++ tests/unit/batches/test_batch_utils.py | 22 ++ .../proxy/hooks/test_managed_files.py | 119 +++++++++++ .../test_bedrock_files_transformation.py | 82 ++++++++ tests/unit/proxy/auth/test_auth_utils.py | 190 +++++++++++++++++- .../proxy/batches_endpoints/test_endpoints.py | 83 ++++++++ .../common_utils/test_check_batch_cost.py | 20 +- .../test_team_admin_field_permissions.py | 5 +- .../test_team_endpoints.py | 61 ++++++ .../test_files_common_utils.py | 17 ++ .../test_files_endpoint.py | 39 ++++ .../test_endpoints_characterization_batch.py | 26 +++ 22 files changed, 850 insertions(+), 44 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 4cfc6d4e081..abc2487a4a5 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -20,6 +20,7 @@ from litellm_enterprise.proxy.common_utils.managed_object_attribution import ( import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs from litellm.constants import ( BATCH_LINE_ITEMS_MAX_PART_BYTES, MANAGED_OBJECT_STALENESS_CUTOFF_DAYS, @@ -765,7 +766,7 @@ class CheckBatchCost: credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {} _file_content = await afile_content( file_id=cast(str, raw_output_file_id), - **cast(FileContentCallOptions, credentials), + **cast(FileContentCallOptions, {**credentials, **stored_s3_bucket_kwargs(cast(str, raw_output_file_id))}), ) content_bytes: Final = await _file_content_bytes(_file_content) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 626a3b0bb83..bbb09e75b47 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -49,6 +49,7 @@ from litellm.constants import ( ) from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.internal_call_metadata import requests_background @@ -396,6 +397,7 @@ class _CursorPageArgs(TypedDict, total=False): class _RouterFileCallKwargs(TypedDict, total=False): client: ReadOnly[AsyncOpenAI | None] custom_llm_provider: ReadOnly[str | None] + stored_s3_bucket_name: ReadOnly[str] _MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str]) @@ -463,7 +465,7 @@ async def _delete_provider_file( await llm_router.afile_delete( model=model_id, file_id=provider_file_id, - **cast(_RouterFileCallKwargs, delete_data), + **cast(_RouterFileCallKwargs, {**delete_data, **stored_s3_bucket_kwargs(provider_file_id)}), ) @@ -1157,7 +1159,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve) retrieve_options: Final = cast( FileRetrieveCallOptions, - {**call_options, "max_retries": 0}, + {**call_options, **stored_s3_bucket_kwargs(provider_file_id), "max_retries": 0}, ) for attempt in range(len(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS) + 1): if attempt > 0: @@ -1795,8 +1797,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if potential_file_id and "llm_output_file_id," in potential_file_id: model_id = self.get_model_id_from_unified_file_id(potential_file_id) if model_id: + provider_file_id: Final = self.get_output_file_id_from_unified_file_id(potential_file_id) data["model"] = model_id - data["file_id"] = self.get_output_file_id_from_unified_file_id(potential_file_id) + data["file_id"] = provider_file_id + data.update(stored_s3_bucket_kwargs(provider_file_id)) elif call_type == CallTypes.acreate_batch.value: input_file_id = cast(Optional[str], data.get("input_file_id")) if input_file_id: @@ -2894,7 +2898,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): await llm_router.afile_content( model=model_id, file_id=provider_file_id, - **cast(_RouterFileCallKwargs, data), + **cast( + _RouterFileCallKwargs, + { + **data, + **( + stored_s3_bucket_kwargs(provider_file_id) + if isinstance(provider_file_id, str) + else {} + ), + }, + ), ), ) except Exception as e: diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index ca072368061..cdae3de47c8 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -11,6 +11,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils import batch_utils as core_batch_utils +from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo @@ -551,7 +552,7 @@ async def fetch_batch_managed_file_content( "file_id": provider_output_file_id, "custom_llm_provider": custom_llm_provider, **cast( # cast-ok: preserve dynamic provider credentials without validation - FileContentCallOptions, credentials + FileContentCallOptions, {**credentials, **stored_s3_bucket_kwargs(provider_output_file_id)} ), } diff --git a/litellm/litellm_core_utils/cloud_storage_security.py b/litellm/litellm_core_utils/cloud_storage_security.py index 477775bb423..d78fbd58a57 100644 --- a/litellm/litellm_core_utils/cloud_storage_security.py +++ b/litellm/litellm_core_utils/cloud_storage_security.py @@ -17,6 +17,28 @@ BEDROCK_MANAGED_S3_PREFIXES: Final = ( BEDROCK_MANAGED_S3_OUTPUT_PREFIX, ) MANAGED_CLOUD_STORAGE_SCHEMES: Final = ("s3://", "gs://") +STORED_S3_BUCKET_PARAM: Final = "stored_s3_bucket_name" +REQUEST_S3_BUCKET_PARAMS: Final = frozenset({"s3_bucket_name", "s3_output_bucket_name"}) + + +def stored_s3_bucket_kwargs(stored_file_id: str) -> Mapping[str, str]: + """The bucket (and any prefix before LiteLLM's own path) of an ``s3://`` file id LiteLLM recorded itself, + as the param that lets Bedrock read it back. + + Only pass ids from LiteLLM's own records (managed file rows, provider batch responses), never one a caller sent. + """ + if not stored_file_id.startswith("s3://"): + return {} + bucket, _, object_key = stored_file_id.removeprefix("s3://").partition("/") + if not bucket: + return {} + segments: Final = object_key.split("/") + prefix_depth: Final = next( + (depth for depth in range(len(segments)) if "/".join(segments[depth:]).startswith(BEDROCK_MANAGED_S3_PREFIXES)), + 0, + ) + prefix: Final = "/".join(segments[:prefix_depth]) + return {STORED_S3_BUCKET_PARAM: f"{bucket}/{prefix}" if prefix else bucket} MAX_FILE_ID_DECODE_PASSES: Final = 8 diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 41c1baffab8..bbbbe9635c9 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -62,6 +62,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "bucket_name", "s3_bucket_name", "s3_output_bucket_name", + "stored_s3_bucket_name", "allow_legacy_cloud_file_ids", "s3_endpoint_url", "s3_region_name", diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 476d1395321..b19ad776051 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.cloud_storage_security import ( BEDROCK_MANAGED_S3_OUTPUT_PREFIX, BEDROCK_MANAGED_S3_PREFIXES, BEDROCK_MANAGED_S3_UPLOAD_PREFIX, + STORED_S3_BUCKET_PARAM, build_managed_cloud_object_name, encode_s3_object_key_for_url, sanitize_cloud_object_component, @@ -313,11 +314,19 @@ def get_configured_s3_bucket_names(litellm_params: Mapping[str, object]) -> tupl Resolve the S3 buckets a Bedrock file id may live in. Bedrock batch outputs land in ``s3_output_bucket_name`` when it differs from - the input bucket, so retrieval validates against both. + the input bucket, so retrieval validates against both, plus the bucket LiteLLM + recorded for a managed file uploaded or written to a request-chosen bucket. """ input_bucket: Final = _input_bucket_name(litellm_params) output_bucket: Final = _output_bucket_name(litellm_params) - buckets: Final = tuple(dict.fromkeys(bucket for bucket in (input_bucket, output_bucket) if bucket)) + stored_bucket: Final = litellm_params.get(STORED_S3_BUCKET_PARAM) + buckets: Final = tuple( + dict.fromkeys( + bucket + for bucket in (input_bucket, output_bucket, stored_bucket if isinstance(stored_bucket, str) else None) + if bucket + ) + ) if not buckets: raise ValueError(_S3_BUCKET_REQUIRED_ERROR) return buckets diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index d3978a58c43..715b98a441a 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -3,7 +3,7 @@ import os import re import sys from collections.abc import Collection, Iterator, Mapping, Sequence -from functools import lru_cache +from functools import lru_cache, partial from itertools import accumulate, chain, repeat from logging import Logger from types import MappingProxyType @@ -22,6 +22,7 @@ from litellm.constants import ( MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS, ) +from litellm.litellm_core_utils.cloud_storage_security import REQUEST_S3_BUCKET_PARAMS, STORED_S3_BUCKET_PARAM from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import ( @@ -444,6 +445,7 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( "s3_endpoint_url", "s3_bucket_name", "s3_output_bucket_name", + STORED_S3_BUCKET_PARAM, "allow_legacy_cloud_file_ids", "sagemaker_base_url", "deployment_url", @@ -494,6 +496,9 @@ _PROXY_ALLOWLIST_SETTING_BY_PARAM: Final[Mapping[str, _ClientSideAllowlistSettin } ) +TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY: Final = "allowed_s3_buckets" +_BUCKET_NAMES: Final = TypeAdapter(tuple[str, ...]) + class _ClientSideStorageAllowlist(BaseModel): model_config = ConfigDict(frozen=True) @@ -511,8 +516,15 @@ class _ClientSideStorageAllowlist(BaseModel): def _banned_param_error(param: str) -> ValueError: setting: Final = _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param) + team_hint: Final = ( + f" or in the team's `metadata.{TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY}`" + if param in REQUEST_S3_BUCKET_PARAMS + else "" + ) allowlist_hint: Final = ( - "" if setting is None else f" To allow specific values only, list them in `general_settings.{setting}`." + "" + if setting is None + else f" To allow specific values only, list them in `general_settings.{setting}`{team_hint}." ) return ValueError( f"Rejected Request: {param} is not allowed in request body. " @@ -532,11 +544,13 @@ def _check_banned_params( *, manages_deployments: bool = False, mcp_server_fields_allowed: bool = False, + defer_s3_buckets: bool = False, ) -> None: """Raise ``ValueError`` if ``body`` carries a banned param without admin opt-in. Shared between the root-level check and the nested-config check so a - new banned param only needs to be added in one place. + new banned param only needs to be added in one place. ``defer_s3_buckets`` lets + bucket params through for handlers that check them against the caller team's allowlist. """ wif_body: Final[Mapping[str, object]] = ( MappingProxyType({key: value for key, value in body.items() if key not in oauth_token_exchange_litellm_params}) @@ -557,6 +571,8 @@ def _check_banned_params( setting := _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param) ) is not None and _ClientSideStorageAllowlist.model_validate(general_settings).admits(setting, body[param]): continue + if defer_s3_buckets and param in REQUEST_S3_BUCKET_PARAMS: + continue if ( _allow_model_level_clientside_configurable_parameters( model=model, @@ -704,6 +720,7 @@ def is_request_body_safe( model: str, *, route: str | None = None, + defer_s3_buckets: bool = False, ) -> bool: """ Check if the request body is safe. @@ -733,33 +750,28 @@ def is_request_body_safe( if "model_list" in request_body: raise ValueError("Rejected Request: model_list is not allowed in the request body.") manages_deployments: Final = route_manages_deployments(route) - _check_banned_params( - request_body, - general_settings, - llm_router, - model, + check_banned_params: Final = partial( + _check_banned_params, + general_settings=general_settings, + llm_router=llm_router, + model=model, manages_deployments=manages_deployments, - mcp_server_fields_allowed=route_manages_mcp_servers(route), + defer_s3_buckets=defer_s3_buckets, ) + check_banned_params(request_body, mcp_server_fields_allowed=route_manages_mcp_servers(route)) for nested_key in _NESTED_CONFIG_KEYS: nested = _coerce_metadata_to_dict(request_body.get(nested_key)) if nested is not None: - _check_banned_params(nested, general_settings, llm_router, model, manages_deployments=manages_deployments) + check_banned_params(nested) for metadata_key in _NESTED_METADATA_KEYS: metadata = _coerce_metadata_to_dict(request_body.get(metadata_key)) if metadata is not None: - _check_banned_params(metadata, general_settings, llm_router, model, manages_deployments=manages_deployments) + check_banned_params(metadata) if any(isinstance(key, str) and key.startswith(f"{metadata_key}[") for key in request_body): - _check_banned_params( - extract_nested_form_metadata(form_data=request_body, prefix=f"{metadata_key}["), - general_settings, - llm_router, - model, - manages_deployments=manages_deployments, - ) + check_banned_params(extract_nested_form_metadata(form_data=request_body, prefix=f"{metadata_key}[")) for target in iter_request_fallback_targets(request_body): if isinstance(target, dict): - _check_banned_params(target, general_settings, llm_router, model, manages_deployments=manages_deployments) + check_banned_params(target) target_model = target.get("model") if isinstance(target_model, str): _reject_url_valued_fallback_target(target_model) @@ -772,9 +784,7 @@ def is_request_body_safe( _reject_request_credential_params(litellm_params, manages_deployments=manages_deployments) litellm_params_metadata: Final = _coerce_metadata_to_dict(litellm_params.get("metadata")) if litellm_params_metadata is not None: - _check_banned_params( - litellm_params_metadata, general_settings, llm_router, model, manages_deployments=manages_deployments - ) + check_banned_params(litellm_params_metadata) for user_config_params in _user_config_litellm_params(request_body): # pyright: ignore[reportUnknownArgumentType] # request_body is an untyped dict reject_api_key_secret_reference(user_config_params) return True @@ -846,6 +856,48 @@ def _coerce_metadata_to_dict(value: object) -> dict[str, object] | None: return None +def _team_s3_buckets(team_metadata: Mapping[str, object] | None) -> tuple[str, ...]: + if team_metadata is None: + return () + try: + return _BUCKET_NAMES.validate_python(team_metadata.get(TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY) or ()) + except ValidationError: + return () + + +_TEAM_S3_BUCKET_ROUTE_TEMPLATES: Final = frozenset( + {"/v1/files", "/files", "/{provider}/v1/files", "/v1/batches", "/batches", "/{provider}/v1/batches"} +) + + +def checks_team_s3_buckets_in_handler(request: Request) -> bool: + """File upload and batch create check request buckets against the caller team's allowlist themselves.""" + return get_request_route_template(request) in _TEAM_S3_BUCKET_ROUTE_TEMPLATES and request.method == "POST" + + +def enforce_team_s3_bucket_allowlist( + request_body: Mapping[str, object], + general_settings: Mapping[str, object], + llm_router: Router | None, + team_metadata: Mapping[str, object] | None, +) -> None: + """Re-run the request body check that auth relaxed for bucket params, now admitting the caller team's buckets.""" + proxy_buckets: Final = _ClientSideStorageAllowlist.model_validate(general_settings).allowed_client_side_s3_buckets + model: Final = request_body.get("model") + try: + is_request_body_safe( + request_body=dict(request_body), + general_settings={ + **general_settings, + "allowed_client_side_s3_buckets": (*(proxy_buckets or ()), *_team_s3_buckets(team_metadata)), + }, + llm_router=llm_router, + model=model if isinstance(model, str) else "", + ) + except ValueError as rejection: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=f"Authentication Error, {rejection}") + + async def pre_db_read_auth_checks( request: Request, request_data: dict, @@ -875,6 +927,7 @@ async def pre_db_read_auth_checks( llm_router=llm_router, model=request_data.get("model", ""), # [TODO] use model passed in url as well (azure openai routes) route=route, + defer_s3_buckets=checks_team_s3_buckets_in_handler(request), ) # Check 3. Check if IP address is allowed @@ -1522,6 +1575,9 @@ class BatchLimitRequest(Protocol): def model_fields_set(self) -> Collection[str]: ... +_ADMIN_ONLY_METADATA_KEYS: Final = (*ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS, TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY) + + def enforce_batch_limits_are_admin_only( data: BatchLimitRequest, existing_metadata: Mapping[str, object] | None, @@ -1532,7 +1588,8 @@ def enforce_batch_limits_are_admin_only( Every key in ``ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS`` caps what the holder may do with batches, so a holder-writable copy would let a caller lift their - own quota. Gated on the resulting value rather than on presence, so a form + own quota, and the S3 bucket list grants access to buckets with the proxy's + own credentials. Gated on the resulting value rather than on presence, so a form resending the stored value stays a no-op. """ if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value: @@ -1542,17 +1599,19 @@ def enforce_batch_limits_are_admin_only( (data.metadata or EMPTY_MAPPING) if "metadata" in data.model_fields_set else stored ) changed: Final = next( - (key for key in ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS if requested.get(key) != stored.get(key)), + (key for key in _ADMIN_ONLY_METADATA_KEYS if requested.get(key) != stored.get(key)), None, ) if changed is None: return + reason: Final = ( + "It lets the holder read and write those buckets with the proxy's AWS credentials." + if changed == TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY + else "It limits what the holder can do with batches, so the holder cannot raise it." + ) raise HTTPException( status_code=403, - detail={ - "error": f"Only proxy admins can set {changed} on a {entity}. " - "It limits what the holder can do with batches, so the holder cannot raise it." - }, + detail={"error": f"Only proxy admins can set {changed} on a {entity}. {reason}"}, ) diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 512a9166e31..00fa7f3c33e 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -12,6 +12,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Protocol, cast from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -20,6 +21,7 @@ from litellm.litellm_core_utils.core_helpers import attribute_or_none, has_attri from litellm.litellm_core_utils.hidden_params import get_hidden_params, set_hidden_param from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config from litellm.proxy._types import * +from litellm.proxy.auth.auth_utils import enforce_team_s3_bucket_allowlist from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit from litellm.proxy.batches_endpoints.litellm_executed_batches import ( @@ -81,6 +83,7 @@ if TYPE_CHECKING: from prisma.models import LiteLLM_ManagedObjectTable router: Final = APIRouter() +_REQUEST_BODY_ADAPTER: Final = TypeAdapter(Mapping[str, object]) class _UserBatchLister(Protocol): @@ -285,6 +288,7 @@ async def create_batch( """ from litellm.proxy.proxy_server import ( general_settings, + general_settings_view, llm_router, proxy_config, proxy_logging_obj, @@ -294,6 +298,12 @@ async def create_batch( data: dict = {} try: data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line + enforce_team_s3_bucket_allowlist( + request_body=_REQUEST_BODY_ADAPTER.validate_python(data), + general_settings=general_settings_view(), + llm_router=llm_router, + team_metadata=user_api_key_dict.team_metadata, + ) _raise_when_routing_headers_disagree(request, data.get("model")) verbose_proxy_logger.debug( "Request received by LiteLLM:\n%s", @@ -384,6 +394,7 @@ async def create_batch( prepare_data_with_credentials( data=_create_batch_data, credentials=credentials, + keep_request_buckets=True, ) # Create batch using model credentials @@ -488,6 +499,7 @@ async def create_batch( prepare_data_with_credentials( data=_create_batch_data, credentials=credentials, + keep_request_buckets=True, ) # Create batch using model credentials @@ -506,6 +518,7 @@ async def create_batch( llm_router=llm_router, user_api_key_dict=user_api_key_dict, custom_llm_provider=custom_llm_provider, + keep_request_buckets=True, ) _raise_not_found_when_openai_fallback_unservable( requested_provider=requested_provider, diff --git a/litellm/proxy/management_endpoints/team_admin_field_permissions.py b/litellm/proxy/management_endpoints/team_admin_field_permissions.py index 0ece14eb90a..08a84ca91c3 100644 --- a/litellm/proxy/management_endpoints/team_admin_field_permissions.py +++ b/litellm/proxy/management_endpoints/team_admin_field_permissions.py @@ -31,6 +31,7 @@ from litellm.proxy._types import ( UpdateKeyRequest, UpdateTeamRequest, ) +from litellm.proxy.auth.auth_utils import TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY from litellm.types.proxy.management_endpoints.scim_v2 import ( SCIM_MANAGED_TEAM_METADATA_KEY, SCIM_TEAM_DATA_METADATA_KEY, @@ -97,6 +98,7 @@ PROXY_CONTROLLED_TEAM_METADATA_KEYS: Final[frozenset[str]] = frozenset( "priority", "zguard_policy_id", "allowed_tools", + TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY, "allow_client_mock_response", "allow_client_message_redaction_opt_out", "allow_client_pricing_override", diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 9a044f1dee3..e5e68a930c9 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -24,6 +24,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.constants import MAX_FILE_LIST_LIMIT from litellm.litellm_core_utils.batch_utils import batch_cost_is_final +from litellm.litellm_core_utils.cloud_storage_security import REQUEST_S3_BUCKET_PARAMS from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.base_llm.managed_resources.utils import is_base64_encoded_unified_id from litellm.proxy._types import ProxyException @@ -884,6 +885,8 @@ def apply_team_provider_credentials( llm_router: Optional["Router"], user_api_key_dict: "UserAPIKeyAuth", custom_llm_provider: str, + *, + keep_request_buckets: bool = False, ) -> None: """ Resolve credentials for a provider-only request (no model pinned) via @@ -901,7 +904,11 @@ def apply_team_provider_credentials( ) if credentials is None: return - prepare_data_with_credentials(data=data, credentials={**credentials, **explicit_request_credentials(data)}) + prepare_data_with_credentials( + data=data, + credentials={**credentials, **explicit_request_credentials(data)}, + keep_request_buckets=keep_request_buckets, + ) def add_deployment_model_info( @@ -925,10 +932,16 @@ def add_deployment_model_info( } +_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_NO_REQUEST_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) + + def prepare_data_with_credentials( data: dict, credentials: dict, file_id: str | None = None, + *, + keep_request_buckets: bool = False, ) -> None: """ Update data dictionary with model credentials (in-place). @@ -937,8 +950,15 @@ def prepare_data_with_credentials( data: Data dictionary to update credentials: Credentials from router file_id: Optional original file_id to set (for decoded file IDs) + keep_request_buckets: Keep an S3 bucket the request already carries over the deployment's. Only + file upload and batch create set it, after checking the bucket against the caller's allowlist. + Reads keep the deployment's bucket so a request cannot widen which buckets a file id may point at. """ - data.update(credentials) + request_fields: Final = _REQUEST_FIELDS.validate_python(data) if keep_request_buckets else _NO_REQUEST_FIELDS + request_buckets: Final = MappingProxyType( + {key: request_fields[key] for key in REQUEST_S3_BUCKET_PARAMS if key in request_fields} + ) + data.update({**credentials, **request_buckets}) data.pop("custom_llm_provider", None) if file_id is not None: diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 5ce0ed8b493..ca1470bc94e 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -40,6 +40,7 @@ from litellm.llms.base_llm.files.litellm_db_storage_backend import LITELLM_DB_ST from litellm.llms.base_llm.files.transformation import BaseFileEndpoints, FileListModelScope from litellm.llms.base_llm.managed_resources.isolation import build_list_page from litellm.proxy._types import * +from litellm.proxy.auth.auth_utils import enforce_team_s3_bucket_allowlist from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.batches_endpoints.litellm_executed_batches import ( litellm_executed_provider_of, @@ -181,6 +182,7 @@ async def _litellm_executed_batch_input_model( _MAX_BATCH_FILE_SIZE_MB_ADAPTER: Final = TypeAdapter(int | None) _LISTED_FILES_ADAPTER: Final = TypeAdapter(list[OpenAIFileObject]) +_REQUEST_BODY_ADAPTER: Final = TypeAdapter(Mapping[str, object]) class UploadedFileInfo(TypedDict): @@ -465,6 +467,7 @@ async def route_create_file( prepare_data_with_credentials( data=_create_file_request, credentials={**credentials, **deployment_drop_params(llm_router, model, _create_file_request)}, + keep_request_buckets=True, ) # Create the file with model credentials @@ -528,6 +531,7 @@ async def route_create_file( llm_router=llm_router, user_api_key_dict=user_api_key_dict, custom_llm_provider=custom_llm_provider, + keep_request_buckets=True, ) # get configs for custom_llm_provider llm_provider_config: Final = get_files_provider_config(custom_llm_provider=custom_llm_provider) @@ -630,6 +634,12 @@ async def create_file( # Extract file creation parameters using utility function request_body: Final = await read_request_body(request=request) or {} + enforce_team_s3_bucket_allowlist( + request_body=_REQUEST_BODY_ADAPTER.validate_python(request_body), + general_settings=general_settings_view(), + llm_router=llm_router, + team_metadata=user_api_key_dict.team_metadata, + ) file_params: Final = await extract_file_creation_params( request=request, request_body=request_body, @@ -707,6 +717,8 @@ async def create_file( # Add litellm_metadata to data if provided (from form field) if extracted_litellm_metadata is not None: data["litellm_metadata"] = extracted_litellm_metadata + if isinstance(s3_bucket_name := form_data_raw.get("s3_bucket_name"), str) and s3_bucket_name: + data["s3_bucket_name"] = s3_bucket_name if expires_after_anchor is not None or expires_after_seconds_str is not None: if expires_after_anchor is None or expires_after_seconds_str is None: diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index 896fc9d975a..bef78554a36 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -1072,6 +1072,28 @@ def respx_interceptable_httpx_client(monkeypatch): litellm.in_memory_llm_clients_cache.flush_cache() +@pytest.mark.asyncio +@respx.mock +async def test_bedrock_output_written_to_a_request_bucket_is_read_from_that_bucket(respx_interceptable_httpx_client): + route: Final = respx.get( + "https://s3.us-west-2.amazonaws.com/team-a-bucket/litellm-batch-outputs/job-1/input.jsonl.out" + ).mock(return_value=httpx.Response(200, content=b'{"recordId": "1"}\n')) + + file_content: Final = await bu.fetch_batch_managed_file_content( + "s3://team-a-bucket/litellm-batch-outputs/job-1/input.jsonl.out", + custom_llm_provider="bedrock", # pyright: ignore[reportArgumentType] # the cost job passes bedrock at runtime + litellm_params={ + "s3_bucket_name": "model-bucket", + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "secret", + "aws_region_name": "us-west-2", + }, + ) + + assert route.call_count == 1 + assert file_content == b'{"recordId": "1"}\n' + + @pytest.mark.asyncio @respx.mock async def test_output_file_content_vertex_managed_uri_accepted_by_real_validation(respx_interceptable_httpx_client): diff --git a/tests/unit/enterprise/proxy/hooks/test_managed_files.py b/tests/unit/enterprise/proxy/hooks/test_managed_files.py index 8c8e970b80d..d8a647fda27 100644 --- a/tests/unit/enterprise/proxy/hooks/test_managed_files.py +++ b/tests/unit/enterprise/proxy/hooks/test_managed_files.py @@ -3520,6 +3520,87 @@ async def test_afile_retrieve_fetches_from_provider_when_file_object_none(): assert managed_file_table.upsert_calls == [] +TEAM_BUCKET_FILE_ID: Final = "s3://team-a-bucket/litellm-batch-outputs/job-1/input.jsonl.out" + + +def _managed_files_holding_a_team_bucket_file() -> PROXY_LiteLLMManagedFiles: + internal_usage_cache: Final = MagicMock() + internal_usage_cache.async_get_cache = AsyncMock( + return_value={ + "unified_file_id": _managed_output_file_id(TEAM_BUCKET_FILE_ID), + "model_mappings": {"model-1": TEAM_BUCKET_FILE_ID}, + "flat_model_file_ids": [TEAM_BUCKET_FILE_ID], + "file_object": None, + "created_by": "user-a", + "updated_by": "user-a", + } + ) + internal_usage_cache.async_set_cache = AsyncMock() + prisma_client: Final = AsyncMock() + prisma_client.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None) + managed_files: Final = PROXY_LiteLLMManagedFiles( + internal_usage_cache=internal_usage_cache, prisma_client=prisma_client + ) + managed_files.delete_unified_file_id = AsyncMock(return_value=None) + return managed_files + + +@pytest.mark.asyncio +async def test_deleting_a_managed_file_in_a_team_bucket_names_that_bucket_to_the_provider(): + router: Final = MagicMock() + router.afile_delete = AsyncMock(return_value=MagicMock()) + + await _managed_files_holding_a_team_bucket_file().afile_delete( + file_id=_managed_output_file_id(TEAM_BUCKET_FILE_ID), litellm_parent_otel_span=None, llm_router=router + ) + + assert router.afile_delete.call_args.kwargs["file_id"] == TEAM_BUCKET_FILE_ID + assert router.afile_delete.call_args.kwargs["stored_s3_bucket_name"] == "team-a-bucket" + + +@pytest.mark.asyncio +async def test_reading_a_managed_file_in_a_team_bucket_names_that_bucket_to_the_provider(): + router: Final = MagicMock() + router.afile_content = AsyncMock(return_value=MagicMock()) + + await _managed_files_holding_a_team_bucket_file().afile_content( + file_id=_managed_output_file_id(TEAM_BUCKET_FILE_ID), litellm_parent_otel_span=None, llm_router=router + ) + + assert router.afile_content.call_args.kwargs["file_id"] == TEAM_BUCKET_FILE_ID + assert router.afile_content.call_args.kwargs["stored_s3_bucket_name"] == "team-a-bucket" + + +@pytest.mark.asyncio +async def test_refreshing_a_managed_file_in_a_team_bucket_names_that_bucket_to_the_provider(): + from litellm.types.llms.openai import OpenAIFileObject + + stored_file: Final = LiteLLM_ManagedFileTable( + unified_file_id="unified-team-file", + file_object=None, + model_mappings={"model-1": TEAM_BUCKET_FILE_ID}, + flat_model_file_ids=[TEAM_BUCKET_FILE_ID], + created_by="user-a", + ) + proxy_managed_files, _ = _managed_files_with_fake_prisma(stored_file) + router: Final = MagicMock() + router.registry.return_value.credentials = MagicMock( + return_value={"custom_llm_provider": "bedrock", "s3_bucket_name": "model-bucket"} + ) + provider_file: Final = OpenAIFileObject( + id=TEAM_BUCKET_FILE_ID, object="file", bytes=1, created_at=1, filename="input.jsonl.out", purpose="batch_output" + ) + + with patch("litellm.afile_retrieve", new_callable=AsyncMock, return_value=provider_file) as provider_retrieve: + refreshed: Final = await proxy_managed_files.afile_retrieve( + file_id="unified-team-file", litellm_parent_otel_span=None, llm_router=router + ) + + assert (refreshed.id, refreshed.filename) == ("unified-team-file", "input.jsonl.out") + assert provider_retrieve.call_args.kwargs["file_id"] == TEAM_BUCKET_FILE_ID + assert provider_retrieve.call_args.kwargs["stored_s3_bucket_name"] == "team-a-bucket" + + @pytest.mark.asyncio async def test_afile_retrieve_raises_error_when_no_router_and_file_object_none(): """ @@ -5665,6 +5746,44 @@ async def test_provider_format_file_without_ownership_row_stays_accessible(): assert result["file_id"] == RAW_PROVIDER_FILE_ID +def _managed_output_file_id(provider_file_id: str) -> str: + unified: Final = ( + "litellm_proxy:application/json;unified_id,u-1;target_model_names,claude-batch;" + f"llm_output_file_id,{provider_file_id};llm_output_file_model_id,model-1" + ) + return base64.urlsafe_b64encode(unified.encode()).decode().rstrip("=") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "provider_file_id,stored_bucket", + [("s3://team-a-bucket/litellm-batch-outputs/job-1/input.jsonl.out", "team-a-bucket"), ("file-openai-1", None)], +) +async def test_owned_managed_file_content_is_read_from_the_bucket_litellm_recorded( + provider_file_id: str, stored_bucket: str | None +): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedfiletable.find_first.return_value = _owned_record( + created_by="user_a", team_id="team_a" + ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(MagicMock(), prisma_client=prisma_client) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(user_id="user_a", team_id="team_a", parent_otel_span=MagicMock()), + cache=MagicMock(), + data={"file_id": _managed_output_file_id(provider_file_id)}, + call_type="afile_content", + ) + + assert (result["model"], result["file_id"], result.get("stored_s3_bucket_name")) == ( + "model-1", + provider_file_id, + stored_bucket, + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("batch_id", [MODEL_ENCODED_BATCH_ID, RAW_PROVIDER_BATCH_ID]) async def test_post_call_batch_create_stores_ownership_row(batch_id): diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index fdd6d5e61bc..f56910ec103 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -2928,6 +2928,88 @@ class TestBedrockFileContentTransformation: assert response.content == b'{"recordId": "x"}' +class TestBedrockFileContentFromTheStoredBucket: + TEAM_OUTPUT_URI: Final = "s3://team-a-bucket/litellm-batch-outputs/job-1/input.jsonl.out" + TEAM_OUTPUT_URL: Final = "https://s3.us-west-2.amazonaws.com/team-a-bucket/litellm-batch-outputs/job-1/input.jsonl.out" + CREDENTIALS: Final = { + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "secret", + "aws_region_name": "us-west-2", + } + + def test_file_in_another_bucket_is_refused_without_the_stored_bucket(self) -> None: + with pytest.raises(Exception, match="does not match the configured storage bucket"): + litellm.file_content( + file_id=self.TEAM_OUTPUT_URI, + custom_llm_provider="bedrock", + s3_bucket_name="model-bucket", + **self.CREDENTIALS, + ) + + def test_file_in_the_stored_bucket_is_read_from_that_bucket(self) -> None: + import httpx + import respx + + with respx.mock: + route: Final = respx.get(self.TEAM_OUTPUT_URL).mock(return_value=httpx.Response(200, content=b"{}")) + response: Final = litellm.file_content( + file_id=self.TEAM_OUTPUT_URI, + custom_llm_provider="bedrock", + s3_bucket_name="model-bucket", + stored_s3_bucket_name="team-a-bucket", + **self.CREDENTIALS, + ) + + assert route.call_count == 1 + assert response.content == b"{}" + + @pytest.mark.parametrize( + "stored_file_id", + [ + "s3://team-a-bucket/tenant/litellm-batch-outputs/job-1/input.jsonl.out", + "s3://team-a-bucket/tenant/litellm-bedrock-files-nova-1.jsonl", + "s3://team-a-bucket/tenant/deeper/litellm-bedrock-files/nova-1.jsonl", + ], + ) + def test_file_written_under_a_prefixed_request_bucket_is_read_back_from_it(self, stored_file_id: str) -> None: + import httpx + import respx + + from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs + + url: Final = f"https://s3.us-west-2.amazonaws.com/{stored_file_id.removeprefix('s3://')}" + with respx.mock: + route: Final = respx.get(url).mock(return_value=httpx.Response(200, content=b"{}")) + litellm.file_content( + file_id=stored_file_id, + custom_llm_provider="bedrock", + s3_bucket_name="model-bucket", + **stored_s3_bucket_kwargs(stored_file_id), + **self.CREDENTIALS, + ) + + assert route.call_count == 1 + + def test_file_under_the_model_bucket_prefix_keeps_its_url_when_a_stored_bucket_is_passed(self) -> None: + import httpx + import respx + + prefixed_url: Final = ( + "https://s3.us-west-2.amazonaws.com/model-bucket/tenant/litellm-batch-outputs/job-1/input.jsonl.out" + ) + with respx.mock: + route: Final = respx.get(prefixed_url).mock(return_value=httpx.Response(200, content=b"{}")) + litellm.file_content( + file_id="s3://model-bucket/tenant/litellm-batch-outputs/job-1/input.jsonl.out", + custom_llm_provider="bedrock", + s3_bucket_name="model-bucket/tenant", + stored_s3_bucket_name="model-bucket", + **self.CREDENTIALS, + ) + + assert route.call_count == 1 + + class TestBedrockFilesS3SignatureEncoding: """ S3 rebuilds the canonical request from the wire path with single percent-encoding, diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 8651d522ac8..82b80ad8da9 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -4,7 +4,8 @@ Unit tests for auth_utils functions related to rate limiting and customer ID ext import base64 import logging -from collections.abc import Callable +from collections.abc import Callable, Mapping +from types import SimpleNamespace from typing import Final, Optional from unittest.mock import MagicMock, patch @@ -12,7 +13,7 @@ import pytest from fastapi import HTTPException, Request import litellm -from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( _allow_model_level_clientside_configurable_parameters, _get_customer_id_from_standard_headers, @@ -33,7 +34,9 @@ from litellm.proxy.auth.auth_utils import ( get_project_model_rpm_limit, get_project_model_tpm_limit, get_request_route_template, + enforce_team_s3_bucket_allowlist, is_request_body_safe, + pre_db_read_auth_checks, ) from litellm.router import Router from litellm.types.utils import oauth_token_exchange_litellm_params, server_owned_wif_litellm_params @@ -5429,6 +5432,189 @@ def test_other_blocked_fields_do_not_point_at_the_storage_allowlist() -> None: assert "allowed_client_side" not in str(error.value) +_TEAM_A_METADATA: Final = {"allowed_s3_buckets": ["team-a-bucket"]} + + +def _enforce_for_team( + body: Mapping[str, object], team_metadata: Mapping[str, object] | None, **general_settings: object +) -> None: + enforce_team_s3_bucket_allowlist( + request_body=body, + general_settings=general_settings, + llm_router=None, + team_metadata=team_metadata, + ) + + +def _rejection_for_team(body: Mapping[str, object], team_metadata: Mapping[str, object] | None) -> HTTPException: + with pytest.raises(HTTPException) as rejection: + _enforce_for_team(body, team_metadata) + assert rejection.value.status_code == 401, rejection.value.detail + return rejection.value + + +@pytest.mark.parametrize("field", ["s3_bucket_name", "s3_output_bucket_name"]) +def test_team_allowlist_admits_its_own_bucket_and_rejects_another_teams(field: str) -> None: + _enforce_for_team({field: "team-a-bucket"}, _TEAM_A_METADATA) + rejection: Final = _rejection_for_team({field: "team-b-bucket"}, _TEAM_A_METADATA) + assert f"{field} is not allowed in request body" in str(rejection.detail) + assert "metadata.allowed_s3_buckets" in str(rejection.detail) + + +@pytest.mark.parametrize( + "team_metadata", + [None, {}, {"allowed_s3_buckets": None}, {"allowed_s3_buckets": "team-a-bucket"}, {"allowed_s3_buckets": [1]}], +) +def test_team_without_a_usable_allowlist_cannot_pick_a_bucket(team_metadata: Mapping[str, object] | None) -> None: + _rejection_for_team({"s3_bucket_name": "team-a-bucket"}, team_metadata) + + +def _team_bucket_admitted( + body: Mapping[str, object], team_metadata: Mapping[str, object] | None, **general_settings: object +) -> bool: + try: + _enforce_for_team(body, team_metadata, **general_settings) + except HTTPException: + return False + return True + + +def test_proxy_wide_allowlist_still_applies_alongside_the_team_allowlist() -> None: + assert _team_bucket_admitted( + {"s3_bucket_name": "team-a-bucket", "s3_output_bucket_name": "shared-bucket"}, + _TEAM_A_METADATA, + allowed_client_side_s3_buckets=["shared-bucket"], + ) + assert not _team_bucket_admitted({"s3_output_bucket_name": "shared-bucket"}, _TEAM_A_METADATA) + + +def test_team_allowlist_is_checked_for_buckets_nested_in_the_body() -> None: + _enforce_for_team({"extra_body": {"s3_bucket_name": "team-a-bucket"}}, _TEAM_A_METADATA) + _rejection_for_team({"litellm_metadata": {"s3_bucket_name": "team-b-bucket"}}, _TEAM_A_METADATA) + + +def test_team_allowlist_admits_no_other_blocked_field() -> None: + rejection: Final = _rejection_for_team({"s3_endpoint_url": "team-a-bucket"}, _TEAM_A_METADATA) + assert "s3_endpoint_url" in str(rejection.detail) + + +def _routed_request(method: str, route_template: str | None, path: str) -> Request: + scope: Final[dict[str, object]] = {"type": "http", "method": method, "path": path, "headers": [], "query_string": b""} + return Request(scope if route_template is None else {**scope, "route": SimpleNamespace(path=route_template)}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route_template,path", + [ + ("/v1/files", "/v1/files"), + ("/files", "/files"), + ("/{provider}/v1/files", "/bedrock/v1/files"), + ("/v1/batches", "/v1/batches"), + ("/batches", "/batches"), + ("/{provider}/v1/batches", "/bedrock/v1/batches"), + ], +) +async def test_auth_leaves_request_buckets_to_the_upload_and_batch_create_handlers( + route_template: str, path: str +) -> None: + assert await _auth_admits( + _routed_request("POST", route_template, path), + {"s3_bucket_name": "team-b-bucket", "s3_output_bucket_name": "team-b-bucket"}, + path, + ) + + +async def _auth_admits(request: Request, request_data: dict[str, object], route: str) -> bool: + try: + await pre_db_read_auth_checks(request=request, request_data=request_data, route=route) + except ValueError: + return False + return True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,route_template,path", + [ + ("GET", "/v1/files", "/v1/files"), + ("POST", "/v1/files/{file_id}/content", "/v1/files/file-1/content"), + ("POST", "/v1/chat/completions", "/v1/chat/completions"), + ("POST", None, "/v1/batches"), + ], +) +async def test_auth_still_rejects_request_buckets_on_every_other_route( + method: str, route_template: str | None, path: str +) -> None: + with pytest.raises(ValueError, match="s3_bucket_name is not allowed"): + await pre_db_read_auth_checks( + request=_routed_request(method, route_template, path), + request_data={"s3_bucket_name": "team-b-bucket"}, + route=path, + ) + + +@pytest.mark.asyncio +async def test_a_caller_can_never_send_the_stored_bucket_param() -> None: + with pytest.raises(ValueError, match="stored_s3_bucket_name is not allowed"): + await pre_db_read_auth_checks( + request=_routed_request("POST", "/v1/batches", "/v1/batches"), + request_data={"stored_s3_bucket_name": "team-b-bucket"}, + route="/v1/batches", + ) + rejection: Final = _rejection_for_team( + {"stored_s3_bucket_name": "team-a-bucket"}, {"allowed_s3_buckets": ["team-a-bucket"]} + ) + assert "stored_s3_bucket_name" in str(rejection.detail) + with pytest.raises(ValueError, match="stored_s3_bucket_name"): + _batches_body_is_safe({"stored_s3_bucket_name": "team-a-bucket"}, _BUCKET_ALLOWLIST) + + +@pytest.mark.asyncio +async def test_auth_still_rejects_other_blocked_fields_on_the_upload_and_batch_create_routes() -> None: + with pytest.raises(ValueError, match="api_base"): + await pre_db_read_auth_checks( + request=_routed_request("POST", "/v1/batches", "/v1/batches"), + request_data={"s3_bucket_name": "team-b-bucket", "api_base": "https://attacker.example"}, + route="/v1/batches", + ) + + +@pytest.mark.parametrize( + "role,stored,sent,refused", + [ + (LitellmUserRoles.PROXY_ADMIN, None, ["any-bucket"], False), + (LitellmUserRoles.INTERNAL_USER, None, ["any-bucket"], True), + (LitellmUserRoles.INTERNAL_USER, ["team-a-bucket"], ["team-a-bucket", "other"], True), + (LitellmUserRoles.INTERNAL_USER, ["team-a-bucket"], None, True), + (LitellmUserRoles.INTERNAL_USER, ["team-a-bucket"], ["team-a-bucket"], False), + ], +) +def test_only_proxy_admins_change_a_teams_s3_bucket_list( + role: LitellmUserRoles, stored: list[str] | None, sent: list[str] | None, refused: bool +) -> None: + from litellm.proxy._types import UpdateTeamRequest + from litellm.proxy.auth.auth_utils import enforce_batch_limits_are_admin_only + + existing: Final = None if stored is None else {"allowed_s3_buckets": stored, "project": "x"} + request: Final = UpdateTeamRequest( + team_id="t1", metadata={"project": "x"} if sent is None else {"allowed_s3_buckets": sent, "project": "x"} + ) + caller: Final = UserAPIKeyAuth(user_role=role, user_id="u1") + + if not refused: + enforce_batch_limits_are_admin_only( + data=request, existing_metadata=existing, user_api_key_dict=caller, entity="team" + ) + return + with pytest.raises(HTTPException) as rejection: + enforce_batch_limits_are_admin_only( + data=request, existing_metadata=existing, user_api_key_dict=caller, entity="team" + ) + assert rejection.value.status_code == 403 + assert "Only proxy admins can set allowed_s3_buckets on a team." in str(rejection.value.detail) + + def _request_with_content_length(content_length: int) -> Request: return Request( { diff --git a/tests/unit/proxy/batches_endpoints/test_endpoints.py b/tests/unit/proxy/batches_endpoints/test_endpoints.py index b5da6f31eb4..cee8a129172 100644 --- a/tests/unit/proxy/batches_endpoints/test_endpoints.py +++ b/tests/unit/proxy/batches_endpoints/test_endpoints.py @@ -85,6 +85,12 @@ CREDS: Dict[str, Dict[str, str]] = { "api_base": "http://vllm.test/v1", "model": "hosted_vllm/qwen", }, + "bedrock-batch": { + "custom_llm_provider": "bedrock", + "model": "bedrock/us.amazon.nova-lite-v1:0", + "s3_bucket_name": "model-bucket", + "s3_output_bucket_name": "model-bucket", + }, } # A real model-encoded file id: decodes to "azure/gpt-4o", strips to "file-original123". @@ -426,6 +432,83 @@ async def test_create__model_from_body(harness): assert resp.id == encode_file_id_with_model("batch-provider-id", "vertex-model", id_type="batch") +_TEAM_A_KEY: Final = UserAPIKeyAuth( + api_key="sk-team-a", team_id="team-a", team_metadata={"allowed_s3_buckets": ["team-a-bucket"]} +) + + +def _batch_body_with_output_bucket(bucket: str) -> dict[str, object]: + return { + "input_file_id": "file-plain", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "model": "vertex-model", + "s3_output_bucket_name": bucket, + } + + +@pytest.mark.asyncio +async def test_create__output_bucket_on_the_team_allowlist_reaches_the_provider(harness): + set_body(harness, _batch_body_with_output_bucket("team-a-bucket")) + + await call_create(harness, user=_TEAM_A_KEY) + + assert harness.acreate_kwargs()["s3_output_bucket_name"] == "team-a-bucket" + + +@pytest.mark.asyncio +async def test_create__team_with_an_allowlist_and_no_request_bucket_leaves_the_bucket_to_the_model(harness): + set_body( + harness, + { + "input_file_id": "file-plain", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "model": "vertex-model", + }, + ) + + await call_create(harness, user=_TEAM_A_KEY) + + assert "s3_output_bucket_name" not in harness.acreate_kwargs() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_bucket,expected_bucket", [("team-a-bucket", "team-a-bucket"), (None, "model-bucket")] +) +@pytest.mark.parametrize( + "routing", + [ + {"input_file_id": "file-plain", "model": "bedrock-batch"}, + {"input_file_id": encode_file_id_with_model("file-plain", "bedrock-batch")}, + ], + ids=["model_in_body", "model_encoded_file_id"], +) +async def test_create__model_routed_batch_keeps_the_team_bucket_over_the_deployment_bucket( + harness, routing: dict[str, str], request_bucket: str | None, expected_bucket: str +): + body: Final = {"endpoint": "/v1/chat/completions", "completion_window": "24h", **routing} + set_body(harness, body if request_bucket is None else {**body, "s3_output_bucket_name": request_bucket}) + + await call_create(harness, user=_TEAM_A_KEY) + + assert harness.acreate_kwargs()["s3_output_bucket_name"] == expected_bucket + + +@pytest.mark.asyncio +async def test_create__output_bucket_off_the_team_allowlist_is_rejected_before_the_provider(harness): + set_body(harness, _batch_body_with_output_bucket("team-b-bucket")) + + with pytest.raises(ProxyException) as rejection: + await call_create(harness, user=_TEAM_A_KEY) + + assert str(rejection.value.code) == "401" + assert "s3_output_bucket_name is not allowed in request body" in rejection.value.message + harness.litellm_acreate.assert_not_called() + harness.router_acreate.assert_not_called() + + @pytest.mark.asyncio async def test_create__model_from_header(harness): set_body( diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 09c76b2dd65..79524cfeda9 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -2297,6 +2297,10 @@ class TestManagedOutputFileIdEncodesPublicModelGroup: return job async def _run(self, job: MagicMock) -> str: + output_file_id, _ = await self._run_capturing(job, self._RAW_OUTPUT_FILE_ID) + return output_file_id + + async def _run_capturing(self, job: MagicMock, raw_output_file_id: str) -> tuple[str, Mapping[str, object]]: from litellm_enterprise.proxy.common_utils.check_batch_cost import ( CheckBatchCost, ) @@ -2342,7 +2346,7 @@ class TestManagedOutputFileIdEncodesPublicModelGroup: object="batch", status="completed", ) - response.output_file_id = self._RAW_OUTPUT_FILE_ID + response.output_file_id = raw_output_file_id file_content = MagicMock() file_content.content = b'{"id":"req-1"}' @@ -2352,7 +2356,7 @@ class TestManagedOutputFileIdEncodesPublicModelGroup: "litellm.files.main.afile_content", new_callable=AsyncMock, return_value=file_content, - ), + ) as provider_file_content, patch( "litellm.batches.batch_utils.get_file_content_as_dictionary", return_value=[{"id": "req-1"}], @@ -2376,7 +2380,17 @@ class TestManagedOutputFileIdEncodesPublicModelGroup: prom_logger=None, ) - return response.output_file_id + return response.output_file_id, provider_file_content.call_args.kwargs + + @pytest.mark.asyncio + async def test_output_written_to_a_team_bucket_is_billed_from_that_bucket(self): + team_output: Final = "s3://team-a-bucket/litellm-batch-outputs/job-1/input.jsonl.out" + + _, download = await self._run_capturing( + self._job(self._managed_input_file_id(self._PUBLIC_MODEL_GROUP)), team_output + ) + + assert (download["file_id"], download["stored_s3_bucket_name"]) == (team_output, "team-a-bucket") @pytest.mark.asyncio async def test_target_model_names_comes_from_input_file_not_provider_model(self): diff --git a/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py b/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py index 4af57a83395..88b02ed1aa1 100644 --- a/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py +++ b/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py @@ -322,6 +322,7 @@ _PROXY_READ_TEAM_METADATA_KEYS: Final = ( "allowed_vector_store_indexes", "enforced_file_expires_after", "enforced_batch_output_expires_after", + "allowed_s3_buckets", "require_trace_id", "allowed_passthrough_routes", "denied_passthrough_routes", @@ -384,7 +385,9 @@ class TestTeamMetadataEdits: verdict: Final = _metadata_verdict(_STORED_METADATA, {**_STORED_METADATA, key: "set-by-team-admin"}) assert verdict == TeamAdminMetadataKeyProxyControlled(key=key) - @pytest.mark.parametrize("key", ["opted_out_global_guardrails", "callback_settings", "priority", "guardrails"]) + @pytest.mark.parametrize( + "key", ["opted_out_global_guardrails", "callback_settings", "priority", "guardrails", "allowed_s3_buckets"] + ) def test_changing_or_removing_a_stored_proxy_controlled_key_names_it(self, key: str): stored: Final = {**_STORED_METADATA, key: ["stored"]} changed: Final = _metadata_verdict(stored, {**stored, key: ["changed"]}) diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 7e54f0893c0..83a121da21a 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -13062,6 +13062,67 @@ async def test_update_team_blocks_org_admin_require_trace_id_change(mock_db_clie assert "Only proxy admins can set `require_trace_id` on a team." in str(exc.value.message) +@pytest.mark.asyncio +async def test_new_team_blocks_org_admin_granting_s3_buckets(mock_db_client: MagicMock) -> None: + mock_db_client.db.litellm_teamtable.count = AsyncMock(return_value=0) + + with ( + _org_admins(("u-team-admin", "org-1")), + patch( + "litellm.proxy.management_endpoints.team_endpoints._should_auto_add_team_creator", + return_value=False, + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints.get_org_object", + new=AsyncMock(return_value=MagicMock()), + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._check_org_team_limits", + new=AsyncMock(return_value=None), + ), + ): + exc: Final[pytest.ExceptionInfo[ProxyException]] + with pytest.raises(ProxyException) as exc: + await new_team( + data=NewTeamRequest( + team_alias="t", + organization_id="org-1", + metadata={"allowed_s3_buckets": ["another-orgs-bucket"]}, + ), + http_request=MagicMock(spec=Request), + user_api_key_dict=_non_admin_auth(), + ) + + assert str(exc.value.code) == "403" + assert "Only proxy admins can set allowed_s3_buckets on a team." in str(exc.value.message) + + +@pytest.mark.asyncio +async def test_update_team_blocks_org_admin_widening_s3_buckets(mock_db_client: MagicMock) -> None: + existing: Final[MagicMock] = MagicMock() + existing.metadata = {"allowed_s3_buckets": ["team-a-bucket"]} + existing.model_dump.return_value = { + "team_id": "t1", + "organization_id": "org-1", + "metadata": {"allowed_s3_buckets": ["team-a-bucket"]}, + } + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=existing) + + with _org_admins(("u-team-admin", "org-1")): + exc: Final[pytest.ExceptionInfo[ProxyException]] + with pytest.raises(ProxyException) as exc: + await update_team( + data=UpdateTeamRequest( + team_id="t1", metadata={"allowed_s3_buckets": ["team-a-bucket", "another-orgs-bucket"]} + ), + http_request=MagicMock(spec=Request), + user_api_key_dict=_non_admin_auth(), + ) + + assert str(exc.value.code) == "403" + assert "Only proxy admins can set allowed_s3_buckets on a team." in str(exc.value.message) + + @pytest.mark.asyncio async def test_update_team_blocks_org_admin_null_metadata_clear_require_trace_id( mock_db_client: MagicMock, diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index bbe7f4383b6..2f469bb8684 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -21,6 +21,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( get_team_provider_deployment, is_litellm_executed_batch, map_raw_file_ids_to_unified, + prepare_data_with_credentials, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.utils import InternalUsageCache, handle_exception_on_proxy @@ -952,3 +953,19 @@ def test_a_provider_wildcard_grant_finds_only_deployments_named_under_it(deploye ) assert (None if credentials is None else credentials["api_key"]) == api_key + + +@pytest.mark.parametrize("field", ["s3_bucket_name", "s3_output_bucket_name"]) +def test_create_paths_keep_a_request_bucket_and_reads_keep_the_deployment_bucket(field: str) -> None: + deployment: Final = {"custom_llm_provider": "bedrock", "api_key": "sk-deployment", field: "model-bucket"} + create_with_bucket: Final[dict[str, object]] = {field: "team-a-bucket"} + create_without_bucket: Final[dict[str, object]] = {} + read_with_bucket: Final[dict[str, object]] = {field: "team-a-bucket"} + + prepare_data_with_credentials(data=create_with_bucket, credentials=dict(deployment), keep_request_buckets=True) + prepare_data_with_credentials(data=create_without_bucket, credentials=dict(deployment), keep_request_buckets=True) + prepare_data_with_credentials(data=read_with_bucket, credentials=dict(deployment)) + + assert create_with_bucket == {field: "team-a-bucket", "api_key": "sk-deployment"} + assert create_without_bucket == {field: "model-bucket", "api_key": "sk-deployment"} + assert read_with_bucket == {field: "model-bucket", "api_key": "sk-deployment"} diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py index 27a65a91c41..1a57fc81a9f 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py @@ -951,6 +951,45 @@ def test_batch_upload_for_a_provider_model_still_goes_to_the_provider(batch_uplo assert provider_upload.call_args.kwargs["custom_llm_provider"] == "gemini" +def _upload_as_team_a(bucket: str | None): + import litellm.proxy.proxy_server as ps + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="sk-team-a", team_id="team-a", team_metadata={"allowed_s3_buckets": ["team-a-bucket"]} + ) + return _upload_batch_file( + {"x-litellm-model": "gemini-2.0-flash"}, {} if bucket is None else {"s3_bucket_name": bucket} + ) + + +def test_batch_upload_from_a_team_with_an_allowlist_and_no_bucket_leaves_the_bucket_to_the_model(batch_upload_seams): + _, provider_upload, _ = batch_upload_seams + + response = _upload_as_team_a(None) + + assert response.status_code == 200, response.text + assert "s3_bucket_name" not in provider_upload.call_args.kwargs + + +def test_batch_upload_sends_a_bucket_on_the_team_allowlist_to_the_provider_upload(batch_upload_seams): + _, provider_upload, _ = batch_upload_seams + + response = _upload_as_team_a("team-a-bucket") + + assert response.status_code == 200, response.text + assert provider_upload.call_args.kwargs["s3_bucket_name"] == "team-a-bucket" + + +def test_batch_upload_rejects_a_bucket_off_the_team_allowlist_before_the_provider_upload(batch_upload_seams): + _, provider_upload, _ = batch_upload_seams + + response = _upload_as_team_a("team-b-bucket") + + assert response.status_code == 401, response.text + assert "s3_bucket_name is not allowed in request body" in response.text + provider_upload.assert_not_awaited() + + @pytest.mark.parametrize( "model, expected_drop_params", [("vertex-batch-drop", True), ("vertex-batch-keep", False)], diff --git a/tests/unit/test_router/core/test_endpoints_characterization_batch.py b/tests/unit/test_router/core/test_endpoints_characterization_batch.py index 6ba298accfe..626f25d00e6 100644 --- a/tests/unit/test_router/core/test_endpoints_characterization_batch.py +++ b/tests/unit/test_router/core/test_endpoints_characterization_batch.py @@ -125,6 +125,32 @@ async def test_acreate_file_sends_target_model_names_to_a_litellm_proxy_deployme assert dict(host.calls[0].kwargs["extra_body"]) == {"keep": 1, "target_model_names": "remote-model"} # pyright: ignore[reportArgumentType] # a mapping +@pytest.mark.parametrize( + "request_bucket,expected_bucket", [({}, "model-bucket"), ({"s3_bucket_name": "team-a-bucket"}, "team-a-bucket")] +) +async def test_acreate_file_prefers_the_request_s3_bucket_over_the_deployment_bucket( + make_core: MakeCore, host: EndpointHost, request_bucket: dict[str, str], expected_bucket: str +): + core: Final = make_core(model_list=[deployment("g", "d1", s3_bucket_name="model-bucket")]) + await core.acreate_file(model="g", file=_jsonl(), purpose="batch", **request_bucket) + assert host.calls[0].kwargs["s3_bucket_name"] == expected_bucket + + +@pytest.mark.parametrize( + "request_bucket,expected_bucket", + [({}, "model-bucket"), ({"s3_output_bucket_name": "team-a-bucket"}, "team-a-bucket")], +) +async def test_acreate_batch_prefers_the_request_output_bucket_over_the_deployment_bucket( + make_core: MakeCore, host: EndpointHost, request_bucket: dict[str, str], expected_bucket: str +): + host.answer("openai/d1", _batch()) + core: Final = make_core(model_list=[deployment("g", "d1", s3_output_bucket_name="model-bucket")]) + await core.acreate_batch( + model="g", input_file_id="f", endpoint="/v1/chat/completions", completion_window="24h", **request_bucket + ) + assert host.calls[0].kwargs["s3_output_bucket_name"] == expected_bucket + + async def test_acreate_file_sends_the_deployment_gcs_bucket_in_litellm_metadata( make_core: MakeCore, host: EndpointHost ): From f0d234c4c6605607d5411679adc7d4dad43fd3aa Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 10 Nov 2026 11:18:30 -0800 Subject: [PATCH 030/189] fix(router): let an explicit routing tag outrank the encrypted content pin (#50975) * fix(router): let an explicit routing tag outrank the encrypted content pin A Responses follow-up that replays encrypted reasoning minted on one deployment, but asks for another with a routing tag, failed with "Not allowed to access model due to tags configuration". The affinity check now runs the router's tag filter on the deployment it is about to pin, and on encryption-boundary peers. When the filter would reject them, it releases the pin and strips the reasoning the tagged deployment cannot decrypt, so the request routes there. Requests the tag filter accepts behave as before. Fixes #50926 * fix(router): strip reasoning against only the boundary peers the tags keep When tags rejected some encryption-boundary peers, reasoning was still judged against every peer, so a blob only a rejected peer could decrypt reached the tagged one. Pin and strip against the tag-kept peers. * fix(router): mark the deployment TypeVar Final * fix(router): keep every boundary peer for the router's tag check Returning only the tag-kept peers hid tags the later tag check needs to fail open, so a request that passed the affinity check was refused. Return all boundary peers and judge the reasoning against the kept ones. --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../encrypted_content_affinity_check.py | 43 +++- .../test_encrypted_content_affinity_check.py | 212 ++++++++++++++++++ 2 files changed, 248 insertions(+), 7 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 2dfe7921c60..8db5a5f0005 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -41,7 +41,7 @@ from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass, field, replace from functools import cache, lru_cache, partial from time import monotonic -from typing import TYPE_CHECKING, Final, Optional, cast +from typing import TYPE_CHECKING, Final, Optional, TypeVar, cast from pydantic import BaseModel, TypeAdapter, ValidationError @@ -49,6 +49,7 @@ import litellm from litellm._logging import verbose_router_logger from litellm.exceptions import AuthenticationError, BadRequestError from litellm.integrations.custom_logger import CustomLogger, Span +from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider, get_llm_provider from litellm.litellm_core_utils.litellm_logging import has_sensitive_keyword from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( @@ -60,6 +61,7 @@ from litellm.litellm_core_utils.routing_common_utils import as_excluded_deployme from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param +from litellm.router_strategy.tag_based_routing import get_deployments_for_tag from litellm.secret_managers.main import secret_manager_would_be_consulted from litellm.types.llms.openai import AllMessageValues from litellm.types.router import Deployment, GenericLiteLLMParams, ModelInfo @@ -70,6 +72,7 @@ if TYPE_CHECKING: from litellm.router_api import RoutingContext _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object]) _HEADERS_ADAPTER: Final = TypeAdapter(Mapping[object, object]) _ACCOUNT_SCOPE_HEADERS: Final = frozenset({"openai-organization", "openai-project", "cookie"}) _SINGLE_VENDOR_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value}) @@ -245,7 +248,9 @@ class EncryptedContentAffinityCheck(CustomLogger): return None if deployment_id is None else str(deployment_id) @staticmethod - def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None: + def _find_deployment_by_model_id( + healthy_deployments: list[dict[str, object]], model_id: str + ) -> dict[str, object] | None: for deployment in healthy_deployments: model_info = deployment.get("model_info") if not isinstance(model_info, dict): @@ -546,6 +551,27 @@ class EncryptedContentAffinityCheck(CustomLogger): # Request routing (pre-call filter) # ------------------------------------------------------------------ + async def _tag_routable( + self, model: str, pinned: Sequence[_DeploymentT], request_kwargs: Mapping[str, object] + ) -> tuple[_DeploymentT, ...]: + """The deployments in ``pinned`` the router's tag filter keeps for this request, none when it rejects + them all, so an explicit routing tag outranks the pin.""" + if self.ctx is None: + return tuple(pinned) + try: + kept: Final = await get_deployments_for_tag( + ctx=self.ctx, + model=model, + healthy_deployments=pinned, # pyright: ignore[reportArgumentType] # deployment dicts + request_kwargs=request_kwargs, # pyright: ignore[reportArgumentType] # the request kwargs + metadata_variable_name=get_metadata_variable_name_from_kwargs(request_kwargs), + request_deployments=self.ctx.registry().deployments(model) or pinned, # pyright: ignore[reportArgumentType] # deployment dicts + ) + except ValueError: + return () + kept_identities: Final = frozenset(map(id, kept)) + return tuple(deployment for deployment in pinned if id(deployment) in kept_identities) + async def async_filter_deployments( self, model: str, @@ -629,7 +655,7 @@ class EncryptedContentAffinityCheck(CustomLogger): healthy_deployments=untried_candidates, model_id=model_id, ) - if deployment is not None: + if deployment is not None and await self._tag_routable(model, (deployment,), typed_request_kwargs): verbose_router_logger.debug( "EncryptedContentAffinityCheck: pinning -> deployment=%s", model_id, @@ -650,19 +676,22 @@ class EncryptedContentAffinityCheck(CustomLogger): healthy_deployments=untried_candidates, model_id=model_id, ) - if boundary_matches: + routable_peers: Final = ( + await self._tag_routable(model, boundary_matches, typed_request_kwargs) if boundary_matches else () + ) + if routable_peers: verbose_router_logger.debug( - "EncryptedContentAffinityCheck: model_id=%s not in healthy_deployments; " + "EncryptedContentAffinityCheck: model_id=%s cannot serve this turn; " "pinning to %d deployment(s) on same encryption boundary", model_id, - len(boundary_matches), + len(routable_peers), ) request_kwargs["_encrypted_content_affinity_pinned"] = True self.strip_reasoning_the_targets_cannot_decrypt( self.ctx, request_input, anthropic_messages, - boundary_matches, + routable_peers, unmarked_origin=model_id, clock=self._clock, ) diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 23417e11948..e908481ca2b 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -3508,3 +3508,215 @@ async def _forwarded_reasoning_after_order_fallback(boundary: str | None) -> tup ) async def test_order_fallback_keeps_reasoning_only_on_a_declared_boundary_peer(boundary, expected_reasoning): assert await _forwarded_reasoning_after_order_fallback(boundary) == ("region-3", expected_reasoning) + + +def _tagged_deployment( + vendor: str, boundary: str | None = None, tags: tuple[str, ...] = ("default",) +) -> dict[str, object]: + return { + "model_name": "tagged-group", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_base": f"https://{vendor}.example.invalid/v1", + "api_key": f"sk-{vendor}", + "tags": [*tags, f"vendor:{vendor}"], + }, + "model_info": {"id": vendor} | ({} if boundary is None else {"encrypted_content_boundary": boundary}), + } + + +async def _tagged_followup( + tags: list[str] | None, peer_boundary: str | None = None, origin_tags: tuple[str, ...] = ("default",) +) -> tuple[str, object]: + router: Final = litellm.Router( + model_list=[_tagged_deployment("a", "org-a", origin_tags), _tagged_deployment("b", peer_boundary)], + enable_tag_filtering=True, + optional_pre_call_checks=["encrypted_content_affinity"], + num_retries=0, + ) + handler: Final = AsyncMock(return_value=_build_mock_response(output_items=[], response_id="resp_tagged")) + with patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new=handler, + ): + response: Final = await router.aresponses( + model="tagged-group", + input=[ + {"role": "user", "content": "first question"}, + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "thought about it"}], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "blob-from-a", "a" + ), + }, + {"role": "user", "content": "follow-up"}, + ], + litellm_metadata={} if tags is None else {"tags": tags}, + ) + return response._hidden_params["model_id"], handler.await_args.kwargs["input"][1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tags", [None, ["vendor:a"]], ids=["untagged", "tag-matches-pin"]) +async def test_tagged_followup_stays_on_the_deployment_that_minted_its_reasoning(tags: list[str] | None) -> None: + served_by, reasoning = await _tagged_followup(tags) + + assert served_by == "a" + assert reasoning["encrypted_content"] == "blob-from-a" + + +@pytest.mark.asyncio +async def test_untagged_followup_stays_pinned_when_only_another_deployment_is_tagged_default() -> None: + served_by, reasoning = await _tagged_followup(None, origin_tags=()) + + assert served_by == "a" + assert reasoning["encrypted_content"] == "blob-from-a" + + +@pytest.mark.asyncio +async def test_routing_tag_that_excludes_the_pin_routes_to_the_tagged_deployment_without_its_reasoning() -> None: + served_by, reasoning = await _tagged_followup(["vendor:b"]) + + assert served_by == "b" + assert "encrypted_content" not in reasoning + assert reasoning["summary"] == [{"type": "summary_text", "text": "thought about it"}] + + +@pytest.mark.asyncio +async def test_routing_tag_that_picks_an_encryption_boundary_peer_keeps_the_reasoning() -> None: + served_by, reasoning = await _tagged_followup(["vendor:b"], peer_boundary="org-a") + + assert served_by == "b" + assert reasoning["encrypted_content"] == "blob-from-a" + + +@pytest.mark.asyncio +async def test_routing_tag_left_to_fail_open_keeps_the_pin_when_only_a_cooled_deployment_opts_in() -> None: + router: Final = litellm.Router( + model_list=[ + _tagged_deployment("a"), + _tagged_deployment("opted-in") | {"model_info": {"id": "opted-in", "allow_fail_open": True}}, + ], + enable_tag_filtering=True, + optional_pre_call_checks=["encrypted_content_affinity"], + num_retries=0, + ) + router.cooldown_cache.add_deployment_to_cooldown( + model_id="opted-in", original_exception=RuntimeError("unavailable"), exception_status=500, cooldown_time=3600 + ) + handler: Final = AsyncMock(return_value=_build_mock_response(output_items=[], response_id="resp_fail_open")) + with patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new=handler, + ): + response: Final = await router.aresponses( + model="tagged-group", + input=[ + { + "type": "reasoning", + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "blob-from-a", "a" + ), + }, + {"role": "user", "content": "follow-up"}, + ], + litellm_metadata={"tags": ["!default"], "inherited_tags": []}, + ) + + assert response._hidden_params["model_id"] == "a" + assert handler.await_args.kwargs["input"][0]["encrypted_content"] == "blob-from-a" + + +@pytest.mark.asyncio +async def test_routing_tag_strips_reasoning_only_a_rejected_boundary_peer_could_decrypt() -> None: + shares_only_a_credentials: Final = _tagged_deployment("a", tags=()) | { + "model_name": "other-group", + "model_info": {"id": "c"}, + } + router: Final = litellm.Router( + model_list=[_tagged_deployment("a", "org-a"), _tagged_deployment("b", "org-a"), shares_only_a_credentials], + enable_tag_filtering=True, + optional_pre_call_checks=["encrypted_content_affinity"], + num_retries=0, + ) + handler: Final = AsyncMock(return_value=_build_mock_response(output_items=[], response_id="resp_peer")) + with patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new=handler, + ): + response: Final = await router.aresponses( + model="tagged-group", + input=[ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "from a"}], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "blob-from-a", "a" + ), + }, + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "from c"}], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "blob-from-c", "c" + ), + }, + {"role": "user", "content": "follow-up"}, + ], + litellm_metadata={"tags": ["vendor:b"]}, + ) + forwarded: Final = handler.await_args.kwargs["input"] + + assert response._hidden_params["model_id"] == "b" + assert [item.get("encrypted_content") for item in forwarded[:2]] == ["blob-from-a", None] + + +def _boundary_peer(name: str, tags: list[str], model_info: Mapping[str, object]) -> dict[str, object]: + return { + "model_name": "tagged-group", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_base": f"https://{name}.example.invalid/v1", + "api_key": f"sk-{name}", + "tags": tags, + }, + "model_info": {"id": name, "encrypted_content_boundary": "org-a", **model_info}, + } + + +@pytest.mark.asyncio +async def test_routing_tags_that_fail_open_onto_a_boundary_peer_are_served_by_it() -> None: + router: Final = litellm.Router( + model_list=[ + _boundary_peer("a", ["z"], {}) | {"model_name": "other-group"}, + _boundary_peer("default-peer", ["default", "x"], {"allow_fail_open": True}), + _boundary_peer("y-peer", ["y"], {}), + ], + enable_tag_filtering=True, + optional_pre_call_checks=["encrypted_content_affinity"], + num_retries=0, + ) + handler: Final = AsyncMock(return_value=_build_mock_response(output_items=[], response_id="resp_fail_open")) + with patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new=handler, + ): + response: Final = await router.aresponses( + model="tagged-group", + input=[ + { + "type": "reasoning", + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "blob-from-a", "a" + ), + }, + {"role": "user", "content": "follow-up"}, + ], + litellm_metadata={"tags": ["&x", "&y"], "inherited_tags": []}, + ) + + assert response._hidden_params["model_id"] == "default-peer" + assert handler.await_args.kwargs["input"][0]["encrypted_content"] == "blob-from-a" From 263281ac89c9b81b17451f0c5c84126ccb2f6213 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:49:15 +0000 Subject: [PATCH 031/189] feat(cost): bill batch cached image input tokens for OpenAI image models (#50997) * feat(cost): bill batch cached image input tokens for OpenAI image models * fix(cost): bill batch cached audio and image tokens without a text cache price * fix(cost): honor modality cache prices in the half-price batch fallback --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: kerry --- .../crates/model-catalog/src/model_info.rs | 1 + litellm/cost_calculator.py | 132 ++++++++++----- .../litellm_core_utils/llm_cost_calc/utils.py | 3 + ...odel_prices_and_context_window_backup.json | 6 + litellm/types/utils.py | 2 + litellm/utils.py | 3 + model_prices_and_context_window.json | 6 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 17 ++ tests/unit/test_cost_calculator.py | 155 +++++++++++++++++- tests/unit/test_utils.py | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 11 files changed, 290 insertions(+), 40 deletions(-) diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index afbbfec59e6..fcd0c027323 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -28,6 +28,7 @@ pub struct ModelInfoFields { pub cache_read_input_audio_token_cost_flex: Option, pub cache_read_input_audio_token_cost_priority: Option, pub cache_read_input_audio_token_cost_batches: Option, + pub cache_read_input_image_token_cost_batches: Option, pub explicit_cache_read_input_token_cost: Option, pub prompt_cache_min_tokens: Option, pub cache_read_input_token_cost_above_200k_tokens: Option, diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 58947f9d0c2..53137b73f8b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -49,7 +49,6 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( BilledTokenRates, CostCalculatorUtils, calculate_cache_writing_cost, - calculate_cost_component, generic_cost_per_character, generic_cost_per_token, get_batch_cost_rates, @@ -3177,6 +3176,21 @@ def default_video_cost_calculator( return 0.0 +def _optional_batch_rate( + model_info: ModelInfo, + key: Literal[ + "input_cost_per_audio_token_batches", + "cache_read_input_audio_token_cost_batches", + "input_cost_per_image_token_batches", + "input_cost_per_video_token_batches", + ], + prompt_tokens: int, +) -> float | None: + context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens) + context_rate: Final = get_cost_per_unit(model_info, context_key, None) + return model_info.get(key) if context_rate is None else context_rate + + def _batch_rate( model_info: ModelInfo, key: Literal[ @@ -3189,9 +3203,7 @@ def _batch_rate( *, prompt_tokens: int = 0, ) -> float: - context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens) - context_rate: Final = get_cost_per_unit(model_info, context_key, None) - rate: Final = model_info.get(key) if context_rate is None else context_rate + rate: Final = _optional_batch_rate(model_info, key, prompt_tokens) return fallback if rate is None else rate @@ -3275,30 +3287,14 @@ def batch_cost_calculator( model_info, batch_rates.input, batch_rates.cache_read, + batch_rates.cache_read_image, batch_rates.cache_creation, batch_rates.cache_creation_rates_by_ttl, ) elif input_cost_per_token is not None: - details: Final = parse_prompt_tokens_details(usage) - cache_read_tokens: Final = details["cache_hit_tokens"] - cache_creation_tokens: Final = details["cache_creation_tokens"] - - # Subtract cached tokens from prompt_tokens before calculating cost - # Fixes issue where cached tokens are being charged again - base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens - total_prompt_cost = ( - base_input_tokens * (input_cost_per_token) / 2 - ) # batch cost is usually half of the regular token cost - - # Add cache read cost if applicable - cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None) - total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2 - - configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") - cache_creation_cost: Final = ( - input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost + total_prompt_cost = _half_price_prompt_cost( + usage, model_info, input_cost_per_token, batch_rates.cache_read_image ) - total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2 text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token) image_rate: Final = _batch_or_half( model_info.get("output_cost_per_image_token_batches"), @@ -3324,15 +3320,54 @@ def _standard_rate(model_info: ModelInfo, key: str, prompt_tokens: int) -> float return get_cost_per_unit(model_info, get_modality_token_cost_key(model_info, key, prompt_tokens), None) -def _cached_audio_batch_fallback( - model_info: ModelInfo, batch_cache_read_rate: float | None, cache_read_rate: float | None, prompt_tokens: int -) -> float: +def _cached_modality_rate( + model_info: ModelInfo, + standard_key: Literal["cache_read_input_audio_token_cost", "cache_read_input_image_token_cost"], + batch_modality_rate: float | None, + batch_cache_read_rate: float | None, + cache_read_rate: float | None, + prompt_tokens: int, +) -> float | None: + if batch_modality_rate is not None: + return batch_modality_rate if batch_cache_read_rate is not None: return batch_cache_read_rate - standard_audio_rate: Final = _standard_rate(model_info, "cache_read_input_audio_token_cost", prompt_tokens) - if standard_audio_rate is not None: - return standard_audio_rate - return 0.0 if cache_read_rate is None else cache_read_rate + standard_modality_rate: Final = _standard_rate(model_info, standard_key, prompt_tokens) + return cache_read_rate if standard_modality_rate is None else standard_modality_rate + + +def _half_price_prompt_cost( + usage: Usage, model_info: ModelInfo, input_cost_per_token: float, batch_cache_read_image_rate: float | None +) -> float: + details: Final = parse_prompt_tokens_details(usage) + cache_creation_tokens: Final = details["cache_creation_tokens"] + base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens + cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None) + half_cache_read_rate: Final = (get_cost_per_unit(model_info, cache_read_cost_key, 0.0) or 0.0) / 2 + cached_audio_rate: Final = _batch_or_half( + _optional_batch_rate(model_info, "cache_read_input_audio_token_cost_batches", usage.prompt_tokens), + _standard_rate(model_info, "cache_read_input_audio_token_cost", usage.prompt_tokens), + default=half_cache_read_rate, + ) + cached_image_rate: Final = _batch_or_half( + batch_cache_read_image_rate, + _standard_rate(model_info, "cache_read_input_image_token_cost", usage.prompt_tokens), + default=half_cache_read_rate, + ) + cached_audio_tokens: Final = details["cache_hit_audio_tokens"] + cached_image_tokens: Final = details["cached_image_tokens"] + cached_text_tokens: Final = max(details["cache_hit_tokens"] - cached_audio_tokens - cached_image_tokens, 0) + configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") + cache_creation_cost: Final = ( + input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost + ) + return ( + base_input_tokens * input_cost_per_token / 2 + + cached_text_tokens * half_cache_read_rate + + cached_audio_tokens * cached_audio_rate + + cached_image_tokens * cached_image_rate + + cache_creation_tokens * cache_creation_cost / 2 + ) def _batch_prompt_cost( @@ -3340,6 +3375,7 @@ def _batch_prompt_cost( model_info: ModelInfo, input_rate: float, batch_cache_read_rate: float | None, + batch_cache_read_image_rate: float | None, cache_creation_rate: float | None, cache_creation_rates_by_ttl: Mapping[str, float], ) -> float: @@ -3349,8 +3385,30 @@ def _batch_prompt_cost( if batch_cache_read_rate is None else batch_cache_read_rate ) - cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 - cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 + cached_audio_rate: Final = _cached_modality_rate( + model_info, + "cache_read_input_audio_token_cost", + _optional_batch_rate(model_info, "cache_read_input_audio_token_cost_batches", usage.prompt_tokens), + batch_cache_read_rate, + cache_read_rate, + usage.prompt_tokens, + ) + cached_image_rate: Final = _cached_modality_rate( + model_info, + "cache_read_input_image_token_cost", + batch_cache_read_image_rate, + batch_cache_read_rate, + cache_read_rate, + usage.prompt_tokens, + ) + cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cached_audio_rate is not None else 0 + cached_image_tokens: Final = details["cached_image_tokens"] if cached_image_rate is not None else 0 + cached_text_tokens: Final = ( + max(details["cache_hit_tokens"] - details["cache_hit_audio_tokens"] - details["cached_image_tokens"], 0) + if cache_read_rate is not None + else 0 + ) + cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl) written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0 audio_tokens, image_tokens, video_tokens = ( @@ -3367,13 +3425,9 @@ def _batch_prompt_cost( * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate, prompt_tokens=usage.prompt_tokens) + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate) + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) - + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) - + cached_audio_tokens - * _batch_rate( - model_info, - "cache_read_input_audio_token_cost_batches", - _cached_audio_batch_fallback(model_info, batch_cache_read_rate, cache_read_rate, usage.prompt_tokens), - ) + + cached_text_tokens * (cache_read_rate or 0.0) + + cached_audio_tokens * (cached_audio_rate or 0.0) + + cached_image_tokens * (cached_image_rate or 0.0) + ( calculate_cache_writing_cost( written_tokens, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index ae7c4b47ece..2eb204d151b 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -112,6 +112,7 @@ _BATCH_RATE_PREFIXES: Final = ( "input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost", + "cache_read_input_image_token_cost", "cache_creation_input_token_cost", ) _BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values()) @@ -326,6 +327,7 @@ class BatchCostRates: cache_read: float | None cache_creation: float | None cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({})) + cache_read_image: float | None = None def _batch_rate(model_info: ModelInfo, key: str) -> float | None: @@ -388,6 +390,7 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide if (rate := _batch_rate_for_prefix(model_info, key, usage, inclusive)) is not None } ), + cache_read_image=_batch_rate_for_prefix(model_info, "cache_read_input_image_token_cost", usage, inclusive), ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a4c16fbeaa3..b37a5197483 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -36449,6 +36449,7 @@ }, "gpt-image-1.5": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", @@ -36497,6 +36498,7 @@ }, "gpt-image-2": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "deprecation_date": "2027-02-10", @@ -36539,6 +36541,7 @@ }, "gpt-image-2.5-flare": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, @@ -36575,6 +36578,7 @@ }, "gpt-image-2.5-sunburst": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, @@ -39936,6 +39940,7 @@ }, "gpt-image-1-mini": { "cache_read_input_image_token_cost": 2.5e-07, + "cache_read_input_image_token_cost_batches": 1.3e-07, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2026-12-01", @@ -60402,6 +60407,7 @@ }, "chatgpt-image-latest": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ffc9e915ea4..6325de96839 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -372,6 +372,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_512k_tokens: float | None cache_read_input_token_cost_batches: ReadOnly[float | None] cache_read_input_audio_token_cost_batches: ReadOnly[float | None] + cache_read_input_image_token_cost_batches: ReadOnly[float | None] cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] @@ -4190,6 +4191,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_read_input_token_cost_above_200k_tokens_batches: float | None = None cache_read_input_token_cost_above_272k_tokens_batches: float | None = None cache_read_input_audio_token_cost_batches: float | None = None + cache_read_input_image_token_cost_batches: float | None = None cache_creation_input_token_cost_batches: float | None = None cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index ca916e049d1..717b1b9009a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7266,6 +7266,9 @@ def get_model_info_helper( cache_read_input_audio_token_cost_batches=_model_info.get( "cache_read_input_audio_token_cost_batches", None ), + cache_read_input_image_token_cost_batches=_model_info.get( + "cache_read_input_image_token_cost_batches", None + ), explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), cache_read_input_image_token_cost=_model_info.get("cache_read_input_image_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a4c16fbeaa3..b37a5197483 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -36449,6 +36449,7 @@ }, "gpt-image-1.5": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", @@ -36497,6 +36498,7 @@ }, "gpt-image-2": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "deprecation_date": "2027-02-10", @@ -36539,6 +36541,7 @@ }, "gpt-image-2.5-flare": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, @@ -36575,6 +36578,7 @@ }, "gpt-image-2.5-sunburst": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, @@ -39936,6 +39940,7 @@ }, "gpt-image-1-mini": { "cache_read_input_image_token_cost": 2.5e-07, + "cache_read_input_image_token_cost_batches": 1.3e-07, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2026-12-01", @@ -60402,6 +60407,7 @@ }, "chatgpt-image-latest": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index fd7ec0d6bca..0aa76deb9d4 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -4163,6 +4163,23 @@ def test_get_batch_cost_rates_parses_string_rates(): assert (rates.input, rates.output, rates.cache_read) == (2e-6, 6e-6, 2e-7) +def test_get_batch_cost_rates_reads_the_batch_cached_image_rate_for_the_crossed_tier(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=2.5e-6, + cache_read_input_image_token_cost=2e-6, + cache_read_input_image_token_cost_batches=1e-6, + cache_read_input_image_token_cost_above_272k_tokens_batches=1.5e-6, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert rates.cache_read_image == 1.5e-6 + + def test_get_batch_cost_rates_falls_back_to_the_flat_rates_when_tier_rates_are_unparsable(): from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 9f3bfc2de86..8af9ae29c67 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -3,6 +3,7 @@ import asyncio import datetime import time from collections.abc import Iterator, Mapping +from pathlib import Path from types import MappingProxyType, SimpleNamespace from typing import Final, Literal, cast @@ -10,7 +11,7 @@ import httpx import pytest import respx from openai.types.completion_usage import CompletionUsage -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm import TranscriptionResponse, model_cost @@ -4580,6 +4581,158 @@ def test_batch_cost_calculator_bills_cached_tokens_at_the_standard_cache_price_w assert prompt_cost == pytest.approx(1000 * expected_rate) +def _cached_image_batch_usage() -> Usage: + return Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=1000, image_tokens=1000, cached_tokens_details=CachedTokensDetails(image_tokens=1000) + ), + ) + + +@pytest.mark.parametrize( + ("image_batch_cache_price", "text_batch_cache_price", "expected_rate"), + [ + pytest.param(1e-6, 6.25e-7, 1e-6, id="the-batch-cached-image-price-wins"), + pytest.param(None, 6.25e-7, 6.25e-7, id="without-it-cached-images-bill-the-batch-cache-price"), + pytest.param(None, None, 2e-6, id="without-any-batch-cache-price-cached-images-bill-the-standard-image-cache-price"), + ], +) +def test_batch_cost_calculator_bills_cached_image_tokens_at_the_batch_cached_image_price( + image_batch_cache_price: float | None, text_batch_cache_price: float | None, expected_rate: float +) -> None: + model_info: Final = cast( + ModelInfo, + { + "input_cost_per_token": 5e-6, + "input_cost_per_image_token": 8e-6, + "cache_read_input_token_cost": 1.25e-6, + "cache_read_input_image_token_cost": 2e-6, + "input_cost_per_token_batches": 2.5e-6, + "input_cost_per_image_token_batches": 4e-6, + "cache_read_input_token_cost_batches": text_batch_cache_price, + "cache_read_input_image_token_cost_batches": image_batch_cache_price, + }, + ) + + prompt_cost, _ = batch_cost_calculator( + usage=_cached_image_batch_usage(), model="gpt-image-2", custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(1000 * expected_rate) + + +_COST_MAP_ROWS: Final = TypeAdapter(dict[str, dict[str, object]]) + + +def _rows_with_a_batch_cached_image_price() -> tuple[tuple[str, str, float, ModelInfo], ...]: + rows: Final = _COST_MAP_ROWS.validate_json( + Path(litellm.__file__).with_name("model_prices_and_context_window_backup.json").read_bytes() + ) + return tuple( + (name, str(entry["litellm_provider"]), rate, cast(ModelInfo, entry)) + for name, entry in rows.items() + if isinstance(rate := entry.get("cache_read_input_image_token_cost_batches"), float) + ) + + +def _batch_cached_image_cost(name: str, provider: str, entry: ModelInfo) -> float: + return batch_cost_calculator(_cached_image_batch_usage(), name, provider, model_info=entry)[0] + + +@pytest.mark.parametrize( + ("usage", "modality_batch_cache_key"), + [ + pytest.param(_cached_audio_batch_usage(), "cache_read_input_audio_token_cost_batches", id="cached-audio"), + pytest.param(_cached_image_batch_usage(), "cache_read_input_image_token_cost_batches", id="cached-image"), + ], +) +def test_batch_cost_calculator_bills_cached_modality_tokens_at_their_batch_cache_price_without_a_text_cache_price( + usage: Usage, modality_batch_cache_key: str +) -> None: + model_info: Final = cast( + ModelInfo, + { + "input_cost_per_token": 5e-6, + "input_cost_per_audio_token": 8e-6, + "input_cost_per_image_token": 8e-6, + "input_cost_per_token_batches": 2.5e-6, + "input_cost_per_audio_token_batches": 4e-6, + "input_cost_per_image_token_batches": 4e-6, + modality_batch_cache_key: 1e-6, + }, + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model="gpt-image-2", custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(1000 * 1e-6) + + +@pytest.mark.parametrize( + ("usage", "modality_cache_key", "expected_prompt_cost"), + [ + pytest.param( + _cached_audio_batch_usage(), + "cache_read_input_audio_token_cost_batches", + 1000 * 1e-6, + id="cached-audio-batch-price", + ), + pytest.param( + _cached_image_batch_usage(), + "cache_read_input_image_token_cost_batches", + 1000 * 1e-6, + id="cached-image-batch-price", + ), + pytest.param( + _cached_audio_batch_usage(), + "cache_read_input_audio_token_cost", + 1000 * 1e-6 / 2, + id="cached-audio-standard-price-halved", + ), + pytest.param( + _cached_image_batch_usage(), + "cache_read_input_image_token_cost", + 1000 * 1e-6 / 2, + id="cached-image-standard-price-halved", + ), + ], +) +def test_batch_cost_calculator_half_price_fallback_bills_cached_modality_tokens_at_their_own_cache_price( + usage: Usage, modality_cache_key: str, expected_prompt_cost: float +) -> None: + model_info: Final = cast( + ModelInfo, + { + "input_cost_per_token": 5e-6, + "cache_read_input_token_cost": 2.5e-6, + modality_cache_key: 1e-6, + }, + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model="gpt-image-2", custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(expected_prompt_cost) + + +def test_batch_cached_image_tokens_bill_the_rows_batch_cached_image_price_on_every_row_that_has_one() -> None: + rows: Final = _rows_with_a_batch_cached_image_price() + assert rows, "no cost map row prices Batch cached image input" + + mispriced: Final = tuple( + (name, _batch_cached_image_cost(name, provider, entry), 1000 * rate) + for name, provider, rate, entry in rows + if _batch_cached_image_cost(name, provider, entry) != pytest.approx(1000 * rate) + ) + + assert mispriced == () + + def test_batch_cost_calculator_bills_long_context_cached_tokens_at_the_standard_long_context_cache_price() -> None: model_info: Final[ModelInfo] = { "input_cost_per_token": 1.25e-6, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index eaa0b3970bd..045d78fc1e7 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1023,6 +1023,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, "cache_read_input_audio_token_cost_batches": {"type": "number"}, + "cache_read_input_image_token_cost_batches": {"type": "number"}, "cache_read_input_audio_token_cost_flex": {"type": "number"}, "cache_read_input_audio_token_cost_priority": {"type": "number"}, "cache_read_input_image_token_cost": {"type": "number"}, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e1b0104e089..ddf6eb21c63 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -36614,6 +36614,8 @@ export interface components { cache_read_input_audio_token_cost_priority?: number | null; /** Cache Read Input Image Token Cost */ cache_read_input_image_token_cost?: number | null; + /** Cache Read Input Image Token Cost Batches */ + cache_read_input_image_token_cost_batches?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; /** Cache Read Input Token Cost Above 100K Tokens */ @@ -52804,6 +52806,8 @@ export interface components { cache_read_input_audio_token_cost_priority?: number | null; /** Cache Read Input Image Token Cost */ cache_read_input_image_token_cost?: number | null; + /** Cache Read Input Image Token Cost Batches */ + cache_read_input_image_token_cost_batches?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; /** Cache Read Input Token Cost Above 100K Tokens */ From c319616c1f8bae5df86193dc01df5a4611dd6b20 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:56:15 -0800 Subject: [PATCH 032/189] Registry audit 2026-11-10, Together, Mistral, Fireworks, Azure AI and Copilot fixes (#51003) Lowers Together Tev1 4B input price to the pricing page value, adds Mistral Leanstral vision and reasoning flags, and absorbs #50991, #50988 and #50958. Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Kerry Lu <326648561+kerry-berri@users.noreply.github.com> Co-authored-by: zoveki <349028329+zoveki@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 195 +++++++++++++++++- model_prices_and_context_window.json | 195 +++++++++++++++++- .../test_fireworks_ai_chat_transformation.py | 16 ++ .../fireworks_ai/latest_router_aliases.py | 13 ++ ...t_fireworks_ai_responses_transformation.py | 22 ++ .../test_model_prices_and_context_window.py | 23 +++ 6 files changed, 452 insertions(+), 12 deletions(-) create mode 100644 tests/unit/llms/fireworks_ai/latest_router_aliases.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b37a5197483..edcea48e4df 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3921,6 +3921,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, "supports_computer_use": true, @@ -3932,6 +3937,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, + "supports_web_search": true, "prompt_cache_min_tokens": 4096, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule" }, @@ -4074,7 +4080,7 @@ "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" }, "azure_ai/claude-fable-latest": { - "alias_of": "azure_ai/claude-fable-5-1" + "alias_of": "azure_ai/claude-fable-5-2" }, "azure_ai/claude-mythos-5-1": { "input_cost_per_token": 1e-05, @@ -4088,6 +4094,11 @@ "max_input_tokens": 1000000, "max_output_tokens": 128000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "supports_adaptive_thinking": true, "thinking_always_on": true, "supports_mid_conversation_system": true, @@ -4105,6 +4116,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": false, + "supports_web_search": true, "prompt_cache_min_tokens": 512, "source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" }, @@ -4229,6 +4241,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, "supports_computer_use": true, @@ -4239,6 +4256,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, + "supports_web_search": true, "prompt_cache_min_tokens": 1024, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule" }, @@ -4290,6 +4308,11 @@ "max_input_tokens": 1000000, "max_output_tokens": 128000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -4302,6 +4325,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, + "supports_web_search": true, "prompt_cache_min_tokens": 1024, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models" }, @@ -28947,6 +28971,24 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/minimax-latest": { + "cache_read_input_token_cost": 6e-08, + "cache_read_input_token_cost_priority": 9e-08, + "input_cost_per_token": 3e-07, + "input_cost_per_token_priority": 4.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 512000, + "max_output_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_priority": 1.8e-06, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/minimax-latest": { "alias_of": "fireworks_ai/accounts/fireworks/models/minimax-m3" }, @@ -33848,7 +33890,7 @@ "thinking_always_on": true }, "github_copilot/claude-fable-latest": { - "alias_of": "github_copilot/claude-fable-5.1" + "alias_of": "github_copilot/claude-fable-5.2" }, "github_copilot/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -65253,7 +65295,9 @@ "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true }, "mistral/mistral-moderation-2603": { "input_cost_per_token": 0.0, @@ -65775,6 +65819,30 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/kimi-latest": { + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_priority": 3.75e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_priority": 3.75e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_priority": 1.875e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/kimi-latest": { "alias_of": "fireworks_ai/accounts/fireworks/models/kimi-k3" }, @@ -66074,6 +66142,7 @@ "supports_function_calling": true, "supports_mid_conversation_system": true, "supports_reasoning": true, + "supports_tool_choice": true, "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/nemotron-lightning-3p5-30b-a3b": { @@ -66137,6 +66206,24 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/qwen-max-latest": { + "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_priority": 3.75e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_priority": 3e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_priority": 9e-06, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/routers/glm-5p2-fast": { "cache_read_input_token_cost": 2.1e-07, "deprecation_date": "2026-09-25", @@ -66194,6 +66281,27 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/kimi-fast-latest": { + "cache_read_input_token_cost": 4.5e-07, + "input_cost_per_token": 4.5e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.25e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/routers/kimi-k3-us": { "cache_read_input_token_cost": 4.5e-07, "cache_read_input_token_cost_priority": 5.625e-07, @@ -69008,7 +69116,9 @@ "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/glm-5p3": { "cache_read_input_token_cost": 2.6e-07, @@ -69029,6 +69139,25 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/routers/glm-latest": { + "cache_read_input_token_cost": 2.6e-07, + "cache_read_input_token_cost_priority": 3.25e-07, + "input_cost_per_token": 1.4e-06, + "input_cost_per_token_priority": 1.75e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "output_cost_per_token_priority": 5.5e-06, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/routers/glm-5p3-us": { "cache_read_input_token_cost": 3.9e-07, "cache_read_input_token_cost_priority": 4.875e-07, @@ -69102,6 +69231,22 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/routers/glm-fast-latest": { + "cache_read_input_token_cost": 3.9e-07, + "input_cost_per_token": 2.1e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/routers/auto": { "litellm_provider": "fireworks_ai", "mode": "chat", @@ -69163,6 +69308,24 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/glm-flash-latest": { + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_priority": 3.75e-08, + "input_cost_per_token": 1.5e-07, + "input_cost_per_token_priority": 1.875e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_priority": 6.25e-07, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/routers/glm-5p3-flash-us": { "cache_read_input_token_cost": 4.5e-08, "cache_read_input_token_cost_priority": 5.625e-08, @@ -71992,12 +72155,12 @@ "source": "https://api.together.xyz/v1/models" }, "together_ai/together/Tev1-4B-experimental": { - "input_cost_per_token": 4.2e-08, + "input_cost_per_token": 4e-08, "litellm_provider": "together_ai", "max_input_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, - "source": "https://api.together.ai/v1/models" + "source": "https://www.together.ai/pricing" }, "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": { "deprecation_date": "2026-02-06", @@ -79437,6 +79600,26 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/deepseek-flash-latest": { + "cache_read_input_token_cost": 6e-09, + "cache_read_input_token_cost_priority": 7.5e-09, + "input_cost_per_token": 3e-07, + "input_cost_per_token_priority": 3.75e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_priority": 1.5e-06, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "github_copilot/claude-sonnet-4.6": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b37a5197483..edcea48e4df 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3921,6 +3921,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, "supports_computer_use": true, @@ -3932,6 +3937,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, + "supports_web_search": true, "prompt_cache_min_tokens": 4096, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule" }, @@ -4074,7 +4080,7 @@ "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" }, "azure_ai/claude-fable-latest": { - "alias_of": "azure_ai/claude-fable-5-1" + "alias_of": "azure_ai/claude-fable-5-2" }, "azure_ai/claude-mythos-5-1": { "input_cost_per_token": 1e-05, @@ -4088,6 +4094,11 @@ "max_input_tokens": 1000000, "max_output_tokens": 128000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "supports_adaptive_thinking": true, "thinking_always_on": true, "supports_mid_conversation_system": true, @@ -4105,6 +4116,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": false, + "supports_web_search": true, "prompt_cache_min_tokens": 512, "source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" }, @@ -4229,6 +4241,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, "supports_computer_use": true, @@ -4239,6 +4256,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, + "supports_web_search": true, "prompt_cache_min_tokens": 1024, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule" }, @@ -4290,6 +4308,11 @@ "max_input_tokens": 1000000, "max_output_tokens": 128000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -4302,6 +4325,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, + "supports_web_search": true, "prompt_cache_min_tokens": 1024, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models" }, @@ -28947,6 +28971,24 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/minimax-latest": { + "cache_read_input_token_cost": 6e-08, + "cache_read_input_token_cost_priority": 9e-08, + "input_cost_per_token": 3e-07, + "input_cost_per_token_priority": 4.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 512000, + "max_output_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_priority": 1.8e-06, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/minimax-latest": { "alias_of": "fireworks_ai/accounts/fireworks/models/minimax-m3" }, @@ -33848,7 +33890,7 @@ "thinking_always_on": true }, "github_copilot/claude-fable-latest": { - "alias_of": "github_copilot/claude-fable-5.1" + "alias_of": "github_copilot/claude-fable-5.2" }, "github_copilot/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, @@ -65253,7 +65295,9 @@ "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true }, "mistral/mistral-moderation-2603": { "input_cost_per_token": 0.0, @@ -65775,6 +65819,30 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/kimi-latest": { + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_priority": 3.75e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_priority": 3.75e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_priority": 1.875e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/kimi-latest": { "alias_of": "fireworks_ai/accounts/fireworks/models/kimi-k3" }, @@ -66074,6 +66142,7 @@ "supports_function_calling": true, "supports_mid_conversation_system": true, "supports_reasoning": true, + "supports_tool_choice": true, "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/nemotron-lightning-3p5-30b-a3b": { @@ -66137,6 +66206,24 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/qwen-max-latest": { + "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_priority": 3.75e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_priority": 3e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_priority": 9e-06, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/routers/glm-5p2-fast": { "cache_read_input_token_cost": 2.1e-07, "deprecation_date": "2026-09-25", @@ -66194,6 +66281,27 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/kimi-fast-latest": { + "cache_read_input_token_cost": 4.5e-07, + "input_cost_per_token": 4.5e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.25e-05, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/routers/kimi-k3-us": { "cache_read_input_token_cost": 4.5e-07, "cache_read_input_token_cost_priority": 5.625e-07, @@ -69008,7 +69116,9 @@ "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", "supports_function_calling": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/glm-5p3": { "cache_read_input_token_cost": 2.6e-07, @@ -69029,6 +69139,25 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/routers/glm-latest": { + "cache_read_input_token_cost": 2.6e-07, + "cache_read_input_token_cost_priority": 3.25e-07, + "input_cost_per_token": 1.4e-06, + "input_cost_per_token_priority": 1.75e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "output_cost_per_token_priority": 5.5e-06, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/routers/glm-5p3-us": { "cache_read_input_token_cost": 3.9e-07, "cache_read_input_token_cost_priority": 4.875e-07, @@ -69102,6 +69231,22 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/routers/glm-fast-latest": { + "cache_read_input_token_cost": 3.9e-07, + "input_cost_per_token": 2.1e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/routers/auto": { "litellm_provider": "fireworks_ai", "mode": "chat", @@ -69163,6 +69308,24 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/glm-flash-latest": { + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_priority": 3.75e-08, + "input_cost_per_token": 1.5e-07, + "input_cost_per_token_priority": 1.875e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_priority": 6.25e-07, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/routers/glm-5p3-flash-us": { "cache_read_input_token_cost": 4.5e-08, "cache_read_input_token_cost_priority": 5.625e-08, @@ -71992,12 +72155,12 @@ "source": "https://api.together.xyz/v1/models" }, "together_ai/together/Tev1-4B-experimental": { - "input_cost_per_token": 4.2e-08, + "input_cost_per_token": 4e-08, "litellm_provider": "together_ai", "max_input_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, - "source": "https://api.together.ai/v1/models" + "source": "https://www.together.ai/pricing" }, "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": { "deprecation_date": "2026-02-06", @@ -79437,6 +79600,26 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/accounts/fireworks/routers/deepseek-flash-latest": { + "cache_read_input_token_cost": 6e-09, + "cache_read_input_token_cost_priority": 7.5e-09, + "input_cost_per_token": 3e-07, + "input_cost_per_token_priority": 3.75e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_priority": 1.5e-06, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "github_copilot/claude-sonnet-4.6": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index dd352aac66d..0968dedd427 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -23,6 +23,7 @@ from litellm.types.utils import ( Message, ModelResponse, ) +from tests.unit.llms.fireworks_ai.latest_router_aliases import LATEST_ROUTER_ALIAS_TARGETS def test_validate_environment_sets_session_affinity_from_litellm_session_id(): @@ -368,6 +369,21 @@ def test_get_supported_openai_params_parallel_tool_calls_without_tool_choice( assert "tool_choice" not in supported_params +@pytest.mark.parametrize(("alias", "target"), LATEST_ROUTER_ALIAS_TARGETS) +def test_latest_router_alias_accepts_the_same_params_as_the_model_it_points_to(alias: str, target: str) -> None: + config: Final = FireworksAIConfig() + alias_params: Final = set(config.get_supported_openai_params(alias)) + target_params: Final = set(config.get_supported_openai_params(target)) + excluded_params: Final = ( + {"reasoning_effort", "reasoning_history", "thinking"} + if alias == "accounts/fireworks/routers/glm-flash-latest" + else set() + ) + + assert alias_params - excluded_params == target_params - excluded_params + assert "tool_choice" in alias_params + + def test_get_provider_info_omits_false_supports_reasoning(monkeypatch): """Test that Fireworks only overrides supports_reasoning for supported models.""" config = FireworksAIConfig() diff --git a/tests/unit/llms/fireworks_ai/latest_router_aliases.py b/tests/unit/llms/fireworks_ai/latest_router_aliases.py new file mode 100644 index 00000000000..6c305034b93 --- /dev/null +++ b/tests/unit/llms/fireworks_ai/latest_router_aliases.py @@ -0,0 +1,13 @@ +from typing import Final + +# https://api.fireworks.ai/v1/serverless/models?format=nested, read 2026-10-29 +LATEST_ROUTER_ALIAS_TARGETS: Final = ( + ("accounts/fireworks/routers/minimax-latest", "accounts/fireworks/models/minimax-m3"), + ("accounts/fireworks/routers/kimi-latest", "accounts/fireworks/models/kimi-k3"), + ("accounts/fireworks/routers/kimi-fast-latest", "accounts/fireworks/routers/kimi-k3-fast"), + ("accounts/fireworks/routers/qwen-max-latest", "accounts/fireworks/models/qwen3p8-max"), + ("accounts/fireworks/routers/glm-latest", "accounts/fireworks/models/glm-5p3"), + ("accounts/fireworks/routers/glm-fast-latest", "accounts/fireworks/routers/glm-5p3-fast"), + ("accounts/fireworks/routers/glm-flash-latest", "accounts/fireworks/models/glm-5p3-flash"), + ("accounts/fireworks/routers/deepseek-flash-latest", "accounts/fireworks/models/deepseek-v4p1-flash-1017"), +) diff --git a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index fe33a26dfd7..9a9c6ed3c82 100644 --- a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -26,6 +26,7 @@ from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage, Resp from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager +from tests.unit.llms.fireworks_ai.latest_router_aliases import LATEST_ROUTER_ALIAS_TARGETS FIREWORKS_RESPONSES_URL: Final = "https://api.fireworks.ai/inference/v1/responses" HTTPX_CLIENT_FACTORY: Final = "litellm.llms.custom_httpx.llm_http_handler.get_httpx_client" @@ -249,6 +250,27 @@ def test_responses_call_keeps_developer_items_in_place_for_a_model_the_cost_map_ ) +@respx.mock +@pytest.mark.parametrize(("alias", "target"), LATEST_ROUTER_ALIAS_TARGETS) +def test_latest_router_alias_places_developer_items_like_the_model_it_points_to(alias: str, target: str) -> None: + route: Final = _fireworks_responses_route(alias) + for model in (alias, target): + litellm.responses( + model=f"fireworks_ai/{model}", + input=[ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": "Paris"}, + {"role": "developer", "content": "Reply with PONG."}, + ], + api_key="fw-test-key", + ) + alias_body, target_body = (json.loads(call.request.content) for call in route.calls) + assert alias_body["model"] == alias + assert {**alias_body, "model": target} == target_body + + @respx.mock def test_responses_call_folds_developer_items_when_the_loaded_cost_map_turns_the_flag_off( declare_mid_conversation_system: Callable[[str, bool], None], diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index b270d385693..165f92e0e59 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -1050,6 +1050,29 @@ def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_azure_ai_claude_rows_bill_web_search_like_their_anthropic_rows(path: Path) -> None: + prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) + pairs: Final = { + key: (row, prices[key.removeprefix("azure_ai/")]) + for key, row in prices.items() + if key.startswith("azure_ai/claude") and isinstance(row, dict) and key.removeprefix("azure_ai/") in prices + } + priced_pairs: Final = { + key: (row, bare) + for key, (row, bare) in pairs.items() + if isinstance(bare, dict) and bare.get("search_context_cost_per_query") + } + assert priced_pairs + drifted: Final = tuple( + key + for key, (row, bare) in priced_pairs.items() + if row.get("search_context_cost_per_query") != bare["search_context_cost_per_query"] + or row.get("supports_web_search") is not True + ) + assert drifted == (), drifted + + @pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) def test_azure_ai_claude_rows_priced_for_web_search_say_they_support_it(path: Path) -> None: prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) From 63afd25823fd994ef834d7eee5292b8331bd4473 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:36:41 +0000 Subject: [PATCH 033/189] fix(cost-map): sync together_ai together/Tev1-4B-experimental input price with /v1/models (#51013) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 ++-- model_prices_and_context_window.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index edcea48e4df..f3f56143964 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -72155,12 +72155,12 @@ "source": "https://api.together.xyz/v1/models" }, "together_ai/together/Tev1-4B-experimental": { - "input_cost_per_token": 4e-08, + "input_cost_per_token": 4.2e-08, "litellm_provider": "together_ai", "max_input_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, - "source": "https://www.together.ai/pricing" + "source": "https://api.together.ai/v1/models" }, "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": { "deprecation_date": "2026-02-06", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index edcea48e4df..f3f56143964 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -72155,12 +72155,12 @@ "source": "https://api.together.xyz/v1/models" }, "together_ai/together/Tev1-4B-experimental": { - "input_cost_per_token": 4e-08, + "input_cost_per_token": 4.2e-08, "litellm_provider": "together_ai", "max_input_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, - "source": "https://www.together.ai/pricing" + "source": "https://api.together.ai/v1/models" }, "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": { "deprecation_date": "2026-02-06", From 88dd18f4ab8275aaaef9ca5a0f0274a0aee26b66 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:38:09 +0000 Subject: [PATCH 034/189] chore(cost-map): add github_copilot/gemini-4-carbon from the Copilot changelog launch post (#51012) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 17 +++++++++++++++++ model_prices_and_context_window.json | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f3f56143964..662909b5cb7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -88711,5 +88711,22 @@ "supports_vision": true, "supports_web_search": true, "web_search_billing_unit": "per_query" + }, + "github_copilot/gemini-4-carbon": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "github_copilot", + "max_input_tokens": 200000, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://github.blog/changelog/2026-11-09-gemini-4-carbon-in-github-copilot", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f3f56143964..662909b5cb7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -88711,5 +88711,22 @@ "supports_vision": true, "supports_web_search": true, "web_search_billing_unit": "per_query" + }, + "github_copilot/gemini-4-carbon": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "github_copilot", + "max_input_tokens": 200000, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://github.blog/changelog/2026-11-09-gemini-4-carbon-in-github-copilot", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_vision": true } } From 4867a56e9aef1310c741b90ee9c1a9e1bbd49e0d Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:50:17 +0000 Subject: [PATCH 035/189] fix(proxy)!: reject client-supplied Vertex location and project unless the admin opts in (#47028) * fix(proxy): reject client-supplied Vertex location and project unless the admin opts in A request body could carry vertex_location, vertex_project, or their vertex_ai_* aliases and override the region and project pinned on a Vertex AI deployment, sending the admin's credentials to a caller-chosen project or region. Add the four fields to the banned request params so they need allow_client_side_credentials or configurable_clientside_auth_params. /vertex_ai/live and /rag/ingest name their Vertex project and location by design and have no deployment pin to override, so they keep those two fields while every other banned param is still rejected there * fix(proxy): close the remaining Vertex target overrides and keep inert fields working /vertex_ai/live now closes with 1008 when the query names a project or location that differs from a use_in_pass_through deployment's pin, unless that deployment or the proxy opts in A model group's opt-in now counts only when every deployment in it carries it, and Google-style routes look the opt-in up on the model named in the URL Vertex target names inside metadata, the live route's vertex_ai_* aliases, and /health/test_connection inheritance go back to the base behavior, since nothing reads them as a target * test(batches): give the batch FakeRouter the model-list lookup the opt-in check reads * fix(proxy): pin default_vertex_config on /vertex_ai/live and the proxy's Vertex project on /rag/ingest * fix(proxy): require the Vertex opt-in on alias targets and fallbacks, and pass the live rewriter flag in main's tests * test(proxy): give the batch tests' fake router the alias and routed-group lookups * fix(proxy): close the Vertex opt-in, RAG corpus, and live setup-frame gaps found in review A Vertex opt-in now counts only when every group the request can reach opts in, router, context window, and content policy fallbacks and one alias hop included, while a group with no Google deployment never needs the fields. A model named in the route path is checked next to the body model. RAG ingest from a non-admin key refuses a bare or dot-segment corpus id, a vertex_ai_project or vertex_ai_location naming another target, and a store a non-admin saved before the upgrade, and reads a corpus named by project number on the proxy's own project. /vertex_ai/live closes 1008 on any caller-named project when the proxy pins none, moves a setup frame sent as a binary message onto the session's target, and resolves the region from default_vertex_config before the model path. * fix(proxy): let a live caller name the proxy's configured Vertex project when nothing pins one With default_vertex_config set but no project pinned, /vertex_ai/live closed every query that named a project, including the one the admin set in VERTEXAI_PROJECT or DEFAULT_VERTEXAI_PROJECT. A query naming one of those projects now connects, and any other project still closes with 1008 before a token is minted. * fix(proxy): keep an admin-created vector store's corpus as saved on RAG ingest A non-admin ingest into a store an admin created named by project number had its corpus moved onto the proxy's own project before the admin-store exemption ran, so the upload went to a different corpus than the admin saved. The exemption now runs first, and a non-admin store still gets the project-number rewrite and the target check. * fix(proxy): authorize a RAG corpus moved onto the proxy's project and check client-side opt-ins once per request * fix(proxy): merge opt-in fallback groups only when their deployments share provider and opt-in list * fix(proxy): check team public models before allowing target overrides * fix(proxy): include team wildcard targets in opt-in checks * fix(proxy): validate overrides against authenticated team routes * fix(proxy): retain URL model during deferred credential checks --- litellm/proxy/auth/auth_utils.py | 401 +++++++++--- litellm/proxy/auth/user_api_key_auth.py | 19 +- litellm/proxy/batches_endpoints/endpoints.py | 1 + .../litellm_executed_batches.py | 5 +- .../health_endpoints/_health_endpoints.py | 8 +- .../openai_files_endpoints/files_endpoints.py | 1 + .../llm_passthrough_endpoints.py | 153 ++++- .../pass_through_endpoints.py | 46 +- .../passthrough_endpoint_router.py | 10 +- litellm/proxy/prompts/prompt_endpoints.py | 1 + litellm/proxy/proxy_server.py | 5 +- litellm/proxy/rag_endpoints/endpoints.py | 119 +++- litellm/rag/ingestion/vertex_ai_ingestion.py | 4 +- .../test_vertex_image_regional_uplift.py | 10 +- tests/unit/proxy/auth/test_auth_utils.py | 591 +++++++++++++++++- .../test_user_api_key_auth_request_flow.py | 92 ++- .../proxy/batches_endpoints/test_endpoints.py | 8 +- .../test_litellm_executed_batches.py | 22 + .../health_endpoints/test_health_endpoints.py | 6 + .../test_llm_pass_through_endpoints.py | 404 +++++++++++- .../test_pass_through_endpoints.py | 97 ++- .../public_v1/test_model_hub.py | 4 + .../proxy/rag_endpoints/test_rag_endpoints.py | 290 ++++++++- 23 files changed, 2163 insertions(+), 134 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 715b98a441a..7413490e44f 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -2,8 +2,9 @@ import importlib.util import os import re import sys -from collections.abc import Collection, Iterator, Mapping, Sequence -from functools import lru_cache, partial +from collections.abc import Callable, Collection, Iterator, Mapping, Sequence +from dataclasses import dataclass +from functools import cache, lru_cache from itertools import accumulate, chain, repeat from logging import Logger from types import MappingProxyType @@ -24,6 +25,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.cloud_storage_security import REQUEST_S3_BUCKET_PARAMS, STORED_S3_BUCKET_PARAM from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference +from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import ( SSRFError, @@ -42,11 +44,13 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( from litellm.types.router import ( CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, Deployment, + DeploymentTypedDict, + LiteLLMParamsTypedDict, reject_server_owned_wif_fields, server_owned_wif_fields_present, ) from litellm.types.router import reject_server_owned_wif_params as _reject_server_owned_wif_params -from litellm.types.utils import CustomPricingLiteLLMParams, oauth_token_exchange_litellm_params +from litellm.types.utils import CustomPricingLiteLLMParams, LlmProviders, oauth_token_exchange_litellm_params from litellm.types.videos.main import DecodedVideoId @@ -212,35 +216,74 @@ def _is_param_allowed( return False +def deployment_clientside_auth_params(litellm_params: LiteLLMParamsTypedDict) -> CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS: + if "configurable_clientside_auth_params" not in litellm_params: + return None + return litellm_params["configurable_clientside_auth_params"] + + +_DEPLOYMENT_LIST: Final = TypeAdapter(tuple[DeploymentTypedDict, ...], config=ConfigDict(arbitrary_types_allowed=True)) + + +def _model_group_deployments( + model: str, llm_router: Router, team_id: str | None = None +) -> Sequence[DeploymentTypedDict]: + registry: Final = llm_router.registry() + provider_group: Final = model.split("/", 1)[0] + if team_id is not None and registry.routing_group(model) is None: + team_deployments: Final = registry.deployments( + model, source="all", team_id=team_id + ) or _DEPLOYMENT_LIST.validate_python(registry.deployments(model, source="wildcard", team_id=team_id)) + if team_deployments: + return team_deployments + return ( + registry.deployments(model, source="routed_group", team_id=team_id) + or (registry.deployments(provider_group, team_id=team_id) if provider_group in provider_list else None) + or () + ) + + +_VERTEX_TARGET_READING_PROVIDERS: Final[frozenset[str]] = frozenset( + (LlmProviders.VERTEX_AI.value, LlmProviders.VERTEX_AI_BETA.value, LlmProviders.GEMINI.value, LlmProviders.GDC.value) +) + + +def _deployment_provider(litellm_params: LiteLLMParamsTypedDict) -> str | None: + match litellm_params: + case {"custom_llm_provider": str(declared)} if declared: + return declared + case {"model": str(model)}: + return inferred_provider(model) + case _: + return None + + +def _model_group_reads_vertex_target(model: str, llm_router: Router | None, team_id: str | None = None) -> bool: + if llm_router is None: + return True + deployments: Final = _model_group_deployments(model, llm_router, team_id) + providers: Final = frozenset(_deployment_provider(deployment["litellm_params"]) for deployment in deployments) + return not deployments or None in providers or bool(providers & _VERTEX_TARGET_READING_PROVIDERS) + + def _allow_model_level_clientside_configurable_parameters( - model: str, param: str, request_body_value: object, llm_router: Router | None + model: str, param: str, request_body_value: object, llm_router: Router | None, team_id: str | None = None ) -> bool: """ - Check if model is allowed to use configurable client-side params - - get matching model - - check if 'clientside_configurable_parameters' is set for model - - + The router, not the caller, picks which deployment serves the request, so an + opt-in counts only when every deployment of the model group, already alias + resolved, carries it """ if llm_router is None: return False - # check if model is set - registry: Final = llm_router.registry() - model_info = registry.group(model) - if model_info is None: - # check if wildcard model is set - if model.split("/", 1)[0] in provider_list: - model_info = registry.group(model.split("/", 1)[0]) - - if model_info is None: - return False - - if model_info is None or model_info.configurable_clientside_auth_params is None: - return False - - return _is_param_allowed( - param=param, - request_body_value=request_body_value, - configurable_clientside_auth_params=model_info.configurable_clientside_auth_params, + deployments: Final = _model_group_deployments(model, llm_router, team_id) + return bool(deployments) and all( + _is_param_allowed( + param=param, + request_body_value=request_body_value, + configurable_clientside_auth_params=deployment_clientside_auth_params(deployment["litellm_params"]), + ) + for deployment in deployments ) @@ -319,6 +362,20 @@ def route_manages_mcp_servers(route: str | None) -> bool: return route is not None and _MCP_SERVER_MANAGEMENT_ROUTE.fullmatch(route) is not None +VERTEX_TARGET_PARAMS: Final[frozenset[str]] = frozenset( + ("vertex_project", "vertex_location", "vertex_ai_project", "vertex_ai_location") +) +_ROUTES_THAT_NAME_THEIR_VERTEX_TARGET: Final[frozenset[str]] = frozenset( + ("/vertex_ai/live", "/rag/ingest", "/v1/rag/ingest") +) + + +def _without_route_named_vertex_target(request_body: Mapping[str, object], route: str | None) -> Mapping[str, object]: + if route not in _ROUTES_THAT_NAME_THEIR_VERTEX_TARGET: + return request_body + return MappingProxyType({key: value for key, value in request_body.items() if key not in VERTEX_TARGET_PARAMS}) + + _NESTED_CONFIG_KEYS: Final[tuple[str, ...]] = ("litellm_embedding_config", "extra_body") # Metadata containers that carry per-request configuration consumed by the @@ -411,6 +468,10 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( "aws_external_id", "aws_session_tags", "vertex_credentials", + "vertex_location", + "vertex_project", + "vertex_ai_location", + "vertex_ai_project", # Azure managed-identity / federated-auth token. The Azure provider # transformer reads ``azure_ad_token`` (top-level or via # ``extra_body``) and resolves it through ``get_secret`` before @@ -483,6 +544,105 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( *sorted(CustomPricingLiteLLMParams.model_fields.keys()), ) +_BANNED_METADATA_PARAMS: Final[tuple[str, ...]] = tuple( + param for param in _BANNED_REQUEST_BODY_PARAMS if param not in VERTEX_TARGET_PARAMS +) + + +def _routed_opt_in_hop( + frontier_and_reached: tuple[frozenset[str], frozenset[str]], router_and_team: tuple[Router, str | None] +) -> tuple[frozenset[str], frozenset[str]]: + frontier, reached = frontier_and_reached + llm_router, team_id = router_and_team + deployments: Final = chain.from_iterable(_model_group_deployments(model, llm_router, team_id) for model in frontier) + routed_groups: Final = frozenset(deployment["model_name"] for deployment in deployments) - reached + fallbacks: Final = frozenset[str]().union( + *(router_fallback_model_names(name, llm_router, _EVERY_FALLBACK_TYPE) for name in routed_groups) + ) + registry: Final = llm_router.registry() + next_hop: Final = ( + routed_groups | frozenset(registry.resolve(name, kind="alias") or name for name in fallbacks) + ) - reached + return next_hop, reached | next_hop + + +def _opt_in_model_groups( + model: str, also_requested_models: tuple[str, ...], llm_router: Router | None, team_id: str | None = None +) -> frozenset[str]: + requested: Final = frozenset((model, *also_requested_models)) + if llm_router is None: + return requested + registry: Final = llm_router.registry() + routed: Final = frozenset(registry.resolve(name, kind="alias") or name for name in requested) + fallbacks: Final = frozenset[str]().union( + *(router_fallback_model_names(name, llm_router, _EVERY_FALLBACK_TYPE) for name in requested | routed) + ) + groups: Final = routed | frozenset(registry.resolve(name, kind="alias") or name for name in fallbacks) + hops: Final = accumulate(repeat((llm_router, team_id)), _routed_opt_in_hop, initial=(groups, groups)) + _, reached = next(hop for hop in hops if not hop[0]) + return reached + + +_MAX_OPT_IN_FALLBACK_MODELS: Final = 100 + + +@dataclass(frozen=True, slots=True) +class _OptInGroups: + every: tuple[str, ...] + vertex_target: tuple[str, ...] + team_id: str | None = None + + +_OptInKey = tuple[str | tuple[str, str | None], ...] + + +def _opt_in_key(litellm_params: LiteLLMParamsTypedDict) -> _OptInKey: + return tuple( + item if isinstance(item, str) else ("api_base", item.get("api_base")) + for item in deployment_clientside_auth_params(litellm_params) or () + ) + + +def _same_answer_key( + model_group: str, llm_router: Router, team_id: str | None +) -> frozenset[tuple[str | None, _OptInKey]] | str: + deployments: Final = _model_group_deployments(model_group, llm_router, team_id) + key: Final = frozenset( + (_deployment_provider(deployment["litellm_params"]), _opt_in_key(deployment["litellm_params"])) + for deployment in deployments + ) + return key or model_group + + +def _distinct_opt_in_groups( + model_groups: frozenset[str], llm_router: Router | None, team_id: str | None +) -> tuple[str, ...]: + if llm_router is None: + return tuple(sorted(model_groups)) + return tuple( + { + _same_answer_key(model_group, llm_router, team_id): model_group for model_group in sorted(model_groups) + }.values() + ) + + +def _resolve_opt_in_groups( + model: str, also_requested_models: tuple[str, ...], llm_router: Router | None, team_id: str | None = None +) -> _OptInGroups: + if len(frozenset(also_requested_models)) > _MAX_OPT_IN_FALLBACK_MODELS: + raise ValueError( + f"Rejected Request: the request names more than {_MAX_OPT_IN_FALLBACK_MODELS} fallback models, " + "too many to check against configurable_clientside_auth_params." + ) + every: Final = _distinct_opt_in_groups( + _opt_in_model_groups(model, also_requested_models, llm_router, team_id), llm_router, team_id + ) + return _OptInGroups( + every=every, + vertex_target=tuple(group for group in every if _model_group_reads_vertex_target(group, llm_router, team_id)), + team_id=team_id, + ) + _ClientSideAllowlistSetting: TypeAlias = Literal[ "allowed_client_side_s3_buckets", "allowed_client_side_s3_endpoint_urls" @@ -538,12 +698,14 @@ def _banned_param_error(param: str) -> ValueError: def _check_banned_params( body: Mapping[str, object], - general_settings: dict, + general_settings: Mapping[str, object], llm_router: Router | None, - model: str, *, manages_deployments: bool = False, mcp_server_fields_allowed: bool = False, + banned_params: tuple[str, ...] = _BANNED_REQUEST_BODY_PARAMS, + opt_in_groups: Callable[[], _OptInGroups], + defer_model_checks: bool = False, defer_s3_buckets: bool = False, ) -> None: """Raise ``ValueError`` if ``body`` carries a banned param without admin opt-in. @@ -560,7 +722,7 @@ def _check_banned_params( reject_server_owned_wif_params(wif_body) if not manages_deployments: reject_federated_credential_reference(body) - for param in _BANNED_REQUEST_BODY_PARAMS: + for param in banned_params: if param not in body: continue if general_settings.get("allow_client_side_credentials") is True: @@ -573,14 +735,20 @@ def _check_banned_params( continue if defer_s3_buckets and param in REQUEST_S3_BUCKET_PARAMS: continue - if ( + if defer_model_checks: + continue + if all( _allow_model_level_clientside_configurable_parameters( - model=model, + model=model_group, param=param, request_body_value=body[param], llm_router=llm_router, + team_id=opt_in_groups().team_id, ) is True + for model_group in ( + opt_in_groups().vertex_target if param in VERTEX_TARGET_PARAMS else opt_in_groups().every + ) ): # Per-param opt-in: only THIS param is permitted by the # deployment's ``configurable_clientside_auth_params``. Skip @@ -664,27 +832,39 @@ def fallback_target_model_name(target: object) -> str | None: return None -def _direct_fallback_model_names(model: str, llm_router: Router) -> frozenset[str]: +_EVERY_FALLBACK_TYPE: Final[tuple[str, ...]] = ("general", "context_window", "content_policy") + + +def _direct_fallback_model_names(model: str, llm_router: Router, fallback_types: tuple[str, ...]) -> frozenset[str]: from litellm.proxy.auth.model_checks import get_all_fallbacks - return frozenset( - name for target in get_all_fallbacks(model, llm_router) if (name := fallback_target_model_name(target)) + targets: Final = chain.from_iterable( + get_all_fallbacks(model, llm_router, fallback_type) for fallback_type in fallback_types ) + return frozenset(name for target in targets if (name := fallback_target_model_name(target))) def _fallback_hop( - frontier_and_reached: tuple[frozenset[str], frozenset[str]], llm_router: Router + frontier_and_reached: tuple[frozenset[str], frozenset[str]], router_and_types: tuple[Router, tuple[str, ...]] ) -> tuple[frozenset[str], frozenset[str]]: frontier, reached = frontier_and_reached + llm_router, fallback_types = router_and_types next_hop: Final = ( - frozenset(chain.from_iterable(_direct_fallback_model_names(model, llm_router) for model in frontier)) - reached + frozenset( + chain.from_iterable(_direct_fallback_model_names(model, llm_router, fallback_types) for model in frontier) + ) + - reached ) return next_hop, reached | next_hop -def router_fallback_model_names(model: str, llm_router: Router) -> frozenset[str]: +def router_fallback_model_names( + model: str, llm_router: Router, fallback_types: tuple[str, ...] = ("general",) +) -> frozenset[str]: """The groups a failure on `model` can reach through the router's fallbacks, chained and generic (`*`) ones.""" - hops: Final = accumulate(repeat(llm_router), _fallback_hop, initial=(frozenset({model}), frozenset({model}))) + hops: Final = accumulate( + repeat((llm_router, fallback_types)), _fallback_hop, initial=(frozenset({model}), frozenset({model})) + ) _, reached = next(hop for hop in hops if not hop[0]) return reached - {model} @@ -714,12 +894,14 @@ def _reject_url_valued_fallback_target(value: str) -> None: def is_request_body_safe( - request_body: dict, - general_settings: dict, + request_body: Mapping[str, object], + general_settings: Mapping[str, object], llm_router: Router | None, model: str, *, route: str | None = None, + team_id: str | None = None, + defer_model_checks: bool = False, defer_s3_buckets: bool = False, ) -> bool: """ @@ -750,28 +932,72 @@ def is_request_body_safe( if "model_list" in request_body: raise ValueError("Rejected Request: model_list is not allowed in the request body.") manages_deployments: Final = route_manages_deployments(route) - check_banned_params: Final = partial( - _check_banned_params, - general_settings=general_settings, - llm_router=llm_router, - model=model, + root_body: Final = _without_route_named_vertex_target(request_body, route) + route_model: Final = _model_from_route_path(route) if route is not None else None + also_requested_models: Final = ( + *request_fallback_model_names(root_body), + *((route_model,) if route_model else ()), + ) + opt_in_groups: Final = cache( + lambda: _resolve_opt_in_groups(model or route_model or "", also_requested_models, llm_router, team_id) + ) + _check_banned_params( + root_body, + general_settings, + llm_router, manages_deployments=manages_deployments, + mcp_server_fields_allowed=route_manages_mcp_servers(route), + opt_in_groups=opt_in_groups, + defer_model_checks=defer_model_checks, defer_s3_buckets=defer_s3_buckets, ) - check_banned_params(request_body, mcp_server_fields_allowed=route_manages_mcp_servers(route)) for nested_key in _NESTED_CONFIG_KEYS: nested = _coerce_metadata_to_dict(request_body.get(nested_key)) if nested is not None: - check_banned_params(nested) + _check_banned_params( + nested, + general_settings, + llm_router, + manages_deployments=manages_deployments, + opt_in_groups=opt_in_groups, + defer_model_checks=defer_model_checks, + defer_s3_buckets=defer_s3_buckets, + ) for metadata_key in _NESTED_METADATA_KEYS: metadata = _coerce_metadata_to_dict(request_body.get(metadata_key)) if metadata is not None: - check_banned_params(metadata) - if any(isinstance(key, str) and key.startswith(f"{metadata_key}[") for key in request_body): - check_banned_params(extract_nested_form_metadata(form_data=request_body, prefix=f"{metadata_key}[")) + _check_banned_params( + metadata, + general_settings, + llm_router, + manages_deployments=manages_deployments, + banned_params=_BANNED_METADATA_PARAMS, + opt_in_groups=opt_in_groups, + defer_model_checks=defer_model_checks, + defer_s3_buckets=defer_s3_buckets, + ) + if any(key.startswith(f"{metadata_key}[") for key in request_body): + _check_banned_params( + extract_nested_form_metadata(form_data=request_body, prefix=f"{metadata_key}["), + general_settings, + llm_router, + manages_deployments=manages_deployments, + banned_params=_BANNED_METADATA_PARAMS, + opt_in_groups=opt_in_groups, + defer_model_checks=defer_model_checks, + defer_s3_buckets=defer_s3_buckets, + ) for target in iter_request_fallback_targets(request_body): if isinstance(target, dict): - check_banned_params(target) + _check_banned_params( + target, + general_settings, + llm_router, + manages_deployments=manages_deployments, + opt_in_groups=opt_in_groups, + defer_model_checks=defer_model_checks, + defer_s3_buckets=defer_s3_buckets, + ) target_model = target.get("model") if isinstance(target_model, str): _reject_url_valued_fallback_target(target_model) @@ -784,8 +1010,17 @@ def is_request_body_safe( _reject_request_credential_params(litellm_params, manages_deployments=manages_deployments) litellm_params_metadata: Final = _coerce_metadata_to_dict(litellm_params.get("metadata")) if litellm_params_metadata is not None: - check_banned_params(litellm_params_metadata) - for user_config_params in _user_config_litellm_params(request_body): # pyright: ignore[reportUnknownArgumentType] # request_body is an untyped dict + _check_banned_params( + litellm_params_metadata, + general_settings, + llm_router, + manages_deployments=manages_deployments, + banned_params=_BANNED_METADATA_PARAMS, + opt_in_groups=opt_in_groups, + defer_model_checks=defer_model_checks, + defer_s3_buckets=defer_s3_buckets, + ) + for user_config_params in _user_config_litellm_params(request_body): reject_api_key_secret_reference(user_config_params) return True @@ -880,6 +1115,8 @@ def enforce_team_s3_bucket_allowlist( general_settings: Mapping[str, object], llm_router: Router | None, team_metadata: Mapping[str, object] | None, + *, + team_id: str | None = None, ) -> None: """Re-run the request body check that auth relaxed for bucket params, now admitting the caller team's buckets.""" proxy_buckets: Final = _ClientSideStorageAllowlist.model_validate(general_settings).allowed_client_side_s3_buckets @@ -893,6 +1130,7 @@ def enforce_team_s3_bucket_allowlist( }, llm_router=llm_router, model=model if isinstance(model, str) else "", + team_id=team_id, ) except ValueError as rejection: raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=f"Authentication Error, {rejection}") @@ -902,6 +1140,8 @@ async def pre_db_read_auth_checks( request: Request, request_data: dict, route: str, + *, + defer_team_checks: bool = False, ): """ 1. Checks if request size is under max_request_size_mb (if set) @@ -921,12 +1161,17 @@ async def pre_db_read_auth_checks( await check_if_request_size_is_safe(request=request) # Check 2. Request body is safe + defer_model_checks: Final = ( + defer_team_checks and llm_router is not None and bool(llm_router.registry().index("team_model")) + ) + request.state.deferred_model_body_check = defer_model_checks # rebind-ok: hand state to auth is_request_body_safe( request_body=request_data, general_settings=general_settings, llm_router=llm_router, - model=request_data.get("model", ""), # [TODO] use model passed in url as well (azure openai routes) + model=request_data.get("model") or _model_from_route_path(route) or "", route=route, + defer_model_checks=defer_model_checks, defer_s3_buckets=checks_team_s3_buckets_in_handler(request), ) @@ -2454,37 +2699,8 @@ def get_model_from_request( team_id=team_id, ) model = _format_model_candidates(candidates) - - # If no explicit model was found, try to extract from route if model is None: - # Parse model from route that follows the pattern /openai/deployments/{model}/* - match: Final = re.match(r"/openai/deployments/([^/]+)", route) - if match: - model = match.group(1) - - # If still not found, extract model from Google generateContent-style routes. - # These routes put the model in the path and allow "/" inside the model id. - # Examples: - # - /v1beta/models/gemini-2.0-flash:generateContent - # - /v1beta/models/bedrock/claude-sonnet-3.7:generateContent - # - /models/custom/ns/model:streamGenerateContent - if model is None and not route.lower().startswith("/vertex"): - google_match = re.search(r"/(?:v1beta|beta)/models/([^:]+):", route) - if google_match: - model = google_match.group(1) - - if model is None and not route.lower().startswith("/vertex"): - google_match = re.search(r"^/models/([^:]+):", route) - if google_match: - model = google_match.group(1) - - # If still not found, extract from Vertex AI passthrough route - # Pattern: /vertex_ai/.../models/{model_id}:* - # Example: /vertex_ai/v1/.../models/gemini-1.5-pro:generateContent - if model is None and route.lower().startswith("/vertex"): - vertex_match: Final = re.search(r"/models/([^:]+)", route) - if vertex_match: - model = vertex_match.group(1) + model = _model_from_route_path(route) if route.lower().startswith("/bedrock"): bedrock_model: Final = _model_from_bedrock_route(route) @@ -2503,6 +2719,23 @@ def get_model_from_request( return model +def _model_from_route_path(route: str) -> str | None: + """ + Google generateContent-style routes allow "/" inside the model id, e.g. + /v1beta/models/bedrock/claude-sonnet-3.7:generateContent + """ + azure_deployment: Final = re.match(r"/openai/deployments/([^/]+)", route) + if azure_deployment: + return azure_deployment.group(1) + if route.lower().startswith("/vertex"): + vertex_model: Final = re.search(r"/models/([^:]+)", route) + return vertex_model.group(1) if vertex_model else None + google_model: Final = re.search(r"/(?:v1beta|beta)/models/([^:]+):", route) or re.search( + r"^/models/([^:]+):", route + ) + return google_model.group(1) if google_model else None + + def _router_model_from_azure_route(route: str, llm_router: Router | None) -> str | None: if llm_router is None: return None diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 1dca6075a3d..5bf4db6a725 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -109,12 +109,14 @@ from litellm.proxy.auth.auth_object_prefetch import ( ) from litellm.proxy.auth.auth_utils import ( abbreviate_api_key, + checks_team_s3_buckets_in_handler, fallback_target_model_name, get_end_user_id_from_request_body, get_model_from_request, get_request_route, get_request_route_template, is_invalid_virtual_key_error, + is_request_body_safe, iter_request_fallback_targets, log_model_access_denial, normalize_request_route, @@ -2276,6 +2278,7 @@ async def user_api_key_auth_builder( azure_apim_header: str | None, request_data: dict[str, object], custom_litellm_key_header: str | None = None, + defer_team_checks: bool = False, ) -> UserAPIKeyAuth: from litellm.proxy.proxy_server import ( general_settings, @@ -2310,6 +2313,7 @@ async def user_api_key_auth_builder( request_data=request_data, request=request, route=route, + defer_team_checks=defer_team_checks, ) pass_through_endpoints: Final[list[dict] | None] = general_settings.get("pass_through_endpoints", None) ## CHECK IF X-LITELM-API-KEY IS PASSED IN - supercedes Authorization header @@ -3684,6 +3688,18 @@ async def run_centralized_common_checks( user_custom_auth, ) + if not sub_call and getattr(request.state, "deferred_model_body_check", False) is True: + body_model: Final = request_data.get("model") + is_request_body_safe( + request_body=request_data, + general_settings=_GENERAL_SETTINGS_ADAPTER.validate_python(general_settings), + llm_router=llm_router, + model=body_model if isinstance(body_model, str) else "", + route=route, + team_id=user_api_key_auth_obj.team_id, + defer_s3_buckets=checks_team_s3_buckets_in_handler(request), + ) + # Public routes (e.g. /health/liveness) are exempt from # auth in the builder — the wrapper must not retroactively apply # authz on top, or k8s readiness probes and other unauthenticated @@ -4520,6 +4536,7 @@ async def user_api_key_auth( azure_apim_header=azure_apim_header, request_data=request_data, custom_litellm_key_header=custom_litellm_key_header, + defer_team_checks=True, ) except Exception: # The body was read first, so a caller who sent both a malformed body and @@ -4999,7 +5016,7 @@ async def authorize_internal_virtual_key( ) ) route: Final = get_request_route(request=request) - await pre_db_read_auth_checks(request_data=request_data, request=request, route=route) + await pre_db_read_auth_checks(request_data=request_data, request=request, route=route, defer_team_checks=True) auth: Final = await validate_resolved_virtual_key( request=request, request_data=request_data, diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 00fa7f3c33e..0b443572fe6 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -303,6 +303,7 @@ async def create_batch( general_settings=general_settings_view(), llm_router=llm_router, team_metadata=user_api_key_dict.team_metadata, + team_id=user_api_key_dict.team_id, ) _raise_when_routing_headers_disagree(request, data.get("model")) verbose_proxy_logger.debug( diff --git a/litellm/proxy/batches_endpoints/litellm_executed_batches.py b/litellm/proxy/batches_endpoints/litellm_executed_batches.py index 4dd6496812e..8c78e650339 100644 --- a/litellm/proxy/batches_endpoints/litellm_executed_batches.py +++ b/litellm/proxy/batches_endpoints/litellm_executed_batches.py @@ -452,7 +452,7 @@ class LiteLLMExecutedBatchRunner: ) -> LiteLLMBatch: endpoint: Final = _validate_endpoint(create_request.get("endpoint")) content: Final = await self._download_input(unified_input_file_id, user_api_key_dict) - parsed: Final = parse_batch_input(content, endpoint, self._body_rejection(model)) + parsed: Final = parse_batch_input(content, endpoint, self._body_rejection(model, user_api_key_dict.team_id)) if isinstance(parsed, InvalidBatchInput): raise batch_error(400, f"Invalid batch input file: {parsed.describe()}") llm_batch_id: Final = f"{LITELLM_EXECUTED_BATCH_ID_PREFIX}{uuid_module.uuid4().hex}" @@ -533,7 +533,7 @@ class LiteLLMExecutedBatchRunner: return failed return await self.batches.load_batch(batch.id) or batch - def _body_rejection(self, model: str) -> BodyRejection: + def _body_rejection(self, model: str, team_id: str | None) -> BodyRejection: def reject(body: Mapping[str, object]) -> str | None: try: is_request_body_safe( @@ -541,6 +541,7 @@ class LiteLLMExecutedBatchRunner: general_settings=dict(self.general_settings), llm_router=self.llm_router, model=model, + team_id=team_id, ) except ValueError as e: return str(e) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 333687b07cd..1211c56975b 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -62,6 +62,7 @@ from litellm.proxy.auth.auth_exception_handler import ( ) from litellm.proxy.auth.auth_utils import ( _BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check + VERTEX_TARGET_PARAMS, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils import http_parsing_utils @@ -156,6 +157,11 @@ _CONFIG_CONNECTION_FIELDS: Final[frozenset[str]] = frozenset( ) +_BANNED_CONNECTION_PARAMS: Final[tuple[str, ...]] = tuple( + param for param in _BANNED_REQUEST_BODY_PARAMS if param not in VERTEX_TARGET_PARAMS +) + + def _request_inherits_config_credentials( config_params: Mapping[str, object], request_params: Mapping[str, object], @@ -181,7 +187,7 @@ def _request_inherits_config_credentials( return False if allow_client_side_credentials: return True - return not any(param in request_params for param in _BANNED_REQUEST_BODY_PARAMS) + return not any(param in request_params for param in _BANNED_CONNECTION_PARAMS) def _admin_api_key_from_request( diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index ca1470bc94e..8b8d9ec81be 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -639,6 +639,7 @@ async def create_file( general_settings=general_settings_view(), llm_router=llm_router, team_metadata=user_api_key_dict.team_metadata, + team_id=user_api_key_dict.team_id, ) file_params: Final = await extract_file_creation_params( request=request, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 5bdb2219194..6ab31e7d4b0 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -93,6 +93,7 @@ from litellm.proxy.auth.auth_checks import ( key_model_aliases_for_auth_check, team_model_aliases_for_auth_check, ) +from litellm.proxy.auth.auth_utils import deployment_clientside_auth_params from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401 # legacy module exports @@ -4241,6 +4242,10 @@ VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON: Final = ( "Vertex AI auth failed: set a use_in_pass_through vertex model, default_vertex_config, or DEFAULT_VERTEXAI_* env" ) +VERTEX_LIVE_UNPINNED_PROJECT_CLOSE_REASON: Final = ( + "Rejected Request: vertex_project must come from the proxy's configuration, see allow_client_side_credentials" +) + VERTEX_PUBLISHER_MODEL_PREFIX: Final = "publishers/google/models/" VERTEX_PUBLISHERS_SEGMENT: Final = "publishers/" @@ -4265,11 +4270,25 @@ def _get_llm_router() -> Router | None: return llm_router -def _resolve_vertex_live_credentials( +@dataclass(frozen=True, slots=True) +class _ConfiguredVertexLiveTarget: + credentials: VertexPassThroughCredentials | None + + +@dataclass(frozen=True, slots=True) +class _DeploymentVertexLiveTarget: + credentials: VertexPassThroughCredentials + deployment_params: LiteLLMParamsTypedDict + + +_VertexLiveTarget = _ConfiguredVertexLiveTarget | _DeploymentVertexLiveTarget + + +def _resolve_vertex_live_target( vertex_project: str | None, vertex_location: str | None, model: str | None, -) -> VertexPassThroughCredentials | None: +) -> _VertexLiveTarget: """ Resolution order: an explicit project/location registration, then ``default_vertex_config`` (which the proxy fills from the ``DEFAULT_VERTEXAI_*`` env vars whenever the yaml leaves it out), then any DB model entry @@ -4283,36 +4302,133 @@ def _resolve_vertex_live_credentials( location=vertex_location, ) if keyed is not None and keyed.vertex_project is not None: - return keyed - from_deployments: Final = passthrough_endpoint_router.get_vertex_credentials_from_router_deployments(model=model) + return _ConfiguredVertexLiveTarget(credentials=keyed) + from_deployments: Final = passthrough_endpoint_router.get_vertex_deployment_from_router(model=model) if from_deployments is not None: - return from_deployments + deployment, credentials = from_deployments + return _DeploymentVertexLiveTarget(credentials=credentials, deployment_params=deployment["litellm_params"]) if keyed is not None: - return keyed + return _ConfiguredVertexLiveTarget(credentials=keyed) passthrough_endpoint_router.set_default_vertex_config() - return passthrough_endpoint_router.get_vertex_credentials( - project_id=vertex_project, - location=vertex_location, + return _ConfiguredVertexLiveTarget( + credentials=passthrough_endpoint_router.get_vertex_credentials( + project_id=vertex_project, + location=vertex_location, + ) ) +def _requested_vertex_live_target( + credentials: VertexPassThroughCredentials, vertex_project: str | None, vertex_location: str | None +) -> tuple[tuple[str, str | None, str | None], ...]: + return ( + ("vertex_project", vertex_project, credentials.vertex_project), + ("vertex_location", vertex_location, credentials.vertex_location), + ) + + +def _proxy_configured_vertex_projects() -> frozenset[str]: + return frozenset( + project + for project in ( + litellm.vertex_project, + get_secret_str("VERTEXAI_PROJECT"), + get_secret_str("DEFAULT_VERTEXAI_PROJECT"), + ) + if project + ) + + +def _names_an_unconfigured_project( + credentials: VertexPassThroughCredentials | None, vertex_project: str | None +) -> bool: + return ( + bool(vertex_project) + and (credentials is None or credentials.vertex_project is None) + and vertex_project not in _proxy_configured_vertex_projects() + ) + + +def _live_target_override_reason( + target: _VertexLiveTarget, + vertex_project: str | None, + vertex_location: str | None, + allow_client_side_credentials: bool, +) -> str | None: + if allow_client_side_credentials: + return None + match target: + case _ConfiguredVertexLiveTarget(credentials=credentials) if _names_an_unconfigured_project( + credentials, vertex_project + ): + return VERTEX_LIVE_UNPINNED_PROJECT_CLOSE_REASON + case _ConfiguredVertexLiveTarget(credentials=VertexPassThroughCredentials() as credentials): + configured_field: Final = next( + ( + name + for name, asked, pinned in _requested_vertex_live_target( + credentials, vertex_project, vertex_location + ) + if asked and pinned and asked != pinned + ), + None, + ) + if configured_field is None: + return None + return ( + f"Rejected Request: {configured_field} is pinned by default_vertex_config, " + "see allow_client_side_credentials" + ) + case _ConfiguredVertexLiveTarget(): + return None + case _DeploymentVertexLiveTarget(credentials=credentials, deployment_params=deployment_params): + opted_in: Final = deployment_clientside_auth_params(deployment_params) or () + deployment_field: Final = next( + ( + name + for name, asked, pinned in _requested_vertex_live_target( + credentials, vertex_project, vertex_location + ) + if asked and asked != pinned and name not in opted_in + ), + None, + ) + if deployment_field is None: + return None + return ( + f"Rejected Request: {deployment_field} is pinned by the deployment, " + "see configurable_clientside_auth_params" + ) + + +_VERTEX_RESOURCE_PROJECT_AND_LOCATION: Final = re.compile(r"projects/[^/]+/locations/(?P[^/]+)/") + + def _build_vertex_live_setup_model_rewriter( vertex_project: str | None, vertex_location: str | None, + configured_location: str | None, llm_router: Router | None, + keeps_caller_resource_path: bool, ) -> Callable[[str], str] | None: """ Rewrite the ``setup`` frame's model into the full Vertex resource path the Live API requires. Clients address the gateway the way they address LiteLLM (bare id or model alias); Vertex reads anything - that is not a ``projects/...`` path as a project name and closes the socket + that is not a ``projects/...`` path as a project name and closes the socket. A ``projects/...`` path + picks the project the proxy's token is spent on, so it is moved onto the connection's project, and onto its + region when one is configured, unless ``allow_client_side_credentials`` lets callers choose them """ if vertex_project is None or vertex_location is None: return None def rewrite(setup_model: str) -> str: if setup_model.startswith("projects/"): - return setup_model + resource_prefix: Final = _VERTEX_RESOURCE_PROJECT_AND_LOCATION.match(setup_model) + if keeps_caller_resource_path or resource_prefix is None: + return setup_model + location: Final = configured_location or resource_prefix.group("location") + return f"projects/{vertex_project}/locations/{location}/{setup_model[resource_prefix.end() :]}" aliased: Final = _resolve_alias_to_upstream_model(setup_model, llm_router) return f"projects/{vertex_project}/locations/{vertex_location}/{_vertex_publisher_model_suffix(aliased)}" @@ -4366,11 +4482,22 @@ async def vertex_ai_live_websocket_passthrough( await websocket.accept() incoming_headers: Final = dict(websocket.headers) - vertex_credentials_config: Final = _resolve_vertex_live_credentials( + live_target: Final = _resolve_vertex_live_target( vertex_project=vertex_project, vertex_location=vertex_location, model=model, ) + allow_client_side_credentials: Final = _proxy_general_settings().get("allow_client_side_credentials") is True + override_reason: Final = _live_target_override_reason( + live_target, + vertex_project=vertex_project, + vertex_location=vertex_location, + allow_client_side_credentials=allow_client_side_credentials, + ) + if override_reason is not None: + await websocket.close(code=1008, reason=override_reason) + return + vertex_credentials_config: Final = live_target.credentials configured_project: Final = vertex_project or ( vertex_credentials_config.vertex_project if vertex_credentials_config is not None else None @@ -4452,7 +4579,9 @@ async def vertex_ai_live_websocket_passthrough( setup_model_rewriter=_build_vertex_live_setup_model_rewriter( vertex_project=resolved_project, vertex_location=resolved_location, + configured_location=configured_location, llm_router=_get_llm_router(), + keeps_caller_resource_path=allow_client_side_credentials, ), ) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 66634688b9a..42a7c458ef2 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2613,6 +2613,21 @@ def _rewrite_vertex_live_setup_model(text_data: str, setup_model_rewriter: Calla return json.dumps({**message, "setup": {**setup, "model": rewritten_model}}) +VERTEX_LIVE_UNDECODABLE_BINARY_CLOSE_REASON: Final = "Rejected Request: binary messages must be UTF-8 encoded JSON" + + +def _rewrite_vertex_live_binary_setup_model( + bytes_data: bytes, setup_model_rewriter: Callable[[str], str] | None +) -> bytes | None: + if setup_model_rewriter is None: + return bytes_data + try: + text_data: Final = bytes_data.decode("utf-8") + except UnicodeDecodeError: + return None + return _rewrite_vertex_live_setup_model(text_data, setup_model_rewriter).encode("utf-8") + + def _resolved_vertex_live_setup( setup_data: Mapping[str, object], setup_model_rewriter: Callable[[str], str] | None ) -> Mapping[str, object]: @@ -3042,6 +3057,28 @@ async def _relay_client_frame( return verdict +_UNDECODABLE_BINARY_CLIENT_FRAME: Final = _RefuseClientFrame( + refusal=WebsocketFrameModelRefused( + message=VERTEX_LIVE_UNDECODABLE_BINARY_CLOSE_REASON, error_type="invalid_request_error" + ) +) + + +async def _relay_client_binary_frame( + websocket: WebSocket, + upstream_ws: ClientConnection, + bytes_data: bytes, + setup_model_rewriter: Callable[[str], str] | None, + gate: WebsocketFrameModelGate | None, + ledger: _WebsocketTurnLedger, +) -> _ForwardClientFrame | _RefuseClientFrame: + rewritten_bytes: Final = _rewrite_vertex_live_binary_setup_model(bytes_data, setup_model_rewriter) + if rewritten_bytes is None: + await websocket.close(code=CloseCode.POLICY_VIOLATION, reason=VERTEX_LIVE_UNDECODABLE_BINARY_CLOSE_REASON) + return _UNDECODABLE_BINARY_CLIENT_FRAME + return await _relay_client_frame(websocket, upstream_ws, bytes_data, rewritten_bytes, gate, ledger) + + _EXPLICIT_SUCCESS_KWARGS: Final = frozenset({"logging_obj", "passthrough_logging_payload", "custom_llm_provider"}) WALKOUT_TURN_DRAIN_SECONDS: Final = 600.0 @@ -3409,8 +3446,13 @@ async def websocket_passthrough_request( metered_total += _metered_client_text(meter, text_data) elif bytes_data is not None: if isinstance( - await _relay_client_frame( - websocket, upstream_ws, bytes_data, bytes_data, client_frame_model_gate, ledger + await _relay_client_binary_frame( + websocket, + upstream_ws, + bytes_data, + setup_model_rewriter, + client_frame_model_gate, + ledger, ), _RefuseClientFrame, ): diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index e793dac4d06..877c4ece6c7 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -223,6 +223,12 @@ class PassthroughEndpointRouter: return provider def get_vertex_credentials_from_router_deployments(self, model: str | None) -> VertexPassThroughCredentials | None: + found: Final = self.get_vertex_deployment_from_router(model) + return found[1] if found is not None else None + + def get_vertex_deployment_from_router( + self, model: str | None + ) -> tuple[DeploymentTypedDict, VertexPassThroughCredentials] | None: """ Resolve vertex pass-through credentials from the live router deployments flagged ``use_in_pass_through``. @@ -242,7 +248,7 @@ class PassthroughEndpointRouter: ) matched: Final = next( ( - credentials + (deployment, credentials) for deployment, credentials in resolved if model is not None and self._deployment_matches_model(deployment, model) ), @@ -260,7 +266,7 @@ class PassthroughEndpointRouter: ) if len(targets) != 1: return None - return resolved[0][1] + return resolved[0] def _resolve_vertex_deployment_credentials( self, litellm_params: LiteLLMParamsTypedDict diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index b7775b99419..04d6c78f08a 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -1111,6 +1111,7 @@ async def test_prompt( general_settings=general_settings, llm_router=llm_router, model=data.get("model", ""), + team_id=user_api_key_dict.team_id, ) # Use ProxyBaseLLMRequestProcessing to go through all proxy logic diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3fddca6f4d8..f69d9bd4724 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16496,7 +16496,9 @@ async def model_info_lookup(model: str, custom_llm_provider: str | None = None): dependencies=[Depends(user_api_key_auth)], response_model=RawRequestTypedDict, ) -async def transform_request(request: TransformRequestBody): +async def transform_request( + request: TransformRequestBody, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] +): from litellm.utils import return_raw_request try: @@ -16505,6 +16507,7 @@ async def transform_request(request: TransformRequestBody): general_settings=general_settings, llm_router=llm_router, model=request.request_body.get("model", ""), + team_id=user_api_key_dict.team_id, ) except ValueError as e: raise HTTPException(status_code=400, detail={"error": str(e)}) diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index f4710fa30e3..47050f01644 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -8,6 +8,7 @@ Provides: import base64 import json +import re from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -21,12 +22,13 @@ from starlette.datastructures import UploadFile import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH, LITELLM_PROXY_ADMIN_NAME from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( LiteLLM_ManagedVectorStore, ) +from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import * -from litellm.proxy.auth.auth_utils import is_request_body_safe +from litellm.proxy.auth.auth_utils import get_request_route, is_request_body_safe from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -70,9 +72,12 @@ from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, with_managed_search_file_ids, ) +from litellm.rag.ingestion.vertex_ai_ingestion import DEFAULT_VERTEX_RAG_LOCATION from litellm.rag.main import get_ingestion_class from litellm.rag.rag_query import RAGQuery from litellm.repositories.table_repositories import ManagedVectorStoresRepository +from litellm.repositories.user_repository import UserRepository +from litellm.secret_managers.main import get_secret_str from litellm.types.rag import RAGIngestResponse from litellm.types.utils import Choices, ModelResponse @@ -192,6 +197,103 @@ def _ingest_provider_error(vector_store_config: Mapping[str, object]) -> str | N return None +_VERTEX_RAG_CORPUS_NAME: Final = re.compile( + r"projects/(?P[^/%\\]+)/locations/(?P[^/%\\]+)/ragCorpora/(?P[^/%\\]+)" +) +_DOT_SEGMENTS: Final = frozenset({".", ".."}) + + +def _clean_vertex_corpus_name(corpus_name: object) -> re.Match[str] | None: + if not isinstance(corpus_name, str) or not _DOT_SEGMENTS.isdisjoint(corpus_name.split("/")): + return None + return _VERTEX_RAG_CORPUS_NAME.fullmatch(corpus_name) + + +def _vertex_target_message(field: str) -> str: + return ( + f"Rejected Request: {field} picks a Vertex project or region other than the proxy's. " + "Use a vector store a proxy admin created, or set general_settings.allow_client_side_credentials" + ) + + +def _vertex_target_is_checked( + vector_store_config: Mapping[str, object], + user_api_key_dict: UserAPIKeyAuth, + general_settings: Mapping[str, object], +) -> bool: + return ( + vector_store_config.get("custom_llm_provider") == "vertex_ai" + and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN + and general_settings.get("allow_client_side_credentials") is not True + ) + + +def _configured_vertex_project() -> str | None: + return litellm.vertex_project or get_secret_str("VERTEXAI_PROJECT") + + +def _corpus_on_configured_project(vector_store_config: Mapping[str, object]) -> Mapping[str, object]: + corpus: Final = _clean_vertex_corpus_name(vector_store_config.get("vector_store_id")) + configured_project: Final = _configured_vertex_project() + if corpus is None or configured_project is None or not corpus["project"].isdigit(): + return vector_store_config + return MappingProxyType( + { + **vector_store_config, + "vector_store_id": ( + f"projects/{configured_project}/locations/{corpus['location']}/ragCorpora/{corpus['corpus']}" + ), + } + ) + + +async def _authorize_rewritten_corpus( + requested: Mapping[str, object], rewritten: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth +) -> None: + rewritten_id: Final = rewritten.get("vector_store_id") + if isinstance(rewritten_id, str) and rewritten_id != requested.get("vector_store_id"): + await assert_user_can_access_vector_store_id(vector_store_id=rewritten_id, user_api_key_dict=user_api_key_dict) + + +def _vertex_target_error(vector_store_config: Mapping[str, object]) -> str | None: + corpus_name: Final = vector_store_config.get("vector_store_id") + corpus: Final = _clean_vertex_corpus_name(corpus_name) + if corpus_name and corpus is None: + return _vertex_target_message("vector_store_id") + configured_project: Final = _configured_vertex_project() + configured_location: Final = VertexBase.safe_get_vertex_ai_location({}) or DEFAULT_VERTEX_RAG_LOCATION + requested: Final = ( + *( + (name, vector_store_config.get(name), configured_project) + for name in ("vertex_project", "vertex_ai_project") + ), + *( + (name, vector_store_config.get(name), configured_location) + for name in ("vertex_location", "vertex_ai_location") + ), + ("vector_store_id", corpus["project"] if corpus is not None else None, configured_project), + ("vector_store_id", corpus["location"] if corpus is not None else None, configured_location), + ) + field: Final = next((name for name, asked, configured in requested if asked and asked != configured), None) + return None if field is None else _vertex_target_message(field) + + +async def _admin_set_store_target( + managed_store: LiteLLM_ManagedVectorStore | None, prisma_client: "PrismaClient | None" +) -> bool: + if managed_store is None: + return False + if managed_store.get("is_config", False): + return True + creator: Final = managed_store.get("user_id") + if creator == LITELLM_PROXY_ADMIN_NAME: + return True + if creator is None or prisma_client is None: + return False + creator_row: Final = await UserRepository(prisma_client).find_by_id(creator) + return creator_row is not None and creator_row.user_role == LitellmUserRoles.PROXY_ADMIN + + _MANAGED_STORE_CALLER_OPTIONS: Final = frozenset( { "vector_store_id", @@ -757,6 +859,7 @@ async def rag_ingest( from litellm.proxy.proxy_server import ( add_litellm_data_to_request, general_settings, + general_settings_view, llm_router, prisma_client, proxy_config, @@ -782,15 +885,25 @@ async def rag_ingest( general_settings=general_settings, llm_router=llm_router, model="", + route=get_request_route(request=request), + team_id=user_api_key_dict.team_id, ) except ValueError as e: raise HTTPException(status_code=400, detail={"error": str(e)}) managed_store: Final = resolved_stores.get(request_vector_store_config.get("vector_store_id")) - base: Final = { + requested_base: Final = { **_caller_vector_store_options(request_vector_store_config, managed_store), **_managed_store_overrides(managed_store), } + vertex_target_checked: Final = _vertex_target_is_checked( + requested_base, user_api_key_dict, general_settings_view() + ) and not await _admin_set_store_target(managed_store, prisma_client) + base: Final = _corpus_on_configured_project(requested_base) if vertex_target_checked else requested_base + await _authorize_rewritten_corpus(requested_base, base, user_api_key_dict) + vertex_target_error: Final = _vertex_target_error(base) if vertex_target_checked else None + if vertex_target_error is not None: + raise HTTPException(status_code=400, detail={"error": vertex_target_error}) request_data: Final[dict[str, Any]] = await add_litellm_data_to_request( data={}, request=request, diff --git a/litellm/rag/ingestion/vertex_ai_ingestion.py b/litellm/rag/ingestion/vertex_ai_ingestion.py index cceddf31117..8157c0c6405 100644 --- a/litellm/rag/ingestion/vertex_ai_ingestion.py +++ b/litellm/rag/ingestion/vertex_ai_ingestion.py @@ -28,6 +28,8 @@ if TYPE_CHECKING: from litellm import Router from litellm.types.rag import RAGIngestOptions +DEFAULT_VERTEX_RAG_LOCATION: Final = "us-central1" + def _present_fields(fields: tuple[tuple[str, object], ...]) -> Mapping[str, object]: return {name: value for name, value in fields if value} @@ -90,7 +92,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): # Get project, location, and credentials using VertexBase methods self.project_id = self.safe_get_vertex_ai_project(litellm_params) - self.location = self.get_vertex_ai_location(litellm_params) or "us-central1" + self.location = self.get_vertex_ai_location(litellm_params) or DEFAULT_VERTEX_RAG_LOCATION self.vertex_credentials = self.safe_get_vertex_ai_credentials(litellm_params) async def embed( diff --git a/tests/integration/spend/test_vertex_image_regional_uplift.py b/tests/integration/spend/test_vertex_image_regional_uplift.py index 6900a6d489a..dfd52d28f1f 100644 --- a/tests/integration/spend/test_vertex_image_regional_uplift.py +++ b/tests/integration/spend/test_vertex_image_regional_uplift.py @@ -100,6 +100,7 @@ def _deployment( location: Mapping[str, JsonValue], multiplier: JsonValue = 1.1, backend: str = _IMAGE_BACKEND, + clientside_params: tuple[str, ...] = (), ) -> str: return scenario.model( model=f"vertex_ai/{backend}", @@ -111,6 +112,7 @@ def _deployment( output_cost_per_token=4e-5, output_cost_per_image=0.05, **({} if multiplier is None else {"regional_endpoint_uplift_multiplier": multiplier}), + **({"configurable_clientside_auth_params": list(clientside_params)} if clientside_params else {}), **location, ) @@ -380,7 +382,13 @@ def test_request_body_location_overrides_the_deployment( gateway: Gateway, deployment_location: str, extra: Mapping[str, JsonValue], expected: float ) -> None: with wire_server(_scripted()) as wire, gateway.scenario() as scenario: - model: Final = _deployment(gateway, scenario, wire, location={"vertex_location": deployment_location}) + model: Final = _deployment( + gateway, + scenario, + wire, + location={"vertex_location": deployment_location}, + clientside_params=("vertex_location", "vertex_ai_location"), + ) call_id, response = _generate(gateway, model, extra=extra) _assert_images(response, call_id, expected) _assert_one_upstream_image_call(wire) diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 82b80ad8da9..73a2c534628 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -4,6 +4,7 @@ Unit tests for auth_utils functions related to rate limiting and customer ID ext import base64 import logging +import time from collections.abc import Callable, Mapping from types import SimpleNamespace from typing import Final, Optional @@ -371,7 +372,11 @@ _SERVICE_ACCOUNT_TOKEN_REFERENCE: Final = "oidc/file//var/run/secrets/kubernetes def _user_config_with(litellm_params: dict[str, str]) -> dict[str, object]: - return {"model_list": [{"model_name": "devstral", "litellm_params": {"model": "mistral/devstral-latest", **litellm_params}}]} + return { + "model_list": [ + {"model_name": "devstral", "litellm_params": {"model": "mistral/devstral-latest", **litellm_params}} + ] + } def _user_config_defaulting_to(litellm_params: dict[str, str]) -> dict[str, object]: @@ -391,7 +396,11 @@ def _user_config_search_tool_with(litellm_params: dict[str, str]) -> dict[str, o "search_tools": [ { "search_tool_name": "web", - "litellm_params": {"search_provider": "tavily", "api_base": "https://attacker.example", **litellm_params}, + "litellm_params": { + "search_provider": "tavily", + "api_base": "https://attacker.example", + **litellm_params, + }, } ] } @@ -512,7 +521,9 @@ def test_a_call_cannot_carry_an_oidc_api_key_in_litellm_params(route: str | None ) -@pytest.mark.parametrize("route", ["/model/new", "/model/update", "/model/deployment-1/update", "/health/test_connection"]) +@pytest.mark.parametrize( + "route", ["/model/new", "/model/update", "/model/deployment-1/update", "/health/test_connection"] +) def test_a_deployment_write_may_carry_an_oidc_api_key_in_litellm_params(route: str): """A proxy admin configures an oidc/ api_key on a deployment through these routes, and the model management role gate refuses anyone else with a 403.""" @@ -3675,6 +3686,7 @@ def test_request_cannot_select_chatgpt_auth_file(allow_client_side_credentials: model="chatgpt-account-a", ) + class TestIsRequestBodySafeBlocksBedrockProjectOverride: """``aws_bedrock_project_id`` pins a deployment to a Bedrock project so that project's data-retention policy applies to its requests. A @@ -3934,7 +3946,7 @@ class TestIsRequestBodySafeBlocksNVCFFunctionOverride: monkeypatch.setattr( auth_utils, "_allow_model_level_clientside_configurable_parameters", - lambda model, param, request_body_value, llm_router: param == "nvcf_function_id", + lambda model, param, request_body_value, llm_router, team_id=None: param == "nvcf_function_id", ) assert ( @@ -4019,7 +4031,7 @@ class TestIsRequestBodySafeBlocksRivaUseSsl: monkeypatch.setattr( auth_utils, "_allow_model_level_clientside_configurable_parameters", - lambda model, param, request_body_value, llm_router: param == "use_ssl", + lambda model, param, request_body_value, llm_router, team_id=None: param == "use_ssl", ) assert ( @@ -4457,7 +4469,7 @@ def test_model_level_allow_does_not_skip_subsequent_banned_params(monkeypatch): monkeypatch.setattr( auth_utils, "_allow_model_level_clientside_configurable_parameters", - lambda model, param, request_body_value, llm_router: param == "api_base", + lambda model, param, request_body_value, llm_router, team_id=None: param == "api_base", ) with pytest.raises(ValueError, match="Rejected Request: langfuse_host is not allowed in request") as exc: @@ -4981,6 +4993,538 @@ class TestIsRequestBodySafeBlocksAwsIdentitySelectors: ) +_VERTEX_TARGET_PARAMS: Final = ("vertex_location", "vertex_project", "vertex_ai_location", "vertex_ai_project") + + +def _vertex_pinned_router(configurable_clientside_auth_params: list[str]) -> Router: + return Router( + model_list=[ + { + "model_name": "gemini-2.5-flash", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-flash", + "vertex_project": "admin-project", + "vertex_location": "europe-west4", + "configurable_clientside_auth_params": configurable_clientside_auth_params, + }, + } + ] + ) + + +class TestIsRequestBodySafeBlocksVertexTargetOverrides: + @pytest.mark.parametrize("param", _VERTEX_TARGET_PARAMS) + @pytest.mark.parametrize( + "placement", + [ + lambda param: {param: "us-central1"}, + lambda param: {"extra_body": {param: "us-central1"}}, + ], + ids=["root", "extra_body"], + ) + def test_vertex_target_is_rejected_without_admin_opt_in( + self, param: str, placement: Callable[[str], dict[str, object]] + ) -> None: + with pytest.raises(ValueError, match=param): + is_request_body_safe( + request_body={"model": "gemini-2.5-flash", **placement(param)}, + general_settings={}, + llm_router=_vertex_pinned_router([]), + model="gemini-2.5-flash", + route="/v1/chat/completions", + ) + + @pytest.mark.parametrize("param", _VERTEX_TARGET_PARAMS) + @pytest.mark.parametrize( + "placement", + [ + lambda param: {"metadata": {param: "us-central1"}}, + lambda param: {"litellm_metadata": {param: "us-central1"}}, + lambda param: {f"metadata[{param}]": "us-central1"}, + lambda param: {"litellm_params": {"metadata": {param: "us-central1"}}}, + ], + ids=["metadata", "litellm_metadata", "form_metadata", "litellm_params_metadata"], + ) + def test_vertex_target_in_metadata_never_reaches_the_provider_and_is_allowed( + self, param: str, placement: Callable[[str], dict[str, object]] + ) -> None: + assert ( + is_request_body_safe( + request_body={"model": "gemini-2.5-flash", **placement(param)}, + general_settings={}, + llm_router=_vertex_pinned_router([]), + model="gemini-2.5-flash", + route="/v1/chat/completions", + ) + is True + ) + + def test_vertex_target_is_allowed_under_proxy_wide_opt_in(self) -> None: + assert ( + is_request_body_safe( + request_body={"model": "gemini-2.5-flash", **{param: "us-central1" for param in _VERTEX_TARGET_PARAMS}}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="gemini-2.5-flash", + route="/v1/chat/completions", + ) + is True + ) + + def test_deployment_opt_in_allows_only_the_listed_vertex_target(self) -> None: + router: Final = _vertex_pinned_router(["vertex_location"]) + assert ( + is_request_body_safe( + request_body={"model": "gemini-2.5-flash", "vertex_location": "us-central1"}, + general_settings={}, + llm_router=router, + model="gemini-2.5-flash", + route="/v1/chat/completions", + ) + is True + ) + with pytest.raises(ValueError, match="vertex_project"): + is_request_body_safe( + request_body={ + "model": "gemini-2.5-flash", + "vertex_location": "us-central1", + "vertex_project": "caller-project", + }, + general_settings={}, + llm_router=router, + model="gemini-2.5-flash", + route="/v1/chat/completions", + ) + + @pytest.mark.parametrize("opted_in_first", [True, False]) + def test_group_opt_in_does_not_count_while_any_deployment_lacks_it(self, opted_in_first: bool) -> None: + opted_in: Final = { + "model_name": "gemini-group", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-flash", + "vertex_location": "us-central1", + "configurable_clientside_auth_params": ["vertex_location"], + }, + } + pinned: Final = { + "model_name": "gemini-group", + "litellm_params": {"model": "vertex_ai/gemini-2.5-flash", "vertex_location": "europe-west4"}, + } + router: Final = Router(model_list=[opted_in, pinned] if opted_in_first else [pinned, opted_in]) + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe( + request_body={"model": "gemini-group", "vertex_location": "us-central1"}, + general_settings={}, + llm_router=router, + model="gemini-group", + route="/v1/chat/completions", + ) + + @pytest.mark.parametrize("model", ["chat", "chat-alias", "entry"]) + @pytest.mark.parametrize("opted_in", [False, True]) + def test_team_public_model_requires_the_team_deployment_opt_in(self, model: str, opted_in: bool) -> None: + router: Final = Router( + model_list=[ + {"model_name": "chat", "litellm_params": {"model": "openai/test-model", "api_key": "test-key"}}, + {"model_name": "entry", "litellm_params": {"model": "openai/test-model", "api_key": "test-key"}}, + { + "model_name": "team-chat", + "model_info": {"team_id": "team-one", "team_public_model_name": "chat"}, + "litellm_params": { + "model": "vertex_ai/test-model", + "vertex_location": "europe-west4", + "configurable_clientside_auth_params": ["vertex_location"] if opted_in else [], + }, + }, + ], + model_group_alias={"chat-alias": "chat"}, + fallbacks=[{"entry": ["chat"]}], + ) + request_body: Final = {"model": model, "vertex_location": "us-central1"} + if opted_in: + assert ( + is_request_body_safe(request_body, {}, router, model, route="/v1/chat/completions", team_id="team-one") + is True + ) + return + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe(request_body, {}, router, model, route="/v1/chat/completions", team_id="team-one") + + @pytest.mark.parametrize( + "model, opted_in, global_exact, is_allowed", + [ + ("chat-test-model", False, False, False), + ("chat-test-model", True, False, True), + ("other-test-model", False, False, True), + ("chat-test-model", False, True, True), + ], + ) + def test_team_public_wildcard_keeps_its_own_opt_in( + self, model: str, opted_in: bool, global_exact: bool, is_allowed: bool + ) -> None: + router: Final = Router( + model_list=[ + {"model_name": "*", "litellm_params": {"model": "openai/*", "api_key": "test-key"}}, + { + "model_name": "team-chat", + "model_info": {"team_id": "team-one", "team_public_model_name": "chat-*"}, + "litellm_params": { + "model": "vertex_ai/*", + "configurable_clientside_auth_params": ["vertex_location"] if opted_in else [], + }, + }, + ] + + ( + [ + { + "model_name": "chat-test-model", + "litellm_params": {"model": "openai/test-model", "api_key": "test-key"}, + } + ] + if global_exact + else [] + ) + ) + request_body: Final = {"model": model, "vertex_location": "us-central1"} + if is_allowed: + assert ( + is_request_body_safe(request_body, {}, router, model, route="/v1/chat/completions", team_id="team-one") + is True + ) + return + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe(request_body, {}, router, model, route="/v1/chat/completions", team_id="team-one") + + @pytest.mark.parametrize("fallback_setting", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) + def test_team_internal_fallback_to_another_public_name_keeps_its_pin(self, fallback_setting: str) -> None: + router: Final = Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": "openai/test-model", "api_key": "test-key"}} + for name in ("chat", "backup") + ] + + [ + { + "model_name": internal, + "model_info": {"team_id": "team-one", "team_public_model_name": public}, + "litellm_params": { + "model": "vertex_ai/test-model", + "configurable_clientside_auth_params": opt_in, + }, + } + for internal, public, opt_in in ( + ("team-chat", "chat", ["vertex_location"]), + ("team-backup", "backup", []), + ) + ], + **{fallback_setting: [{"team-chat": ["backup"]}]}, + ) + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe( + {"model": "chat", "vertex_location": "us-central1"}, + {}, + router, + "chat", + route="/v1/chat/completions", + team_id="team-one", + ) + + def test_model_with_no_deployment_has_no_opt_in(self) -> None: + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe( + request_body={"model": "not-configured", "vertex_location": "us-central1"}, + general_settings={}, + llm_router=_vertex_pinned_router(["vertex_location"]), + model="not-configured", + route="/v1/chat/completions", + ) + + def test_group_opt_in_counts_when_every_deployment_carries_it(self) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "gemini-group", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-flash", + "vertex_location": location, + "configurable_clientside_auth_params": ["vertex_location"], + }, + } + for location in ("us-central1", "europe-west4") + ] + ) + assert ( + is_request_body_safe( + request_body={"model": "gemini-group", "vertex_location": "us-central1"}, + general_settings={}, + llm_router=router, + model="gemini-group", + route="/v1/chat/completions", + ) + is True + ) + + @pytest.mark.parametrize( + "model, is_allowed", [("public-alias", False), ("public-gemini-2.5-flash", True)], ids=["alias", "direct"] + ) + def test_alias_opt_in_comes_from_the_group_the_alias_routes_to(self, model: str, is_allowed: bool) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "gemini-eu/*", + "litellm_params": {"model": "vertex_ai/*", "vertex_location": "europe-west4"}, + }, + { + "model_name": "public-*", + "litellm_params": { + "model": "vertex_ai/*", + "vertex_location": "europe-west4", + "configurable_clientside_auth_params": ["vertex_location"], + }, + }, + ], + model_group_alias={"public-alias": "gemini-eu/gemini-2.5-flash"}, + ) + request_body: Final = {"model": model, "vertex_location": "us-central1"} + if is_allowed: + assert is_request_body_safe(request_body, {}, router, model, route="/v1/chat/completions") is True + return + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe(request_body, {}, router, model, route="/v1/chat/completions") + + @staticmethod + def _opted_in_and_pinned_router( + router_fallbacks: list[dict[str, list[str]]], + fallback_setting: str = "fallbacks", + model_group_alias: dict[str, str] | None = None, + ) -> Router: + return Router( + model_list=[ + { + "model_name": name, + "litellm_params": { + "model": "vertex_ai/gemini-2.5-flash", + "vertex_location": "europe-west4", + "configurable_clientside_auth_params": opt_in, + }, + } + for name, opt_in in ( + ("gemini-optin", ["vertex_location"]), + ("gemini-optin-other", ["vertex_location"]), + ("gemini-pinned", []), + ) + ], + model_group_alias=model_group_alias or {}, + **{fallback_setting: router_fallbacks}, + ) + + @pytest.mark.parametrize( + "fields, is_allowed", + [ + ({"vertex_location": "us-central1", "fallbacks": ["gemini-pinned"]}, False), + ({"fallbacks": [{"model": "gemini-pinned", "vertex_location": "us-central1"}]}, False), + ({"vertex_location": "us-central1", "fallbacks": ["gemini-optin-other"]}, True), + ({"fallbacks": [{"model": "gemini-optin-other", "vertex_location": "us-central1"}]}, True), + ], + ids=["pinned", "pinned_entry_with_field", "opted_in", "opted_in_entry_with_field"], + ) + def test_request_fallback_needs_the_same_opt_in(self, fields: dict[str, object], is_allowed: bool) -> None: + router: Final = self._opted_in_and_pinned_router([]) + request_body: Final = {"model": "gemini-optin", **fields} + if is_allowed: + assert is_request_body_safe(request_body, {}, router, "gemini-optin", route="/v1/chat/completions") is True + return + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe(request_body, {}, router, "gemini-optin", route="/v1/chat/completions") + + @pytest.mark.parametrize("fallback_setting", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) + @pytest.mark.parametrize( + "fallback_group, is_allowed", + [("gemini-pinned", False), ("gemini-optin-other", True)], + ids=["pinned", "opted_in"], + ) + def test_router_fallback_needs_the_same_opt_in( + self, fallback_setting: str, fallback_group: str, is_allowed: bool + ) -> None: + router: Final = self._opted_in_and_pinned_router([{"gemini-optin": [fallback_group]}], fallback_setting) + request_body: Final = {"model": "gemini-optin", "vertex_location": "us-central1"} + if is_allowed: + assert is_request_body_safe(request_body, {}, router, "gemini-optin", route="/v1/chat/completions") is True + return + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe(request_body, {}, router, "gemini-optin", route="/v1/chat/completions") + + @pytest.mark.parametrize( + "model, fields, is_allowed", + [ + ("gemini-optin", {"vertex_location": "us-central1"}, True), + ("openai-group", {"vertex_location": "us-central1"}, True), + ("gemini-optin", {"api_base": "https://eu.example.com"}, False), + ("studio-group", {"vertex_location": "us-central1"}, False), + ], + ids=["vertex_field_to_openai_fallback", "vertex_field_on_openai", "api_base_to_openai_fallback", "gemini"], + ) + def test_only_google_groups_need_the_vertex_opt_in( + self, model: str, fields: dict[str, object], is_allowed: bool + ) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "gemini-optin", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-flash", + "configurable_clientside_auth_params": ["vertex_location", {"api_base": r"https://eu\..*"}], + }, + }, + {"model_name": "openai-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}, + {"model_name": "studio-group", "litellm_params": {"model": "gemini/gemini-2.5-flash", "api_key": "k"}}, + ], + fallbacks=[{"gemini-optin": ["openai-group"]}, {"studio-group": ["openai-group"]}], + ) + request_body: Final = {"model": model, **fields} + if is_allowed: + assert is_request_body_safe(request_body, {}, router, model, route="/v1/chat/completions") is True + return + with pytest.raises(ValueError, match=r"vertex_location|api_base"): + is_request_body_safe(request_body, {}, router, model, route="/v1/chat/completions") + + def test_many_fallback_entries_on_one_wildcard_are_checked_in_linear_time(self) -> None: + router: Final = Router( + model_list=[{"model_name": "*", "litellm_params": {"model": "openai/*", "api_key": "k"}}] + ) + request_body: Final = { + "model": "gpt-4o-mini", + "fallbacks": [{"model": f"model-{index % 100}", "vertex_location": f"r{index}"} for index in range(5000)], + } + + started: Final = time.perf_counter() + assert is_request_body_safe(request_body, {}, router, "gpt-4o-mini", route="/v1/chat/completions") is True + assert time.perf_counter() - started < 10 + + def test_too_many_fallback_models_for_an_opt_in_check_are_rejected(self) -> None: + router: Final = Router( + model_list=[{"model_name": "*", "litellm_params": {"model": "openai/*", "api_key": "k"}}] + ) + request_body: Final = { + "model": "gpt-4o-mini", + "vertex_location": "us-central1", + "fallbacks": [f"model-{index}" for index in range(101)], + } + + with pytest.raises(ValueError, match="more than 100 fallback models"): + is_request_body_safe(request_body, {}, router, "gpt-4o-mini", route="/v1/chat/completions") + + def test_one_wildcard_serving_two_providers_checks_the_google_one(self) -> None: + router: Final = Router(model_list=[{"model_name": "*", "litellm_params": {"model": "*"}}]) + request_body: Final = { + "model": "vertex_ai/gemini-2.5-flash", + "vertex_location": "us-central1", + "fallbacks": ["xai/grok-3"], + } + + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe(request_body, {}, router, "vertex_ai/gemini-2.5-flash", route="/v1/chat/completions") + + def test_a_pinned_group_sharing_a_deployment_id_with_an_opted_in_group_is_still_checked(self) -> None: + router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": { + "model": "vertex_ai/gemini-2.5-flash", + "vertex_location": "europe-west4", + "configurable_clientside_auth_params": opt_in, + }, + "model_info": {"id": "shared-id"}, + } + for name, opt_in in (("a-pinned", []), ("z-optin", ["vertex_location"])) + ] + ) + request_body: Final = {"model": "a-pinned", "vertex_location": "us-central1", "fallbacks": ["z-optin"]} + + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe(request_body, {}, router, "a-pinned", route="/v1/chat/completions") + + @pytest.mark.parametrize( + "model, is_allowed", [("public-alias", True), ("gemini-optin", False)], ids=["alias", "aliased_target"] + ) + def test_alias_chain_is_checked_one_hop_like_the_router_serves_it(self, model: str, is_allowed: bool) -> None: + router: Final = self._opted_in_and_pinned_router( + [], model_group_alias={"public-alias": "gemini-optin", "gemini-optin": "gemini-pinned"} + ) + request_body: Final = {"model": model, "vertex_location": "us-central1"} + if is_allowed: + assert is_request_body_safe(request_body, {}, router, model, route="/v1/chat/completions") is True + return + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe(request_body, {}, router, model, route="/v1/chat/completions") + + @pytest.mark.parametrize( + "route", + [ + "/openai/deployments/gemini-pinned/chat/completions", + "/v1beta/models/gemini-pinned:generateContent", + ], + ) + def test_model_named_in_the_route_needs_the_opt_in_too(self, route: str) -> None: + router: Final = self._opted_in_and_pinned_router([]) + request_body: Final = {"model": "gemini-optin", "vertex_location": "us-central1"} + with pytest.raises(ValueError, match="vertex_location"): + is_request_body_safe(request_body, {}, router, "gemini-optin", route=route) + + @pytest.mark.asyncio + @pytest.mark.parametrize("opted_in", [True, False]) + async def test_opt_in_of_the_model_named_in_a_google_route_decides(self, monkeypatch, opted_in: bool) -> None: + route: Final = "/v1beta/models/gemini-2.5-flash:generateContent" + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_router", _vertex_pinned_router(["vertex_location"] if opted_in else []) + ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + request: Final = Request({"type": "http", "method": "POST", "path": route, "headers": [], "query_string": b""}) + if opted_in: + await pre_db_read_auth_checks(request=request, request_data={"vertex_location": "us-central1"}, route=route) + return + with pytest.raises(ValueError, match="vertex_location"): + await pre_db_read_auth_checks(request=request, request_data={"vertex_location": "us-central1"}, route=route) + + @pytest.mark.parametrize( + "route, kept_params", + [ + ("/vertex_ai/live", _VERTEX_TARGET_PARAMS), + ("/rag/ingest", _VERTEX_TARGET_PARAMS), + ("/v1/rag/ingest", _VERTEX_TARGET_PARAMS), + ], + ) + def test_routes_that_name_their_vertex_target_keep_it(self, route: str, kept_params: tuple[str, ...]) -> None: + assert ( + is_request_body_safe( + request_body={param: "caller-chosen" for param in kept_params}, + general_settings={}, + llm_router=None, + model="", + route=route, + ) + is True + ) + + @pytest.mark.parametrize( + "route, param", + [ + ("/vertex_ai/live", "api_base"), + ("/rag/ingest", "api_base"), + ("/v1/rag/ingest", "api_base"), + ], + ) + def test_routes_that_name_their_vertex_target_reject_every_other_banned_param(self, route: str, param: str) -> None: + with pytest.raises(ValueError, match=param): + is_request_body_safe( + request_body={"vertex_location": "us-central1", param: "caller-chosen"}, + general_settings={}, + llm_router=None, + model="", + route=route, + ) + + @pytest.mark.parametrize( "allowed_param, input_value, should_return_true", [ @@ -5446,6 +5990,29 @@ def _enforce_for_team( ) +@pytest.mark.parametrize("opted_in", [False, True]) +def test_team_bucket_validation_preserves_the_authenticated_model_opt_in(opted_in: bool) -> None: + router: Final = Router( + model_list=[ + {"model_name": "chat", "litellm_params": {"model": "openai/test-model"}}, + { + "model_name": "team-chat", + "model_info": {"team_id": "team-one", "team_public_model_name": "chat"}, + "litellm_params": { + "model": "vertex_ai/test-model", + "configurable_clientside_auth_params": ["vertex_location"] if opted_in else [], + }, + }, + ] + ) + body: Final = {"model": "chat", "s3_bucket_name": "team-a-bucket", "vertex_location": "us-central1"} + if opted_in: + enforce_team_s3_bucket_allowlist(body, {}, router, _TEAM_A_METADATA, team_id="team-one") + return + with pytest.raises(HTTPException, match="vertex_location"): + enforce_team_s3_bucket_allowlist(body, {}, router, _TEAM_A_METADATA, team_id="team-one") + + def _rejection_for_team(body: Mapping[str, object], team_metadata: Mapping[str, object] | None) -> HTTPException: with pytest.raises(HTTPException) as rejection: _enforce_for_team(body, team_metadata) @@ -5499,7 +6066,13 @@ def test_team_allowlist_admits_no_other_blocked_field() -> None: def _routed_request(method: str, route_template: str | None, path: str) -> Request: - scope: Final[dict[str, object]] = {"type": "http", "method": method, "path": path, "headers": [], "query_string": b""} + scope: Final[dict[str, object]] = { + "type": "http", + "method": method, + "path": path, + "headers": [], + "query_string": b"", + } return Request(scope if route_template is None else {**scope, "route": SimpleNamespace(path=route_template)}) @@ -5661,7 +6234,9 @@ async def test_a_non_numeric_max_request_size_mb_is_ignored_without_premium(): def test_chatgpt_model_configuration_reaches_role_authorization( allow_client_side_credentials: bool, route: str ) -> None: - body: Final = {"litellm_params": {"model": "chatgpt/gpt-6-luna", "chatgpt_auth_file": "/tokens/account-a/auth.json"}} + body: Final = { + "litellm_params": {"model": "chatgpt/gpt-6-luna", "chatgpt_auth_file": "/tokens/account-a/auth.json"} + } assert is_request_body_safe( request_body=body, general_settings={"allow_client_side_credentials": allow_client_side_credentials}, diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 1489afc02af..8a5d4f7969b 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -3279,7 +3279,9 @@ class TestJWTOAuth2Coexistence: async def test_auto_register_passes_validated_org_context_to_generated_key(self): jwt_token = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.signature" general_settings = {"enable_jwt_auth": True} - user_api_key_cache: Final = _validated_jwt_admission_cache(org_id="validated-org", end_user_id="validated-end-user") + user_api_key_cache: Final = _validated_jwt_admission_cache( + org_id="validated-org", end_user_id="validated-end-user" + ) prisma_client = MagicMock() jwt_handler = MagicMock() jwt_handler.is_jwt.return_value = True @@ -4851,6 +4853,92 @@ def _proxy_attrs_for_centralized_checks(user_custom_auth=None, flag=False, maste } +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route, include_model, path_params", + [ + ("/v1/files", True, {}), + ("/v1/batches", True, {}), + ("/v1/chat/completions", True, {}), + ("/v1beta/models/chat:generateContent", False, {"model_name": "chat"}), + ("/openai/deployments/chat/chat/completions", False, {"model": "chat"}), + ], +) +@pytest.mark.parametrize("team_id, is_allowed", [(None, True), ("other-team", True), ("pinned-team", False)]) +async def test_deferred_body_check_uses_only_the_authenticated_teams_deployments( + monkeypatch: pytest.MonkeyPatch, + team_id: str | None, + is_allowed: bool, + route: str, + include_model: bool, + path_params: Mapping[str, str], +) -> None: + from litellm.proxy.utils import ProxyLogging + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + team_id=team_id, api_key=api_key, user_role=LitellmUserRoles.INTERNAL_USER, models=["chat"] + ) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "chat", + "litellm_params": { + "model": "vertex_ai/test-model", + "configurable_clientside_auth_params": ["vertex_location"], + }, + }, + { + "model_name": "private-chat", + "model_info": {"team_id": "pinned-team", "team_public_model_name": "chat"}, + "litellm_params": {"model": "vertex_ai/test-model", "vertex_location": "europe-west4"}, + }, + ] + ) + key_cache: Final = UserApiKeyCache() + attributes: Final = { + **_proxy_attrs_for_centralized_checks(user_custom_auth=authenticate), + "llm_router": router, + "llm_model_list": None, + "user_api_key_cache": key_cache, + "proxy_logging_obj": ProxyLogging(user_api_key_cache=key_cache), + } + for name, value in attributes.items(): + monkeypatch.setattr(litellm.proxy.proxy_server, name, value) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": route, + "headers": [], + "path_params": path_params, + "route": SimpleNamespace(path=route), + }, + receive=AsyncMock( + return_value={ + "type": "http.request", + "more_body": False, + "body": json.dumps( + { + "vertex_location": "us-central1", + **({"model": "chat"} if include_model else {}), + **({"s3_bucket_name": "team-bucket"} if route in {"/v1/files", "/v1/batches"} else {}), + } + ).encode(), + } + ), + ) + if is_allowed: + identity: Final = await user_api_key_auth(request=request, api_key="Bearer external-token") + assert identity.team_id == team_id + return + with pytest.raises(ProxyException) as rejection: + await user_api_key_auth(request=request, api_key="Bearer external-token") + assert int(rejection.value.code) == 401 + assert "vertex_location" in str(rejection.value) + + @pytest.mark.asyncio async def test_centralized_common_checks_runs_for_standard_auth(): """Regardless of which _user_api_key_auth_builder path returned, the @@ -5556,8 +5644,6 @@ async def _run_centralized_checks_for_end_user( setattr(_proxy_server_mod, attribute, value) - - @pytest.mark.parametrize( ("make_token", "make_end_user", "gpt_4o_spend", "model", "route", "must_raise"), [ diff --git a/tests/unit/proxy/batches_endpoints/test_endpoints.py b/tests/unit/proxy/batches_endpoints/test_endpoints.py index cee8a129172..127cf85710b 100644 --- a/tests/unit/proxy/batches_endpoints/test_endpoints.py +++ b/tests/unit/proxy/batches_endpoints/test_endpoints.py @@ -221,6 +221,7 @@ def harness(monkeypatch: pytest.MonkeyPatch): registry.access_groups = MagicMock(return_value={}) registry.resolve = MagicMock(side_effect=lambda name, **_: name) registry.all = MagicMock(return_value=[]) + registry.deployments = MagicMock(return_value=[]) creds_resolver = MagicMock(side_effect=_registry_creds_lookup) registry.credentials = _route_credentials({"with_provider": creds_resolver}) router.model_list = [] @@ -474,9 +475,7 @@ async def test_create__team_with_an_allowlist_and_no_request_bucket_leaves_the_b @pytest.mark.asyncio -@pytest.mark.parametrize( - "request_bucket,expected_bucket", [("team-a-bucket", "team-a-bucket"), (None, "model-bucket")] -) +@pytest.mark.parametrize("request_bucket,expected_bucket", [("team-a-bucket", "team-a-bucket"), (None, "model-bucket")]) @pytest.mark.parametrize( "routing", [ @@ -1564,6 +1563,7 @@ def retrieve_harness(): registry.access_groups = MagicMock(return_value={}) registry.resolve = MagicMock(side_effect=lambda name, **_: name) registry.all = MagicMock(return_value=[]) + registry.deployments = MagicMock(return_value=[]) creds_resolver = MagicMock(side_effect=_registry_creds_lookup) router.model_list = [] router.aretrieve_batch = AsyncMock(return_value=make_batch()) @@ -2151,6 +2151,7 @@ def list_harness(): registry.access_groups = MagicMock(return_value={}) registry.resolve = MagicMock(side_effect=lambda name, **_: name) registry.all = MagicMock(return_value=[]) + registry.deployments = MagicMock(return_value=[]) creds_resolver = MagicMock(side_effect=_registry_creds_lookup) registry.credentials = _route_credentials({"with_provider": creds_resolver}) router.model_list = [] @@ -2576,6 +2577,7 @@ def cancel_harness(): registry.access_groups = MagicMock(return_value={}) registry.resolve = MagicMock(side_effect=lambda name, **_: name) registry.all = MagicMock(return_value=[]) + registry.deployments = MagicMock(return_value=[]) creds_resolver = MagicMock(side_effect=_registry_creds_lookup) registry.credentials = _route_credentials({"with_provider": creds_resolver}) router.model_list = [] diff --git a/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py b/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py index f9918d5a1ee..ce2e740c39c 100644 --- a/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py +++ b/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py @@ -252,11 +252,20 @@ class FakeRouter: def registry(self) -> "FakeRegistry": return FakeRegistry() + def settings(self) -> "FakeSettings": + return FakeSettings() + def get_model_group_info(self, model_group: str) -> None: return None class FakeRegistry: + def routing_group(self, name: str) -> None: + return None + + def index(self, name: Literal["team_model"]) -> Mapping[tuple[str, str], Sequence[int]]: + return MappingProxyType({}) + def names(self, model_name: str | None = None, *, kind: str = "groups") -> list[str]: assert kind == "ids" return [DEPLOYMENT_ID] if model_name == BATCH_MODEL else [] @@ -264,6 +273,19 @@ class FakeRegistry: def group(self, name: str, *, field: str = "info") -> None: return None + def resolve(self, name: str, *, kind: str = "group") -> None: + return None + + def deployments( + self, model_name: str | None = None, *, source: str = "group", team_id: str | None = None + ) -> list[object]: + return [] + + +class FakeSettings: + def is_set(self, name: str) -> bool: + return False + class FakeStorageBackend: def __init__(self, contents: Mapping[str, bytes]) -> None: diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 2258eb95fb8..4aff6adadd3 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -5271,6 +5271,12 @@ class TestConfigBaseForHealthCheck: assert "vertex_credentials" not in base assert base["rpm"] == 100 + @pytest.mark.parametrize("param", ["vertex_project", "vertex_location", "vertex_ai_project", "vertex_ai_location"]) + def test_request_naming_a_vertex_target_still_inherits_config_credentials(self, param: str) -> None: + base = self._base(self.CONFIG, {"model": "openai/gpt-4o", param: "us-central1"}, selected_by_id=False) + assert base["vertex_credentials"] == "configured-creds" + assert base["api_key"] == "sk-configured" + def test_add_model_flow_keeps_its_own_credentials(self): """Adding a second deployment for an already-configured name sends a complete connection; it is tested as sent, not as configured.""" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 70f5e855e16..6e551f8ff0a 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -6399,7 +6399,11 @@ class TestVertexAILiveWebsocketPassthrough: ), ( "projects/other/locations/us-central1/publishers/google/models/gemini-2.0-flash", - "projects/other/locations/us-central1/publishers/google/models/gemini-2.0-flash", + "projects/proj-db/locations/global/publishers/google/models/gemini-2.0-flash", + ), + ( + "projects/other/locations/us-central1/endpoints/1234", + "projects/proj-db/locations/global/endpoints/1234", ), ], ) @@ -6425,7 +6429,9 @@ class TestVertexAILiveWebsocketPassthrough: rewriter = passthrough_module._build_vertex_live_setup_model_rewriter( vertex_project="proj-db", vertex_location="global", + configured_location="global", llm_router=llm_router, + keeps_caller_resource_path=False, ) assert rewriter is not None @@ -6517,6 +6523,402 @@ class TestVertexAILiveWebsocketPassthrough: assert "default_vertex_config" in close_kwargs["reason"] assert len(close_kwargs["reason"].encode("utf-8")) <= 123 + def _pinned_live_router(self, extra_params: Mapping[str, object]) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "gemini-live", + "litellm_params": { + "model": "vertex_ai/gemini-live-2.5-flash", + "use_in_pass_through": True, + "vertex_project": "proj-db", + "vertex_location": "global", + "vertex_credentials": '{"type": "service_account"}', + **extra_params, + }, + } + ] + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "query, field", + [ + ({"vertex_location": "us-central1"}, "vertex_location"), + ({"vertex_project": "caller-project"}, "vertex_project"), + ({"vertex_project": "proj-db", "vertex_location": "us-central1"}, "vertex_location"), + ], + ) + async def test_query_overriding_a_pinned_deployment_closes_before_minting_a_token( + self, monkeypatch, query: Mapping[str, str], field: str + ): + from litellm.proxy.pass_through_endpoints import ( + llm_passthrough_endpoints as passthrough_module, + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", self._pinned_live_router({})) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + monkeypatch.setattr(passthrough_module.passthrough_endpoint_router, "default_vertex_config", None) + self._clear_vertex_env(monkeypatch) + websocket = self._websocket() + ensure_token = AsyncMock(return_value=("token-abc", "proj-db")) + ws_passthrough = AsyncMock() + + with ( + patch.object(passthrough_module.vertex_llm_base, "ensure_access_token_async", ensure_token), + patch.object(passthrough_module, "websocket_passthrough_request", ws_passthrough), + ): + await passthrough_module.vertex_ai_live_websocket_passthrough( + websocket=websocket, + user_api_key_dict=UserAPIKeyAuth(), + **query, + ) + + close_kwargs = websocket.close.await_args.kwargs + assert close_kwargs["code"] == 1008 + assert field in close_kwargs["reason"] + assert len(close_kwargs["reason"].encode("utf-8")) <= 123 + ensure_token.assert_not_awaited() + ws_passthrough.assert_not_awaited() + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "extra_params, general_settings, query, expected_location", + [ + ({}, {}, {"vertex_project": "proj-db", "vertex_location": "global"}, "global"), + ( + {"configurable_clientside_auth_params": ["vertex_location"]}, + {}, + {"vertex_location": "us-central1"}, + "us-central1", + ), + ({}, {"allow_client_side_credentials": True}, {"vertex_location": "us-central1"}, "us-central1"), + ], + ids=["matches_the_pin", "deployment_opt_in", "proxy_wide_opt_in"], + ) + async def test_query_the_admin_allows_reaches_vertex( + self, + monkeypatch, + extra_params: Mapping[str, object], + general_settings: Mapping[str, object], + query: Mapping[str, str], + expected_location: str, + ): + from litellm.proxy.pass_through_endpoints import ( + llm_passthrough_endpoints as passthrough_module, + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", self._pinned_live_router(extra_params)) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", dict(general_settings)) + monkeypatch.setattr(passthrough_module.passthrough_endpoint_router, "default_vertex_config", None) + self._clear_vertex_env(monkeypatch) + websocket = self._websocket() + ensure_token = AsyncMock(return_value=("token-abc", "proj-db")) + ws_passthrough = AsyncMock() + + with ( + patch.object(passthrough_module.vertex_llm_base, "ensure_access_token_async", ensure_token), + patch.object(passthrough_module, "websocket_passthrough_request", ws_passthrough), + ): + await passthrough_module.vertex_ai_live_websocket_passthrough( + websocket=websocket, + user_api_key_dict=UserAPIKeyAuth(), + **query, + ) + + websocket.close.assert_not_awaited() + rewriter = ws_passthrough.await_args.kwargs["setup_model_rewriter"] + assert rewriter("gemini-live") == ( + f"projects/proj-db/locations/{expected_location}/publishers/google/models/gemini-live-2.5-flash" + ) + + def _use_default_vertex_config(self, monkeypatch, general_settings: Mapping[str, object], vertex_location): + from litellm.proxy.pass_through_endpoints import ( + llm_passthrough_endpoints as passthrough_module, + ) + from litellm.types.passthrough_endpoints.vertex_ai import ( + VertexPassThroughCredentials, + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", dict(general_settings)) + monkeypatch.setattr( + passthrough_module.passthrough_endpoint_router, + "default_vertex_config", + VertexPassThroughCredentials( + vertex_project="proj-env", + vertex_location=vertex_location, + vertex_credentials='{"type": "env_account"}', + ), + ) + monkeypatch.setattr(passthrough_module.passthrough_endpoint_router, "deployment_key_to_vertex_credentials", {}) + self._clear_vertex_env(monkeypatch) + + async def _open_live(self, query: Mapping[str, str]): + from litellm.proxy.pass_through_endpoints import ( + llm_passthrough_endpoints as passthrough_module, + ) + + websocket = self._websocket() + ensure_token = AsyncMock( + side_effect=lambda credentials, project_id, custom_llm_provider: ("token-abc", project_id or "proj-env") + ) + ws_passthrough = AsyncMock() + with ( + patch.object(passthrough_module.vertex_llm_base, "ensure_access_token_async", ensure_token), + patch.object(passthrough_module, "websocket_passthrough_request", ws_passthrough), + ): + await passthrough_module.vertex_ai_live_websocket_passthrough( + websocket=websocket, + user_api_key_dict=UserAPIKeyAuth(), + **query, + ) + return websocket, ensure_token, ws_passthrough + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "query, field", + [ + ({"vertex_project": "caller-project"}, "vertex_project"), + ({"vertex_location": "us-central1"}, "vertex_location"), + ({"vertex_project": "proj-env", "vertex_location": "us-central1"}, "vertex_location"), + ], + ) + async def test_query_moving_default_vertex_config_closes_before_minting_a_token( + self, monkeypatch, query: Mapping[str, str], field: str + ): + self._use_default_vertex_config(monkeypatch, {}, vertex_location="europe-west4") + + websocket, ensure_token, ws_passthrough = await self._open_live(query) + + close_kwargs = websocket.close.await_args.kwargs + assert close_kwargs["code"] == 1008 + assert f"{field} is pinned by default_vertex_config" in close_kwargs["reason"] + assert len(close_kwargs["reason"].encode("utf-8")) <= 123 + ensure_token.assert_not_awaited() + ws_passthrough.assert_not_awaited() + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "general_settings, default_location, query, expected_project, expected_location", + [ + ( + {}, + "europe-west4", + {"vertex_project": "proj-env", "vertex_location": "europe-west4"}, + "proj-env", + "europe-west4", + ), + ({}, None, {"vertex_location": "us-central1"}, "proj-env", "us-central1"), + ( + {"allow_client_side_credentials": True}, + "europe-west4", + {"vertex_project": "caller-project", "vertex_location": "us-central1"}, + "caller-project", + "us-central1", + ), + ], + ids=["matches_the_pin", "location_left_unset", "proxy_wide_opt_in"], + ) + async def test_default_vertex_config_serves_the_queries_it_allows( + self, + monkeypatch, + general_settings: Mapping[str, object], + default_location, + query: Mapping[str, str], + expected_project: str, + expected_location: str, + ): + self._use_default_vertex_config(monkeypatch, general_settings, vertex_location=default_location) + + websocket, ensure_token, ws_passthrough = await self._open_live(query) + + websocket.close.assert_not_awaited() + assert ensure_token.await_args.kwargs["project_id"] == expected_project + rewriter = ws_passthrough.await_args.kwargs["setup_model_rewriter"] + assert rewriter("gemini-live-2.5-flash") == ( + f"projects/{expected_project}/locations/{expected_location}/publishers/google/models/gemini-live-2.5-flash" + ) + + @pytest.mark.asyncio + async def test_a_registered_project_and_location_pair_still_serves_its_query(self, monkeypatch): + from litellm.proxy.pass_through_endpoints import ( + llm_passthrough_endpoints as passthrough_module, + ) + from litellm.types.passthrough_endpoints.vertex_ai import ( + VertexPassThroughCredentials, + ) + + self._use_default_vertex_config(monkeypatch, {}, vertex_location="europe-west4") + monkeypatch.setattr( + passthrough_module.passthrough_endpoint_router, + "deployment_key_to_vertex_credentials", + { + "caller-project-us-central1": VertexPassThroughCredentials( + vertex_project="caller-project", + vertex_location="us-central1", + vertex_credentials='{"type": "registered_account"}', + ) + }, + ) + + websocket, ensure_token, _ = await self._open_live( + {"vertex_project": "caller-project", "vertex_location": "us-central1"} + ) + + websocket.close.assert_not_awaited() + ensure_token.assert_awaited_once_with( + credentials='{"type": "registered_account"}', + project_id="caller-project", + custom_llm_provider="vertex_ai_beta", + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "general_settings, expected", + [ + ({}, "projects/proj-env/locations/europe-west4/publishers/google/models/gemini-2.0-flash"), + ( + {"allow_client_side_credentials": True}, + "projects/caller-project/locations/us-central1/publishers/google/models/gemini-2.0-flash", + ), + ], + ids=["pinned", "proxy_wide_opt_in"], + ) + async def test_setup_frame_resource_path_cannot_move_the_session_to_another_project( + self, monkeypatch, general_settings: Mapping[str, object], expected: str + ): + self._use_default_vertex_config(monkeypatch, general_settings, vertex_location="europe-west4") + + _, _, ws_passthrough = await self._open_live({}) + + rewriter = ws_passthrough.await_args.kwargs["setup_model_rewriter"] + assert ( + rewriter("projects/caller-project/locations/us-central1/publishers/google/models/gemini-2.0-flash") + == expected + ) + + def _pin_no_vertex_project( + self, + monkeypatch, + general_settings: Mapping[str, object], + default_vertex_config: VertexPassThroughCredentials | None, + ): + from litellm.proxy.pass_through_endpoints import ( + llm_passthrough_endpoints as passthrough_module, + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", dict(general_settings)) + monkeypatch.setattr( + passthrough_module.passthrough_endpoint_router, "default_vertex_config", default_vertex_config + ) + monkeypatch.setattr(passthrough_module.passthrough_endpoint_router, "deployment_key_to_vertex_credentials", {}) + self._clear_vertex_env(monkeypatch) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "default_vertex_config", + [None, VertexPassThroughCredentials(vertex_location="europe-west4")], + ids=["no_default_vertex_config", "default_vertex_config_without_a_project"], + ) + async def test_caller_named_project_closes_before_any_upstream_connect_when_nothing_pins_one( + self, monkeypatch, default_vertex_config: VertexPassThroughCredentials | None + ): + self._pin_no_vertex_project(monkeypatch, {}, default_vertex_config) + + websocket, ensure_token, ws_passthrough = await self._open_live({"vertex_project": "other-project"}) + + ws_passthrough.assert_not_awaited() + ensure_token.assert_not_awaited() + close_kwargs = websocket.close.await_args.kwargs + assert close_kwargs["code"] == 1008 + assert "vertex_project" in close_kwargs["reason"] + assert len(close_kwargs["reason"].encode("utf-8")) <= 123 + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "configured_by, default_vertex_config", + [ + ("VERTEXAI_PROJECT", None), + ("DEFAULT_VERTEXAI_PROJECT", VertexPassThroughCredentials(vertex_location="europe-west4")), + ], + ids=["vertexai_project_env", "default_vertexai_project_env"], + ) + async def test_with_nothing_pinned_the_caller_may_name_the_project_the_admin_configured( + self, monkeypatch, configured_by: str, default_vertex_config: VertexPassThroughCredentials | None + ): + self._pin_no_vertex_project(monkeypatch, {}, default_vertex_config) + monkeypatch.setattr(litellm, "vertex_project", None) + monkeypatch.delenv("VERTEXAI_PROJECT", raising=False) + monkeypatch.setenv(configured_by, "proj-configured") + + websocket, _, ws_passthrough = await self._open_live( + {"vertex_project": "proj-configured", "vertex_location": "europe-west4"} + ) + + websocket.close.assert_not_awaited() + assert ws_passthrough.await_args.kwargs["custom_headers"]["x-goog-user-project"] == "proj-configured" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "general_settings, query, expected_project, expected_location", + [ + ({}, {"vertex_location": "europe-west4"}, "proj-env", "europe-west4"), + ( + {"allow_client_side_credentials": True}, + {"vertex_project": "other-project", "vertex_location": "europe-west4"}, + "other-project", + "europe-west4", + ), + ], + ids=["caller_region", "proxy_wide_opt_in"], + ) + async def test_with_nothing_pinned_the_caller_keeps_the_region_and_the_opt_in_keeps_the_project( + self, + monkeypatch, + general_settings: Mapping[str, object], + query: Mapping[str, str], + expected_project: str, + expected_location: str, + ): + self._pin_no_vertex_project(monkeypatch, general_settings, None) + + websocket, _, ws_passthrough = await self._open_live(query) + + websocket.close.assert_not_awaited() + passthrough_kwargs = ws_passthrough.await_args.kwargs + assert passthrough_kwargs["custom_headers"]["x-goog-user-project"] == expected_project + assert passthrough_kwargs["setup_model_rewriter"]("gemini-2.0-flash") == ( + f"projects/{expected_project}/locations/{expected_location}/publishers/google/models/gemini-2.0-flash" + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "setup_model, expected", + [ + ( + "projects/proj-env/locations/europe-west4/publishers/google/models/gemini-2.0-flash", + "projects/proj-env/locations/europe-west4/publishers/google/models/gemini-2.0-flash", + ), + ( + "projects/caller-project/locations/europe-west4/publishers/google/models/gemini-2.0-flash", + "projects/proj-env/locations/europe-west4/publishers/google/models/gemini-2.0-flash", + ), + ], + ids=["own_project", "other_project"], + ) + async def test_setup_frame_keeps_its_region_when_no_region_is_pinned( + self, monkeypatch, setup_model: str, expected: str + ): + self._use_default_vertex_config(monkeypatch, {}, vertex_location=None) + + websocket, _, ws_passthrough = await self._open_live({}) + + websocket.close.assert_not_awaited() + rewriter = ws_passthrough.await_args.kwargs["setup_model_rewriter"] + assert rewriter(setup_model) == expected + @pytest.mark.asyncio @pytest.mark.parametrize("model, expected_model", [("gemini-3.8-live", "gemini-3.8-live"), (None, "")]) async def test_credential_failure_logs_the_named_model_under_vertex_ai( diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 4f73e8935e8..8222ca09b24 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -7712,7 +7712,7 @@ async def test_websocket_passthrough_keeps_normal_upstream_close_normal(): websocket.close.assert_awaited_once_with() -async def _run_setup_rewrite_passthrough(setup_model: str, llm_router) -> str: +async def _run_setup_rewrite_passthrough(setup_model: str, llm_router, keeps_caller_resource_path: bool = False) -> str: from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _build_vertex_live_setup_model_rewriter, ) @@ -7740,7 +7740,9 @@ async def _run_setup_rewrite_passthrough(setup_model: str, llm_router) -> str: setup_model_rewriter=_build_vertex_live_setup_model_rewriter( vertex_project="proj-db", vertex_location="global", + configured_location="global", llm_router=llm_router, + keeps_caller_resource_path=keeps_caller_resource_path, ), ) @@ -7778,6 +7780,67 @@ async def test_websocket_passthrough_rewrites_gateway_alias_setup_model(): assert sent_setup["model"] == "projects/proj-db/locations/global/publishers/google/models/gemini-live-2.5-flash" +async def _run_binary_client_frame_passthrough(frame: bytes): + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + _build_vertex_live_setup_model_rewriter, + ) + + upstream_ws = RecordingUpstreamWebSocket() + websocket = _client_websocket( + AsyncMock( + side_effect=[ + {"type": "websocket.receive", "bytes": frame}, + {"type": "websocket.disconnect"}, + ] + ) + ) + + with _patched_websocket_passthrough_environment(upstream_ws): + await websocket_passthrough_request( + websocket=websocket, + target="wss://aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent", + custom_headers={"Authorization": "Bearer token"}, + user_api_key_dict=UserAPIKeyAuth(), + forward_headers=False, + endpoint="/vertex_ai/live", + accept_websocket=False, + setup_model_rewriter=_build_vertex_live_setup_model_rewriter( + vertex_project="proj-a", + vertex_location="europe-west4", + configured_location="europe-west4", + llm_router=None, + keeps_caller_resource_path=False, + ), + ) + + return websocket, upstream_ws + + +@pytest.mark.asyncio +async def test_websocket_passthrough_moves_a_binary_setup_frame_onto_the_pinned_project(): + setup_frame = json.dumps( + {"setup": {"model": "projects/proj-b/locations/europe-west4/publishers/google/models/gemini-2.0-flash"}} + ).encode("utf-8") + + _, upstream_ws = await _run_binary_client_frame_passthrough(setup_frame) + + upstream_ws.send.assert_awaited_once() + sent_frame = upstream_ws.send.await_args.args[0] + assert isinstance(sent_frame, bytes), "a binary client message must stay binary upstream" + assert json.loads(sent_frame)["setup"]["model"] == ( + "projects/proj-a/locations/europe-west4/publishers/google/models/gemini-2.0-flash" + ) + + +@pytest.mark.asyncio +async def test_websocket_passthrough_closes_on_a_binary_frame_that_is_not_utf8(): + websocket, upstream_ws = await _run_binary_client_frame_passthrough(b"\xff\xfe\x00projects/proj-b") + + upstream_ws.send.assert_not_awaited() + websocket.close.assert_awaited_once() + assert websocket.close.await_args.kwargs["code"] == 1008 + + @pytest.mark.parametrize( "setup_model", ["gemini-live-2.5-flash", "models/gemini-live-2.5-flash", "publishers/google/models/gemini-live-2.5-flash"], @@ -7799,7 +7862,11 @@ def test_vertex_live_setup_model_resolves_before_extraction(setup_model): ) rewriter = _build_vertex_live_setup_model_rewriter( - vertex_project="proj-db", vertex_location="global", llm_router=None + vertex_project="proj-db", + vertex_location="global", + configured_location="global", + llm_router=None, + keeps_caller_resource_path=False, ) setup_data = {"model": setup_model} @@ -7849,7 +7916,11 @@ async def test_websocket_passthrough_logs_a_bare_alias_setup_model(): endpoint="/vertex_ai/live", accept_websocket=False, setup_model_rewriter=_build_vertex_live_setup_model_rewriter( - vertex_project="proj-db", vertex_location="global", llm_router=None + vertex_project="proj-db", + vertex_location="global", + configured_location="global", + llm_router=None, + keeps_caller_resource_path=False, ), ) @@ -7914,14 +7985,24 @@ async def test_websocket_passthrough_rewrites_alias_of_unrecognised_upstream_mod @pytest.mark.asyncio -async def test_websocket_passthrough_leaves_full_resource_setup_model_untouched(): - full_resource = "projects/other/locations/us-central1/publishers/google/models/gemini-2.0-flash-live-preview-04-09" +@pytest.mark.parametrize( + "keeps_caller_resource_path, expected_prefix", + [(True, "projects/other/locations/us-central1/"), (False, "projects/proj-db/locations/global/")], + ids=["caller_path_kept", "moved_onto_session_target"], +) +async def test_websocket_passthrough_full_resource_setup_model( + keeps_caller_resource_path: bool, expected_prefix: str +) -> None: + model_suffix = "publishers/google/models/gemini-2.0-flash-live-preview-04-09" - sent_frame = await _run_setup_rewrite_passthrough(full_resource, llm_router=None) + sent_frame = await _run_setup_rewrite_passthrough( + f"projects/other/locations/us-central1/{model_suffix}", + llm_router=None, + keeps_caller_resource_path=keeps_caller_resource_path, + ) - assert json.loads(sent_frame)["setup"]["model"] == full_resource assert sent_frame == json.dumps( - {"setup": {"model": full_resource, "generationConfig": {"responseModalities": ["TEXT"]}}} + {"setup": {"model": f"{expected_prefix}{model_suffix}", "generationConfig": {"responseModalities": ["TEXT"]}}} ) diff --git a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py index 143f5bfb83b..7facd6a62d6 100644 --- a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py +++ b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py @@ -2,6 +2,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -26,6 +27,7 @@ class _FakeRouter: infos: Mapping[str, ModelGroupInfo] model_group_alias: Mapping[str, str] = MappingProxyType({}) + team_model_to_deployment_indices: Mapping[tuple[str, str], Sequence[int]] = MappingProxyType({}) def get_model_group_info(self, model_group: str, team_id: str | None = None) -> ModelGroupInfo | None: return self.infos.get(model_group) @@ -351,6 +353,8 @@ def test_no_router_answers_with_a_problem_rather_than_the_openai_error_shape(mon def test_an_unexpected_router_failure_answers_as_a_problem_not_the_openai_error_shape(monkeypatch): class _Exploding: + team_model_to_deployment_indices: Final[Mapping[tuple[str, str], Sequence[int]]] = MappingProxyType({}) + def get_model_group_info(self, model_group: str, team_id: str | None = None) -> ModelGroupInfo: raise RuntimeError("router blew up") diff --git a/tests/unit/proxy/rag_endpoints/test_rag_endpoints.py b/tests/unit/proxy/rag_endpoints/test_rag_endpoints.py index 6eff950dddf..1d5a1f8fa30 100644 --- a/tests/unit/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/unit/proxy/rag_endpoints/test_rag_endpoints.py @@ -18,7 +18,8 @@ from fastapi.testclient import TestClient import litellm from litellm import Router -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.constants import LITELLM_PROXY_ADMIN_NAME +from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.openai_files_endpoints.common_utils import ( TeamProviderDeployment, @@ -292,6 +293,7 @@ VERTEX_REGISTRY_STORE = { "vector_store_id": "projects/registry-project/locations/us-central1/ragCorpora/42", "custom_llm_provider": "vertex_ai", "litellm_params": {"vertex_project": "registry-project", "vertex_location": "us-central1"}, + "is_config": True, } UNSUPPORTED_INGEST_PROVIDER_ERROR = ( "Provider '{provider}' is not supported for RAG ingestion. " @@ -1501,6 +1503,292 @@ def test_rag_ingest_unmanaged_store_keeps_the_callers_full_config(client_interna assert mock_aingest.await_args.kwargs["ingest_options"]["vector_store"] == caller_config +_PROXY_VERTEX_PROJECT: Final = "proxy-project" +_PROXY_VERTEX_LOCATION: Final = "europe-west4" + + +@pytest.fixture +def proxy_vertex_target(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "vertex_project", None) + monkeypatch.setattr(litellm, "vertex_location", None) + monkeypatch.setenv("VERTEXAI_PROJECT", _PROXY_VERTEX_PROJECT) + monkeypatch.setenv("VERTEXAI_LOCATION", _PROXY_VERTEX_LOCATION) + + +def _vertex_ingest(client: TestClient, vector_store: Mapping[str, str]) -> tuple[httpx.Response, AsyncMock]: + aingest_patch, registry_patch = _patched_ingest_boundary( + None, {"vector_store_id": "projects/p/locations/l/ragCorpora/7", "file_id": "file_123"} + ) + with aingest_patch as mock_aingest, registry_patch, _patched_prisma_client(None): + response = client.post("/v1/rag/ingest", **_ingest_form({"custom_llm_provider": "vertex_ai", **vector_store})) + return response, mock_aingest + + +@pytest.mark.parametrize( + "vector_store, field", + [ + ({"vertex_project": "other-project"}, "vertex_project"), + ({"vertex_location": "us-east5"}, "vertex_location"), + ( + {"vector_store_id": f"projects/other-project/locations/{_PROXY_VERTEX_LOCATION}/ragCorpora/7"}, + "vector_store_id", + ), + ( + { + "vertex_project": _PROXY_VERTEX_PROJECT, + "vector_store_id": f"projects/other-project/locations/{_PROXY_VERTEX_LOCATION}/ragCorpora/7", + }, + "vector_store_id", + ), + ({"vector_store_id": f"projects/{_PROXY_VERTEX_PROJECT}/locations/us-east5/ragCorpora/7"}, "vector_store_id"), + ({"vertex_ai_project": "other-project"}, "vertex_ai_project"), + ({"vertex_ai_location": "us-east5"}, "vertex_ai_location"), + ({"vector_store_id": "projects/123456789/locations/us-east5/ragCorpora/7"}, "vector_store_id"), + ], + ids=[ + "project_field", + "location_field", + "corpus_project", + "corpus_project_behind_matching_field", + "corpus_region", + "project_alias_field", + "location_alias_field", + "corpus_region_by_project_number", + ], +) +@pytest.mark.usefixtures("proxy_vertex_target") +def test_rag_ingest_refuses_a_key_pointing_the_proxys_vertex_credentials_elsewhere( + client_internal_user, vector_store: Mapping[str, str], field: str +): + response, mock_aingest = _vertex_ingest(client_internal_user, vector_store) + + assert response.status_code == 400, response.json() + assert response.json()["detail"]["error"].startswith(f"Rejected Request: {field} picks a Vertex project") + mock_aingest.assert_not_awaited() + + +def test_rag_ingest_refuses_a_key_naming_a_vertex_project_when_the_proxy_names_none( + client_internal_user, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "vertex_project", None) + monkeypatch.delenv("VERTEXAI_PROJECT", raising=False) + + response, mock_aingest = _vertex_ingest(client_internal_user, {"vertex_project": "other-project"}) + + assert response.status_code == 400, response.json() + mock_aingest.assert_not_awaited() + + +@pytest.mark.parametrize( + "vector_store", + [ + {}, + {"vertex_project": _PROXY_VERTEX_PROJECT, "vertex_location": _PROXY_VERTEX_LOCATION}, + {"vector_store_id": f"projects/{_PROXY_VERTEX_PROJECT}/locations/{_PROXY_VERTEX_LOCATION}/ragCorpora/7"}, + {"vertex_ai_project": _PROXY_VERTEX_PROJECT, "vertex_ai_location": _PROXY_VERTEX_LOCATION}, + ], + ids=["proxy_defaults", "same_fields", "corpus_in_the_proxy_project", "same_alias_fields"], +) +@pytest.mark.usefixtures("proxy_vertex_target") +def test_rag_ingest_lets_a_key_use_the_proxys_own_vertex_target(client_internal_user, vector_store: Mapping[str, str]): + response, mock_aingest = _vertex_ingest(client_internal_user, vector_store) + + assert response.status_code == 200, response.json() + assert mock_aingest.await_args.kwargs["ingest_options"]["vector_store"] == { + "custom_llm_provider": "vertex_ai", + **vector_store, + } + + +@pytest.mark.parametrize( + "project_field, location_field", + [("vertex_project", "vertex_location"), ("vertex_ai_project", "vertex_ai_location")], + ids=["fields", "alias_fields"], +) +@pytest.mark.parametrize( + "user_role, general_settings", + [ + (LitellmUserRoles.PROXY_ADMIN, {}), + (LitellmUserRoles.INTERNAL_USER, {"allow_client_side_credentials": True}), + ], + ids=["proxy_admin", "proxy_wide_opt_in"], +) +@pytest.mark.usefixtures("proxy_vertex_target") +def test_rag_ingest_lets_the_admin_choose_another_vertex_target( + user_role: LitellmUserRoles, general_settings: Mapping[str, object], project_field: str, location_field: str +): + caller: Final = UserAPIKeyAuth(user_id="vertex-caller", user_role=user_role) + original_overrides: Final = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: caller + vector_store: Final = { + project_field: "other-project", + location_field: "us-east5", + "vector_store_id": "projects/other-project/locations/us-east5/ragCorpora/7", + } + try: + with patch.dict("litellm.proxy.proxy_server.general_settings", general_settings): + response, mock_aingest = _vertex_ingest(TestClient(app), vector_store) + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 200, response.json() + assert mock_aingest.await_args.kwargs["ingest_options"]["vector_store"] == { + "custom_llm_provider": "vertex_ai", + **vector_store, + } + + +@pytest.mark.usefixtures("proxy_vertex_target") +def test_rag_ingest_reads_a_corpus_named_by_project_number_on_the_proxys_own_project(client_internal_user): + response, mock_aingest = _vertex_ingest( + client_internal_user, {"vector_store_id": f"projects/123456789/locations/{_PROXY_VERTEX_LOCATION}/ragCorpora/7"} + ) + + assert response.status_code == 200, response.json() + assert mock_aingest.await_args.kwargs["ingest_options"]["vector_store"]["vector_store_id"] == ( + f"projects/{_PROXY_VERTEX_PROJECT}/locations/{_PROXY_VERTEX_LOCATION}/ragCorpora/7" + ) + + +_PROXY_CORPUS_PREFIX: Final = f"projects/{_PROXY_VERTEX_PROJECT}/locations/{_PROXY_VERTEX_LOCATION}/ragCorpora" +_FOREIGN_CORPUS: Final = f"projects/other-project/locations/{_PROXY_VERTEX_LOCATION}/ragCorpora/1" + + +@pytest.mark.usefixtures("proxy_vertex_target") +def test_rag_ingest_refuses_another_teams_store_named_by_a_project_number(client_internal_user): + other_teams_store: Final = LiteLLM_ManagedVectorStore( + vector_store_id=f"{_PROXY_CORPUS_PREFIX}/1", + custom_llm_provider="vertex_ai", + litellm_params={}, + user_id="other-user", + team_id="other-team", + ) + registry: Final = MagicMock() + registry.get_litellm_managed_vector_store_from_registry.side_effect = lambda vector_store_id: ( + other_teams_store if vector_store_id == other_teams_store["vector_store_id"] else None + ) + with ( + patch( # test-quality-ok: aingest is the endpoint's downstream boundary; the test asserts it is never called + "litellm.proxy.rag_endpoints.endpoints.litellm.aingest", + new=AsyncMock(return_value={"vector_store_id": f"{_PROXY_CORPUS_PREFIX}/1", "file_id": "file_123"}), + ) as mock_aingest, + patch.object(litellm, "vector_store_registry", registry), # test-quality-ok: seeds the saved stores by id + _patched_prisma_client(None), + ): + response = client_internal_user.post( + "/v1/rag/ingest", + **_ingest_form( + { + "custom_llm_provider": "vertex_ai", + "vector_store_id": f"projects/0/locations/{_PROXY_VERTEX_LOCATION}/ragCorpora/1", + } + ), + ) + + assert response.status_code == 403, response.json() + mock_aingest.assert_not_awaited() + + +@pytest.mark.parametrize( + "vector_store_id", + [ + f"./{_FOREIGN_CORPUS}", + f"x/../{_FOREIGN_CORPUS}", + f"{_PROXY_CORPUS_PREFIX}/1{'/..' * 6}/{_FOREIGN_CORPUS}", + f"{_PROXY_CORPUS_PREFIX}/..", + f"{_PROXY_CORPUS_PREFIX}/1%2F..%2F..", + f"{_PROXY_CORPUS_PREFIX}/1\\..\\..", + ], + ids=["dot_prefix", "dot_dot_prefix", "own_corpus_then_dot_dots", "dot_dot_corpus", "percent", "backslash"], +) +@pytest.mark.usefixtures("proxy_vertex_target") +def test_rag_ingest_refuses_a_vertex_corpus_name_with_path_tricks(client_internal_user, vector_store_id: str): + response, mock_aingest = _vertex_ingest(client_internal_user, {"vector_store_id": vector_store_id}) + + assert response.status_code == 400, response.json() + assert response.json()["detail"]["error"].startswith("Rejected Request: vector_store_id picks a Vertex project") + mock_aingest.assert_not_awaited() + + +_FOREIGN_CORPUS_BY_NUMBER: Final = f"projects/987654321/locations/{_PROXY_VERTEX_LOCATION}/ragCorpora/1" + + +def _foreign_vertex_store(created_by: str | None, corpus: str = _FOREIGN_CORPUS) -> LiteLLM_ManagedVectorStore: + return LiteLLM_ManagedVectorStore( + vector_store_id=corpus, custom_llm_provider="vertex_ai", litellm_params={}, user_id=created_by + ) + + +def _prisma_with_user_row(user_row: LiteLLM_UserTable | None) -> MagicMock: + prisma_client: Final = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=user_row) + prisma_client.db.litellm_managedvectorstorestable.find_unique = AsyncMock( + return_value=MagicMock(vector_store_metadata={}) + ) + prisma_client.db.litellm_managedvectorstorestable.update = AsyncMock() + return prisma_client + + +def _ingest_into_saved_store( + store: LiteLLM_ManagedVectorStore, creator_row: LiteLLM_UserTable | None +) -> tuple[httpx.Response, AsyncMock]: + caller: Final = UserAPIKeyAuth(user_id="vertex-caller", user_role=LitellmUserRoles.INTERNAL_USER) + original_overrides: Final = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: caller + aingest_patch, registry_patch = _patched_ingest_boundary( + store, {"vector_store_id": store["vector_store_id"], "file_id": "file_123"} + ) + try: + with aingest_patch as mock_aingest, registry_patch, _patched_prisma_client(_prisma_with_user_row(creator_row)): + response = TestClient(app).post( + "/v1/rag/ingest", **_ingest_form({"vector_store_id": store["vector_store_id"]}) + ) + finally: + app.dependency_overrides = original_overrides + return response, mock_aingest + + +@pytest.mark.parametrize( + "creator_row", + [LiteLLM_UserTable(user_id="vertex-caller", user_role=LitellmUserRoles.INTERNAL_USER.value), None], + ids=["saved_by_an_internal_user", "saved_by_a_key_with_no_user"], +) +@pytest.mark.usefixtures("proxy_vertex_target") +def test_rag_ingest_refuses_a_saved_vertex_store_a_non_admin_pointed_elsewhere(creator_row: LiteLLM_UserTable | None): + store: Final = _foreign_vertex_store(creator_row.user_id if creator_row is not None else None) + + response, mock_aingest = _ingest_into_saved_store(store, creator_row) + + assert response.status_code == 400, response.json() + assert response.json()["detail"]["error"].startswith("Rejected Request: vector_store_id picks a Vertex project") + mock_aingest.assert_not_awaited() + + +@pytest.mark.parametrize("corpus", [_FOREIGN_CORPUS, _FOREIGN_CORPUS_BY_NUMBER], ids=["by_id", "by_number"]) +@pytest.mark.parametrize( + "created_by, is_config, creator_row", + [ + (LITELLM_PROXY_ADMIN_NAME, False, None), + ("admin-user", False, LiteLLM_UserTable(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN.value)), + (None, True, None), + ], + ids=["master_key_created", "admin_user_created", "config_store"], +) +@pytest.mark.usefixtures("proxy_vertex_target") +def test_rag_ingest_lets_a_key_use_a_vertex_store_an_admin_pointed_elsewhere( + corpus: str, created_by: str | None, is_config: bool, creator_row: LiteLLM_UserTable | None +): + store: Final = LiteLLM_ManagedVectorStore(**_foreign_vertex_store(created_by, corpus), is_config=is_config) + + response, mock_aingest = _ingest_into_saved_store(store, creator_row) + + assert response.status_code == 200, response.json() + assert mock_aingest.await_args.kwargs["ingest_options"]["vector_store"] == { + "vector_store_id": corpus, + "custom_llm_provider": "vertex_ai", + } + + def test_rag_ingest_db_managed_store_drops_the_callers_credential_name(client_internal_user): aingest_patch, registry_patch = _patched_ingest_boundary( DB_MANAGED_STORE, {"vector_store_id": "db-store", "file_id": "file_123"} From 8fb2b97d5b2b09d0265122c8120951548f017d1d Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 10 Nov 2026 12:55:49 -0800 Subject: [PATCH 036/189] fix(router): sync responses() applies the Router's num_retries and fallbacks (#51011) * fix(router): sync responses() applies the Router's num_retries and fallbacks Sync Router.responses() made one attempt and raised, while aresponses() retried and fell back on the same Router. The sync plan now walks retries and fallbacks like the async one, stamps the deployment's num_retries on the error the same way, and the SDK no longer nests its own retries under the Router's for sync generic calls Fixes #49938 * fix(router): sync generate_content() retries and falls back through the Router Reading litellm_metadata in the SDK retry check left sync generate_content with no retries at all, since its Router plan made one attempt and the inner completion() call stopped retrying. Sync generate_content and generate_content_stream now walk the Router's retries and fallbacks like their async versions, so retries are neither lost nor nested --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/router_rust/invoke/endpoints.py | 22 ++++-- litellm/utils.py | 16 ++--- ...test_endpoints_characterization_generic.py | 2 +- .../core/test_parity_regress5_errors.py | 43 +++++++++++- tests/unit/test_utils.py | 68 +++++++++++++------ 5 files changed, 111 insertions(+), 40 deletions(-) diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index d728583cb31..a5307ff8af9 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -293,7 +293,7 @@ class Plan: return get_router_metadata_variable_name(function_name=self.function_name) -def _generic(endpoint: str, host_op: str | None = None, *, sync: bool = False) -> Plan: +def _generic(endpoint: str, host_op: str | None = None, *, sync: bool = False, single_attempt: bool = True) -> Plan: if sync: return Plan( endpoint=endpoint, @@ -303,9 +303,9 @@ def _generic(endpoint: str, host_op: str | None = None, *, sync: bool = False) - prepare_field="litellm_metadata", function_name="generic_api_call", splits_stream_timeout=_handler_name(host_op or endpoint) in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, - stamp="none", + stamp="none" if single_attempt else "deployment", fail_key="group", - single_attempt=True, + single_attempt=single_attempt, ) return Plan( endpoint=endpoint, @@ -483,7 +483,7 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType( "aanthropic_messages": _generic("aanthropic_messages", "anthropic_messages"), "anthropic_messages": _generic("anthropic_messages", "anthropic_messages"), "aresponses": _generic("aresponses"), - "responses": _generic("responses", sync=True), + "responses": _generic("responses", sync=True, single_attempt=False), } ) @@ -583,6 +583,8 @@ _SYNC_GENERIC: Final[Mapping[str, str]] = MappingProxyType( } ) +_ROUTER_RETRIED_SYNC_GENERIC: Final = frozenset({"generate_content", "generate_content_stream"}) + _CONTAINER_FILE_ROUTED: Final = frozenset( { "aupload_container_file", @@ -708,7 +710,12 @@ def generic_plan(endpoint: str) -> Plan | None: if endpoint in _VIDEO_DIRECT: return _generic(endpoint, f"videos.{endpoint}") if endpoint in _SYNC_GENERIC: - return _generic(endpoint, _SYNC_GENERIC[endpoint], sync=True) + return _generic( + endpoint, + _SYNC_GENERIC[endpoint], + sync=True, + single_attempt=endpoint not in _ROUTER_RETRIED_SYNC_GENERIC, + ) return None @@ -1658,6 +1665,7 @@ class RustEndpoints: def _sync_factory(self, name: str, plan: Plan) -> Callable[..., object]: is_vector_store: Final = name.startswith("vector_store_") + attempts: Final = _SINGLE_ATTEMPT if plan.single_attempt else None def sync_wrapper( custom_llm_provider: str | None = None, @@ -1669,7 +1677,7 @@ class RustEndpoints: raise TypeError( "_generic_api_call_with_fallbacks() missing 1 required positional argument: 'model'" ) - return self._core.call_sync(plan, kwargs, _SINGLE_ATTEMPT) + return self._core.call_sync(plan, kwargs, attempts) provider_kwargs: dict[ # mutable-ok: the request kwargs the Python router mutates in place # rebind-ok: rebound as the Python router rebinds it str, object ] = ( @@ -1683,7 +1691,7 @@ class RustEndpoints: "_direct_vector_store_embedding_executor": self._embedding_executor(kwargs), } if provider_kwargs.get("model"): - return self._core.call_sync(plan, provider_kwargs, _SINGLE_ATTEMPT) + return self._core.call_sync(plan, provider_kwargs, attempts) if name == "vector_store_search": return self._invoke(plan.host_op)(**{**provider_kwargs, "router": self._core.public}) return self._invoke(plan.host_op)(**provider_kwargs) diff --git a/litellm/utils.py b/litellm/utils.py index 717b1b9009a..a69d3c063de 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2060,11 +2060,9 @@ def _get_call_type_from_public_name(function_name: str) -> str: return function_name -def _is_litellm_router_call(kwargs: Mapping[str, object], *, is_async: bool) -> bool: - """Router completion uses metadata. Async generic calls retry with litellm_metadata; sync generic calls need SDK retries.""" - metadata_buckets: Final = ( - (kwargs.get("metadata"), kwargs.get("litellm_metadata")) if is_async else (kwargs.get("metadata"),) - ) +def _is_litellm_router_call(kwargs: Mapping[str, object]) -> bool: + """Router completion stamps the group on metadata, generic endpoints on litellm_metadata.""" + metadata_buckets: Final = (kwargs.get("metadata"), kwargs.get("litellm_metadata")) return any(isinstance(bucket, Mapping) and "model_group" in bucket for bucket in metadata_buckets) @@ -2322,7 +2320,7 @@ def client(original_function): litellm.num_retries = None # set retries to None to prevent infinite loops context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {}) - is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) + is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs) if ( num_retries and not is_completion_litellm_router_call @@ -2360,7 +2358,7 @@ def client(original_function): if not is_resent_failure(e): litellm.num_retries = None # set retries to None to prevent infinite loops - is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) + is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs) if ( num_retries and not is_responses_litellm_router_call @@ -2664,7 +2662,7 @@ def client(original_function): if retry_call_type == CallTypes.acompletion.value: context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {}) - is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) + is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs) if ( num_retries @@ -2700,7 +2698,7 @@ def client(original_function): result = await original_function(*args, **kwargs) return result elif retry_call_type == CallTypes.aresponses.value: - is_aresponses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) + is_aresponses_litellm_router_call: Final = _is_litellm_router_call(kwargs) if ( num_retries diff --git a/tests/unit/test_router/core/test_endpoints_characterization_generic.py b/tests/unit/test_router/core/test_endpoints_characterization_generic.py index b1a356cf8eb..f7669bff80c 100644 --- a/tests/unit/test_router/core/test_endpoints_characterization_generic.py +++ b/tests/unit/test_router/core/test_endpoints_characterization_generic.py @@ -195,7 +195,7 @@ async def test_sync_generic_endpoints_set_no_routing_hidden_params( assert response._hidden_params == {} -@pytest.mark.parametrize("endpoint", ["ocr", "generate_content"]) +@pytest.mark.parametrize("endpoint", ["ocr", "video_generation"]) async def test_sync_generic_endpoints_are_not_retried(make_core: MakeCore, host: EndpointHost, endpoint: str): host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) _, request = ROUTED_SYNC[endpoint] diff --git a/tests/unit/test_router/core/test_parity_regress5_errors.py b/tests/unit/test_router/core/test_parity_regress5_errors.py index 3e8537af878..bfeca17b3a2 100644 --- a/tests/unit/test_router/core/test_parity_regress5_errors.py +++ b/tests/unit/test_router/core/test_parity_regress5_errors.py @@ -114,12 +114,51 @@ async def test_a_providers_bad_request_falls_back_and_notes_the_failed_fallback( def test_a_single_attempt_endpoint_raises_with_no_note(make_core: MakeCore, host: EndpointHost) -> None: - core: Final = build(make_core, fallbacks=[{"a": ["b"]}]) + core: Final = build(make_core, fallbacks=[{"a": ["b"]}], num_retries=1) show_router_debug(host) host.answer("openai/a", server_error()) with pytest.raises(litellm.InternalServerError) as raised: - core.responses(model="a", input="hi") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint + core.ocr(model="a", document={}) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # an endpoint assert NOTE not in str(raised.value) + assert host.deployments_called() == ["a"] + + +ROUTER_RETRIED: Final[dict[str, tuple[str, str, Mapping[str, object]]]] = { + "responses": ("responses", "aresponses", {"input": "hi"}), + "generate_content": ("generate_content", "agenerate_content", {"contents": []}), +} + + +async def call_routed(core: RouterCore, endpoint: str, mode: str) -> object: + sync_name, async_name, request = ROUTER_RETRIED[endpoint] + if mode == "sync": + return getattr(core, sync_name)(model="a", **request) + return await getattr(core, async_name)(model="a", **request) + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize("endpoint", sorted(ROUTER_RETRIED)) +async def test_a_router_retried_endpoint_retries_a_failed_attempt_with_the_router_num_retries( + make_core: MakeCore, host: EndpointHost, endpoint: str, mode: str +) -> None: + core: Final = build(make_core, num_retries=1) + recovered: Final = completion_response(content="recovered") + host.answer("openai/a", server_error(), recovered) + assert await call_routed(core, endpoint, mode) is recovered + assert host.deployments_called() == ["a", "a"] + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize("endpoint", sorted(ROUTER_RETRIED)) +async def test_a_router_retried_endpoint_falls_back_to_the_next_group_when_its_group_fails( + make_core: MakeCore, host: EndpointHost, endpoint: str, mode: str +) -> None: + core: Final = build(make_core, fallbacks=[{"a": ["b"]}]) + fallback: Final = completion_response(content="from b") + host.answer("openai/a", server_error()) + host.answer("openai/b", fallback) + assert await call_routed(core, endpoint, mode) is fallback + assert host.deployments_called() == ["a", "b"] async def test_an_empty_request_list_says_no_fallbacks_are_configured(make_core: MakeCore, host: EndpointHost) -> None: diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 045d78fc1e7..da8c3c6a451 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -5706,20 +5706,17 @@ async def test_wrapper_async_logs_converted_responses_stream_with_standard_loggi @pytest.mark.parametrize( - "kwargs, is_async, expected", + "kwargs, expected", [ - ({"metadata": {"model_group": "g"}}, False, True), - ({"metadata": {"model_group": "g"}}, True, True), - ({"litellm_metadata": {"model_group": "g"}}, False, False), - ({"litellm_metadata": {"model_group": "g"}}, True, True), - ({}, False, False), - ({}, True, False), - ({"metadata": None}, False, False), - ({"metadata": None}, True, False), + ({"metadata": {"model_group": "g"}}, True), + ({"litellm_metadata": {"model_group": "g"}}, True), + ({"metadata": {"user": "u"}, "litellm_metadata": {"user": "u"}}, False), + ({}, False), + ({"metadata": None}, False), ], ) -def test_is_litellm_router_call_is_async_aware(kwargs: Mapping[str, object], is_async: bool, expected: bool) -> None: - assert _is_litellm_router_call(kwargs, is_async=is_async) is expected +def test_is_litellm_router_call_reads_both_metadata_buckets(kwargs: Mapping[str, object], expected: bool) -> None: + assert _is_litellm_router_call(kwargs) is expected @pytest.mark.asyncio @@ -5758,11 +5755,24 @@ async def test_router_aresponses_does_not_run_sdk_retries(monkeypatch: pytest.Mo litellm.in_memory_llm_clients_cache.flush_cache() -def test_router_responses_keeps_sdk_retries_for_sync_router_call( - monkeypatch: pytest.MonkeyPatch, -) -> None: +SYNC_ROUTER_RETRIED_CALLS: Final[dict[str, tuple[str, Callable[[litellm.Router], object]]]] = { + "responses": ("/v1/responses", lambda router: router.responses(model="responses-retry", input="hi")), + "generate_content": ( + "/v1/chat/completions", + lambda router: router.generate_content( + model="responses-retry", contents=[{"role": "user", "parts": [{"text": "hi"}]}] + ), + ), +} + + +def _count_sync_router_attempts( + monkeypatch: pytest.MonkeyPatch, endpoint: str, router_num_retries: int, deployment_num_retries: int | None +) -> int: + path, call = SYNC_ROUTER_RETRIED_CALLS[endpoint] monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() + retries: Final = {} if deployment_num_retries is None else {"num_retries": deployment_num_retries} model_list: Final = [ { "model_name": "responses-retry", @@ -5770,15 +5780,16 @@ def test_router_responses_keeps_sdk_retries_for_sync_router_call( "model": "openai/gpt-4o-mini", "api_key": "sk-test", "api_base": "https://responses-retry.local/v1", - "num_retries": 2, + **retries, }, } ] - router: Final = litellm.Router(model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True) - + router: Final = litellm.Router( + model_list=model_list, num_retries=router_num_retries, retry_after=0, disable_cooldowns=True + ) try: with respx.mock(assert_all_called=True) as respx_mock: - upstream: Final = respx_mock.post("https://responses-retry.local/v1/responses").mock( + upstream: Final = respx_mock.post(f"https://responses-retry.local{path}").mock( return_value=httpx.Response( 503, headers={"retry-after": "0"}, @@ -5786,14 +5797,29 @@ def test_router_responses_keeps_sdk_retries_for_sync_router_call( ) ) with pytest.raises(litellm.ServiceUnavailableError): - router.responses(model="responses-retry", input="hi") - - assert upstream.call_count == 3 + call(router) + return upstream.call_count finally: router.discard() litellm.in_memory_llm_clients_cache.flush_cache() +@pytest.mark.parametrize("router_num_retries", [0, 1]) +@pytest.mark.parametrize("endpoint", sorted(SYNC_ROUTER_RETRIED_CALLS)) +def test_sync_router_call_retries_as_often_as_the_deployment_num_retries( + monkeypatch: pytest.MonkeyPatch, router_num_retries: int, endpoint: str +) -> None: + assert _count_sync_router_attempts(monkeypatch, endpoint, router_num_retries, deployment_num_retries=2) == 3 + + +@pytest.mark.parametrize("endpoint", sorted(SYNC_ROUTER_RETRIED_CALLS)) +def test_sync_router_call_does_not_nest_the_global_num_retries_under_router_retries( + monkeypatch: pytest.MonkeyPatch, endpoint: str +) -> None: + monkeypatch.setattr(litellm, "num_retries", 2) + assert _count_sync_router_attempts(monkeypatch, endpoint, router_num_retries=1, deployment_num_retries=None) == 2 + + @pytest.mark.asyncio async def test_aresponses_uses_sdk_retries_without_router(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) From 894552205b1749f843db91880593d9b921000fdd Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 10 Nov 2026 13:02:26 -0800 Subject: [PATCH 037/189] feat(spend): report guardrail spend separately in daily activity (#51010) * feat(spend): report guardrail spend separately in daily activity Model and key spend in /user/daily/activity/aggregated include the cost of guardrails like Bedrock guardrails, and nothing in the response says how much. A customer that bills tenants per key and per model could not get model cost alone, so tenants that pay AWS for their own guardrails were charged for them twice Daily spend rows now carry a guardrail_spend column next to spend, and every breakdown in the daily activity responses exposes it, so model cost is spend - guardrail_spend. spend keeps its meaning, so budgets still count guardrail cost as before The value comes from the request's cost_breakdown.guardrail_cost, capped at the row's spend because an allowance discount or a repeat job poll can charge less than the guardrail cost the breakdown records. The delivery ledger freezes it into the accounting payload next to accounting_savings, since the stored payload does not keep cost_breakdown, and the legacy daily queue writes the same value The migration only adds the column with a 0 default, so rows written before it read 0 * fix(spend): count guardrail spend on rows billed without a cost breakdown A request blocked before the LLM call is billed for its guardrails straight from guardrail_information, so it carries no cost_breakdown and its guardrail_spend read 0. Both writers now price guardrail_spend from guardrail_information with guardrail_information_cost, the same function success and failure rows use to add guardrail cost to spend Also fills guardrail_spend in the client usage view built from request logs, qualifies the field description for rows recorded before it existed, and adds the lock timeout the migration checks require * fix(ui): avoid a nested ternary when reading guardrail entries from request logs --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../migration.sql | 22 +++++ .../litellm_proxy_extras/schema.prisma | 7 ++ litellm/accounting/projection.py | 1 + litellm/accounting/snapshot.py | 2 + .../llm_cost_calc/guardrail_cost.py | 6 ++ litellm/proxy/_lazy_openapi_snapshot.json | 6 ++ litellm/proxy/_types.py | 1 + litellm/proxy/db/daily_spend_bulk_upsert.py | 1 + litellm/proxy/db/db_spend_update_writer.py | 5 ++ .../daily_spend_update_queue.py | 4 + .../common_daily_activity.py | 6 ++ litellm/proxy/schema.prisma | 7 ++ .../repositories/daily_activity_repository.py | 1 + litellm/repositories/daily_activity_sql.py | 1 + .../common_daily_activity.py | 6 ++ litellm/types/repositories/daily_activity.py | 2 + schema.prisma | 7 ++ .../database/test_spendlogs_ledger_parity.py | 2 + .../spend/fixtures/daily_activity_team.json | 2 +- .../spend/fixtures/daily_activity_user.json | 2 +- .../daily_activity_team_aggregated.json | 42 ++++++++++ .../golden/daily_activity_team_paginated.json | 42 ++++++++++ .../daily_activity_user_aggregated.json | 42 ++++++++++ .../golden/daily_activity_user_paginated.json | 42 ++++++++++ .../spend/test_daily_activity_routes.py | 1 + .../spend/test_daily_rollup_retry.py | 1 + tests/unit/accounting/test_projection.py | 19 ++++- .../llm_cost_calc/test_guardrail_cost.py | 13 +++ .../test_daily_spend_update_queue.py | 2 + .../proxy/db/test_daily_spend_bulk_upsert.py | 7 +- .../test_common_daily_activity.py | 80 ++++++++++++++++++- .../test_daily_activity_routes.py | 5 ++ .../test_team_endpoints.py | 1 + .../test_daily_activity_repository.py | 1 + .../repositories/test_daily_activity_sql.py | 1 + .../_components/CacheLeakageCard.test.tsx | 1 + .../_components/hooks/clientUsageData.test.ts | 20 +++++ .../_components/hooks/clientUsageData.ts | 22 +++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 ++ 39 files changed, 432 insertions(+), 7 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261110120000_add_daily_spend_guardrail_spend/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261110120000_add_daily_spend_guardrail_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261110120000_add_daily_spend_guardrail_spend/migration.sql new file mode 100644 index 00000000000..c2c264325a6 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261110120000_add_daily_spend_guardrail_spend/migration.sql @@ -0,0 +1,22 @@ +SET LOCAL lock_timeout = '10s'; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyGlobalSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 37783275d9b..5a5d56cc6e2 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -883,6 +883,7 @@ model LiteLLM_DailyUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -919,6 +920,7 @@ model LiteLLM_DailyGlobalSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -952,6 +954,7 @@ model LiteLLM_DailyOrganizationSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -990,6 +993,7 @@ model LiteLLM_DailyEndUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1027,6 +1031,7 @@ model LiteLLM_DailyAgentSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1064,6 +1069,7 @@ model LiteLLM_DailyTeamSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1104,6 +1110,7 @@ model LiteLLM_DailyTagSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/accounting/projection.py b/litellm/accounting/projection.py index dd52b49f101..9d04d39884e 100644 --- a/litellm/accounting/projection.py +++ b/litellm/accounting/projection.py @@ -133,6 +133,7 @@ def _daily_counters(row: Mapping[str, object], metadata: Mapping[str, object]) - ("prompt_caching_savings_spend", _number(savings.get("prompt_caching"))), ("gateway_injected_caching_savings_spend", _number(savings.get("gateway_injected_caching"))), ("autorouter_savings_spend", 0.0 if internal else _number(savings.get("autorouter"))), + ("guardrail_spend", _number(metadata.get("accounting_guardrail_spend"))), ("total_response_time_ms", _number(duration) if timed else 0), ("timed_requests", int(timed)), ) diff --git a/litellm/accounting/snapshot.py b/litellm/accounting/snapshot.py index ab85df08320..874b90110c1 100644 --- a/litellm/accounting/snapshot.py +++ b/litellm/accounting/snapshot.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_share_of_spend from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure from litellm.proxy._types import SpendLogsPayload from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens @@ -73,6 +74,7 @@ def build_accounting_payload( **{key: _accounting_metadata_field(key, value) for key, value in metadata.items() if key in _METADATA_FIELDS}, "accounting_entities": entities, "accounting_savings": savings._asdict(), + "accounting_guardrail_spend": guardrail_share_of_spend(metadata, payload["spend"]), "accounting_model_access_groups": model_access_groups, "accounting_windows": windows, } diff --git a/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py b/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py index e4ad36f50bc..03f6794889c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py +++ b/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py @@ -311,6 +311,12 @@ def guardrail_information_cost(guardrail_information: object) -> float: return _validated_entry_cost(guardrail_information) +def guardrail_share_of_spend(spend_log_metadata: Mapping[str, object], spend: float) -> float: + """The part of a spend row's ``spend`` billed for its guardrails, priced the way success and failure rows add + it to spend and capped at the row's spend, since an allowance discount or a repeat job poll can charge less.""" + return min(guardrail_information_cost(spend_log_metadata.get("guardrail_information")), max(spend, 0.0)) + + def cost_breakdown_with_guardrail(cost_breakdown: CostBreakdown | None, guardrail_cost: float) -> CostBreakdown | None: if guardrail_cost <= 0.0: return cost_breakdown diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 2cadfd6c4a3..36e2cb46e90 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -4882,6 +4882,12 @@ "title": "Gateway Injected Caching Savings Spend", "type": "number" }, + "guardrail_spend": { + "default": 0.0, + "description": "Guardrail spend in USD, already included in spend, so model spend is spend - guardrail_spend. Rows recorded before this field existed report 0 even when their spend includes guardrail cost", + "title": "Guardrail Spend", + "type": "number" + }, "prompt_caching_savings_spend": { "default": 0.0, "title": "Prompt Caching Savings Spend", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 228e80da4bd..736c09277f1 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5335,6 +5335,7 @@ class BaseDailySpendTransaction(TypedDict): # missing value to zero, so requiring it here would describe a shape the aggregation # is explicitly tested against. autorouter_savings_spend: NotRequired[float] + guardrail_spend: NotRequired[float] # writable-ok: the rollup queue accumulates into this key in place # request level metrics spend: float diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py index 2d1770f4c1d..38b132e7e10 100644 --- a/litellm/proxy/db/daily_spend_bulk_upsert.py +++ b/litellm/proxy/db/daily_spend_bulk_upsert.py @@ -98,6 +98,7 @@ _SPEND_COLUMNS: Final = ( "prompt_caching_savings_spend", "gateway_injected_caching_savings_spend", "autorouter_savings_spend", + "guardrail_spend", ) _CASTS: Final[Mapping[str, str]] = MappingProxyType( diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 4a1871ff1f5..fc280dd4f73 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -39,6 +39,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.batch_utils import BATCH_COST_REQUEST_ID_SUFFIX from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_share_of_spend from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.proxy._types import ( DB_RETRY_SAFE_ERROR_TYPES, @@ -3164,6 +3165,10 @@ class DBSpendUpdateWriter: prompt_caching_savings_spend=savings_spend.prompt_caching, gateway_injected_caching_savings_spend=savings_spend.gateway_injected_caching, autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter, + guardrail_spend=guardrail_share_of_spend( + _metadata, + payload["spend"], # pyright: ignore[reportUnknownArgumentType] # legacy dict | SpendLogsPayload union + ), total_response_time_ms=timed_duration_ms or 0, timed_requests=0 if timed_duration_ms is None else 1, ) diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 288c85c3513..70a9aaa5512 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -154,6 +154,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("autorouter_savings_spend", 0) or 0 ) + daily_transaction.get("autorouter_savings_spend", 0) + daily_transaction["guardrail_spend"] = ( + payload.get("guardrail_spend", 0) or 0 + ) + daily_transaction.get("guardrail_spend", 0) + daily_transaction["total_response_time_ms"] = ( payload.get("total_response_time_ms", 0) or 0 ) + daily_transaction.get("total_response_time_ms", 0) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index a0db1b00ec4..80d873587c3 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -163,6 +163,9 @@ class DailySpendRecord(Protocol): @property def autorouter_savings_spend(self) -> float: ... + @property + def guardrail_spend(self) -> float: ... + @property def api_requests(self) -> int: ... @@ -250,6 +253,7 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) -> existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0 existing_metrics.gateway_injected_caching_savings_spend += record.gateway_injected_caching_savings_spend or 0 existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0 + existing_metrics.guardrail_spend += record.guardrail_spend or 0 # rebind-ok: callers rely on in-place accumulation existing_metrics.api_requests += record.api_requests or 0 existing_metrics.successful_requests += record.successful_requests or 0 existing_metrics.failed_requests += record.failed_requests or 0 @@ -736,6 +740,7 @@ def sum_spend_metrics(metrics: Sequence[SpendMetrics]) -> SpendMetrics: prompt_caching_savings_spend=sum(metric.prompt_caching_savings_spend for metric in metrics), gateway_injected_caching_savings_spend=sum(metric.gateway_injected_caching_savings_spend for metric in metrics), autorouter_savings_spend=sum(metric.autorouter_savings_spend for metric in metrics), + guardrail_spend=sum(metric.guardrail_spend for metric in metrics), total_tokens=sum(metric.total_tokens for metric in metrics), successful_requests=sum(metric.successful_requests for metric in metrics), failed_requests=sum(metric.failed_requests for metric in metrics), @@ -766,6 +771,7 @@ def _record_to_spend_metrics(record: RollupMetricsRow) -> SpendMetrics: prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0, gateway_injected_caching_savings_spend=record.gateway_injected_caching_savings_spend or 0, autorouter_savings_spend=record.autorouter_savings_spend or 0, + guardrail_spend=record.guardrail_spend or 0, api_requests=record.api_requests or 0, successful_requests=record.successful_requests or 0, failed_requests=record.failed_requests or 0, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 37783275d9b..5a5d56cc6e2 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -883,6 +883,7 @@ model LiteLLM_DailyUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -919,6 +920,7 @@ model LiteLLM_DailyGlobalSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -952,6 +954,7 @@ model LiteLLM_DailyOrganizationSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -990,6 +993,7 @@ model LiteLLM_DailyEndUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1027,6 +1031,7 @@ model LiteLLM_DailyAgentSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1064,6 +1069,7 @@ model LiteLLM_DailyTeamSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1104,6 +1110,7 @@ model LiteLLM_DailyTagSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 809f78e378c..9aa0950785b 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -132,6 +132,7 @@ class _CustomerDailyRow(BaseModel): prompt_caching_savings_spend: float gateway_injected_caching_savings_spend: float autorouter_savings_spend: float + guardrail_spend: float spend: float api_requests: int successful_requests: int diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 5d1f85658ea..dc3ca19bb30 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -223,6 +223,7 @@ def _rollup_metric_select(table: DailyActivityTable) -> str: SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend, SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend, SUM(autorouter_savings_spend)::float AS autorouter_savings_spend, + SUM(guardrail_spend)::float AS guardrail_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, SUM(failed_requests)::bigint AS failed_requests, diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 19791b7f620..c4816e7d1ea 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -31,6 +31,11 @@ class SpendMetrics(LiteLLMBaseModel): prompt_caching_savings_spend: float = Field(default=0.0) gateway_injected_caching_savings_spend: float = Field(default=0.0) autorouter_savings_spend: float = Field(default=0.0) + guardrail_spend: float = Field( + default=0.0, + description="Guardrail spend in USD, already included in spend, so model spend is spend - guardrail_spend. " + "Rows recorded before this field existed report 0 even when their spend includes guardrail cost", + ) total_tokens: int = Field(default=0) successful_requests: int = Field(default=0) failed_requests: int = Field(default=0) @@ -259,6 +264,7 @@ class LiteLLM_DailyUserSpend(LiteLLMBaseModel): prompt_caching_savings_spend: float = 0.0 gateway_injected_caching_savings_spend: float = 0.0 autorouter_savings_spend: float = 0.0 + guardrail_spend: float = 0.0 spend: float = 0.0 api_requests: int = 0 successful_requests: int = 0 diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index 27e169cec5a..8d68d92a336 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -144,6 +144,7 @@ class RollupMetricsRow: prompt_caching_savings_spend: float | None gateway_injected_caching_savings_spend: float | None autorouter_savings_spend: float | None + guardrail_spend: float | None = field(default=None, kw_only=True) api_requests: int | None successful_requests: int | None failed_requests: int | None @@ -219,6 +220,7 @@ class DailyActivityRow(Protocol): prompt_caching_savings_spend: float gateway_injected_caching_savings_spend: float autorouter_savings_spend: float + guardrail_spend: float spend: float api_requests: int successful_requests: int diff --git a/schema.prisma b/schema.prisma index 37783275d9b..5a5d56cc6e2 100644 --- a/schema.prisma +++ b/schema.prisma @@ -883,6 +883,7 @@ model LiteLLM_DailyUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -919,6 +920,7 @@ model LiteLLM_DailyGlobalSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -952,6 +954,7 @@ model LiteLLM_DailyOrganizationSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -990,6 +993,7 @@ model LiteLLM_DailyEndUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1027,6 +1031,7 @@ model LiteLLM_DailyAgentSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1064,6 +1069,7 @@ model LiteLLM_DailyTeamSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1104,6 +1110,7 @@ model LiteLLM_DailyTagSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/tests/integration/database/test_spendlogs_ledger_parity.py b/tests/integration/database/test_spendlogs_ledger_parity.py index 7a8c57d234e..3019d75a2a7 100644 --- a/tests/integration/database/test_spendlogs_ledger_parity.py +++ b/tests/integration/database/test_spendlogs_ledger_parity.py @@ -424,6 +424,7 @@ async def _assert_legacy_database_parity( "guardrail_status": "success", "guardrail_usage": {"text": 2}, "guardrail_cost_by_unit": {"text": 0.02}, + "guardrail_cost": 0.02, "guardrail_cost_in_spend": True, } ], @@ -579,6 +580,7 @@ async def _assert_legacy_database_parity( assert daily["autorouter_savings_spend"] == pytest.approx( (0 if origin == "auto_router" else 0.125) + (0.2 if correction_order else 0) ) + assert daily["guardrail_spend"] == pytest.approx(min(0.02, cost)) group_spend = {row["access_group_name"]: row["spend"] for row in results[-1]["LiteLLM_ModelAccessGroupBudgetTable"]} assert group_spend == {"group": cost + prior_spend, **({"unserved-group": prior_spend} if fallback_narrowing else {})} assert results[-1]["LiteLLM_DailyGuardrailUsageUnits"] diff --git a/tests/integration/spend/fixtures/daily_activity_team.json b/tests/integration/spend/fixtures/daily_activity_team.json index 8f4956dd7fe..cf42f7ae2dd 100644 --- a/tests/integration/spend/fixtures/daily_activity_team.json +++ b/tests/integration/spend/fixtures/daily_activity_team.json @@ -1 +1 @@ -{"results":[{"date":"2026-06-01","metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"breakdown":{"mcp_servers":{},"models":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"model_groups":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"providers":{"provider-a":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"endpoints":{"/v1/chat/completions":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"api_keys":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}},"entities":{"team-1":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{"team_alias":"Usage Team"},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}}},"guardrails":{}}}],"metadata":{"total_spend":1273.0,"total_flat_cost":0.0,"total_ptu_hours":0.0,"total_prompt_tokens":1014,"total_completion_tokens":10,"total_tokens":1024,"total_api_requests":5,"total_successful_requests":5,"total_failed_requests":0,"total_cache_read_input_tokens":6,"total_cache_creation_input_tokens":0,"total_compression_saved_tokens":0,"total_compression_savings_spend":0.0,"total_prompt_caching_savings_spend":0.0,"total_gateway_injected_caching_savings_spend":0.0,"total_autorouter_savings_spend":0.0,"total_response_time_ms":0,"total_timed_requests":0,"page":1,"total_pages":1,"has_more":false,"api_key_limit":100,"total_api_keys":5,"entity_total_api_keys":{"team-1":5}}} +{"results":[{"date":"2026-06-01","metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"breakdown":{"mcp_servers":{},"models":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"model_groups":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"providers":{"provider-a":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"endpoints":{"/v1/chat/completions":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"api_keys":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}},"entities":{"team-1":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{"team_alias":"Usage Team"},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}}},"guardrails":{}}}],"metadata":{"total_spend":1273.0,"total_flat_cost":0.0,"total_ptu_hours":0.0,"total_prompt_tokens":1014,"total_completion_tokens":10,"total_tokens":1024,"total_api_requests":5,"total_successful_requests":5,"total_failed_requests":0,"total_cache_read_input_tokens":6,"total_cache_creation_input_tokens":0,"total_compression_saved_tokens":0,"total_compression_savings_spend":0.0,"total_prompt_caching_savings_spend":0.0,"total_gateway_injected_caching_savings_spend":0.0,"total_autorouter_savings_spend":0.0,"total_response_time_ms":0,"total_timed_requests":0,"page":1,"total_pages":1,"has_more":false,"api_key_limit":100,"total_api_keys":5,"entity_total_api_keys":{"team-1":5}}} diff --git a/tests/integration/spend/fixtures/daily_activity_user.json b/tests/integration/spend/fixtures/daily_activity_user.json index 058e12a5044..53f3bfd5487 100644 --- a/tests/integration/spend/fixtures/daily_activity_user.json +++ b/tests/integration/spend/fixtures/daily_activity_user.json @@ -1 +1 @@ -{"results":[{"date":"2026-06-01","metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"breakdown":{"mcp_servers":{},"models":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"model_groups":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"providers":{"provider-a":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"endpoints":{"/v1/chat/completions":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"api_keys":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}},"entities":{"user-1":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{"user_email":"user@example.com","user_alias":null},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}}},"guardrails":{}}}],"metadata":{"total_spend":1273.0,"total_flat_cost":0.0,"total_ptu_hours":0.0,"total_prompt_tokens":1014,"total_completion_tokens":10,"total_tokens":1024,"total_api_requests":5,"total_successful_requests":5,"total_failed_requests":0,"total_cache_read_input_tokens":6,"total_cache_creation_input_tokens":0,"total_compression_saved_tokens":0,"total_compression_savings_spend":0.0,"total_prompt_caching_savings_spend":0.0,"total_gateway_injected_caching_savings_spend":0.0,"total_autorouter_savings_spend":0.0,"total_response_time_ms":0,"total_timed_requests":0,"page":1,"total_pages":1,"has_more":false,"api_key_limit":100,"total_api_keys":5,"entity_total_api_keys":{"user-1":5}}} +{"results":[{"date":"2026-06-01","metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"breakdown":{"mcp_servers":{},"models":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"model_groups":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"providers":{"provider-a":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"endpoints":{"/v1/chat/completions":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"api_keys":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}},"entities":{"user-1":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{"user_email":"user@example.com","user_alias":null},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"guardrail_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}}},"guardrails":{}}}],"metadata":{"total_spend":1273.0,"total_flat_cost":0.0,"total_ptu_hours":0.0,"total_prompt_tokens":1014,"total_completion_tokens":10,"total_tokens":1024,"total_api_requests":5,"total_successful_requests":5,"total_failed_requests":0,"total_cache_read_input_tokens":6,"total_cache_creation_input_tokens":0,"total_compression_saved_tokens":0,"total_compression_savings_spend":0.0,"total_prompt_caching_savings_spend":0.0,"total_gateway_injected_caching_savings_spend":0.0,"total_autorouter_savings_spend":0.0,"total_response_time_ms":0,"total_timed_requests":0,"page":1,"total_pages":1,"has_more":false,"api_key_limit":100,"total_api_keys":5,"entity_total_api_keys":{"user-1":5}}} diff --git a/tests/integration/spend/golden/daily_activity_team_aggregated.json b/tests/integration/spend/golden/daily_activity_team_aggregated.json index 91a5d33391b..ce8ccbcaafb 100644 --- a/tests/integration/spend/golden/daily_activity_team_aggregated.json +++ b/tests/integration/spend/golden/daily_activity_team_aggregated.json @@ -39,6 +39,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -68,6 +69,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -97,6 +99,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -126,6 +129,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -155,6 +159,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -188,6 +193,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -217,6 +223,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -246,6 +253,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -275,6 +283,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -304,6 +313,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -327,6 +337,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -360,6 +371,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -389,6 +401,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -418,6 +431,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -447,6 +461,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -476,6 +491,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -501,6 +517,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -535,6 +552,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -558,6 +576,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -589,6 +608,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -618,6 +638,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -647,6 +668,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -670,6 +692,7 @@ "metrics": { "api_requests": 3, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 3, "completion_tokens": 6, @@ -694,6 +717,7 @@ "metrics": { "api_requests": 0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "completion_tokens": 0, @@ -725,6 +749,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -748,6 +773,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -781,6 +807,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -804,6 +831,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -835,6 +863,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -864,6 +893,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -893,6 +923,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -916,6 +947,7 @@ "metrics": { "api_requests": 3, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 3, "completion_tokens": 6, @@ -940,6 +972,7 @@ "metrics": { "api_requests": 0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "completion_tokens": 0, @@ -971,6 +1004,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -994,6 +1028,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -1027,6 +1062,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1056,6 +1092,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1085,6 +1122,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1114,6 +1152,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1143,6 +1182,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -1166,6 +1206,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -1190,6 +1231,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, diff --git a/tests/integration/spend/golden/daily_activity_team_paginated.json b/tests/integration/spend/golden/daily_activity_team_paginated.json index 18ee30a74e6..43aee6bdfb6 100644 --- a/tests/integration/spend/golden/daily_activity_team_paginated.json +++ b/tests/integration/spend/golden/daily_activity_team_paginated.json @@ -39,6 +39,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -68,6 +69,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -97,6 +99,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -126,6 +129,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -155,6 +159,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -188,6 +193,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -217,6 +223,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -246,6 +253,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -275,6 +283,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -304,6 +313,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -327,6 +337,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -360,6 +371,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -389,6 +401,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -418,6 +431,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -447,6 +461,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -476,6 +491,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -501,6 +517,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -535,6 +552,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -558,6 +576,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -589,6 +608,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -618,6 +638,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -647,6 +668,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -670,6 +692,7 @@ "metrics": { "api_requests": 3, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 3, "completion_tokens": 6, @@ -694,6 +717,7 @@ "metrics": { "api_requests": 0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "completion_tokens": 0, @@ -725,6 +749,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -748,6 +773,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -781,6 +807,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -804,6 +831,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -835,6 +863,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -864,6 +893,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -893,6 +923,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -916,6 +947,7 @@ "metrics": { "api_requests": 3, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 3, "completion_tokens": 6, @@ -940,6 +972,7 @@ "metrics": { "api_requests": 0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "completion_tokens": 0, @@ -971,6 +1004,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -994,6 +1028,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -1027,6 +1062,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1056,6 +1092,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1085,6 +1122,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1114,6 +1152,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1143,6 +1182,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -1166,6 +1206,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -1190,6 +1231,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, diff --git a/tests/integration/spend/golden/daily_activity_user_aggregated.json b/tests/integration/spend/golden/daily_activity_user_aggregated.json index 3189c000035..35d7e57eba8 100644 --- a/tests/integration/spend/golden/daily_activity_user_aggregated.json +++ b/tests/integration/spend/golden/daily_activity_user_aggregated.json @@ -39,6 +39,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -68,6 +69,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -97,6 +99,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -126,6 +129,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -155,6 +159,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -188,6 +193,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -217,6 +223,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -246,6 +253,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -275,6 +283,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -304,6 +313,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -327,6 +337,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -360,6 +371,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -389,6 +401,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -418,6 +431,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -447,6 +461,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -476,6 +491,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -502,6 +518,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -536,6 +553,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -559,6 +577,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -590,6 +609,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -619,6 +639,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -648,6 +669,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -671,6 +693,7 @@ "metrics": { "api_requests": 3, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 3, "completion_tokens": 6, @@ -695,6 +718,7 @@ "metrics": { "api_requests": 0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "completion_tokens": 0, @@ -726,6 +750,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -749,6 +774,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -782,6 +808,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -805,6 +832,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -836,6 +864,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -865,6 +894,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -894,6 +924,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -917,6 +948,7 @@ "metrics": { "api_requests": 3, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 3, "completion_tokens": 6, @@ -941,6 +973,7 @@ "metrics": { "api_requests": 0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "completion_tokens": 0, @@ -972,6 +1005,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -995,6 +1029,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -1028,6 +1063,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1057,6 +1093,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1086,6 +1123,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1115,6 +1153,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1144,6 +1183,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -1167,6 +1207,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -1191,6 +1232,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, diff --git a/tests/integration/spend/golden/daily_activity_user_paginated.json b/tests/integration/spend/golden/daily_activity_user_paginated.json index 4f7b62b3147..9ce806e48b5 100644 --- a/tests/integration/spend/golden/daily_activity_user_paginated.json +++ b/tests/integration/spend/golden/daily_activity_user_paginated.json @@ -39,6 +39,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -68,6 +69,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -97,6 +99,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -126,6 +129,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -155,6 +159,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -188,6 +193,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -217,6 +223,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -246,6 +253,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -275,6 +283,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -304,6 +313,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -327,6 +337,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -360,6 +371,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -389,6 +401,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -418,6 +431,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -447,6 +461,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -476,6 +491,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -502,6 +518,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -536,6 +553,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -559,6 +577,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -590,6 +609,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -619,6 +639,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -648,6 +669,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -671,6 +693,7 @@ "metrics": { "api_requests": 3, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 3, "completion_tokens": 6, @@ -695,6 +718,7 @@ "metrics": { "api_requests": 0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "completion_tokens": 0, @@ -726,6 +750,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -749,6 +774,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -782,6 +808,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -805,6 +832,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -836,6 +864,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -865,6 +894,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -894,6 +924,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -917,6 +948,7 @@ "metrics": { "api_requests": 3, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 3, "completion_tokens": 6, @@ -941,6 +973,7 @@ "metrics": { "api_requests": 0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "completion_tokens": 0, @@ -972,6 +1005,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -995,6 +1029,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -1028,6 +1063,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1057,6 +1093,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1086,6 +1123,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1115,6 +1153,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 1, "completion_tokens": 2, @@ -1144,6 +1183,7 @@ "metrics": { "api_requests": 1, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 2, "completion_tokens": 2, @@ -1167,6 +1207,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, @@ -1191,6 +1232,7 @@ "metrics": { "api_requests": 5, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 6, "completion_tokens": 10, diff --git a/tests/integration/spend/test_daily_activity_routes.py b/tests/integration/spend/test_daily_activity_routes.py index 9bfe4bb50ce..972d2d87098 100644 --- a/tests/integration/spend/test_daily_activity_routes.py +++ b/tests/integration/spend/test_daily_activity_routes.py @@ -402,6 +402,7 @@ def _daily_tag_metrics( "prompt_caching_savings_spend": 0.0, "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "total_tokens": prompt_tokens + completion_tokens, "successful_requests": api_requests, "failed_requests": 0, diff --git a/tests/integration/spend/test_daily_rollup_retry.py b/tests/integration/spend/test_daily_rollup_retry.py index 9ed712cdd06..47a8fb8d7bf 100644 --- a/tests/integration/spend/test_daily_rollup_retry.py +++ b/tests/integration/spend/test_daily_rollup_retry.py @@ -152,6 +152,7 @@ def test_failed_daily_user_rollup_commit_is_retried_so_spend_report_and_daily_ac "prompt_caching_savings_spend": pytest.approx(0.0), "gateway_injected_caching_savings_spend": pytest.approx(0.0), "autorouter_savings_spend": pytest.approx(0.0), + "guardrail_spend": pytest.approx(0.0), "total_tokens": 40, "successful_requests": 1, "failed_requests": 0, diff --git a/tests/unit/accounting/test_projection.py b/tests/unit/accounting/test_projection.py index c78d6bec757..e618c706fac 100644 --- a/tests/unit/accounting/test_projection.py +++ b/tests/unit/accounting/test_projection.py @@ -23,8 +23,23 @@ _FIELDS: Final = TypeAdapter(dict[str, object]) @pytest.mark.parametrize("status", ["success", "failure"]) @pytest.mark.parametrize("origin", [None, "auto_router", "billed_retry"]) @pytest.mark.parametrize("spend", [0.0, 0.25]) +@pytest.mark.parametrize( + ("guardrail_information", "billed_guardrail_cost"), + [ + (None, 0.0), + ([{"guardrail_name": "guard", "guardrail_cost": 0.1}], 0.1), + ([{"guardrail_name": "guard", "guardrail_cost": 0.5}], 0.5), + ([{"guardrail_name": "guard", "guardrail_cost": 0.1, "guardrail_cost_in_spend": False}], 0.0), + ], +) @pytest.mark.asyncio -async def test_daily_projection_matches_legacy_writer(status: str, origin: str | None, spend: float) -> None: +async def test_daily_projection_matches_legacy_writer( + status: str, + origin: str | None, + spend: float, + guardrail_information: list[dict[str, object]] | None, + billed_guardrail_cost: float, +) -> None: payload: Final = get_logging_payload( kwargs={ "model": "projection-test", @@ -43,6 +58,7 @@ async def test_daily_projection_matches_legacy_writer(status: str, origin: str | **_FIELDS.validate_json(payload["metadata"]), **({BILLED_RETRY_ATTEMPT_METADATA_KEY: True} if origin == "billed_retry" else {}), **({INTERNAL_CALL_ORIGIN_METADATA_KEY: origin} if origin not in (None, "billed_retry") else {}), + **({"guardrail_information": guardrail_information} if guardrail_information is not None else {}), } row: Final = {**payload, "spend": spend, "metadata": json.dumps(metadata), "request_duration_ms": 25} legacy: Final = await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction( @@ -60,6 +76,7 @@ async def test_daily_projection_matches_legacy_writer(status: str, origin: str | restored: Final = _FIELDS.validate_json(json.dumps(accounting, default=str)) stored_daily: Final = next(item for item in project(restored) if item.table == "LiteLLM_DailyUserSpend") assert dict(stored_daily.increments) == {name: legacy[name] for name, _ in stored_daily.increments} + assert dict(stored_daily.increments)["guardrail_spend"] == min(billed_guardrail_cost, spend) assert dict(stored_daily.key)["date"] == legacy["date"] assert dict(stored_daily.key)["api_key"] == legacy["api_key"] assert not {"messages", "response", "proxy_server_request", "requester_ip_address"}.intersection(accounting) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py index 949f55b8a82..35813996a4f 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py @@ -13,6 +13,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( cost_breakdown_with_guardrail, guardrail_cost_total, guardrail_information_cost, + guardrail_share_of_spend, ) @@ -349,3 +350,15 @@ def test_native_trace_entry_redacts_the_pii_matches_before_they_reach_spend_logs assert entry["guardrail_status"] == "guardrail_intervened" assert "123-45-6789" not in json.dumps(entry["guardrail_response"]) assert entry["guardrail_usage"]["sensitiveInformationPolicyUnits"] == 2 + + + +@pytest.mark.parametrize(("spend", "expected"), [(1.0, 0.3), (0.2, 0.2), (-1.0, 0.0)]) +def test_guardrail_share_of_spend_is_the_billed_guardrail_cost_capped_at_the_row_spend( + spend: float, expected: float +) -> None: + information: Final = [ + {"guardrail_name": "billed", "guardrail_cost": 0.3}, + {"guardrail_name": "report-only", "guardrail_cost": 0.4, "guardrail_cost_in_spend": False}, + ] + assert guardrail_share_of_spend({"guardrail_information": information}, spend) == pytest.approx(expected) diff --git a/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index 79447d0fc59..497ebd3093a 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -209,6 +209,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "prompt_caching_savings_spend": 0, "gateway_injected_caching_savings_spend": 0, "autorouter_savings_spend": 0, + "guardrail_spend": 0, "total_response_time_ms": 0, "timed_requests": 0, } @@ -263,6 +264,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "prompt_caching_savings_spend": 0, "gateway_injected_caching_savings_spend": 0, "autorouter_savings_spend": 0, + "guardrail_spend": 0, "total_response_time_ms": 0, "timed_requests": 0, } diff --git a/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py b/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py index 5bce9137666..1b5dd90205c 100644 --- a/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py +++ b/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py @@ -88,10 +88,10 @@ def test_one_statement_carries_every_row_in_the_batch(): assert sql.count("INSERT INTO") == 1 assert len(re.findall(r"ON CONFLICT", sql)) == 1 - # 25 bound columns per row plus the inlined updated_at, so the row count is what + # 26 bound columns per row plus the inlined updated_at, so the row count is what # separates one multi-row statement from a hundred single-row ones. - assert len(params) == 100 * 25 - assert "$2500::text" in sql + assert len(params) == 100 * 26 + assert "$2600::text" in sql assert sql.count("(NOW() AT TIME ZONE 'UTC')") == 100 + 1 @@ -111,6 +111,7 @@ def test_conflict_target_is_the_full_unique_constraint(): "prompt_tokens", "completion_tokens", "spend", + "guardrail_spend", "api_requests", "successful_requests", "failed_requests", diff --git a/tests/unit/proxy/management_endpoints/test_common_daily_activity.py b/tests/unit/proxy/management_endpoints/test_common_daily_activity.py index bba294b571e..d38f71ee8bc 100644 --- a/tests/unit/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/unit/proxy/management_endpoints/test_common_daily_activity.py @@ -710,6 +710,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "prompt_caching_savings_spend": 0.0, "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "total_response_time_ms": 0, "timed_requests": 0, "failed_requests": 0, @@ -1294,6 +1295,7 @@ def test_update_breakdown_metrics_includes_user_email(): prompt_caching_savings_spend=0, gateway_injected_caching_savings_spend=0, autorouter_savings_spend=0, + guardrail_spend=0, total_response_time_ms=0, timed_requests=0, total_tokens=2, @@ -1375,6 +1377,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.prompt_caching_savings_spend = 0.0 mock_record_1.gateway_injected_caching_savings_spend = 0.0 mock_record_1.autorouter_savings_spend = 0.0 + mock_record_1.guardrail_spend = 0.0 mock_record_1.total_response_time_ms = 18_000 mock_record_1.timed_requests = 9 mock_record_1.api_requests = 10 @@ -1401,6 +1404,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.prompt_caching_savings_spend = 0.0 mock_record_2.gateway_injected_caching_savings_spend = 0.0 mock_record_2.autorouter_savings_spend = 0.0 + mock_record_2.guardrail_spend = 0.0 mock_record_2.total_response_time_ms = 2_500 mock_record_2.timed_requests = 5 mock_record_2.api_requests = 5 @@ -1473,6 +1477,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "prompt_caching_savings_spend": 0.0, "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "total_response_time_ms": 0, "timed_requests": 0, "failed_requests": 0, @@ -1569,6 +1574,7 @@ async def test_aggregated_activity_flags_only_keys_that_key_info_can_still_resol "prompt_caching_savings_spend": 0.0, "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "total_response_time_ms": 0, "timed_requests": 0, "api_requests": 1, @@ -1618,7 +1624,7 @@ async def test_aggregated_activity_flags_only_keys_that_key_info_can_still_resol assert key_breakdown["deleted-key"].metadata.key_alias == "deleted" -def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_group="gpt-4"): +def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_group="gpt-4", guardrail_spend=0.0): """A LiteLLM_DailyUserSpend row as the per-user breakdown reads it.""" return SimpleNamespace( date="2024-01-01", @@ -1639,6 +1645,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr prompt_caching_savings_spend=0.0, gateway_injected_caching_savings_spend=0.0, autorouter_savings_spend=0.0, + guardrail_spend=guardrail_spend, total_response_time_ms=0, timed_requests=0, api_requests=1, @@ -1647,6 +1654,38 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr ) +@pytest.mark.asyncio +async def test_paginated_activity_separates_guardrail_spend_per_model_and_key(): + records = [ + _daily_user_spend_record(user_id="u", api_key="key", spend=0.0076916, model="guarded", guardrail_spend=0.00015), + _daily_user_spend_record(user_id="u", api_key="key", spend=0.002, model="unguarded"), + ] + mock_prisma = MagicMock() + mock_prisma.db.litellm_dailyuserspend.count = AsyncMock(return_value=len(records)) + mock_prisma.db.litellm_dailyuserspend.find_many = AsyncMock(return_value=records) + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + result = await get_daily_activity( + prisma_client=mock_prisma, + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id=None, + entity_metadata_field=None, + start_date="2024-01-01", + end_date="2024-01-01", + model=None, + api_key=None, + page=1, + page_size=1000, + ) + + models = result.results[0].breakdown.models + guarded = models["guarded"].api_key_breakdown["key"].metrics + assert guarded.spend - guarded.guardrail_spend == pytest.approx(0.0075416) + assert models["unguarded"].api_key_breakdown["key"].metrics.guardrail_spend == 0.0 + assert result.results[0].metrics.guardrail_spend == pytest.approx(0.00015) + + @pytest.mark.asyncio async def test_get_daily_activity_applies_resolve_entity_metadata_to_breakdown(): """Regression for LIT-3889: the Spend Per User chart showed raw UUIDs. @@ -1788,6 +1827,7 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "prompt_caching_savings_spend": None, "gateway_injected_caching_savings_spend": None, "autorouter_savings_spend": None, + "guardrail_spend": None, "total_response_time_ms": None, "timed_requests": None, "api_requests": None, @@ -1836,6 +1876,7 @@ def _no_spend_record(): prompt_caching_savings_spend=None, gateway_injected_caching_savings_spend=None, autorouter_savings_spend=None, + guardrail_spend=None, total_response_time_ms=None, timed_requests=None, api_requests=None, @@ -1975,6 +2016,7 @@ def _spend_record(api_key, *, model="gpt-4o-mini-ptu", spend=0.0, ptu_flat_cost= prompt_caching_savings_spend=0, gateway_injected_caching_savings_spend=0, autorouter_savings_spend=0, + guardrail_spend=0, total_response_time_ms=0, timed_requests=0, total_tokens=0, @@ -2021,6 +2063,7 @@ def _grouping_row( endpoint=None, spend=0.0, ptu_flat_cost=0.0, + guardrail_spend=None, ): return GroupingSetsRow( date="2024-01-01", @@ -2043,6 +2086,7 @@ def _grouping_row( prompt_caching_savings_spend=0.0, gateway_injected_caching_savings_spend=0.0, autorouter_savings_spend=0.0, + guardrail_spend=guardrail_spend, total_response_time_ms=0, timed_requests=0, api_requests=0, @@ -2088,6 +2132,38 @@ def test_grouping_sets_dispatcher_excludes_ptu_sentinel_from_key_breakdowns(ptu_ assert "real-key" in model_bucket.api_key_breakdown +def test_model_key_breakdown_separates_guardrail_spend_from_model_spend(): + from litellm.proxy.management_endpoints.common_daily_activity import ( + _GROUP_DATE_API_KEY, + _GROUP_DATE_MODEL, + _GROUP_DATE_MODEL_API_KEY, + _GROUP_GRAND_TOTAL, + _aggregate_grouping_sets_records_sync, + ) + + records = [ + _grouping_row(_GROUP_DATE_MODEL, model="guarded", spend=0.0076916, guardrail_spend=0.00015), + _grouping_row(_GROUP_DATE_MODEL, model="unguarded", spend=0.002), + _grouping_row( + _GROUP_DATE_MODEL_API_KEY, model="guarded", api_key="key", spend=0.0076916, guardrail_spend=0.00015 + ), + _grouping_row(_GROUP_DATE_MODEL_API_KEY, model="unguarded", api_key="key", spend=0.002), + _grouping_row(_GROUP_DATE_API_KEY, api_key="key", spend=0.0096916, guardrail_spend=0.00015), + _grouping_row(_GROUP_GRAND_TOTAL, spend=0.0096916, guardrail_spend=0.00015), + ] + + aggregated = _aggregate_grouping_sets_records_sync(records=records, api_key_metadata={}) + day = aggregated["results"][0] + + guarded = day.breakdown.models["guarded"].api_key_breakdown["key"].metrics + unguarded = day.breakdown.models["unguarded"].api_key_breakdown["key"].metrics + assert guarded.spend - guarded.guardrail_spend == pytest.approx(0.0075416) + assert unguarded.guardrail_spend == 0.0 + assert day.breakdown.models["guarded"].metrics.guardrail_spend == pytest.approx(0.00015) + assert day.breakdown.api_keys["key"].metrics.guardrail_spend == pytest.approx(0.00015) + assert aggregated["totals"].guardrail_spend == pytest.approx(0.00015) + + def test_grouping_sets_dispatcher_populates_every_breakdown_level(ptu_cost_attribution_enabled): """Every GROUPING SETS level lands in its bucket, and the flat-cost sentinel is kept out of the model_group and provider api_key sub-breakdowns too.""" @@ -2205,6 +2281,7 @@ def test_update_breakdown_metrics_covers_mcp_endpoint_and_entity(ptu_cost_attrib prompt_caching_savings_spend=0, gateway_injected_caching_savings_spend=0, autorouter_savings_spend=0, + guardrail_spend=0, total_response_time_ms=0, timed_requests=0, total_tokens=0, @@ -2560,6 +2637,7 @@ async def test_get_daily_activity_aggregated_with_entity_breakdown(): "prompt_caching_savings_spend": 0.0, "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "total_response_time_ms": 0, "timed_requests": 0, "failed_requests": 0, diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index 9cd2bab07f0..23ad9efdc70 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -67,6 +67,7 @@ class _Activity: prompt_caching_savings_spend: float gateway_injected_caching_savings_spend: float autorouter_savings_spend: float + guardrail_spend: float api_requests: int successful_requests: int failed_requests: int @@ -109,6 +110,7 @@ def _activity_for_entity( prompt_caching_savings_spend=0.2, gateway_injected_caching_savings_spend=0.3, autorouter_savings_spend=0.4, + guardrail_spend=0.5, api_requests=1, successful_requests=1, failed_requests=0, @@ -165,6 +167,7 @@ def _seeded_activity() -> tuple[_Activity, ...]: prompt_caching_savings_spend=0.0, gateway_injected_caching_savings_spend=0.0, autorouter_savings_spend=0.0, + guardrail_spend=0.0, api_requests=1, successful_requests=1, failed_requests=0, @@ -188,6 +191,7 @@ def _metrics(rows: Sequence[_Activity]) -> Mapping[str, int | float]: "prompt_caching_savings_spend": sum(row.prompt_caching_savings_spend for row in rows), "gateway_injected_caching_savings_spend": sum(row.gateway_injected_caching_savings_spend for row in rows), "autorouter_savings_spend": sum(row.autorouter_savings_spend for row in rows), + "guardrail_spend": sum(row.guardrail_spend for row in rows), "api_requests": sum(row.api_requests for row in rows), "successful_requests": sum(row.successful_requests for row in rows), "failed_requests": sum(row.failed_requests for row in rows), @@ -791,6 +795,7 @@ def test_search_folds_each_entity_key_across_days( "prompt_caching_savings_spend": 0.4, "gateway_injected_caching_savings_spend": 0.6, "autorouter_savings_spend": 0.8, + "guardrail_spend": 1.0, "total_tokens": 30, "successful_requests": 2, "failed_requests": 0, diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 83a121da21a..a79ebd307fe 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -19308,6 +19308,7 @@ def _db_with_one_ptu_hour_of_team_spend(team_id: str) -> MagicMock: prompt_caching_savings_spend=0.0, gateway_injected_caching_savings_spend=0.0, autorouter_savings_spend=0.0, + guardrail_spend=0.0, total_response_time_ms=0, timed_requests=0, api_requests=3, diff --git a/tests/unit/repositories/test_daily_activity_repository.py b/tests/unit/repositories/test_daily_activity_repository.py index b68bb948b7b..bca18c15d61 100644 --- a/tests/unit/repositories/test_daily_activity_repository.py +++ b/tests/unit/repositories/test_daily_activity_repository.py @@ -675,6 +675,7 @@ async def test_customer_team_page_returns_loaded_metrics_and_retains_count_on_em "prompt_caching_savings_spend": 0.25, "gateway_injected_caching_savings_spend": 0.125, "autorouter_savings_spend": 0.0, + "guardrail_spend": 0.0, "total_response_time_ms": 10, "timed_requests": 1, "created_at": "2026-01-01T00:00:00Z", diff --git a/tests/unit/repositories/test_daily_activity_sql.py b/tests/unit/repositories/test_daily_activity_sql.py index f8375b93dad..051db54a161 100644 --- a/tests/unit/repositories/test_daily_activity_sql.py +++ b/tests/unit/repositories/test_daily_activity_sql.py @@ -284,6 +284,7 @@ def test_aggregated_query_uses_the_caller_date_bounds(offset_minutes: int | None def test_aggregate_query_sums_all_savings_drivers_and_response_time() -> None: query = build_aggregated_sql(_scope(), api_key_limit=constants.USAGE_TOP_API_KEYS_DEFAULT) fields: Final = tuple(field for field in SpendMetrics.model_fields if field.endswith("_savings_spend")) + ( + "guardrail_spend", "total_response_time_ms", "timed_requests", ) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx index 95abc4e22cb..e87fbbf79d1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx @@ -36,6 +36,7 @@ const baseMetrics = (overrides: Partial): components["schemas"]["S prompt_caching_savings_spend: 0, gateway_injected_caching_savings_spend: 0, autorouter_savings_spend: 0, + guardrail_spend: 0, total_response_time_ms: 0, timed_requests: 0, ...overrides, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.test.ts index 1a92f1d1bc8..3f17685f295 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.test.ts @@ -88,6 +88,26 @@ describe("client usage from request logs", () => { expect(day?.date).toBe(localDay); }); + it("should report guardrail spend inside model spend, capped at what each request was charged", () => { + const guarded = (request_id: string, spend: number, guardrail_information: unknown) => + log({ request_id, spend, metadata: { user_agent: desktopAgent, guardrail_information } }); + const result = clientUsageFromLogs( + page([ + guarded("billed", 0.0076916, [ + { guardrail_cost: 0.00015 }, + { guardrail_cost: 0.0004, guardrail_cost_in_spend: false }, + ]), + guarded("capped", 0.0001, { guardrail_cost: 0.0003 }), + guarded("malformed", 0.002, [{ guardrail_cost: "free" }]), + log(), + ]), + scope, + ); + expect(result.error).toBeNull(); + const metrics = result.data.results?.[0]?.breakdown?.models?.["provider-model"].metrics; + expect(metrics?.guardrail_spend).toBeCloseTo(0.00025, 10); + }); + it("should deduplicate request ids without double counting bare and full user-agent tags", () => { const result = clientUsageFromLogs(page([log(), log({ spend: 0.02 })]), scope); expect(result.data.metadata?.total_api_requests).toBe(1); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.ts index 998dfd2b271..996efd06061 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.ts @@ -31,11 +31,19 @@ const usageFields = { input_tokens_details: cacheDetailsSchema.nullish(), }; const usageSchema = z.object(usageFields); +const guardrailEntrySchema = z.object({ + guardrail_cost: z.number().finite().nullish().catch(null), + guardrail_cost_in_spend: z.boolean().nullish().catch(null), +}); const metadataFields = { user_agent: z.string().nullish(), status: z.string().nullish(), additional_usage_values: usageSchema.nullish(), usage_object: usageSchema.nullish(), + guardrail_information: z + .union([z.array(guardrailEntrySchema), guardrailEntrySchema]) + .nullish() + .catch(null), }; const logFields = { request_id: z.string().min(1), @@ -127,6 +135,19 @@ const cacheTokens = (row: ClientLog) => { cacheCreationTokens: additional.cacheCreationTokens ?? usage.cacheCreationTokens, }; }; +const billedGuardrailCost = (entry: z.infer) => + entry.guardrail_cost_in_spend === false ? 0 : Math.max(entry.guardrail_cost ?? 0, 0); +type GuardrailInformation = NonNullable["guardrail_information"]>; +const guardrailEntries = (information: GuardrailInformation) => + Array.isArray(information) ? information : [information]; +const guardrailSpend = (row: ClientLog) => { + const information = row.metadata?.guardrail_information; + const entries = information == null ? [] : guardrailEntries(information); + return Math.min( + entries.reduce((total, entry) => total + billedGuardrailCost(entry), 0), + Math.max(row.spend, 0), + ); +}; const sum = (rows: readonly ClientLog[], value: (row: ClientLog) => number) => rows.reduce((total, row) => total + value(row), 0); const metricsFor = (rows: readonly ClientLog[]): SpendMetrics => ({ @@ -137,6 +158,7 @@ const metricsFor = (rows: readonly ClientLog[]): SpendMetrics => ({ prompt_caching_savings_spend: 0, gateway_injected_caching_savings_spend: 0, autorouter_savings_spend: 0, + guardrail_spend: sum(rows, guardrailSpend), spend: sum(rows, (row) => row.spend), prompt_tokens: sum(rows, (row) => row.prompt_tokens), completion_tokens: sum(rows, (row) => row.completion_tokens), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ddf6eb21c63..bd5e87bcbdd 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -47697,6 +47697,12 @@ export interface components { * @default 0 */ gateway_injected_caching_savings_spend: number; + /** + * Guardrail Spend + * @description Guardrail spend in USD, already included in spend, so model spend is spend - guardrail_spend. Rows recorded before this field existed report 0 even when their spend includes guardrail cost + * @default 0 + */ + guardrail_spend: number; /** * Prompt Caching Savings Spend * @default 0 From 9fb29c7e385c99dbe52150dc61b41c17a6eec839 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:04:06 +0000 Subject: [PATCH 038/189] fix(tool_choice): accept the Responses API's flat allowed_tools layout on chat completions (#50467) * fix(tool_choice): accept the Responses API's flat allowed_tools layout on chat completions An allowed_tools tool_choice in the flat Responses layout ({"type": "allowed_tools", "mode": ..., "tools": [...]}) answered 400 on litellm.completion, acompletion, Router and the proxy's chat and /v1/responses-to-chat routes. One parser now reads both layouts into the nested Chat layout, and the existing Responses bridge flattens it back for /v1/responses, so either layout reaches OpenAI in the shape each endpoint expects * fix(tool_choice): drop an allowed_tools choice under drop_params and refuse conflicting flat entries An allowed_tools tool_choice a provider can't enforce, or a malformed one, now drops under drop_params and answers 400 without it, the same rule LiteLLM uses for other unsupported values. That covers providers outside OpenAI and Azure, malformed choices on every provider, and Azure api versions older than 2023-12-01 A flat allowed_tools entry that carries both name and a nested function or custom key answers 400, since the two names can disagree, matching what flattening already refuses * fix(tool-choice): keep the 400 when LiteLLM runs the tools, report auto after a drop Under drop_params an allowed_tools choice a provider can't enforce was dropped, so on MCP gateway tools and in run_tool_loop and arun_tool_loop LiteLLM could run a tool the allow list excluded. Those paths now keep the 400 they answered before the flat layout was accepted, and a malformed choice stays a 400 on MCP gateway tools. /v1/responses served through chat echoed a choice it had dropped, on sync, async and streamed responses. It now reports auto, and a choice the provider received is still echoed as sent. * fix(tool-choice): refuse an excluded tool right before LiteLLM runs it Checking the provider up front missed a model alias, an old Azure api_version, a litellm_proxy hop, and SDK fallbacks. The tool loop and the MCP gateway now check the allowed_tools list right before they run a tool call, so an excluded call answers 400 and never runs. Under drop_params the choice now drops only when the caller runs every tool, so a provider-run MCP server or built-in keeps the 400. The Azure checks read the request's api_version, which also fixes the /v1/responses echo there. * fix(tool-choice): drop the choice under drop_params and hand an excluded MCP call back unrun The drop now applies to every provider that cannot take an allowed_tools tool_choice, whatever the request's tools hold. LiteLLM never runs a call the choice excludes: on MCP paths the whole round goes back to the caller in a 200, the same shape as require_approval, and the tool loop keeps its 400 because it only returns final content. A malformed choice stays a 400 before any request on the tool loop and on chat with MCP gateway tools. * fix(tool-choice): refuse an allowed_tools entry whose name is not a string An mcp or other non-function entry with a list or dict name passed the parser and crashed when the allow list was hashed. The parser now treats it as malformed on both layouts, so it answers the same 400 as an empty allow list, or drops under drop_params. * fix(tool-choice): honor server-level and null-name mcp entries and stop hooks running excluded calls * fix(tool-choice): match named mcp entries by server label and let built-in entries allow their interception call * fix(tool-choice): leave a malformed allowed_tools choice unrewritten in the interception hooks * fix(tool-choice): resolve mcp entries like the gateway and keep the choice on every hook path Under drop_params a malformed allowed_tools choice now acts as if it was never sent, and only a choice that parses is stored and enforced. Without drop_params it answers 400 before any provider request, on /v1/responses with gateway tools before MCP listing An mcp entry resolves its label the way the gateway picks servers to list (server name in any case, alias, access group, toolset), and a named entry matches the tool's own name or the gateway's prefixed name. The gateway rewrites mcp entries into function entries in the caller's layout and sends "none" when nothing is left The web search follow-up keeps an allowed_tools choice, compression's litellm_content_retrieve is never excluded and joins the provider-facing choice, and the native /v1/responses hook loop applies the exclusion check and stores the choice * fix(allowed_tools): stop web search and code interpreter follow-ups forcing another call A kept allowed_tools choice now goes to the follow-up in mode auto, so a required choice no longer loops to the agentic loop limit. The MCP label scope dataclass uses a default_factory so the module imports on Python 3.11 * fix(allowed_tools): stop the Responses web search follow-up forcing another call The Responses-shaped follow-up after a web search left tool_choice out of its patch, so the caller's required choice went out again through the chat to Responses bridge and looped to the loop limit. It now carries a kept allowed_tools choice in mode auto, like the chat follow-up * fix(mcp): scope each MCP entry to its own allowed_tools and keep the allowlist on follow-ups A label now covers only the tools its own allowed_tools lists, so a second label on the same server no longer widens it. Responses MCP follow-ups, streamed or not, keep an allowed_tools choice in auto mode instead of dropping it. The new Responses tests fake only the upstream MCP listing, so LiteLLM's own discovery runs * fix(mcp): match an entry's allowed_tools against the tool name the gateway lists A server with a display name override lists the display name, and the listing filter matches allowed_tools against it, but the label scope only checked the original name, so no spelling worked. The scope now also accepts the listed name and its unprefixed part, like the listing filter. Tests now cover the whole gateway and toolset entries and a display-named tool. * fix(mcp): match an entry's allowed_tools only against a server's own prefix, not the first hyphen * fix(mcp): match a tool no server resolves only by its exact listed name in an entry's allowed_tools * fix(mcp): refuse a tool whose display name only looks like an allowed name, and let the chat follow-up answer An MCP entry's allowed_tools now matches a tool only by its exact listed name or a spelling routing accepts for its real name, so a display name that carries the server's own prefix no longer passes for another tool. The chat MCP follow-up keeps an allowed_tools choice in auto mode, so the model can answer with text after the tool ran. * fix(mcp): match a toolset label's tools by exact native name A toolset label served any tool whose name a routing spelling of a granted name matched, so a toolset granting the literal name "server-tool" also served the real tool "tool" that only another toolset grants. It now serves a tool only when the toolset grants its exact native name, the same check tool_is_granted uses when the gateway lists tools --- .../transformation.py | 4 +- .../code_interpreter_interception/handler.py | 24 +- .../compression_interception/handler.py | 5 + .../websearch_interception/handler.py | 84 ++- .../agentic_loop_refusal.py | 10 + .../allowed_tools_tool_choice.py | 328 +++++++++++- .../chat_completion_agentic_loop.py | 137 +++-- .../litellm_core_utils/get_litellm_params.py | 3 +- litellm/llms/azure/chat/gpt_transformation.py | 21 +- litellm/llms/custom_httpx/llm_http_handler.py | 16 +- litellm/main.py | 33 +- .../handler.py | 30 +- litellm/responses/main.py | 25 +- .../responses/mcp/chat_completions_handler.py | 46 +- .../mcp/litellm_proxy_mcp_handler.py | 189 ++++++- .../responses/mcp/mcp_streaming_iterator.py | 27 +- litellm/tool_loop.py | 24 +- litellm/types/integrations/custom_logger.py | 5 + litellm/types/litellm_params.py | 1 + litellm/utils.py | 35 +- ...responses_transformation_transformation.py | 4 + .../test_handler.py | 73 +++ .../test_compression_interception_handler.py | 47 ++ .../test_websearch_chat_completion.py | 92 ++++ .../test_websearch_responses.py | 65 +++ .../test_allowed_tools_tool_choice.py | 500 +++++++++++++++++- .../test_chat_completion_agentic_loop.py | 255 ++++++++- .../test_azure_chat_gpt_transformation.py | 19 +- .../custom_httpx/test_llm_http_handler.py | 89 ++++ .../test_anthropic_responses_bridge.py | 167 ++++++ .../test_handler.py | 34 ++ .../mcp/test_aresponses_api_with_mcp.py | 331 +++++++++++- .../mcp/test_chat_completions_handler.py | 375 ++++++++++++- .../mcp/test_litellm_proxy_mcp_handler.py | 279 +++++++++- .../test_responses_api_bridge_flag.py | 38 ++ tests/unit/test_main.py | 103 +++- tests/unit/test_tool_loop.py | 215 +++++++- tests/unit/test_utils.py | 27 +- tests/unit/types/test_litellm_params.py | 1 + 39 files changed, 3537 insertions(+), 224 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index a2de41a0c45..dd1ef626a69 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -36,7 +36,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import ( ) from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.allowed_tools_tool_choice import ( - parse_chat_allowed_tools_tool_choice, + parse_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, ) from litellm.litellm_core_utils.core_helpers import ( @@ -900,7 +900,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"): return choice_type if choice_type == "allowed_tools": - parsed: Final = parse_chat_allowed_tools_tool_choice( + parsed: Final = parse_allowed_tools_tool_choice( cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic object, tool_choice ) diff --git a/litellm/integrations/code_interpreter_interception/handler.py b/litellm/integrations/code_interpreter_interception/handler.py index b035732afa6..ce5cbc15902 100644 --- a/litellm/integrations/code_interpreter_interception/handler.py +++ b/litellm/integrations/code_interpreter_interception/handler.py @@ -18,6 +18,10 @@ from pydantic import ValidationError import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + allowed_tools_tool_choice_in_auto_mode, + allowed_tools_tool_choice_with_builtin_as_function, +) from litellm.llms.base_llm.sandbox.transformation import ( CodeExecutionResult, ContainerHandle, @@ -175,6 +179,10 @@ def _sandbox_unavailable(step: str, error: Exception, output: str = _SANDBOX_UNA return _SandboxUnavailable(output=output) +def _is_code_interpreter_tool(tool: Mapping[str, object]) -> bool: + return tool.get("type") == "code_interpreter" + + def _output_item_type(item: object) -> object: if isinstance(item, dict): item_mapping: Final[dict[str, object]] = item @@ -343,8 +351,8 @@ class CodeInterpreterInterceptionLogger(CustomLogger): (function_tool if isinstance(tool, dict) and tool.get("type") == "code_interpreter" else tool) for tool in tools ] - if self._tool_choice_targets_code_interpreter(kwargs.get("tool_choice")): - kwargs["tool_choice"] = self._get_function_tool_choice(call_type=call_type) + if "tool_choice" in kwargs: + kwargs["tool_choice"] = self._intercepted_tool_choice(kwargs["tool_choice"], call_type) return kwargs @staticmethod @@ -428,6 +436,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger): or (isinstance(function, dict) and function.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME) ) + @classmethod + def _intercepted_tool_choice(cls, tool_choice: object, call_type: CallTypes | None) -> object: + if cls._tool_choice_targets_code_interpreter(tool_choice): + return cls._get_function_tool_choice(call_type=call_type) + return allowed_tools_tool_choice_with_builtin_as_function( + tool_choice, _is_code_interpreter_tool, LITELLM_CODE_EXECUTION_TOOL_NAME + ) + def _resolve_provider(self, kwargs: dict[str, object]) -> str | None: provider: Final = kwargs.get("custom_llm_provider") if isinstance(provider, str) and provider: @@ -694,7 +710,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger): def _get_followup_optional_params(self, optional_params: dict[str, object]) -> dict[str, object]: drop_tool_choice: Final = self._tool_choice_targets_code_interpreter(optional_params.get("tool_choice")) return { - k: v for k, v in optional_params.items() if k != "tools" and not (k == "tool_choice" and drop_tool_choice) + k: allowed_tools_tool_choice_in_auto_mode(v) if k == "tool_choice" else v + for k, v in optional_params.items() + if k != "tools" and not (k == "tool_choice" and drop_tool_choice) } async def async_post_agentic_loop_response_hook( diff --git a/litellm/integrations/compression_interception/handler.py b/litellm/integrations/compression_interception/handler.py index 5352ce6b6a0..6c04d16f817 100644 --- a/litellm/integrations/compression_interception/handler.py +++ b/litellm/integrations/compression_interception/handler.py @@ -15,6 +15,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger from litellm.compression import compress from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.allowed_tools_tool_choice import allowed_tools_tool_choice_with_function from litellm.litellm_core_utils.asyncify import asyncify from litellm.types.integrations.compression_interception import ( CompressionInterceptionConfig, @@ -180,6 +181,10 @@ class CompressionInterceptionLogger(CustomLogger): existing_tools=cast(list[dict[str, object]] | None, kwargs.get("tools")), compressed_tools=compressed_tools, ) + if "tool_choice" in kwargs: + kwargs["tool_choice"] = allowed_tools_tool_choice_with_function( + kwargs["tool_choice"], LITELLM_CONTENT_RETRIEVE_TOOL_NAME + ) call_id = cast(str | None, kwargs.get("litellm_call_id")) if not call_id: call_id = str(uuid.uuid4()) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index b61bb6cdc90..b386c020a1b 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -9,11 +9,12 @@ server-side using litellm router's search tools. import asyncio import math import uuid -from collections.abc import AsyncIterator, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast +from pydantic import TypeAdapter from typing_extensions import Never, ReadOnly import litellm @@ -38,6 +39,10 @@ from litellm.integrations.websearch_interception.transformation import ( from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + allowed_tools_tool_choice_kept_on_follow_up, + allowed_tools_tool_choice_with_builtin_as_function, +) from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.types.integrations.custom_logger import ( @@ -100,6 +105,8 @@ WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks" _RESPONSE_CONTENT_FIELD: Final = "content" +_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object]) + _CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX: Final = "Perform a web search for the query:" _ResponseT: Final = TypeVar("_ResponseT") @@ -263,6 +270,22 @@ class _ParentRequestCorrelation: _NO_ACOMPLETION_NAMED: Final[_AcompletionNamedParams] = {} +def _tool_choice_naming_litellm_web_search( + kwargs: Mapping[str, object], is_search_tool: Callable[[Mapping[str, object]], bool] +) -> Mapping[str, object]: + if "tool_choice" not in kwargs: + return MappingProxyType({}) + return { + "tool_choice": allowed_tools_tool_choice_with_builtin_as_function( + kwargs["tool_choice"], is_search_tool, LITELLM_WEB_SEARCH_TOOL_NAME + ) + } + + +def _kept_allowed_tools_tool_choice(optional_params: Mapping[str, object]) -> Mapping[str, object]: + return allowed_tools_tool_choice_kept_on_follow_up(optional_params.get("tool_choice")) + + class WebSearchInterceptionLogger(CustomLogger): """ CustomLogger that intercepts WebSearch tool calls for models that don't @@ -514,13 +537,17 @@ class WebSearchInterceptionLogger(CustomLogger): # Keep other tools as-is converted_tools.append(tool) - kwargs["tools"] = converted_tools + converted_kwargs: Final[dict[str, object]] = { + **kwargs, + "tools": converted_tools, + **_tool_choice_naming_litellm_web_search(kwargs, lambda tool: is_search_tool(dict(tool))), + } if kwargs.get("stream"): verbose_logger.debug("WebSearchInterception: deployment hook converting stream=True to stream=False") - return as_converted_stream(kwargs, WEBSEARCH_INTERCEPTION_PREFIX) + return as_converted_stream(converted_kwargs, WEBSEARCH_INTERCEPTION_PREFIX) - return kwargs + return converted_kwargs def _convert_responses_tools( self, kwargs: Mapping[str, object], tools: Sequence[dict[str, object]] @@ -535,7 +562,11 @@ class WebSearchInterceptionLogger(CustomLogger): get_litellm_web_search_tool_responses() if is_web_search_tool_responses(tool) else tool for tool in tools ] - converted_kwargs: Final = {**kwargs, "tools": converted_tools} + converted_kwargs: Final = { + **kwargs, + "tools": converted_tools, + **_tool_choice_naming_litellm_web_search(kwargs, is_web_search_tool_responses), + } if kwargs.get("stream"): verbose_logger.debug("WebSearchInterception: deployment hook converting stream=True to stream=False") @@ -1256,9 +1287,13 @@ class WebSearchInterceptionLogger(CustomLogger): tools_param: Final = optional_params.get("tools") optional_params_clean: Final = { - k: v - for k, v in optional_params.items() - if k not in {"tools", "tool_choice", "stream", "model_alias_map", "stream_response", "custom_prompt_dict"} + **{ + k: v + for k, v in optional_params.items() + if k + not in {"tools", "tool_choice", "stream", "model_alias_map", "stream_response", "custom_prompt_dict"} + }, + **_kept_allowed_tools_tool_choice(_OPTIONAL_PARAMS.validate_python(optional_params)), } kwargs_for_followup: Final = { @@ -1843,9 +1878,11 @@ class WebSearchInterceptionLogger(CustomLogger): ) if request_patch.messages is None: raise ValueError("WebSearchInterception: missing follow-up messages") - params: Final = dict(optional_params) - params.update(request_patch.optional_params) - params.pop("tool_choice", None) + merged_params: Final = {**optional_params, **request_patch.optional_params} + params: Final = { + **{key: value for key, value in merged_params.items() if key != "tool_choice"}, + **_kept_allowed_tools_tool_choice(merged_params), + } patch_kwargs: Final = dict[str, object](request_patch.kwargs) return await litellm.acompletion( model=request_patch.model or model, @@ -1958,17 +1995,20 @@ class WebSearchInterceptionLogger(CustomLogger): tools_param: Final = optional_params.get("tools") optional_params_clean: Final = { - k: v - for k, v in optional_params.items() - if k - not in { - "tools", - "tool_choice", - "extra_body", - "model_alias_map", - "stream_response", - "custom_prompt_dict", - } + **{ + k: v + for k, v in optional_params.items() + if k + not in { + "tools", + "tool_choice", + "extra_body", + "model_alias_map", + "stream_response", + "custom_prompt_dict", + } + }, + **_kept_allowed_tools_tool_choice(_OPTIONAL_PARAMS.validate_python(optional_params)), } if tools_param is not None: optional_params_clean["tools"] = tools_param diff --git a/litellm/litellm_core_utils/agentic_loop_refusal.py b/litellm/litellm_core_utils/agentic_loop_refusal.py index f3ee15dffdd..828d5fa03ad 100644 --- a/litellm/litellm_core_utils/agentic_loop_refusal.py +++ b/litellm/litellm_core_utils/agentic_loop_refusal.py @@ -40,6 +40,16 @@ async def every_claimed_tool_call( return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))}) +_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call", "tool_use"}) + + +def called_tool_names(response: object) -> frozenset[str]: + items: Final = (*parts_of(message_field(response, "output")), *parts_of(message_field(response, "content"))) + return _strings( + message_field(item, "name") for item in items if message_field(item, "type") in _TOOL_CALL_ITEM_TYPES + ) + + def _call_ids(call: object) -> frozenset[str]: return _strings((message_field(call, "id"), message_field(call, "call_id"))) diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py index f4c4ecbbb26..4e6824e51e9 100644 --- a/litellm/litellm_core_utils/allowed_tools_tool_choice.py +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -1,12 +1,17 @@ -from collections.abc import Mapping +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from types import MappingProxyType from typing import Final, Literal +from openai.types.chat.chat_completion_allowed_tool_choice_param import ChatCompletionAllowedToolChoiceParam +from openai.types.chat.chat_completion_allowed_tools_param import ChatCompletionAllowedToolsParam from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from litellm.exceptions import BadRequestError ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"}) +_FIRST_AZURE_API_VERSION_WITH_TOOL_CHOICE: Final = "2023-12-01" HOSTED_TOOL_CHOICE_TYPES: Final = frozenset( { @@ -29,6 +34,9 @@ HOSTED_TOOL_CHOICE_TYPES: Final = frozenset( CHAT_ALLOWED_TOOLS_LAYOUT: Final = ( '{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": []}}' ) +RESPONSES_ALLOWED_TOOLS_LAYOUT: Final = ( + '{"type": "allowed_tools", "mode": "auto" | "required", "tools": []}' +) class ChatAllowedTools(BaseModel): @@ -45,7 +53,17 @@ class ChatAllowedToolsToolChoice(BaseModel): allowed_tools: ChatAllowedTools -_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice) +class ResponsesAllowedToolsToolChoice(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + type: Literal["allowed_tools"] + mode: Literal["auto", "required"] + tools: tuple[Mapping[str, object], ...] = Field(min_length=1) + + +_ALLOWED_TOOLS_TOOL_CHOICE: Final[TypeAdapter[ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice]] = ( + TypeAdapter(ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice) +) _SELECTOR: Final = TypeAdapter(dict[str, object]) @@ -67,21 +85,22 @@ def is_allowed_tools_tool_choice(tool_choice: object) -> bool: return False -def raise_if_allowed_tools_tool_choice_unsupported( +def drops_unsupported_allowed_tools_tool_choice( tool_choice: object, custom_llm_provider: str, model: str, -) -> None: - """Providers outside the set would silently send the request with no tool restriction at all, so the call stops - here with a 400. ``drop_params`` is deliberately not consulted: dropping a restriction widens what the model may call.""" + drop_params: bool, +) -> bool: if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: - return + return False + if drop_params: + return True raise BadRequestError( message=( "tool_choice type 'allowed_tools' is only supported for " f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. " - "drop_params does not drop it, since that would let the model call tools the request excluded. " - "Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead." + "Send only the allowed tools in `tools` with tool_choice='auto' or 'required', or set drop_params=True " + "to drop the tool_choice, which lets the model call any tool in `tools`." ), model=model, llm_provider=custom_llm_provider, @@ -119,18 +138,295 @@ def raise_if_hosted_tool_choice_unsupported( ) -def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None: - try: - return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) - except ValidationError: - return None - - def _has_name(selector: Mapping[str, object]) -> bool: name: Final = selector.get("name") return isinstance(name, str) and name != "" +def _names_a_tool_or_nothing(tool: Mapping[str, object]) -> bool: + tool_type: Final = tool.get("type") + if not isinstance(tool_type, str) or tool_type not in ("function", "custom"): + return tool.get("name") is None or _has_name(tool) + match tool.get(tool_type): + case {"name": str()}: + return True + case {"name": object()}: + return False + case _: + return True + + +def _names_are_strings(tools: Iterable[Mapping[str, object]]) -> bool: + return all(map(_names_a_tool_or_nothing, tools)) + + +def _chat_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None: + tool_type: Final = tool.get("type") + if not isinstance(tool_type, str) or tool_type not in ("function", "custom"): + return tool + if tool_type in tool: + return None if "name" in tool else tool + if not _has_name(tool): + return None + return {**{key: value for key, value in tool.items() if key != "name"}, tool_type: {"name": tool["name"]}} + + +def _chat_layout(choice: ResponsesAllowedToolsToolChoice) -> ChatAllowedToolsToolChoice | None: + nested: Final = tuple(_chat_allowed_tool(tool) for tool in choice.tools) + if any(tool is None for tool in nested): + return None + return ChatAllowedToolsToolChoice( + type="allowed_tools", + allowed_tools=ChatAllowedTools(mode=choice.mode, tools=tuple(tool for tool in nested if tool is not None)), + ) + + +def parse_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None: + try: + parsed: Final = _ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) + except ValidationError: + return None + match parsed: + case ChatAllowedToolsToolChoice(): + return parsed if _names_are_strings(parsed.allowed_tools.tools) else None + case ResponsesAllowedToolsToolChoice(): + return _chat_layout(parsed) if _names_are_strings(parsed.tools) else None + + +def azure_api_version_takes_tool_choice(api_version: str) -> bool: + return api_version.count("-") < 2 or api_version >= _FIRST_AZURE_API_VERSION_WITH_TOOL_CHOICE + + +def _azure_request_takes_tool_choice(api_version: str | None) -> bool: + import litellm + from litellm.secret_managers.main import get_secret_str + + return azure_api_version_takes_tool_choice( + api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION + ) + + +def _enforces_allowed_tools(custom_llm_provider: str | None, api_version: str | None) -> bool: + if custom_llm_provider == "azure": + return _azure_request_takes_tool_choice(api_version) + return custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS + + +def unenforced_allowed_tools_tool_choice( + tool_choice: object, custom_llm_provider: str | None, api_version: str | None = None +) -> bool: + return is_allowed_tools_tool_choice(tool_choice) and ( + not _enforces_allowed_tools(custom_llm_provider, api_version) + or parse_allowed_tools_tool_choice(tool_choice) is None + ) + + +def _allowed_tool_names(tool_choice: object) -> frozenset[object]: + parsed: Final = parse_allowed_tools_tool_choice(tool_choice) + if parsed is None: + return frozenset() + flattened: Final = tuple(_responses_allowed_tool(tool) for tool in parsed.allowed_tools.tools) + return frozenset(tool.get("name") for tool in flattened if tool is not None and _server_entry(tool) is None) + + +def parse_allowed_tools_tool_choice_or_refuse( + tool_choice: object, model: str, drop_params: bool +) -> ChatAllowedToolsToolChoice | None: + parsed: Final = parse_allowed_tools_tool_choice(tool_choice) + if parsed is not None or drop_params: + return parsed + raise BadRequestError( + message=( + f"Invalid tool choice, tool_choice={tool_choice}. An allowed_tools tool_choice must be " + f"{CHAT_ALLOWED_TOOLS_LAYOUT} or {RESPONSES_ALLOWED_TOOLS_LAYOUT}" + ), + model=model, + llm_provider="", + ) + + +def allowed_tools_tool_choice_in_effect(tool_choice: object, model: str, drop_params: bool) -> object: + if not is_allowed_tools_tool_choice(tool_choice): + return tool_choice + return None if parse_allowed_tools_tool_choice_or_refuse(tool_choice, model, drop_params) is None else tool_choice + + +def _server_entry(tool: Mapping[str, object]) -> tuple[str, str | None] | None: + label: Final = tool.get("server_label") + name: Final = tool.get("name") + if tool.get("type") != "mcp" or not isinstance(label, str): + return None + return label, name if isinstance(name, str) else None + + +def _server_entries(tool_choice: object) -> frozenset[tuple[str, str | None]]: + parsed: Final = parse_allowed_tools_tool_choice(tool_choice) + entries: Final = () if parsed is None else tuple(map(_server_entry, parsed.allowed_tools.tools)) + return frozenset(entry for entry in entries if entry is not None) + + +@dataclass(frozen=True, slots=True) +class GatewayTool: + names: frozenset[str] + server_labels: frozenset[str] + + +@dataclass(frozen=True, slots=True) +class GatewayTools: + server_labels: frozenset[str] + tools: Mapping[str, GatewayTool] + + +NO_GATEWAY_TOOLS: Final = GatewayTools(server_labels=frozenset(), tools=MappingProxyType({})) + + +def _server_entry_allows(entry: tuple[str, str | None], tool: GatewayTool | None) -> bool: + label, name = entry + return tool is not None and label in tool.server_labels and (name is None or name in tool.names) + + +def _a_server_entry_allows(entries: Iterable[tuple[str, str | None]], tool: GatewayTool | None) -> bool: + return any(_server_entry_allows(entry, tool) for entry in entries) + + +def tool_calls_the_choice_excludes( + tool_choice: object, tool_names: Iterable[str], gateway: GatewayTools = NO_GATEWAY_TOOLS +) -> tuple[str, ...]: + if not is_allowed_tools_tool_choice(tool_choice): + return () + allowed: Final = _allowed_tool_names(tool_choice) + server_entries: Final = _server_entries(tool_choice) + return tuple( + name + for name in tool_names + if name not in allowed and not _a_server_entry_allows(server_entries, gateway.tools.get(name)) + ) + + +def refuse_tool_calls_the_choice_excludes(tool_choice: object, tool_names: Iterable[str], model: str) -> None: + excluded: Final = tool_calls_the_choice_excludes(tool_choice, tool_names) + if excluded: + raise BadRequestError( + message=( + f"The model called {', '.join(excluded)}, which the allowed_tools tool_choice excludes, because this " + "request did not reach a model that enforces the choice. LiteLLM did not run the call. Send only the " + "allowed tools in `tools` with tool_choice='auto' or 'required'." + ), + model=model, + llm_provider="", + ) + + +def _with_builtin_as( + tool: Mapping[str, object], is_builtin: Callable[[Mapping[str, object]], bool], function_entry: Mapping[str, object] +) -> Mapping[str, object]: + return function_entry if tool.get("type") not in ("function", "custom", "mcp") and is_builtin(tool) else tool + + +def _function_entry( + layout: ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice, name: str +) -> Mapping[str, object]: + match layout: + case ChatAllowedToolsToolChoice(): + return {"type": "function", "function": {"name": name}} + case ResponsesAllowedToolsToolChoice(): + return {"type": "function", "name": name} + + +def _entries_of( + layout: ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice, +) -> tuple[Mapping[str, object], ...]: + match layout: + case ChatAllowedToolsToolChoice(): + return layout.allowed_tools.tools + case ResponsesAllowedToolsToolChoice(): + return layout.tools + + +def _with_entries( + layout: ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice, tools: Iterable[Mapping[str, object]] +) -> Mapping[str, object]: + entries: Final = [dict(tool) for tool in tools] + match layout: + case ChatAllowedToolsToolChoice(): + return {"type": "allowed_tools", "allowed_tools": {"mode": layout.allowed_tools.mode, "tools": entries}} + case ResponsesAllowedToolsToolChoice(): + return {"type": "allowed_tools", "mode": layout.mode, "tools": entries} + + +def _callers_layout(tool_choice: object) -> ChatAllowedToolsToolChoice | ResponsesAllowedToolsToolChoice | None: + if parse_allowed_tools_tool_choice(tool_choice) is None: + return None + return _ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) + + +def allowed_tools_tool_choice_with_builtin_as_function( + tool_choice: object, is_builtin: Callable[[Mapping[str, object]], bool], function_name: str +) -> object: + layout: Final = _callers_layout(tool_choice) + if layout is None: + return tool_choice + entry: Final = _function_entry(layout, function_name) + return _with_entries(layout, (_with_builtin_as(tool, is_builtin, entry) for tool in _entries_of(layout))) + + +def allowed_tools_tool_choice_with_function(tool_choice: object, function_name: str) -> object: + layout: Final = _callers_layout(tool_choice) + if layout is None or function_name in _allowed_tool_names(tool_choice): + return tool_choice + return _with_entries(layout, (*_entries_of(layout), _function_entry(layout, function_name))) + + +def allowed_tools_tool_choice_in_auto_mode(tool_choice: object) -> object: + layout: Final = _callers_layout(tool_choice) + match layout: + case None: + return tool_choice + case ChatAllowedToolsToolChoice(): + return layout.model_copy( + update={"allowed_tools": layout.allowed_tools.model_copy(update={"mode": "auto"})} + ).model_dump(mode="json") + case ResponsesAllowedToolsToolChoice(): + return layout.model_copy(update={"mode": "auto"}).model_dump(mode="json") + + +def allowed_tools_tool_choice_kept_on_follow_up(tool_choice: object) -> Mapping[str, object]: + if not is_allowed_tools_tool_choice(tool_choice): + return MappingProxyType({}) + return MappingProxyType({"tool_choice": allowed_tools_tool_choice_in_auto_mode(tool_choice)}) + + +def _gateway_entry(tool: Mapping[str, object], gateway_labels: frozenset[str]) -> tuple[str, str | None] | None: + entry: Final = _server_entry(tool) + return entry if entry is not None and entry[0] in gateway_labels else None + + +def allowed_tools_tool_choice_with_gateway_tools_as_functions(tool_choice: object, gateway: GatewayTools) -> object: + layout: Final = _callers_layout(tool_choice) + entries: Final = () if layout is None else _entries_of(layout) + gateway_entries: Final = tuple( + entry for entry in (_gateway_entry(tool, gateway.server_labels) for tool in entries) if entry is not None + ) + if layout is None or not gateway_entries: + return tool_choice + kept: Final = tuple(tool for tool in entries if _gateway_entry(tool, gateway.server_labels) is None) + allowed: Final = tuple( + name for name, tool in gateway.tools.items() if _a_server_entry_allows(gateway_entries, tool) + ) + if not kept and not allowed: + return "none" + return _with_entries(layout, (*kept, *(_function_entry(layout, name) for name in allowed))) + + +def chat_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ChatCompletionAllowedToolChoiceParam: + return ChatCompletionAllowedToolChoiceParam( + type="allowed_tools", + allowed_tools=ChatCompletionAllowedToolsParam( + mode=choice.allowed_tools.mode, tools=[dict(tool) for tool in choice.allowed_tools.tools] + ), + ) + + def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None: tool_type: Final = tool.get("type") if tool_type not in ("function", "custom"): diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index ce98d2cbb01..53451b04515 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -1,7 +1,7 @@ # this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api import json -from collections.abc import Awaitable, Iterator, Mapping, Sequence +from collections.abc import Awaitable, Iterable, Iterator, Mapping, Sequence from itertools import chain from types import MappingProxyType from typing import Final, TypeVar @@ -18,7 +18,12 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, ) +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + parse_allowed_tools_tool_choice, + tool_calls_the_choice_excludes, +) from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model +from litellm.litellm_core_utils.get_litellm_params import AGENTIC_LOOP_TOOL_CHOICE_KEY from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict, object_items @@ -34,7 +39,11 @@ from litellm.types.integrations.custom_logger import ( is_interception_internal_key, stashed_stream_options, ) -from litellm.types.utils import ModelResponse +from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, + ChatCompletionMessageToolCall, + ModelResponse, +) from litellm.utils import CustomStreamWrapper _DispatchResult: Final = TypeVar("_DispatchResult") @@ -181,6 +190,40 @@ def _end_refused_turn( return finalized if replay is None else replay +def _tool_call_name(call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall) -> str: + match call: + case ChatCompletionMessageCustomToolCall(): + return call.custom.name + case ChatCompletionMessageToolCall(): + return call.function.name or "" + + +def _called_tool_names(response: ModelResponse) -> tuple[str, ...]: + calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in response.choices) + return tuple(map(_tool_call_name, calls)) + + +def _server_fulfilled_tool_names(callbacks: Iterable[CustomLogger]) -> frozenset[str]: + return frozenset(chain.from_iterable(callback.server_fulfilled_tool_names for callback in callbacks)) + + +def holds_a_call_the_choice_excludes( + tool_names: Iterable[str], kwargs: Mapping[str, object], callbacks: Iterable[CustomLogger] +) -> bool: + server_fulfilled: Final = _server_fulfilled_tool_names(callbacks) + client_tool_names: Final = (name for name in tool_names if name not in server_fulfilled) + return bool(tool_calls_the_choice_excludes(kwargs.get(AGENTIC_LOOP_TOOL_CHOICE_KEY), client_tool_names)) + + +def with_caller_tool_choice( + kwargs: Mapping[str, object], optional_params: Mapping[str, object] +) -> Mapping[str, object]: + tool_choice: Final = optional_params.get("tool_choice") + if AGENTIC_LOOP_TOOL_CHOICE_KEY in kwargs or parse_allowed_tools_tool_choice(tool_choice) is None: + return kwargs + return {**kwargs, AGENTIC_LOOP_TOOL_CHOICE_KEY: tool_choice} + + def _request_tools(optional_params: Mapping[str, object]) -> list[dict[str, object]]: return [tool for tool in object_items(optional_params.get("tools")) if is_str_keyed_dict(tool)] @@ -238,6 +281,35 @@ async def _refused_chat_turn( return finalize_refused_chat_turn(response, claimed) +def _fingerprint_of_a_turn_to_run( + *, + response: ModelResponse, + request_kwargs: Mapping[str, object], + callbacks: Sequence[CustomLogger], + tool_calls: object, + fingerprints: Sequence[str], + depth: int, + max_loops: int, + model: str, + stream: bool, +) -> str | None: + if holds_a_call_the_choice_excludes(_called_tool_names(response), request_kwargs, callbacks): + return None + try: + return _check_agentic_loop_safety( + tool_calls=tool_calls, + fingerprints=fingerprints, + depth=depth, + max_loops=max_loops, + model=model, + ) + except AgenticLoopSafetyError as e: + if stream: + raise + verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__) + return None + + def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]: metadata: Final = kwargs_for_followup.get("litellm_metadata") return MappingProxyType( @@ -371,7 +443,8 @@ async def maybe_run_chat_completion_agentic_loop( stream: bool, ) -> ModelResponse | CustomStreamWrapper | None: callbacks: Final = _custom_loggers(logging_obj) - depth, max_loops, fingerprints = _agentic_loop_settings(kwargs) + request_kwargs: Final = with_caller_tool_choice(kwargs, optional_params) + depth, max_loops, fingerprints = _agentic_loop_settings(request_kwargs) tools: Final = _request_tools(optional_params) for callback in callbacks: @@ -379,7 +452,7 @@ async def maybe_run_chat_completion_agentic_loop( continue hook_kwargs = { - **kwargs, + **request_kwargs, "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, "custom_llm_provider": custom_llm_provider, } @@ -403,18 +476,18 @@ async def maybe_run_chat_completion_agentic_loop( if not should_run: continue - try: - fingerprint = _check_agentic_loop_safety( - tool_calls=tool_calls, - fingerprints=fingerprints, - depth=depth, - max_loops=max_loops, - model=model, - ) - except AgenticLoopSafetyError as e: - if stream: - raise - verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__) + fingerprint = _fingerprint_of_a_turn_to_run( + response=response, + request_kwargs=request_kwargs, + callbacks=callbacks, + tool_calls=tool_calls, + fingerprints=fingerprints, + depth=depth, + max_loops=max_loops, + model=model, + stream=stream, + ) + if fingerprint is None: return _end_refused_turn( await _refused_chat_turn( callbacks, @@ -425,9 +498,9 @@ async def maybe_run_chat_completion_agentic_loop( messages=messages, tools=tools, custom_llm_provider=custom_llm_provider, - kwargs=kwargs, + kwargs=request_kwargs, ), - kwargs=kwargs, + kwargs=request_kwargs, depth=depth, model=model, custom_llm_provider=custom_llm_provider, @@ -471,7 +544,7 @@ async def maybe_run_chat_completion_agentic_loop( callback=callback, model=model, optional_params=optional_params, - kwargs=kwargs, + kwargs=dict(request_kwargs), logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, depth=depth, @@ -482,7 +555,7 @@ async def maybe_run_chat_completion_agentic_loop( return _converted_stream_replay( response, - kwargs=kwargs, + kwargs=request_kwargs, depth=depth, model=model, custom_llm_provider=custom_llm_provider, @@ -526,18 +599,18 @@ async def run_chat_completion_agentic_hooks( if not should_run: continue - try: - fingerprint = _check_agentic_loop_safety( - tool_calls=tool_calls, - fingerprints=fingerprints, - depth=depth, - max_loops=max_loops, - model=model, - ) - except AgenticLoopSafetyError as e: - if stream: - raise - verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__) + fingerprint = _fingerprint_of_a_turn_to_run( + response=response, + request_kwargs=kwargs, + callbacks=callbacks, + tool_calls=tool_calls, + fingerprints=fingerprints, + depth=depth, + max_loops=max_loops, + model=model, + stream=stream, + ) + if fingerprint is None: return await _refused_chat_turn( callbacks, callback, diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index bbbbe9635c9..198a3cd0234 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -32,8 +32,9 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset( ) PROVIDER_AFFINITY_HEADER_KWARG_KEY: Final = "provider_affinity_header" +AGENTIC_LOOP_TOOL_CHOICE_KEY: Final = "_agentic_loop_tool_choice" AGENTIC_LOOP_CONTROL_KWARGS_KEYS: Final = frozenset( - {"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops"} + {"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops", AGENTIC_LOOP_TOOL_CHOICE_KEY} ) OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS: Final = frozenset( { diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index ca036fa4bdf..5eae465134f 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -1,17 +1,12 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import ( - TYPE_CHECKING, - Any, - Final, - cast, # noqa: TID251 # non_default_params reaches the mapper as an untyped dict -) +from typing import TYPE_CHECKING, Any, Final from httpx._models import Headers, Response import litellm from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_cache_breakpoint_support -from litellm.litellm_core_utils.allowed_tools_tool_choice import is_allowed_tools_tool_choice +from litellm.litellm_core_utils.allowed_tools_tool_choice import azure_api_version_takes_tool_choice from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_tool_reference_parts_from_tool_messages, @@ -250,17 +245,11 @@ class AzureOpenAIConfig(BaseConfig): if api_version_year is None or api_version_month is None or api_version_day is None: optional_params["tool_choice"] = value else: - if ( - api_version_year < "2023" - or (api_version_year == "2023" and api_version_month < "12") - or (api_version_year == "2023" and api_version_month == "12" and api_version_day < "01") - ): - if is_allowed_tools_tool_choice( - cast(object, value) # cast-ok: widening an untyped dict value to object is always sound - ) or not (litellm.drop_params is True or (drop_params is not None and drop_params is True)): + if not azure_api_version_takes_tool_choice(api_version): + if not (litellm.drop_params is True or drop_params is True): raise UnsupportedParamsError( status_code=400, - message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. drop_params never drops an allowed_tools tool_choice, since that would let the model call tools the request excluded. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""", + message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""", ) elif value == "required" and ( api_version_year == "2024" and api_version_month <= "05" diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 72c086b74ed..5b6a64d2377 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -48,6 +48,7 @@ from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAG from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs from litellm.litellm_core_utils.agentic_loop_refusal import ( + called_tool_names, every_claimed_tool_call, finalize_refused_responses_turn, is_refused_item, @@ -64,7 +65,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( ) from litellm.litellm_core_utils.chat_completion_agentic_loop import ( bill_first_round_on_failure, + holds_a_call_the_choice_excludes, run_chat_completion_agentic_hooks, + with_caller_tool_choice, ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value @@ -160,6 +163,7 @@ from litellm.types.integrations.custom_logger import ( AgenticLoopPlan, AgenticLoopRequestPatch, AgenticLoopSafetyError, + AgenticLoopToolChoiceError, converted_stream_requested, is_interception_internal_key, ) @@ -6406,7 +6410,13 @@ class BaseLLMHTTPHandler: tools: Final = anthropic_messages_optional_request_params.get("tools", []) depth, max_loops, fingerprints = self._get_agentic_loop_settings(kwargs=kwargs) - hook_kwargs: Final = {**kwargs, "_agentic_loop_api_surface": api_surface} + hook_kwargs: Final = { + **with_caller_tool_choice( + _OPTIONAL_PARAMS.validate_python(kwargs), + _OPTIONAL_PARAMS.validate_python(anthropic_messages_optional_request_params), + ), + "_agentic_loop_api_surface": api_surface, + } for callback in callbacks: if not isinstance(callback, CustomLogger): @@ -6445,6 +6455,10 @@ class BaseLLMHTTPHandler: # Safety guards must run OUTSIDE the callback try/except — they are # bounded-loop / cycle-break rails, not callback bugs. try: + if holds_a_call_the_choice_excludes( + called_tool_names(response), hook_kwargs, (cb for cb in callbacks if isinstance(cb, CustomLogger)) + ): + raise AgenticLoopToolChoiceError(model) fingerprint = self._check_agentic_loop_safety( tool_calls=tool_calls, fingerprints=fingerprints, diff --git a/litellm/main.py b/litellm/main.py index febf9503a7e..ce71a41d516 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -75,7 +75,8 @@ from litellm.constants import ( from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.allowed_tools_tool_choice import ( - raise_if_allowed_tools_tool_choice_unsupported, + drops_unsupported_allowed_tools_tool_choice, + parse_allowed_tools_tool_choice, raise_if_hosted_tool_choice_unsupported, ) from litellm.litellm_core_utils.asyncify import asyncify, run_async_function @@ -91,6 +92,7 @@ from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, Str from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_litellm_params import ( AGENTIC_LOOP_CONTROL_KWARGS_KEYS, + AGENTIC_LOOP_TOOL_CHOICE_KEY, AWS_CREDENTIAL_KWARGS_KEYS, OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, PROVIDER_AFFINITY_HEADER_KWARG_KEY, @@ -5612,7 +5614,16 @@ def completion( messages = validate_and_fix_openai_messages(messages=messages) tools = validate_and_fix_openai_tools(tools=tools) # validate tool_choice - tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice, model=model) + requested_tool_choice: Final = cast( # cast-ok: widening an untyped dict to object is always sound + object, tool_choice + ) + tool_choice = validate_chat_completion_tool_choice( + tool_choice=tool_choice, + model=model, + drop_params=drops_unsupported_params( + cast("Mapping[str, object]", kwargs).get("drop_params") # cast-ok: kwargs is the untyped request dict + ), + ) # validate optional params stop = validate_openai_optional_params(stop=stop) thinking = validate_and_fix_thinking_param(thinking=thinking) @@ -5909,12 +5920,13 @@ def completion( if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): tools = _drop_anthropic_only_tool_keys(tools=tools) - raise_if_allowed_tools_tool_choice_unsupported( + drops_tool_choice: Final = drops_unsupported_allowed_tools_tool_choice( tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic object, tool_choice ), custom_llm_provider=custom_llm_provider, model=model, + drop_params=drop_unsupported_reasoning, ) raise_if_hosted_tool_choice_unsupported( tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic @@ -6039,7 +6051,7 @@ def completion( "response_format": response_format, "seed": seed, "tools": tools, - "tool_choice": tool_choice, + "tool_choice": None if drops_tool_choice else tool_choice, "max_retries": max_retries, "logprobs": logprobs, "top_logprobs": top_logprobs, @@ -6228,7 +6240,18 @@ def completion( if key in kwargs }, ) - litellm_params: Final = with_control_options(requested_litellm_params, control_options) + litellm_params: Final = with_control_options( + { + **requested_litellm_params, + **( + {AGENTIC_LOOP_TOOL_CHOICE_KEY: requested_tool_choice} + if AGENTIC_LOOP_TOOL_CHOICE_KEY not in kwargs + and parse_allowed_tools_tool_choice(requested_tool_choice) is not None + else {} + ), + }, + control_options, + ) affinity_api_base: Final = ( str(client.base_url) if isinstance(client, openai.OpenAI | openai.AsyncOpenAI) diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 802ed6bf9c5..aa5c7028240 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -10,6 +10,7 @@ from typing import Final from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.allowed_tools_tool_choice import unenforced_allowed_tools_tool_choice from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools @@ -72,6 +73,22 @@ def _bridged_request( ) +def _with_the_tool_choice_sent( + responses_api_request: ResponsesAPIOptionalRequestParams, + custom_llm_provider: str | None, + request_kwargs: Mapping[str, object], +) -> ResponsesAPIOptionalRequestParams: + api_version: Final = request_kwargs.get("api_version") + tool_choice: Final[object] = responses_api_request.get( # pyright: ignore[reportUnknownMemberType] # ToolChoice has an untyped member + "tool_choice" + ) + if not unenforced_allowed_tools_tool_choice( + tool_choice, custom_llm_provider, api_version if isinstance(api_version, str) else None + ): + return responses_api_request + return {**responses_api_request, "tool_choice": None} + + class LiteLLMCompletionTransformationHandler: def response_api_handler( self, @@ -127,13 +144,16 @@ class LiteLLMCompletionTransformationHandler: ), **_chat_verbosity(bridged_request), } + echoed_request: Final = _with_the_tool_choice_sent( + bridged_request, custom_llm_provider, _REQUEST_KWARGS_ADAPTER.validate_python(kwargs) + ) if _is_async: return self.async_response_api_handler( previous_response_client_ids, litellm_completion_request=litellm_completion_request, request_input=bridged_input, - responses_api_request=bridged_request, + responses_api_request=echoed_request, **kwargs, ) @@ -146,7 +166,7 @@ class LiteLLMCompletionTransformationHandler: LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( chat_completion_response=litellm_completion_response, request_input=bridged_input, - responses_api_request=bridged_request, + responses_api_request=echoed_request, ) ) @@ -157,7 +177,7 @@ class LiteLLMCompletionTransformationHandler: model=model, litellm_custom_stream_wrapper=litellm_completion_response, request_input=bridged_input, - responses_api_request=bridged_request, + responses_api_request=echoed_request, custom_llm_provider=custom_llm_provider, litellm_metadata=kwargs.get("litellm_metadata", {}), tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id( @@ -209,7 +229,9 @@ class LiteLLMCompletionTransformationHandler: **_chat_verbosity(bridged.responses_api_request), }, request_input=bridged.input, - responses_api_request=bridged.responses_api_request, + responses_api_request=_with_the_tool_choice_sent( + bridged.responses_api_request, custom_llm_provider, request_kwargs + ), **request_kwargs, ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 7eab3985f59..78a3966f3b0 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -29,6 +29,10 @@ from litellm.integrations.anthropic_cache_control_hook import ( configured_injection_points, routed_model_info, ) +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + allowed_tools_tool_choice_in_effect, + allowed_tools_tool_choice_with_gateway_tools_as_functions, +) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params, require_attributes @@ -316,6 +320,11 @@ async def aresponses_api_with_mcp( LiteLLM_Proxy_MCP_Handler, ) + request_kwargs: Final = _STR_MAPPING.validate_python(kwargs) + tool_choice_in_effect: Final = allowed_tools_tool_choice_in_effect( + tool_choice, model, drops_unsupported_params(request_kwargs.get("drop_params")) + ) + # Parse MCP tools and separate from other tools ( mcp_tools_with_litellm_proxy, @@ -334,14 +343,13 @@ async def aresponses_api_with_mcp( else (None, None, None, None) ) - request_kwargs: Final = _STR_MAPPING.validate_python(kwargs) logging_obj: Final = request_kwargs.get("litellm_logging_obj") joins_request_trace: Final = isinstance(logging_obj, LiteLLMLoggingObj) bridged_first_turn: Final = ( joins_request_trace and not stream and previous_response_id is None - and not provider_stores_the_conversation(model, custom_llm_provider, kwargs) + and not provider_stores_the_conversation(model, custom_llm_provider, request_kwargs) ) listing_trace_id: Final = ( @@ -365,6 +373,9 @@ async def aresponses_api_with_mcp( raw_headers=discovery_raw_headers, ) openai_tools: Final = LiteLLM_Proxy_MCP_Handler.transform_mcp_tools_to_openai(original_mcp_tools) + gateway: Final = await LiteLLM_Proxy_MCP_Handler.gateway_tools( + tool_choice_in_effect, tool_server_map, mcp_tools_with_litellm_proxy, user_api_key_auth + ) # Combine with other tools all_tools: Final = openai_tools + other_tools if (openai_tools or other_tools) else None @@ -383,7 +394,7 @@ async def aresponses_api_with_mcp( "stream": stream, "temperature": temperature, "text": text, - "tool_choice": tool_choice, + "tool_choice": allowed_tools_tool_choice_with_gateway_tools_as_functions(tool_choice_in_effect, gateway), "top_p": top_p, "truncation": truncation, "user": user, @@ -422,6 +433,8 @@ async def aresponses_api_with_mcp( previous_response_id=previous_response_id, tool_server_map=tool_server_map, served_tools=original_mcp_tools, + tool_choice=tool_choice_in_effect, + gateway=gateway, **kwargs, ) await mcp_streaming_response.start_initial_response() @@ -460,7 +473,11 @@ async def aresponses_api_with_mcp( ######################################################### if should_auto_execute and isinstance(response, ResponsesAPIResponse): with first_round_billed_on_failure(logging_obj, response): - tool_calls: Final = LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_response(response=response) + tool_calls: Final = LiteLLM_Proxy_MCP_Handler.tool_calls_to_run( + LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_response(response=response), + tool_choice_in_effect, + gateway, + ) if tool_calls: user_api_key_auth = kwargs.get("litellm_metadata", {}).get("user_api_key_auth") diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index f8a52227d1c..90895df3537 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -2,6 +2,7 @@ import asyncio import logging +from collections.abc import Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast @@ -10,6 +11,10 @@ from pydantic import TypeAdapter from typing_extensions import TypedDict, Unpack from litellm._internal_context import internal_sub_call +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + allowed_tools_tool_choice_kept_on_follow_up, + allowed_tools_tool_choice_with_gateway_tools_as_functions, +) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.responses.mcp.chat_stream_round_billing import ChatStreamRoundBilling, engaged_round_billing @@ -38,7 +43,7 @@ class _MCPCompletionKwargs(TypedDict, total=False, extra_items=object): def _add_mcp_metadata_to_response( response: ModelResponse | CustomStreamWrapper, openai_tools: list | None, - tool_calls: list | None = None, + tool_calls: Sequence[object] | None = None, tool_results: list | None = None, ) -> None: """ @@ -162,6 +167,9 @@ async def acompletion_with_mcp( deduplicated_mcp_tools, target_format="chat", ) + gateway: Final = await LiteLLM_Proxy_MCP_Handler.gateway_tools( + kwargs.get("tool_choice"), tool_server_map, mcp_tools_with_litellm_proxy, user_api_key_auth + ) # Combine with other tools all_tools: Final = openai_tools + other_tools if (openai_tools or other_tools) else None @@ -181,6 +189,7 @@ async def acompletion_with_mcp( "tools": all_tools, "_skip_mcp_handler": True, # Prevent recursion **clean_kwargs, + "tool_choice": allowed_tools_tool_choice_with_gateway_tools_as_functions(kwargs.get("tool_choice"), gateway), } # If not auto-executing, just make the call with transformed tools @@ -253,7 +262,7 @@ async def acompletion_with_mcp( self.base_call_args = base_call_args self.request_tags = request_tags self.collected_chunks: list[ModelResponseStream] = [] - self.tool_calls: list | None = None + self.tool_calls: Sequence[object] | None = None self.tool_results: list | None = None self.complete_response: ModelResponse | None = None self.stream_exhausted = False @@ -490,8 +499,10 @@ async def acompletion_with_mcp( if isinstance(complete_response, ModelResponse): self.complete_response = complete_response # Extract tool calls from complete response - self.tool_calls = LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response( - response=complete_response + self.tool_calls = LiteLLM_Proxy_MCP_Handler.tool_calls_to_run( + LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response=complete_response), + kwargs.get("tool_choice"), + gateway, ) if self.tool_calls: @@ -527,11 +538,13 @@ async def acompletion_with_mcp( ) # Make follow-up call with streaming - follow_up_call_args: Final = dict(self.base_call_args) - follow_up_call_args["messages"] = follow_up_messages - follow_up_call_args["stream"] = True - # Ensure follow-up call doesn't trigger MCP handler again - follow_up_call_args["_skip_mcp_handler"] = True + follow_up_call_args: Final[dict[str, object]] = { + **self.base_call_args, + **allowed_tools_tool_choice_kept_on_follow_up(self.base_call_args.get("tool_choice")), + "messages": follow_up_messages, + "stream": True, + "_skip_mcp_handler": True, + } # Import litellm here to ensure we get the patched version # This ensures the patch works correctly in tests @@ -670,7 +683,11 @@ async def acompletion_with_mcp( if not isinstance(initial_response, ModelResponse): return initial_response - tool_calls: Final = LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response=initial_response) + tool_calls: Final = LiteLLM_Proxy_MCP_Handler.tool_calls_to_run( + LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response=initial_response), + kwargs.get("tool_choice"), + gateway, + ) if not tool_calls: _add_mcp_metadata_to_response( @@ -709,9 +726,12 @@ async def acompletion_with_mcp( tool_results=tool_results, ) - follow_up_call_args: Final = dict(base_call_args) - follow_up_call_args["messages"] = follow_up_messages - follow_up_call_args["stream"] = stream + follow_up_call_args: Final = { + **base_call_args, + **allowed_tools_tool_choice_kept_on_follow_up(base_call_args.get("tool_choice")), + "messages": follow_up_messages, + "stream": stream, + } with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None): final_response: Final = await litellm_acompletion(**follow_up_call_args) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index d045689cb56..c3166b26a74 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -1,20 +1,33 @@ import traceback from collections.abc import Awaitable, Callable, Collection, Iterable, Mapping, Sequence +from dataclasses import dataclass, field from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, TypedDict, overload from openai.types.chat import ChatCompletionToolParam from openai.types.responses.function_tool_param import FunctionToolParam +from pydantic import TypeAdapter, ValidationError from typing_extensions import TypedDict as ExtraItemsTypedDict from typing_extensions import Unpack from litellm._logging import verbose_logger from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + NO_GATEWAY_TOOLS, + GatewayTool, + GatewayTools, + allowed_tools_tool_choice_kept_on_follow_up, + parse_allowed_tools_tool_choice, + tool_calls_the_choice_excludes, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager from litellm.proxy.mcp_server.utils import ( iter_known_server_prefixes, + iter_known_tool_name_spellings, logging_safe_mcp_headers, + match_known_tool_name, split_server_prefix_from_name, strip_known_server_prefix, ) @@ -117,6 +130,64 @@ async def _served_mcp_path_names( return await served_names(names) if names else frozenset[str]() +def _served_tool(tool_name: str, server_name: str) -> tuple[MCPServer | None, str]: + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy.mcp_server.server import ( + _resolve_display_name_to_original, + ) + + mcp_server: Final = global_mcp_server_manager.get_mcp_server_by_name( + server_name + ) or global_mcp_server_manager.get_mcp_server_from_tool_name(tool_name) + resolved_tool_name: Final = _resolve_display_name_to_original(tool_name, [mcp_server]) if mcp_server else tool_name + return mcp_server, strip_known_server_prefix(resolved_tool_name, mcp_server) + + +@dataclass(frozen=True, slots=True) +class _LabelScope: + label: str + whole_gateway: bool = False + server_ids: frozenset[str] = frozenset() + toolset_tools: Mapping[str, tuple[str, ...]] = field(default_factory=lambda: MappingProxyType({})) + allowed_tools: frozenset[str] | None = None + + def serves(self, server: MCPServer | None, tool_name: str, listed_name: str) -> bool: + if not self._allows(server, tool_name, listed_name): + return False + if self.whole_gateway: + return True + if server is None: + return False + toolset_names: Final = self.toolset_tools.get(server.server_id) + return server.server_id in self.server_ids or (toolset_names is not None and tool_name in toolset_names) + + def _allows(self, server: MCPServer | None, tool_name: str, listed_name: str) -> bool: + if self.allowed_tools is None or listed_name in self.allowed_tools: + return True + return server is not None and match_known_tool_name(tool_name, server, self.allowed_tools) is not None + + +_OBJECT_LIST: Final = TypeAdapter(list[object]) + + +def _listed_allowed_tools(tool: Mapping[str, object]) -> frozenset[str] | None: + try: + allowed_tools: Final = _OBJECT_LIST.validate_python(tool.get("allowed_tools")) + except ValidationError: + return None + return frozenset(name for name in allowed_tools if isinstance(name, str)) if allowed_tools else None + + +def _gateway_tool(name: str, server: MCPServer | None, tool_name: str, scopes: Iterable[_LabelScope]) -> GatewayTool: + spellings: Final = () if server is None else tuple(iter_known_tool_name_spellings(tool_name, server)) + return GatewayTool( + names=frozenset((name, tool_name, *spellings)), + server_labels=frozenset(scope.label for scope in scopes if scope.serves(server, tool_name, name)), + ) + + class LiteLLM_Proxy_MCP_Handler: """ Helper class with static methods for MCP integration with Responses API. @@ -608,6 +679,88 @@ class LiteLLM_Proxy_MCP_Handler: _should_auto_execute_tools = should_auto_execute_tools + @staticmethod + async def _label_scope( + tool: Mapping[str, object], + servers: Sequence[MCPServer], + user_api_key_auth: "UserAPIKeyAuth | None", + granted_toolsets: "GrantedToolsetIds | None", + ) -> _LabelScope | None: + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy.mcp_server.server import ( + _get_allowed_mcp_servers_from_mcp_server_names, + ) + + label: Final = tool.get("server_label") + server_url: Final = tool.get("server_url") + if not isinstance(label, str) or not isinstance(server_url, str): + return None + allowed_tools: Final = _listed_allowed_tools(tool) + if names_whole_gateway(tool): + return _LabelScope(label=label, whole_gateway=True, allowed_tools=allowed_tools) + kind, value = await LiteLLM_Proxy_MCP_Handler._resolve_proxy_mcp_name( + server_url.split("/")[-1], user_api_key_auth, granted_toolsets + ) + match kind: + case "server": + resolved: Final = await _get_allowed_mcp_servers_from_mcp_server_names([value], list(servers)) + return _LabelScope( + label=label, + server_ids=frozenset(server.server_id for server in resolved), + allowed_tools=allowed_tools, + ) + case "toolset": + permissions: Final = await global_mcp_server_manager.resolve_toolset_tool_permissions( + toolset_ids=[value] + ) + return _LabelScope( + label=label, + toolset_tools=MappingProxyType({key: tuple(names) for key, names in permissions.items()}), + allowed_tools=allowed_tools, + ) + case "denied_toolset": + return _LabelScope(label=label) + + @staticmethod + async def gateway_tools( + tool_choice: object, + tool_server_map: Mapping[str, str], + mcp_tools_with_litellm_proxy: Iterable[Mapping[str, object]], + user_api_key_auth: "UserAPIKeyAuth | None", + granted_toolsets: "GrantedToolsetIds | None" = None, + ) -> GatewayTools: + if parse_allowed_tools_tool_choice(tool_choice) is None: + return NO_GATEWAY_TOOLS + served: Final = {name: _served_tool(name, server_name) for name, server_name in tool_server_map.items()} + servers: Final = {server.server_id: server for server, _ in served.values() if server is not None} + scopes: Final = tuple( + [ + await LiteLLM_Proxy_MCP_Handler._label_scope( + tool, tuple(servers.values()), user_api_key_auth, granted_toolsets + ) + for tool in mcp_tools_with_litellm_proxy + ] + ) + label_scopes: Final = tuple(scope for scope in scopes if scope is not None) + return GatewayTools( + server_labels=frozenset(scope.label for scope in label_scopes), + tools=MappingProxyType( + { + name: _gateway_tool(name, server, tool_name, label_scopes) + for name, (server, tool_name) in served.items() + } + ), + ) + + @staticmethod + def tool_calls_to_run(tool_calls: Sequence[object], tool_choice: object, gateway: GatewayTools) -> Sequence[object]: + names: Final = tuple( + LiteLLM_Proxy_MCP_Handler.extract_tool_call_details(tool_call)[0] or "" for tool_call in tool_calls + ) + return () if tool_calls_the_choice_excludes(tool_choice, names, gateway) else tool_calls + @staticmethod def extract_tool_calls_from_response(response: ResponsesAPIResponse) -> list[object]: """Extract tool calls from the response output.""" @@ -749,9 +902,6 @@ class LiteLLM_Proxy_MCP_Handler: from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy.mcp_server.server import ( - _resolve_display_name_to_original, - ) from litellm.proxy.proxy_server import proxy_logging_obj tool_results: Final[list[MCPToolResult]] = [] @@ -780,13 +930,7 @@ class LiteLLM_Proxy_MCP_Handler: server_name = tool_server_map[tool_name] - mcp_server = global_mcp_server_manager.get_mcp_server_by_name( - server_name - ) or global_mcp_server_manager.get_mcp_server_from_tool_name(tool_name) - resolved_tool_name = ( - _resolve_display_name_to_original(tool_name, [mcp_server]) if mcp_server else tool_name - ) - sanitized_tool_name = strip_known_server_prefix(resolved_tool_name, mcp_server) + mcp_server, sanitized_tool_name = _served_tool(tool_name, server_name) start_time = datetime.now() logging_input: Sequence[Mapping[str, object]] = [ @@ -1226,6 +1370,8 @@ class LiteLLM_Proxy_MCP_Handler: previous_response_id: str | None, tool_server_map: dict[str, str], served_tools: Sequence[MCPTool] | None = None, + tool_choice: object = None, + gateway: GatewayTools = NO_GATEWAY_TOOLS, **kwargs, ) -> "MCPEnhancedStreamingIterator": """ @@ -1260,6 +1406,8 @@ class LiteLLM_Proxy_MCP_Handler: user_api_key_auth=kwargs.get("user_api_key_auth") or kwargs.get("litellm_metadata", {}).get("user_api_key_auth"), original_request_params=request_params, + tool_choice=tool_choice, + gateway=gateway, ) @staticmethod @@ -1370,21 +1518,12 @@ class LiteLLM_Proxy_MCP_Handler: def prepare_follow_up_call_params( call_params: Mapping[str, object], original_stream_setting: bool ) -> _AresponsesOptions: - """ - Prepare call parameters for the follow-up LLM call after tool execution. - - Restores the original streaming setting and removes tool_choice since - we're now providing tool results, not requesting tool calls. - """ - follow_up_params: Final = _AresponsesOptions(**call_params) - - # Restore original streaming setting for follow-up call - follow_up_params["stream"] = original_stream_setting - - # Remove tool_choice since we're providing results, not requesting tool calls - follow_up_params.pop("tool_choice", None) - - return follow_up_params + """Prepare call parameters for the follow-up LLM call after tool execution.""" + return _AresponsesOptions( + **{key: value for key, value in call_params.items() if key not in ("stream", "tool_choice")}, + **allowed_tools_tool_choice_kept_on_follow_up(call_params.get("tool_choice")), + stream=original_stream_setting, + ) _prepare_follow_up_call_params = prepare_follow_up_call_params diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index ea6064b554d..c28f40c3d2b 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -7,6 +7,11 @@ from typing_extensions import TypedDict as ExtraItemsTypedDict from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.agentic_loop_settings import validated_max_agentic_loops +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + NO_GATEWAY_TOOLS, + GatewayTools, + allowed_tools_tool_choice_kept_on_follow_up, +) from litellm.responses.litellm_completion_transformation.streaming_iterator import LiteLLMCompletionStreamingIterator from litellm.responses.mcp.request_context import MCPRequestContext from litellm.responses.mcp.stream_round_billing import StreamedRoundBilling @@ -294,6 +299,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): user_api_key_auth: "UserAPIKeyAuth | None" = None, original_request_params: dict[str, Any] | None = None, served_tools: Sequence[MCPTool] | None = None, + tool_choice: object = None, + gateway: GatewayTools = NO_GATEWAY_TOOLS, ): # MCP setup self.mcp_tools_with_litellm_proxy = mcp_tools_with_litellm_proxy or [] @@ -318,6 +325,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self.mcp_discovery_generated = True # Events are already generated self.mcp_events = mcp_events # Store the initial MCP events for backward compatibility self.tool_server_map = tool_server_map + self.tool_choice: Final = tool_choice + self.gateway: Final = gateway self.served_tools = tuple(served_tools) if served_tools is not None else None # Iterator references @@ -851,10 +860,15 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): try: # Extract tool calls from the response - if self.collected_response is not None: - tool_calls = LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_response(self.collected_response) - else: - tool_calls = [] + tool_calls: Final = ( + LiteLLM_Proxy_MCP_Handler.tool_calls_to_run( + LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_response(self.collected_response), + self.tool_choice, + self.gateway, + ) + if self.collected_response is not None + else () + ) if not tool_calls: return self.tool_call_round += 1 @@ -1038,7 +1052,6 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): ) else: return - # Remove tool_choice to avoid forcing more tool calls follow_up_params.pop("tool_choice", None) if self.tool_call_round >= self.max_tool_call_rounds: @@ -1050,6 +1063,10 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): "MCP auto-execute hit max_agentic_loops=%s; forcing a text-only follow-up.", self.max_tool_call_rounds, ) + else: + follow_up_params.update( + allowed_tools_tool_choice_kept_on_follow_up(self.original_request_params.get("tool_choice")) + ) follow_up_response: Final = await aresponses(**_FollowUpRequest(**follow_up_params)) diff --git a/litellm/tool_loop.py b/litellm/tool_loop.py index 554fdff4fce..3fc9210b1ba 100644 --- a/litellm/tool_loop.py +++ b/litellm/tool_loop.py @@ -6,6 +6,11 @@ from typing import Final, TypeAlias, cast from typing_extensions import TypedDict, Unpack from litellm.constants import DEFAULT_TOOL_LOOP_MAX_ROUNDS +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + allowed_tools_tool_choice_in_effect, + refuse_tool_calls_the_choice_excludes, +) +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, @@ -32,11 +37,14 @@ class ToolLoopMaxRoundsExceeded(RuntimeError): super().__init__(f"model still requested tool calls on round {max_rounds} of {max_rounds}") -def _validate_tool_loop_args(max_rounds: int, completion_kwargs: _ToolLoopCompletionKwargs) -> None: +def _tool_choice_in_effect(max_rounds: int, model: str, completion_kwargs: _ToolLoopCompletionKwargs) -> object: if max_rounds < 1: raise ValueError(f"max_rounds must be >= 1, got {max_rounds}") if completion_kwargs.get("stream"): raise ValueError("run_tool_loop requires whole responses; stream=True is not supported") + return allowed_tools_tool_choice_in_effect( + completion_kwargs.get("tool_choice"), model, drops_unsupported_params(completion_kwargs.get("drop_params")) + ) def _expect_model_response(response: object) -> ModelResponse: @@ -114,6 +122,14 @@ def _client_tool_calls( ) +def _refuse_tool_calls_the_choice_excludes( + tool_calls: tuple[ChatCompletionMessageToolCall, ...], model: str, tool_choice: object +) -> None: + refuse_tool_calls_the_choice_excludes( + tool_choice, tuple(tool_call.function.name or "" for tool_call in tool_calls), model + ) + + def _replays_provider_specific_fields( message: Message, tool_calls: tuple[ChatCompletionMessageToolCall, ...], @@ -151,7 +167,7 @@ def run_tool_loop( """ import litellm - _validate_tool_loop_args(max_rounds, completion_kwargs) + tool_choice: Final = _tool_choice_in_effect(max_rounds, model, completion_kwargs) history: tuple[AllMessageValues, ...] = tuple(messages) # rebind-ok: rounds append new turns for round_number in range(1, max_rounds + 1): message = ( @@ -169,6 +185,7 @@ def run_tool_loop( return message.content if round_number == max_rounds: break + _refuse_tool_calls_the_choice_excludes(client_tool_calls, model, tool_choice) tool_results = tuple(execute_tool(tool_call) for tool_call in client_tool_calls) history = ( *history, @@ -202,7 +219,7 @@ async def arun_tool_loop( """ import litellm - _validate_tool_loop_args(max_rounds, completion_kwargs) + tool_choice: Final = _tool_choice_in_effect(max_rounds, model, completion_kwargs) history: tuple[AllMessageValues, ...] = tuple(messages) # rebind-ok: rounds append new turns for round_number in range(1, max_rounds + 1): message = ( @@ -220,6 +237,7 @@ async def arun_tool_loop( return message.content if round_number == max_rounds: break + _refuse_tool_calls_the_choice_excludes(client_tool_calls, model, tool_choice) tool_results = tuple([await execute_tool(tool_call) for tool_call in client_tool_calls]) history = ( *history, diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index 14ed719a880..127b3d61753 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -81,6 +81,11 @@ class AgenticLoopLimitError(AgenticLoopSafetyError): pass +class AgenticLoopToolChoiceError(AgenticLoopSafetyError): + def __init__(self, model: str) -> None: + super().__init__(f"The allowed_tools tool_choice excludes a tool call the model made, model={model}") + + class StandardCustomLoggerInitParams(LiteLLMBaseModel): """ Params for initializing a CustomLogger. diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index c04430322c4..87f8e210782 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -362,6 +362,7 @@ class AgenticLoopState: api_surface: Literal["chat_completions", "responses"] | None = field( default=None, metadata=wire("_agentic_loop_api_surface") ) + tool_choice: Mapping[str, object] | None = field(default=None, metadata=wire("_agentic_loop_tool_choice")) code_interpreter_active: bool | None = field(default=None, metadata=wire("_code_interpreter_interception_active")) code_interpreter_sandbox_key: str | None = field( default=None, metadata=wire("_code_interpreter_interception_sandbox_key") diff --git a/litellm/utils.py b/litellm/utils.py index a69d3c063de..a3a1fe5ede4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -85,9 +85,9 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.allowed_tools_tool_choice import ( - CHAT_ALLOWED_TOOLS_LAYOUT, HOSTED_TOOL_CHOICE_TYPES, - parse_chat_allowed_tools_tool_choice, + chat_allowed_tools_tool_choice, + parse_allowed_tools_tool_choice_or_refuse, ) from litellm.litellm_core_utils.attempt_settlement import mark_attempt_answered from litellm.litellm_core_utils.core_helpers import ( @@ -9580,6 +9580,7 @@ def validate_chat_completion_user_messages(messages: list[AllMessageValues]): def validate_chat_completion_tool_choice( tool_choice: dict | str | None, model: str = "", + drop_params: bool = False, ) -> dict | str | None: """ Confirm the tool choice is passed in the OpenAI format. @@ -9592,35 +9593,21 @@ def validate_chat_completion_tool_choice( tool_choice_type = tool_choice.get("type") if tool_choice_type in ("auto", "none", "required") and "function" not in tool_choice: return tool_choice_type - if ( - tool_choice_type == "allowed_tools" - and ( - parse_chat_allowed_tools_tool_choice( - cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic - object, tool_choice - ) - ) - is None - ) - ): - raise BadRequestError( - message=( - f"Invalid tool choice, tool_choice={tool_choice}. On chat completions an allowed_tools " - f"tool_choice must be {CHAT_ALLOWED_TOOLS_LAYOUT}" + if tool_choice_type == "allowed_tools": + allowed_tools: Final = parse_allowed_tools_tool_choice_or_refuse( + cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic + object, tool_choice ), - model=model, - llm_provider="", + model, + drop_params, ) + return None if allowed_tools is None else dict(chat_allowed_tools_tool_choice(allowed_tools)) # Standard OpenAI format: {"type": "function", "function": {...}} forces_hosted_tool: Final[bool] = ( isinstance(tool_choice_type, str) and tool_choice_type in HOSTED_TOOL_CHOICE_TYPES ) - if ( - tool_choice_type != "allowed_tools" - and not forces_hosted_tool - and (tool_choice_type is None or tool_choice.get("function") is None) - ): + if not forces_hosted_tool and (tool_choice_type is None or tool_choice.get("function") is None): raise BadRequestError( message=f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec", model=model, diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 7ba833aad14..c06c2f771c1 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -4938,6 +4938,10 @@ def test_output_item_done_with_stream_map_keeps_empty_delta(): }, {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]}, ), + ( + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]}, + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]}, + ), ], ) async def test_acompletion_bridge_normalizes_tool_choice_on_the_wire( diff --git a/tests/unit/integrations/code_interpreter_interception/test_handler.py b/tests/unit/integrations/code_interpreter_interception/test_handler.py index f4f6a127fc7..2a0712f1a53 100644 --- a/tests/unit/integrations/code_interpreter_interception/test_handler.py +++ b/tests/unit/integrations/code_interpreter_interception/test_handler.py @@ -12,6 +12,7 @@ from typing import Final import httpx import pytest +from pydantic import TypeAdapter import respx import litellm @@ -27,6 +28,7 @@ from litellm.integrations.code_interpreter_interception.handler import ( LITELLM_CODE_EXECUTION_TOOL_NAME, CodeInterpreterInterceptionLogger, ) +from litellm.litellm_core_utils.allowed_tools_tool_choice import tool_calls_the_choice_excludes from litellm.llms.base_llm.sandbox.transformation import CodeExecutionResult from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, @@ -726,6 +728,45 @@ async def test_chat_completion_build_plan_runs_code_and_appends_tool_message(): assert plan.metadata["code_interpreter_calls"][0]["code"] == "print(40 + 2)" +@pytest.mark.asyncio +async def test_chat_completion_build_plan_keeps_an_allowed_tools_choice_without_forcing_another_call(): + allowed: Final = [ + {"type": "function", "function": {"name": LITELLM_CODE_EXECUTION_TOOL_NAME}}, + {"type": "function", "function": {"name": "get_time"}}, + ] + logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox(stdout="42")) + + plan: Final = await logger.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + {"id": "call_1", "name": LITELLM_CODE_EXECUTION_TOOL_NAME, "arguments": '{"code":"print(1)"}'} + ] + }, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + response={"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]}, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={ + "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}], + "tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": allowed}}, + }, + logging_obj=FakeLogging(litellm_call_id="k1"), + stream=False, + kwargs={ + "acompletion": True, + "litellm_call_id": "k1", + _ACTIVE_KEY: True, + _SANDBOX_KEY: "sbxkey1", + "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, + }, + ) + + assert plan.request_patch is not None + assert plan.request_patch.optional_params == { + "tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": allowed}} + } + + @pytest.mark.asyncio async def test_pre_call_strips_client_forged_marker_on_initial_request(): """A client cannot pre-set the active marker on the original request: with no @@ -1203,6 +1244,38 @@ async def test_extract_tool_calls_reads_object_attributes(): # --------------------------------------------------------------------------- +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "tool_choice"), + [ + (CallTypes.aresponses, {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "code_interpreter"}]}), + ( + CallTypes.acompletion, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "code_interpreter"}]}}, + ), + ], +) +async def test_pre_call_lets_an_allowed_tools_choice_listing_code_interpreter_allow_the_run( + call_type: CallTypes, tool_choice: dict[str, object] +): + logger: Final = CodeInterpreterInterceptionLogger() + + result: Final = TypeAdapter(dict[str, object]).validate_python( + await logger.async_pre_call_deployment_hook( + { + "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}], + "tool_choice": tool_choice, + "custom_llm_provider": "openai", + }, + call_type, + ) + ) + + assert tool_calls_the_choice_excludes(result["tool_choice"], (LITELLM_CODE_EXECUTION_TOOL_NAME, "get_weather")) == ( + "get_weather", + ) + + @pytest.mark.asyncio async def test_pre_call_uses_session_id_from_metadata_as_sandbox_key(): """When session_id is in request metadata, it becomes the sandbox key so the diff --git a/tests/unit/integrations/compression_interception/test_compression_interception_handler.py b/tests/unit/integrations/compression_interception/test_compression_interception_handler.py index d7a1d6f14e1..549c0363942 100644 --- a/tests/unit/integrations/compression_interception/test_compression_interception_handler.py +++ b/tests/unit/integrations/compression_interception/test_compression_interception_handler.py @@ -548,3 +548,50 @@ async def test_pre_call_hook_counts_tokens_off_the_event_loop(): assert result["messages"] is messages assert "tools" not in result assert_loop_stayed_free(took, lags) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tool_choice", "retrieval_entry"), + [ + ( + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "existing_tool"}]}, + {"type": "function", "name": "litellm_content_retrieve"}, + ), + ( + { + "type": "allowed_tools", + "allowed_tools": {"mode": "auto", "tools": [{"type": "function", "function": {"name": "existing_tool"}}]}, + }, + {"type": "function", "function": {"name": "litellm_content_retrieve"}}, + ), + ], +) +async def test_pre_call_hook_lets_an_allowed_tools_choice_allow_the_retrieval_tool_it_adds( + monkeypatch, tool_choice, retrieval_entry +): + monkeypatch.setattr( + "litellm.integrations.compression_interception.handler.compress", + lambda **_: { + "messages": [{"role": "user", "content": "stubbed"}], + "original_tokens": 12000, + "compressed_tokens": 5000, + "cache": {"auth.py": "full file content"}, + "tools": [{"type": "function", "function": {"name": "litellm_content_retrieve", "parameters": {}}}], + }, + ) + + result = await CompressionInterceptionLogger().async_pre_call_deployment_hook( + kwargs={ + "model": "anthropic/claude-sonnet-4-5", + "messages": [{"role": "user", "content": "very large context"}], + "tools": [{"type": "function", "function": {"name": "existing_tool", "parameters": {"type": "object"}}}], + "tool_choice": tool_choice, + }, + call_type=CallTypes.anthropic_messages, + ) + + assert result is not None + entries = result["tool_choice"].get("tools") or result["tool_choice"]["allowed_tools"]["tools"] + assert entries[-1] == retrieval_entry + assert entries[:-1] == (tool_choice.get("tools") or tool_choice["allowed_tools"]["tools"]) diff --git a/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py b/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py index fc415374096..d49da255c67 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py @@ -6,6 +6,7 @@ litellm.acompletion() for transparent server-side web search execution. """ import datetime +import json from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -422,6 +423,97 @@ async def test_execute_chat_completion_agentic_loop_strips_tool_choice(): ) +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["auto", "required"]) +async def test_execute_chat_completion_agentic_loop_keeps_an_allowed_tools_choice_without_forcing_another_call( + mode: str, + perplexity_search: respx.Route, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "test-openai-key") + litellm.in_memory_llm_clients_cache.flush_cache() + followup: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").respond( + json={ + "id": "followup", + "object": "chat.completion", + "created": 0, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "done"}, "finish_reason": "stop"}], + } + ) + tool_choice: Final = { + "type": "allowed_tools", + "allowed_tools": { + "mode": mode, + "tools": [{"type": "function", "function": {"name": "litellm_web_search"}}], + }, + } + + await WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI])._execute_chat_completion_agentic_loop( + model="gpt-4o", + messages=[{"role": "user", "content": "What is Bitcoin price?"}], + tool_calls=[{"id": "call_1", "name": "litellm_web_search", "input": {"query": "bitcoin price"}}], + optional_params={ + "tools": [ + {"type": "function", "function": {"name": "litellm_web_search"}}, + {"type": "function", "function": {"name": "get_time"}}, + ], + "tool_choice": tool_choice, + }, + logging_obj=MagicMock(), + stream=False, + kwargs={}, + ) + + assert perplexity_search.call_count == 1 + assert json.loads(followup.calls.last.request.content)["tool_choice"] == { + "type": "allowed_tools", + "allowed_tools": {"mode": "auto", "tools": [{"type": "function", "function": {"name": "litellm_web_search"}}]}, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tool_choice", "followup_tool_choice"), + [ + ( + { + "type": "allowed_tools", + "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, + }, + { + "type": "allowed_tools", + "allowed_tools": {"mode": "auto", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, + }, + ), + ({"type": "function", "function": {"name": "litellm_web_search"}}, None), + ("required", None), + ], +) +async def test_chat_followup_plan_keeps_only_an_allowed_tools_choice( + tool_choice: object, followup_tool_choice: object, perplexity_search: respx.Route +) -> None: + plan: Final = await WebSearchInterceptionLogger().async_build_chat_completion_agentic_loop_plan( + tools={ + "tool_calls": [{"id": "call_abc", "name": "litellm_web_search", "input": {"query": "latest ai news"}}], + "tool_type": "websearch", + "provider": "openai", + "response_format": "openai", + }, + model="gpt-4o", + messages=[{"role": "user", "content": "What's the latest AI news?"}], + response=MagicMock(), + optional_params={"tool_choice": tool_choice, "max_tokens": 512}, + logging_obj=None, + stream=False, + kwargs={"custom_llm_provider": "openai"}, + ) + + assert plan.request_patch is not None + assert plan.request_patch.optional_params.get("tool_choice") == followup_tool_choice + + if __name__ == "__main__": # Run with: pytest test_websearch_chat_completion.py -v -s pytest.main([__file__, "-v", "-s"]) diff --git a/tests/unit/integrations/websearch_interception/test_websearch_responses.py b/tests/unit/integrations/websearch_interception/test_websearch_responses.py index 6ca812a5e40..01267496668 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_responses.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_responses.py @@ -21,6 +21,7 @@ from litellm.integrations.websearch_interception.handler import ( from litellm.types.integrations.custom_logger import ( RESPONSES_AGENTIC_SURFACE, ) +from litellm.litellm_core_utils.allowed_tools_tool_choice import tool_calls_the_choice_excludes from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.utils import CallTypes, LlmProviders @@ -363,6 +364,37 @@ async def test_responses_followup_keeps_the_caller_model_across_the_chat_complet assert followup_model == requested_model +@pytest.mark.asyncio +@pytest.mark.usefixtures("perplexity_search") +@pytest.mark.parametrize( + ("tool_choice", "followup_tool_choice"), + [ + ( + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "litellm_web_search"}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "litellm_web_search"}]}, + ), + ({"type": "function", "name": "litellm_web_search"}, None), + ("required", None), + ], +) +async def test_responses_followup_keeps_only_an_allowed_tools_choice_without_forcing_another_call( + tool_choice: object, followup_tool_choice: object +) -> None: + plan: Final = await WebSearchInterceptionLogger().async_build_responses_agentic_loop_plan( + tools={**_WEB_SEARCH_FUNCTION_CALL, "provider": "azure"}, + model="gpt-5.4-mini", + messages=[{"role": "user", "content": "What's the latest AI news?"}], + response=_responses_output_with_web_search(), + optional_params={"tools": [{"type": "function", "name": "litellm_web_search"}], "tool_choice": tool_choice}, + logging_obj=None, + stream=False, + kwargs={"custom_llm_provider": "azure", "_agentic_loop_api_surface": RESPONSES_AGENTIC_SURFACE}, + ) + + assert plan.request_patch is not None + assert plan.request_patch.optional_params.get("tool_choice") == followup_tool_choice + + @pytest.mark.asyncio async def test_deployment_hook_converts_native_responses_web_search_tool(): """async_pre_call_deployment_hook converts a native Responses web_search tool @@ -390,6 +422,39 @@ async def test_deployment_hook_converts_native_responses_web_search_tool(): assert tool["parameters"]["required"] == ["query"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "tool_choice"), + [ + (CallTypes.aresponses, {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "web_search"}]}), + (CallTypes.acompletion, {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "web_search"}]}), + ( + CallTypes.acompletion, + {"type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": [{"type": "web_search"}]}}, + ), + ], +) +async def test_deployment_hook_lets_an_allowed_tools_choice_listing_web_search_allow_the_search( + call_type: CallTypes, tool_choice: dict[str, object] +): + logger: Final = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI]) + + result: Final = await logger.async_pre_call_deployment_hook( + kwargs={ + "model": "gpt-4o", + "custom_llm_provider": "openai", + "tools": [{"type": "web_search"}, {"type": "function", "name": "get_weather"}], + "tool_choice": tool_choice, + }, + call_type=call_type, + ) + + assert result is not None + assert tool_calls_the_choice_excludes(result["tool_choice"], ("litellm_web_search", "get_weather")) == ( + "get_weather", + ) + + @pytest.mark.asyncio async def test_deployment_hook_responses_returns_none_without_web_search(): """No web search tool in a responses request -> deployment hook makes no change.""" diff --git a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py index d020c7fcd85..7171bd9e0e5 100644 --- a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py +++ b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py @@ -4,16 +4,45 @@ from typing import Final import pytest +import litellm from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.allowed_tools_tool_choice import ( ChatAllowedToolsToolChoice, + GatewayTool, + GatewayTools, + allowed_tools_tool_choice_in_auto_mode, + allowed_tools_tool_choice_in_effect, + allowed_tools_tool_choice_with_builtin_as_function, + allowed_tools_tool_choice_with_function, + allowed_tools_tool_choice_with_gateway_tools_as_functions, + drops_unsupported_allowed_tools_tool_choice, hosted_tool_choice_type, is_allowed_tools_tool_choice, - parse_chat_allowed_tools_tool_choice, + chat_allowed_tools_tool_choice, + parse_allowed_tools_tool_choice, raise_if_hosted_tool_choice_unsupported, + refuse_tool_calls_the_choice_excludes, + tool_calls_the_choice_excludes, + unenforced_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, ) +_CHOICES_WITH_A_NAME_THAT_IS_NOT_A_STRING: Final = ( + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "name": ["get_time"]}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "name": {"first": "get_time"}}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "name": 7}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "name": ""}]}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "mcp", "name": ["get_time"]}]}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "mcp", "name": 7}]}}, + { + "type": "allowed_tools", + "allowed_tools": {"mode": "auto", "tools": [{"type": "function", "function": {"name": ["get_time"]}}]}, + }, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "function": {"name": 7}}]}, +) + +_WHOLE_DOCS_SERVER: Final = {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "server_label": "docs"}]} + _CHAT_LAYOUT: Final[dict[str, object]] = { "type": "allowed_tools", "allowed_tools": { @@ -29,7 +58,7 @@ _CHAT_LAYOUT: Final[dict[str, object]] = { def _parsed(tool_choice: Mapping[str, object]) -> ChatAllowedToolsToolChoice: - parsed: Final = parse_chat_allowed_tools_tool_choice(tool_choice) + parsed: Final = parse_allowed_tools_tool_choice(tool_choice) assert parsed is not None return parsed @@ -75,18 +104,64 @@ def test_responses_layout_refuses_to_rewrite_a_malformed_or_conflicting_entry(to assert responses_allowed_tools_tool_choice(_parsed(tool_choice)) is None +_RESPONSES_LAYOUT: Final[dict[str, object]] = { + "type": "allowed_tools", + "mode": "required", + "tools": [ + {"type": "function", "name": "get_time"}, + {"type": "custom", "name": "apply_patch", "x-vendor": 1}, + {"type": "mcp", "server_label": "docs"}, + {"type": "function", "function": {"name": "already_nested"}}, + ], +} + + +def test_responses_layout_is_read_into_the_chat_layout(): + assert chat_allowed_tools_tool_choice(_parsed(_RESPONSES_LAYOUT)) == { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [ + {"type": "function", "function": {"name": "get_time"}}, + {"type": "custom", "x-vendor": 1, "custom": {"name": "apply_patch"}}, + {"type": "mcp", "server_label": "docs"}, + {"type": "function", "function": {"name": "already_nested"}}, + ], + }, + } + + +def test_responses_layout_round_trips_back_to_what_the_caller_sent(): + flat_only: Final = {**_RESPONSES_LAYOUT, "tools": list(_RESPONSES_LAYOUT["tools"])[:3]} + assert responses_allowed_tools_tool_choice(_parsed(flat_only)) == flat_only + + @pytest.mark.parametrize( "tool_choice", [ - {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}, {"type": "allowed_tools", "allowed_tools": {"mode": "any", "tools": [{"type": "function"}]}}, {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}, "extra": 1}, + {"type": "allowed_tools", "mode": "auto", "tools": []}, + {"type": "allowed_tools", "mode": "none", "tools": [{"type": "function", "name": "get_time"}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function"}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "custom", "name": ""}]}, + { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "function", "name": "get_time", "function": {"name": "get_weather"}}], + }, + { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_time"}], + "allowed_tools": {"mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + }, {"type": "function", "function": {"name": "get_time"}}, ], ) -def test_parse_rejects_anything_but_the_chat_layout(tool_choice: dict[str, object]): - assert parse_chat_allowed_tools_tool_choice(tool_choice) is None +def test_parse_rejects_a_malformed_choice_in_either_layout(tool_choice: dict[str, object]): + assert parse_allowed_tools_tool_choice(tool_choice) is None @pytest.mark.parametrize( @@ -103,6 +178,421 @@ def test_is_allowed_tools_tool_choice(tool_choice: Mapping[str, object] | str | assert is_allowed_tools_tool_choice(tool_choice) is expected +_FLAT_CHOICE: Final = {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]} + + +@pytest.mark.parametrize( + ("tool_choice", "custom_llm_provider"), + [ + (_FLAT_CHOICE, "anthropic"), + (_CHAT_LAYOUT, "hosted_vllm"), + ({"type": "allowed_tools", "mode": "required", "tools": []}, "openai"), + ({"type": "allowed_tools", "tools": [{"type": "function", "name": "get_time"}]}, "azure"), + ], +) +def test_a_choice_nothing_would_enforce_counts_as_unenforced(tool_choice: object, custom_llm_provider: str): + assert unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider) + + +@pytest.mark.parametrize( + ("tool_choice", "custom_llm_provider"), + [ + (_FLAT_CHOICE, "openai"), + (_CHAT_LAYOUT, "azure"), + (_FLAT_CHOICE, "litellm_proxy"), + ("required", "anthropic"), + ({"type": "function", "function": {"name": "get_time"}}, "anthropic"), + (None, "anthropic"), + ], +) +def test_a_choice_the_provider_enforces_or_any_other_choice_counts_as_enforced( + tool_choice: object, custom_llm_provider: str +): + assert not unenforced_allowed_tools_tool_choice(tool_choice, custom_llm_provider) + + +@pytest.mark.parametrize( + ("tool_choice", "tool_names"), + [(_FLAT_CHOICE, ("get_time",)), (_CHAT_LAYOUT, ("get_time", "apply_patch"))], +) +def test_a_tool_call_the_choice_lists_passes(tool_choice: object, tool_names: tuple[str, ...]): + assert refuse_tool_calls_the_choice_excludes(tool_choice, tool_names, "the-model") is None + + +@pytest.mark.parametrize( + "tool_choice", [_FLAT_CHOICE, _CHAT_LAYOUT, {"type": "allowed_tools", "mode": "required", "tools": []}] +) +def test_a_tool_call_the_choice_does_not_list_is_refused(tool_choice: object): + with pytest.raises(BadRequestError, match=r"The model called .*get_weather") as exc: + refuse_tool_calls_the_choice_excludes(tool_choice, ("get_time", "get_weather"), "the-model") + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize("tool_choice", ["required", None, {"type": "function", "function": {"name": "get_time"}}]) +def test_any_other_choice_leaves_the_tool_calls_alone(tool_choice: object): + assert refuse_tool_calls_the_choice_excludes(tool_choice, ("get_weather",), "the-model") is None + + +@pytest.mark.parametrize( + ("api_version", "unenforced"), + [ + ("2023-07-01-preview", True), + ("2023-11-30", True), + ("2023-12-01-preview", False), + ("2025-04-01-preview", False), + ("v1", False), + ], +) +def test_azure_counts_as_enforcing_only_on_a_version_that_takes_tool_choice(api_version: str, unenforced: bool): + assert unenforced_allowed_tools_tool_choice(_FLAT_CHOICE, "azure", api_version) is unenforced + + +def test_azure_reads_the_same_api_version_fallback_as_the_request(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "api_version", "2023-07-01-preview") + + assert unenforced_allowed_tools_tool_choice(_FLAT_CHOICE, "azure") + + +def test_drop_params_drops_a_choice_the_provider_cannot_take(): + assert drops_unsupported_allowed_tools_tool_choice(_FLAT_CHOICE, "anthropic", "the-model", True) + + +def test_a_choice_the_provider_cannot_take_answers_400_without_drop_params(): + with pytest.raises(BadRequestError, match="only supported for") as exc: + drops_unsupported_allowed_tools_tool_choice(_FLAT_CHOICE, "anthropic", "the-model", False) + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "required", "tools": []}, + {"type": "allowed_tools", "tools": [{"type": "function", "name": "get_time"}]}, + *_CHOICES_WITH_A_NAME_THAT_IS_NOT_A_STRING, + ], +) +def test_a_malformed_choice_is_refused_without_drop_params_and_dropped_with_it(tool_choice: object): + with pytest.raises(BadRequestError, match="Invalid tool choice") as exc: + allowed_tools_tool_choice_in_effect(tool_choice, "the-model", False) + + assert exc.value.status_code == 400 + assert allowed_tools_tool_choice_in_effect(tool_choice, "the-model", True) is None + + +@pytest.mark.parametrize("tool_choice", _CHOICES_WITH_A_NAME_THAT_IS_NOT_A_STRING) +def test_a_name_that_is_not_a_string_lists_no_tool(tool_choice: object): + assert tool_calls_the_choice_excludes(tool_choice, ("get_time",)) == ("get_time",) + + +@pytest.mark.parametrize("tool_choice", [_FLAT_CHOICE, _CHAT_LAYOUT, "required", None]) +@pytest.mark.parametrize("drop_params", [True, False]) +def test_a_valid_choice_or_any_other_choice_stays_in_effect(tool_choice: object, drop_params: bool): + assert allowed_tools_tool_choice_in_effect(tool_choice, "the-model", drop_params) is tool_choice + + +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "server_label": "docs", "name": None}]}, + { + "type": "allowed_tools", + "allowed_tools": {"mode": "auto", "tools": [{"type": "mcp", "server_label": "docs", "name": None}]}, + }, + ], +) +def test_an_mcp_entry_with_a_null_name_is_a_valid_choice(tool_choice: object): + assert parse_allowed_tools_tool_choice(tool_choice) is not None + assert allowed_tools_tool_choice_in_effect(tool_choice, "the-model", False) is tool_choice + + +_DOCS_SEARCH: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "mcp", "server_label": "docs", "name": "search"}], +} + + +def _gateway_serving_search(*server_labels: str) -> GatewayTools: + return GatewayTools( + server_labels=frozenset({"docs", "other"}), + tools={ + "docs_server-search": GatewayTool( + names=frozenset({"docs_server-search", "search"}), server_labels=frozenset(server_labels) + ) + }, + ) + + +@pytest.mark.parametrize("name", ["search", "docs_server-search"]) +def test_a_named_mcp_entry_allows_its_call_on_a_server_its_label_serves(name: str): + tool_choice: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "mcp", "server_label": "docs", "name": name}], + } + + assert ( + tool_calls_the_choice_excludes(tool_choice, ("docs_server-search",), _gateway_serving_search("docs", "other")) + == () + ) + + +def test_an_mcp_entry_without_a_name_allows_every_call_its_server_serves(): + assert ( + tool_calls_the_choice_excludes(_WHOLE_DOCS_SERVER, ("docs_server-search",), _gateway_serving_search("docs")) + == () + ) + + +@pytest.mark.parametrize( + ("tool_choice", "server_labels"), + [ + (_WHOLE_DOCS_SERVER, frozenset({"other"})), + (_WHOLE_DOCS_SERVER, frozenset()), + ( + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "server_label": "docs", "name": "read"}]}, + frozenset({"docs"}), + ), + (_DOCS_SEARCH, frozenset({"other"})), + (_DOCS_SEARCH, frozenset()), + ], +) +def test_an_mcp_entry_allows_no_call_outside_its_server_or_its_name( + tool_choice: object, server_labels: frozenset[str] +): + assert tool_calls_the_choice_excludes( + tool_choice, ("docs_server-search",), _gateway_serving_search(*server_labels) + ) == ("docs_server-search",) + + +def _is_web_search(tool: Mapping[str, object]) -> bool: + return tool.get("type") == "web_search" + + +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "web_search"}, {"type": "function", "name": "get_time"}]}, + { + "type": "allowed_tools", + "allowed_tools": { + "mode": "auto", + "tools": [{"type": "web_search"}, {"type": "function", "function": {"name": "get_time"}}], + }, + }, + ], +) +def test_a_builtin_entry_allows_the_function_that_replaces_it_in_the_same_layout(tool_choice: dict[str, object]): + rewritten: Final = allowed_tools_tool_choice_with_builtin_as_function( + tool_choice, _is_web_search, "litellm_web_search" + ) + + assert tool_calls_the_choice_excludes(tool_choice, ("litellm_web_search", "get_time")) == ("litellm_web_search",) + assert tool_calls_the_choice_excludes(rewritten, ("litellm_web_search", "get_time", "get_weather")) == ( + "get_weather", + ) + assert isinstance(rewritten, dict) and rewritten.keys() == tool_choice.keys() + + +def test_a_function_or_mcp_entry_is_never_taken_for_a_builtin(): + tool_choice: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "web_search"}, {"type": "mcp", "server_label": "docs", "name": "web_search"}], + } + + rewritten: Final = allowed_tools_tool_choice_with_builtin_as_function( + tool_choice, lambda tool: tool.get("name") == "web_search", "litellm_web_search" + ) + + assert parse_allowed_tools_tool_choice(rewritten) == parse_allowed_tools_tool_choice(tool_choice) + + +@pytest.mark.parametrize( + "tool_choice", + [ + "auto", + None, + {"type": "allowed_tools", "mode": "auto", "tools": []}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "web_search", "name": []}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "web_search"}, {"type": "function", "name": 3}]}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "web_search", "name": {}}]}}, + ], +) +def test_any_other_or_malformed_choice_is_left_as_sent(tool_choice: object): + assert allowed_tools_tool_choice_with_builtin_as_function(tool_choice, _is_web_search, "litellm_web_search") is tool_choice + + +_GATEWAY_WITH_TWO_SERVERS: Final = GatewayTools( + server_labels=frozenset({"docs", "weather"}), + tools={ + "docs_server-search": GatewayTool( + names=frozenset({"docs_server-search", "search"}), server_labels=frozenset({"docs"}) + ), + "docs_server-read": GatewayTool(names=frozenset({"docs_server-read", "read"}), server_labels=frozenset({"docs"})), + "weather_server-forecast": GatewayTool( + names=frozenset({"weather_server-forecast", "forecast"}), server_labels=frozenset({"weather"}) + ), + }, +) + + +@pytest.mark.parametrize( + ("tool_choice", "expected"), + [ + ( + { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "mcp", "server_label": "docs"}, {"type": "function", "name": "get_time"}], + }, + { + "type": "allowed_tools", + "mode": "required", + "tools": [ + {"type": "function", "name": "get_time"}, + {"type": "function", "name": "docs_server-search"}, + {"type": "function", "name": "docs_server-read"}, + ], + }, + ), + ( + { + "type": "allowed_tools", + "allowed_tools": { + "mode": "auto", + "tools": [{"type": "mcp", "server_label": "weather", "name": "forecast"}], + }, + }, + { + "type": "allowed_tools", + "allowed_tools": { + "mode": "auto", + "tools": [{"type": "function", "function": {"name": "weather_server-forecast"}}], + }, + }, + ), + ], +) +def test_gateway_mcp_entries_become_the_functions_they_allow_in_the_callers_layout( + tool_choice: dict[str, object], expected: dict[str, object] +): + assert allowed_tools_tool_choice_with_gateway_tools_as_functions(tool_choice, _GATEWAY_WITH_TWO_SERVERS) == expected + + +def test_an_mcp_entry_of_a_label_the_gateway_does_not_serve_is_forwarded_as_sent(): + tool_choice: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "mcp", "server_label": "remote"}, {"type": "mcp", "server_label": "docs", "name": "read"}], + } + + assert allowed_tools_tool_choice_with_gateway_tools_as_functions(tool_choice, _GATEWAY_WITH_TWO_SERVERS) == { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "mcp", "server_label": "remote"}, {"type": "function", "name": "docs_server-read"}], + } + + +@pytest.mark.parametrize("mode", ["auto", "required"]) +def test_gateway_entries_that_allow_no_converted_tool_send_none_instead_of_an_empty_list(mode: str): + tool_choice: Final = { + "type": "allowed_tools", + "mode": mode, + "tools": [{"type": "mcp", "server_label": "docs", "name": "delete"}], + } + + assert allowed_tools_tool_choice_with_gateway_tools_as_functions(tool_choice, _GATEWAY_WITH_TWO_SERVERS) == "none" + + +@pytest.mark.parametrize("tool_choice", ["auto", None, _FLAT_CHOICE, {"type": "allowed_tools", "mode": "auto", "tools": []}]) +def test_a_choice_without_gateway_entries_is_forwarded_as_sent(tool_choice: object): + assert allowed_tools_tool_choice_with_gateway_tools_as_functions(tool_choice, _GATEWAY_WITH_TWO_SERVERS) is tool_choice + + +@pytest.mark.parametrize( + ("tool_choice", "expected"), + [ + ( + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_time"}, {"type": "function", "name": "retrieve"}], + }, + ), + ( + {"type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": [{"type": "mcp", "server_label": "x"}]}}, + { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [{"type": "mcp", "server_label": "x"}, {"type": "function", "function": {"name": "retrieve"}}], + }, + }, + ), + ], +) +def test_an_added_function_joins_the_choice_in_the_callers_layout( + tool_choice: dict[str, object], expected: dict[str, object] +): + assert allowed_tools_tool_choice_with_function(tool_choice, "retrieve") == expected + + +@pytest.mark.parametrize( + "tool_choice", + [ + "auto", + None, + {"type": "function", "function": {"name": "get_time"}}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "retrieve"}]}, + ], +) +def test_a_choice_that_is_not_allowed_tools_or_already_lists_the_function_is_left_as_sent(tool_choice: object): + assert allowed_tools_tool_choice_with_function(tool_choice, "retrieve") is tool_choice + + +@pytest.mark.parametrize( + ("tool_choice", "expected"), + [ + ( + { + "type": "allowed_tools", + "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, + }, + { + "type": "allowed_tools", + "allowed_tools": {"mode": "auto", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, + }, + ), + ( + { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "function", "name": "get_time"}, {"type": "mcp", "server_label": "docs"}], + }, + { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_time"}, {"type": "mcp", "server_label": "docs"}], + }, + ), + (_CHAT_LAYOUT, _CHAT_LAYOUT), + ("required", "required"), + ( + {"type": "function", "function": {"name": "get_time"}}, + {"type": "function", "function": {"name": "get_time"}}, + ), + ], +) +def test_auto_mode_lifts_only_the_requirement_and_keeps_the_callers_layout_and_tools( + tool_choice: object, expected: object +): + assert allowed_tools_tool_choice_in_auto_mode(tool_choice) == expected + + @pytest.mark.parametrize( ("tool_choice", "expected"), [ diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index d26da679246..7d41cbd6628 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -22,8 +22,8 @@ they stay out of the body even without it. import datetime import json -from collections.abc import Mapping, Sequence -from typing import Any, Dict, Final, List, Optional, Tuple +from collections.abc import Callable, Mapping, Sequence +from typing import Any, ClassVar, Dict, Final, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -38,6 +38,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.chat_completion_agentic_loop import ( _execute_chat_completion_agentic_plan, maybe_run_chat_completion_agentic_loop, + run_chat_completion_agentic_hooks, ) from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper @@ -67,6 +68,7 @@ _INTERNAL_CONTROL_FIELDS = ( "_code_interpreter_interception_active", "_code_interpreter_interception_sandbox_key", "_code_interpreter_interception_converted_stream", + "_agentic_loop_tool_choice", "litellm_metadata", ) @@ -132,6 +134,27 @@ def _tool_call_model_response() -> ModelResponse: ) +def _custom_tool_call_model_response() -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_abc", + "type": "custom", + "custom": {"name": "litellm_code_execution", "input": "print(6*7)"}, + } + ], + ), + ) + ] + ) + + def _plain_model_response(content: str = "The answer is 42") -> ModelResponse: return ModelResponse( choices=[ @@ -1090,3 +1113,231 @@ class TestAgenticFollowUpKeepsTheProviderPrefix: response: Final = await self._run_followup("openai/gpt-4o", "openrouter", patched_model="openai/gpt-4o") assert response._hidden_params["custom_llm_provider"] == "openrouter" + + +_ALLOWS_ONLY_GET_WEATHER: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_weather"}], +} +_ALLOWS_THE_HOOKS_CALL: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "litellm_code_execution"}], +} + + +async def _dispatch_under_the_choice(tool_choice: object, acompletion_mock: AsyncMock) -> Any: + gate: Final = _GateOnlyLogger( + plan=AgenticLoopPlan( + run_agentic_loop=True, request_patch=AgenticLoopRequestPatch(messages=_patched_messages()) + ), + tool_calls=_REFUSED_GATE_TOOL_CALLS, + ) + with patch.object(litellm, "acompletion", acompletion_mock): + return await maybe_run_chat_completion_agentic_loop( + response=_tool_call_model_response(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={"tool_choice": tool_choice}, + kwargs={}, + logging_obj=_LoggingWithCallbacks((gate,)), + custom_llm_provider="openai", + stream=False, + ) + + +@pytest.mark.asyncio +async def test_an_intercepted_call_the_choice_excludes_ends_the_turn_without_running(restore_callbacks): + acompletion_mock: Final = AsyncMock() + + result: Final = await _dispatch_under_the_choice(_ALLOWS_ONLY_GET_WEATHER, acompletion_mock) + + acompletion_mock.assert_not_awaited() + assert isinstance(result, ModelResponse) + assert not result.choices[0].message.tool_calls + assert result.choices[0].finish_reason == "stop" + + +@pytest.mark.asyncio +async def test_an_intercepted_call_the_choice_lists_still_runs(restore_callbacks): + followup: Final = _plain_model_response("done") + acompletion_mock: Final = AsyncMock(return_value=followup) + + result: Final = await _dispatch_under_the_choice(_ALLOWS_THE_HOOKS_CALL, acompletion_mock) + + acompletion_mock.assert_awaited_once() + assert result is followup + + +class _ChatHookLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.runs = 0 + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: list[dict], + tools: list[dict] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict, + ) -> tuple[bool, dict]: + return True, {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]} + + async def async_run_chat_completion_agentic_loop( + self, + tools: dict, + model: str, + messages: list[dict], + response: object, + optional_params: dict, + logging_obj: Any, + stream: bool, + kwargs: dict, + ) -> object: + self.runs += 1 + return _plain_model_response("ran the hook") + + +def _logging_with(hook: CustomLogger) -> Logging: + return Logging( + model="gpt-4o-mini", + messages=[], + stream=False, + call_type="acompletion", + start_time=datetime.datetime.now(), + litellm_call_id="call-test", + function_id="function-test", + dynamic_success_callbacks=[hook], + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_response", [_tool_call_model_response, _custom_tool_call_model_response]) +async def test_a_chat_hook_never_runs_a_call_the_choice_excludes(model_response: Callable[[], ModelResponse]): + hook: Final = _ChatHookLogger() + + result: Final = await run_chat_completion_agentic_hooks( + response=model_response(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={}, + kwargs={"_agentic_loop_tool_choice": _ALLOWS_ONLY_GET_WEATHER}, + logging_obj=_logging_with(hook), + custom_llm_provider="openai", + stream=False, + ) + + assert hook.runs == 0 + assert isinstance(result, ModelResponse) + assert not result.choices[0].message.tool_calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_response", "tool_type"), + [(_tool_call_model_response, "function"), (_custom_tool_call_model_response, "custom")], +) +async def test_a_chat_hook_runs_a_call_the_choice_lists(model_response: Callable[[], ModelResponse], tool_type: str): + hook: Final = _ChatHookLogger() + + result: Final = await run_chat_completion_agentic_hooks( + response=model_response(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={}, + kwargs={ + "_agentic_loop_tool_choice": { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": tool_type, "name": "litellm_code_execution"}], + } + }, + logging_obj=_logging_with(hook), + custom_llm_provider="openai", + stream=False, + ) + + assert hook.runs == 1 + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "ran the hook" + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("tool_choice", "hook_runs"), [(_ALLOWS_ONLY_GET_WEATHER, 0), (_ALLOWS_THE_HOOKS_CALL, 1)]) +async def test_acompletion_hands_the_callers_choice_to_the_chat_hooks(tool_choice: dict[str, object], hook_runs: int): + hook: Final = _ChatHookLogger() + mock_client: Final = MagicMock() + mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_model_response())) + + response: Final = await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + tool_choice=tool_choice, + api_key="sk-test", + client=mock_client, + success_callback=[hook], + ) + + assert (hook.runs, mock_client.post.await_count, response.choices[0].message.tool_calls) == (hook_runs, 1, None) + + + +class _RetrievalHookLogger(_ChatHookLogger): + server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({"litellm_code_execution"}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_response", [_tool_call_model_response, _custom_tool_call_model_response]) +async def test_a_chat_hook_runs_a_tool_litellm_serves_itself_that_no_choice_can_list( + model_response: Callable[[], ModelResponse], +): + hook: Final = _RetrievalHookLogger() + + result: Final = await run_chat_completion_agentic_hooks( + response=model_response(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={}, + kwargs={"_agentic_loop_tool_choice": _ALLOWS_ONLY_GET_WEATHER}, + logging_obj=_logging_with(hook), + custom_llm_provider="openai", + stream=False, + ) + + assert hook.runs == 1 + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "ran the hook" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "auto", "tools": []}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "name": ["get_weather"]}]}, + ], +) +async def test_a_malformed_choice_dropped_by_drop_params_never_holds_back_the_chat_hooks( + tool_choice: dict[str, object], +): + hook: Final = _ChatHookLogger() + mock_client: Final = MagicMock() + mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_model_response())) + + response: Final = await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + tool_choice=tool_choice, + drop_params=True, + api_key="sk-test", + client=mock_client, + success_callback=[hook], + ) + + assert (hook.runs, response.choices[0].message.content) == (1, "ran the hook") diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index 6358d6976c4..2cb8b9cf6c5 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -99,10 +99,21 @@ def test_map_openai_params_drop_params_still_drops_a_named_tool_choice_on_old_ap assert "tool_choice" not in mapped -def test_map_openai_params_drop_params_never_drops_allowed_tools_on_old_api_version(): - with pytest.raises(litellm.UnsupportedParamsError, match="allowed_tools") as exc: - _map_tool_choice_on_pre_tool_choice_api_version( - {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}} +def test_map_openai_params_drop_params_drops_allowed_tools_on_old_api_version(): + mapped: Final = _map_tool_choice_on_pre_tool_choice_api_version( + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}} + ) + assert "tool_choice" not in mapped + + +def test_map_openai_params_allowed_tools_on_old_api_version_is_a_400_without_drop_params(): + with pytest.raises(litellm.UnsupportedParamsError, match="2023-12-01-preview") as exc: + AzureOpenAIConfig().map_openai_params( + {"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}}}, + {}, + "gpt-4o", + drop_params=False, + api_version="2023-07-01-preview", ) assert exc.value.status_code == 400 diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index ba8fb3bf107..4d8ba26d8d8 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -455,6 +455,95 @@ def test_response_api_handler_sends_no_stream_options_when_web_search_forces_a_n assert "stream_options" not in recorder.seen_kwargs +class _LookupLoop(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.runs = 0 + + async def async_should_run_agentic_loop( + self, + response: object, + model: str, + messages: list[dict[str, object]], + tools: list[dict[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict[str, object], + ) -> tuple[bool, dict[str, object]]: + return True, {"tool_calls": [{"id": "call_lookup", "name": "lookup"}]} + + async def async_run_agentic_loop( + self, + tools: dict, + model: str, + messages: list[dict], + response: object, + anthropic_messages_provider_config: object, + anthropic_messages_optional_request_params: dict, + logging_obj: object, + stream: bool, + kwargs: dict, + ) -> object: + self.runs += 1 + return response + + +def _a_lookup_call(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-4o", + "output": [ + { + "type": "function_call", + "id": "fc_1", + "call_id": "call_lookup", + "name": "lookup", + "arguments": "{}", + "status": "completed", + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + + +@pytest.mark.parametrize("allowed_name,runs", [("get_time", 0), ("lookup", 1)]) +def test_response_api_handler_runs_no_hook_call_the_allowed_tools_choice_excludes(allowed_name: str, runs: int): + hook: Final = _LookupLoop() + + response: Final = BaseLLMHTTPHandler().response_api_handler( + model="gpt-4o", + input="what time is it?", + responses_api_provider_config=AzureOpenAIResponsesAPIConfig(), + response_api_optional_request_params={ + "tools": [ + {"type": "function", "name": "get_time", "parameters": {"type": "object", "properties": {}}}, + {"type": "function", "name": "lookup", "parameters": {"type": "object", "properties": {}}}, + ], + "tool_choice": { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": allowed_name}], + }, + }, + custom_llm_provider="azure", + litellm_params=GenericLiteLLMParams( + api_key="sk-test", api_base="https://example.openai.azure.com", api_version="2025-04-01-preview" + ), + logging_obj=Mock(litellm_call_id="call-1", dynamic_success_callbacks=[hook], model_call_details={}), + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_a_lookup_call))), + ) + + assert hook.runs == runs + assert isinstance(response, ResponsesAPIResponse) + assert len(response.output) == runs + + @pytest.mark.asyncio async def test_async_response_api_handler_streams_when_provider_transform_adds_stream(): handler = BaseLLMHTTPHandler() diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index cbd26618312..de30c68c3c8 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -2457,3 +2457,170 @@ async def test_codex_tool_search_follow_up_text_verbosity_follows_provider_param assert dropped == without_text assert allowed == {**without_text, "verbosity": "low"} + + +_FLAT_ALLOWED_WEATHER: Final = { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "function", "name": "get_weather"}], +} +_RESPONSES_TOOLS: Final = ( + {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}, + {"type": "function", "name": "get_time", "parameters": {"type": "object", "properties": {}}}, +) + + +def _chat_answer() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-echo", + "object": "chat.completion", + "created": 1739462947, + "model": "the-model", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "done"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.asyncio +async def test_responses_reports_auto_for_an_allowed_tools_choice_it_dropped( + respx_mock: respx.MockRouter, use_async: bool +) -> None: + route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + return_value=_anthropic_message({"type": "text", "text": "done"}) + ) + call: Final = { + "model": "anthropic/claude-haiku-4-5", + "input": "What is the weather?", + "tools": list(_RESPONSES_TOOLS), + "tool_choice": _FLAT_ALLOWED_WEATHER, + "drop_params": True, + "api_key": "sk-ant-fake", + } + + response: Final = await litellm.aresponses(**call) if use_async else litellm.responses(**call) + + assert "tool_choice" not in json.loads(route.calls[0].request.content) + assert response.tool_choice == "auto" + + +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.asyncio +async def test_responses_drop_the_choice_beside_a_tool_the_provider_runs(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + return_value=_anthropic_message({"type": "text", "text": "done"}) + ) + + response: Final = await litellm.aresponses( + model="anthropic/claude-haiku-4-5", + input="What is the weather?", + tools=[{"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com/sse"}], + tool_choice=_FLAT_ALLOWED_WEATHER, + drop_params=True, + api_key="sk-ant-fake", + ) + + assert "tool_choice" not in json.loads(route.calls[0].request.content) + assert response.tool_choice == "auto" + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_responses_report_auto_when_an_old_azure_api_version_dropped_the_choice(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(url__regex=r"https://azure-bridge\.example\.com/.*chat/completions.*").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-old-azure", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [ + {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "done"}} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + + response: Final = litellm.responses( + model="azure/gpt-4o", + input="What is the weather?", + tools=list(_RESPONSES_TOOLS), + tool_choice=_FLAT_ALLOWED_WEATHER, + drop_params=True, + use_chat_completions_api=True, + api_key="fake", + api_base="https://azure-bridge.example.com", + api_version="2023-07-01-preview", + ) + + assert "tool_choice" not in json.loads(route.calls[0].request.content) + assert response.tool_choice == "auto" + + +def test_streamed_responses_report_auto_for_an_allowed_tools_choice_it_dropped(respx_mock: respx.MockRouter) -> None: + respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=_anthropic_sse( + { + "type": "message_start", + "message": { + "id": "msg_dropped_choice", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [], + "usage": {"input_tokens": 9, "output_tokens": 1}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "done"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4}}, + {"type": "message_stop"}, + ), + ) + ) + + events: Final = tuple( + litellm.responses( + model="anthropic/claude-haiku-4-5", + input="What is the weather?", + tools=list(_RESPONSES_TOOLS), + tool_choice=_FLAT_ALLOWED_WEATHER, + drop_params=True, + stream=True, + api_key="sk-ant-fake", + ) + ) + + reported: Final = tuple( + event.response.tool_choice + for event in events + if event.type in (ResponsesAPIStreamEvents.RESPONSE_CREATED, ResponsesAPIStreamEvents.RESPONSE_COMPLETED) + ) + assert reported == ("auto", "auto") + + +def test_responses_through_chat_keeps_reporting_a_choice_the_provider_received(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("http://localhost:12346/v1/chat/completions").mock(return_value=_chat_answer()) + + response: Final = litellm.responses( + model="openai/gpt-5.5", + input="What is the weather?", + tools=list(_RESPONSES_TOOLS), + tool_choice=_FLAT_ALLOWED_WEATHER, + use_chat_completions_api=True, + api_base="http://localhost:12346/v1", + api_key="sk-fake", + ) + + assert json.loads(route.calls[0].request.content)["tool_choice"]["type"] == "allowed_tools" + assert response.tool_choice["type"] == "allowed_tools" + assert list(response.tool_choice["tools"]) == _FLAT_ALLOWED_WEATHER["tools"] diff --git a/tests/unit/responses/litellm_completion_transformation/test_handler.py b/tests/unit/responses/litellm_completion_transformation/test_handler.py index 4794f3701c5..1611225dfe9 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_handler.py @@ -332,6 +332,40 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h ] +@pytest.mark.asyncio +async def test_bridged_follow_up_after_a_client_tool_search_reports_auto_for_an_allowed_tools_choice_it_dropped(): + previous_response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="azure_ai", model_id="deployment-a", response_id="chatcmpl-turn-one" + ) + provider: Final = _RecordingAnthropicHandler(_ANTHROPIC_MESSAGE_PAYLOAD) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider)) + + with patch( + "litellm.proxy.proxy_server.prisma_client", + SimpleNamespace(db=_StoredTurns((_stored_mcp_turn(previous_response_id),))), + ): + response = await litellm.aresponses( + model="azure_ai/claude-sonnet-4-6", + api_base="https://fake-foundry-resource.services.ai.azure.com", + api_key="fake-api-key", + input="Double it", + tools=[ + {"type": "tool_search", "execution": "client"}, + {"type": "function", "name": "get_time", "parameters": {"type": "object", "properties": {}}}, + ], + tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + drop_params=True, + previous_response_id=previous_response_id, + client=client, + ) + + assert provider.request_body is not None, "the bridged turn never reached the provider" + assert "tool_choice" not in provider.request_body + assert isinstance(response, ResponsesAPIResponse) + assert response.tool_choice == "auto" + + @pytest.mark.asyncio @pytest.mark.parametrize("client_field", ["caller_input", "names_non_string_fields"]) async def test_bridged_follow_up_after_a_client_tool_search_with_a_client_field_named_like_a_bridge_keyword_reaches_the_provider( diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index a39bc09cee6..24f5173d6a3 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -5,7 +5,7 @@ import logging import sys import warnings from collections.abc import Callable, Iterator, Mapping, Sequence -from contextlib import contextmanager, nullcontext +from contextlib import AbstractContextManager, contextmanager, nullcontext from datetime import datetime from types import MappingProxyType, SimpleNamespace from typing import Any, Final, List, cast @@ -25,16 +25,18 @@ from litellm.integrations.langfuse.langfuse import langfuse_trace_id, langfuse_t from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.xai.common_utils import xai_reported_cost_in_usd from litellm.proxy.mcp_server import mcp_server_manager -from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk +from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing, ServerListOk from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator from litellm.types.llms.openai import ( OpenAIMcpServerTool, + ResponseCompletedEvent, ResponsesAPIResponse, ResponsesAPIStreamingResponse, ToolParam, ) +from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.types.utils import ModelResponse, StandardLoggingPayload, Usage @@ -925,7 +927,7 @@ async def test_mcp_parameter_preparation_helpers(): ), "Stream should be restored to original setting" assert ( "tool_choice" not in follow_up_params - ), "tool_choice should be removed for follow-up call" + ), "a tool_choice that is not allowed_tools should be removed for follow-up call" assert follow_up_params["temperature"] == 0.7, "Other params should be preserved" assert ( base_call_params["tool_choice"] == "required" @@ -1703,9 +1705,16 @@ async def _auto_execute_echo( return await _auto_execute(logging_obj, _ECHO_MCP_TOOL, "PING") -async def _auto_execute_through_mcp_server( - monkeypatch: pytest.MonkeyPatch, call_tool: AsyncMock, logging_obj: Logging -) -> None: +_DEMO_MCP_SERVER: Final = MCPServer( + server_id="demo", name="demo", server_name="demo", transport="http", url="https://demo.example/mcp", auth_type="none" +) + + +def _demo_server_named(name: str) -> MCPServer | None: + return _DEMO_MCP_SERVER if name == "demo" else None + + +def _served_by_a_recording_mcp_server(monkeypatch: pytest.MonkeyPatch, call_tool: AsyncMock) -> None: monkeypatch.setitem( sys.modules, "litellm.proxy.proxy_server", SimpleNamespace(proxy_logging_obj=None, prisma_client=None) ) @@ -1716,18 +1725,35 @@ async def _auto_execute_through_mcp_server( get_registry=MagicMock(return_value={}), call_tool=call_tool, get_mcp_server_from_tool_name=MagicMock(return_value=None), - get_mcp_server_by_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(side_effect=_demo_server_named), + get_allowed_mcp_servers=AsyncMock(return_value=[_DEMO_MCP_SERVER.server_id]), + get_mcp_servers_from_ids=MagicMock(return_value=[_DEMO_MCP_SERVER]), ), ) - echo_tool: Final = MCPTool.model_validate( - {"name": "echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}}, - by_name=False, + + +def _echo_tool_listed() -> AbstractContextManager[object]: + return patch( + "litellm.proxy.mcp_server.server._get_tools_from_mcp_servers", + AsyncMock( + return_value=AggregateToolListing( + tools=[ + MCPTool.model_validate( + {"name": "echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}}, + by_name=False, + ) + ], + outcomes={}, + ) + ), ) - with patch.object( - LiteLLM_Proxy_MCP_Handler, - "process_mcp_tools_without_openai_transform", - AsyncMock(return_value=([echo_tool], {"echo": "demo"})), - ): + + +async def _auto_execute_through_mcp_server( + monkeypatch: pytest.MonkeyPatch, call_tool: AsyncMock, logging_obj: Logging +) -> None: + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + with _echo_tool_listed(): await litellm.aresponses( model="openai/gpt-5.6", input="Call echo with PING", @@ -1739,6 +1765,281 @@ async def _auto_execute_through_mcp_server( ) +_ALLOWED_ONLY_ANOTHER_TOOL: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "another_tool"}], +} + + +async def _call_with_a_choice_excluding_echo(stream: bool) -> tuple[object, ...]: + with _echo_tool_listed(): + response: Final = await litellm.aresponses( + model="openai/gpt-5.6", + input="Call echo with PING", + tools=[dict(_ECHO_MCP_TOOL)], + tool_choice=_ALLOWED_ONLY_ANOTHER_TOOL, + api_key="sk-mcp-rounds", + api_base="https://mcp-rounds.example.com/v1", + stream=stream, + ) + return await _drain(response) if stream else (response,) + + +@pytest.mark.asyncio +async def test_an_mcp_tool_the_choice_excludes_never_runs_when_the_model_ignored_the_choice( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + _serve_rounds(respx_mock, monkeypatch, (_ROUND_ONE_BODY,)) + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + (response,) = await _call_with_a_choice_excluding_echo(stream=False) + + call_tool.assert_not_awaited() + assert isinstance(response, ResponsesAPIResponse) + assert response.status == "completed" + assert _output_types(response.output) == ("function_call",) + + +def _anthropic_call_to_echo() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "msg_echo", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "tool_use", "id": "toolu_echo", "name": "echo", "input": {"value": "PING"}}], + "stop_reason": "tool_use", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) + + +async def _call_echo_on_anthropic_under_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[httpx.Response], tool_choice: object +) -> tuple[object, respx.Route]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(side_effect=list(rounds)) + with _echo_tool_listed(): + response: Final = await litellm.aresponses( + model="anthropic/claude-haiku-4-5", + input="Call echo with PING", + tools=[dict(_ECHO_MCP_TOOL)], + tool_choice=tool_choice, + drop_params=True, + api_key="sk-ant-fake", + ) + return response, route + + +@pytest.mark.asyncio +async def test_a_dropped_choice_hands_back_an_mcp_call_it_excludes( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + response, route = await _call_echo_on_anthropic_under_drop_params( + respx_mock, monkeypatch, (_anthropic_call_to_echo(),), _ALLOWED_ONLY_ANOTHER_TOOL + ) + + call_tool.assert_not_awaited() + assert "tool_choice" not in json.loads(route.calls[0].request.content) + assert isinstance(response, ResponsesAPIResponse) + assert response.tool_choice == "auto" + assert _output_types(response.output) == ("function_call",) + + +def _anthropic_answer() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "msg_answer", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "PING back"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "auto", "tools": []}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "name": ["echo"]}]}, + ], +) +async def test_a_malformed_choice_under_drop_params_runs_an_mcp_call_as_if_never_sent( + tool_choice: dict[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + call_tool: Final = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="PING")])) + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + response, route = await _call_echo_on_anthropic_under_drop_params( + respx_mock, monkeypatch, (_anthropic_call_to_echo(), _anthropic_answer()), tool_choice + ) + + assert call_tool.await_count == 1 + assert all("tool_choice" not in json.loads(call.request.content) for call in route.calls) + assert isinstance(response, ResponsesAPIResponse) + assert "message" in _output_types(response.output) + + +@pytest.mark.asyncio +async def test_a_malformed_choice_without_drop_params_answers_400_before_listing_or_calling_a_model( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + listing: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, AsyncMock()) + monkeypatch.setattr("litellm.proxy.mcp_server.server._get_tools_from_mcp_servers", listing) + + with pytest.raises(litellm.BadRequestError, match="Invalid tool choice"): + await litellm.aresponses( + model="openai/gpt-5.6", + input="Call echo with PING", + tools=[dict(_ECHO_MCP_TOOL)], + tool_choice={"type": "allowed_tools", "mode": "auto", "tools": []}, + api_key="sk-mcp-rounds", + api_base="https://mcp-rounds.example.com/v1", + ) + + listing.assert_not_awaited() + assert not respx_mock.calls + + +_DEMO_LABELED_ECHO_TOOL: Final = MappingProxyType({**_ECHO_MCP_TOOL, "server_label": "demo"}) + + +async def _call_echo_with_the_demo_label(tool_choice: object, stream: bool) -> tuple[object, ...]: + with _echo_tool_listed(): + response: Final = await litellm.aresponses( + model="openai/gpt-5.6", + input="Call echo with PING", + tools=[dict(_DEMO_LABELED_ECHO_TOOL)], + tool_choice=tool_choice, + api_key="sk-mcp-rounds", + api_base="https://mcp-rounds.example.com/v1", + stream=stream, + ) + return await _drain(response) if stream else (response,) + + +@pytest.mark.asyncio +async def test_the_provider_gets_the_converted_function_in_place_of_a_gateway_mcp_entry( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + route: Final = _serve_rounds(respx_mock, monkeypatch, (_ROUND_ONE_BODY, _ROUND_TWO_BODY)) + call_tool: Final = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="PING")])) + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + await _call_echo_with_the_demo_label( + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "mcp", "server_label": "demo", "name": "echo"}]}, + stream=False, + ) + + assert json.loads(route.calls[0].request.content)["tool_choice"] == { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "function", "name": "echo"}], + } + assert call_tool.await_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_the_follow_up_after_a_gateway_call_keeps_the_converted_allowlist_without_forcing_a_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + route: Final = _serve_responses( + respx_mock, + monkeypatch, + tuple( + _streamed(body) if stream else httpx.Response(200, content=json.dumps(body, default=dict)) + for body in (_ROUND_ONE_BODY, _ROUND_TWO_BODY) + ), + ) + _served_by_a_recording_mcp_server( + monkeypatch, AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="PING")])) + ) + + await _call_echo_with_the_demo_label( + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "mcp", "server_label": "demo", "name": "echo"}]}, + stream=stream, + ) + + assert json.loads(route.calls[1].request.content)["tool_choice"] == { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "echo"}], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_a_gateway_mcp_entry_that_allows_no_converted_tool_sends_none_and_runs_nothing( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + route: Final = _serve_responses( + respx_mock, + monkeypatch, + (_streamed(_ROUND_ONE_BODY) if stream else httpx.Response(200, content=json.dumps(_ROUND_ONE_BODY, default=dict)),), + ) + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + await _call_echo_with_the_demo_label( + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "server_label": "demo", "name": "reverse"}]}, + stream=stream, + ) + + assert json.loads(route.calls[0].request.content)["tool_choice"] == "none" + call_tool.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_an_mcp_tool_the_choice_lists_still_runs(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + _serve_rounds(respx_mock, monkeypatch, (_ROUND_ONE_BODY, _ROUND_TWO_BODY)) + call_tool: Final = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="PING")])) + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + with _echo_tool_listed(): + await litellm.aresponses( + model="openai/gpt-5.6", + input="Call echo with PING", + tools=[dict(_ECHO_MCP_TOOL)], + tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "echo"}]}, + api_key="sk-mcp-rounds", + api_base="https://mcp-rounds.example.com/v1", + ) + + assert call_tool.await_count == 1 + + +@pytest.mark.asyncio +async def test_a_streamed_mcp_tool_the_choice_excludes_never_runs_when_the_model_ignored_the_choice( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + _serve_responses(respx_mock, monkeypatch, (_streamed(_ROUND_ONE_BODY),)) + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + events: Final = await _call_with_a_choice_excluding_echo(stream=True) + + call_tool.assert_not_awaited() + last: Final = events[-1] + assert isinstance(last, ResponseCompletedEvent) + assert _output_types(last.response.output) == ("function_call",) + assert not any("mcp_call" in str(getattr(event, "type", "")) for event in events) + + def _rounds_logging_obj(recorder: _RoundRecorder, call_id: str) -> Logging: return Logging( model="openai/gpt-5.6", diff --git a/tests/unit/responses/mcp/test_chat_completions_handler.py b/tests/unit/responses/mcp/test_chat_completions_handler.py index c0d32742e8a..aaeb30769fb 100644 --- a/tests/unit/responses/mcp/test_chat_completions_handler.py +++ b/tests/unit/responses/mcp/test_chat_completions_handler.py @@ -7,9 +7,9 @@ import threading import types import warnings from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from datetime import datetime -from types import MappingProxyType +from types import MappingProxyType, SimpleNamespace from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -17,6 +17,7 @@ import httpx import pytest import respx from httpx import Response +from mcp.types import CallToolResult, TextContent from mcp.types import Tool as MCPTool from starlette.responses import StreamingResponse from starlette.types import Message @@ -25,6 +26,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.responses.mcp import chat_completions_handler from litellm.responses.mcp.chat_completions_handler import ( acompletion_with_mcp, @@ -33,6 +35,7 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.types.utils import ModelResponse, ModelResponseStream, StandardLoggingPayload, Usage from litellm.utils import CustomStreamWrapper @@ -2744,3 +2747,371 @@ def test_stream_auto_execute_read_synchronously_and_dropped_after_its_loop_close assert [raised for raised in unraisable.raised if _names_chat_stream_billing(raised)] == [] assert [str(warning.message) for warning in caught if _names_chat_stream_billing(str(warning.message))] == [] + + +_MCP_ECHO_TOOL: Final = MCPTool.model_validate( + {"name": "echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}}, + by_name=False, +) +_FLAT_ALLOWED_ECHO: Final = {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "echo"}]} + + +_ALLOWED_ONLY_ANOTHER_TOOL: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "another_tool"}], +} + + +_DEMO_MCP_SERVER: Final = MCPServer( + server_id="demo", + name="demo", + server_name="demo", + transport="http", + url="https://demo.example/mcp", + auth_type="none", +) + + +def _demo_server_named(name: str) -> MCPServer | None: + return _DEMO_MCP_SERVER if name.lower() == "demo" else None + + +def _served_by_a_recording_mcp_server( + monkeypatch: pytest.MonkeyPatch, + call_tool: AsyncMock, + listed: MCPTool = _MCP_ECHO_TOOL, + catalog: SimpleNamespace = SimpleNamespace(operation=nullcontext), +) -> None: + monkeypatch.setattr( + "litellm.proxy.mcp_server.server._get_tools_from_mcp_servers", + AsyncMock(return_value=AggregateToolListing(tools=[listed], outcomes={})), + ) + monkeypatch.setitem( + sys.modules, "litellm.proxy.proxy_server", SimpleNamespace(proxy_logging_obj=None, prisma_client=None) + ) + monkeypatch.setattr( + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", + SimpleNamespace( + catalog=catalog, + get_registry=MagicMock(return_value={}), + call_tool=call_tool, + get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(side_effect=_demo_server_named), + get_allowed_mcp_servers=AsyncMock(return_value=[_DEMO_MCP_SERVER.server_id]), + get_mcp_servers_from_ids=MagicMock(return_value=[_DEMO_MCP_SERVER]), + ), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_an_mcp_tool_the_choice_excludes_never_runs_when_the_model_ignored_the_choice( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE) if stream else _ok(_CHAT_ROUND_ONE),)) + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + response: Final = await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + tool_choice=_ALLOWED_ONLY_ANOTHER_TOOL, + api_key="sk-mcp-chat-rounds", + api_base="https://mcp-chat-rounds.example.com/v1", + stream=stream, + ) + chunks: Final = await _read_all(response) if stream else () + + handed_back: Final = ( + tuple(call.function.name for chunk in chunks for call in chunk.choices[0].delta.tool_calls or ()) + if stream + else tuple(call.function.name for call in response.choices[0].message.tool_calls) + ) + assert handed_back == ("echo",) + call_tool.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_a_required_choice_asks_the_follow_up_in_auto_mode_for_the_same_tools_once_its_mcp_call_ran( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +): + rounds: Final = (_CHAT_ROUND_ONE, _CHAT_ROUND_TWO) + _serve_chat_rounds(respx_mock, monkeypatch, tuple(_streamed(body) if stream else _ok(body) for body in rounds)) + call_tool: Final = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="PING")])) + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + response: Final = await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + tool_choice={"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "echo"}]}, + api_key="sk-mcp-chat-rounds", + api_base="https://mcp-chat-rounds.example.com/v1", + stream=stream, + ) + if stream: + await _read_all(response) + + first, follow_up = (json.loads(call.request.content)["tool_choice"] for call in respx_mock.calls) + assert call_tool.await_count == 1 + assert first["allowed_tools"]["mode"] == "required" + assert follow_up == {**first, "allowed_tools": {**first["allowed_tools"], "mode": "auto"}} + + +def _anthropic_call_to_echo(name: str = "echo") -> Response: + return Response( + 200, + json={ + "id": "msg_echo", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "tool_use", "id": "toolu_echo", "name": name, "input": {"value": "PING"}}], + "stop_reason": "tool_use", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) + + +def _anthropic_answer() -> Response: + return Response( + 200, + json={ + "id": "msg_answer", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "PING back"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) + + +_WHOLE_DEMO_SERVER: Final = {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "server_label": "demo"}]} +_WHOLE_OTHER_SERVER: Final = {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "server_label": "other"}]} + + +async def _call_echo_on_anthropic_under_drop_params( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + rounds: Sequence[Response], + tool_choice: object, + server_url: str = "litellm_proxy/mcp/demo", +) -> tuple[ModelResponse, respx.Route]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(side_effect=list(rounds)) + response: Final = await litellm.acompletion( + model="anthropic/claude-haiku-4-5", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_label": "demo", "server_url": server_url, "require_approval": "never"}], + tool_choice=tool_choice, + drop_params=True, + api_key="sk-ant-fake", + ) + return response, route + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tool_choice", [_ALLOWED_ONLY_ANOTHER_TOOL, _WHOLE_OTHER_SERVER]) +async def test_a_dropped_choice_hands_back_an_mcp_call_it_excludes( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tool_choice: object +): + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + response, route = await _call_echo_on_anthropic_under_drop_params( + respx_mock, monkeypatch, (_anthropic_call_to_echo(),), tool_choice + ) + + call_tool.assert_not_awaited() + assert "tool_choice" not in json.loads(route.calls[0].request.content) + assert [call.function.name for call in response.choices[0].message.tool_calls] == ["echo"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tool_choice", "server_url"), + [ + (_FLAT_ALLOWED_ECHO, "litellm_proxy/mcp/demo"), + (_WHOLE_DEMO_SERVER, "litellm_proxy/mcp/demo"), + (_WHOLE_DEMO_SERVER, "litellm_proxy"), + ], +) +async def test_a_dropped_choice_still_runs_an_mcp_call_it_lists( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tool_choice: object, server_url: str +): + call_tool: Final = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="PING")])) + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + response, _route = await _call_echo_on_anthropic_under_drop_params( + respx_mock, monkeypatch, (_anthropic_call_to_echo(), _anthropic_answer()), tool_choice, server_url + ) + + assert call_tool.await_count == 1 + assert response.choices[0].message.content == "PING back" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "auto", "tools": []}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "name": ["echo"]}]}, + ], +) +async def test_a_malformed_choice_under_drop_params_runs_an_mcp_call_as_if_never_sent( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tool_choice: object +): + call_tool: Final = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="PING")])) + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + response, route = await _call_echo_on_anthropic_under_drop_params( + respx_mock, monkeypatch, (_anthropic_call_to_echo(), _anthropic_answer()), tool_choice + ) + + assert call_tool.await_count == 1 + assert response.choices[0].message.content == "PING back" + assert all("tool_choice" not in json.loads(call.request.content) for call in route.calls) + + +_DEMO_ECHO_TOOL: Final = MCPTool.model_validate( + {"name": "demo-echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}}, + by_name=False, +) +_DEMO_IN_AN_ACCESS_GROUP: Final = _DEMO_MCP_SERVER.model_copy(update={"access_groups": ["demo_group"]}) + + +def _choice_of(*tools: Mapping[str, object]) -> Mapping[str, object]: + return {"type": "allowed_tools", "mode": "auto", "tools": list(tools)} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tool_choice", "server_url", "listed"), + [ + (_choice_of({"type": "mcp", "server_label": "demo", "name": "echo"}), "litellm_proxy/mcp/demo", _DEMO_ECHO_TOOL), + ( + _choice_of({"type": "mcp", "server_label": "demo", "name": "demo-echo"}), + "litellm_proxy/mcp/demo", + _DEMO_ECHO_TOOL, + ), + (_WHOLE_DEMO_SERVER, "litellm_proxy/mcp/DEMO", _MCP_ECHO_TOOL), + (_WHOLE_DEMO_SERVER, "litellm_proxy/mcp/demo_group", _MCP_ECHO_TOOL), + ], +) +async def test_an_mcp_entry_allows_the_tool_the_gateway_resolves_for_its_label_and_name( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + tool_choice: object, + server_url: str, + listed: MCPTool, +): + call_tool: Final = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="PING")])) + _served_by_a_recording_mcp_server( + monkeypatch, + call_tool, + listed, + SimpleNamespace( + operation=nullcontext, + current=lambda: SimpleNamespace(servers={_DEMO_MCP_SERVER.server_id: _DEMO_IN_AN_ACCESS_GROUP}), + ), + ) + + response, _route = await _call_echo_on_anthropic_under_drop_params( + respx_mock, monkeypatch, (_anthropic_call_to_echo(listed.name), _anthropic_answer()), tool_choice, server_url + ) + + assert call_tool.await_count == 1 + assert response.choices[0].message.content == "PING back" + + +@pytest.mark.asyncio +async def test_a_named_mcp_entry_allows_no_other_tool_on_its_server( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool, _DEMO_ECHO_TOOL) + + response, _route = await _call_echo_on_anthropic_under_drop_params( + respx_mock, + monkeypatch, + (_anthropic_call_to_echo("demo-echo"),), + _choice_of({"type": "mcp", "server_label": "demo", "name": "reverse"}), + ) + + call_tool.assert_not_awaited() + assert [call.function.name for call in response.choices[0].message.tool_calls] == ["demo-echo"] + + +async def _call_echo_on_openai_with_the_demo_server( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response], tool_choice: object +) -> tuple[object, ...]: + _serve_chat_rounds(respx_mock, monkeypatch, rounds) + await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[ + { + "type": "mcp", + "server_label": "demo", + "server_url": "litellm_proxy/mcp/demo", + "require_approval": "never", + }, + {"type": "function", "function": {"name": "get_time", "parameters": {"type": "object"}}}, + ], + tool_choice=tool_choice, + api_key="sk-mcp-chat-rounds", + api_base="https://mcp-chat-rounds.example.com/v1", + ) + return tuple(json.loads(call.request.content).get("tool_choice") for call in respx_mock.calls) + + +@pytest.mark.asyncio +async def test_the_provider_gets_the_converted_function_in_place_of_a_gateway_mcp_entry( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + call_tool: Final = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="PING")])) + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + sent: Final = await _call_echo_on_openai_with_the_demo_server( + respx_mock, + monkeypatch, + (_ok(_CHAT_ROUND_ONE), _ok(_CHAT_ROUND_TWO)), + _choice_of({"type": "mcp", "server_label": "demo"}, {"type": "function", "name": "get_time"}), + ) + + assert sent[0] == { + "type": "allowed_tools", + "allowed_tools": { + "mode": "auto", + "tools": [ + {"type": "function", "function": {"name": "get_time"}}, + {"type": "function", "function": {"name": "echo"}}, + ], + }, + } + assert call_tool.await_count == 1 + + +@pytest.mark.asyncio +async def test_a_gateway_mcp_entry_that_allows_no_converted_tool_sends_none_and_runs_nothing( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + call_tool: Final = AsyncMock() + _served_by_a_recording_mcp_server(monkeypatch, call_tool) + + sent: Final = await _call_echo_on_openai_with_the_demo_server( + respx_mock, + monkeypatch, + (_ok(_CHAT_ROUND_ONE),), + _choice_of({"type": "mcp", "server_label": "demo", "name": "reverse"}), + ) + + assert sent == ("none",) + call_tool.assert_not_awaited() diff --git a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py index 44ba9007d92..537fb672e16 100644 --- a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -1725,6 +1725,14 @@ TOOLSET_ROWS: Final = ( ), MCPToolset(toolset_id="ts-shouty", toolset_name="CALC", tools=[{"server_id": "srv-2", "tool_name": "lookup"}]), MCPToolset(toolset_id="ts-group", toolset_name="reference", tools=[{"server_id": "srv-1", "tool_name": "add"}]), + MCPToolset( + toolset_id="ts-prefixed", + toolset_name="prefixed-toolset", + tools=[{"server_id": "srv-1", "tool_name": "add"}, {"server_id": "srv-1", "tool_name": "calc-subtract"}], + ), + MCPToolset( + toolset_id="ts-subtract", toolset_name="subtract-toolset", tools=[{"server_id": "srv-1", "tool_name": "subtract"}] + ), ) GATEWAY_SERVERS: Final = ( MCPServer(server_id="srv-1", name="calc", alias="calc", transport=MCPTransport.http, access_groups=["wiki"]), @@ -1753,9 +1761,9 @@ def _toolset_table(lookup_error: Exception | None = None) -> types.SimpleNamespa return types.SimpleNamespace(find_first=find_first, find_many=find_many) -def _team_grants(toolset_id: str) -> GrantedToolsetIds: +def _team_grants(*toolset_ids: str) -> GrantedToolsetIds: async def team_permission(context: UserAPIKeyAuth) -> LiteLLM_ObjectPermissionTable: - return LiteLLM_ObjectPermissionTable(object_permission_id="op-team", mcp_toolsets=[toolset_id]) + return LiteLLM_ObjectPermissionTable(object_permission_id="op-team", mcp_toolsets=list(toolset_ids)) async def granted_through_team(context: UserAPIKeyAuth) -> frozenset[str]: return await granted_toolset_ids(context, team_object_permission=team_permission, require_key_access=False) @@ -1767,19 +1775,20 @@ async def _no_toolset_grants(context: UserAPIKeyAuth) -> frozenset[str]: return frozenset() -async def _tools_listed_through_the_gateway( +async def _listing_through_the_gateway( monkeypatch: pytest.MonkeyPatch, user_api_key_auth: UserAPIKeyAuth, - server_urls: tuple[str, ...], + mcp_tools: Sequence[Mapping[str, object]], granted_toolsets: GrantedToolsetIds, toolset_table: types.SimpleNamespace | None = None, agent_permission: LiteLLM_ObjectPermissionTable | None = None, -) -> list[str]: + servers: Sequence[MCPServer] = GATEWAY_SERVERS, +) -> tuple[list[MCPTool], list[str]]: async def upstream_listing(_client: object, server_name: str) -> list[MCPTool]: return list(UPSTREAM_TOOLS[server_name]) manager: Final = MCPServerManager() - manager.registry = {server.server_id: server for server in GATEWAY_SERVERS} + manager.registry = {server.server_id: server for server in servers} manager.config_mcp_servers = dict(manager.registry) manager.create_mcp_client = AsyncMock(return_value=object()) manager._fetch_tools_with_timeout = AsyncMock(side_effect=upstream_listing) @@ -1800,17 +1809,269 @@ async def _tools_listed_through_the_gateway( monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"supported_db_objects": []}) try: - tools, _server_names = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager( + return await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager( user_api_key_auth=user_api_key_auth, - mcp_tools_with_litellm_proxy=[{"type": "mcp", "server_url": url} for url in server_urls], + mcp_tools_with_litellm_proxy=mcp_tools, granted_toolsets=granted_toolsets, ) finally: - for server in GATEWAY_SERVERS: + for server in servers: manager._drop_listed_tools(server.server_id) + + +async def _tools_listed_through_the_gateway( + monkeypatch: pytest.MonkeyPatch, + user_api_key_auth: UserAPIKeyAuth, + server_urls: tuple[str, ...], + granted_toolsets: GrantedToolsetIds, + toolset_table: types.SimpleNamespace | None = None, + agent_permission: LiteLLM_ObjectPermissionTable | None = None, +) -> list[str]: + tools, _server_names = await _listing_through_the_gateway( + monkeypatch, + user_api_key_auth, + [{"type": "mcp", "server_url": url} for url in server_urls], + granted_toolsets, + toolset_table, + agent_permission, + ) return sorted(tool.name for tool in tools) +async def _calls_the_gateway_runs( + monkeypatch: pytest.MonkeyPatch, + labeled_server_urls: Mapping[str, str], + allowed_label: str, + called: tuple[str, ...], + unresolvable_tools: Mapping[str, str] = MappingProxyType({}), +) -> tuple[str, ...]: + return await _calls_the_entries_run( + monkeypatch, + UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=LitellmUserRoles.INTERNAL_USER), + [{"type": "mcp", "server_label": label, "server_url": url} for label, url in labeled_server_urls.items()], + allowed_label, + called, + unresolvable_tools, + ) + + +async def _calls_the_entries_run( + monkeypatch: pytest.MonkeyPatch, + user: UserAPIKeyAuth, + mcp_tools: Sequence[Mapping[str, object]], + allowed_label: str, + called: tuple[str, ...], + unresolvable_tools: Mapping[str, str] = MappingProxyType({}), + servers: Sequence[MCPServer] = GATEWAY_SERVERS, + granted_toolsets: tuple[str, ...] = ("ts-granted",), +) -> tuple[str, ...]: + tools, server_names = await _listing_through_the_gateway( + monkeypatch, user, mcp_tools, _team_grants(*granted_toolsets), servers=servers + ) + _deduplicated, tool_server_map = LiteLLM_Proxy_MCP_Handler._deduplicate_mcp_tools(tools, server_names) + tool_choice: Final = {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "server_label": allowed_label}]} + gateway: Final = await LiteLLM_Proxy_MCP_Handler.gateway_tools( + tool_choice, {**tool_server_map, **unresolvable_tools}, mcp_tools, user, _team_grants(*granted_toolsets) + ) + return tuple( + name + for name in called + if LiteLLM_Proxy_MCP_Handler.tool_calls_to_run( + ({"type": "function_call", "name": name, "call_id": f"call_{name}", "arguments": "{}"},), + tool_choice, + gateway, + ) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("labeled_server_urls", "allowed_label", "runs"), + [ + ({"team": "litellm_proxy/mcp/team-toolset", "docs": "litellm_proxy/mcp/docs"}, "team", ("calc-add",)), + ({"team": "litellm_proxy/mcp/team-toolset", "docs": "litellm_proxy/mcp/docs"}, "docs", ("docs-lookup",)), + ({"team": "litellm_proxy/mcp/team-toolset", "other": "litellm_proxy/mcp/other-toolset"}, "other", ()), + ], + ids=["granted_toolset", "server_beside_a_toolset", "ungranted_toolset"], +) +async def test_an_mcp_entry_allows_only_the_tools_its_labels_toolset_or_server_serves( + monkeypatch: pytest.MonkeyPatch, + labeled_server_urls: Mapping[str, str], + allowed_label: str, + runs: tuple[str, ...], +): + ran: Final = await _calls_the_gateway_runs( + monkeypatch, labeled_server_urls, allowed_label, ("calc-add", "docs-lookup") + ) + + assert ran == runs + + +_READER_AND_WRITER_ON_CALC: Final = ( + {"type": "mcp", "server_label": "reader", "server_url": "litellm_proxy/mcp/calc", "allowed_tools": ["add"]}, + {"type": "mcp", "server_label": "writer", "server_url": "litellm_proxy/mcp/calc", "allowed_tools": ["subtract"]}, +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("entries", "allowed_label", "runs"), + [ + (_READER_AND_WRITER_ON_CALC, "reader", ("calc-add",)), + (_READER_AND_WRITER_ON_CALC, "writer", ("calc-subtract",)), + ( + ({"type": "mcp", "server_label": "all", "server_url": "litellm_proxy", "allowed_tools": ["add"]},), + "all", + ("calc-add",), + ), + ], + ids=["reader_on_a_shared_server", "writer_on_a_shared_server", "whole_gateway"], +) +async def test_an_mcp_entry_allows_only_the_tools_its_own_allowed_tools_lists( + monkeypatch: pytest.MonkeyPatch, + entries: tuple[Mapping[str, object], ...], + allowed_label: str, + runs: tuple[str, ...], +): + ran: Final = await _calls_the_entries_run( + monkeypatch, + _key_holding("srv-1"), + entries, + allowed_label, + ("calc-add", "calc-subtract", "ghost-remove"), + MappingProxyType({"ghost-remove": "ghost"}), + ) + + assert ran == runs + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("allowed_tools", "runs"), [(["add"], ("calc-add",)), (["subtract"], ())]) +async def test_a_toolset_entry_allows_only_the_toolset_tools_its_own_allowed_tools_lists( + monkeypatch: pytest.MonkeyPatch, allowed_tools: list[str], runs: tuple[str, ...] +): + ran: Final = await _calls_the_entries_run( + monkeypatch, + UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=LitellmUserRoles.INTERNAL_USER), + ( + { + "type": "mcp", + "server_label": "team", + "server_url": "litellm_proxy/mcp/team-toolset", + "allowed_tools": allowed_tools, + }, + ), + "team", + ("calc-add", "calc-subtract"), + ) + + assert ran == runs + + +@pytest.mark.asyncio +async def test_a_toolset_label_allows_no_tool_whose_prefixed_name_only_matches_a_tool_the_toolset_grants( + monkeypatch: pytest.MonkeyPatch, +): + ran: Final = await _calls_the_entries_run( + monkeypatch, + UserAPIKeyAuth(api_key="sk-team", team_id="team-1", user_role=LitellmUserRoles.INTERNAL_USER), + ( + {"type": "mcp", "server_label": "team", "server_url": "litellm_proxy/mcp/prefixed-toolset"}, + {"type": "mcp", "server_label": "writer", "server_url": "litellm_proxy/mcp/subtract-toolset"}, + ), + "team", + ("calc-add", "calc-subtract"), + granted_toolsets=("ts-prefixed", "ts-subtract"), + ) + + assert ran == ("calc-add",) + + +@pytest.mark.asyncio +async def test_an_mcp_entry_allows_a_tool_its_allowed_tools_names_by_display_name(monkeypatch: pytest.MonkeyPatch): + calc_showing_add_as_plus: Final = GATEWAY_SERVERS[0].model_copy( + update={"tool_name_to_display_name": {"add": "plus"}} + ) + + ran: Final = await _calls_the_entries_run( + monkeypatch, + _key_holding("srv-1"), + ({"type": "mcp", "server_label": "reader", "server_url": "litellm_proxy/mcp/calc", "allowed_tools": ["plus"]},), + "reader", + ("plus", "calc-subtract"), + servers=(calc_showing_add_as_plus, GATEWAY_SERVERS[1]), + ) + + assert ran == ("plus",) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("server_url", ["litellm_proxy/mcp/calc", "litellm_proxy"], ids=["one_server", "whole_gateway"]) +async def test_an_mcp_entry_allows_no_tool_whose_hyphenated_display_name_ends_in_a_name_it_lists( + monkeypatch: pytest.MonkeyPatch, server_url: str +): + calc_showing_add_as_not_subtract: Final = GATEWAY_SERVERS[0].model_copy( + update={"tool_name_to_display_name": {"add": "not-subtract"}} + ) + + ran: Final = await _calls_the_entries_run( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ({"type": "mcp", "server_label": "reader", "server_url": server_url, "allowed_tools": ["subtract"]},), + "reader", + ("not-subtract", "calc-subtract"), + servers=(calc_showing_add_as_not_subtract, GATEWAY_SERVERS[1]), + ) + + assert ran == ("calc-subtract",) + + +@pytest.mark.asyncio +async def test_an_mcp_entry_allows_no_tool_whose_display_name_carries_a_server_prefix_before_a_name_it_lists( + monkeypatch: pytest.MonkeyPatch, +): + calc_showing_add_as_srv_1_subtract: Final = GATEWAY_SERVERS[0].model_copy( + update={"tool_name_to_display_name": {"add": "srv-1-subtract"}} + ) + + ran: Final = await _calls_the_entries_run( + monkeypatch, + _key_holding("srv-1", "srv-2"), + ( + { + "type": "mcp", + "server_label": "reader", + "server_url": "litellm_proxy/mcp/calc", + "allowed_tools": ["subtract"], + }, + { + "type": "mcp", + "server_label": "writer", + "server_url": "litellm_proxy/mcp/calc", + "allowed_tools": ["srv-1-subtract"], + }, + ), + "reader", + ("srv-1-subtract", "calc-subtract"), + servers=(calc_showing_add_as_srv_1_subtract, GATEWAY_SERVERS[1]), + ) + + assert ran == ("calc-subtract",) + + +@pytest.mark.asyncio +async def test_an_mcp_entry_allows_no_call_on_a_server_the_gateway_cannot_resolve(monkeypatch: pytest.MonkeyPatch): + ran: Final = await _calls_the_gateway_runs( + monkeypatch, + {"team": "litellm_proxy/mcp/team-toolset"}, + "team", + ("calc-add", "ghost-add"), + MappingProxyType({"ghost-add": "ghost"}), + ) + + assert ran == ("calc-add",) + + def _key_holding(*server_ids: str, role: LitellmUserRoles = LitellmUserRoles.INTERNAL_USER) -> UserAPIKeyAuth: return UserAPIKeyAuth( api_key="sk-team", diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index f6084529ed6..e483b180129 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -305,6 +305,44 @@ class TestUseResponsesApiBridgeFlag: assert request_body["messages"] == [{"role": "user", "content": "Hello"}] assert response.output[0].content[0].text == "Answer" + def test_bridge_sends_a_responses_layout_allowed_tools_tool_choice_in_the_chat_layout( + self, respx_mock: respx.MockRouter + ): + upstream: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + litellm.responses( + model="openai/my-custom-model", + input="What time is it?", + use_chat_completions_api=True, + tools=[ + {"type": "function", "name": name, "parameters": {"type": "object", "properties": {}}} + for name in ("get_time", "get_weather") + ], + tool_choice={"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]}, + api_key="fake-provider-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read())["tool_choice"] == { + "type": "allowed_tools", + "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, + } + def test_bridge_merges_instructions_and_developer_input_for_databricks(self, respx_mock: respx.MockRouter): upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( return_value=httpx.Response( diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index d2035b6a682..b300562b029 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -557,15 +557,21 @@ _ALLOWED_TOOLS_TOOL_CHOICE: Final = { "type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, } +_RESPONSES_LAYOUT_ALLOWED_TOOLS_TOOL_CHOICE: Final = { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "function", "name": "get_time"}], +} _TWO_TOOLS: Final = [ {"type": "function", "function": {"name": "get_time", "parameters": {"type": "object", "properties": {}}}}, {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}}, ] +@pytest.mark.parametrize("tool_choice", [_ALLOWED_TOOLS_TOOL_CHOICE, _RESPONSES_LAYOUT_ALLOWED_TOOLS_TOOL_CHOICE]) @pytest.mark.parametrize("model", ["openai/gpt-5.5", "azure/my-gpt-deployment"]) def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat( - respx_mock: respx.MockRouter, openai_api_response: dict[str, object], model: str + respx_mock: respx.MockRouter, openai_api_response: dict[str, object], model: str, tool_choice: dict[str, object] ): mock_route: Final = respx_mock.post(url__regex=r"http://localhost:12346/.*chat/completions.*").mock( return_value=httpx.Response(status_code=200, json=openai_api_response) @@ -575,7 +581,7 @@ def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat model=model, messages=[{"role": "user", "content": "What time is it?"}], tools=_TWO_TOOLS, - tool_choice=_ALLOWED_TOOLS_TOOL_CHOICE, + tool_choice=tool_choice, api_base="http://localhost:12346/v1", api_key="fake_api_key", api_version="2025-04-01-preview", @@ -587,10 +593,10 @@ def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat assert [tool["function"]["name"] for tool in sent["tools"]] == ["get_time", "get_weather"] -@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("tool_choice", [_ALLOWED_TOOLS_TOOL_CHOICE, _RESPONSES_LAYOUT_ALLOWED_TOOLS_TOOL_CHOICE]) @pytest.mark.parametrize("model", ["anthropic/claude-sonnet-4-5", "gemini/gemini-2.5-flash", "bedrock/converse/x"]) def test_completion_allowed_tools_tool_choice_is_a_400_where_the_provider_cannot_enforce_it( - respx_mock: respx.MockRouter, model: str, drop_params: bool + respx_mock: respx.MockRouter, model: str, tool_choice: dict[str, object] ): with pytest.raises( litellm.BadRequestError, match=r"allowed_tools.*only supported for azure, litellm_proxy, openai" @@ -599,8 +605,7 @@ def test_completion_allowed_tools_tool_choice_is_a_400_where_the_provider_cannot model=model, messages=[{"role": "user", "content": "What time is it?"}], tools=_TWO_TOOLS, - tool_choice=_ALLOWED_TOOLS_TOOL_CHOICE, - drop_params=drop_params, + tool_choice=tool_choice, api_key="fake_api_key", aws_access_key_id="fake", aws_secret_access_key="fake", @@ -610,6 +615,64 @@ def test_completion_allowed_tools_tool_choice_is_a_400_where_the_provider_cannot assert not respx_mock.calls +@pytest.mark.parametrize( + ("model", "tool_choice"), + [ + ("hosted_vllm/my-model", _ALLOWED_TOOLS_TOOL_CHOICE), + ("hosted_vllm/my-model", _RESPONSES_LAYOUT_ALLOWED_TOOLS_TOOL_CHOICE), + ("openai/gpt-5.5", {"type": "allowed_tools", "mode": "required", "tools": []}), + ("openai/gpt-5.5", {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function"}]}), + ], +) +def test_completion_drop_params_drops_an_allowed_tools_tool_choice_it_cannot_send( + respx_mock: respx.MockRouter, openai_api_response: dict[str, object], model: str, tool_choice: dict[str, object] +): + mock_route: Final = respx_mock.post(url__regex=r"http://localhost:12346/.*chat/completions.*").mock( + return_value=httpx.Response(status_code=200, json=openai_api_response) + ) + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "What time is it?"}], + tools=_TWO_TOOLS, + tool_choice=tool_choice, + drop_params=True, + api_base="http://localhost:12346/v1", + api_key="fake_api_key", + ) + + assert mock_route.called + sent: Final = json.loads(respx_mock.calls[0].request.content) + assert "tool_choice" not in sent + assert [tool["function"]["name"] for tool in sent["tools"]] == ["get_time", "get_weather"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "required", "tools": []}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "name": ["get_time"]}]}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "mcp", "name": ["get_time"]}]}}, + ], +) +async def test_a_malformed_allowed_tools_tool_choice_is_a_400_on_mcp_gateway_tools_without_drop_params( + tool_choice: dict[str, object], respx_mock: respx.MockRouter +): + with pytest.raises(litellm.BadRequestError, match="Invalid tool choice") as exc: + await litellm.acompletion( + model="openai/gpt-5.5", + messages=[{"role": "user", "content": "What time is it?"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + tool_choice=tool_choice, + api_base="http://localhost:12346/v1", + api_key="fake_api_key", + ) + + assert exc.value.status_code == 400 + assert not respx_mock.calls + + _HOSTED_TOOL_CHOICE: Final = {"type": "web_search"} _HOSTED_TOOLS: Final[list[dict[str, object]]] = [*_TWO_TOOLS, {"type": "web_search"}] @@ -673,6 +736,34 @@ def test_completion_hosted_tool_choice_is_a_400_where_the_provider_cannot_run_it assert not respx_mock.calls +def test_drop_params_drops_the_choice_beside_a_tool_the_provider_runs(respx_mock: respx.MockRouter): + route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "done"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + ) + + litellm.completion( + model="anthropic/claude-haiku-4-5", + messages=[{"role": "user", "content": "What time is it?"}], + tools=[{"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com/sse"}], + tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + drop_params=True, + api_key="sk-ant-fake", + ) + + assert "tool_choice" not in json.loads(route.calls[0].request.content) + + _CLAUDE_TOOL_SEARCH_CALL: Final = { "id": "srvtoolu_01search", "type": "function", diff --git a/tests/unit/test_tool_loop.py b/tests/unit/test_tool_loop.py index de8f62803cc..f56a88e8899 100644 --- a/tests/unit/test_tool_loop.py +++ b/tests/unit/test_tool_loop.py @@ -11,6 +11,7 @@ from litellm.types.llms.openai import ChatCompletionToolMessage from litellm.types.utils import ChatCompletionMessageToolCall OPENAI_CHAT_COMPLETIONS_URL: Final = "https://api.openai.com/v1/chat/completions" +ANTHROPIC_MESSAGES_URL: Final = "https://api.anthropic.com/v1/messages" WEATHER_TOOLS: Final = ( { "type": "function", @@ -248,6 +249,219 @@ def test_stream_rejected_before_any_request(respx_mock: respx.MockRouter) -> Non assert route.call_count == 0 +_ALLOWED_WEATHER_ONLY: Final = { + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "get_weather"}], +} +_EMPTY_ALLOWED_TOOLS: Final = {"type": "allowed_tools", "mode": "auto", "tools": []} +_TIME_CALL: Final = _tool_call("call_1", "get_time", {}) + + +def _anthropic_message(*content: dict[str, object]) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "msg_tool_loop", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": list(content), + "stop_reason": "tool_use" if content[0]["type"] == "tool_use" else "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) + + +def _anthropic_tool_use(name: str) -> httpx.Response: + return _anthropic_message({"type": "tool_use", "id": "toolu_1", "name": name, "input": {}}) + + +def _tools_run_before_the_refusal( + tool_choice: dict[str, object], **completion_kwargs: object +) -> list[ChatCompletionMessageToolCall]: + executed: Final[list[ChatCompletionMessageToolCall]] = [] + + def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executed.append(tool_call) + return _tool_result(tool_call) + + with pytest.raises(litellm.BadRequestError, match="LiteLLM did not run the call"): + litellm.run_tool_loop( + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=execute, + tool_choice=tool_choice, + api_key="sk-test", + **completion_kwargs, + ) + return executed + + +def test_a_tool_the_dropped_choice_excludes_never_runs(respx_mock: respx.MockRouter) -> None: + respx_mock.post(ANTHROPIC_MESSAGES_URL).mock(return_value=_anthropic_tool_use("get_time")) + + assert not _tools_run_before_the_refusal( + _ALLOWED_WEATHER_ONLY, model="anthropic/claude-haiku-4-5", drop_params=True + ) + assert "tool_choice" not in _request_bodies(respx_mock)[0] + + +def test_a_tool_the_dropped_choice_lists_still_runs(respx_mock: respx.MockRouter) -> None: + respx_mock.post(ANTHROPIC_MESSAGES_URL).mock( + side_effect=[_anthropic_tool_use("get_weather"), _anthropic_message({"type": "text", "text": "done"})] + ) + + assert ( + litellm.run_tool_loop( + model="anthropic/claude-haiku-4-5", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + tool_choice=_ALLOWED_WEATHER_ONLY, + drop_params=True, + api_key="sk-test", + ) + == "done" + ) + + +def test_a_tool_the_choice_excludes_never_runs_when_the_model_ignored_the_choice( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response(None, [_TIME_CALL])) + ) + + assert not _tools_run_before_the_refusal(_ALLOWED_WEATHER_ONLY, model="openai/gpt-5-mini") + + +@pytest.mark.parametrize( + "tool_choice", + [_EMPTY_ALLOWED_TOOLS, {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "name": ["get_time"]}]}], +) +def test_a_malformed_choice_is_refused_before_any_request( + tool_choice: dict[str, object], respx_mock: respx.MockRouter +) -> None: + with pytest.raises(litellm.BadRequestError, match="Invalid tool choice"): + litellm.run_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + tool_choice=tool_choice, + api_key="sk-test", + ) + + assert not respx_mock.calls + + +@pytest.mark.parametrize( + "tool_choice", + [_EMPTY_ALLOWED_TOOLS, {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "mcp", "name": ["get_time"]}]}], +) +def test_a_malformed_choice_under_drop_params_runs_the_loop_as_if_never_sent( + tool_choice: dict[str, object], respx_mock: respx.MockRouter +) -> None: + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + side_effect=[ + httpx.Response(200, json=_openai_response(None, [_TIME_CALL])), + httpx.Response(200, json=_openai_response("done")), + ] + ) + executed: Final[list[str | None]] = [] + + def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executed.append(tool_call.function.name) + return _tool_result(tool_call) + + answer: Final = litellm.run_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=execute, + tool_choice=tool_choice, + drop_params=True, + api_key="sk-test", + ) + + assert (answer, executed) == ("done", ["get_time"]) + assert all("tool_choice" not in body for body in _request_bodies(respx_mock)) + + +def test_a_choice_on_a_model_alias_reaches_the_loop( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "model_alias_map", {"the-alias": "openai/gpt-5-mini"}) + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response("done")) + ) + + assert ( + litellm.run_tool_loop( + model="the-alias", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + tool_choice=_ALLOWED_WEATHER_ONLY, + api_key="sk-test", + ) + == "done" + ) + + +@pytest.mark.asyncio +async def test_async_loop_never_runs_a_tool_the_choice_excludes( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler()) + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response(None, [_TIME_CALL])) + ) + executed: Final[list[ChatCompletionMessageToolCall]] = [] + + async def execute(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executed.append(tool_call) + return _tool_result(tool_call) + + with pytest.raises(litellm.BadRequestError, match="LiteLLM did not run the call"): + await litellm.arun_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=execute, + tool_choice=_ALLOWED_WEATHER_ONLY, + drop_params=True, + api_key="sk-test", + ) + + assert not executed + + +def test_allowed_tools_choice_the_provider_enforces_reaches_it_every_round(respx_mock: respx.MockRouter) -> None: + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + side_effect=[ + httpx.Response(200, json=_openai_response(None, [_tool_call("call_1", "get_weather", {"city": "Paris"})])), + httpx.Response(200, json=_openai_response("done")), + ] + ) + + answer: Final = litellm.run_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_weather"}]}, + api_key="sk-test", + ) + + assert answer == "done" + assert [body["tool_choice"]["type"] for body in _request_bodies(respx_mock)] == ["allowed_tools", "allowed_tools"] + + def test_custom_tool_call_raises_type_error_without_executing(respx_mock: respx.MockRouter) -> None: custom_response: Final = _openai_response( None, @@ -481,7 +695,6 @@ async def test_arun_tool_loop_two_rounds(respx_mock: respx.MockRouter, monkeypat assert messages == messages_snapshot -ANTHROPIC_MESSAGES_URL: Final = "https://api.anthropic.com/v1/messages" TOOL_SEARCH_CALL: Final = { "type": "server_tool_use", "id": "srvtoolu_01Search", diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index da8c3c6a451..59c772f2993 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8864,22 +8864,29 @@ def test_validate_tool_choice_forcing_a_hosted_tool_passes_through(tool_choice: @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("mode", ["auto", "required"]) -def test_validate_tool_choice_allowed_tools_chat_layout_passes_through(mode: str): - tool_choice: Final = { +def test_validate_tool_choice_allowed_tools_either_layout_comes_back_in_the_chat_layout(mode: str): + chat_layout: Final = { "type": "allowed_tools", "allowed_tools": { "mode": mode, "tools": [{"type": "function", "function": {"name": "get_time"}}, {"type": "mcp", "server_label": "docs"}], }, } - assert validate_chat_completion_tool_choice(tool_choice, model=MODEL) is tool_choice + responses_layout: Final = { + "type": "allowed_tools", + "mode": mode, + "tools": [{"type": "function", "name": "get_time"}, {"type": "mcp", "server_label": "docs"}], + } + assert validate_chat_completion_tool_choice(chat_layout, model=MODEL) == chat_layout + assert validate_chat_completion_tool_choice(responses_layout, model=MODEL) == chat_layout @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize( "tool_choice", [ - {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]}, + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function"}]}, + {"type": "allowed_tools", "mode": "required", "tools": []}, {"type": "allowed_tools"}, {"type": "allowed_tools", "allowed_tools": {"mode": "required"}}, {"type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": []}}, @@ -8896,6 +8903,18 @@ def test_validate_tool_choice_allowed_tools_malformed_is_a_400(tool_choice: dict assert '"mode": "auto" | "required"' in str(exc_info.value) +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "required", "tools": []}, + {"type": "allowed_tools", "allowed_tools": {"mode": "none", "tools": [{"type": "function"}]}}, + ], +) +def test_validate_tool_choice_allowed_tools_malformed_is_dropped_under_drop_params(tool_choice: dict[str, object]): + assert validate_chat_completion_tool_choice(tool_choice, model=MODEL, drop_params=True) is None + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("tool_choice", [123, []]) def test_validate_tool_choice_invalid_type_is_a_400(tool_choice): diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index aae5dfe6472..6bedc240029 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -238,6 +238,7 @@ AGENTIC_LOOP_STATE_NAMES: Final = ( "_agentic_loop_depth", "_agentic_loop_fingerprints", "_agentic_loop_api_surface", + "_agentic_loop_tool_choice", "_code_interpreter_interception_active", "_code_interpreter_interception_sandbox_key", "_code_interpreter_interception_session_scoped", From 1d5e0c4be5d0cbbcef990242ae7e825daa114346 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:19:57 +0000 Subject: [PATCH 039/189] test(integration): keep owned peers off ephemeral ports (#51019) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/README.md | 2 +- tests/integration/_support/asgi.py | 4 +++- tests/integration/_support/async_server.py | 4 +++- .../integration/_support/aws_control_plane.py | 4 +++- tests/integration/_support/device_login.py | 8 +++---- tests/integration/_support/forward_proxy.py | 6 ++++-- tests/integration/_support/idle_gateway.py | 4 +++- tests/integration/_support/mail.py | 4 +++- .../integration/_support/responses_ws_peer.py | 7 ++++++- tests/integration/_support/riva.py | 4 +++- tests/integration/_support/wire.py | 4 +++- tests/unit/integration_support/test_wire.py | 21 ++++++++++++++++++- 12 files changed, 56 insertions(+), 16 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index c5d52922de6..a1d224749b3 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -28,7 +28,7 @@ Owned proxies disable `.env` loading by default. A contract testing dotenv behav Owned proxy workers configure a 32-thread executor before application startup, preserving additional worker startup hooks. Held synchronous provider calls must not consume a CPU-dependent executor limit before the rest of a fault burst can start. The proxy entrypoint imports LiteLLM inside `main`, so spawned workers begin supervision before importing the application. Worker healthchecks have a five-second deadline by default, so uvicorn kills a worker that stays paused longer than that. Tests never signal proxy processes themselves. `OwnedProxy.kill_worker` SIGKILLs a worker and returns once its replacement answers readiness with its own pid, `replacing_worker` does the same for a kill that has to land mid-burst, and `suspended` pauses a worker or the supervisor, which only works on a proxy launched with `healthcheck=WorkerHealthcheck.LENIENT`. The `kill()` that `replacing_worker` yields returns as soon as the worker is dead, so a test can release requests it holds on the surviving worker right away. Uvicorn may take a full worker healthcheck to notice the death, so the block waits for the supervisor to reap the worker and for the replacement only when it exits. Helper processes that are not the proxy stop through `stop_child`, `stop_processes` and `stop_process_group` and pause through `paused`. `scripts/check_test_quality.py` rule TQ010 rejects a raw `os.kill`, SIGKILL / SIGSTOP / SIGCONT, `.kill()`, `.terminate()`, `.suspend()` or `.send_signal()` anywhere under `tests/integration` outside `_support`, and TQ011 rejects an assertion that caps an elapsed time with a fixed number. Bound timings with `_support/timing.py` instead, relative to a measured control or to the injected delay. `run.py` measures how long this box takes to import the proxy application before it starts pytest, records it in `boot-calibration.json` beside the results and exports it as `INTEGRATION_IMPORT_SECONDS`. Readiness deadlines, graceful stops and the default per-test timeout are multiples of that measurement (`_support/budget.py`), so a slower box gets longer budgets and a two-worker proxy gets twice the readiness deadline of a one-worker one. Readiness requires responsive worker identities that are still alive and a positively identified owned listener. Unreadable socket records cannot establish ownership. Crash tests own their client tasks with `TaskGroup`, release held peers in `finally` and verify that the intended worker dies. A test requiring traffic on both workers prepares that traffic explicitly instead of assuming OS load balancing, for example by sending each half while the other worker is suspended -Any port the harness picks before the server that listens on it has started comes from `allocate_port()` in `_support/ports.py`. That covers owned proxies, scripted upstreams, fake providers, OTLP sinks, database relays and Redis. A server that binds port 0 itself and keeps that socket, such as `wire_server` without a port, needs no allocation. Each pytest process claims a block of 200 ports between 10000 and 32767 by holding an exclusive lock file per block in the system temp directory, so two runs on one machine, such as parallel worktrees, never share a block, and the OS drops the lock when the process exits. The range sits above the fixed service ports and below the Linux and macOS ephemeral ranges, so a port-0 bind or an outbound connection never lands in a block. Ports inside a block go out in order and are never handed out twice in a run, a port that something outside the harness already holds is skipped, and a process that uses up its block claims the next free one. A test that restarts a server on the same port allocates that port up front instead of reading it back from the first server. TQ012 in `scripts/check_test_quality.py` rejects reading a port back from a port-0 bind anywhere under `tests/integration` +Every owned listener's default port comes from `allocate_port()` in `_support/ports.py`. That covers owned proxies, scripted upstreams, fake providers, OTLP sinks, database relays and Redis, including peers such as `wire_server` that bind and keep their own sockets. Keeping those listeners out of the ephemeral range prevents them from taking a port Prisma selected with a bind-and-close probe before its query engine starts. Each pytest process claims a block of 200 ports between 10000 and 32767 by holding an exclusive lock file per block in the system temp directory, so two runs on one machine, such as parallel worktrees, never share a block, and the OS drops the lock when the process exits. The range sits above the fixed service ports and below the Linux and macOS ephemeral ranges, so a port-0 bind or an outbound connection never lands in a block. Ports inside a block go out in order and are never handed out twice in a run, a port that something outside the harness already holds is skipped, and a process that uses up its block claims the next free one. A test that restarts a server on the same port allocates that port up front instead of reading it back from the first server. TQ012 in `scripts/check_test_quality.py` rejects reading a port back from a port-0 bind anywhere under `tests/integration` The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. `GITHUB_FILES` in `run.py` lists the files excluded from the `run_integration.sh` selection diff --git a/tests/integration/_support/asgi.py b/tests/integration/_support/asgi.py index 30eb9f21760..4dcf3614836 100644 --- a/tests/integration/_support/asgi.py +++ b/tests/integration/_support/asgi.py @@ -12,13 +12,15 @@ from typing import Final import uvicorn from starlette.types import ASGIApp +from tests.integration._support.ports import allocate_port + @contextmanager def asgi_server( app: ASGIApp, *, before_stop: Callable[[], None] | None = None, port: int = 0, keepalive_seconds: int = 30 ) -> Iterator[str]: with socket.socket() as listener: - listener.bind(("127.0.0.1", port)) + listener.bind(("127.0.0.1", port or allocate_port())) bound_port: Final = listener.getsockname()[1] server: Final = uvicorn.Server( uvicorn.Config( diff --git a/tests/integration/_support/async_server.py b/tests/integration/_support/async_server.py index 53dcbc8d92d..73851b1b4a6 100644 --- a/tests/integration/_support/async_server.py +++ b/tests/integration/_support/async_server.py @@ -7,6 +7,8 @@ from contextlib import suppress from threading import Thread from typing import Final +from tests.integration._support.ports import allocate_port + Handler = Callable[[asyncio.StreamReader, asyncio.StreamWriter], Awaitable[None]] @@ -19,7 +21,7 @@ async def close_writer(writer: asyncio.StreamWriter) -> None: class LoopbackServer: def __init__(self, port: int, handler: Handler, ssl_context: ssl.SSLContext | None = None) -> None: self.loop: Final = asyncio.new_event_loop() - self._port: Final = port + self._port: Final = port or allocate_port() self._handler: Final = handler self._ssl_context: Final = ssl_context self._server: asyncio.Server | None = None diff --git a/tests/integration/_support/aws_control_plane.py b/tests/integration/_support/aws_control_plane.py index 8cef78f8bee..1c984cdf03e 100644 --- a/tests/integration/_support/aws_control_plane.py +++ b/tests/integration/_support/aws_control_plane.py @@ -18,6 +18,8 @@ from integration._support.sigv4 import canonical_query, signature from integration._support.wire import Reply from pydantic import JsonValue +from tests.integration._support.ports import allocate_port + _AUTHORIZATION: Final = re.compile( r"^AWS4-HMAC-SHA256 Credential=(?P[^/]+)/(?P[^,]+), " r"SignedHeaders=(?P[^,]+), Signature=(?P[0-9a-f]{64})$" @@ -267,7 +269,7 @@ def control_plane(directory: Path, hosts: tuple[str, ...]) -> Generator[ControlP block_on_close = False request_queue_size = 128 - with Server(("127.0.0.1", 0), Proxy) as server: + with Server(("127.0.0.1", allocate_port()), Proxy) as server: thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) thread.start() try: diff --git a/tests/integration/_support/device_login.py b/tests/integration/_support/device_login.py index 13a7169a01a..349021a99e9 100644 --- a/tests/integration/_support/device_login.py +++ b/tests/integration/_support/device_login.py @@ -22,6 +22,8 @@ from integration._support.tls import server_context, write_self_signed_cert from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue +from tests.integration._support.ports import allocate_port + CHATGPT_AUTH_HOST: Final = "auth.openai.com" GITHUB_HOST: Final = "github.com" GITHUB_API_HOST: Final = "api.github.com" @@ -366,7 +368,7 @@ def _tunnel( _pipe(upstream, self.request) outbound.join(timeout=12) - with _TunnelServer(("127.0.0.1", 0), Tunnel) as server: + with _TunnelServer(("127.0.0.1", allocate_port()), Tunnel) as server: thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) thread.start() try: @@ -378,9 +380,7 @@ def _tunnel( @contextmanager -def device_login_peers( - directory: Path, provider: Callable[[Request], Reply] | None = None -) -> Generator[Peers]: +def device_login_peers(directory: Path, provider: Callable[[Request], Reply] | None = None) -> Generator[Peers]: cert, key = write_self_signed_cert(directory, (*AUTH_HOSTS, COPILOT_API_HOST)) switches: Final = Switches(threading.Event(), threading.Event(), threading.Event(), threading.Event()) authorities: Final[SimpleQueue[str]] = SimpleQueue() diff --git a/tests/integration/_support/forward_proxy.py b/tests/integration/_support/forward_proxy.py index 0d92c3d0719..98c440c3f6c 100644 --- a/tests/integration/_support/forward_proxy.py +++ b/tests/integration/_support/forward_proxy.py @@ -10,6 +10,8 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from queue import SimpleQueue from typing import Final +from tests.integration._support.ports import allocate_port + @dataclass(frozen=True, slots=True) class Tunnel: @@ -61,7 +63,7 @@ def refusing_forward_proxy(port: int = 0) -> Generator[ForwardProxy, None, None] def log_message(self, format: str, *args: object) -> None: pass - with ThreadingHTTPServer(("127.0.0.1", port), Handler) as server: + with ThreadingHTTPServer(("127.0.0.1", port or allocate_port()), Handler) as server: thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) thread.start() try: @@ -105,7 +107,7 @@ def tunnelling_forward_proxy(destination_port: int, port: int = 0) -> Generator[ def log_message(self, format: str, *args: object) -> None: pass - with ThreadingHTTPServer(("127.0.0.1", port), Handler) as server: + with ThreadingHTTPServer(("127.0.0.1", port or allocate_port()), Handler) as server: thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) thread.start() try: diff --git a/tests/integration/_support/idle_gateway.py b/tests/integration/_support/idle_gateway.py index 5a86d7e0a35..42cbf385174 100644 --- a/tests/integration/_support/idle_gateway.py +++ b/tests/integration/_support/idle_gateway.py @@ -18,6 +18,8 @@ from typing import Final from integration._support.wire import Reply, Request +from tests.integration._support.ports import allocate_port + MARKER: Final = re.compile(rb"idle-[0-9a-f]{16}") _LOOPBACK: Final = "127.0.0.1" @@ -193,7 +195,7 @@ def idle_gateway( port: int = 0, ) -> Generator[IdleGateway]: arrivals: Final[SimpleQueue[Arrival]] = SimpleQueue() - with _Server(port, respond, drop_after, drop, arrivals) as server: + with _Server(port or allocate_port(), respond, drop_after, drop, arrivals) as server: thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) thread.start() try: diff --git a/tests/integration/_support/mail.py b/tests/integration/_support/mail.py index 3894baeccc3..08562d9a8c3 100644 --- a/tests/integration/_support/mail.py +++ b/tests/integration/_support/mail.py @@ -10,6 +10,8 @@ from email.message import Message from queue import SimpleQueue from typing import Final +from tests.integration._support.ports import allocate_port + @dataclass(frozen=True, slots=True) class Delivery: @@ -112,7 +114,7 @@ def smtp_sink() -> Generator[Mailbox, None, None]: allow_reuse_address = True daemon_threads = False - with OwnedServer(("127.0.0.1", 0), Handler) as server: + with OwnedServer(("127.0.0.1", allocate_port()), Handler) as server: mailbox: Final = Mailbox("127.0.0.1", server.server_address[1]) thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) thread.start() diff --git a/tests/integration/_support/responses_ws_peer.py b/tests/integration/_support/responses_ws_peer.py index 1bf3e1fcbae..fb02c10f39d 100644 --- a/tests/integration/_support/responses_ws_peer.py +++ b/tests/integration/_support/responses_ws_peer.py @@ -21,6 +21,8 @@ from integration._support.tls import server_context from pydantic import JsonValue from websockets.asyncio.server import ServerConnection, serve +from tests.integration._support.ports import allocate_port + PROVIDER_MODEL: Final = "ws-peer-model" USAGE: Final = {"input_tokens": 5, "output_tokens": 2, "total_tokens": 7} TERMINAL: Final = frozenset({"response.completed", "response.failed", "error"}) @@ -105,7 +107,10 @@ async def _serve( script: Script, ) -> None: async with serve( - lambda connection: _answer(connection, paths, credentials, frames, script), "127.0.0.1", 0, ssl=tls + lambda connection: _answer(connection, paths, credentials, frames, script), + "127.0.0.1", + allocate_port(), + ssl=tls, ) as server: ports.put(next(iter(server.sockets)).getsockname()[1]) await stop.wait() diff --git a/tests/integration/_support/riva.py b/tests/integration/_support/riva.py index 438d877386b..f50c28b705d 100644 --- a/tests/integration/_support/riva.py +++ b/tests/integration/_support/riva.py @@ -15,6 +15,8 @@ from typing import Final import grpc from riva.client.proto import riva_asr_pb2, riva_asr_pb2_grpc +from tests.integration._support.ports import allocate_port + SAMPLE_RATE_HZ: Final = 16000 OFFLINE_ONLY_REFUSAL: Final = ( "Error: Unavailable model requested given these parameters: language_code=en-US; sample_rate=16000; type=online;" @@ -139,7 +141,7 @@ def riva_server( options: Final = () if max_receive_bytes is None else (("grpc.max_receive_message_length", max_receive_bytes),) server: Final = grpc.server(futures.ThreadPoolExecutor(max_workers=8), options=options) riva_asr_pb2_grpc.add_RivaSpeechRecognitionServicer_to_server(_Asr(words, offline_only, stall, calls), server) - port: Final = server.add_insecure_port("127.0.0.1:0") + port: Final = server.add_insecure_port(f"127.0.0.1:{allocate_port()}") server.start() try: yield RivaServer(f"127.0.0.1:{port}", calls) diff --git a/tests/integration/_support/wire.py b/tests/integration/_support/wire.py index d4cbccfc43e..343430db4a6 100644 --- a/tests/integration/_support/wire.py +++ b/tests/integration/_support/wire.py @@ -15,6 +15,8 @@ from queue import SimpleQueue from types import MappingProxyType from typing import TYPE_CHECKING, BinaryIO, Final +from tests.integration._support.ports import allocate_port + if TYPE_CHECKING: from collections.abc import Buffer @@ -334,7 +336,7 @@ def wire_server( if tls is not None: self.socket = tls.wrap_socket(self.socket, server_side=True) - with OwnedHTTPServer(("127.0.0.1", port), Handler) as server: + with OwnedHTTPServer(("127.0.0.1", port or allocate_port()), Handler) as server: thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": _POLL_SECONDS}) thread.start() try: diff --git a/tests/unit/integration_support/test_wire.py b/tests/unit/integration_support/test_wire.py index c786895720f..7dd168eb212 100644 --- a/tests/unit/integration_support/test_wire.py +++ b/tests/unit/integration_support/test_wire.py @@ -7,7 +7,7 @@ import ssl import time from collections.abc import Callable, Iterator from concurrent.futures import ThreadPoolExecutor -from contextlib import ExitStack +from contextlib import ExitStack, closing from http.client import HTTPConnection, HTTPResponse, HTTPSConnection from pathlib import Path from typing import Final @@ -19,6 +19,7 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec from cryptography.x509.oid import NameOID +from tests.integration._support.ports import FIRST_PORT, LAST_PORT, allocate_port from tests.integration._support.wire import Reply, Request, wire_server PAUSE_SECONDS: Final = 6 @@ -28,6 +29,24 @@ LATE: Final = b'{"call": "late"}' UNREAD: Final = b"x" * (32 * 1024 * 1024) +@pytest.mark.parametrize("port", (None, 0), ids=("default", "zero")) +def test_default_listener_uses_the_harness_port_range(port: int | None) -> None: + with wire_server(_echo, **({} if port is None else {"port": port})) as wire: + bound_port: Final = urlsplit(wire.url).port + assert bound_port is not None and FIRST_PORT <= bound_port <= LAST_PORT + with closing(_client(wire.url)) as connection: + connection.request("POST", "/echo", FIRST) + response: Final = connection.getresponse() + assert (response.status, response.read()) == (200, FIRST) + + +def test_explicit_listener_port_is_preserved() -> None: + port: Final = allocate_port() + with wire_server(_echo, port=port) as wire: + assert urlsplit(wire.url).port == port + assert wire.drain() == () + + def _self_signed_server_context(directory: Path) -> ssl.SSLContext: key: Final = ec.generate_private_key(ec.SECP256R1()) name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "wire peer")]) From 845f10952405dce252e0ce98d5fc96ebd825009d Mon Sep 17 00:00:00 2001 From: norkelbyte <348963847+norkelbyte@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:28:49 -0800 Subject: [PATCH 040/189] fix(bedrock): forward x-pass- headers on the /bedrock/model pass-through routes (#50728) * fix(bedrock): forward x-pass- headers on the /bedrock/model pass-through routes The /bedrock/model/{model}/invoke, invoke-with-response-stream, converse and converse-stream routes built the pass-through call from the request body, so x-pass- headers never reached Bedrock. The processor now hands the caller's x-pass- headers to the pass-through route, keeps them out of the logged body snapshot, and on Converse drops the invoke-only guardrail headers when the deployment sets guardrailConfig and the request metadata header when the proxy owns request metadata * fix(bedrock): bill the served service tier on invoke pass-through and scope x-pass- forwarding to the Bedrock routes * test(bedrock): assert forwarded x-pass- headers on the request Bedrock receives * test(bedrock): gate the router-model x-pass- forwarding on the request Bedrock receives * fix(bedrock): bill a caller's forwarded invoke guardrail and keep x-pass- headers out of guardrail vendor payloads * fix(bedrock): bill a caller guardrail on every invoke model and replace a trace value Bedrock rejects * fix(bedrock): keep guardrail traces out of chat responses, forward the trace header as sent, and drop credential x-pass- headers --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../base_llm/passthrough/transformation.py | 9 +- .../bedrock/passthrough/transformation.py | 95 +++- litellm/passthrough/main.py | 16 +- litellm/passthrough/utils.py | 34 +- litellm/proxy/litellm_pre_call_utils.py | 4 + .../llm_passthrough_endpoints.py | 9 + ...drock_passthrough_guardrail_config_wire.py | 26 +- ...drock_passthrough_request_metadata_wire.py | 61 ++- .../test_base_invoke_transformation.py | 136 ++++++ ...test_bedrock_passthrough_transformation.py | 444 +++++++++++++++++- tests/unit/passthrough/test_utils.py | 78 +++ .../test_llm_pass_through_endpoints.py | 180 +++++++ .../unit/proxy/test_litellm_pre_call_utils.py | 32 ++ 13 files changed, 1081 insertions(+), 43 deletions(-) create mode 100644 tests/unit/passthrough/test_utils.py diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index 7e58cc6e71b..898033bff8f 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -4,6 +4,7 @@ import re from abc import abstractmethod from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol, TypeAlias from pydantic import ConfigDict, TypeAdapter, ValidationError @@ -29,6 +30,7 @@ if TYPE_CHECKING: RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) _DECODED_JSON: Final = TypeAdapter(object) +_NO_RESPONSE_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) class PassthroughMetadata(LiteLLMBaseModel): @@ -236,7 +238,12 @@ class BasePassthroughConfig(BaseLLMModelInfo): return None def create_stream_collector( - self, model: str, custom_llm_provider: str, endpoint: str + self, + model: str, + custom_llm_provider: str, + endpoint: str, + *, + response_headers: Mapping[str, str] = _NO_RESPONSE_HEADERS, ) -> PassthroughStreamCollector: return RawBytesStreamCollector( provider_config=self, model=model, custom_llm_provider=custom_llm_provider, endpoint=endpoint diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index 4a2bb57ce5a..901c16afaa7 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -3,16 +3,21 @@ from collections.abc import Callable, Iterator, Mapping, Sequence from functools import partial from itertools import chain from types import MappingProxyType -from typing import TYPE_CHECKING, ClassVar, Final, Optional, cast +from typing import TYPE_CHECKING, ClassVar, Final, Optional, TypeVar, cast import httpx from httpx import Response -from pydantic import TypeAdapter +from pydantic import BaseModel, TypeAdapter from litellm._logging import verbose_logger from litellm.constants import AWS_ERROR_TYPE_HEADER from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping +from litellm.llms.base_llm.passthrough.transformation import ( + BasePassthroughConfig, + PassthroughStreamCollector, + relayed_json_object, +) from litellm.llms.bedrock.chat.converse_to_openai import as_json_array, as_json_object from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( bedrock_invoke_guardrail_headers, @@ -21,7 +26,12 @@ from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletio from litellm.types.utils import ModelResponseStream from ..base_aws_llm import BaseAWSLLM -from ..common_utils import BedrockError, BedrockEventStreamDecoderBase, BedrockModelInfo +from ..common_utils import ( + BEDROCK_INVOKE_TRACE_FIELD, + BedrockError, + BedrockEventStreamDecoderBase, + BedrockModelInfo, +) from ..request_metadata import ( BEDROCK_REQUEST_METADATA_HEADER, BedrockRequestMetadataViolation, @@ -49,6 +59,9 @@ _BEDROCK_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stre _BEDROCK_INVOKE_GUARDRAIL_HEADERS: Final = frozenset( {"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"} ) +_BEDROCK_SERVICE_TIER_HEADER: Final = "x-amzn-bedrock-service-tier" +_NO_RESPONSE_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) +_TieredResponse: Final = TypeVar("_TieredResponse", bound=BaseModel) def _bedrock_endpoint_action(endpoint: str) -> str: @@ -111,7 +124,9 @@ def _with_request_metadata( litellm_params: Mapping[str, object], ) -> tuple[Mapping[str, str], Mapping[str, object]]: if action in _BEDROCK_CONVERSE_ACTIONS: - return headers, _with_body_request_metadata(action, body, litellm_params) + return _without_owned_request_metadata_header(headers), _with_body_request_metadata( + action, body, litellm_params + ) if action in _BEDROCK_INVOKE_ACTIONS and "requestMetadata" in body: return _without_owned_request_metadata_header(headers), _with_body_request_metadata( action, body, litellm_params @@ -137,6 +152,10 @@ def _converse_guardrail_config(action: str, guardrail_config: object) -> object: return {key: value for key, value in fields.items() if key != "streamProcessingMode"} +def _without_invoke_guardrail_headers(headers: Mapping[str, str]) -> Mapping[str, str]: + return {name: value for name, value in headers.items() if name.lower() not in _BEDROCK_INVOKE_GUARDRAIL_HEADERS} + + def _with_deployment_guardrail( action: str, headers: Mapping[str, str], @@ -146,7 +165,10 @@ def _with_deployment_guardrail( if guardrail_config is None: return headers, body if action in _BEDROCK_CONVERSE_ACTIONS: - return headers, {**body, "guardrailConfig": _converse_guardrail_config(action, guardrail_config)} + return _without_invoke_guardrail_headers(headers), { + **body, + "guardrailConfig": _converse_guardrail_config(action, guardrail_config), + } if action in _BEDROCK_INVOKE_ACTIONS: guardrail_headers: Final = tuple(bedrock_invoke_guardrail_headers(guardrail_config).items()) return merge_bedrock_invoke_headers( @@ -266,6 +288,19 @@ class _CoalescedChunks: return self._chunks +_BEDROCK_EVENT: Final = TypeAdapter(dict[str, object]) + + +def _with_invoke_trace(chunk: ModelResponseStream, trace: Mapping[str, object] | None) -> ModelResponseStream: + if trace is None: + return chunk + return chunk.model_copy( + update={ + "provider_specific_fields": {**(chunk.provider_specific_fields or {}), BEDROCK_INVOKE_TRACE_FIELD: trace} + } + ) + + def _translate_message(decoder: "AWSEventStreamDecoder", message: str) -> ModelResponseStream | None: from litellm.litellm_core_utils.streaming_handler import ( convert_generic_chunk_to_model_response_stream, @@ -273,11 +308,15 @@ def _translate_message(decoder: "AWSEventStreamDecoder", message: str) -> ModelR ) from litellm.types.utils import GenericStreamingChunk - translated_chunk: Final = decoder.chunk_parser(chunk_data=json.loads(message)) + event: Final = _BEDROCK_EVENT.validate_json(message) + trace: Final = object_mapping(event.get(BEDROCK_INVOKE_TRACE_FIELD)) + translated_chunk: Final = decoder.chunk_parser(chunk_data=event) if isinstance(translated_chunk, ModelResponseStream): - return translated_chunk + return _with_invoke_trace(translated_chunk, trace) if generic_chunk_has_all_required_fields(cast(dict, translated_chunk)): - return convert_generic_chunk_to_model_response_stream(cast(GenericStreamingChunk, translated_chunk)) + return _with_invoke_trace( + convert_generic_chunk_to_model_response_stream(cast(GenericStreamingChunk, translated_chunk)), trace + ) return None @@ -332,15 +371,33 @@ def _count_prompt_tokens(model: str, request_body: object) -> int: return token_counter(model=model, messages=messages) +def _invoke_guardrail_trace(httpx_response: Response) -> Mapping[str, object]: + reply: Final = relayed_json_object(httpx_response) or {} + return {BEDROCK_INVOKE_TRACE_FIELD: reply.get(BEDROCK_INVOKE_TRACE_FIELD)} + + +def _served_service_tier(response_headers: Mapping[str, str]) -> str | None: + headers: Final = httpx.Headers(response_headers) + if _BEDROCK_SERVICE_TIER_HEADER not in headers: + return None + return headers[_BEDROCK_SERVICE_TIER_HEADER] + + +def _with_served_service_tier(response: _TieredResponse, service_tier: str | None) -> _TieredResponse: + if service_tier is None: + return response + return response.model_copy(update={"service_tier": service_tier}) + + def _build_logged_response( - chunks: Sequence[ModelResponseStream], litellm_logging_obj: "LiteLLMLoggingObj" + chunks: Sequence[ModelResponseStream], litellm_logging_obj: "LiteLLMLoggingObj", service_tier: str | None ) -> Optional["CostResponseTypes"]: from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder if len(chunks) == 0: return None return stream_chunk_builder( - chunks=list(chunks), + chunks=[*chunks[:-1], _with_served_service_tier(chunks[-1], service_tier)], logging_obj=litellm_logging_obj, count_prompt_tokens=partial( _count_prompt_tokens, litellm_logging_obj.model, litellm_logging_obj.model_call_details.get("input") @@ -355,11 +412,13 @@ class BedrockEventStreamCollector: self, parse_event: Callable[["EventStreamMessage"], str | None], decoder: Optional["AWSEventStreamDecoder"], + service_tier: str | None = None, ) -> None: from botocore.eventstream import EventStreamBuffer self._parse_event = parse_event self._decoder = decoder + self._service_tier: Final = service_tier self._event_stream_buffer: Final[EventStreamBuffer] = EventStreamBuffer() self._chunks: Final = _CoalescedChunks() @@ -377,7 +436,7 @@ class BedrockEventStreamCollector: self._chunks.add(translated) def build_logged_response(self, litellm_logging_obj: "LiteLLMLoggingObj") -> Optional["CostResponseTypes"]: - return _build_logged_response(self._chunks.chunks(), litellm_logging_obj) + return _build_logged_response(self._chunks.chunks(), litellm_logging_obj, self._service_tier) class BedrockAgentRuntimeAWS(BaseAWSLLM): @@ -542,15 +601,23 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD request_data=request_data, encoding=encoding, ) + if "invoke" in endpoint: + logging_obj.record_native_bedrock_guardrail(_invoke_guardrail_trace(httpx_response)) - return litellm_model_response + return _with_served_service_tier(litellm_model_response, _served_service_tier(httpx_response.headers)) def create_stream_collector( - self, model: str, custom_llm_provider: str, endpoint: str + self, + model: str, + custom_llm_provider: str, + endpoint: str, + *, + response_headers: Mapping[str, str] = _NO_RESPONSE_HEADERS, ) -> PassthroughStreamCollector: return BedrockEventStreamCollector( parse_event=self._parse_message_from_event, decoder=self._get_event_stream_decoder(model=model, endpoint=endpoint), + service_tier=_served_service_tier(response_headers), ) def _get_event_stream_decoder(self, model: str, endpoint: str) -> Optional["AWSEventStreamDecoder"]: diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index 4e94b1bb910..77b2cf9b5b5 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -57,11 +57,17 @@ def _as_generator(iterable: Iterator[bytes]) -> Generator[bytes, bytes, None]: class _SpendCollection: """Feeds relayed chunks to the provider's stream collector without letting spend tracking break the relay.""" - def __init__(self, provider_config: BasePassthroughConfig, litellm_logging_obj: LiteLLMLoggingObj) -> None: + def __init__( + self, + provider_config: BasePassthroughConfig, + litellm_logging_obj: LiteLLMLoggingObj, + response_headers: httpx.Headers, + ) -> None: self.collector: Final[PassthroughStreamCollector] = provider_config.create_stream_collector( model=litellm_logging_obj.model, custom_llm_provider=litellm_logging_obj.model_call_details.get("custom_llm_provider", ""), endpoint=litellm_logging_obj.model_call_details.get("endpoint", ""), + response_headers=response_headers, ) self.chunk_count = 0 self._failed = False @@ -97,7 +103,8 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]): self._response: httpx.Response self._iterator: AsyncGenerator[bytes, bytes] self._litellm_logging_obj = litellm_logging_obj - self._spend = _SpendCollection(provider_config, litellm_logging_obj) + self._provider_config: Final = provider_config + self._spend: _SpendCollection self._flush_scheduled = False self._background_tasks: set[asyncio.Task[bool]] = set() # mutable-ok: instance set for background task tracking self._flush: asyncio.Task[bool] | None = None @@ -129,6 +136,7 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]): self._response = await self._response_coro self.headers = self._response.headers self.status_code = self._response.status_code + self._spend = _SpendCollection(self._provider_config, self._litellm_logging_obj, self._response.headers) self._initialized = True try: raise_for_provider_status(self._response) @@ -148,7 +156,7 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]): return _init().__await__() def _start_flush(self) -> None: - if self._flush_scheduled or not self._spend.should_flush: + if self._flush_scheduled or not self._initialized or not self._spend.should_flush: return self._flush_scheduled = True @@ -237,7 +245,7 @@ class PassthroughStreamingResponse(Generator[bytes, bytes, None]): self.status_code = response.status_code self._litellm_logging_obj = litellm_logging_obj self._iterator: Generator[bytes, bytes, None] = _as_generator(response.iter_bytes()) - self._spend = _SpendCollection(provider_config, litellm_logging_obj) + self._spend = _SpendCollection(provider_config, litellm_logging_obj, response.headers) self._flush_scheduled = False def _start_flush(self) -> None: diff --git a/litellm/passthrough/utils.py b/litellm/passthrough/utils.py index 452c9c7de9d..1bf5f08b1c1 100644 --- a/litellm/passthrough/utils.py +++ b/litellm/passthrough/utils.py @@ -25,7 +25,16 @@ _PASS_THROUGH_PROTECTED_HEADERS: Final[frozenset] = frozenset( ) # Header name prefix used to block AWS SigV4 signing headers from being overridden. -_PASS_THROUGH_PROTECTED_HEADER_PREFIXES: Final[tuple] = ("x-amz-",) +_PASS_THROUGH_PROTECTED_HEADER_PREFIXES: Final[tuple[str, ...]] = ("x-amz-",) + + +def _is_forwarded_pass_through_header(header_name: str) -> bool: + if not header_name.lower().startswith(PASS_THROUGH_HEADER_PREFIX): + return False + forwarded_name: Final = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower() + return forwarded_name not in _PASS_THROUGH_PROTECTED_HEADERS and not any( + forwarded_name.startswith(prefix) for prefix in _PASS_THROUGH_PROTECTED_HEADER_PREFIXES + ) class BasePassthroughUtils: @@ -42,18 +51,11 @@ class BasePassthroughUtils: # parse_qs returns a dict where each value is a list, so let's flatten it updated_existing_query_params: Final = {k: v[0] if len(v) == 1 else v for k, v in existing_query_params.items()} - # Start with default query params (lowest priority) - merged_params: Final = {} - if default_query_params: - merged_params.update(default_query_params) + return {**(default_query_params or {}), **updated_existing_query_params, **request_query_params} - # Override with existing URL query params (medium priority) - merged_params.update(updated_existing_query_params) - - # Override with request query params (highest priority - client can override anything) - merged_params.update(request_query_params) - - return merged_params + @staticmethod + def pass_through_prefixed_headers(request_headers: Mapping[str, str]) -> Mapping[str, str]: + return {name: value for name, value in request_headers.items() if _is_forwarded_pass_through_header(name)} @staticmethod def forward_headers_from_request( @@ -88,16 +90,14 @@ class BasePassthroughUtils: # Credential and protocol-level headers are excluded from this mechanism. for header_name, header_value in request_headers.items(): if header_name.lower().startswith(PASS_THROUGH_HEADER_PREFIX): - # Strip the 'x-pass-' prefix and normalize to lowercase - actual_header_name = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower() - if actual_header_name in _PASS_THROUGH_PROTECTED_HEADERS or any( - actual_header_name.startswith(p) for p in _PASS_THROUGH_PROTECTED_HEADER_PREFIXES - ): + if not _is_forwarded_pass_through_header(header_name): verbose_logger.debug( "x-pass- header %s maps to a protected header name; skipping", header_name, ) continue + # Strip the 'x-pass-' prefix and normalize to lowercase + actual_header_name = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower() headers[actual_header_name] = header_value return headers diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 22399ced718..d5eac099fb6 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1656,6 +1656,10 @@ def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: } +def without_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: + return {header: value for header, value in headers.items() if not _is_credential_header(header)} + + class TenantLoggingMetadata(TypedDict): user_api_key_metadata: ReadOnly[Mapping[str, object] | None] user_api_key_team_metadata: ReadOnly[Mapping[str, object] | None] diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 6ab31e7d4b0..a2e2e36d458 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -86,6 +86,7 @@ from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, valid from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.llms.vllm.common_utils import endpoint_under_api_base from litellm.passthrough.main import AsyncPassthroughStreamingResponse, is_provider_error +from litellm.passthrough.utils import BasePassthroughUtils from litellm.proxy._types import * from litellm.proxy.auth.auth_checks import ( can_agent_call_model, @@ -1423,6 +1424,12 @@ async def bedrock_passthrough_error_response( raise +def _bedrock_pass_through_headers(request: Request) -> Mapping[str, str]: + from litellm.proxy.litellm_pre_call_utils import without_credential_headers + + return without_credential_headers(BasePassthroughUtils.pass_through_prefixed_headers(request.headers)) + + async def handle_bedrock_passthrough_router_model( model: str, endpoint: str, @@ -1478,6 +1485,7 @@ async def handle_bedrock_passthrough_router_model( "endpoint": endpoint, "data": request_body, "custom_llm_provider": "bedrock", + "request_headers": _bedrock_pass_through_headers(request), } data: Final[dict[str, object]] = guardrail_request_data_with_streaming( MappingProxyType(bedrock_payload), @@ -1684,6 +1692,7 @@ async def bedrock_llm_proxy_route( "endpoint": endpoint, "data": request_body, "custom_llm_provider": "bedrock", + "request_headers": _bedrock_pass_through_headers(request), } data: Final[dict[str, object]] = guardrail_request_data_with_streaming( MappingProxyType(passthrough_payload), diff --git a/tests/integration/providers/test_bedrock_passthrough_guardrail_config_wire.py b/tests/integration/providers/test_bedrock_passthrough_guardrail_config_wire.py index f995a7ac83f..d630fa00138 100644 --- a/tests/integration/providers/test_bedrock_passthrough_guardrail_config_wire.py +++ b/tests/integration/providers/test_bedrock_passthrough_guardrail_config_wire.py @@ -404,6 +404,13 @@ def _expected_on_the_wire(action: str, request: Request) -> Applied: return Applied("deployment-guardrail", "3", "ENABLED", None) +def _caller_guardrail_on_the_wire(request: Request) -> Applied: + assert {"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"} <= ( + _signed_header_names(request) + ), request.headers["authorization"] + return Applied("caller-guardrail", "9", _CALLER_INVOKE_HEADERS["x-pass-x-amzn-bedrock-trace"], None) + + @pytest.mark.parametrize("action", _ACTIONS) def test_passthrough_on_a_guardrailed_deployment_blocks_with_the_deployment_guardrail( rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str @@ -478,7 +485,10 @@ def test_deployment_without_a_guardrail_leaves_the_caller_guardrail_as_sent( rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str ) -> None: if action.startswith("invoke"): - pytest.skip("BUG: the router-model invoke pass-through drops the caller's X-Amzn-Bedrock-Guardrail* headers") + pytest.skip( + "BUG: the router-model invoke pass-through drops the unprefixed X-Amzn-Bedrock-Guardrail* headers" + " that boto3's native guardrailIdentifier / guardrailVersion / trace args send" + ) marker: Final = uuid.uuid4().hex runtime: Final = rig.runtime(monkeypatch) @@ -491,6 +501,20 @@ def test_deployment_without_a_guardrail_leaves_the_caller_guardrail_as_sent( ), request.body +@pytest.mark.parametrize("action", ("invoke", "invoke-with-response-stream")) +def test_deployment_without_a_guardrail_forwards_the_caller_guardrail_sent_as_x_pass_headers( + rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str +) -> None: + marker: Final = uuid.uuid4().hex + runtime: Final = rig.runtime(monkeypatch, _CALLER_INVOKE_HEADERS) + + outcome: Final = _send(runtime, action, rig.unguarded_model, f"{marker} say {_BLOCKED_WORD}", caller_guardrail=False) + + assert outcome == Outcome(_BLOCKED_TEXT, intervened=True, traced_guardrail=None), outcome + (request,) = rig.received(rig.wire, marker) + assert _applied(request) == _caller_guardrail_on_the_wire(request), (request.headers, request.body) + + def test_each_load_balanced_request_carries_the_guardrail_of_the_deployment_it_reached( rig: Rig, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/integration/providers/test_bedrock_passthrough_request_metadata_wire.py b/tests/integration/providers/test_bedrock_passthrough_request_metadata_wire.py index 5fd12405ab2..992e18a6d5d 100644 --- a/tests/integration/providers/test_bedrock_passthrough_request_metadata_wire.py +++ b/tests/integration/providers/test_bedrock_passthrough_request_metadata_wire.py @@ -5,6 +5,7 @@ from urllib.parse import unquote from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import Final import boto3 @@ -25,6 +26,7 @@ _MODEL_ID: Final = "anthropic.claude-sonnet-5-v1:0" _EVENT_STREAM: Final = "application/vnd.amazon.eventstream" _METADATA_HEADER: Final = "x-amzn-bedrock-request-metadata" _HEADER_MAX_CHARS: Final = 8500 +_LATENCY_HEADER: Final = "x-amzn-bedrock-performanceconfig-latency" _LONG_SPEND_LOGS_METADATA: Final = tuple((f"{index:02d}".ljust(256, "k"), "\n" * 256) for index in range(12)) _CONVERSE_REPLY: Final[Mapping[str, JsonValue]] = { "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, @@ -105,9 +107,11 @@ class Rig: long_key: str long_identity: tuple[tuple[str, str], ...] - def runtime(self, key: str, monkeypatch: pytest.MonkeyPatch): # noqa: ANN201 # boto3 clients are untyped + def runtime( # noqa: ANN201 # boto3 clients are untyped + self, key: str, monkeypatch: pytest.MonkeyPatch, extra_headers: Mapping[str, str] = MappingProxyType({}) + ): monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", key) - return boto3.client( + client: Final = boto3.client( "bedrock-runtime", endpoint_url=f"{str(self.proxy.client.base_url).rstrip('/')}/bedrock", region_name="us-east-1", @@ -117,6 +121,11 @@ class Rig: parameter_validation=False, ), ) + client.meta.events.register( + "before-send.bedrock-runtime.*", + lambda request, **_: request.headers.update(extra_headers), + ) + return client def received(self, marker: str) -> tuple[Request, ...]: return tuple(request for request in self.wire.drain() if marker.encode() in request.body) @@ -252,6 +261,54 @@ def test_boto3_passthrough_calls_carry_the_key_identity_in_signed_request_metada ] +def test_caller_request_metadata_header_sent_through_x_pass_never_replaces_the_key_identity( + rig: Rig, monkeypatch: pytest.MonkeyPatch +) -> None: + marker: Final = f"forged-header-{uuid.uuid4().hex}" + runtime: Final = rig.runtime( + rig.key, + monkeypatch, + { + f"x-pass-{_METADATA_HEADER}": json.dumps({"user_api_key_team_alias": "forged", "ticket": "T-42"}), + f"x-pass-{_LATENCY_HEADER}": "optimized", + }, + ) + model: Final = rig.strict_model + + runtime.converse(modelId=model, messages=_messages(marker)) + tuple(runtime.converse_stream(modelId=model, messages=_messages(marker))["stream"]) + runtime.invoke_model(modelId=model, body=json.dumps(_invoke_body(marker)))["body"].read() + tuple(runtime.invoke_model_with_response_stream(modelId=model, body=json.dumps(_invoke_body(marker)))["body"]) + + received: Final = rig.received(marker) + assert [request.target.rsplit("/", 1)[-1] for request in received] == [ + "converse", + "converse-stream", + "invoke", + "invoke-with-response-stream", + ], received + assert all( + request.headers.get(_LATENCY_HEADER) == "optimized" and _LATENCY_HEADER in _signed_header_names(request) + for request in received + ), [request.headers for request in received] + converse_requests: Final = received[:2] + invoke_requests: Final = received[2:] + assert [_METADATA_HEADER in request.headers for request in converse_requests] == [False, False], [ + request.headers for request in converse_requests + ] + assert [json.loads(request.body)["requestMetadata"] for request in converse_requests] == [ + rig.identity, + rig.identity, + ], [request.body for request in converse_requests] + assert [json.loads(request.headers[_METADATA_HEADER]) for request in invoke_requests] == [ + rig.identity, + rig.identity, + ], [request.headers for request in invoke_requests] + assert all(_METADATA_HEADER in _signed_header_names(request) for request in invoke_requests), [ + request.headers["authorization"] for request in invoke_requests + ] + + def test_invoke_metadata_too_long_for_the_header_signs_the_leading_pairs_that_fit( rig: Rig, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index c36644ba536..f6728e2ed10 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -1,4 +1,7 @@ +import base64 import json +import struct +from binascii import crc32 from collections.abc import Mapping from types import MappingProxyType from typing import Final @@ -518,3 +521,136 @@ def test_imported_llama_model_arn_is_encoded_and_max_tokens_become_max_gen_len(r assert response.choices[0].message.content == "Here's a joke..." assert str(route.calls[0].request.url) == expected_url assert json.loads(route.calls[0].request.content) == {"prompt": "Tell me a joke", "max_gen_len": 100} + + +GUARDED_INVOKE_TRACE: Final = MappingProxyType( + { + "guardrail": { + "input": {"caller-guardrail": {"invocationMetrics": {"usage": {"topicPolicyUnits": 1}}}}, + "actionReason": "No action.", + } + } +) +GUARDED_INVOKE_METRICS: Final = MappingProxyType( + {"inputTokenCount": 25, "outputTokenCount": 12, "invocationLatency": 300, "firstByteLatency": 90} +) +NOVA_INVOKE_MODEL: Final = "us.amazon.nova-2-lite-v1:0" +LLAMA_INVOKE_MODEL: Final = "meta.llama3-8b-instruct-v1:0" + + +def _event_frame(payload: Mapping[str, object]) -> bytes: + def header(name: str, value: str) -> bytes: + name_b, value_b = name.encode(), value.encode() + return ( + struct.pack("!B", len(name_b)) + name_b + struct.pack("!B", 7) + struct.pack("!H", len(value_b)) + value_b + ) + + payload_b: Final = json.dumps( + {"bytes": base64.b64encode(json.dumps(payload).encode()).decode()}, separators=(",", ":") + ).encode() + headers_b: Final = ( + header(":event-type", "chunk") + header(":content-type", "application/json") + header(":message-type", "event") + ) + prelude: Final = struct.pack("!II", 12 + len(headers_b) + len(payload_b) + 4, len(headers_b)) + prelude_crc: Final = crc32(prelude) & 0xFFFFFFFF + message: Final = struct.pack("!I", prelude_crc) + headers_b + payload_b + return prelude + message + struct.pack("!I", crc32(message, prelude_crc) & 0xFFFFFFFF) + + +def _guarded_invoke_completion(model: str, reply: httpx.Response, stream: bool) -> object: + return litellm.completion( + model=f"bedrock/invoke/{model}", + messages=[{"role": "user", "content": "hi"}], + max_tokens=20, + stream=stream, + guardrailConfig={"guardrailIdentifier": "caller-guardrail", "guardrailVersion": "1", "trace": "enabled"}, + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(lambda _request: reply))), + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + +@pytest.mark.parametrize( + ("model", "reply_body"), + [ + pytest.param( + NOVA_INVOKE_MODEL, + { + "output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 25, "outputTokens": 12, "totalTokens": 37}, + }, + id="nova", + ), + pytest.param( + LLAMA_INVOKE_MODEL, + {"generation": "hi", "prompt_token_count": 25, "generation_token_count": 12, "stop_reason": "stop"}, + id="llama", + ), + ], +) +def test_guarded_non_claude_invoke_completion_keeps_the_guardrail_trace_from_the_client( + model: str, reply_body: Mapping[str, object] +) -> None: + reply: Final = httpx.Response(200, json={**reply_body, "amazon-bedrock-trace": dict(GUARDED_INVOKE_TRACE)}) + + response: Final = _guarded_invoke_completion(model, reply, stream=False) + + assert isinstance(response, litellm.ModelResponse) + assert response.choices[0].message.content == "hi" + assert "amazon-bedrock-trace" not in response.model_dump(), response.model_dump() + + +@pytest.mark.parametrize( + ("model", "events"), + [ + pytest.param( + NOVA_INVOKE_MODEL, + ( + {"messageStart": {"role": "assistant"}}, + {"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "hi"}}}, + {"contentBlockStop": {"contentBlockIndex": 0}}, + {"messageStop": {"stopReason": "end_turn"}}, + { + "metadata": {"usage": {"inputTokens": 25, "outputTokens": 12, "totalTokens": 37}, "metrics": {}}, + "amazon-bedrock-invocationMetrics": dict(GUARDED_INVOKE_METRICS), + "amazon-bedrock-guardrailAction": "NONE", + "amazon-bedrock-trace": dict(GUARDED_INVOKE_TRACE), + }, + ), + id="nova", + ), + pytest.param( + LLAMA_INVOKE_MODEL, + ( + {"generation": "hi", "prompt_token_count": 25, "generation_token_count": 1, "stop_reason": None}, + { + "generation": "", + "prompt_token_count": None, + "generation_token_count": 12, + "stop_reason": "stop", + "amazon-bedrock-invocationMetrics": dict(GUARDED_INVOKE_METRICS), + "amazon-bedrock-trace": dict(GUARDED_INVOKE_TRACE), + }, + ), + id="llama", + ), + ], +) +def test_guarded_non_claude_invoke_stream_yields_only_the_text_and_stop_chunks( + model: str, events: tuple[Mapping[str, object], ...] +) -> None: + reply: Final = httpx.Response( + 200, + headers={"content-type": "application/vnd.amazon.eventstream"}, + content=b"".join(_event_frame(event) for event in events), + ) + + stream: Final = _guarded_invoke_completion(model, reply, stream=True) + + assert isinstance(stream, litellm.CustomStreamWrapper) + assert [ + ([(choice.delta.content, choice.finish_reason) for choice in chunk.choices], chunk.provider_specific_fields or {}) + for chunk in stream + ] == [([("hi", None)], {}), ([(None, "stop")], {})] diff --git a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index 44c0e82c006..c4706b17294 100644 --- a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -23,7 +23,8 @@ from litellm.llms.bedrock.passthrough.transformation import ( BedrockPassthroughConfig, is_bedrock_streaming_endpoint, ) -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.passthrough.main import AsyncPassthroughStreamingResponse, PassthroughStreamingResponse from litellm.types.utils import ModelResponse CONVERSE_MODEL = "anthropic.claude-sonnet-4-5-20250929-v1:0" @@ -597,9 +598,13 @@ def _invoke_chunk(payload: dict) -> bytes: return _event_frame("chunk", {"bytes": base64.b64encode(json.dumps(payload).encode()).decode()}) -def _stream_logging_obj(endpoint: str, dynamic_async_success_callbacks: list[CustomLogger] | None = None) -> Logging: +def _stream_logging_obj( + endpoint: str, + dynamic_async_success_callbacks: list[CustomLogger] | None = None, + model: str = CONVERSE_MODEL, +) -> Logging: logging_obj = Logging( - model=f"bedrock/{CONVERSE_MODEL}", + model=f"bedrock/{model}", messages=[], stream=True, call_type="pass_through_endpoint", @@ -902,6 +907,137 @@ def test_stream_collector_logs_nothing_for_an_unrecognized_endpoint(): assert collector.build_logged_response(_converse_stream_logging_obj()) is None +TIERED_MODEL: Final = "amazon.nova-pro-v1:0" +TIERED_INVOKE_ENDPOINT: Final = f"/model/{TIERED_MODEL}/invoke" +TIERED_INVOKE_STREAM_ENDPOINT: Final = f"/model/{TIERED_MODEL}/invoke-with-response-stream" +SERVICE_TIER_HEADER: Final = "x-amzn-bedrock-service-tier" +TIERED_PROMPT_TOKENS: Final = 25 +TIERED_COMPLETION_TOKENS: Final = 12 +SERVED_TIER_PRICES: Final = ( + pytest.param({SERVICE_TIER_HEADER: "priority"}, "_priority", id="priority"), + pytest.param({}, "", id="standard"), +) + + +def _tier_priced_cost(price_suffix: str) -> float: + prices: Final = litellm.model_cost[TIERED_MODEL] + return ( + TIERED_PROMPT_TOKENS * prices[f"input_cost_per_token{price_suffix}"] + + TIERED_COMPLETION_TOKENS * prices[f"output_cost_per_token{price_suffix}"] + ) + + +def _nova_invoke_text_stream(texts: list[str]) -> bytes: + return b"".join( + _invoke_chunk(event) + for event in ( + {"messageStart": {"role": "assistant"}}, + *({"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": text}}} for text in texts), + {"contentBlockStop": {"contentBlockIndex": 0}}, + {"messageStop": {"stopReason": "end_turn"}}, + { + "metadata": { + "usage": { + "inputTokens": TIERED_PROMPT_TOKENS, + "outputTokens": TIERED_COMPLETION_TOKENS, + "totalTokens": TIERED_PROMPT_TOKENS + TIERED_COMPLETION_TOKENS, + }, + "metrics": {"latencyMs": 120}, + } + }, + ) + ) + + +def _provider_stream_reply(reply_headers: Mapping[str, str], stream: bytes) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "application/vnd.amazon.eventstream", **reply_headers}, + content=stream, + request=httpx.Request("POST", f"https://bedrock-runtime.us-east-1.amazonaws.com{TIERED_INVOKE_STREAM_ENDPOINT}"), + ) + + +@pytest.mark.parametrize(("reply_headers", "price_suffix"), SERVED_TIER_PRICES) +def test_invoke_reply_is_priced_at_the_service_tier_bedrock_served( + reply_headers: Mapping[str, str], price_suffix: str +) -> None: + reply: Final = httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": "Invoke response"}]}}, + "stopReason": "end_turn", + "usage": { + "inputTokens": TIERED_PROMPT_TOKENS, + "outputTokens": TIERED_COMPLETION_TOKENS, + "totalTokens": TIERED_PROMPT_TOKENS + TIERED_COMPLETION_TOKENS, + }, + }, + headers=reply_headers, + ) + + logged: Final = BedrockPassthroughConfig().logging_non_streaming_response( + model=TIERED_MODEL, + custom_llm_provider="bedrock", + httpx_response=reply, + request_data={}, + logging_obj=_stream_logging_obj(TIERED_INVOKE_ENDPOINT, model=TIERED_MODEL), + endpoint=TIERED_INVOKE_ENDPOINT, + ) + + assert litellm.completion_cost(model=f"bedrock/{TIERED_MODEL}", completion_response=logged) == pytest.approx( + _tier_priced_cost(price_suffix) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("reply_headers", "price_suffix"), SERVED_TIER_PRICES) +async def test_invoke_stream_relay_is_priced_at_the_service_tier_bedrock_served( + reply_headers: Mapping[str, str], price_suffix: str +) -> None: + callback: Final = _CaptureAsyncSuccessCallback() + stream: Final = _nova_invoke_text_stream(["Invoke ", "response"]) + reply: Final = _provider_stream_reply(reply_headers, stream) + + async def provider_reply() -> httpx.Response: + return reply + + relay: Final = AsyncPassthroughStreamingResponse( + response=provider_reply(), + litellm_logging_obj=_stream_logging_obj(TIERED_INVOKE_STREAM_ENDPOINT, [callback], model=TIERED_MODEL), + provider_config=BedrockPassthroughConfig(), + ) + relayed: Final = b"".join([chunk async for chunk in relay]) + + assert relayed == stream + assert await relay.wait_for_spend_flush() is True + assert callback.kwargs is not None + assert callback.kwargs["response_cost"] == pytest.approx(_tier_priced_cost(price_suffix)) + + +class _ImmediateExecutor: + def submit(self, fn: Callable[..., object], /, *args: object, **kwargs: object) -> None: + fn(*args, **kwargs) + + +@pytest.mark.parametrize(("reply_headers", "price_suffix"), SERVED_TIER_PRICES) +def test_sync_invoke_stream_relay_is_priced_at_the_service_tier_bedrock_served( + reply_headers: Mapping[str, str], price_suffix: str +) -> None: + logging_obj: Final = _stream_logging_obj(TIERED_INVOKE_STREAM_ENDPOINT, model=TIERED_MODEL) + stream: Final = _nova_invoke_text_stream(["Invoke ", "response"]) + + with patch("litellm.utils.executor", _ImmediateExecutor()): + relayed: Final = b"".join( + PassthroughStreamingResponse( + _provider_stream_reply(reply_headers, stream), logging_obj, BedrockPassthroughConfig() + ) + ) + + assert relayed == stream + assert logging_obj.model_call_details["response_cost"] == pytest.approx(_tier_priced_cost(price_suffix)) + + REQUEST_METADATA_HEADER = "x-amzn-bedrock-request-metadata" REQUEST_METADATA_FIELDS = ["user_api_key_alias", "user_api_key_team_alias", "spend_logs_metadata"] PROXY_IDENTITY = { @@ -977,7 +1113,11 @@ async def test_converse_passthrough_signs_proxy_identity_into_request_metadata( caller_body: Final = {**CONVERSE_BODY, "requestMetadata": {"user_api_key_team_alias": "forged", "ticket": "T-42"}} request: Final = await _sent_passthrough_request( - action, caller_body, PROXY_IDENTITY, request_query_params=query_params + action, + caller_body, + PROXY_IDENTITY, + request_query_params=query_params, + request_headers={f"x-pass-{REQUEST_METADATA_HEADER.upper()}": '{"user_api_key_team_alias":"forged"}'}, ) assert json.loads(request.content) == { @@ -1281,12 +1421,14 @@ async def test_deployment_guardrail_replaces_the_caller_converse_guardrail( {**CONVERSE_BODY, "guardrailConfig": caller_guardrail}, {}, deployment_params={"guardrailConfig": {"guardrailIdentifier": "deployment-guardrail"}}, + request_headers=CALLER_INVOKE_GUARDRAIL_HEADERS, ) assert json.loads(request.content) == { **CONVERSE_BODY, "guardrailConfig": {"guardrailIdentifier": "deployment-guardrail"}, } + assert _invoke_guardrail_headers(request) == {} @pytest.mark.asyncio @@ -1328,6 +1470,59 @@ async def test_passthrough_without_a_deployment_guardrail_sends_the_caller_guard } +CALLER_GUARDRAIL_IDENTITY_HEADERS: Final[Mapping[str, str]] = { + "x-pass-X-Amzn-Bedrock-GuardrailIdentifier": "caller-guardrail", + "x-pass-X-Amzn-Bedrock-GuardrailVersion": "9", +} +SENT_CALLER_GUARDRAIL_IDENTITY: Final[Mapping[str, str]] = { + "x-amzn-bedrock-guardrailidentifier": "caller-guardrail", + "x-amzn-bedrock-guardrailversion": "9", +} + + +SDK_INVOKE_MODEL: Final = "us.anthropic.claude-haiku-4-5-20251001-v1:0" + + +def _sdk_passthrough_call( + model: str, action: str, request_headers: Mapping[str, str], reply_body: Mapping[str, object] +) -> tuple[httpx.Request, object]: + sent: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response(200, json=reply_body, request=request) + + response: Final = litellm.llm_passthrough_route( + model=f"bedrock/{model}", + method="POST", + endpoint=f"/model/{model}/{action}", + json=INVOKE_BODY, + request_headers=dict(request_headers), + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + aws_access_key_id="AKIASDKKEY", + aws_secret_access_key="sdk-secret", + aws_region_name="us-east-1", + ) + return sent[0], response + + +@pytest.mark.parametrize("action", ["invoke", "invoke-with-response-stream"]) +@pytest.mark.parametrize("caller_trace", [None, "DISABLED", "enabled"]) +def test_sdk_invoke_passthrough_sends_the_caller_guardrail_trace_exactly_as_sent( + action: str, caller_trace: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", None) + caller_trace_header: Final = {} if caller_trace is None else {"x-pass-X-Amzn-Bedrock-Trace": caller_trace} + sent_trace_header: Final = {} if caller_trace is None else {"x-amzn-bedrock-trace": caller_trace} + + request, _ = _sdk_passthrough_call( + SDK_INVOKE_MODEL, action, {**CALLER_GUARDRAIL_IDENTITY_HEADERS, **caller_trace_header}, {"ok": True} + ) + + assert _invoke_guardrail_headers(request) == {**SENT_CALLER_GUARDRAIL_IDENTITY, **sent_trace_header} + assert json.loads(request.content) == INVOKE_BODY + + @pytest.mark.asyncio @pytest.mark.parametrize("action", ["invoke", "invoke-with-response-stream"]) @pytest.mark.parametrize("guardrail_config", ["deployment-guardrail", {"guardrailVersion": "3"}]) @@ -1368,3 +1563,244 @@ async def test_passthrough_leaves_actions_without_a_guardrail_field_as_sent(monk assert json.loads(request.content) == count_tokens_body assert _invoke_guardrail_headers(request) == {} + + +GUARDRAIL_UNIT_PRICE: Final = 0.00015 +GUARDRAIL_PRICING: Final = { + "litellm_provider": "bedrock", + "mode": "guardrail", + "guardrail_cost_per_unit": {"topicPolicyUnits": GUARDRAIL_UNIT_PRICE, "contentPolicyUnits": GUARDRAIL_UNIT_PRICE}, +} +_ASSESSMENT_USAGE: Final = {"invocationMetrics": {"usage": {"topicPolicyUnits": 1, "contentPolicyUnits": 1}}} +INVOKE_GUARDRAIL_TRACE: Final = { + "guardrail": { + "input": {"caller-guardrail": _ASSESSMENT_USAGE}, + "outputs": [{"caller-guardrail": _ASSESSMENT_USAGE}], + "actionReason": "No action.", + } +} +BILLED_GUARDRAIL_USAGE: Final = {"topicPolicyUnits": 2, "contentPolicyUnits": 2} +BILLED_GUARDRAIL_COST: Final = 4 * GUARDRAIL_UNIT_PRICE +NOVA_INVOKE_MODEL: Final = "us.amazon.nova-2-lite-v1:0" +LLAMA_INVOKE_MODEL: Final = "meta.llama3-8b-instruct-v1:0" +INVOKE_METRICS: Final = { + "inputTokenCount": 25, + "outputTokenCount": 12, + "invocationLatency": 300, + "firstByteLatency": 90, +} + +INVOKE_REPLIES_WITH_A_GUARDRAIL_TRACE: Final = ( + pytest.param( + NOVA_INVOKE_MODEL, + { + "output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 25, "outputTokens": 12, "totalTokens": 37}, + "amazon-bedrock-guardrailAction": "NONE", + }, + id="nova", + ), + pytest.param( + LLAMA_INVOKE_MODEL, + {"generation": "hi", "prompt_token_count": 25, "generation_token_count": 12, "stop_reason": "stop"}, + id="llama", + ), + pytest.param( + "mistral.mistral-7b-instruct-v0:2", {"outputs": [{"text": "hi", "stop_reason": "stop"}]}, id="mistral" + ), + pytest.param( + CONVERSE_MODEL, + { + "id": "msg-1", + "type": "message", + "role": "assistant", + "model": CONVERSE_MODEL, + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 25, "output_tokens": 12}, + }, + id="claude", + ), +) + +INVOKE_STREAMS_WITH_A_GUARDRAIL_TRACE: Final = ( + pytest.param( + NOVA_INVOKE_MODEL, + ( + {"messageStart": {"role": "assistant"}}, + {"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "hi"}}}, + {"contentBlockStop": {"contentBlockIndex": 0}}, + {"messageStop": {"stopReason": "end_turn"}}, + { + "metadata": { + "usage": {"inputTokens": 25, "outputTokens": 12, "totalTokens": 37}, + "metrics": {}, + "trace": {}, + }, + "amazon-bedrock-invocationMetrics": INVOKE_METRICS, + "amazon-bedrock-guardrailAction": "NONE", + "amazon-bedrock-trace": INVOKE_GUARDRAIL_TRACE, + }, + ), + id="nova", + ), + pytest.param( + LLAMA_INVOKE_MODEL, + ( + {"generation": "hi", "prompt_token_count": 25, "generation_token_count": 1, "stop_reason": None}, + { + "generation": "", + "prompt_token_count": None, + "generation_token_count": 12, + "stop_reason": "stop", + "amazon-bedrock-invocationMetrics": INVOKE_METRICS, + "amazon-bedrock-trace": INVOKE_GUARDRAIL_TRACE, + }, + ), + id="llama", + ), + pytest.param( + CONVERSE_MODEL, + ( + { + "type": "message_start", + "message": { + "id": "msg-1", + "type": "message", + "role": "assistant", + "model": CONVERSE_MODEL, + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 25, "output_tokens": 1}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 12}}, + { + "type": "message_stop", + "amazon-bedrock-invocationMetrics": INVOKE_METRICS, + "amazon-bedrock-trace": INVOKE_GUARDRAIL_TRACE, + }, + ), + id="claude", + ), +) + + +def _billed_guardrails(standard_logging_object: object) -> list[tuple[object, object]]: + assert isinstance(standard_logging_object, dict) + return [ + (entry["guardrail_usage"], entry["guardrail_cost"]) + for entry in standard_logging_object["guardrail_information"] or [] + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "reply_body"), INVOKE_REPLIES_WITH_A_GUARDRAIL_TRACE) +async def test_invoke_reply_bills_the_guardrail_bedrock_traced( + model: str, reply_body: Mapping[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock/guardrails", GUARDRAIL_PRICING) + endpoint: Final = f"/model/{model}/invoke" + callback: Final = _CaptureAsyncSuccessCallback() + logging_obj: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="allm_passthrough_route", + start_time=datetime(2025, 1, 1), + litellm_call_id="call-1", + function_id="fn-1", + dynamic_async_success_callbacks=[callback], + ) + logging_obj.update_environment_variables(litellm_params={}, optional_params={}) + logging_obj.model_call_details["custom_llm_provider"] = "bedrock" + logging_obj.model_call_details["endpoint"] = endpoint + reply: Final = httpx.Response( + 200, + json={**reply_body, "amazon-bedrock-trace": INVOKE_GUARDRAIL_TRACE}, + request=httpx.Request("POST", f"https://bedrock-runtime.us-east-1.amazonaws.com{endpoint}"), + ) + + await logging_obj.async_success_handler( + result=reply, start_time=datetime(2025, 1, 1), end_time=datetime(2025, 1, 1, 0, 0, 1) + ) + + assert callback.kwargs is not None + assert _billed_guardrails(callback.kwargs["standard_logging_object"]) == [ + (BILLED_GUARDRAIL_USAGE, pytest.approx(BILLED_GUARDRAIL_COST)) + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "events"), INVOKE_STREAMS_WITH_A_GUARDRAIL_TRACE) +async def test_invoke_stream_relay_bills_the_guardrail_bedrock_traced( + model: str, events: tuple[dict[str, object], ...], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem(litellm.model_cost, "bedrock/guardrails", GUARDRAIL_PRICING) + endpoint: Final = f"/model/{model}/invoke-with-response-stream" + callback: Final = _CaptureAsyncSuccessCallback() + stream: Final = b"".join(_invoke_chunk(event) for event in events) + + async def provider_reply() -> httpx.Response: + return _provider_stream_reply({}, stream) + + relay: Final = AsyncPassthroughStreamingResponse( + response=provider_reply(), + litellm_logging_obj=_stream_logging_obj(endpoint, [callback], model=model), + provider_config=BedrockPassthroughConfig(), + ) + relayed: Final = b"".join([chunk async for chunk in relay]) + + assert relayed == stream + assert await relay.wait_for_spend_flush() is True + assert callback.kwargs is not None + assert _billed_guardrails(callback.kwargs["standard_logging_object"]) == [ + (BILLED_GUARDRAIL_USAGE, pytest.approx(BILLED_GUARDRAIL_COST)) + ] + + +class _CaptureSyncSuccessCallback(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.standard_logging_objects: list[object] = [] + + def log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.standard_logging_objects.append(kwargs.get("standard_logging_object")) + + +@pytest.mark.parametrize( + ("model", "reply_body"), + [param for param in INVOKE_REPLIES_WITH_A_GUARDRAIL_TRACE if param.id in {"nova", "claude"}], +) +def test_sdk_invoke_passthrough_bills_the_caller_guardrail_bedrock_traced_once( + model: str, reply_body: Mapping[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", None) + monkeypatch.setitem(litellm.model_cost, "bedrock/guardrails", GUARDRAIL_PRICING) + callback: Final = _CaptureSyncSuccessCallback() + monkeypatch.setattr(litellm, "callbacks", [callback]) + traced_reply: Final = {**reply_body, "amazon-bedrock-trace": INVOKE_GUARDRAIL_TRACE} + + with patch("litellm.utils.executor", _ImmediateExecutor()): + request, response = _sdk_passthrough_call( + model, + "invoke", + {**CALLER_GUARDRAIL_IDENTITY_HEADERS, "x-pass-X-Amzn-Bedrock-Trace": "ENABLED"}, + traced_reply, + ) + + assert isinstance(response, httpx.Response) + assert response.json() == traced_reply + assert _invoke_guardrail_headers(request) == { + **SENT_CALLER_GUARDRAIL_IDENTITY, + "x-amzn-bedrock-trace": "ENABLED", + } + assert [_billed_guardrails(logged) for logged in callback.standard_logging_objects] == [ + [(BILLED_GUARDRAIL_USAGE, pytest.approx(BILLED_GUARDRAIL_COST))] + ] diff --git a/tests/unit/passthrough/test_utils.py b/tests/unit/passthrough/test_utils.py new file mode 100644 index 00000000000..02a1c70f8cb --- /dev/null +++ b/tests/unit/passthrough/test_utils.py @@ -0,0 +1,78 @@ +from typing import Final + +import httpx +import pytest + +from litellm.passthrough.utils import _PASS_THROUGH_PROTECTED_HEADERS, BasePassthroughUtils + + +def test_pass_through_prefixed_headers_keeps_only_the_x_pass_headers_forwarded_upstream() -> None: + inbound: Final = { + "authorization": "Bearer sk-litellm-key", + "x-litellm-api-key": "sk-litellm-key", + "content-type": "application/json", + "x-amzn-bedrock-guardrailidentifier": "native-guardrail", + "x-passthrough-id": "not-a-pass-header", + "x-pass-authorization": "Bearer upstream-secret", + "X-Pass-X-Api-Key": "upstream-api-key", + "x-pass-X-Amz-Security-Token": "forged-session-token", + "x-pass-x-amzn-bedrock-trace": "DISABLED", + "X-Pass-X-Amzn-Bedrock-GuardrailIdentifier": "caller-guardrail", + "X-PASS-ANTHROPIC-BETA": "beta-1", + } + + selected: Final = BasePassthroughUtils.pass_through_prefixed_headers(inbound) + + assert dict(selected) == { + "x-pass-x-amzn-bedrock-trace": "DISABLED", + "X-Pass-X-Amzn-Bedrock-GuardrailIdentifier": "caller-guardrail", + "X-PASS-ANTHROPIC-BETA": "beta-1", + } + + +@pytest.mark.parametrize( + "header_name", + [ + *(f"x-pass-{name}" for name in sorted(_PASS_THROUGH_PROTECTED_HEADERS)), + "X-Pass-Authorization", + "x-pass-X-Amz-Date", + "x-pass-x-amz-security-token", + "x-pass-X-Amzn-Bedrock-Trace", + "x-pass-anthropic-beta", + "x-pass-cookie", + "x-passthrough-id", + "authorization", + ], +) +def test_pass_through_prefixed_headers_select_exactly_the_headers_forwarded_upstream(header_name: str) -> None: + inbound: Final = {header_name: "caller-value"} + + selected: Final = BasePassthroughUtils.pass_through_prefixed_headers(inbound) + forwarded: Final = BasePassthroughUtils.forward_headers_from_request( + request_headers=dict(inbound), headers={}, forward_headers=False + ) + + assert {name[len("x-pass-") :].lower(): value for name, value in selected.items()} == forwarded + + +@pytest.mark.parametrize( + ("default_query_params", "expected"), + [ + pytest.param( + {"api-version": "default", "region": "default"}, + {"api-version": "url", "region": "default", "shared": "request", "tag": ["a", "b"]}, + id="with_defaults", + ), + pytest.param(None, {"api-version": "url", "shared": "request", "tag": ["a", "b"]}, id="without_defaults"), + ], +) +def test_merged_query_parameters_let_the_request_override_the_url_and_the_url_override_defaults( + default_query_params: dict[str, str | list] | None, expected: dict[str, str | list[str]] +) -> None: + merged: Final = BasePassthroughUtils.get_merged_query_parameters( + existing_url=httpx.URL("https://upstream.test/v1?api-version=url&shared=url&tag=a&tag=b"), + request_query_params={"shared": "request"}, + default_query_params=default_query_params, + ) + + assert merged == expected diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 6e551f8ff0a..8e88fd7449a 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -1350,11 +1350,23 @@ async def test_mistral_passthrough_accepts_multipart_without_json_parsing(): assert captured_kwargs["custom_headers"] == {"Authorization": "Bearer mistral-test-key"} +BEDROCK_PROXY_MODEL: Final = "anthropic.claude-sonnet-5-5" +BEDROCK_PROXY_CALLER_BEARER: Final = "sk-caller-virtual-key" +BEDROCK_PROXY_ROUTES: Final = pytest.mark.parametrize( + ("path_model", "uses_router"), + [ + pytest.param("bedrock-sonnet", True, id="router_model"), + pytest.param(BEDROCK_PROXY_MODEL, False, id="direct_model"), + ], +) + + class TestBedrockLLMProxyRoute: @pytest.mark.asyncio async def test_bedrock_llm_proxy_route_application_inference_profile(self): mock_request = Mock() mock_request.method = "POST" + mock_request.headers = {} mock_response = Mock() mock_user_api_key_dict = Mock() mock_request_body = {"messages": [{"role": "user", "content": "test"}]} @@ -1397,6 +1409,7 @@ class TestBedrockLLMProxyRoute: async def test_bedrock_llm_proxy_route_regular_model(self): mock_request = Mock() mock_request.method = "POST" + mock_request.headers = {} mock_response = Mock() mock_user_api_key_dict = Mock() mock_request_body = {"messages": [{"role": "user", "content": "test"}]} @@ -1444,6 +1457,7 @@ class TestBedrockLLMProxyRoute: ) -> None: mock_request: Final = Mock() mock_request.method = "POST" + mock_request.headers = {} mock_processor: Final = Mock() mock_processor.base_passthrough_process_llm_request = AsyncMock(return_value="success") request_body: Final = {"messages": [{"role": "user", "content": "test"}]} @@ -1484,6 +1498,7 @@ class TestBedrockLLMProxyRoute: ) -> None: mock_request: Final = Mock() mock_request.method = "POST" + mock_request.headers = {} mock_processor: Final = Mock() mock_processor.base_passthrough_process_llm_request = AsyncMock(return_value="success") request_body: Final = {"messages": [{"role": "user", "content": "test"}]} @@ -1515,6 +1530,91 @@ class TestBedrockLLMProxyRoute: processing_data: Final = processor_constructor.call_args.kwargs["data"] _assert_bedrock_processing_data_classification(processing_data, is_streaming) + @BEDROCK_PROXY_ROUTES + def test_invoke_signs_the_caller_x_pass_header_and_sends_no_caller_credential( + self, monkeypatch: pytest.MonkeyPatch, httpx_transport: None, path_model: str, uses_router: bool + ) -> None: + caller_cookie: Final = "proxy_session=caller-secret" + + sent, _ = _bedrock_invoke_through_proxy( + monkeypatch, + path_model, + uses_router, + { + "x-pass-X-Amzn-Bedrock-Service-Tier": "priority", + "Authorization": f"Bearer {BEDROCK_PROXY_CALLER_BEARER}", + "Cookie": caller_cookie, + }, + ) + + signed_headers: Final = sent.headers["authorization"].split("SignedHeaders=")[1].split(",")[0].split(";") + assert sent.headers["x-amzn-bedrock-service-tier"] == "priority" + assert "x-amzn-bedrock-service-tier" in signed_headers + assert [name for name in sent.headers if name.startswith("x-pass-") or name == "cookie"] == [] + assert [ + value for value in sent.headers.values() if BEDROCK_PROXY_CALLER_BEARER in value or caller_cookie in value + ] == [] + + @BEDROCK_PROXY_ROUTES + @pytest.mark.parametrize("caller_trace", [None, "DISABLED", "enabled"]) + def test_invoke_sends_the_caller_guardrail_trace_exactly_as_sent( + self, + monkeypatch: pytest.MonkeyPatch, + httpx_transport: None, + path_model: str, + uses_router: bool, + caller_trace: str | None, + ) -> None: + trace_header: Final = {} if caller_trace is None else {"x-pass-X-Amzn-Bedrock-Trace": caller_trace} + + sent, _ = _bedrock_invoke_through_proxy( + monkeypatch, + path_model, + uses_router, + { + "x-pass-X-Amzn-Bedrock-GuardrailIdentifier": "caller-guardrail", + "x-pass-X-Amzn-Bedrock-GuardrailVersion": "9", + **trace_header, + }, + ) + + assert ( + sent.headers["x-amzn-bedrock-guardrailidentifier"], + sent.headers["x-amzn-bedrock-guardrailversion"], + sent.headers.get("x-amzn-bedrock-trace"), + ) == ("caller-guardrail", "9", caller_trace) + + @BEDROCK_PROXY_ROUTES + def test_invoke_keeps_protected_x_pass_values_from_bedrock_guardrails_and_the_logged_body( + self, monkeypatch: pytest.MonkeyPatch, httpx_transport: None, path_model: str, uses_router: bool + ) -> None: + protected_values: Final = ( + "Bearer upstream-secret", + "upstream-api-key", + "forged-session-token", + "session=upstream-cookie", + "Basic upstream-proxy-secret", + ) + + sent, guardrail_data = _bedrock_invoke_through_proxy( + monkeypatch, + path_model, + uses_router, + { + "x-pass-authorization": protected_values[0], + "x-pass-x-api-key": protected_values[1], + "x-pass-X-Amz-Security-Token": protected_values[2], + "x-pass-cookie": protected_values[3], + "x-pass-proxy-authorization": protected_values[4], + "x-pass-X-Amzn-Bedrock-GuardrailIdentifier": "caller-guardrail", + }, + ) + + logged_body: Final = json.dumps(guardrail_data["proxy_server_request"]["body"], default=str) + assert sent.headers["x-amzn-bedrock-guardrailidentifier"] == "caller-guardrail" + assert guardrail_data["request_headers"] == {"x-pass-x-amzn-bedrock-guardrailidentifier": "caller-guardrail"} + assert [value for value in protected_values if value in str(sent.headers.raw) or value in logged_body] == [] + @pytest.mark.asyncio @pytest.mark.parametrize( ("model_id", "action"), @@ -1526,6 +1626,7 @@ class TestBedrockLLMProxyRoute: async def test_bedrock_direct_model_id_does_not_imply_streaming(self, model_id: str, action: str) -> None: mock_request: Final = Mock() mock_request.method = "POST" + mock_request.headers = {} mock_processor: Final = Mock() mock_processor.base_passthrough_process_llm_request = AsyncMock(return_value="success") request_body: Final = {"messages": [{"role": "user", "content": "test"}]} @@ -1562,6 +1663,7 @@ class TestBedrockLLMProxyRoute: async def test_bedrock_router_model_id_does_not_imply_streaming(self, model_id: str, action: str) -> None: mock_request: Final = Mock() mock_request.method = "POST" + mock_request.headers = {} mock_processor: Final = Mock() mock_processor.base_passthrough_process_llm_request = AsyncMock(return_value="success") request_body: Final = {"messages": [{"role": "user", "content": "test"}]} @@ -1865,6 +1967,84 @@ class TestBedrockLLMProxyRoute: assert "Blocked by guardrail" in str(exc_info.value.detail) +class _RequestDataCaptureGuardrail(CustomGuardrail): + def __init__(self) -> None: + super().__init__(guardrail_name="request-data-capture", default_on=True, event_hook="pre_call") + self.data: dict[str, object] = {} + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict[str, object], + call_type: CallTypesLiteral, + ) -> dict[str, object]: + self.data = data + return data + + +def _bedrock_invoke_through_proxy( + monkeypatch: pytest.MonkeyPatch, path_model: str, uses_router: bool, headers: Mapping[str, str] +) -> tuple[httpx.Request, dict[str, object]]: + from litellm.proxy import proxy_server + + for ambient in (*TestBedrockAgentRuntimePassthroughVirtualKeyLeak.AMBIENT_AWS_ENV, "AWS_BEDROCK_RUNTIME_ENDPOINT"): + monkeypatch.delenv(ambient, raising=False) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATESTACCESSKEY") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + router: Final = ( + litellm.Router( + model_list=[ + { + "model_name": "bedrock-sonnet", + "litellm_params": {"model": f"bedrock/{BEDROCK_PROXY_MODEL}", "aws_region_name": "us-east-1"}, + } + ] + ) + if uses_router + else None + ) + guardrail: Final = _RequestDataCaptureGuardrail() + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + monkeypatch.setitem( + proxy_server.app.dependency_overrides, + user_api_key_auth, + lambda: UserAPIKeyAuth(api_key=BEDROCK_PROXY_CALLER_BEARER), + ) + + with respx.mock(assert_all_called=True) as upstream: + invoke: Final = upstream.post( + f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{BEDROCK_PROXY_MODEL}/invoke" + ).mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_bedrock", + "type": "message", + "role": "assistant", + "model": BEDROCK_PROXY_MODEL, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 5, "output_tokens": 1}, + }, + ) + ) + response: Final = TestClient(proxy_server.app).post( + f"/bedrock/model/{path_model}/invoke", + json={ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + }, + headers=dict(headers), + ) + + assert response.status_code == 200, response.text + return invoke.calls.last.request, guardrail.data + + _THROTTLE_TYPE: Final = "ThrottlingException:http://internal.amazon.com/coral/com.amazon.bedrock/" _THROTTLE_BODY: Final = b'{"message":"Too many requests, please wait before trying again."}' diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 28d2a4fa7ad..e8ee31b27f5 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -934,6 +934,38 @@ async def test_add_litellm_data_to_request_body_snapshot_excludes_secret_fields( ) +@pytest.mark.asyncio +async def test_chat_completions_body_field_named_request_headers_stays_in_the_logged_body(): + from litellm.proxy.proxy_server import ProxyConfig + + client_field: Final = {"x-client-trace": "client-supplied"} + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/chat/completions", + "scheme": "http", + "server": ("localhost", 4000), + "query_string": b"", + "headers": [(b"authorization", b"Bearer sk-litellm-key")], + } + ) + + enriched: Final = await add_litellm_data_to_request( + data={ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hello"}], + "request_headers": client_field, + }, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=ProxyConfig(), + general_settings={}, + ) + + assert enriched["proxy_server_request"]["body"]["request_headers"] == client_field + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_body_snapshot_excludes_proxy_server_request(): """Regression: the body snapshot used to include the proxy_server_request From a7696a2bc2fe18b18e8617a3231ebc6a01c04e68 Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:29:32 -0800 Subject: [PATCH 041/189] fix(proxy): send the streamed Responses fallback metadata as a typed Responses event (#50804) * fix(proxy): send the streamed Responses fallback metadata as a typed Responses event With expose_fallback_errors_to_caller on and include_fallback_errors set, a streamed /v1/responses call that fell back sent its fallback metadata as a chat.completion.chunk with no type, so the OpenAI SDK read it as an audio delta event with type None. On the Responses route the frame is now {"type": "litellm.fallback_metadata", "litellm_fallback": {...}} and goes through the same Responses framing as every other event, so it gets an event: line whenever its neighbours do. The chat route keeps its chat chunk unchanged * test(proxy): stop pinning the Responses stream terminator in the fallback metadata test Main ends Responses streams without data: [DONE] since #47980, so the test reads the event frames whether or not a terminator follows them, and passes both before and after that change merges * fix(proxy): keep the top-level model on the Responses fallback metadata event * fix(proxy): keep the fallback metadata id on the Responses event and omit an empty model * fix(proxy): leave the top-level model off the Responses fallback event when raw model names are on The complexity router's return_raw_model_name leaves every other event unstamped, so the fallback event now matches. A new test streams a raw upstream frame before the event to cover how main names it. * chore: push again after GitHub dropped the previous push event * fix(proxy): send the Responses fallback event without an event line so older OpenAI SDKs read it --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 2 +- litellm/proxy/proxy_server.py | 32 +++-- .../test_include_fallback_errors_wire.py | 39 +++++- .../proxy_server/test_streaming_helpers.py | 123 ++++++++++++++++++ 4 files changed, 184 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9e2b49ef692..bfa559f86c5 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -381,7 +381,7 @@ def resolve_litellm_call_id(client_call_id: str | None) -> str: return str(uuid.uuid4()) -def should_return_raw_model_name(request_data: dict[str, object]) -> bool: +def should_return_raw_model_name(request_data: Mapping[str, object]) -> bool: return any( isinstance(metadata, dict) and metadata.get(RETURN_RAW_MODEL_NAME_METADATA_KEY) is True for metadata in (request_data.get("metadata"), request_data.get("litellm_metadata")) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f69d9bd4724..be90b028cf4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10303,23 +10303,34 @@ def _get_streaming_fallback_metadata( return True, None, fallback_errors +_FALLBACK_METADATA_ID: Final = "litellm-fallback-metadata" +_RESPONSES_FALLBACK_METADATA_EVENT_TYPE: Final = "litellm.fallback_metadata" + + def _format_fallback_metadata_sse_event( *, fallback_model: str | None, fallback_errors: list[dict[str, object]], + responses_stream: bool, + return_raw_model_name: bool, ) -> str: - import time - + litellm_fallback: Final = {"fallback_model": fallback_model, "errors": fallback_errors} + if responses_stream: + stamped_model: Final = None if return_raw_model_name else fallback_model + event: Final = { + "type": _RESPONSES_FALLBACK_METADATA_EVENT_TYPE, + "id": _FALLBACK_METADATA_ID, + **({"model": stamped_model} if stamped_model else {}), + "litellm_fallback": litellm_fallback, + } + return f"data: {json.dumps(event)}\n\n" payload: Final = { - "id": "litellm-fallback-metadata", + "id": _FALLBACK_METADATA_ID, "object": "chat.completion.chunk", "created": int(time.time()), "model": fallback_model or "", "choices": [], - "litellm_fallback": { - "fallback_model": fallback_model, - "errors": fallback_errors, - }, + "litellm_fallback": litellm_fallback, } return f"data: {json.dumps(payload)}\n\n" @@ -10328,7 +10339,7 @@ def _restamp_streaming_chunk_model( *, chunk: object, requested_model_from_client: str, - request_data: dict, + request_data: Mapping[str, object], model_mismatch_logged: bool, fallback_was_attempted: bool = False, fallback_model_from_metadata: str | None = None, @@ -10906,6 +10917,7 @@ async def async_data_generator( model_mismatch_logged = False fallback_metadata_event_sent = False include_fallback_errors: Final = _should_include_fallback_errors(request_data) + return_raw_model_name: Final = should_return_raw_model_name(request_data) # Use a running string instead of list + join to avoid O(n^2) overhead. # Previously "".join(str_so_far_parts) was called every chunk, re-joining # the entire accumulated response. String += is O(n) amortized total. @@ -11003,6 +11015,8 @@ async def async_data_generator( yield _format_fallback_metadata_sse_event( fallback_model=fallback_model_from_metadata, fallback_errors=fallback_errors, + responses_stream=error_state is not None, + return_raw_model_name=return_raw_model_name, ) fallback_metadata_event_sent = True continue @@ -11059,6 +11073,8 @@ async def async_data_generator( yield _format_fallback_metadata_sse_event( fallback_model=fallback_model_from_metadata, fallback_errors=fallback_errors, + responses_stream=error_state is not None, + return_raw_model_name=return_raw_model_name, ) fallback_metadata_event_sent = True diff --git a/tests/integration/routing/test_include_fallback_errors_wire.py b/tests/integration/routing/test_include_fallback_errors_wire.py index 2f48bc43a22..a41fefd5ce1 100644 --- a/tests/integration/routing/test_include_fallback_errors_wire.py +++ b/tests/integration/routing/test_include_fallback_errors_wire.py @@ -16,7 +16,14 @@ from urllib.parse import unquote import httpx import pytest import yaml -from integration._support.anthropic_sse import error_body, message_json, message_stream, parse_sse, stream_reply +from integration._support.anthropic_sse import ( + SseEvent, + error_body, + message_json, + message_stream, + parse_sse, + stream_reply, +) from integration._support.client import Gateway, eventually, gateway_from_environment from integration._support.database import read_rows from integration._support.openai_wire import chat_reply, responses_reply @@ -34,6 +41,7 @@ _API_KEY: Final = "synthetic-fallback-errors-key" _ROUTER_ONLY_KEYS: Final = ("include_fallback_errors", "silent_model") _ANSWER: Final = "answered by" _UNAUTHORIZED_MESSAGE: Final = "scripted 401: the primary key was revoked" +_RESPONSES_FALLBACK_METADATA_EVENT: Final = "litellm.fallback_metadata" _NONCE: Final = re.compile(r"nonce=([0-9a-f]{32})") _JSON: Final = TypeAdapter(dict[str, JsonValue]) _ERRORS: Final = TypeAdapter(list[dict[str, JsonValue]]) @@ -445,11 +453,25 @@ def exposed(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Exposed]: yield _Exposed(proxy, wire) +def _mention_the_outage(errors: list[dict[str, JsonValue]]) -> bool: + return len(errors) == 1 and _UNAUTHORIZED_MESSAGE in str(errors[0]["message"]) + + def _errors_mention_the_outage(errors_header: str | None) -> bool: if errors_header is None: return False - errors: Final = _ERRORS.validate_json(errors_header) - return len(errors) == 1 and _UNAUTHORIZED_MESSAGE in str(errors[0]["message"]) + return _mention_the_outage(_ERRORS.validate_json(errors_header)) + + +def _fallback_reports(events: tuple[SseEvent, ...]) -> tuple[SseEvent, ...]: + return tuple(event for event in events if "litellm_fallback" in event.data) + + +def _reports_the_outage_served_by_the_backup(report: SseEvent) -> bool: + fallback: Final = _JSON.validate_python(report.data["litellm_fallback"]) + return fallback["fallback_model"] == _EXPOSED_OPENAI_BACKUP and _mention_the_outage( + _ERRORS.validate_python(fallback["errors"]) + ) @pytest.mark.parametrize("stream", _STREAMING) @@ -473,6 +495,10 @@ def test_exposed_chat_fallback_reports_the_errors_without_putting_the_flag_on_th if not stream: assert observed.attempted == "1", observed assert _errors_mention_the_outage(observed.errors_header), observed + return + reports: Final = _fallback_reports(parse_sse(observed.text)) + assert tuple(report.data.get("object") for report in reports) == ("chat.completion.chunk",), observed + assert _reports_the_outage_served_by_the_backup(reports[0]), observed def test_exposed_chat_without_a_fallback_keeps_the_flag_off_the_wire_and_the_errors_header_off( @@ -561,6 +587,13 @@ def test_exposed_responses_fallback_keeps_the_flag_off_the_wire(exposed: _Expose assert observed.served_id == _identity("responses", _EXPOSED_OPENAI_BACKUP, sent.nonce), observed assert observed.hit == (_EXPOSED_OPENAI_PRIMARY, _EXPOSED_OPENAI_BACKUP), observed assert observed.leaks == (), observed + if not stream: + return + events: Final = parse_sse(observed.text) + reports: Final = _fallback_reports(events) + assert tuple(report.data.get("type") for report in reports) == (_RESPONSES_FALLBACK_METADATA_EVENT,), observed + assert _reports_the_outage_served_by_the_backup(reports[0]), observed + assert all("type" in event.data for event in events if event.data), observed @pytest.mark.parametrize("stream", _STREAMING) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index fdf58ec2fd7..4136a4acb16 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -874,6 +874,8 @@ def test_format_fallback_metadata_sse_event(): event = _format_fallback_metadata_sse_event( fallback_model="fallback-model", fallback_errors=fallback_errors, + responses_stream=False, + return_raw_model_name=False, ) assert isinstance(event, str) @@ -1469,6 +1471,127 @@ async def test_async_data_generator_emits_fallback_error_metadata_event(monkeypa assert isinstance(metadata_payload["created"], int) +@pytest.mark.asyncio +async def test_chat_stream_sends_fallback_metadata_in_place_of_a_stripped_usage_artifact( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_logging_flags(monkeypatch) + monkeypatch.setitem(ps.general_settings, "expose_fallback_errors_to_caller", True) + fallback_errors: Final = [ + {"message": "litellm.RateLimitError: upstream limited request", "type": "RateLimitError", "code": "429"} + ] + usage_artifact: Final = ModelResponseStream( + id="chatcmpl-test", choices=[], created=0, model="openai/internal-fallback", object="chat.completion.chunk" + ) + upstream: Final = _FakeStream( + [usage_artifact, _simple_chunk(model="openai/internal-fallback", content="hello")], + hidden_params={ + "additional_headers": { + "x-litellm-attempted-fallbacks": 1, + "x-litellm-model-group": "fallback-model", + "x-litellm-fallback-errors": json.dumps(fallback_errors), + } + }, + ) + + frames: Final = tuple( + [ + frame.decode() if isinstance(frame, bytes) else frame + async for frame in async_data_generator( + response=upstream, + user_api_key_dict=_user_auth(), + request_data={ + "model": "primary-model", + "include_fallback_errors": True, + "_litellm_strip_stream_usage": True, + }, + ) + ] + ) + payloads: Final = tuple(json.loads(frame.removeprefix("data: ")) for frame in frames[:-1]) + + assert frames[-1] == "data: [DONE]\n\n" + assert len(payloads) == 2 + assert payloads[0]["object"] == "chat.completion.chunk" + assert payloads[0]["choices"] == [] + assert payloads[0]["litellm_fallback"] == {"fallback_model": "fallback-model", "errors": fallback_errors} + assert payloads[1]["choices"][0]["delta"]["content"] == "hello" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fallback_model", ["fallback-model", None]) +@pytest.mark.parametrize("return_raw_model_name", [False, True]) +async def test_responses_stream_sends_fallback_metadata_as_a_typed_event_without_an_event_line( + monkeypatch: pytest.MonkeyPatch, + fallback_model: str | None, + return_raw_model_name: bool, +) -> None: + _patch_logging_flags(monkeypatch) + monkeypatch.setitem(ps.general_settings, "expose_fallback_errors_to_caller", True) + fallback_errors: Final = [ + { + "message": "litellm.AuthenticationError: Incorrect API key provided", + "type": "AuthenticationError", + "param": None, + "code": "401", + } + ] + response: Final = ResponsesAPIResponse(id="resp_fallback", created_at=1, model="fallback-model", output=[]) + upstream: Final = _FakeStream( + [ + ResponseCreatedEvent.model_validate( + {"type": "response.created", "sequence_number": 0, "response": response} + ), + ResponseCompletedEvent.model_validate( + {"type": "response.completed", "sequence_number": 1, "response": response} + ), + ], + hidden_params={ + "additional_headers": { + "x-litellm-attempted-fallbacks": 1, + **({"x-litellm-model-group": fallback_model} if fallback_model else {}), + "x-litellm-fallback-errors": json.dumps(fallback_errors), + } + }, + ) + + frames: Final = tuple( + [ + frame.decode() if isinstance(frame, bytes) else frame + async for frame in async_data_generator( + response=upstream, + user_api_key_dict=_user_auth(), + request_data={ + "model": "primary-model", + "include_fallback_errors": True, + "metadata": {RETURN_RAW_MODEL_NAME_METADATA_KEY: return_raw_model_name}, + }, + responses_stream_errors=True, + ) + ] + ) + event_frames: Final = tuple(frame for frame in frames if frame != "data: [DONE]\n\n") + payloads: Final = tuple( + json.loads(next(line.removeprefix("data: ") for line in frame.splitlines() if line.startswith("data: "))) + for frame in event_frames + ) + + assert tuple(payload.get("type") for payload in payloads) == ( + "response.created", + "litellm.fallback_metadata", + "response.completed", + ) + stamped_model: Final = None if return_raw_model_name else fallback_model + assert payloads[1] == { + "type": "litellm.fallback_metadata", + "id": "litellm-fallback-metadata", + **({"model": stamped_model} if stamped_model else {}), + "litellm_fallback": {"fallback_model": fallback_model, "errors": fallback_errors}, + } + assert {payload.get("model") for payload in payloads} == {stamped_model} + assert event_frames[1] == f"data: {json.dumps(payloads[1])}\n\n" + + @pytest.mark.asyncio async def test_async_data_generator_skips_fallback_error_event_without_opt_in( monkeypatch, From 0fd6dd4168dd2c95c929c7d36d7100dee589dd9b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:55:53 +0000 Subject: [PATCH 042/189] fix(proxy): recheck missing users when resolving session grants (#51030) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/resolvers/grants.py | 3 ++ .../unit/proxy/auth/test_resolvers_grants.py | 42 +++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/auth/resolvers/grants.py b/litellm/proxy/auth/resolvers/grants.py index c66761a8459..9e445cee090 100644 --- a/litellm/proxy/auth/resolvers/grants.py +++ b/litellm/proxy/auth/resolvers/grants.py @@ -28,6 +28,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_checks import ( TeamNotFoundError, + forget_missing_user, get_team_membership, get_team_object, get_user_object, @@ -246,6 +247,8 @@ class GrantResolver: return ResolvedGrants(user_object, team_object, membership, effective_user_id) async def _user(self, lookup: UserLookup) -> LiteLLM_UserTable | None: + if lookup.user_id is not None: + forget_missing_user(lookup.user_id) return await self._load_user( user_id=lookup.user_id, prisma_client=self._prisma, diff --git a/tests/unit/proxy/auth/test_resolvers_grants.py b/tests/unit/proxy/auth/test_resolvers_grants.py index e61269ec1c7..e0e33da5191 100644 --- a/tests/unit/proxy/auth/test_resolvers_grants.py +++ b/tests/unit/proxy/auth/test_resolvers_grants.py @@ -1,6 +1,10 @@ -from fastapi import HTTPException +from types import SimpleNamespace +from typing import TYPE_CHECKING, Final, cast +from unittest.mock import AsyncMock + import httpx import pytest +from fastapi import HTTPException from litellm.proxy._types import ( LiteLLM_TeamMembership, @@ -8,8 +12,7 @@ from litellm.proxy._types import ( LiteLLM_UserTable, ProxyException, ) -from litellm.proxy.auth.auth_checks import TeamNotFoundError, UserNotFoundError -from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.auth.auth_checks import TeamNotFoundError, UserNotFoundError, forget_missing_user from litellm.proxy.auth.resolvers.grants import ( GrantResolver, LookupDegraded, @@ -21,10 +24,43 @@ from litellm.proxy.auth.resolvers.grants import ( raise_public, user_models, ) +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache USER_ID = "user-1" TEAM_ID = "team-1" +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +@pytest.mark.parametrize("user_exists", (True, False), ids=("created-on-another-worker", "still-missing")) +async def test_session_grants_recheck_a_recently_missing_user( + user_exists: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("time.time", lambda: 1_700_000_000.0) + user_id: Final = "session-user-after-miss" + row: Final = LiteLLM_UserTable(user_id=user_id, user_role="internal_user") + read: Final = AsyncMock(return_value=None) + database: Final = cast( + "PrismaClient", + SimpleNamespace( + db=SimpleNamespace( + litellm_usertable=SimpleNamespace(find_unique=read, find_first=AsyncMock(return_value=None)) + ) + ), + ) + resolver: Final = GrantResolver(database, UserApiKeyCache()) + forget_missing_user(user_id) + try: + assert await resolver.resolve(UserLookup(user_id=user_id), team_id=None) == UserGone(user_id=user_id) + read.return_value = row if user_exists else None + + outcome: Final = await resolver.resolve(UserLookup(user_id=user_id), team_id=None) + + assert outcome == (ResolvedGrants(row, None, None, user_id) if user_exists else UserGone(user_id=user_id)) + finally: + forget_missing_user(user_id) + class _Loaders: """Fake row readers standing in for the ``auth_checks`` loaders, recording every call they receive.""" From e95bf288f395120746d1a1509b4c09bb768d8ce9 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 10 Nov 2026 14:10:24 -0800 Subject: [PATCH 043/189] feat(cli): connect a per-user GitHub Copilot credential from the command line (#50863) * feat(cli): connect a per-user GitHub Copilot credential from the command line lite credentials connect runs the GitHub device login with the caller's own key, so users can connect Copilot without opening the LiteLLM UI. * fix(cli): stop connect before the GitHub code expires and explain lost connections The connect command kept polling until the proxy rejected the expired code with a raw HTTP 400, so it now stops one interval before expiry and prints its own expired message. A dropped connection or timeout while waiting now prints a short rerun message instead of a traceback. * fix(cli): keep checking for approval until the GitHub code actually expires Connect stopped one interval early, so an approval in the last seconds was missed. The countdown now starts when the code is requested, which is never later than the proxy's own cutoff, and polling continues while another poll fits. --- litellm/proxy/client/cli/README.md | 10 ++ .../proxy/client/cli/commands/credentials.py | 103 ++++++++++- litellm/proxy/client/credentials.py | 32 ++++ .../client/cli/test_credentials_commands.py | 161 +++++++++++++++++- 4 files changed, 297 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index 1fdf8da7d2b..52d367d7333 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -260,6 +260,16 @@ Delete a credential: lite credentials delete ``` +#### Connect a GitHub Copilot Credential + +Connect your own GitHub Copilot account to a per-user GitHub Copilot credential, the same as the Connect button in the UI. Run it with your own virtual key, since the connection belongs to the user that key resolves to: + +```bash +lite --api-key sk-your-key credentials connect +``` + +It prints a code and a GitHub link, then waits. Open the link, enter the code, and approve. The command exits 0 once GitHub confirms the login, and exits 1 if the code expires, the login is denied, or the GitHub account has no Copilot seat. + ### Keys Management The CLI provides commands for managing API keys on your LiteLLM proxy server: diff --git a/litellm/proxy/client/cli/commands/credentials.py b/litellm/proxy/client/cli/commands/credentials.py index 2028bdb5599..40bcc6f6eea 100644 --- a/litellm/proxy/client/cli/commands/credentials.py +++ b/litellm/proxy/client/cli/commands/credentials.py @@ -1,5 +1,8 @@ import json -from collections.abc import Sequence +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import Final, Literal import click @@ -8,9 +11,30 @@ import rich from rich.table import Table from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm.models.credentials import UserConnectionPollResponse, UserConnectionPollStatus + from ...credentials import CredentialsManagementClient +from ...exceptions import UnauthorizedError from ._cli_context import cli_context_values +_SLOW_DOWN_STEP_SECONDS: Final = 5 +_CONNECT_FAILURES: Final[MappingProxyType[UserConnectionPollStatus, str]] = MappingProxyType( + { + "expired": "The code expired before it was approved. Run the command again for a new code", + "denied": "The GitHub login was denied", + "no_copilot_seat": "That GitHub account has no GitHub Copilot seat", + } +) + + +@dataclass(frozen=True, slots=True) +class PollClock: + now: Callable[[], float] + sleep: Callable[[float], None] + + +SYSTEM_CLOCK: Final = PollClock(now=time.monotonic, sleep=time.sleep) + class _CredentialInfo(TypedDict): custom_llm_provider: ReadOnly[NotRequired[str]] @@ -179,3 +203,80 @@ def get(ctx: click.Context, credential_name: str): client: Final = CredentialsManagementClient(context["base_url"], context["api_key"]) response: Final = client.get(credential_name) _print_json(response) + + +def _next_poll_interval(poll: UserConnectionPollResponse, current: int) -> int: + if poll.interval is not None: + return poll.interval + return current + _SLOW_DOWN_STEP_SECONDS if poll.status == "slow_down" else current + + +def _poll_until_settled( + client: CredentialsManagementClient, + credential_name: str, + flow_handle: str, + first_interval: int, + expires_at: float, + clock: PollClock, +) -> UserConnectionPollResponse: + interval = first_interval # rebind-ok: GitHub can ask for a longer wait on any poll + while clock.now() + interval < expires_at: + clock.sleep(interval) + if (poll := client.poll_user_connection(credential_name, flow_handle)).status not in ("pending", "slow_down"): + return poll + interval = _next_poll_interval(poll, interval) + return UserConnectionPollResponse(status="expired") + + +def connect_user_credential( + client: CredentialsManagementClient, + credential_name: str, + clock: PollClock, +) -> UserConnectionPollResponse: + """Run the GitHub device login for the calling key's user and return the final poll result. + + Polling stops once the next poll would land after the code expires, since the proxy rejects polls on an expired + code. The countdown starts before the code is requested, so it never runs past the proxy's own.""" + requested_at: Final = clock.now() + started: Final = client.start_user_connection(credential_name) + click.echo(f"Go to {started.verification_uri} and enter code {started.user_code}") + click.echo("Waiting for approval...") + expires_at: Final = requested_at + started.expires_in + return _poll_until_settled(client, credential_name, started.flow_handle, started.interval, expires_at, clock) + + +def _report_http_error(response: requests.Response) -> None: + click.echo(f"Error: HTTP {response.status_code}", err=True) + try: + error_body: Final[_JsonBodyView] = {"body": response.json()} + _print_json(error_body["body"]) + except json.JSONDecodeError: + click.echo(response.text, err=True) + + +@credentials.command() +@click.argument("credential_name") +@click.pass_context +def connect(ctx: click.Context, credential_name: str) -> None: + """Connect your own GitHub Copilot account to a per-user credential, without the UI""" + context: Final = cli_context_values(ctx) + client: Final = CredentialsManagementClient(context["base_url"], context["api_key"]) + try: + result: Final = connect_user_credential(client, credential_name, SYSTEM_CLOCK) + except UnauthorizedError as e: + if isinstance(e.orig_exception, requests.exceptions.HTTPError) and e.orig_exception.response is not None: + _report_http_error(e.orig_exception.response) + raise click.Abort() + except requests.exceptions.HTTPError as e: + _report_http_error(e.response) + raise click.Abort() + except requests.exceptions.RequestException as e: + click.echo( + f"Error: could not reach the proxy at {context['base_url']} ({type(e).__name__}). Run the command again", + err=True, + ) + raise click.Abort() + if result.status != "connected": + click.echo(f"Error: {_CONNECT_FAILURES.get(result.status, result.status)}", err=True) + raise click.exceptions.Exit(1) + click.echo(f"Connected as {result.github_login} for credential '{credential_name}'") diff --git a/litellm/proxy/client/credentials.py b/litellm/proxy/client/credentials.py index 920aa238ace..2bd8ee8268a 100644 --- a/litellm/proxy/client/credentials.py +++ b/litellm/proxy/client/credentials.py @@ -5,6 +5,8 @@ from urllib.parse import quote import requests from pydantic import TypeAdapter +from litellm.models.credentials import UserConnectionPollResponse, UserConnectionStartResponse + from .exceptions import UnauthorizedError _DECODED_JSON: Final = TypeAdapter(object) @@ -217,3 +219,33 @@ class CredentialsManagementClient: if e.response.status_code == 401: raise UnauthorizedError(e) raise + + def _send_json(self, request: requests.Request) -> object: + try: + response: Final = requests.Session().send(request.prepare(), timeout=self._timeout) + response.raise_for_status() + return _DECODED_JSON.validate_python(response.json()) + except requests.exceptions.HTTPError as e: + if e.response.status_code == 401: + raise UnauthorizedError(e) + raise + + def _user_connection_url(self, credential_name: str, action: str) -> str: + return f"{self._base_url}/credentials/{quote(credential_name, safe='')}/user_connection/{action}" + + def start_user_connection(self, credential_name: str) -> UserConnectionStartResponse: + """Start the GitHub device login that connects the calling key's user to a per-user credential.""" + request: Final = requests.Request( + "POST", self._user_connection_url(credential_name, "start"), headers=self._get_headers() + ) + return UserConnectionStartResponse.model_validate(self._send_json(request)) + + def poll_user_connection(self, credential_name: str, flow_handle: str) -> UserConnectionPollResponse: + """Check once whether the user approved the device login that ``start_user_connection`` began.""" + request: Final = requests.Request( + "POST", + self._user_connection_url(credential_name, "poll"), + headers=self._get_headers(), + json={"flow_handle": flow_handle}, + ) + return UserConnectionPollResponse.model_validate(self._send_json(request)) diff --git a/tests/unit/proxy/client/cli/test_credentials_commands.py b/tests/unit/proxy/client/cli/test_credentials_commands.py index cdbd73057a0..1e319ab4de3 100644 --- a/tests/unit/proxy/client/cli/test_credentials_commands.py +++ b/tests/unit/proxy/client/cli/test_credentials_commands.py @@ -3,11 +3,12 @@ from unittest.mock import MagicMock import pytest import requests +import responses from click.testing import CliRunner - - +from litellm.proxy.client.cli.commands.credentials import PollClock, connect_user_credential from litellm.proxy.client.cli.main import cli +from litellm.proxy.client.credentials import CredentialsManagementClient @pytest.fixture @@ -136,9 +137,7 @@ def test_acreate_credential_http_error(cli_runner, mock_credentials_client): mock_error_response = MagicMock() mock_error_response.status_code = 400 mock_error_response.json.return_value = {"error": "Invalid request"} - mock_instance.create.side_effect = requests.exceptions.HTTPError( - response=mock_error_response - ) + mock_instance.create.side_effect = requests.exceptions.HTTPError(response=mock_error_response) # Run command result = cli_runner.invoke( @@ -182,9 +181,7 @@ def test_adelete_credential_http_error(cli_runner, mock_credentials_client): mock_error_response = MagicMock() mock_error_response.status_code = 404 mock_error_response.json.return_value = {"error": "Credential not found"} - mock_instance.delete.side_effect = requests.exceptions.HTTPError( - response=mock_error_response - ) + mock_instance.delete.side_effect = requests.exceptions.HTTPError(response=mock_error_response) # Run command result = cli_runner.invoke(cli, ["credentials", "delete", "test-cred"]) @@ -264,3 +261,151 @@ def test_update_credential_requires_exactly_one_display_name_flag(cli_runner, mo assert result.exit_code != 0 assert "exactly one" in result.output mock_credentials_client.return_value.update_display_name.assert_not_called() + + +PROXY = "http://proxy.test" +CONNECT_URL = f"{PROXY}/credentials/team%2Fcopilot/user_connection" + + +def _start_body(interval: int = 0, expires_in: int = 900) -> dict[str, object]: + return { + "user_code": "ABCD-1234", + "verification_uri": "https://github.com/login/device", + "expires_in": expires_in, + "interval": interval, + "flow_handle": "sealed-handle", + } + + +class _FakeClock: + def __init__(self) -> None: + self.elapsed = 0.0 + self.waits: tuple[float, ...] = () + + def sleep(self, seconds: float) -> None: + self.elapsed += seconds + self.waits = (*self.waits, seconds) + + def as_poll_clock(self) -> PollClock: + return PollClock(now=lambda: self.elapsed, sleep=self.sleep) + + +def _run_connect(cli_runner): + return cli_runner.invoke( + cli, ["--base-url", PROXY, "--api-key", "sk-user", "credentials", "connect", "team/copilot"] + ) + + +@responses.activate +def test_connect_shows_the_github_code_and_reports_the_connected_login(cli_runner): + responses.add(responses.POST, f"{CONNECT_URL}/start", json=_start_body()) + responses.add(responses.POST, f"{CONNECT_URL}/poll", json={"status": "pending"}) + responses.add(responses.POST, f"{CONNECT_URL}/poll", json={"status": "connected", "github_login": "octocat"}) + + result = _run_connect(cli_runner) + + assert result.exit_code == 0, result.output + assert "https://github.com/login/device" in result.output + assert "ABCD-1234" in result.output + assert "Connected as octocat" in result.output + polls = [call for call in responses.calls if call.request.url.endswith("/poll")] + assert len(polls) == 2 + assert all(json.loads(call.request.body) == {"flow_handle": "sealed-handle"} for call in polls) + assert all(call.request.headers["Authorization"] == "Bearer sk-user" for call in responses.calls) + + +@pytest.mark.parametrize( + ("status", "message"), + [("expired", "expired"), ("denied", "denied"), ("no_copilot_seat", "no GitHub Copilot seat")], +) +@responses.activate +def test_connect_exits_nonzero_when_the_login_does_not_complete(cli_runner, status, message): + responses.add(responses.POST, f"{CONNECT_URL}/start", json=_start_body()) + responses.add(responses.POST, f"{CONNECT_URL}/poll", json={"status": status}) + + result = _run_connect(cli_runner) + + assert result.exit_code == 1 + assert message in result.output + assert "Connected as" not in result.output + + +@pytest.mark.parametrize( + ("status_code", "detail"), + [(401, "An authenticated user is required"), (404, "Credential not found")], +) +@responses.activate +def test_connect_surfaces_the_proxy_error_and_never_polls(cli_runner, status_code, detail): + responses.add(responses.POST, f"{CONNECT_URL}/start", status=status_code, json={"detail": detail}) + + result = _run_connect(cli_runner) + + assert result.exit_code != 0 + assert f"HTTP {status_code}" in result.output + assert detail in result.output + assert not any(call.request.url.endswith("/poll") for call in responses.calls) + + +@responses.activate +def test_connect_waits_the_interval_github_asks_for_and_backs_off_on_slow_down(): + responses.add(responses.POST, f"{CONNECT_URL}/start", json=_start_body(interval=5)) + for poll in ( + {"status": "slow_down"}, + {"status": "pending"}, + {"status": "slow_down", "interval": 20}, + {"status": "connected", "github_login": "octocat"}, + ): + responses.add(responses.POST, f"{CONNECT_URL}/poll", json=poll) + clock = _FakeClock() + + result = connect_user_credential( + CredentialsManagementClient(PROXY, "sk-user"), "team/copilot", clock.as_poll_clock() + ) + + assert result.status == "connected" + assert clock.waits == (5, 10, 10, 20) + + +@responses.activate +def test_connect_stops_polling_once_the_next_poll_would_land_after_the_code_expires(): + responses.add(responses.POST, f"{CONNECT_URL}/start", json=_start_body(interval=5, expires_in=19)) + for _ in range(3): + responses.add(responses.POST, f"{CONNECT_URL}/poll", json={"status": "pending"}) + responses.add(responses.POST, f"{CONNECT_URL}/poll", status=400, json={"detail": "invalid or expired flow_handle"}) + clock = _FakeClock() + + result = connect_user_credential( + CredentialsManagementClient(PROXY, "sk-user"), "team/copilot", clock.as_poll_clock() + ) + + assert result.status == "expired" + assert clock.waits == (5, 5, 5) + assert sum(call.request.url.endswith("/poll") for call in responses.calls) == 3 + + +@responses.activate +def test_connect_still_collects_an_approval_on_the_last_poll_before_the_code_expires(): + responses.add(responses.POST, f"{CONNECT_URL}/start", json=_start_body(interval=5, expires_in=19)) + responses.add(responses.POST, f"{CONNECT_URL}/poll", json={"status": "pending"}) + responses.add(responses.POST, f"{CONNECT_URL}/poll", json={"status": "pending"}) + responses.add(responses.POST, f"{CONNECT_URL}/poll", json={"status": "connected", "github_login": "octocat"}) + clock = _FakeClock() + + result = connect_user_credential( + CredentialsManagementClient(PROXY, "sk-user"), "team/copilot", clock.as_poll_clock() + ) + + assert result.status == "connected" + assert clock.elapsed == 15 + + +@responses.activate +def test_connect_reports_a_lost_connection_while_waiting_and_asks_to_rerun(cli_runner): + responses.add(responses.POST, f"{CONNECT_URL}/start", json=_start_body()) + responses.add(responses.POST, f"{CONNECT_URL}/poll", body=requests.exceptions.ConnectionError("connection reset")) + + result = _run_connect(cli_runner) + + assert result.exit_code == 1 + assert "ABCD-1234" in result.output + assert f"could not reach the proxy at {PROXY} (ConnectionError). Run the command again" in result.output From 8207a57d8440e2035292958431cc65be8604c97f Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 14:24:15 -0800 Subject: [PATCH 044/189] fix(responses): keep the namespace == name shape for deferred top-level tools (#50602) * fix(responses): keep the namespace == name shape for deferred top-level tools The OpenAI Agents SDK matches a call to a deferred top-level function (defer_loading=True) only when the function_call carries namespace == name, the shape it reserves for deferred top-level tools. The Responses bridge returned no namespace, so after a native tool search the SDK failed with "Tool not found". A client that adds namespace == name then hit the echo path, which joined it into "__" for the provider. - namespace_tool_name_map maps deferred top-level functions to (name, name), so non-streaming and streaming replies carry namespace == name - the function_call echo does not qualify a name whose namespace equals it Co-Authored-By: Claude Opus 5.5 * fix(responses): keep the qualified name for a namespace member that shares its namespace's name A namespace "search" with a member "search" is advertised as the chat tool "search__search" and comes back as name == namespace == "search". The echo rule from the previous commit skipped the join for every namespace == name, so that replay reached the provider as "search", a tool it was never given. The bare-name echo now applies only to the request's deferred top-level tools (deferred_top_level_names), and the reply map uses the same set. A deferred top-level name that a self-named namespace member also uses is left out of both directions, so the namespace member keeps its round trip. The lookup reads defer_loading is True instead of validating it, so callers that only read the input (token counting under drop_params) do not raise on a malformed flag. Co-Authored-By: Claude Opus 5.5 * fix(responses): keep the bare name for a namespace == name echo replayed without request tools Stored history (previous_response_id) is rebuilt with no request tools, so the echo of a deferred call carrying namespace == name used to join to name__name. Only a namespace that holds a member of its own name still joins. The lookup also tolerates a tools value that is not a list, and the comprehensions stay within one for and one if clause. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): read request tools through one validated accessor The deferred-name lookup and the response-side tool maps take the request's tools from one TypeAdapter-backed accessor, so a tools value that is not a list of objects reads as no tools and the type gate does not grow. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): type the request mapping handed to the input transformer transform_responses_api_input_to_messages takes dict[str, object] instead of a bare dict, so the tools accessor's argument is known to the type gate and the instructions value is narrowed to str. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): build the namespace tool map from the validated member names namespace_tool_name_map reads each namespace's members through the same validated accessor as the self-named check, which keeps the type gate's per-rule counts at their merge-base values. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): replay stored turns with the tools they were sent with Stored previous_response_id history is rebuilt with each turn's own tools, so a namespace member that shares its namespace's name keeps the joined name__name form on replay, while a deferred top-level call keeps the bare name. Adds the session handler regression. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): keep the object entries of a tools list and read them on the streaming path too One accessor hands the request's tools to the request path and the streaming iterator, dropping only the entries that are not objects instead of the whole list, so both paths see the same custom, namespace and deferred tools. Trims the explanatory comments. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): count input tokens with the request's tools in hand /v1/responses/input_tokens hands the request params, tools included, to the input transformer, so an echoed call for a namespace member named after its namespace counts as name__name there too. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): hand the request tools to every input conversion that reads an echoed call The guardrail provenance pass, the prompt caching pre-call check and the Rust precall token count convert Responses input with the request's tools, so a namespace member named after its namespace keeps name__name there as it does on the request path. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): keep namespace == name for deferred tools loaded by a client-side tool search A tool the client's own tool search loaded is still a deferred top-level tool to the Agents SDK, so the bridge keeps its deferral for the response-side name map and the route strips it again before the provider sees the request. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Homen Shum <317650355+homenshum-cheiron@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrail_translation/handler.py | 21 +- .../proxy/response_api_endpoints/endpoints.py | 14 +- .../client_tool_search.py | 25 +- .../session_handler.py | 10 +- .../streaming_iterator.py | 4 +- .../transformation.py | 95 +++++- litellm/router_rust/steps/precall.py | 2 +- .../prompt_caching_deployment_check.py | 2 +- ...test_openai_responses_guardrail_handler.py | 20 ++ .../response_api_endpoints/test_endpoints.py | 22 +- .../test_client_tool_search.py | 23 ++ .../test_litellm_completion_responses.py | 279 +++++++++++++++++- .../test_session_handler.py | 56 +++- 13 files changed, 528 insertions(+), 45 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 5b3b8e88834..8e9f2a0a0bd 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -111,7 +111,6 @@ from litellm.types.llms.openai import ( ResponseAPIUsage, ResponseCompactionItem, ResponseCompletedEvent, - ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, @@ -299,8 +298,6 @@ _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( {"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"} ) -_EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {} - def _item_rewrite_field(item: Mapping[str, object]) -> str | None: item_type: Final = item.get("type") @@ -522,13 +519,15 @@ def _provenance_unit_bounds( def _input_item_provenance( raw_input: Sequence[object], expected_messages: Sequence[object], + request_tools: object = None, ) -> tuple[Mapping[int, int], frozenset[int]] | None: if not all(isinstance(item, Mapping) for item in raw_input): return None + request_params: Final = {"tools": request_tools} solo_conversions: Final = tuple( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=cast("ResponseInputParam", [item]), # cast-ok: items checked as Mappings above - responses_api_request=_EMPTY_RESPONSES_REQUEST, + responses_api_request=request_params, replay_tool_search=True, ) for item in raw_input @@ -536,7 +535,7 @@ def _input_item_provenance( full_conversion: Final = tuple( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=cast("ResponseInputParam", list(raw_input)), # cast-ok: items checked as Mappings above - responses_api_request=_EMPTY_RESPONSES_REQUEST, + responses_api_request=request_params, replay_tool_search=True, ) ) @@ -549,7 +548,7 @@ def _input_item_provenance( else tuple( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=cast("ResponseInputParam", list(raw_input[start:end])), # cast-ok: checked as Mappings above - responses_api_request=_EMPTY_RESPONSES_REQUEST, + responses_api_request=request_params, replay_tool_search=True, ) ) @@ -708,11 +707,12 @@ def _patched_request_fields( instructions: object, original_messages: Sequence[object], structured_messages: Sequence[object], + request_tools: object = None, ) -> _RequestFields | None: if not isinstance(raw_input, list) or len(original_messages) != len(structured_messages): return None offset: Final = 1 if instructions else 0 - provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:]) + provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:], request_tools) if provenance is None: return None item_for_message, tainted = provenance @@ -755,10 +755,13 @@ def _patch_or_convert_request_fields( instructions: object, original_messages: Sequence[object], structured_messages: Sequence[AllMessageValues], + request_tools: object = None, ) -> _RequestFields | None: if not isinstance(structured_messages, list): return None - patched: Final = _patched_request_fields(raw_input, instructions, original_messages, structured_messages) + patched: Final = _patched_request_fields( + raw_input, instructions, original_messages, structured_messages, request_tools + ) if patched is not None: return patched input_items, converted_instructions = ( @@ -1042,7 +1045,7 @@ class OpenAIResponsesHandler(BaseTranslation): ) ) return _patch_or_convert_request_fields( - data.get("input"), data.get("instructions"), structured_messages, merged + data.get("input"), data.get("instructions"), structured_messages, merged, data.get("tools") ) def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]: diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 574a677061d..11a0b947abe 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -110,7 +110,6 @@ from litellm.router import Router from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( REASONING_EFFORT, - ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ) from litellm.types.responses.main import DeleteResponseResult @@ -1709,16 +1708,18 @@ def _missing_responses_param_response(param: str) -> JSONResponse: def _responses_input_as_token_count_messages( input_value: str | ResponseInputParam, - instructions: str | None, + request_params: Mapping[str, object], ) -> tuple[Mapping[str, object], ...]: from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) - request_params: Final[ResponsesAPIOptionalRequestParams] = {"instructions": instructions} transformed: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=input_value, - responses_api_request=request_params, + responses_api_request={ + "instructions": request_params.get("instructions"), + "tools": request_params.get("tools"), + }, only_since_compaction=True, ) return tuple( @@ -1799,10 +1800,7 @@ async def responses_input_tokens( try: payload: Final[_TokenCountPayload] = { "model": model_name, - "messages": _responses_input_as_token_count_messages( - input_value=input_value, - instructions=data.get("instructions"), - ), + "messages": _responses_input_as_token_count_messages(input_value=input_value, request_params=data), "tools": token_fields.get("tools"), } token_request: Final = TokenCountRequest.model_validate(payload) diff --git a/litellm/responses/litellm_completion_transformation/client_tool_search.py b/litellm/responses/litellm_completion_transformation/client_tool_search.py index 030bb014b7a..1c715fc1191 100644 --- a/litellm/responses/litellm_completion_transformation/client_tool_search.py +++ b/litellm/responses/litellm_completion_transformation/client_tool_search.py @@ -143,20 +143,33 @@ def _without_deferral(spec: Mapping[str, object]) -> Mapping[str, object]: return {key: value for key, value in spec.items() if key != "defer_loading"} -def _loaded_tools(tool: Mapping[str, object]) -> tuple[_LoadedTool, ...]: +def _loaded_tools(tool: Mapping[str, object], deferred_names: frozenset[str] = frozenset()) -> tuple[_LoadedTool, ...]: spec: Final = _tool_spec(tool) if spec is None: return () if spec.type != "namespace": - return (_LoadedTool(namespace=None, namespace_description="", spec=_without_deferral(tool)),) + # A loaded deferred tool keeps its deferral so its calls read back as namespace == name + deferred: Final = spec.defer_loading is True or spec.name in deferred_names + loaded_spec: Final = {**_without_deferral(tool), "defer_loading": True} if deferred else _without_deferral(tool) + return (_LoadedTool(namespace=None, namespace_description="", spec=loaded_spec),) return tuple( _LoadedTool(namespace=spec.name, namespace_description=spec.description, spec=_without_deferral(member)) for member in spec.tools ) -def _all_loaded_tools(tools: Iterable[Mapping[str, object]]) -> tuple[_LoadedTool, ...]: - return tuple(chain.from_iterable(map(_loaded_tools, tools))) +def _all_loaded_tools( + tools: Iterable[Mapping[str, object]], deferred_names: frozenset[str] = frozenset() +) -> tuple[_LoadedTool, ...]: + return tuple(chain.from_iterable(_loaded_tools(tool, deferred_names) for tool in tools)) + + +def _deferred_top_level_names(tools: Iterable[Mapping[str, object]]) -> frozenset[str]: + return frozenset( + spec.name + for tool in tools + if (spec := _tool_spec(tool)) is not None and spec.type != "namespace" and _is_deferred(tool) + ) def _callable_names(tool: object) -> tuple[str, ...]: @@ -256,7 +269,9 @@ def bridge_client_tool_search( if searches_on_client else existing ) - loaded: Final = _all_loaded_tools((*loaded_earlier, *client_tool_search_loaded_tools(input))) + loaded: Final = _all_loaded_tools( + (*loaded_earlier, *client_tool_search_loaded_tools(input)), _deferred_top_level_names(existing) + ) taken: Final = frozenset(chain.from_iterable(map(_callable_names, eager))) bridged_input: Final = ( input if isinstance(input, str) else [bridged for item in input if (bridged := _bridged_item(item)) is not None] diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 6dbd7d69d11..2d40092c7b1 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -279,6 +279,11 @@ def _instruction_items_before( ) +def _stored_request_tools(proxy_server_request: Mapping[str, object] | None) -> object | None: + """The stored turn's ``tools``, so its calls read the same namespace shapes it was sent with.""" + return proxy_server_request.get("tools") if proxy_server_request is not None else None + + def _stored_instructions(proxy_server_request: Mapping[str, object] | None) -> str | None: instructions: Final = None if proxy_server_request is None else proxy_server_request.get("instructions") return instructions if isinstance(instructions, str) and instructions else None @@ -443,6 +448,7 @@ class ResponsesSessionHandler: ) response_input_param: str | ResponseInputParam | None = None + stored_tools: Final = _stored_request_tools(proxy_server_request_dict) ############################################################ # Add Input messages for this Spend Log @@ -463,7 +469,7 @@ class ResponsesSessionHandler: chat_completion_message_history.extend( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=bridge_client_tool_search(response_input_param, None).input, - responses_api_request={}, + responses_api_request={"tools": stored_tools} if stored_tools is not None else {}, replay_reasoning=True, item_cache_control=item_cache_control, tool_call_provider_fields=tool_call_provider_fields, @@ -480,7 +486,7 @@ class ResponsesSessionHandler: chat_completion_message_history.extend( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=bridge_client_tool_search(responses_api_output, None).input, - responses_api_request={}, + responses_api_request={"tools": stored_tools} if stored_tools is not None else {}, replay_reasoning=True, item_cache_control=item_cache_control, tool_call_provider_fields=tool_call_provider_fields, diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 040e2af2a08..489ab8a317d 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -241,7 +241,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._accumulated_provider_specific_fields: dict[str, object] = {} request_tools: Final = self.responses_api_request.get("tools") self._custom_tool_names: set[str] = extract_custom_tool_names(request_tools) - self._namespace_tool_names = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(request_tools) + self._namespace_tool_names = LiteLLMCompletionResponsesConfig.namespace_tool_name_map( + LiteLLMCompletionResponsesConfig.request_tools(self.responses_api_request) + ) self._web_search_calls: dict[str, object] = {} # mutable-ok: latest call by provider id self._queued_web_search_call_ids: set[str] = set() # mutable-ok: emitted call ids self._tool_search = tool_search_routing(request_tools) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 3c1abbfe492..22f4c218879 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -185,6 +185,7 @@ from .tool_search import ( answers_tool_search, anthropic_tool_search_chat_tool, build_tool_search_call, + has_client_tool_search, has_server_tool_search, holds_tool_search_use, is_anthropic_tool_search_chat_tool, @@ -317,6 +318,21 @@ _TEXT_ADAPTER: Final = TypeAdapter(str) _RESPONSES_API_TOOL_CHOICE_ADAPTER: Final = TypeAdapter(ToolChoice) +def _tool_mapping(entry: object) -> Mapping[str, object] | None: + try: + return _STR_KEY_DICT_ADAPTER.validate_python(entry) + except ValidationError: + return None + + +def _tool_mappings(raw_tools: object) -> tuple[Mapping[str, object], ...]: + try: + entries: Final = _OBJECT_LIST_ADAPTER.validate_python(raw_tools) + except ValidationError: + return () + return tuple(tool for entry in entries if (tool := _tool_mapping(entry)) is not None) + + def _validate_responses_media(adapter: TypeAdapter[_MediaT], value: object) -> _MediaT: try: return adapter.validate_python(value) @@ -1182,6 +1198,8 @@ class LiteLLMCompletionResponsesConfig: ) -> Sequence[ChatToolParam]: if not any("defer_loading" in tool for tool in tools): return tools + if has_client_tool_search(callers_tools): + return [LiteLLMCompletionResponsesConfig._without_deferral(tool) for tool in tools] if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): return tools if not has_server_tool_search(callers_tools) and provider_builds_anthropic_request(model, custom_llm_provider): @@ -1498,6 +1516,9 @@ class LiteLLMCompletionResponsesConfig: LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message, replay_reasoning=replay_reasoning, native_compaction=native_compaction, + self_named_namespace_members=LiteLLMCompletionResponsesConfig.self_named_namespace_members( + LiteLLMCompletionResponsesConfig.request_tools(responses_api_request) + ), ) try: converted: Final = ( @@ -1679,6 +1700,7 @@ class LiteLLMCompletionResponsesConfig: input: str | ResponseInputParam, replay_reasoning: bool = False, native_compaction: bool = False, + self_named_namespace_members: frozenset[str] = frozenset(), ) -> list[ AllMessageValues | GenericChatCompletionMessage | ChatCompletionMessageToolCall | ChatCompletionResponseMessage ]: @@ -1705,6 +1727,7 @@ class LiteLLMCompletionResponsesConfig: input_item=_input, replay_reasoning=replay_reasoning, native_compaction=native_compaction, + self_named_namespace_members=self_named_namespace_members, ) ) @@ -2417,6 +2440,7 @@ class LiteLLMCompletionResponsesConfig: input_item: Mapping[str, object], replay_reasoning: bool = False, native_compaction: bool = False, + self_named_namespace_members: frozenset[str] = frozenset(), ) -> list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionResponseMessage]: """ Transform a Responses API input item into a Chat Completion message @@ -2470,7 +2494,8 @@ class LiteLLMCompletionResponsesConfig: return LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( function_call=cast( # cast-ok: callee coerces every field it reads with `or ""` / str() Mapping[str, str], input_item - ) + ), + self_named_namespace_members=self_named_namespace_members, ) elif input_item.get("type") == "compaction": return list( @@ -2739,6 +2764,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_responses_api_function_call_to_chat_completion_message( function_call: Mapping[str, str], + self_named_namespace_members: frozenset[str] = frozenset(), ) -> list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionResponseMessage]: """ Transform a Responses API function_call into a Chat Completion message with tool calls @@ -2768,7 +2794,8 @@ class LiteLLMCompletionResponsesConfig: raw_arguments = json.dumps({"content": raw_input}) if raw_input else "" raw_name: Final = function_call.get("name") or "" namespace: Final = function_call.get("namespace") or "" - qualify: Final = bool(namespace) and function_call.get("type") != "custom_tool_call" + deferred_top_level: Final = namespace == raw_name and raw_name not in self_named_namespace_members + qualify: Final = bool(namespace) and not deferred_top_level and function_call.get("type") != "custom_tool_call" tool_call_id: Final = function_call.get("call_id") or function_call.get("id") or "" tool_call_function: Final = ChatCompletionToolCallFunctionChunk( name=f"{namespace}__{raw_name}" if qualify else raw_name, @@ -3151,15 +3178,57 @@ class LiteLLMCompletionResponsesConfig: return result @staticmethod - def namespace_tool_name_map(tools: ResponseTools) -> NamespaceNameMap: - namespace_entries: Final = tuple( - (str(tool.get("name") or ""), str(namespace_tool.get("name") or "")) + def request_tools(responses_api_request: Mapping[str, object] | None) -> ResponseTools: + """The request's tools, keeping the entries that are objects.""" + raw_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None + return _tool_mappings(raw_tools) if raw_tools is not None else None + + @staticmethod + def _namespace_member_names(tool: Mapping[str, object]) -> tuple[str, ...]: + return tuple( + str(member.get("name") or "") + for member in _tool_mappings(tool.get("tools")) + if member.get("type") == "function" + ) + + @staticmethod + def self_named_namespace_members(tools: ResponseTools) -> frozenset[str]: + """Names of ``namespace`` tools that hold a function tool of their own name. + + Such a call comes back as namespace == name too, so it keeps the joined ``name__name`` form, + and a deferred top-level tool of that name does not get the namespace == name shape. + """ + return frozenset( + str(tool.get("name") or "") for tool in tools or () if tool.get("type") == "namespace" - for namespace_tools in (tool.get("tools"),) - if isinstance(namespace_tools, Sequence) and not isinstance(namespace_tools, (str, bytes)) - for namespace_tool in namespace_tools - if isinstance(namespace_tool, Mapping) and namespace_tool.get("type") == "function" + and str(tool.get("name") or "") in LiteLLMCompletionResponsesConfig._namespace_member_names(tool) + ) + + @staticmethod + def deferred_top_level_names(tools: ResponseTools) -> frozenset[str]: + """Names of deferred top-level function tools that travel as namespace == name. + + Only ``defer_loading is True`` counts here: the request path rejects other values, and + callers that only read the input (token counting under drop_params) must not raise. + """ + self_named: Final = LiteLLMCompletionResponsesConfig.self_named_namespace_members(tools) + return frozenset( + str(tool.get("name") or "") + for tool in tools or () + if tool.get("type") == "function" + and tool.get("defer_loading") is True + and tool.get("name") + and str(tool.get("name") or "") not in self_named + ) + + @staticmethod + def namespace_tool_name_map(tools: ResponseTools) -> NamespaceNameMap: + namespace_entries: Final = tuple( + (str(tool.get("name") or ""), member_name) + for tool in tools or () + if tool.get("type") == "namespace" + for member_name in LiteLLMCompletionResponsesConfig._namespace_member_names(tool) ) top_level_function_names: Final = frozenset( str(tool.get("name") or "") for tool in tools or () if tool.get("type") == "function" @@ -3178,7 +3247,11 @@ class LiteLLMCompletionResponsesConfig: qualified_entries: Final = tuple( (f"{namespace}__{tool_name}", (namespace, tool_name)) for namespace, tool_name in namespace_entries ) - return MappingProxyType(dict(qualified_entries + unambiguous_entries)) + # The OpenAI Agents SDK reserves namespace == name for a deferred top-level tool (agents/_tool_identity.py) + deferred_top_level_entries: Final = tuple( + (name, (name, name)) for name in LiteLLMCompletionResponsesConfig.deferred_top_level_names(tools) + ) + return MappingProxyType(dict(qualified_entries + unambiguous_entries + deferred_top_level_entries)) @staticmethod def _restore_namespace_tool_name(tool_name: str, names: NamespaceNameMap) -> tuple[str, str | None]: @@ -3219,7 +3292,7 @@ class LiteLLMCompletionResponsesConfig: value=tool_call, ) - request_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None + request_tools: Final = LiteLLMCompletionResponsesConfig.request_tools(responses_api_request) custom_tool_names: Final = extract_custom_tool_names(request_tools) namespace_tool_names: Final = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(request_tools) tool_search: Final = tool_search_routing(request_tools) diff --git a/litellm/router_rust/steps/precall.py b/litellm/router_rust/steps/precall.py index c68c5fd5982..9de5c47b820 100644 --- a/litellm/router_rust/steps/precall.py +++ b/litellm/router_rust/steps/precall.py @@ -115,7 +115,7 @@ def _input_tokens( instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # its input type is untyped input=text_input, # pyright: ignore[reportArgumentType] # str | list, as the request carries it - responses_api_request={"instructions": instructions} if instructions is not None else {}, + responses_api_request={"instructions": instructions, "tools": extras.get("tools")}, only_since_compaction=True, ) return counter(model=tokenizer_model, messages=input_messages, tools=_tools(extras)) diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index 2f0cbc0d474..47818f997b3 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -119,7 +119,7 @@ def _responses_prompt(request_data: Mapping[str, object]) -> _ResponsesPrompt | ) messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # its input type is untyped input=cast(str | ResponseInputParam, request_input), # cast-ok: a bad item raises and is caught here - responses_api_request={"instructions": request_data.get("instructions")}, + responses_api_request={"instructions": request_data.get("instructions"), "tools": request_tools}, item_cache_control=True, only_since_compaction=True, ) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 8902cc59f0f..b2bc886a6b7 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -4224,6 +4224,26 @@ class TestProvenancePatching: assert _input_item_provenance(["not a mapping"], []) is None + def test_provenance_reads_a_self_named_namespace_echo_with_the_request_tools(self): + from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + tools = [{"type": "namespace", "name": "search", "tools": [{"type": "function", "name": "search"}]}] + raw_input = [ + {"role": "user", "content": "Search for the release notes."}, + {"type": "function_call", "name": "search", "namespace": "search", "call_id": "call_1", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "found"}, + ] + expected = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=raw_input, responses_api_request={"tools": tools} + ) + + assert expected[1]["tool_calls"][0]["function"]["name"] == "search__search" + assert _input_item_provenance(raw_input, expected, tools) is not None + assert _input_item_provenance(raw_input, expected) is None + def test_provenance_bails_when_expected_messages_disagree(self): from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 2c23305d5c9..e4e4b27e1c2 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -6173,6 +6173,24 @@ _SUMMARY_INPUT: Final = [{"type": "message", "role": "user", "content": _COMPACT def test_input_token_count_reads_a_compaction_item_as_its_summary(): from litellm.proxy.response_api_endpoints.endpoints import _responses_input_as_token_count_messages - assert _responses_input_as_token_count_messages(_COMPACTED_INPUT, None) == ( - _responses_input_as_token_count_messages(_SUMMARY_INPUT, None) + assert _responses_input_as_token_count_messages(_COMPACTED_INPUT, {}) == ( + _responses_input_as_token_count_messages(_SUMMARY_INPUT, {}) ) + + +def test_input_token_count_keeps_the_joined_name_of_a_self_named_namespace_member(): + from litellm.proxy.response_api_endpoints.endpoints import _responses_input_as_token_count_messages + + echo: Final = [ + {"role": "user", "content": "Search for the release notes."}, + {"type": "function_call", "name": "search", "namespace": "search", "call_id": "call_1", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "found"}, + ] + namespace_tools: Final = [{"type": "namespace", "name": "search", "tools": [{"type": "function", "name": "search"}]}] + + def called_names(tools: object) -> list[str]: + messages = _responses_input_as_token_count_messages(cast(Any, echo), {"tools": tools}) + return [call["function"]["name"] for message in messages for call in message.get("tool_calls") or []] + + assert called_names(namespace_tools) == ["search__search"] + assert called_names([{"type": "function", "name": "search", "defer_loading": True}]) == ["search"] diff --git a/tests/unit/responses/litellm_completion_transformation/test_client_tool_search.py b/tests/unit/responses/litellm_completion_transformation/test_client_tool_search.py index dc975b3d112..03027d07976 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_client_tool_search.py +++ b/tests/unit/responses/litellm_completion_transformation/test_client_tool_search.py @@ -1,6 +1,7 @@ from typing import Final from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search +from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam _CODEX_TOOL_SEARCH: Final = {"type": "tool_search", "execution": "client"} @@ -66,3 +67,25 @@ def test_deferred_request_tools_are_held_back_only_when_the_client_searches() -> assert with_search.tools == (_CODEX_TOOL_SEARCH, _SHELL) assert without_search.tools == (_SHELL, deferred) + + +def test_a_loaded_deferred_tool_keeps_its_deferral_so_its_calls_read_back_as_namespace_equals_name() -> None: + deferred: Final = {**_SHELL, "name": "deploy", "defer_loading": True} + loaded_plain: Final = {**_SHELL, "name": "deploy", "description": "loaded copy"} + + bridged: Final = bridge_client_tool_search( + [_SEARCH_CALL, _search_output(loaded_plain)], [_CODEX_TOOL_SEARCH, _SHELL, deferred] + ) + + assert bridged.tools == (_CODEX_TOOL_SEARCH, _SHELL, {**loaded_plain, "defer_loading": True}) + assert LiteLLMCompletionResponsesConfig.namespace_tool_name_map( + LiteLLMCompletionResponsesConfig.request_tools({"tools": list(bridged.tools)}) + ) == {"deploy": ("deploy", "deploy")} + + +def test_a_loaded_tool_that_was_never_deferred_stays_undeferred() -> None: + loaded_lint: Final = {**_SHELL, "name": "lint"} + + bridged: Final = bridge_client_tool_search([_SEARCH_CALL, _search_output(loaded_lint)], [_CODEX_TOOL_SEARCH]) + + assert bridged.tools == (_CODEX_TOOL_SEARCH, loaded_lint) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 8fb066d437f..0e16276ffa5 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -910,6 +910,118 @@ class TestLiteLLMCompletionResponsesConfig: assert tool_calls[0].namespace == "collaboration" assert tool_calls[0].arguments == '{"message":"hello"}' + def test_transform_chat_completion_response_deferred_top_level_tool_call_has_namespace_equal_to_name(self): + """A deferred top-level function (defer_loading=True) comes back with namespace == name, + the shape the OpenAI Agents SDK reserves for it. Without it the SDK raises + "Tool get_reporting_categories not found" after a native tool search.""" + tool_call_id = "call_deferred_top_level" + chat_completion_response = ModelResponse( + id="test-response-id", + created=1234567890, + model="bedrock/invoke/anthropic.claude-opus-5", + object="chat.completion", + choices=[ + Choices( + finish_reason="tool_calls", + index=0, + message=Message( + content=None, + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + id=tool_call_id, + type="function", + function=Function(name="get_reporting_categories", arguments='{"market":"TR-TITCK"}'), + ) + ], + ), + ) + ], + ) + + try: + responses_api_response = ( + LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Is CHG-TR-0042 Type IB in TR-TITCK?", + responses_api_request={ + "tools": [ + { + "type": "function", + "name": "get_reporting_categories", + "defer_loading": True, + "parameters": {"type": "object", "properties": {"market": {"type": "string"}}}, + } + ] + }, + chat_completion_response=chat_completion_response, + ) + ) + finally: + TOOL_CALLS_CACHE.delete_cache(key=tool_call_id) + + tool_calls = [item for item in responses_api_response.output if item.type == "function_call"] + assert [(c.name, c.namespace, c.arguments) for c in tool_calls] == [ + ("get_reporting_categories", "get_reporting_categories", '{"market":"TR-TITCK"}') + ] + + def test_transform_chat_completion_response_tool_loaded_by_client_side_search_has_namespace_equal_to_name(self): + """A deferred tool the client's own tool search loaded is still a deferred top-level tool to the + Agents SDK, so its call comes back as namespace == name through the client-search bridge too.""" + from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search + + tool_call_id = "call_client_searched_deploy" + bridged = bridge_client_tool_search( + [ + {"type": "tool_search_call", "call_id": "s1", "execution": "client", "arguments": {"query": "deploy"}}, + { + "type": "tool_search_output", + "call_id": "s1", + "status": "completed", + "execution": "client", + "tools": [{"type": "function", "name": "deploy", "parameters": {"type": "object"}}], + }, + ], + [ + {"type": "tool_search", "execution": "client"}, + {"type": "function", "name": "deploy", "defer_loading": True, "parameters": {"type": "object"}}, + ], + ) + chat_completion_response = ModelResponse( + id="test-response-id", + created=1234567890, + model="bedrock/invoke/anthropic.claude-opus-5", + object="chat.completion", + choices=[ + Choices( + finish_reason="tool_calls", + index=0, + message=Message( + content=None, + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + id=tool_call_id, type="function", function=Function(name="deploy", arguments="{}") + ) + ], + ), + ) + ], + ) + + try: + responses_api_response = ( + LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input=bridged.input, + responses_api_request={"tools": list(bridged.tools)}, + chat_completion_response=chat_completion_response, + ) + ) + finally: + TOOL_CALLS_CACHE.delete_cache(key=tool_call_id) + + tool_calls = [item for item in responses_api_response.output if item.type == "function_call"] + assert [(c.name, c.namespace) for c in tool_calls] == [("deploy", "deploy")] + def test_transform_chat_completion_response_plain_tool_call_has_no_namespace(self): """A non-namespace function call must not gain a namespace attribute, matching the streaming path which only sets it when a namespace was restored.""" @@ -1999,6 +2111,30 @@ class TestToolTransformation: assert [tool["function"]["name"] for tool in result["tools"]] == ["get_weather", "finance__get_stock_price"] assert all("defer_loading" not in tool for tool in result["tools"]) + @pytest.mark.parametrize( + ("model", "provider"), + [ + ("anthropic/claude-sonnet-5-5", "anthropic"), + ("bedrock/invoke/us.anthropic.claude-opus-5", "bedrock"), + ("gemini/gemini-2.5-flash", "gemini"), + ], + ) + def test_tools_loaded_by_a_client_side_search_go_out_undeferred(self, model: str, provider: str) -> None: + tools: Final = [ + {"type": "tool_search", "execution": "client"}, + {"type": "function", "name": "deploy", "defer_loading": True, "parameters": {"type": "object"}}, + ] + + result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input="Deploy the service.", + responses_api_request={"tools": tools}, + custom_llm_provider=provider, + ) + + assert [tool["function"]["name"] for tool in result["tools"]] == ["tool_search", "deploy"] + assert all("defer_loading" not in tool for tool in result["tools"]) + def test_native_bm25_search_tool_keeps_the_other_tools_deferred(self) -> None: tools: Final = [ {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}, @@ -3144,6 +3280,104 @@ class TestToolTransformation: assert messages[0]["tool_calls"][0]["function"]["name"] == "mcp__everything__get_sum" + def test_function_call_echo_with_namespace_equal_to_name_keeps_bare_name(self): + """The Agents SDK echoes a deferred top-level call with namespace == name. The provider + was given the bare tool name, so the outbound chat tool_call must not become + "get_reporting_categories__get_reporting_categories".""" + messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=[ + { + "type": "function_call", + "name": "get_reporting_categories", + "namespace": "get_reporting_categories", + "call_id": "call_3", + "arguments": '{"market": "KR-MFDS"}', + } + ], + responses_api_request={ + "tools": [{"type": "function", "name": "get_reporting_categories", "defer_loading": True}] + }, + ) + + assert [call["function"]["name"] for call in messages[0]["tool_calls"]] == ["get_reporting_categories"] + + def test_function_call_echo_with_namespace_equal_to_name_keeps_bare_name_without_request_tools(self): + """Stored history (previous_response_id) is rebuilt with no request tools. A deferred call echoed + there still carries namespace == name and must go to the provider under its bare name, not + "get_reporting_categories__get_reporting_categories".""" + messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=[ + { + "type": "function_call", + "name": "get_reporting_categories", + "namespace": "get_reporting_categories", + "call_id": "call_7", + "arguments": "{}", + } + ], + responses_api_request={}, + ) + + assert [call["function"]["name"] for call in messages[0]["tool_calls"]] == ["get_reporting_categories"] + + def test_function_call_echo_from_namespace_with_member_of_same_name_keeps_qualified_name(self): + """A namespace "search" with a member "search" is advertised as the chat tool + "search__search" and comes back as name == namespace == "search". Its echo must + still join, or the provider gets a call to a tool it was never given.""" + tools = [ + { + "type": "namespace", + "name": "search", + "description": "Search tools", + "tools": [{"type": "function", "name": "search", "defer_loading": True}], + } + ] + messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=[ + { + "type": "function_call", + "name": "search", + "namespace": "search", + "call_id": "call_4", + "arguments": "{}", + } + ], + responses_api_request={"tools": tools}, + ) + + assert [call["function"]["name"] for call in messages[0]["tool_calls"]] == ["search__search"] + assert LiteLLMCompletionResponsesConfig.deferred_top_level_names(tools) == frozenset() + + def test_deferred_top_level_name_shared_with_self_named_namespace_member_stays_qualified(self): + """When a deferred top-level "search" and a namespace "search" with a member "search" + are both in the request, namespace == name cannot tell them apart. Neither direction + uses that shape for the top-level tool, so the namespace member keeps its round trip.""" + tools = [ + {"type": "function", "name": "search", "defer_loading": True}, + {"type": "function", "name": "lookup", "defer_loading": True}, + { + "type": "namespace", + "name": "search", + "description": "Search tools", + "tools": [{"type": "function", "name": "search"}], + }, + ] + + assert LiteLLMCompletionResponsesConfig.deferred_top_level_names(tools) == frozenset({"lookup"}) + assert dict(LiteLLMCompletionResponsesConfig.namespace_tool_name_map(tools)) == { + "search__search": ("search", "search"), + "lookup": ("lookup", "lookup"), + } + messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=[ + {"type": "function_call", "name": "search", "namespace": "search", "call_id": "c5", "arguments": "{}"}, + {"type": "function_call", "name": "lookup", "namespace": "lookup", "call_id": "c6", "arguments": "{}"}, + ], + responses_api_request={"tools": tools}, + ) + + assert [call["function"]["name"] for call in messages[0]["tool_calls"]] == ["search__search", "lookup"] + def test_custom_tool_call_echo_keeps_short_name(self): """Custom tools stay advertised under their short name, so a namespace on a custom_tool_call echo is routing metadata and must not be prefixed.""" @@ -3302,6 +3536,27 @@ class TestToolTransformation: assert result["gamma__run"] == ("gamma", "run") assert "run" not in result + def test_namespace_tool_name_map_keeps_the_object_tools_beside_a_malformed_entry(self): + tools = LiteLLMCompletionResponsesConfig.request_tools( + { + "tools": [ + { + "type": "namespace", + "name": "ns", + "tools": [{"type": "function", "name": "a"}, "junk"], + }, + "junk", + {"type": "function", "name": "top", "defer_loading": True}, + ] + } + ) + + assert LiteLLMCompletionResponsesConfig.namespace_tool_name_map(tools) == { + "ns__a": ("ns", "a"), + "a": ("ns", "a"), + "top": ("top", "top"), + } + def test_namespace_tool_name_map_drops_top_level_function_collision(self): tools = [ {"type": "function", "name": "run", "parameters": {"type": "object"}}, @@ -5051,7 +5306,8 @@ class TestEnsureOutputItemContentPartAdded: iterator._responses_namespace_tool_call_fields("admin__run") iterator._responses_namespace_tool_call_fields("admin__run") - namespace_map.assert_called_once_with(request["tools"]) + namespace_map.assert_called_once() + assert namespace_map.call_args.args[0] == tuple(request["tools"]) def test_emit_terminal_event_uses_stream_finish_reason(self): """ @@ -5453,11 +5709,21 @@ class TestHostedWebSearchReplay: ("server_tool_use", "expected_input"), ( ( - {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}}, + { + "type": "server_tool_use", + "id": "srvtoolu_01Pending", + "name": "web_fetch", + "input": {"url": _FETCHED_URL}, + }, {"url": _FETCHED_URL}, ), ( - {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}}, + { + "type": "server_tool_use", + "id": "srvtoolu_01Pending", + "name": "web_search", + "input": {"query": "python"}, + }, {"query": "python"}, ), ), @@ -5486,7 +5752,12 @@ class TestHostedWebSearchReplay: assert [turn["role"] for turn in anthropic_turns] == ["user", "assistant", "user"] assert [block for block in anthropic_turns[1]["content"] if block["type"] != "text"] == [ - {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": server_tool_use["name"], "input": expected_input}, + { + "type": "server_tool_use", + "id": "srvtoolu_01Pending", + "name": server_tool_use["name"], + "input": expected_input, + }, {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, ] assert [(block["type"], block["tool_use_id"]) for block in anthropic_turns[2]["content"]] == [ diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 9a8fa19e2e4..3e2a2c2cf34 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -888,6 +888,57 @@ async def test_message_history_replays_real_key_named_tool_payloads() -> None: assert json.loads(tool_message["content"]) == function_output +async def test_message_history_keeps_the_joined_name_of_a_stored_self_named_namespace_member() -> None: + request_id: Final = "chatcmpl-self-named-namespace" + responses_request_body: Final = { + "model": "anthropic/claude-sonnet-4-5", + "tools": [ + { + "type": "namespace", + "name": "search", + "tools": [{"type": "function", "name": "search", "parameters": {"type": "object", "properties": {}}}], + } + ], + "input": [ + {"role": "user", "content": "Search for the release notes."}, + {"type": "function_call", "call_id": "call_1", "name": "search", "namespace": "search", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "release notes found"}, + ], + } + + with patch( + "litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs", + return_value=True, + ): + proxy_server_request: Final = json.loads( + _get_proxy_server_request_for_spend_logs_payload( + metadata={}, + litellm_params={"proxy_server_request": {"body": responses_request_body}}, + kwargs={}, + ) + ) + + spend_log: Final = { + "request_id": request_id, + "call_type": "aresponses", + "session_id": "session-self-named-namespace", + "proxy_server_request": proxy_server_request, + "response": _chat_completion_response(request_id, "Found them."), + } + + with patch.object( + ResponsesSessionHandler, + "get_all_spend_logs_for_previous_response_id", + new_callable=AsyncMock, + ) as mock_get_spend_logs: + mock_get_spend_logs.return_value = [spend_log] + result: Final = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + request_id + ) + + assert [call["function"]["name"] for call in result["messages"][1]["tool_calls"]] == ["search__search"] + + def _turn_spend_log( turn_input: str | list[dict[str, str]], answer: str, compaction_blocks: list[dict[str, str]] | None = None ) -> dict[str, object]: @@ -1162,7 +1213,10 @@ async def test_an_item_a_poll_caught_half_written_replays_its_finished_text_once "resp_poll_done", "aget_responses", "completed", - [_identified_output_message("msg_answer", "One, two, three"), _identified_output_message("msg_more", "Done")], + [ + _identified_output_message("msg_answer", "One, two, three"), + _identified_output_message("msg_more", "Done"), + ], ), ) From ac0d2fcd68405310baaafbf32334d3d1ffc27877 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:40:03 +0000 Subject: [PATCH 045/189] chore(cost-map): add fal_ai image and video rows priced from the fal.ai model pages (first fal-ai sync) (#51035) * chore(cost-map): add fal_ai image and video rows priced from the fal.ai model pages (first fal-ai sync) * chore(cost-map): add 18 more fal_ai rows whose page states a per-image, per-megapixel or per-second price despite a generic API unit * chore(cost-map): add 11 promotional fal_ai video rows at the rate live on 2026-11-10 (promotional rates apply until the promotion ends) * test: exempt the three fal_ai seedance-2.5/us rows whose 1080p rate is above $1 per second from the cost ceiling check --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 8350 ++++++++++++++++- model_prices_and_context_window.json | 8350 ++++++++++++++++- tests/unit/test_utils.py | 3 + 3 files changed, 16681 insertions(+), 22 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 662909b5cb7..40ff675de19 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26158,8 +26158,11 @@ }, "fal_ai/fal-ai/flux-pro/v1.1": { "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pro/v1.1 at $0.04 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, "mode": "image_generation", "output_cost_per_image": 0.04, + "output_cost_per_pixel": 3.814697265625e-08, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1.1", "supported_endpoints": [ "/v1/images/generations" ] @@ -26168,14 +26171,18 @@ "litellm_provider": "fal_ai", "mode": "image_generation", "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra", "supported_endpoints": [ "/v1/images/generations" ] }, "fal_ai/fal-ai/flux/schnell": { "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/schnell at $0.003 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, "mode": "image_generation", "output_cost_per_image": 0.003, + "output_cost_per_pixel": 2.86102294921875e-09, + "source": "https://fal.ai/models/fal-ai/flux/schnell", "supported_endpoints": [ "/v1/images/generations" ] @@ -26251,6 +26258,7 @@ }, "fal_ai/minimax/h3/text-to-video": { "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, "mode": "video_generation", "output_cost_per_second": 0.13, "output_cost_per_second_480p": 0.05, @@ -26360,6 +26368,7 @@ "litellm_provider": "fal_ai", "mode": "image_generation", "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3", "supported_endpoints": [ "/v1/images/generations" ] @@ -26398,8 +26407,11 @@ }, "fal_ai/fal-ai/stable-diffusion-v35-medium": { "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/stable-diffusion-v35-medium at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, "mode": "image_generation", - "output_cost_per_image": 0.0398, + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/stable-diffusion-v35-medium", "supported_endpoints": [ "/v1/images/generations" ] @@ -26407,7 +26419,8 @@ "fal_ai/fal-ai/nano-banana": { "litellm_provider": "fal_ai", "mode": "image_generation", - "output_cost_per_image": 0.039, + "output_cost_per_image": 0.0398, + "source": "https://fal.ai/models/fal-ai/nano-banana", "supported_endpoints": [ "/v1/images/generations" ] @@ -26415,7 +26428,8 @@ "fal_ai/fal-ai/gemini-25-flash-image": { "litellm_provider": "fal_ai", "mode": "image_generation", - "output_cost_per_image": 0.039, + "output_cost_per_image": 0.0398, + "source": "https://fal.ai/models/fal-ai/gemini-25-flash-image", "supported_endpoints": [ "/v1/images/generations" ] @@ -28346,9 +28360,7 @@ }, "fal_ai/fal-ai/flux/dev": { "litellm_provider": "fal_ai", - "metadata": { - "notes": "fal bills FLUX.1 [dev] at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price is the fallback when dimensions are unavailable" - }, + "metadata": {"notes":"fal bills fal-ai/flux/dev at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, "mode": "image_generation", "output_cost_per_image": 0.025, "output_cost_per_pixel": 2.384185791015625e-08, @@ -28380,16 +28392,16 @@ }, "fal_ai/fal-ai/flux-lora-depth": { "litellm_provider": "fal_ai", - "metadata": { - "notes": "fal bills fal-ai/flux-lora-depth at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries" - }, + "metadata": {"notes":"fal bills fal-ai/flux-lora-depth at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, "mode": "image_generation", "output_cost_per_image": 0.035, "output_cost_per_pixel": 3.337860107421875e-08, "source": "https://fal.ai/models/fal-ai/flux-lora-depth", "supported_endpoints": [ - "/v1/images/edits" - ] + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true }, "fal_ai/fal-ai/moondream3-preview/query": { "input_cost_per_token": 4e-07, @@ -88728,5 +88740,8321 @@ "supports_parallel_function_calling": true, "supports_reasoning": true, "supports_vision": true + }, + "fal_ai/alibaba/happy-horse/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/happy-horse/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/happy-horse/v1.1/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.18, + "output_cost_per_second_1080p": 0.18, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/v1.1/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/happy-horse/v1.1/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.18, + "output_cost_per_second_1080p": 0.18, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/v1.1/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/happy-horse/v1.1/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.18, + "output_cost_per_second_1080p": 0.18, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/v1.1/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0-prime/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_480p": 0.068, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/wan-3.0-prime/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0-prime/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_480p": 0.068, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/wan-3.0-prime/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0-prime/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_480p": 0.068, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/wan-3.0-prime/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2, + "output_cost_per_second_1080p": 0.2, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/alibaba/wan-3.0/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2, + "output_cost_per_second_1080p": 0.2, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/alibaba/wan-3.0/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2, + "output_cost_per_second_1080p": 0.2, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/alibaba/wan-3.0/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/draft-enhance": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.29, + "output_cost_per_second_1080p": 0.29, + "source": "https://fal.ai/models/blackforestlabs/flux-3/draft-enhance", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/edit-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_720p": 0.03, + "source": "https://fal.ai/models/blackforestlabs/flux-3/edit-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/extend-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.41, + "output_cost_per_second_1080p": 0.53, + "output_cost_per_second_720p": 0.41, + "source": "https://fal.ai/models/blackforestlabs/flux-3/extend-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/extend-video/draft": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.12, + "output_cost_per_second_720p": 0.12, + "source": "https://fal.ai/models/blackforestlabs/flux-3/extend-video/draft", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/first-last-frame-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_720p": 0.17, + "source": "https://fal.ai/models/blackforestlabs/flux-3/first-last-frame-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/first-last-frame-to-video/draft": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/blackforestlabs/flux-3/first-last-frame-to-video/draft", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/image-to-video/draft": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/blackforestlabs/flux-3/image-to-video/draft", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/keyframes-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_720p": 0.17, + "source": "https://fal.ai/models/blackforestlabs/flux-3/keyframes-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/keyframes-to-video/draft": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/blackforestlabs/flux-3/keyframes-to-video/draft", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/klein/base/edit-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills blackforestlabs/flux-3/klein/base/edit-image/lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/base/edit-image/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/blackforestlabs/flux-3/klein/base/text-to-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills blackforestlabs/flux-3/klein/base/text-to-image/lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/base/text-to-image/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/blackforestlabs/flux-3/klein/edit-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills blackforestlabs/flux-3/klein/edit-image/lora at $0.012 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.012, + "output_cost_per_pixel": 1.1444091796875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/edit-image/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/blackforestlabs/flux-3/klein/text-to-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills blackforestlabs/flux-3/klein/text-to-image/lora at $0.012 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.012, + "output_cost_per_pixel": 1.1444091796875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/text-to-image/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/blackforestlabs/flux-3/text-to-video/draft": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/blackforestlabs/flux-3/text-to-video/draft", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/bria_video_eraser/erase/keypoints": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/bria_video_eraser/erase/keypoints", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/bria_video_eraser/erase/mask": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/bria_video_eraser/erase/mask", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/bria_video_eraser/erase/prompt": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/bria_video_eraser/erase/prompt", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/embed-product": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/embed-product", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/extract-object": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://fal.ai/models/bria/extract-object", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-2/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-2/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-bbq-preview/generate": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-bbq-preview/generate", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bria/fibo-edit-1.5/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit-1.5/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit-1.5/expand": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit-1.5/expand", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit-1.5/product-holding": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit-1.5/product-holding", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit-1.5/virtual-try-on": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit-1.5/virtual-try-on", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/add_object_by_text": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/add_object_by_text", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/blend": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/blend", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/colorize": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/colorize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/erase_by_text": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/erase_by_text", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/relight": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/relight", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/replace_object_by_text": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/replace_object_by_text", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/reseason": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/reseason", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/restore": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/restore", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/restyle": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/restyle", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/rewrite_text": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/rewrite_text", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/sketch_to_colored_image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/sketch_to_colored_image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-gen-1.5/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-gen-1.5/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bria/fibo-lite/generate": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.036, + "source": "https://fal.ai/models/bria/fibo-lite/generate", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bria/fibo/generate": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo/generate", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bria/genfill/v2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills bria/genfill/v2 at $0.04 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "output_cost_per_pixel": 3.814697265625e-08, + "source": "https://fal.ai/models/bria/genfill/v2", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/guided-remove-background": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/bria/guided-remove-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/increase-resolution": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/increase-resolution", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/product-dimensions": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/product-dimensions", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/replace-background": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/replace-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/upscale/creative": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/upscale/creative", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/video/background-removal": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/video/background-removal", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/background-removal/green-screen-despill": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/bria/video/background-removal/green-screen-despill", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/background-removal/realtime": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0042, + "source": "https://fal.ai/models/bria/video/background-removal/realtime", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/background-removal/v3": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/bria/video/background-removal/v3", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/erase/keypoints": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/video/erase/keypoints", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/erase/mask": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/video/erase/mask", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/erase/prompt": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/video/erase/prompt", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/increase-resolution": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/video/increase-resolution", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/omnihuman/v2": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.105, + "source": "https://fal.ai/models/bytedance/omnihuman/v2", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/fast/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2419, + "output_cost_per_second_720p": 0.2419, + "source": "https://fal.ai/models/bytedance/seedance-2.0/fast/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/fast/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2419, + "output_cost_per_second_720p": 0.2419, + "source": "https://fal.ai/models/bytedance/seedance-2.0/fast/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/mini/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.1547, + "output_cost_per_second_480p": 0.0721, + "output_cost_per_second_720p": 0.1547, + "source": "https://fal.ai/models/bytedance/seedance-2.0/mini/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/mini/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 480p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.007, + "output_cost_per_second_480p": 0.007, + "source": "https://fal.ai/models/bytedance/seedance-2.0/mini/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/mini/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.1547, + "output_cost_per_second_480p": 0.0721, + "output_cost_per_second_720p": 0.1547, + "source": "https://fal.ai/models/bytedance/seedance-2.0/mini/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/us/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.3034, + "output_cost_per_second_1080p": 0.682, + "output_cost_per_second_720p": 0.3034, + "source": "https://fal.ai/models/bytedance/seedance-2.0/us/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/us/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.37, + "output_cost_per_second_480p": 0.1731, + "output_cost_per_second_720p": 0.37, + "source": "https://fal.ai/models/bytedance/seedance-2.0/us/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.5/us/draft/complete": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 1.164, + "source": "https://fal.ai/models/bytedance/seedance-2.5/us/draft/complete", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.5/us/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.473, + "output_cost_per_second_1080p": 1.164, + "output_cost_per_second_480p": 0.2205, + "output_cost_per_second_720p": 0.473, + "source": "https://fal.ai/models/bytedance/seedance-2.5/us/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.5/us/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.473, + "output_cost_per_second_1080p": 1.164, + "output_cost_per_second_480p": 0.2205, + "output_cost_per_second_720p": 0.473, + "source": "https://fal.ai/models/bytedance/seedance-2.5/us/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedream/v5/flash/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.027, + "source": "https://fal.ai/models/bytedance/seedream/v5/flash/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/flash/layerize": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.027, + "source": "https://fal.ai/models/bytedance/seedream/v5/flash/layerize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/flash/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.027, + "source": "https://fal.ai/models/bytedance/seedream/v5/flash/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bytedance/seedream/v5/lite/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/bytedance/seedream/v5/lite/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/lite/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/bytedance/seedream/v5/lite/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/clarityai/crystal-upscaler": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills clarityai/crystal-upscaler at $0.016 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.016, + "output_cost_per_pixel": 1.52587890625e-08, + "source": "https://fal.ai/models/clarityai/crystal-upscaler", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/decart/lucy-2-5/realtime": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/decart/lucy-2-5/realtime", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/decart/lucy-2-6/realtime": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/decart/lucy-2-6/realtime", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/decart/lucy-restyle": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/decart/lucy-restyle", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/decart/lucy2-vton/realtime": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/decart/lucy2-vton/realtime", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ai-avatar/multi": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/ai-avatar/multi", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ai-avatar/multi-text": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/ai-avatar/multi-text", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ai-avatar/single-text": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/ai-avatar/single-text", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/bagel": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/bagel", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/bagel/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/bagel/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ben/v2/image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ben/v2/image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/ben/v2/image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/boogu-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/boogu-image at $0.04 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "output_cost_per_pixel": 3.814697265625e-08, + "source": "https://fal.ai/models/fal-ai/boogu-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/boogu-image/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/boogu-image/edit at $0.04 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "output_cost_per_pixel": 3.814697265625e-08, + "source": "https://fal.ai/models/fal-ai/boogu-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/background/replace": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/background/replace", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/eraser": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/eraser", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/expand": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/expand", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/genfill": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/genfill", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/product-shot": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/product-shot", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/reimagine": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/reimagine", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/text-to-image/base": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/text-to-image/base", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/bria/text-to-image/fast": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/bria/text-to-image/fast", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/bria/text-to-image/hd": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/text-to-image/hd", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/bytedance/dreamactor/v2": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/bytedance/dreamactor/v2", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/bytedance/omnihuman": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/fal-ai/bytedance/omnihuman", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/bytedance/omnihuman/v1.5": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.16, + "source": "https://fal.ai/models/fal-ai/bytedance/omnihuman/v1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/bytedance/seedream/v4.5/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bytedance/seedream/v4.5/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bytedance/seedream/v4.5/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bytedance/seedream/v4.5/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/bytedance/seedream/v4/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/bytedance/seedream/v4/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bytedance/seedream/v4/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/bytedance/seedream/v4/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/clarity-upscaler": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/clarity-upscaler at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/clarity-upscaler", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/codeformer": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/codeformer at $0.0021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0021, + "output_cost_per_pixel": 2.002716064453125e-09, + "source": "https://fal.ai/models/fal-ai/codeformer", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/control-light": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/control-light at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/control-light", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/davinci-magihuman": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/davinci-magihuman", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ddcolor": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ddcolor at $0.001 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.001, + "output_cost_per_pixel": 9.5367431640625e-10, + "source": "https://fal.ai/models/fal-ai/ddcolor", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/depth-anything-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://fal.ai/models/fal-ai/depth-anything-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/echomimic-v3": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/echomimic-v3", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ernie-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ernie-image at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/ernie-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ernie-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ernie-image/lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/ernie-image/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ernie-image/lora/turbo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ernie-image/lora/turbo at $0.015 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.015, + "output_cost_per_pixel": 1.430511474609375e-08, + "source": "https://fal.ai/models/fal-ai/ernie-image/lora/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ernie-image/turbo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ernie-image/turbo at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/ernie-image/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/fashn/tryon/v1.5": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://fal.ai/models/fal-ai/fashn/tryon/v1.5", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/fashn/tryon/v1.6": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://fal.ai/models/fal-ai/fashn/tryon/v1.6", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/feynobg": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/feynobg", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ffmpeg-api/compose": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0002, + "source": "https://fal.ai/models/fal-ai/ffmpeg-api/compose", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ffmpeg-api/merge-audio-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0002, + "source": "https://fal.ai/models/fal-ai/ffmpeg-api/merge-audio-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/finegrain-eraser": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.27, + "source": "https://fal.ai/models/fal-ai/finegrain-eraser", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/finegrain-eraser/bbox": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.18, + "source": "https://fal.ai/models/fal-ai/finegrain-eraser/bbox", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/finegrain-eraser/mask": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.13, + "source": "https://fal.ai/models/fal-ai/finegrain-eraser/mask", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/firered-image-edit-v1.1": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/firered-image-edit-v1.1 at $0.0325 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0325, + "output_cost_per_pixel": 3.0994415283203126e-08, + "source": "https://fal.ai/models/fal-ai/firered-image-edit-v1.1", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flashhead": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.005, + "source": "https://fal.ai/models/fal-ai/flashhead", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/flux-1/dev": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/dev at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/dev", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-1/dev/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/dev/image-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/dev/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-1/dev/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/dev/redux at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/dev/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-1/krea": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/krea at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/krea", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-1/krea/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/krea/image-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/krea/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-1/krea/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/krea/redux at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/krea/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-1/schnell": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/schnell at $0.003 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.003, + "output_cost_per_pixel": 2.86102294921875e-09, + "source": "https://fal.ai/models/fal-ai/flux-1/schnell", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-1/schnell/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/schnell/redux at $0.003 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.003, + "output_cost_per_pixel": 2.86102294921875e-09, + "source": "https://fal.ai/models/fal-ai/flux-1/schnell/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-1/srpo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/srpo at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/srpo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-1/srpo/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/srpo/image-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/srpo/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2 at $0.012 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.012, + "output_cost_per_pixel": 1.1444091796875e-08, + "source": "https://fal.ai/models/fal-ai/flux-2", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-flex": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-flex at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-flex", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-flex/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-flex/edit at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-flex/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-lora-gallery/add-background": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/add-background at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/add-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-lora-gallery/apartment-staging": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/apartment-staging at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/apartment-staging", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-lora-gallery/ballpoint-pen-sketch": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/ballpoint-pen-sketch at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/ballpoint-pen-sketch", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/digital-comic-art": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/digital-comic-art at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/digital-comic-art", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/face-to-full-portrait": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/face-to-full-portrait at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/face-to-full-portrait", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-lora-gallery/hdr-style": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/hdr-style at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/hdr-style", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/multiple-angles": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/multiple-angles at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/multiple-angles", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-lora-gallery/realism": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/realism at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/realism", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/satellite-view-style": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/satellite-view-style at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/satellite-view-style", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/sepia-vintage": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/sepia-vintage at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/sepia-vintage", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/virtual-tryon": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/virtual-tryon at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/virtual-tryon", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/edit at $0.012 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.012, + "output_cost_per_pixel": 1.1444091796875e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/flash": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/flash at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/flash", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/flash/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/flash/edit at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/flash/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/4b": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/4b/base": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/base at $0.009 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.009, + "output_cost_per_pixel": 8.58306884765625e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/base", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/4b/base/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/base/edit at $0.009 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.009, + "output_cost_per_pixel": 8.58306884765625e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/base/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/4b/base/edit/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/base/edit/lora at $0.016 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.016, + "output_cost_per_pixel": 1.52587890625e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/base/edit/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/4b/base/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/base/lora at $0.016 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.016, + "output_cost_per_pixel": 1.52587890625e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/base/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/4b/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/edit at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/4b/edit/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/edit/lora at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/edit/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/4b/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/lora at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/9b": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b at $0.006 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.006, + "output_cost_per_pixel": 5.7220458984375e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/9b/base": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/base at $0.011 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.011, + "output_cost_per_pixel": 1.049041748046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/base", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/9b/base/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/base/edit at $0.011 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.011, + "output_cost_per_pixel": 1.049041748046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/base/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/9b/base/edit/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/base/edit/lora at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/base/edit/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/9b/base/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/base/lora at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/base/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/9b/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/edit at $0.011 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.011, + "output_cost_per_pixel": 1.049041748046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/9b/edit/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/edit/lora at $0.015 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.015, + "output_cost_per_pixel": 1.430511474609375e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/edit/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/9b/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/lora at $0.015 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.015, + "output_cost_per_pixel": 1.430511474609375e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/lora at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/lora/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/lora/edit at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/lora/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/turbo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/turbo at $0.008 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.008, + "output_cost_per_pixel": 7.62939453125e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/turbo/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/turbo/edit at $0.008 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.008, + "output_cost_per_pixel": 7.62939453125e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/turbo/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-general": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-general at $0.075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.075, + "output_cost_per_pixel": 7.152557373046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-general", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-general/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-general/image-to-image at $0.075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.075, + "output_cost_per_pixel": 7.152557373046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-general/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-general/inpainting": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-general/inpainting at $0.075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.075, + "output_cost_per_pixel": 7.152557373046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-general/inpainting", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-kontext-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-kontext-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-kontext-lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-kontext-lora/inpaint": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-kontext-lora/inpaint at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-kontext-lora/inpaint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-kontext-lora/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-kontext-lora/text-to-image at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-kontext-lora/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-kontext/dev": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-kontext/dev at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-kontext/dev", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-krea-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-krea-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-krea-lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-krea-lora/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-krea-lora/image-to-image at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-krea-lora/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-krea-lora/inpainting": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-krea-lora/inpainting at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-krea-lora/inpainting", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-krea-lora/stream": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-krea-lora/stream at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-krea-lora/stream", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-lora-canny": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora-canny at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora-canny", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-lora-fill": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora-fill at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora-fill", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-lora/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora/image-to-image at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-lora/inpainting": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora/inpainting at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora/inpainting", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-lora/stream": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora/stream at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora/stream", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-pro/kontext": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/kontext/max": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext/max", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/kontext/max/multi": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext/max/multi", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/kontext/max/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext/max/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-pro/kontext/multi": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext/multi", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/kontext/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-pro/v1.1-ultra-finetuned": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.07, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra-finetuned", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-pro/v1.1-ultra/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pro/v1.1-ultra/redux at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/v1.1/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pro/v1.1/redux at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1.1/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/v1/fill": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pro/v1/fill at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1/fill", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/v1/fill-finetuned": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pro/v1/fill-finetuned at $0.06 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.06, + "output_cost_per_pixel": 5.7220458984375e-08, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1/fill-finetuned", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pulid": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pulid at $0.0333 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0333, + "output_cost_per_pixel": 3.175735473632813e-08, + "source": "https://fal.ai/models/fal-ai/flux-pulid", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-vision-upscaler": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-vision-upscaler at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/flux-vision-upscaler", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/dev/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/dev/image-to-image at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/flux/dev/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/dev/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/dev/redux at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/dev/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/krea": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/krea at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/krea", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux/krea/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/krea/image-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/krea/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/krea/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/krea/redux at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/krea/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/schnell/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/schnell/redux at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/schnell/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/srpo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/srpo at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/srpo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux/srpo/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/srpo/image-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/srpo/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/framepack": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0333, + "source": "https://fal.ai/models/fal-ai/framepack", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/framepack/f1": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0333, + "source": "https://fal.ai/models/fal-ai/framepack/f1", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/framepack/flf2v": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0333, + "source": "https://fal.ai/models/fal-ai/framepack/flf2v", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/gemini-25-flash-image/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.0398, + "source": "https://fal.ai/models/fal-ai/gemini-25-flash-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/gemini-3-pro-image-preview": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/gemini-3-pro-image-preview", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/gemini-3-pro-image-preview/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/gemini-3-pro-image-preview/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/gemini-3.1-flash-image-preview": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/gemini-3.1-flash-image-preview", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/gemini-3.1-flash-image-preview/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/gemini-3.1-flash-image-preview/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/glm-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/glm-image at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/glm-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/glm-image/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/glm-image/image-to-image at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/glm-image/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/heygen/avatar3/digital-twin": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.034, + "source": "https://fal.ai/models/fal-ai/heygen/avatar3/digital-twin", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/avatar4/digital-twin": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/heygen/avatar4/digital-twin", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/avatar4/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/heygen/avatar4/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/avatar5/digital-twin": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/heygen/avatar5/digital-twin", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v2/translate/precision": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/heygen/v2/translate/precision", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v2/translate/speed": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/heygen/v2/translate/speed", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v2/video-agent": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.034, + "source": "https://fal.ai/models/fal-ai/heygen/v2/video-agent", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v3/filler-word-removal": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/fal-ai/heygen/v3/filler-word-removal", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v3/lipsync/precision": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/heygen/v3/lipsync/precision", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v3/lipsync/speed": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/heygen/v3/lipsync/speed", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v3/video-agent": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.034, + "source": "https://fal.ai/models/fal-ai/heygen/v3/video-agent", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/hidream-i1-fast": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-i1-fast at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/hidream-i1-fast", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hidream-i1-full": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-i1-full at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/hidream-i1-full", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hidream-i1-full/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-i1-full/image-to-image at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/hidream-i1-full/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/hidream-o1-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-o1-image at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/hidream-o1-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hidream-o1-image/dev": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-o1-image/dev at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/hidream-o1-image/dev", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hidream-o1-image/dev/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-o1-image/dev/edit at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/hidream-o1-image/dev/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/hidream-o1-image/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-o1-image/edit at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/hidream-o1-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/hunyuan_world": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/hunyuan_world", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/hunyuan-image/v2.1/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hunyuan-image/v2.1/text-to-image at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/hunyuan-image/v2.1/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hunyuan-image/v3/instruct/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hunyuan-image/v3/instruct/edit at $0.09 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.09, + "output_cost_per_pixel": 8.58306884765625e-08, + "source": "https://fal.ai/models/fal-ai/hunyuan-image/v3/instruct/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/hunyuan-image/v3/instruct/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hunyuan-image/v3/instruct/text-to-image at $0.09 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.09, + "output_cost_per_pixel": 8.58306884765625e-08, + "source": "https://fal.ai/models/fal-ai/hunyuan-image/v3/instruct/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hunyuan-image/v3/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hunyuan-image/v3/text-to-image at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/hunyuan-image/v3/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hunyuan-video-v1.5/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.075, + "source": "https://fal.ai/models/fal-ai/hunyuan-video-v1.5/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/hunyuan-video-v1.5/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.075, + "source": "https://fal.ai/models/fal-ai/hunyuan-video-v1.5/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/iclight-v2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/iclight-v2 at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/iclight-v2", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/id-v2v": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/id-v2v", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/id-v2v/relight": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/id-v2v/relight", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ideogram/character": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/ideogram/character", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/character/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/ideogram/character/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/character/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/ideogram/character/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/object-removal": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/ideogram/object-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/remove-background": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/fal-ai/ideogram/remove-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/upscale": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/upscale", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/ideogram/v2", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ideogram/v2/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/ideogram/v2/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/ideogram/v2/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2/turbo": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/ideogram/v2/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ideogram/v2/turbo/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/ideogram/v2/turbo/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2/turbo/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/ideogram/v2/turbo/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2a": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/ideogram/v2a", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ideogram/v2a/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/ideogram/v2a/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2a/turbo": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://fal.ai/models/fal-ai/ideogram/v2a/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ideogram/v2a/turbo/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://fal.ai/models/fal-ai/ideogram/v2a/turbo/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v3/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v3/generate-transparent": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/generate-transparent", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ideogram/v3/layerize-text": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.09, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/layerize-text", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v3/reframe": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/reframe", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v3/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v3/replace-background": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/replace-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/age-modify": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/age-modify", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/city-teleport": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/city-teleport", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/expression-change": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/expression-change", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/hair-change": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/hair-change", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/headshot-photo": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/headshot-photo", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/makeup-application": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/makeup-application", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/object-removal": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/object-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/outpaint": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/image-apps-v2/outpaint at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/outpaint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/perspective": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/perspective", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/photo-restoration": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/photo-restoration", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/photography-effects": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.4, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/photography-effects", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/portrait-enhance": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.4, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/portrait-enhance", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/product-holding": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/product-holding", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/product-photography": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/product-photography", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/relighting": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/relighting", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/style-transfer": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/style-transfer", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/texture-transform": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/texture-transform", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/virtual-try-on": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/virtual-try-on", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/age-progression": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/age-progression", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/baby-version": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/baby-version", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/background-change": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/background-change", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/broccoli-haircut": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/broccoli-haircut", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/cartoonify": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/cartoonify", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/color-correction": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/color-correction", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/expression-change": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/expression-change", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/face-enhancement": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/face-enhancement", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/hair-change": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/hair-change", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/object-removal": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/object-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/photo-restoration": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/photo-restoration", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/plushie-style": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/plushie-style", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/professional-photo": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/professional-photo", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/realism": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/realism", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/reframe": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/reframe", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/retouch": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/retouch", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/scene-composition": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/scene-composition", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/style-transfer": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/style-transfer", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/text-removal": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/text-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/time-of-day": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/time-of-day", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/weather-effect": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/weather-effect", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/wojak-style": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/wojak-style", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/youtube-thumbnails": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/youtube-thumbnails", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image2svg": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.005, + "source": "https://fal.ai/models/fal-ai/image2svg", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/instant-character": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/instant-character at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/instant-character", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/kling-image/o1": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/kling-image/o1", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/kling-image/o3/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/kling-image/o3/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/kling-image/o3/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/kling-image/o3/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/kling-image/v3/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/kling-image/v3/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/kling-image/v3/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/kling-image/v3/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/kling-video/ai-avatar/v2/pro": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.115, + "source": "https://fal.ai/models/fal-ai/kling-video/ai-avatar/v2/pro", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/ai-avatar/v2/standard": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0562, + "source": "https://fal.ai/models/fal-ai/kling-video/ai-avatar/v2/standard", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.112, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/reference-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.112, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/standard/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.084, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/standard/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/standard/reference-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.084, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/standard/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/standard/video-to-video/edit": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.126, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/standard/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/standard/video-to-video/reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.126, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/standard/video-to-video/reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/video-to-video/edit": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.168, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/video-to-video/reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.168, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/video-to-video/reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/4k/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/4k/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/4k/reference-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/4k/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/4k/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/4k/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/4k/video-to-video/edit": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/4k/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/4k/video-to-video/reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/4k/video-to-video/reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/pro/video-to-video/edit": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.168, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/pro/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/pro/video-to-video/reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.168, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/pro/video-to-video/reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/standard/video-to-video/edit": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.126, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/standard/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/standard/video-to-video/reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.126, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/standard/video-to-video/reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1.5/pro/effects": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/kling-video/v1.5/pro/effects", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1.6/pro/effects": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.098, + "source": "https://fal.ai/models/fal-ai/kling-video/v1.6/pro/effects", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1.6/standard/effects": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.056, + "source": "https://fal.ai/models/fal-ai/kling-video/v1.6/standard/effects", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1/pro/ai-avatar": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.115, + "source": "https://fal.ai/models/fal-ai/kling-video/v1/pro/ai-avatar", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1/standard/ai-avatar": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0562, + "source": "https://fal.ai/models/fal-ai/kling-video/v1/standard/ai-avatar", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1/standard/effects": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.045, + "source": "https://fal.ai/models/fal-ai/kling-video/v1/standard/effects", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v2.6/pro/motion-control": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.112, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.6/pro/motion-control", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v2.6/standard/motion-control": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.6/standard/motion-control", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/4k/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/4k/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/4k/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/4k/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/pro/motion-control": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.168, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/pro/motion-control", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/standard/motion-control": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.126, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/standard/motion-control", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/turbo/pro/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/turbo/pro/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/turbo/pro/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/turbo/pro/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/turbo/standard/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.112, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/turbo/standard/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/turbo/standard/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.112, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/turbo/standard/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/krea-2/turbo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/krea-2/turbo at $0.008 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.008, + "output_cost_per_pixel": 7.62939453125e-09, + "source": "https://fal.ai/models/fal-ai/krea-2/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/krea-2/turbo/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/krea-2/turbo/lora at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/krea-2/turbo/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/krea-2/turbo/style": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/krea-2/turbo/style at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/krea-2/turbo/style", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/krea-wan-14b/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.025, + "source": "https://fal.ai/models/fal-ai/krea-wan-14b/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/krea-wan-14b/video-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.025, + "source": "https://fal.ai/models/fal-ai/krea-wan-14b/video-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/latentsync": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.005, + "source": "https://fal.ai/models/fal-ai/latentsync", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/leffa/pose-transfer": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/leffa/pose-transfer", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/leffa/virtual-tryon": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/leffa/virtual-tryon", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/lightx/recamera": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/lightx/recamera", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/lightx/relight": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/lightx/relight", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/longcat-image at $0.13 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.13, + "output_cost_per_pixel": 1.239776611328125e-07, + "source": "https://fal.ai/models/fal-ai/longcat-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/longcat-image/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/longcat-image/edit at $0.15 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.15, + "output_cost_per_pixel": 1.430511474609375e-07, + "source": "https://fal.ai/models/fal-ai/longcat-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/longcat-video/distilled/image-to-video/480p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.005, + "source": "https://fal.ai/models/fal-ai/longcat-video/distilled/image-to-video/480p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/distilled/image-to-video/720p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/fal-ai/longcat-video/distilled/image-to-video/720p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/distilled/text-to-video/480p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.005, + "source": "https://fal.ai/models/fal-ai/longcat-video/distilled/text-to-video/480p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/distilled/text-to-video/720p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/fal-ai/longcat-video/distilled/text-to-video/720p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/image-to-video/480p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.025, + "source": "https://fal.ai/models/fal-ai/longcat-video/image-to-video/480p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/image-to-video/720p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://fal.ai/models/fal-ai/longcat-video/image-to-video/720p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/text-to-video/480p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.025, + "source": "https://fal.ai/models/fal-ai/longcat-video/text-to-video/480p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/text-to-video/720p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://fal.ai/models/fal-ai/longcat-video/text-to-video/720p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/audio-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/audio-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/extend-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/extend-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.16, 2160p $0.32"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.08, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/image-to-video/fast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.12, 2160p $0.24"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/image-to-video/fast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/reframe": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2, + "output_cost_per_second_1080p": 0.2, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/reframe", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/retake-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/retake-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.16, 2160p $0.32"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.08, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/text-to-video/fast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.12, 2160p $0.24"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/text-to-video/fast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltxv-13b-098-distilled": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/fal-ai/ltxv-13b-098-distilled", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltxv-13b-098-distilled/extend": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/fal-ai/ltxv-13b-098-distilled/extend", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltxv-13b-098-distilled/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/fal-ai/ltxv-13b-098-distilled/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltxv-13b-098-distilled/multiconditioning": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/fal-ai/ltxv-13b-098-distilled/multiconditioning", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/lumina-image/v2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/lumina-image/v2 at $0.075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.075, + "output_cost_per_pixel": 7.152557373046875e-08, + "source": "https://fal.ai/models/fal-ai/lumina-image/v2", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/marigold-v2": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/marigold-v2", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/minimax/hailuo-02-fast/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 512p rate; tiers without a catalog field: 512p $0.017"}, + "mode": "video_generation", + "output_cost_per_second": 0.017, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-02-fast/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/hailuo-02/pro/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-02/pro/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/hailuo-02/pro/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-02/pro/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/hailuo-02/standard/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.045, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-02/standard/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/image-01/subject-reference": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/fal-ai/minimax/image-01/subject-reference", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/mmaudio-v2": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.001, + "source": "https://fal.ai/models/fal-ai/mmaudio-v2", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/moss-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/moss-image at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/moss-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/nafnet/deblur": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/nafnet/deblur at $0.0225 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0225, + "output_cost_per_pixel": 2.1457672119140624e-08, + "source": "https://fal.ai/models/fal-ai/nafnet/deblur", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/nafnet/denoise": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/nafnet/denoise at $0.0225 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0225, + "output_cost_per_pixel": 2.1457672119140624e-08, + "source": "https://fal.ai/models/fal-ai/nafnet/denoise", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/nano-banana-2/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/nano-banana-2/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/nano-banana-pro/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/nano-banana-pro/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/nano-banana/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.0398, + "source": "https://fal.ai/models/fal-ai/nano-banana/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/nucleus-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/nucleus-image at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/nucleus-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/omnigen-v2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/omnigen-v2 at $0.15 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.15, + "output_cost_per_pixel": 1.430511474609375e-07, + "source": "https://fal.ai/models/fal-ai/omnigen-v2", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/one-to-all-animation/1.3b": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 480p rate; tiers without a catalog field: 580p $0.045"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_480p": 0.03, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/fal-ai/one-to-all-animation/1.3b", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/one-to-all-animation/14b": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 480p rate; tiers without a catalog field: 580p $0.09"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_480p": 0.06, + "output_cost_per_second_720p": 0.12, + "source": "https://fal.ai/models/fal-ai/one-to-all-animation/14b", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/patina": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/patina at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/patina", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/phota/enhance": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.13, + "source": "https://fal.ai/models/fal-ai/phota/enhance", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/blur": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/blur", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/chromatic-aberration": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/chromatic-aberration", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/color-correction": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/color-correction", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/color-tint": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/color-tint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/desaturate": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/desaturate", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/dissolve": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/dissolve", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/dodge-burn": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/dodge-burn", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/grain": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/grain", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/parabolize": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/parabolize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/sharpen": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/sharpen", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/solarize": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/solarize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/vignette": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/vignette", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/qwen-image-2512": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-2512 at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-2512", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/qwen-image-2512/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-2512/lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-2512/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/qwen-image-edit-2509": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509 at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/add-background": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/add-background at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/add-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/face-to-full-portrait": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/face-to-full-portrait at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/face-to-full-portrait", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/group-photo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/group-photo at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/group-photo", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/integrate-product": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/integrate-product at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/integrate-product", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/lighting-restoration": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/lighting-restoration at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/lighting-restoration", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/multiple-angles": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/multiple-angles at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/multiple-angles", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/next-scene": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/next-scene at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/next-scene", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/remove-element": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/remove-element at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/remove-element", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/remove-lighting": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/remove-lighting at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/remove-lighting", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/shirt-design": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/shirt-design at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/shirt-design", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2511": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2511 at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2511", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2511-multiple-angles": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2511-multiple-angles at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2511-multiple-angles", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2511/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2511/lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2511/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/add-background": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/add-background at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/add-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/face-to-full-portrait": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/face-to-full-portrait at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/face-to-full-portrait", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/group-photo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/group-photo at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/group-photo", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/integrate-product": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/integrate-product at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/integrate-product", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/lighting-restoration": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/lighting-restoration at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/lighting-restoration", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/multiple-angles": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/multiple-angles at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/multiple-angles", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/next-scene": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/next-scene at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/next-scene", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/remove-element": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/remove-element at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/remove-element", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/remove-lighting": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/remove-lighting at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/remove-lighting", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/shirt-design": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/shirt-design at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/shirt-design", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit/image-to-image at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit/inpaint": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit/inpaint at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit/inpaint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-layered": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/qwen-image-layered", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-layered/lora": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/qwen-image-layered/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image/image-to-image at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/recraft/upscale/creative": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://fal.ai/models/fal-ai/recraft/upscale/creative", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/recraft/upscale/crisp": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.004, + "source": "https://fal.ai/models/fal-ai/recraft/upscale/crisp", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/recraft/v4.1/pro/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.21, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/pro/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4.1/pro/text-to-vector": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/pro/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4.1/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4.1/text-to-vector": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4.1/utility/pro/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.21, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/utility/pro/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4.1/utility/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/utility/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4/pro/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://fal.ai/models/fal-ai/recraft/v4/pro/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4/pro/text-to-vector": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://fal.ai/models/fal-ai/recraft/v4/pro/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/recraft/v4/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4/text-to-vector": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/recraft/v4/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/vectorize": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/fal-ai/recraft/vectorize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/sana": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/sana at $0.001 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.001, + "output_cost_per_pixel": 9.5367431640625e-10, + "source": "https://fal.ai/models/fal-ai/sana", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/sana/sprint": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/sana/sprint at $0.0025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0025, + "output_cost_per_pixel": 2.384185791015625e-09, + "source": "https://fal.ai/models/fal-ai/sana/sprint", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/sana/v1.5/1.6b": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/sana/v1.5/1.6b at $0.0075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0075, + "output_cost_per_pixel": 7.152557373046875e-09, + "source": "https://fal.ai/models/fal-ai/sana/v1.5/1.6b", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/sana/v1.5/4.8b": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/sana/v1.5/4.8b at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/sana/v1.5/4.8b", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/seedvr/upscale/image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/seedvr/upscale/image at $0.001 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.001, + "output_cost_per_pixel": 9.5367431640625e-10, + "source": "https://fal.ai/models/fal-ai/seedvr/upscale/image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/seedvr/upscale/image/seamless": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/seedvr/upscale/image/seamless at $0.0025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0025, + "output_cost_per_pixel": 2.384185791015625e-09, + "source": "https://fal.ai/models/fal-ai/seedvr/upscale/image/seamless", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/stable-diffusion-v3-medium": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/fal-ai/stable-diffusion-v3-medium", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/stable-diffusion-v35-large": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/stable-diffusion-v35-large at $0.065 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.065, + "output_cost_per_pixel": 6.198883056640625e-08, + "source": "https://fal.ai/models/fal-ai/stable-diffusion-v35-large", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/stepx-edit2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/stepx-edit2 at $0.2 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.2, + "output_cost_per_pixel": 1.9073486328125e-07, + "source": "https://fal.ai/models/fal-ai/stepx-edit2", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/telestyle-v2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/telestyle-v2 at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/telestyle-v2", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/uso": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/uso at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/uso", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/vidu/q2/reference-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/vidu/q2/reference-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/vidu/q2/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/vidu/q2/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/vidu/reference-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/vidu/reference-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/wan-22-vace-fun-a14b/depth": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-22-vace-fun-a14b/depth", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-22-vace-fun-a14b/inpainting": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-22-vace-fun-a14b/inpainting", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-22-vace-fun-a14b/outpainting": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-22-vace-fun-a14b/outpainting", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-22-vace-fun-a14b/reframe": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-22-vace-fun-a14b/reframe", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-25-preview/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/wan-25-preview/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/wan-25-preview/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-25-preview/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-25-preview/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/wan-25-preview/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan-25-preview/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-25-preview/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b/depth": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b/depth", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b/inpainting": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b/inpainting", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b/outpainting": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b/outpainting", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b/pose": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b/pose", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b/reframe": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b/reframe", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-apps/video-edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-vace-apps/video-edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-14b/speech-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 480p rate; tiers without a catalog field: 580p $0.15"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.1, + "output_cost_per_second_720p": 0.2, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-14b/speech-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-5b/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.016, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-5b/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/wan/v2.2-a14b/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/image-to-video/lora": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/image-to-video/lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/text-to-image/lora": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/text-to-image/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/text-to-video/lora": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/text-to-video/lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/video-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/video-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.7/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/wan/v2.7/edit-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/edit-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.7/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.7/pro/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/pro/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/wan/v2.7/pro/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/pro/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan/v2.7/reference-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.7/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan/v2.7/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/z-image/base": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/base at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/z-image/base", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/z-image/base/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/base/lora at $0.012 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.012, + "output_cost_per_pixel": 1.1444091796875e-08, + "source": "https://fal.ai/models/fal-ai/z-image/base/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/z-image/turbo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/z-image/turbo/controlnet": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/controlnet at $0.0065 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0065, + "output_cost_per_pixel": 6.198883056640625e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/controlnet", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/controlnet/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/controlnet/lora at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/controlnet/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/image-to-image at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/image-to-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/image-to-image/lora at $0.0085 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0085, + "output_cost_per_pixel": 8.106231689453126e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/image-to-image/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/inpaint": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/inpaint at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/inpaint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/inpaint/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/inpaint/lora at $0.0115 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0115, + "output_cost_per_pixel": 1.0967254638671875e-08, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/inpaint/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/lora at $0.0085 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0085, + "output_cost_per_pixel": 8.106231689453126e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/z-image/turbo/tiling": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/tiling at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/tiling", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/z-image/turbo/tiling/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/tiling/lora at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/tiling/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.1/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 360p $0.03"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.1/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.1/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 360p $0.03"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.1/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.1/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 360p $0.03"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.1/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.1/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 360p $0.03"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.1/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/flash/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.094, + "output_cost_per_second_720p": 0.094, + "source": "https://fal.ai/models/kling/v4/flash/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/flash/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.094, + "output_cost_per_second_720p": 0.094, + "source": "https://fal.ai/models/kling/v4/flash/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/flash/video-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.063, + "output_cost_per_second_720p": 0.063, + "source": "https://fal.ai/models/kling/v4/flash/video-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.235, + "output_cost_per_second_1080p": 0.235, + "output_cost_per_second_720p": 0.157, + "source": "https://fal.ai/models/kling/v4/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.235, + "output_cost_per_second_1080p": 0.235, + "output_cost_per_second_720p": 0.157, + "source": "https://fal.ai/models/kling/v4/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/video-to-video/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.141, + "output_cost_per_second_1080p": 0.141, + "output_cost_per_second_720p": 0.094, + "source": "https://fal.ai/models/kling/v4/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/krea/v3/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills krea/v3/edit at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/krea/v3/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/krea/v3/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills krea/v3/text-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/krea/v3/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/luma/agent/uni-1/v1/max": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.102, + "source": "https://fal.ai/models/luma/agent/uni-1/v1/max", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/luma/agent/uni-1/v1/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.042, + "source": "https://fal.ai/models/luma/agent/uni-1/v1/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/meta/muse-image/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/meta/muse-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/meta/muse-image/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/meta/muse-image/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/minimax/h3-max-turbo/extend-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_1080p": 0.08, + "output_cost_per_second_2k": 0.16, + "output_cost_per_second_480p": 0.025, + "output_cost_per_second_768p": 0.04, + "source": "https://fal.ai/models/minimax/h3-max-turbo/extend-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max-turbo/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.16, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max-turbo/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/extend-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.16, + "output_cost_per_second_2k": 0.32, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/extend-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/insert-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.06, + "source": "https://fal.ai/models/minimax/h3-max/insert-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/lip-sync/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.16, + "output_cost_per_second_2k": 0.32, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/lip-sync/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.13, + "output_cost_per_second_2k": 0.13, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_4k": 0.16, + "output_cost_per_second_768p": 0.06, + "source": "https://fal.ai/models/minimax/h3/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3/image-to-video/lora": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.1625, + "output_cost_per_second_2k": 0.1625, + "output_cost_per_second_480p": 0.0625, + "output_cost_per_second_4k": 0.2, + "output_cost_per_second_768p": 0.075, + "source": "https://fal.ai/models/minimax/h3/image-to-video/lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3/reference-to-video/lora": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.1625, + "output_cost_per_second_2k": 0.1625, + "output_cost_per_second_480p": 0.0625, + "output_cost_per_second_4k": 0.2, + "output_cost_per_second_768p": 0.075, + "source": "https://fal.ai/models/minimax/h3/reference-to-video/lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3/text-to-video/lora": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.1625, + "output_cost_per_second_2k": 0.1625, + "output_cost_per_second_480p": 0.0625, + "output_cost_per_second_4k": 0.2, + "output_cost_per_second_768p": 0.075, + "source": "https://fal.ai/models/minimax/h3/text-to-video/lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/nvidia/cosmos-3-super/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/nvidia/cosmos-3-super/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/pixelcut/background-removal": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.016, + "source": "https://fal.ai/models/pixelcut/background-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/pixelcut/looping-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.32, + "output_cost_per_second_1080p": 0.32, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_768p": 0.16, + "source": "https://fal.ai/models/pixelcut/looping-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/pixelcut/product-photo": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.024, + "source": "https://fal.ai/models/pixelcut/product-photo", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/pixverse/music-video/vibemv": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.09, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/pixverse/music-video/vibemv", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/recraft/v4.1/flash/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.007, + "source": "https://fal.ai/models/recraft/v4.1/flash/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/rundiffusion-fal/juggernaut-flux/base": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/juggernaut-flux/base at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/rundiffusion-fal/juggernaut-flux/base", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/rundiffusion-fal/juggernaut-flux/base/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/juggernaut-flux/base/image-to-image at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/rundiffusion-fal/juggernaut-flux/base/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/rundiffusion-fal/juggernaut-flux/lightning": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/juggernaut-flux/lightning at $0.006 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.006, + "output_cost_per_pixel": 5.7220458984375e-09, + "source": "https://fal.ai/models/rundiffusion-fal/juggernaut-flux/lightning", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/rundiffusion-fal/juggernaut-flux/pro": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/juggernaut-flux/pro at $0.055 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.055, + "output_cost_per_pixel": 5.245208740234375e-08, + "source": "https://fal.ai/models/rundiffusion-fal/juggernaut-flux/pro", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/rundiffusion-fal/juggernaut-flux/pro/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/juggernaut-flux/pro/image-to-image at $0.055 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.055, + "output_cost_per_pixel": 5.245208740234375e-08, + "source": "https://fal.ai/models/rundiffusion-fal/juggernaut-flux/pro/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/rundiffusion-fal/rundiffusion-photo-flux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/rundiffusion-photo-flux at $0.045 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.045, + "output_cost_per_pixel": 4.291534423828125e-08, + "source": "https://fal.ai/models/rundiffusion-fal/rundiffusion-photo-flux", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/smoretalk-ai/rembg-enhance": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.006, + "source": "https://fal.ai/models/smoretalk-ai/rembg-enhance", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/sonilo/v1.1/video-to-video-music": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.009, + "source": "https://fal.ai/models/sonilo/v1.1/video-to-video-music", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/sonilo/v1.1/video-to-video-sound-effects": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.009, + "source": "https://fal.ai/models/sonilo/v1.1/video-to-video-sound-effects", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/tbc/mischief-1/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.01, + "output_cost_per_second_480p": 0.005, + "output_cost_per_second_720p": 0.01, + "source": "https://fal.ai/models/tbc/mischief-1/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/veed/fabric-1.0": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p)"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.15, + "source": "https://fal.ai/models/veed/fabric-1.0", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/veed/fabric-1.0/fast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p)"}, + "mode": "video_generation", + "output_cost_per_second": 0.2, + "output_cost_per_second_480p": 0.1, + "output_cost_per_second_720p": 0.2, + "source": "https://fal.ai/models/veed/fabric-1.0/fast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/veed/fabric-1.0/text": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p)"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.15, + "source": "https://fal.ai/models/veed/fabric-1.0/text", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/wan/v2.6/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/wan/v2.6/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/wan/v2.6/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/wan/v2.6/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/wan/v2.6/image-to-video/flash": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.075, + "output_cost_per_second_1080p": 0.075, + "output_cost_per_second_720p": 0.05, + "source": "https://fal.ai/models/wan/v2.6/image-to-video/flash", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/wan/v2.6/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/wan/v2.6/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/wan/v2.6/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/wan/v2.6/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/wan/v2.6/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/wan/v2.6/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://fal.ai/models/xai/grok-imagine-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/xai/grok-imagine-video/v1.5/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.14, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/xai/grok-imagine-video/v1.5/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/happy-horse/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/creatify/boreal": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.01, + "output_cost_per_second_1080p": 0.03, + "output_cost_per_second_2k": 0.12, + "output_cost_per_second_720p": 0.01, + "source": "https://fal.ai/models/creatify/boreal", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/creatify/aurora": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.14, + "output_cost_per_second_480p": 0.07, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/fal-ai/creatify/aurora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/vidu/q4/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 540p $0.0315"}, + "mode": "video_generation", + "output_cost_per_second": 0.0665, + "output_cost_per_second_1080p": 0.084, + "output_cost_per_second_2k": 0.133, + "output_cost_per_second_4k": 0.273, + "output_cost_per_second_720p": 0.0665, + "source": "https://fal.ai/models/fal-ai/vidu/q4/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/vidu/q4/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 540p $0.0315"}, + "mode": "video_generation", + "output_cost_per_second": 0.0665, + "output_cost_per_second_1080p": 0.084, + "output_cost_per_second_2k": 0.133, + "output_cost_per_second_4k": 0.273, + "output_cost_per_second_720p": 0.0665, + "source": "https://fal.ai/models/fal-ai/vidu/q4/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/ideogram/v4.5": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/ideogram/v4.5", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/ideogram/v4.5/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/ideogram/v4.5/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/ideogram/v4/instant": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills ideogram/v4/instant at $0.0075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0075, + "output_cost_per_pixel": 7.152557373046875e-09, + "source": "https://fal.ai/models/ideogram/v4/instant", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/lightricks/ltx-2.5/image-to-video/fast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.19"}, + "mode": "video_generation", + "output_cost_per_second": 0.13, + "output_cost_per_second_1080p": 0.13, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.09, + "source": "https://fal.ai/models/lightricks/ltx-2.5/image-to-video/fast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/lightricks/ltx-2.5/image-to-video/pro": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.17, + "output_cost_per_second_720p": 0.12, + "source": "https://fal.ai/models/lightricks/ltx-2.5/image-to-video/pro", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/lightricks/ltx-2.5/text-to-video/fast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.19"}, + "mode": "video_generation", + "output_cost_per_second": 0.13, + "output_cost_per_second_1080p": 0.13, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.09, + "source": "https://fal.ai/models/lightricks/ltx-2.5/text-to-video/fast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/lightricks/ltx-2.5/text-to-video/pro": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.17, + "output_cost_per_second_720p": 0.12, + "source": "https://fal.ai/models/lightricks/ltx-2.5/text-to-video/pro", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/recast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.45, + "output_cost_per_second_1080p": 0.45, + "output_cost_per_second_768p": 0.3, + "source": "https://fal.ai/models/minimax/h3-max/recast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/reframe-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/reframe-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/relight": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.16, + "output_cost_per_second_2k": 0.32, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/relight", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/wan/v2.6/reference-to-video/flash": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/wan/v2.6/reference-to-video/flash", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/v1.5/lite/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.14, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_720p": 0.03, + "source": "https://fal.ai/models/xai/grok-imagine-video/v1.5/lite/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/v1.5/lite/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.14, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_720p": 0.03, + "source": "https://fal.ai/models/xai/grok-imagine-video/v1.5/lite/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3-max/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; promotional 50% off rate through 2026-11-22 23:59 UTC per the page; regular rates $0.17 (720p) and $0.29 (1080p) per second resume 2026-11-23 00:00 UTC (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.085, + "output_cost_per_second_1080p": 0.145, + "output_cost_per_second_720p": 0.085, + "source": "https://fal.ai/models/blackforestlabs/flux-3-max/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3-max/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; promotional 50% off rate through 2026-11-22 23:59 UTC per the page; regular rates $0.17 (720p) and $0.29 (1080p) per second resume 2026-11-23 00:00 UTC (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.085, + "output_cost_per_second_1080p": 0.145, + "output_cost_per_second_720p": 0.085, + "source": "https://fal.ai/models/blackforestlabs/flux-3-max/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3-max/keyframes-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; regular rate; the page's 75% promotion ran October 15 to November 7, 2026 and has ended (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_720p": 0.17, + "source": "https://fal.ai/models/blackforestlabs/flux-3-max/keyframes-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/heygen/heygen-video/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; regular rate; the page's 50% promotion ended October 31, 2026 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.02, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.06, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_768p": 0.02, + "source": "https://fal.ai/models/heygen/heygen-video/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/heygen/heygen-video/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; regular rate; the page's 50% promotion ended October 31, 2026 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.02, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.06, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_768p": 0.02, + "source": "https://fal.ai/models/heygen/heygen-video/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/heygen/heygen-video/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; regular rate; the page's 50% promotion ended October 31, 2026 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.02, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.06, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_768p": 0.02, + "source": "https://fal.ai/models/heygen/heygen-video/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max-turbo/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; promotional 30% off rate through 2026-11-15 per the page; standard rates $0.025 (480p), $0.04 (768p), $0.08 (1080p) per second from 2026-11-16 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.028, + "output_cost_per_second_1080p": 0.056, + "output_cost_per_second_480p": 0.0175, + "output_cost_per_second_768p": 0.028, + "source": "https://fal.ai/models/minimax/h3-max-turbo/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max-turbo/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; promotional 30% off rate through 2026-11-15 per the page; standard rates $0.025 (480p), $0.04 (768p), $0.08 (1080p) per second from 2026-11-16 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.028, + "output_cost_per_second_1080p": 0.056, + "output_cost_per_second_480p": 0.0175, + "output_cost_per_second_768p": 0.028, + "source": "https://fal.ai/models/minimax/h3-max-turbo/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/camera-controls": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 480p rate; promotional 30% off rate through 2026-11-15 per the page; standard rates $0.05 (480p), $0.08 (768p), $0.16 (1080p) per second from 2026-11-16 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.035, + "output_cost_per_second_1080p": 0.112, + "output_cost_per_second_480p": 0.035, + "output_cost_per_second_768p": 0.056, + "source": "https://fal.ai/models/minimax/h3-max/camera-controls", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; promotional 30% off rate through 2026-11-15 per the page; standard rates $0.05 (480p), $0.08 (768p), $0.16 (1080p) per second from 2026-11-16 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.056, + "output_cost_per_second_1080p": 0.112, + "output_cost_per_second_480p": 0.035, + "output_cost_per_second_768p": 0.056, + "source": "https://fal.ai/models/minimax/h3-max/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; promotional 30% off rate through 2026-11-15 per the page; standard rates $0.05 (480p), $0.08 (768p), $0.16 (1080p) per second from 2026-11-16 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.056, + "output_cost_per_second_1080p": 0.112, + "output_cost_per_second_480p": 0.035, + "output_cost_per_second_768p": 0.056, + "source": "https://fal.ai/models/minimax/h3-max/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 662909b5cb7..40ff675de19 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26158,8 +26158,11 @@ }, "fal_ai/fal-ai/flux-pro/v1.1": { "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pro/v1.1 at $0.04 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, "mode": "image_generation", "output_cost_per_image": 0.04, + "output_cost_per_pixel": 3.814697265625e-08, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1.1", "supported_endpoints": [ "/v1/images/generations" ] @@ -26168,14 +26171,18 @@ "litellm_provider": "fal_ai", "mode": "image_generation", "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra", "supported_endpoints": [ "/v1/images/generations" ] }, "fal_ai/fal-ai/flux/schnell": { "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/schnell at $0.003 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, "mode": "image_generation", "output_cost_per_image": 0.003, + "output_cost_per_pixel": 2.86102294921875e-09, + "source": "https://fal.ai/models/fal-ai/flux/schnell", "supported_endpoints": [ "/v1/images/generations" ] @@ -26251,6 +26258,7 @@ }, "fal_ai/minimax/h3/text-to-video": { "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, "mode": "video_generation", "output_cost_per_second": 0.13, "output_cost_per_second_480p": 0.05, @@ -26360,6 +26368,7 @@ "litellm_provider": "fal_ai", "mode": "image_generation", "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3", "supported_endpoints": [ "/v1/images/generations" ] @@ -26398,8 +26407,11 @@ }, "fal_ai/fal-ai/stable-diffusion-v35-medium": { "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/stable-diffusion-v35-medium at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, "mode": "image_generation", - "output_cost_per_image": 0.0398, + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/stable-diffusion-v35-medium", "supported_endpoints": [ "/v1/images/generations" ] @@ -26407,7 +26419,8 @@ "fal_ai/fal-ai/nano-banana": { "litellm_provider": "fal_ai", "mode": "image_generation", - "output_cost_per_image": 0.039, + "output_cost_per_image": 0.0398, + "source": "https://fal.ai/models/fal-ai/nano-banana", "supported_endpoints": [ "/v1/images/generations" ] @@ -26415,7 +26428,8 @@ "fal_ai/fal-ai/gemini-25-flash-image": { "litellm_provider": "fal_ai", "mode": "image_generation", - "output_cost_per_image": 0.039, + "output_cost_per_image": 0.0398, + "source": "https://fal.ai/models/fal-ai/gemini-25-flash-image", "supported_endpoints": [ "/v1/images/generations" ] @@ -28346,9 +28360,7 @@ }, "fal_ai/fal-ai/flux/dev": { "litellm_provider": "fal_ai", - "metadata": { - "notes": "fal bills FLUX.1 [dev] at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price is the fallback when dimensions are unavailable" - }, + "metadata": {"notes":"fal bills fal-ai/flux/dev at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, "mode": "image_generation", "output_cost_per_image": 0.025, "output_cost_per_pixel": 2.384185791015625e-08, @@ -28380,16 +28392,16 @@ }, "fal_ai/fal-ai/flux-lora-depth": { "litellm_provider": "fal_ai", - "metadata": { - "notes": "fal bills fal-ai/flux-lora-depth at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries" - }, + "metadata": {"notes":"fal bills fal-ai/flux-lora-depth at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, "mode": "image_generation", "output_cost_per_image": 0.035, "output_cost_per_pixel": 3.337860107421875e-08, "source": "https://fal.ai/models/fal-ai/flux-lora-depth", "supported_endpoints": [ - "/v1/images/edits" - ] + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true }, "fal_ai/fal-ai/moondream3-preview/query": { "input_cost_per_token": 4e-07, @@ -88728,5 +88740,8321 @@ "supports_parallel_function_calling": true, "supports_reasoning": true, "supports_vision": true + }, + "fal_ai/alibaba/happy-horse/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/happy-horse/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/happy-horse/v1.1/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.18, + "output_cost_per_second_1080p": 0.18, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/v1.1/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/happy-horse/v1.1/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.18, + "output_cost_per_second_1080p": 0.18, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/v1.1/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/happy-horse/v1.1/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.18, + "output_cost_per_second_1080p": 0.18, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/v1.1/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0-prime/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_480p": 0.068, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/wan-3.0-prime/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0-prime/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_480p": 0.068, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/wan-3.0-prime/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0-prime/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_480p": 0.068, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/wan-3.0-prime/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2, + "output_cost_per_second_1080p": 0.2, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/alibaba/wan-3.0/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2, + "output_cost_per_second_1080p": 0.2, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/alibaba/wan-3.0/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/wan-3.0/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2, + "output_cost_per_second_1080p": 0.2, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/alibaba/wan-3.0/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/draft-enhance": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.29, + "output_cost_per_second_1080p": 0.29, + "source": "https://fal.ai/models/blackforestlabs/flux-3/draft-enhance", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/edit-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_720p": 0.03, + "source": "https://fal.ai/models/blackforestlabs/flux-3/edit-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/extend-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.41, + "output_cost_per_second_1080p": 0.53, + "output_cost_per_second_720p": 0.41, + "source": "https://fal.ai/models/blackforestlabs/flux-3/extend-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/extend-video/draft": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.12, + "output_cost_per_second_720p": 0.12, + "source": "https://fal.ai/models/blackforestlabs/flux-3/extend-video/draft", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/first-last-frame-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_720p": 0.17, + "source": "https://fal.ai/models/blackforestlabs/flux-3/first-last-frame-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/first-last-frame-to-video/draft": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/blackforestlabs/flux-3/first-last-frame-to-video/draft", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/image-to-video/draft": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/blackforestlabs/flux-3/image-to-video/draft", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/keyframes-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_720p": 0.17, + "source": "https://fal.ai/models/blackforestlabs/flux-3/keyframes-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/keyframes-to-video/draft": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/blackforestlabs/flux-3/keyframes-to-video/draft", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/klein/base/edit-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills blackforestlabs/flux-3/klein/base/edit-image/lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/base/edit-image/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/blackforestlabs/flux-3/klein/base/text-to-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills blackforestlabs/flux-3/klein/base/text-to-image/lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/base/text-to-image/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/blackforestlabs/flux-3/klein/edit-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills blackforestlabs/flux-3/klein/edit-image/lora at $0.012 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.012, + "output_cost_per_pixel": 1.1444091796875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/edit-image/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/blackforestlabs/flux-3/klein/text-to-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills blackforestlabs/flux-3/klein/text-to-image/lora at $0.012 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.012, + "output_cost_per_pixel": 1.1444091796875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/text-to-image/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/blackforestlabs/flux-3/text-to-video/draft": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/blackforestlabs/flux-3/text-to-video/draft", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/bria_video_eraser/erase/keypoints": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/bria_video_eraser/erase/keypoints", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/bria_video_eraser/erase/mask": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/bria_video_eraser/erase/mask", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/bria_video_eraser/erase/prompt": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/bria_video_eraser/erase/prompt", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/embed-product": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/embed-product", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/extract-object": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://fal.ai/models/bria/extract-object", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-2/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-2/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-bbq-preview/generate": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-bbq-preview/generate", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bria/fibo-edit-1.5/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit-1.5/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit-1.5/expand": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit-1.5/expand", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit-1.5/product-holding": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit-1.5/product-holding", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit-1.5/virtual-try-on": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit-1.5/virtual-try-on", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/add_object_by_text": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/add_object_by_text", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/blend": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/blend", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/colorize": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/colorize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/erase_by_text": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/erase_by_text", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/relight": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/relight", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/replace_object_by_text": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/replace_object_by_text", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/reseason": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/reseason", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/restore": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/restore", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/restyle": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/restyle", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/rewrite_text": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/rewrite_text", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-edit/sketch_to_colored_image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-edit/sketch_to_colored_image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/fibo-gen-1.5/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-gen-1.5/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bria/fibo-lite/generate": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.036, + "source": "https://fal.ai/models/bria/fibo-lite/generate", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bria/fibo/generate": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo/generate", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bria/genfill/v2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills bria/genfill/v2 at $0.04 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "output_cost_per_pixel": 3.814697265625e-08, + "source": "https://fal.ai/models/bria/genfill/v2", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/guided-remove-background": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/bria/guided-remove-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/increase-resolution": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/increase-resolution", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/product-dimensions": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/product-dimensions", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/replace-background": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/replace-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/upscale/creative": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/upscale/creative", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bria/video/background-removal": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/video/background-removal", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/background-removal/green-screen-despill": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/bria/video/background-removal/green-screen-despill", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/background-removal/realtime": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0042, + "source": "https://fal.ai/models/bria/video/background-removal/realtime", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/background-removal/v3": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/bria/video/background-removal/v3", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/erase/keypoints": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/video/erase/keypoints", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/erase/mask": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/video/erase/mask", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/erase/prompt": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/video/erase/prompt", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/video/increase-resolution": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/bria/video/increase-resolution", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/omnihuman/v2": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.105, + "source": "https://fal.ai/models/bytedance/omnihuman/v2", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/fast/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2419, + "output_cost_per_second_720p": 0.2419, + "source": "https://fal.ai/models/bytedance/seedance-2.0/fast/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/fast/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2419, + "output_cost_per_second_720p": 0.2419, + "source": "https://fal.ai/models/bytedance/seedance-2.0/fast/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/mini/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.1547, + "output_cost_per_second_480p": 0.0721, + "output_cost_per_second_720p": 0.1547, + "source": "https://fal.ai/models/bytedance/seedance-2.0/mini/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/mini/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 480p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.007, + "output_cost_per_second_480p": 0.007, + "source": "https://fal.ai/models/bytedance/seedance-2.0/mini/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/mini/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.1547, + "output_cost_per_second_480p": 0.0721, + "output_cost_per_second_720p": 0.1547, + "source": "https://fal.ai/models/bytedance/seedance-2.0/mini/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/us/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.3034, + "output_cost_per_second_1080p": 0.682, + "output_cost_per_second_720p": 0.3034, + "source": "https://fal.ai/models/bytedance/seedance-2.0/us/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/us/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.37, + "output_cost_per_second_480p": 0.1731, + "output_cost_per_second_720p": 0.37, + "source": "https://fal.ai/models/bytedance/seedance-2.0/us/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.5/us/draft/complete": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 1.164, + "source": "https://fal.ai/models/bytedance/seedance-2.5/us/draft/complete", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.5/us/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.473, + "output_cost_per_second_1080p": 1.164, + "output_cost_per_second_480p": 0.2205, + "output_cost_per_second_720p": 0.473, + "source": "https://fal.ai/models/bytedance/seedance-2.5/us/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.5/us/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.473, + "output_cost_per_second_1080p": 1.164, + "output_cost_per_second_480p": 0.2205, + "output_cost_per_second_720p": 0.473, + "source": "https://fal.ai/models/bytedance/seedance-2.5/us/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedream/v5/flash/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.027, + "source": "https://fal.ai/models/bytedance/seedream/v5/flash/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/flash/layerize": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.027, + "source": "https://fal.ai/models/bytedance/seedream/v5/flash/layerize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/flash/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.027, + "source": "https://fal.ai/models/bytedance/seedream/v5/flash/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bytedance/seedream/v5/lite/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/bytedance/seedream/v5/lite/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/lite/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/bytedance/seedream/v5/lite/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/clarityai/crystal-upscaler": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills clarityai/crystal-upscaler at $0.016 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.016, + "output_cost_per_pixel": 1.52587890625e-08, + "source": "https://fal.ai/models/clarityai/crystal-upscaler", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/decart/lucy-2-5/realtime": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/decart/lucy-2-5/realtime", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/decart/lucy-2-6/realtime": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/decart/lucy-2-6/realtime", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/decart/lucy-restyle": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/decart/lucy-restyle", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/decart/lucy2-vton/realtime": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/decart/lucy2-vton/realtime", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ai-avatar/multi": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/ai-avatar/multi", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ai-avatar/multi-text": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/ai-avatar/multi-text", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ai-avatar/single-text": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/ai-avatar/single-text", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/bagel": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/bagel", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/bagel/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/bagel/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ben/v2/image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ben/v2/image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/ben/v2/image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/boogu-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/boogu-image at $0.04 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "output_cost_per_pixel": 3.814697265625e-08, + "source": "https://fal.ai/models/fal-ai/boogu-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/boogu-image/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/boogu-image/edit at $0.04 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "output_cost_per_pixel": 3.814697265625e-08, + "source": "https://fal.ai/models/fal-ai/boogu-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/background/replace": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/background/replace", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/eraser": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/eraser", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/expand": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/expand", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/genfill": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/genfill", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/product-shot": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/product-shot", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/reimagine": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/reimagine", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bria/text-to-image/base": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/text-to-image/base", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/bria/text-to-image/fast": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/bria/text-to-image/fast", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/bria/text-to-image/hd": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bria/text-to-image/hd", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/bytedance/dreamactor/v2": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/bytedance/dreamactor/v2", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/bytedance/omnihuman": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/fal-ai/bytedance/omnihuman", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/bytedance/omnihuman/v1.5": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.16, + "source": "https://fal.ai/models/fal-ai/bytedance/omnihuman/v1.5", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/bytedance/seedream/v4.5/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bytedance/seedream/v4.5/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bytedance/seedream/v4.5/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/bytedance/seedream/v4.5/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/bytedance/seedream/v4/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/bytedance/seedream/v4/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bytedance/seedream/v4/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/bytedance/seedream/v4/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/clarity-upscaler": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/clarity-upscaler at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/clarity-upscaler", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/codeformer": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/codeformer at $0.0021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0021, + "output_cost_per_pixel": 2.002716064453125e-09, + "source": "https://fal.ai/models/fal-ai/codeformer", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/control-light": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/control-light at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/control-light", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/davinci-magihuman": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/davinci-magihuman", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ddcolor": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ddcolor at $0.001 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.001, + "output_cost_per_pixel": 9.5367431640625e-10, + "source": "https://fal.ai/models/fal-ai/ddcolor", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/depth-anything-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://fal.ai/models/fal-ai/depth-anything-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/echomimic-v3": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/echomimic-v3", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ernie-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ernie-image at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/ernie-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ernie-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ernie-image/lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/ernie-image/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ernie-image/lora/turbo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ernie-image/lora/turbo at $0.015 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.015, + "output_cost_per_pixel": 1.430511474609375e-08, + "source": "https://fal.ai/models/fal-ai/ernie-image/lora/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ernie-image/turbo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/ernie-image/turbo at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/ernie-image/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/fashn/tryon/v1.5": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://fal.ai/models/fal-ai/fashn/tryon/v1.5", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/fashn/tryon/v1.6": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://fal.ai/models/fal-ai/fashn/tryon/v1.6", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/feynobg": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/feynobg", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ffmpeg-api/compose": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0002, + "source": "https://fal.ai/models/fal-ai/ffmpeg-api/compose", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ffmpeg-api/merge-audio-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0002, + "source": "https://fal.ai/models/fal-ai/ffmpeg-api/merge-audio-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/finegrain-eraser": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.27, + "source": "https://fal.ai/models/fal-ai/finegrain-eraser", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/finegrain-eraser/bbox": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.18, + "source": "https://fal.ai/models/fal-ai/finegrain-eraser/bbox", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/finegrain-eraser/mask": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.13, + "source": "https://fal.ai/models/fal-ai/finegrain-eraser/mask", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/firered-image-edit-v1.1": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/firered-image-edit-v1.1 at $0.0325 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0325, + "output_cost_per_pixel": 3.0994415283203126e-08, + "source": "https://fal.ai/models/fal-ai/firered-image-edit-v1.1", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flashhead": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.005, + "source": "https://fal.ai/models/fal-ai/flashhead", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/flux-1/dev": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/dev at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/dev", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-1/dev/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/dev/image-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/dev/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-1/dev/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/dev/redux at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/dev/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-1/krea": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/krea at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/krea", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-1/krea/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/krea/image-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/krea/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-1/krea/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/krea/redux at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/krea/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-1/schnell": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/schnell at $0.003 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.003, + "output_cost_per_pixel": 2.86102294921875e-09, + "source": "https://fal.ai/models/fal-ai/flux-1/schnell", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-1/schnell/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/schnell/redux at $0.003 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.003, + "output_cost_per_pixel": 2.86102294921875e-09, + "source": "https://fal.ai/models/fal-ai/flux-1/schnell/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-1/srpo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/srpo at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/srpo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-1/srpo/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-1/srpo/image-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-1/srpo/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2 at $0.012 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.012, + "output_cost_per_pixel": 1.1444091796875e-08, + "source": "https://fal.ai/models/fal-ai/flux-2", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-flex": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-flex at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-flex", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-flex/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-flex/edit at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-flex/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-lora-gallery/add-background": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/add-background at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/add-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-lora-gallery/apartment-staging": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/apartment-staging at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/apartment-staging", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-lora-gallery/ballpoint-pen-sketch": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/ballpoint-pen-sketch at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/ballpoint-pen-sketch", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/digital-comic-art": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/digital-comic-art at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/digital-comic-art", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/face-to-full-portrait": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/face-to-full-portrait at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/face-to-full-portrait", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-lora-gallery/hdr-style": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/hdr-style at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/hdr-style", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/multiple-angles": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/multiple-angles at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/multiple-angles", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-lora-gallery/realism": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/realism at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/realism", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/satellite-view-style": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/satellite-view-style at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/satellite-view-style", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/sepia-vintage": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/sepia-vintage at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/sepia-vintage", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-lora-gallery/virtual-tryon": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2-lora-gallery/virtual-tryon at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2-lora-gallery/virtual-tryon", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/edit at $0.012 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.012, + "output_cost_per_pixel": 1.1444091796875e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/flash": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/flash at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/flash", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/flash/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/flash/edit at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/flash/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/4b": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/4b/base": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/base at $0.009 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.009, + "output_cost_per_pixel": 8.58306884765625e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/base", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/4b/base/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/base/edit at $0.009 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.009, + "output_cost_per_pixel": 8.58306884765625e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/base/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/4b/base/edit/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/base/edit/lora at $0.016 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.016, + "output_cost_per_pixel": 1.52587890625e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/base/edit/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/4b/base/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/base/lora at $0.016 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.016, + "output_cost_per_pixel": 1.52587890625e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/base/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/4b/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/edit at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/4b/edit/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/edit/lora at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/edit/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/4b/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/4b/lora at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/4b/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/9b": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b at $0.006 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.006, + "output_cost_per_pixel": 5.7220458984375e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/9b/base": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/base at $0.011 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.011, + "output_cost_per_pixel": 1.049041748046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/base", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/9b/base/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/base/edit at $0.011 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.011, + "output_cost_per_pixel": 1.049041748046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/base/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/9b/base/edit/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/base/edit/lora at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/base/edit/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/9b/base/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/base/lora at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/base/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/klein/9b/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/edit at $0.011 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.011, + "output_cost_per_pixel": 1.049041748046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/9b/edit/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/edit/lora at $0.015 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.015, + "output_cost_per_pixel": 1.430511474609375e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/edit/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/klein/9b/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/klein/9b/lora at $0.015 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.015, + "output_cost_per_pixel": 1.430511474609375e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/klein/9b/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/lora at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/lora/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/lora/edit at $0.021 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.021, + "output_cost_per_pixel": 2.002716064453125e-08, + "source": "https://fal.ai/models/fal-ai/flux-2/lora/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2/turbo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/turbo at $0.008 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.008, + "output_cost_per_pixel": 7.62939453125e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2/turbo/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-2/turbo/edit at $0.008 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries (input megapixels are billed too and are not modeled)"}, + "mode": "image_generation", + "output_cost_per_image": 0.008, + "output_cost_per_pixel": 7.62939453125e-09, + "source": "https://fal.ai/models/fal-ai/flux-2/turbo/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-general": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-general at $0.075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.075, + "output_cost_per_pixel": 7.152557373046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-general", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-general/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-general/image-to-image at $0.075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.075, + "output_cost_per_pixel": 7.152557373046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-general/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-general/inpainting": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-general/inpainting at $0.075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.075, + "output_cost_per_pixel": 7.152557373046875e-08, + "source": "https://fal.ai/models/fal-ai/flux-general/inpainting", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-kontext-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-kontext-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-kontext-lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-kontext-lora/inpaint": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-kontext-lora/inpaint at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-kontext-lora/inpaint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-kontext-lora/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-kontext-lora/text-to-image at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-kontext-lora/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-kontext/dev": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-kontext/dev at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux-kontext/dev", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-krea-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-krea-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-krea-lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-krea-lora/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-krea-lora/image-to-image at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-krea-lora/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-krea-lora/inpainting": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-krea-lora/inpainting at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-krea-lora/inpainting", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-krea-lora/stream": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-krea-lora/stream at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-krea-lora/stream", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-lora-canny": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora-canny at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora-canny", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-lora-fill": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora-fill at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora-fill", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-lora/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora/image-to-image at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-lora/inpainting": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora/inpainting at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora/inpainting", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-lora/stream": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-lora/stream at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora/stream", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-pro/kontext": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/kontext/max": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext/max", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/kontext/max/multi": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext/max/multi", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/kontext/max/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext/max/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-pro/kontext/multi": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext/multi", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/kontext/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/flux-pro/kontext/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-pro/v1.1-ultra-finetuned": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.07, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra-finetuned", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-pro/v1.1-ultra/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pro/v1.1-ultra/redux at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/v1.1/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pro/v1.1/redux at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1.1/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/v1/fill": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pro/v1/fill at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1/fill", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/v1/fill-finetuned": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pro/v1/fill-finetuned at $0.06 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.06, + "output_cost_per_pixel": 5.7220458984375e-08, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1/fill-finetuned", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pulid": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-pulid at $0.0333 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0333, + "output_cost_per_pixel": 3.175735473632813e-08, + "source": "https://fal.ai/models/fal-ai/flux-pulid", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-vision-upscaler": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux-vision-upscaler at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/flux-vision-upscaler", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/dev/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/dev/image-to-image at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/flux/dev/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/dev/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/dev/redux at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/dev/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/krea": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/krea at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/krea", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux/krea/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/krea/image-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/krea/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/krea/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/krea/redux at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/krea/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/schnell/redux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/schnell/redux at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/schnell/redux", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux/srpo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/srpo at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/srpo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux/srpo/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/flux/srpo/image-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/flux/srpo/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/framepack": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0333, + "source": "https://fal.ai/models/fal-ai/framepack", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/framepack/f1": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0333, + "source": "https://fal.ai/models/fal-ai/framepack/f1", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/framepack/flf2v": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0333, + "source": "https://fal.ai/models/fal-ai/framepack/flf2v", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/gemini-25-flash-image/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.0398, + "source": "https://fal.ai/models/fal-ai/gemini-25-flash-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/gemini-3-pro-image-preview": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/gemini-3-pro-image-preview", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/gemini-3-pro-image-preview/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/gemini-3-pro-image-preview/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/gemini-3.1-flash-image-preview": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/gemini-3.1-flash-image-preview", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/gemini-3.1-flash-image-preview/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/gemini-3.1-flash-image-preview/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/glm-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/glm-image at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/glm-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/glm-image/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/glm-image/image-to-image at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/glm-image/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/heygen/avatar3/digital-twin": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.034, + "source": "https://fal.ai/models/fal-ai/heygen/avatar3/digital-twin", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/avatar4/digital-twin": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/heygen/avatar4/digital-twin", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/avatar4/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/heygen/avatar4/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/avatar5/digital-twin": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/heygen/avatar5/digital-twin", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v2/translate/precision": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/heygen/v2/translate/precision", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v2/translate/speed": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/heygen/v2/translate/speed", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v2/video-agent": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.034, + "source": "https://fal.ai/models/fal-ai/heygen/v2/video-agent", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v3/filler-word-removal": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/fal-ai/heygen/v3/filler-word-removal", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v3/lipsync/precision": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/heygen/v3/lipsync/precision", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v3/lipsync/speed": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/heygen/v3/lipsync/speed", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/heygen/v3/video-agent": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.034, + "source": "https://fal.ai/models/fal-ai/heygen/v3/video-agent", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/hidream-i1-fast": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-i1-fast at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/hidream-i1-fast", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hidream-i1-full": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-i1-full at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/hidream-i1-full", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hidream-i1-full/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-i1-full/image-to-image at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/hidream-i1-full/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/hidream-o1-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-o1-image at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/hidream-o1-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hidream-o1-image/dev": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-o1-image/dev at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/hidream-o1-image/dev", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hidream-o1-image/dev/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-o1-image/dev/edit at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/hidream-o1-image/dev/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/hidream-o1-image/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hidream-o1-image/edit at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/hidream-o1-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/hunyuan_world": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/hunyuan_world", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/hunyuan-image/v2.1/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hunyuan-image/v2.1/text-to-image at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/hunyuan-image/v2.1/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hunyuan-image/v3/instruct/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hunyuan-image/v3/instruct/edit at $0.09 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.09, + "output_cost_per_pixel": 8.58306884765625e-08, + "source": "https://fal.ai/models/fal-ai/hunyuan-image/v3/instruct/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/hunyuan-image/v3/instruct/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hunyuan-image/v3/instruct/text-to-image at $0.09 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.09, + "output_cost_per_pixel": 8.58306884765625e-08, + "source": "https://fal.ai/models/fal-ai/hunyuan-image/v3/instruct/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hunyuan-image/v3/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/hunyuan-image/v3/text-to-image at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/hunyuan-image/v3/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/hunyuan-video-v1.5/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.075, + "source": "https://fal.ai/models/fal-ai/hunyuan-video-v1.5/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/hunyuan-video-v1.5/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.075, + "source": "https://fal.ai/models/fal-ai/hunyuan-video-v1.5/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/iclight-v2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/iclight-v2 at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/iclight-v2", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/id-v2v": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/id-v2v", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/id-v2v/relight": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/id-v2v/relight", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ideogram/character": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/ideogram/character", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/character/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/ideogram/character/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/character/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/ideogram/character/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/object-removal": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/ideogram/object-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/remove-background": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/fal-ai/ideogram/remove-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/upscale": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/upscale", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/ideogram/v2", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ideogram/v2/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/ideogram/v2/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/ideogram/v2/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2/turbo": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/ideogram/v2/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ideogram/v2/turbo/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/ideogram/v2/turbo/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2/turbo/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/ideogram/v2/turbo/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2a": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/ideogram/v2a", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ideogram/v2a/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/ideogram/v2a/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v2a/turbo": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://fal.ai/models/fal-ai/ideogram/v2a/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ideogram/v2a/turbo/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://fal.ai/models/fal-ai/ideogram/v2a/turbo/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v3/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v3/generate-transparent": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/generate-transparent", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/ideogram/v3/layerize-text": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.09, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/layerize-text", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v3/reframe": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/reframe", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v3/remix": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/remix", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/ideogram/v3/replace-background": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/ideogram/v3/replace-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/age-modify": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/age-modify", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/city-teleport": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/city-teleport", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/expression-change": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/expression-change", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/hair-change": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/hair-change", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/headshot-photo": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/headshot-photo", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/makeup-application": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/makeup-application", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/object-removal": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/object-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/outpaint": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/image-apps-v2/outpaint at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/outpaint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/perspective": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/perspective", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/photo-restoration": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/photo-restoration", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/photography-effects": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.4, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/photography-effects", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/portrait-enhance": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.4, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/portrait-enhance", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/product-holding": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/product-holding", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/product-photography": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/product-photography", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/relighting": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/relighting", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/style-transfer": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/style-transfer", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/texture-transform": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/texture-transform", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-apps-v2/virtual-try-on": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-apps-v2/virtual-try-on", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/age-progression": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/age-progression", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/baby-version": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/baby-version", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/background-change": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/background-change", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/broccoli-haircut": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/broccoli-haircut", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/cartoonify": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/cartoonify", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/color-correction": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/color-correction", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/expression-change": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/expression-change", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/face-enhancement": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/face-enhancement", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/hair-change": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/hair-change", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/object-removal": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/object-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/photo-restoration": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/photo-restoration", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/plushie-style": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/plushie-style", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/professional-photo": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/professional-photo", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/realism": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/realism", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/reframe": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/reframe", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/retouch": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/retouch", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/scene-composition": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/scene-composition", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/style-transfer": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/style-transfer", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/text-removal": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/text-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/time-of-day": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/time-of-day", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/weather-effect": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/weather-effect", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/wojak-style": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/wojak-style", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image-editing/youtube-thumbnails": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/image-editing/youtube-thumbnails", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/image2svg": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.005, + "source": "https://fal.ai/models/fal-ai/image2svg", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/instant-character": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/instant-character at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/instant-character", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/kling-image/o1": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/kling-image/o1", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/kling-image/o3/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/kling-image/o3/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/kling-image/o3/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/kling-image/o3/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/kling-image/v3/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/kling-image/v3/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/kling-image/v3/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.028, + "source": "https://fal.ai/models/fal-ai/kling-image/v3/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/kling-video/ai-avatar/v2/pro": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.115, + "source": "https://fal.ai/models/fal-ai/kling-video/ai-avatar/v2/pro", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/ai-avatar/v2/standard": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0562, + "source": "https://fal.ai/models/fal-ai/kling-video/ai-avatar/v2/standard", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.112, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/reference-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.112, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/standard/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.084, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/standard/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/standard/reference-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.084, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/standard/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/standard/video-to-video/edit": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.126, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/standard/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/standard/video-to-video/reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.126, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/standard/video-to-video/reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/video-to-video/edit": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.168, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o1/video-to-video/reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.168, + "source": "https://fal.ai/models/fal-ai/kling-video/o1/video-to-video/reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/4k/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/4k/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/4k/reference-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/4k/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/4k/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/4k/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/4k/video-to-video/edit": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/4k/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/4k/video-to-video/reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/4k/video-to-video/reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/pro/video-to-video/edit": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.168, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/pro/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/pro/video-to-video/reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.168, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/pro/video-to-video/reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/standard/video-to-video/edit": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.126, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/standard/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/o3/standard/video-to-video/reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.126, + "source": "https://fal.ai/models/fal-ai/kling-video/o3/standard/video-to-video/reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1.5/pro/effects": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/kling-video/v1.5/pro/effects", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1.6/pro/effects": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.098, + "source": "https://fal.ai/models/fal-ai/kling-video/v1.6/pro/effects", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1.6/standard/effects": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.056, + "source": "https://fal.ai/models/fal-ai/kling-video/v1.6/standard/effects", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1/pro/ai-avatar": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.115, + "source": "https://fal.ai/models/fal-ai/kling-video/v1/pro/ai-avatar", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1/standard/ai-avatar": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.0562, + "source": "https://fal.ai/models/fal-ai/kling-video/v1/standard/ai-avatar", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v1/standard/effects": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.045, + "source": "https://fal.ai/models/fal-ai/kling-video/v1/standard/effects", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v2.6/pro/motion-control": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.112, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.6/pro/motion-control", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v2.6/standard/motion-control": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.6/standard/motion-control", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/4k/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/4k/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/4k/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.42, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/4k/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/pro/motion-control": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.168, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/pro/motion-control", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/standard/motion-control": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.126, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/standard/motion-control", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/turbo/pro/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/turbo/pro/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/turbo/pro/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/turbo/pro/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/turbo/standard/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.112, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/turbo/standard/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v3/turbo/standard/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.112, + "source": "https://fal.ai/models/fal-ai/kling-video/v3/turbo/standard/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/krea-2/turbo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/krea-2/turbo at $0.008 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.008, + "output_cost_per_pixel": 7.62939453125e-09, + "source": "https://fal.ai/models/fal-ai/krea-2/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/krea-2/turbo/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/krea-2/turbo/lora at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/krea-2/turbo/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/krea-2/turbo/style": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/krea-2/turbo/style at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/krea-2/turbo/style", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/krea-wan-14b/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.025, + "source": "https://fal.ai/models/fal-ai/krea-wan-14b/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/krea-wan-14b/video-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.025, + "source": "https://fal.ai/models/fal-ai/krea-wan-14b/video-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/latentsync": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.005, + "source": "https://fal.ai/models/fal-ai/latentsync", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/leffa/pose-transfer": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/leffa/pose-transfer", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/leffa/virtual-tryon": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/leffa/virtual-tryon", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/lightx/recamera": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/lightx/recamera", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/lightx/relight": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/lightx/relight", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/longcat-image at $0.13 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.13, + "output_cost_per_pixel": 1.239776611328125e-07, + "source": "https://fal.ai/models/fal-ai/longcat-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/longcat-image/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/longcat-image/edit at $0.15 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.15, + "output_cost_per_pixel": 1.430511474609375e-07, + "source": "https://fal.ai/models/fal-ai/longcat-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/longcat-video/distilled/image-to-video/480p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.005, + "source": "https://fal.ai/models/fal-ai/longcat-video/distilled/image-to-video/480p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/distilled/image-to-video/720p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/fal-ai/longcat-video/distilled/image-to-video/720p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/distilled/text-to-video/480p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.005, + "source": "https://fal.ai/models/fal-ai/longcat-video/distilled/text-to-video/480p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/distilled/text-to-video/720p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/fal-ai/longcat-video/distilled/text-to-video/720p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/image-to-video/480p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.025, + "source": "https://fal.ai/models/fal-ai/longcat-video/image-to-video/480p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/image-to-video/720p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://fal.ai/models/fal-ai/longcat-video/image-to-video/720p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/text-to-video/480p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.025, + "source": "https://fal.ai/models/fal-ai/longcat-video/text-to-video/480p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/longcat-video/text-to-video/720p": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://fal.ai/models/fal-ai/longcat-video/text-to-video/720p", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/audio-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/audio-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/extend-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/extend-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.16, 2160p $0.32"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.08, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/image-to-video/fast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.12, 2160p $0.24"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/image-to-video/fast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/reframe": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.2, + "output_cost_per_second_1080p": 0.2, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/reframe", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/retake-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/retake-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.16, 2160p $0.32"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.08, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-2.3/text-to-video/fast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.12, 2160p $0.24"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "source": "https://fal.ai/models/fal-ai/ltx-2.3/text-to-video/fast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltxv-13b-098-distilled": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/fal-ai/ltxv-13b-098-distilled", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltxv-13b-098-distilled/extend": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/fal-ai/ltxv-13b-098-distilled/extend", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltxv-13b-098-distilled/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/fal-ai/ltxv-13b-098-distilled/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltxv-13b-098-distilled/multiconditioning": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://fal.ai/models/fal-ai/ltxv-13b-098-distilled/multiconditioning", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/lumina-image/v2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/lumina-image/v2 at $0.075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.075, + "output_cost_per_pixel": 7.152557373046875e-08, + "source": "https://fal.ai/models/fal-ai/lumina-image/v2", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/marigold-v2": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/marigold-v2", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/minimax/hailuo-02-fast/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 512p rate; tiers without a catalog field: 512p $0.017"}, + "mode": "video_generation", + "output_cost_per_second": 0.017, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-02-fast/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/hailuo-02/pro/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-02/pro/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/hailuo-02/pro/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-02/pro/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/hailuo-02/standard/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.045, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-02/standard/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/image-01/subject-reference": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/fal-ai/minimax/image-01/subject-reference", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/mmaudio-v2": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.001, + "source": "https://fal.ai/models/fal-ai/mmaudio-v2", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/moss-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/moss-image at $0.05 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "output_cost_per_pixel": 4.76837158203125e-08, + "source": "https://fal.ai/models/fal-ai/moss-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/nafnet/deblur": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/nafnet/deblur at $0.0225 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0225, + "output_cost_per_pixel": 2.1457672119140624e-08, + "source": "https://fal.ai/models/fal-ai/nafnet/deblur", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/nafnet/denoise": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/nafnet/denoise at $0.0225 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0225, + "output_cost_per_pixel": 2.1457672119140624e-08, + "source": "https://fal.ai/models/fal-ai/nafnet/denoise", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/nano-banana-2/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/nano-banana-2/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/nano-banana-pro/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://fal.ai/models/fal-ai/nano-banana-pro/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/nano-banana/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.0398, + "source": "https://fal.ai/models/fal-ai/nano-banana/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/nucleus-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/nucleus-image at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/nucleus-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/omnigen-v2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/omnigen-v2 at $0.15 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.15, + "output_cost_per_pixel": 1.430511474609375e-07, + "source": "https://fal.ai/models/fal-ai/omnigen-v2", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/one-to-all-animation/1.3b": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 480p rate; tiers without a catalog field: 580p $0.045"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_480p": 0.03, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/fal-ai/one-to-all-animation/1.3b", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/one-to-all-animation/14b": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 480p rate; tiers without a catalog field: 580p $0.09"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_480p": 0.06, + "output_cost_per_second_720p": 0.12, + "source": "https://fal.ai/models/fal-ai/one-to-all-animation/14b", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/patina": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/patina at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/patina", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/phota/enhance": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.13, + "source": "https://fal.ai/models/fal-ai/phota/enhance", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/blur": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/blur", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/chromatic-aberration": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/chromatic-aberration", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/color-correction": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/color-correction", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/color-tint": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/color-tint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/desaturate": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/desaturate", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/dissolve": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/dissolve", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/dodge-burn": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/dodge-burn", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/grain": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/grain", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/parabolize": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/parabolize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/sharpen": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/sharpen", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/solarize": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/solarize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/post-processing/vignette": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://fal.ai/models/fal-ai/post-processing/vignette", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/qwen-image-2512": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-2512 at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-2512", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/qwen-image-2512/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-2512/lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-2512/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/qwen-image-edit-2509": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509 at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/add-background": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/add-background at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/add-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/face-to-full-portrait": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/face-to-full-portrait at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/face-to-full-portrait", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/group-photo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/group-photo at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/group-photo", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/integrate-product": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/integrate-product at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/integrate-product", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/lighting-restoration": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/lighting-restoration at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/lighting-restoration", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/multiple-angles": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/multiple-angles at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/multiple-angles", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/next-scene": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/next-scene at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/next-scene", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/remove-element": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/remove-element at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/remove-element", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/remove-lighting": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/remove-lighting at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/remove-lighting", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2509-lora-gallery/shirt-design": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2509-lora-gallery/shirt-design at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2509-lora-gallery/shirt-design", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2511": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2511 at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2511", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2511-multiple-angles": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2511-multiple-angles at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2511-multiple-angles", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-2511/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-2511/lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-2511/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/add-background": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/add-background at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/add-background", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/face-to-full-portrait": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/face-to-full-portrait at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/face-to-full-portrait", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/group-photo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/group-photo at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/group-photo", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/integrate-product": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/integrate-product at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/integrate-product", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/lighting-restoration": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/lighting-restoration at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/lighting-restoration", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/multiple-angles": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/multiple-angles at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/multiple-angles", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/next-scene": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/next-scene at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/next-scene", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/remove-element": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/remove-element at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/remove-element", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/remove-lighting": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/remove-lighting at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/remove-lighting", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit-plus-lora-gallery/shirt-design": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit-plus-lora-gallery/shirt-design at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit-plus-lora-gallery/shirt-design", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit/image-to-image at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-edit/inpaint": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image-edit/inpaint at $0.03 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "output_cost_per_pixel": 2.86102294921875e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image-edit/inpaint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-layered": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/qwen-image-layered", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image-layered/lora": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/fal-ai/qwen-image-layered/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/qwen-image/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/qwen-image/image-to-image at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/qwen-image/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/recraft/upscale/creative": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://fal.ai/models/fal-ai/recraft/upscale/creative", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/recraft/upscale/crisp": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.004, + "source": "https://fal.ai/models/fal-ai/recraft/upscale/crisp", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/recraft/v4.1/pro/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.21, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/pro/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4.1/pro/text-to-vector": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/pro/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4.1/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4.1/text-to-vector": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4.1/utility/pro/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.21, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/utility/pro/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4.1/utility/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/fal-ai/recraft/v4.1/utility/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4/pro/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://fal.ai/models/fal-ai/recraft/v4/pro/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4/pro/text-to-vector": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://fal.ai/models/fal-ai/recraft/v4/pro/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/recraft/v4/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/v4/text-to-vector": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/recraft/v4/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/recraft/vectorize": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/fal-ai/recraft/vectorize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/sana": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/sana at $0.001 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.001, + "output_cost_per_pixel": 9.5367431640625e-10, + "source": "https://fal.ai/models/fal-ai/sana", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/sana/sprint": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/sana/sprint at $0.0025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0025, + "output_cost_per_pixel": 2.384185791015625e-09, + "source": "https://fal.ai/models/fal-ai/sana/sprint", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/sana/v1.5/1.6b": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/sana/v1.5/1.6b at $0.0075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0075, + "output_cost_per_pixel": 7.152557373046875e-09, + "source": "https://fal.ai/models/fal-ai/sana/v1.5/1.6b", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/sana/v1.5/4.8b": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/sana/v1.5/4.8b at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/sana/v1.5/4.8b", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/seedvr/upscale/image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/seedvr/upscale/image at $0.001 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.001, + "output_cost_per_pixel": 9.5367431640625e-10, + "source": "https://fal.ai/models/fal-ai/seedvr/upscale/image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/seedvr/upscale/image/seamless": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/seedvr/upscale/image/seamless at $0.0025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0025, + "output_cost_per_pixel": 2.384185791015625e-09, + "source": "https://fal.ai/models/fal-ai/seedvr/upscale/image/seamless", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/stable-diffusion-v3-medium": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/fal-ai/stable-diffusion-v3-medium", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/stable-diffusion-v35-large": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/stable-diffusion-v35-large at $0.065 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.065, + "output_cost_per_pixel": 6.198883056640625e-08, + "source": "https://fal.ai/models/fal-ai/stable-diffusion-v35-large", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/stepx-edit2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/stepx-edit2 at $0.2 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.2, + "output_cost_per_pixel": 1.9073486328125e-07, + "source": "https://fal.ai/models/fal-ai/stepx-edit2", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/telestyle-v2": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/telestyle-v2 at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/telestyle-v2", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/uso": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/uso at $0.1 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "output_cost_per_pixel": 9.5367431640625e-08, + "source": "https://fal.ai/models/fal-ai/uso", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/vidu/q2/reference-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/vidu/q2/reference-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/vidu/q2/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/vidu/q2/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/vidu/reference-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/fal-ai/vidu/reference-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/wan-22-vace-fun-a14b/depth": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-22-vace-fun-a14b/depth", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-22-vace-fun-a14b/inpainting": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-22-vace-fun-a14b/inpainting", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-22-vace-fun-a14b/outpainting": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-22-vace-fun-a14b/outpainting", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-22-vace-fun-a14b/reframe": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-22-vace-fun-a14b/reframe", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-25-preview/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/wan-25-preview/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/wan-25-preview/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-25-preview/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-25-preview/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/wan-25-preview/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan-25-preview/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-25-preview/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b/depth": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b/depth", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b/inpainting": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b/inpainting", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b/outpainting": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b/outpainting", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b/pose": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b/pose", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-14b/reframe": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan-vace-14b/reframe", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-apps/video-edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan-vace-apps/video-edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-14b/speech-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 480p rate; tiers without a catalog field: 580p $0.15"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_480p": 0.1, + "output_cost_per_second_720p": 0.2, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-14b/speech-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-5b/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.016, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-5b/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/wan/v2.2-a14b/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/image-to-video/lora": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/image-to-video/lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/text-to-image/lora": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/text-to-image/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/text-to-video/lora": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/text-to-video/lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-a14b/video-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 580p $0.06"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-a14b/video-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.7/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/wan/v2.7/edit-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/edit-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.7/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.7/pro/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/pro/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/wan/v2.7/pro/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/pro/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan/v2.7/reference-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.7/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/wan/v2.7/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/fal-ai/wan/v2.7/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/z-image/base": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/base at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/z-image/base", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/z-image/base/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/base/lora at $0.012 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.012, + "output_cost_per_pixel": 1.1444091796875e-08, + "source": "https://fal.ai/models/fal-ai/z-image/base/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/z-image/turbo": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/z-image/turbo/controlnet": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/controlnet at $0.0065 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0065, + "output_cost_per_pixel": 6.198883056640625e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/controlnet", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/controlnet/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/controlnet/lora at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/controlnet/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/image-to-image at $0.005 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.005, + "output_cost_per_pixel": 4.76837158203125e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/image-to-image/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/image-to-image/lora at $0.0085 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0085, + "output_cost_per_pixel": 8.106231689453126e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/image-to-image/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/inpaint": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/inpaint at $0.01 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "output_cost_per_pixel": 9.5367431640625e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/inpaint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/inpaint/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/inpaint/lora at $0.0115 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0115, + "output_cost_per_pixel": 1.0967254638671875e-08, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/inpaint/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/z-image/turbo/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/lora at $0.0085 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0085, + "output_cost_per_pixel": 8.106231689453126e-09, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/z-image/turbo/tiling": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/tiling at $0.02 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "output_cost_per_pixel": 1.9073486328125e-08, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/tiling", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/z-image/turbo/tiling/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills fal-ai/z-image/turbo/tiling/lora at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/fal-ai/z-image/turbo/tiling/lora", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.1/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 360p $0.03"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.1/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.1/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 360p $0.03"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.1/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.1/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 360p $0.03"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.1/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.1/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 360p $0.03"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.1/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/flash/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.094, + "output_cost_per_second_720p": 0.094, + "source": "https://fal.ai/models/kling/v4/flash/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/flash/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.094, + "output_cost_per_second_720p": 0.094, + "source": "https://fal.ai/models/kling/v4/flash/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/flash/video-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.063, + "output_cost_per_second_720p": 0.063, + "source": "https://fal.ai/models/kling/v4/flash/video-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.235, + "output_cost_per_second_1080p": 0.235, + "output_cost_per_second_720p": 0.157, + "source": "https://fal.ai/models/kling/v4/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.235, + "output_cost_per_second_1080p": 0.235, + "output_cost_per_second_720p": 0.157, + "source": "https://fal.ai/models/kling/v4/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/video-to-video/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.141, + "output_cost_per_second_1080p": 0.141, + "output_cost_per_second_720p": 0.094, + "source": "https://fal.ai/models/kling/v4/video-to-video/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/krea/v3/edit": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills krea/v3/edit at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/krea/v3/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/krea/v3/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills krea/v3/text-to-image at $0.025 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.025, + "output_cost_per_pixel": 2.384185791015625e-08, + "source": "https://fal.ai/models/krea/v3/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/luma/agent/uni-1/v1/max": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.102, + "source": "https://fal.ai/models/luma/agent/uni-1/v1/max", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/luma/agent/uni-1/v1/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.042, + "source": "https://fal.ai/models/luma/agent/uni-1/v1/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/meta/muse-image/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/meta/muse-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/meta/muse-image/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/meta/muse-image/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/minimax/h3-max-turbo/extend-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_1080p": 0.08, + "output_cost_per_second_2k": 0.16, + "output_cost_per_second_480p": 0.025, + "output_cost_per_second_768p": 0.04, + "source": "https://fal.ai/models/minimax/h3-max-turbo/extend-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max-turbo/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.16, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max-turbo/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/extend-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.16, + "output_cost_per_second_2k": 0.32, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/extend-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/insert-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.06, + "source": "https://fal.ai/models/minimax/h3-max/insert-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/lip-sync/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.16, + "output_cost_per_second_2k": 0.32, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/lip-sync/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.13, + "output_cost_per_second_2k": 0.13, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_4k": 0.16, + "output_cost_per_second_768p": 0.06, + "source": "https://fal.ai/models/minimax/h3/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3/image-to-video/lora": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.1625, + "output_cost_per_second_2k": 0.1625, + "output_cost_per_second_480p": 0.0625, + "output_cost_per_second_4k": 0.2, + "output_cost_per_second_768p": 0.075, + "source": "https://fal.ai/models/minimax/h3/image-to-video/lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3/reference-to-video/lora": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.1625, + "output_cost_per_second_2k": 0.1625, + "output_cost_per_second_480p": 0.0625, + "output_cost_per_second_4k": 0.2, + "output_cost_per_second_768p": 0.075, + "source": "https://fal.ai/models/minimax/h3/reference-to-video/lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3/text-to-video/lora": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.1625, + "output_cost_per_second_2k": 0.1625, + "output_cost_per_second_480p": 0.0625, + "output_cost_per_second_4k": 0.2, + "output_cost_per_second_768p": 0.075, + "source": "https://fal.ai/models/minimax/h3/text-to-video/lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/nvidia/cosmos-3-super/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/nvidia/cosmos-3-super/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/pixelcut/background-removal": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.016, + "source": "https://fal.ai/models/pixelcut/background-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/pixelcut/looping-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.32, + "output_cost_per_second_1080p": 0.32, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_768p": 0.16, + "source": "https://fal.ai/models/pixelcut/looping-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/pixelcut/product-photo": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.024, + "source": "https://fal.ai/models/pixelcut/product-photo", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/pixverse/music-video/vibemv": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.09, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/pixverse/music-video/vibemv", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/recraft/v4.1/flash/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.007, + "source": "https://fal.ai/models/recraft/v4.1/flash/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/rundiffusion-fal/juggernaut-flux/base": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/juggernaut-flux/base at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/rundiffusion-fal/juggernaut-flux/base", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/rundiffusion-fal/juggernaut-flux/base/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/juggernaut-flux/base/image-to-image at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/rundiffusion-fal/juggernaut-flux/base/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/rundiffusion-fal/juggernaut-flux/lightning": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/juggernaut-flux/lightning at $0.006 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.006, + "output_cost_per_pixel": 5.7220458984375e-09, + "source": "https://fal.ai/models/rundiffusion-fal/juggernaut-flux/lightning", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/rundiffusion-fal/juggernaut-flux/pro": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/juggernaut-flux/pro at $0.055 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.055, + "output_cost_per_pixel": 5.245208740234375e-08, + "source": "https://fal.ai/models/rundiffusion-fal/juggernaut-flux/pro", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/rundiffusion-fal/juggernaut-flux/pro/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/juggernaut-flux/pro/image-to-image at $0.055 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.055, + "output_cost_per_pixel": 5.245208740234375e-08, + "source": "https://fal.ai/models/rundiffusion-fal/juggernaut-flux/pro/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/rundiffusion-fal/rundiffusion-photo-flux": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills rundiffusion-fal/rundiffusion-photo-flux at $0.045 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.045, + "output_cost_per_pixel": 4.291534423828125e-08, + "source": "https://fal.ai/models/rundiffusion-fal/rundiffusion-photo-flux", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/smoretalk-ai/rembg-enhance": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.006, + "source": "https://fal.ai/models/smoretalk-ai/rembg-enhance", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/sonilo/v1.1/video-to-video-music": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.009, + "source": "https://fal.ai/models/sonilo/v1.1/video-to-video-music", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/sonilo/v1.1/video-to-video-sound-effects": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.009, + "source": "https://fal.ai/models/sonilo/v1.1/video-to-video-sound-effects", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/tbc/mischief-1/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.01, + "output_cost_per_second_480p": 0.005, + "output_cost_per_second_720p": 0.01, + "source": "https://fal.ai/models/tbc/mischief-1/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/veed/fabric-1.0": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p)"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.15, + "source": "https://fal.ai/models/veed/fabric-1.0", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/veed/fabric-1.0/fast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p)"}, + "mode": "video_generation", + "output_cost_per_second": 0.2, + "output_cost_per_second_480p": 0.1, + "output_cost_per_second_720p": 0.2, + "source": "https://fal.ai/models/veed/fabric-1.0/fast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/veed/fabric-1.0/text": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p)"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.15, + "source": "https://fal.ai/models/veed/fabric-1.0/text", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/wan/v2.6/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/wan/v2.6/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/wan/v2.6/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/wan/v2.6/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/wan/v2.6/image-to-video/flash": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.075, + "output_cost_per_second_1080p": 0.075, + "output_cost_per_second_720p": 0.05, + "source": "https://fal.ai/models/wan/v2.6/image-to-video/flash", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/wan/v2.6/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/wan/v2.6/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/wan/v2.6/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/wan/v2.6/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/wan/v2.6/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/wan/v2.6/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://fal.ai/models/xai/grok-imagine-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/xai/grok-imagine-video/v1.5/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.14, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/xai/grok-imagine-video/v1.5/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/alibaba/happy-horse/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/alibaba/happy-horse/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/creatify/boreal": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.01, + "output_cost_per_second_1080p": 0.03, + "output_cost_per_second_2k": 0.12, + "output_cost_per_second_720p": 0.01, + "source": "https://fal.ai/models/creatify/boreal", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/creatify/aurora": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.14, + "output_cost_per_second_480p": 0.07, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/fal-ai/creatify/aurora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/vidu/q4/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 540p $0.0315"}, + "mode": "video_generation", + "output_cost_per_second": 0.0665, + "output_cost_per_second_1080p": 0.084, + "output_cost_per_second_2k": 0.133, + "output_cost_per_second_4k": 0.273, + "output_cost_per_second_720p": 0.0665, + "source": "https://fal.ai/models/fal-ai/vidu/q4/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/vidu/q4/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; tiers without a catalog field: 540p $0.0315"}, + "mode": "video_generation", + "output_cost_per_second": 0.0665, + "output_cost_per_second_1080p": 0.084, + "output_cost_per_second_2k": 0.133, + "output_cost_per_second_4k": 0.273, + "output_cost_per_second_720p": 0.0665, + "source": "https://fal.ai/models/fal-ai/vidu/q4/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/ideogram/v4.5": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/ideogram/v4.5", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/ideogram/v4.5/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://fal.ai/models/ideogram/v4.5/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/ideogram/v4/instant": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"fal bills ideogram/v4/instant at $0.0075 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries"}, + "mode": "image_generation", + "output_cost_per_image": 0.0075, + "output_cost_per_pixel": 7.152557373046875e-09, + "source": "https://fal.ai/models/ideogram/v4/instant", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/lightricks/ltx-2.5/image-to-video/fast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.19"}, + "mode": "video_generation", + "output_cost_per_second": 0.13, + "output_cost_per_second_1080p": 0.13, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.09, + "source": "https://fal.ai/models/lightricks/ltx-2.5/image-to-video/fast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/lightricks/ltx-2.5/image-to-video/pro": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.17, + "output_cost_per_second_720p": 0.12, + "source": "https://fal.ai/models/lightricks/ltx-2.5/image-to-video/pro", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/lightricks/ltx-2.5/text-to-video/fast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate; tiers without a catalog field: 1440p $0.19"}, + "mode": "video_generation", + "output_cost_per_second": 0.13, + "output_cost_per_second_1080p": 0.13, + "output_cost_per_second_4k": 0.3, + "output_cost_per_second_720p": 0.09, + "source": "https://fal.ai/models/lightricks/ltx-2.5/text-to-video/fast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/lightricks/ltx-2.5/text-to-video/pro": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.17, + "output_cost_per_second_720p": 0.12, + "source": "https://fal.ai/models/lightricks/ltx-2.5/text-to-video/pro", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/recast": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.45, + "output_cost_per_second_1080p": 0.45, + "output_cost_per_second_768p": 0.3, + "source": "https://fal.ai/models/minimax/h3-max/recast", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/reframe-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/reframe-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/relight": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.16, + "output_cost_per_second_2k": 0.32, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/relight", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/wan/v2.6/reference-to-video/flash": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 1080p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://fal.ai/models/wan/v2.6/reference-to-video/flash", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/v1.5/lite/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.14, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_720p": 0.03, + "source": "https://fal.ai/models/xai/grok-imagine-video/v1.5/lite/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/v1.5/lite/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.14, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_720p": 0.03, + "source": "https://fal.ai/models/xai/grok-imagine-video/v1.5/lite/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3-max/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; promotional 50% off rate through 2026-11-22 23:59 UTC per the page; regular rates $0.17 (720p) and $0.29 (1080p) per second resume 2026-11-23 00:00 UTC (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.085, + "output_cost_per_second_1080p": 0.145, + "output_cost_per_second_720p": 0.085, + "source": "https://fal.ai/models/blackforestlabs/flux-3-max/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3-max/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; promotional 50% off rate through 2026-11-22 23:59 UTC per the page; regular rates $0.17 (720p) and $0.29 (1080p) per second resume 2026-11-23 00:00 UTC (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.085, + "output_cost_per_second_1080p": 0.145, + "output_cost_per_second_720p": 0.085, + "source": "https://fal.ai/models/blackforestlabs/flux-3-max/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3-max/keyframes-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; regular rate; the page's 75% promotion ran October 15 to November 7, 2026 and has ended (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_720p": 0.17, + "source": "https://fal.ai/models/blackforestlabs/flux-3-max/keyframes-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/heygen/heygen-video/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; regular rate; the page's 50% promotion ended October 31, 2026 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.02, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.06, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_768p": 0.02, + "source": "https://fal.ai/models/heygen/heygen-video/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/heygen/heygen-video/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; regular rate; the page's 50% promotion ended October 31, 2026 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.02, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.06, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_768p": 0.02, + "source": "https://fal.ai/models/heygen/heygen-video/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/heygen/heygen-video/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; regular rate; the page's 50% promotion ended October 31, 2026 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.02, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.06, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_768p": 0.02, + "source": "https://fal.ai/models/heygen/heygen-video/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max-turbo/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; promotional 30% off rate through 2026-11-15 per the page; standard rates $0.025 (480p), $0.04 (768p), $0.08 (1080p) per second from 2026-11-16 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.028, + "output_cost_per_second_1080p": 0.056, + "output_cost_per_second_480p": 0.0175, + "output_cost_per_second_768p": 0.028, + "source": "https://fal.ai/models/minimax/h3-max-turbo/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max-turbo/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; promotional 30% off rate through 2026-11-15 per the page; standard rates $0.025 (480p), $0.04 (768p), $0.08 (1080p) per second from 2026-11-16 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.028, + "output_cost_per_second_1080p": 0.056, + "output_cost_per_second_480p": 0.0175, + "output_cost_per_second_768p": 0.028, + "source": "https://fal.ai/models/minimax/h3-max-turbo/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/camera-controls": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 480p rate; promotional 30% off rate through 2026-11-15 per the page; standard rates $0.05 (480p), $0.08 (768p), $0.16 (1080p) per second from 2026-11-16 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.035, + "output_cost_per_second_1080p": 0.112, + "output_cost_per_second_480p": 0.035, + "output_cost_per_second_768p": 0.056, + "source": "https://fal.ai/models/minimax/h3-max/camera-controls", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; promotional 30% off rate through 2026-11-15 per the page; standard rates $0.05 (480p), $0.08 (768p), $0.16 (1080p) per second from 2026-11-16 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.056, + "output_cost_per_second_1080p": 0.112, + "output_cost_per_second_480p": 0.035, + "output_cost_per_second_768p": 0.056, + "source": "https://fal.ai/models/minimax/h3-max/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768p rate; promotional 30% off rate through 2026-11-15 per the page; standard rates $0.05 (480p), $0.08 (768p), $0.16 (1080p) per second from 2026-11-16 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10)"}, + "mode": "video_generation", + "output_cost_per_second": 0.056, + "output_cost_per_second_1080p": 0.112, + "output_cost_per_second_480p": 0.035, + "output_cost_per_second_768p": 0.056, + "source": "https://fal.ai/models/minimax/h3-max/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] } } diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 59c772f2993..8ff7f8bd251 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1453,6 +1453,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "fal_ai/bytedance/seedance-2.0/text-to-video", "fal_ai/bytedance/seedance-2.0/image-to-video", "fal_ai/bytedance/seedance-2.0/reference-to-video", + "fal_ai/bytedance/seedance-2.5/us/draft/complete", # 1080p is $1.164/second per fal.ai + "fal_ai/bytedance/seedance-2.5/us/image-to-video", # 1080p is $1.164/second per fal.ai + "fal_ai/bytedance/seedance-2.5/us/text-to-video", # 1080p is $1.164/second per fal.ai ] is_valid, violations = validate_model_cost_values(actual_json, exceptions) From 53e1e9b87b36dabed967b17338948535020b7abc Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:41:59 -0800 Subject: [PATCH 046/189] fix(vertex_ai): bill gemini image edits by token usage (#51038) * fix(vertex_ai): bill Gemini image edit token usage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(vertex_ai): validate image edit usage before normalization Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kenneth Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../gemini/image_generation/transformation.py | 2 +- .../llms/gemini/image_usage_transformation.py | 6 +- .../vertex_gemini_transformation.py | 18 ++++- ...est_vertex_ai_image_edit_transformation.py | 40 ++++++++++ ...tex_ai_image_generation_cost_calculator.py | 78 +++++++++++++++++++ 5 files changed, 137 insertions(+), 7 deletions(-) diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index 1f986b11af2..6d1e1cf3931 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -170,7 +170,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): return request_body_obj.model_dump(exclude_none=True) def _transform_image_usage(self, usage_metadata: dict): - return transform_gemini_image_usage(usage_metadata) + return transform_gemini_image_usage(_JSON_DICT.validate_python(usage_metadata)) def transform_image_generation_response( self, diff --git a/litellm/llms/gemini/image_usage_transformation.py b/litellm/llms/gemini/image_usage_transformation.py index 5f08d8403af..3ef403f79ee 100644 --- a/litellm/llms/gemini/image_usage_transformation.py +++ b/litellm/llms/gemini/image_usage_transformation.py @@ -1,5 +1,7 @@ from typing import Final +from pydantic import TypeAdapter + from litellm.types.utils import ImageUsage, ImageUsageInputTokensDetails @@ -34,12 +36,12 @@ def _sum_modality_token_details(usage_metadata: dict, *details_keys: str) -> Ima return tokens_details -def transform_gemini_image_usage(usage_metadata: dict) -> ImageUsage: +def transform_gemini_image_usage(usage_metadata: dict[str, object]) -> ImageUsage: """ Transform Gemini usageMetadata to ImageUsage format. """ input_tokens_details = _sum_modality_token_details(usage_metadata, "promptTokensDetails", "prompt_tokens_details") - output_tokens: Final = usage_metadata.get("candidatesTokenCount", 0) + output_tokens: Final = TypeAdapter(int).validate_python(usage_metadata.get("candidatesTokenCount", 0)) output_tokens_details: Final = _sum_modality_token_details( usage_metadata, "candidatesTokensDetails", "candidates_tokens_details" ) diff --git a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py index 3633bb68b50..8e9bef97085 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py @@ -6,10 +6,12 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, cast import httpx from httpx._types import RequestFiles +from pydantic import ConfigDict, TypeAdapter import litellm from litellm.images.utils import ImageEditRequestUtils from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.gemini.image_usage_transformation import transform_gemini_image_usage from litellm.llms.vertex_ai.common_utils import get_vertex_base_url, map_vertex_image_size from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.secret_managers.main import get_secret_str @@ -20,7 +22,9 @@ from litellm.types.llms.vertex_ai import ( HttpxPartType, ) from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage +from litellm.types.utils import FileTypes, ImageObject, ImageResponse + +_JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -202,7 +206,6 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, ) -> ImageResponse: - model_response: Final = ImageResponse() try: response_json: Final = _generate_content_payload(raw_response) except Exception as exc: @@ -223,8 +226,15 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): if (inline_data := part.get("inlineData")) and (b64_json := inline_data.get("data")) ] - model_response.data = cast(list[OpenAIImage], data_list) - return model_response + usage_metadata: Final = _JSON_DICT.validate_python(response_json.get("usageMetadata") or {}) + traffic_type: Final = usage_metadata.get("trafficType") if usage_metadata else None + return ImageResponse( + data=data_list, + usage=transform_gemini_image_usage(usage_metadata) if usage_metadata else None, + hidden_params={"provider_specific_fields": {"traffic_type": traffic_type}} + if isinstance(traffic_type, str) and traffic_type + else {}, + ) def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[HttpxPartType]: images: Final[list[FileTypes]] = image if isinstance(image, list) else [image] diff --git a/tests/unit/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py b/tests/unit/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py index f36480ae7df..e311313c136 100644 --- a/tests/unit/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py +++ b/tests/unit/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py @@ -24,6 +24,46 @@ from litellm.llms.vertex_ai.image_edit.vertex_imagen_transformation import ( from litellm.types.router import GenericLiteLLMParams +@pytest.mark.parametrize("traffic_type", ("ON_DEMAND_PRIORITY", "", None)) +def test_vertex_gemini_edit_preserves_usage_and_traffic_type(traffic_type: str | None) -> None: + response: Final = VertexAIGeminiImageEditConfig().transform_image_edit_response( + model="gemini-3.1-flash-image", + raw_response=httpx.Response( + 200, + json={ + "candidates": [], + "usageMetadata": { + "promptTokenCount": 1131, + "candidatesTokenCount": 0, + "totalTokenCount": 1131, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 7}, + {"modality": "IMAGE", "tokenCount": 1124}, + ], + **({"trafficType": traffic_type} if traffic_type is not None else {}), + }, + }, + ), + logging_obj=Logging( + model="gemini-3.1-flash-image", + messages=[], + stream=False, + call_type="image_edit", + start_time=datetime(2025, 1, 1), + litellm_call_id="edit", + function_id="edit", + ), + ) + assert response.usage is not None + assert response.usage.input_tokens == 1131 + assert response.usage.output_tokens == 0 + assert response.usage.input_tokens_details.text_tokens == 7 + assert response.usage.input_tokens_details.image_tokens == 1124 + assert response.hidden_params.get("provider_specific_fields", {}) == ( + {"traffic_type": traffic_type} if traffic_type else {} + ) + + @pytest.mark.parametrize("size", ("1536x1024", "1024x1536", "3072x2048", "256x256", "512x512", "4096x4096")) @pytest.mark.parametrize("model", ("gemini-3.1-flash-image", "gemini-3-pro-image", "gemini-3.1-flash-lite-image")) def test_vertex_gemini_edit_size_reaches_request(size: str, model: str) -> None: diff --git a/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py b/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py index e54e25cbd18..65d6e97f8e5 100644 --- a/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py +++ b/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py @@ -1,7 +1,16 @@ import os +from datetime import datetime +from typing import Final + +import httpx +import pytest import litellm +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.vertex_ai.gemini.cost_calculator import cost_per_web_search_request +from litellm.llms.vertex_ai.image_edit.vertex_gemini_transformation import ( + VertexAIGeminiImageEditConfig, +) from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_image_generation_cost_calculator, ) @@ -15,6 +24,75 @@ from litellm.types.utils import ( ) +@pytest.mark.parametrize("call_type", ("image_edit", "aimage_edit")) +@pytest.mark.parametrize("has_image, has_usage", ((True, True), (False, True), (True, False))) +def test_vertex_image_edit_cost_uses_input_tokens_and_retains_image_fallback( + call_type: str, + has_image: bool, + has_usage: bool, +) -> None: + model: Final = "gemini-3.1-flash-image" + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") + response: Final = VertexAIGeminiImageEditConfig().transform_image_edit_response( + model=model, + raw_response=httpx.Response( + 200, + json={ + "candidates": [ + { + "finishReason": "STOP" if has_image else "IMAGE_RECITATION", + **( + {"content": {"parts": [{"inlineData": {"mimeType": "image/png", "data": "aW1n"}}]}} + if has_image + else {} + ), + } + ], + **( + { + "usageMetadata": { + "promptTokenCount": 1131, + "candidatesTokenCount": 100 if has_image else 0, + "totalTokenCount": 1231 if has_image else 1131, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 7}, + {"modality": "IMAGE", "tokenCount": 1124}, + ], + } + } + if has_usage + else {} + ), + }, + ), + logging_obj=Logging( + model=model, + messages=[], + stream=False, + call_type=call_type, + start_time=datetime(2025, 1, 1), + litellm_call_id="edit", + function_id="edit", + ), + ) + cost: Final = litellm.completion_cost( + model=model, + custom_llm_provider="vertex_ai", + completion_response=response, + call_type=call_type, + ) + expected: Final = ( + 7 * (model_info["input_cost_per_token"] or 0) + + 1124 * (model_info.get("input_cost_per_image_token") or model_info["input_cost_per_token"] or 0) + + (100 if has_image else 0) * (model_info.get("output_cost_per_image_token") or 0) + if has_usage + else model_info["output_cost_per_image"] + ) + assert expected is not None and expected > 0 + assert cost == pytest.approx(expected) + assert vertex_image_generation_cost_calculator(model=model, image_response=response) == pytest.approx(expected) + + def _image_response_with_web_search(web_search_requests): usage = ImageUsage( input_tokens=20, From 9ef55423392dd042e6e2b256b328520d3f77349f Mon Sep 17 00:00:00 2001 From: giladk1221 Date: Wed, 11 Nov 2026 00:43:36 +0200 Subject: [PATCH 047/189] feat(search): add Microsoft Web IQ provider (#50883) * feat(search): add Microsoft Web IQ provider * fix(search): require HTTPS for Web IQ credentials * fix(search): include Web IQ in the packaged endpoint catalog * fix(search): register Microsoft Web IQ query pricing * test(search): fix Web IQ exception assertion lint * fix(search): type Web IQ response passthrough and search error headers The type-check gate failed on 8 new unknown-type errors in the Web IQ transformation. Build passthrough fields from a typed helper and validate the response models directly, and type the base search error headers as dict[str, str] so subclasses no longer inherit an unknown parameter. * fix(search): raise Web IQ parse errors without widening the base header type Typing the base search error headers as dict[str, str] made the shared HTTP handler report one more argument-type error. Restore the base signature and raise the same BaseLLMException directly from Web IQ. --------- Co-authored-by: Gilad Katz <29776916+giladk1221@users.noreply.github.com> Co-authored-by: ryan-crabbe-berri --- .github/workflows/test-unit.yml | 1 + litellm/llms/webiq/__init__.py | 0 litellm/llms/webiq/search/__init__.py | 0 litellm/llms/webiq/search/transformation.py | 156 ++++++++ ...odel_prices_and_context_window_backup.json | 5 + .../provider_endpoints_support_backup.json | 26 ++ litellm/types/utils.py | 1 + litellm/utils.py | 2 + model_prices_and_context_window.json | 5 + provider_endpoints_support.json | 26 ++ tests/unit/llms/webiq/__init__.py | 0 tests/unit/llms/webiq/search/__init__.py | 0 .../test_webiq_search_transformation.py | 351 ++++++++++++++++++ .../test_search_tool_management.py | 11 + .../public_endpoints/test_public_endpoints.py | 16 +- tests/unit/search/test_main.py | 13 + 16 files changed, 611 insertions(+), 2 deletions(-) create mode 100644 litellm/llms/webiq/__init__.py create mode 100644 litellm/llms/webiq/search/__init__.py create mode 100644 litellm/llms/webiq/search/transformation.py create mode 100644 tests/unit/llms/webiq/__init__.py create mode 100644 tests/unit/llms/webiq/search/__init__.py create mode 100644 tests/unit/llms/webiq/search/test_webiq_search_transformation.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index fb29a92e306..d3f4821a05b 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -397,6 +397,7 @@ jobs: tests/unit/llms/voyage tests/unit/llms/wandb tests/unit/llms/watsonx + tests/unit/llms/webiq tests/unit/llms/xai tests/unit/llms/xinference tests/unit/llms/you_com diff --git a/litellm/llms/webiq/__init__.py b/litellm/llms/webiq/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/webiq/search/__init__.py b/litellm/llms/webiq/search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/webiq/search/transformation.py b/litellm/llms/webiq/search/transformation.py new file mode 100644 index 00000000000..85d122a21af --- /dev/null +++ b/litellm/llms/webiq/search/transformation.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final +from urllib.parse import urlsplit + +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.search.transformation import BaseSearchConfig, SearchResponse, SearchResult + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + +class _WebIQResult(BaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + title: str + url: str + content: str + lastUpdatedAt: str | None = None + + +class _WebIQResponse(BaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + webResults: tuple[_WebIQResult, ...] + + +_DOMAINS: Final = TypeAdapter(tuple[str, ...]) + + +def _extra_fields(model: BaseModel, reserved: Mapping[str, object]) -> MappingProxyType[str, object]: + extra: Final[Mapping[str, object]] = model.model_extra or {} + return MappingProxyType({key: value for key, value in extra.items() if key not in reserved}) + + +def _query_with_domains(query: str, domains: object) -> str: + if domains is None: + return query + parsed: Final = _DOMAINS.validate_python(domains) + included: Final = tuple(f"site:{domain}" for domain in parsed if domain and not domain.startswith("-")) + excluded: Final = tuple(f"-site:{domain[1:]}" for domain in parsed if domain.startswith("-") and len(domain) > 1) + include_clause: Final = f"({' OR '.join(included)})" if included else "" + if not included and not excluded: + return query + return " ".join(part for part in (f"({query})", include_clause, *excluded) if part) + + +class WebIQSearchConfig(BaseSearchConfig): + DEFAULT_API_BASE: Final = "https://api.microsoft.ai/v3" + + @staticmethod + def ui_friendly_name() -> str: + return "Microsoft Web IQ" + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseSearchConfig interface + api_key: str | None = None, + api_base: str | None = None, + **kwargs: object, # kwargs-ok: BaseSearchConfig interface + ) -> dict[str, str]: # mutable-ok: HTTP handler requires a dict + if urlsplit(api_base or self.DEFAULT_API_BASE).scheme != "https": + raise ValueError("Microsoft Web IQ search requires an HTTPS api_base") + resolved_key: Final = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("WEBIQ_API_KEY",), + base_env_var=None, + default_api_base=self.DEFAULT_API_BASE, + ) + if not resolved_key: + raise ValueError("Set WEBIQ_API_KEY or pass api_key for Microsoft Web IQ search") + return { + **headers, + "x-apikey": resolved_key, + "Content-Type": "application/json", + } + + def get_complete_url( + self, + api_base: str | None, + optional_params: dict[str, object], # mutable-ok: BaseSearchConfig interface + data: dict[str, object] | list[dict[str, object]] | None = None, # mutable-ok: BaseSearchConfig interface + **kwargs: object, # kwargs-ok: BaseSearchConfig interface + ) -> str: + base: Final = (api_base or self.DEFAULT_API_BASE).rstrip("/") + return base if base.endswith("/search/web") else f"{base}/search/web" + + def transform_search_request( + self, + query: str | list[str], # mutable-ok: BaseSearchConfig interface + optional_params: dict[str, object], # mutable-ok: BaseSearchConfig interface + **kwargs: object, # kwargs-ok: BaseSearchConfig interface + ) -> dict[str, object]: # mutable-ok: HTTP handler requires a JSON dict + country: Final = optional_params.get("country") + mapped: Final = MappingProxyType( + { + "maxResults": optional_params.get("max_results"), + "region": country.upper() if isinstance(country, str) else None, + } + ) + native: Final = MappingProxyType( + { + key: value + for key, value in optional_params.items() + if key not in self.get_supported_perplexity_optional_params() + } + ) + return { + "contentFormat": "passage", + "maxLength": 5000, + **MappingProxyType({key: value for key, value in mapped.items() if value is not None}), + **native, + "query": _query_with_domains( + " ".join(query) if isinstance(query, list) else query, + optional_params.get("search_domain_filter"), + ), + } + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs: object, # kwargs-ok: BaseSearchConfig interface + ) -> SearchResponse: + try: + parsed: Final = _WebIQResponse.model_validate_json(raw_response.content) + except ValidationError as exc: + raise BaseLLMException( + status_code=502, + message=f"Microsoft Web IQ returned an invalid search response: {exc}", + headers=dict(raw_response.headers), + ) from exc + return SearchResponse.model_validate( + { + **_extra_fields(parsed, SearchResponse.model_fields), + "results": [ + SearchResult.model_validate( + { + **_extra_fields(result, SearchResult.model_fields), + "title": result.title, + "url": result.url, + "snippet": result.content, + "date": result.lastUpdatedAt or None, + "last_updated": result.lastUpdatedAt or None, + } + ) + for result in parsed.webResults + ], + } + ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 40ff675de19..83ab91f2bc0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25306,6 +25306,11 @@ "notes": "Firecrawl search pricing: $83 for 100,000 credits, 2 credits per 10 results. Cost = ceiling(limit/10) * 2 * $0.00083" } }, + "webiq/search": { + "input_cost_per_query": 0.007, + "litellm_provider": "webiq", + "mode": "search" + }, "perplexity/search": { "input_cost_per_query": 0.005, "litellm_provider": "perplexity", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 00fee0a0d5a..3431cc97969 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -4098,6 +4098,32 @@ "generateContent": false } }, + "webiq": { + "display_name": "Microsoft Web IQ (`webiq`)", + "url": "https://learn.microsoft.com/en-us/web-iq/reference/web-search", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, "tavily": { "display_name": "Tavily (`tavily`)", "url": "https://docs.litellm.ai/docs/search/tavily", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6325de96839..addd642eca7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4869,6 +4869,7 @@ class SearchProviders(str, Enum): AGENTCORE = "agentcore" NIMBLE = "nimble" BING_GROUNDING = "bing_grounding" + WEBIQ = "webiq" DEEPSEEK = "deepseek" XAI = "xai" diff --git a/litellm/utils.py b/litellm/utils.py index a3a1fe5ede4..4a4f2e8edfe 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -11357,6 +11357,7 @@ class ProviderConfigManager: from litellm.llms.serper.search.transformation import SerperSearchConfig from litellm.llms.tavily.search.transformation import TavilySearchConfig from litellm.llms.tinyfish.search.transformation import TinyfishSearchConfig + from litellm.llms.webiq.search.transformation import WebIQSearchConfig from litellm.llms.xai.search.transformation import XAISearchConfig from litellm.llms.you_com.search.transformation import YouComSearchConfig @@ -11381,6 +11382,7 @@ class ProviderConfigManager: SearchProviders.AGENTCORE: AgentCoreSearchConfig, SearchProviders.NIMBLE: NimbleSearchConfig, SearchProviders.BING_GROUNDING: BingGroundingSearchConfig, + SearchProviders.WEBIQ: WebIQSearchConfig, SearchProviders.DEEPSEEK: DeepSeekSearchConfig, SearchProviders.XAI: XAISearchConfig, } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 40ff675de19..83ab91f2bc0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25306,6 +25306,11 @@ "notes": "Firecrawl search pricing: $83 for 100,000 credits, 2 credits per 10 results. Cost = ceiling(limit/10) * 2 * $0.00083" } }, + "webiq/search": { + "input_cost_per_query": 0.007, + "litellm_provider": "webiq", + "mode": "search" + }, "perplexity/search": { "input_cost_per_query": 0.005, "litellm_provider": "perplexity", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 00fee0a0d5a..3431cc97969 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -4098,6 +4098,32 @@ "generateContent": false } }, + "webiq": { + "display_name": "Microsoft Web IQ (`webiq`)", + "url": "https://learn.microsoft.com/en-us/web-iq/reference/web-search", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, "tavily": { "display_name": "Tavily (`tavily`)", "url": "https://docs.litellm.ai/docs/search/tavily", diff --git a/tests/unit/llms/webiq/__init__.py b/tests/unit/llms/webiq/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/webiq/search/__init__.py b/tests/unit/llms/webiq/search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/webiq/search/test_webiq_search_transformation.py b/tests/unit/llms/webiq/search/test_webiq_search_transformation.py new file mode 100644 index 00000000000..64ef26bacae --- /dev/null +++ b/tests/unit/llms/webiq/search/test_webiq_search_transformation.py @@ -0,0 +1,351 @@ +import json +from collections.abc import Iterator +from typing import Final + +import openai +import pytest +import respx + +import litellm +from litellm.llms.base_llm.search.transformation import SearchResponse +from litellm.llms.webiq.search.transformation import WebIQSearchConfig + + +@pytest.fixture(autouse=True) +def webiq_environment(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.delenv("WEBIQ_API_KEY", raising=False) + monkeypatch.setattr( # test-quality-ok: select HTTPX so respx can intercept the external HTTP boundary + litellm, "disable_aiohttp_transport", True + ) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +def test_search_maps_request_and_preserves_sources(respx_mock: respx.MockRouter) -> None: + endpoint: Final = f"{WebIQSearchConfig.DEFAULT_API_BASE}/search/web" + route: Final = respx_mock.post(endpoint).respond( + json={ + "webResults": [ + { + "title": "A source", + "url": "https://example.com/article", + "content": "Relevant passage", + "lastUpdatedAt": "2026-01-02T00:00:00Z", + "crawledAt": "2026-01-03T00:00:00Z", + "language": "en", + "contentTier": "standard", + "instrumentationSuffix": "source-1", + } + ], + "traceId": "trace-1", + "querySignals": {"freshness": True}, + } + ) + + response: Final = litellm.search( + query="RAG research", + search_provider="webiq", + api_key="test-key", + max_results=3, + country="gb", + search_domain_filter=["example.com", "example.org", "-excluded.example"], + language="en", + ) + + assert json.loads(route.calls.last.request.content) == { + "query": "(RAG research) (site:example.com OR site:example.org) -site:excluded.example", + "maxResults": 3, + "region": "GB", + "contentFormat": "passage", + "maxLength": 5000, + "language": "en", + } + assert route.calls.last.request.headers["x-apikey"] == "test-key" + assert response.model_dump() == { + "object": "search", + "results": [ + { + "title": "A source", + "url": "https://example.com/article", + "snippet": "Relevant passage", + "date": "2026-01-02T00:00:00Z", + "last_updated": "2026-01-02T00:00:00Z", + "crawledAt": "2026-01-03T00:00:00Z", + "language": "en", + "contentTier": "standard", + "instrumentationSuffix": "source-1", + } + ], + "traceId": "trace-1", + "querySignals": {"freshness": True}, + } + + +@pytest.mark.parametrize("api_base", [None, "https://api.microsoft.ai/v3", "https://api.microsoft.ai/v3/search/web"]) +async def test_asearch_uses_environment_credentials_at_default_endpoint( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, api_base: str | None +) -> None: + monkeypatch.setenv("WEBIQ_API_KEY", "environment-key") + route: Final = respx_mock.post(f"{WebIQSearchConfig.DEFAULT_API_BASE}/search/web").respond(json={"webResults": []}) + + response: Final = await litellm.asearch(query="no matches", search_provider="webiq", api_base=api_base) + + assert response.model_dump() == {"object": "search", "results": []} + assert route.calls.last.request.headers["x-apikey"] == "environment-key" + assert json.loads(route.calls.last.request.content) == { + "query": "no matches", + "contentFormat": "passage", + "maxLength": 5000, + } + + +async def test_asearch_uses_explicit_credentials_at_custom_endpoint(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://webiq.example/v3/search/web").respond(json={"webResults": []}) + + response: Final = await litellm.asearch( + query="no matches", search_provider="webiq", api_key="custom-key", api_base="https://webiq.example/v3/" + ) + + assert response.model_dump() == {"object": "search", "results": []} + assert route.calls.last.request.headers["x-apikey"] == "custom-key" + assert json.loads(route.calls.last.request.content) == { + "query": "no matches", + "contentFormat": "passage", + "maxLength": 5000, + } + + +@pytest.mark.parametrize("updated", [None, "", "2026-02-01T00:00:00Z"]) +def test_optional_updated_date_never_uses_crawl_time(respx_mock: respx.MockRouter, updated: str | None) -> None: + respx_mock.post(f"{WebIQSearchConfig.DEFAULT_API_BASE}/search/web").respond( + json={ + "webResults": [ + { + "title": "Title", + "url": "https://example.com", + "content": "Passage", + "lastUpdatedAt": updated, + "crawledAt": "2026-03-01T00:00:00Z", + } + ] + } + ) + response: Final = litellm.search(query="query", search_provider="webiq", api_key="key") + assert (response.results[0].date, response.results[0].last_updated) == (updated or None, updated or None) + + +def test_missing_optional_dates_and_result_order(respx_mock: respx.MockRouter) -> None: + respx_mock.post(f"{WebIQSearchConfig.DEFAULT_API_BASE}/search/web").respond( + json={ + "webResults": [ + {"title": title, "url": f"https://example.com/{title}", "content": title} + for title in ("first", "second") + ] + } + ) + response: Final = litellm.search(query="query", search_provider="webiq", api_key="key") + assert [(r.title, r.snippet, r.date, r.last_updated) for r in response.results] == [ + ("first", "first", None, None), + ("second", "second", None, None), + ] + + +@pytest.mark.parametrize("body", ["not json", "{}", '{"webResults": null}', '{"webResults": [{}]}']) +def test_malformed_response_is_not_reported_as_empty_success(respx_mock: respx.MockRouter, body: str) -> None: + respx_mock.post(f"{WebIQSearchConfig.DEFAULT_API_BASE}/search/web").respond(text=body) + with pytest.raises(openai.APIError, match="invalid search response") as error: + litellm.search(query="query", search_provider="webiq", api_key="key") + assert error.value.status_code == 502 + + +@pytest.mark.parametrize("status", [401, 403, 429, 503]) +def test_upstream_http_errors_keep_status(respx_mock: respx.MockRouter, status: int) -> None: + respx_mock.post(f"{WebIQSearchConfig.DEFAULT_API_BASE}/search/web").respond( + status_code=status, json={"error": {"message": "upstream failure"}}, headers={"Retry-After": "2"} + ) + with pytest.raises(openai.APIError, match="upstream failure") as error: + litellm.search(query="query", search_provider="webiq", api_key="key") + assert error.value.status_code == status + + +def test_native_options_override_mapped_defaults_without_mutating_input() -> None: + params: Final = { + "max_results": 3, + "maxResults": 7, + "country": "us", + "region": "JP", + "maxLength": 2000, + "max_tokens_per_page": 100, + "contentFormat": "markdown", + "safeSearch": "strict", + } + before: Final = params.copy() + assert WebIQSearchConfig().transform_search_request(["first", "second"], params) == { + "query": "first second", + "maxResults": 7, + "region": "JP", + "maxLength": 2000, + "contentFormat": "markdown", + "safeSearch": "strict", + } + assert params == before + + +@pytest.mark.parametrize( + "domains, expected", + [ + ([], "query"), + ([""], "query"), + (["-"], "query"), + (["-example.com"], "(query) -site:example.com"), + (["example.com"], "(query) (site:example.com)"), + ], +) +def test_domain_filter_queries(domains: list[str], expected: str) -> None: + assert WebIQSearchConfig().transform_search_request("query", {"search_domain_filter": domains}) == { + "query": expected, + "contentFormat": "passage", + "maxLength": 5000, + } + + +@pytest.mark.parametrize("suffix", ["", "/", "/search/web", "/search/web/"]) +def test_endpoint_is_appended_once(suffix: str) -> None: + assert WebIQSearchConfig().get_complete_url(f"https://webiq.example/v3{suffix}", {}) == ( + "https://webiq.example/v3/search/web" + ) + + +def test_environment_key_cannot_be_sent_to_caller_selected_host(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("WEBIQ_API_KEY", "server-key") + with pytest.raises(ValueError, match="Refusing to send"): + WebIQSearchConfig().validate_environment({}, api_base="https://untrusted.example/v3") + + +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("api_key", [None, "caller-key"]) +async def test_http_endpoint_rejected_before_sending_credentials( + monkeypatch: pytest.MonkeyPatch, use_async: bool, api_key: str | None +) -> None: + monkeypatch.setenv("WEBIQ_API_KEY", "server-key") + with respx.mock(assert_all_called=False) as transport: + transport.post("http://api.microsoft.ai/v3/search/web").respond(json={"webResults": []}) + if use_async: + with pytest.raises(openai.APIError, match="HTTPS"): + await litellm.asearch( + query="query", search_provider="webiq", api_base="http://api.microsoft.ai/v3", api_key=api_key + ) + else: + with pytest.raises(openai.APIError, match="HTTPS"): + litellm.search( + query="query", search_provider="webiq", api_base="http://api.microsoft.ai/v3", api_key=api_key + ) + assert len(transport.calls) == 0 + + +def test_explicit_key_wins_and_headers_are_not_mutated(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("WEBIQ_API_KEY", "server-key") + config: Final = WebIQSearchConfig() + original: Final = {"X-Request-ID": "request-1"} + result: Final = config.validate_environment(original, api_key="caller-key", api_base="https://custom.example/v3") + assert result == {"X-Request-ID": "request-1", "x-apikey": "caller-key", "Content-Type": "application/json"} + assert original == {"X-Request-ID": "request-1"} + assert config.validate_environment(result, api_key="caller-key") == result + + +def test_missing_key_fails_before_network() -> None: + with pytest.raises(ValueError, match="WEBIQ_API_KEY"): + WebIQSearchConfig().validate_environment({}) + + +@pytest.mark.parametrize( + "result_metadata, response_metadata", + [ + ({"snippet": "upstream", "date": "upstream", "last_updated": "upstream"}, {}), + ({}, {"results": "upstream", "object": "upstream"}), + ], +) +def test_metadata_cannot_replace_standard_search_fields( + respx_mock: respx.MockRouter, result_metadata: dict[str, str], response_metadata: dict[str, str] +) -> None: + respx_mock.post(f"{WebIQSearchConfig.DEFAULT_API_BASE}/search/web").respond( + json={ + "webResults": [ + { + "title": "Title", + "url": "https://example.com", + "content": "Passage", + "lastUpdatedAt": "2026-01-02T00:00:00Z", + "language": "en", + **result_metadata, + } + ], + "traceId": "trace-1", + **response_metadata, + } + ) + + response: Final = litellm.search(query="query", search_provider="webiq", api_key="key") + + assert response.model_dump() == { + "object": "search", + "results": [ + { + "title": "Title", + "url": "https://example.com", + "snippet": "Passage", + "date": "2026-01-02T00:00:00Z", + "last_updated": "2026-01-02T00:00:00Z", + "language": "en", + } + ], + "traceId": "trace-1", + } + + +@pytest.mark.parametrize("tool_parameter", ["model", "search_tool_name"]) +async def test_router_resolves_named_webiq_tool_and_forwards_its_configuration( + respx_mock: respx.MockRouter, tool_parameter: str +) -> None: + route: Final = respx_mock.post("https://webiq.example/v3/search/web").respond( + json={"webResults": [{"title": "Source", "url": "https://example.com", "content": "Grounding"}]} + ) + router: Final = litellm.Router( + model_list=[], + search_tools=[ + { + "search_tool_name": "webiq-search", + "litellm_params": { + "search_provider": "webiq", + "api_key": "tool-key", + "api_base": "https://webiq.example/v3", + "maxLength": 1500, + }, + } + ], + num_retries=0, + ) + + response: Final = await router.asearch(**{tool_parameter: "webiq-search"}, query="ground this", max_results=2) + + assert isinstance(response, SearchResponse) + assert response.model_dump() == { + "object": "search", + "results": [ + { + "title": "Source", + "url": "https://example.com", + "snippet": "Grounding", + "date": None, + "last_updated": None, + } + ], + } + assert route.calls.last.request.headers["x-apikey"] == "tool-key" + assert json.loads(route.calls.last.request.content) == { + "query": "ground this", + "contentFormat": "passage", + "maxLength": 1500, + "maxResults": 2, + } diff --git a/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py b/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py index 65ebece406a..39d8f12be53 100644 --- a/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py +++ b/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py @@ -3,6 +3,7 @@ import json from collections.abc import Callable from datetime import datetime from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -27,6 +28,16 @@ from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stub client = TestClient(app) +def test_dashboard_discovers_webiq_with_its_display_name() -> None: + with _override_auth(UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)): + response: Final = client.get("/search_tools/ui/available_providers") + + assert response.status_code == 200, response.text + assert [provider for provider in response.json()["providers"] if provider["provider_name"] == "webiq"] == [ + {"provider_name": "webiq", "ui_friendly_name": "Microsoft Web IQ"} + ] + + @pytest.mark.asyncio async def test_list_search_tools_db_only(monkeypatch): """Test listing search tools when only DB tools exist""" diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 27ba8968200..66096feb570 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1,5 +1,3 @@ -from litellm.proxy.deps.repositories import get_plugin_upload_repository -from litellm.repositories.plugin_upload_repository import PluginUploadRepository import asyncio import json import re @@ -20,7 +18,9 @@ from pydantic import TypeAdapter import litellm from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.health_check_latest import LatestHealthCheckRow +from litellm.proxy.deps.repositories import get_plugin_upload_repository from litellm.proxy.public_endpoints import router +from litellm.repositories.plugin_upload_repository import PluginUploadRepository from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets from litellm.types.proxy.common_proxy_errors import CommonProxyErrors from litellm.types.proxy.management_endpoints.model_management_endpoints import ( @@ -1285,6 +1285,18 @@ def test_get_supported_endpoints_lists_streamlake_on_chat_messages_and_responses assert "embeddings" not in streamlake_endpoints +def test_get_supported_endpoints_lists_webiq_search(reset_endpoints_cache: None) -> None: + app: Final = FastAPI() + app.include_router(router) + with TestClient(app) as client: + response: Final = client.get("/public/endpoints") + assert response.status_code == 200, response.text + + search: Final = next(endpoint for endpoint in response.json()["endpoints"] if endpoint["key"] == "search") + assert search["endpoint"] == "/search" + assert {"slug": "webiq", "display_name": "Microsoft Web IQ"} in search["providers"] + + def test_get_supported_endpoints_display_names_have_no_slug_suffix( reset_endpoints_cache, ): diff --git a/tests/unit/search/test_main.py b/tests/unit/search/test_main.py index e30fd2eb6f8..0201311d04b 100644 --- a/tests/unit/search/test_main.py +++ b/tests/unit/search/test_main.py @@ -56,6 +56,19 @@ def _proxy_style_logging_obj() -> Logging: ) +@pytest.mark.usefixtures("_local_model_cost_map") +def test_webiq_search_reports_the_catalog_query_cost(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.microsoft.ai/v3/search/web").respond( + json={"webResults": [{"title": "A source", "url": "https://example.com", "content": "A passage"}]} + ) + response: Final = litellm.search(query="litellm release notes", search_provider="webiq", api_key="test-webiq-key") + + assert response._hidden_params["response_cost"] > 0 + assert response._hidden_params["response_cost"] == pytest.approx( + litellm.model_cost["webiq/search"]["input_cost_per_query"] + ) + + @pytest.mark.usefixtures("_local_model_cost_map") def test_search_bills_search_tool_input_cost_per_query(respx_mock: respx.MockRouter) -> None: respx_mock.post(EXA_SEARCH_URL).respond(json={"results": [{"title": "t", "url": "https://example.com"}]}) From bc3a69ce35077d19ace47b24b34c5c691ad56883 Mon Sep 17 00:00:00 2001 From: taxfree <42464993+taxfree-python@users.noreply.github.com> Date: Wed, 11 Nov 2026 07:50:39 +0900 Subject: [PATCH 048/189] fix(otel): serialize pydantic Message objects in langfuse_otel, weave_otel and langtrace (#46845) langfuse_otel, weave_otel and langtrace now encode litellm.Message objects, and any value JSON cannot encode, when they write the prompt, so spans keep their token usage and model parameters. Under LITELLM_OTEL_V2 the payload builder dumps Message objects instead of dropping them --- .../langfuse/langfuse_otel_attributes.py | 3 +- litellm/integrations/langtrace.py | 3 +- litellm/integrations/otel/model/payloads.py | 25 +++++- litellm/integrations/weave/weave_otel.py | 8 +- litellm/litellm_core_utils/safe_json_dumps.py | 13 +++- .../otel/test_otel_v2_vendor_mappers.py | 74 ++++++++++++++++++ tests/unit/integrations/test_langfuse_otel.py | 62 ++++++++++++++- tests/unit/integrations/test_langtrace.py | 76 +++++++++++++++++++ tests/unit/integrations/test_weave_otel.py | 69 ++++++++++++++++- 9 files changed, 323 insertions(+), 10 deletions(-) create mode 100644 tests/unit/integrations/test_langtrace.py diff --git a/litellm/integrations/langfuse/langfuse_otel_attributes.py b/litellm/integrations/langfuse/langfuse_otel_attributes.py index 28530076c57..a4677683d59 100644 --- a/litellm/integrations/langfuse/langfuse_otel_attributes.py +++ b/litellm/integrations/langfuse/langfuse_otel_attributes.py @@ -17,6 +17,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor safe_set_attribute, ) from litellm.litellm_core_utils.core_helpers import require_attributes +from litellm.litellm_core_utils.safe_json_dumps import jsonable_or_str from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse from litellm.types.utils import ( EmbeddingResponse, @@ -103,7 +104,7 @@ class LangfuseLLMObsOTELAttributes(BaseLLMObsOTELAttributes): prompt["tools"] = tools input: Final = prompt - safe_set_attribute(span, "langfuse.observation.input", json.dumps(input)) + safe_set_attribute(span, "langfuse.observation.input", json.dumps(input, default=jsonable_or_str)) @staticmethod @override diff --git a/litellm/integrations/langtrace.py b/litellm/integrations/langtrace.py index ea411baee7e..8556880f7c6 100644 --- a/litellm/integrations/langtrace.py +++ b/litellm/integrations/langtrace.py @@ -1,6 +1,7 @@ import json from typing import TYPE_CHECKING, Any, Final +from litellm.litellm_core_utils.safe_json_dumps import jsonable_or_str from litellm.types.integrations.otel_span_attributes import SpanAttributes if TYPE_CHECKING: @@ -60,7 +61,7 @@ class LangtraceAttributes: if prompts: span.add_event( name="gen_ai.content.prompt", - attributes={SpanAttributes.LLM_PROMPTS.value: json.dumps(prompts)}, + attributes={SpanAttributes.LLM_PROMPTS.value: json.dumps(prompts, default=jsonable_or_str)}, ) self.set_span_attributes(span, span_attributes) diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 4238d72fc5e..aa05691dc28 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast from urllib.parse import urlsplit from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm.integrations.otel.model.metadata import ( @@ -519,7 +519,7 @@ class LLMCallSpanData: promoted_metadata=promoted_metadata, is_streaming=as_bool(payload.get("stream")), tools=_extract_tools(params), - messages_in=_dicts(payload.get("messages")) if capture_content else (), + messages_in=_message_dicts(payload.get("messages")) if capture_content else (), choices_out=choices_out if capture_content else (), system_fingerprint=as_str(response.get("system_fingerprint")), time_to_first_chunk_seconds=time_to_first_chunk_seconds, @@ -783,6 +783,27 @@ def _dicts(value: object) -> tuple[Mapping[str, object], ...]: return tuple(item for item in value if isinstance(item, dict)) +_OBJECTS: Final = TypeAdapter(tuple[object, ...]) +_ANY_KEY_MAPPING: Final = TypeAdapter(Mapping[object, object]) + + +def _message_dict(message: object) -> Mapping[str, object] | None: + if isinstance(message, dict): + return {str(key): value for key, value in _ANY_KEY_MAPPING.validate_python(message).items()} + if not isinstance(message, BaseModel): + return None + try: + return message.model_dump(mode="json", fallback=str) + except ValueError: + return None + + +def _message_dicts(value: object) -> tuple[Mapping[str, object], ...]: + if not isinstance(value, list): + return () + return tuple(message for item in _OBJECTS.validate_python(value) if (message := _message_dict(item)) is not None) + + def _finish_reasons(choices: tuple[Mapping[str, object], ...]) -> tuple[str, ...]: """Non-empty ``finish_reason`` of each response choice.""" return tuple(r for c in choices if (r := as_str(c.get("finish_reason")))) diff --git a/litellm/integrations/weave/weave_otel.py b/litellm/integrations/weave/weave_otel.py index 68a9dc3e162..eb8472dbd16 100644 --- a/litellm/integrations/weave/weave_otel.py +++ b/litellm/integrations/weave/weave_otel.py @@ -21,7 +21,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor safe_set_attribute, ) from litellm.litellm_core_utils.core_helpers import has_attributes, require_attributes -from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.safe_json_dumps import jsonable_or_str, safe_dumps from litellm.types.integrations.weave_otel import WeaveOtelConfig, WeaveSpanAttributes from litellm.types.utils import StandardCallbackDynamicParams @@ -74,7 +74,11 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes): prompt["functions"] = functions if tools is not None: prompt["tools"] = tools - safe_set_attribute(span, OpenInferenceSpanAttributes.INPUT_VALUE, json.dumps(prompt)) + safe_set_attribute( + span, + OpenInferenceSpanAttributes.INPUT_VALUE, + json.dumps(prompt, default=jsonable_or_str), + ) def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object): diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index ea26d549326..b6783327884 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -3,7 +3,8 @@ import math from collections.abc import Callable from typing import Any, Final -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter +from pydantic_core import to_jsonable_python from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -113,3 +114,13 @@ def safe_dumps( return json.dumps( safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str ) + + +_JSONABLE: Final = TypeAdapter(object) + + +def jsonable_or_str(value: object) -> object: + try: + return _JSONABLE.validate_python(to_jsonable_python(value, fallback=str)) + except ValueError: + return str(value) diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index 611e828cf84..9551c22a185 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -15,6 +15,7 @@ import pytest from litellm.integrations.otel import GenAIOperation from litellm.integrations.otel.mappers import ( + GenAIMapper, LangfuseMapper, LangtraceMapper, OpenInferenceMapper, @@ -33,6 +34,7 @@ from litellm.integrations.otel.model.payloads import ( ToolDefinition, ) from litellm.integrations.otel.model.trace_controls import TraceControls +from litellm.types.utils import Message from tests.unit.integrations.otel.test_otel_v2_sources_of_truth import _responses_payload @@ -708,6 +710,78 @@ def test_langfuse_mapper_keeps_chat_output_when_no_embedding_summary(): assert json.loads(attrs["langfuse.observation.output"]) == [{"role": "assistant", "content": "Sunny."}] +LITELLM_USER_MESSAGE: Final = Message(role="user", content="weather in sf?") +NOT_JSON_ENCODABLE: Final = object() + + +@pytest.mark.parametrize( + ("message", "expected_prompt"), + [ + ({"role": "user", "content": "weather in sf?"}, {"role": "user", "content": "weather in sf?"}), + ( + {"role": "user", "content": "weather in sf?", 1: "one"}, + {"role": "user", "content": "weather in sf?", "1": "one"}, + ), + (LITELLM_USER_MESSAGE, LITELLM_USER_MESSAGE.model_dump(mode="json")), + ( + Message(role="user", content="weather in sf?", trace_handle=NOT_JSON_ENCODABLE), + {**LITELLM_USER_MESSAGE.model_dump(mode="json"), "trace_handle": str(NOT_JSON_ENCODABLE)}, + ), + ], + ids=[ + "dict_message", + "dict_message_with_int_key", + "litellm_message_object", + "litellm_message_with_value_json_cannot_encode", + ], +) +def test_vendor_mappers_keep_the_prompt_of_litellm_message_objects( + message: Mapping[str, object] | Message, expected_prompt: Mapping[str, object] +) -> None: + payload: Final = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-5.4-nano", + "messages": [message], + "response": {"choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Sunny."}}]}, + } + data: Final = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True) + + assert { + "genai": json.loads(str(GenAIMapper().map(data)["gen_ai.input.messages"])), + "langfuse": json.loads(str(LangfuseMapper().map(data)["langfuse.observation.input"])), + "langtrace": json.loads(str(LangtraceMapper().map(data)["llm.prompts"])), + "openinference": json.loads(str(OpenInferenceMapper().map(data)["input.value"])), + } == { + "genai": [expected_prompt], + "langfuse": [expected_prompt], + "langtrace": [expected_prompt], + "openinference": [{"role": "user", "content": "weather in sf?"}], + } + + +def test_vendor_mappers_keep_the_span_when_a_litellm_message_cannot_be_dumped() -> None: + payload: Final = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-5.4-nano", + "messages": [ + {"role": "system", "content": "be brief"}, + Message(role="user", content="weather in sf?", audio_bytes=b"\xff\xfe"), + ], + "response": {"choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Sunny."}}]}, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True)) + + assert { + "input": json.loads(str(attrs["langfuse.observation.input"])), + "output": json.loads(str(attrs["langfuse.observation.output"])), + } == { + "input": [{"role": "system", "content": "be brief"}], + "output": [{"role": "assistant", "content": "Sunny."}], + } + + def test_langfuse_mapper_renders_a_responses_api_call_from_the_standard_logging_payload(): payload = { "call_type": "aresponses", diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index aa9a51a93ee..e35adca7fd8 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -20,6 +20,7 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanE from opentelemetry.trace import Span import litellm +from litellm.integrations._types.open_inference import SpanAttributes from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, REDACTED_BY_LITELLM from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger from litellm.integrations.langfuse.langfuse_otel_attributes import LangfuseLLMObsOTELAttributes @@ -28,7 +29,20 @@ from litellm.integrations.otel.model.destination import OtelDestination from litellm.integrations.otel.plumbing.context import set_request_destinations from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.utils import Choices, ModelResponse, StandardCallbackDynamicParams +from litellm.types.utils import Choices, Message, ModelResponse, StandardCallbackDynamicParams + +CHAT_MESSAGE_DICTS: Final = ({"role": "user", "content": "hello"}, {"role": "assistant", "content": "hi back"}) +CHAT_MESSAGE_OBJECTS: Final = tuple( + Message(role=message["role"], content=message["content"]) for message in CHAT_MESSAGE_DICTS +) +NOT_JSON_ENCODABLE: Final = object() +NON_UTF8_BYTES: Final = b"\x80" +GENERATION_USAGE_AND_PARAM_KEYS: Final = ( + SpanAttributes.LLM_TOKEN_COUNT_PROMPT, + SpanAttributes.LLM_TOKEN_COUNT_COMPLETION, + SpanAttributes.LLM_TOKEN_COUNT_TOTAL, + SpanAttributes.LLM_INVOCATION_PARAMETERS, +) class TestLangfuseOtelIntegration: @@ -140,6 +154,52 @@ class TestLangfuseOtelIntegration: "langfuse.observation.type", "generation" ) + @pytest.mark.parametrize( + ("messages", "expected_input"), + [ + (CHAT_MESSAGE_DICTS, list(CHAT_MESSAGE_DICTS)), + (CHAT_MESSAGE_OBJECTS, [message.model_dump() for message in CHAT_MESSAGE_OBJECTS]), + ( + (Message(role="user", content="hello", trace_handle=NOT_JSON_ENCODABLE),), + [{**CHAT_MESSAGE_OBJECTS[0].model_dump(), "trace_handle": str(NOT_JSON_ENCODABLE)}], + ), + ( + (Message(role="user", content="hello", audio_bytes=NON_UTF8_BYTES),), + [{**CHAT_MESSAGE_OBJECTS[0].model_dump(), "audio_bytes": str(NON_UTF8_BYTES)}], + ), + ], + ids=[ + "dict_messages", + "litellm_message_objects", + "litellm_message_with_value_json_cannot_encode", + "litellm_message_with_non_utf8_bytes", + ], + ) + def test_generation_keeps_input_model_params_and_token_usage( + self, messages: Sequence[object], expected_input: Sequence[object] + ) -> None: + span: Final = TracerProvider().get_tracer("langfuse-otel-tests").start_span("litellm_request") + response: Final = ModelResponse( + model="gpt-4o", + choices=[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "done"}}], + usage={"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10}, + ) + + LangfuseOtelLogger.set_langfuse_otel_attributes( + span, + {"messages": messages, "standard_logging_object": {"model_parameters": {"temperature": 0.2}}}, + response, + ) + + attributes: Final = span.attributes or {} + assert json.loads(str(attributes["langfuse.observation.input"])) == expected_input + assert {key: attributes.get(key) for key in GENERATION_USAGE_AND_PARAM_KEYS} == { + SpanAttributes.LLM_TOKEN_COUNT_PROMPT: 7, + SpanAttributes.LLM_TOKEN_COUNT_COMPLETION: 3, + SpanAttributes.LLM_TOKEN_COUNT_TOTAL: 10, + SpanAttributes.LLM_INVOCATION_PARAMETERS: json.dumps({"temperature": 0.2}), + } + def test_set_langfuse_environment_attribute(self): """Test that Langfuse environment is set correctly when environment variable is present.""" mock_span = MagicMock() diff --git a/tests/unit/integrations/test_langtrace.py b/tests/unit/integrations/test_langtrace.py new file mode 100644 index 00000000000..a5615b58eb4 --- /dev/null +++ b/tests/unit/integrations/test_langtrace.py @@ -0,0 +1,76 @@ +import json +from collections.abc import Sequence +from typing import Final + +import pytest +from opentelemetry.sdk.trace import TracerProvider + +from litellm.integrations.langtrace import LangtraceAttributes +from litellm.types.integrations.otel_span_attributes import SpanAttributes +from litellm.types.utils import Message, ModelResponse + +CHAT_MESSAGE_DICTS: Final = ({"role": "user", "content": "weather in SF?"}, {"role": "assistant", "content": "Sunny."}) +CHAT_MESSAGE_OBJECTS: Final = tuple( + Message(role=message["role"], content=message["content"]) for message in CHAT_MESSAGE_DICTS +) +NOT_JSON_ENCODABLE: Final = object() +NON_UTF8_BYTES: Final = b"\x80" + + +@pytest.mark.parametrize( + ("messages", "expected_prompts"), + [ + (CHAT_MESSAGE_DICTS, list(CHAT_MESSAGE_DICTS)), + (CHAT_MESSAGE_OBJECTS, [message.model_dump() for message in CHAT_MESSAGE_OBJECTS]), + ( + ({"role": "user", "content": "before\x00after", "metadata": {1: "one"}},), + [{"role": "user", "content": "before\x00after", "metadata": {"1": "one"}}], + ), + ( + (Message(role="user", content="weather in SF?", trace_handle=NOT_JSON_ENCODABLE),), + [{**CHAT_MESSAGE_OBJECTS[0].model_dump(), "trace_handle": str(NOT_JSON_ENCODABLE)}], + ), + ( + (Message(role="user", content="weather in SF?", audio_bytes=NON_UTF8_BYTES),), + [str(Message(role="user", content="weather in SF?", audio_bytes=NON_UTF8_BYTES))], + ), + ], + ids=[ + "dict_messages", + "litellm_message_objects", + "dict_message_with_nul_and_int_key", + "litellm_message_with_value_json_cannot_encode", + "litellm_message_with_non_utf8_bytes", + ], +) +def test_langtrace_span_keeps_prompt_and_token_usage( + messages: Sequence[object], expected_prompts: Sequence[object] +) -> None: + span: Final = TracerProvider().get_tracer("langtrace-tests").start_span("litellm_request") + response: Final = ModelResponse( + model="gpt-4o", + choices=[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Still sunny."}}], + usage={"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10}, + ) + + LangtraceAttributes().set_langtrace_attributes( + span, {"messages": messages, "litellm_params": {"custom_llm_provider": "openai"}}, response + ) + + assert [ + json.loads(str((event.attributes or {})[SpanAttributes.LLM_PROMPTS.value])) + for event in span.events + if event.name == "gen_ai.content.prompt" + ] == [expected_prompts] + assert { + key: (span.attributes or {}).get(key) + for key in ( + SpanAttributes.LLM_USAGE_PROMPT_TOKENS.value, + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS.value, + SpanAttributes.LLM_USAGE_TOTAL_TOKENS.value, + ) + } == { + SpanAttributes.LLM_USAGE_PROMPT_TOKENS.value: 7, + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS.value: 3, + SpanAttributes.LLM_USAGE_TOTAL_TOKENS.value: 10, + } diff --git a/tests/unit/integrations/test_weave_otel.py b/tests/unit/integrations/test_weave_otel.py index a3a022d514b..e279abbd1d9 100644 --- a/tests/unit/integrations/test_weave_otel.py +++ b/tests/unit/integrations/test_weave_otel.py @@ -1,6 +1,6 @@ import json import os -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Final from unittest.mock import MagicMock, patch @@ -12,9 +12,10 @@ from litellm.integrations.weave.weave_otel import ( WeaveLLMObsOTELAttributes, _set_weave_specific_attributes, get_weave_otel_config, + set_weave_otel_attributes, ) from litellm.types.integrations.weave_otel import WeaveOtelConfig, WeaveSpanAttributes -from litellm.types.utils import ModelResponse +from litellm.types.utils import Message, ModelResponse def test_get_weave_otel_config(): @@ -205,3 +206,67 @@ def test_weave_output_value_is_the_dumped_model_response() -> None: output: Final = json.loads(str(span.attributes[OpenInferenceSpanAttributes.OUTPUT_VALUE])) assert output["id"] == "chatcmpl-1" assert output["choices"][0]["message"]["content"] == "Sunny." + + +NOT_JSON_ENCODABLE: Final = object() +NON_UTF8_BYTES: Final = b"\x80" + + +@pytest.mark.parametrize( + ("messages", "expected_messages"), + [ + ( + (Message(role="user", content="weather in SF?"), Message(role="assistant", content="Sunny.")), + [ + Message(role="user", content="weather in SF?").model_dump(), + Message(role="assistant", content="Sunny.").model_dump(), + ], + ), + ( + (Message(role="user", content="weather in SF?", trace_handle=NOT_JSON_ENCODABLE),), + [{**Message(role="user", content="weather in SF?").model_dump(), "trace_handle": str(NOT_JSON_ENCODABLE)}], + ), + ( + (Message(role="user", content="weather in SF?", audio_bytes=NON_UTF8_BYTES),), + [str(Message(role="user", content="weather in SF?", audio_bytes=NON_UTF8_BYTES))], + ), + ], + ids=["litellm_messages", "litellm_message_with_value_json_cannot_encode", "litellm_message_with_non_utf8_bytes"], +) +def test_weave_span_keeps_input_and_token_usage_when_messages_are_litellm_messages( + messages: tuple[Message, ...], expected_messages: Sequence[object] +) -> None: + span: Final = _TRACER.start_span("litellm_request") + response: Final = ModelResponse( + model="gpt-4o", + choices=[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Still sunny."}}], + usage={"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10}, + ) + + set_weave_otel_attributes(span, {"messages": messages, "standard_logging_object": {}}, response) + + attributes: Final = span.attributes or {} + assert json.loads(str(attributes[OpenInferenceSpanAttributes.INPUT_VALUE])) == {"messages": expected_messages} + assert { + key: attributes.get(key) + for key in ( + OpenInferenceSpanAttributes.LLM_TOKEN_COUNT_PROMPT, + OpenInferenceSpanAttributes.LLM_TOKEN_COUNT_COMPLETION, + OpenInferenceSpanAttributes.LLM_TOKEN_COUNT_TOTAL, + ) + } == { + OpenInferenceSpanAttributes.LLM_TOKEN_COUNT_PROMPT: 7, + OpenInferenceSpanAttributes.LLM_TOKEN_COUNT_COMPLETION: 3, + OpenInferenceSpanAttributes.LLM_TOKEN_COUNT_TOTAL: 10, + } + + +def test_weave_span_input_keeps_dict_message_content_exactly_as_sent() -> None: + span: Final = _TRACER.start_span("litellm_request") + messages: Final = ({"role": "user", "content": "before\x00after", "metadata": {1: "one"}},) + + set_weave_otel_attributes(span, {"messages": messages, "standard_logging_object": {}}, ModelResponse(model="gpt-4o")) + + assert json.loads(str((span.attributes or {})[OpenInferenceSpanAttributes.INPUT_VALUE])) == { + "messages": [{"role": "user", "content": "before\x00after", "metadata": {"1": "one"}}] + } From 05c04d6261df9369d880bcda8d9c21b4e5ee113b Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 15:04:08 -0800 Subject: [PATCH 049/189] perf(spend): narrow user and team spend log reads to the owner's api_keys so the (api_key, startTime) index is used (#51026) * perf(spend): narrow user and team spend log reads to the owner's api_keys so the (api_key, startTime) index is used Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend): satisfy the type discipline gate for the api_key scope helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend): run the api_key lookup under the statement timeout, skip it for search and empty key lists Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/team_endpoints.py | 20 +- .../spend_tracking/spend_log_api_key_scope.py | 83 +++++++ .../spend_management_endpoints.py | 32 +++ .../test_team_endpoints.py | 62 ++++- .../test_spend_management_endpoints.py | 214 +++++++++++++++++- 5 files changed, 396 insertions(+), 15 deletions(-) create mode 100644 litellm/proxy/spend_tracking/spend_log_api_key_scope.py diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index f14f3d93b79..0704d2c2444 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -18,7 +18,7 @@ import traceback from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass -from datetime import datetime, timedelta, timezone +from datetime import date, datetime, timedelta, timezone from itertools import chain from types import MappingProxyType from typing import ( @@ -228,6 +228,7 @@ from litellm.proxy.management_helpers.utils import ( management_endpoint_wrapper, ) from litellm.proxy.spend_tracking.committed_spend import drop_committed_spend +from litellm.proxy.spend_tracking.spend_log_api_key_scope import spend_log_api_keys_for_teams from litellm.proxy.utils import ( PrismaClient, ProxyLogging, @@ -7463,9 +7464,11 @@ def aggregated_date_range_error(start_date: str | None, end_date: str | None) -> return None -def _team_user_spend_sql(*, team_count: int, restrict_to_user: bool) -> str: +def _team_user_spend_sql(*, team_count: int, restrict_to_user: bool, narrow_to_api_keys: bool) -> str: team_placeholders: Final = ", ".join(f"${i}" for i in range(3, 3 + team_count)) user_clause: Final = f' AND sl."user" = ${3 + team_count}' if restrict_to_user else "" + api_key_placeholder: Final = 3 + team_count + (1 if restrict_to_user else 0) + api_key_clause: Final = f" AND sl.api_key = ANY(${api_key_placeholder}::text[])" if narrow_to_api_keys else "" return f""" SELECT sl.team_id, @@ -7483,7 +7486,7 @@ def _team_user_spend_sql(*, team_count: int, restrict_to_user: bool) -> str: LEFT JOIN "LiteLLM_UserTable" u ON u.user_id = sl."user" WHERE sl."startTime" >= $1::timestamp AND sl."startTime" < $2::timestamp + INTERVAL '1 day' - AND sl.team_id IN ({team_placeholders}){user_clause} + AND sl.team_id IN ({team_placeholders}){user_clause}{api_key_clause} GROUP BY sl.team_id, sl."user", u.user_email, u.user_alias ORDER BY spend DESC, sl.team_id, sl."user" """ @@ -7555,15 +7558,24 @@ async def get_team_spend_by_user( own_user_only: Final = scope.api_key_filter is not None user_param: Final = (user_api_key_dict.user_id or "",) if own_user_only else () + scoped_api_keys: Final = await spend_log_api_keys_for_teams( + prisma_client, scoped_team_ids, date.fromisoformat(start_date), date.fromisoformat(end_date) + ) + api_key_param: Final = (list(scoped_api_keys),) if scoped_api_keys is not None else () async with db_span("team_user_spend", "LiteLLM_SpendLogs"): rows: Final = _TEAM_USER_SPEND_ROWS.validate_python( await query_raw_within_statement_timeout( prisma_client, - _team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only), + _team_user_spend_sql( + team_count=len(scoped_team_ids), + restrict_to_user=own_user_only, + narrow_to_api_keys=scoped_api_keys is not None, + ), start_date, end_date, *scoped_team_ids, *user_param, + *api_key_param, what="Spend per user", ) ) diff --git a/litellm/proxy/spend_tracking/spend_log_api_key_scope.py b/litellm/proxy/spend_tracking/spend_log_api_key_scope.py new file mode 100644 index 00000000000..6507011a977 --- /dev/null +++ b/litellm/proxy/spend_tracking/spend_log_api_key_scope.py @@ -0,0 +1,83 @@ +"""``LiteLLM_SpendLogs`` has no index on ``"user"`` or ``team_id``, so a read filtered by owner scans the +whole date window. The owner's ``LiteLLM_Daily*Spend`` rows (kept after key deletion) plus their live keys +name every api_key that produced those logs, and ``(api_key, "startTime")`` is indexed.""" + +from collections.abc import Mapping, Sequence +from datetime import date, timedelta +from typing import Final + +from pydantic import ConfigDict, TypeAdapter + +from litellm.proxy.utils import PrismaClient, query_raw_within_statement_timeout +from litellm.types.llms.base import LiteLLMBaseModel + +SPEND_LOG_API_KEY_SCOPE_MAX_KEYS: Final = 5000 +_WHAT: Final = "Spend log key lookup" + +_USER_API_KEYS_SQL: Final = """ +SELECT DISTINCT api_key FROM ( + SELECT api_key FROM "LiteLLM_DailyUserSpend" + WHERE user_id = $1 AND date >= $2 AND date <= $3 + UNION ALL + SELECT token AS api_key FROM "LiteLLM_VerificationToken" WHERE user_id = $1 +) owned +LIMIT $4 +""" + +_TEAM_API_KEYS_SQL: Final = """ +SELECT DISTINCT api_key FROM ( + SELECT api_key FROM "LiteLLM_DailyTeamSpend" + WHERE team_id = ANY($1::text[]) AND date >= $2 AND date <= $3 + UNION ALL + SELECT token AS api_key FROM "LiteLLM_VerificationToken" WHERE team_id = ANY($1::text[]) +) owned +LIMIT $4 +""" + + +class _ApiKeyRow(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + api_key: str + + +_API_KEY_ROWS: Final = TypeAdapter(tuple[_ApiKeyRow, ...]) + + +def _narrowing_keys(rows: Sequence[Mapping[str, object]]) -> tuple[str, ...] | None: + """None when the list cannot narrow the read: nothing known about the owner, or too many keys to help.""" + keys: Final = tuple(row.api_key for row in _API_KEY_ROWS.validate_python(rows)) + return None if not keys or len(keys) > SPEND_LOG_API_KEY_SCOPE_MAX_KEYS else keys + + +def _daily_dates_around(start: date, end: date) -> tuple[str, str]: + return (start - timedelta(days=1)).isoformat(), (end + timedelta(days=1)).isoformat() + + +async def spend_log_api_keys_for_user( + prisma_client: PrismaClient, user_id: str, start: date, end: date +) -> tuple[str, ...] | None: + """The api_keys that can carry this user's spend logs in the window, or None when there are too many to help.""" + first, last = _daily_dates_around(start, end) + return _narrowing_keys( + await query_raw_within_statement_timeout( + prisma_client, _USER_API_KEYS_SQL, user_id, first, last, SPEND_LOG_API_KEY_SCOPE_MAX_KEYS + 1, what=_WHAT + ) + ) + + +async def spend_log_api_keys_for_teams( + prisma_client: PrismaClient, team_ids: Sequence[str], start: date, end: date +) -> tuple[str, ...] | None: + first, last = _daily_dates_around(start, end) + return _narrowing_keys( + await query_raw_within_statement_timeout( + prisma_client, + _TEAM_API_KEYS_SQL, + list(team_ids), + first, + last, + SPEND_LOG_API_KEY_SCOPE_MAX_KEYS + 1, + what=_WHAT, + ) + ) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 7d9a0dcc971..6d87b39afb3 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -70,6 +70,10 @@ from litellm.proxy.spend_tracking.spend_capture_rate import ( # NOTE: Avoid module-level import from common_utils: proxy_server imports this # module while common_utils may pull proxy_server during init, which can leave # those names undefined. Import the helpers locally where they are used. +from litellm.proxy.spend_tracking.spend_log_api_key_scope import ( + spend_log_api_keys_for_teams, + spend_log_api_keys_for_user, +) from litellm.proxy.spend_tracking.spend_tracking_utils import ( get_spend_by_team, get_spend_by_team_and_customer, @@ -257,6 +261,22 @@ async def _query_raw(prisma_client: PrismaClient, sql_query: str, *args: object) return await prisma_client.db.query_raw(sql_query, *args) +async def _scope_api_keys( + prisma_client: PrismaClient, *, where_conditions: Mapping[str, object], start: datetime | None, end: datetime | None +) -> tuple[str, ...] | None: + """The api_keys behind a ``"user"`` / ``team_id`` filter, so the unindexed column is only checked on rows the + ``(api_key, "startTime")`` index already picked.""" + if start is None or end is None or isinstance(where_conditions.get("api_key"), str): + return None + user_id: Final = where_conditions.get("user") + if isinstance(user_id, str): + return await spend_log_api_keys_for_user(prisma_client, user_id, start.date(), end.date()) + team_id: Final = where_conditions.get("team_id") + if isinstance(team_id, str): + return await spend_log_api_keys_for_teams(prisma_client, (team_id,), start.date(), end.date()) + return None + + async def _query_raw_or_none(prisma_client: PrismaClient, sql_query: str, *args: object) -> Sequence[_RowT] | None: """``_query_raw`` for the call sites that guard the result against ``None``.""" return await _query_raw(prisma_client, sql_query, *args) @@ -2950,6 +2970,18 @@ async def ui_view_spend_logs( sql_params.append(f"%{like_escaped_session_id}%") p += 1 + scope_api_keys: Final = ( + await _scope_api_keys( + prisma_client, where_conditions=where_conditions, start=start_date_obj, end=end_date_obj + ) + if not search_owns_window + else None + ) + if scope_api_keys is not None: + sql_conditions.append(f"api_key = ANY(${p}::text[])") + sql_params.append(list(scope_api_keys)) + p += 1 # rebind-ok: advances the file's shared $N placeholder counter + # Status filter if status_filter is not None and not (group_by_session is True and not is_search_lookup): if status_filter == "success": diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index a79ebd307fe..ac46ccc1cfc 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -17341,6 +17341,19 @@ def _team_spend_by_user_caller(user_id: str, teams: list[str]) -> LiteLLM_UserTa ) +def _team_spend_reads(api_keys: list[str], spend_rows: list[dict] | Exception) -> AsyncMock: + """Raw reads inside the statement-timeout transaction: the key lookup, then the spend log aggregate.""" + + async def read(sql: str, *params: object) -> list[dict]: + if 'FROM "LiteLLM_DailyTeamSpend"' in sql: + return [{"api_key": api_key} for api_key in api_keys] + if isinstance(spend_rows, Exception): + raise spend_rows + return spend_rows + + return AsyncMock(side_effect=read) + + def _team_spend_by_user_db_row(team_id: str, user_id: str, spend: float, requests: int) -> dict: return { "team_id": team_id, @@ -17367,12 +17380,13 @@ async def test_get_team_spend_by_user_admin_groups_spend_logs_by_team_and_user(m mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha, beta]) transaction = statement_timeout_reads( mock_db_client, - AsyncMock( - return_value=[ + _team_spend_reads( + ["sk-alpha", "sk-beta"], + [ _team_spend_by_user_db_row("team-alpha", "alice", 0.5, 3), _team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2), _team_spend_by_user_db_row("team-beta", "alice", 0.1, 1), - ] + ], ), ) @@ -17386,11 +17400,15 @@ async def test_get_team_spend_by_user_admin_groups_spend_logs_by_team_and_user(m assert any("statement_timeout = 30000" in statement for statement in transaction.statements) mock_db_client.db.tx.assert_not_called() sql, *params = transaction.query_raw.call_args.args - assert params == ["2026-09-01", "2026-09-04", "team-alpha", "team-beta"] + assert params == ["2026-09-01", "2026-09-04", "team-alpha", "team-beta", ["sk-alpha", "sk-beta"]] assert 'FROM "LiteLLM_SpendLogs" sl' in sql assert 'sl."startTime" >= $1::timestamp' in sql assert "sl.\"startTime\" < $2::timestamp + INTERVAL '1 day'" in sql assert "sl.team_id IN ($3, $4)" in sql + assert "sl.api_key = ANY($5::text[])" in sql + lookup_sql, *lookup_params = transaction.query_raw.call_args_list[0].args + assert 'FROM "LiteLLM_DailyTeamSpend"' in lookup_sql and 'FROM "LiteLLM_VerificationToken"' in lookup_sql + assert lookup_params == [["team-alpha", "team-beta"], "2026-08-31", "2026-09-05", 5001] assert 'GROUP BY sl.team_id, sl."user"' in sql assert 'sl."user" = $' not in sql @@ -17412,7 +17430,7 @@ async def test_get_team_spend_by_user_team_admin_sees_every_member(mock_db_clien caller = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER) alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), []) mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha]) - transaction = statement_timeout_reads(mock_db_client, AsyncMock(return_value=[])) + transaction = statement_timeout_reads(mock_db_client, _team_spend_reads(["sk-alpha"], [])) mock_db_client.writer_db.litellm_usertable.find_unique = AsyncMock( return_value=_team_spend_by_user_caller("alice", ["team-alpha"]) ) @@ -17422,8 +17440,9 @@ async def test_get_team_spend_by_user_team_admin_sees_every_member(mock_db_clien ) sql, *params = transaction.query_raw.call_args.args - assert params == ["2026-09-01", "2026-09-04", "team-alpha"] + assert params == ["2026-09-01", "2026-09-04", "team-alpha", ["sk-alpha"]] assert 'sl."user" = $' not in sql + assert "sl.api_key = ANY($4::text[])" in sql @pytest.mark.asyncio @@ -17435,7 +17454,7 @@ async def test_get_team_spend_by_user_plain_member_only_sees_own_row(mock_db_cli mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha]) mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) transaction = statement_timeout_reads( - mock_db_client, AsyncMock(return_value=[_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2)]) + mock_db_client, _team_spend_reads(["sk-bob"], [_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2)]) ) mock_db_client.writer_db.litellm_usertable.find_unique = AsyncMock( return_value=_team_spend_by_user_caller("bob", ["team-alpha"]) @@ -17446,9 +17465,10 @@ async def test_get_team_spend_by_user_plain_member_only_sees_own_row(mock_db_cli ) sql, *params = transaction.query_raw.call_args.args - assert params == ["2026-09-01", "2026-09-04", "team-alpha", "bob"] + assert params == ["2026-09-01", "2026-09-04", "team-alpha", "bob", ["sk-bob"]] assert "sl.team_id IN ($3)" in sql assert 'AND sl."user" = $4' in sql + assert "sl.api_key = ANY($5::text[])" in sql assert [(r.user_id, r.spend) for r in response.results] == [("bob", 0.25)] @@ -17506,7 +17526,7 @@ async def test_get_team_spend_by_user_statement_timeout_returns_503(mock_db_clie admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), []) mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha]) - statement_timeout_reads(mock_db_client, AsyncMock(side_effect=statement_timeout_error())) + statement_timeout_reads(mock_db_client, _team_spend_reads(["sk-alpha"], statement_timeout_error())) with pytest.raises(HTTPException) as exc_info: await get_team_spend_by_user( @@ -17520,6 +17540,30 @@ async def test_get_team_spend_by_user_statement_timeout_returns_503(mock_db_clie assert "longer than 30s" in str(exc_info.value.detail) +@pytest.mark.asyncio +@pytest.mark.parametrize("known_key_count", [0, 5001]) +async def test_get_team_spend_by_user_without_a_usable_key_list_keeps_the_plain_team_filter( + mock_db_client, known_key_count: int +): + from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user + + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), []) + mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha]) + transaction = statement_timeout_reads( + mock_db_client, _team_spend_reads([f"sk-{i}" for i in range(known_key_count)], []) + ) + + await get_team_spend_by_user( + user_api_key_dict=admin, team_ids="team-alpha", start_date="2026-09-01", end_date="2026-09-04" + ) + + sql, *params = transaction.query_raw.call_args.args + assert params == ["2026-09-01", "2026-09-04", "team-alpha"] + assert "sl.team_id IN ($3)" in sql + assert "sl.api_key = ANY(" not in sql + + class _TeamRowWithOrganization(LiteLLM_TeamTable): litellm_organization_table: LiteLLM_OrganizationTable | None = None diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 4a418d87f22..29e2f97d11c 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -6,7 +6,7 @@ import hashlib import json import re import sqlite3 -from collections.abc import AsyncIterator, Awaitable, Mapping, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from datetime import timezone from functools import partial from types import SimpleNamespace @@ -141,6 +141,7 @@ def _reconstruct_ui_where_from_sql(sql_query, params): sess = re.fullmatch(r"session_id LIKE \$(\d+)", cond) status = re.fullmatch(r"status = \$(\d+)", cond) api_key_not_in = re.fullmatch(r"api_key NOT IN \(\$(\d+), \$(\d+)\)", cond) + api_key_any = re.fullmatch(r"api_key = ANY\(\$(\d+)::text\[\]\)", cond) req_or_call = re.fullmatch(r"\(request_id = \$(\d+) OR litellm_call_id = \$\1\)", cond) if req_or_call: where["request_id_or_call_id"] = params[int(req_or_call.group(1)) - 1] @@ -169,6 +170,8 @@ def _reconstruct_ui_where_from_sql(sql_query, params): where["session_id"] = {"contains": str(params[int(sess.group(1)) - 1]).strip("%")} elif status: where["status"] = {"equals": params[int(status.group(1)) - 1]} + elif api_key_any: + where["api_key_in"] = list(params[int(api_key_any.group(1)) - 1]) elif api_key_not_in: where["api_key_not_in"] = [ params[int(api_key_not_in.group(1)) - 1], @@ -218,7 +221,18 @@ def _reconstruct_ui_where_from_sql(sql_query, params): return where -def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=None, query_observer=None): +def _emulate_owner_api_key_lookup(mock_spend_logs: list[dict], sql_query: str, params: tuple) -> list[dict]: + """The daily spend tables and key table name every api_key an owner's requests ran under, so the + lookup answers with the distinct api_keys of the owner's logs.""" + owner_col = "user" if 'FROM "LiteLLM_DailyUserSpend"' in sql_query else "team_id" + owners = {params[0]} if isinstance(params[0], str) else set(params[0]) + api_keys = {log["api_key"] for log in mock_spend_logs if log.get(owner_col) in owners} + return [{"api_key": api_key} for api_key in sorted(api_keys)] + + +def make_ui_spend_logs_mock_prisma( + mock_spend_logs, filter_fn, team_lookup_fn=None, query_observer=None, owner_api_keys_fn=None +): """ Create a MockPrismaClient for /spend/logs/ui endpoint tests. @@ -228,6 +242,8 @@ def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=No where_conditions, returns the filtered list of logs. team_lookup_fn: Optional async callable for team RBAC (find_unique). If provided, adds litellm_teamtable to db. + owner_api_keys_fn: Optional Callable[[tuple], list[dict]] answering the + user/team -> api_key lookup instead of deriving it from the logs. """ class MockDB: @@ -251,6 +267,10 @@ def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=No return [] if 'SELECT DISTINCT "user", team_id' in sql_query: return _emulate_spend_log_owner_lookup(mock_spend_logs, sql_query, params) + if 'FROM "LiteLLM_DailyUserSpend"' in sql_query or 'FROM "LiteLLM_DailyTeamSpend"' in sql_query: + if owner_api_keys_fn is not None: + return owner_api_keys_fn(params) + return _emulate_owner_api_key_lookup(mock_spend_logs, sql_query, params) filtered = filter_fn(_reconstruct_ui_where_from_sql(sql_query, params)) total = len(filtered) if "COUNT(*)" in sql_query: @@ -845,6 +865,196 @@ async def test_ui_view_spend_logs_with_user_id(client, monkeypatch): assert data["data"][0]["user"] == "test_user_1" +def _owned_log(request_id: str, api_key: str, user: str, team_id: str) -> dict: + return { + "id": request_id, + "request_id": request_id, + "api_key": api_key, + "user": user, + "team_id": team_id, + "spend": 0.01, + "startTime": "2026-09-02T10:00:00+00:00", + "model": "gpt-4o", + } + + +def _owner_filtered_logs(logs: list[dict]) -> Callable[[dict], list[dict]]: + """What Postgres returns for the page query: the owner filter and, when present, the api_key list both apply.""" + + def filter_fn(where: dict) -> list[dict]: + return [ + log + for log in logs + if log["user"] == where.get("user", log["user"]) + and log["team_id"] == where.get("team_id", log["team_id"]) + and log["api_key"] in where.get("api_key_in", [log["api_key"]]) + ] + + return filter_fn + + +def _owner_api_keys(daily_rows: dict[str, set[str]], live_keys: dict[str, set[str]]) -> Callable[[tuple], list[dict]]: + """The key lookup's two sources, keyed by owner: api_keys on daily spend rows and api_keys of live keys.""" + + def lookup(params: tuple) -> list[dict]: + owners = {params[0]} if isinstance(params[0], str) else set(params[0]) + api_keys = set().union(*(daily_rows.get(o, set()) | live_keys.get(o, set()) for o in owners)) + return [{"api_key": api_key} for api_key in sorted(api_keys)] + + return lookup + + +def _spend_logs_page_query(queries: list[tuple[str, tuple]]) -> tuple[str, tuple, list[str] | None]: + sql, params = next((sql, params) for sql, params in queries if '"LiteLLM_SpendLogs"' in sql and "ORDER BY" in sql) + key_param = re.search(r"api_key = ANY\(\$(\d+)::text\[\]\)", sql) + return sql, params, None if key_param is None else params[int(key_param.group(1)) - 1] + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_user_filter_reads_spend_logs_through_the_users_api_keys(client, monkeypatch): + from litellm.proxy.spend_tracking.spend_log_api_key_scope import SPEND_LOG_API_KEY_SCOPE_MAX_KEYS + + logs = [ + _owned_log("req1", "sk-live", "alice", "team1"), + _owned_log("req2", "sk-deleted", "alice", "team1"), + _owned_log("req3", "sk-bob", "bob", "team1"), + ] + queries: list[tuple[str, tuple]] = [] + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + make_ui_spend_logs_mock_prisma( + logs, + _owner_filtered_logs(logs), + query_observer=lambda sql, params: queries.append((sql, params)), + owner_api_keys_fn=_owner_api_keys( + daily_rows={"alice": {"sk-deleted"}, "bob": {"sk-bob"}}, live_keys={"alice": {"sk-live"}} + ), + ), + ) + + response = client.get( + "/spend/logs/ui", + params={"user_id": "alice", "start_date": "2026-09-01 00:00:00", "end_date": "2026-09-03 23:59:59"}, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200, response.text + assert {row["request_id"] for row in response.json()["data"]} == {"req1", "req2"} + lookup_sql, lookup_params = next((sql, params) for sql, params in queries if '"LiteLLM_DailyUserSpend"' in sql) + assert '"LiteLLM_VerificationToken"' in lookup_sql + assert lookup_params == ("alice", "2026-08-31", "2026-09-04", SPEND_LOG_API_KEY_SCOPE_MAX_KEYS + 1) + page_sql, _, scoped_api_keys = _spend_logs_page_query(queries) + assert scoped_api_keys == ["sk-deleted", "sk-live"] + assert '"user" = $' in page_sql + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_team_filter_reads_spend_logs_through_the_teams_api_keys(client, monkeypatch): + logs = [ + _owned_log("req1", "sk-alpha-1", "alice", "team-alpha"), + _owned_log("req2", "sk-alpha-2", "bob", "team-alpha"), + _owned_log("req3", "sk-beta", "carol", "team-beta"), + ] + queries: list[tuple[str, tuple]] = [] + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + make_ui_spend_logs_mock_prisma( + logs, + _owner_filtered_logs(logs), + query_observer=lambda sql, params: queries.append((sql, params)), + owner_api_keys_fn=_owner_api_keys( + daily_rows={"team-alpha": {"sk-alpha-1"}, "team-beta": {"sk-beta"}}, + live_keys={"team-alpha": {"sk-alpha-2"}}, + ), + ), + ) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", + lambda user_api_key_dict: True, + ) + monkeypatch.setitem( + app.dependency_overrides, + ps.user_api_key_auth, + lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user"), + ) + + response = client.get( + "/spend/logs/ui", + params={"team_id": "team-alpha", "start_date": "2026-09-01 00:00:00", "end_date": "2026-09-03 23:59:59"}, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200, response.text + assert {row["request_id"] for row in response.json()["data"]} == {"req1", "req2"} + _, lookup_params = next((sql, params) for sql, params in queries if '"LiteLLM_DailyTeamSpend"' in sql) + assert lookup_params[:3] == (["team-alpha"], "2026-08-31", "2026-09-04") + page_sql, _, scoped_api_keys = _spend_logs_page_query(queries) + assert scoped_api_keys == ["sk-alpha-1", "sk-alpha-2"] + assert "team_id = $" in page_sql + + +@pytest.mark.asyncio +@pytest.mark.parametrize("known_keys", [set(), {f"sk-{i}" for i in range(5001)}]) +async def test_ui_view_spend_logs_without_a_usable_key_list_keeps_the_plain_user_filter( + client, monkeypatch, known_keys: set[str] +): + logs = [_owned_log("req1", "sk-live", "alice", "team1")] + queries: list[tuple[str, tuple]] = [] + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + make_ui_spend_logs_mock_prisma( + logs, + _owner_filtered_logs(logs), + query_observer=lambda sql, params: queries.append((sql, params)), + owner_api_keys_fn=_owner_api_keys(daily_rows={"alice": known_keys}, live_keys={}), + ), + ) + + response = client.get( + "/spend/logs/ui", + params={"user_id": "alice", "start_date": "2026-09-01 00:00:00", "end_date": "2026-09-03 23:59:59"}, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200, response.text + assert [row["request_id"] for row in response.json()["data"]] == ["req1"] + page_sql, _, scoped_api_keys = _spend_logs_page_query(queries) + assert scoped_api_keys is None + assert '"user" = $' in page_sql + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_search_with_user_filter_is_not_narrowed_to_the_windows_keys(client, monkeypatch): + logs = [_owned_log("req-old", "sk-retired", "alice", "team1")] + queries: list[tuple[str, tuple]] = [] + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + make_ui_spend_logs_mock_prisma( + logs, + _owner_filtered_logs(logs), + query_observer=lambda sql, params: queries.append((sql, params)), + owner_api_keys_fn=_owner_api_keys(daily_rows={}, live_keys={"alice": {"sk-live"}}), + ), + ) + + response = client.get( + "/spend/logs/ui", + params={ + "user_id": "alice", + "search": "req-old", + "start_date": "2026-09-01 00:00:00", + "end_date": "2026-09-03 23:59:59", + }, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200, response.text + assert [row["request_id"] for row in response.json()["data"]] == ["req-old"] + page_sql, _, scoped_api_keys = _spend_logs_page_query(queries) + assert scoped_api_keys is None + assert "request_id = $" in page_sql and '"user" = $' in page_sql + + @pytest.mark.asyncio @pytest.mark.parametrize( "session_id_query,expected_request_ids", From 37144b5e27ca2acbe41b3b99118ca8c36a5f1b66 Mon Sep 17 00:00:00 2001 From: shrey Date: Tue, 10 Nov 2026 15:41:04 -0800 Subject: [PATCH 050/189] fix(mcp): challenge a single-server listing whose credential the upstream rejected (#50389) --- litellm/proxy/mcp_server/catalog.py | 33 +- litellm/proxy/mcp_server/exceptions.py | 17 +- litellm/proxy/mcp_server/legacy_callbacks.py | 7 + litellm/proxy/mcp_server/scope_challenge.py | 31 +- litellm/proxy/mcp_server/server.py | 74 ++++- .../types/mcp_server/mcp_server_manager.py | 6 + tests/unit/proxy/mcp_server/test_catalog.py | 303 +++++++++++++++++- .../test_mcp_oauth_passthrough_tools.py | 77 ++++- .../unit/proxy/mcp_server/test_mcp_server.py | 137 ++++++++ .../test_mcp_server_tool_calls_and_headers.py | 23 ++ 10 files changed, 665 insertions(+), 43 deletions(-) diff --git a/litellm/proxy/mcp_server/catalog.py b/litellm/proxy/mcp_server/catalog.py index d2ff322ba5c..e30f26abf41 100644 --- a/litellm/proxy/mcp_server/catalog.py +++ b/litellm/proxy/mcp_server/catalog.py @@ -826,6 +826,20 @@ async def prepare_tool_listing_headers( return server_auth_header, extra_headers, catalog_auth_header +def challenge_rejected_credential(server: MCPServer, outcome: ServerOutcome, listed: Sequence[MCPServer]) -> None: + """Answer a listing of this one server with the gateway's 401 when the caller can fix the rejection.""" + from litellm.proxy.mcp_server.legacy_callbacks import get_listing_scope_response + + scope_response: Final = get_listing_scope_response() + if ( + scope_response is not None + and len(listed) == 1 + and outcome.tag == "auth_required" + and server.reauthorizes_in_client + ): + scope_response.deny_reauth() + + async def get_filtered_server_tools( server: MCPServer, *, @@ -943,14 +957,11 @@ async def get_filtered_server_tools( ) return page.model_copy(update={"tools": filtered_tools}), ServerListOk(tool_count=len(filtered_tools)) except MCPUpstreamAuthError as e: - # Absorb so one unauthenticated server does not empty every other server's - # tools. Surfacing the upstream 401 to the client as a re-auth challenge is - # intentionally not done here: raising from this list handler cannot produce a - # 401 + WWW-Authenticate (the MCP session manager serializes it as a JSON-RPC - # error). Single-server routes surface it via the request-scope preemptive - # check in _raise_preemptive_401_for_unauthenticated_servers instead. + # Absorbed so peers keep their tools; a raise here would reach the client as a JSON-RPC error, not a 401 verbose_logger.debug("MCP list_tools: omitting %s; it needs upstream auth", server.name) - return ListToolsResult(tools=[]), classify_list_exception(e) + outcome: Final = classify_list_exception(e) + challenge_rejected_credential(server, outcome, allowed_mcp_servers) + return ListToolsResult(tools=[]), outcome except Exception as e: verbose_logger.exception("Error getting tools from server %s: %s", server.name, e) return ListToolsResult(tools=[]), classify_list_exception(e) @@ -1199,6 +1210,8 @@ async def list_gateway_catalog( update={"meta": {key: value for key, value in page_meta.items() if key != SERVER_OUTCOMES_META_KEY}} ) except Exception as error: + outcome: Final = classify_list_exception(error) + challenge_rejected_credential(server, outcome, allowed) if cursor is not None: raise MCPError( code=INVALID_PARAMS, message="Upstream continuation failed; start a fresh listing" @@ -1207,11 +1220,7 @@ async def list_gateway_catalog( request, (), None, - { - "litellm.ai/server_outcomes": { - _aggregate_server_key(server): classify_list_exception(error).model_dump(mode="json") - } - }, + {"litellm.ai/server_outcomes": {_aggregate_server_key(server): outcome.model_dump(mode="json")}}, ) pages, next_cursor, outcomes = await paginate_catalog( diff --git a/litellm/proxy/mcp_server/exceptions.py b/litellm/proxy/mcp_server/exceptions.py index 8be3ae78d15..db9bc4fc0ee 100644 --- a/litellm/proxy/mcp_server/exceptions.py +++ b/litellm/proxy/mcp_server/exceptions.py @@ -1,10 +1,11 @@ """Exceptions raised by the LiteLLM MCP proxy.""" from typing import Final +from urllib.parse import quote from fastapi import HTTPException -from litellm.proxy.mcp_server.oauth_utils import encode_upstream_scope +from litellm.proxy.mcp_server.oauth_utils import encode_upstream_scope, well_known_root_suffix TOKEN_EXCHANGE_SUBJECT_ERROR: Final = ( "Missing or invalid subject token. Authenticate with your identity provider and send its token " @@ -101,12 +102,14 @@ class MCPUpstreamAuthError(Exception): challenge: str | None = self.www_authenticate if challenge is None and (self.status_code == 401 or self.required_scope is not None) and base_url: prefix: Final = base_url.rstrip("/") - if aggregate_step_up: - resource_metadata_url = f"{prefix}/.well-known/oauth-protected-resource" - elif request_path and request_path.startswith(f"/{self.server_name}/mcp"): - resource_metadata_url = f"{prefix}/.well-known/oauth-protected-resource/{self.server_name}/mcp" - else: - resource_metadata_url = f"{prefix}/.well-known/oauth-protected-resource/mcp/{self.server_name}" + name: Final = quote(self.server_name, safe="/:@!$&'()*+,;=") + resource_metadata_url: Final = ( + f"{prefix}/.well-known/oauth-protected-resource" + if aggregate_step_up + else f"{prefix}/.well-known/oauth-protected-resource{well_known_root_suffix()}/{name}/mcp" + if request_path and request_path.startswith(f"/{self.server_name}/mcp") + else f"{prefix}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp/{name}" + ) challenge = f'Bearer resource_metadata="{resource_metadata_url}"' scoped_challenge: Final = ( (f'{challenge}, error="insufficient_scope"' if challenge else 'Bearer error="insufficient_scope"') diff --git a/litellm/proxy/mcp_server/legacy_callbacks.py b/litellm/proxy/mcp_server/legacy_callbacks.py index 3d2f97492bb..82d98f39604 100644 --- a/litellm/proxy/mcp_server/legacy_callbacks.py +++ b/litellm/proxy/mcp_server/legacy_callbacks.py @@ -17,6 +17,7 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.mcp_server.contracts import OperationContext from litellm.proxy.mcp_server.mcp_context import get_active_mcp_request_ctx from litellm.proxy.mcp_server.scope_challenge import ( + LISTING_METHODS, OAuthScopeResponse, is_tool_call_request, scope_response_from_request, @@ -113,6 +114,12 @@ def get_scope_response() -> OAuthScopeResponse | None: return scope_response_from_request(context.request) if context is not None else None +def get_listing_scope_response() -> OAuthScopeResponse | None: + """The response gate of the active request, when the SDK parsed that request as a listing.""" + context: Final = get_active_mcp_request_ctx() + return get_scope_response() if context is not None and context.method in LISTING_METHODS else None + + async def record_upstream_tool_authorization(response: httpx2.Response, *, pending: OAuthScopeResponse | None) -> None: if pending is None or not pending.pending or not response.is_success or response.request.method != "POST": return diff --git a/litellm/proxy/mcp_server/scope_challenge.py b/litellm/proxy/mcp_server/scope_challenge.py index 128b27b0f91..927cb38c42e 100644 --- a/litellm/proxy/mcp_server/scope_challenge.py +++ b/litellm/proxy/mcp_server/scope_challenge.py @@ -19,18 +19,31 @@ SCOPE_RESPONSE_KEY: Final = "litellm.mcp.scope_response" _REQUEST: Final = TypeAdapter[dict[str, object]](dict[str, object]) -def is_tool_call_request(body: bytes) -> bool: +LISTING_METHODS: Final = frozenset({"tools/list", "prompts/list", "resources/list", "resources/templates/list"}) + + +def _request_method(body: bytes) -> object: try: - return _REQUEST.validate_json(body).get("method") == "tools/call" + return _REQUEST.validate_json(body).get("method") except ValidationError: - return False + return None + + +def is_tool_call_request(body: bytes) -> bool: + return _request_method(body) == "tools/call" + + +def is_listing_request(body: bytes) -> bool: + method: Final = _request_method(body) + return isinstance(method, str) and method in LISTING_METHODS class OAuthScopeResponse: - def __init__(self, send: Send, base_url: str, request_path: str | None) -> None: + def __init__(self, send: Send, base_url: str, request_path: str | None, reauthorize_as: str | None = None) -> None: self._send = send self._base_url = base_url self._request_path = request_path + self._reauthorize_as = reauthorize_as self._ready = asyncio.Event() self._lock = asyncio.Lock() self._challenge: MCPUpstreamAuthError | None = None @@ -44,7 +57,15 @@ class OAuthScopeResponse: self._ready.set() def deny(self, error: MCPUpstreamAuthError) -> None: - if self.pending and error.status_code == 403 and error.required_scope is not None: + if error.status_code == 403 and error.required_scope is not None: + self._refuse(error) + + def deny_reauth(self) -> None: + if self._reauthorize_as is not None: + self._refuse(MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name=self._reauthorize_as)) + + def _refuse(self, error: MCPUpstreamAuthError) -> None: + if self.pending: self._challenge = error self._ready.set() diff --git a/litellm/proxy/mcp_server/server.py b/litellm/proxy/mcp_server/server.py index bbbbd09e527..c255eddb214 100644 --- a/litellm/proxy/mcp_server/server.py +++ b/litellm/proxy/mcp_server/server.py @@ -10,6 +10,7 @@ import contextvars import hashlib import json import os +import re import time import types from collections import Counter @@ -81,9 +82,11 @@ from litellm.proxy.mcp_server.oauth_utils import ( well_known_root_suffix, ) from litellm.proxy.mcp_server.scope_challenge import ( + LISTING_METHODS, SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response, + is_listing_request, is_tool_call_request, ) from litellm.proxy.mcp_server.ui_session_utils import ( @@ -225,16 +228,13 @@ _SESSION_MANAGERS_INITIALIZED = False _INITIALIZATION_LOCK: Final = asyncio.Lock() -def _jsonrpc_text_has_top_level_method(text: str) -> bool: - """Whether a (possibly truncated) JSON-RPC envelope has a ``method`` key at - the root object's top level. +def _jsonrpc_text_top_level_key_end(text: str, key: str) -> int | None: + """Where the root object's top-level ``key`` ends in a (possibly truncated) JSON-RPC envelope. - Used to tell a request/notification (carries ``method``) apart from a - response (carries ``result``/``error`` and no top-level ``method``). A - response payload can itself nest a ``method`` field, so only keys at the + A response payload can itself nest a ``method`` field, so only keys at the root object's depth are inspected rather than searching the whole string. - Returns ``True`` only when a top-level ``method`` key is positively found; - truncation that hides it yields ``False``. + Returns an index only when the key is positively found; truncation that + hides it yields ``None``. """ depth = 0 in_string = False @@ -243,7 +243,7 @@ def _jsonrpc_text_has_top_level_method(text: str) -> bool: reading_key = False expect_key = False key_chars: list[str] = [] - for ch in text: + for index, ch in enumerate(text): if in_string: if escaped: escaped = False @@ -251,8 +251,8 @@ def _jsonrpc_text_has_top_level_method(text: str) -> bool: escaped = True elif ch == '"': in_string = False - if reading_key and depth == 1 and "".join(key_chars) == "method": - return True + if reading_key and depth == 1 and "".join(key_chars) == key: + return index + 1 elif reading_key: key_chars.append(ch) continue @@ -276,7 +276,32 @@ def _jsonrpc_text_has_top_level_method(text: str) -> bool: expect_key = bool(in_object) and in_object[-1] elif ch == ":": expect_key = False - return False + return None + + +def _jsonrpc_text_has_top_level_method(text: str) -> bool: + """Whether a (possibly truncated) JSON-RPC envelope has a top-level ``method`` key. + + Used to tell a request/notification (carries ``method``) apart from a + response (carries ``result``/``error`` and no top-level ``method``). + """ + return _jsonrpc_text_top_level_key_end(text, "method") is not None + + +_JSONRPC_STRING_VALUE: Final = re.compile(r'\s*:\s*("(?:[^"\\]|\\.)*")') + + +def _jsonrpc_text_method(text: str) -> str | None: + """The top-level ``method`` of a (possibly truncated) JSON-RPC envelope, when the cut leaves it whole.""" + key_end: Final = _jsonrpc_text_top_level_key_end(text, "method") + value: Final = _JSONRPC_STRING_VALUE.match(text, key_end) if key_end is not None else None + if value is None: + return None + try: + method: Final = json.loads(value.group(1)) + except ValueError: + return None + return method if isinstance(method, str) else None def _mcp_meta_trace_carrier(req_ctx: object) -> dict[str, str] | None: @@ -2470,6 +2495,27 @@ if MCP_AVAILABLE: ), ) + def _is_listing_body(body: bytes) -> bool: + """A listing request, including one the routing peek cut off after its method.""" + if is_listing_request(body): + return True + peek_cut: Final = len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES + return peek_cut and _jsonrpc_text_method(body.decode("utf-8", errors="replace")) in LISTING_METHODS + + def _server_to_reauthorize( + mcp_servers: Sequence[str] | None, scoped_server_endpoint: bool, body: bytes, client_ip: str | None + ) -> str | None: + """The server a client addressed alone for a listing it can fix by authenticating again.""" + if not scoped_server_endpoint or mcp_servers is None or len(mcp_servers) != 1 or not _is_listing_body(body): + return None + server: Final = operations.global_mcp_server_manager.get_mcp_server_by_identifier( + mcp_servers[0], client_ip=client_ip + ) + reauthorizes: Final = ( + server is not None and MCPServerManager.resolve_oauth2_flow_for_request(server).reauthorizes_in_client + ) + return mcp_servers[0] if reauthorizes else None + async def handle_streamable_http_mcp(scope: Scope, receive: Receive, send: Send) -> None: """Handle MCP requests through StreamableHTTP.""" try: @@ -2638,13 +2684,15 @@ if MCP_AVAILABLE: await rejection(scope, receive, send) return + reauthorize_as: Final = _server_to_reauthorize(mcp_servers, scoped_server_endpoint, body, _client_ip) scope_response: Final = ( OAuthScopeResponse( send, get_request_base_url(StarletteRequest(scope)), get_route_relative_request_path(scope), + reauthorize_as=reauthorize_as, ) - if is_tool_call_request(body) or len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES + if is_tool_call_request(body) or len(body) >= _MCP_ROUTING_PEEK_MAX_BYTES or reauthorize_as is not None else None ) if scope_response is not None: diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 1d3d5911eb1..ed0f050a76a 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -383,6 +383,12 @@ class MCPServer(LiteLLMBaseModel): door and is never a precondition for opening a session.""" return self.is_true_passthrough or self.is_oauth_delegate + @property + def reauthorizes_in_client(self) -> bool: + """True when a caller can fix an upstream 401 by authenticating again: the client-forwarded + token modes and gateway-vaulted per-user OAuth.""" + return self.is_client_forwarded_token or (self.is_gateway_managed_oauth2 and self.needs_user_oauth_token) + @property def is_dcr_bridge(self) -> bool: """True when this client-forwarded-token server serves the gateway-hosted DCR front door diff --git a/tests/unit/proxy/mcp_server/test_catalog.py b/tests/unit/proxy/mcp_server/test_catalog.py index 45c648b81d2..3c78e8f2561 100644 --- a/tests/unit/proxy/mcp_server/test_catalog.py +++ b/tests/unit/proxy/mcp_server/test_catalog.py @@ -2,9 +2,11 @@ import asyncio from collections.abc import Mapping, Sequence from types import SimpleNamespace from typing import Final, Literal -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock +import httpx2 import pytest +from mcp.server.context import ServerRequestContext from mcp.shared.exceptions import MCPError from mcp.types import ( ListPromptsRequest, @@ -18,17 +20,24 @@ from mcp.types import ( PaginatedRequestParams, Tool, ) +from starlette.requests import Request from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.proxy.mcp_server import catalog +from litellm.proxy.mcp_server import operations as mcp_operations from litellm.proxy.mcp_server.contracts import OperationContext +from litellm.proxy.mcp_server.exceptions import MCPUpstreamAuthError from litellm.proxy.mcp_server.faults.list_outcomes import ( SERVER_OUTCOMES_META_KEY, AggregateToolListing, + ServerListFault, ServerListOk, ) -from litellm.types.mcp import MCPTransport +from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var, mcp_gateway_server_name +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager +from litellm.proxy.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse +from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer CatalogKind = Literal["tools", "prompts", "resources", "templates"] @@ -772,3 +781,293 @@ def test_partial_optional_catalog_never_advertises_freshness() -> None: assert result.ttl_ms == 0 assert result.cache_scope == "private" + +_GATEWAY: Final = "https://gateway.example" +_UNAUTHORIZED: Final = ServerListFault(tag="auth_required", status_code=401) + + +def _server( + auth_type: MCPAuth, + name: str = "docs", + oauth2_flow: str | None = None, + delegate_auth_to_upstream: bool = False, + authentication_token: str | None = None, + alias: str | None = None, +) -> MCPServer: + return MCPServer( + server_id=f"{name}-id", + name=name, + alias=alias or name, + url="https://upstream.example/mcp", + transport=MCPTransport.http, + auth_type=auth_type, + oauth2_flow=oauth2_flow, + delegate_auth_to_upstream=delegate_auth_to_upstream, + authentication_token=authentication_token, + ) + + +_VAULTED: Final = _server(MCPAuth.oauth2, oauth2_flow="authorization_code") +_PEER: Final = _server(MCPAuth.none, name="wiki") +_STATIC: Final = _server(MCPAuth.api_key, authentication_token="k") +_REAUTHORIZABLE: Final = [ + pytest.param(_VAULTED, id="vaulted-per-user-oauth"), + pytest.param(_server(MCPAuth.true_passthrough), id="true-passthrough"), + pytest.param(_server(MCPAuth.oauth_delegate), id="oauth-delegate"), +] +_NOT_REAUTHORIZABLE: Final = [ + pytest.param(_server(MCPAuth.none), id="no-auth"), + pytest.param(_STATIC, id="static-api-key"), + pytest.param(_server(MCPAuth.bearer_token, authentication_token="k"), id="static-bearer"), + pytest.param(_server(MCPAuth.oauth2, oauth2_flow="client_credentials"), id="client-credentials"), + pytest.param(_server(MCPAuth.oauth2, delegate_auth_to_upstream=True), id="legacy-upstream-delegate"), +] + + +def _request_context(gate: OAuthScopeResponse, method: str) -> ServerRequestContext: + return ServerRequestContext( + session=MagicMock(), + lifespan_context={}, + protocol_version="2025-11-25", + method=method, + request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate}), + ) + + +def _challenged( + server: MCPServer, + outcome: ServerListFault | ServerListOk, + listed: Sequence[MCPServer] | None = None, + reauthorize_as: str | None = "docs", + method: str = "tools/list", +) -> bool: + gate: Final = OAuthScopeResponse(AsyncMock(), _GATEWAY, "/docs/mcp", reauthorize_as=reauthorize_as) + token: Final = active_mcp_request_ctx_var.set(_request_context(gate, method)) + try: + catalog.challenge_rejected_credential(server, outcome, [server] if listed is None else listed) + finally: + active_mcp_request_ctx_var.reset(token) + return not gate.pending + + +@pytest.mark.parametrize("server", _REAUTHORIZABLE) +def test_a_rejected_credential_the_caller_can_renew_is_challenged(server: MCPServer) -> None: + assert _challenged(server, _UNAUTHORIZED) + + +@pytest.mark.parametrize("server", _NOT_REAUTHORIZABLE) +def test_a_rejected_credential_the_caller_cannot_renew_stays_absorbed(server: MCPServer) -> None: + assert not _challenged(server, _UNAUTHORIZED) + + +@pytest.mark.parametrize( + "outcome", + [ + ServerListOk(tool_count=3), + ServerListFault(tag="forbidden", status_code=403), + ServerListFault(tag="upstream_error", status_code=500), + ServerListFault(tag="unreachable"), + ], +) +def test_only_an_unauthorized_outcome_is_challenged(outcome: ServerListFault | ServerListOk) -> None: + assert not _challenged(_VAULTED, outcome) + + +def test_a_listing_that_spans_several_servers_is_not_challenged() -> None: + assert not _challenged(_VAULTED, _UNAUTHORIZED, [_VAULTED, _PEER]) + + +def test_a_request_that_is_not_a_single_server_listing_is_not_challenged() -> None: + assert not _challenged(_VAULTED, _UNAUTHORIZED, reauthorize_as=None) + + +@pytest.mark.parametrize("method", ["tools/call", "server/discover", "initialize"]) +def test_a_request_the_sdk_parsed_as_something_other_than_a_listing_is_not_challenged(method: str) -> None: + assert not _challenged(_VAULTED, _UNAUTHORIZED, method=method) + + +@pytest.mark.asyncio +async def test_a_rejected_stored_token_is_answered_with_the_gateways_own_challenge() -> None: + send: Final = AsyncMock() + gate: Final = OAuthScopeResponse(send, _GATEWAY, "/docs/mcp", reauthorize_as="docs") + token: Final = active_mcp_request_ctx_var.set(_request_context(gate, "tools/list")) + try: + started: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) + await asyncio.sleep(0) + catalog.challenge_rejected_credential(_VAULTED, _UNAUTHORIZED, [_VAULTED]) + await asyncio.wait_for(started, 1) + finally: + active_mcp_request_ctx_var.reset(token) + + start: Final = send.await_args_list[0].args[0] + assert start["status"] == 401 + assert dict(start["headers"])[b"www-authenticate"].decode() == ( + 'Bearer resource_metadata="https://gateway.example/.well-known/oauth-protected-resource/docs/mcp"' + ) + + +def _upstream_rejecting_the_token(_request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(401, headers={"www-authenticate": 'Bearer error="invalid_token"'}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("server", "peers", "challenged"), + [ + pytest.param(_server(MCPAuth.true_passthrough), (), True, id="true-passthrough"), + pytest.param(_server(MCPAuth.oauth_delegate), (), True, id="oauth-delegate"), + pytest.param(_STATIC, (), False, id="static-api-key"), + pytest.param(_server(MCPAuth.true_passthrough), (_PEER,), False, id="selected-group-of-two"), + ], +) +async def test_tools_listing_challenges_a_single_server_whose_token_the_upstream_rejected( + monkeypatch: pytest.MonkeyPatch, server: MCPServer, peers: Sequence[MCPServer], challenged: bool +) -> None: + manager: Final = MCPServerManager(upstream_transport=httpx2.MockTransport(_upstream_rejecting_the_token)) + monkeypatch.setattr(mcp_operations, "global_mcp_server_manager", manager) + gate: Final = OAuthScopeResponse(AsyncMock(), _GATEWAY, "/docs/mcp", reauthorize_as="docs") + context: Final = mcp_operations.prepare_context( + UserAPIKeyAuth(token="h", user_id="u1"), oauth2_headers={"Authorization": "Bearer rejected"} + ) + request_token: Final = active_mcp_request_ctx_var.set(_request_context(gate, "tools/list")) + name_token: Final = mcp_gateway_server_name.set("docs") + try: + listing, outcome = await catalog.get_filtered_server_tools( + server, context=context, allowed_mcp_servers=[server, *peers], prefetched_oauth_creds={} + ) + finally: + mcp_gateway_server_name.reset(name_token) + active_mcp_request_ctx_var.reset(request_token) + + assert listing.tools == [] + assert outcome.tag == "auth_required" + assert (not gate.pending) is challenged + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_name", ["ListPromptsRequest", "ListResourcesRequest", "ListResourceTemplatesRequest"]) +@pytest.mark.parametrize( + ("server", "challenged"), + [ + pytest.param(_VAULTED, True, id="vaulted-per-user-oauth"), + pytest.param(_STATIC, False, id="static-api-key"), + ], +) +async def test_optional_listing_challenges_a_server_whose_stored_token_was_rejected( + monkeypatch: pytest.MonkeyPatch, request_name: str, server: MCPServer, challenged: bool +) -> None: + from mcp import types + + from litellm.proxy import proxy_server + + monkeypatch.setenv("LITELLM_SALT_KEY", "listing-reauth-test") + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(mcp_operations.global_mcp_server_manager, "registry", {server.server_id: server}) + monkeypatch.setattr(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])) + monkeypatch.setattr( + catalog, "fetch_optional_catalog_page", AsyncMock(side_effect=MCPUpstreamAuthError(401, None, "docs")) + ) + gate: Final = OAuthScopeResponse(AsyncMock(), _GATEWAY, "/docs/mcp", reauthorize_as="docs") + token: Final = active_mcp_request_ctx_var.set(_request_context(gate, "prompts/list")) + try: + result: Final = await catalog.list_gateway_catalog( + mcp_operations.prepare_context(), getattr(types, request_name)() + ) + finally: + active_mcp_request_ctx_var.reset(token) + + assert next(iter(result.meta[SERVER_OUTCOMES_META_KEY].values())) == {"status": "auth_required", "http_status": 401} + assert (not gate.pending) is challenged + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_name", "result_name", "items"), + [ + ("ListPromptsRequest", "ListPromptsResult", "prompts"), + ("ListResourcesRequest", "ListResourcesResult", "resources"), + ("ListResourceTemplatesRequest", "ListResourceTemplatesResult", "resource_templates"), + ], +) +async def test_optional_listing_challenges_when_the_token_is_rejected_on_a_later_page( + monkeypatch: pytest.MonkeyPatch, request_name: str, result_name: str, items: str +) -> None: + from mcp import types + + from litellm.proxy import proxy_server + + monkeypatch.setenv("LITELLM_SALT_KEY", "listing-reauth-test") + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(mcp_operations.global_mcp_server_manager, "registry", {_VAULTED.server_id: _VAULTED}) + monkeypatch.setattr(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[_VAULTED])) + first_page: Final = getattr(types, result_name)(**{items: [], "next_cursor": "next"}) + monkeypatch.setattr( + catalog, + "fetch_optional_catalog_page", + AsyncMock(side_effect=[first_page, MCPUpstreamAuthError(401, None, "docs")]), + ) + gate: Final = OAuthScopeResponse(AsyncMock(), _GATEWAY, "/docs/mcp", reauthorize_as="docs") + token: Final = active_mcp_request_ctx_var.set(_request_context(gate, "prompts/list")) + request: Final = getattr(types, request_name) + try: + first: Final = await catalog.list_gateway_catalog(mcp_operations.prepare_context(), request()) + assert first.next_cursor + assert gate.pending + with pytest.raises(MCPError, match="Upstream continuation failed"): + await catalog.list_gateway_catalog( + mcp_operations.prepare_context(), + request(params=types.PaginatedRequestParams(cursor=first.next_cursor)), + ) + finally: + active_mcp_request_ctx_var.reset(token) + + assert not gate.pending + + +@pytest.mark.parametrize( + ("request_path", "resource"), + [("/mcp/docs", "litellm/mcp/docs"), ("/docs/mcp", "litellm/docs/mcp")], +) +def test_the_challenge_names_the_metadata_route_under_a_server_root_path( + monkeypatch: pytest.MonkeyPatch, request_path: str, resource: str +) -> None: + monkeypatch.setenv("SERVER_ROOT_PATH", "/litellm") + + challenge: Final = MCPUpstreamAuthError(401, None, "docs").to_http_exception(_GATEWAY, request_path) + + assert (challenge.headers or {})["www-authenticate"] == ( + f'Bearer resource_metadata="https://gateway.example/.well-known/oauth-protected-resource/{resource}"' + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_path", "addressed", "resource"), + [ + pytest.param("/docs/mcp", "docs", "docs/mcp", id="per-server-path"), + pytest.param("/Docs/mcp", "Docs", "Docs/mcp", id="other-case"), + pytest.param("/docs_internal/mcp", "docs_internal", "docs_internal/mcp", id="name-not-alias"), + pytest.param("/mcp/Docs", "Docs", "mcp/Docs", id="standard-path-other-case"), + pytest.param("/mcp/a:b@c+d,e", "a:b@c+d,e", "mcp/a:b@c+d,e", id="path-legal-characters-stay-as-typed"), + pytest.param("/mcp/a b%c", "a b%c", "mcp/a%20b%25c", id="path-illegal-characters-are-encoded"), + pytest.param("/mcp/日本", "日本", "mcp/%E6%97%A5%E6%9C%AC", id="non-ascii-name"), + pytest.param("/日本/mcp", "日本", "%E6%97%A5%E6%9C%AC/mcp", id="non-ascii-name-legacy-path"), + ], +) +async def test_the_challenge_names_the_url_the_client_addressed(request_path: str, addressed: str, resource: str) -> None: + send: Final = AsyncMock() + gate: Final = OAuthScopeResponse(send, _GATEWAY, request_path, reauthorize_as=addressed) + aliased: Final = _server(MCPAuth.oauth2, name="docs_internal", oauth2_flow="authorization_code", alias="docs") + token: Final = active_mcp_request_ctx_var.set(_request_context(gate, "tools/list")) + try: + started: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) + await asyncio.sleep(0) + catalog.challenge_rejected_credential(aliased, _UNAUTHORIZED, [aliased]) + await asyncio.wait_for(started, 1) + finally: + active_mcp_request_ctx_var.reset(token) + + start: Final = send.await_args_list[0].args[0] + assert dict(start["headers"])[b"www-authenticate"].decode() == ( + f'Bearer resource_metadata="https://gateway.example/.well-known/oauth-protected-resource/{resource}"' + ) diff --git a/tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_tools.py b/tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_tools.py index 03a7373c908..5054af4a2d4 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_tools.py +++ b/tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_tools.py @@ -25,6 +25,7 @@ from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, _extract_upstream_auth_failure, ) +from litellm.proxy.mcp_server.scope_challenge import OAuthScopeResponse, is_listing_request from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -375,12 +376,10 @@ async def test_aggregate_list_tools_absorbs_one_unauthenticated_server(): @pytest.mark.asyncio -async def test_single_server_route_also_absorbs_upstream_auth_error(): +async def test_single_server_route_also_absorbs_upstream_auth_error() -> None: """A single-server route (//mcp) absorbs an upstream-auth error just like the aggregate: the failing server contributes no tools and an "auth_required" outcome rather than re-raising. - Surfacing it to the client as a 401 + WWW-Authenticate challenge cannot be done from this list - handler — the MCP session manager serializes a raise into a JSON-RPC error, not an HTTP 401 — so - re-auth surfacing is handled by a request-scope preemptive check, tracked separately.""" + The 401 a single-server request receives is sent through its scope response, not raised from here.""" from unittest.mock import patch from litellm.proxy._types import UserAPIKeyAuth @@ -836,3 +835,73 @@ def test_managed_step_up_metadata_matches_the_sdk_configured_resource(path): requested_resource=f"https://gateway.test{path}", configured_resource=f"https://gateway.test{resource_path}", ), "the SDK must accept the advertised resource before it can reauthorize" + + +@pytest.mark.parametrize( + ("body", "expected"), + [ + (b'{"jsonrpc":"2.0","id":1,"method":"tools/list"}', True), + (b'{"jsonrpc":"2.0","id":1,"method":"prompts/list"}', True), + (b'{"jsonrpc":"2.0","id":1,"method":"resources/list"}', True), + (b'{"jsonrpc":"2.0","id":1,"method":"resources/templates/list"}', True), + (b'{"jsonrpc":"2.0","id":1,"method":"tools/call"}', False), + (b'{"jsonrpc":"2.0","id":1,"method":"initialize"}', False), + (b'{"jsonrpc":"2.0","id":1,"method":["tools/list"]}', False), + (b'{"jsonrpc":"2.0","id":1}', False), + (b"not json", False), + ], +) +def test_listing_requests_are_recognised_by_method(body: bytes, expected: bool) -> None: + assert is_listing_request(body) is expected + + +async def _pending_listing_response(gate: OAuthScopeResponse, send: AsyncMock) -> "asyncio.Task[None]": + task: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) + await asyncio.sleep(0) + send.assert_not_awaited() + return task + + +@pytest.mark.asyncio +async def test_reauth_denial_replaces_the_pending_listing_response_with_the_gateway_challenge() -> None: + send: Final = AsyncMock() + gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/docs/mcp", reauthorize_as="docs") + started: Final = await _pending_listing_response(gate, send) + + gate.deny_reauth() + await asyncio.wait_for(started, 1) + + start, body = [call.args[0] for call in send.await_args_list] + assert start["status"] == 401 + assert dict(start["headers"])[b"www-authenticate"].decode() == ( + 'Bearer resource_metadata="https://gateway.example/.well-known/oauth-protected-resource/docs/mcp"' + ) + assert body["body"] == b'{"detail":"Unauthorized"}' + + +@pytest.mark.asyncio +async def test_a_gate_that_does_not_reauthorize_ignores_a_reauth_denial() -> None: + send: Final = AsyncMock() + gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/docs/mcp") + started: Final = await _pending_listing_response(gate, send) + + gate.deny_reauth() + assert gate.pending + gate.allow() + await asyncio.wait_for(started, 1) + + assert [call.args[0]["status"] for call in send.await_args_list] == [200] + + +@pytest.mark.asyncio +async def test_scope_step_up_denial_still_ignores_an_unauthorized_response() -> None: + send: Final = AsyncMock() + gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/docs/mcp", reauthorize_as="docs") + started: Final = await _pending_listing_response(gate, send) + + gate.deny(MCPUpstreamAuthError(401, None, "docs")) + assert gate.pending + gate.allow() + await asyncio.wait_for(started, 1) + + assert [call.args[0]["status"] for call in send.await_args_list] == [200] diff --git a/tests/unit/proxy/mcp_server/test_mcp_server.py b/tests/unit/proxy/mcp_server/test_mcp_server.py index 456b1994e2d..b42f913f078 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server.py @@ -2,6 +2,8 @@ import asyncio import json import os +import queue +from collections.abc import Sequence from contextlib import asynccontextmanager, nullcontext from typing import Final from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -17,6 +19,7 @@ from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, MCPTransport, ) +from litellm.types.mcp import MCPAuth mcp_server_manager = MCPServerManager() @@ -3209,6 +3212,140 @@ async def test_mcp_routing_stashes_only_method_for_auth_and_replays_full_body(me ) +def _docs_server(auth_type: str, legacy_client_credentials: bool = False) -> MCPServer: + return MCPServer( + server_id="docs-id", + name="docs", + alias="docs", + url="https://upstream.example/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth(auth_type), + token_url="https://idp.example/token" if legacy_client_credentials else None, + client_id="client" if legacy_client_credentials else None, + client_secret="secret" if legacy_client_credentials else None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("path", "method", "shape", "selected_servers", "docs", "expected"), + [ + ("/mcp/docs", "tools/list", "plain", ["docs"], _docs_server("oauth_delegate"), "reauthorizes:/mcp/docs"), + ("/mcp/docs", "prompts/list", "plain", ["docs"], _docs_server("oauth_delegate"), "reauthorizes:/mcp/docs"), + ("/mcp/docs", "resources/list", "plain", ["docs"], _docs_server("oauth_delegate"), "reauthorizes:/mcp/docs"), + ("/mcp/docs", "resources/templates/list", "plain", ["docs"], _docs_server("oauth_delegate"), "reauthorizes:/mcp/docs"), + ("/mcp/DOCS", "tools/list", "plain", ["DOCS"], _docs_server("oauth_delegate"), "reauthorizes:/mcp/DOCS"), + ("/mcp/docs", "tools/list", "plain", ["docs"], _docs_server("api_key"), "none"), + ("/mcp/docs", "tools/list", "plain", ["docs"], _docs_server("oauth2", legacy_client_credentials=True), "none"), + ("/mcp/docs", "tools/call", "plain", ["docs"], _docs_server("oauth_delegate"), "holds"), + ("/mcp/docs", "tools/list", "plain", ["docs", "wiki"], _docs_server("oauth_delegate"), "none"), + ("/mcp/docs", "tools/list", "plain", ["unregistered_group"], _docs_server("oauth_delegate"), "none"), + ("/mcp/docs", "tools/list", "plain", None, _docs_server("oauth_delegate"), "none"), + ("/mcp/docs", "ping", "plain", ["docs"], _docs_server("oauth_delegate"), "none"), + ("/docs/mcp", "tools/list", "plain", ["docs"], _docs_server("oauth_delegate"), "reauthorizes:/docs/mcp"), + ("/Docs/mcp", "tools/list", "plain", ["Docs"], _docs_server("oauth_delegate"), "reauthorizes:/Docs/mcp"), + ("/mcp/docs-id", "tools/list", "plain", ["docs-id"], _docs_server("oauth_delegate"), "reauthorizes:/mcp/docs-id"), + ("/mcp", "tools/list", "plain", ["docs"], _docs_server("oauth_delegate"), "none"), + ("/toolset/team/mcp", "tools/list", "plain", ["docs"], _docs_server("oauth_delegate"), "none"), + ("/mcp/docs", "tools/list", "cursor-after-method", ["docs"], _docs_server("oauth_delegate"), "reauthorizes:/mcp/docs"), + ("/mcp/docs", "prompts/list", "cursor-after-method", ["docs"], _docs_server("oauth_delegate"), "reauthorizes:/mcp/docs"), + ("/mcp/docs", "tools/list", "cursor-before-method", ["docs"], _docs_server("oauth_delegate"), "holds"), + ("/mcp/docs", "tools/call", "cursor-after-method", ["docs"], _docs_server("oauth_delegate"), "holds"), + ("/mcp/docs", "ping", "duplicate-method", ["docs"], _docs_server("oauth_delegate"), "none"), + ], +) +async def test_mcp_routing_holds_a_single_servers_listing_response_for_a_reauth_challenge( + path: str, method: str, shape: str, selected_servers: Sequence[str] | None, docs: MCPServer, expected: str +) -> None: + try: + from litellm.proxy.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse + from litellm.proxy.mcp_server.server import ( + handle_streamable_http_mcp, + session_manager_stateful, + session_manager_stateless, + ) + except ImportError: + pytest.skip("MCP server not available") + + async def gate_kind(gate: object) -> str: + if not isinstance(gate, OAuthScopeResponse): + return "none" + gate.deny_reauth() + if gate.pending: + return "holds" + await gate({"type": "http.response.start", "status": 200, "headers": []}) + challenge: Final = dict(sent.await_args_list[0].args[0]["headers"])[b"www-authenticate"].decode() + return "reauthorizes:" + challenge.split("oauth-protected-resource")[1].rstrip('"') + + from litellm.proxy.mcp_server import operations + + envelope: Final = {"jsonrpc": "2.0", "id": 2, "method": method, "params": {"cursor": "c" * 6000}} + bodies: Final = { + "plain": json.dumps({**envelope, "params": {}}).encode(), + "cursor-after-method": json.dumps(envelope).encode(), + "cursor-before-method": json.dumps({"params": envelope["params"], **envelope}).encode(), + "duplicate-method": b'{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{},"method":"ping"}', + } + jsonrpc_body: Final = bodies[shape] + scope: Final = { + "type": "http", + "method": "POST", + "path": path, + "scheme": "https", + "server": ("gateway.example", 443), + "headers": [ + (b"host", b"gateway.example"), + (b"content-type", b"application/json"), + (b"authorization", b"Bearer test-key"), + ], + } + receive: Final = AsyncMock(side_effect=[{"type": "http.request", "body": jsonrpc_body, "more_body": False}]) + held: Final = queue.SimpleQueue[object]() + sent: Final = AsyncMock() + + async def stateless_handle(handled_scope: Scope, _receive: Receive, _send: Send) -> None: + held.put(handled_scope.get(SCOPE_RESPONSE_KEY)) + + with ( + patch( # test-quality-ok: the ASGI handler reads auth from a module-level helper; the suite's only seam + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=(MagicMock(), None, selected_servers, None, None, None), + ), + patch( # test-quality-ok: registry is empty in unit tests; key owns one server + "litellm.proxy.mcp_server.server._get_allowed_mcp_servers", + new_callable=AsyncMock, + return_value=[MagicMock()], + ), + patch( # test-quality-ok: init flag is a module global; no injection seam + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + True, + ), + patch.object( # test-quality-ok: session managers are module-level singletons; the suite's only seam + session_manager_stateless, "handle_request", side_effect=stateless_handle + ), + patch.object( # test-quality-ok: session managers are module-level singletons; the suite's only seam + session_manager_stateful, "handle_request", side_effect=stateless_handle + ), + patch.object( # test-quality-ok: the registry is a module-level singleton's attribute; the suite's only seam + operations.global_mcp_server_manager, "registry", {docs.server_id: docs} + ), + patch( # test-quality-ok: the missing-token challenge runs before the gate and is not under test here + "litellm.proxy.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", + new_callable=AsyncMock, + ), + patch.object( # test-quality-ok: session managers are module-level singletons; the suite's only seam + session_manager_stateless, "_server_instances", {} + ), + patch.object( # test-quality-ok: session managers are module-level singletons; the suite's only seam + session_manager_stateful, "_server_instances", {} + ), + ): + await handle_streamable_http_mcp(scope, receive, sent) + + assert await gate_kind(held.get_nowait()) == expected + + @pytest.mark.asyncio async def test_mcp_routing_batch_body_is_not_stashed_for_auth(): try: diff --git a/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py index 621a7b65005..4b08fa918cf 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -4190,6 +4190,29 @@ def test_jsonrpc_text_has_top_level_method_ignores_nested_method(): assert _jsonrpc_text_has_top_level_method(truncated) is False +@pytest.mark.parametrize( + ("text", "method"), + [ + pytest.param('{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}', "tools/list", id="complete"), + pytest.param('{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{"cursor":"' + "c" * 5000, "tools/list", id="cut-after-method"), + pytest.param('{"jsonrpc":"2.0","params":{"cursor":"' + "c" * 5000, None, id="cut-before-method"), + pytest.param('{"jsonrpc":"2.0","id":1,"method":"tools/li', None, id="cut-inside-method"), + pytest.param('{"jsonrpc":"2.0","params":{"method":"tools/list"},"method":"ping"}', "ping", id="nested-method-first"), + pytest.param('{"jsonrpc":"2.0","id":1,"method":"tools\\/list","params":{"cursor":"' + "c" * 5000, "tools/list", id="escaped-slash"), + pytest.param('{"jsonrpc":"2.0","id":1,"method":"\\u0074ools/list","params":{"cursor":"' + "c" * 5000, "tools/list", id="unicode-escape"), + pytest.param('{"jsonrpc":"2.0","id":1,"method":"bad\\x","params":{}}', None, id="invalid-escape"), + pytest.param('{"jsonrpc":"2.0","id":1,"method":7,"params":{}}', None, id="not-a-string"), + pytest.param('{"jsonrpc":"2.0","id":1,"result":{"method":"tools/list"}}', None, id="response"), + ], +) +def test_jsonrpc_text_method_reads_the_top_level_method_when_the_cut_leaves_it_whole( + text: str, method: str | None +) -> None: + from litellm.proxy.mcp_server.server import _jsonrpc_text_method + + assert _jsonrpc_text_method(text) == method + + @pytest.mark.asyncio @pytest.mark.parametrize("response_field", ("result", "error")) @pytest.mark.parametrize(("character", "bytes_before_cap"), (("", 0), ("x", 0), ("é", 1), ("中", 2), ("😀", 3))) From ebd5a15b22ba012d2f5a5ee654afc93cd51add5e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:52:07 -0800 Subject: [PATCH 051/189] fix(ci): install tzdata so zoneinfo tests pass on runners without legacy tz aliases (#51057) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- pyproject.toml | 1 + uv.lock | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b7db2cc1d7b..6145db57902 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -277,6 +277,7 @@ proxy-dev = [ ] ci = [ "psutil==7.2.2", + "tzdata==2026.2", # These are lazily imported at call sites; keep them out of core deps to # avoid bloating the base SDK install (google-generativeai pulls grpcio + # protobuf, Pillow is a compiled C extension). diff --git a/uv.lock b/uv.lock index 62187d596ec..2dec937800c 100644 --- a/uv.lock +++ b/uv.lock @@ -5230,6 +5230,7 @@ ci = [ { name = "pytest-retry" }, { name = "tenacity" }, { name = "traceloop-sdk" }, + { name = "tzdata" }, ] dev = [ { name = "basedpyright" }, @@ -5468,6 +5469,7 @@ ci = [ { name = "pytest-retry", specifier = "==1.7.0" }, { name = "tenacity", specifier = "==8.5.0" }, { name = "traceloop-sdk", specifier = "==0.34.0" }, + { name = "tzdata", specifier = "==2026.2" }, ] dev = [ { name = "basedpyright", specifier = "==1.39.7" }, From 2c9068dbab010603f2367f765000217156342e30 Mon Sep 17 00:00:00 2001 From: Joseph Barker <156112794+seph-barker@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:30:41 -0800 Subject: [PATCH 052/189] fix(guardrails/rubrik): moderate text-completion prompts and forward rac_agent_id (#49912) * fix(guardrails/rubrik): moderate text-completion prompts and forward rac_agent_id _moderate_prompt synthesized a user message from texts for /v1/completions requests but built the payload from structured_messages, so the webhook received an empty messages list and the prompt went unmoderated. Pass the resolved messages into _build_prompt_moderation_payload. Forward metadata.rac_agent_id to the before_prompt and after_completion webhooks so agent-scoped policies apply. Only that key is forwarded; the rest of request_data metadata carries non-serializable objects and inbound headers * fix(guardrails/rubrik): also read rac_agent_id from litellm_metadata /v1/messages, /v1/responses and other LITELLM_METADATA_ROUTES keep proxy metadata under litellm_metadata, so a caller on those routes may set rac_agent_id there. Read metadata first, then litellm_metadata, forwarding only a non-empty string * fix(guardrails/rubrik): take rac_agent_id from key or team metadata only The Rubrik webhooks use rac_agent_id to choose which agent-scoped policies apply. Forwarding it from the request body let a caller pick any agent's policies, including one a policy exempts. Read it from the caller's admin-configured key metadata, falling back to team metadata, via CustomGuardrail._get_admin_metadata. The proxy writes those from the authenticated key and strips client-supplied user_api_key_* fields from both metadata dicts, so a caller cannot set or override the value --------- Co-authored-by: yucheng-berri --- litellm/integrations/rubrik.py | 42 +++++- tests/unit/integrations/test_rubrik.py | 179 ++++++++++++++++++++++++- 2 files changed, 209 insertions(+), 12 deletions(-) diff --git a/litellm/integrations/rubrik.py b/litellm/integrations/rubrik.py index 57e8323cdc3..7d04b1df95e 100644 --- a/litellm/integrations/rubrik.py +++ b/litellm/integrations/rubrik.py @@ -31,7 +31,7 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionUserMessage from litellm.types.utils import ( ChatCompletionMessageToolCall, Function, @@ -103,6 +103,8 @@ class _RequestContextSource(Protocol): @overload def get(self, key: Literal["optional_params"], /) -> Mapping[str, object] | None: ... @overload + def get(self, key: Literal["metadata", "litellm_metadata"], /) -> Mapping[str, object] | None: ... + @overload def get(self, key: str, /) -> object | None: ... def __contains__(self, key: object, /) -> bool: ... def __getitem__(self, key: str, /) -> object: ... @@ -441,7 +443,7 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): ) -> GenericGuardrailAPIInputs: """Send the (normalized) prompt to the before_prompt webhook and raise if the prompt is blocked.""" - messages = inputs.get("structured_messages") + messages: Sequence[AllMessageValues] | None = inputs.get("structured_messages") if not messages: # For non-chat request types (e.g. /v1/completions), litellm # supplies the prompt as ``texts`` with no structured_messages. @@ -450,11 +452,11 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): if texts: joined: Final = "\n".join(t for t in texts if t) if joined: - messages = [{"role": "user", "content": joined}] + messages = (ChatCompletionUserMessage(role="user", content=joined),) if not messages: return inputs - payload: Final = self._build_prompt_moderation_payload(inputs, request_data) + payload: Final = self._build_prompt_moderation_payload(inputs, messages, request_data) service_response: Final = await self._post_to_prompt_moderation_endpoint(payload) refusal: Final = self._extract_prompt_refusal(service_response) if refusal is None: @@ -616,7 +618,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): @staticmethod def _build_prompt_moderation_payload( inputs: GenericGuardrailAPIInputs, - request_data: Mapping[str, object], + messages: Sequence[AllMessageValues | None], + request_data: _RequestContextSource, ) -> Mapping[str, object]: """Build the bare OpenAI request the before_prompt webhook consumes. @@ -637,10 +640,11 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): correlation_key: Final = request_data.get("correlation_key") or request_data.get("litellm_call_id") return { "model": inputs.get("model") or request_data.get("model") or "", - "messages": RubrikLogger._flatten_messages_for_moderation(inputs.get("structured_messages")), + "messages": RubrikLogger._flatten_messages_for_moderation(messages), **({"tools": tools} if tools is not None else _EMPTY_MAPPING), **({"user": user} if user else _EMPTY_MAPPING), **({"correlation_key": correlation_key} if correlation_key else _EMPTY_MAPPING), + **RubrikLogger._rac_agent_metadata(request_data), } @staticmethod @@ -671,7 +675,8 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): else: tools = optional_params.get("tools") - # The response moderation service consumes only messages/model/tools. + # The response moderation service consumes only messages/model/tools + # (plus metadata.rac_agent_id when set). # Don't forward proxy_server_request -- in litellm >=1.83 its ``body`` # snapshot carries a UserAPIKeyAuth instance that breaks json.dumps, # silently fail-opening the guardrail. @@ -679,8 +684,31 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): "messages": call_details.get("messages"), "model": call_details.get("model"), "tools": tools, + **RubrikLogger._rac_agent_metadata(request_data), } + @staticmethod + def _rac_agent_metadata(request_data: _RequestContextSource) -> Mapping[str, object]: + """Forward ``rac_agent_id`` from the caller's key or team metadata. + + The webhooks use it to choose which agent-scoped policies apply, so it + must come from configuration an admin controls, never from the request + body: a caller who could set it would select any agent's policies, + including one a policy exempts. ``_get_admin_metadata`` reads only the + ``user_api_key_metadata`` / ``user_api_key_team_metadata`` the proxy + writes from the authenticated key (key overrides team); the proxy strips + client-supplied ``user_api_key_*`` fields from both metadata dicts. + """ + rac_agent_id: Final[object] = RubrikLogger._admin_rac_agent_id(request_data) + if not isinstance(rac_agent_id, str) or not rac_agent_id: + return _EMPTY_MAPPING + return {"metadata": {"rac_agent_id": rac_agent_id}} + + @staticmethod + def _admin_rac_agent_id(request_data: _RequestContextSource) -> object: + """``rac_agent_id`` from the merged key/team admin metadata, untyped.""" + return RubrikLogger._get_admin_metadata(request_data).get("rac_agent_id") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportArgumentType] # upstream helper takes and returns a bare dict; caller narrows to str + @staticmethod def _sanitize_proxy_server_request(proxy_server_request: Mapping[str, object] | str | None) -> object: """Allowlist only routing fields (``url``, ``method``) when forwarding diff --git a/tests/unit/integrations/test_rubrik.py b/tests/unit/integrations/test_rubrik.py index f8aec70a2f7..cc9e03ba4b9 100644 --- a/tests/unit/integrations/test_rubrik.py +++ b/tests/unit/integrations/test_rubrik.py @@ -5,6 +5,7 @@ Covers initialization, apply_guardrail (prompt moderation + response/tool blocking), batch logging, and Anthropic format handling. """ +import json import os from typing import Any, Dict from unittest.mock import AsyncMock, Mock, patch @@ -534,6 +535,22 @@ def _mock_service_response(response_json): return mock_client +def _capturing_service(response_json): + """Like ``_mock_service_response`` but also records each posted JSON body.""" + captured: list = [] + + async def mock_post(*_args, **kwargs): + captured.append(kwargs.get("json")) + mock_resp = Mock() + mock_resp.json.return_value = response_json + mock_resp.raise_for_status = Mock() + return mock_resp + + mock_client = AsyncMock() + mock_client.post = mock_post + return mock_client, captured + + def _echo_service(): """Create a mock tool blocking client that echoes the payload back.""" @@ -1230,6 +1247,35 @@ class TestPromptModeration: ) assert result is inputs + async def test_text_completion_prompt_is_sent_as_user_message(self, handler): + """/v1/completions requests carry the prompt as ``texts`` with no + structured_messages; the webhook must still receive it.""" + handler.moderation_client, captured = _capturing_service({}) + + await handler.apply_guardrail( + inputs={"texts": ["write a poem", "", "about cats"]}, + request_data={}, + input_type="request", + ) + + assert list(captured[0]["messages"]) == [ + {"role": "user", "content": "write a poem\nabout cats"} + ] + + async def test_structured_messages_take_precedence_over_texts(self, handler): + handler.moderation_client, captured = _capturing_service({}) + + await handler.apply_guardrail( + inputs={ + "texts": ["ignored"], + "structured_messages": [{"role": "user", "content": "hello"}], + }, + request_data={}, + input_type="request", + ) + + assert list(captured[0]["messages"]) == [{"role": "user", "content": "hello"}] + async def test_prompt_moderation_stashes_logging_obj_on_block(self, handler): """On a prompt block, _stash_block_context must set the blocked flag.""" inputs = { @@ -1378,7 +1424,7 @@ class TestBuildPromptModerationPayload: "structured_messages": [{"role": "user", "content": "hi"}], "tools": [{"type": "function", "function": {"name": "fn"}}], } - payload = RubrikLogger._build_prompt_moderation_payload(inputs, {}) + payload = RubrikLogger._build_prompt_moderation_payload(inputs, inputs["structured_messages"], {}) assert payload["tools"] == [{"type": "function", "function": {"name": "fn"}}] def test_payload_includes_user_when_present(self): @@ -1386,7 +1432,7 @@ class TestBuildPromptModerationPayload: "structured_messages": [{"role": "user", "content": "hi"}], } request_data = {"user": "alice"} - payload = RubrikLogger._build_prompt_moderation_payload(inputs, request_data) + payload = RubrikLogger._build_prompt_moderation_payload(inputs, inputs["structured_messages"], request_data) assert payload["user"] == "alice" def test_payload_uses_explicit_correlation_key(self): @@ -1395,22 +1441,145 @@ class TestBuildPromptModerationPayload: "correlation_key": "corr-123", "litellm_call_id": "litellm-456", } - payload = RubrikLogger._build_prompt_moderation_payload(inputs, request_data) + payload = RubrikLogger._build_prompt_moderation_payload(inputs, inputs["structured_messages"], request_data) assert payload["correlation_key"] == "corr-123" def test_payload_falls_back_to_litellm_call_id(self): """When correlation_key is absent, litellm_call_id is used.""" inputs = {"structured_messages": [{"role": "user", "content": "hi"}]} request_data = {"litellm_call_id": "litellm-789"} - payload = RubrikLogger._build_prompt_moderation_payload(inputs, request_data) + payload = RubrikLogger._build_prompt_moderation_payload(inputs, inputs["structured_messages"], request_data) assert payload["correlation_key"] == "litellm-789" def test_payload_omits_optional_fields_when_absent(self): inputs = {"structured_messages": [{"role": "user", "content": "hi"}]} - payload = RubrikLogger._build_prompt_moderation_payload(inputs, {}) + payload = RubrikLogger._build_prompt_moderation_payload(inputs, inputs["structured_messages"], {}) assert "tools" not in payload assert "user" not in payload assert "correlation_key" not in payload + assert "metadata" not in payload + + +# -- rac_agent_id forwarding -------------------------------------------------- + + +def _proxy_metadata( + key_meta: Dict[str, Any] | None = None, + team_meta: Dict[str, Any] | None = None, + **caller_fields: Any, +) -> Dict[str, Any]: + """Request metadata as the proxy hands it to guardrails: the authenticated + key/team metadata under ``user_api_key_*``, next to fields the caller sent""" + return { + **caller_fields, + "headers": {"authorization": "Bearer sk-secret"}, + "litellm_parent_otel_span": object(), + "user_api_key_metadata": key_meta or {}, + "user_api_key_team_metadata": team_meta or {}, + } + + +def _prompt_and_response_metadata(request_data: Dict[str, Any]): + inputs = {"structured_messages": [{"role": "user", "content": "hi"}]} + prompt_payload = RubrikLogger._build_prompt_moderation_payload( + inputs, inputs["structured_messages"], request_data + ) + response_request = RubrikLogger._extract_request_data( + {"model": "gpt-4"}, request_data + ) + return prompt_payload.get("metadata"), response_request.get("metadata") + + +class TestRacAgentIdForwarding: + def test_forwards_rac_agent_id_from_key_metadata_only(self): + request_data = {"metadata": _proxy_metadata(key_meta={"rac_agent_id": "agent-42", "other": 1})} + assert _prompt_and_response_metadata(request_data) == ( + {"rac_agent_id": "agent-42"}, + {"rac_agent_id": "agent-42"}, + ) + + def test_forwards_rac_agent_id_from_team_metadata(self): + request_data = {"metadata": _proxy_metadata(team_meta={"rac_agent_id": "team-agent"})} + assert _prompt_and_response_metadata(request_data) == ( + {"rac_agent_id": "team-agent"}, + {"rac_agent_id": "team-agent"}, + ) + + def test_key_metadata_overrides_team_metadata(self): + request_data = { + "metadata": _proxy_metadata( + key_meta={"rac_agent_id": "key-agent"}, + team_meta={"rac_agent_id": "team-agent"}, + ) + } + assert _prompt_and_response_metadata(request_data)[0] == {"rac_agent_id": "key-agent"} + + def test_admin_metadata_under_litellm_metadata(self): + """/v1/messages and /v1/responses put proxy metadata under litellm_metadata""" + request_data = {"litellm_metadata": _proxy_metadata(key_meta={"rac_agent_id": "agent-42"})} + assert _prompt_and_response_metadata(request_data)[0] == {"rac_agent_id": "agent-42"} + + @pytest.mark.parametrize("container", ["metadata", "litellm_metadata"]) + def test_caller_supplied_rac_agent_id_is_ignored(self, container): + """A caller must not choose the policy scope: the request-body value is + never forwarded, even when the key has no agent configured""" + request_data = {container: _proxy_metadata(rac_agent_id="exempt-agent")} + assert _prompt_and_response_metadata(request_data) == (None, None) + + def test_caller_cannot_override_the_keys_agent(self): + request_data = { + "metadata": _proxy_metadata( + key_meta={"rac_agent_id": "assigned-agent"}, + rac_agent_id="exempt-agent", + ), + "litellm_metadata": {"rac_agent_id": "exempt-agent"}, + } + assert _prompt_and_response_metadata(request_data) == ( + {"rac_agent_id": "assigned-agent"}, + {"rac_agent_id": "assigned-agent"}, + ) + + @pytest.mark.parametrize( + "request_data", + [ + {}, + {"metadata": None}, + {"metadata": "not-a-dict"}, + {"metadata": _proxy_metadata()}, + {"metadata": _proxy_metadata(key_meta={"rac_agent_id": ""})}, + {"metadata": _proxy_metadata(key_meta={"rac_agent_id": {"nested": "object"}})}, + {"metadata": _proxy_metadata(key_meta={"rac_agent_id": 42})}, + ], + ) + def test_metadata_omitted_without_a_valid_admin_rac_agent_id(self, request_data): + assert _prompt_and_response_metadata(request_data) == (None, None) + + @pytest.mark.asyncio + async def test_both_webhooks_receive_rac_agent_id_and_payloads_serialize( + self, handler + ): + request_data = {"metadata": _proxy_metadata(key_meta={"rac_agent_id": "agent-42"})} + handler.moderation_client, prompt_captured = _capturing_service({}) + await handler.apply_guardrail( + inputs={"structured_messages": [{"role": "user", "content": "hi"}]}, + request_data=request_data, + input_type="request", + ) + handler.moderation_client, response_captured = _capturing_service( + {"choices": [{"message": {"role": "assistant", "content": "hello"}}]} + ) + await handler.apply_guardrail( + inputs={"texts": ["hello"]}, + request_data=request_data, + input_type="response", + ) + + assert prompt_captured[0]["metadata"] == {"rac_agent_id": "agent-42"} + assert response_captured[0]["request"]["metadata"] == { + "rac_agent_id": "agent-42" + } + json.dumps(prompt_captured[0]) + json.dumps(response_captured[0]) # -- _extract_request_data tools preference ----------------------------------- From 4957007b0b9a88b95a3cebd791bd870fdeed4f16 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Wed, 11 Nov 2026 08:31:26 +0800 Subject: [PATCH 053/189] fix(model-prices): use current Hy4 promotional rates (#51072) --- ...odel_prices_and_context_window_backup.json | 20 +++++++++---------- model_prices_and_context_window.json | 20 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 83ab91f2bc0..6ffc20e9e71 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -64701,15 +64701,15 @@ "supports_vision": false }, "tencent/hy4": { - "cache_read_input_token_cost": 4.2e-08, - "comment": "TokenHub bills Hy4 at 50% of list until 2026-12-31 23:59:59 Beijing time; these are the list rates.", - "input_cost_per_token": 8.34e-07, - "input_cost_per_token_cache_hit": 4.2e-08, + "cache_read_input_token_cost": 2.1e-08, + "comment": "These are the 50% promotional rates. Update on 2026-12-31 when the promotion ends at 23:59:59 Beijing time. List rates per 1M tokens are $0.834 input, $2.501 output and $0.042 cached input.", + "input_cost_per_token": 4.17e-07, + "input_cost_per_token_cache_hit": 2.1e-08, "litellm_provider": "tencent", "max_input_tokens": 960000, "max_output_tokens": 64000, "mode": "chat", - "output_cost_per_token": 2.501e-06, + "output_cost_per_token": 1.2505e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", "supports_function_calling": true, "supports_prompt_caching": true, @@ -64718,15 +64718,15 @@ "supports_vision": true }, "tencent/hy4-preview": { - "cache_read_input_token_cost": 4.2e-08, - "comment": "TokenHub bills Hy4 preview at 50% of list until 2026-12-31 23:59:59 Beijing time; these are the list rates.", - "input_cost_per_token": 8.34e-07, - "input_cost_per_token_cache_hit": 4.2e-08, + "cache_read_input_token_cost": 2.1e-08, + "comment": "These are the 50% promotional rates. Update on 2026-12-31 when the promotion ends at 23:59:59 Beijing time. List rates per 1M tokens are $0.834 input, $2.501 output and $0.042 cached input.", + "input_cost_per_token": 4.17e-07, + "input_cost_per_token_cache_hit": 2.1e-08, "litellm_provider": "tencent", "max_input_tokens": 960000, "max_output_tokens": 64000, "mode": "chat", - "output_cost_per_token": 2.501e-06, + "output_cost_per_token": 1.2505e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", "supports_function_calling": true, "supports_prompt_caching": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 83ab91f2bc0..6ffc20e9e71 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -64701,15 +64701,15 @@ "supports_vision": false }, "tencent/hy4": { - "cache_read_input_token_cost": 4.2e-08, - "comment": "TokenHub bills Hy4 at 50% of list until 2026-12-31 23:59:59 Beijing time; these are the list rates.", - "input_cost_per_token": 8.34e-07, - "input_cost_per_token_cache_hit": 4.2e-08, + "cache_read_input_token_cost": 2.1e-08, + "comment": "These are the 50% promotional rates. Update on 2026-12-31 when the promotion ends at 23:59:59 Beijing time. List rates per 1M tokens are $0.834 input, $2.501 output and $0.042 cached input.", + "input_cost_per_token": 4.17e-07, + "input_cost_per_token_cache_hit": 2.1e-08, "litellm_provider": "tencent", "max_input_tokens": 960000, "max_output_tokens": 64000, "mode": "chat", - "output_cost_per_token": 2.501e-06, + "output_cost_per_token": 1.2505e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", "supports_function_calling": true, "supports_prompt_caching": true, @@ -64718,15 +64718,15 @@ "supports_vision": true }, "tencent/hy4-preview": { - "cache_read_input_token_cost": 4.2e-08, - "comment": "TokenHub bills Hy4 preview at 50% of list until 2026-12-31 23:59:59 Beijing time; these are the list rates.", - "input_cost_per_token": 8.34e-07, - "input_cost_per_token_cache_hit": 4.2e-08, + "cache_read_input_token_cost": 2.1e-08, + "comment": "These are the 50% promotional rates. Update on 2026-12-31 when the promotion ends at 23:59:59 Beijing time. List rates per 1M tokens are $0.834 input, $2.501 output and $0.042 cached input.", + "input_cost_per_token": 4.17e-07, + "input_cost_per_token_cache_hit": 2.1e-08, "litellm_provider": "tencent", "max_input_tokens": 960000, "max_output_tokens": 64000, "mode": "chat", - "output_cost_per_token": 2.501e-06, + "output_cost_per_token": 1.2505e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", "supports_function_calling": true, "supports_prompt_caching": true, From 285ddf82e1825438f7d87d96dfac793d73a6e918 Mon Sep 17 00:00:00 2001 From: Nurlan Mammadli <116812760+nurlanmnn@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:32:49 -0500 Subject: [PATCH 054/189] fix(proxy): allow management_routes keys to call PATCH /model/{model_id}/update (#50979) The PATCH model update endpoint was missing from the management_routes group, so virtual keys restricted to management_routes got a 403 on it while the legacy POST /model/update worked. Fixes #50918. Co-authored-by: Claude Opus 5.5 --- litellm/proxy/_types.py | 1 + tests/unit/proxy/auth/test_route_checks.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 736c09277f1..0d40faf55fb 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -713,6 +713,7 @@ class LiteLLMRoutes(enum.Enum): # model "/model/new", "/model/update", + "/model/{model_id}/update", "/model/delete", "/model/info", "/model/{model_id}/kubernetes_pods", diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index c1011310af3..d36e9558c86 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -387,6 +387,25 @@ def test_virtual_key_allowed_routes_with_litellm_routes_member_name_allowed(grou assert result is True +@pytest.mark.parametrize( + "route,method", + [ + ("/model/update", "POST"), + ("/model/123e4567-e89b-12d3-a456-426614174000/update", "PATCH"), + ], +) +def test_virtual_key_management_routes_allows_model_update(route: str, method: str) -> None: + valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=["management_routes"]) + + result: Final = RouteChecks.is_virtual_key_allowed_to_call_route( + route=route, + valid_token=valid_token, + request=_mock_request(method), + ) + + assert result is True + + def test_virtual_key_mcp_routes_allows_v1_mcp_server(): """Regression test for #20325: allow virtual keys to list MCP servers.""" From 7894ec8434424361217d8e5dad79edf8a3a6d7d6 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:35:13 -0800 Subject: [PATCH 055/189] fix(proxy): drop the beta label from the /v1/messages swagger tags (#51046) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/anthropic_endpoints/endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 7e990e3c71b..1688992f6fa 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -128,7 +128,7 @@ def _strip_total_tokens_from_anthropic_response(response: object) -> None: @router.post( "/v1/messages", - tags=["[beta] Anthropic `/v1/messages`"], + tags=["Anthropic `/v1/messages`"], dependencies=[Depends(user_api_key_auth)], ) async def anthropic_response( @@ -319,7 +319,7 @@ def _provider_context_management(response: TokenCountResponse) -> Mapping[str, o @router.post( "/v1/messages/count_tokens", - tags=["[beta] Anthropic Messages Token Counting"], + tags=["Anthropic Messages Token Counting"], dependencies=[Depends(user_api_key_auth)], ) async def count_tokens( From e2625d0509c8e42253849a53bbf1a831dfcaf3bd Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 16:36:10 -0800 Subject: [PATCH 056/189] fix(router): re-check and reserve per-model rate limits on fallback targets (#50994) * fix(router): re-check and reserve per-model rate limits on fallback targets A provider error fallback handed the caller the fallback model group without its per-model TPM/RPM limits ever being checked. The router gets a fallback_rate_limit_check hook next to fallback_budget_check, the proxy wires it to the v3 limiter, which reserves the target model's per-model descriptors before each fallback hop and skips targets that are over their limit. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): dedupe a repeated fallback target before the reserving rate limit check Both walks now consult the attempted set before the rate limit check, so a chain like ["B", "B"] reserves B once. The deprecated settings parity test also learns the new setting. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): a bidirectional fallback chain never re-reserves the requested model group Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): match the surfaced failure in the fallback reservation tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(rate-limit): keep the admission stash untouched while building fallback descriptors Building the fallback target's descriptors could rewrite the request stash's PTU ceiling, which post-call settlement reads for the model the caller named. The fallback check now hides the stash while it builds, like the frame check. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + .../routing_common_utils.py | 14 ++- litellm/proxy/auth/fallback_rate_limit.py | 48 ++++++++ .../common_utils/proxy_rate_limit_error.py | 1 + .../hooks/parallel_request_limiter_v3.py | 22 ++++ litellm/proxy/proxy_server.py | 7 ++ litellm/router.py | 3 + litellm/router_api.py | 10 +- litellm/router_deprecated.py | 1 + litellm/router_rust/core.py | 10 ++ .../router_utils/fallback_event_handlers.py | 59 +++++++-- litellm/types/router.py | 14 +++ .../routing/test_fallback_rate_limit_check.py | 77 ++++++++++++ .../proxy/auth/test_fallback_rate_limit.py | 77 ++++++++++++ .../test_fallback_event_handlers.py | 114 +++++++++++++++++- .../core/test_deprecated_members.py | 1 + 16 files changed, 445 insertions(+), 14 deletions(-) create mode 100644 litellm/proxy/auth/fallback_rate_limit.py create mode 100644 tests/integration/routing/test_fallback_rate_limit_check.py create mode 100644 tests/unit/proxy/auth/test_fallback_rate_limit.py diff --git a/litellm/constants.py b/litellm/constants.py index b29c22810e7..25eb5d9b854 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -89,6 +89,7 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset( "ignore_invalid_deployments", "fallback_access_check", "fallback_budget_check", + "fallback_rate_limit_check", "auto_router_capability_limit", } ) diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index 89d6729766f..154c3942edc 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -299,9 +299,13 @@ def format_fallback_outcome_message( over_budget_targets: Final = tuple( dict.fromkeys(target.target for target in skipped if target.reason is FallbackSkipReason.OVER_BUDGET) ) + rate_limited_targets: Final = tuple( + dict.fromkeys(target.target for target in skipped if target.reason is FallbackSkipReason.RATE_LIMITED) + ) attempted_text: Final = ", ".join(str(mask_sensitive_structure(target)) for target in attempted_targets) not_authorized_text: Final = ", ".join(str(mask_sensitive_structure(target)) for target in not_authorized_targets) over_budget_text: Final = ", ".join(str(mask_sensitive_structure(target)) for target in over_budget_targets) + rate_limited_text: Final = ", ".join(str(mask_sensitive_structure(target)) for target in rate_limited_targets) attempted_failure: Final = ( (f"Fallback to {attempted_text} also failed: {fallback_failure_detail}",) if fallback_failure_detail and attempted_targets @@ -323,7 +327,15 @@ def format_fallback_outcome_message( if over_budget_targets else () ) - sentences: Final = attempted_failure + not_authorized_sentence + over_budget_sentence + rate_limited_sentence: Final = ( + ( + f"Fallback to {rate_limited_text} was skipped because the caller is over its per-model rate limit for " + f"{'it' if len(rate_limited_targets) == 1 else 'them'}.", + ) + if rate_limited_targets + else () + ) + sentences: Final = attempted_failure + not_authorized_sentence + over_budget_sentence + rate_limited_sentence if sentences: return f"{lead} {' '.join(sentences)}" diff --git a/litellm/proxy/auth/fallback_rate_limit.py b/litellm/proxy/auth/fallback_rate_limit.py new file mode 100644 index 00000000000..1d4a33b2994 --- /dev/null +++ b/litellm/proxy/auth/fallback_rate_limit.py @@ -0,0 +1,48 @@ +""" +Re-check per-model rate limits before the router takes a fallback. + +The v3 limiter admits a request once, against the model group the caller asked for. When that +group fails at the provider and the router moves to a fallback group, nothing re-checked the +caller's per-model limits on the fallback, so a provider outage on one model removed the cap on +the other for every caller at once. `RouterFallbackRateLimitCheck` runs the limiter's +model-scoped descriptors for the fallback target and reserves capacity on them, so the router +skips a target the caller has exhausted. +""" + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Final + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.auth.fallback_budget import ( + _user_api_key_auth_from_request, # pyright: ignore[reportPrivateUsage] # shared auth lookup for fallback checks +) +from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + PROXY_MaxParallelRequestsHandler_v3, # pyright: ignore[reportPrivateUsage] # the proxy's limiter type +) +from litellm.router_api import RoutingContext + + +@dataclass(frozen=True, slots=True) +class RouterFallbackRateLimitCheck: + """ + `FallbackRateLimitCheck` for the proxy's router: a fallback target is attempted only when the + caller still has per-model rate limit capacity on it, and the attempt reserves that capacity. + Requests that carry no key (for example internal health checks) and proxies running without + the v3 limiter are not restricted. + """ + + limiter: Callable[[], object] + + async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: RoutingContext) -> bool: + limiter: Final = self.limiter() + if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): + return True + valid_token: Final = _user_api_key_auth_from_request(request_kwargs) + if valid_token is None: + return True + try: + return await limiter.async_fallback_target_allowed(valid_token, model, request_kwargs) + except Exception as e: # noqa: BLE001 # fail closed: a limiter failure must not hand out unchecked capacity + verbose_proxy_logger.warning("Skipping fallback to model=%s: rate limit check failed: %s", model, e) + return False diff --git a/litellm/proxy/common_utils/proxy_rate_limit_error.py b/litellm/proxy/common_utils/proxy_rate_limit_error.py index 02da6f706d1..28d28a5c9bd 100644 --- a/litellm/proxy/common_utils/proxy_rate_limit_error.py +++ b/litellm/proxy/common_utils/proxy_rate_limit_error.py @@ -49,6 +49,7 @@ PER_MODEL_RATE_LIMIT_DESCRIPTOR_KEYS: Final = frozenset( { "model_per_key", "model_per_team", + "model_per_end_user", "model_per_organization", "model_per_project", "model_per_project_itpm", diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index ae9fa548cc9..a3227bca235 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -79,6 +79,7 @@ from litellm.proxy.auth.budget_throttle import throttled_limit from litellm.proxy.common_utils.callback_utils import get_model_group_from_litellm_kwargs from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body from litellm.proxy.common_utils.proxy_rate_limit_error import ( + PER_MODEL_RATE_LIMIT_DESCRIPTOR_KEYS, ProxyRateLimitError, map_v3_rate_limit_type, ) @@ -3777,6 +3778,27 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if response["overall_code"] == "OK": await self._release_parallel_request_slots(acquisition, user_api_key_dict.parent_otel_span) + async def async_fallback_target_allowed( + self, + user_api_key_dict: UserAPIKeyAuth, + model: str, + request_data: Mapping[str, object], + ) -> bool: + """Reserve the caller's per-model limits on a router fallback target. False when any is exhausted.""" + data: Final = MappingProxyType({**request_data, "model": model}) + hidden_stash: Final = _request_stash.set(None) + try: + built: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None) + finally: + _request_stash.reset(hidden_stash) + descriptors: Final = [ + descriptor for descriptor in built if descriptor["key"] in PER_MODEL_RATE_LIMIT_DESCRIPTOR_KEYS + ] + if not descriptors: + return True + response: Final = await self.should_rate_limit(descriptors, parent_otel_span=user_api_key_dict.parent_otel_span) + return response["overall_code"] == "OK" + @asynccontextmanager async def request_capacity( self, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index be90b028cf4..888276f7c31 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -454,6 +454,7 @@ from litellm.proxy.auth.auth_utils import ( ) from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck, router_fallback_budget_check from litellm.proxy.auth.fallback_model_access import router_fallback_access_check +from litellm.proxy.auth.fallback_rate_limit import RouterFallbackRateLimitCheck from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.legacy_empty_models_check import warn_about_legacy_empty_models from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY, LicenseCheck @@ -7656,6 +7657,9 @@ class ProxyConfig: is_enforced=router_fallback_budget_check.is_enforced, team_member_model_budget_limiter=model_max_budget_limiter, ), + fallback_rate_limit_check=RouterFallbackRateLimitCheck( + limiter=lambda: proxy_logging_obj.get_proxy_hook("parallel_request_limiter"), + ), auto_router_capability_limit=_license_check.auto_router_capability_limit, ) ) @@ -8162,6 +8166,9 @@ class ProxyConfig: is_enforced=router_fallback_budget_check.is_enforced, team_member_model_budget_limiter=model_max_budget_limiter, ), + fallback_rate_limit_check=RouterFallbackRateLimitCheck( + limiter=lambda: proxy_logging_obj.get_proxy_hook("parallel_request_limiter"), + ), auto_router_capability_limit=_license_check.auto_router_capability_limit, ) ) diff --git a/litellm/router.py b/litellm/router.py index 0d46326b904..98b016f590f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -34,6 +34,7 @@ from litellm.types.router import ( DeploymentTypedDict, FallbackAccessCheck, FallbackBudgetCheck, + FallbackRateLimitCheck, GuardrailTypedDict, LiteLLM_Params, ModelGroupInfo, @@ -120,6 +121,7 @@ def _router_init( enable_mid_stream_fallback_continuation: bool = False, fallback_access_check: FallbackAccessCheck | None = None, fallback_budget_check: FallbackBudgetCheck | None = None, + fallback_rate_limit_check: FallbackRateLimitCheck | None = None, auto_router_capability_limit: AutoRouterCapabilityLimit | None = None, session_affinity_repin_on_failover: bool = False, _host: PythonHost = python_host, @@ -166,6 +168,7 @@ def _router_init( enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False. fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted). fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback). + fallback_rate_limit_check (Optional[FallbackRateLimitCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over its per-model rate limit is skipped. Defaults to None (rate limits are not re-checked on fallback). Returns: Router: An instance of the litellm.Router class. diff --git a/litellm/router_api.py b/litellm/router_api.py index c85a764151f..a75397ea43d 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -80,6 +80,7 @@ if TYPE_CHECKING: DeploymentTypedDict, FallbackAccessCheck, FallbackBudgetCheck, + FallbackRateLimitCheck, GuardrailTypedDict, ModelGroupInfo, PreRoutingHookResponse, @@ -273,6 +274,7 @@ ROUTER_INIT_ARGS: Final[tuple[str, ...]] = ( "enable_mid_stream_fallback_continuation", "fallback_access_check", "fallback_budget_check", + "fallback_rate_limit_check", "auto_router_capability_limit", "session_affinity_repin_on_failover", ) @@ -1204,6 +1206,7 @@ SETTING_FIELDS: Final[tuple[str, ...]] = ( "provider_default_deployment_ids", "fallback_access_check", "fallback_budget_check", + "fallback_rate_limit_check", "kubernetes_pod_discovery", "search_tools", "guardrail_list", @@ -1264,6 +1267,7 @@ class RouterSettings: provider_default_deployment_ids: list[str] = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) # mutable-ok: the router's live list fallback_access_check: "FallbackAccessCheck | None" = None fallback_budget_check: "FallbackBudgetCheck | None" = None + fallback_rate_limit_check: "FallbackRateLimitCheck | None" = None kubernetes_pod_discovery: "KubernetesPodDiscovery" = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) search_tools: "list[SearchToolTypedDict]" = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) # mutable-ok: the router's live list guardrail_list: "list[GuardrailTypedDict]" = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) # mutable-ok: the router's live list @@ -1522,7 +1526,7 @@ def _takes_legacy_llm_router(check: object) -> bool: async def _run_fallback_check( - check: "FallbackAccessCheck | FallbackBudgetCheck", + check: "FallbackAccessCheck | FallbackBudgetCheck | FallbackRateLimitCheck", ctx: "RoutingContext", *, model: str, @@ -1666,6 +1670,10 @@ class RoutingContext: check: Final = self._router.settings().fallback_budget_check return check is None or await _run_fallback_check(check, self, model=model, request_kwargs=request_kwargs) + async def fallback_rate_limit_allowed(self, *, model: str, request_kwargs: Mapping[str, object]) -> bool: + check: Final = self._router.settings().fallback_rate_limit_check + return check is None or await _run_fallback_check(check, self, model=model, request_kwargs=request_kwargs) + def pre_call_checks( self, model: str, diff --git a/litellm/router_deprecated.py b/litellm/router_deprecated.py index 618ac4321d8..2cf111da2a5 100644 --- a/litellm/router_deprecated.py +++ b/litellm/router_deprecated.py @@ -345,6 +345,7 @@ class DeprecatedRouterMembers: ) fallback_access_check = _setting("fallback_access_check", lambda r: r.settings().fallback_access_check) fallback_budget_check = _setting("fallback_budget_check", lambda r: r.settings().fallback_budget_check) + fallback_rate_limit_check = _setting("fallback_rate_limit_check", lambda r: r.settings().fallback_rate_limit_check) # These three are declared with the attribute type Router gave them rather than the descriptor's: # protocols in router_utils and the proxy match Router on them, and protocol matching does not # see through a descriptor. Reads and writes still go through _setting at runtime. diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index 682aa557007..aef4db05ca9 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -330,6 +330,7 @@ _SERVED_ARGUMENTS: Final = frozenset( "default_litellm_params", "fallback_access_check", "fallback_budget_check", + "fallback_rate_limit_check", "enable_mid_stream_fallback_continuation", "redis_url", "redis_host", @@ -418,6 +419,7 @@ _DYNAMIC_SETTINGS: Final = frozenset( "default_litellm_params", "fallback_access_check", "fallback_budget_check", + "fallback_rate_limit_check", "routing_strategy_args", "enable_tag_filtering", "tag_filtering_match_any", @@ -664,6 +666,7 @@ class Resolved: default_litellm_params: Mapping[str, object] fallback_access_check: object fallback_budget_check: object + fallback_rate_limit_check: object enable_mid_stream_fallback_continuation: bool = False routing_groups: "tuple[RoutingGroup, ...]" = () enable_tag_filtering: bool = False @@ -779,6 +782,7 @@ class Resolved: default_litellm_params=MappingProxyType(dict(as_str_mapping(get("default_litellm_params")) or {})), fallback_access_check=get("fallback_access_check"), fallback_budget_check=get("fallback_budget_check"), + fallback_rate_limit_check=get("fallback_rate_limit_check"), enable_mid_stream_fallback_continuation=continuation_flag(get("enable_mid_stream_fallback_continuation")), enable_tag_filtering=bool(get("enable_tag_filtering")), tag_filtering_match_any=_true_unless_given(get("tag_filtering_match_any")), @@ -1911,6 +1915,7 @@ class RustCore( tag_routing_prefix=r.read("tag_routing_prefix"), # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it fallback_access_check=r.fallback_access_check, # pyright: ignore[reportArgumentType] # the check as passed fallback_budget_check=r.fallback_budget_check, # pyright: ignore[reportArgumentType] # the check as passed + fallback_rate_limit_check=r.fallback_rate_limit_check, # pyright: ignore[reportArgumentType] # the check as passed cache_responses=self._endpoints.cache_responses, search_tools=self._endpoints.search_tools, # pyright: ignore[reportArgumentType] # the configured entries, as passed guardrail_list=self._endpoints.guardrail_list, # pyright: ignore[reportArgumentType] # same @@ -5414,6 +5419,11 @@ class _Driver: if not await ctx.fallback_budget_allowed(model=target, request_kwargs=request): self._walk.targets().record_skip(target, FallbackSkipReason.OVER_BUDGET) return False + if target not in self._walk.targets() and not await ctx.fallback_rate_limit_allowed( + model=target, request_kwargs=request + ): + self._walk.targets().record_skip(target, FallbackSkipReason.RATE_LIMITED) + return False except Exception as error: # noqa: BLE001 # handed to the Rust call loop, which ends the chain self._level_errors[self._level + 1] = error return {**attempt_failure(error), "raised": True} diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 068782f7a08..88687269d4b 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -753,6 +753,47 @@ async def _is_fallback_target_within_budget( return False +async def _is_fallback_target_admitted( + ctx: "RoutingContext", + fallback_entry: str | Mapping[str, object], + original_model_group: str, + kwargs: Mapping[str, object], + *, + attempted: AttemptedFallbackTargets, +) -> bool: + return ( + await _is_fallback_target_authorized(ctx, fallback_entry, original_model_group, kwargs, attempted=attempted) + and await _is_fallback_target_within_budget( + ctx, fallback_entry, original_model_group, kwargs, attempted=attempted + ) + and await _is_fallback_target_within_rate_limit( + ctx, fallback_entry, original_model_group, kwargs, attempted=attempted + ) + ) + + +async def _is_fallback_target_within_rate_limit( + ctx: "RoutingContext", + fallback_entry: str | Mapping[str, object], + original_model_group: str, + kwargs: Mapping[str, object], + *, + attempted: AttemptedFallbackTargets, +) -> bool: + rate_limit_check: Final = ctx.settings().fallback_rate_limit_check + target: Final = _get_fallback_target_model_group(fallback_entry) + if rate_limit_check is None or target is None or target == original_model_group: + return True + if await ctx.fallback_rate_limit_allowed(model=target, request_kwargs=kwargs): + return True + verbose_router_logger.info( + "Skipping fallback to model_group = %s: caller is over its per-model rate limit", + mask_sensitive_structure(fallback_entry), + ) + attempted.record_skip(target, FallbackSkipReason.RATE_LIMITED) + return False + + def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: """ True when a file, batch, or fine-tuning job operation names an id that only exists @@ -863,18 +904,16 @@ async def run_async_fallback( original_model_group, ) continue - if not await _is_fallback_target_authorized(ctx, mg, original_model_group, kwargs, attempted=attempted): - continue - if not await _is_fallback_target_within_budget(ctx, mg, original_model_group, kwargs, attempted=attempted): - continue attempt_key = fallback_attempt_key(mg) + if attempt_key is not None and attempt_key in attempted: + verbose_router_logger.info( + "Skipping fallback to model_group = %s, already attempted for this request", + mask_sensitive_structure(mg), + ) + continue + if not await _is_fallback_target_admitted(ctx, mg, original_model_group, kwargs, attempted=attempted): + continue if attempt_key is not None: - if attempt_key in attempted: - verbose_router_logger.info( - "Skipping fallback to model_group = %s, already attempted for this request", - mask_sensitive_structure(mg), - ) - continue attempted.record(attempt_key) try: # LOGGING diff --git a/litellm/types/router.py b/litellm/types/router.py index 6f5652db771..bc715759c45 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -89,6 +89,7 @@ class SilentExperimentFlag(TypedDict, total=False): class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" + RATE_LIMITED = "rate_limited" @dataclass(frozen=True, slots=True) @@ -1352,6 +1353,19 @@ class FallbackBudgetCheck(Protocol): async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: "RouterContext") -> bool: ... +class FallbackRateLimitCheck(Protocol): + """ + Decides whether the caller behind `request_kwargs` still has per-model rate limit capacity on fallback `model`. + + The limiter admits a request once, against the *requested* model group, so a provider failure would hand + the caller a fallback group whose per-model limits were never checked. The router runs this before every + cross-model-group fallback attempt, skips rejected targets, and an implementation reserves capacity when + it admits so concurrent failovers cannot all pass. + """ + + async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: "RouterContext") -> bool: ... + + class AutoRouterCapabilityLimit(Protocol): """ Resolves how many complexity routers may claim each licensed capability right now; None means unlimited. diff --git a/tests/integration/routing/test_fallback_rate_limit_check.py b/tests/integration/routing/test_fallback_rate_limit_check.py new file mode 100644 index 00000000000..e7db24f5464 --- /dev/null +++ b/tests/integration/routing/test_fallback_rate_limit_check.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from contextlib import ExitStack +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.routing.test_payment_required_mapping import is_model_info_probe, model_list_reply +from integration.routing.test_per_model_rate_limit_fallbacks import ( + assert_served, + chat, + error_message, + healthy_reply, + model_entry, + proxy_config, +) + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + primary: str + fallback: str + fallback_wire: Wire + + +def provider_error_reply(request: Request) -> Reply: + if is_model_info_probe(request): + return model_list_reply() + return Reply(status=500, body=b'{"error": {"message": "scripted provider outage", "type": "server_error"}}') + + +def served_requests(wire: Wire) -> int: + return sum(1 for request in wire.drain() if not is_model_info_probe(request)) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + suffix: Final = uuid.uuid4().hex[:10] + primary: Final = f"fallback-check-primary-{suffix}" + fallback: Final = f"fallback-check-fallback-{suffix}" + directory: Final = tmp_path_factory.mktemp("fallback-rate-limit-check") + with ExitStack() as stack: + rig_gateway: Final = stack.enter_context(gateway_from_environment()) + primary_wire: Final = stack.enter_context(wire_server(provider_error_reply)) + fallback_wire: Final = stack.enter_context(wire_server(healthy_reply)) + config: Final = proxy_config( + directory, + model_list=(model_entry(primary, primary_wire), model_entry(fallback, fallback_wire)), + fallbacks=({primary: [fallback]},), + general_settings={}, + ) + owned: Final = stack.enter_context(owned_proxy_process(rig_gateway, directory, {}, config=config, workers=2)) + yield Rig(owned.gateway, primary, fallback, fallback_wire) + + +def test_provider_error_fallback_honors_the_keys_per_model_rpm_limit_on_the_fallback_model(rig: Rig) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(model_rpm_limit={rig.fallback: 1}) + served_requests(rig.fallback_wire) + + first: Final = chat(rig.gateway, key, rig.primary, f"pmrl-{uuid.uuid4().hex}") + assert_served(first, rig.fallback, "served pmrl-") + + second: Final = chat(rig.gateway, key, rig.primary, f"pmrl-{uuid.uuid4().hex}") + assert second.status_code == 500, (second.status_code, second.text) + assert f"Fallback to {rig.fallback} was skipped because the caller is over its per-model rate limit" in ( + error_message(second.text) + ), second.text + assert served_requests(rig.fallback_wire) == 1 diff --git a/tests/unit/proxy/auth/test_fallback_rate_limit.py b/tests/unit/proxy/auth/test_fallback_rate_limit.py new file mode 100644 index 00000000000..b1173a758cd --- /dev/null +++ b/tests/unit/proxy/auth/test_fallback_rate_limit.py @@ -0,0 +1,77 @@ +from typing import Final + +import pytest + +from litellm import Router +from litellm.caching.dual_cache import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.fallback_rate_limit import RouterFallbackRateLimitCheck +from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 +from litellm.proxy.utils import InternalUsageCache +from litellm.router_api import RoutingContext + +PRIMARY: Final = {"model_name": "primary", "litellm_params": {"model": "openai/gpt-4o", "api_key": "k"}} +FALLBACK: Final = {"model_name": "fallback", "litellm_params": {"model": "openai/gpt-4o", "api_key": "k"}} + + +def _router() -> Router: + return Router(model_list=[PRIMARY, FALLBACK], fallbacks=[{"primary": ["fallback"]}]) + + +def _limiter() -> PROXY_MaxParallelRequestsHandler_v3: + return PROXY_MaxParallelRequestsHandler_v3(internal_usage_cache=InternalUsageCache(DualCache())) + + +def _request(token: UserAPIKeyAuth) -> dict[str, object]: + return { + "model": "primary", + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"user_api_key_auth": token}, + } + + +@pytest.mark.asyncio +async def test_fallback_target_is_admitted_until_the_keys_per_model_rpm_limit_is_spent() -> None: + limiter: Final = _limiter() + check: Final = RouterFallbackRateLimitCheck(limiter=lambda: limiter) + token: Final = UserAPIKeyAuth(api_key="hashed", token="hashed", metadata={"model_rpm_limit": {"fallback": 2}}) + ctx: Final = RoutingContext(_router()) + verdicts: Final = tuple([await check(model="fallback", request_kwargs=_request(token), ctx=ctx) for _ in range(3)]) + assert verdicts == (True, True, False) + + +@pytest.mark.asyncio +async def test_fallback_target_without_a_per_model_limit_is_always_admitted() -> None: + limiter: Final = _limiter() + check: Final = RouterFallbackRateLimitCheck(limiter=lambda: limiter) + token: Final = UserAPIKeyAuth(api_key="hashed", token="hashed", metadata={"model_rpm_limit": {"other": 1}}) + ctx: Final = RoutingContext(_router()) + verdicts: Final = tuple([await check(model="fallback", request_kwargs=_request(token), ctx=ctx) for _ in range(3)]) + assert verdicts == (True, True, True) + + +@pytest.mark.asyncio +async def test_request_without_a_key_is_unrestricted() -> None: + check: Final = RouterFallbackRateLimitCheck(limiter=_limiter) + assert await check(model="fallback", request_kwargs={"model": "primary"}, ctx=RoutingContext(_router())) is True + + +@pytest.mark.asyncio +async def test_proxy_without_the_v3_limiter_is_unrestricted() -> None: + check: Final = RouterFallbackRateLimitCheck(limiter=lambda: None) + token: Final = UserAPIKeyAuth(api_key="hashed", token="hashed", metadata={"model_rpm_limit": {"fallback": 0}}) + assert await check(model="fallback", request_kwargs=_request(token), ctx=RoutingContext(_router())) is True + + +@pytest.mark.asyncio +async def test_a_failing_limiter_fails_closed() -> None: + class BrokenLimiter(PROXY_MaxParallelRequestsHandler_v3): + async def async_fallback_target_allowed( + self, user_api_key_dict: object, model: str, request_data: object + ) -> bool: + raise RuntimeError("redis down") + + limiter: Final = BrokenLimiter(internal_usage_cache=InternalUsageCache(DualCache())) + check: Final = RouterFallbackRateLimitCheck(limiter=lambda: limiter) + token: Final = UserAPIKeyAuth(api_key="hashed", token="hashed", metadata={"model_rpm_limit": {"fallback": 5}}) + assert await check(model="fallback", request_kwargs=_request(token), ctx=RoutingContext(_router())) is False diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 6263292fb40..9c034b77fe7 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -5,7 +5,7 @@ import time from collections.abc import AsyncIterator, Mapping from datetime import datetime, timedelta from types import MappingProxyType -from typing import Any, Final, Literal, NoReturn +from typing import Any, Final, NoReturn from unittest.mock import MagicMock, patch import httpx @@ -20,7 +20,6 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.router_api import RouterSettings, RoutingContext from litellm.router_utils.cooldown_handlers import async_get_cooldown_deployments, mark_advisor_orchestration_failure -from litellm.types.utils import Choices, Message, ModelResponse, StandardLoggingPayload, Usage from litellm.router_utils.fallback_event_handlers import ( MID_STREAM_FALLBACK_CONTROLS_KEY, PRE_ROUTING_SELECTED_MODEL_KEY, @@ -45,6 +44,7 @@ from litellm.router_utils.fallback_event_handlers import ( run_async_fallback, ) from litellm.types.router import FallbackSkipReason, SkippedFallbackTarget +from litellm.types.utils import Choices, Message, ModelResponse, StandardLoggingPayload, Usage from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -2501,3 +2501,113 @@ def test_is_bad_request_without_fallback_only_stops_on_a_real_400_when_the_flag_ ) -> None: assert is_bad_request_without_fallback(exception, fallback_on_unsupported_params=False) is stops_fallbacks assert is_bad_request_without_fallback(exception, fallback_on_unsupported_params=True) is False + + +@pytest.mark.asyncio +async def test_run_async_fallback_skips_a_target_the_rate_limit_check_rejects() -> None: + checked: Final = [] # mutable-ok: records the targets the check saw + + async def refuse(*, model: str, request_kwargs: Mapping[str, object], ctx: object) -> bool: + checked.append(model) + return False + + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "http://127.0.0.1:9"}, + }, + { + "model_name": "fallback", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "mock_response": "served"}, + }, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + fallback_rate_limit_check=refuse, + ) + with pytest.raises( + Exception, match="Fallback to fallback was skipped because the caller is over its per-model rate limit" + ): + await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}]) + assert checked == ["fallback"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_serves_a_target_the_rate_limit_check_admits() -> None: + async def admit(*, model: str, request_kwargs: Mapping[str, object], ctx: object) -> bool: + return True + + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "http://127.0.0.1:9"}, + }, + { + "model_name": "fallback", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "mock_response": "served"}, + }, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + fallback_rate_limit_check=admit, + ) + response: Final = await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}]) + assert response.choices[0].message.content == "served" + + +@pytest.mark.asyncio +async def test_run_async_fallback_reserves_a_repeated_target_once() -> None: + checked: Final = [] # mutable-ok: records the targets the check saw + + async def admit(*, model: str, request_kwargs: Mapping[str, object], ctx: object) -> bool: + checked.append(model) + return True + + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "http://127.0.0.1:9"}, + }, + { + "model_name": "fallback", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "http://127.0.0.1:9"}, + }, + ], + fallbacks=[{"primary": ["fallback", "fallback"]}], + num_retries=0, + fallback_rate_limit_check=admit, + ) + with pytest.raises(Exception, match="model group 'primary' failed"): + await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}]) + assert checked == ["fallback"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_never_re_reserves_the_requested_model_group() -> None: + checked: Final = [] # mutable-ok: records the targets the check saw + + async def admit(*, model: str, request_kwargs: Mapping[str, object], ctx: object) -> bool: + checked.append(model) + return True + + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "http://127.0.0.1:9"}, + }, + { + "model_name": "fallback", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "http://127.0.0.1:9"}, + }, + ], + fallbacks=[{"primary": ["fallback"]}, {"fallback": ["primary"]}], + num_retries=0, + fallback_rate_limit_check=admit, + ) + with pytest.raises(Exception, match="model group 'primary' failed"): + await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}]) + assert checked == ["fallback"] diff --git a/tests/unit/test_router/core/test_deprecated_members.py b/tests/unit/test_router/core/test_deprecated_members.py index 01cc6b3f6d9..7d5f634cda5 100644 --- a/tests/unit/test_router/core/test_deprecated_members.py +++ b/tests/unit/test_router/core/test_deprecated_members.py @@ -103,6 +103,7 @@ _CONFIGURED_SETTINGS: Final[dict[str, object]] = { # mutable-ok: expected value "provider_default_deployment_ids": ["w"], "fallback_access_check": None, "fallback_budget_check": None, + "fallback_rate_limit_check": None, "search_tools": [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily"}}], "guardrail_list": [{"guardrail_name": "g", "litellm_params": {"guardrail": "x"}}], "cache_responses": True, From 2605aa1c841bdd820febb3d357a8db204090c035 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:39:17 +0000 Subject: [PATCH 057/189] chore(cost-map): add 61 fal_ai image and video rows priced from the fal.ai model pages (second fal-ai sync) (#51077) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 971 ++++++++++++++++++ model_prices_and_context_window.json | 971 ++++++++++++++++++ tests/unit/test_utils.py | 1 + 3 files changed, 1943 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6ffc20e9e71..68b77e3b785 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -97061,5 +97061,976 @@ "supported_output_modalities": [ "video" ] + }, + "fal_ai/blackforestlabs/flux-3/edit-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"the page states only the 1K (1 MP) price of $0.048 and the pricing API bills in megapixels at $0.048, so the per-pixel rate is $0.048 per megapixel and the per-image price is the default 1k output; 768sq, 1.5k, 2k and 4k outputs are priced by the per-pixel rate"}, + "mode": "image_generation", + "output_cost_per_image": 0.048, + "output_cost_per_pixel": 4.57763671875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/edit-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/blackforestlabs/flux-3/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; the pricing API reports a $0.085 per second base price that the page does not state"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_2k": 0.32, + "output_cost_per_second_4k": 0.47, + "output_cost_per_second_720p": 0.17, + "source": "https://fal.ai/models/blackforestlabs/flux-3/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/klein/edit-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per output image by resolution; output_cost_per_image is the 1K rate (image_size defaults to the input size, 1K is the page example); 768 x 768 is $0.008 and 1.5K is $0.011 per image, no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.009, + "output_cost_per_image_1K": 0.009, + "output_cost_per_image_2K": 0.014, + "output_cost_per_image_4K": 0.036, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/edit-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/blackforestlabs/flux-3/klein/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per output image by resolution; output_cost_per_image is the 1K rate (default image_size square_hd); 768 x 768 is $0.008 and 1.5K is $0.011 per image, no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.009, + "output_cost_per_image_1K": 0.009, + "output_cost_per_image_2K": 0.014, + "output_cost_per_image_4K": 0.036, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/blackforestlabs/flux-3/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"the page states only the 1K (1 MP) price of $0.048 and the pricing API bills in megapixels at $0.048, so the per-pixel rate is $0.048 per megapixel and the per-image price is the default 1k output; 768sq, 1.5k, 2k and 4k outputs are priced by the per-pixel rate"}, + "mode": "image_generation", + "output_cost_per_image": 0.048, + "output_cost_per_pixel": 4.57763671875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/blackforestlabs/flux-3/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; the pricing API reports a $0.085 per second base price that the page does not state"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_2k": 0.32, + "output_cost_per_second_4k": 0.47, + "output_cost_per_second_720p": 0.17, + "source": "https://fal.ai/models/blackforestlabs/flux-3/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-video-upscale": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution in the default creative mode (creativity=1); output_cost_per_second is the highest tier (4K) because the output resolution depends on the source and upscale_factor; precise mode (creativity=0) is $0.084 (1080p), $0.15 (2K) and $0.33 (4K) per second and has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.474, + "output_cost_per_second_1080p": 0.12, + "output_cost_per_second_2k": 0.21, + "output_cost_per_second_4k": 0.474, + "source": "https://fal.ai/models/blackforestlabs/flux-video-upscale", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/fibo-2/generate": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image at the default 1MP output; the 4MP output is $0.06 per image and has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-2/generate", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bytedance/seedance-2.0/fast/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video at the default resolution 720p, the only per-second rate the page states; fal bills tokens at $0.0112 per 1000 tokens, counts input video duration, and multiplies by 0.6 with video inputs"}, + "mode": "video_generation", + "output_cost_per_second": 0.2419, + "output_cost_per_second_720p": 0.2419, + "source": "https://fal.ai/models/bytedance/seedance-2.0/fast/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/us/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; 1080p and 4k are not priced on the page; fal bills tokens at $0.0168 per 1000 tokens, counts input video duration, and multiplies by 0.6 with video inputs"}, + "mode": "video_generation", + "output_cost_per_second": 0.37, + "output_cost_per_second_480p": 0.1731, + "output_cost_per_second_720p": 0.37, + "source": "https://fal.ai/models/bytedance/seedance-2.0/us/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.5/us/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; with video references fal bills input and output duration and multiplies by 0.6"}, + "mode": "video_generation", + "output_cost_per_second": 0.473, + "output_cost_per_second_1080p": 1.164, + "output_cost_per_second_480p": 0.2205, + "output_cost_per_second_720p": 0.473, + "source": "https://fal.ai/models/bytedance/seedance-2.5/us/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/bria/background/remove": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image without a prompt (prompt is optional); $0.03 per image when a prompt is given has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.018, + "source": "https://fal.ai/models/fal-ai/bria/background/remove", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bytedance-upscaler/upscale/video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by target_resolution at 30fps in normal mode; output_cost_per_second is the default target_resolution 1080p rate; 6k $0.0574 and 8k $0.1148 per second have no catalog field; 30-60fps doubles, over 60fps quadruples and pro mode multiplies by 10, none of which is modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.0072, + "output_cost_per_second_1080p": 0.0072, + "output_cost_per_second_2k": 0.0144, + "output_cost_per_second_4k": 0.0288, + "source": "https://fal.ai/models/fal-ai/bytedance-upscaler/upscale/video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/emu-3.5-image/edit-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default 720p resolution; 480p is $0.15 per image and has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://fal.ai/models/fal-ai/emu-3.5-image/edit-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/emu-3.5-image/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default 720p resolution; 480p is $0.15 per image and has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://fal.ai/models/fal-ai/emu-3.5-image/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/kling-video/v2.5-turbo/pro/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.5-turbo/pro/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v2.5-turbo/pro/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.5-turbo/pro/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v2.5-turbo/standard/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.042, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.5-turbo/standard/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/hailuo-02/standard/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768P rate; 512P is roughly $0.017 per second and has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.045, + "output_cost_per_second_768p": 0.045, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-02/standard/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/image-01": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/fal-ai/minimax/image-01", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/object-removal": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default model best_quality; low_quality $0.006, medium_quality $0.012 and high_quality $0.018 per image have no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.024, + "source": "https://fal.ai/models/fal-ai/object-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/object-removal/bbox": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default model best_quality; low_quality $0.006, medium_quality $0.012 and high_quality $0.018 per image have no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.024, + "source": "https://fal.ai/models/fal-ai/object-removal/bbox", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/object-removal/mask": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default model best_quality; low_quality $0.006, medium_quality $0.012 and high_quality $0.018 per image have no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.024, + "source": "https://fal.ai/models/fal-ai/object-removal/mask", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/recraft/v3/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default realistic_image style; vector styles are $0.08 per image and have no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/recraft/v3/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/scail-2": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/scail-2", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-apps/long-reframe": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution, converted from the page rate per 2 seconds ($0.08 per 2s at 720p, $0.04 per 2s at 480p); output_cost_per_second is the 720p rate because the default resolution is auto; 580p ($0.06 per 2s) has no catalog field, 240p and 360p are not priced on the page; seconds are rounded up to a multiple of 2"}, + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_720p": 0.04, + "source": "https://fal.ai/models/fal-ai/wan-vace-apps/long-reframe", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-14b/animate/move": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per video second of output by resolution, where a video second is 16 frames; output_cost_per_second is the default resolution 480p rate; 580p is $0.06 per video second and has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-14b/animate/move", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-14b/animate/replace": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per video second of output by resolution, where a video second is 16 frames; output_cost_per_second is the default resolution 480p rate; 580p is $0.06 per video second and has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-14b/animate/replace", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.2/edit": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; 360p is $0.0273 per second and has no catalog field; input tokens (text, image, video) are billed at $1.50 per million"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.1229, + "output_cost_per_second_4k": 0.2457, + "output_cost_per_second_720p": 0.0819, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.2/extend": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; 360p is $0.0273 per second and has no catalog field; input tokens (text, image, video) are billed at $1.50 per million"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.1229, + "output_cost_per_second_4k": 0.2457, + "output_cost_per_second_720p": 0.0819, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/extend", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.2/image-to-video": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; 360p is $0.0273 per second and has no catalog field; input tokens (text, image, video) are billed at $1.50 per million"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.1229, + "output_cost_per_second_4k": 0.2457, + "output_cost_per_second_720p": 0.0819, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.2/reference-to-video": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; 360p is $0.0273 per second and has no catalog field; input tokens (text, image, video) are billed at $1.50 per million"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.1229, + "output_cost_per_second_4k": 0.2457, + "output_cost_per_second_720p": 0.0819, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.2/text-to-video": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; 360p is $0.0273 per second and has no catalog field; input tokens (text, image, video) are billed at $1.50 per million"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.1229, + "output_cost_per_second_4k": 0.2457, + "output_cost_per_second_720p": 0.0819, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/virtual-try-on": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://fal.ai/models/google/virtual-try-on", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/ideogram/v4/tiling": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per megapixel at the default rendering_speed BALANCED; TURBO is $0.03 and QUALITY is $0.10 per megapixel, no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.06, + "output_cost_per_pixel": 5.7220458984375e-08, + "source": "https://fal.ai/models/ideogram/v4/tiling", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/ideogram/v4/tiling/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per megapixel at the default rendering_speed BALANCED; TURBO is $0.03375 and QUALITY is $0.1125 per megapixel, no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.0675, + "output_cost_per_pixel": 6.437301635742188e-08, + "source": "https://fal.ai/models/ideogram/v4/tiling/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/kling/v4/flash/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video at 720p (the only resolution) with image references; with video references fal bills $0.063 per second on the output duration plus the greater of the input and output durations, which has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.094, + "output_cost_per_second_720p": 0.094, + "source": "https://fal.ai/models/kling/v4/flash/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution with image references; output_cost_per_second is the default resolution 1080p rate; with video references or video elements fal bills $0.094 (720p) or $0.141 (1080p) per second on the input plus output length, which has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.235, + "output_cost_per_second_1080p": 0.235, + "output_cost_per_second_720p": 0.157, + "source": "https://fal.ai/models/kling/v4/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/luma/agent/ray/v3.2/reframe": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per started source second by resolution; output_cost_per_second is the default resolution 540p rate ($0.06), which has no tier field"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.36, + "output_cost_per_second_720p": 0.12, + "source": "https://fal.ai/models/luma/agent/ray/v3.2/reframe", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of requested output duration by resolution; output_cost_per_second is the default resolution 768P rate; the page states no promotion for this endpoint; reference tokens above the 4,096 included per request are $0.02 per 1,000 tokens and are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.16, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/styles/16bit-pixel": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/styles/16bit-pixel", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/styles/hand-drawn": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/styles/hand-drawn", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/styles/low-poly": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/styles/low-poly", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/styles/retro-toon-70s": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/styles/retro-toon-70s", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/styles/vhs": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/styles/vhs", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/mirelo-ai/sfx2.0/video-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated audio per sample; promotional 33% off rate through 2026-12-13 per the page; the standard rate of $0.015 per second applies from 2026-12-14 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10); mode and endpoint follow fal_ai/sonilo/v1.1/video-to-video-sound-effects"}, + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/mirelo-ai/sfx2.0/video-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/nvidia/cosmos-3-super/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image; the optional $0.02 per request prompt expansion surcharge is not modeled, and agentic generation bills every candidate image"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/nvidia/cosmos-3-super/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/recraft/v4/style/pro/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image when a style_id is provided; the one-time $0.005 style creation surcharge when no style_id is given is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/recraft/v4/style/pro/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/recraft/v4/style/pro/text-to-vector": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image when a style_id is provided; the one-time $0.005 style creation surcharge when no style_id is given is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.12, + "source": "https://fal.ai/models/recraft/v4/style/pro/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/recraft/v4/style/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image when a style_id is provided; the one-time $0.005 style creation surcharge when no style_id is given is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/recraft/v4/style/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/recraft/v4/style/text-to-vector": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image when a style_id is provided; the one-time $0.005 style creation surcharge when no style_id is given is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/recraft/v4/style/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/vast/eden-live-14b-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of video generated in a live session; the 10 billable second ($0.10) minimum per session is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/vast/eden-live-14b-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/vast/eden-live-5b-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of video generated in a live session; the 10 billable second ($0.035) minimum per session is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.0035, + "source": "https://fal.ai/models/vast/eden-live-5b-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-image/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.022, + "source": "https://fal.ai/models/xai/grok-imagine-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/xai/grok-imagine-video/edit-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the 720p rate because the default resolution is auto; the $0.01 per second of input video charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://fal.ai/models/xai/grok-imagine-video/edit-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/extend-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the 720p rate because the endpoint has no resolution field; the $0.01 per second of input video charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://fal.ai/models/xai/grok-imagine-video/extend-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; the $0.002 per request image input charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://fal.ai/models/xai/grok-imagine-video/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 480p rate; the $0.002 per request image input charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://fal.ai/models/xai/grok-imagine-video/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://fal.ai/models/xai/grok-imagine-video/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/v1.5/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; the $0.01 per reference image charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.14, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/xai/grok-imagine-video/v1.5/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/v1.5/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 480p rate; the $0.01 per reference image charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/xai/grok-imagine-video/v1.5/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6ffc20e9e71..68b77e3b785 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -97061,5 +97061,976 @@ "supported_output_modalities": [ "video" ] + }, + "fal_ai/blackforestlabs/flux-3/edit-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"the page states only the 1K (1 MP) price of $0.048 and the pricing API bills in megapixels at $0.048, so the per-pixel rate is $0.048 per megapixel and the per-image price is the default 1k output; 768sq, 1.5k, 2k and 4k outputs are priced by the per-pixel rate"}, + "mode": "image_generation", + "output_cost_per_image": 0.048, + "output_cost_per_pixel": 4.57763671875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/edit-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/blackforestlabs/flux-3/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; the pricing API reports a $0.085 per second base price that the page does not state"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_2k": 0.32, + "output_cost_per_second_4k": 0.47, + "output_cost_per_second_720p": 0.17, + "source": "https://fal.ai/models/blackforestlabs/flux-3/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-3/klein/edit-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per output image by resolution; output_cost_per_image is the 1K rate (image_size defaults to the input size, 1K is the page example); 768 x 768 is $0.008 and 1.5K is $0.011 per image, no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.009, + "output_cost_per_image_1K": 0.009, + "output_cost_per_image_2K": 0.014, + "output_cost_per_image_4K": 0.036, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/edit-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/blackforestlabs/flux-3/klein/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per output image by resolution; output_cost_per_image is the 1K rate (default image_size square_hd); 768 x 768 is $0.008 and 1.5K is $0.011 per image, no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.009, + "output_cost_per_image_1K": 0.009, + "output_cost_per_image_2K": 0.014, + "output_cost_per_image_4K": 0.036, + "source": "https://fal.ai/models/blackforestlabs/flux-3/klein/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/blackforestlabs/flux-3/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"the page states only the 1K (1 MP) price of $0.048 and the pricing API bills in megapixels at $0.048, so the per-pixel rate is $0.048 per megapixel and the per-image price is the default 1k output; 768sq, 1.5k, 2k and 4k outputs are priced by the per-pixel rate"}, + "mode": "image_generation", + "output_cost_per_image": 0.048, + "output_cost_per_pixel": 4.57763671875e-08, + "source": "https://fal.ai/models/blackforestlabs/flux-3/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/blackforestlabs/flux-3/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; the pricing API reports a $0.085 per second base price that the page does not state"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_2k": 0.32, + "output_cost_per_second_4k": 0.47, + "output_cost_per_second_720p": 0.17, + "source": "https://fal.ai/models/blackforestlabs/flux-3/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/blackforestlabs/flux-video-upscale": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution in the default creative mode (creativity=1); output_cost_per_second is the highest tier (4K) because the output resolution depends on the source and upscale_factor; precise mode (creativity=0) is $0.084 (1080p), $0.15 (2K) and $0.33 (4K) per second and has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.474, + "output_cost_per_second_1080p": 0.12, + "output_cost_per_second_2k": 0.21, + "output_cost_per_second_4k": 0.474, + "source": "https://fal.ai/models/blackforestlabs/flux-video-upscale", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bria/fibo-2/generate": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image at the default 1MP output; the 4MP output is $0.06 per image and has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/bria/fibo-2/generate", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bytedance/seedance-2.0/fast/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video at the default resolution 720p, the only per-second rate the page states; fal bills tokens at $0.0112 per 1000 tokens, counts input video duration, and multiplies by 0.6 with video inputs"}, + "mode": "video_generation", + "output_cost_per_second": 0.2419, + "output_cost_per_second_720p": 0.2419, + "source": "https://fal.ai/models/bytedance/seedance-2.0/fast/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.0/us/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; 1080p and 4k are not priced on the page; fal bills tokens at $0.0168 per 1000 tokens, counts input video duration, and multiplies by 0.6 with video inputs"}, + "mode": "video_generation", + "output_cost_per_second": 0.37, + "output_cost_per_second_480p": 0.1731, + "output_cost_per_second_720p": 0.37, + "source": "https://fal.ai/models/bytedance/seedance-2.0/us/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedance-2.5/us/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 720p rate; with video references fal bills input and output duration and multiplies by 0.6"}, + "mode": "video_generation", + "output_cost_per_second": 0.473, + "output_cost_per_second_1080p": 1.164, + "output_cost_per_second_480p": 0.2205, + "output_cost_per_second_720p": 0.473, + "source": "https://fal.ai/models/bytedance/seedance-2.5/us/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/bria/background/remove": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image without a prompt (prompt is optional); $0.03 per image when a prompt is given has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.018, + "source": "https://fal.ai/models/fal-ai/bria/background/remove", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/bytedance-upscaler/upscale/video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by target_resolution at 30fps in normal mode; output_cost_per_second is the default target_resolution 1080p rate; 6k $0.0574 and 8k $0.1148 per second have no catalog field; 30-60fps doubles, over 60fps quadruples and pro mode multiplies by 10, none of which is modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.0072, + "output_cost_per_second_1080p": 0.0072, + "output_cost_per_second_2k": 0.0144, + "output_cost_per_second_4k": 0.0288, + "source": "https://fal.ai/models/fal-ai/bytedance-upscaler/upscale/video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/emu-3.5-image/edit-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default 720p resolution; 480p is $0.15 per image and has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://fal.ai/models/fal-ai/emu-3.5-image/edit-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/emu-3.5-image/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default 720p resolution; 480p is $0.15 per image and has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://fal.ai/models/fal-ai/emu-3.5-image/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/kling-video/v2.5-turbo/pro/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.5-turbo/pro/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v2.5-turbo/pro/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.5-turbo/pro/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/kling-video/v2.5-turbo/standard/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.042, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.5-turbo/standard/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/hailuo-02/standard/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 768P rate; 512P is roughly $0.017 per second and has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.045, + "output_cost_per_second_768p": 0.045, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-02/standard/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/image-01": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://fal.ai/models/fal-ai/minimax/image-01", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/object-removal": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default model best_quality; low_quality $0.006, medium_quality $0.012 and high_quality $0.018 per image have no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.024, + "source": "https://fal.ai/models/fal-ai/object-removal", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/object-removal/bbox": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default model best_quality; low_quality $0.006, medium_quality $0.012 and high_quality $0.018 per image have no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.024, + "source": "https://fal.ai/models/fal-ai/object-removal/bbox", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/object-removal/mask": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default model best_quality; low_quality $0.006, medium_quality $0.012 and high_quality $0.018 per image have no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.024, + "source": "https://fal.ai/models/fal-ai/object-removal/mask", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/recraft/v3/image-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default realistic_image style; vector styles are $0.08 per image and have no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/fal-ai/recraft/v3/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/scail-2": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.2, + "source": "https://fal.ai/models/fal-ai/scail-2", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-vace-apps/long-reframe": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution, converted from the page rate per 2 seconds ($0.08 per 2s at 720p, $0.04 per 2s at 480p); output_cost_per_second is the 720p rate because the default resolution is auto; 580p ($0.06 per 2s) has no catalog field, 240p and 360p are not priced on the page; seconds are rounded up to a multiple of 2"}, + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_720p": 0.04, + "source": "https://fal.ai/models/fal-ai/wan-vace-apps/long-reframe", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-14b/animate/move": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per video second of output by resolution, where a video second is 16 frames; output_cost_per_second is the default resolution 480p rate; 580p is $0.06 per video second and has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-14b/animate/move", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-14b/animate/replace": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per video second of output by resolution, where a video second is 16 frames; output_cost_per_second is the default resolution 480p rate; 580p is $0.06 per video second and has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_480p": 0.04, + "output_cost_per_second_720p": 0.08, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-14b/animate/replace", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.2/edit": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; 360p is $0.0273 per second and has no catalog field; input tokens (text, image, video) are billed at $1.50 per million"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.1229, + "output_cost_per_second_4k": 0.2457, + "output_cost_per_second_720p": 0.0819, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/edit", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.2/extend": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; 360p is $0.0273 per second and has no catalog field; input tokens (text, image, video) are billed at $1.50 per million"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.1229, + "output_cost_per_second_4k": 0.2457, + "output_cost_per_second_720p": 0.0819, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/extend", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.2/image-to-video": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; 360p is $0.0273 per second and has no catalog field; input tokens (text, image, video) are billed at $1.50 per million"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.1229, + "output_cost_per_second_4k": 0.2457, + "output_cost_per_second_720p": 0.0819, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.2/reference-to-video": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; 360p is $0.0273 per second and has no catalog field; input tokens (text, image, video) are billed at $1.50 per million"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.1229, + "output_cost_per_second_4k": 0.2457, + "output_cost_per_second_720p": 0.0819, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/gemini-omni-flash/v1.2/text-to-video": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; 360p is $0.0273 per second and has no catalog field; input tokens (text, image, video) are billed at $1.50 per million"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.1229, + "output_cost_per_second_4k": 0.2457, + "output_cost_per_second_720p": 0.0819, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/google/virtual-try-on": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://fal.ai/models/google/virtual-try-on", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/ideogram/v4/tiling": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per megapixel at the default rendering_speed BALANCED; TURBO is $0.03 and QUALITY is $0.10 per megapixel, no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.06, + "output_cost_per_pixel": 5.7220458984375e-08, + "source": "https://fal.ai/models/ideogram/v4/tiling", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/ideogram/v4/tiling/lora": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per megapixel at the default rendering_speed BALANCED; TURBO is $0.03375 and QUALITY is $0.1125 per megapixel, no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.0675, + "output_cost_per_pixel": 6.437301635742188e-08, + "source": "https://fal.ai/models/ideogram/v4/tiling/lora", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/kling/v4/flash/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video at 720p (the only resolution) with image references; with video references fal bills $0.063 per second on the output duration plus the greater of the input and output durations, which has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.094, + "output_cost_per_second_720p": 0.094, + "source": "https://fal.ai/models/kling/v4/flash/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/kling/v4/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution with image references; output_cost_per_second is the default resolution 1080p rate; with video references or video elements fal bills $0.094 (720p) or $0.141 (1080p) per second on the input plus output length, which has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.235, + "output_cost_per_second_1080p": 0.235, + "output_cost_per_second_720p": 0.157, + "source": "https://fal.ai/models/kling/v4/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/luma/agent/ray/v3.2/reframe": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per started source second by resolution; output_cost_per_second is the default resolution 540p rate ($0.06), which has no tier field"}, + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.36, + "output_cost_per_second_720p": 0.12, + "source": "https://fal.ai/models/luma/agent/ray/v3.2/reframe", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of requested output duration by resolution; output_cost_per_second is the default resolution 768P rate; the page states no promotion for this endpoint; reference tokens above the 4,096 included per request are $0.02 per 1,000 tokens and are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.16, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/styles/16bit-pixel": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/styles/16bit-pixel", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/styles/hand-drawn": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/styles/hand-drawn", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/styles/low-poly": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/styles/low-poly", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/styles/retro-toon-70s": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/styles/retro-toon-70s", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/minimax/h3-max/styles/vhs": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_768p": 0.08, + "source": "https://fal.ai/models/minimax/h3-max/styles/vhs", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/mirelo-ai/sfx2.0/video-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated audio per sample; promotional 33% off rate through 2026-12-13 per the page; the standard rate of $0.015 per second applies from 2026-12-14 (ruling fal-ai-promotional-rates-apply-until-promotion-ends, 2026-11-10); mode and endpoint follow fal_ai/sonilo/v1.1/video-to-video-sound-effects"}, + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/mirelo-ai/sfx2.0/video-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/nvidia/cosmos-3-super/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image; the optional $0.02 per request prompt expansion surcharge is not modeled, and agentic generation bills every candidate image"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://fal.ai/models/nvidia/cosmos-3-super/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/recraft/v4/style/pro/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image when a style_id is provided; the one-time $0.005 style creation surcharge when no style_id is given is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://fal.ai/models/recraft/v4/style/pro/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/recraft/v4/style/pro/text-to-vector": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image when a style_id is provided; the one-time $0.005 style creation surcharge when no style_id is given is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.12, + "source": "https://fal.ai/models/recraft/v4/style/pro/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/recraft/v4/style/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image when a style_id is provided; the one-time $0.005 style creation surcharge when no style_id is given is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://fal.ai/models/recraft/v4/style/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/recraft/v4/style/text-to-vector": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per generated image when a style_id is provided; the one-time $0.005 style creation surcharge when no style_id is given is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://fal.ai/models/recraft/v4/style/text-to-vector", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/vast/eden-live-14b-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of video generated in a live session; the 10 billable second ($0.10) minimum per session is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.01, + "source": "https://fal.ai/models/vast/eden-live-14b-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/vast/eden-live-5b-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of video generated in a live session; the 10 billable second ($0.035) minimum per session is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.0035, + "source": "https://fal.ai/models/vast/eden-live-5b-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-image/edit": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.022, + "source": "https://fal.ai/models/xai/grok-imagine-image/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/xai/grok-imagine-video/edit-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the 720p rate because the default resolution is auto; the $0.01 per second of input video charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://fal.ai/models/xai/grok-imagine-video/edit-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/extend-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the 720p rate because the endpoint has no resolution field; the $0.01 per second of input video charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://fal.ai/models/xai/grok-imagine-video/extend-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; the $0.002 per request image input charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://fal.ai/models/xai/grok-imagine-video/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 480p rate; the $0.002 per request image input charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://fal.ai/models/xai/grok-imagine-video/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate"}, + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://fal.ai/models/xai/grok-imagine-video/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/v1.5/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 720p rate; the $0.01 per reference image charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.14, + "output_cost_per_second_1080p": 0.25, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/xai/grok-imagine-video/v1.5/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/xai/grok-imagine-video/v1.5/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video by resolution; output_cost_per_second is the default resolution 480p rate; the $0.01 per reference image charge is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.14, + "source": "https://fal.ai/models/xai/grok-imagine-video/v1.5/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] } } diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 8ff7f8bd251..8d8c123ef49 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1455,6 +1455,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "fal_ai/bytedance/seedance-2.0/reference-to-video", "fal_ai/bytedance/seedance-2.5/us/draft/complete", # 1080p is $1.164/second per fal.ai "fal_ai/bytedance/seedance-2.5/us/image-to-video", # 1080p is $1.164/second per fal.ai + "fal_ai/bytedance/seedance-2.5/us/reference-to-video", # 1080p is $1.164/second per fal.ai "fal_ai/bytedance/seedance-2.5/us/text-to-video", # 1080p is $1.164/second per fal.ai ] From 076ded32be1b000070f07c11b5cbdb4463d6b96f Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 10 Nov 2026 17:07:11 -0800 Subject: [PATCH 058/189] feat(auto_router): report a session's whole spend and name why savings stay unestimated (#51066) * feat(auto_router): report a session's whole spend on /auto_router/session Claude Code sends its background calls straight to Haiku under the same session id, so those requests never reached the auto-router session rollup and the status line under-reported the session. Every billed spend row in a caller-supplied session now increments a key-scoped Redis counter whose one-hour expiry each write resets, and the endpoint reports it as total_spend, with spend unchanged. The increment runs where the spend row is accepted, so it counts each row exactly as often as the key's spend counters do. It reads as no total without a shared Redis * fix(auto_router): name the unsupported provider when a turn routes off Anthropic A turn routed to a non-Anthropic tier with no native wire response, such as a Cloudflare DeepSeek model, failed strict native validation and was logged as observation_unavailable, which reads like a broken capture. It now reports unsupported_selected_provider. Turns with a native wire response keep their existing reasons, and no estimate is added, so savings for these turns stay unknown * docs(auto_router): explain when session savings stay unestimated saved_spend on /auto_router/session now says a turn inside the longest prompt-cache TTL of the session's first observed turn, or of its last incomplete observation, is not estimated, so a null there is expected for that window * fix(auto_router): say why a system-only cache prefix has no savings estimate Claude Code puts its cache breakpoint on the system prompt, so the baseline prefix count sends Anthropic a body with no messages, which it rejects, and every turn read as token_count_unavailable. A count that fails only on system or tools prefixes now reports uncountable_system_or_tools_prefix. Gateways that accept the empty body still count it. A live check against claude-opus-5-5 showed a differential count with a placeholder message is off by -2 tokens for system-only and +68 for tools, so these turns stay unestimated and the savings gap for Claude Code is not fixed here * fix(auto_router): declare the session total's Redis key family and test its expiry The session total read and wrote Redis without a service target, so its calls showed up as bare redis spans. Both now run under session_spend_totals. The key no longer hashes the api key, which is already a hashed token, and only digests the session id. A new test drives Redis expiry on a test clock to show an active session keeps its total and an idle one restarts * docs(auto_router): note which turns are still estimated inside the history window Turns that took the identical baseline path, and turns whose whole cached prefix the session had already written, are estimated even inside the first cache TTL window, so saved_spend now says so --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/constants.py | 1 + .../llms/anthropic/prompt_cache_prediction.py | 6 +- litellm/proxy/db/db_spend_update_writer.py | 17 +++ .../proxy/hooks/autorouter_baseline_cache.py | 12 ++ .../auto_router_endpoints.py | 8 +- .../spend_tracking/session_spend_total.py | 59 ++++++++ .../spend_tracking/spend_tracking_utils.py | 25 +++- litellm/proxy/utils.py | 4 +- .../auto_router_endpoints.py | 14 +- .../test_anthropic_prompt_cache_prediction.py | 18 ++- .../proxy/db/test_db_spend_update_writer.py | 126 +++++++++++++++++- .../hooks/test_autorouter_baseline_cache.py | 52 ++++++++ .../test_auto_router_endpoints.py | 37 +++++ .../test_session_spend_total.py | 90 +++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 7 +- 15 files changed, 461 insertions(+), 15 deletions(-) create mode 100644 litellm/proxy/spend_tracking/session_spend_total.py create mode 100644 tests/unit/proxy/spend_tracking/test_session_spend_total.py diff --git a/litellm/constants.py b/litellm/constants.py index 25eb5d9b854..931ac4f2b81 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1713,6 +1713,7 @@ RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id" BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt" MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response" SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" +SESSION_SPEND_TOTAL_TTL_SECONDS: Final = 3600 SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted" CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id" COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY: Final = "_complexity_router_child_session_id" diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 28896c99eaf..6429b3b402c 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -208,6 +208,7 @@ class UnsupportedCachePlan: "invalid_cache_ttl_order", "unsupported_thinking_cache_semantics", "token_count_unavailable", + "uncountable_system_or_tools_prefix", "inconsistent_prefix_token_count", ] @@ -583,7 +584,10 @@ async def count_cache_plan( counts: Final = tuple( await asyncio.gather(*(token_counter(model, api_key, marker.prefix_body) for marker in plan.breakpoints)) ) - if any(value is None for value in counts): + failed: Final = tuple(marker for marker, value in zip(plan.breakpoints, counts, strict=True) if value is None) + if failed and all(not marker.prefix_body.get("messages") for marker in failed): + return UnsupportedCachePlan("uncountable_system_or_tools_prefix") + if failed: return UnsupportedCachePlan("token_count_unavailable") known: Final = tuple(value for value in counts if value is not None) if any(value < 0 for value in (total, *known)) or any( diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index fc280dd4f73..fd47b2fbb6b 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -92,6 +92,7 @@ from litellm.proxy.spend_tracking.savings import ( extract_cache_read_tokens, marks_gateway_injection, ) +from litellm.proxy.spend_tracking.session_spend_total import SessionSpendTotals from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.repositories.prisma_protocols import BatchTable from litellm.responses.streaming_iterator import spawn_logging_task @@ -685,8 +686,10 @@ class DBSpendUpdateWriter: self, redis_cache: RedisCache | None = None, accounting_clock: Callable[[], float] = time.monotonic, + session_spend_totals: SessionSpendTotals | None = None, ): self.redis_cache = redis_cache + self.session_spend_totals: Final = session_spend_totals self.accounting_storage_checks = Cadence(accounting_clock) self.accounting_delay_alerts = Cadence(accounting_clock) self.redis_update_buffer = RedisUpdateBuffer(redis_cache=self.redis_cache) @@ -819,6 +822,7 @@ class DBSpendUpdateWriter: prisma_client=prisma_client, spend_logs_kept=keep_spend_logs, ) + await self._record_session_spend(hashed_token, billable_cost or 0.0, request_fields) if keep_spend_logs: await self._enqueue_tool_usage_transaction( payload=payload, @@ -874,6 +878,19 @@ class DBSpendUpdateWriter: ) return True + async def _record_session_spend( + self, api_key: str | None, spend: float, request_fields: Mapping[str, object] + ) -> None: + from litellm.proxy.spend_tracking.spend_tracking_utils import caller_session_for_accounting + + totals: Final = self.session_spend_totals + if totals is None or not api_key or spend <= 0: + return + session_id: Final = caller_session_for_accounting(request_fields) + if session_id is None: + return + await totals.record(api_key=api_key, session_id=session_id, spend=spend) + async def _record_spend_log( self, payload: SpendLogsPayload, diff --git a/litellm/proxy/hooks/autorouter_baseline_cache.py b/litellm/proxy/hooks/autorouter_baseline_cache.py index b7dbebbe8d8..8e445589942 100644 --- a/litellm/proxy/hooks/autorouter_baseline_cache.py +++ b/litellm/proxy/hooks/autorouter_baseline_cache.py @@ -312,6 +312,18 @@ async def _capture_native( ) -> CapturedBaselineObservation: capture: Final = context.capture original: Final = capture.observation + details: Final = logging_obj.model_call_details + if details.get("custom_llm_provider") != "anthropic" and not isinstance( + details.get("httpx_response"), httpx.Response + ): + return capture.with_observation( + original.model_copy( + update={ + "available_at": max(original.started_at, context.collector.clock()), + "reason": context.invalidated or "unsupported_selected_provider", + } + ) + ) event: Final = _WireEvent.model_validate(logging_obj.model_call_details) wire: Final = event.httpx_response.request usage: Final = _ResponseUsage.model_validate(response_obj).usage diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 1e43a8375a6..ca10487a8a3 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -1052,7 +1052,7 @@ async def get_auto_router_session( fills a moment after each turn; a session with no flushed auto-routed turn yet is a 404. The id is bounded the way the writer bounded it, so an oversized client id still finds its row. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) @@ -1063,6 +1063,11 @@ async def get_auto_router_session( raise HTTPException( status_code=404, detail=f"No auto-routed turns recorded for session {session_id!r} under this key" ) + total: Final = ( + await proxy_logging_obj.session_spend_totals.read(user_api_key_dict.api_key, session_id) + if user_api_key_dict.api_key + else None + ) saved_spend, baseline_spend = _savings_cohort(row.turns, row.savings_estimated_turns, row.spend, row.saved_spend) _, estimated_baseline_spend = _savings_cohort( row.turns, row.savings_estimated_turns, row.savings_estimated_actual_spend, row.savings_estimated_saved_spend @@ -1074,6 +1079,7 @@ async def get_auto_router_session( turns=row.turns, last_model=row.last_model, spend=row.spend, + total_spend=total, savings_estimated_turns=row.savings_estimated_turns, savings_estimated_actual_spend=row.savings_estimated_actual_spend, saved_spend=saved_spend, diff --git a/litellm/proxy/spend_tracking/session_spend_total.py b/litellm/proxy/spend_tracking/session_spend_total.py new file mode 100644 index 00000000000..af13a8940a7 --- /dev/null +++ b/litellm/proxy/spend_tracking/session_spend_total.py @@ -0,0 +1,59 @@ +import hashlib +import logging +from collections.abc import Callable +from typing import Final, Protocol + +from pydantic import TypeAdapter + +from litellm._internal_context import with_service_target +from litellm._logging import verbose_proxy_logger +from litellm.caching.redis_cache import log_redis_failure +from litellm.constants import SESSION_SPEND_TOTAL_TTL_SECONDS + +_STORED_SPEND: Final[TypeAdapter[float | str | bytes | None]] = TypeAdapter(float | str | bytes | None) + + +class SharedCounterStore(Protocol): + async def async_increment( + self, key: str, value: float, *, ttl: int | None = None, refresh_ttl: bool = False + ) -> float: ... + + async def async_get_cache(self, key: str) -> object: ... + + +def _key(api_key: str, session_id: str) -> str: + session_digest: Final = hashlib.sha256(session_id.encode("utf-8", errors="surrogatepass")).hexdigest() + return f"session_spend_total:{api_key}:{session_digest}" + + +class SessionSpendTotals: + """Spend billed to one key's caller-supplied session, kept in the shared Redis on a TTL that every write slides.""" + + def __init__(self, shared_store: Callable[[], SharedCounterStore | None]) -> None: + self.shared_store: Final = shared_store + + @with_service_target("session_spend_totals") + async def record(self, api_key: str, session_id: str, spend: float) -> None: + store: Final = self.shared_store() + if store is None: + return + try: + await store.async_increment( + _key(api_key, session_id), spend, ttl=SESSION_SPEND_TOTAL_TTL_SECONDS, refresh_ttl=True + ) + except Exception as record_error: # noqa: BLE001 # a lost session total must never fail the spend write + log_redis_failure(verbose_proxy_logger, logging.WARNING, "Session spend total write failed", record_error) + + @with_service_target("session_spend_totals") + async def read(self, api_key: str, session_id: str) -> float | None: + store: Final = self.shared_store() + if store is None: + return None + try: + stored: Final = _STORED_SPEND.validate_python(await store.async_get_cache(_key(api_key, session_id))) + except Exception as read_error: # noqa: BLE001 # an unreadable total is reported as no total + log_redis_failure(verbose_proxy_logger, logging.WARNING, "Session spend total read failed", read_error) + return None + if stored is None: + return None + return float(stored.decode() if isinstance(stored, bytes) else stored) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 15b2d18eeae..f431763c049 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -37,7 +37,7 @@ from litellm.constants import ( MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, ) from litellm.integrations.otel.model.metadata import caller_session_id -from litellm.integrations.otel.model.trace_controls import TraceControls +from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls from litellm.litellm_core_utils.batch_utils import ( BATCH_COST_REQUEST_ID_SUFFIX, charged_poll_cost, @@ -1050,12 +1050,31 @@ def _get_batch_trace_session_id(call_type: str | None, request_id: str | None) - return request_id.removesuffix(BATCH_COST_REQUEST_ID_SUFFIX) -def _caller_session_for_spend_log(kwargs: Mapping[str, object], metadata: Mapping[str, object] | None) -> str | None: +def caller_session_for_accounting(kwargs: Mapping[str, object]) -> str | None: + params: Final = as_str_mapping(kwargs.get("litellm_params")) or MappingProxyType({}) + preserved: Final = next( + ( + session + for key in ("litellm_metadata", "metadata") + if isinstance(session := (as_str_mapping(params.get(key)) or {}).get(CALLER_SESSION_ID_METADATA_KEY), str) + ), + None, + ) + return _caller_session_for_spend_log( + kwargs, + MappingProxyType({CALLER_SESSION_ID_METADATA_KEY: preserved}) if preserved else None, + trace=caller_trace_controls(kwargs), + ) + + +def _caller_session_for_spend_log( + kwargs: Mapping[str, object], metadata: Mapping[str, object] | None, trace: TraceControls | None = None +) -> str | None: beside_trace_header: Final = metadata.get(CALLER_SESSION_ID_METADATA_KEY) if metadata else None caller_session: Final = ( beside_trace_header if isinstance(beside_trace_header, str) and beside_trace_header - else caller_session_id(kwargs, TraceControls()) + else caller_session_id(kwargs, trace or TraceControls()) ) return caller_session if caller_session and "\x00" not in caller_session else None diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index f78e68220b1..4468a724d2e 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -285,6 +285,7 @@ from litellm.proxy.policy_engine.pipeline_executor import ( ) from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_resolver import PolicyResolver +from litellm.proxy.spend_tracking.session_spend_total import SessionSpendTotals from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.spend_log_insert import RejectedSpendLogRow @@ -1983,7 +1984,8 @@ class ProxyLogging: ) self.premium_user = premium_user self.service_logging_obj = ServiceLogging() - self.db_spend_update_writer = DBSpendUpdateWriter() + self.session_spend_totals: Final = SessionSpendTotals(lambda: self.internal_usage_cache.dual_cache.redis_cache) + self.db_spend_update_writer = DBSpendUpdateWriter(session_spend_totals=self.session_spend_totals) self.proxy_hook_mapping: dict[str, CustomLogger] = {} # Guard flags to prevent duplicate background tasks diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index dfcac3b6bde..e244aba3e3c 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -176,12 +176,24 @@ class AutoRouterSessionResponse(LiteLLMBaseModel): turns: int = Field(description="Auto-routed turns the rollup has recorded for this session so far") last_model: str = Field(description="The deployment model the most recent turn was routed to") spend: float = Field(description="What the session's routed traffic actually cost, classifier calls included") + total_spend: float | None = Field( + default=None, + description="Spend billed to this key under the session id, routed or not, from a shared Redis counter " + "whose one-hour expiry every billed request resets. None without a shared Redis or once the counter " + "expired. A best-effort figure, not a lifetime total: a session idle past the expiry restarts from zero, " + "and a failed Redis write, eviction or flush drops spend without notice", + ) savings_estimated_turns: int = Field(description="Requests whose savings estimate recorded its baseline cost") savings_estimated_actual_spend: float = Field( description="Actual spend, including classifier cost, for requests whose estimate recorded its baseline cost" ) saved_spend: float | None = Field( - description="Recorded historical savings plus newer estimates, net of classifier cost" + description="Recorded historical savings plus newer estimates, net of classifier cost. None until a turn is " + "estimated. A turn that starts within the longest prompt-cache TTL its breakpoints use (5 minutes, or 1 hour " + "with a 1h breakpoint) of the session's first observed turn, or of the last turn whose observation was " + "incomplete, is not estimated, because the baseline's cache state is unknown then. Two kinds of turn are " + "still estimated inside that window: turns that took the identical baseline path, and turns whose whole " + "cached prefix the session had already written" ) baseline_spend: float | None = Field(description="Estimated single-model cost: spend plus recorded savings") savings_estimated_baseline_spend: float | None = Field( diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index 89ea412af0c..6a946c2e75c 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -515,10 +515,15 @@ async def test_cache_plan_count_conserves_total_and_rejects_unknown( @pytest.mark.asyncio @pytest.mark.parametrize("section", ["system", "tools"]) -@pytest.mark.parametrize("rejects_prefix", (False, True)) +@pytest.mark.parametrize( + ("rejected", "reason"), + [(None, None), (1, "uncountable_system_or_tools_prefix"), (2, "token_count_unavailable")], + ids=["counted", "system-or-tools-prefix-rejected", "message-prefix-rejected"], +) async def test_native_count_preserves_settings_and_requires_every_prefix( section: str, - rejects_prefix: bool, + rejected: int | None, + reason: str | None, monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -572,9 +577,9 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( endpoint: Final = "https://gateway.example/v1/messages/count_tokens" routes: Final = tuple( upstream.post(endpoint, json={**body, "model": _MODEL}).respond( - 400 if rejects_prefix and index == 1 else 200, + 400 if index == rejected else 200, json={"detail": {"error": "messages parameter is required"}} - if rejects_prefix and index == 1 + if index == rejected else {"input_tokens": tokens}, ) for index, (body, tokens) in enumerate( @@ -588,8 +593,9 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( unexpected: Final = upstream.post(endpoint).respond(200, json={"input_tokens": 1}) result: Final = await count_cache_plan(target.model, target.api_key, plan, count) - if rejects_prefix: - assert result == UnsupportedCachePlan("token_count_unavailable") + if reason is not None: + assert isinstance(result, UnsupportedCachePlan) + assert result.reason == reason else: assert isinstance(result, CountedPromptCachePlan) assert result.total_tokens == 6000 diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index e2debe05c03..8bf7cd36a8c 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -5,7 +5,7 @@ import itertools import json import logging import re -from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping from contextlib import AbstractAsyncContextManager, asynccontextmanager, contextmanager from datetime import datetime, timedelta, timezone from pathlib import Path @@ -39,6 +39,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( build_window_spend_transaction, ) from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL +from litellm.proxy.spend_tracking.session_spend_total import SessionSpendTotals from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -5887,3 +5888,126 @@ async def test_request_error_counts_survive_the_ledger_path_switch(monkeypatch: assert accumulator.drain() == {RequestErrorKey( date="2026-11-02", api_key=prisma.spend_log_transactions[0]["api_key"], team_id="", user_id="user", model_group="ledger-test", status_code=503, ): 1} + + +class _SessionTotalsRedis: + def __init__(self) -> None: + self.values: dict[str, float] = {} # mutable-ok: stands in for Redis state + + async def async_increment(self, key: str, value: float, ttl: int | None = None, refresh_ttl: bool = False) -> float: + self.values[key] = self.values.get(key, 0.0) + value + return self.values[key] + + async def async_get_cache(self, key: str) -> object: + return self.values.get(key) + + +def _session_writer() -> tuple[DBSpendUpdateWriter, SessionSpendTotals]: + redis: Final = _SessionTotalsRedis() + totals: Final = SessionSpendTotals(lambda: redis) + writer: Final = DBSpendUpdateWriter(session_spend_totals=totals) + writer._batch_database_updates = AsyncMock() # pyright: ignore[reportPrivateUsage] # the batch rollup needs a database this test does not stand up + return writer, totals + + +async def _record_spend_row(writer: DBSpendUpdateWriter, spend: float, litellm_params: Mapping[str, object]) -> None: + payload: Final[dict[str, object]] = { + **_minimal_spend_payload(), + "request_id": "req-1", + "call_type": "anthropic_messages", + "api_key": "hashed-key", + "spend": spend, + "status": "success", + } + with ( + patch( # test-quality-ok: update_database reads this proxy_server global at call time, no seam + "litellm.proxy.proxy_server.disable_spend_logs", False + ), + patch( # test-quality-ok: update_database reads this proxy_server global at call time, no seam + "litellm.proxy.proxy_server.prisma_client", _tool_usage_prisma() + ), + patch( # test-quality-ok: update_database reads this proxy_server global at call time, no seam + "litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget" + ), + patch( # test-quality-ok: update_database imports the payload builder inside its body, no seam + "litellm.proxy.spend_tracking.spend_tracking_utils.get_logging_payload", + return_value=payload, + ), + ): + await writer.update_database( + token="hashed-key", + user_id="test-user", + end_user_id=None, + team_id=None, + org_id=None, + kwargs={"model": "claude-haiku-5-5", "litellm_params": dict(litellm_params)}, + completion_response=None, + start_time=datetime(2026, 11, 10, 10, 0, tzinfo=timezone.utc), + end_time=datetime(2026, 11, 10, 10, 0, 1, tzinfo=timezone.utc), + response_cost=spend, + ) + + +_CALLER_SESSION: Final[Mapping[str, object]] = { + "litellm_session_id": "cc-session-1", + "metadata": {"session_id": "cc-session-1"}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "litellm_params", + [ + pytest.param(_CALLER_SESSION, id="direct-call"), + pytest.param( + { + "litellm_session_id": "cc-session-1", + "metadata": {"session_id": "cc-session-1", "trace_id": "cc-session-1"}, + }, + id="claude-code-session-header", + ), + pytest.param( + { + **_CALLER_SESSION, + "metadata": {"session_id": "cc-session-1", "internal_call_origin": "autorouter_classifier"}, + }, + id="classifier-sub-call", + ), + ], +) +async def test_a_billed_row_in_a_caller_session_adds_its_spend_to_that_sessions_total( + litellm_params: Mapping[str, object], +): + writer, totals = _session_writer() + + await _record_spend_row(writer, 0.25, litellm_params) + + assert await totals.read("hashed-key", "cc-session-1") == 0.25 + assert await totals.read("another-key", "cc-session-1") is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("litellm_params", "spend"), + [ + pytest.param( + { + "litellm_session_id": "minted-1", + "metadata": {"session_id": "minted-1", "litellm_session_id_generated": True}, + }, + 0.25, + id="generated-session", + ), + pytest.param({"metadata": {}}, 0.25, id="no-session"), + pytest.param(_CALLER_SESSION, 0.0, id="free-row"), + ], +) +async def test_a_row_without_a_caller_session_or_cost_adds_nothing( + litellm_params: Mapping[str, object], spend: float +): + writer, totals = _session_writer() + + await _record_spend_row(writer, spend, litellm_params) + + assert await totals.read("hashed-key", "cc-session-1") is None + assert await totals.read("hashed-key", "minted-1") is None diff --git a/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py b/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py index 1fe2cb2ca68..12aa850afb6 100644 --- a/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py +++ b/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py @@ -1098,3 +1098,55 @@ async def test_tier_pins_never_enter_the_caller_snapshot_on_later_routing_passes assert context is not None and context.baseline_body is not None assert context.baseline_body.get("max_tokens") == 16 assert "output_config" not in context.baseline_body and "thinking" not in context.baseline_body + + +_NON_NATIVE_MODELS: Final = _MESSAGES.validate_json("""[ + {"model_name":"test-router","litellm_params":{"model":"auto_router/complexity_router", + "complexity_router_config":{"tiers":{"SIMPLE":"flash","MEDIUM":"flash","COMPLEX":"flash", + "REASONING":"opus"},"session_affinity":false}}}, + {"model_name":"flash","litellm_params":{"model":"cloudflare/@cf/deepseek-ai/deepseek-v4.1-flash","api_key":"test-selected", + "api_base":"https://chat.example.test/v1"},"model_info":{"id":"selected"}}, + {"model_name":"opus","litellm_params":{"model":"anthropic/claude-opus-5","api_key":"test-selected"}, + "model_info":{"id":"baseline"}}]""") + + +def _chat_upstream(request: httpx.Request) -> httpx.Response: + stream: Final = _JSON_OBJECT.validate_json(request.content).get("stream") is True + chunk: Final = { + "id": "chatcmpl-1", + "object": "chat.completion.chunk", + "created": 1, + "model": "deepseek-v4.1-flash", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 2, "total_tokens": 14}, + } + completion: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "deepseek-v4.1-flash", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 2, "total_tokens": 14}, + } + content: Final = ( + f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode() if stream else json.dumps(completion).encode() + ) + return httpx.Response( + 200, + content=content, + request=request, + headers=MappingProxyType({"content-type": "text/event-stream" if stream else "application/json"}), + ) + + +@pytest.mark.parametrize("stream", (False, True)) +async def test_a_turn_routed_to_a_non_anthropic_tier_says_its_provider_is_unsupported( + monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + rig: Final = _Rig(monkeypatch, models=_NON_NATIVE_MODELS) + with respx.mock() as transport: + transport.route(host="chat.example.test").mock(side_effect=_chat_upstream) + await _call(rig.router, rig.logging(stream)) + captured: Final = _observation(await rig.capture.payload()) + assert captured.observation.outcome == "uncertain" + assert captured.observation.reason == "unsupported_selected_provider" diff --git a/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py index 59e12a97899..ba4b3b8cf4e 100644 --- a/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py @@ -1339,6 +1339,7 @@ class TestAutoRouterSession: "turns": turns, "last_model": "anthropic/claude-sonnet-5", "spend": spend, + "total_spend": None, "saved_spend": 0.24, "savings_estimated_turns": 3 if estimated else 0, "savings_estimated_actual_spend": 0.14 if estimated else 0.0, @@ -1350,6 +1351,42 @@ class TestAutoRouterSession: "baseline_models": {"anthropic/claude-opus-5": 3}, } + @pytest.mark.asyncio + @pytest.mark.parametrize(("recorded", "expected"), [((0.5, 0.25), 0.75), ((), None)], ids=["counted", "none"]) + async def test_the_session_reports_the_spend_billed_to_its_key_under_the_session( + self, monkeypatch: pytest.MonkeyPatch, recorded: tuple[float, ...], expected: float | None + ) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session + from litellm.proxy.spend_tracking.session_spend_total import SessionSpendTotals + + class _Redis: + def __init__(self) -> None: + self.values: dict[str, float] = {} # mutable-ok: stands in for Redis state + + async def async_increment( + self, key: str, value: float, ttl: int | None = None, refresh_ttl: bool = False + ) -> float: + self.values[key] = self.values.get(key, 0.0) + value + return self.values[key] + + async def async_get_cache(self, key: str) -> object: + return self.values.get(key) + + redis: Final = _Redis() + totals: Final = SessionSpendTotals(lambda: redis) + caller = UserAPIKeyAuth(api_key="sk-caller") + for spend in recorded: + await totals.record(caller.api_key or "", "sess-1", spend) + await totals.record(UserAPIKeyAuth(api_key="sk-other").api_key or "", "sess-1", 9.0) + monkeypatch.setattr(proxy_server.proxy_logging_obj, "session_spend_totals", totals) + self._rig(monkeypatch, [{**self.ROW, "api_key": caller.api_key, "session_id": "sess-1"}]) + + response = await get_auto_router_session(user_api_key_dict=caller, session_id="sess-1") + + assert response.total_spend == expected + assert response.spend == self.ROW["spend"] + @pytest.mark.asyncio async def test_another_keys_session_is_a_404_even_for_an_admin(self, monkeypatch: pytest.MonkeyPatch): # The scope is the caller's own key hash, exactly what the spend writer keyed the row under; diff --git a/tests/unit/proxy/spend_tracking/test_session_spend_total.py b/tests/unit/proxy/spend_tracking/test_session_spend_total.py new file mode 100644 index 00000000000..de208b7947f --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_session_spend_total.py @@ -0,0 +1,90 @@ +from typing import Final + +import pytest + +from litellm.proxy.spend_tracking.session_spend_total import SessionSpendTotals + + +class _Redis: + """The two RedisCache calls the session total makes, over a dict with Redis expiry on a test clock.""" + + def __init__(self, failing: bool = False) -> None: + self.failing = failing + self.now = 0.0 + self.values: dict[str, tuple[float, float]] = {} # mutable-ok: stands in for Redis state + + def _live(self, key: str) -> tuple[float, float] | None: + entry: Final = self.values.get(key) + return entry if entry is not None and entry[1] > self.now else None + + async def async_increment( + self, key: str, value: float, *, ttl: int | None = None, refresh_ttl: bool = False + ) -> float: + if self.failing: + raise ConnectionError("Redis dropped the write") + live: Final = self._live(key) + expires_at: Final = self.now + (ttl or 0) if live is None or refresh_ttl else live[1] + total: Final = (live[0] if live is not None else 0.0) + value + self.values[key] = (total, expires_at) + return total + + async def async_get_cache(self, key: str) -> str | None: + if self.failing: + raise ConnectionError("Redis dropped the read") + live: Final = self._live(key) + return str(live[0]) if live is not None else None + + +def _totals(redis: _Redis) -> SessionSpendTotals: + return SessionSpendTotals(lambda: redis) + + +@pytest.mark.asyncio +async def test_a_session_total_sums_its_own_keys_rows_only() -> None: + totals: Final = _totals(_Redis()) + + await totals.record("key-a", "session-1", 0.5) + await totals.record("key-a", "session-1", 0.25) + await totals.record("key-b", "session-1", 4.0) + await totals.record("key-a", "session-2", 8.0) + + assert await totals.read("key-a", "session-1") == 0.75 + assert await totals.read("key-b", "session-1") == 4.0 + assert await totals.read("key-a", "session-3") is None + + +@pytest.mark.asyncio +async def test_a_redis_outage_never_fails_the_spend_write_and_reads_as_no_total() -> None: + totals: Final = _totals(_Redis(failing=True)) + + await totals.record("key", "session", 0.5) + + assert await totals.read("key", "session") is None + + +@pytest.mark.asyncio +async def test_without_a_shared_redis_there_is_no_total() -> None: + totals: Final = SessionSpendTotals(lambda: None) + + await totals.record("key", "session", 0.5) + + assert await totals.read("key", "session") is None + + +@pytest.mark.asyncio +async def test_an_active_session_keeps_its_total_and_an_idle_one_restarts() -> None: + redis: Final = _Redis() + totals: Final = _totals(redis) + + await totals.record("key", "session", 0.5) + redis.now = 3000.0 + await totals.record("key", "session", 0.25) + redis.now = 6000.0 + active: Final = await totals.read("key", "session") + redis.now = 6000.0 + 3601.0 + idle: Final = await totals.read("key", "session") + await totals.record("key", "session", 0.125) + + assert active == 0.75 + assert idle is None + assert await totals.read("key", "session") == 0.125 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bd5e87bcbdd..4f946a4be89 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27922,7 +27922,7 @@ export interface components { router_type: string; /** * Saved Spend - * @description Recorded historical savings plus newer estimates, net of classifier cost + * @description Recorded historical savings plus newer estimates, net of classifier cost. None until a turn is estimated. A turn that starts within the longest prompt-cache TTL its breakpoints use (5 minutes, or 1 hour with a 1h breakpoint) of the session's first observed turn, or of the last turn whose observation was incomplete, is not estimated, because the baseline's cache state is unknown then. Two kinds of turn are still estimated inside that window: turns that took the identical baseline path, and turns whose whole cached prefix the session had already written */ saved_spend: number | null; /** @@ -27947,6 +27947,11 @@ export interface components { * @description What the session's routed traffic actually cost, classifier calls included */ spend: number; + /** + * Total Spend + * @description Spend billed to this key under the session id, routed or not, from a shared Redis counter whose one-hour expiry every billed request resets. None without a shared Redis or once the counter expired. A best-effort figure, not a lifetime total: a session idle past the expiry restarts from zero, and a failed Redis write, eviction or flush drops spend without notice + */ + total_spend?: number | null; /** * Turns * @description Auto-routed turns the rollup has recorded for this session so far From 665f56d64a1e9ad5ccfccb30e1e82b5520d93701 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:07:16 +0000 Subject: [PATCH 059/189] chore(pricing): sync replicate model prices from replicate.com model pages (#51067) * chore(pricing): sync replicate model prices from replicate.com model pages * chore(pricing): sync replicate model prices from replicate.com model pages * chore(pricing): sync replicate model prices from replicate.com model pages * chore(pricing): sync replicate model prices from replicate.com model pages * chore(pricing): regenerate provider endpoint support for new replicate modes --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 1951 ++++++++++++++++- .../provider_endpoints_support_backup.json | 10 +- model_prices_and_context_window.json | 1951 ++++++++++++++++- provider_endpoints_support.json | 10 +- 4 files changed, 3906 insertions(+), 16 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 68b77e3b785..0ee57858b17 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48622,6 +48622,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, + "source": "https://replicate.com/meta/llama-2-13b", "supports_tool_choice": true }, "replicate/meta/llama-2-13b-chat": { @@ -48631,6 +48632,8 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, + "source": "https://replicate.com/meta/llama-2-13b-chat", + "supports_system_messages": true, "supports_tool_choice": true }, "replicate/meta/llama-2-70b": { @@ -48640,6 +48643,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, + "source": "https://replicate.com/meta/llama-2-70b", "supports_tool_choice": true }, "replicate/meta/llama-2-70b-chat": { @@ -48649,6 +48653,8 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, + "source": "https://replicate.com/meta/llama-2-70b-chat", + "supports_system_messages": true, "supports_tool_choice": true }, "replicate/meta/llama-2-7b": { @@ -48658,6 +48664,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/meta/llama-2-7b", "supports_tool_choice": true }, "replicate/meta/llama-2-7b-chat": { @@ -48667,6 +48674,8 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/meta/llama-2-7b-chat", + "supports_system_messages": true, "supports_tool_choice": true }, "replicate/meta/llama-3-70b": { @@ -48721,6 +48730,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/mistralai/mistral-7b-v0.1", "supports_tool_choice": true }, "replicate/mistralai/mixtral-8x7b-instruct-v0.1": { @@ -48737,8 +48747,10 @@ "output_cost_per_token": 1e-05, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-5", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_reasoning": true, "supports_vision": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -48749,6 +48761,7 @@ "output_cost_per_token": 3.6e-07, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-oss-20b", "supports_function_calling": true, "supports_system_messages": true }, @@ -48764,13 +48777,15 @@ "supports_tool_choice": true, "supports_response_schema": true, "supports_prompt_caching": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://replicate.com/anthropic/claude-4.5-haiku" }, "replicate/ibm-granite/granite-3.3-8b-instruct": { "input_cost_per_token": 3e-08, "output_cost_per_token": 2.5e-07, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/ibm-granite/granite-3.3-8b-instruct", "supports_function_calling": true, "supports_system_messages": true }, @@ -48779,6 +48794,7 @@ "output_cost_per_token": 1e-05, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-4o", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -48820,6 +48836,7 @@ "output_cost_per_token": 6e-07, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-4o-mini", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -48832,6 +48849,7 @@ "output_cost_per_token": 1.06e-06, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/qwen/qwen3-235b-a22b-instruct-2507", "supports_function_calling": true, "supports_system_messages": true }, @@ -48856,6 +48874,7 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, + "source": "https://replicate.com/deepseek-ai/deepseek-v3", "supports_function_calling": true, "supports_system_messages": true }, @@ -48911,7 +48930,9 @@ "supports_response_schema": false, "input_cost_per_token_above_200k_tokens": 4e-06, "output_cost_per_token_above_200k_tokens": 1.8e-05, + "source": "https://replicate.com/google/gemini-3-pro", "supports_audio_input": true, + "supports_reasoning": true, "supports_video_input": true }, "replicate/anthropic/claude-4.5-sonnet": { @@ -48926,13 +48947,15 @@ "supports_tool_choice": true, "supports_response_schema": true, "supports_prompt_caching": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://replicate.com/anthropic/claude-4.5-sonnet" }, "replicate/openai/gpt-4.1": { "input_cost_per_token": 2e-06, "output_cost_per_token": 8e-06, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-4.1", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -48953,6 +48976,7 @@ "output_cost_per_token": 1.6e-06, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-4.1-mini", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -48965,16 +48989,21 @@ "output_cost_per_token": 4e-07, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-5-nano", "supports_function_calling": true, - "supports_system_messages": true + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true }, "replicate/openai/gpt-5-mini": { "input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-5-mini", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_reasoning": true, "supports_vision": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -48985,6 +49014,7 @@ "output_cost_per_token": 2.5e-06, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/google/gemini-2.5-flash", "supports_function_calling": false, "supports_parallel_function_calling": false, "supports_vision": true, @@ -49000,6 +49030,7 @@ "output_cost_per_token": 7.2e-07, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-oss-120b", "supports_function_calling": true, "supports_system_messages": true }, @@ -49010,6 +49041,7 @@ "mode": "chat", "max_input_tokens": 163840, "max_output_tokens": 163840, + "source": "https://replicate.com/deepseek-ai/deepseek-v3.1", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true @@ -49030,6 +49062,7 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, + "source": "https://replicate.com/deepseek-ai/deepseek-r1", "supports_reasoning": true, "supports_system_messages": true }, @@ -97062,6 +97095,1918 @@ "video" ] }, + "replicate/alibaba/happyhorse-1.0": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_720p": 0.14, + "source": "https://replicate.com/alibaba/happyhorse-1.0" + }, + "replicate/alibaba/happyhorse-1.1": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.18, + "output_cost_per_second_1080p": 0.18, + "output_cost_per_second_720p": 0.14, + "source": "https://replicate.com/alibaba/happyhorse-1.1" + }, + "replicate/alibaba/qwen-image-3": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/alibaba/qwen-image-3" + }, + "replicate/alibaba/qwen-image-3-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/alibaba/qwen-image-3-pro" + }, + "replicate/alibaba/wan-3": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "output_cost_per_second_1080p": 0.14, + "output_cost_per_second_480p": 0.035, + "output_cost_per_second_720p": 0.07, + "source": "https://replicate.com/alibaba/wan-3" + }, + "replicate/alibaba/wan-3-prime": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_480p": 0.068, + "output_cost_per_second_720p": 0.14, + "source": "https://replicate.com/alibaba/wan-3-prime" + }, + "replicate/anthropic/claude-fable-5": { + "input_cost_per_token": 1e-05, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 5e-05, + "source": "https://replicate.com/anthropic/claude-fable-5", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-fable-5.2": { + "input_cost_per_token": 8e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 4e-05, + "source": "https://replicate.com/anthropic/claude-fable-5.2", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-opus-4.6": { + "input_cost_per_token": 5e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://replicate.com/anthropic/claude-opus-4.6", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-opus-4.7": { + "input_cost_per_token": 5e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://replicate.com/anthropic/claude-opus-4.7", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-opus-5.5": { + "input_cost_per_token": 4e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2e-05, + "source": "https://replicate.com/anthropic/claude-opus-5.5", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-sonnet-4.6": { + "input_cost_per_token": 3e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://replicate.com/anthropic/claude-sonnet-4.6", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-sonnet-5": { + "input_cost_per_token": 2e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://replicate.com/anthropic/claude-sonnet-5", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/black-forest-labs/flux-1.1-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/black-forest-labs/flux-1.1-pro" + }, + "replicate/black-forest-labs/flux-1.1-pro-ultra": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://replicate.com/black-forest-labs/flux-1.1-pro-ultra" + }, + "replicate/black-forest-labs/flux-1.1-pro-ultra-finetuned": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.07, + "source": "https://replicate.com/black-forest-labs/flux-1.1-pro-ultra-finetuned" + }, + "replicate/black-forest-labs/flux-2-flex": { + "input_cost_per_pixel": 6e-08, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 6e-08, + "source": "https://replicate.com/black-forest-labs/flux-2-flex" + }, + "replicate/black-forest-labs/flux-2-klein-4b": { + "input_cost_per_pixel": 1e-09, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 1e-09, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-4b" + }, + "replicate/black-forest-labs/flux-2-klein-4b-base": { + "input_cost_per_pixel": 9e-09, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 9e-09, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-4b-base" + }, + "replicate/black-forest-labs/flux-2-klein-4b-base-lora": { + "input_cost_per_pixel": 9e-09, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 1.9e-08, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-4b-base-lora" + }, + "replicate/black-forest-labs/flux-2-klein-9b": { + "input_cost_per_pixel": 2e-09, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 1.5e-08, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-9b" + }, + "replicate/black-forest-labs/flux-2-klein-9b-base": { + "input_cost_per_pixel": 1.1e-08, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 1.1e-08, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-9b-base" + }, + "replicate/black-forest-labs/flux-2-klein-9b-base-lora": { + "input_cost_per_pixel": 1.1e-08, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 2.1e-08, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-9b-base-lora" + }, + "replicate/black-forest-labs/flux-canny-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-canny-dev" + }, + "replicate/black-forest-labs/flux-canny-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://replicate.com/black-forest-labs/flux-canny-pro" + }, + "replicate/black-forest-labs/flux-depth-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-depth-dev" + }, + "replicate/black-forest-labs/flux-depth-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://replicate.com/black-forest-labs/flux-depth-pro" + }, + "replicate/black-forest-labs/flux-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-dev" + }, + "replicate/black-forest-labs/flux-dev-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.032, + "source": "https://replicate.com/black-forest-labs/flux-dev-lora" + }, + "replicate/black-forest-labs/flux-fill-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/black-forest-labs/flux-fill-dev" + }, + "replicate/black-forest-labs/flux-fill-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://replicate.com/black-forest-labs/flux-fill-pro" + }, + "replicate/black-forest-labs/flux-kontext-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-kontext-dev" + }, + "replicate/black-forest-labs/flux-kontext-dev-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.032, + "source": "https://replicate.com/black-forest-labs/flux-kontext-dev-lora" + }, + "replicate/black-forest-labs/flux-kontext-max": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/black-forest-labs/flux-kontext-max" + }, + "replicate/black-forest-labs/flux-kontext-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/black-forest-labs/flux-kontext-pro" + }, + "replicate/black-forest-labs/flux-krea-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-krea-dev" + }, + "replicate/black-forest-labs/flux-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.055, + "source": "https://replicate.com/black-forest-labs/flux-pro" + }, + "replicate/black-forest-labs/flux-pro-finetuned": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://replicate.com/black-forest-labs/flux-pro-finetuned" + }, + "replicate/black-forest-labs/flux-redux-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-redux-dev" + }, + "replicate/black-forest-labs/flux-redux-schnell": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.003, + "source": "https://replicate.com/black-forest-labs/flux-redux-schnell" + }, + "replicate/black-forest-labs/flux-schnell": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.003, + "source": "https://replicate.com/black-forest-labs/flux-schnell" + }, + "replicate/black-forest-labs/flux-schnell-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://replicate.com/black-forest-labs/flux-schnell-lora" + }, + "replicate/bria/eraser": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/eraser" + }, + "replicate/bria/expand-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/expand-image" + }, + "replicate/bria/fibo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/fibo" + }, + "replicate/bria/fibo-edit": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/fibo-edit" + }, + "replicate/bria/generate-background": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/generate-background" + }, + "replicate/bria/genfill": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/genfill" + }, + "replicate/bria/image-3.2": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/image-3.2" + }, + "replicate/bria/increase-resolution": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/increase-resolution" + }, + "replicate/bria/product-cutout": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/product-cutout" + }, + "replicate/bria/product-packshot": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/product-packshot" + }, + "replicate/bria/product-shadow": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/product-shadow" + }, + "replicate/bria/remove-background": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.018, + "source": "https://replicate.com/bria/remove-background" + }, + "replicate/bria/video-erase-object": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/bria/video-erase-object" + }, + "replicate/bria/video-increase-resolution": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/bria/video-increase-resolution" + }, + "replicate/bria/video-remove-background": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/bria/video-remove-background" + }, + "replicate/bytedance/dreamactor-m2.0": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/bytedance/dreamactor-m2.0" + }, + "replicate/bytedance/dreamina-3.1": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/bytedance/dreamina-3.1" + }, + "replicate/bytedance/omni-human": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://replicate.com/bytedance/omni-human" + }, + "replicate/bytedance/omni-human-1.5": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.16, + "source": "https://replicate.com/bytedance/omni-human-1.5" + }, + "replicate/bytedance/seedance-1-lite": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.036, + "output_cost_per_second_1080p": 0.072, + "output_cost_per_second_480p": 0.018, + "output_cost_per_second_720p": 0.036, + "source": "https://replicate.com/bytedance/seedance-1-lite" + }, + "replicate/bytedance/seedance-1-pro": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_480p": 0.03, + "output_cost_per_second_720p": 0.06, + "source": "https://replicate.com/bytedance/seedance-1-pro" + }, + "replicate/bytedance/seedance-1-pro-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_480p": 0.015, + "output_cost_per_second_720p": 0.025, + "source": "https://replicate.com/bytedance/seedance-1-pro-fast" + }, + "replicate/bytedance/seedream-3": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/bytedance/seedream-3" + }, + "replicate/bytedance/seedream-4": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/bytedance/seedream-4" + }, + "replicate/bytedance/seedream-4.5": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bytedance/seedream-4.5" + }, + "replicate/bytedance/seedream-5-lite": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://replicate.com/bytedance/seedream-5-lite" + }, + "replicate/datalab-to/ocr": { + "litellm_provider": "replicate", + "mode": "ocr", + "ocr_cost_per_page": 0.002, + "source": "https://replicate.com/datalab-to/ocr" + }, + "replicate/decart/lucy-edit-2": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://replicate.com/decart/lucy-edit-2" + }, + "replicate/easel/ai-avatars": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://replicate.com/easel/ai-avatars" + }, + "replicate/elevenlabs/dubbing": { + "litellm_provider": "replicate", + "mode": "audio_speech", + "output_cost_per_second": 0.0366667, + "source": "https://replicate.com/elevenlabs/dubbing" + }, + "replicate/elevenlabs/flash-v2.5": { + "input_cost_per_character": 5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/elevenlabs/flash-v2.5" + }, + "replicate/elevenlabs/turbo-v2.5": { + "input_cost_per_character": 5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/elevenlabs/turbo-v2.5" + }, + "replicate/elevenlabs/v2-multilingual": { + "input_cost_per_character": 0.0001, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/elevenlabs/v2-multilingual" + }, + "replicate/elevenlabs/v3": { + "input_cost_per_character": 0.0001, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/elevenlabs/v3" + }, + "replicate/flux-kontext-apps/cartoonify": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/cartoonify" + }, + "replicate/flux-kontext-apps/change-haircut": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/change-haircut" + }, + "replicate/flux-kontext-apps/depth-of-field": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/depth-of-field" + }, + "replicate/flux-kontext-apps/face-to-many-kontext": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/face-to-many-kontext" + }, + "replicate/flux-kontext-apps/filters": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/filters" + }, + "replicate/flux-kontext-apps/iconic-locations": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/iconic-locations" + }, + "replicate/flux-kontext-apps/impossible-scenarios": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/impossible-scenarios" + }, + "replicate/flux-kontext-apps/multi-image-kontext-max": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/flux-kontext-apps/multi-image-kontext-max" + }, + "replicate/flux-kontext-apps/multi-image-kontext-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/multi-image-kontext-pro" + }, + "replicate/flux-kontext-apps/multi-image-list": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/flux-kontext-apps/multi-image-list" + }, + "replicate/flux-kontext-apps/portrait-series": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/portrait-series" + }, + "replicate/flux-kontext-apps/professional-headshot": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/professional-headshot" + }, + "replicate/flux-kontext-apps/restore-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/restore-image" + }, + "replicate/flux-kontext-apps/restyle-video-frame": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/restyle-video-frame" + }, + "replicate/flux-kontext-apps/text-removal": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/text-removal" + }, + "replicate/google/gemini-2.5-flash-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.039, + "source": "https://replicate.com/google/gemini-2.5-flash-image" + }, + "replicate/google/gemini-3-flash": { + "input_cost_per_token": 5e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://replicate.com/google/gemini-3-flash", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/google/gemini-3.1-flash-tts": { + "input_cost_per_token": 2e-06, + "litellm_provider": "replicate", + "mode": "audio_speech", + "output_cost_per_token": 4e-05, + "source": "https://replicate.com/google/gemini-3.1-flash-tts" + }, + "replicate/google/gemini-3.1-pro": { + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "source": "https://replicate.com/google/gemini-3.1-pro", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/google/gemini-3.5-flash": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 9e-06, + "source": "https://replicate.com/google/gemini-3.5-flash", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/google/gemini-3.8-flash-lite-tts": { + "input_cost_per_token": 5e-07, + "litellm_provider": "replicate", + "mode": "audio_speech", + "output_cost_per_token": 6e-06, + "source": "https://replicate.com/google/gemini-3.8-flash-lite-tts" + }, + "replicate/google/gemini-3.8-flash-tts": { + "input_cost_per_token": 5e-07, + "litellm_provider": "replicate", + "mode": "audio_speech", + "output_cost_per_token": 9e-06, + "source": "https://replicate.com/google/gemini-3.8-flash-tts" + }, + "replicate/google/nano-banana": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.039, + "source": "https://replicate.com/google/nano-banana" + }, + "replicate/google/nano-banana-2": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.067, + "output_cost_per_image_1K": 0.067, + "output_cost_per_image_2K": 0.101, + "output_cost_per_image_4K": 0.151, + "source": "https://replicate.com/google/nano-banana-2" + }, + "replicate/google/nano-banana-2-lite": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.034, + "source": "https://replicate.com/google/nano-banana-2-lite" + }, + "replicate/google/nano-banana-2.1": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_1K": 0.0336, + "output_cost_per_image_2K": 0.0504, + "output_cost_per_image_4K": 0.1134, + "source": "https://replicate.com/google/nano-banana-2.1" + }, + "replicate/heygen/avatar-iv": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.075, + "source": "https://replicate.com/heygen/avatar-iv" + }, + "replicate/heygen/avatar-v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.075, + "source": "https://replicate.com/heygen/avatar-v" + }, + "replicate/heygen/lipsync-precision": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.0667, + "source": "https://replicate.com/heygen/lipsync-precision" + }, + "replicate/heygen/lipsync-speed": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.0333, + "source": "https://replicate.com/heygen/lipsync-speed" + }, + "replicate/heygen/video-agent": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.0333, + "source": "https://replicate.com/heygen/video-agent" + }, + "replicate/ibm-granite/granite-20b-code-instruct-8k": { + "input_cost_per_token": 1e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://replicate.com/ibm-granite/granite-20b-code-instruct-8k", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-3.0-2b-instruct": { + "input_cost_per_token": 3e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-3.0-2b-instruct", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-3.0-8b-instruct": { + "input_cost_per_token": 5e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-3.0-8b-instruct", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-3.1-2b-instruct": { + "input_cost_per_token": 3e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-3.1-2b-instruct", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-3.1-8b-instruct": { + "input_cost_per_token": 3e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-3.1-8b-instruct", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-3.2-8b-instruct": { + "input_cost_per_token": 3e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-3.2-8b-instruct", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-4.0-h-small": { + "input_cost_per_token": 6e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-4.0-h-small", + "supports_function_calling": true, + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-4.1-8b": { + "input_cost_per_token": 6e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-4.1-8b", + "supports_function_calling": true, + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-4.2-8b": { + "input_cost_per_token": 6e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-4.2-8b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-8b-code-instruct-128k": { + "input_cost_per_token": 5e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-8b-code-instruct-128k", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-speech-3.3-8b": { + "litellm_provider": "replicate", + "mode": "audio_transcription", + "output_cost_per_token": 4.9e-06, + "source": "https://replicate.com/ibm-granite/granite-speech-3.3-8b" + }, + "replicate/ibm-granite/granite-speech-4.1-2b": { + "input_cost_per_token": 3e-08, + "litellm_provider": "replicate", + "mode": "audio_transcription", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-speech-4.1-2b" + }, + "replicate/ibm-granite/granite-vision-3.3-2b": { + "input_cost_per_token": 0.0, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-vision-3.3-2b", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/ibm-granite/granite-vision-4.1-4b": { + "input_cost_per_token": 6e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-vision-4.1-4b", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/ideogram-ai/ideogram-v2": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/ideogram-ai/ideogram-v2" + }, + "replicate/ideogram-ai/ideogram-v2-turbo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://replicate.com/ideogram-ai/ideogram-v2-turbo" + }, + "replicate/ideogram-ai/ideogram-v2a": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/ideogram-ai/ideogram-v2a" + }, + "replicate/ideogram-ai/ideogram-v2a-turbo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/ideogram-ai/ideogram-v2a-turbo" + }, + "replicate/ideogram-ai/ideogram-v3-balanced": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://replicate.com/ideogram-ai/ideogram-v3-balanced" + }, + "replicate/ideogram-ai/ideogram-v3-quality": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.09, + "source": "https://replicate.com/ideogram-ai/ideogram-v3-quality" + }, + "replicate/ideogram-ai/ideogram-v3-turbo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/ideogram-ai/ideogram-v3-turbo" + }, + "replicate/ideogram-ai/ideogram-v4-balanced": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://replicate.com/ideogram-ai/ideogram-v4-balanced" + }, + "replicate/ideogram-ai/ideogram-v4-quality": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://replicate.com/ideogram-ai/ideogram-v4-quality" + }, + "replicate/ideogram-ai/layerize": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.09, + "source": "https://replicate.com/ideogram-ai/layerize" + }, + "replicate/intelligent-utilities/html-to-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://replicate.com/intelligent-utilities/html-to-image" + }, + "replicate/inworld/realtime-tts-1.5-max": { + "input_cost_per_character": 3.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/inworld/realtime-tts-1.5-max" + }, + "replicate/inworld/realtime-tts-1.5-mini": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/inworld/realtime-tts-1.5-mini" + }, + "replicate/inworld/realtime-tts-2": { + "input_cost_per_character": 2.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/inworld/realtime-tts-2" + }, + "replicate/kwaivgi/kling-lip-sync": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.014, + "source": "https://replicate.com/kwaivgi/kling-lip-sync" + }, + "replicate/kwaivgi/kling-v2.1-master": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.28, + "source": "https://replicate.com/kwaivgi/kling-v2.1-master" + }, + "replicate/kwaivgi/kling-v2.5-turbo-pro": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "source": "https://replicate.com/kwaivgi/kling-v2.5-turbo-pro" + }, + "replicate/lightricks/audio-to-video": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://replicate.com/lightricks/audio-to-video" + }, + "replicate/lightricks/ltx-2-distilled": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://replicate.com/lightricks/ltx-2-distilled" + }, + "replicate/lightricks/ltx-2-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_1080p": 0.04, + "output_cost_per_second_2k": 0.08, + "output_cost_per_second_4k": 0.16, + "source": "https://replicate.com/lightricks/ltx-2-fast" + }, + "replicate/lightricks/ltx-2-pro": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.12, + "output_cost_per_second_4k": 0.24, + "source": "https://replicate.com/lightricks/ltx-2-pro" + }, + "replicate/lightricks/ltx-2-retake": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://replicate.com/lightricks/ltx-2-retake" + }, + "replicate/lightricks/ltx-2.3-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.12, + "output_cost_per_second_4k": 0.24, + "source": "https://replicate.com/lightricks/ltx-2.3-fast" + }, + "replicate/lightricks/ltx-2.3-pro": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.08, + "output_cost_per_second_2k": 0.16, + "output_cost_per_second_4k": 0.32, + "source": "https://replicate.com/lightricks/ltx-2.3-pro" + }, + "replicate/lightricks/ltx-2.5-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.12, + "output_cost_per_second_4k": 0.24, + "output_cost_per_second_720p": 0.03, + "source": "https://replicate.com/lightricks/ltx-2.5-fast" + }, + "replicate/luma/photon": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/luma/photon" + }, + "replicate/luma/photon-flash": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/luma/photon-flash" + }, + "replicate/luma/reframe-video": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.06, + "source": "https://replicate.com/luma/reframe-video" + }, + "replicate/meta/llama-4-maverick-instruct": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 9.5e-07, + "source": "https://replicate.com/meta/llama-4-maverick-instruct", + "supports_system_messages": true + }, + "replicate/meta/llama-4-scout-instruct": { + "input_cost_per_token": 1.7e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 6.5e-07, + "source": "https://replicate.com/meta/llama-4-scout-instruct", + "supports_system_messages": true + }, + "replicate/meta/llama-guard-4-12b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://replicate.com/meta/llama-guard-4-12b", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/meta/meta-llama-3-70b": { + "input_cost_per_token": 6.5e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.75e-06, + "source": "https://replicate.com/meta/meta-llama-3-70b" + }, + "replicate/meta/meta-llama-3-70b-instruct": { + "input_cost_per_token": 6.5e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.75e-06, + "source": "https://replicate.com/meta/meta-llama-3-70b-instruct", + "supports_system_messages": true + }, + "replicate/meta/meta-llama-3-8b": { + "input_cost_per_token": 5e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/meta/meta-llama-3-8b" + }, + "replicate/meta/meta-llama-3-8b-instruct": { + "input_cost_per_token": 5e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/meta/meta-llama-3-8b-instruct", + "supports_system_messages": true + }, + "replicate/minimax/h3": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.13, + "output_cost_per_second_2k": 0.13, + "output_cost_per_second_768p": 0.08, + "source": "https://replicate.com/minimax/h3" + }, + "replicate/minimax/h3-highspeed": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.03, + "source": "https://replicate.com/minimax/h3-highspeed" + }, + "replicate/minimax/h3-lite": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "output_cost_per_second_480p": 0.0125, + "output_cost_per_second_768p": 0.02, + "source": "https://replicate.com/minimax/h3-lite" + }, + "replicate/minimax/image-01": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/minimax/image-01" + }, + "replicate/minimax/speech-02-hd": { + "input_cost_per_token": 0.0001, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-02-hd" + }, + "replicate/minimax/speech-02-turbo": { + "input_cost_per_token": 6e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-02-turbo" + }, + "replicate/minimax/speech-2.6-hd": { + "input_cost_per_token": 0.0001, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-2.6-hd" + }, + "replicate/minimax/speech-2.6-turbo": { + "input_cost_per_token": 6e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-2.6-turbo" + }, + "replicate/minimax/speech-2.8-hd": { + "input_cost_per_token": 0.0001, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-2.8-hd" + }, + "replicate/minimax/speech-2.8-turbo": { + "input_cost_per_token": 6e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-2.8-turbo" + }, + "replicate/moonshotai/kimi-k2-thinking": { + "input_cost_per_token": 6.6e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.7e-06, + "source": "https://replicate.com/moonshotai/kimi-k2-thinking" + }, + "replicate/moonshotai/kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "source": "https://replicate.com/moonshotai/kimi-k2.5", + "supports_vision": true + }, + "replicate/moonshotai/kimi-k2.6": { + "input_cost_per_token": 9.5e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://replicate.com/moonshotai/kimi-k2.6", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/nightmareai/real-esrgan": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.002, + "source": "https://replicate.com/nightmareai/real-esrgan" + }, + "replicate/openai/gpt-4o-mini-transcribe": { + "input_cost_per_token": 3e-06, + "litellm_provider": "replicate", + "mode": "audio_transcription", + "output_cost_per_token": 5e-06, + "source": "https://replicate.com/openai/gpt-4o-mini-transcribe" + }, + "replicate/openai/gpt-4o-transcribe": { + "input_cost_per_token": 6e-06, + "litellm_provider": "replicate", + "mode": "audio_transcription", + "output_cost_per_token": 1e-05, + "source": "https://replicate.com/openai/gpt-4o-transcribe" + }, + "replicate/openai/gpt-5-pro": { + "input_cost_per_token": 1.5e-05, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 0.00012, + "source": "https://replicate.com/openai/gpt-5-pro", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.1": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://replicate.com/openai/gpt-5.1", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.2": { + "input_cost_per_token": 1.75e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "source": "https://replicate.com/openai/gpt-5.2", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.4": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://replicate.com/openai/gpt-5.4", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.6-luna": { + "input_cost_per_token": 1e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://replicate.com/openai/gpt-5.6-luna", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.6-sol": { + "input_cost_per_token": 2e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://replicate.com/openai/gpt-5.6-sol", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.6-terra": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://replicate.com/openai/gpt-5.6-terra", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/perceptron-ai-inc/isaac-0.1": { + "input_cost_per_token": 2.6e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.17e-06, + "source": "https://replicate.com/perceptron-ai-inc/isaac-0.1", + "supports_vision": true + }, + "replicate/philz1337x/clarity-pro-upscaler": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 3e-08, + "source": "https://replicate.com/philz1337x/clarity-pro-upscaler" + }, + "replicate/pixverse/lipsync": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://replicate.com/pixverse/lipsync" + }, + "replicate/playht/play-dialog": { + "litellm_provider": "replicate", + "mode": "audio_speech", + "output_cost_per_second": 0.001, + "source": "https://replicate.com/playht/play-dialog" + }, + "replicate/prunaai/flux-fast": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.005, + "source": "https://replicate.com/prunaai/flux-fast" + }, + "replicate/prunaai/flux-kontext-fast": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/prunaai/flux-kontext-fast" + }, + "replicate/prunaai/hidream-l1-fast": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.005, + "source": "https://replicate.com/prunaai/hidream-l1-fast" + }, + "replicate/prunaai/p-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.005, + "source": "https://replicate.com/prunaai/p-image" + }, + "replicate/prunaai/p-image-edit": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/prunaai/p-image-edit" + }, + "replicate/prunaai/p-image-edit-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/prunaai/p-image-edit-lora" + }, + "replicate/prunaai/p-image-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.005, + "source": "https://replicate.com/prunaai/p-image-lora" + }, + "replicate/prunaai/p-video-2-pro-avatar": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.08, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_4k": 0.12, + "output_cost_per_second_768p": 0.04, + "source": "https://replicate.com/prunaai/p-video-2-pro-avatar" + }, + "replicate/prunaai/p-video-2-pro-replace": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.07, + "output_cost_per_second_480p": 0.025, + "output_cost_per_second_4k": 0.09, + "output_cost_per_second_768p": 0.05, + "source": "https://replicate.com/prunaai/p-video-2-pro-replace" + }, + "replicate/prunaai/p-video-animate": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_720p": 0.03, + "source": "https://replicate.com/prunaai/p-video-animate" + }, + "replicate/prunaai/p-video-avatar": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.025, + "output_cost_per_second_1080p": 0.045, + "output_cost_per_second_720p": 0.025, + "source": "https://replicate.com/prunaai/p-video-avatar" + }, + "replicate/prunaai/p-video-replace": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_720p": 0.03, + "source": "https://replicate.com/prunaai/p-video-replace" + }, + "replicate/prunaai/wan-2.2-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://replicate.com/prunaai/wan-2.2-image" + }, + "replicate/quiverai/arrow-1.1": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.2, + "source": "https://replicate.com/quiverai/arrow-1.1" + }, + "replicate/quiverai/arrow-1.1-max": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://replicate.com/quiverai/arrow-1.1-max" + }, + "replicate/quiverai/arrow-1.1-max-vectorize": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.2, + "source": "https://replicate.com/quiverai/arrow-1.1-max-vectorize" + }, + "replicate/quiverai/arrow-1.1-vectorize": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://replicate.com/quiverai/arrow-1.1-vectorize" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-fusion": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-fusion" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-next-scene": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-next-scene" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-photo-to-anime": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-photo-to-anime" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-relight": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-relight" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-skin": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-skin" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-upscale": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-upscale" + }, + "replicate/qwen/qwen-edit-multiangle": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen/qwen-edit-multiangle" + }, + "replicate/qwen/qwen-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/qwen/qwen-image" + }, + "replicate/qwen/qwen-image-2": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://replicate.com/qwen/qwen-image-2" + }, + "replicate/qwen/qwen-image-2-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://replicate.com/qwen/qwen-image-2-pro" + }, + "replicate/qwen/qwen-image-2512": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://replicate.com/qwen/qwen-image-2512" + }, + "replicate/qwen/qwen-image-edit": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen/qwen-image-edit" + }, + "replicate/qwen/qwen-image-edit-2511": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen/qwen-image-edit-2511" + }, + "replicate/qwen/qwen-image-edit-plus": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen/qwen-image-edit-plus" + }, + "replicate/qwen/qwen-image-edit-plus-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen/qwen-image-edit-plus-lora" + }, + "replicate/qwen/qwen3-7-plus": { + "input_cost_per_token": 2.76e-07, + "input_cost_per_token_above_256k_tokens": 8.26e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.101e-06, + "output_cost_per_token_above_256k_tokens": 3.301e-06, + "source": "https://replicate.com/qwen/qwen3-7-plus", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/qwen/qwen3-tts": { + "input_cost_per_character": 2e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/qwen/qwen3-tts" + }, + "replicate/recraft-ai/recraft-20b": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.022, + "source": "https://replicate.com/recraft-ai/recraft-20b" + }, + "replicate/recraft-ai/recraft-20b-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.044, + "source": "https://replicate.com/recraft-ai/recraft-20b-svg" + }, + "replicate/recraft-ai/recraft-creative-upscale": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://replicate.com/recraft-ai/recraft-creative-upscale" + }, + "replicate/recraft-ai/recraft-crisp-upscale": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.006, + "source": "https://replicate.com/recraft-ai/recraft-crisp-upscale" + }, + "replicate/recraft-ai/recraft-remove-background": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/recraft-ai/recraft-remove-background" + }, + "replicate/recraft-ai/recraft-v3": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/recraft-ai/recraft-v3" + }, + "replicate/recraft-ai/recraft-v3-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/recraft-ai/recraft-v3-svg" + }, + "replicate/recraft-ai/recraft-v4": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/recraft-ai/recraft-v4" + }, + "replicate/recraft-ai/recraft-v4-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://replicate.com/recraft-ai/recraft-v4-pro" + }, + "replicate/recraft-ai/recraft-v4-pro-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://replicate.com/recraft-ai/recraft-v4-pro-svg" + }, + "replicate/recraft-ai/recraft-v4-styles-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://replicate.com/recraft-ai/recraft-v4-styles-pro" + }, + "replicate/recraft-ai/recraft-v4-styles-pro-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.12, + "source": "https://replicate.com/recraft-ai/recraft-v4-styles-pro-svg" + }, + "replicate/recraft-ai/recraft-v4-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/recraft-ai/recraft-v4-svg" + }, + "replicate/recraft-ai/recraft-v4.1": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/recraft-ai/recraft-v4.1" + }, + "replicate/recraft-ai/recraft-v4.1-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://replicate.com/recraft-ai/recraft-v4.1-pro" + }, + "replicate/recraft-ai/recraft-v4.1-pro-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://replicate.com/recraft-ai/recraft-v4.1-pro-svg" + }, + "replicate/recraft-ai/recraft-v4.1-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/recraft-ai/recraft-v4.1-svg" + }, + "replicate/recraft-ai/recraft-v4.1-utility": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/recraft-ai/recraft-v4.1-utility" + }, + "replicate/recraft-ai/recraft-v4.1-utility-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://replicate.com/recraft-ai/recraft-v4.1-utility-pro" + }, + "replicate/recraft-ai/recraft-vectorize": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/recraft-ai/recraft-vectorize" + }, + "replicate/resemble-ai/chatterbox": { + "input_cost_per_character": 2.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/resemble-ai/chatterbox" + }, + "replicate/resemble-ai/chatterbox-pro": { + "input_cost_per_character": 4e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/resemble-ai/chatterbox-pro" + }, + "replicate/resemble-ai/chatterbox-turbo": { + "input_cost_per_character": 2.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/resemble-ai/chatterbox-turbo" + }, + "replicate/retro-diffusion/rd-animation": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.07, + "source": "https://replicate.com/retro-diffusion/rd-animation" + }, + "replicate/runwayml/aleph-2": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.336, + "source": "https://replicate.com/runwayml/aleph-2" + }, + "replicate/runwayml/gen-4.5": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.12, + "source": "https://replicate.com/runwayml/gen-4.5" + }, + "replicate/runwayml/gen4-image-turbo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/runwayml/gen4-image-turbo" + }, + "replicate/runwayml/gen4-turbo": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/runwayml/gen4-turbo" + }, + "replicate/snowflake/snowflake-arctic-instruct": { + "input_cost_per_token": 2e-05, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2e-05, + "source": "https://replicate.com/snowflake/snowflake-arctic-instruct", + "supports_system_messages": true + }, + "replicate/stability-ai/stable-diffusion-3": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://replicate.com/stability-ai/stable-diffusion-3" + }, + "replicate/stability-ai/stable-diffusion-3.5-large": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.065, + "source": "https://replicate.com/stability-ai/stable-diffusion-3.5-large" + }, + "replicate/stability-ai/stable-diffusion-3.5-large-turbo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/stability-ai/stable-diffusion-3.5-large-turbo" + }, + "replicate/stability-ai/stable-diffusion-3.5-medium": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://replicate.com/stability-ai/stable-diffusion-3.5-medium" + }, + "replicate/sync/lipsync-2": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/sync/lipsync-2" + }, + "replicate/sync/lipsync-2-pro": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.08325, + "source": "https://replicate.com/sync/lipsync-2-pro" + }, + "replicate/sync/react-1": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.167, + "source": "https://replicate.com/sync/react-1" + }, + "replicate/tencent/hunyuan-image-2.1": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://replicate.com/tencent/hunyuan-image-2.1" + }, + "replicate/tencent/hunyuan-image-3": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/tencent/hunyuan-image-3" + }, + "replicate/veed/fabric-1.0": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.15, + "source": "https://replicate.com/veed/fabric-1.0" + }, + "replicate/wan-video/wan-2.2-s2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://replicate.com/wan-video/wan-2.2-s2v" + }, + "replicate/wan-video/wan-2.5-i2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://replicate.com/wan-video/wan-2.5-i2v" + }, + "replicate/wan-video/wan-2.5-i2v-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.068, + "output_cost_per_second_1080p": 0.102, + "output_cost_per_second_720p": 0.068, + "source": "https://replicate.com/wan-video/wan-2.5-i2v-fast" + }, + "replicate/wan-video/wan-2.5-t2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://replicate.com/wan-video/wan-2.5-t2v" + }, + "replicate/wan-video/wan-2.5-t2v-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.068, + "output_cost_per_second_1080p": 0.102, + "output_cost_per_second_720p": 0.068, + "source": "https://replicate.com/wan-video/wan-2.5-t2v-fast" + }, + "replicate/wan-video/wan-2.6-i2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://replicate.com/wan-video/wan-2.6-i2v" + }, + "replicate/wan-video/wan-2.6-t2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://replicate.com/wan-video/wan-2.6-t2v" + }, + "replicate/wan-video/wan-2.7-i2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://replicate.com/wan-video/wan-2.7-i2v" + }, + "replicate/wan-video/wan-2.7-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/wan-video/wan-2.7-image" + }, + "replicate/wan-video/wan-2.7-image-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/wan-video/wan-2.7-image-pro" + }, + "replicate/wan-video/wan-2.7-r2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://replicate.com/wan-video/wan-2.7-r2v" + }, + "replicate/wan-video/wan-2.7-t2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://replicate.com/wan-video/wan-2.7-t2v" + }, + "replicate/wan-video/wan-2.7-videoedit": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://replicate.com/wan-video/wan-2.7-videoedit" + }, + "replicate/wan-video/wan2.6-i2v-flash": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_1080p": 0.075, + "output_cost_per_second_720p": 0.05, + "source": "https://replicate.com/wan-video/wan2.6-i2v-flash" + }, + "replicate/wavespeedai/qwen-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/wavespeedai/qwen-image" + }, + "replicate/wavespeedai/wan-2.1-i2v-480p": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.09, + "source": "https://replicate.com/wavespeedai/wan-2.1-i2v-480p" + }, + "replicate/wavespeedai/wan-2.1-i2v-720p": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.25, + "source": "https://replicate.com/wavespeedai/wan-2.1-i2v-720p" + }, + "replicate/wavespeedai/wan-2.1-t2v-480p": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "source": "https://replicate.com/wavespeedai/wan-2.1-t2v-480p" + }, + "replicate/wavespeedai/wan-2.1-t2v-720p": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.24, + "source": "https://replicate.com/wavespeedai/wan-2.1-t2v-720p" + }, + "replicate/xai/grok-imagine-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://replicate.com/xai/grok-imagine-image" + }, + "replicate/xai/grok-imagine-image-quality": { + "input_cost_per_pixel": 1e-08, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://replicate.com/xai/grok-imagine-image-quality" + }, + "replicate/xai/grok-imagine-r2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/xai/grok-imagine-r2v" + }, + "replicate/xai/grok-imagine-video": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://replicate.com/xai/grok-imagine-video" + }, + "replicate/xai/grok-imagine-video-1.5": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "source": "https://replicate.com/xai/grok-imagine-video-1.5" + }, + "replicate/xai/grok-imagine-video-extension": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/xai/grok-imagine-video-extension" + }, + "replicate/xai/grok-text-to-speech": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/xai/grok-text-to-speech" + }, "fal_ai/blackforestlabs/flux-3/edit-image": { "litellm_provider": "fal_ai", "metadata": {"notes":"the page states only the 1K (1 MP) price of $0.048 and the pricing API bills in megapixels at $0.048, so the per-pixel rate is $0.048 per megapixel and the per-image price is the default 1k output; 768sq, 1.5k, 2k and 4k outputs are priced by the per-pixel rate"}, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 3431cc97969..b35037ee651 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -3508,16 +3508,16 @@ "responses": true, "text_completion": true, "embeddings": false, - "image_generations": false, + "image_generations": true, "image_edits": false, - "audio_transcriptions": false, - "audio_speech": false, + "audio_transcriptions": true, + "audio_speech": true, "rerank": false, - "ocr": false, + "ocr": true, "search": false, "moderations": false, "realtime": false, - "video_generations": false, + "video_generations": true, "batches": false, "messages": true, "interactions": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 68b77e3b785..0ee57858b17 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48622,6 +48622,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, + "source": "https://replicate.com/meta/llama-2-13b", "supports_tool_choice": true }, "replicate/meta/llama-2-13b-chat": { @@ -48631,6 +48632,8 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, + "source": "https://replicate.com/meta/llama-2-13b-chat", + "supports_system_messages": true, "supports_tool_choice": true }, "replicate/meta/llama-2-70b": { @@ -48640,6 +48643,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, + "source": "https://replicate.com/meta/llama-2-70b", "supports_tool_choice": true }, "replicate/meta/llama-2-70b-chat": { @@ -48649,6 +48653,8 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, + "source": "https://replicate.com/meta/llama-2-70b-chat", + "supports_system_messages": true, "supports_tool_choice": true }, "replicate/meta/llama-2-7b": { @@ -48658,6 +48664,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/meta/llama-2-7b", "supports_tool_choice": true }, "replicate/meta/llama-2-7b-chat": { @@ -48667,6 +48674,8 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/meta/llama-2-7b-chat", + "supports_system_messages": true, "supports_tool_choice": true }, "replicate/meta/llama-3-70b": { @@ -48721,6 +48730,7 @@ "max_output_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/mistralai/mistral-7b-v0.1", "supports_tool_choice": true }, "replicate/mistralai/mixtral-8x7b-instruct-v0.1": { @@ -48737,8 +48747,10 @@ "output_cost_per_token": 1e-05, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-5", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_reasoning": true, "supports_vision": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -48749,6 +48761,7 @@ "output_cost_per_token": 3.6e-07, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-oss-20b", "supports_function_calling": true, "supports_system_messages": true }, @@ -48764,13 +48777,15 @@ "supports_tool_choice": true, "supports_response_schema": true, "supports_prompt_caching": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://replicate.com/anthropic/claude-4.5-haiku" }, "replicate/ibm-granite/granite-3.3-8b-instruct": { "input_cost_per_token": 3e-08, "output_cost_per_token": 2.5e-07, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/ibm-granite/granite-3.3-8b-instruct", "supports_function_calling": true, "supports_system_messages": true }, @@ -48779,6 +48794,7 @@ "output_cost_per_token": 1e-05, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-4o", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -48820,6 +48836,7 @@ "output_cost_per_token": 6e-07, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-4o-mini", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -48832,6 +48849,7 @@ "output_cost_per_token": 1.06e-06, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/qwen/qwen3-235b-a22b-instruct-2507", "supports_function_calling": true, "supports_system_messages": true }, @@ -48856,6 +48874,7 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, + "source": "https://replicate.com/deepseek-ai/deepseek-v3", "supports_function_calling": true, "supports_system_messages": true }, @@ -48911,7 +48930,9 @@ "supports_response_schema": false, "input_cost_per_token_above_200k_tokens": 4e-06, "output_cost_per_token_above_200k_tokens": 1.8e-05, + "source": "https://replicate.com/google/gemini-3-pro", "supports_audio_input": true, + "supports_reasoning": true, "supports_video_input": true }, "replicate/anthropic/claude-4.5-sonnet": { @@ -48926,13 +48947,15 @@ "supports_tool_choice": true, "supports_response_schema": true, "supports_prompt_caching": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://replicate.com/anthropic/claude-4.5-sonnet" }, "replicate/openai/gpt-4.1": { "input_cost_per_token": 2e-06, "output_cost_per_token": 8e-06, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-4.1", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -48953,6 +48976,7 @@ "output_cost_per_token": 1.6e-06, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-4.1-mini", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -48965,16 +48989,21 @@ "output_cost_per_token": 4e-07, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-5-nano", "supports_function_calling": true, - "supports_system_messages": true + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true }, "replicate/openai/gpt-5-mini": { "input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-5-mini", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_reasoning": true, "supports_vision": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -48985,6 +49014,7 @@ "output_cost_per_token": 2.5e-06, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/google/gemini-2.5-flash", "supports_function_calling": false, "supports_parallel_function_calling": false, "supports_vision": true, @@ -49000,6 +49030,7 @@ "output_cost_per_token": 7.2e-07, "litellm_provider": "replicate", "mode": "chat", + "source": "https://replicate.com/openai/gpt-oss-120b", "supports_function_calling": true, "supports_system_messages": true }, @@ -49010,6 +49041,7 @@ "mode": "chat", "max_input_tokens": 163840, "max_output_tokens": 163840, + "source": "https://replicate.com/deepseek-ai/deepseek-v3.1", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true @@ -49030,6 +49062,7 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, + "source": "https://replicate.com/deepseek-ai/deepseek-r1", "supports_reasoning": true, "supports_system_messages": true }, @@ -97062,6 +97095,1918 @@ "video" ] }, + "replicate/alibaba/happyhorse-1.0": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_720p": 0.14, + "source": "https://replicate.com/alibaba/happyhorse-1.0" + }, + "replicate/alibaba/happyhorse-1.1": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.18, + "output_cost_per_second_1080p": 0.18, + "output_cost_per_second_720p": 0.14, + "source": "https://replicate.com/alibaba/happyhorse-1.1" + }, + "replicate/alibaba/qwen-image-3": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/alibaba/qwen-image-3" + }, + "replicate/alibaba/qwen-image-3-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/alibaba/qwen-image-3-pro" + }, + "replicate/alibaba/wan-3": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "output_cost_per_second_1080p": 0.14, + "output_cost_per_second_480p": 0.035, + "output_cost_per_second_720p": 0.07, + "source": "https://replicate.com/alibaba/wan-3" + }, + "replicate/alibaba/wan-3-prime": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.28, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_480p": 0.068, + "output_cost_per_second_720p": 0.14, + "source": "https://replicate.com/alibaba/wan-3-prime" + }, + "replicate/anthropic/claude-fable-5": { + "input_cost_per_token": 1e-05, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 5e-05, + "source": "https://replicate.com/anthropic/claude-fable-5", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-fable-5.2": { + "input_cost_per_token": 8e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 4e-05, + "source": "https://replicate.com/anthropic/claude-fable-5.2", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-opus-4.6": { + "input_cost_per_token": 5e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://replicate.com/anthropic/claude-opus-4.6", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-opus-4.7": { + "input_cost_per_token": 5e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://replicate.com/anthropic/claude-opus-4.7", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-opus-5.5": { + "input_cost_per_token": 4e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2e-05, + "source": "https://replicate.com/anthropic/claude-opus-5.5", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-sonnet-4.6": { + "input_cost_per_token": 3e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://replicate.com/anthropic/claude-sonnet-4.6", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/anthropic/claude-sonnet-5": { + "input_cost_per_token": 2e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://replicate.com/anthropic/claude-sonnet-5", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/black-forest-labs/flux-1.1-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/black-forest-labs/flux-1.1-pro" + }, + "replicate/black-forest-labs/flux-1.1-pro-ultra": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://replicate.com/black-forest-labs/flux-1.1-pro-ultra" + }, + "replicate/black-forest-labs/flux-1.1-pro-ultra-finetuned": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.07, + "source": "https://replicate.com/black-forest-labs/flux-1.1-pro-ultra-finetuned" + }, + "replicate/black-forest-labs/flux-2-flex": { + "input_cost_per_pixel": 6e-08, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 6e-08, + "source": "https://replicate.com/black-forest-labs/flux-2-flex" + }, + "replicate/black-forest-labs/flux-2-klein-4b": { + "input_cost_per_pixel": 1e-09, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 1e-09, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-4b" + }, + "replicate/black-forest-labs/flux-2-klein-4b-base": { + "input_cost_per_pixel": 9e-09, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 9e-09, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-4b-base" + }, + "replicate/black-forest-labs/flux-2-klein-4b-base-lora": { + "input_cost_per_pixel": 9e-09, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 1.9e-08, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-4b-base-lora" + }, + "replicate/black-forest-labs/flux-2-klein-9b": { + "input_cost_per_pixel": 2e-09, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 1.5e-08, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-9b" + }, + "replicate/black-forest-labs/flux-2-klein-9b-base": { + "input_cost_per_pixel": 1.1e-08, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 1.1e-08, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-9b-base" + }, + "replicate/black-forest-labs/flux-2-klein-9b-base-lora": { + "input_cost_per_pixel": 1.1e-08, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 2.1e-08, + "source": "https://replicate.com/black-forest-labs/flux-2-klein-9b-base-lora" + }, + "replicate/black-forest-labs/flux-canny-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-canny-dev" + }, + "replicate/black-forest-labs/flux-canny-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://replicate.com/black-forest-labs/flux-canny-pro" + }, + "replicate/black-forest-labs/flux-depth-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-depth-dev" + }, + "replicate/black-forest-labs/flux-depth-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://replicate.com/black-forest-labs/flux-depth-pro" + }, + "replicate/black-forest-labs/flux-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-dev" + }, + "replicate/black-forest-labs/flux-dev-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.032, + "source": "https://replicate.com/black-forest-labs/flux-dev-lora" + }, + "replicate/black-forest-labs/flux-fill-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/black-forest-labs/flux-fill-dev" + }, + "replicate/black-forest-labs/flux-fill-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://replicate.com/black-forest-labs/flux-fill-pro" + }, + "replicate/black-forest-labs/flux-kontext-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-kontext-dev" + }, + "replicate/black-forest-labs/flux-kontext-dev-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.032, + "source": "https://replicate.com/black-forest-labs/flux-kontext-dev-lora" + }, + "replicate/black-forest-labs/flux-kontext-max": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/black-forest-labs/flux-kontext-max" + }, + "replicate/black-forest-labs/flux-kontext-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/black-forest-labs/flux-kontext-pro" + }, + "replicate/black-forest-labs/flux-krea-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-krea-dev" + }, + "replicate/black-forest-labs/flux-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.055, + "source": "https://replicate.com/black-forest-labs/flux-pro" + }, + "replicate/black-forest-labs/flux-pro-finetuned": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://replicate.com/black-forest-labs/flux-pro-finetuned" + }, + "replicate/black-forest-labs/flux-redux-dev": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/black-forest-labs/flux-redux-dev" + }, + "replicate/black-forest-labs/flux-redux-schnell": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.003, + "source": "https://replicate.com/black-forest-labs/flux-redux-schnell" + }, + "replicate/black-forest-labs/flux-schnell": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.003, + "source": "https://replicate.com/black-forest-labs/flux-schnell" + }, + "replicate/black-forest-labs/flux-schnell-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://replicate.com/black-forest-labs/flux-schnell-lora" + }, + "replicate/bria/eraser": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/eraser" + }, + "replicate/bria/expand-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/expand-image" + }, + "replicate/bria/fibo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/fibo" + }, + "replicate/bria/fibo-edit": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/fibo-edit" + }, + "replicate/bria/generate-background": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/generate-background" + }, + "replicate/bria/genfill": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/genfill" + }, + "replicate/bria/image-3.2": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/image-3.2" + }, + "replicate/bria/increase-resolution": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/increase-resolution" + }, + "replicate/bria/product-cutout": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/product-cutout" + }, + "replicate/bria/product-packshot": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/product-packshot" + }, + "replicate/bria/product-shadow": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bria/product-shadow" + }, + "replicate/bria/remove-background": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.018, + "source": "https://replicate.com/bria/remove-background" + }, + "replicate/bria/video-erase-object": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/bria/video-erase-object" + }, + "replicate/bria/video-increase-resolution": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/bria/video-increase-resolution" + }, + "replicate/bria/video-remove-background": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/bria/video-remove-background" + }, + "replicate/bytedance/dreamactor-m2.0": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/bytedance/dreamactor-m2.0" + }, + "replicate/bytedance/dreamina-3.1": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/bytedance/dreamina-3.1" + }, + "replicate/bytedance/omni-human": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://replicate.com/bytedance/omni-human" + }, + "replicate/bytedance/omni-human-1.5": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.16, + "source": "https://replicate.com/bytedance/omni-human-1.5" + }, + "replicate/bytedance/seedance-1-lite": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.036, + "output_cost_per_second_1080p": 0.072, + "output_cost_per_second_480p": 0.018, + "output_cost_per_second_720p": 0.036, + "source": "https://replicate.com/bytedance/seedance-1-lite" + }, + "replicate/bytedance/seedance-1-pro": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_480p": 0.03, + "output_cost_per_second_720p": 0.06, + "source": "https://replicate.com/bytedance/seedance-1-pro" + }, + "replicate/bytedance/seedance-1-pro-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_480p": 0.015, + "output_cost_per_second_720p": 0.025, + "source": "https://replicate.com/bytedance/seedance-1-pro-fast" + }, + "replicate/bytedance/seedream-3": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/bytedance/seedream-3" + }, + "replicate/bytedance/seedream-4": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/bytedance/seedream-4" + }, + "replicate/bytedance/seedream-4.5": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/bytedance/seedream-4.5" + }, + "replicate/bytedance/seedream-5-lite": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://replicate.com/bytedance/seedream-5-lite" + }, + "replicate/datalab-to/ocr": { + "litellm_provider": "replicate", + "mode": "ocr", + "ocr_cost_per_page": 0.002, + "source": "https://replicate.com/datalab-to/ocr" + }, + "replicate/decart/lucy-edit-2": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://replicate.com/decart/lucy-edit-2" + }, + "replicate/easel/ai-avatars": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://replicate.com/easel/ai-avatars" + }, + "replicate/elevenlabs/dubbing": { + "litellm_provider": "replicate", + "mode": "audio_speech", + "output_cost_per_second": 0.0366667, + "source": "https://replicate.com/elevenlabs/dubbing" + }, + "replicate/elevenlabs/flash-v2.5": { + "input_cost_per_character": 5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/elevenlabs/flash-v2.5" + }, + "replicate/elevenlabs/turbo-v2.5": { + "input_cost_per_character": 5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/elevenlabs/turbo-v2.5" + }, + "replicate/elevenlabs/v2-multilingual": { + "input_cost_per_character": 0.0001, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/elevenlabs/v2-multilingual" + }, + "replicate/elevenlabs/v3": { + "input_cost_per_character": 0.0001, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/elevenlabs/v3" + }, + "replicate/flux-kontext-apps/cartoonify": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/cartoonify" + }, + "replicate/flux-kontext-apps/change-haircut": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/change-haircut" + }, + "replicate/flux-kontext-apps/depth-of-field": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/depth-of-field" + }, + "replicate/flux-kontext-apps/face-to-many-kontext": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/face-to-many-kontext" + }, + "replicate/flux-kontext-apps/filters": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/filters" + }, + "replicate/flux-kontext-apps/iconic-locations": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/iconic-locations" + }, + "replicate/flux-kontext-apps/impossible-scenarios": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/impossible-scenarios" + }, + "replicate/flux-kontext-apps/multi-image-kontext-max": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/flux-kontext-apps/multi-image-kontext-max" + }, + "replicate/flux-kontext-apps/multi-image-kontext-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/multi-image-kontext-pro" + }, + "replicate/flux-kontext-apps/multi-image-list": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/flux-kontext-apps/multi-image-list" + }, + "replicate/flux-kontext-apps/portrait-series": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/portrait-series" + }, + "replicate/flux-kontext-apps/professional-headshot": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/professional-headshot" + }, + "replicate/flux-kontext-apps/restore-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/restore-image" + }, + "replicate/flux-kontext-apps/restyle-video-frame": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/restyle-video-frame" + }, + "replicate/flux-kontext-apps/text-removal": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/flux-kontext-apps/text-removal" + }, + "replicate/google/gemini-2.5-flash-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.039, + "source": "https://replicate.com/google/gemini-2.5-flash-image" + }, + "replicate/google/gemini-3-flash": { + "input_cost_per_token": 5e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://replicate.com/google/gemini-3-flash", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/google/gemini-3.1-flash-tts": { + "input_cost_per_token": 2e-06, + "litellm_provider": "replicate", + "mode": "audio_speech", + "output_cost_per_token": 4e-05, + "source": "https://replicate.com/google/gemini-3.1-flash-tts" + }, + "replicate/google/gemini-3.1-pro": { + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "source": "https://replicate.com/google/gemini-3.1-pro", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/google/gemini-3.5-flash": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 9e-06, + "source": "https://replicate.com/google/gemini-3.5-flash", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/google/gemini-3.8-flash-lite-tts": { + "input_cost_per_token": 5e-07, + "litellm_provider": "replicate", + "mode": "audio_speech", + "output_cost_per_token": 6e-06, + "source": "https://replicate.com/google/gemini-3.8-flash-lite-tts" + }, + "replicate/google/gemini-3.8-flash-tts": { + "input_cost_per_token": 5e-07, + "litellm_provider": "replicate", + "mode": "audio_speech", + "output_cost_per_token": 9e-06, + "source": "https://replicate.com/google/gemini-3.8-flash-tts" + }, + "replicate/google/nano-banana": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.039, + "source": "https://replicate.com/google/nano-banana" + }, + "replicate/google/nano-banana-2": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.067, + "output_cost_per_image_1K": 0.067, + "output_cost_per_image_2K": 0.101, + "output_cost_per_image_4K": 0.151, + "source": "https://replicate.com/google/nano-banana-2" + }, + "replicate/google/nano-banana-2-lite": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.034, + "source": "https://replicate.com/google/nano-banana-2-lite" + }, + "replicate/google/nano-banana-2.1": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_1K": 0.0336, + "output_cost_per_image_2K": 0.0504, + "output_cost_per_image_4K": 0.1134, + "source": "https://replicate.com/google/nano-banana-2.1" + }, + "replicate/heygen/avatar-iv": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.075, + "source": "https://replicate.com/heygen/avatar-iv" + }, + "replicate/heygen/avatar-v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.075, + "source": "https://replicate.com/heygen/avatar-v" + }, + "replicate/heygen/lipsync-precision": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.0667, + "source": "https://replicate.com/heygen/lipsync-precision" + }, + "replicate/heygen/lipsync-speed": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.0333, + "source": "https://replicate.com/heygen/lipsync-speed" + }, + "replicate/heygen/video-agent": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.0333, + "source": "https://replicate.com/heygen/video-agent" + }, + "replicate/ibm-granite/granite-20b-code-instruct-8k": { + "input_cost_per_token": 1e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://replicate.com/ibm-granite/granite-20b-code-instruct-8k", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-3.0-2b-instruct": { + "input_cost_per_token": 3e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-3.0-2b-instruct", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-3.0-8b-instruct": { + "input_cost_per_token": 5e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-3.0-8b-instruct", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-3.1-2b-instruct": { + "input_cost_per_token": 3e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-3.1-2b-instruct", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-3.1-8b-instruct": { + "input_cost_per_token": 3e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-3.1-8b-instruct", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-3.2-8b-instruct": { + "input_cost_per_token": 3e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-3.2-8b-instruct", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-4.0-h-small": { + "input_cost_per_token": 6e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-4.0-h-small", + "supports_function_calling": true, + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-4.1-8b": { + "input_cost_per_token": 6e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-4.1-8b", + "supports_function_calling": true, + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-4.2-8b": { + "input_cost_per_token": 6e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-4.2-8b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-8b-code-instruct-128k": { + "input_cost_per_token": 5e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-8b-code-instruct-128k", + "supports_system_messages": true + }, + "replicate/ibm-granite/granite-speech-3.3-8b": { + "litellm_provider": "replicate", + "mode": "audio_transcription", + "output_cost_per_token": 4.9e-06, + "source": "https://replicate.com/ibm-granite/granite-speech-3.3-8b" + }, + "replicate/ibm-granite/granite-speech-4.1-2b": { + "input_cost_per_token": 3e-08, + "litellm_provider": "replicate", + "mode": "audio_transcription", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-speech-4.1-2b" + }, + "replicate/ibm-granite/granite-vision-3.3-2b": { + "input_cost_per_token": 0.0, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-vision-3.3-2b", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/ibm-granite/granite-vision-4.1-4b": { + "input_cost_per_token": 6e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/ibm-granite/granite-vision-4.1-4b", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/ideogram-ai/ideogram-v2": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/ideogram-ai/ideogram-v2" + }, + "replicate/ideogram-ai/ideogram-v2-turbo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://replicate.com/ideogram-ai/ideogram-v2-turbo" + }, + "replicate/ideogram-ai/ideogram-v2a": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/ideogram-ai/ideogram-v2a" + }, + "replicate/ideogram-ai/ideogram-v2a-turbo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/ideogram-ai/ideogram-v2a-turbo" + }, + "replicate/ideogram-ai/ideogram-v3-balanced": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://replicate.com/ideogram-ai/ideogram-v3-balanced" + }, + "replicate/ideogram-ai/ideogram-v3-quality": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.09, + "source": "https://replicate.com/ideogram-ai/ideogram-v3-quality" + }, + "replicate/ideogram-ai/ideogram-v3-turbo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/ideogram-ai/ideogram-v3-turbo" + }, + "replicate/ideogram-ai/ideogram-v4-balanced": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "source": "https://replicate.com/ideogram-ai/ideogram-v4-balanced" + }, + "replicate/ideogram-ai/ideogram-v4-quality": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://replicate.com/ideogram-ai/ideogram-v4-quality" + }, + "replicate/ideogram-ai/layerize": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.09, + "source": "https://replicate.com/ideogram-ai/layerize" + }, + "replicate/intelligent-utilities/html-to-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.001, + "source": "https://replicate.com/intelligent-utilities/html-to-image" + }, + "replicate/inworld/realtime-tts-1.5-max": { + "input_cost_per_character": 3.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/inworld/realtime-tts-1.5-max" + }, + "replicate/inworld/realtime-tts-1.5-mini": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/inworld/realtime-tts-1.5-mini" + }, + "replicate/inworld/realtime-tts-2": { + "input_cost_per_character": 2.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/inworld/realtime-tts-2" + }, + "replicate/kwaivgi/kling-lip-sync": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.014, + "source": "https://replicate.com/kwaivgi/kling-lip-sync" + }, + "replicate/kwaivgi/kling-v2.1-master": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.28, + "source": "https://replicate.com/kwaivgi/kling-v2.1-master" + }, + "replicate/kwaivgi/kling-v2.5-turbo-pro": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "source": "https://replicate.com/kwaivgi/kling-v2.5-turbo-pro" + }, + "replicate/lightricks/audio-to-video": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://replicate.com/lightricks/audio-to-video" + }, + "replicate/lightricks/ltx-2-distilled": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://replicate.com/lightricks/ltx-2-distilled" + }, + "replicate/lightricks/ltx-2-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_1080p": 0.04, + "output_cost_per_second_2k": 0.08, + "output_cost_per_second_4k": 0.16, + "source": "https://replicate.com/lightricks/ltx-2-fast" + }, + "replicate/lightricks/ltx-2-pro": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.12, + "output_cost_per_second_4k": 0.24, + "source": "https://replicate.com/lightricks/ltx-2-pro" + }, + "replicate/lightricks/ltx-2-retake": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://replicate.com/lightricks/ltx-2-retake" + }, + "replicate/lightricks/ltx-2.3-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.12, + "output_cost_per_second_4k": 0.24, + "source": "https://replicate.com/lightricks/ltx-2.3-fast" + }, + "replicate/lightricks/ltx-2.3-pro": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_1080p": 0.08, + "output_cost_per_second_2k": 0.16, + "output_cost_per_second_4k": 0.32, + "source": "https://replicate.com/lightricks/ltx-2.3-pro" + }, + "replicate/lightricks/ltx-2.5-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.06, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.12, + "output_cost_per_second_4k": 0.24, + "output_cost_per_second_720p": 0.03, + "source": "https://replicate.com/lightricks/ltx-2.5-fast" + }, + "replicate/luma/photon": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/luma/photon" + }, + "replicate/luma/photon-flash": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/luma/photon-flash" + }, + "replicate/luma/reframe-video": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.06, + "source": "https://replicate.com/luma/reframe-video" + }, + "replicate/meta/llama-4-maverick-instruct": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 9.5e-07, + "source": "https://replicate.com/meta/llama-4-maverick-instruct", + "supports_system_messages": true + }, + "replicate/meta/llama-4-scout-instruct": { + "input_cost_per_token": 1.7e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 6.5e-07, + "source": "https://replicate.com/meta/llama-4-scout-instruct", + "supports_system_messages": true + }, + "replicate/meta/llama-guard-4-12b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://replicate.com/meta/llama-guard-4-12b", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/meta/meta-llama-3-70b": { + "input_cost_per_token": 6.5e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.75e-06, + "source": "https://replicate.com/meta/meta-llama-3-70b" + }, + "replicate/meta/meta-llama-3-70b-instruct": { + "input_cost_per_token": 6.5e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.75e-06, + "source": "https://replicate.com/meta/meta-llama-3-70b-instruct", + "supports_system_messages": true + }, + "replicate/meta/meta-llama-3-8b": { + "input_cost_per_token": 5e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/meta/meta-llama-3-8b" + }, + "replicate/meta/meta-llama-3-8b-instruct": { + "input_cost_per_token": 5e-08, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://replicate.com/meta/meta-llama-3-8b-instruct", + "supports_system_messages": true + }, + "replicate/minimax/h3": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.13, + "output_cost_per_second_2k": 0.13, + "output_cost_per_second_768p": 0.08, + "source": "https://replicate.com/minimax/h3" + }, + "replicate/minimax/h3-highspeed": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.03, + "source": "https://replicate.com/minimax/h3-highspeed" + }, + "replicate/minimax/h3-lite": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "output_cost_per_second_480p": 0.0125, + "output_cost_per_second_768p": 0.02, + "source": "https://replicate.com/minimax/h3-lite" + }, + "replicate/minimax/image-01": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/minimax/image-01" + }, + "replicate/minimax/speech-02-hd": { + "input_cost_per_token": 0.0001, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-02-hd" + }, + "replicate/minimax/speech-02-turbo": { + "input_cost_per_token": 6e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-02-turbo" + }, + "replicate/minimax/speech-2.6-hd": { + "input_cost_per_token": 0.0001, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-2.6-hd" + }, + "replicate/minimax/speech-2.6-turbo": { + "input_cost_per_token": 6e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-2.6-turbo" + }, + "replicate/minimax/speech-2.8-hd": { + "input_cost_per_token": 0.0001, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-2.8-hd" + }, + "replicate/minimax/speech-2.8-turbo": { + "input_cost_per_token": 6e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/minimax/speech-2.8-turbo" + }, + "replicate/moonshotai/kimi-k2-thinking": { + "input_cost_per_token": 6.6e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.7e-06, + "source": "https://replicate.com/moonshotai/kimi-k2-thinking" + }, + "replicate/moonshotai/kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "source": "https://replicate.com/moonshotai/kimi-k2.5", + "supports_vision": true + }, + "replicate/moonshotai/kimi-k2.6": { + "input_cost_per_token": 9.5e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://replicate.com/moonshotai/kimi-k2.6", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/nightmareai/real-esrgan": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.002, + "source": "https://replicate.com/nightmareai/real-esrgan" + }, + "replicate/openai/gpt-4o-mini-transcribe": { + "input_cost_per_token": 3e-06, + "litellm_provider": "replicate", + "mode": "audio_transcription", + "output_cost_per_token": 5e-06, + "source": "https://replicate.com/openai/gpt-4o-mini-transcribe" + }, + "replicate/openai/gpt-4o-transcribe": { + "input_cost_per_token": 6e-06, + "litellm_provider": "replicate", + "mode": "audio_transcription", + "output_cost_per_token": 1e-05, + "source": "https://replicate.com/openai/gpt-4o-transcribe" + }, + "replicate/openai/gpt-5-pro": { + "input_cost_per_token": 1.5e-05, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 0.00012, + "source": "https://replicate.com/openai/gpt-5-pro", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.1": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://replicate.com/openai/gpt-5.1", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.2": { + "input_cost_per_token": 1.75e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "source": "https://replicate.com/openai/gpt-5.2", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.4": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://replicate.com/openai/gpt-5.4", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.6-luna": { + "input_cost_per_token": 1e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://replicate.com/openai/gpt-5.6-luna", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.6-sol": { + "input_cost_per_token": 2e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://replicate.com/openai/gpt-5.6-sol", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/openai/gpt-5.6-terra": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://replicate.com/openai/gpt-5.6-terra", + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/perceptron-ai-inc/isaac-0.1": { + "input_cost_per_token": 2.6e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2.17e-06, + "source": "https://replicate.com/perceptron-ai-inc/isaac-0.1", + "supports_vision": true + }, + "replicate/philz1337x/clarity-pro-upscaler": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_pixel": 3e-08, + "source": "https://replicate.com/philz1337x/clarity-pro-upscaler" + }, + "replicate/pixverse/lipsync": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://replicate.com/pixverse/lipsync" + }, + "replicate/playht/play-dialog": { + "litellm_provider": "replicate", + "mode": "audio_speech", + "output_cost_per_second": 0.001, + "source": "https://replicate.com/playht/play-dialog" + }, + "replicate/prunaai/flux-fast": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.005, + "source": "https://replicate.com/prunaai/flux-fast" + }, + "replicate/prunaai/flux-kontext-fast": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/prunaai/flux-kontext-fast" + }, + "replicate/prunaai/hidream-l1-fast": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.005, + "source": "https://replicate.com/prunaai/hidream-l1-fast" + }, + "replicate/prunaai/p-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.005, + "source": "https://replicate.com/prunaai/p-image" + }, + "replicate/prunaai/p-image-edit": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/prunaai/p-image-edit" + }, + "replicate/prunaai/p-image-edit-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/prunaai/p-image-edit-lora" + }, + "replicate/prunaai/p-image-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.005, + "source": "https://replicate.com/prunaai/p-image-lora" + }, + "replicate/prunaai/p-video-2-pro-avatar": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.04, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.08, + "output_cost_per_second_480p": 0.02, + "output_cost_per_second_4k": 0.12, + "output_cost_per_second_768p": 0.04, + "source": "https://replicate.com/prunaai/p-video-2-pro-avatar" + }, + "replicate/prunaai/p-video-2-pro-replace": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_2k": 0.07, + "output_cost_per_second_480p": 0.025, + "output_cost_per_second_4k": 0.09, + "output_cost_per_second_768p": 0.05, + "source": "https://replicate.com/prunaai/p-video-2-pro-replace" + }, + "replicate/prunaai/p-video-animate": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_720p": 0.03, + "source": "https://replicate.com/prunaai/p-video-animate" + }, + "replicate/prunaai/p-video-avatar": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.025, + "output_cost_per_second_1080p": 0.045, + "output_cost_per_second_720p": 0.025, + "source": "https://replicate.com/prunaai/p-video-avatar" + }, + "replicate/prunaai/p-video-replace": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.06, + "output_cost_per_second_720p": 0.03, + "source": "https://replicate.com/prunaai/p-video-replace" + }, + "replicate/prunaai/wan-2.2-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://replicate.com/prunaai/wan-2.2-image" + }, + "replicate/quiverai/arrow-1.1": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.2, + "source": "https://replicate.com/quiverai/arrow-1.1" + }, + "replicate/quiverai/arrow-1.1-max": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://replicate.com/quiverai/arrow-1.1-max" + }, + "replicate/quiverai/arrow-1.1-max-vectorize": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.2, + "source": "https://replicate.com/quiverai/arrow-1.1-max-vectorize" + }, + "replicate/quiverai/arrow-1.1-vectorize": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.15, + "source": "https://replicate.com/quiverai/arrow-1.1-vectorize" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-fusion": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-fusion" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-next-scene": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-next-scene" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-photo-to-anime": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-photo-to-anime" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-relight": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-relight" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-skin": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-skin" + }, + "replicate/qwen-edit-apps/qwen-image-edit-plus-lora-upscale": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen-edit-apps/qwen-image-edit-plus-lora-upscale" + }, + "replicate/qwen/qwen-edit-multiangle": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen/qwen-edit-multiangle" + }, + "replicate/qwen/qwen-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/qwen/qwen-image" + }, + "replicate/qwen/qwen-image-2": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://replicate.com/qwen/qwen-image-2" + }, + "replicate/qwen/qwen-image-2-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.075, + "source": "https://replicate.com/qwen/qwen-image-2-pro" + }, + "replicate/qwen/qwen-image-2512": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://replicate.com/qwen/qwen-image-2512" + }, + "replicate/qwen/qwen-image-edit": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen/qwen-image-edit" + }, + "replicate/qwen/qwen-image-edit-2511": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen/qwen-image-edit-2511" + }, + "replicate/qwen/qwen-image-edit-plus": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen/qwen-image-edit-plus" + }, + "replicate/qwen/qwen-image-edit-plus-lora": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/qwen/qwen-image-edit-plus-lora" + }, + "replicate/qwen/qwen3-7-plus": { + "input_cost_per_token": 2.76e-07, + "input_cost_per_token_above_256k_tokens": 8.26e-07, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 1.101e-06, + "output_cost_per_token_above_256k_tokens": 3.301e-06, + "source": "https://replicate.com/qwen/qwen3-7-plus", + "supports_system_messages": true, + "supports_vision": true + }, + "replicate/qwen/qwen3-tts": { + "input_cost_per_character": 2e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/qwen/qwen3-tts" + }, + "replicate/recraft-ai/recraft-20b": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.022, + "source": "https://replicate.com/recraft-ai/recraft-20b" + }, + "replicate/recraft-ai/recraft-20b-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.044, + "source": "https://replicate.com/recraft-ai/recraft-20b-svg" + }, + "replicate/recraft-ai/recraft-creative-upscale": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://replicate.com/recraft-ai/recraft-creative-upscale" + }, + "replicate/recraft-ai/recraft-crisp-upscale": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.006, + "source": "https://replicate.com/recraft-ai/recraft-crisp-upscale" + }, + "replicate/recraft-ai/recraft-remove-background": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/recraft-ai/recraft-remove-background" + }, + "replicate/recraft-ai/recraft-v3": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/recraft-ai/recraft-v3" + }, + "replicate/recraft-ai/recraft-v3-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/recraft-ai/recraft-v3-svg" + }, + "replicate/recraft-ai/recraft-v4": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/recraft-ai/recraft-v4" + }, + "replicate/recraft-ai/recraft-v4-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://replicate.com/recraft-ai/recraft-v4-pro" + }, + "replicate/recraft-ai/recraft-v4-pro-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "source": "https://replicate.com/recraft-ai/recraft-v4-pro-svg" + }, + "replicate/recraft-ai/recraft-v4-styles-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.1, + "source": "https://replicate.com/recraft-ai/recraft-v4-styles-pro" + }, + "replicate/recraft-ai/recraft-v4-styles-pro-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.12, + "source": "https://replicate.com/recraft-ai/recraft-v4-styles-pro-svg" + }, + "replicate/recraft-ai/recraft-v4-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/recraft-ai/recraft-v4-svg" + }, + "replicate/recraft-ai/recraft-v4.1": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/recraft-ai/recraft-v4.1" + }, + "replicate/recraft-ai/recraft-v4.1-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://replicate.com/recraft-ai/recraft-v4.1-pro" + }, + "replicate/recraft-ai/recraft-v4.1-pro-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://replicate.com/recraft-ai/recraft-v4.1-pro-svg" + }, + "replicate/recraft-ai/recraft-v4.1-svg": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/recraft-ai/recraft-v4.1-svg" + }, + "replicate/recraft-ai/recraft-v4.1-utility": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/recraft-ai/recraft-v4.1-utility" + }, + "replicate/recraft-ai/recraft-v4.1-utility-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.25, + "source": "https://replicate.com/recraft-ai/recraft-v4.1-utility-pro" + }, + "replicate/recraft-ai/recraft-vectorize": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://replicate.com/recraft-ai/recraft-vectorize" + }, + "replicate/resemble-ai/chatterbox": { + "input_cost_per_character": 2.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/resemble-ai/chatterbox" + }, + "replicate/resemble-ai/chatterbox-pro": { + "input_cost_per_character": 4e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/resemble-ai/chatterbox-pro" + }, + "replicate/resemble-ai/chatterbox-turbo": { + "input_cost_per_character": 2.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/resemble-ai/chatterbox-turbo" + }, + "replicate/retro-diffusion/rd-animation": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.07, + "source": "https://replicate.com/retro-diffusion/rd-animation" + }, + "replicate/runwayml/aleph-2": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.336, + "source": "https://replicate.com/runwayml/aleph-2" + }, + "replicate/runwayml/gen-4.5": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.12, + "source": "https://replicate.com/runwayml/gen-4.5" + }, + "replicate/runwayml/gen4-image-turbo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/runwayml/gen4-image-turbo" + }, + "replicate/runwayml/gen4-turbo": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/runwayml/gen4-turbo" + }, + "replicate/snowflake/snowflake-arctic-instruct": { + "input_cost_per_token": 2e-05, + "litellm_provider": "replicate", + "mode": "chat", + "output_cost_per_token": 2e-05, + "source": "https://replicate.com/snowflake/snowflake-arctic-instruct", + "supports_system_messages": true + }, + "replicate/stability-ai/stable-diffusion-3": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://replicate.com/stability-ai/stable-diffusion-3" + }, + "replicate/stability-ai/stable-diffusion-3.5-large": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.065, + "source": "https://replicate.com/stability-ai/stable-diffusion-3.5-large" + }, + "replicate/stability-ai/stable-diffusion-3.5-large-turbo": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://replicate.com/stability-ai/stable-diffusion-3.5-large-turbo" + }, + "replicate/stability-ai/stable-diffusion-3.5-medium": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://replicate.com/stability-ai/stable-diffusion-3.5-medium" + }, + "replicate/sync/lipsync-2": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/sync/lipsync-2" + }, + "replicate/sync/lipsync-2-pro": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.08325, + "source": "https://replicate.com/sync/lipsync-2-pro" + }, + "replicate/sync/react-1": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.167, + "source": "https://replicate.com/sync/react-1" + }, + "replicate/tencent/hunyuan-image-2.1": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://replicate.com/tencent/hunyuan-image-2.1" + }, + "replicate/tencent/hunyuan-image-3": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://replicate.com/tencent/hunyuan-image-3" + }, + "replicate/veed/fabric-1.0": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_480p": 0.08, + "output_cost_per_second_720p": 0.15, + "source": "https://replicate.com/veed/fabric-1.0" + }, + "replicate/wan-video/wan-2.2-s2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.02, + "source": "https://replicate.com/wan-video/wan-2.2-s2v" + }, + "replicate/wan-video/wan-2.5-i2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://replicate.com/wan-video/wan-2.5-i2v" + }, + "replicate/wan-video/wan-2.5-i2v-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.068, + "output_cost_per_second_1080p": 0.102, + "output_cost_per_second_720p": 0.068, + "source": "https://replicate.com/wan-video/wan-2.5-i2v-fast" + }, + "replicate/wan-video/wan-2.5-t2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.1, + "source": "https://replicate.com/wan-video/wan-2.5-t2v" + }, + "replicate/wan-video/wan-2.5-t2v-fast": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.068, + "output_cost_per_second_1080p": 0.102, + "output_cost_per_second_720p": 0.068, + "source": "https://replicate.com/wan-video/wan-2.5-t2v-fast" + }, + "replicate/wan-video/wan-2.6-i2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://replicate.com/wan-video/wan-2.6-i2v" + }, + "replicate/wan-video/wan-2.6-t2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://replicate.com/wan-video/wan-2.6-t2v" + }, + "replicate/wan-video/wan-2.7-i2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_720p": 0.1, + "source": "https://replicate.com/wan-video/wan-2.7-i2v" + }, + "replicate/wan-video/wan-2.7-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/wan-video/wan-2.7-image" + }, + "replicate/wan-video/wan-2.7-image-pro": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://replicate.com/wan-video/wan-2.7-image-pro" + }, + "replicate/wan-video/wan-2.7-r2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://replicate.com/wan-video/wan-2.7-r2v" + }, + "replicate/wan-video/wan-2.7-t2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://replicate.com/wan-video/wan-2.7-t2v" + }, + "replicate/wan-video/wan-2.7-videoedit": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://replicate.com/wan-video/wan-2.7-videoedit" + }, + "replicate/wan-video/wan2.6-i2v-flash": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_1080p": 0.075, + "output_cost_per_second_720p": 0.05, + "source": "https://replicate.com/wan-video/wan2.6-i2v-flash" + }, + "replicate/wavespeedai/qwen-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.025, + "source": "https://replicate.com/wavespeedai/qwen-image" + }, + "replicate/wavespeedai/wan-2.1-i2v-480p": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.09, + "source": "https://replicate.com/wavespeedai/wan-2.1-i2v-480p" + }, + "replicate/wavespeedai/wan-2.1-i2v-720p": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.25, + "source": "https://replicate.com/wavespeedai/wan-2.1-i2v-720p" + }, + "replicate/wavespeedai/wan-2.1-t2v-480p": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "source": "https://replicate.com/wavespeedai/wan-2.1-t2v-480p" + }, + "replicate/wavespeedai/wan-2.1-t2v-720p": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.24, + "source": "https://replicate.com/wavespeedai/wan-2.1-t2v-720p" + }, + "replicate/xai/grok-imagine-image": { + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://replicate.com/xai/grok-imagine-image" + }, + "replicate/xai/grok-imagine-image-quality": { + "input_cost_per_pixel": 1e-08, + "litellm_provider": "replicate", + "mode": "image_generation", + "output_cost_per_image": 0.06, + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://replicate.com/xai/grok-imagine-image-quality" + }, + "replicate/xai/grok-imagine-r2v": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/xai/grok-imagine-r2v" + }, + "replicate/xai/grok-imagine-video": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_480p": 0.05, + "output_cost_per_second_720p": 0.07, + "source": "https://replicate.com/xai/grok-imagine-video" + }, + "replicate/xai/grok-imagine-video-1.5": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.08, + "source": "https://replicate.com/xai/grok-imagine-video-1.5" + }, + "replicate/xai/grok-imagine-video-extension": { + "litellm_provider": "replicate", + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://replicate.com/xai/grok-imagine-video-extension" + }, + "replicate/xai/grok-text-to-speech": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "replicate", + "mode": "audio_speech", + "source": "https://replicate.com/xai/grok-text-to-speech" + }, "fal_ai/blackforestlabs/flux-3/edit-image": { "litellm_provider": "fal_ai", "metadata": {"notes":"the page states only the 1K (1 MP) price of $0.048 and the pricing API bills in megapixels at $0.048, so the per-pixel rate is $0.048 per megapixel and the per-image price is the default 1k output; 768sq, 1.5k, 2k and 4k outputs are priced by the per-pixel rate"}, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 3431cc97969..b35037ee651 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3508,16 +3508,16 @@ "responses": true, "text_completion": true, "embeddings": false, - "image_generations": false, + "image_generations": true, "image_edits": false, - "audio_transcriptions": false, - "audio_speech": false, + "audio_transcriptions": true, + "audio_speech": true, "rerank": false, - "ocr": false, + "ocr": true, "search": false, "moderations": false, "realtime": false, - "video_generations": false, + "video_generations": true, "batches": false, "messages": true, "interactions": true, From 4b498d1bba768bd81e4a434a14ea7ae2870dd01f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:08:48 +0000 Subject: [PATCH 060/189] chore(databricks): add claude-fable-5-2, date glm-5-2 retirement and fix llama-3-1-8b context from the supported-models page (#51076) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 59 ++++++++++++++++++- model_prices_and_context_window.json | 59 ++++++++++++++++++- 2 files changed, 112 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0ee57858b17..092987ce69e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -21878,7 +21878,7 @@ "thinking_always_on": true }, "databricks/claude-fable-latest": { - "alias_of": "databricks/databricks-claude-fable-5-1" + "alias_of": "databricks/databricks-claude-fable-5-2" }, "databricks/databricks-claude-haiku-4-5": { "cache_creation_input_token_cost": 1.24999e-06, @@ -22804,7 +22804,10 @@ "input_cost_per_token_priority": 2.45e-06, "output_cost_per_token_priority": 7.7e-06, "cache_read_input_token_cost_priority": 4.55e-07, - "cache_creation_input_token_cost_priority": 2.45e-06 + "cache_creation_input_token_cost_priority": 2.45e-06, + "deprecation_date": "2027-02-04", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "successor_model": "databricks/databricks-glm-5-3" }, "databricks/databricks-glm-5-3": { "cache_creation_input_token_cost": 1.4e-06, @@ -23471,7 +23474,7 @@ "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", - "max_input_tokens": 200000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." @@ -99977,5 +99980,55 @@ "supported_output_modalities": [ "video" ] + }, + "databricks/databricks-claude-fable-5-2": { + "cache_creation_input_token_cost": 8e-06, + "cache_read_input_token_cost": 8e-06, + "input_cost_per_token": 8e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "metadata": {"notes":"Claude Fable 5.2 on Databricks is available only through Unity Gateway with the model service name system.ai.claude-fable-5-2, and Databricks hosts it through Amazon Bedrock. The Databricks pricing page lists no row for it yet, so input and output are inherited from the catalog Bedrock row anthropic.claude-fable-5-2, no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "mode": "chat", + "output_cost_per_token": 4e-05, + "prompt_cache_min_tokens": 512.0, + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "supports_adaptive_thinking": true, + "supports_anthropic_thinking_payload": true, + "supports_assistant_prefill": false, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "thinking_always_on": true + }, + "databricks/system.ai.claude-fable-5-2": { + "cache_creation_input_token_cost": 8e-06, + "cache_read_input_token_cost": 8e-06, + "input_cost_per_token": 8e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "metadata": {"notes":"Claude Fable 5.2 on Databricks is available only through Unity Gateway with the model service name system.ai.claude-fable-5-2, and Databricks hosts it through Amazon Bedrock. The Databricks pricing page lists no row for it yet, so input and output are inherited from the catalog Bedrock row anthropic.claude-fable-5-2, no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "mode": "chat", + "output_cost_per_token": 4e-05, + "prompt_cache_min_tokens": 512.0, + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "supports_adaptive_thinking": true, + "supports_anthropic_thinking_payload": true, + "supports_assistant_prefill": false, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "thinking_always_on": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0ee57858b17..092987ce69e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -21878,7 +21878,7 @@ "thinking_always_on": true }, "databricks/claude-fable-latest": { - "alias_of": "databricks/databricks-claude-fable-5-1" + "alias_of": "databricks/databricks-claude-fable-5-2" }, "databricks/databricks-claude-haiku-4-5": { "cache_creation_input_token_cost": 1.24999e-06, @@ -22804,7 +22804,10 @@ "input_cost_per_token_priority": 2.45e-06, "output_cost_per_token_priority": 7.7e-06, "cache_read_input_token_cost_priority": 4.55e-07, - "cache_creation_input_token_cost_priority": 2.45e-06 + "cache_creation_input_token_cost_priority": 2.45e-06, + "deprecation_date": "2027-02-04", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "successor_model": "databricks/databricks-glm-5-3" }, "databricks/databricks-glm-5-3": { "cache_creation_input_token_cost": 1.4e-06, @@ -23471,7 +23474,7 @@ "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", - "max_input_tokens": 200000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." @@ -99977,5 +99980,55 @@ "supported_output_modalities": [ "video" ] + }, + "databricks/databricks-claude-fable-5-2": { + "cache_creation_input_token_cost": 8e-06, + "cache_read_input_token_cost": 8e-06, + "input_cost_per_token": 8e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "metadata": {"notes":"Claude Fable 5.2 on Databricks is available only through Unity Gateway with the model service name system.ai.claude-fable-5-2, and Databricks hosts it through Amazon Bedrock. The Databricks pricing page lists no row for it yet, so input and output are inherited from the catalog Bedrock row anthropic.claude-fable-5-2, no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "mode": "chat", + "output_cost_per_token": 4e-05, + "prompt_cache_min_tokens": 512.0, + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "supports_adaptive_thinking": true, + "supports_anthropic_thinking_payload": true, + "supports_assistant_prefill": false, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "thinking_always_on": true + }, + "databricks/system.ai.claude-fable-5-2": { + "cache_creation_input_token_cost": 8e-06, + "cache_read_input_token_cost": 8e-06, + "input_cost_per_token": 8e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "metadata": {"notes":"Claude Fable 5.2 on Databricks is available only through Unity Gateway with the model service name system.ai.claude-fable-5-2, and Databricks hosts it through Amazon Bedrock. The Databricks pricing page lists no row for it yet, so input and output are inherited from the catalog Bedrock row anthropic.claude-fable-5-2, no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "mode": "chat", + "output_cost_per_token": 4e-05, + "prompt_cache_min_tokens": 512.0, + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "supports_adaptive_thinking": true, + "supports_anthropic_thinking_payload": true, + "supports_assistant_prefill": false, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_mid_conversation_system": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "thinking_always_on": true } } From 11af2eb22dc799dff033c2d3ba71be991b6a8201 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Wed, 11 Nov 2026 10:57:54 +0900 Subject: [PATCH 061/189] refactor(rust_bridge): route every Rust-to-Python call through host modules (#50822) * refactor(rust_bridge): route every Rust-to-Python call through host modules Rust now reaches Python only through litellm/rust_bridge/host/ (plus the execution driver), and no Python module outside host/ imports it. Library calls that reproduce Python behavior (httpx and OSError error text, huggingface_hub downloads, the random generator, os.environ) go through host functions too. The rule is written into host/AGENTS.md The live litellm.model_cost table, its version and every Python writer move to litellm_core_utils/model_cost_table.py, and host/model_cost.py keeps only the functions Rust calls. The cost map fetch passes use_http_settings instead of handing Rust an http_settings() snapshot * refactor(rust_bridge): read REDIS_USERNAME from the process environment in Rust The Redis bridge asked Python for all of os.environ to read one key. CPython's os.environ writes through to the process environment, so Rust reads it with ProcessEnvironment like the rest of the module. Also drops doc comments that restated their code. Co-Authored-By: Claude Opus 5.5 * ci(rust): raise the native extension size cap to 55 MB Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/scripts/verify_linux_native_wheel.py | 2 +- litellm-rust/crates/python-bridge/AGENTS.md | 1 + litellm-rust/crates/python-bridge/src/http.rs | 15 +- .../crates/python-bridge/src/logger/mod.rs | 2 +- .../python-bridge/src/model_cost/fetch.rs | 9 +- .../crates/python-bridge/src/preflight.rs | 10 +- .../python-bridge/src/python_settings.rs | 9 +- .../crates/python-bridge/src/redis/breaker.rs | 2 +- .../src/redis/client/credentials.rs | 4 +- .../python-bridge/src/redis/client/mod.rs | 65 +----- .../crates/python-bridge/src/router/host.rs | 10 +- .../python-bridge/src/routes/messages/host.rs | 17 +- .../src/routes/responses/host.rs | 9 +- .../python-bridge/src/secrets/callback.rs | 38 ++-- .../crates/python-bridge/src/secrets/mod.rs | 1 + .../python-bridge/src/secrets/mutation.rs | 28 +-- .../python-bridge/src/secrets/python.rs | 4 +- .../python-bridge/src/secrets/runtime.rs | 4 +- .../crates/python-bridge/src/secrets/vault.rs | 32 +-- .../crates/python-bridge/src/tokenizer.rs | 24 +-- litellm/litellm_core_utils/AGENTS.md | 16 +- .../litellm_core_utils/get_model_cost_map.py | 27 ++- .../litellm_core_utils/model_cost_table.py | 112 ++++++++++ .../bedrock/audio_transcription/__init__.py | 3 +- .../llms/fireworks_ai/chat/transformation.py | 2 +- .../proxy/common_utils/model_deprecation.py | 2 +- litellm/proxy/proxy_server.py | 2 +- litellm/rust_bridge/AGENTS.md | 2 +- litellm/rust_bridge/_native.pyi | 29 +-- litellm/rust_bridge/host/AGENTS.md | 50 +++-- litellm/rust_bridge/{ => host}/logger.py | 0 .../rust_bridge/host/model_capabilities.py | 2 +- litellm/rust_bridge/host/model_cost.py | 117 +---------- litellm/rust_bridge/{ => host}/preflight.py | 0 litellm/rust_bridge/host/randomness.py | 11 + litellm/rust_bridge/host/routes/responses.py | 6 + litellm/rust_bridge/host/secret_manager.py | 193 ++++-------------- litellm/rust_bridge/host/timeouts.py | 10 +- litellm/rust_bridge/host/tokenizers.py | 17 ++ litellm/rust_bridge/public_call.py | 10 + litellm/rust_bridge/responses/detectors.py | 10 +- litellm/rust_bridge/responses/websocket.py | 2 +- litellm/secret_managers/native_config.py | 159 +++++++++++++++ litellm/utils.py | 14 +- ...edrock_native_output_register_model_sdk.py | 2 +- .../sdk/test_model_cost_fetch_sdk.py | 2 +- .../unit/caching/test_llm_caching_handler.py | 2 +- tests/unit/conftest.py | 2 +- tests/unit/integrations/arize/test_arize.py | 2 +- .../integrations/arize/test_arize_phoenix.py | 2 +- tests/unit/integrations/opik/test_opik.py | 2 +- .../test_anthropic_cache_control_hook.py | 2 +- tests/unit/integrations/test_helicone.py | 2 +- ...llm_core_utils_prompt_templates_factory.py | 2 +- .../test_get_llm_provider_logic.py | 2 +- .../test_model_cost_table.py | 57 ++++++ .../native/test_anthropic_messages_handler.py | 2 +- .../test_azure_chat_gpt_transformation.py | 2 +- .../test_azure_ai_flux2_image_generation.py | 2 +- .../test_azure_ai_fw_models_metadata.py | 2 +- .../test_azure_ai_kimi_k26_metadata.py | 2 +- .../test_amazon_nova_canvas_image_edit.py | 2 +- .../llms/custom_httpx/test_http_handler.py | 2 +- tests/unit/llms/openai/test_openai.py | 2 +- .../parallel_ai/test_parallel_ai_search.py | 2 +- tests/unit/llms/test_custom_llm.py | 2 +- ...test_vertex_and_google_ai_studio_gemini.py | 2 +- .../common_utils/test_config_sync_pubsub.py | 2 +- .../common_utils/test_http_parsing_utils.py | 2 +- .../common_utils/test_model_deprecation.py | 2 +- .../proxy/hooks/test_dynamic_rate_limiter.py | 2 +- .../hooks/test_prompt_injection_detection.py | 2 +- .../test_sso_helper_utils.py | 2 +- .../proxy_server/test_routes_model_info.py | 2 +- .../proxy/proxy_server/test_routes_utils.py | 2 +- ...test_proxy_server_endpoints_and_startup.py | 2 +- tests/unit/responses/test_responses_utils.py | 2 +- .../unit/router_strategy/test_lowest_cost.py | 2 +- .../router_strategy/test_lowest_tpm_rpm_v2.py | 2 +- .../unit/rust_bridge/host/test_model_cost.py | 61 +----- .../rust_bridge/{ => host}/test_preflight.py | 6 +- .../rust_bridge/host/test_secret_manager.py | 3 +- tests/unit/rust_bridge/test_logger.py | 3 +- tests/unit/test_cost_calculator.py | 2 +- tests/unit/test_main.py | 2 +- tests/unit/test_model_cost_aliases.py | 2 +- .../test_no_top_level_test_invocations.py | 2 +- tests/unit/test_pydantic_namespaces.py | 2 +- .../test_register_model_custom_pricing.py | 4 +- tests/unit/test_router/test_router.py | 2 +- .../unit/test_router_model_cost_isolation.py | 2 +- tests/unit/test_scheduler.py | 2 +- tests/unit/test_utils.py | 18 +- tests/unit/test_utils_get_model_info.py | 2 +- tests/unit/types/test_files.py | 2 +- 95 files changed, 701 insertions(+), 621 deletions(-) create mode 100644 litellm/litellm_core_utils/model_cost_table.py rename litellm/rust_bridge/{ => host}/logger.py (100%) rename litellm/rust_bridge/{ => host}/preflight.py (100%) create mode 100644 litellm/rust_bridge/host/randomness.py create mode 100644 litellm/rust_bridge/host/tokenizers.py create mode 100644 litellm/secret_managers/native_config.py create mode 100644 tests/unit/litellm_core_utils/test_model_cost_table.py rename tests/unit/rust_bridge/{ => host}/test_preflight.py (92%) diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index 4c4cb399dce..571092263ed 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -214,7 +214,7 @@ def main( native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") - native_size_limit: Final = 48_000_000 + native_size_limit: Final = 55_000_000 native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index 26858b2afe2..b18a1e2ca19 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -81,6 +81,7 @@ GIL handling to `litellm-host-python`. ## Bridge Shape +- Reach Python only through `litellm/rust_bridge/host/` and the `litellm.rust_bridge.execution` driver. Never import another LiteLLM module or read a `litellm` global. `litellm/rust_bridge/host/AGENTS.md` lists what may be imported directly - Prefer one stable method per top-level LiteLLM route, for example `messages(...)`, calling the matching `litellm-inference-` entrypoint. - Do not add one exported PyO3 function per provider helper unless there is a diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index c6a8b98747c..57c58c3f7cd 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -112,19 +112,22 @@ pub(crate) fn call_config( Ok(resolved(py, settings)) } -/// The client config for a value of `litellm.rust_bridge.host.settings.http_settings()` the caller handed in, under -/// the process environment. `None` reads the environment alone pub(crate) fn settings_config( py: Python<'_>, - http_settings: Option<&Bound<'_, PyAny>>, + use_http_settings: bool, ) -> PyResult { - let configured = match http_settings { - Some(value) => configured(&PythonSettings::Http.snapshot(value.clone()))?, + let snapshot = if use_http_settings { + PythonSettings::Http.read_or_unset(py)? + } else { + None + }; + let layer = match snapshot { + Some(snapshot) => configured(&snapshot)?, None => HttpSettingsLayer::default(), }; let settings = HttpSettings::from_layers([ HttpSettingsLayer::from_environment(&ProcessEnvironment), - configured, + layer, ]); Ok(resolved(py, settings)) } diff --git a/litellm-rust/crates/python-bridge/src/logger/mod.rs b/litellm-rust/crates/python-bridge/src/logger/mod.rs index 8f40d94f458..5032b418404 100644 --- a/litellm-rust/crates/python-bridge/src/logger/mod.rs +++ b/litellm-rust/crates/python-bridge/src/logger/mod.rs @@ -7,7 +7,7 @@ use litellm_tracing::{DiagnosticInput, Level, Logger, Metadata, Policy, Processo use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; -const MODULE: &str = "litellm.rust_bridge.logger"; +const MODULE: &str = "litellm.rust_bridge.host.logger"; type NativeDiagnosticOutput = (String, Option, Option, Vec, bool); #[pyclass] diff --git a/litellm-rust/crates/python-bridge/src/model_cost/fetch.rs b/litellm-rust/crates/python-bridge/src/model_cost/fetch.rs index 77ec88cc1b9..dff6296cf91 100644 --- a/litellm-rust/crates/python-bridge/src/model_cost/fetch.rs +++ b/litellm-rust/crates/python-bridge/src/model_cost/fetch.rs @@ -20,13 +20,12 @@ pub(crate) struct NativeCostMapSource { #[pymethods] impl NativeCostMapSource { - /// `http_settings` is `litellm.rust_bridge.host.settings.http_settings()`, or `None` for the environment alone #[new] - #[pyo3(signature = (url, *, http_settings))] - fn new(py: Python<'_>, url: String, http_settings: Option>) -> PyResult { + #[pyo3(signature = (url, *, use_http_settings))] + fn new(py: Python<'_>, url: String, use_http_settings: bool) -> PyResult { Ok(Self { url, - http: crate::http::settings_config(py, http_settings.as_ref())?, + http: crate::http::settings_config(py, use_http_settings)?, }) } @@ -226,7 +225,7 @@ mod tests { }); Python::initialize(); Python::attach(|py| { - let source = NativeCostMapSource::new(py, url, None).unwrap(); + let source = NativeCostMapSource::new(py, url, false).unwrap(); let policy = Bound::new(py, NativeFetchPolicy::new(3, 30.0, 5.0, 2.0).unwrap()).unwrap(); let outcome = source.fetch_blocking(py, policy.borrow(), None).unwrap(); diff --git a/litellm-rust/crates/python-bridge/src/preflight.rs b/litellm-rust/crates/python-bridge/src/preflight.rs index bc692498880..2a07723b94c 100644 --- a/litellm-rust/crates/python-bridge/src/preflight.rs +++ b/litellm-rust/crates/python-bridge/src/preflight.rs @@ -7,7 +7,7 @@ use pyo3::{ }; use strum::{IntoStaticStr, VariantArray}; -const MODULE: &str = "litellm.rust_bridge.preflight"; +const MODULE: &str = "litellm.rust_bridge.host.preflight"; /// The litellm globals the preflight still reads through Python. `preflight_contract.json` /// pins each function's parameters on both sides. @@ -115,7 +115,7 @@ mod tests { /// tests that install it run one at a time. static PREFLIGHT_MODULE: Mutex<()> = Mutex::new(()); - /// A fresh stand-in for `litellm.rust_bridge.preflight` that records every call, then + /// A fresh stand-in for `litellm.rust_bridge.host.preflight` that records every call, then /// `script` run against it with the module bound as `preflight`. fn preflight_stubs<'py>(py: Python<'py>, script: &std::ffi::CStr) -> Bound<'py, PyDict> { let locals = PyDict::new(py); @@ -124,15 +124,15 @@ mod tests { import sys import types -for name in ('litellm', 'litellm.rust_bridge'): +for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host'): sys.modules.setdefault(name, types.ModuleType(name)) -preflight = types.ModuleType('litellm.rust_bridge.preflight') +preflight = types.ModuleType('litellm.rust_bridge.host.preflight') preflight.warnings = [] preflight.checked = [] preflight.credential_list = lambda: [] preflight.warn_unknown_credential = lambda name, loaded: preflight.warnings.append((name, loaded)) preflight.check_limits = lambda kwargs: preflight.checked.append(kwargs) -sys.modules['litellm.rust_bridge.preflight'] = preflight +sys.modules['litellm.rust_bridge.host.preflight'] = preflight ", Some(&locals), Some(&locals), diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index 93377abb7e9..1cb1813049c 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -26,10 +26,15 @@ pub(crate) struct Snapshot<'py> { value: Bound<'py, PyAny>, } -impl Snapshot<'_> { +impl<'py> Snapshot<'py> { pub(crate) fn read(&self, spec: &FieldSpec) -> Result { spec.read(&self.value, self.group.into()) } + + pub(crate) fn attribute(&self, name: &str) -> PyResult>> { + let value = self.value.getattr(name)?; + Ok((!value.is_none()).then_some(value)) + } } impl PythonSettings { @@ -56,7 +61,7 @@ impl PythonSettings { } } - /// A snapshot of a value Python handed in, read with the same field rules as the accessor's + #[cfg(test)] pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> { Snapshot { group: self, value } } diff --git a/litellm-rust/crates/python-bridge/src/redis/breaker.rs b/litellm-rust/crates/python-bridge/src/redis/breaker.rs index 733cf1bd969..5fdac76cb02 100644 --- a/litellm-rust/crates/python-bridge/src/redis/breaker.rs +++ b/litellm-rust/crates/python-bridge/src/redis/breaker.rs @@ -72,7 +72,7 @@ impl NativeRedisBreaker { redis_kwargs: RedisSpec, settings: BreakerSettings, ) -> PyResult { - let endpoint = resolved_endpoint(py, &redis_kwargs)?; + let endpoint = resolved_endpoint(&redis_kwargs); Self::build(py, &settings, endpoint) } diff --git a/litellm-rust/crates/python-bridge/src/redis/client/credentials.rs b/litellm-rust/crates/python-bridge/src/redis/client/credentials.rs index 21a37947c30..6995981a03e 100644 --- a/litellm-rust/crates/python-bridge/src/redis/client/credentials.rs +++ b/litellm-rust/crates/python-bridge/src/redis/client/credentials.rs @@ -5,7 +5,7 @@ use litellm_redis_connect::{CredentialSource, CredentialStage, Error, Expiring, use litellm_secret_value::SecretValue; use pyo3::prelude::*; -const MODULE: &str = "litellm.rust_bridge.host.redis_credentials"; +const CREDENTIALS_MODULE: &str = "litellm.rust_bridge.host.redis_credentials"; /// A redis-py `CredentialProvider` the SDK caller passed in `cache_kwargs`, asked through the host /// module the way redis-py's async connection asks it. @@ -29,7 +29,7 @@ impl CredentialSource for PythonCredentialSource { let context = self.context.clone(); Box::pin(async move { let (username, password) = attach_blocking(context, move |py| { - py.import(MODULE)? + py.import(CREDENTIALS_MODULE)? .getattr("credentials")? .call1((provider.bind(py),))? .extract::<(Option, String)>() diff --git a/litellm-rust/crates/python-bridge/src/redis/client/mod.rs b/litellm-rust/crates/python-bridge/src/redis/client/mod.rs index 989f3dc9247..5e5ab7e6f5a 100644 --- a/litellm-rust/crates/python-bridge/src/redis/client/mod.rs +++ b/litellm-rust/crates/python-bridge/src/redis/client/mod.rs @@ -3,7 +3,7 @@ mod credentials; use std::{collections::BTreeMap, sync::Arc, time::Duration}; use litellm_circuit_breaker::Config as BreakerConfig; -use litellm_core_utils::settings::ProcessEnvironment; +use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; use litellm_host_python::PythonContext; use litellm_python_compat::number::{self, SecondsError}; use litellm_redis_client::{RedisClient, RedisEvents}; @@ -92,7 +92,7 @@ fn shared_client( ) -> PyResult> { let environment: SharedEnvironment = Arc::new(ProcessEnvironment); let logger = crate::logger::capture(py); - let resolved = match logger.scope(|| resolve_spec(py, &spec))? { + let resolved = match logger.scope(|| resolve_spec(&spec)) { Ok(resolution) => { logger.scope(|| { for ignored in &resolution.ignored { @@ -174,39 +174,23 @@ impl NativeRedisClient { } /// The Redis server `spec` names, whatever its db, credentials or namespace, or none when the Rust client cannot reach it. -pub(crate) fn resolved_endpoint(py: Python<'_>, spec: &RedisSpec) -> PyResult> { - Ok(resolve_spec(py, spec)? +pub(crate) fn resolved_endpoint(spec: &RedisSpec) -> Option { + resolve_spec(spec) .ok() - .and_then(|resolution| resolution.target.endpoint())) + .and_then(|resolution| resolution.target.endpoint()) } // A live registry entry's client keeps its provider alive, so no live entry shares its address -fn resolve_spec( - py: Python<'_>, - spec: &RedisSpec, -) -> PyResult> { +fn resolve_spec(spec: &RedisSpec) -> Result { let handle = spec .credential_provider .as_ref() .map(|provider| ProviderHandle::new(provider.as_ptr() as usize as u64)); - resolve(&spec.kwargs, handle, &py.import("os")?.getattr("environ")?) -} - -/// What redis-py makes of the kwargs, with `REDIS_USERNAME` read from `environ` the way Python's Azure AD auth -/// reads `os.environ`. -fn resolve( - kwargs: &Kwargs, - provider: Option, - environ: &Bound<'_, PyAny>, -) -> PyResult> { - let redis_username: Option = environ - .call_method1("get", ("REDIS_USERNAME",))? - .extract()?; - Ok(litellm_redis_resolve::interpret( - kwargs, - provider, - redis_username.as_deref(), - )) + litellm_redis_resolve::interpret( + &spec.kwargs, + handle, + ProcessEnvironment.get("REDIS_USERNAME").as_deref(), + ) } fn unresolved( @@ -257,9 +241,6 @@ fn seconds_error(error: SecondsError) -> PyErr { #[cfg(test)] mod tests { - use std::ffi::CStr; - - use litellm_redis_types::Auth; use rstest::rstest; use serde_json::json; @@ -270,30 +251,6 @@ mod tests { litellm_redis_resolve::kwargs(fields) } - #[rstest] - #[case::set(c"environ = {'REDIS_USERNAME': 'azure-user'}", Some("azure-user"))] - #[case::empty_is_unset(c"environ = {'REDIS_USERNAME': ''}", None)] - fn azure_auth_takes_its_username_from_the_environ_like_python( - #[case] source: &CStr, - #[case] expected: Option<&str>, - ) { - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run(source, Some(&locals), Some(&locals)).unwrap(); - let environ = locals.get_item("environ").unwrap().unwrap(); - let kwargs = kwargs(json!({"host": "redis", "azure_redis_ad_token": "true"})); - - let resolved = resolve(&kwargs, None, &environ).unwrap().unwrap(); - - assert!( - matches!(&resolved.auth, Auth::AzureAd(azure) if azure.username.as_deref() == expected), - "{:?}", - resolved.auth - ); - }); - } - #[rstest] #[case::no_target(json!({"port": 6379}), Some("ValueError"))] #[case::unenforceable(json!({"host": "redis", "ssl": true, "ssl_validate_ocsp": true}), Some("TypeError"))] diff --git a/litellm-rust/crates/python-bridge/src/router/host.rs b/litellm-rust/crates/python-bridge/src/router/host.rs index 27f9fe347e0..04d19e70edc 100644 --- a/litellm-rust/crates/python-bridge/src/router/host.rs +++ b/litellm-rust/crates/python-bridge/src/router/host.rs @@ -717,6 +717,8 @@ pub(crate) fn deployment_record(deployment: &Deployment) -> Value { deployment.record() } +const RANDOMNESS_MODULE: &str = "litellm.rust_bridge.host.randomness"; + /// The process's `random` module: a Python router draws its shuffle and retry jitter there, so /// a seeded process picks the same deployments on either core. The bare module functions share /// one generator, so these calls interleave with every other draw the process makes. @@ -725,7 +727,9 @@ pub(crate) struct PythonRandomness; impl Randomness for PythonRandomness { fn next_f64(&self) -> f64 { Python::attach(|py| -> PyResult { - py.import("random")?.call_method0("random")?.extract() + py.import(RANDOMNESS_MODULE)? + .call_method0("next_float")? + .extract() }) .unwrap_or_else(|_| ThreadRandomness.next_f64()) } @@ -733,8 +737,8 @@ impl Randomness for PythonRandomness { fn index_below(&self, len: usize) -> usize { // `random.choice` is `seq[self._randbelow(len(seq))]`, and so is `randrange(len)`. Python::attach(|py| -> PyResult { - py.import("random")? - .call_method1("randrange", (len,))? + py.import(RANDOMNESS_MODULE)? + .call_method1("index_below", (len,))? .extract() }) .unwrap_or_else(|_| ThreadRandomness.index_below(len)) diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 2a5f79ca393..c975ac339c2 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -24,11 +24,12 @@ use crate::{ ArgumentError, optional_timeout, project_optional_fields, public_response, python_timeout_seconds, }, - python_settings::missing_module, + python_settings::PythonSettings, routes::RequestView, }; const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages"; +const MODEL_CAPABILITIES_MODULE: &str = "litellm.rust_bridge.host.model_capabilities"; fn merge_headers( forwarded: Option>, @@ -40,16 +41,10 @@ fn merge_headers( (!merged.is_empty()).then_some(merged) } -/// Reads `litellm.`, the module global a param spec names, the way -/// `VertexBase.get_vertex_ai_project` reads `litellm.vertex_project`. fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult>> { - let module = match py.import("litellm") { - Ok(module) => module, - Err(error) if missing_module(py, &error, "litellm")? => return Ok(None), - Err(error) => return Err(error), - }; - let value = module.getattr(name)?; - Ok((!value.is_none()).then_some(value)) + PythonSettings::ProviderDefaults + .read_or_unset(py)? + .map_or(Ok(None), |defaults| defaults.attribute(name)) } /// The caller's litellm params, read from the kwargs by the names the typed params declare, @@ -173,7 +168,7 @@ impl MessagesPythonHost { ) -> PyResult { let module = py.import(ROUTE_HOST_MODULE)?; let capabilities: MessagesModelCapabilities = from_py( - &py.import("litellm.rust_bridge.host.model_capabilities")? + &py.import(MODEL_CAPABILITIES_MODULE)? .getattr("anthropic_model_capabilities")? .call1((model, custom_llm_provider))?, )?; diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs index b797e334c6a..019f8452737 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs @@ -44,12 +44,9 @@ pub(super) fn project( let value = match (name.as_str(), value) { ("previous_response_id", serde_json::Value::String(id)) => { let decoded: String = py - .import("litellm.responses.utils")? - .getattr("ResponsesAPIRequestUtils")? - .call_method1( - "decode_previous_response_id_to_original_previous_response_id", - (id,), - )? + .import(super::ROUTE_HOST_MODULE)? + .getattr("original_previous_response_id")? + .call1((id,))? .extract()?; serde_json::Value::String(decoded) } diff --git a/litellm-rust/crates/python-bridge/src/secrets/callback.rs b/litellm-rust/crates/python-bridge/src/secrets/callback.rs index af6485a56e1..a2afb42b540 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/callback.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/callback.rs @@ -15,7 +15,6 @@ use pyo3::{ use super::error::{external_error, read_error}; -const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler"; const DIAGNOSTICS_MODULE: &str = "litellm.rust_bridge.host.diagnostics"; /// A secret manager whose reads execute in Python: a custom manager, a legacy compatible @@ -61,8 +60,8 @@ impl PythonClient { None => kwargs.set_item("key_management_settings", py.None())?, } let result = py - .import(HANDLER_MODULE)? - .getattr("get_secret_from_manager")? + .import(super::SECRET_MANAGER_MODULE)? + .getattr("read_from_manager")? .call((), Some(&kwargs))?; if result.is_instance_of::() { result.extract().map(Some) @@ -133,7 +132,8 @@ mod tests { use litellm_host_python::PythonContext; - use super::{DIAGNOSTICS_MODULE, HANDLER_MODULE, PythonSecretManager, python_name}; + use super::{DIAGNOSTICS_MODULE, PythonSecretManager, python_name}; + use crate::secrets::SECRET_MANAGER_MODULE; use crate::secrets::python_error; /// `sys.modules` is interpreter-global, so tests that install or rely on the handler module @@ -169,12 +169,12 @@ class Manager: raise failure manager = Manager() import sys, types -for name in ('litellm', 'litellm.secret_managers'): +for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host'): sys.modules.setdefault(name, types.ModuleType(name)) -handler = sys.modules.setdefault('litellm.secret_managers.secret_manager_handler', types.ModuleType('litellm.secret_managers.secret_manager_handler')) -def get_secret_from_manager(**kwargs): +handler = sys.modules.setdefault('litellm.rust_bridge.host.secret_manager', types.ModuleType('litellm.rust_bridge.host.secret_manager')) +def read_from_manager(**kwargs): return kwargs['client'].sync_read_secret(kwargs['secret_name']) -handler.get_secret_from_manager = get_secret_from_manager +handler.read_from_manager = read_from_manager ", Some(&locals), Some(&locals), @@ -317,7 +317,7 @@ sys.modules.setdefault('litellm._logging', logging) }); } - /// Installs a fake `get_secret_from_manager` that records its kwargs, runs `body`, and + /// Installs a fake `read_from_manager` that records its kwargs, runs `body`, and /// removes the fake handler again; parent package stubs persist for concurrent tests. /// Callers hold `handler_guard` before attaching so the GIL is never held while waiting on it. fn with_fake_handler<'py>(py: Python<'py>, body: impl FnOnce(&Bound<'py, PyDict>)) { @@ -325,16 +325,16 @@ sys.modules.setdefault('litellm._logging', logging) py.run( c" import sys, types -previous_handler = sys.modules.get('litellm.secret_managers.secret_manager_handler') +previous_handler = sys.modules.get('litellm.rust_bridge.host.secret_manager') calls = [] -def get_secret_from_manager(**kwargs): +def read_from_manager(**kwargs): calls.append(kwargs) return 'handled-' + kwargs['secret_name'] -handler = types.ModuleType('litellm.secret_managers.secret_manager_handler') -handler.get_secret_from_manager = get_secret_from_manager -for name in ('litellm', 'litellm.secret_managers'): +handler = types.ModuleType('litellm.rust_bridge.host.secret_manager') +handler.read_from_manager = read_from_manager +for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host'): sys.modules.setdefault(name, types.ModuleType(name)) -sys.modules['litellm.secret_managers.secret_manager_handler'] = handler +sys.modules['litellm.rust_bridge.host.secret_manager'] = handler ", Some(&locals), Some(&locals), @@ -344,9 +344,9 @@ sys.modules['litellm.secret_managers.secret_manager_handler'] = handler py.run( c" if previous_handler is None: - sys.modules.pop('litellm.secret_managers.secret_manager_handler', None) + sys.modules.pop('litellm.rust_bridge.host.secret_manager', None) else: - sys.modules['litellm.secret_managers.secret_manager_handler'] = previous_handler + sys.modules['litellm.rust_bridge.host.secret_manager'] = previous_handler ", Some(&locals), Some(&locals), @@ -366,7 +366,7 @@ else: with_fake_handler(py, |locals| { locals.set_item("expression", expression).unwrap(); py.run( - c"handler.get_secret_from_manager = lambda **kwargs: eval(expression)", + c"handler.read_from_manager = lambda **kwargs: eval(expression)", Some(locals), Some(locals), ) @@ -417,7 +417,7 @@ else: reader.client.read(py, "API_KEY").unwrap().as_deref(), Some("handled-API_KEY") ); - assert!(py.import(HANDLER_MODULE).is_ok()); + assert!(py.import(SECRET_MANAGER_MODULE).is_ok()); let calls = locals.get_item("calls").unwrap().unwrap(); let call = calls.get_item(0).unwrap().cast_into::().unwrap(); assert!(call.get_item("client").unwrap().unwrap().is(&client)); diff --git a/litellm-rust/crates/python-bridge/src/secrets/mod.rs b/litellm-rust/crates/python-bridge/src/secrets/mod.rs index 57042e6a32e..0840ecdfc6c 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/mod.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/mod.rs @@ -21,6 +21,7 @@ use resolved::ResolvedSecrets; use crate::{coercion::FieldSpec, python_settings::PythonSettings}; +const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager"; const NATIVE: FieldSpec = FieldSpec::new("native", |field| field.schema_bool()); /// Where a Rust route reads provider secrets from. Python's `get_secret_str` when the settings diff --git a/litellm-rust/crates/python-bridge/src/secrets/mutation.rs b/litellm-rust/crates/python-bridge/src/secrets/mutation.rs index b04060651ac..47cd36c4015 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/mutation.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/mutation.rs @@ -2,7 +2,7 @@ use super::operations::{PythonMutationError, PythonMutationResponse}; use litellm_error::Report; use litellm_host_python::{json_loads, to_py}; use litellm_secrets::cyberark; -use pyo3::{prelude::*, types::PyDict}; +use pyo3::prelude::*; pub(super) fn mutation_value( result: Result, @@ -62,14 +62,11 @@ fn cyberark_message( http_message(py, "POST", url, status)? } cyberark::Error::Http(source) if authentication => match os_error_code(&source) { - Some(code) => { - let reason = py.import("os")?.getattr("strerror")?.call1((code,))?; - py.import("builtins")? - .getattr("OSError")? - .call1((code, reason))? - .str()? - .extract()? - } + Some(code) => py + .import(super::SECRET_MANAGER_MODULE)? + .getattr("os_error_message")? + .call1((code,))? + .extract()?, None => Report::new(&cyberark::Error::Http(source)).to_string(), }, cyberark::Error::Http(source) if source.is_connect() => { @@ -103,13 +100,8 @@ pub(super) fn http_message( url: &str, status: u16, ) -> PyResult { - let httpx = py.import("httpx")?; - let request = httpx.getattr("Request")?.call1((method, url))?; - let kwargs = PyDict::new(py); - kwargs.set_item("request", request)?; - let response = httpx.getattr("Response")?.call((status,), Some(&kwargs))?; - match response.call_method0("raise_for_status") { - Err(error) => error.value(py).str()?.extract(), - Ok(_) => Ok(format!("HTTP {status}")), - } + py.import(super::SECRET_MANAGER_MODULE)? + .getattr("http_status_message")? + .call1((method, url, status))? + .extract() } diff --git a/litellm-rust/crates/python-bridge/src/secrets/python.rs b/litellm-rust/crates/python-bridge/src/secrets/python.rs index c5d9e3565b3..a92cf9a497a 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/python.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/python.rs @@ -9,8 +9,6 @@ use pyo3::prelude::*; use super::error::external_error; -const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager"; - /// Reads each secret through the host module's `read_secret_str`, so the configured manager, the key /// management settings and the environment fallback behave exactly as they do in Python. pub(super) struct PythonSecrets { @@ -21,7 +19,7 @@ pub(super) struct PythonSecrets { impl PythonSecrets { pub(super) fn new(py: Python<'_>) -> PyResult { Ok(Self::reading_with( - py.import(SECRET_MANAGER_MODULE)? + py.import(super::SECRET_MANAGER_MODULE)? .getattr("read_secret_str")? .unbind(), PythonContext::capture(py)?, diff --git a/litellm-rust/crates/python-bridge/src/secrets/runtime.rs b/litellm-rust/crates/python-bridge/src/secrets/runtime.rs index 0d52b0aa629..8061d1682ed 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/runtime.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/runtime.rs @@ -12,8 +12,6 @@ use pyo3::{exceptions::PyAttributeError, prelude::*, types::PyDict}; use super::ManagerError; use crate::execution::{run_async_value, run_sync_value}; -const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager"; - #[derive(Clone, PartialEq)] struct Configuration { system: KeyManagementSystem, @@ -98,7 +96,7 @@ impl NativeSecretManager { return Ok(Some(native)); } let config = py - .import(SECRET_MANAGER_MODULE)? + .import(super::SECRET_MANAGER_MODULE)? .getattr("native_secret_manager_config")? .call1((client,))?; if config.is_none() { diff --git a/litellm-rust/crates/python-bridge/src/secrets/vault.rs b/litellm-rust/crates/python-bridge/src/secrets/vault.rs index 30e3bd2c413..5f14727abaa 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/vault.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/vault.rs @@ -134,31 +134,21 @@ fn timeout_message( String::new() }); } - let kwargs = pyo3::types::PyDict::new(py); - kwargs.set_item( - "message", - transport_timeout_message( - TimeoutPhase::Read, - context.timeout, - Some(elapsed.as_secs_f64()), - ), - )?; - kwargs.set_item("model", "default-model-name")?; - kwargs.set_item("llm_provider", "litellm-httpx-handler")?; - py.import("litellm.exceptions")? - .getattr("Timeout")? - .call((), Some(&kwargs))? - .str()? + let message = transport_timeout_message( + TimeoutPhase::Read, + context.timeout, + Some(elapsed.as_secs_f64()), + ); + py.import(super::SECRET_MANAGER_MODULE)? + .getattr("timeout_message")? + .call1((message,))? .extract() } fn response_text(py: Python<'_>, body: &[u8]) -> PyResult { - let kwargs = pyo3::types::PyDict::new(py); - kwargs.set_item("content", pyo3::types::PyBytes::new(py, body))?; - py.import("httpx")? - .getattr("Response")? - .call((200,), Some(&kwargs))? - .getattr("text")? + py.import(super::SECRET_MANAGER_MODULE)? + .getattr("response_text")? + .call1((pyo3::types::PyBytes::new(py, body),))? .extract() } diff --git a/litellm-rust/crates/python-bridge/src/tokenizer.rs b/litellm-rust/crates/python-bridge/src/tokenizer.rs index 2026b2fdc42..11b21bd5673 100644 --- a/litellm-rust/crates/python-bridge/src/tokenizer.rs +++ b/litellm-rust/crates/python-bridge/src/tokenizer.rs @@ -38,16 +38,19 @@ use litellm_token_counter::huggingface::{ #[cfg(feature = "tiktoken")] use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary}; +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +const HOST_MODULE: &str = "litellm.rust_bridge.host.tokenizers"; + #[cfg(feature = "tiktoken")] pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { enter_native()?; - let resource: std::path::PathBuf = - PyModule::import(py, "litellm.litellm_core_utils.tokenizers")? - .getattr("__file__")? - .extract()?; + let resources: std::path::PathBuf = PyModule::import(py, HOST_MODULE)? + .getattr("tiktoken_resource_dir")? + .call0()? + .extract()?; release_gil(py, || { TiktokenTokenizer::from_cached_ranks(encoding, |file| { - std::fs::read_to_string(resource.with_file_name(file)) + std::fs::read_to_string(resources.join(file)) }) }) .map_err(|error| token_count_error_to_pyerr(error.into())) @@ -197,14 +200,9 @@ impl Tokenizer { #[cfg(feature = "huggingface")] { enter_native()?; - let kwargs = PyDict::new(py); - kwargs.set_item("repo_id", identifier)?; - kwargs.set_item("filename", "tokenizer.json")?; - kwargs.set_item("revision", revision)?; - kwargs.set_item("token", token)?; - let path: String = PyModule::import(py, "huggingface_hub")? - .getattr("hf_hub_download")? - .call((), Some(&kwargs))? + let path: String = PyModule::import(py, HOST_MODULE)? + .getattr("download_tokenizer")? + .call1((identifier, revision, token))? .extract()?; let json = release_gil(py, || std::fs::read_to_string(path)) .map_err(TokenizerError::Download)?; diff --git a/litellm/litellm_core_utils/AGENTS.md b/litellm/litellm_core_utils/AGENTS.md index e817eca0ac4..9cfeccba44c 100644 --- a/litellm/litellm_core_utils/AGENTS.md +++ b/litellm/litellm_core_utils/AGENTS.md @@ -3,7 +3,7 @@ - This module decides when to fetch and install. Fetching, parsing, integrity checks and alias expansion run in Rust - The install order is publish, `add_known_models`, router replay, then `model_cost_inputs.reapply` - Registrations land by lookup in the published table, so they are written after publishing -- Startup and `register_model(url)` fetch with `http_settings=None`, which reads env vars only like main's bare httpx. Reload passes `http_settings()` +- Startup and `register_model(url)` fetch with `use_http_settings=False`, which reads env vars only like main's bare httpx. Reload passes `use_http_settings=True`, so Rust layers the configured HTTP settings on top - Startup falls back to the bundled map on any failure. Reload never falls back and keeps the current map - `register_model(url)` makes one attempt and registers nothing on failure - `utils._register_on_current_map` reruns the whole pass on the new table after `NativeTableSwapped`, and its last attempt passes `rows=None`, so no entry is lost @@ -12,4 +12,16 @@ - `GetModelCostMap.validate_model_cost_map` and `_check_*`, since Rust checks fetched documents - `ModelCostMapReloaded.model_cost_map`, use `GetModelCostMap.loaded_model_cost_map()` - the `sleep`, `rng` and `client` parameters, which are ignored with a `DeprecationWarning` - - `AliasedCostEntry` re-exported here, which now lives in `rust_bridge/host/model_cost.py` + - `AliasedCostEntry` re-exported here, which now lives in `model_cost_table.py` + +## model_cost_table.py + +- This module owns the live `litellm.model_cost` table. Python code reads and writes it here, and `rust_bridge/host/model_cost.py` calls it for Rust +- `publish`, `write` and `changed_in_place` are the only writers. Each moves `ModelCostVersion` +- The version is `(id(table), len(table), writes)`, so replacing the dict or changing its size moves it without a writer + - An in-place value edit does not, so code that edits rows in place calls `changed_in_place()` +- Anything derived from the table is cached under the version (`@VersionCached`, `model_cost_version()`), never cleared by hand +- Compat names exist only so code written against main keeps working. New code never calls them, and they never gain behavior of their own + - `utils._invalidate_model_cost_lowercase_map` forwards to `changed_in_place` + - `utils.get_model_cost_mutation_generation` forwards to `model_cost_version().writes` + - `utils._model_cost_lowercase_map` is always `None` diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 80393a72631..6f4405066ca 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -32,9 +32,13 @@ from litellm.constants import ( MODEL_COST_MAP_MAX_SHRINK_RATIO, MODEL_COST_MAP_MIN_MODEL_COUNT, ) +from litellm.litellm_core_utils import model_cost_table from litellm.litellm_core_utils.fallback_generalizations import ( set_fallback_generalizations, ) +from litellm.litellm_core_utils.model_cost_table import ( + AliasedCostEntry as AliasedCostEntry, # noqa: PLC0414 # main exported the row type from here +) from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.rust_bridge._native import ( NativeCostMapSource, @@ -44,11 +48,6 @@ from litellm.rust_bridge._native import ( NativeModelCostMap, NativeRejected, ) -from litellm.rust_bridge.host import model_cost as model_cost_host -from litellm.rust_bridge.host.model_cost import ( - AliasedCostEntry as AliasedCostEntry, # noqa: PLC0414 # main exported the row type from here -) -from litellm.rust_bridge.host.settings import HttpSettings _CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) _CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) @@ -259,8 +258,8 @@ def get_model_cost_map_loaded_at() -> "datetime | None": return _LastLoad.current.loaded_at -def cost_map_source(url: str, http_settings: HttpSettings | None) -> NativeCostMapSource: - return NativeCostMapSource(url, http_settings=http_settings) +def cost_map_source(url: str, use_http_settings: bool) -> NativeCostMapSource: + return NativeCostMapSource(url, use_http_settings=use_http_settings) def fetch_policy(timeout: float, max_attempts: int) -> NativeFetchPolicy: @@ -288,7 +287,7 @@ def adopt_model_cost_map(new_model_cost_map: dict) -> int: # mutable-ok: main's import litellm from litellm import utils - model_cost_host.publish(new_model_cost_map) + model_cost_table.publish(new_model_cost_map) litellm.add_known_models(model_cost_map=new_model_cost_map) utils.reapply_runtime_model_cost_registrations() return len(new_model_cost_map) @@ -354,15 +353,13 @@ async def refetch_model_cost_map( on failure it returns ``ModelCostMapReloadUnavailable`` and the map already installed stays. """ - from litellm.rust_bridge.host.settings import http_settings - _warn_ignored_fetch_hooks(sleep, rng, client) if _uses_bundled_map_only(): _mark_rows_with_rules_set("local", None, True, None, datetime.now(timezone.utc)) return ModelCostMapReloaded(model_count=_install(model_cost_inputs.install_bundled())) - fetched: Final = await cost_map_source(url, http_settings()).fetch(fetch_policy(timeout, max_attempts)) + fetched: Final = await cost_map_source(url, use_http_settings=True).fetch(fetch_policy(timeout, max_attempts)) if isinstance(fetched, NativeFetchFailed): verbose_logger.warning( "LiteLLM: model cost map reload failed: %s. Keeping the currently loaded map.", fetched.reason @@ -382,7 +379,9 @@ def _retry_remote_fetch_in_background( url: str, timeout: int, max_attempts: int, first_failure: NativeFetchFailed ) -> None: try: - fetched: Final = cost_map_source(url, None).fetch_blocking(fetch_policy(timeout, max_attempts), first_failure) + fetched: Final = cost_map_source(url, use_http_settings=False).fetch_blocking( + fetch_policy(timeout, max_attempts), first_failure + ) if isinstance(fetched, NativeFetchFailed): verbose_logger.warning( "LiteLLM: Failed to fetch remote model cost map from %s after %d attempts; keeping local backup", @@ -425,7 +424,7 @@ def get_model_cost_map( _mark_rows_with_rules_set("local", None, True, None, started) return _rows_with_rules_set(model_cost_inputs.install_bundled()) - fetched: Final = cost_map_source(url, None).fetch_blocking(fetch_policy(timeout, 1)) + fetched: Final = cost_map_source(url, use_http_settings=False).fetch_blocking(fetch_policy(timeout, 1)) if isinstance(fetched, NativeFetchFailed): if fetched.retryable and max_attempts > 1: threading.Thread( @@ -459,7 +458,7 @@ def get_model_cost_map( def fetch_registration_document(url: str) -> dict[str, dict[str, object]] | None: """The document behind ``register_model(url)``, fetched in one attempt, or ``None`` when the fetch failed.""" - fetched: Final = cost_map_source(url, None).fetch_blocking(fetch_policy(5, 1)) + fetched: Final = cost_map_source(url, use_http_settings=False).fetch_blocking(fetch_policy(5, 1)) if isinstance(fetched, NativeFetchFailed): verbose_logger.warning("LiteLLM: register_model could not fetch %s: %s", url, fetched.reason) return None diff --git a/litellm/litellm_core_utils/model_cost_table.py b/litellm/litellm_core_utils/model_cost_table.py new file mode 100644 index 00000000000..c28098d674c --- /dev/null +++ b/litellm/litellm_core_utils/model_cost_table.py @@ -0,0 +1,112 @@ +"""The live ``litellm.model_cost`` table, every write to it, and its version.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from functools import lru_cache +from typing import Final, Generic, ParamSpec, TypeVar + +from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE + +P = ParamSpec("P") +R = TypeVar("R") + + +class AliasedCostEntry(dict[str, object]): + """A ``litellm.model_cost`` row, the one object every alias name of a model is served under.""" + + __slots__ = ("names", "registered_fields") + + def __init__( + self, entry: Mapping[str, object], names: tuple[str, ...], registered_fields: frozenset[str] = frozenset() + ) -> None: + super().__init__(entry) + self.names: tuple[str, ...] = names + self.registered_fields: frozenset[str] = registered_fields + + +@dataclass(frozen=True, slots=True) +class ModelCostVersion: + table: int + size: int + writes: int + + +class _Writes: + __slots__ = ("count",) + + def __init__(self) -> None: + self.count: int = 0 + + def record(self) -> None: + self.count += 1 + + +_WRITES: Final = _Writes() + + +def model_cost_version() -> ModelCostVersion: + import litellm + + table: Final = litellm.model_cost + return ModelCostVersion(table=id(table), size=len(table), writes=_WRITES.count) + + +def publish(table: dict[str, object]) -> None: + """Makes ``table`` the live ``litellm.model_cost``.""" + import litellm + + litellm.model_cost = table + _WRITES.record() + + +def write(rows: dict[str, object], written: Mapping[str, object]) -> None: + """Writes one registration's rows into ``rows``, the published table.""" + rows.update(written) + _WRITES.record() + + +def changed_in_place() -> None: + """Moves the version and drops the name index after an in-place edit of ``litellm.model_cost``.""" + from litellm.litellm_core_utils.get_model_cost_map import model_cost_inputs + + model_cost_inputs.invalidate() + _WRITES.record() + + +class VersionCached(Generic[P, R]): + """Decorates ``function`` with an ``lru_cache`` keyed on the ``litellm.model_cost`` version each call read, and + exposes that cache's ``cache_clear`` and ``cache_info``.""" + + __slots__ = ("_at", "cache_clear", "cache_info") + + def __init__(self, function: Callable[P, R]) -> None: + def at( + _version: ModelCostVersion, + *args: P.args, + **kwargs: P.kwargs, # kwargs-ok: forwards a ParamSpec unchanged + ) -> R: + return function(*args, **kwargs) + + cached: Final = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)(at) + self._at: Final = cached + self.cache_clear: Final = cached.cache_clear + self.cache_info: Final = cached.cache_info + + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R: # kwargs-ok: forwards a ParamSpec unchanged + return self._at(model_cost_version(), *args, **kwargs) + + +@dataclass(frozen=True, slots=True) +class Landing: + key: str + base: Mapping[str, object] + matched_builtin: bool + + +def published() -> dict[str, object]: + """The table published as ``litellm.model_cost`` now.""" + import litellm + + return litellm.model_cost diff --git a/litellm/llms/bedrock/audio_transcription/__init__.py b/litellm/llms/bedrock/audio_transcription/__init__.py index bc74e9c38ab..dcda72c9a60 100644 --- a/litellm/llms/bedrock/audio_transcription/__init__.py +++ b/litellm/llms/bedrock/audio_transcription/__init__.py @@ -10,8 +10,7 @@ from pydantic import BaseModel, ConfigDict from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.rust_bridge import runtime from litellm.rust_bridge.catalog import Route, RouteContext -from litellm.rust_bridge.host.timeouts import timeout_to_seconds -from litellm.rust_bridge.public_call import NativeCall +from litellm.rust_bridge.public_call import NativeCall, timeout_to_seconds from litellm.rust_bridge.transcription.native import ( NATIVE_ATRANSCRIPTION, NATIVE_TRANSCRIPTION, diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 9db1d50d730..55a75423042 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -13,13 +13,13 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) +from litellm.litellm_core_utils.model_cost_table import ModelCostVersion, model_cost_version from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_litellm_response_only_fields, unpack_legacy_defs, ) from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.rust_bridge.host.model_cost import ModelCostVersion, model_cost_version from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index b50890089e4..f33bb82b49b 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Final, TypeAlias import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.rust_bridge.host.model_cost import AliasedCostEntry +from litellm.litellm_core_utils.model_cost_table import AliasedCostEntry from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_WARN_DAYS, DeprecationStatus, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 888276f7c31..f7a27e4896c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7896,7 +7896,7 @@ class ProxyConfig: else: raise ValueError("Invalid Key Management System selected") - from litellm.rust_bridge.host.secret_manager import capture_secret_manager + from litellm.secret_managers.native_config import capture_secret_manager if litellm.secret_manager_client is not previous_client: capture_secret_manager(litellm.secret_manager_client, key_management_system) diff --git a/litellm/rust_bridge/AGENTS.md b/litellm/rust_bridge/AGENTS.md index c2a030ca4d9..167b91c9b89 100644 --- a/litellm/rust_bridge/AGENTS.md +++ b/litellm/rust_bridge/AGENTS.md @@ -9,7 +9,7 @@ - provider execution, which is Rust core - CPython runtime mechanics, which is `litellm-host-python` - legacy callback sharing and dispatch policy, which is `callbacks-legacy-python` -- `host/` holds the modules Rust imports by name to read Python-owned state. `host/AGENTS.md` states that contract +- `host/` holds the modules Rust imports by name to reach Python-owned state and behavior. Only Rust calls them, and Rust reaches Python nowhere else. `host/AGENTS.md` states that contract # Routing diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 221968e07ed..ebbd45c7382 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -8,7 +8,6 @@ from typing_extensions import disjoint_base from litellm.redis.store_settings import RedisBreakerSettings, RedisStoreSettings from litellm.rust_bridge.execution import Execution -from litellm.rust_bridge.host.settings import HttpSettings from litellm.rust_bridge.public_call import NativeCall _T_co = TypeVar("_T_co", covariant=True) @@ -22,10 +21,9 @@ class NativeRedisStoreError(Exception): ... def trace_encode_error(message: str) -> bytes: ... def bundled_model_cost_map() -> bytes: ... def native_redis_breaker_states() -> list[str]: ... - @final class NativeCostMapSource: - def __new__(cls, url: str, *, http_settings: HttpSettings | None) -> NativeCostMapSource: ... + def __new__(cls, url: str, *, use_http_settings: bool) -> NativeCostMapSource: ... def fetch(self, policy: NativeFetchPolicy) -> Future[NativeFetched | NativeFetchFailed]: ... def fetch_blocking( self, policy: NativeFetchPolicy, resume_from: NativeFetchFailed | None = None @@ -230,9 +228,7 @@ class NativeRouterUsage: def attach(self, client: NativeRedisClient) -> bool: ... @property def has_redis(self) -> bool: ... - def increment( - self, counter: _UsageCounter, value: float, ttl_seconds: int - ) -> Future[float]: ... + def increment(self, counter: _UsageCounter, value: float, ttl_seconds: int) -> Future[float]: ... def increment_deployment( self, deployment_id: str, @@ -241,12 +237,8 @@ class NativeRouterUsage: tokens: float, requests: int, ) -> Future[None]: ... - def get_many( - self, counters: Sequence[_UsageCounter] - ) -> Future[list[float | None]]: ... - def increment_sync( - self, counter: _UsageCounter, value: float, ttl_seconds: int - ) -> float: ... + def get_many(self, counters: Sequence[_UsageCounter]) -> Future[list[float | None]]: ... + def increment_sync(self, counter: _UsageCounter, value: float, ttl_seconds: int) -> float: ... def increment_deployment_sync( self, deployment_id: str, @@ -255,9 +247,7 @@ class NativeRouterUsage: tokens: float, requests: int, ) -> None: ... - def get_many_sync( - self, counters: Sequence[_UsageCounter] - ) -> list[float | None]: ... + def get_many_sync(self, counters: Sequence[_UsageCounter]) -> list[float | None]: ... def flush_local_counters(self) -> None: ... def snapshot(self) -> dict[str, float]: ... @@ -433,7 +423,9 @@ class NativeAuthCache: def get_sync(self, key: str) -> object: ... def set(self, key: str, value: object, ttl_seconds: float | None = None) -> Future[None]: ... def set_sync(self, key: str, value: object, ttl_seconds: float | None = None) -> None: ... - def declare_set(self, key: str, value: object, ttl_seconds: float | None, send: NativeSend) -> NativePending[None]: ... + def declare_set( + self, key: str, value: object, ttl_seconds: float | None, send: NativeSend + ) -> NativePending[None]: ... def declare_delete(self, key: str, send: NativeSend) -> NativePending[None]: ... def set_many(self, entries: Sequence[tuple[str, object]], ttl_seconds: float | None = None) -> Future[None]: ... def write_back(self, entries: Sequence[tuple[str, object, float | None]], send: NativeSend) -> Future[None]: ... @@ -550,7 +542,6 @@ def messages( def amessages( call: NativeCall, ) -> Execution: ... - @final class ResponsesWebSocketConnection: def __new__(cls, _uninstantiable: Never, /) -> Never: ... @@ -1189,7 +1180,9 @@ class RouterCoreHandle: callbacks: Mapping[str, bool] | None = None, ) -> RouterCall[_SuccessReportSteps]: ... # Count a success against any id, driven like `report_failure`; `done` carries `key`. - def count_success(self, id: str, callbacks: Mapping[str, bool] | None = None) -> RouterCall[_SuccessReportSteps]: ... + def count_success( + self, id: str, callbacks: Mapping[str, bool] | None = None + ) -> RouterCall[_SuccessReportSteps]: ... # [(id, tpm_key, rpm_key)] for this minute, or None for a group with no deployment. def usage_keys(self, model_group: str) -> list[tuple[str, str, str]] | None: ... # Every deployment as a `model_list` entry, in config order. diff --git a/litellm/rust_bridge/host/AGENTS.md b/litellm/rust_bridge/host/AGENTS.md index 4cb87a85c28..c249eb82ee0 100644 --- a/litellm/rust_bridge/host/AGENTS.md +++ b/litellm/rust_bridge/host/AGENTS.md @@ -1,24 +1,48 @@ # Rust host modules -Python state Rust needs but does not own, behind one module per question. Rust imports a module here by path and calls a function by name +Python behavior Rust needs but does not own, behind one module per question. Rust imports a module here by path and calls a function by name -- One module answers one question: `model_capabilities` (what a model can do), `callbacks_legacy_python` (the legacy callback registries), `settings` (configured snapshots), `cache` (the live configured cache), `diagnostics` (legacy failure logging), `secret_manager` (the configured secret manager and native config capture), `http_transport` (which async transport is in use), `timeouts` (timeout to seconds), `redis_events` (reporting native Redis calls and breaker transitions to `ServiceLogging`), `redis_credentials` (what a caller's redis-py `CredentialProvider` authenticates with), `model_cost` (the live `litellm.model_cost` table, every write to it and its version) -- A module is the only place Rust reads that Python state. Rust never does `py.import("litellm").getattr(...)` on a global; it asks here +## Boundary + +- This package is the only place Rust reaches into Python. `python-bridge` and `callbacks-legacy-python` import no other LiteLLM module, and never do `py.import("litellm").getattr(...)` on a global + - The one exception is `litellm.rust_bridge.execution`, the driver's step classes. It is runtime protocol, not Python state, and the Python entrypoints import it too +- Only Rust calls these modules. No Python module outside `host/` imports one. Tests may, to set up the Python state Rust reads + - Logic both sides need lives with its Python owner (`public_call`, `secret_managers`, the route's types), and the host module calls it +- A Python call that reproduces Python behavior goes through a host function, including standard library and third-party calls: error text built from `httpx`, `OSError` or `litellm.exceptions`, `huggingface_hub` downloads, the process `random` generator + - Rust reads the process environment itself (`ProcessEnvironment`). CPython's `os.environ` writes through to it, so there is nothing to ask Python for + - Rust imports a module directly only to get a type or a Python builtin it applies itself: `isinstance` checks (`os.PathLike`, `ssl.SSLContext`), exception classes it raises (`redis.exceptions`), `builtins.object` and `inspect.getattr_static` for Python attribute semantics + - `litellm-host-python` is the generic runtime and keeps its own `asyncio`, `contextvars` and `json` imports +- Check these invariants by review whenever a Rust file adds a `py.import`, an `import_exception!` or a `*_MODULE` constant, or a Python file imports from `litellm.rust_bridge.host` + +## Modules + +- One module answers one question + - `model_capabilities`: what a model can do + - `callbacks_legacy_python`: the legacy callback registries + - `settings`: configured snapshots + - `cache`: the live configured cache + - `diagnostics`: legacy failure logging + - `logger`: where native diagnostics are emitted + - `preflight`: the SDK globals the native preflight borrows + - `secret_manager`: the configured secret manager, its native config and the error text Python's managers produce + - `http_transport`: which async transport is in use + - `timeouts`: timeout to seconds + - `tokenizers`: bundled tiktoken files and Hugging Face tokenizer downloads + - `randomness`: the process `random` generator the Python router draws from + - `redis_events`: reporting native Redis calls and breaker transitions to `ServiceLogging` + - `redis_credentials`: what a caller's redis-py `CredentialProvider` authenticates with + - `model_cost`: where a `register_model` entry lands in the published table and the row it writes +- `routes/.py` holds the questions one route asks, under the same function names on every route (`response`, `map_failure`, `connection_defaults`), so Rust names only the module. A question two routes ask goes in a top-level module here instead - Each module reuses the Python implementation the Python route runs (`AnthropicModelInfo`, `Logging`). Do not re-derive those rules here or in Rust; one resolver, two callers -- The module path and function signature are a contract. The Rust side pins the path in a `const MODULE` in the crate that asks (`python-bridge/src/routes/messages/host.rs`, `python-bridge/src/secrets/`, `callbacks-legacy-python/src/python.rs`); change both sides together and keep `tests/unit/rust_bridge/host/` green + +## Contract + +- The module path and function signature are a contract. Each Rust crate declares a host module's path once, in a `const *_MODULE` next to the code that asks (`python-bridge/src/secrets/mod.rs`, `python-bridge/src/routes//`, `callbacks-legacy-python/src/python.rs`). Change both sides together and keep `tests/unit/rust_bridge/host/` green - Inputs and outputs use values and frozen snapshots. Retain caller-owned clients and caches as opaque references when identity is required; Rust never discovers them through module globals - Moving a question into Rust means replacing the function body with a native call, not adding a second resolver. The Python signature stays -- `routes/.py` holds the questions one route asks, under the same function names on every route (`response`, `map_failure`, `connection_defaults`), so Rust names only the module. A question two routes ask goes in a top-level module here instead ## model_cost -- `publish`, `write` and `changed_in_place` are the only writers. Each moves `ModelCostVersion` -- The version is `(id(table), len(table), writes)`, so replacing the dict or changing its size moves it without the host - - An in-place value edit does not, so code that edits rows in place calls `changed_in_place()` -- Anything derived from the table is cached under the version (`@VersionCached`, `model_cost_version()`), never cleared by hand +- `model_cost` answers what Rust needs while registering models: `entry`, `published`, `write`, `resolve`, `merged` and `kept`. The live table, its version and every Python writer belong to `litellm_core_utils/model_cost_table.py` - `resolve`, `merged` and `kept` are shims until `get_model_info` and a merge generic over Python values move to Rust. Then their bodies become native calls and the signatures stay - `resolve` returns `None` when it was handed a table that is no longer published. That is how a pass learns a reload swapped the table -- Compat names exist only so code written against main keeps working. New code never calls them, and they never gain behavior of their own - - `utils._invalidate_model_cost_lowercase_map` forwards to `changed_in_place` - - `utils.get_model_cost_mutation_generation` forwards to `model_cost_version().writes` - - `utils._model_cost_lowercase_map` is always `None` diff --git a/litellm/rust_bridge/logger.py b/litellm/rust_bridge/host/logger.py similarity index 100% rename from litellm/rust_bridge/logger.py rename to litellm/rust_bridge/host/logger.py diff --git a/litellm/rust_bridge/host/model_capabilities.py b/litellm/rust_bridge/host/model_capabilities.py index 6fd8b42520e..00a00b9e79b 100644 --- a/litellm/rust_bridge/host/model_capabilities.py +++ b/litellm/rust_bridge/host/model_capabilities.py @@ -6,7 +6,7 @@ from typing import Final, TypeAlias import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.rust_bridge.host.model_cost import ModelCostVersion, model_cost_version +from litellm.litellm_core_utils.model_cost_table import ModelCostVersion, model_cost_version class _Held: diff --git a/litellm/rust_bridge/host/model_cost.py b/litellm/rust_bridge/host/model_cost.py index 328e0c6252e..cfedbb64592 100644 --- a/litellm/rust_bridge/host/model_cost.py +++ b/litellm/rust_bridge/host/model_cost.py @@ -1,118 +1,21 @@ -"""The live ``litellm.model_cost`` table, every write to it, and its version.""" +"""The ``litellm.model_cost`` questions Rust asks while registering models.""" from __future__ import annotations -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from functools import lru_cache -from typing import Final, Generic, ParamSpec, TypeVar +from collections.abc import Mapping +from typing import Final -from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE - -P = ParamSpec("P") -R = TypeVar("R") +from litellm.litellm_core_utils import model_cost_table +from litellm.litellm_core_utils.model_cost_table import AliasedCostEntry, Landing -class AliasedCostEntry(dict[str, object]): - """A ``litellm.model_cost`` row, the one object every alias name of a model is served under.""" - - __slots__ = ("names", "registered_fields") - - def __init__( - self, entry: Mapping[str, object], names: tuple[str, ...], registered_fields: frozenset[str] = frozenset() - ) -> None: - super().__init__(entry) - self.names: tuple[str, ...] = names - self.registered_fields: frozenset[str] = registered_fields +def entry(fields: Mapping[str, object], names: tuple[str, ...]) -> AliasedCostEntry: + """The row object every name in ``names`` shares, the owner first.""" + return AliasedCostEntry(fields, names) -entry: Final = AliasedCostEntry - - -@dataclass(frozen=True, slots=True) -class ModelCostVersion: - table: int - size: int - writes: int - - -class _Writes: - __slots__ = ("count",) - - def __init__(self) -> None: - self.count: int = 0 - - def record(self) -> None: - self.count += 1 - - -_WRITES: Final = _Writes() - - -def model_cost_version() -> ModelCostVersion: - import litellm - - table: Final = litellm.model_cost - return ModelCostVersion(table=id(table), size=len(table), writes=_WRITES.count) - - -def publish(table: dict[str, object]) -> None: - """Makes ``table`` the live ``litellm.model_cost``.""" - import litellm - - litellm.model_cost = table - _WRITES.record() - - -def write(rows: dict[str, object], written: Mapping[str, object]) -> None: - """Writes one registration's rows into ``rows``, the published table.""" - rows.update(written) - _WRITES.record() - - -def changed_in_place() -> None: - """Moves the version and drops the name index after an in-place edit of ``litellm.model_cost``.""" - from litellm.litellm_core_utils.get_model_cost_map import model_cost_inputs - - model_cost_inputs.invalidate() - _WRITES.record() - - -class VersionCached(Generic[P, R]): - """Decorates ``function`` with an ``lru_cache`` keyed on the ``litellm.model_cost`` version each call read, and - exposes that cache's ``cache_clear`` and ``cache_info``.""" - - __slots__ = ("_at", "cache_clear", "cache_info") - - def __init__(self, function: Callable[P, R]) -> None: - def at( - _version: ModelCostVersion, - *args: P.args, - **kwargs: P.kwargs, # kwargs-ok: forwards a ParamSpec unchanged - ) -> R: - return function(*args, **kwargs) - - cached: Final = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)(at) - self._at: Final = cached - self.cache_clear: Final = cached.cache_clear - self.cache_info: Final = cached.cache_info - - def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R: # kwargs-ok: forwards a ParamSpec unchanged - return self._at(model_cost_version(), *args, **kwargs) - - -@dataclass(frozen=True, slots=True) -class Landing: - key: str - base: Mapping[str, object] - matched_builtin: bool - - -def published() -> dict[str, object]: - """The table published as ``litellm.model_cost`` now.""" - import litellm - - return litellm.model_cost +published: Final = model_cost_table.published +write: Final = model_cost_table.write def resolve( diff --git a/litellm/rust_bridge/preflight.py b/litellm/rust_bridge/host/preflight.py similarity index 100% rename from litellm/rust_bridge/preflight.py rename to litellm/rust_bridge/host/preflight.py diff --git a/litellm/rust_bridge/host/randomness.py b/litellm/rust_bridge/host/randomness.py new file mode 100644 index 00000000000..71d841833e4 --- /dev/null +++ b/litellm/rust_bridge/host/randomness.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import random + + +def next_float() -> float: + return random.random() + + +def index_below(length: int) -> int: + return random.randrange(length) diff --git a/litellm/rust_bridge/host/routes/responses.py b/litellm/rust_bridge/host/routes/responses.py index 22e8cbbf3b0..0568326d18b 100644 --- a/litellm/rust_bridge/host/routes/responses.py +++ b/litellm/rust_bridge/host/routes/responses.py @@ -20,6 +20,12 @@ def connection_defaults(provider: str) -> tuple[str | None, str | None]: return litellm.api_key or provider_key, litellm.api_base +def original_previous_response_id(previous_response_id: str) -> str: + from litellm.responses.utils import ResponsesAPIRequestUtils + + return ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(previous_response_id) + + def response(value: Mapping[str, object]) -> ResponsesAPIResponse: return ResponsesAPIResponse.model_validate(value) diff --git a/litellm/rust_bridge/host/secret_manager.py b/litellm/rust_bridge/host/secret_manager.py index 4fb3350165c..b236b4ee19d 100644 --- a/litellm/rust_bridge/host/secret_manager.py +++ b/litellm/rust_bridge/host/secret_manager.py @@ -1,165 +1,58 @@ from __future__ import annotations import os -from collections.abc import Mapping -from dataclasses import dataclass, field -from importlib import import_module from typing import Final -from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem +import httpx - -@dataclass(frozen=True, slots=True) -class NativeSecretManagerConfig: - system: str - environment: tuple[tuple[str, str], ...] = field(repr=False) - settings: Mapping[str, object] = field(repr=False) - enterprise_enabled: bool - owner_type: type[object] - environment_attributes: tuple[tuple[str, str], ...] - settings_attributes: tuple[str, ...] - methods: tuple[tuple[str, object], ...] = field(repr=False) - - -@dataclass(frozen=True, slots=True) -class _ClientAdapter: - system: KeyManagementSystem - module: str - name: str - methods: tuple[str, ...] - environment_attributes: tuple[tuple[str, str], ...] = () - settings_attributes: tuple[str, ...] = () - enterprise_enabled: bool = False - - -_ADAPTERS: Final = ( - _ClientAdapter( - KeyManagementSystem.AWS_SECRET_MANAGER, - "litellm.secret_managers.aws_secret_manager_v2", - "AWSSecretsManagerV2", - ("sync_read_secret", "async_read_secret"), - settings_attributes=( - "aws_region_name", - "aws_role_name", - "aws_session_name", - "aws_external_id", - "aws_profile_name", - "aws_web_identity_token", - "aws_sts_endpoint", - "replica_regions", - "kms_key_id", - ), - ), - _ClientAdapter( - KeyManagementSystem.HASHICORP_VAULT, - "litellm.secret_managers.hashicorp_secret_manager", - "HashicorpSecretManager", - ("sync_read_secret", "async_read_secret", "async_write_secret", "async_delete_secret", "async_rotate_secret"), - environment_attributes=( - ("HCP_VAULT_ADDR", "vault_addr"), - ("HCP_VAULT_TOKEN", "vault_token"), - ("HCP_VAULT_NAMESPACE", "vault_namespace"), - ("HCP_VAULT_LOGIN_NAMESPACE", "login_namespace_override"), - ("HCP_VAULT_SECRET_NAMESPACE", "secret_namespace_override"), - ("HCP_VAULT_MOUNT_NAME", "vault_mount_name"), - ("HCP_VAULT_PATH_PREFIX", "vault_path_prefix"), - ("HCP_VAULT_CLIENT_CERT", "tls_cert_path"), - ("HCP_VAULT_CLIENT_KEY", "tls_key_path"), - ("HCP_VAULT_CERT_ROLE", "vault_cert_role"), - ("HCP_VAULT_APPROLE_ROLE_ID", "approle_role_id"), - ("HCP_VAULT_APPROLE_SECRET_ID", "approle_secret_id"), - ("HCP_VAULT_APPROLE_MOUNT_PATH", "approle_mount_path"), - ("HCP_VAULT_REFRESH_INTERVAL", "cache.default_ttl"), - ), - enterprise_enabled=True, - ), - _ClientAdapter( - KeyManagementSystem.CYBERARK, - "litellm.secret_managers.cyberark_secret_manager", - "CyberArkSecretManager", - ("sync_read_secret", "async_read_secret", "async_write_secret", "async_delete_secret", "async_rotate_secret"), - environment_attributes=( - ("CYBERARK_API_BASE", "conjur_addr"), - ("CYBERARK_ACCOUNT", "conjur_account"), - ("CYBERARK_USERNAME", "conjur_username"), - ("CYBERARK_API_KEY", "conjur_api_key"), - ("CYBERARK_CLIENT_CERT", "tls_cert_path"), - ("CYBERARK_CLIENT_KEY", "tls_key_path"), - ("CYBERARK_SSL_VERIFY", "ssl_verify"), - ("CYBERARK_REFRESH_INTERVAL", "cache.default_ttl"), - ), - enterprise_enabled=True, - ), - _ClientAdapter( - KeyManagementSystem.GOOGLE_SECRET_MANAGER, - "litellm.secret_managers.google_secret_manager", - "GoogleSecretManager", - ("get_secret_from_google_secret_manager",), - environment_attributes=( - ("GOOGLE_SECRET_MANAGER_PROJECT_ID", "PROJECT_ID"), - ("GOOGLE_SECRET_MANAGER_REFRESH_INTERVAL", "cache.default_ttl"), - ("GOOGLE_SECRET_MANAGER_ALWAYS_READ_SECRET_MANAGER", "always_read_secret_manager"), - ), - enterprise_enabled=True, - ), -) - -_SDK_ADAPTERS: Final = ( - _ClientAdapter(KeyManagementSystem.AZURE_KEY_VAULT, "azure.keyvault.secrets", "SecretClient", ("get_secret",)), - _ClientAdapter(KeyManagementSystem.GOOGLE_KMS, "google.cloud.kms_v1", "KeyManagementServiceClient", ("decrypt",)), -) - - -def _capture(client: object, adapter: _ClientAdapter) -> NativeSecretManagerConfig: - prefixes: Final = ("AWS_", "AZURE_", "GOOGLE_", "VERTEX_", "GCS_", "HCP_VAULT_", "CYBERARK_") - config: Final = NativeSecretManagerConfig( - system=adapter.system.value, - environment=tuple( - (name, value) - for name, value in os.environ.items() - if name.startswith(prefixes) or name == "SECRET_MANAGER_REFRESH_INTERVAL" - ), - settings=KeyManagementSettings().model_dump(mode="json"), - enterprise_enabled=adapter.enterprise_enabled, - owner_type=type(client), - environment_attributes=adapter.environment_attributes, - settings_attributes=adapter.settings_attributes, - methods=tuple((name, getattr(type(client), name)) for name in adapter.methods), - ) - vars(client)["_litellm_native_secret_config"] = config - return config - - -def capture_secret_manager(client: object, system: str) -> None: - for adapter in (*_ADAPTERS, *_SDK_ADAPTERS): - if ( - adapter.system.value == system - and type(client).__name__ == adapter.name - and type(client) is getattr(import_module(adapter.module), adapter.name) - ): - _capture(client, adapter) - return - if ( - system == KeyManagementSystem.AWS_KMS.value - and type(client).__module__ == "botocore.client" - and type(client).__name__ == "KMS" - ): - _capture(client, _ClientAdapter(KeyManagementSystem.AWS_KMS, "botocore.client", "KMS", ("decrypt",))) +from litellm.secret_managers.native_config import NativeSecretManagerConfig, captured_config +from litellm.types.secret_managers.main import KeyManagementSettings def native_secret_manager_config(client: object) -> NativeSecretManagerConfig | None: - captured: Final = getattr(client, "_litellm_native_secret_config", None) - if isinstance(captured, NativeSecretManagerConfig): - return captured - for adapter in _ADAPTERS: - if type(client).__module__ == adapter.module and type(client) is getattr( - import_module(adapter.module), adapter.name - ): - return _capture(client, adapter) - return None + return captured_config(client) def read_secret_str(name: str) -> str | None: from litellm.secret_managers.main import get_secret_str return get_secret_str(name) + + +def read_from_manager( + client: object, + key_manager: str, + secret_name: str, + key_management_settings: KeyManagementSettings | None, +) -> str | None: + from litellm.secret_managers.secret_manager_handler import get_secret_from_manager + + return get_secret_from_manager( + client=client, + key_manager=key_manager, + secret_name=secret_name, + key_management_settings=key_management_settings, + ) + + +def timeout_message(message: str) -> str: + from litellm.exceptions import Timeout + + return str(Timeout(message=message, model="default-model-name", llm_provider="litellm-httpx-handler")) + + +def response_text(body: bytes) -> str: + return httpx.Response(200, content=body).text + + +def http_status_message(method: str, url: str, status: int) -> str: + response: Final = httpx.Response(status, request=httpx.Request(method, url)) + try: + _ = response.raise_for_status() + except httpx.HTTPStatusError as error: + return str(error) + return f"HTTP {status}" + + +def os_error_message(code: int) -> str: + return str(OSError(code, os.strerror(code))) diff --git a/litellm/rust_bridge/host/timeouts.py b/litellm/rust_bridge/host/timeouts.py index 5fde4397b55..fe9abd71c1e 100644 --- a/litellm/rust_bridge/host/timeouts.py +++ b/litellm/rust_bridge/host/timeouts.py @@ -1,13 +1,9 @@ -"""Shared timeout conversion for the native Rust bridges.""" - from __future__ import annotations import httpx +from litellm.rust_bridge import public_call + def timeout_to_seconds(timeout: float | httpx.Timeout | None) -> float | None: - if timeout is None: - return None - if isinstance(timeout, httpx.Timeout): - return timeout.read - return float(timeout) + return public_call.timeout_to_seconds(timeout) diff --git a/litellm/rust_bridge/host/tokenizers.py b/litellm/rust_bridge/host/tokenizers.py new file mode 100644 index 00000000000..e86e9b54aa7 --- /dev/null +++ b/litellm/rust_bridge/host/tokenizers.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from pathlib import Path + + +def tiktoken_resource_dir() -> str: + from litellm.litellm_core_utils import tokenizers + + return str(Path(tokenizers.__file__).parent) + + +def download_tokenizer(repo_id: str, revision: str, token: str | None) -> str: + import huggingface_hub + + return huggingface_hub.hf_hub_download( # pyright: ignore[reportUnknownMemberType] # untyped user_agent dict in its signature + repo_id=repo_id, filename="tokenizer.json", revision=revision, token=token + ) diff --git a/litellm/rust_bridge/public_call.py b/litellm/rust_bridge/public_call.py index 1ab3d5e10c9..0572d498b7d 100644 --- a/litellm/rust_bridge/public_call.py +++ b/litellm/rust_bridge/public_call.py @@ -9,6 +9,8 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Final, TypeVar, cast # noqa: TID251 # narrows caller-owned containers without copying them +import httpx + from litellm.utils import ( _get_wrapper_timeout, # pyright: ignore[reportPrivateUsage] # the @client wrapper's own failure stamp ) @@ -49,6 +51,14 @@ def optional_sequence(value: object) -> Sequence[object] | None: return cast("Sequence[object]", value) # cast-ok: the same caller-owned object is handed on unchanged +def timeout_to_seconds(timeout: float | httpx.Timeout | None) -> float | None: + if timeout is None: + return None + if isinstance(timeout, httpx.Timeout): + return timeout.read + return float(timeout) + + def known_params(names: Collection[str], request: Mapping[str, object]) -> Mapping[str, object]: return {name: value for name, value in request.items() if name in names and value is not None} diff --git a/litellm/rust_bridge/responses/detectors.py b/litellm/rust_bridge/responses/detectors.py index 2fdb2c44b54..f8d5b4462a1 100644 --- a/litellm/rust_bridge/responses/detectors.py +++ b/litellm/rust_bridge/responses/detectors.py @@ -12,11 +12,12 @@ import litellm from litellm.llms.opencode.common_utils import is_opencode_api_base from litellm.rust_bridge.detectors import api_base, call_callbacks, overridden, present from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams if TYPE_CHECKING: from openai.types.responses import ResponseInputParam - from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponsesAPIOptionalRequestParams + from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS def client_headers(request: NativeCall) -> bool: @@ -160,9 +161,12 @@ _PARAMS: Final = TypeAdapter(dict[str, object]) def model_param_gating(request: NativeCall) -> bool: """Python clamps, drops or remaps Responses params according to the model's capabilities.""" from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig - from litellm.rust_bridge.host.routes.responses import PARAMETERS - params: Final = {name: request.resolved[name] for name in PARAMETERS if request.resolved.get(name) is not None} + params: Final = { + name: request.resolved[name] + for name in ResponsesAPIOptionalRequestParams.__annotations__ + if request.resolved.get(name) is not None + } mapped: Final = _PARAMS.validate_python( OpenAIResponsesAPIConfig().map_openai_params( # pyright: ignore[reportUnknownMemberType] # returns a bare dict cast("ResponsesAPIOptionalRequestParams", params), # cast-ok: the handler passes these kwargs diff --git a/litellm/rust_bridge/responses/websocket.py b/litellm/rust_bridge/responses/websocket.py index 83a1fa53376..e27e88fa9aa 100644 --- a/litellm/rust_bridge/responses/websocket.py +++ b/litellm/rust_bridge/responses/websocket.py @@ -10,7 +10,7 @@ import httpx from websockets.exceptions import ConnectionClosedOK from litellm.rust_bridge import _native -from litellm.rust_bridge.host.timeouts import timeout_to_seconds +from litellm.rust_bridge.public_call import timeout_to_seconds class RustResponsesWebSocket(Protocol): diff --git a/litellm/secret_managers/native_config.py b/litellm/secret_managers/native_config.py new file mode 100644 index 00000000000..c75fe94bae0 --- /dev/null +++ b/litellm/secret_managers/native_config.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import os +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib import import_module +from typing import Final + +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem + + +@dataclass(frozen=True, slots=True) +class NativeSecretManagerConfig: + system: str + environment: tuple[tuple[str, str], ...] = field(repr=False) + settings: Mapping[str, object] = field(repr=False) + enterprise_enabled: bool + owner_type: type[object] + environment_attributes: tuple[tuple[str, str], ...] + settings_attributes: tuple[str, ...] + methods: tuple[tuple[str, object], ...] = field(repr=False) + + +@dataclass(frozen=True, slots=True) +class _ClientAdapter: + system: KeyManagementSystem + module: str + name: str + methods: tuple[str, ...] + environment_attributes: tuple[tuple[str, str], ...] = () + settings_attributes: tuple[str, ...] = () + enterprise_enabled: bool = False + + +_ADAPTERS: Final = ( + _ClientAdapter( + KeyManagementSystem.AWS_SECRET_MANAGER, + "litellm.secret_managers.aws_secret_manager_v2", + "AWSSecretsManagerV2", + ("sync_read_secret", "async_read_secret"), + settings_attributes=( + "aws_region_name", + "aws_role_name", + "aws_session_name", + "aws_external_id", + "aws_profile_name", + "aws_web_identity_token", + "aws_sts_endpoint", + "replica_regions", + "kms_key_id", + ), + ), + _ClientAdapter( + KeyManagementSystem.HASHICORP_VAULT, + "litellm.secret_managers.hashicorp_secret_manager", + "HashicorpSecretManager", + ("sync_read_secret", "async_read_secret", "async_write_secret", "async_delete_secret", "async_rotate_secret"), + environment_attributes=( + ("HCP_VAULT_ADDR", "vault_addr"), + ("HCP_VAULT_TOKEN", "vault_token"), + ("HCP_VAULT_NAMESPACE", "vault_namespace"), + ("HCP_VAULT_LOGIN_NAMESPACE", "login_namespace_override"), + ("HCP_VAULT_SECRET_NAMESPACE", "secret_namespace_override"), + ("HCP_VAULT_MOUNT_NAME", "vault_mount_name"), + ("HCP_VAULT_PATH_PREFIX", "vault_path_prefix"), + ("HCP_VAULT_CLIENT_CERT", "tls_cert_path"), + ("HCP_VAULT_CLIENT_KEY", "tls_key_path"), + ("HCP_VAULT_CERT_ROLE", "vault_cert_role"), + ("HCP_VAULT_APPROLE_ROLE_ID", "approle_role_id"), + ("HCP_VAULT_APPROLE_SECRET_ID", "approle_secret_id"), + ("HCP_VAULT_APPROLE_MOUNT_PATH", "approle_mount_path"), + ("HCP_VAULT_REFRESH_INTERVAL", "cache.default_ttl"), + ), + enterprise_enabled=True, + ), + _ClientAdapter( + KeyManagementSystem.CYBERARK, + "litellm.secret_managers.cyberark_secret_manager", + "CyberArkSecretManager", + ("sync_read_secret", "async_read_secret", "async_write_secret", "async_delete_secret", "async_rotate_secret"), + environment_attributes=( + ("CYBERARK_API_BASE", "conjur_addr"), + ("CYBERARK_ACCOUNT", "conjur_account"), + ("CYBERARK_USERNAME", "conjur_username"), + ("CYBERARK_API_KEY", "conjur_api_key"), + ("CYBERARK_CLIENT_CERT", "tls_cert_path"), + ("CYBERARK_CLIENT_KEY", "tls_key_path"), + ("CYBERARK_SSL_VERIFY", "ssl_verify"), + ("CYBERARK_REFRESH_INTERVAL", "cache.default_ttl"), + ), + enterprise_enabled=True, + ), + _ClientAdapter( + KeyManagementSystem.GOOGLE_SECRET_MANAGER, + "litellm.secret_managers.google_secret_manager", + "GoogleSecretManager", + ("get_secret_from_google_secret_manager",), + environment_attributes=( + ("GOOGLE_SECRET_MANAGER_PROJECT_ID", "PROJECT_ID"), + ("GOOGLE_SECRET_MANAGER_REFRESH_INTERVAL", "cache.default_ttl"), + ("GOOGLE_SECRET_MANAGER_ALWAYS_READ_SECRET_MANAGER", "always_read_secret_manager"), + ), + enterprise_enabled=True, + ), +) + +_SDK_ADAPTERS: Final = ( + _ClientAdapter(KeyManagementSystem.AZURE_KEY_VAULT, "azure.keyvault.secrets", "SecretClient", ("get_secret",)), + _ClientAdapter(KeyManagementSystem.GOOGLE_KMS, "google.cloud.kms_v1", "KeyManagementServiceClient", ("decrypt",)), +) + + +def _capture(client: object, adapter: _ClientAdapter) -> NativeSecretManagerConfig: + prefixes: Final = ("AWS_", "AZURE_", "GOOGLE_", "VERTEX_", "GCS_", "HCP_VAULT_", "CYBERARK_") + config: Final = NativeSecretManagerConfig( + system=adapter.system.value, + environment=tuple( + (name, value) + for name, value in os.environ.items() + if name.startswith(prefixes) or name == "SECRET_MANAGER_REFRESH_INTERVAL" + ), + settings=KeyManagementSettings().model_dump(mode="json"), + enterprise_enabled=adapter.enterprise_enabled, + owner_type=type(client), + environment_attributes=adapter.environment_attributes, + settings_attributes=adapter.settings_attributes, + methods=tuple((name, getattr(type(client), name)) for name in adapter.methods), + ) + vars(client)["_litellm_native_secret_config"] = config + return config + + +def capture_secret_manager(client: object, system: str) -> None: + for adapter in (*_ADAPTERS, *_SDK_ADAPTERS): + if ( + adapter.system.value == system + and type(client).__name__ == adapter.name + and type(client) is getattr(import_module(adapter.module), adapter.name) + ): + _capture(client, adapter) + return + if ( + system == KeyManagementSystem.AWS_KMS.value + and type(client).__module__ == "botocore.client" + and type(client).__name__ == "KMS" + ): + _capture(client, _ClientAdapter(KeyManagementSystem.AWS_KMS, "botocore.client", "KMS", ("decrypt",))) + + +def captured_config(client: object) -> NativeSecretManagerConfig | None: + captured: Final = getattr(client, "_litellm_native_secret_config", None) + if isinstance(captured, NativeSecretManagerConfig): + return captured + for adapter in _ADAPTERS: + if type(client).__module__ == adapter.module and type(client) is getattr( + import_module(adapter.module), adapter.name + ): + return _capture(client, adapter) + return None diff --git a/litellm/utils.py b/litellm/utils.py index 4a4f2e8edfe..c6ade2603c4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -107,6 +107,13 @@ from litellm.litellm_core_utils.get_model_cost_map import ( fetch_registration_document, model_cost_inputs, ) +from litellm.litellm_core_utils.model_cost_table import ( + AliasedCostEntry, + Landing, + VersionCached, + changed_in_place, + model_cost_version, +) from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired from litellm.litellm_core_utils.offline_mode import is_offline_mode @@ -121,13 +128,6 @@ from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.rust_bridge._native import NativeModelCostMap, NativeRegistered from litellm.rust_bridge.catalog import decision from litellm.rust_bridge.configuration import Decision -from litellm.rust_bridge.host.model_cost import ( - AliasedCostEntry, - Landing, - VersionCached, - changed_in_place, - model_cost_version, -) _CachingHandlerResponse = None _LLMCachingHandler = None diff --git a/tests/integration/sdk/test_bedrock_native_output_register_model_sdk.py b/tests/integration/sdk/test_bedrock_native_output_register_model_sdk.py index 921552455b7..0aa36635c65 100644 --- a/tests/integration/sdk/test_bedrock_native_output_register_model_sdk.py +++ b/tests/integration/sdk/test_bedrock_native_output_register_model_sdk.py @@ -10,7 +10,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm import ModelResponse, Router -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish _OPUS_4_8: Final = "us.anthropic.claude-opus-4-8" _JSON_TOOL: Final = "json_tool_call" diff --git a/tests/integration/sdk/test_model_cost_fetch_sdk.py b/tests/integration/sdk/test_model_cost_fetch_sdk.py index 206ffa99909..fadd8386d17 100644 --- a/tests/integration/sdk/test_model_cost_fetch_sdk.py +++ b/tests/integration/sdk/test_model_cost_fetch_sdk.py @@ -28,7 +28,7 @@ from litellm.litellm_core_utils.get_model_cost_map import ( model_cost_inputs, refetch_model_cost_map, ) -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests.test_litellm_rust.support.child_interpreter import run_child_interpreter FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" diff --git a/tests/unit/caching/test_llm_caching_handler.py b/tests/unit/caching/test_llm_caching_handler.py index be0c5e311da..16ad2235a57 100644 --- a/tests/unit/caching/test_llm_caching_handler.py +++ b/tests/unit/caching/test_llm_caching_handler.py @@ -28,7 +28,7 @@ from litellm.llms.custom_httpx.http_handler import( HTTPHandler, ) from litellm.types.utils import LlmProviders -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from tests.unit.caching._event_loop_reuse import runs_until_a_loop_address_repeats diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index fe21cb23caf..88784814ba4 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -134,7 +134,7 @@ os.environ["PYTHON_DOTENV_DISABLED"] = "1" os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" import litellm # noqa: E402 # litellm reads LITELLM_LOCAL_MODEL_COST_MAP at import -from litellm.rust_bridge.host.model_cost import publish # noqa: E402 # same import-time dependency +from litellm.litellm_core_utils.model_cost_table import publish # noqa: E402 # same import-time dependency from litellm._logging import ALL_LOGGERS # noqa: E402 # same import-time dependency from litellm.anthropic_beta_headers_manager import ( reload_beta_headers_config, # noqa: E402 # same import-time dependency diff --git a/tests/unit/integrations/arize/test_arize.py b/tests/unit/integrations/arize/test_arize.py index 42dd5e01ce9..331e0503db0 100644 --- a/tests/unit/integrations/arize/test_arize.py +++ b/tests/unit/integrations/arize/test_arize.py @@ -16,7 +16,7 @@ from litellm.integrations.arize.arize import ArizeConfig, ArizeLogger from litellm.integrations.opentelemetry import OpenTelemetryConfig from litellm._logging import verbose_logger, verbose_proxy_logger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome diff --git a/tests/unit/integrations/arize/test_arize_phoenix.py b/tests/unit/integrations/arize/test_arize_phoenix.py index 73e6b62ec4c..08aa400c339 100644 --- a/tests/unit/integrations/arize/test_arize_phoenix.py +++ b/tests/unit/integrations/arize/test_arize_phoenix.py @@ -17,7 +17,7 @@ from litellm.integrations.arize.arize_phoenix import ( ) from litellm._logging import verbose_logger, verbose_proxy_logger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome diff --git a/tests/unit/integrations/opik/test_opik.py b/tests/unit/integrations/opik/test_opik.py index 0bf35ffdabf..30d6fffee78 100644 --- a/tests/unit/integrations/opik/test_opik.py +++ b/tests/unit/integrations/opik/test_opik.py @@ -12,7 +12,7 @@ import pytest import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 4b6f59a78ab..c6f99a36ec4 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -19,7 +19,7 @@ from openai.types.chat import ChatCompletionMessage from pydantic import BaseModel, ConfigDict import litellm -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from litellm import Router from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( diff --git a/tests/unit/integrations/test_helicone.py b/tests/unit/integrations/test_helicone.py index 90fc2f97ee8..f979090dc60 100644 --- a/tests/unit/integrations/test_helicone.py +++ b/tests/unit/integrations/test_helicone.py @@ -7,7 +7,7 @@ from collections.abc import Iterator from pathlib import Path from typing import Final from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from unittest.mock import MagicMock diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index fafc098ed5d..01b94f01891 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -86,7 +86,7 @@ from litellm.utils import ( function_setup, validate_and_fix_openai_messages, ) -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome READ_FILE: Final[list[object]] = [{"name": "read_file", "description": "Read the contents of a file."}] diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index ea270fbf7c8..3eb87947f0c 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -21,7 +21,7 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.model_latest_aliases import get_model_latest_aliases from litellm.types.router import LiteLLM_Params -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from unittest.mock import patch diff --git a/tests/unit/litellm_core_utils/test_model_cost_table.py b/tests/unit/litellm_core_utils/test_model_cost_table.py new file mode 100644 index 00000000000..1873dab8129 --- /dev/null +++ b/tests/unit/litellm_core_utils/test_model_cost_table.py @@ -0,0 +1,57 @@ +from typing import Final + +import pytest + +import litellm +from litellm.cost_calculator import cost_per_token +from litellm.litellm_core_utils.get_model_cost_map import model_cost_inputs +from litellm.litellm_core_utils.model_cost_table import changed_in_place, publish, write + +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + + +_MODEL: Final = "lit-table-model" + + +def _row(price: float) -> dict[str, object]: + return {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": price, "output_cost_per_token": price} + + +def _billed() -> float: + prompt, completion = cost_per_token( + model=_MODEL, custom_llm_provider="openai", prompt_tokens=1, completion_tokens=0 + ) + return prompt + completion + + +def test_a_published_table_is_read_in_place_of_what_was_cached_for_the_one_before() -> None: + publish({**litellm.model_cost, _MODEL: _row(1.0)}) + assert litellm.get_model_info(_MODEL)["input_cost_per_token"] == 1.0 + assert _billed() == 1.0 + + publish({**litellm.model_cost, _MODEL: _row(2.0)}) + + assert litellm.get_model_info(_MODEL)["input_cost_per_token"] == 2.0 + assert _billed() == 2.0 + + +def test_rows_written_into_the_live_table_replace_a_cached_value_of_the_same_size() -> None: + publish({**litellm.model_cost, _MODEL: _row(1.0)}) + assert _billed() == 1.0 + + write(litellm.model_cost, {_MODEL: _row(3.0)}) + + assert litellm.get_model_info(_MODEL)["input_cost_per_token"] == 3.0 + assert _billed() == 3.0 + + +def test_a_provider_changed_in_place_regroups_the_key_once_the_change_is_reported( + monkeypatch: pytest.MonkeyPatch, +) -> None: + assert "gpt-4o" not in model_cost_inputs.keys_in(litellm.model_cost, "lit-moved") + + monkeypatch.setitem(litellm.model_cost["gpt-4o"], "litellm_provider", "lit-moved") + changed_in_place() + + assert model_cost_inputs.keys_in(litellm.model_cost, "lit-moved") == ["gpt-4o"] + assert "gpt-4o" not in model_cost_inputs.keys_in(litellm.model_cost, "openai") diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index b4bf1120935..f6300b2e70c 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -32,7 +32,7 @@ from litellm.types.utils import ( StandardLoggingPayloadErrorInformation, StreamingChoices, ) -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish import unittest.mock diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index 2cb8b9cf6c5..cec8c07ca1a 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -27,7 +27,7 @@ from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5 from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig from litellm.router import Router from litellm.utils import get_optional_params -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome _MAPPED_PARAMS: Final = TypeAdapter(dict[str, object]) diff --git a/tests/unit/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py b/tests/unit/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py index 5ead4ad3986..90a8939e2dd 100644 --- a/tests/unit/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py +++ b/tests/unit/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py @@ -16,7 +16,7 @@ from litellm.llms.azure_ai.image_generation.flux_transformation import ( ) from litellm.types.utils import ImageObject, ImageResponse from litellm.utils import get_optional_params_image_gen -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish @pytest.fixture(autouse=True) diff --git a/tests/unit/llms/azure_ai/test_azure_ai_fw_models_metadata.py b/tests/unit/llms/azure_ai/test_azure_ai_fw_models_metadata.py index 927833b530d..9dc29e9860d 100644 --- a/tests/unit/llms/azure_ai/test_azure_ai_fw_models_metadata.py +++ b/tests/unit/llms/azure_ai/test_azure_ai_fw_models_metadata.py @@ -19,7 +19,7 @@ def use_local_model_cost_map(): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") import litellm - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish original_model_cost = litellm.model_cost litellm.model_cost = json.loads( diff --git a/tests/unit/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py b/tests/unit/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py index e23be01df55..be5e3a5ce18 100644 --- a/tests/unit/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py +++ b/tests/unit/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py @@ -14,7 +14,7 @@ def use_local_model_cost_map(): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") import litellm - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish original_model_cost = litellm.model_cost litellm.model_cost = json.loads( diff --git a/tests/unit/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py b/tests/unit/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py index d17c2569b92..690350223ea 100644 --- a/tests/unit/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py +++ b/tests/unit/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py @@ -27,7 +27,7 @@ def ensure_nova_canvas_image_edit_model_cost_flags(monkeypatch): Full ``model_prices_and_context_window.json`` includes these flags, but CI or alternate cost maps may omit them—merge minimal entries so tests match production. """ - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish for key in ( "amazon.nova-canvas-v1:0", diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index b0071287746..4e031531180 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -42,7 +42,7 @@ from litellm.types.llms.custom_http import VerifyTypes from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from litellm.exceptions import Timeout as LitellmTimeout from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index bdeff226af2..1936fc1d2df 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -42,7 +42,7 @@ from litellm.types.utils import ( PromptTokensDetails, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from openai.types.beta.assistant import Assistant from openai.types.beta.assistant_deleted import AssistantDeleted from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome diff --git a/tests/unit/llms/parallel_ai/test_parallel_ai_search.py b/tests/unit/llms/parallel_ai/test_parallel_ai_search.py index 1400fcc7d4e..d067aa49352 100644 --- a/tests/unit/llms/parallel_ai/test_parallel_ai_search.py +++ b/tests/unit/llms/parallel_ai/test_parallel_ai_search.py @@ -57,7 +57,7 @@ def bundled_cost_map(monkeypatch): litellm caches model-info lookups, so swapping ``model_cost`` only takes effect once those caches are invalidated -- on the way in and back out. """ - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) publish(litellm.model_cost) diff --git a/tests/unit/llms/test_custom_llm.py b/tests/unit/llms/test_custom_llm.py index e731d79ed99..a9668ebb485 100644 --- a/tests/unit/llms/test_custom_llm.py +++ b/tests/unit/llms/test_custom_llm.py @@ -36,7 +36,7 @@ from litellm.types.utils import ( StreamingChoices, ) from litellm.utils import ModelResponseIterator -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 8c2cb643943..531c5d0b1e6 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -37,7 +37,7 @@ from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata from litellm.types.utils import ChoiceLogprobs, ModelInfo, ModelResponseStream, Usage from litellm.utils import CustomStreamWrapper -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.integrations.custom_logger import CustomLogger from litellm.llms.vertex_ai.gemini.transformation import ( diff --git a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py index feda96c46b7..bd5941b880e 100644 --- a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py @@ -1086,7 +1086,7 @@ async def test_model_cost_map_reload_does_not_publish_config_change() -> None: from litellm.proxy import proxy_server from litellm.proxy.proxy_server import ProxyConfig, _set_redis_usage_cache from litellm.proxy.utils import litellm_config_cache - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish litellm_config_cache.flush_cache() prisma_client = _reload_config_prisma_client() diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index 588411cd57a..8423dee9886 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -34,7 +34,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) from fastapi import Request as Request_http_parsing from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from starlette.types import Message from starlette.websockets import WebSocket from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome diff --git a/tests/unit/proxy/common_utils/test_model_deprecation.py b/tests/unit/proxy/common_utils/test_model_deprecation.py index 57839d65cfb..8961c0a1a34 100644 --- a/tests/unit/proxy/common_utils/test_model_deprecation.py +++ b/tests/unit/proxy/common_utils/test_model_deprecation.py @@ -15,7 +15,7 @@ import pytest import litellm from litellm import Router from litellm._logging import verbose_logger -from litellm.rust_bridge.host.model_cost import AliasedCostEntry +from litellm.litellm_core_utils.model_cost_table import AliasedCostEntry from litellm.proxy.common_utils.model_deprecation import ( _classify, _parse_deprecation_date, diff --git a/tests/unit/proxy/hooks/test_dynamic_rate_limiter.py b/tests/unit/proxy/hooks/test_dynamic_rate_limiter.py index c24d04b3a8b..87bb193dfbf 100644 --- a/tests/unit/proxy/hooks/test_dynamic_rate_limiter.py +++ b/tests/unit/proxy/hooks/test_dynamic_rate_limiter.py @@ -14,7 +14,7 @@ from litellm import DualCache as DualCache_dynamic_rate, Router from litellm._uuid import uuid from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.proxy._types import UserAPIKeyAuth -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from typing import Optional diff --git a/tests/unit/proxy/hooks/test_prompt_injection_detection.py b/tests/unit/proxy/hooks/test_prompt_injection_detection.py index 1b50de938b6..1db5e9e6e18 100644 --- a/tests/unit/proxy/hooks/test_prompt_injection_detection.py +++ b/tests/unit/proxy/hooks/test_prompt_injection_detection.py @@ -17,7 +17,7 @@ from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm import Router as Router_prompt_injection from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome diff --git a/tests/unit/proxy/management_endpoints/test_sso_helper_utils.py b/tests/unit/proxy/management_endpoints/test_sso_helper_utils.py index 0b488618636..1e5bb5a8531 100644 --- a/tests/unit/proxy/management_endpoints/test_sso_helper_utils.py +++ b/tests/unit/proxy/management_endpoints/test_sso_helper_utils.py @@ -9,7 +9,7 @@ import os import pytest import litellm from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from litellm.proxy._types import LitellmUserRoles diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index a988250159f..f0459e7b927 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -27,7 +27,7 @@ from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy import proxy_server from litellm.proxy._types import LiteLLM_ProxyModelTable, UserAPIKeyAuth -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from .conftest import normalize, route_entry_points_to_stubs # type: ignore[import-not-found] diff --git a/tests/unit/proxy/proxy_server/test_routes_utils.py b/tests/unit/proxy/proxy_server/test_routes_utils.py index 551c4b76766..cdbdb70e3ac 100644 --- a/tests/unit/proxy/proxy_server/test_routes_utils.py +++ b/tests/unit/proxy/proxy_server/test_routes_utils.py @@ -17,7 +17,7 @@ import pytest import respx import litellm -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from litellm.litellm_core_utils import get_llm_provider_logic from litellm.proxy import proxy_server diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 9bbdc7165dd..5b423421458 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -59,7 +59,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendUpdateQueue from litellm.proxy.hooks.parallel_request_limiter_v3 import RequestRateLimiterStash from litellm.proxy.proxy_server import app, aws_error_type_exception_handler, initialize, openai_exception_handler -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from litellm.proxy.common_utils.banner import LITELLM_BANNER from tests._master_key import MASTER_KEY from litellm.redis.stores.connection import connect diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index e3eb9961228..94f60fa5b30 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -13,7 +13,7 @@ from litellm.responses.litellm_completion_transformation.transformation import L from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse from litellm.types.utils import ModelInfo, Usage -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish class TestResponsesAPIRequestUtils: diff --git a/tests/unit/router_strategy/test_lowest_cost.py b/tests/unit/router_strategy/test_lowest_cost.py index f71d1ad9144..e23b7628c35 100644 --- a/tests/unit/router_strategy/test_lowest_cost.py +++ b/tests/unit/router_strategy/test_lowest_cost.py @@ -7,7 +7,7 @@ import litellm from litellm.caching.caching import DualCache from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome DEPLOYMENT_ID = "9876" diff --git a/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py b/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py index 1139de6c0fc..60cad628598 100644 --- a/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py +++ b/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py @@ -11,7 +11,7 @@ import traceback from typing import Dict, Final, Iterator from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from unittest.mock import patch diff --git a/tests/unit/rust_bridge/host/test_model_cost.py b/tests/unit/rust_bridge/host/test_model_cost.py index 82a52a941a1..66f411da83e 100644 --- a/tests/unit/rust_bridge/host/test_model_cost.py +++ b/tests/unit/rust_bridge/host/test_model_cost.py @@ -3,16 +3,8 @@ from typing import Final import pytest import litellm -from litellm.cost_calculator import cost_per_token -from litellm.litellm_core_utils.get_model_cost_map import model_cost_inputs -from litellm.rust_bridge.host.model_cost import ( - changed_in_place, - entry, - merged, - publish, - resolve, - write, -) +from litellm.litellm_core_utils.model_cost_table import AliasedCostEntry +from litellm.rust_bridge.host.model_cost import merged, resolve pytestmark = pytest.mark.usefixtures("local_model_cost_map") @@ -25,56 +17,9 @@ def test_resolve_lands_on_the_row_a_lookup_in_the_published_table_resolves() -> def test_merged_keeps_the_alias_names_and_adds_only_the_fields_a_registration_set() -> None: - current: Final = entry({"mode": "chat", "x": 1}, ("owner", "alias"), frozenset({"mode"})) + current: Final = AliasedCostEntry({"mode": "chat", "x": 1}, ("owner", "alias"), frozenset({"mode"})) row: Final = merged(current, {"x": 2}, {"litellm_provider": "openai", "output_cost_per_token": None}) assert row == {"mode": "chat", "x": 2, "litellm_provider": "openai"} assert (row.names, row.registered_fields) == (("owner", "alias"), frozenset({"mode", "litellm_provider"})) - - -_MODEL: Final = "lit-table-model" - - -def _row(price: float) -> dict[str, object]: - return {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": price, "output_cost_per_token": price} - - -def _billed() -> float: - prompt, completion = cost_per_token( - model=_MODEL, custom_llm_provider="openai", prompt_tokens=1, completion_tokens=0 - ) - return prompt + completion - - -def test_a_published_table_is_read_in_place_of_what_was_cached_for_the_one_before() -> None: - publish({**litellm.model_cost, _MODEL: _row(1.0)}) - assert litellm.get_model_info(_MODEL)["input_cost_per_token"] == 1.0 - assert _billed() == 1.0 - - publish({**litellm.model_cost, _MODEL: _row(2.0)}) - - assert litellm.get_model_info(_MODEL)["input_cost_per_token"] == 2.0 - assert _billed() == 2.0 - - -def test_rows_written_into_the_live_table_replace_a_cached_value_of_the_same_size() -> None: - publish({**litellm.model_cost, _MODEL: _row(1.0)}) - assert _billed() == 1.0 - - write(litellm.model_cost, {_MODEL: _row(3.0)}) - - assert litellm.get_model_info(_MODEL)["input_cost_per_token"] == 3.0 - assert _billed() == 3.0 - - -def test_a_provider_changed_in_place_regroups_the_key_once_the_change_is_reported( - monkeypatch: pytest.MonkeyPatch, -) -> None: - assert "gpt-4o" not in model_cost_inputs.keys_in(litellm.model_cost, "lit-moved") - - monkeypatch.setitem(litellm.model_cost["gpt-4o"], "litellm_provider", "lit-moved") - changed_in_place() - - assert model_cost_inputs.keys_in(litellm.model_cost, "lit-moved") == ["gpt-4o"] - assert "gpt-4o" not in model_cost_inputs.keys_in(litellm.model_cost, "openai") diff --git a/tests/unit/rust_bridge/test_preflight.py b/tests/unit/rust_bridge/host/test_preflight.py similarity index 92% rename from tests/unit/rust_bridge/test_preflight.py rename to tests/unit/rust_bridge/host/test_preflight.py index a8b1a40a00b..ebf8638f72e 100644 --- a/tests/unit/rust_bridge/test_preflight.py +++ b/tests/unit/rust_bridge/host/test_preflight.py @@ -8,8 +8,8 @@ import pytest from pydantic import TypeAdapter import litellm -from litellm.rust_bridge import preflight -from litellm.rust_bridge.preflight import check_limits +from litellm.rust_bridge.host import preflight +from litellm.rust_bridge.host.preflight import check_limits @pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) @@ -47,7 +47,7 @@ def test_check_limits_refuses_a_call_over_the_budget(monkeypatch: pytest.MonkeyP check_limits({"model": "mistral/mistral-ocr-latest"}) -CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/python-bridge/preflight_contract.json" +CONTRACT_PATH: Final = Path(__file__).parents[4] / "litellm-rust/crates/python-bridge/preflight_contract.json" _SHIMS: Final[MappingProxyType[str, Callable[..., object]]] = MappingProxyType( { "credential_list": preflight.credential_list, diff --git a/tests/unit/rust_bridge/host/test_secret_manager.py b/tests/unit/rust_bridge/host/test_secret_manager.py index 1f8acc5792e..0705b3c806d 100644 --- a/tests/unit/rust_bridge/host/test_secret_manager.py +++ b/tests/unit/rust_bridge/host/test_secret_manager.py @@ -18,7 +18,8 @@ from pydantic import JsonValue import litellm from litellm.rust_bridge import _native as native -from litellm.rust_bridge.host.secret_manager import capture_secret_manager, native_secret_manager_config +from litellm.rust_bridge.host.secret_manager import native_secret_manager_config +from litellm.secret_managers.native_config import capture_secret_manager from litellm.secret_managers.aws_secret_manager_v2 import AWSSecretsManagerV2 from litellm.secret_managers.cyberark_secret_manager import CyberArkSecretManager from litellm.secret_managers.hashicorp_secret_manager import HashicorpSecretManager diff --git a/tests/unit/rust_bridge/test_logger.py b/tests/unit/rust_bridge/test_logger.py index df3aedf4cb9..97a32609538 100644 --- a/tests/unit/rust_bridge/test_logger.py +++ b/tests/unit/rust_bridge/test_logger.py @@ -18,7 +18,8 @@ from litellm.litellm_core_utils.secret_redaction import ( python_redact_string, python_redact_structured_value, ) -from litellm.rust_bridge import diagnostics, logger +from litellm.rust_bridge import diagnostics +from litellm.rust_bridge.host import logger def test_native_records_preserve_metadata_and_redact_before_custom_handlers( diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 8af9ae29c67..07b6111aa67 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -39,7 +39,7 @@ from litellm.types.rerank import RerankResponse from litellm.types.utils import CacheCreationTokenDetails, CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage from litellm.types.videos.main import VideoObject from litellm.utils import cached_get_model_info_helper, supports_prompt_caching -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish import os import threading from unittest.mock import MagicMock, patch diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index b300562b029..c4879587038 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5653,7 +5653,7 @@ def _priced_at(prompt_tokens, completion_tokens): def local_cost_map(monkeypatch): """The prices these tests assert are the checked-in ones. Setting the environment variable alone does not reload the map, so pin the map itself.""" - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) diff --git a/tests/unit/test_model_cost_aliases.py b/tests/unit/test_model_cost_aliases.py index c01e4d07e3e..3dfee102004 100644 --- a/tests/unit/test_model_cost_aliases.py +++ b/tests/unit/test_model_cost_aliases.py @@ -12,7 +12,7 @@ from typing import Final import pytest from litellm.rust_bridge._native import NativeInstalled, NativeModelCostMap, NativeRejected -from litellm.rust_bridge.host.model_cost import AliasedCostEntry +from litellm.litellm_core_utils.model_cost_table import AliasedCostEntry def _install(document: dict[str, dict[str, object]]) -> NativeInstalled: diff --git a/tests/unit/test_no_top_level_test_invocations.py b/tests/unit/test_no_top_level_test_invocations.py index 386c71d779e..eae026bbf71 100644 --- a/tests/unit/test_no_top_level_test_invocations.py +++ b/tests/unit/test_no_top_level_test_invocations.py @@ -8,7 +8,7 @@ import pytest import litellm from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome LOCAL_TESTING_DIR = Path(__file__).parents[1] / "local_testing" diff --git a/tests/unit/test_pydantic_namespaces.py b/tests/unit/test_pydantic_namespaces.py index fc21ea501db..3a4c1459d14 100644 --- a/tests/unit/test_pydantic_namespaces.py +++ b/tests/unit/test_pydantic_namespaces.py @@ -7,7 +7,7 @@ import pytest import litellm from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index d4f591e7291..969ca0269cf 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -28,7 +28,7 @@ from litellm.litellm_core_utils.get_model_cost_map import ( model_cost_inputs, ) from litellm.rust_bridge._native import NativeTableSwapped -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from litellm.types.utils import ( Choices, CompletionTokensDetailsWrapper, @@ -258,7 +258,7 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa litellm.model_cost.pop(registered_key, None) litellm.model_cost = original_model_cost os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish publish(litellm.model_cost) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 6557db7be32..e84d187f937 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -101,7 +101,7 @@ from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, ) from litellm.types.utils import CallTypes, CredentialItem, Usage -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE from tests.large_text import text diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 8d15ba02a38..8bc201b030a 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -48,7 +48,7 @@ from litellm.types.router import ( from litellm.utils import ( reapply_runtime_model_cost_registrations, ) -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish _COPILOT_TOKEN_EXPIRES_AT: Final = 4_102_444_800 diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index 5856caf7740..832b73a0730 100644 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -21,7 +21,7 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.scheduler import FlowItem, Scheduler, SchedulerCacheKeys -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 8d8c123ef49..6fece4d8c66 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -120,7 +120,7 @@ from litellm.utils import ( validate_environment, validate_chat_completion_tool_choice, ) -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish @pytest.mark.parametrize("root_base_model", [42, ["gemini-3.5-flash"], {"model": "gemini-3.5-flash"}]) @@ -2529,7 +2529,7 @@ def test_register_model_with_scientific_notation(): test_model_name = f"test-scientific-notation-model-{uuid.uuid4().hex[:12]}" # Clear LRU caches that might have stale data - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish publish(litellm.model_cost) model_cost_dict = { @@ -5173,7 +5173,7 @@ def test_reapply_runtime_registrations_replays_register_model_overrides( from litellm.utils import ( reapply_runtime_model_cost_registrations, ) - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish # Only the recorded half is under test here; the live-router rebuild is covered # in test_router_model_cost_isolation.py. Routers built by earlier tests in this @@ -5230,7 +5230,7 @@ def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkey from litellm.utils import ( reapply_runtime_model_cost_registrations, ) - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish saved_model_cost = litellm.model_cost try: @@ -5257,7 +5257,7 @@ def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkey def test_model_cost_walks_survive_a_registration_landing_mid_walk(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish class _EntryRegisteredMidWalk(dict): def get(self, field: str, default: object = None) -> object: @@ -5306,7 +5306,7 @@ def test_request_path_cost_map_walks_survive_a_registration_mid_walk( from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig from litellm.llms.ollama.common_utils import OllamaModelInfo from litellm.llms.openai.chat.gpt_reasoning_transformation import _catalogue_declares_default_effort - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish walks: Final[Mapping[str, Callable[[], object]]] = { "fal_ai": lambda: _keyed_rows("lit-5853-model", "standard"), @@ -8537,7 +8537,7 @@ def test_get_model_info_gemini(monkeypatch): def test_register_model_without_a_provider_never_inherits_another_providers_entry( monkeypatch: pytest.MonkeyPatch, ) -> None: - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish key: Final = "azure/claude-sonnet-4-6" monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) @@ -8570,7 +8570,7 @@ def test_register_model_without_a_provider_never_inherits_another_providers_entr def test_register_model_over_a_builtin_row_bills_explicit_cache_hits( monkeypatch: pytest.MonkeyPatch, registered_prices: dict[str, float], billed_cache_read_rate: float ) -> None: - from litellm.rust_bridge.host.model_cost import publish + from litellm.litellm_core_utils.model_cost_table import publish key: Final = "dashscope/qwen-register-explicit-cache-test" monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) @@ -11659,7 +11659,7 @@ def test_the_catalog_with_register_model_entries_lists_each_model_once_and_leave ) def test_runtime_model_cost_registrations_resolve_bedrock_routing(monkeypatch: pytest.MonkeyPatch, key: str) -> None: from litellm import utils as model_utils - from litellm.rust_bridge.host.model_cost import AliasedCostEntry + from litellm.litellm_core_utils.model_cost_table import AliasedCostEntry rates: Final = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002} alias: Final = "openai.test-converse-alias" diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index cde0a8978c0..0ebe291cf62 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -14,7 +14,7 @@ from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.utils import ModelInfoBase from litellm.utils import supports_native_structured_output -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish import os from unittest.mock import MagicMock, patch diff --git a/tests/unit/types/test_files.py b/tests/unit/types/test_files.py index 5b2c82b69db..b55dde3d44b 100644 --- a/tests/unit/types/test_files.py +++ b/tests/unit/types/test_files.py @@ -16,7 +16,7 @@ from litellm.types.files import ( get_file_mime_type_from_extension, get_file_type_from_extension, ) -from litellm.rust_bridge.host.model_cost import publish +from litellm.litellm_core_utils.model_cost_table import publish from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome From 35e501974e227acccd203289737c6d31857e764c Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:19:17 -0800 Subject: [PATCH 062/189] fix(spend): write a background response's settling read in-process when the spend collector is on (#50957) * fix(passthrough): bill a background response once however many times it is read * fix(passthrough): keep the background response flag on spend events sent to the collector * test(spend): type the background response test helper and its settlement parameter * fix(proxy): keep a background response's settling read in-process instead of adding a spend event field An older collector refuses a spend event with a field it does not know and drops the whole charge, so a newer gateway sending to an older collector lost the background response's cost. The read that settles a background response now stays in-process, the same way video status polls do, and the spend event keeps its shape * fix(proxy): keep a batch retrieve in-process when message redaction hides the logged batch * refactor(proxy): split the is_offloadable_success docstring into one line per reason --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../proxy/hooks/proxy_track_cost_callback.py | 2 +- litellm/proxy/spend_tracking/spend_event.py | 22 ++++++++-- .../hooks/test_proxy_track_cost_callback.py | 40 ++++++++++++++++--- .../proxy/spend_tracking/test_spend_event.py | 26 +++++++++--- 4 files changed, 75 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index dd7a24ae879..554fc7ed1d5 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -425,7 +425,7 @@ class _ProxyDBLogger(CustomLogger): async def async_log_success_event( self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime ) -> None: - if self.spend_event_producer is None or not is_offloadable_success(response_obj, kwargs.get("call_type")): + if self.spend_event_producer is None or not is_offloadable_success(response_obj, kwargs): await self._PROXY_track_cost_callback(dict(kwargs), response_obj, start_time, end_time) return event: Final = build_spend_event( diff --git a/litellm/proxy/spend_tracking/spend_event.py b/litellm/proxy/spend_tracking/spend_event.py index d054519cb9c..52b9acf44c0 100644 --- a/litellm/proxy/spend_tracking/spend_event.py +++ b/litellm/proxy/spend_tracking/spend_event.py @@ -23,10 +23,15 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm +from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.db.spend_log_tool_index import response_tool_call_names from litellm.types.interactions import InteractionsAPIResponse from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) from litellm.types.utils import LiteLLMBatch, Usage from litellm.types.videos.utils import is_video_status_call_type @@ -206,10 +211,19 @@ class SpendEventDecodeError: reason: str -def is_offloadable_success(response_obj: object, call_type: object) -> bool: - """Batch retrieves and interaction polls branch on the concrete response class, and a video status poll on - the status and video id the event leaves out, so they stay in-process.""" - return not isinstance(response_obj, _NOT_OFFLOADED_RESPONSE_TYPES) and not is_video_status_call_type(call_type) +def is_offloadable_success(response_obj: object, kwargs: ObjectMapping) -> bool: + """A success stays in-process when its cost logic reads something the spend event leaves out. + + Batches and interaction polls need the concrete response class. A batch retrieve needs the polled batch id, + which message redaction can hide. A video status poll needs the status and the video id. A background + response's settling read needs the settlement flag.""" + call_type: Final = kwargs.get("call_type") + return ( + not isinstance(response_obj, _NOT_OFFLOADED_RESPONSE_TYPES) + and not is_batch_retrieve_call_type(call_type) + and not is_video_status_call_type(call_type) + and kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is not BackgroundResponseSettlement.SETTLES + ) def _json_fallback(value: object) -> str: diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 43f529faa6b..dcec934188f 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -18,7 +18,9 @@ from fastapi import FastAPI, HTTPException import litellm from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache +from litellm.litellm_core_utils.batch_utils import POLLED_BATCH_COST_IS_FINAL_KWARG, POLLED_BATCH_ID_KWARG from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY +from litellm.litellm_core_utils.redact_messages import perform_redaction from litellm.proxy._types import ProxyException, SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.collector import SpendEventConsumer from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded @@ -47,6 +49,10 @@ from litellm.proxy.utils import ( ) from litellm.types.proxy.carried_budget_state import CommittedBudgetSpend, EntitySpendCounterKeys from litellm.types.router import Deployment, LiteLLM_Params +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage from litellm.types.videos.main import VideoObject @@ -3692,17 +3698,41 @@ def _completed_batch() -> LiteLLMBatch: ) +def _redacted_completed_batch() -> object: + return perform_redaction({}, _completed_batch()) + + def _completed_video() -> VideoObject: return VideoObject(id="video_abc", object="video", status="completed") @pytest.mark.asyncio @pytest.mark.parametrize( - ("call_type", "polled_job"), - [(CallTypes.aretrieve_batch.value, _completed_batch), ("avideo_status", _completed_video)], + ("job_kwargs", "polled_job"), + [ + ({"call_type": CallTypes.aretrieve_batch.value}, _completed_batch), + ( + { + "call_type": CallTypes.aretrieve_batch.value, + POLLED_BATCH_ID_KWARG: "batch_abc", + POLLED_BATCH_COST_IS_FINAL_KWARG: True, + }, + _redacted_completed_batch, + ), + ({"call_type": "avideo_status"}, _completed_video), + ( + { + "call_type": "pass_through_endpoint", + SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES, + }, + _offload_response, + ), + ], + ids=["batch_retrieve", "redacted_batch_retrieve", "video_status_poll", "background_response_settling_read"], ) -async def test_async_log_success_event_keeps_job_polls_in_process(call_type, polled_job): - """The spend event leaves out the polled video's status and id, which decide whether the poll charges the job.""" +async def test_async_log_success_event_keeps_job_polls_in_process(job_kwargs, polled_job): + """The spend event leaves out the polled video's status and id and a background response's settlement flag, + which decide whether the poll charges the job and which spend row it writes.""" producer = SpendEventProducer( address=UnixAddress(path="/nonexistent/spend.sock"), on_unavailable="drop", @@ -3711,7 +3741,7 @@ async def test_async_log_success_event_keeps_job_polls_in_process(call_type, pol fallback=_no_fallback, ) logger = ProxyDBLogger(producer) - kwargs = {**_offload_kwargs(), "call_type": call_type} + kwargs = {**_offload_kwargs(), **job_kwargs} completed_job = polled_job() with ( diff --git a/tests/unit/proxy/spend_tracking/test_spend_event.py b/tests/unit/proxy/spend_tracking/test_spend_event.py index e1912d36d91..52422533213 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_event.py +++ b/tests/unit/proxy/spend_tracking/test_spend_event.py @@ -16,6 +16,10 @@ from litellm.proxy.spend_tracking.spend_event import ( is_offloadable_success, spend_event_callback_args, ) +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) from litellm.types.utils import LiteLLMBatch, ModelResponse, Usage _BIG_PROMPT: Final = "x" * 20_000 @@ -194,11 +198,23 @@ def test_undecodable_line_is_an_error_value(): assert isinstance(decode_spend_event(b"not json\n"), SpendEventDecodeError) +@pytest.mark.parametrize( + ("settlement", "offloadable"), + [(BackgroundResponseSettlement.SETTLES, False), (None, True), ("settles", True)], + ids=["settling_read_stays_in_process", "other_read_is_offloaded", "lookalike_string_is_offloaded"], +) +def test_only_a_background_response_settling_read_stays_in_process( + settlement: BackgroundResponseSettlement | str | None, offloadable: bool +) -> None: + kwargs: Final = {"call_type": "pass_through_endpoint", SETTLES_BACKGROUND_RESPONSE_KWARG: settlement} + assert is_offloadable_success(None, kwargs) is offloadable + + def test_batch_retrieves_and_video_status_polls_stay_in_process(): - assert is_offloadable_success(_response(), "acompletion") is True - assert is_offloadable_success(None, None) is True - assert is_offloadable_success(None, "avideo_generation") is True - assert is_offloadable_success(None, "avideo_status") is False + assert is_offloadable_success(_response(), {"call_type": "acompletion"}) is True + assert is_offloadable_success(None, {}) is True + assert is_offloadable_success(None, {"call_type": "avideo_generation"}) is True + assert is_offloadable_success(None, {"call_type": "avideo_status"}) is False assert ( is_offloadable_success( LiteLLMBatch( @@ -210,7 +226,7 @@ def test_batch_retrieves_and_video_status_polls_stay_in_process(): object="batch", status="completed", ), - "aretrieve_batch", + {"call_type": "aretrieve_batch"}, ) is False ) From 8092edc7a1fe953e3aaa71a70e6b2fd41699025b Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 10 Nov 2026 18:28:49 -0800 Subject: [PATCH 063/189] fix(proxy): judge the zero-cost budget waiver by the model a key or team alias serves (#51058) * fix(proxy): judge the zero-cost budget waiver by the model a key or team alias serves The zero-cost budget waiver judged the name the client sent, but key `aliases` and team `model_aliases` are applied later during request preparation. An over-budget key got a 422 for an alias of a $0 model, and an alias named like a $0 group that points at a paid model was waived and served past the budget. For a string body `model` on chat, completions, responses and messages, the key-path, centralized and custom-auth waivers now judge the model request preparation serves. That model comes from the same team, key, global and router alias functions that request preparation uses, so the two cannot drift. Other routes and the JWT path keep judging the requested model. common_checks no longer overrides a caller's explicit decision not to skip budgets, and proxy-made sub-calls are judged by their own model. Resolves LIT-10283 * fix(proxy): waive only once auth has applied the router aliases, and keep empty team alias targets The served-model waiver now applies only after auth's router-alias rewrite has run on the request body. Custom auth without common checks runs its waiver before that rewrite, so it keeps judging the requested model instead of a key or team alias of a name the router rewrites elsewhere. The waiver also keeps an empty-string team alias target, matching request preparation. Resolves LIT-10283 * fix(proxy): waive by the served model only for a client's own body-model request The waiver resolved aliases for any request on a chat, completions, responses or messages path, including the HTTP request the Responses WebSocket builds for first-frame admission. That session dispatches the model as an endpoint model, which skips the team alias, so a team alias of a paid model to a free one waived the paid model's budget. The waiver now resolves aliases only when FastAPI matched the request to one of the body model routes, which requests built inside the proxy and path model routes never are. The waiver tests are folded into one table, share a priced router fixture, and cover the stale team alias flag cache. Resolves LIT-10283 --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../routing_common_utils.py | 25 --- litellm/proxy/auth/auth_checks.py | 20 +- litellm/proxy/auth/user_api_key_auth.py | 107 +++++++++-- litellm/proxy/common_request_processing.py | 37 ++-- litellm/proxy/litellm_pre_call_utils.py | 178 ++++++++++++------ .../auth/test_custom_auth_end_user_budget.py | 64 ++++++- tests/unit/proxy/auth/test_model_checks.py | 55 ++++++ .../test_user_api_key_auth_request_flow.py | 93 ++++++++- tests/unit/proxy/conftest.py | 24 +++ .../unit/proxy/test_litellm_pre_call_utils.py | 58 ++++++ .../test_zero_cost_model_budget_bypass.py | 170 ++++++++++++++++- .../test_router_utils_common_utils.py | 20 -- 12 files changed, 685 insertions(+), 166 deletions(-) diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index 154c3942edc..363854f70de 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -38,7 +38,6 @@ _V = TypeVar("_V") MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation" ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG}) -_OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) @@ -215,30 +214,6 @@ def resolve_model_group_alias(model_group_alias: object, model: str) -> str | No return target -def resolve_served_model( - model: str, - model_alias_map: object, - key_aliases: object, - model_group_alias: object, -) -> str: - """Model the request preparation serves for `model`. - - Apply the global alias map, then key aliases, then `router_settings.model_group_alias`. - """ - global_aliases: Final = ( - _OBJECT_MAPPING_ADAPTER.validate_python(model_alias_map) if isinstance(model_alias_map, Mapping) else None - ) - global_target: Final = global_aliases.get(model) if global_aliases is not None else None - after_global: Final = global_target if isinstance(global_target, str) else model - key_alias_mapping: Final = ( - _OBJECT_MAPPING_ADAPTER.validate_python(key_aliases) if isinstance(key_aliases, Mapping) else None - ) - key_target: Final = key_alias_mapping.get(after_global) if key_alias_mapping is not None else None - after_key: Final = key_target if isinstance(key_target, str) else after_global - router_target: Final = resolve_model_group_alias(model_group_alias, after_key) - return router_target if router_target is not None else after_key - - def truncate_fallback_error_detail(detail: str) -> str: """ Bound a fallback failure detail before it is logged or appended to an exception message. diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f0c9632837d..fa331b37950 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -557,7 +557,7 @@ def _get_router_zero_cost_cache(llm_router: RegistryReader) -> dict[str, bool] | return cache if isinstance(cache, dict) else None -def is_model_cost_zero(model: str | list[str] | None, llm_router: RegistryReader | None) -> bool: +def is_model_cost_zero(model: str | Sequence[str] | None, llm_router: RegistryReader | None) -> bool: """ Check if a model has zero cost (no configured pricing). @@ -1122,7 +1122,7 @@ def route_skips_budget_checks(route: str) -> bool: def request_skips_budget_checks( route: str, - model: str | list[str] | None, + model: str | Sequence[str] | None, llm_router: Router | None, *, request_body: Mapping[str, object] | None = None, @@ -1146,7 +1146,7 @@ async def common_checks( proxy_logging_obj: ProxyLogging, valid_token: UserAPIKeyAuth | None, request: Request, - skip_budget_checks: bool = False, + skip_budget_checks: bool | None = None, project_object: LiteLLM_ProjectTableCachedObj | None = None, ) -> bool: """ @@ -1181,11 +1181,15 @@ async def common_checks( team_id=valid_token.team_id if valid_token is not None else None, ) - skip_all_budget_checks: Final = skip_budget_checks or request_skips_budget_checks( - route=route, - model=_model, - llm_router=llm_router, - request_body=_typed_request_body(request_body), + skip_all_budget_checks: Final = ( + skip_budget_checks + if skip_budget_checks is not None + else request_skips_budget_checks( + route=route, + model=_model, + llm_router=llm_router, + request_body=_typed_request_body(request_body), + ) ) membership_user_id: Final = ( diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 5bf4db6a725..d7a8d4a0700 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -41,7 +41,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value -from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias, resolve_served_model +from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias from litellm.models.end_user import LiteLLM_EndUserTable from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers @@ -95,6 +95,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo route_skips_budget_checks, stamp_matched_model_access_groups, team_member_max_budget_alert_check, + team_model_aliases_for_auth_check, virtual_key_max_budget_alert_check, virtual_key_max_budget_check, virtual_key_soft_budget_check, @@ -173,7 +174,11 @@ from litellm.proxy.common_utils.user_api_key_cache import ( from litellm.proxy.db.db_lookup_gate import bounded_db_lookup from litellm.proxy.db.db_span import db_span from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + pre_call_served_model, + request_served_model, +) from litellm.proxy.spend_tracking.carried_budget_state import ( carry_committed_budget_spend, carry_team_and_user_budget_state, @@ -332,6 +337,65 @@ def _is_codex_native_litellm_turn(route: str, model: object, llm_router: litellm return routes_to_litellm(model, llm_router, team_id) +_SERVED_MODEL_BUDGET_WAIVER_ROUTES: Final = frozenset( + { + "/chat/completions", + "/v1/chat/completions", + "/completions", + "/v1/completions", + "/responses", + "/v1/responses", + "/messages", + "/v1/messages", + } +) + + +def _inference_model_overridden() -> bool: + from litellm.proxy import proxy_server + + cli_model: Final[object] = proxy_server.user_model # pyright: ignore[reportUnknownMemberType] # legacy untyped CLI global + completion_model: Final = proxy_server.general_settings_view().get("completion_model") + return (isinstance(completion_model, str) and completion_model != "") or ( + isinstance(cli_model, str) and cli_model != "" + ) + + +async def budget_served_model( + model: str | Sequence[str] | None, + request_data: Mapping[str, object], + request: Request | None, + llm_router: litellm.Router | None, + valid_token: UserAPIKeyAuth, +) -> str | Sequence[str] | None: + """The model the zero-cost budget waiver judges. For a client's own HTTP request to a chat, completions, + responses or messages route that names a string body `model`, it is the model request preparation serves it + with through the caller's team, key, global and router aliases. Everywhere else it stays the requested model: + requests built inside the proxy, a path or query `model`, a configured `completion_model` or CLI `--model`, + and requests whose body auth has not yet passed through the router aliases.""" + if request is None or llm_router is None or not isinstance(model, str): + return model + if get_request_route_template(request) not in _SERVED_MODEL_BUDGET_WAIVER_ROUTES: + return model + if request.scope.get(MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY) is not True or request_data.get("model") != model: + return model + if "model" in safe_get_request_query_params(request=request) or _inference_model_overridden(): + return model + if not ( + key_model_aliases_for_auth_check(valid_token) + or team_model_aliases_for_auth_check(valid_token) + or model in litellm.model_alias_map + ): + return model + from litellm.proxy.proxy_server import prisma_client, proxy_config, proxy_logging_obj + + settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict=valid_token, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) + group_alias: Final[object] = settings.get("model_group_alias") if isinstance(settings, Mapping) else None + return request_served_model(model, valid_token, group_alias, llm_router) + + async def _resolve_router_settings_model_group_alias( request_data: dict[str, object], # mutable-ok: the request body is rewritten in place for every downstream reader valid_token: UserAPIKeyAuth, @@ -971,9 +1035,6 @@ async def _try_team_member_model_budget_fallback( model_group_aliases: Final[object] = ( router_settings.get("model_group_alias") if router_settings is not None else None ) - key_aliases: Final[object] = cast( # cast-ok: auth aliases are dynamically typed at runtime - object, valid_token.aliases - ) resolved_fallback_candidates: Final[tuple[tuple[str, str], ...]] = tuple( ( candidate, @@ -985,12 +1046,7 @@ async def _try_team_member_model_budget_fallback( ( fallback_candidate, fallback_model, - resolve_served_model( - fallback_model, - litellm.model_alias_map, - key_aliases, - model_group_aliases, - ), + pre_call_served_model(fallback_model, valid_token, model_group_aliases), ) for fallback_candidate, fallback_model in resolved_fallback_candidates ) @@ -3137,7 +3193,10 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e team_id=valid_token.team_id, ) skip_budget_checks: Final = request_skips_budget_checks( - route=route, model=model, llm_router=llm_router, request_body=request_data + route=route, + model=await budget_served_model(model, request_data, request, llm_router, valid_token), + llm_router=llm_router, + request_body=request_data, ) # Check 3. Check if user is in their team budget @@ -3968,12 +4027,13 @@ async def run_centralized_common_checks( dict(end_user_model_max_budget) if end_user_model_max_budget else None ) - skip_budget_checks: Final = _should_skip_budget_checks( + skip_budget_checks: Final = await _should_skip_budget_checks( request_data=request_data, route=route, request=request, llm_router=llm_router, - team_id=user_api_key_auth_obj.team_id, + valid_token=user_api_key_auth_obj, + sub_call=sub_call, ) typed_team_object: Final[LiteLLM_TeamTable | None] = ( team_object if isinstance(team_object, LiteLLM_TeamTable) else None @@ -4216,21 +4276,27 @@ async def _reserve_budget_after_common_checks( request.state.budget_reservation = reservation # rebind-ok: read by the release middleware -def _should_skip_budget_checks( +async def _should_skip_budget_checks( request_data: dict[str, object], route: str, request: Request | None, llm_router: litellm.Router | None, - team_id: str | None = None, + valid_token: UserAPIKeyAuth, + sub_call: bool = False, ) -> bool: model: Final = _get_model_from_request_context( request_data=request_data, route=route, request=request, llm_router=llm_router, - team_id=team_id, + team_id=valid_token.team_id, + ) + return request_skips_budget_checks( + route=route, + model=(model if sub_call else await budget_served_model(model, request_data, request, llm_router, valid_token)), + llm_router=llm_router, + request_body=request_data, ) - return request_skips_budget_checks(route=route, model=model, llm_router=llm_router, request_body=request_data) def _resolve_request_principal(request: Request, valid_token: UserAPIKeyAuth) -> Principal: @@ -4908,7 +4974,10 @@ async def _run_post_custom_auth_checks( # every budget check for these; this path did not, so the same request could # be refused under custom auth and served under the other two. skip_budget_checks: Final = ( - is_model_cost_zero(model=current_model, llm_router=llm_router) + is_model_cost_zero( + model=await budget_served_model(current_model, request_data, request, llm_router, valid_token), + llm_router=llm_router, + ) if current_model is not None and llm_router is not None else False ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index bfa559f86c5..46a9f9910d7 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -89,7 +89,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) -from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias, resolve_served_model +from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.served_output_texts import ( record_served_output_texts, @@ -310,6 +310,8 @@ from litellm.proxy.litellm_pre_call_utils import ( LiteLLMProxyRequestSetup, add_litellm_data_to_request, get_metadata_variable_name, + global_then_key_alias_target, + pre_call_served_model, refresh_proxy_server_request_body_snapshot, reject_url_valued_destination, sanitize_for_log, @@ -1017,13 +1019,11 @@ _ROUTER_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = Type def _fallback_probe_data( - pristine: object, fallback_model: str, key_aliases: object, group_alias: object + pristine: object, fallback_model: str, user_api_key_dict: UserAPIKeyAuth, group_alias: object ) -> Mapping[str, object]: return { **_REQUEST_FIELDS.validate_python(pristine), - "model": resolve_served_model( - fallback_model, _DECODED_JSON.validate_python(litellm.model_alias_map), key_aliases, group_alias - ), + "model": pre_call_served_model(fallback_model, user_api_key_dict, group_alias), } @@ -2246,13 +2246,9 @@ async def _team_member_within_fallback_model_budget( if proxy_config is not None and llm_router is not None else None ) - raw_aliases: Final[object] = cast( # cast-ok: auth aliases are dynamically typed at runtime - object, user_api_key_dict.aliases - ) - served_model: Final = resolve_served_model( + served_model: Final = pre_call_served_model( model, - litellm.model_alias_map, - raw_aliases, + user_api_key_dict, router_settings.get("model_group_alias") if router_settings is not None else None, ) if is_model_cost_zero(model=served_model, llm_router=llm_router): @@ -2714,17 +2710,9 @@ class ProxyBaseLLMRequestProcessing: ### MODEL ALIAS MAPPING ### # check if model name in model alias map # get the actual model name - if isinstance(self.data["model"], str) and self.data["model"] in litellm.model_alias_map: - self.data["model"] = litellm.model_alias_map[self.data["model"]] - - # Check key-specific aliases - if ( - isinstance(self.data["model"], str) - and user_api_key_dict.aliases - and isinstance(user_api_key_dict.aliases, dict) - and self.data["model"] in user_api_key_dict.aliases - ): - self.data["model"] = user_api_key_dict.aliases[self.data["model"]] + prepared_model: Final = self.data["model"] + if isinstance(prepared_model, str): + self.data["model"] = global_then_key_alias_target(prepared_model, user_api_key_dict) if llm_router is not None and proxy_config is not None: self.data.update( @@ -2945,9 +2933,6 @@ class ProxyBaseLLMRequestProcessing: if isinstance(raw_router_settings, Mapping) else None ) - key_aliases: Final[object] = cast( # cast-ok: auth aliases are dynamically typed at runtime - object, user_api_key_dict.aliases - ) try: for fallback_model in fallback_models: if fallback_model == original_model: @@ -2965,7 +2950,7 @@ class ProxyBaseLLMRequestProcessing: try: if endpoint_unsupported_by_model( route_type, - _fallback_probe_data(pristine, fallback_model, key_aliases, group_alias), + _fallback_probe_data(pristine, fallback_model, user_api_key_dict, group_alias), llm_router, user_api_key_dict.team_id, ): diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d5eac099fb6..447b41f9cfd 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -14,7 +14,6 @@ from typing import ( Any, Final, Literal, - Protocol, TypeAlias, cast, # noqa: TID251 # narrows untyped deployment and credential payloads ) @@ -57,7 +56,6 @@ from litellm.litellm_core_utils.core_helpers import ( attribute_or_none, is_codex_user_agent, normalize_drop_params, - require_attributes, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -325,6 +323,7 @@ def sanitize_for_log(value: object) -> str: return _sanitize_for_log(value) +from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias from litellm.router import Router from litellm.secret_managers.main import get_secret_bool from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES, STREAM_USAGE_INJECTED_FIELD @@ -3604,12 +3603,8 @@ def _warn_stale_team_alias_once(warning_key: str, message: str, *args: str) -> N verbose_proxy_logger.warning(message, *args) -class _SupportsStartswith(Protocol): - def startswith(self, prefix: str, /) -> bool: ... - - def _update_model_if_team_alias_exists( - data: dict, + data: dict[str, object], # mutable-ok: request preparation rewrites the body model in place user_api_key_dict: UserAPIKeyAuth, ) -> None: """ @@ -3634,63 +3629,91 @@ def _update_model_if_team_alias_exists( Keeping the requested model name lets it resolve against the deployments that still exist (e.g. a gateway-level model group shared with the team). """ - _model: Final = data.get("model") - team_model_aliases: Final = as_str_mapping(user_api_key_dict.team_model_aliases) - if not _model or not team_model_aliases or _model not in team_model_aliases: - return - from litellm.proxy.proxy_server import llm_router - # Skip alias rewrite if this model resolves to team-specific deployments - # (team models use team_public_model_name, not model_aliases) - aliased_target: Final = team_model_aliases[_model] + rewrite: Final = _team_alias_rewrite(data.get("model"), user_api_key_dict, llm_router) + if rewrite.warning is not None: + _warn_stale_team_alias_once(rewrite.warning.key, rewrite.warning.message, *rewrite.warning.args) + if rewrite.target is not None: + data["model"] = rewrite.target - # Optional bypass for stale aliases from pre-PR deployments: - # only enabled via feature flag to preserve backwards compatibility. - # Cached at module level to avoid hot-path secret lookups on every request. - global _ENABLE_TEAM_STALE_ALIAS_BYPASS - if _ENABLE_TEAM_STALE_ALIAS_BYPASS is None: - _ENABLE_TEAM_STALE_ALIAS_BYPASS = get_secret_bool("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", False) - enable_stale_alias_bypass: Final = _ENABLE_TEAM_STALE_ALIAS_BYPASS - # Check if the alias points to a team-scoped UUID name - # (format: "model_name_{team_id}_{uuid}") - is_stale_team_alias: Final = require_attributes(aliased_target, _SupportsStartswith, "startswith").startswith( - f"model_name_{user_api_key_dict.team_id}_" - ) - if is_stale_team_alias and llm_router: - if aliased_target not in llm_router.registry().index("model_name"): - _warn_stale_team_alias_once( - f"deleted:{user_api_key_dict.team_id}:{_model}:{aliased_target}", + +@dataclass(frozen=True, slots=True) +class _StaleTeamAliasWarning: + key: str + message: str + args: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class _TeamAliasRewrite: + target: str | None + warning: _StaleTeamAliasWarning | None = None + + +_NO_TEAM_ALIAS_REWRITE: Final = _TeamAliasRewrite(target=None) + + +def team_alias_target(model: object, user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None) -> str | None: + return _team_alias_rewrite(model, user_api_key_dict, llm_router).target + + +def _team_alias_rewrite( + model: object, user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None +) -> _TeamAliasRewrite: + team_model_aliases: Final = as_str_mapping(user_api_key_dict.team_model_aliases) + if not isinstance(model, str) or not model or not team_model_aliases or model not in team_model_aliases: + return _NO_TEAM_ALIAS_REWRITE + aliased_target: Final = team_model_aliases[model] + if not isinstance(aliased_target, str): + return _NO_TEAM_ALIAS_REWRITE + is_stale_team_alias: Final = aliased_target.startswith(f"model_name_{user_api_key_dict.team_id}_") + if not is_stale_team_alias or not llm_router: + return _TeamAliasRewrite(target=aliased_target) + if aliased_target not in llm_router.registry().index("model_name"): + return _TeamAliasRewrite( + target=None, + warning=_StaleTeamAliasWarning( + f"deleted:{user_api_key_dict.team_id}:{model}:{aliased_target}", "Team model alias for model='%s', team_id='%s' targets '%s', which has no live " "deployment. Routing with the requested model name instead; remove the stale " "entry from the team's model_aliases to silence this warning.", - _sanitize_for_log(_model), - _sanitize_for_log(user_api_key_dict.team_id), - _sanitize_for_log(aliased_target), - ) - return - # This is a stale alias from pre-PR deployments. - # Check if current team deployments exist for the public name. - key: Final = (user_api_key_dict.team_id, _model) - if key in llm_router.registry().index("team_model"): - if enable_stale_alias_bypass: - # Team deployments exist; skip stale alias - return - _warn_stale_team_alias_once( - f"{user_api_key_dict.team_id}:{_model}:{aliased_target}", - "Stale team model alias detected for model='%s', team_id='%s'. " - "New sibling deployments may be unreachable. " - "Set LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true to enable " - "team-scoped sibling routing.", - _sanitize_for_log(_model), - _sanitize_for_log(user_api_key_dict.team_id), - ) + ( + _sanitize_for_log(model), + _sanitize_for_log(user_api_key_dict.team_id), + _sanitize_for_log(aliased_target), + ), + ), + ) + if (user_api_key_dict.team_id, model) not in llm_router.registry().index("team_model"): + return _TeamAliasRewrite(target=aliased_target) + if _stale_team_alias_bypass_enabled(): + return _NO_TEAM_ALIAS_REWRITE + return _TeamAliasRewrite( + target=aliased_target, + warning=_StaleTeamAliasWarning( + f"{user_api_key_dict.team_id}:{model}:{aliased_target}", + "Stale team model alias detected for model='%s', team_id='%s'. " + "New sibling deployments may be unreachable. " + "Set LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true to enable " + "team-scoped sibling routing.", + (_sanitize_for_log(model), _sanitize_for_log(user_api_key_dict.team_id)), + ), + ) - data["model"] = aliased_target + +def _stale_team_alias_bypass_enabled() -> bool: + global _ENABLE_TEAM_STALE_ALIAS_BYPASS + cached: Final = _ENABLE_TEAM_STALE_ALIAS_BYPASS + if cached is not None: + return cached + enabled: Final = get_secret_bool("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", False) is True + _ENABLE_TEAM_STALE_ALIAS_BYPASS = enabled + return enabled def _update_model_if_key_alias_exists( - data: dict, + data: dict[str, object], # mutable-ok: request preparation rewrites the body model in place user_api_key_dict: UserAPIKeyAuth, ) -> None: """ @@ -3705,14 +3728,45 @@ def _update_model_if_key_alias_exists( } - requested_model = "xai/grok-4-fast-non-reasoning" """ - _model: Final = data.get("model") - if ( - _model - and user_api_key_dict.aliases - and isinstance(user_api_key_dict.aliases, dict) - and _model in user_api_key_dict.aliases - ): - data["model"] = user_api_key_dict.aliases[_model] + requested: Final = data.get("model") + if isinstance(requested, str): + data["model"] = key_alias_target(requested, user_api_key_dict) + + +def key_alias_target(model: str, user_api_key_dict: UserAPIKeyAuth) -> str: + from litellm.proxy.auth.auth_checks import ( # noqa: PLC0415 # auth checks import request preparation + key_model_aliases_for_auth_check, + ) + + key_aliases: Final = key_model_aliases_for_auth_check(user_api_key_dict) + return key_aliases.get(model, model) if key_aliases else model + + +def caller_alias_target(model: str, user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None) -> str: + """The model `add_litellm_data_to_request` leaves after the team's and then the key's aliases.""" + team_target: Final = team_alias_target(model, user_api_key_dict, llm_router) + return key_alias_target(team_target if team_target is not None else model, user_api_key_dict) + + +def global_then_key_alias_target(model: str, user_api_key_dict: UserAPIKeyAuth) -> str: + """The model `common_processing_pre_call_logic` leaves after the global `model_alias_map` and then the + key's aliases, ahead of the key's or team's `router_settings.model_group_alias`.""" + return key_alias_target(litellm.model_alias_map.get(model, model), user_api_key_dict) + + +def pre_call_served_model(model: str, user_api_key_dict: UserAPIKeyAuth, model_group_alias: object) -> str: + """The model group `common_processing_pre_call_logic` sends `model` to.""" + prepared: Final = global_then_key_alias_target(model, user_api_key_dict) + return resolve_model_group_alias(model_group_alias, prepared) or prepared + + +def request_served_model( + model: str, user_api_key_dict: UserAPIKeyAuth, model_group_alias: object, llm_router: Router | None +) -> str: + """The model group a request whose body names `model` is sent to, through every alias rewrite.""" + return pre_call_served_model( + caller_alias_target(model, user_api_key_dict, llm_router), user_api_key_dict, model_group_alias + ) def _apply_credential_overrides_from_model_config( diff --git a/tests/unit/proxy/auth/test_custom_auth_end_user_budget.py b/tests/unit/proxy/auth/test_custom_auth_end_user_budget.py index 63ded55c630..e75a77325c6 100644 --- a/tests/unit/proxy/auth/test_custom_auth_end_user_budget.py +++ b/tests/unit/proxy/auth/test_custom_auth_end_user_budget.py @@ -1,8 +1,9 @@ from typing import Final -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest from starlette.requests import Request +from starlette.routing import Route import litellm from litellm.proxy._types import ( @@ -451,3 +452,64 @@ def test_apply_budget_limits_copies_per_model_limits_onto_end_user_params(): "end_user_model_tpm_limit": {"gpt-4o": 400000}, "end_user_model_rpm_limit": {"gpt-4o": 200}, } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model, key_aliases, run_common_checks, expect_enforced", + [ + ("free-key", {"free-key": "free-model"}, True, False), + ("free-model", {"free-model": "paid-model"}, True, True), + ("free-key", {"free-key": "free-model"}, False, True), + ], +) +async def test_custom_auth_judges_the_zero_cost_waiver_by_the_key_alias_target( + requested_model: str, + key_aliases: dict[str, str], + run_common_checks: bool, + expect_enforced: bool, + free_and_paid_router: litellm.Router, +) -> None: + valid_token: Final = UserAPIKeyAuth( + token="test_token", + aliases=key_aliases, + end_user_id="end-user", + end_user_model_max_budget={requested_model: {"budget_limit": 1.0, "time_period": "1d"}}, + ) + limiter: Final = MagicMock() + limiter.is_end_user_within_model_budget = AsyncMock( + side_effect=litellm.BudgetExceededError(current_cost=5.0, max_budget=1.0) + ) + http_request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "query_string": b"", + "route": Route("/v1/chat/completions", endpoint=lambda request: None), + } + ) + + async def run() -> UserAPIKeyAuth: + return await _run_post_custom_auth_checks( + valid_token=valid_token, + request=http_request, + request_data={"model": requested_model}, + route="/v1/chat/completions", + parent_otel_span=None, + ) + + with ( + patch("litellm.proxy.proxy_server.llm_router", free_and_paid_router), + patch("litellm.proxy.proxy_server.model_max_budget_limiter", limiter), + patch("litellm.proxy.proxy_server.general_settings", {"custom_auth_run_common_checks": run_common_checks}), + ): + if expect_enforced: + with pytest.raises(litellm.BudgetExceededError): + await run() + assert limiter.is_end_user_within_model_budget.await_args.kwargs["model"] == requested_model + else: + result: Final = await run() + assert result.token == "test_token" + limiter.is_end_user_within_model_budget.assert_not_awaited() diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index cd5410ffd18..1c576b1ee24 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -2484,3 +2484,58 @@ def test_prefixed_alias_over_a_bare_catch_all_lists_ids_that_route_through_it(mo ] assert f"team-partial-route/{_ROUTE_OPENAI_ID}" in v1_models assert unrouted_ids == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model, token_aliases", + [ + ("free-key", {"aliases": {"free-key": "free-model"}}), + ("free-team", {"team_model_aliases": {"free-team": "free-model"}}), + ("free-team", {"team_model_aliases": {"free-team": "free-model"}, "aliases": {"free-model": "paid-model"}}), + ("free-model", {"aliases": {"paid-model": "free-model"}}), + ("free-team", {"team_model_aliases": {"free-team": ""}}), + ( + "free-team", + {"team_id": "t1", "team_model_aliases": {"free-team": "model_name_t1_deleted"}}, + ), + ], +) +async def test_caller_alias_target_matches_the_model_add_litellm_data_to_request_dispatches( + requested_model: str, token_aliases: Mapping[str, object] +) -> None: + from unittest.mock import MagicMock + + from fastapi import Request + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import caller_alias_target + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + router: Final = Router( + model_list=[ + {"model_name": "free-model", "litellm_params": {"model": "openai/free-backend", "api_key": "sk-fixture"}}, + {"model_name": "paid-model", "litellm_params": {"model": "openai/paid-backend", "api_key": "sk-fixture"}}, + ] + ) + valid_token: Final = UserAPIKeyAuth(api_key="hashed-key", **token_aliases) + request: Final = MagicMock(spec=Request) + request.url = MagicMock() + request.url.path = "/v1/chat/completions" + request.method = "POST" + request.query_params = {} + request.headers = {"Content-Type": "application/json"} + request.client = MagicMock() + request.client.host = "127.0.0.1" + + with patch("litellm.proxy.proxy_server.llm_router", router): + dispatched: Final = await add_litellm_data_to_request( + data={"model": requested_model}, + request=request, + user_api_key_dict=valid_token, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + served: Final = caller_alias_target(requested_model, valid_token, router) + assert served == dispatched["model"] diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 8a5d4f7969b..0d2433d9707 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -21,6 +21,7 @@ from unittest.mock import ANY, AsyncMock, MagicMock, patch import fastapi import orjson import pytest +from starlette.routing import Route from litellm.redis.stores.connection import connect from litellm.redis.stores.spend_counters import SpendCounterStore @@ -13915,6 +13916,89 @@ async def test_exhausted_key_budget_spares_only_zero_spend_mcp_discovery( assert result.spend == 5.0 +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model, key_aliases, team_model_aliases, expect_blocked", + [ + ("free-key", {"free-key": "free-model"}, None, False), + ("free-team", None, {"free-team": "free-model"}, False), + ("free-model", {"free-model": "paid-model"}, None, True), + ("free-model", None, {"free-model": "paid-model"}, True), + ("free-model", None, None, False), + ("paid-model", None, None, True), + ], +) +async def test_exhausted_key_budget_is_waived_only_when_the_served_model_is_free( + requested_model: str, + key_aliases: dict[str, str] | None, + team_model_aliases: dict[str, str] | None, + expect_blocked: bool, + free_and_paid_router: litellm.Router, +) -> None: + from litellm.proxy.utils import hash_token + + api_key: Final = "sk-exhausted-aliased-free-model" + hashed_token: Final = hash_token(api_key) + user_api_key_cache: Final = DualCache() + await cache_key_object( + hashed_token=hashed_token, + user_api_key_obj=UserAPIKeyAuth( + token=hashed_token, + max_budget=2.0, + spend=5.0, + aliases=key_aliases or {}, + team_model_aliases=team_model_aliases, + ), + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=None, + ) + + mock_request: Final = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.method = "POST" + mock_request.headers = {"authorization": f"Bearer {api_key}"} + mock_request.query_params = {} + mock_request.state = SimpleNamespace() + mock_request.scope = {"route": Route("/v1/chat/completions", endpoint=lambda request: None)} + + proxy_logging_obj: Final = MagicMock() + proxy_logging_obj.budget_alerts = AsyncMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + + async def authenticate() -> UserAPIKeyAuth: + return await user_api_key_auth_builder( + request=mock_request, + api_key=f"Bearer {api_key}", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={"model": requested_model}, + ) + + with ( + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.llm_router", free_and_paid_router), + patch("litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache), + patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj), + patch( + "litellm.proxy.auth.user_api_key_auth.virtual_key_max_budget_alert_check", + new_callable=AsyncMock, + ), + ): + if expect_blocked: + with pytest.raises(ProxyException) as exc_info: + await authenticate() + assert "Budget has been exceeded! Key=key Current cost: 5.0, Max budget: 2.0" in exc_info.value.message + else: + result: Final = await authenticate() + assert result.token == hashed_token + assert result.spend == 5.0 + + +@pytest.mark.asyncio @pytest.mark.parametrize( "route, request_data, expected", [ @@ -13924,9 +14008,14 @@ async def test_exhausted_key_budget_spares_only_zero_spend_mcp_discovery( ("/mcp-rest/tools/call", {}, False), ], ) -def test_should_skip_budget_checks_spares_only_zero_spend_mcp_discovery( +async def test_should_skip_budget_checks_spares_only_zero_spend_mcp_discovery( route: str, request_data: dict[str, object], expected: bool ) -> None: from litellm.proxy.auth.user_api_key_auth import _should_skip_budget_checks - assert _should_skip_budget_checks(request_data=request_data, route=route, request=None, llm_router=None) is expected + assert ( + await _should_skip_budget_checks( + request_data=request_data, route=route, request=None, llm_router=None, valid_token=UserAPIKeyAuth() + ) + is expected + ) diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 8d8781be0a8..7ff922179ce 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -546,3 +546,27 @@ def pbkdf2_on_event_loop(monkeypatch: pytest.MonkeyPatch) -> list[bool]: monkeypatch.setattr(hashlib, "pbkdf2_hmac", recording_pbkdf2) return on_event_loop + + +def _per_token_priced(model_name: str, backend: str, input_cost: float, output_cost: float) -> dict[str, object]: + return { + "model_name": model_name, + "litellm_params": { + "model": backend, + "api_key": "sk-fixture", + "input_cost_per_token": input_cost, + "output_cost_per_token": output_cost, + }, + } + + +@pytest.fixture +def free_and_paid_router() -> "litellm.Router": + import litellm + + return litellm.Router( + model_list=[ + _per_token_priced("free-model", "openai/free-backend", 0.0, 0.0), + _per_token_priced("paid-model", "openai/paid-backend", 1e-6, 2e-6), + ] + ) diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index e8ee31b27f5..f4a836b4b4b 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -11607,3 +11607,61 @@ def test_caller_tags_reach_litellm_metadata_whether_metadata_is_a_dict_or_a_json LiteLLMProxyRequestSetup.merge_caller_body_tags(request_data=data) assert set(data["litellm_metadata"]["tags"]) == {"key-tag", "client-tag"} assert data["metadata"] == caller_metadata + + +@pytest.mark.parametrize( + ("model", "global_aliases", "key_aliases", "group_alias", "expected"), + [ + pytest.param("a", {"a": "b"}, {"b": "c"}, {"c": "d"}, "d", id="global-then-key-then-router"), + pytest.param("a", {}, {}, {"a": {"model": "x"}}, "x", id="router-item-entry"), + pytest.param("a", {"a": "b"}, {}, None, "b", id="global-only"), + pytest.param("a", {}, {}, None, "a", id="no-alias"), + ], +) +def test_pre_call_served_model_applies_aliases_in_request_preparation_order( + monkeypatch: pytest.MonkeyPatch, + model: str, + global_aliases: dict[str, str], + key_aliases: dict[str, str], + group_alias: object, + expected: str, +) -> None: + from litellm.proxy.litellm_pre_call_utils import pre_call_served_model + + monkeypatch.setattr(litellm, "model_alias_map", global_aliases) + + assert pre_call_served_model(model, UserAPIKeyAuth(aliases=key_aliases), group_alias) == expected + + +@pytest.mark.parametrize( + ("token_fields", "expected"), + [ + pytest.param({"aliases": {"requested": 7}}, "requested", id="non-string-key-alias"), + pytest.param({"team_model_aliases": {"requested": 7}}, "requested", id="non-string-team-alias"), + pytest.param({"aliases": {"mid": 7}, "team_model_aliases": {"requested": "mid"}}, "mid", id="after-team"), + ], +) +def test_request_served_model_ignores_a_malformed_alias_like_request_preparation_does( + token_fields: dict[str, object], expected: str +) -> None: + from litellm.proxy.litellm_pre_call_utils import _update_model_if_key_alias_exists, _update_model_if_team_alias_exists + from litellm.proxy.litellm_pre_call_utils import request_served_model + + valid_token: Final = UserAPIKeyAuth(**token_fields) + prepared: Final[dict[str, object]] = {"model": "requested"} # mutable-ok: the rewrite functions edit the body + with patch("litellm.proxy.proxy_server.llm_router", None): + _update_model_if_team_alias_exists(data=prepared, user_api_key_dict=valid_token) + _update_model_if_key_alias_exists(data=prepared, user_api_key_dict=valid_token) + + assert request_served_model("requested", valid_token, None, None) == prepared["model"] == expected + + +def test_stale_team_alias_bypass_flag_is_read_once_per_process(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm.proxy.litellm_pre_call_utils as pre_call_utils + + monkeypatch.setattr(pre_call_utils, "_ENABLE_TEAM_STALE_ALIAS_BYPASS", None) + monkeypatch.setenv("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", "true") + first: Final = pre_call_utils._stale_team_alias_bypass_enabled() + monkeypatch.setenv("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", "false") + + assert (first, pre_call_utils._stale_team_alias_bypass_enabled()) == (True, True) diff --git a/tests/unit/proxy/test_zero_cost_model_budget_bypass.py b/tests/unit/proxy/test_zero_cost_model_budget_bypass.py index ce1dfce1646..35e75a85520 100644 --- a/tests/unit/proxy/test_zero_cost_model_budget_bypass.py +++ b/tests/unit/proxy/test_zero_cost_model_budget_bypass.py @@ -6,12 +6,15 @@ to models with zero cost (e.g., on-premises models). """ import asyncio -from typing import Optional +from typing import Final, Optional from unittest.mock import MagicMock, patch import pytest +from starlette.requests import Request as StarletteRequest +from starlette.routing import Route import litellm +from litellm.constants import MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY from litellm.caching.caching import DualCache from litellm.proxy._types import ( LiteLLM_BudgetTable, @@ -653,8 +656,8 @@ class TestOverBudgetRequestThroughModelGroupAlias: router = self._router() request_data = {"model": model} - skip_budget_checks = _should_skip_budget_checks( - request_data=request_data, route=self.ROUTE, request=None, llm_router=router + skip_budget_checks = await _should_skip_budget_checks( + request_data=request_data, route=self.ROUTE, request=None, llm_router=router, valid_token=UserAPIKeyAuth() ) return await common_checks( request_body=request_data, @@ -692,3 +695,164 @@ class TestOverBudgetRequestThroughModelGroupAlias: async def test_over_budget_request_for_paid_model_is_blocked(self, mock_proxy_logging): with pytest.raises(litellm.BudgetExceededError): await self._request("paid-model", mock_proxy_logging) + + +_CHAT: Final = "/v1/chat/completions" + + +def _routed_request( + template: str | None = _CHAT, query: bytes = b"", router_aliases_applied: bool = True +) -> StarletteRequest: + scope: Final[dict[str, object]] = { # mutable-ok: ASGI scope literal + "type": "http", + "method": "POST", + "path": _CHAT, + "headers": [], + "query_string": query, + MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: router_aliases_applied, + } + if template is not None: + scope["route"] = Route(template, endpoint=lambda request: None) + return StarletteRequest(scope) + + +_KEY_FREE: Final = {"aliases": {"free-key": "free-model"}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("body_model", "token_fields", "request_kwargs", "settings", "expected_skip"), + [ + pytest.param("free-key", _KEY_FREE, {}, {}, True, id="key-alias-to-free"), + pytest.param("free-team", {"team_model_aliases": {"free-team": "free-model"}}, {}, {}, True, id="team-to-free"), + pytest.param("free-model", {"aliases": {"free-model": "paid-model"}}, {}, {}, False, id="key-alias-to-paid"), + pytest.param( + "free-model", {"team_model_aliases": {"free-model": "paid-model"}}, {}, {}, False, id="team-alias-to-paid" + ), + pytest.param( + "free-team", + {"team_model_aliases": {"free-team": "free-model"}, "aliases": {"free-model": "paid-model"}}, + {}, + {}, + False, + id="team-then-key", + ), + pytest.param( + "free-team", + { + "team_model_aliases": {"free-team": "free-model"}, + "router_settings": {"model_group_alias": {"free-model": "paid-model"}}, + }, + {}, + {}, + False, + id="router-alias-after-team", + ), + pytest.param(["free-key"], _KEY_FREE, {}, {}, False, id="list-body"), + pytest.param("free-key", _KEY_FREE, {"query": b"model=free-key"}, {}, False, id="query-model"), + pytest.param( + "free-key", _KEY_FREE, {"template": "/engines/{model:path}/chat/completions"}, {}, False, id="path" + ), + pytest.param("free-key", _KEY_FREE, {"template": None}, {}, False, id="request-built-inside-the-proxy"), + pytest.param("free-key", _KEY_FREE, {"router_aliases_applied": False}, {}, False, id="before-router-alias"), + pytest.param("free-key", _KEY_FREE, {}, {"completion_model": "free-model"}, False, id="completion-model-set"), + ], +) +async def test_budget_waiver_judges_the_model_request_preparation_serves( + body_model: str | list[str], + token_fields: dict[str, object], + request_kwargs: dict[str, object], + settings: dict[str, object], + expected_skip: bool, + free_and_paid_router: Router, +) -> None: + from litellm.proxy.auth.user_api_key_auth import _should_skip_budget_checks + + with patch("litellm.proxy.proxy_server.general_settings", settings): + skip: Final = await _should_skip_budget_checks( + request_data={"model": body_model}, + route=_CHAT, + request=_routed_request(**request_kwargs), + llm_router=free_and_paid_router, + valid_token=UserAPIKeyAuth(token="t", **token_fields), + ) + + assert skip is expected_skip + + +@pytest.mark.asyncio +async def test_sub_call_is_judged_by_its_own_model_not_the_callers_aliases(free_and_paid_router: Router) -> None: + from litellm.proxy.auth.user_api_key_auth import _should_skip_budget_checks + + assert ( + await _should_skip_budget_checks( + request_data={"model": "paid-model"}, + route=_CHAT, + request=_routed_request(), + llm_router=free_and_paid_router, + valid_token=UserAPIKeyAuth(token="t", aliases={"paid-model": "free-model"}), + sub_call=True, + ) + is False + ) + + +@pytest.mark.asyncio +async def test_common_checks_honours_the_callers_decision_not_to_skip_budgets( + mock_proxy_logging: ProxyLogging, free_and_paid_router: Router +) -> None: + with pytest.raises(litellm.BudgetExceededError): + await common_checks( + request_body={"model": "free-model"}, + team_object=None, + user_object=LiteLLM_UserTable(user_id="test-user", spend=100.0, max_budget=50.0), + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route=_CHAT, + llm_router=free_and_paid_router, + proxy_logging_obj=mock_proxy_logging, + valid_token=UserAPIKeyAuth(token="t", aliases={"free-model": "paid-model"}), + request=MagicMock(), + skip_budget_checks=False, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("bypass_flag", "expected_skip"), [("false", True), ("true", False)]) +async def test_waiver_skips_a_stale_team_alias_exactly_when_dispatch_does( + monkeypatch: pytest.MonkeyPatch, bypass_flag: str, expected_skip: bool +) -> None: + import litellm.proxy.litellm_pre_call_utils as pre_call_utils + from litellm.proxy.auth.user_api_key_auth import _should_skip_budget_checks + + monkeypatch.setenv("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", bypass_flag) + monkeypatch.setattr(pre_call_utils, "_ENABLE_TEAM_STALE_ALIAS_BYPASS", None) + team_deployment: Final = {"api_key": "sk-fixture", "output_cost_per_token": 0.0} + router: Final = Router( + model_list=[ + { + "model_name": "model_name_team-1_legacy", + "litellm_params": {"model": "openai/free-backend", "input_cost_per_token": 0.0, **team_deployment}, + "model_info": {"team_id": "team-1", "team_public_model_name": "legacy"}, + }, + { + "model_name": "model_name_team-1_current", + "litellm_params": {"model": "openai/paid-backend", "input_cost_per_token": 1e-6, **team_deployment}, + "model_info": {"team_id": "team-1", "team_public_model_name": "team-model"}, + }, + ] + ) + + assert ( + await _should_skip_budget_checks( + request_data={"model": "team-model"}, + route=_CHAT, + request=_routed_request(), + llm_router=router, + valid_token=UserAPIKeyAuth( + token="t", team_id="team-1", team_model_aliases={"team-model": "model_name_team-1_legacy"} + ), + ) + is expected_skip + ) diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 42b21692a8f..ea384d837fd 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -19,7 +19,6 @@ from litellm.litellm_core_utils.routing_common_utils import ( format_fallback_outcome_message, provider_for_generic_call, resolve_model_group_alias, - resolve_served_model, truncate_fallback_error_detail, PROVIDER_SCOPED_CREDENTIAL_PARAMS, warn_on_provider_credential_mismatch, @@ -579,25 +578,6 @@ class TestResolveModelGroupAlias: assert router.get_model_from_alias("group-b") is None -@pytest.mark.parametrize( - "model, model_alias_map, key_aliases, model_group_alias, expected", - [ - pytest.param("a", {"a": "b"}, {"b": "c"}, {"c": "d"}, "d", id="global-key-router-order"), - pytest.param("a", {}, ["a", "b"], {}, "a", id="non-mapping-key-aliases-ignored"), - pytest.param("a", {}, {}, {"a": {"model": "x"}}, "x", id="router-item-entry"), - pytest.param("a", {}, {}, {}, "a", id="no-alias"), - ], -) -def test_resolves_served_model_aliases_in_order( - model: str, - model_alias_map: object, - key_aliases: object, - model_group_alias: object, - expected: str, -) -> None: - assert resolve_served_model(model, model_alias_map, key_aliases, model_group_alias) == expected - - class TestTruncateFallbackErrorDetail: def test_short_detail_is_returned_unchanged(self): assert truncate_fallback_error_detail("boom") == "boom" From 9a4a587e75e589294f073ba94ad53243018332ea Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:32:16 -0800 Subject: [PATCH 064/189] feat(docker)!: one shipped image and one entrypoint beside the Dockerfile (#43332) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/config.yml | 2 +- .dockerignore | 1 - .github/ci-coverage-allowlist.yml | 9 - .github/workflows/compat-matrix-image.yml | 33 +- .github/workflows/image-scan.yml | 338 +++++------ .github/workflows/test-merge-smoke.yml | 64 +- CONTRIBUTING.md | 4 +- Dockerfile | 61 +- backend/Dockerfile | 146 ----- ci_cd/security_scans_readme.md | 1 - .../litellm-ollama-docker-image/Dockerfile | 25 - .../requirements.txt | 1 - cookbook/litellm-ollama-docker-image/start.sh | 2 - cookbook/litellm-ollama-docker-image/test.py | 35 -- docker-compose.hardened.yml | 44 -- docker-compose.yml | 84 ++- docker-entrypoint.sh | 109 ++++ docker/Dockerfile.database | 172 ------ docker/Dockerfile.non_root | 208 ------- docker/README.md | 95 +-- .../build_from_pip/Dockerfile.build_from_pip | 64 -- docker/build_from_pip/Readme.md | 9 - docker/component_entrypoint.sh | 16 - docker/entrypoint.sh | 16 - docker/install_auto_router.sh | 4 - docker/prod_entrypoint.sh | 16 - docker/tests/nonroot.yaml | 18 - gateway/Dockerfile | 170 ------ litellm/proxy/start.sh | 2 - migrations/Dockerfile | 152 ----- .../build_from_pip_config.yaml | 4 +- tests/e2e/claude_code/cron_vm/Dockerfile | 34 -- tests/e2e/claude_code/cron_vm/README.md | 67 ++- tests/e2e/claude_code/cron_vm/run_daily.sh | 4 +- tests/integration/_support/process.py | 4 +- .../test_collector_image_serves_socket.py | 174 ++++++ .../test_component_image_serves_offline.py | 9 +- .../image/test_offline_image_migration.py | 11 +- .../image/test_ui_image_serves_offline.py | 6 +- .../test_prometheus_series_cap_chaos.py | 2 +- tests/integration/run.py | 1 + tests/unit/test_component_entrypoint.py | 491 --------------- tests/unit/test_docker_entrypoint.py | 557 ++++++++++++++++++ .../test_dockerfile_bedrock_realtime_extra.py | 9 +- tests/unit/test_dockerfile_non_root.py | 40 +- ui/Dockerfile | 48 -- ui/nginx.conf | 11 +- 47 files changed, 1283 insertions(+), 2090 deletions(-) delete mode 100644 backend/Dockerfile delete mode 100644 cookbook/litellm-ollama-docker-image/Dockerfile delete mode 100644 cookbook/litellm-ollama-docker-image/requirements.txt delete mode 100644 cookbook/litellm-ollama-docker-image/start.sh delete mode 100644 cookbook/litellm-ollama-docker-image/test.py delete mode 100644 docker-compose.hardened.yml create mode 100755 docker-entrypoint.sh delete mode 100644 docker/Dockerfile.database delete mode 100644 docker/Dockerfile.non_root delete mode 100644 docker/build_from_pip/Dockerfile.build_from_pip delete mode 100644 docker/build_from_pip/Readme.md delete mode 100755 docker/component_entrypoint.sh delete mode 100755 docker/entrypoint.sh delete mode 100755 docker/install_auto_router.sh delete mode 100644 docker/prod_entrypoint.sh delete mode 100644 docker/tests/nonroot.yaml delete mode 100644 gateway/Dockerfile delete mode 100755 litellm/proxy/start.sh delete mode 100644 migrations/Dockerfile rename docker/build_from_pip/litellm_config.yaml => tests/basic_proxy_startup_tests/build_from_pip_config.yaml (77%) delete mode 100644 tests/e2e/claude_code/cron_vm/Dockerfile create mode 100644 tests/integration/database/image/test_collector_image_serves_socket.py delete mode 100644 tests/unit/test_component_entrypoint.py create mode 100644 tests/unit/test_docker_entrypoint.py delete mode 100644 ui/Dockerfile diff --git a/.circleci/config.yml b/.circleci/config.yml index 95ee0a6412c..cf1a7b94fe5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1099,7 +1099,7 @@ jobs: docker build \ --label org.opencontainers.image.revision="$(git rev-parse HEAD)" \ -t litellm-docker-database:ci \ - -f docker/Dockerfile.database . + -f Dockerfile . fi python3 .circleci/scripts/run_migration_tests.py record-image diff --git a/.dockerignore b/.dockerignore index f3a80fee3e4..8459d0cd482 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,7 +7,6 @@ tests .devcontainer *.tgz log.txt -docker/Dockerfile.* # Claude Flow generated files (must be excluded from Docker build) .claude/ diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index 5dffa78776d..76f8e3ef7e7 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -57,15 +57,6 @@ test_paths: - tests/integration/test_oci_proxy_integration.py dockerfiles: - - reason: >- - The dashboard container is a static Next.js export served by nginx, and the dashboard build - and lint workflows already exercise that output, so building the image adds no signal about it - paths: - - ui/Dockerfile - - reason: >- - An example image under cookbook/ that is documentation rather than a shipped artifact - paths: - - cookbook/litellm-ollama-docker-image/Dockerfile - reason: >- The Rust gateway image compiles the whole workspace in release mode, which is too slow for a per-pull-request job while the gateway binary is still being assembled; the Rust lint, diff --git a/.github/workflows/compat-matrix-image.yml b/.github/workflows/compat-matrix-image.yml index ddd2aa1a50b..73a77c1e861 100644 --- a/.github/workflows/compat-matrix-image.yml +++ b/.github/workflows/compat-matrix-image.yml @@ -26,17 +26,24 @@ jobs: with: persist-credentials: false - - name: Build the Render cron image - run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e - - - name: Resolve and install the Claude Code CLI as the cron user + - name: Install the pinned uv the cron image ships + env: + UV_VERSION: 0.10.9 + UV_SHA256: 20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594 run: | - docker run --rm compat-matrix:${{ github.sha }} bash -c ' - set -euo pipefail - whoami - gh --version - uv --version - version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)" - /opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli - /tmp/claude-cli/claude --version - ' + set -euo pipefail + curl -fsSLo "$RUNNER_TEMP/uv.tar.gz" "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" + echo "${UV_SHA256} $RUNNER_TEMP/uv.tar.gz" | sha256sum -c - + mkdir -p "$RUNNER_TEMP/bin" + tar -xzf "$RUNNER_TEMP/uv.tar.gz" -C "$RUNNER_TEMP/bin" --strip-components=1 uv-x86_64-unknown-linux-gnu/uv + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + + - name: Resolve and install the Claude Code CLI as an unprivileged user + run: | + set -euo pipefail + whoami + gh --version + uv --version + version="$(uv run --no-project --python 3.12 python tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)" + tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" "$RUNNER_TEMP/claude-cli" + "$RUNNER_TEMP/claude-cli/claude" --version diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml index d769b6ab8af..6e13b49eb15 100644 --- a/.github/workflows/image-scan.yml +++ b/.github/workflows/image-scan.yml @@ -8,31 +8,25 @@ on: - "litellm_**" paths: - Dockerfile - - docker/Dockerfile.non_root - - docker/Dockerfile.database - - migrations/Dockerfile + - .dockerignore + - docker-entrypoint.sh - migrations/run.py - - gateway/Dockerfile - - gateway/launch.py - gateway/main.py + - gateway/launch.py - gateway/routes/allowlist.py - litellm_cli/** - - backend/Dockerfile - backend/main.py + - backend/routes/allowlist.py - litellm-rust/** - litellm/proxy/lens/** - - docker/component_entrypoint.sh - - docker/entrypoint.sh - litellm/proxy/prisma_migration.py - litellm/proxy/admin_mcp.py - litellm/proxy/proxy_server.py - - backend/routes/allowlist.py - pyproject.toml - litellm-proxy-extras/** - tests/integration/database/image/** - uv.lock - ui/litellm-dashboard/package-lock.json - - ui/Dockerfile - ui/nginx.conf - .github/workflows/image-scan.yml - .grype.yaml @@ -46,9 +40,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + IMAGE: litellm-image-scan:${{ github.sha }} + ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }} + jobs: - image-scan: - name: image-scan + build: + name: build runs-on: ubuntu-latest if: >- github.event_name != 'pull_request' || @@ -61,7 +59,129 @@ jobs: with: persist-credentials: false + - name: Build the image + run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -t "$IMAGE" . + + - name: Save the image for the verify matrix + run: docker save "$IMAGE" | zstd -T0 -3 -o "$RUNNER_TEMP/litellm-image.tar.zst" + + - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: litellm-image + path: ${{ runner.temp }}/litellm-image.tar.zst + retention-days: 1 + compression-level: 0 + + - name: Build the Admin MCP schema image + run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci --target builder -t "$ADMIN_MCP_SCHEMA_IMAGE" . + + - name: Save the Admin MCP schema image for the verify matrix + run: docker save "$ADMIN_MCP_SCHEMA_IMAGE" | zstd -T0 -3 -o "$RUNNER_TEMP/litellm-admin-mcp-schema.tar.zst" + + - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: litellm-admin-mcp-schema + path: ${{ runner.temp }}/litellm-admin-mcp-schema.tar.zst + retention-days: 1 + compression-level: 0 + + verify: + name: verify (${{ matrix.check }}) + needs: build + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + check: [scan, migrations, gateway, backend, ui, collector] + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: litellm-image + path: ${{ runner.temp }} + + - name: Load the image + run: zstd -d --stdout "$RUNNER_TEMP/litellm-image.tar.zst" | docker load + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + if: matrix.check == 'migrations' || matrix.check == 'backend' + with: + name: litellm-admin-mcp-schema + path: ${{ runner.temp }} + + - name: Load the Admin MCP schema image + if: matrix.check == 'migrations' || matrix.check == 'backend' + run: zstd -d --stdout "$RUNNER_TEMP/litellm-admin-mcp-schema.tar.zst" | docker load + + - name: Set up Python + if: matrix.check != 'scan' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install pytest + if: matrix.check != 'scan' + run: python -m pip install "pytest==9.0.3" + + # The prisma bake must migrate a fresh DB with no egress as an arbitrary + # non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyFilesystem). + # `docker run` as the default uid with network hides a broken bake because + # the migration entrypoint exits 0 even when it applied nothing; asserting + # the schema was created is what catches it. + - name: Verify offline migration as a non-root uid + if: matrix.check == 'migrations' + env: + LITELLM_IMAGE: ${{ env.IMAGE }} + LITELLM_ADMIN_MCP_SCHEMA_IMAGE: ${{ env.ADMIN_MCP_SCHEMA_IMAGE }} + run: | + python -m pytest --noconftest -v \ + tests/integration/database/image/test_offline_image_migration.py \ + tests/integration/database/image/test_image_bedrock_realtime_extra.py \ + tests/integration/database/image/test_image_admin_mcp.py + + - name: Verify the gateway component serves offline as a non-root uid + if: matrix.check == 'gateway' + env: + LITELLM_IMAGE: ${{ env.IMAGE }} + LITELLM_COMPONENT: gateway + LITELLM_IMAGE_COMPONENT: gateway + run: | + python -m pytest --noconftest -v \ + tests/integration/database/image/test_component_image_serves_offline.py \ + tests/integration/database/image/test_image_admin_mcp.py + + - name: Verify the backend component serves offline as a non-root uid + if: matrix.check == 'backend' + env: + LITELLM_IMAGE: ${{ env.IMAGE }} + LITELLM_COMPONENT: backend + LITELLM_IMAGE_COMPONENT: backend + LITELLM_ADMIN_MCP_SCHEMA_IMAGE: ${{ env.ADMIN_MCP_SCHEMA_IMAGE }} + run: | + python -m pytest --noconftest -v \ + tests/integration/database/image/test_component_image_serves_offline.py \ + tests/integration/database/image/test_image_admin_mcp.py + + - name: Verify the ui component serves as an arbitrary uid with a read-only root fs + if: matrix.check == 'ui' + env: + LITELLM_IMAGE: ${{ env.IMAGE }} + run: python -m pytest --noconftest -v tests/integration/database/image/test_ui_image_serves_offline.py + + - name: Verify the collector creates its socket as the image user + if: matrix.check == 'collector' + env: + LITELLM_IMAGE: ${{ env.IMAGE }} + run: python -m pytest --noconftest -v tests/integration/database/image/test_collector_image_serves_socket.py + - name: Download Grype v0.114.0 + if: matrix.check == 'scan' run: | curl -fsSL --retry 3 -o "$RUNNER_TEMP/grype.tar.gz" \ https://github.com/anchore/grype/releases/download/v0.114.0/grype_0.114.0_linux_amd64.tar.gz @@ -69,33 +189,6 @@ jobs: tar xzf "$RUNNER_TEMP/grype.tar.gz" -C "$RUNNER_TEMP" grype chmod +x "$RUNNER_TEMP/grype" - # Dockerfile.non_root is the rootless variant we ship. The other - # Dockerfiles share the same wolfi base and apk set, so OS-layer coverage - # is the same; matrix-scan if those variants ever diverge. - - name: Build runtime image - run: docker build -f docker/Dockerfile.non_root -t litellm-image-scan:${{ github.sha }} . - - - name: Tag the cached builder for Admin MCP schema setup - run: docker build --target builder -f docker/Dockerfile.non_root -t litellm-admin-mcp-schema:${{ github.sha }} . - - # The prisma bake must migrate a fresh DB with no egress as an arbitrary - # non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyRootFilesystem). - # `docker run` as the default uid with network hides a broken bake because - # the migration entrypoint exits 0 even when it applied nothing; asserting - # the schema was created is what catches it. - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify offline migration as a non-root uid - env: - LITELLM_IMAGE: litellm-image-scan:${{ github.sha }} - LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }} - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v - # Scans the whole shipped artifact: OS/apk plus every language package # baked into the image, including ones no lockfile declares (e.g. prisma's # vendored node engine) that osv-scan cannot see. osv-scan stays the fast @@ -103,175 +196,12 @@ jobs: # free OSS, run as a pinned, checksum-verified binary; no GitHub Action # dependency and no vendor SaaS callout. - name: Scan image for fixable HIGH/CRITICAL CVEs + if: matrix.check == 'scan' env: GRYPE_MATCH_PYTHON_USING_CPES: "true" run: | - "$RUNNER_TEMP/grype" litellm-image-scan:${{ github.sha }} \ + "$RUNNER_TEMP/grype" "$IMAGE" \ --config .grype.yaml \ --only-fixed \ --fail-on high \ --output table - - runtime-image: - name: runtime-image (${{ matrix.dockerfile }}) - runs-on: ubuntu-latest - if: >- - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 45 - permissions: - contents: read - strategy: - fail-fast: false - matrix: - dockerfile: [Dockerfile, docker/Dockerfile.database] - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Build runtime image - run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f "${{ matrix.dockerfile }}" -t litellm-runtime-scan:${{ github.sha }} . - - - name: Tag the cached builder for Admin MCP schema setup - run: docker build --target builder -f "${{ matrix.dockerfile }}" -t litellm-admin-mcp-schema:${{ github.sha }} . - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify offline migration as a non-root uid - env: - LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }} - LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }} - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v - - - migrations-image: - name: migrations-image - runs-on: ubuntu-latest - if: >- - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 30 - permissions: - contents: read - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Build migrations image - run: docker build -f migrations/Dockerfile -t litellm-migrations-scan:${{ github.sha }} . - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify offline migration as a non-root uid - env: - LITELLM_IMAGE: litellm-migrations-scan:${{ github.sha }} - LITELLM_MIGRATION_INTERPRETER: python3 - LITELLM_MIGRATION_SCRIPT: /app/run.py - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py -v - - gateway-image: - name: gateway-image - runs-on: ubuntu-latest - if: >- - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 30 - permissions: - contents: read - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Build gateway image - run: docker build -f gateway/Dockerfile -t litellm-gateway-scan:${{ github.sha }} . - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify the gateway serves offline as a non-root uid - env: - LITELLM_IMAGE: litellm-gateway-scan:${{ github.sha }} - LITELLM_COMPONENT_PORT: "4000" - LITELLM_IMAGE_COMPONENT: gateway - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v - - ui-image: - name: ui-image - runs-on: ubuntu-latest - if: >- - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 30 - permissions: - contents: read - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Build UI image - run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} . - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify the UI serves offline as an arbitrary uid with a read-only root fs - env: - LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }} - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_ui_image_serves_offline.py -v - - backend-image: - name: backend-image - runs-on: ubuntu-latest - if: >- - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 30 - permissions: - contents: read - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Build backend image - run: docker build -f backend/Dockerfile -t litellm-backend-scan:${{ github.sha }} . - - - name: Tag the cached builder for Admin MCP schema setup - run: docker build --target builder -f backend/Dockerfile -t litellm-admin-mcp-schema:${{ github.sha }} . - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify the backend serves offline as a non-root uid - env: - LITELLM_IMAGE: litellm-backend-scan:${{ github.sha }} - LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }} - LITELLM_COMPONENT_PORT: "4001" - LITELLM_IMAGE_COMPONENT: backend - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_admin_mcp.py -v diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index 99dbd1c101b..912fc90fbe5 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -200,8 +200,36 @@ jobs: - name: Install the test client dependencies run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --extra proxy --group dev --python 3.12 - - name: Build the image from the published package - run: docker build -t litellm-build-from-pip:smoke -f docker/build_from_pip/Dockerfile.build_from_pip . + - name: Cache Prisma binaries + timeout-minutes: 3 + uses: ./.github/actions/cache-prisma-binaries + + - name: Install the published proxy into its own venv + run: | + uv --no-config venv --python 3.13 /tmp/litellm-pip + uv --no-config pip install --python /tmp/litellm-pip/bin/python \ + "litellm[proxy,proxy-runtime]==1.83.0" \ + "google-cloud-aiplatform==1.133.0" \ + "google-genai==1.37.0" \ + "anthropic[vertex]==0.84.0" \ + "grpcio==1.78.0" \ + "prometheus-client==0.20.0" \ + "langfuse==2.59.7" \ + "opentelemetry-api==1.28.0" \ + "opentelemetry-sdk==1.28.0" \ + "opentelemetry-exporter-otlp==1.28.0" \ + "ddtrace==4.11.0" \ + "sentry-sdk==2.21.0" \ + "mangum==0.17.0" \ + "azure-ai-contentsafety==1.0.0" \ + "azure-storage-file-datalake==12.20.0" \ + "pypdf==6.7.5" \ + "llm-sandbox==0.3.31" \ + "detect-secrets==1.5.0" \ + "prisma==0.11.0" \ + "openai==2.24.0" + export PATH="/tmp/litellm-pip/bin:$PATH" + /tmp/litellm-pip/bin/prisma generate --schema=schema.prisma - name: Start the fake OpenAI endpoint run: | @@ -219,23 +247,21 @@ jobs: echo "::add-mask::${key}" echo "LITELLM_MASTER_KEY=${key}" >> "${GITHUB_ENV}" - - name: Start the proxy container + - name: Start the published proxy + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/circle_test + LITELLM_LICENSE: bad-license + FAKE_OPENAI_API_BASE: http://localhost:8190 + OTEL_EXPORTER: in_memory + USE_DDTRACE: "True" + DD_TRACE_OPENAI_ENABLED: "False" + GCS_FLUSH_INTERVAL: "1" + LITELLM_LOG: ERROR run: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e LITELLM_MASTER_KEY \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e OTEL_EXPORTER=in_memory \ - -e USE_DDTRACE=True \ - -e GCS_FLUSH_INTERVAL=1 \ - -e LITELLM_LOG=ERROR \ - --add-host host.docker.internal:host-gateway \ - --name litellm-build-from-pip \ - -v "${GITHUB_WORKSPACE}/docker/build_from_pip/litellm_config.yaml:/app/config.yaml" \ - litellm-build-from-pip:smoke \ - --config /app/config.yaml \ - --port 4000 + cd /tmp/litellm-pip + nohup /tmp/litellm-pip/bin/ddtrace-run /tmp/litellm-pip/bin/litellm \ + --config "${GITHUB_WORKSPACE}/tests/basic_proxy_startup_tests/build_from_pip_config.yaml" \ + --port 4000 > /tmp/litellm-pip.log 2>&1 & for _ in $(seq 1 150); do curl -sf --max-time 5 -o /dev/null http://localhost:4000 && exit 0 sleep 2 @@ -270,7 +296,7 @@ jobs: - name: Show the proxy logs if: always() - run: docker logs litellm-build-from-pip || true + run: cat /tmp/litellm-pip.log || true smoke-passed: name: smoke passed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e956b27d5da..426cf986d4c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -305,8 +305,8 @@ uv run litellm --config your_config.yaml If you want to build the Docker image yourself: ```bash -# Build using the non-root Dockerfile -docker build -f docker/Dockerfile.non_root -t litellm_dev . +# Build the image (runs as a non-root user by default) +docker build -t litellm_dev . # Generate a master key. Requests send it as the bearer token export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)" diff --git a/Dockerfile b/Dockerfile index 94557bcf68c..205f0cc981d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,8 +119,12 @@ RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh +RUN sed -i 's/\r$//' docker-entrypoint.sh && chmod +x docker-entrypoint.sh + +RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \ + cp -r litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \ + cp litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \ + touch /var/lib/litellm/ui/.litellm_ui_ready FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder COPY --from=uvbin /uv /usr/local/bin/uv @@ -145,24 +149,35 @@ USER root # https://github.com/BerriAI/litellm/issues/33518 RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories -# node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent +RUN for i in 1 2 3; do \ + apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libatomic libevent nginx && break; \ + [ "$i" = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer WORKDIR /app + ENV PATH="/app/.venv/bin:${PATH}" \ + PYTHONPATH="/app" \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + HOME=/app \ + LITELLM_NON_ROOT=true \ PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ + PRISMA_SKIP_POSTINSTALL_GENERATE=1 \ + PRISMA_HIDE_UPDATE_MESSAGE=1 \ + PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \ PRISMA_OFFLINE_MODE=true # Copy only what runtime needs. The application is installed inside the venv; # the rest of the builder's /app is source and build metadata that must not # ship (manifest-scanning tools attribute everything in it to this image). -# entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path. COPY --from=builder /app/.venv /app/.venv COPY --from=liteadmin-builder /opt/liteadmin /opt/liteadmin -COPY --from=builder /app/docker /app/docker +COPY --from=builder /app/docker-entrypoint.sh /app/docker-entrypoint.sh COPY --from=builder /app/schema.prisma /app/schema.prisma COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py # enterprise/ is imported by source path at runtime (proxy_cli puts the @@ -170,21 +185,33 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr # enterprise.enterprise_hooks from it) COPY --from=builder /app/enterprise /app/enterprise COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -# Prisma CLI + engines are baked under /opt/prisma, a fixed path every -# runtime uid can read and that no cache volume mount shadows. The paths are -# pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and recorded into the -# generated client at build time, so `prisma migrate deploy` on a fresh -# database needs no npm and no network access (#33650, #24554). +COPY --from=builder /app/gateway /app/gateway +COPY --from=builder /app/backend /app/backend +COPY --from=builder /app/migrations /app/migrations COPY --from=builder /opt/prisma /opt/prisma +COPY --from=builder /var/lib/litellm /var/lib/litellm +COPY ui/nginx.conf /etc/nginx/nginx.conf +COPY ui/docker-entrypoint.d/40-litellm-runtime-config.sh /app/ui/docker-entrypoint.d/40-litellm-runtime-config.sh -RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete && \ +RUN find /app/.venv -depth -type d -path "*/tornado/test" -exec rm -rf {} + && \ + mkdir -p /app/.cache /var/run/litellm && \ + chown -R 65532:0 /app /var/lib/litellm /var/run/litellm && \ + chmod -R g=u,g+w /app/.cache /var/lib/litellm /var/run/litellm && \ + PRISMA_PATH="$(python -c 'import os, prisma; print(os.path.dirname(prisma.__file__))')" && \ + PROXY_EXTRAS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))')" && \ + chmod -R g=u,g+w "$PRISMA_PATH" "$PROXY_EXTRAS_PATH" && \ chmod -R a+rX /opt/prisma && \ test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" + ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null && \ + python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \ + python -c "from litellm.rust_bridge import _native" && \ + python -c "import gateway.launch, backend.main" -EXPOSE 4000/tcp +USER 65532:65532 -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] +RUN nginx -t + +EXPOSE 4000/tcp 4001/tcp 3000/tcp + +ENTRYPOINT ["/app/docker-entrypoint.sh"] diff --git a/backend/Dockerfile b/backend/Dockerfile deleted file mode 100644 index aad59ba17c6..00000000000 --- a/backend/Dockerfile +++ /dev/null @@ -1,146 +0,0 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -# ---------- Rust extension ---------- -# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes -FROM $LITELLM_BUILD_IMAGE AS rust-native - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /uvx /usr/local/bin/ - -RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -ENV UV_PYTHON_DOWNLOADS=0 - -COPY pyproject.toml README.md rust-toolchain.toml ./ -COPY .cargo ./.cargo -COPY litellm-rust ./litellm-rust -COPY model_prices_and_context_window.json ./ - -RUN --mount=type=cache,target=/root/.cache/uv \ - mkdir -p litellm/rust_bridge /wheels && \ - touch litellm/__init__.py litellm/rust_bridge/__init__.py && \ - uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \ - python -c "import maturin; maturin.build_wheel('/wheels')" - -# ---------- Builder ---------- -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /uvx /usr/local/bin/ - -# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE -# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi -# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. -RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start. -# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a -# BuildKit cache mount (different filesystem). -# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of -# silently pulling a managed interpreter. -# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't -# silently re-enable nodeenv's Node download. -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - UV_COMPILE_BYTECODE=1 \ - UV_PYTHON_DOWNLOADS=0 \ - PRISMA_USE_GLOBAL_NODE=true \ - PATH="/app/.venv/bin:${PATH}" - -# Stage 1 — install dependencies only. -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - --mount=type=bind,source=uv.lock,target=uv.lock \ - --mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \ - --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \ - uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --python python3.13 - -# Stage 2 — copy source and install the project + workspace members. -COPY . . - -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --python python3.13 - -RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \ - uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl - -RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ - -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh - -# ---------- Runtime ---------- -FROM $LITELLM_RUNTIME_IMAGE AS runtime -ARG LITELLM_RELEASE_TAG="" -ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} - -USER root - -RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with -# /home/nonroot. We run the backend as that user -WORKDIR /app -ENV HOME=/home/nonroot \ - PATH="/app/.venv/bin:${PATH}" \ - PYTHONPATH="/app" \ - PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries - -COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv -COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm -COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise -COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend -COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh -COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma -COPY --from=builder /opt/prisma /opt/prisma - -RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete && \ - chmod -R a+rX /opt/prisma && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" - -USER nonroot - -EXPOSE 4001/tcp - -ENTRYPOINT ["/app/docker/component_entrypoint.sh", "uvicorn", "backend.main:app"] -CMD ["--host", "0.0.0.0", "--port", "4001"] diff --git a/ci_cd/security_scans_readme.md b/ci_cd/security_scans_readme.md index dd64b01c296..3003ea1d105 100644 --- a/ci_cd/security_scans_readme.md +++ b/ci_cd/security_scans_readme.md @@ -4,6 +4,5 @@ - Trivy scan on `./docs/` (HIGH/CRITICAL/MEDIUM) - Trivy scan on `./ui/` (HIGH/CRITICAL/MEDIUM) -- Grype scan on `Dockerfile.database` (fails on CRITICAL) - Grype scan on main `Dockerfile` (fails on CRITICAL) - Grype CVSS ≥ 4.0 scan on main `Dockerfile` (fails any vulnerabilities with CVSS ≥ 4.0) diff --git a/cookbook/litellm-ollama-docker-image/Dockerfile b/cookbook/litellm-ollama-docker-image/Dockerfile deleted file mode 100644 index be237a4df77..00000000000 --- a/cookbook/litellm-ollama-docker-image/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM ollama/ollama as ollama - -RUN echo "auto installing llama2" - -# auto install ollama/llama2 -RUN ollama serve & sleep 2 && ollama pull llama2 - -RUN echo "installing litellm" - -RUN apt-get update - -# Install Python -RUN apt-get install -y python3 python3-pip - -# Set the working directory in the container -WORKDIR /app - -# Copy the current directory contents into the container at /app -COPY . /app - -# Install any needed packages specified in requirements.txt - -RUN python3 -m pip install litellm -COPY start.sh /start.sh -ENTRYPOINT [ "/bin/bash", "/start.sh" ] diff --git a/cookbook/litellm-ollama-docker-image/requirements.txt b/cookbook/litellm-ollama-docker-image/requirements.txt deleted file mode 100644 index 9b9181b2360..00000000000 --- a/cookbook/litellm-ollama-docker-image/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -litellm==1.83.14 diff --git a/cookbook/litellm-ollama-docker-image/start.sh b/cookbook/litellm-ollama-docker-image/start.sh deleted file mode 100644 index ecc03ce73c5..00000000000 --- a/cookbook/litellm-ollama-docker-image/start.sh +++ /dev/null @@ -1,2 +0,0 @@ -ollama serve & -litellm \ No newline at end of file diff --git a/cookbook/litellm-ollama-docker-image/test.py b/cookbook/litellm-ollama-docker-image/test.py deleted file mode 100644 index 93b9c6ac4a7..00000000000 --- a/cookbook/litellm-ollama-docker-image/test.py +++ /dev/null @@ -1,35 +0,0 @@ -import openai - -api_base = "http://0.0.0.0:8000" - -openai.api_base = api_base -openai.api_key = "temp-key" -print(openai.api_base) - - -print("LiteLLM: response from proxy with streaming") -response = openai.ChatCompletion.create( - model="ollama/llama2", - messages=[ - { - "role": "user", - "content": "this is a test request, acknowledge that you got it", - } - ], - stream=True, -) - -for chunk in response: - print(f"LiteLLM: streaming response from proxy {chunk}") - -response = openai.ChatCompletion.create( - model="ollama/llama2", - messages=[ - { - "role": "user", - "content": "this is a test request, acknowledge that you got it", - } - ], -) - -print(f"LiteLLM: response from proxy {response}") diff --git a/docker-compose.hardened.yml b/docker-compose.hardened.yml deleted file mode 100644 index 84a23faa054..00000000000 --- a/docker-compose.hardened.yml +++ /dev/null @@ -1,44 +0,0 @@ -services: - # Hardened stack: for testing the proxy under non-root, read-only, proxy-enforced constraints. - # Keep this file focused on hardening/QA scenarios; leave the main docker-compose.yml for default dev usage. - litellm: - build: - context: . - dockerfile: docker/Dockerfile.non_root - target: runtime - depends_on: - - squid - user: "101:101" - group_add: - - "2345" - read_only: true - cap_drop: - - ALL - security_opt: - - no-new-privileges:true - tmpfs: - - /app/cache:rw,noexec,nosuid,nodev,size=128m,uid=101,gid=101,mode=1777 - - /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777 - volumes: - - ./proxy_server_config.yaml:/app/config.yaml:ro - environment: - LITELLM_NON_ROOT: "true" - PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries" - XDG_CACHE_HOME: "/app/cache" - LITELLM_MIGRATION_DIR: "/app/migrations" - HTTP_PROXY: "http://squid:3128" - HTTPS_PROXY: "http://squid:3128" - NO_PROXY: "localhost,127.0.0.1,db" - command: - - "--port" - - "4000" - - "--config" - - "/app/config.yaml" - squid: - image: sameersbn/squid:3.5.27-2 - restart: unless-stopped - ports: - - "3128:3128" - tmpfs: - - /var/spool/squid:rw,noexec,nosuid,nodev,size=64m - - /var/log/squid:rw,noexec,nosuid,nodev,size=16m diff --git a/docker-compose.yml b/docker-compose.yml index 80e1f289aad..b012c3763de 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,52 +1,71 @@ +# LiteLLM plus a Postgres database that stores models, virtual keys and spend +# logs. Used by https://docs.litellm.ai/docs/proxy/docker_quick_start +# +# curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker-compose.yml +# printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env +# docker compose up -d +# +# `docker compose up` runs the published image. From a checkout of the repo, +# `docker compose up --build` builds the Dockerfile next to this file instead. +# `docker compose --profile monitoring up -d` also starts Prometheus on 9090. +# +# Compose reads .env from this directory. Keep it: regenerating LITELLM_SALT_KEY +# makes credentials already stored in the database unreadable. A database +# created without LITELLM_SALT_KEY encrypted them with the master key, so when +# upgrading one, write only LITELLM_MASTER_KEY (the existing one) to .env. For anything +# beyond local evaluation, pin the image to a specific release tag. services: litellm: + image: docker.litellm.ai/berriai/litellm:main-stable build: context: . - args: - target: runtime - image: docker.litellm.ai/berriai/litellm:main-stable - ######################################### - ## Uncomment these lines to start proxy with a config.yaml file ## - # volumes: - # - ./config.yaml:/app/config.yaml - # command: - # - "--config=/app/config.yaml" - ############################################## + target: runtime + pull_policy: missing ports: - - "4000:4000" # Map the container port to the host, change the host port if necessary + - "4000:4000" + # The same image runs every component. The first word of `command` picks + # it: proxy (default), gateway, backend, ui, migrations, metrics, + # collector. See docker/README.md + # command: ["--config", "/app/config.yaml"] + # volumes: + # - ./config.yaml:/app/config.yaml:ro environment: - DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" - # Optional: route read-only queries (find_*, count, group_by, query_raw/_first) - # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for - # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL - # is auto-refreshed alongside the writer. - # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm" - STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env, see the header of this file} + LITELLM_MIGRATION_DIR: /tmp/litellm-migrations + DATABASE_URL: postgresql://llmproxy:dbpassword9090@db:5432/litellm + STORE_MODEL_IN_DB: "True" env_file: - - .env # Load local .env file + - path: .env + required: false + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 + - /app/.cache:rw,noexec,nosuid,nodev,size=128m,mode=1777 depends_on: - - db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first - healthcheck: # Defines the health check configuration for the container + db: + condition: service_healthy + healthcheck: test: - CMD-SHELL - - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" # Command to execute for health check - interval: 30s # Perform health check every 30 seconds - timeout: 10s # Health check command times out after 10 seconds - retries: 3 # Retry up to 3 times if health check fails - start_period: 40s # Wait 40 seconds after container start before beginning health checks + - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s db: image: postgres:16 restart: always - container_name: litellm_db environment: POSTGRES_DB: litellm POSTGRES_USER: llmproxy POSTGRES_PASSWORD: dbpassword9090 - ports: - - "5432:5432" volumes: - - postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts + - postgres_data:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"] interval: 1s @@ -55,6 +74,8 @@ services: prometheus: image: prom/prometheus + profiles: + - monitoring volumes: - prometheus_data:/prometheus - ./prometheus.yml:/etc/prometheus/prometheus.yml @@ -68,6 +89,5 @@ services: volumes: prometheus_data: - driver: local postgres_data: - name: litellm_postgres_data # Named volume for Postgres data persistence + name: litellm_postgres_data diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000000..7bcb62253a7 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,109 @@ +#!/bin/sh +# LiteLLM image entrypoint: docker-entrypoint.sh [COMPONENT] [ARGS...] +# +# COMPONENT selects the process this container runs; it can also be given as +# LITELLM_COMPONENT when the command carries no component. Anything after it is +# passed to that process. A first argument starting with "-" (or no argument at +# all) keeps the historical behaviour of running the monolithic proxy. +# +# proxy litellm ARGS everything in one process (default) +# gateway python -m gateway.launch ARGS inference routes, 0.0.0.0:4000 +# backend uvicorn backend.main:app ARGS management routes, 0.0.0.0:4001 +# ui nginx serving the admin UI port 3000 +# migrations python migrations/run.py prisma migrate deploy, then exit +# metrics python -m litellm.proxy.prometheus_metrics_server ARGS +# collector python -m litellm.proxy.collector ARGS +# --admin-agent the bundled LiteAdmin Slack agent (litellm-admin-agent --web ARGS) +# +# gateway and backend get their host and port defaults first, so ARGS such as +# --port 8080 override them. An unknown first word is executed as-is (docker run +# sh), through ddtrace-run like every component when USE_DDTRACE=true. PgBouncer is not a component: LITELLM_PGBOUNCER_ENABLED=true +# starts it inside proxy and gateway. Components that write Prometheus samples +# start with an empty PROMETHEUS_MULTIPROC_DIR; metrics, collector and raw +# readers keep the workers' files, other raw commands clear stale samples. +set -eu + +usage() { + awk 'NR == 1 { next } /^#/ { sub(/^# ?/, ""); print; next } { exit }' "$0" >&2 +} + +component="${LITELLM_COMPONENT:-proxy}" +case "${1:-}" in + proxy|gateway|backend|ui|migrations|metrics|collector) + component="$1" + shift + ;; + --admin-agent) + shift + export CONNECTION_AUTH_MODE=native + exec /opt/liteadmin/bin/litellm-admin-agent --web "$@" + ;; + ""|-*) + ;; + help) + usage + exit 0 + ;; + *) + if [ -n "${PROMETHEUS_MULTIPROC_DIR:-}" ]; then + mkdir -p "$PROMETHEUS_MULTIPROC_DIR" + case "$*" in + *litellm.proxy.prometheus_metrics_server*|*litellm.proxy.collector*) ;; + *) rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_* ;; + esac + fi + component=raw + ;; +esac + +case "$component" in + raw) + ;; + proxy) + set -- litellm "$@" + ;; + gateway) + set -- python -m gateway.launch --workers "${NUM_WORKERS:-1}" --host 0.0.0.0 --port 4000 "$@" + ;; + backend) + set -- uvicorn backend.main:app --host 0.0.0.0 --port 4001 "$@" + ;; + ui) + sh "$(dirname "$0")/ui/docker-entrypoint.d/40-litellm-runtime-config.sh" + exec nginx -g 'daemon off;' "$@" + ;; + migrations) + set -- python /app/migrations/run.py "$@" + ;; + metrics) + set -- python -m litellm.proxy.prometheus_metrics_server "$@" + ;; + collector) + set -- python -m litellm.proxy.collector "$@" + ;; + *) + echo "docker-entrypoint.sh: unknown LITELLM_COMPONENT '$component'" >&2 + usage + exit 64 + ;; +esac + +if [ -n "${PROMETHEUS_MULTIPROC_DIR:-}" ]; then + case "$component" in + raw) ;; + metrics|collector) mkdir -p "$PROMETHEUS_MULTIPROC_DIR" ;; + *) + mkdir -p "$PROMETHEUS_MULTIPROC_DIR" + rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_* + ;; + esac +fi + +case "${USE_DDTRACE:-}" in + [Tt][Rr][Uu][Ee]) + export DD_TRACE_OPENAI_ENABLED="False" + exec ddtrace-run "$@" + ;; +esac + +exec "$@" diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database deleted file mode 100644 index 2ebffc50b5e..00000000000 --- a/docker/Dockerfile.database +++ /dev/null @@ -1,172 +0,0 @@ -# syntax=docker/dockerfile:1.7 - -# Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 - -# Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a -# Pinned by digest like the other base images; bump explicitly on Node upgrades. -ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 -# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) -ARG PGBOUNCER_VERSION=1.26.0 -ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder -ARG PGBOUNCER_VERSION -ARG PGBOUNCER_SHA256 -USER root -RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl -WORKDIR /build -RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ - echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ - tar xzf pgbouncer.tar.gz --strip-components=1 && \ - ./configure --prefix=/usr/local --with-openssl=/usr && \ - make -j"$(nproc)" pgbouncer && \ - install -m 0755 pgbouncer /usr/local/bin/pgbouncer - -# Admin UI builder. Pinned to the build platform so the architecture-independent -# Next.js static export compiles once natively even in a multi-arch build, -# instead of once per target arch under QEMU. -FROM --platform=$BUILDPLATFORM $UI_BUILD_IMAGE AS ui-builder - -ENV NEXT_TELEMETRY_DISABLED=1 \ - npm_config_fund=false \ - npm_config_audit=false - -WORKDIR /ui - -COPY ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json ./ -COPY ui/litellm-dashboard/vendor/ ./vendor/ -RUN --mount=type=cache,target=/root/.npm npm ci --prefer-offline - -COPY ui/litellm-dashboard/ ./ -RUN npm run build - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apk add --no-cache \ - bash \ - gcc \ - python-3.13 \ - python-3.13-dev \ - openssl \ - libssl3 \ - openssl-dev \ - nodejs \ - npm \ - libsndfile - -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - UV_PYTHON_DOWNLOADS=0 \ - PATH="/app/.venv/bin:${PATH}" - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ -COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --extra bedrock-realtime \ - --python python3.13 - -# Copy full source tree -COPY . . - -# Replace the committed UI bundle with the one built from this exact source. -# Clearing first drops the committed bundle's content-hashed chunks that COPY -# would otherwise leave behind alongside the fresh ones. -RUN rm -rf litellm/proxy/_experimental/out -COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/ - -# Build Admin UI before final sync (applies the enterprise color override when present) -RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh - -# Install project and workspace packages (fast - deps already cached) -RUN uv build --package litellm-proxy --wheel --out-dir /wheels -RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --extra bedrock-realtime \ - --python python3.13 - -RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl - -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime -ARG LITELLM_RELEASE_TAG="" -ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} - -USER root - -# node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent -COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer - -WORKDIR /app -ENV PATH="/app/.venv/bin:${PATH}" \ - PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ - PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ - PRISMA_OFFLINE_MODE=true - -# Copy only what runtime needs. The application is installed inside the venv; -# the rest of the builder's /app is source and build metadata that must not -# ship (manifest-scanning tools attribute everything in it to this image). -# entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path. -COPY --from=builder /app/.venv /app/.venv -COPY --from=builder /app/docker /app/docker -COPY --from=builder /app/schema.prisma /app/schema.prisma -COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py -# enterprise/ is imported by source path at runtime (proxy_cli puts the -# working directory on sys.path; litellm/proxy/hooks resolves -# enterprise.enterprise_hooks from it) -COPY --from=builder /app/enterprise /app/enterprise -COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -# Prisma CLI + engines are baked under /opt/prisma, a fixed path every -# runtime uid can read and that no cache volume mount shadows (unlike -# /app/.cache or $HOME/.cache in readOnlyRootFilesystem + emptyDir setups). -# The paths are pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and -# recorded into the generated client at build time, so `prisma migrate -# deploy` on a fresh database needs no npm and no network access -# (#33650, #24554). -COPY --from=builder /opt/prisma /opt/prisma - -RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete && \ - chmod -R a+rX /opt/prisma && \ - test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ - test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" - -EXPOSE 4000/tcp - -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root deleted file mode 100644 index 90bcc523d9e..00000000000 --- a/docker/Dockerfile.non_root +++ /dev/null @@ -1,208 +0,0 @@ -# syntax=docker/dockerfile:1.7 - -# Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a -# Pinned by digest like the other base images; bump explicitly on Node upgrades. -ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 -# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) -ARG PGBOUNCER_VERSION=1.26.0 -ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder -ARG PGBOUNCER_VERSION -ARG PGBOUNCER_SHA256 -USER root -RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl -WORKDIR /build -RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ - echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ - tar xzf pgbouncer.tar.gz --strip-components=1 && \ - ./configure --prefix=/usr/local --with-openssl=/usr && \ - make -j"$(nproc)" pgbouncer && \ - install -m 0755 pgbouncer /usr/local/bin/pgbouncer - -# Admin UI builder. Pinned to the build platform so the architecture-independent -# Next.js static export compiles once natively even in a multi-arch build, -# instead of once per target arch under QEMU. -FROM --platform=$BUILDPLATFORM $UI_BUILD_IMAGE AS ui-builder - -ENV NEXT_TELEMETRY_DISABLED=1 \ - npm_config_fund=false \ - npm_config_audit=false - -WORKDIR /ui - -COPY ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json ./ -COPY ui/litellm-dashboard/vendor/ ./vendor/ -RUN --mount=type=cache,target=/root/.npm npm ci --prefer-offline - -COPY ui/litellm-dashboard/ ./ -RUN npm run build - -FROM $LITELLM_BUILD_IMAGE AS builder -WORKDIR /app -USER root - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN for i in 1 2 3; do \ - apk add --no-cache \ - python-3.13 \ - python-3.13-dev \ - gcc \ - rust \ - bash \ - coreutils \ - curl \ - openssl \ - libssl3 \ - libsndfile \ - nodejs \ - npm && break || sleep 5; \ - done - -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - UV_PYTHON_DOWNLOADS=0 \ - PATH="/app/.venv/bin:${PATH}" \ - LITELLM_NON_ROOT=true \ - XDG_CACHE_HOME=/app/.cache - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ - uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --extra bedrock-realtime \ - --python python3.13 - -# Copy full source tree -COPY . . - -# Replace the committed UI bundle with the one built from this exact source. -# Clearing first drops the committed bundle's content-hashed chunks that COPY -# would otherwise leave behind alongside the fresh ones. -RUN rm -rf litellm/proxy/_experimental/out -COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/ - -# Set non-root flag for build time consistency -ENV LITELLM_NON_ROOT=true - -RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \ - cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \ - cp /app/litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \ - touch /var/lib/litellm/ui/.litellm_ui_ready - -RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ - uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --extra bedrock-realtime \ - --python python3.13 - -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime -ARG LITELLM_RELEASE_TAG="" -ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} -WORKDIR /app -USER root - -RUN for i in 1 2 3; do \ - apk upgrade --no-cache && break || sleep 5; \ - done && \ - for i in 1 2 3; do \ - apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \ - done -COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer - -# Copy only what runtime needs. The application is installed inside the venv; -# the rest of the builder's /app is source and build metadata that must not -# ship (manifest-scanning tools attribute everything in it to this image). -# entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path. -COPY --from=builder /app/.venv /app/.venv -COPY --from=builder /app/docker /app/docker -COPY --from=builder /app/schema.prisma /app/schema.prisma -COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py -# enterprise/ is imported by source path at runtime (proxy_cli puts the -# working directory on sys.path; litellm/proxy/hooks resolves -# enterprise.enterprise_hooks from it) -COPY --from=builder /app/enterprise /app/enterprise -COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -# Prisma CLI + engines are baked under /opt/prisma, a fixed path every runtime -# uid can read and that no cache volume mount shadows (unlike /app/.cache or -# $HOME/.cache under readOnlyRootFilesystem + emptyDir or arbitrary-uid setups). -# PRISMA_CLI_QUERY_ENGINE_TYPE=binary makes the CLI use the baked binary query -# engine directly, so `prisma migrate deploy` on a fresh database needs no npm -# and no network access; without it the CLI looks for the library engine, which -# prisma stopped baking, and falls back to a download that fails offline or as a -# non-writable uid (#33650, #24554). -COPY --from=builder /opt/prisma /opt/prisma -COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui -COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets - -# XDG_CACHE_HOME is intentionally left unset so it falls back to $HOME/.cache -# (/app/.cache, writable by the runtime uid). The prisma bake at the read-only -# /opt/prisma is anchored by PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH, so -# nothing needs XDG to point there; pointing it at the read-only bake would -# deny any XDG-aware library that writes a cache at runtime. -ENV PATH="/app/.venv/bin:${PATH}" \ - PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ - PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ - HOME=/app \ - LITELLM_NON_ROOT=true \ - PRISMA_SKIP_POSTINSTALL_GENERATE=1 \ - PRISMA_HIDE_UPDATE_MESSAGE=1 \ - PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \ - PRISMA_OFFLINE_MODE=true - -RUN mkdir -p /nonexistent /app/.cache /var/lib/litellm/assets /var/lib/litellm/ui && \ - chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \ - PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ - chown -R nobody:nogroup "$PRISMA_PATH" && \ - LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \ - [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup "$LITELLM_PKG_MIGRATIONS_PATH" || true && \ - LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \ - chgrp -R 0 "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 "$LITELLM_PROXY_EXTRAS_PATH" || true && \ - chmod -R g=u "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u "$LITELLM_PROXY_EXTRAS_PATH" || true && \ - chmod -R g+w "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \ - chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ - chmod -R a+rX /opt/prisma && \ - test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ - test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \ - ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1 && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" - -USER 65534 - -EXPOSE 4000/tcp - -ENTRYPOINT ["/app/docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/README.md b/docker/README.md index ac82c6caebf..3ec26219420 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,15 +2,15 @@ This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose. -> **Just want to run LiteLLM?** This guide builds from source. To run the published -> image instead, use `docker-compose.quickstart.yml` in this directory — the -> two-service stack (gateway + Postgres) that the -> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents: +> **Just want to run LiteLLM?** `docker-compose.yml` in the repository root runs +> the published image with a Postgres database, the stack the +> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents, +> and it works on its own outside a checkout: > > ```bash -> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml +> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker-compose.yml > printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env -> docker compose -f docker-compose.quickstart.yml up -d +> docker compose up -d > ``` ## Prerequisites @@ -20,33 +20,52 @@ This guide provides instructions for building and running the LiteLLM applicatio ## Building and Running the Application -To build and run the application, you will use the `docker-compose.yml` file located in the root of the project. This file is configured to use the `Dockerfile.non_root` for a secure, non-root container environment. +The same `docker-compose.yml` builds from source when you pass `--build`: it builds the `Dockerfile` in the repository root, the one image LiteLLM ships. `docker-entrypoint.sh` next to it is the only entrypoint script; every container starts through it -### 1. Set the Master Key +## One image, many components -The application requires a `LITELLM_MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application. +Every LiteLLM container runs the same image. The first word of the container command (or the `LITELLM_COMPONENT` environment variable when the command carries only flags) picks the process the container runs, and everything after it is handed to that process unchanged: -Create a `.env` file in the root of the project and add the following line: +| Component | Runs | Port | +|--------------|-----------------------------------------------------|------| +| `proxy` | `litellm ...` (everything in one process, default) | 4000 | +| `gateway` | `python -m gateway.launch ...` (inference routes) | 4000 | +| `backend` | `uvicorn backend.main:app ...` (management routes) | 4001 | +| `ui` | nginx serving the static admin UI | 3000 | +| `migrations` | `python migrations/run.py`, `prisma migrate deploy` then exit | | +| `metrics` | `python -m litellm.proxy.prometheus_metrics_server ...` | `--port` | +| `collector` | `python -m litellm.proxy.collector ...` | | -``` -LITELLM_MASTER_KEY=your-secret-key +```bash +docker run -p 4000:4000 litellm --config /app/config.yaml # proxy, exactly as before +docker run -p 4000:4000 litellm gateway --port 4000 # componentized data plane +docker run -p 4001:4001 -e LITELLM_COMPONENT=backend litellm # same, chosen through the env +docker run -p 3000:3000 --read-only --tmpfs /tmp litellm ui # admin UI behind nginx +docker run -e DATABASE_URL=... litellm migrations # one-off schema migration job +docker run -it litellm sh # anything else runs verbatim ``` -Replace `your-secret-key` with a strong, randomly generated secret. +PgBouncer is not a separate component: `LITELLM_PGBOUNCER_ENABLED=true` starts an in-container PgBouncer in front of `DATABASE_URL` inside `proxy` and `gateway`. `USE_DDTRACE=true` wraps whichever component runs with `ddtrace-run`, and `PROMETHEUS_MULTIPROC_DIR` is emptied of stale samples before any workers fork (the `metrics` and `collector` sidecars only read it, so their restart keeps the live samples) + +The image runs as uid `65532` (`nonroot` in the Wolfi base) and also works as an arbitrary uid in gid 0, the shape OpenShift `restricted-v2` assigns, because everything it writes at runtime lives under `/app/.cache`, `/var/lib/litellm` and `/tmp`. Mount those (or set `readOnlyRootFilesystem` with emptyDirs there) for a read-only root filesystem. Prisma's CLI and engines are baked under `/opt/prisma`, so migrations need neither network nor a writable home + +### 1. Set the Master and Salt Keys + +The proxy signs virtual keys with `LITELLM_MASTER_KEY` and encrypts stored provider credentials with `LITELLM_SALT_KEY`. Compose reads both from a `.env` file in the directory you run it from, so generate them once: + +```bash +printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env +``` + +Keep the file: regenerating `LITELLM_SALT_KEY` makes credentials already stored in the database unreadable. A database created without `LITELLM_SALT_KEY` encrypted them with the master key, so when upgrading one, write only the existing `LITELLM_MASTER_KEY` to `.env` and leave `LITELLM_SALT_KEY` out. Provider keys such as `OPENAI_API_KEY` go in the same file, and the whole file is passed to the container ### 2. Build and Run the Containers -Once you have set the `LITELLM_MASTER_KEY`, you can build and run the containers using the following command: - ```bash docker compose up -d --build ``` -This command will: - -- Build the Docker image using `Dockerfile.non_root`. -- Start the `litellm`, `litellm_db`, and `prometheus` services in detached mode (`-d`). -- The `--build` flag ensures that the image is rebuilt if there are any changes to the Dockerfile or the application code. +This command builds the image from the root `Dockerfile` and starts the `litellm` and `db` services in detached mode. Without `--build`, `docker compose up` pulls the published `main-stable` image instead. Add `--profile monitoring` to also start Prometheus on port 9090, scraping the proxy with the root `prometheus.yml` ### 3. Verifying the Application is Running @@ -80,13 +99,13 @@ LITELLM_LICENSE="your-enterprise-license" PROXY_BASE_URL=https://gateway.example.com ``` -For the unified source deployment described above, put them in its `.env` file and rebuild: +For the compose deployment described above, put them in its `.env` file and rebuild: ```bash docker compose up -d --build ``` -In componentized deployments, set the flag and license on the backend container and route `/admin/mcp` to the backend service. The gateway component excludes this endpoint. The unified, database, non-root, and backend image builds bundle the connector +In componentized deployments, set the flag and license on the `backend` component and route `/admin/mcp` to it. The `gateway` component excludes this endpoint. The image bundles the connector for every component Hosting is disabled by default. Opting in requires a valid base Enterprise license; an unlicensed opt-in or invalid flag value prevents startup. Enabling it reserves `/admin`, so rename any MCP server alias called `admin` first @@ -96,34 +115,18 @@ Embedded responses default to `full`; selecting `LITELLM_ADMIN_RESPONSE_VIEW=com See the [LiteAdmin MCP guide](https://docs.litellm.ai/docs/proxy/liteadmin_mcp#run-liteadmin-mcp-inside-litellm) for client configuration, tool restrictions, and verification -## Hardened / Offline Testing +## Hardening -To ensure changes are safe for non-root, read-only root filesystems and restricted egress, always validate with the hardened compose file: +The compose file runs the proxy the way a locked-down cluster would: as the image's non-root user with a read-only root filesystem, every capability dropped, `no-new-privileges` set, and tmpfs mounts only at `/tmp` and `/app/.cache`. The image is built for that, so a change that makes the proxy write anywhere else fails here before it fails in Kubernetes. To try an arbitrary uid in gid 0, the shape OpenShift `restricted-v2` assigns, add `user: "101:0"` to the `litellm` service + +Prisma's CLI and engines are baked under `/opt/prisma`, so migrations run without network access. Verify with: ```bash -docker compose -f docker-compose.yml -f docker-compose.hardened.yml build --no-cache -docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d +docker run --rm --network none --entrypoint prisma docker.litellm.ai/berriai/litellm:main-stable --version ``` -This setup: -- Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image. -- Runs the proxy as a non-root user with a read-only rootfs and only writable tmpfs mounts: - - `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`) - - `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`) -- Pre-builds and serves the admin UI from read-only paths: - - `/var/lib/litellm/ui` (pre-restructured Next.js UI with `.litellm_ui_ready` marker) - - `/var/lib/litellm/assets` (UI logos and assets) -- Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines. - -You should also verify offline Prisma behaviour with: - -```bash -docker run --rm --network none --entrypoint prisma ghcr.io/berriai/litellm:main-stable --version -``` - -This command should succeed (showing engine versions) even with `--network none`, confirming that Prisma binaries are available without network access. - ## Troubleshooting -- **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project. -- **`Master key is not initialized`**: This error means the `LITELLM_MASTER_KEY` environment variable is not set. Make sure you have created a `.env` file in the project root with the `LITELLM_MASTER_KEY` defined. +- **`required variable LITELLM_MASTER_KEY is missing a value`**: Compose did not find a `.env` file with `LITELLM_MASTER_KEY` in the directory you ran it from. Generate one as shown above. +- **`password authentication failed for user "llmproxy"`**: the `litellm_postgres_data` volume was initialised by an older compose file with different credentials. `docker compose down -v` drops it and the next `up` recreates the database. +- **`build_admin_ui.sh: not found`**: the build context is wrong. Run `docker compose` from the root of the repository. diff --git a/docker/build_from_pip/Dockerfile.build_from_pip b/docker/build_from_pip/Dockerfile.build_from_pip deleted file mode 100644 index 25a7ed2fa97..00000000000 --- a/docker/build_from_pip/Dockerfile.build_from_pip +++ /dev/null @@ -1,64 +0,0 @@ -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82 -FROM $UV_IMAGE AS uvbin - -FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d - -ARG LITELLM_VERSION=1.83.0 - -WORKDIR /app - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apt-get update && \ - apt-get install -y --no-install-recommends gcc libffi-dev nodejs npm && \ - rm -rf /var/lib/apt/lists/* - -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - PATH="/app/.venv/bin:${PATH}" - -COPY schema.prisma . - -# This image is specifically for validating/installing the published PyPI -# artifact, not the checked-out source tree. -# Keep the moved proxy-runtime packages explicit until the published PyPI -# artifact includes that extra; newer releases will simply dedupe these. -RUN uv venv --python python && \ - uv pip install --python /app/.venv/bin/python \ - "litellm[proxy,proxy-runtime]==${LITELLM_VERSION}" \ - "google-cloud-aiplatform==1.133.0" \ - "google-genai==1.37.0" \ - "anthropic[vertex]==0.84.0" \ - "grpcio==1.84.1" \ - "prometheus-client==0.20.0" \ - "langfuse==2.59.7" \ - "opentelemetry-api==1.28.0" \ - "opentelemetry-sdk==1.28.0" \ - "opentelemetry-exporter-otlp==1.28.0" \ - "ddtrace==4.11.0" \ - "sentry-sdk==2.21.0" \ - "mangum==0.17.0" \ - "azure-ai-contentsafety==1.0.0" \ - "azure-storage-file-datalake==12.20.0" \ - "pypdf==6.7.5" \ - "llm-sandbox==0.3.31" \ - "detect-secrets==1.5.0" \ - "prisma==0.11.0" \ - "openai==2.24.0" - -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma && \ - chmod -R a+rX /opt/prisma && \ - python -c "import sys; from prisma.client import BINARY_PATHS; bad = sorted(p for group in BINARY_PATHS.model_dump().values() for p in group.values() if not p.startswith('/opt/prisma/')); sys.exit('prisma engines baked outside /opt/prisma: %r' % bad) if bad else None" - -ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries - -COPY docker/prod_entrypoint.sh /app/docker/prod_entrypoint.sh -RUN sed -i 's/\r$//' /app/docker/prod_entrypoint.sh && chmod +x /app/docker/prod_entrypoint.sh - -EXPOSE 4000/tcp - -ENTRYPOINT ["/app/docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/build_from_pip/Readme.md b/docker/build_from_pip/Readme.md deleted file mode 100644 index ad043588f33..00000000000 --- a/docker/build_from_pip/Readme.md +++ /dev/null @@ -1,9 +0,0 @@ -# Docker to build LiteLLM Proxy from litellm pip package - -### When to use this ? - -If you need to build LiteLLM Proxy from litellm pip package, you can use this Dockerfile as a reference. - -### Why build from pip package ? - -- If your company has a strict requirement around security / building images you can follow steps outlined here \ No newline at end of file diff --git a/docker/component_entrypoint.sh b/docker/component_entrypoint.sh deleted file mode 100755 index 3cd530e3009..00000000000 --- a/docker/component_entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -# stale samples from a previous container incarnation would be summed into the aggregate -if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then - mkdir -p "$PROMETHEUS_MULTIPROC_DIR" - rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_* -fi - -case "$USE_DDTRACE" in - [Tt][Rr][Uu][Ee]) - export DD_TRACE_OPENAI_ENABLED="False" - exec ddtrace-run "$@" - ;; -esac - -exec "$@" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100755 index 003d9b21db8..00000000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -euo pipefail - -REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -VENV_PYTHON="$REPO_ROOT/.venv/bin/python" -MIGRATION_SCRIPT="$REPO_ROOT/litellm/proxy/prisma_migration.py" - -if [ -x "$VENV_PYTHON" ]; then - "$VENV_PYTHON" "$MIGRATION_SCRIPT" -elif command -v uv >/dev/null 2>&1; then - (cd "$REPO_ROOT" && uv run --no-sync python "$MIGRATION_SCRIPT") -else - python3 "$MIGRATION_SCRIPT" -fi - -echo "Migration script ran successfully!" diff --git a/docker/install_auto_router.sh b/docker/install_auto_router.sh deleted file mode 100755 index 4fedf201b41..00000000000 --- a/docker/install_auto_router.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# semantic-router dependencies are installed via `uv sync`. diff --git a/docker/prod_entrypoint.sh b/docker/prod_entrypoint.sh deleted file mode 100644 index 4386be65a32..00000000000 --- a/docker/prod_entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -if [ "$1" = "--admin-agent" ]; then - shift - export CONNECTION_AUTH_MODE=native - exec /opt/liteadmin/bin/litellm-admin-agent --web "$@" -fi - -case "$USE_DDTRACE" in - [Tt][Rr][Uu][Ee]) - export DD_TRACE_OPENAI_ENABLED="False" - exec ddtrace-run litellm "$@" - ;; -esac - -exec litellm "$@" diff --git a/docker/tests/nonroot.yaml b/docker/tests/nonroot.yaml deleted file mode 100644 index 36118ca8c59..00000000000 --- a/docker/tests/nonroot.yaml +++ /dev/null @@ -1,18 +0,0 @@ -schemaVersion: 2.0.0 - -metadataTest: - entrypoint: ["docker/prod_entrypoint.sh"] - user: "65534" - workdir: "/app" - -fileExistenceTests: - - name: "Prisma Folder" - path: "/usr/local/lib/python3.13/site-packages/prisma/" - shouldExist: true - uid: 65534 - gid: 65534 - - name: "Prisma Schema" - path: "/usr/local/lib/python3.13/site-packages/prisma/schema.prisma" - shouldExist: true - uid: 65534 - gid: 65534 diff --git a/gateway/Dockerfile b/gateway/Dockerfile deleted file mode 100644 index 1e1c7d73091..00000000000 --- a/gateway/Dockerfile +++ /dev/null @@ -1,170 +0,0 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a -# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) -ARG PGBOUNCER_VERSION=1.26.0 -ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder -ARG PGBOUNCER_VERSION -ARG PGBOUNCER_SHA256 -USER root -RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl -WORKDIR /build -RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ - echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ - tar xzf pgbouncer.tar.gz --strip-components=1 && \ - ./configure --prefix=/usr/local --with-openssl=/usr && \ - make -j"$(nproc)" pgbouncer && \ - install -m 0755 pgbouncer /usr/local/bin/pgbouncer - -# ---------- Rust extension ---------- -# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes -FROM $LITELLM_BUILD_IMAGE AS rust-native - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /uvx /usr/local/bin/ - -RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -ENV UV_PYTHON_DOWNLOADS=0 - -COPY pyproject.toml README.md rust-toolchain.toml ./ -COPY .cargo ./.cargo -COPY litellm-rust ./litellm-rust -COPY model_prices_and_context_window.json ./ -COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/ - -RUN --mount=type=cache,target=/root/.cache/uv \ - mkdir -p litellm/rust_bridge /wheels && \ - touch litellm/__init__.py litellm/rust_bridge/__init__.py && \ - uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \ - python -c "import maturin; maturin.build_wheel('/wheels')" && \ - cd packaging/litellm-proxy && \ - uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \ - python -c "import maturin; maturin.build_wheel('/wheels')" - -# ---------- Builder ---------- -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /uvx /usr/local/bin/ - -# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE -# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi -# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. -RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start. -# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a -# BuildKit cache mount (different filesystem). -# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of -# silently pulling a managed interpreter. -# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't -# silently re-enable nodeenv's Node download. -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - UV_COMPILE_BYTECODE=1 \ - UV_PYTHON_DOWNLOADS=0 \ - PRISMA_USE_GLOBAL_NODE=true \ - PATH="/app/.venv/bin:${PATH}" - -# Stage 1 — install dependencies only. -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - --mount=type=bind,source=uv.lock,target=uv.lock \ - --mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \ - --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \ - --mount=type=bind,source=packaging/litellm-proxy/pyproject.toml,target=packaging/litellm-proxy/pyproject.toml \ - uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra bedrock-realtime \ - --python python3.13 - -# Stage 2 — copy source and install the workspace members. -COPY . . - -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project --no-install-package litellm-proxy --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra bedrock-realtime \ - --python python3.13 - -RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \ - uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl /wheels/litellm_proxy-*.whl - -# PYTHONPATH=/app makes the source tree shadow the installed package, so the -# compiled Rust extension must live next to the source or it is never imported. -RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ - -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh - -# ---------- Runtime ---------- -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -USER root - -RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with -# /home/nonroot. We run the proxy as that user. -WORKDIR /app -ENV HOME=/home/nonroot \ - PATH="/app/.venv/bin:${PATH}" \ - PYTHONPATH="/app" \ - PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries - -COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv -COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm -COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise -COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway -COPY --from=builder --chown=nonroot:nonroot /app/litellm_cli /app/litellm_cli -COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh -COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma -COPY --from=builder /opt/prisma /opt/prisma -COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer - -RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete && \ - chmod -R a+rX /opt/prisma && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \ - python -c "import litellm, litellm.rust_bridge._native; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__" - -USER nonroot - -EXPOSE 4000/tcp - -ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh python -m gateway.launch --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"] -CMD ["--host", "0.0.0.0", "--port", "4000"] diff --git a/litellm/proxy/start.sh b/litellm/proxy/start.sh deleted file mode 100755 index 44df50aaabf..00000000000 --- a/litellm/proxy/start.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -python3 proxy_cli.py \ No newline at end of file diff --git a/migrations/Dockerfile b/migrations/Dockerfile deleted file mode 100644 index 3ffa9140543..00000000000 --- a/migrations/Dockerfile +++ /dev/null @@ -1,152 +0,0 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -# ---------- Builder ---------- -# -# Minimal install for `prisma migrate deploy`. We deliberately skip the heavy -# `proxy-runtime` (otel, sentry, ddtrace, pypdf, google-genai, anthropic-vertex, -# ...) and `semantic-router` extras that the gateway/backend pull in — the -# migration engine doesn't need them. We DO install `--extra proxy` so the -# DB-URL helper from `litellm.proxy.auth.rds_iam_token` is importable, which -# is how the gateway and backend assemble `DATABASE_URL` at pod startup when -# `IAM_TOKEN_DB_AUTH=true` (see backend/main.py:17, gateway/main.py:22). And -# `--extra extra_proxy` provides the `prisma` CLI + the secret-manager -# backends `litellm.secret_managers.main` lazily imports. -# -# `prisma generate` runs once at BUILD time to (a) install the Node-based -# Prisma CLI into the binary cache and (b) download the migration / query -# engine binaries. The Python client it also produces is unused by this -# image's runtime entrypoint — that's fine, it's a few hundred KB and the -# alternative (`prisma py fetch`) doesn't reliably trigger engine downloads -# under nodeenv. Crucially we do NOT run `prisma generate` at RUNTIME; the -# old migration job did, on every pod start, which is the wasteful behaviour -# the componentization is fixing. -# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes -FROM $LITELLM_BUILD_IMAGE AS rust-native - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /uvx /usr/local/bin/ - -RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -ENV UV_PYTHON_DOWNLOADS=0 - -COPY pyproject.toml README.md rust-toolchain.toml ./ -COPY .cargo ./.cargo -COPY litellm-rust ./litellm-rust -COPY model_prices_and_context_window.json ./ - -RUN --mount=type=cache,target=/root/.cache/uv \ - mkdir -p litellm/rust_bridge /wheels && \ - touch litellm/__init__.py litellm/rust_bridge/__init__.py && \ - uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \ - python -c "import maturin; maturin.build_wheel('/wheels')" - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /uvx /usr/local/bin/ - -# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE -# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi -# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. -RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - UV_COMPILE_BYTECODE=1 \ - UV_PYTHON_DOWNLOADS=0 \ - PRISMA_USE_GLOBAL_NODE=true \ - PATH="/app/.venv/bin:${PATH}" - -# Stage 1 — install third-party deps only (cached by pyproject.toml/uv.lock). -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - --mount=type=bind,source=uv.lock,target=uv.lock \ - --mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \ - --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \ - uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra extra_proxy \ - --python python3.13 - -# Stage 2 — copy source and install the workspace members. -COPY . . - -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project --no-default-groups --no-editable \ - --extra proxy \ - --extra extra_proxy \ - --python python3.13 - -RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \ - uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl - -RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ - -COPY migrations/run.py /app/run.py - -# Pre-warm the Prisma binary cache so the Job pod doesn't reach the -# internet on first start. This matches what the backend Dockerfile does: -# `prisma generate` runs nodeenv (downloads Node), installs the prisma npm -# CLI, downloads the engine binaries for each `binaryTarget` in -# schema.prisma, AND emits the generated Python client. We don't need the -# client at runtime — the migration job invokes `prisma migrate deploy` -# via subprocess — but having it cached is harmless and the alternative -# (`prisma py fetch`) doesn't reliably trigger engine downloads. -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma - -# ---------- Runtime ---------- -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -USER root - -RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl libssl3 tzdata python-3.13 nodejs libsndfile libatomic && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532). The -# Prisma engine binaries are dynamically linked against libssl/libcrypto, so -# openssl stays in the runtime layer. -WORKDIR /app -ENV HOME=/home/nonroot \ - PATH="/app/.venv/bin:${PATH}" \ - PYTHONPATH="/app" \ - PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ - PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ - PRISMA_OFFLINE_MODE=true - -COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv -COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm -COPY --from=builder --chown=nonroot:nonroot /app/run.py /app/run.py -COPY --from=builder /opt/prisma /opt/prisma - -RUN chmod -R a+rX /opt/prisma && \ - test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ - test -f /opt/prisma/binaries/node_modules/prisma/build/index.js - -USER nonroot - -ENTRYPOINT ["python3", "/app/run.py"] diff --git a/docker/build_from_pip/litellm_config.yaml b/tests/basic_proxy_startup_tests/build_from_pip_config.yaml similarity index 77% rename from docker/build_from_pip/litellm_config.yaml rename to tests/basic_proxy_startup_tests/build_from_pip_config.yaml index f54647853ef..ff0db25e33f 100644 --- a/docker/build_from_pip/litellm_config.yaml +++ b/tests/basic_proxy_startup_tests/build_from_pip_config.yaml @@ -5,5 +5,5 @@ model_list: api_key: fake-key api_base: os.environ/FAKE_OPENAI_API_BASE -general_settings: - alerting: ["slack"] \ No newline at end of file +general_settings: + alerting: ["slack"] diff --git a/tests/e2e/claude_code/cron_vm/Dockerfile b/tests/e2e/claude_code/cron_vm/Dockerfile deleted file mode 100644 index 05c1387c5a2..00000000000 --- a/tests/e2e/claude_code/cron_vm/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 - -ARG GH_VERSION=2.101.0 -ARG GH_SHA256=9bca2d1c16825f109907a23307628a2f0698fbf99662b73a5cf0b020293072b8 -ARG UV_VERSION=0.10.9 -ARG UV_SHA256=20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594 - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates curl git jq procps iproute2 \ - && rm -rf /var/lib/apt/lists/* - -RUN curl -fsSLo /tmp/gh.tar.gz "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" \ - && echo "${GH_SHA256} /tmp/gh.tar.gz" | sha256sum -c - \ - && tar -xzf /tmp/gh.tar.gz -C /usr/local/bin --strip-components=2 "gh_${GH_VERSION}_linux_amd64/bin/gh" \ - && rm /tmp/gh.tar.gz - -RUN curl -fsSLo /tmp/uv.tar.gz "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \ - && echo "${UV_SHA256} /tmp/uv.tar.gz" | sha256sum -c - \ - && tar -xzf /tmp/uv.tar.gz -C /usr/local/bin --strip-components=1 uv-x86_64-unknown-linux-gnu/uv \ - && rm /tmp/uv.tar.gz - -RUN groupadd --gid 1000 populator && useradd --uid 1000 --gid 1000 --create-home populator - -ENV HOME=/home/populator \ - LITELLM_REPO=/opt/litellm \ - DISABLE_AUTOUPDATER=1 - -COPY --chown=populator:populator . /opt/litellm/tests/e2e/ - -USER populator -WORKDIR /home/populator -CMD ["/opt/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh"] diff --git a/tests/e2e/claude_code/cron_vm/README.md b/tests/e2e/claude_code/cron_vm/README.md index 7451e86e09b..4e47426e860 100644 --- a/tests/e2e/claude_code/cron_vm/README.md +++ b/tests/e2e/claude_code/cron_vm/README.md @@ -1,12 +1,12 @@ # Render cron job for the Claude Code compatibility-matrix populator The populator runs daily as the Render cron job `litellm-compat-matrix` -(Docker runtime, built from the `Dockerfile` in this directory) rather +(Docker runtime; the image recipe is in [Image](#image) below) rather than as a GitHub Action or on a dedicated VM. Trade-offs: -- ✅ No machine to keep on or patch. Render builds the image from this - directory on every push to `main` that touches `tests/e2e/**` and - runs it on the schedule. +- ✅ No machine to keep on or patch. Render builds the image on every + push to `main` that touches `tests/e2e/**` and runs it on the + schedule. - ✅ Credentials live in Render env vars and secret files, scoped to this one service, instead of on a VM filesystem. - ✅ The publish token still uses the `mateo-berri` account, which is a @@ -25,7 +25,6 @@ than as a GitHub Action or on a dedicated VM. Trade-offs: | File | Purpose | | --- | --- | -| `Dockerfile` | The image Render builds: Debian bookworm-slim plus pinned, checksum-verified `gh` and `uv`, with this `tests/e2e/` tree copied to `/opt/litellm/tests/e2e/`. Runs as the non-root user `populator` (uid/gid 1000, which is what Render's secret files are readable by). | | `run_daily.sh` | The actual cron job. Resolves versions, clones the worktree, installs the Claude Code CLI under test, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. | | `install_claude_code.sh` | Downloads one Claude Code release (` `) from the vendor's native release channel, verifies it against the sha256 in that release's `manifest.json`, and refuses a binary whose `--version` disagrees. Run by the cron and by the `compat-matrix-image` GitHub workflow. | | `build_matrix.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. | @@ -106,7 +105,7 @@ the same values if it ever has to be rebuilt. | Workspace | Litellm (the one that already builds the other litellm services) | | Type | Cron job, Docker runtime | | Repo / branch | `BerriAI/litellm` @ `main` | -| Dockerfile path | `tests/e2e/claude_code/cron_vm/Dockerfile` | +| Image | Built from the recipe under [Image](#image). Render only builds a Dockerfile that lives in the connected repo, and this repo ships exactly one Dockerfile (the LiteLLM image in its root), so the service has to point at a copy of the recipe kept with the service or at a prebuilt image | | Docker build context | `tests/e2e` (the repo root `.dockerignore` excludes `tests`, so the context has to start below it) | | Build filter | included paths `tests/e2e/**` | | Schedule | `0 6 * * *` (06:00 UTC daily) | @@ -117,7 +116,7 @@ the same values if it ever has to be rebuilt. Render mounts secret files at `/etc/secrets/`, which is where `CREDENTIALS_DIRECTORY` and `GOOGLE_APPLICATION_CREDENTIALS` in the env example point. Render also passes env vars to `docker build` as build -args, which is why the `Dockerfile` declares no `ARG` that could ever +args, which is why the image recipe declares no `ARG` that could ever be given a secret's name. Creating it through the API looks like this (fill `envVars` and @@ -145,13 +144,59 @@ curl -fsS https://api.render.com/v1/services \ "plan": "4c-16g", "region": "oregon", "envSpecificDetails": { - "dockerfilePath": "tests/e2e/claude_code/cron_vm/Dockerfile", + "dockerfilePath": "", "dockerContext": "tests/e2e" } } }' ``` +## Image + +Debian bookworm-slim plus pinned, checksum-verified `gh` and `uv`, with +this `tests/e2e/` tree copied to `/opt/litellm/tests/e2e/`. Runs as the +non-root user `populator` (uid/gid 1000, which is what Render's secret +files are readable by). This is the recipe the Render service builds; +it lives with the service because the repo ships exactly one Dockerfile, +the LiteLLM image in its root + +```dockerfile +FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 + +ARG GH_VERSION=2.101.0 +ARG GH_SHA256=9bca2d1c16825f109907a23307628a2f0698fbf99662b73a5cf0b020293072b8 +ARG UV_VERSION=0.10.9 +ARG UV_SHA256=20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git jq procps iproute2 \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSLo /tmp/gh.tar.gz "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" \ + && echo "${GH_SHA256} /tmp/gh.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/gh.tar.gz -C /usr/local/bin --strip-components=2 "gh_${GH_VERSION}_linux_amd64/bin/gh" \ + && rm /tmp/gh.tar.gz + +RUN curl -fsSLo /tmp/uv.tar.gz "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \ + && echo "${UV_SHA256} /tmp/uv.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/uv.tar.gz -C /usr/local/bin --strip-components=1 uv-x86_64-unknown-linux-gnu/uv \ + && rm /tmp/uv.tar.gz + +RUN groupadd --gid 1000 populator && useradd --uid 1000 --gid 1000 --create-home populator + +ENV HOME=/home/populator \ + LITELLM_REPO=/opt/litellm \ + DISABLE_AUTOUPDATER=1 + +COPY --chown=populator:populator . /opt/litellm/tests/e2e/ + +USER populator +WORKDIR /home/populator +CMD ["/opt/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh"] +``` + ## Operating it ```bash @@ -185,7 +230,7 @@ curl -fsS "https://api.render.com/v1/services/${CRON_ID}/deploys?limit=1" \ # Build and run the image locally (docker on Apple silicon needs the # platform flag; the context is tests/e2e, see the table above). docker build --platform linux/amd64 \ - -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix tests/e2e + -f /path/to/compat-matrix.Dockerfile -t compat-matrix tests/e2e docker run --rm --platform linux/amd64 \ --env-file litellm-compat-matrix.env -e SKIP_PUBLISH=1 \ -v "$PWD/secrets:/etc/secrets:ro" compat-matrix @@ -234,8 +279,8 @@ docker run --rm --platform linux/amd64 \ A CLI release that breaks a cell shows up as a green→red flip, which withholds auto-merge on that day's docs PR for review. To rerun the matrix on one specific CLI, set `CLAUDE_CODE_VERSION` on the run. - `gh` and `uv` stay pinned in the `Dockerfile`; bump them in a PR with - the checksum from the release's `gh__checksums.txt` and the + `gh` and `uv` stay pinned in the image recipe; bump them with the + checksum from the release's `gh__checksums.txt` and the tarball's `.sha256` sidecar respectively. - **A local build on Apple silicon only proves the image assembles.** Under QEMU the Claude Code binary (a Bun executable) dies with diff --git a/tests/e2e/claude_code/cron_vm/run_daily.sh b/tests/e2e/claude_code/cron_vm/run_daily.sh index 749d5166a34..31b97a9ab7a 100755 --- a/tests/e2e/claude_code/cron_vm/run_daily.sh +++ b/tests/e2e/claude_code/cron_vm/run_daily.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Daily Claude Code compatibility-matrix populator. # -# Runs daily as the Render cron job `litellm-compat-matrix`, built from -# the Dockerfile in this directory (see README.md). The flow is: +# Runs daily as the Render cron job `litellm-compat-matrix` (see +# README.md for the image it runs in). The flow is: # # 1. Resolve the latest LiteLLM final release tag from the GitHub # Releases API. diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 262114ee32b..491fc87dce2 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -945,7 +945,7 @@ def configless_proxy_process( def owned_gateway_image( gateway: Gateway, directory: Path, overrides: Mapping[str, str], *, config: Path, workers: int ) -> Iterator[OwnedProxy]: - """The componentized gateway started the way its image starts it: `docker/component_entrypoint.sh` running + """The componentized gateway started the way its image starts it: `docker-entrypoint.sh` running `python -m gateway.launch`, with the config handed over as `CONFIG_FILE_PATH`. It serves the data plane only, so keys come from a proxy that shares its database.""" root: Final = _proxy_root() @@ -953,7 +953,7 @@ def owned_gateway_image( output: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(directory))) output.mkdir(parents=True, exist_ok=True) command: Final = ( - str(root / "docker" / "component_entrypoint.sh"), + str(root / "docker-entrypoint.sh"), sys.executable, "-m", "gateway.launch", diff --git a/tests/integration/database/image/test_collector_image_serves_socket.py b/tests/integration/database/image/test_collector_image_serves_socket.py new file mode 100644 index 00000000000..a9b1d128e79 --- /dev/null +++ b/tests/integration/database/image/test_collector_image_serves_socket.py @@ -0,0 +1,174 @@ +import os +import shutil +import subprocess +import time +import uuid +from collections.abc import Iterator +from typing import Final + +import pytest + +from tests._master_key import MASTER_KEY + +IMAGE: Final = os.getenv("LITELLM_IMAGE") +POSTGRES_IMAGE: Final = os.getenv("LITELLM_TEST_POSTGRES_IMAGE", "postgres:16-alpine") +NON_ROOT_UID: Final = "12345:0" +STARTUP_TIMEOUT_SECONDS: Final = int(os.getenv("LITELLM_COMPONENT_STARTUP_TIMEOUT", "180")) + +pytestmark = [ + pytest.mark.skipif(IMAGE is None, reason="requires a built image (set LITELLM_IMAGE)"), + pytest.mark.skipif(shutil.which("docker") is None, reason="requires the docker CLI"), +] + + +def _docker(*args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run(["docker", *args], capture_output=True, text=True, check=check) + + +def _wait_until_postgres_ready(pg: str, attempts: int = 60) -> None: + for _ in range(attempts): + running = _docker( + "ps", + "--filter", + f"name={pg}", + "--filter", + "status=running", + "--format", + "{{.Names}}", + check=False, + ).stdout + if pg not in running: + logs = _docker("logs", pg, check=False) + pytest.fail(f"postgres container is not running:\n{logs.stdout}\n{logs.stderr}") + ready = _docker("exec", pg, "pg_isready", "-U", "postgres", "-d", "litellm", check=False) + if ready.returncode == 0: + return + time.sleep(1) + pytest.fail(f"postgres never became ready after {attempts}s") + + +def _container_logs(container: str) -> str: + logs = _docker("logs", container, check=False) + return f"stdout:\n{logs.stdout}\nstderr:\n{logs.stderr}" + + +def _is_running(container: str) -> bool: + return bool( + _docker( + "ps", + "--filter", + f"name={container}", + "--filter", + "status=running", + "--format", + "{{.Names}}", + check=False, + ).stdout.strip() + ) + + +@pytest.fixture() +def collector_stack(request: pytest.FixtureRequest) -> Iterator[str]: + user, mount_socket_volume = request.param + run_id = f"collectorsock-{uuid.uuid4().hex[:8]}" + network = f"{run_id}-net" + pg = f"{run_id}-pg" + collector = f"{run_id}-app" + volume = f"{run_id}-sock" + + _docker("network", "create", network) + try: + _docker( + "run", + "-d", + "--name", + pg, + "--network", + network, + "-e", + "POSTGRES_PASSWORD=pw", + "-e", + "POSTGRES_DB=litellm", + POSTGRES_IMAGE, + ) + _wait_until_postgres_ready(pg) + assert IMAGE is not None + _docker( + "run", + "--rm", + "--name", + f"{run_id}-migrations", + "--network", + network, + "-e", + f"DATABASE_URL=postgresql://postgres:pw@{pg}:5432/litellm", + IMAGE, + "migrations", + ) + run_args = [ + "run", + "-d", + "--name", + collector, + "--network", + network, + "-e", + f"DATABASE_URL=postgresql://postgres:pw@{pg}:5432/litellm", + "-e", + f"LITELLM_MASTER_KEY={MASTER_KEY}", + "-e", + "LITELLM_LOCAL_MODEL_COST_MAP=True", + "-e", + "DISABLE_SCHEMA_UPDATE=true", + ] + if user is not None: + run_args += ["--user", user] + if mount_socket_volume: + run_args += ["-v", f"{volume}:/var/run/litellm"] + _docker(*run_args, IMAGE, "collector") + yield collector + finally: + _docker("rm", "-f", collector, check=False) + _docker("rm", "-f", pg, check=False) + _docker("volume", "rm", "-f", volume, check=False) + _docker("network", "rm", network, check=False) + + +@pytest.mark.parametrize( + "collector_stack", + [ + pytest.param((None, False), id="default-user-no-mount"), + pytest.param((None, True), id="default-user-named-volume"), + pytest.param((NON_ROOT_UID, False), id="arbitrary-uid-gid0-no-mount"), + ], + indirect=True, +) +def test_collector_creates_its_socket_as_the_image_user(collector_stack: str) -> None: + collector = collector_stack + + deadline = time.time() + STARTUP_TIMEOUT_SECONDS + socket_seen = False + while time.time() < deadline: + if not _is_running(collector): + pytest.fail(f"the collector exited before creating its socket.\n{_container_logs(collector)}") + probe = _docker( + "exec", + collector, + "sh", + "-c", + "test -S /var/run/litellm/collector.sock", + check=False, + ) + if probe.returncode == 0: + socket_seen = True + break + time.sleep(2) + + inspect = _docker("inspect", "-f", "{{.State.Running}}", collector, check=False).stdout.strip() + assert inspect == "true", ( + f"the collector container is not running (inspect State.Running={inspect!r}).\n{_container_logs(collector)}" + ) + assert socket_seen, ( + f"/var/run/litellm/collector.sock never appeared within {STARTUP_TIMEOUT_SECONDS}s.\n" + f"{_container_logs(collector)}" + ) diff --git a/tests/integration/database/image/test_component_image_serves_offline.py b/tests/integration/database/image/test_component_image_serves_offline.py index c2f0d1e5406..bf6cae5ef9d 100644 --- a/tests/integration/database/image/test_component_image_serves_offline.py +++ b/tests/integration/database/image/test_component_image_serves_offline.py @@ -1,4 +1,4 @@ -"""Image-level regression net for the prisma bake in the componentized images. +"""Image-level regression net for the prisma bake in the `gateway` and `backend` components. The gateway and backend serve requests; they never shell out to the Prisma CLI (``PrismaManager.setup_database`` is reachable only from ``proxy_cli.py``, which @@ -37,7 +37,8 @@ from tests._master_key import MASTER_KEY IMAGE = os.getenv("LITELLM_IMAGE") POSTGRES_IMAGE = os.getenv("LITELLM_TEST_POSTGRES_IMAGE", "postgres:16-alpine") CURL_IMAGE = os.getenv("LITELLM_TEST_CURL_IMAGE", "curlimages/curl:8.11.1") -COMPONENT_PORT = os.getenv("LITELLM_COMPONENT_PORT", "4000") +COMPONENT = os.getenv("LITELLM_COMPONENT", "gateway") +COMPONENT_PORT = os.getenv("LITELLM_COMPONENT_PORT", {"gateway": "4000", "backend": "4001"}[COMPONENT]) NON_ROOT_UID = "12345:0" STARTUP_TIMEOUT_SECONDS = int(os.getenv("LITELLM_COMPONENT_STARTUP_TIMEOUT", "180")) @@ -65,7 +66,7 @@ def offline_stack(): The container runs with DISABLE_SCHEMA_UPDATE, since applying the schema is the migration job's responsibility in this topology and needs the Prisma CLI - these images deliberately omit, and with LITELLM_LOCAL_MODEL_COST_MAP, or the + these components never invoke, and with LITELLM_LOCAL_MODEL_COST_MAP, or the proxy spends the whole startup budget timing out on a cost-map fetch over the network it does not have. @@ -94,7 +95,7 @@ def offline_stack(): "-e", f"LITELLM_MASTER_KEY={MASTER_KEY}", "-e", "DISABLE_SCHEMA_UPDATE=true", "-e", "LITELLM_LOCAL_MODEL_COST_MAP=True", - IMAGE, + IMAGE, COMPONENT, ) yield network, component finally: diff --git a/tests/integration/database/image/test_offline_image_migration.py b/tests/integration/database/image/test_offline_image_migration.py index be2f0537576..2463fd0fd99 100644 --- a/tests/integration/database/image/test_offline_image_migration.py +++ b/tests/integration/database/image/test_offline_image_migration.py @@ -1,6 +1,6 @@ """Image-level regression net for the prisma bake in the shipped runtime image. -Boots a built image's migration entrypoint the way an OpenShift / air-gapped +Boots a built image's `migrations` component the way an OpenShift / air-gapped deployment does (an internal-only network with no egress, an arbitrary non-root uid in GID 0) against a brand-new Postgres, and asserts the schema was created. @@ -14,6 +14,7 @@ the normal unit-test run and exercised only where an image has been built (the image-scan workflow). Requires a working docker CLI. """ +import shlex import shutil import subprocess import uuid @@ -26,10 +27,7 @@ POSTGRES_IMAGE = os.getenv("LITELLM_TEST_POSTGRES_IMAGE", "postgres:16-alpine") MIN_TABLES = int(os.getenv("LITELLM_TEST_MIN_TABLES", "20")) NON_ROOT_UID = "12345:0" # arbitrary uid in GID 0, as OpenShift restricted-v2 assigns -MIGRATION_INTERPRETER = os.getenv("LITELLM_MIGRATION_INTERPRETER", "python") -MIGRATION_SCRIPT = os.getenv( - "LITELLM_MIGRATION_SCRIPT", "litellm/proxy/prisma_migration.py" -) +MIGRATION_ARGS = tuple(shlex.split(os.getenv("LITELLM_MIGRATION_ARGS", "migrations"))) pytestmark = [ pytest.mark.skipif(IMAGE is None, reason="requires a built image (set LITELLM_IMAGE)"), @@ -113,8 +111,7 @@ def test_migration_offline_as_non_root_uid(offline_postgres): "-e", f"DATABASE_URL=postgresql://postgres:pw@{pg}:5432/litellm", "-e", "LITELLM_MASTER_KEY=sk-offline-migration-test", "-e", "DISABLE_SCHEMA_UPDATE=false", - "-w", "/app", "--entrypoint", MIGRATION_INTERPRETER, - IMAGE, MIGRATION_SCRIPT, + IMAGE, *MIGRATION_ARGS, check=False, ) tables = _table_count(pg) diff --git a/tests/integration/database/image/test_ui_image_serves_offline.py b/tests/integration/database/image/test_ui_image_serves_offline.py index 5ff68effd7f..b3092887b75 100644 --- a/tests/integration/database/image/test_ui_image_serves_offline.py +++ b/tests/integration/database/image/test_ui_image_serves_offline.py @@ -1,7 +1,7 @@ -"""Image-level regression net for arbitrary-uid boot of the UI image. +"""Image-level regression net for arbitrary-uid boot of the `ui` component. OpenShift ``restricted-v2`` ignores the image ``USER`` and assigns an -arbitrary uid in GID 0. The stock nginx base expects to start as root, so +arbitrary uid in GID 0. A stock nginx install expects to start as root, so its cache (``/var/cache/nginx``) and pid (``/run``) paths are root-owned 755 and the master process dies at startup with ``mkdir() "/var/cache/nginx/client_temp" failed (13: Permission denied)``. @@ -63,7 +63,7 @@ def ui_container() -> Iterator[tuple[str, str]]: "run", "-d", "--name", container, "--network", network, "--user", ARBITRARY_UID, "--read-only", "--tmpfs", "/tmp", - IMAGE, + IMAGE, "ui", ) yield network, container finally: diff --git a/tests/integration/observability/test_prometheus_series_cap_chaos.py b/tests/integration/observability/test_prometheus_series_cap_chaos.py index c62c5b4fb0f..fd4e5e9a673 100644 --- a/tests/integration/observability/test_prometheus_series_cap_chaos.py +++ b/tests/integration/observability/test_prometheus_series_cap_chaos.py @@ -422,7 +422,7 @@ def _fill_the_cap(image: CapRig, cell: str) -> frozenset[str]: @pytest.mark.timeout(420) def test_gateway_image_restart_on_a_kept_directory_starts_the_cap_over(tmp_path: Path) -> None: - """D1: the gateway image's launcher (`docker/component_entrypoint.sh` running `python -m gateway.launch`, two + """D1: the gateway image's launcher (`docker-entrypoint.sh` running `python -m gateway.launch`, two workers) restarted on a kept PROMETHEUS_MULTIPROC_DIR: the entrypoint removes the previous container's samples and admitted series before the workers fork, so the second boot shows only its own three keys and a fourth lands on `other`.""" diff --git a/tests/integration/run.py b/tests/integration/run.py index 7ab8d59c02c..b28d9f4e0c6 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -28,6 +28,7 @@ GROUPS: Final = MappingProxyType( GITHUB_FILES: Final = frozenset( { "tests/integration/database/test_roi_observed.py", + "tests/integration/database/image/test_collector_image_serves_socket.py", "tests/integration/database/image/test_component_image_serves_offline.py", "tests/integration/database/image/test_image_admin_mcp.py", "tests/integration/database/image/test_image_bedrock_realtime_extra.py", diff --git a/tests/unit/test_component_entrypoint.py b/tests/unit/test_component_entrypoint.py deleted file mode 100644 index 335c7e54416..00000000000 --- a/tests/unit/test_component_entrypoint.py +++ /dev/null @@ -1,491 +0,0 @@ -"""Unit tests for `docker/component_entrypoint.sh` and its wiring into the -componentized `gateway` / `backend` images and Terraform deployments.""" - -import json -import os -import re -import stat -import subprocess -from pathlib import Path - -import pytest - -from litellm.constants import PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX - -REPO_ROOT = Path(__file__).resolve().parents[2] -COMPONENT_ENTRYPOINT = REPO_ROOT / "docker" / "component_entrypoint.sh" -PROD_ENTRYPOINT = REPO_ROOT / "docker" / "prod_entrypoint.sh" -GATEWAY_DOCKERFILE = REPO_ROOT / "gateway" / "Dockerfile" -BACKEND_DOCKERFILE = REPO_ROOT / "backend" / "Dockerfile" -BUILD_FROM_PIP_DOCKERFILE = REPO_ROOT / "docker" / "build_from_pip" / "Dockerfile.build_from_pip" -TERRAFORM_ECS = REPO_ROOT / "terraform" / "litellm" / "aws" / "ecs.tf" -TERRAFORM_CLOUDRUN = REPO_ROOT / "terraform" / "litellm" / "gcp" / "cloudrun.tf" - -IMAGE_ENTRYPOINT_PATH = "/app/docker/component_entrypoint.sh" - -TRUTHY_USE_DDTRACE = ("true", "True", "TRUE", "tRuE") -FALSY_USE_DDTRACE = (None, "", "false", "False", "1", "yes", "on", "truex") - -PYTHONPATH_SENTINEL = "/lit-entrypoint-sentinel:/app" - -_STUB_TEMPLATE = """#!/bin/sh -{{ - echo "exec={name}" - echo "args=$*" - echo "DD_TRACE_OPENAI_ENABLED=${{DD_TRACE_OPENAI_ENABLED-}}" - echo "PYTHONPATH=${{PYTHONPATH-}}" -}} >> "$RECORD" -""" - -_ENTRYPOINT_RE = re.compile(r"^ENTRYPOINT\s+(\[.*\])\s*$", re.MULTILINE) -_CMD_RE = re.compile(r"^CMD\s+(\[.*\])\s*$", re.MULTILINE) -_COPY_RE = re.compile(r"^COPY\s+(?!--from)(\S+)\s+(\S+)\s*$", re.MULTILINE) -_APP_TARGET_RE = re.compile(r"(?:gateway|backend)\.main:app|gateway\.launch") -_TF_STRING_LOCAL_RE = re.compile(r'^\s*(\w+)\s*=\s*"((?:[^"\\]|\\.)*)"\s*$', re.MULTILINE) -_TF_INTERPOLATION_RE = re.compile(r"\$\{(local|var)\.(\w+)\}") - -TERRAFORM_LAUNCH_SITES = {TERRAFORM_ECS: 2, TERRAFORM_CLOUDRUN: 2} -TERRAFORM_VAR_STUBS = {"gateway_num_workers": "2"} -COMPONENT_LAUNCHERS = { - "gateway": ("python", "-m", "gateway.launch"), - "backend": ("uvicorn", "backend.main:app"), -} -_MAX_INTERPOLATION_PASSES = 5 - - -def _write_stubs(bin_dir: Path, names: tuple[str, ...]) -> None: - for name in names: - stub = bin_dir / name - stub.write_text(_STUB_TEMPLATE.format(name=name)) - stub.chmod(0o755) - - -def _run_entrypoint( - script: Path, - argv: tuple[str, ...], - use_ddtrace: str | None, - tmp_path: Path, -) -> tuple[str, ...]: - """Run `script` with stubbed executables on PATH and return the recorded lines.""" - bin_dir = tmp_path / "bin" - bin_dir.mkdir(parents=True) - _write_stubs(bin_dir, ("ddtrace-run", "uvicorn", "python", "litellm")) - record = tmp_path / "record.txt" - - env = { - **os.environ, - "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", - "RECORD": str(record), - "PYTHONPATH": PYTHONPATH_SENTINEL, - } - env.pop("USE_DDTRACE", None) - env.pop("DD_TRACE_OPENAI_ENABLED", None) - if use_ddtrace is not None: - env["USE_DDTRACE"] = use_ddtrace - - result = subprocess.run( - ["sh", str(script), *argv], - env=env, - capture_output=True, - text=True, - check=False, - ) - assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" - return tuple(record.read_text().splitlines()) if record.exists() else () - - -def _run_shell_command(command: str, bin_dir: Path, record: Path, use_ddtrace: str | None) -> tuple[str, ...]: - """Run a resolved Terraform launch command through `sh -c` and return the recorded lines.""" - env = { - **os.environ, - "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", - "RECORD": str(record), - "PYTHONPATH": PYTHONPATH_SENTINEL, - } - env.pop("USE_DDTRACE", None) - env.pop("DD_TRACE_OPENAI_ENABLED", None) - if use_ddtrace is not None: - env["USE_DDTRACE"] = use_ddtrace - - result = subprocess.run(["sh", "-c", command], env=env, capture_output=True, text=True, check=False) - assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" - return tuple(record.read_text().splitlines()) if record.exists() else () - - -def _resolve_tf_local(terraform_file: Path, name: str) -> str: - """Read a string local out of a .tf file and expand its `local.` / `var.` interpolations.""" - values = { - m.group(1): m.group(2).replace('\\"', '"') for m in _TF_STRING_LOCAL_RE.finditer(terraform_file.read_text()) - } - - assert name in values, f"{terraform_file} defines no {name} local" - resolved = values[name] - for _ in range(_MAX_INTERPOLATION_PASSES): - if "${" not in resolved: - return resolved - resolved = _TF_INTERPOLATION_RE.sub( - lambda m: values[m.group(2)] if m.group(1) == "local" else TERRAFORM_VAR_STUBS[m.group(2)], - resolved, - ) - raise AssertionError(f"{terraform_file}:{name} still has unresolved interpolations: {resolved}") - - -def _entrypoint_argv(dockerfile: Path) -> tuple[str, ...]: - matches = _ENTRYPOINT_RE.findall(dockerfile.read_text()) - assert matches, f"no exec-form ENTRYPOINT found in {dockerfile}" - parsed = json.loads(matches[-1]) - return tuple(str(part) for part in parsed) - - -def _cmd_argv(dockerfile: Path) -> tuple[str, ...]: - matches = _CMD_RE.findall(dockerfile.read_text()) - assert matches, f"no exec-form CMD found in {dockerfile}" - return tuple(str(part) for part in json.loads(matches[-1])) - - -def test_ddtrace_enabled_wraps_the_command_and_disables_the_openai_integration( - tmp_path: Path, -) -> None: - """`USE_DDTRACE=true` must prefix the command with `ddtrace-run`, turn the openai - integration off, and leave PYTHONPATH alone. - - `ddtrace-run` installs its instrumentation by PREPENDING a bootstrap directory to - PYTHONPATH, and the images set PYTHONPATH=/app so the app package is importable. A - wrapper that assigned PYTHONPATH instead of inheriting it would either drop the - bootstrap (silently disabling tracing) or drop /app (breaking the import), so the - recorded value is asserted verbatim. - - PYTHONPATH_SENTINEL deliberately differs from the images' own /app: with /app as the - fixture value, a wrapper that overwrote PYTHONPATH with /app would still satisfy this - assertion and the check would prove nothing. - - The openai integration is disabled because under `ddtrace-run` the bootstrap patches - it before any litellm code runs, so litellm's in-process `patch_all(..., openai=False)` - can no longer suppress it; leaving it on double-reports every LLM call. - """ - recorded = _run_entrypoint( - COMPONENT_ENTRYPOINT, - ("uvicorn", "gateway.main:app", "--workers", "2", "--port", "4000"), - use_ddtrace="true", - tmp_path=tmp_path, - ) - - assert recorded == ( - "exec=ddtrace-run", - "args=uvicorn gateway.main:app --workers 2 --port 4000", - "DD_TRACE_OPENAI_ENABLED=False", - f"PYTHONPATH={PYTHONPATH_SENTINEL}", - ) - - -def test_ddtrace_disabled_execs_the_command_directly(tmp_path: Path) -> None: - recorded = _run_entrypoint( - COMPONENT_ENTRYPOINT, - ("uvicorn", "backend.main:app", "--port", "4001"), - use_ddtrace=None, - tmp_path=tmp_path, - ) - - assert recorded == ( - "exec=uvicorn", - "args=backend.main:app --port 4001", - "DD_TRACE_OPENAI_ENABLED=", - f"PYTHONPATH={PYTHONPATH_SENTINEL}", - ) - - -@pytest.mark.parametrize( - "use_ddtrace, traced", - [*((v, True) for v in TRUTHY_USE_DDTRACE), *((v, False) for v in FALSY_USE_DDTRACE)], -) -def test_gating_matches_the_monolithic_entrypoint_and_get_secret_bool( - use_ddtrace: str | None, traced: bool, tmp_path: Path -) -> None: - """Both entrypoints must accept exactly the spellings `get_secret_bool` accepts. - - `ProxyStartupEvent._init_dd_tracer` reads `USE_DDTRACE` through `get_secret_bool`, which - matches `true` case-insensitively. If the shell gate were stricter, `USE_DDTRACE=True` would - give in-process LLM spans without `ddtrace-run` HTTP spans, a half-enabled state. - """ - component = _run_entrypoint( - COMPONENT_ENTRYPOINT, - ("uvicorn", "gateway.main:app"), - use_ddtrace=use_ddtrace, - tmp_path=tmp_path / "component", - ) - monolith = _run_entrypoint( - PROD_ENTRYPOINT, - ("--port", "4000"), - use_ddtrace=use_ddtrace, - tmp_path=tmp_path / "monolith", - ) - - expected_exec = "exec=ddtrace-run" if traced else "exec=uvicorn" - expected_openai = "DD_TRACE_OPENAI_ENABLED=False" if traced else "DD_TRACE_OPENAI_ENABLED=" - assert component[0] == expected_exec - assert component[2] == expected_openai - assert monolith[0] == ("exec=ddtrace-run" if traced else "exec=litellm") - assert monolith[2] == expected_openai - assert monolith[1] == ("args=litellm --port 4000" if traced else "args=--port 4000") - - -def test_wipes_the_prometheus_multiproc_dir_before_uvicorn_forks(tmp_path: Path) -> None: - """A restarted container inherits the emptyDir of its predecessor, whose worker pids it may reuse, so the - stale .db files must be gone before any worker opens the one carrying its own pid. The admitted-series - files go with them, or the restarted workers would keep counting new label sets on `other` for the - label sets the previous container admitted.""" - multiproc_dir = tmp_path / "multiproc" - multiproc_dir.mkdir() - (multiproc_dir / "gauge_livesum_7.db").write_bytes(b"stale") - (multiproc_dir / "counter_7.db").write_bytes(b"stale") - (multiproc_dir / f"{PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX}litellm_requests_metric").write_bytes(b"stale") - (multiproc_dir / "keep.txt").write_text("not a sample") - - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - _write_stubs(bin_dir, ("uvicorn",)) - record = tmp_path / "record.txt" - env = { - **os.environ, - "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", - "RECORD": str(record), - "PROMETHEUS_MULTIPROC_DIR": str(multiproc_dir), - } - env.pop("USE_DDTRACE", None) - result = subprocess.run( - ["sh", str(COMPONENT_ENTRYPOINT), "uvicorn", "gateway.main:app"], - env=env, - capture_output=True, - text=True, - check=False, - ) - - assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" - assert sorted(p.name for p in multiproc_dir.iterdir()) == ["keep.txt"] - assert record.read_text().splitlines()[0] == "exec=uvicorn" - - -def test_creates_a_missing_prometheus_multiproc_dir(tmp_path: Path) -> None: - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - _write_stubs(bin_dir, ("uvicorn",)) - missing = tmp_path / "multiproc" - env = { - **os.environ, - "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", - "RECORD": str(tmp_path / "record.txt"), - "PROMETHEUS_MULTIPROC_DIR": str(missing), - } - env.pop("USE_DDTRACE", None) - result = subprocess.run( - ["sh", str(COMPONENT_ENTRYPOINT), "uvicorn", "gateway.main:app"], env=env, capture_output=True, text=True - ) - - assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" - assert missing.is_dir() - - -def _copied_script(dockerfile: Path, image_path: str) -> Path: - """Resolve the repo file a Dockerfile `COPY`s to `image_path`, so tests run what the image ships.""" - matches = _COPY_RE.findall(dockerfile.read_text()) - sources = tuple(src for src, dst in matches if dst == image_path) - assert sources, f"{dockerfile} never COPYs anything to {image_path}" - source = REPO_ROOT / sources[-1] - assert source.is_file(), f"{dockerfile} COPYs {sources[-1]}, which does not exist in the build context" - return source - - -@pytest.mark.parametrize( - "use_ddtrace, traced", - [*((v, True) for v in TRUTHY_USE_DDTRACE), *((v, False) for v in FALSY_USE_DDTRACE)], -) -def test_build_from_pip_image_launches_litellm_through_the_prod_entrypoint( - use_ddtrace: str | None, traced: bool, tmp_path: Path -) -> None: - """Run the build_from_pip image's ENTRYPOINT + CMD through the script it actually COPYs. - - The image used to `ENTRYPOINT ["litellm"]`, so `USE_DDTRACE` was inert there at any spelling. - Resolving the ENTRYPOINT path back to its COPY source and executing it with the Dockerfile's - CMD checks the launch the container performs, not just that the Dockerfile mentions the script. - """ - entrypoint = _entrypoint_argv(BUILD_FROM_PIP_DOCKERFILE) - assert len(entrypoint) == 1, ( - f"{BUILD_FROM_PIP_DOCKERFILE} ENTRYPOINT must be the bare script so CMD reaches litellm" - ) - script = _copied_script(BUILD_FROM_PIP_DOCKERFILE, entrypoint[0]) - assert script == PROD_ENTRYPOINT, f"{BUILD_FROM_PIP_DOCKERFILE} bypasses the ddtrace-aware entrypoint" - assert f"chmod +x {entrypoint[0]}" in BUILD_FROM_PIP_DOCKERFILE.read_text() - - cmd = _cmd_argv(BUILD_FROM_PIP_DOCKERFILE) - recorded = _run_entrypoint(script, cmd, use_ddtrace=use_ddtrace, tmp_path=tmp_path) - - cmd_str = " ".join(cmd) - assert recorded == ( - "exec=ddtrace-run" if traced else "exec=litellm", - f"args=litellm {cmd_str}" if traced else f"args={cmd_str}", - "DD_TRACE_OPENAI_ENABLED=False" if traced else "DD_TRACE_OPENAI_ENABLED=", - f"PYTHONPATH={PYTHONPATH_SENTINEL}", - ) - - -def test_entrypoint_script_is_executable() -> None: - mode = COMPONENT_ENTRYPOINT.stat().st_mode - assert mode & stat.S_IXUSR, "entrypoint must be committed executable to run as the image ENTRYPOINT" - assert mode & stat.S_IXOTH, "entrypoint must be executable by the unprivileged `nonroot` user" - - -def test_entrypoint_script_has_no_carriage_returns() -> None: - assert b"\r" not in COMPONENT_ENTRYPOINT.read_bytes() - - -@pytest.mark.parametrize( - "dockerfile, launcher", - [ - (GATEWAY_DOCKERFILE, "python -m gateway.launch"), - (BACKEND_DOCKERFILE, "uvicorn backend.main:app"), - ], -) -def test_component_images_launch_uvicorn_through_the_entrypoint(dockerfile: Path, launcher: str) -> None: - entrypoint = " ".join(_entrypoint_argv(dockerfile)) - - assert IMAGE_ENTRYPOINT_PATH in entrypoint, f"{dockerfile} bypasses the ddtrace-aware entrypoint" - assert launcher in entrypoint - assert entrypoint.index(IMAGE_ENTRYPOINT_PATH) < entrypoint.index(launcher), ( - f"{dockerfile} must invoke uvicorn through the entrypoint, not the other way around" - ) - - -@pytest.mark.parametrize( - "use_ddtrace, num_workers, expected_exec, expected_args", - [ - (None, "4", "exec=python", "args=-m gateway.launch --workers 4 --host 0.0.0.0 --port 4000"), - (None, None, "exec=python", "args=-m gateway.launch --workers 1 --host 0.0.0.0 --port 4000"), - ("true", "4", "exec=ddtrace-run", "args=python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000"), - ], -) -def test_gateway_image_execs_the_supervisor_with_its_worker_count( - use_ddtrace: str | None, num_workers: str | None, expected_exec: str, expected_args: str, tmp_path: Path -) -> None: - """Run the gateway image's ENTRYPOINT + CMD and record what the container execs. - - The Dockerfile's `/app/...` script path is resolved to the checked-in script and `python` - is stubbed on PATH, so the assertion is on the argv `gateway.launch` receives, not on the - Dockerfile text. - """ - entrypoint = tuple( - part.replace(IMAGE_ENTRYPOINT_PATH, str(COMPONENT_ENTRYPOINT)) for part in _entrypoint_argv(GATEWAY_DOCKERFILE) - ) - bin_dir = tmp_path / "bin" - bin_dir.mkdir(parents=True) - _write_stubs(bin_dir, ("ddtrace-run", "python", "uvicorn")) - record = tmp_path / "record.txt" - overrides = {"USE_DDTRACE": use_ddtrace, "NUM_WORKERS": num_workers} - env = { - **{k: v for k, v in os.environ.items() if k not in ("DD_TRACE_OPENAI_ENABLED", *overrides)}, - **{k: v for k, v in overrides.items() if v is not None}, - "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", - "RECORD": str(record), - "PYTHONPATH": PYTHONPATH_SENTINEL, - } - - result = subprocess.run( - [*entrypoint, *_cmd_argv(GATEWAY_DOCKERFILE)], env=env, capture_output=True, text=True, check=False - ) - - assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" - assert tuple(record.read_text().splitlines())[:2] == (expected_exec, expected_args) - - -@pytest.mark.parametrize("dockerfile", [GATEWAY_DOCKERFILE, BACKEND_DOCKERFILE]) -def test_component_images_make_the_entrypoint_executable(dockerfile: Path) -> None: - body = dockerfile.read_text() - assert "chmod +x docker/component_entrypoint.sh" in body - - -@pytest.mark.parametrize("terraform_file", TERRAFORM_LAUNCH_SITES, ids=lambda p: p.parent.name) -@pytest.mark.parametrize("component", ["gateway", "backend"]) -@pytest.mark.parametrize("use_ddtrace", [*TRUTHY_USE_DDTRACE, *FALSY_USE_DDTRACE]) -def test_terraform_launch_command_matches_the_script_contract( - terraform_file: Path, component: str, use_ddtrace: str | None, tmp_path: Path -) -> None: - """The Terraform command and `docker/component_entrypoint.sh` must decide identically. - - The decision deliberately lives in two places. The script is what the image ENTRYPOINT runs; - the Terraform strings are what runs when a deployment overrides that ENTRYPOINT, and they - cannot call the script because the caller supplies the image tag and it may predate the file. - Both modules default to a tag that does. So instead of asserting a shared path, this runs both - implementations under the same environment and asserts they agree on which binary is exec'd - and on whether the openai integration is disabled. - """ - launcher = COMPONENT_LAUNCHERS[component] - app_target = " ".join(launcher[1:]) - command = _resolve_tf_local(terraform_file, f"{component}_launch_cmd") - - bin_dir = tmp_path / "bin" - bin_dir.mkdir(parents=True) - _write_stubs(bin_dir, ("ddtrace-run", "uvicorn", "python")) - from_terraform = _run_shell_command(command, bin_dir, tmp_path / "terraform.txt", use_ddtrace) - - from_script = _run_entrypoint( - COMPONENT_ENTRYPOINT, - launcher, - use_ddtrace=use_ddtrace, - tmp_path=tmp_path / "script", - ) - - assert from_terraform[0] == from_script[0], ( - f"{terraform_file} disagrees with the script on USE_DDTRACE={use_ddtrace}" - ) - assert from_terraform[2] == from_script[2], f"{terraform_file} disagrees with the script on the openai integration" - assert app_target in from_terraform[1] - assert "gateway.main:app" not in from_terraform[1], f"{terraform_file} bypasses the gateway.launch supervisor" - - if use_ddtrace in TRUTHY_USE_DDTRACE: - assert from_terraform[0] == "exec=ddtrace-run" - assert from_terraform[1].startswith(f"args={launcher[0]} ") - assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=False" - else: - assert from_terraform[0] == f"exec={launcher[0]}" - assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=" - - -@pytest.mark.parametrize("terraform_file", TERRAFORM_LAUNCH_SITES, ids=lambda p: p.parent.name) -def test_terraform_routes_every_launch_site_through_a_traced_command(terraform_file: Path) -> None: - """Every place Terraform names a component ASGI target has to honor `USE_DDTRACE`. - - The count is pinned as well: a launch site that is deleted or renamed would otherwise drop - out of the scan and let this pass while covering less than it claims. - """ - launch_sites = tuple(line for line in terraform_file.read_text().splitlines() if _APP_TARGET_RE.search(line)) - - assert len(launch_sites) == TERRAFORM_LAUNCH_SITES[terraform_file], ( - f"{terraform_file} launch-site count changed; re-check each one honors USE_DDTRACE" - ) - for line in launch_sites: - assert "ddtrace-run" in line, f"{terraform_file} launches uvicorn without honoring USE_DDTRACE: {line.strip()}" - - body = terraform_file.read_text() - for component in ("gateway", "backend"): - assert f"local.{component}_launch_cmd" in body, ( - f"{terraform_file} defines a {component} launch command but never uses it" - ) - - -@pytest.mark.parametrize("terraform_file", TERRAFORM_LAUNCH_SITES, ids=lambda p: p.parent.name) -def test_terraform_does_not_depend_on_the_entrypoint_script(terraform_file: Path) -> None: - """Terraform must not reference a file the caller-supplied image may not contain. - - Both modules default to an image tag published before the script existed, so exec'ing that - path would fail at container start rather than degrade to an untraced process. - """ - assert IMAGE_ENTRYPOINT_PATH not in terraform_file.read_text(), ( - f"{terraform_file} depends on a script that images predating it do not ship" - ) - - -def test_gateway_keeps_its_worker_count_and_backend_keeps_a_single_process() -> None: - gateway = " ".join(_entrypoint_argv(GATEWAY_DOCKERFILE)) + " " + " ".join(_cmd_argv(GATEWAY_DOCKERFILE)) - backend = " ".join(_entrypoint_argv(BACKEND_DOCKERFILE)) + " " + " ".join(_cmd_argv(BACKEND_DOCKERFILE)) - - assert "--workers" in gateway and "NUM_WORKERS" in gateway - assert "--workers" not in backend and "NUM_WORKERS" not in backend diff --git a/tests/unit/test_docker_entrypoint.py b/tests/unit/test_docker_entrypoint.py new file mode 100644 index 00000000000..1d8da4a9b62 --- /dev/null +++ b/tests/unit/test_docker_entrypoint.py @@ -0,0 +1,557 @@ +import json +import os +import re +import shlex +import stat +import subprocess +from pathlib import Path +from typing import Final + +import pytest + +from litellm.constants import PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX + +REPO_ROOT: Final = Path(__file__).resolve().parents[2] +DOCKER_ENTRYPOINT: Final = REPO_ROOT / "docker-entrypoint.sh" +DOCKERFILE: Final = REPO_ROOT / "Dockerfile" +TERRAFORM_ECS: Final = REPO_ROOT / "terraform" / "litellm" / "aws" / "ecs.tf" +TERRAFORM_CLOUDRUN: Final = REPO_ROOT / "terraform" / "litellm" / "gcp" / "cloudrun.tf" + +IMAGE_ENTRYPOINT_PATH: Final = "/app/docker-entrypoint.sh" +STUBBED_EXECUTABLES: Final = ("ddtrace-run", "uvicorn", "python", "litellm", "nginx") + +TRUTHY_USE_DDTRACE: Final = ("true", "True", "TRUE", "tRuE") +FALSY_USE_DDTRACE: Final = (None, "", "false", "False", "1", "yes", "on", "truex") + +PYTHONPATH_SENTINEL: Final = "/lit-entrypoint-sentinel:/app" + +_STUB_TEMPLATE: Final = """#!/bin/sh +{{ + echo "exec={name}" + echo "args=$*" + echo "DD_TRACE_OPENAI_ENABLED=${{DD_TRACE_OPENAI_ENABLED-}}" + echo "PYTHONPATH=${{PYTHONPATH-}}" +}} >> "$RECORD" +""" + +_ENTRYPOINT_RE: Final = re.compile(r"^ENTRYPOINT\s+(\[.*\])\s*$", re.MULTILINE) +_CMD_RE: Final = re.compile(r"^CMD\s+(\[.*\])\s*$", re.MULTILINE) +_APP_TARGET_RE: Final = re.compile(r"(?:gateway|backend)\.main:app|gateway\.launch") +_TF_STRING_LOCAL_RE: Final = re.compile(r'^\s*(\w+)\s*=\s*"((?:[^"\\]|\\.)*)"\s*$', re.MULTILINE) +_TF_INTERPOLATION_RE: Final = re.compile(r"\$\{(local|var)\.(\w+)\}") + +TERRAFORM_LAUNCH_SITES: Final = {TERRAFORM_ECS: 2, TERRAFORM_CLOUDRUN: 2} +TERRAFORM_VAR_STUBS: Final = {"gateway_num_workers": "2"} +COMPONENT_LAUNCHERS: Final = { + "gateway": ("python", "-m", "gateway.launch"), + "backend": ("uvicorn", "backend.main:app"), +} +_MAX_INTERPOLATION_PASSES: Final = 5 + + +def _write_stubs(bin_dir: Path, names: tuple[str, ...]) -> None: + for name in names: + stub = bin_dir / name + stub.write_text(_STUB_TEMPLATE.format(name=name)) + stub.chmod(0o755) + + +def _entrypoint_env(bin_dir: Path, record: Path, overrides: dict[str, str | None]) -> dict[str, str]: + cleared = ("USE_DDTRACE", "DD_TRACE_OPENAI_ENABLED", "LITELLM_COMPONENT", "NUM_WORKERS", *overrides) + return { + **{k: v for k, v in os.environ.items() if k not in cleared}, + **{k: v for k, v in overrides.items() if v is not None}, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "RECORD": str(record), + "PYTHONPATH": PYTHONPATH_SENTINEL, + } + + +def _run_entrypoint( + argv: tuple[str, ...], + tmp_path: Path, + use_ddtrace: str | None = None, + *, + entrypoint: Path = DOCKER_ENTRYPOINT, + **overrides: str | None, +) -> tuple[str, ...]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + _write_stubs(bin_dir, STUBBED_EXECUTABLES) + record = tmp_path / "record.txt" + env = _entrypoint_env(bin_dir, record, {"USE_DDTRACE": use_ddtrace, **overrides}) + + result = subprocess.run(["sh", str(entrypoint), *argv], env=env, capture_output=True, text=True, check=False) + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + return tuple(record.read_text().splitlines()) if record.exists() else () + + +def _entrypoint_with_runtime_config(tmp_path: Path) -> tuple[Path, Path]: + entrypoint: Final = tmp_path / "docker-entrypoint.sh" + entrypoint.write_text(DOCKER_ENTRYPOINT.read_text()) + hook: Final = tmp_path / "ui" / "docker-entrypoint.d" / "40-litellm-runtime-config.sh" + hook.parent.mkdir(parents=True) + runtime_dir: Final = tmp_path / "runtime" + runtime_dir.mkdir() + hook.write_text( + (REPO_ROOT / "ui" / "docker-entrypoint.d" / hook.name) + .read_text() + .replace("runtime_dir=/tmp/litellm-ui", f"runtime_dir={shlex.quote(str(runtime_dir))}") + ) + return entrypoint, runtime_dir / "runtime-config.json" + + +@pytest.mark.parametrize("backend_url", ("https://backend.example.com///", "http://backend.example.com", None, "", "invalid")) +def test_ui_runtime_config_is_refreshed_before_nginx_starts(tmp_path: Path, backend_url: str | None) -> None: + entrypoint, runtime_file = _entrypoint_with_runtime_config(tmp_path) + runtime_file.write_text('{"backend_url":"https://stale.example.com"}') + recorded: Final = _run_entrypoint(("ui",), tmp_path, entrypoint=entrypoint, LITELLM_BACKEND_URL=backend_url) + assert recorded[0] == "exec=nginx" + assert (json.loads(runtime_file.read_text()) if runtime_file.exists() else None) == ( + {"backend_url": backend_url.rstrip("/")} if backend_url and backend_url.startswith(("https://", "http://")) else None + ) + + +def test_backend_leaves_ui_runtime_config_unchanged(tmp_path: Path) -> None: + entrypoint, runtime_file = _entrypoint_with_runtime_config(tmp_path) + runtime_file.write_text('{"backend_url":"https://stale.example.com"}') + recorded: Final = _run_entrypoint( + ("backend",), tmp_path, entrypoint=entrypoint, LITELLM_BACKEND_URL="https://new.example.com" + ) + assert recorded[0] == "exec=uvicorn" + assert json.loads(runtime_file.read_text()) == {"backend_url": "https://stale.example.com"} + + +def _run_shell_command(command: str, bin_dir: Path, record: Path, use_ddtrace: str | None) -> tuple[str, ...]: + env = _entrypoint_env(bin_dir, record, {"USE_DDTRACE": use_ddtrace}) + result = subprocess.run(["sh", "-c", command], env=env, capture_output=True, text=True, check=False) + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + return tuple(record.read_text().splitlines()) if record.exists() else () + + +def _resolve_tf_local(terraform_file: Path, name: str) -> str: + values = { + m.group(1): m.group(2).replace('\\"', '"') for m in _TF_STRING_LOCAL_RE.finditer(terraform_file.read_text()) + } + + assert name in values, f"{terraform_file} defines no {name} local" + resolved = values[name] + for _ in range(_MAX_INTERPOLATION_PASSES): + if "${" not in resolved: + return resolved + resolved = _TF_INTERPOLATION_RE.sub( + lambda m: values[m.group(2)] if m.group(1) == "local" else TERRAFORM_VAR_STUBS[m.group(2)], + resolved, + ) + raise AssertionError(f"{terraform_file}:{name} still has unresolved interpolations: {resolved}") + + +def _entrypoint_argv(dockerfile: Path) -> tuple[str, ...]: + matches = _ENTRYPOINT_RE.findall(dockerfile.read_text()) + assert matches, f"no exec-form ENTRYPOINT found in {dockerfile}" + return tuple(str(part) for part in json.loads(matches[-1])) + + +@pytest.mark.parametrize( + "argv, expected_exec, expected_args", + [ + (("proxy", "--config", "/app/config.yaml"), "exec=litellm", "args=--config /app/config.yaml"), + (("gateway",), "exec=python", "args=-m gateway.launch --workers 1 --host 0.0.0.0 --port 4000"), + ( + ("gateway", "--port", "8080"), + "exec=python", + "args=-m gateway.launch --workers 1 --host 0.0.0.0 --port 4000 --port 8080", + ), + (("backend",), "exec=uvicorn", "args=backend.main:app --host 0.0.0.0 --port 4001"), + (("backend", "--port", "9001"), "exec=uvicorn", "args=backend.main:app --host 0.0.0.0 --port 4001 --port 9001"), + (("ui",), "exec=nginx", "args=-g daemon off;"), + (("migrations",), "exec=python", "args=/app/migrations/run.py"), + (("metrics", "--port", "9090"), "exec=python", "args=-m litellm.proxy.prometheus_metrics_server --port 9090"), + (("collector",), "exec=python", "args=-m litellm.proxy.collector"), + ], +) +def test_the_first_argument_selects_the_component( + argv: tuple[str, ...], expected_exec: str, expected_args: str, tmp_path: Path +) -> None: + recorded = _run_entrypoint(argv, tmp_path) + + assert recorded[:2] == (expected_exec, expected_args) + + +@pytest.mark.parametrize( + "argv, expected", + [ + ((), ("exec=litellm", "args=")), + (("--port", "4000"), ("exec=litellm", "args=--port 4000")), + ( + ("--config", "/app/config.yaml", "--detailed_debug"), + ("exec=litellm", "args=--config /app/config.yaml --detailed_debug"), + ), + ], +) +def test_flags_alone_still_run_the_monolithic_proxy( + argv: tuple[str, ...], expected: tuple[str, str], tmp_path: Path +) -> None: + assert _run_entrypoint(argv, tmp_path)[:2] == expected + + +def test_the_dockerfile_leaves_the_command_empty_so_the_env_var_can_pick_the_component(tmp_path: Path) -> None: + dockerfile_text = DOCKERFILE.read_text() + assert _entrypoint_argv(DOCKERFILE) == (IMAGE_ENTRYPOINT_PATH,), "the image ENTRYPOINT must be the bare dispatcher" + assert not _CMD_RE.search(dockerfile_text), "the Dockerfile must not set a CMD" + + assert _run_entrypoint((), tmp_path)[:2] == ("exec=litellm", "args=") + + +@pytest.mark.parametrize( + "component, argv, expected_exec, expected_args", + [ + ("backend", (), "exec=uvicorn", "args=backend.main:app --host 0.0.0.0 --port 4001"), + ( + "gateway", + ("--port", "4100"), + "exec=python", + "args=-m gateway.launch --workers 1 --host 0.0.0.0 --port 4000 --port 4100", + ), + ("ui", (), "exec=nginx", "args=-g daemon off;"), + ], +) +def test_litellm_component_env_selects_the_component_when_the_command_has_none( + component: str, argv: tuple[str, ...], expected_exec: str, expected_args: str, tmp_path: Path +) -> None: + recorded = _run_entrypoint(argv, tmp_path, LITELLM_COMPONENT=component) + + assert recorded[:2] == (expected_exec, expected_args) + + +def test_an_explicit_component_argument_beats_the_env_var(tmp_path: Path) -> None: + recorded = _run_entrypoint(("backend",), tmp_path, LITELLM_COMPONENT="gateway") + + assert recorded[0] == "exec=uvicorn" + + +def test_the_gateway_honours_num_workers(tmp_path: Path) -> None: + recorded = _run_entrypoint(("gateway",), tmp_path, NUM_WORKERS="4") + + assert recorded[1] == "args=-m gateway.launch --workers 4 --host 0.0.0.0 --port 4000" + + +def test_an_unknown_first_word_is_run_as_the_command(tmp_path: Path) -> None: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_stubs(bin_dir, ("some-tool",)) + record = tmp_path / "record.txt" + env = _entrypoint_env(bin_dir, record, {}) + + result = subprocess.run( + ["sh", str(DOCKER_ENTRYPOINT), "some-tool", "--flag"], env=env, capture_output=True, text=True, check=False + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + assert record.read_text().splitlines()[:2] == ["exec=some-tool", "args=--flag"] + + +@pytest.mark.parametrize( + "use_ddtrace, traced", + [*((v, True) for v in TRUTHY_USE_DDTRACE), *((v, False) for v in FALSY_USE_DDTRACE)], +) +def test_ddtrace_wraps_a_raw_command_like_a_component(use_ddtrace: str | None, traced: bool, tmp_path: Path) -> None: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_stubs(bin_dir, ("some-tool", "ddtrace-run")) + record = tmp_path / "record.txt" + env = _entrypoint_env(bin_dir, record, {"USE_DDTRACE": use_ddtrace}) + + result = subprocess.run( + ["sh", str(DOCKER_ENTRYPOINT), "some-tool", "--flag"], env=env, capture_output=True, text=True, check=False + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + assert record.read_text().splitlines()[:2] == ( + ["exec=ddtrace-run", "args=some-tool --flag"] if traced else ["exec=some-tool", "args=--flag"] + ) + + +def test_an_unknown_litellm_component_fails_fast_instead_of_guessing(tmp_path: Path) -> None: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_stubs(bin_dir, STUBBED_EXECUTABLES) + record = tmp_path / "record.txt" + env = _entrypoint_env(bin_dir, record, {"LITELLM_COMPONENT": "gatway"}) + + result = subprocess.run(["sh", str(DOCKER_ENTRYPOINT)], env=env, capture_output=True, text=True, check=False) + + assert result.returncode == 64 + assert "gatway" in result.stderr + assert not record.exists(), "nothing may be exec'd when the component is unknown" + + +def test_admin_agent_flag_runs_the_bundled_liteadmin_agent_not_the_proxy(tmp_path: Path) -> None: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_stubs(bin_dir, STUBBED_EXECUTABLES) + record = tmp_path / "record.txt" + env = _entrypoint_env(bin_dir, record, {}) + + result = subprocess.run( + ["sh", str(DOCKER_ENTRYPOINT), "--admin-agent", "--flag"], env=env, capture_output=True, text=True, check=False + ) + + assert "/opt/liteadmin/bin/litellm-admin-agent" in result.stderr + assert result.returncode == 127 + assert not record.exists(), "--admin-agent must not fall through to the proxy" + + +def test_ddtrace_enabled_wraps_the_command_and_disables_the_openai_integration(tmp_path: Path) -> None: + recorded = _run_entrypoint(("gateway",), tmp_path, use_ddtrace="true", NUM_WORKERS="2") + + assert recorded == ( + "exec=ddtrace-run", + "args=python -m gateway.launch --workers 2 --host 0.0.0.0 --port 4000", + "DD_TRACE_OPENAI_ENABLED=False", + f"PYTHONPATH={PYTHONPATH_SENTINEL}", + ) + + +def test_ddtrace_disabled_execs_the_command_directly(tmp_path: Path) -> None: + recorded = _run_entrypoint(("backend", "--workers", "2"), tmp_path) + + assert recorded == ( + "exec=uvicorn", + "args=backend.main:app --host 0.0.0.0 --port 4001 --workers 2", + "DD_TRACE_OPENAI_ENABLED=", + f"PYTHONPATH={PYTHONPATH_SENTINEL}", + ) + + +@pytest.mark.parametrize( + "use_ddtrace, traced", + [*((v, True) for v in TRUTHY_USE_DDTRACE), *((v, False) for v in FALSY_USE_DDTRACE)], +) +def test_ddtrace_gating_matches_get_secret_bool_for_every_component( + use_ddtrace: str | None, traced: bool, tmp_path: Path +) -> None: + component = _run_entrypoint(("gateway",), tmp_path / "component", use_ddtrace=use_ddtrace) + monolith = _run_entrypoint(("--port", "4000"), tmp_path / "monolith", use_ddtrace=use_ddtrace) + + expected_openai = "DD_TRACE_OPENAI_ENABLED=False" if traced else "DD_TRACE_OPENAI_ENABLED=" + assert component[0] == ("exec=ddtrace-run" if traced else "exec=python") + assert component[2] == expected_openai + assert monolith[0] == ("exec=ddtrace-run" if traced else "exec=litellm") + assert monolith[1] == ("args=litellm --port 4000" if traced else "args=--port 4000") + assert monolith[2] == expected_openai + + +def test_wipes_the_prometheus_multiproc_dir_before_uvicorn_forks(tmp_path: Path) -> None: + multiproc_dir = tmp_path / "multiproc" + multiproc_dir.mkdir() + (multiproc_dir / "gauge_livesum_7.db").write_bytes(b"stale") + (multiproc_dir / "counter_7.db").write_bytes(b"stale") + (multiproc_dir / f"{PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX}litellm_requests_metric").write_bytes(b"stale") + (multiproc_dir / "keep.txt").write_text("not a sample") + + recorded = _run_entrypoint(("gateway",), tmp_path, PROMETHEUS_MULTIPROC_DIR=str(multiproc_dir)) + + assert sorted(p.name for p in multiproc_dir.iterdir()) == ["keep.txt"] + assert recorded[0] == "exec=python" + + +@pytest.mark.parametrize( + "argv", + [ + ("python", "-m", "litellm.proxy.prometheus_metrics_server"), + ("python", "-m", "litellm.proxy.collector"), + ("sh", "-c", "exec python -m litellm.proxy.collector --port 4010"), + ], +) +def test_a_raw_metrics_reader_keeps_the_workers_samples(argv: tuple[str, ...], tmp_path: Path) -> None: + multiproc_dir = tmp_path / "multiproc" + multiproc_dir.mkdir() + (multiproc_dir / "counter_7.db").write_bytes(b"live") + (multiproc_dir / f"{PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX}litellm_requests_metric").write_bytes(b"live") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_stubs(bin_dir, ("python",)) + record = tmp_path / "record.txt" + env = _entrypoint_env(bin_dir, record, {"PROMETHEUS_MULTIPROC_DIR": str(multiproc_dir)}) + + result = subprocess.run( + ["sh", str(DOCKER_ENTRYPOINT), *argv], + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + assert sorted(p.name for p in multiproc_dir.iterdir()) == [ + "counter_7.db", + f"{PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX}litellm_requests_metric", + ] + assert record.read_text().splitlines()[0] == "exec=python" + + +@pytest.mark.parametrize( + "argv", + [ + ("python", "-m", "gateway.launch"), + ("litellm", "--config", "/app/config.yaml"), + ("uvicorn", "backend.main:app"), + ], +) +def test_a_raw_writer_starts_with_an_empty_prometheus_multiproc_dir(argv: tuple[str, ...], tmp_path: Path) -> None: + multiproc_dir = tmp_path / "multiproc" + multiproc_dir.mkdir() + (multiproc_dir / "counter_7.db").write_bytes(b"stale") + (multiproc_dir / "gauge_livesum_7.db").write_bytes(b"stale") + (multiproc_dir / f"{PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX}litellm_requests_metric").write_bytes(b"stale") + (multiproc_dir / "keep.txt").write_text("not a sample") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_stubs(bin_dir, STUBBED_EXECUTABLES) + record = tmp_path / "record.txt" + env = _entrypoint_env(bin_dir, record, {"PROMETHEUS_MULTIPROC_DIR": str(multiproc_dir)}) + + result = subprocess.run( + ["sh", str(DOCKER_ENTRYPOINT), *argv], + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + assert sorted(p.name for p in multiproc_dir.iterdir()) == ["keep.txt"] + assert record.read_text().splitlines()[0] == f"exec={argv[0]}" + + +def test_a_raw_command_still_gets_the_prometheus_multiproc_dir_created(tmp_path: Path) -> None: + multiproc_dir = tmp_path / "not-yet" / "multiproc" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + _write_stubs(bin_dir, ("python",)) + record = tmp_path / "record.txt" + env = _entrypoint_env(bin_dir, record, {"PROMETHEUS_MULTIPROC_DIR": str(multiproc_dir)}) + + result = subprocess.run( + ["sh", str(DOCKER_ENTRYPOINT), "python", "-m", "gateway.launch"], + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + assert multiproc_dir.is_dir() + assert record.read_text().splitlines()[0] == "exec=python" + + +@pytest.mark.parametrize("component", ["metrics", "collector"]) +def test_readers_of_the_prometheus_multiproc_dir_keep_the_workers_samples(component: str, tmp_path: Path) -> None: + multiproc_dir = tmp_path / "multiproc" + multiproc_dir.mkdir() + (multiproc_dir / "counter_12.db").write_bytes(b"live") + (multiproc_dir / "histogram_13.db").write_bytes(b"live") + (multiproc_dir / f"{PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX}litellm_requests_metric").write_bytes(b"live") + + recorded = _run_entrypoint((component,), tmp_path, PROMETHEUS_MULTIPROC_DIR=str(multiproc_dir)) + + assert sorted(p.name for p in multiproc_dir.iterdir()) == [ + "counter_12.db", + "histogram_13.db", + f"{PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX}litellm_requests_metric", + ] + assert recorded[0] == "exec=python" + + +def test_creates_a_missing_prometheus_multiproc_dir(tmp_path: Path) -> None: + missing = tmp_path / "multiproc" + + _run_entrypoint(("backend",), tmp_path, PROMETHEUS_MULTIPROC_DIR=str(missing)) + + assert missing.is_dir() + + +@pytest.mark.parametrize("terraform_file", TERRAFORM_LAUNCH_SITES, ids=lambda p: p.parent.name) +@pytest.mark.parametrize("component", ["gateway", "backend"]) +@pytest.mark.parametrize("use_ddtrace", [*TRUTHY_USE_DDTRACE, *FALSY_USE_DDTRACE]) +def test_terraform_launch_command_matches_the_script_contract( + terraform_file: Path, component: str, use_ddtrace: str | None, tmp_path: Path +) -> None: + launcher = COMPONENT_LAUNCHERS[component] + app_target = " ".join(launcher[1:]) + command = _resolve_tf_local(terraform_file, f"{component}_launch_cmd") + + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + _write_stubs(bin_dir, ("ddtrace-run", "uvicorn", "python")) + from_terraform = _run_shell_command(command, bin_dir, tmp_path / "terraform.txt", use_ddtrace) + + from_script = _run_entrypoint((component,), tmp_path / "script", use_ddtrace=use_ddtrace) + + assert from_terraform[0] == from_script[0], ( + f"{terraform_file} disagrees with the script on USE_DDTRACE={use_ddtrace}" + ) + assert from_terraform[2] == from_script[2], f"{terraform_file} disagrees with the script on the openai integration" + assert app_target in from_terraform[1] + assert app_target in from_script[1] + assert "gateway.main:app" not in from_terraform[1], f"{terraform_file} bypasses the gateway.launch supervisor" + + if use_ddtrace in TRUTHY_USE_DDTRACE: + assert from_terraform[0] == "exec=ddtrace-run" + assert from_terraform[1].startswith(f"args={launcher[0]} ") + assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=False" + else: + assert from_terraform[0] == f"exec={launcher[0]}" + assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=" + + +@pytest.mark.parametrize("terraform_file", TERRAFORM_LAUNCH_SITES, ids=lambda p: p.parent.name) +def test_terraform_routes_every_launch_site_through_a_traced_command(terraform_file: Path) -> None: + launch_sites = tuple(line for line in terraform_file.read_text().splitlines() if _APP_TARGET_RE.search(line)) + + assert len(launch_sites) == TERRAFORM_LAUNCH_SITES[terraform_file], ( + f"{terraform_file} launch-site count changed; re-check each one honors USE_DDTRACE" + ) + for line in launch_sites: + assert "ddtrace-run" in line, f"{terraform_file} launches uvicorn without honoring USE_DDTRACE: {line.strip()}" + + body = terraform_file.read_text() + for component in ("gateway", "backend"): + assert f"local.{component}_launch_cmd" in body, ( + f"{terraform_file} defines a {component} launch command but never uses it" + ) + + +@pytest.mark.parametrize("terraform_file", TERRAFORM_LAUNCH_SITES, ids=lambda p: p.parent.name) +def test_terraform_does_not_depend_on_the_entrypoint_script(terraform_file: Path) -> None: + assert IMAGE_ENTRYPOINT_PATH not in terraform_file.read_text(), ( + f"{terraform_file} depends on a script that images predating it do not ship" + ) + + +def test_help_prints_the_header_comment_block_without_shell_source(tmp_path: Path) -> None: + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + _write_stubs(bin_dir, STUBBED_EXECUTABLES) + env = _entrypoint_env(bin_dir, tmp_path / "record.txt", {}) + + result = subprocess.run( + ["sh", str(DOCKER_ENTRYPOINT), "help"], env=env, capture_output=True, text=True, check=False + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + assert "--admin-agent" in result.stderr + assert "collector" in result.stderr + assert "set -eu" not in result.stderr + assert result.stderr.rstrip().endswith("other raw commands clear stale samples.") + + +def test_entrypoint_script_is_executable() -> None: + mode = DOCKER_ENTRYPOINT.stat().st_mode + assert mode & stat.S_IXUSR, "entrypoint must be committed executable to run as the image ENTRYPOINT" + assert mode & stat.S_IXOTH, "entrypoint must be executable by the unprivileged `nonroot` user" + + +def test_entrypoint_script_has_no_carriage_returns() -> None: + assert b"\r" not in DOCKER_ENTRYPOINT.read_bytes() diff --git a/tests/unit/test_dockerfile_bedrock_realtime_extra.py b/tests/unit/test_dockerfile_bedrock_realtime_extra.py index e157c982105..f4863b5ff77 100644 --- a/tests/unit/test_dockerfile_bedrock_realtime_extra.py +++ b/tests/unit/test_dockerfile_bedrock_realtime_extra.py @@ -1,5 +1,5 @@ """ -Static checks that every proxy Docker image installs the `bedrock-realtime` extra. +Static checks that the shipped Docker image installs the `bedrock-realtime` extra. Bedrock Nova Sonic speech-to-speech (`/v1/realtime`) needs `aws-sdk-bedrock-runtime`, which only ships in the `bedrock-realtime` extra. An image whose `uv sync` stages @@ -23,12 +23,7 @@ else: REPO_ROOT: Final = os.path.join(os.path.dirname(__file__), "..", "..") -PROXY_DOCKERFILES: Final = ( - "Dockerfile", - os.path.join("docker", "Dockerfile.non_root"), - os.path.join("docker", "Dockerfile.database"), - os.path.join("gateway", "Dockerfile"), -) +PROXY_DOCKERFILES: Final = ("Dockerfile",) CONTINUED_LINE_RE: Final = re.compile(r"(?:\\\n|[^\n])+") UV_SYNC_BOUNDARY_RE: Final = re.compile(r"(?=uv sync)") diff --git a/tests/unit/test_dockerfile_non_root.py b/tests/unit/test_dockerfile_non_root.py index 694da6368e7..d37d06c337d 100644 --- a/tests/unit/test_dockerfile_non_root.py +++ b/tests/unit/test_dockerfile_non_root.py @@ -1,39 +1,26 @@ """ -Static checks on docker/Dockerfile.non_root. +Static checks on the shipped Dockerfile's runtime user. -The non_root image is intended for deployment into hardened Kubernetes -clusters where `securityContext.runAsNonRoot: true` is enforced. The -kubelet validates non-root status by parsing the image's USER field as -an integer — a string name like "nobody" is rejected with -CreateContainerConfigError because the kubelet cannot resolve -/etc/passwd inside the image at admission time. +The image is deployed into hardened Kubernetes clusters where +`securityContext.runAsNonRoot: true` is enforced. The kubelet validates +non-root status by parsing the image's USER field as an integer: a string +name like "nonroot" is rejected with CreateContainerConfigError because the +kubelet cannot resolve /etc/passwd inside the image at admission time. """ import os import re -import pytest - -DOCKERFILE_PATH = os.path.join( - os.path.dirname(__file__), - "..", - "..", - "docker", - "Dockerfile.non_root", -) +DOCKERFILE_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "Dockerfile") def _final_user_directive(dockerfile_text: str) -> str: - """Return the value of the last `USER` directive in the file.""" + """Return the uid of the last `USER` directive in the file (`USER uid` or `USER uid:gid`).""" matches = re.findall(r"^USER\s+(\S+)\s*$", dockerfile_text, re.MULTILINE) - assert matches, "Dockerfile.non_root has no USER directive" - return matches[-1] + assert matches, "Dockerfile has no USER directive" + return matches[-1].split(":", 1)[0] -@pytest.mark.skipif( - not os.path.exists(DOCKERFILE_PATH), - reason="Dockerfile.non_root not present in this checkout", -) def test_final_user_directive_is_numeric(): """The runtime USER must be a numeric UID so kubelet's runAsNonRoot admission check (strconv.Atoi) succeeds.""" @@ -43,12 +30,9 @@ def test_final_user_directive_is_numeric(): final_user = _final_user_directive(contents) assert final_user.isdigit(), ( - f"Dockerfile.non_root final USER is {final_user!r}; must be a numeric UID " + f"Dockerfile final USER is {final_user!r}; must be a numeric UID " "so Kubernetes' runAsNonRoot admission check can verify non-root status. " "See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/" ) - assert int(final_user) != 0, ( - f"Dockerfile.non_root final USER is {final_user} (root); the non_root image " - "must run as a non-zero UID." - ) + assert int(final_user) != 0, f"Dockerfile final USER is {final_user} (root); the image must run as a non-zero UID." diff --git a/ui/Dockerfile b/ui/Dockerfile deleted file mode 100644 index d78a155b874..00000000000 --- a/ui/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -# syntax=docker/dockerfile:1.7 - -# UI container — Next.js static export served by nginx. - -ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 -ARG NGINX_VERSION=1.31.8-alpine3.24@sha256:1ef7ca94d7879df82421428ce13bd1eaa412404953859cf7a5b6512c25e67a60 - -# ---------- builder ---------- -FROM ${UI_BUILD_IMAGE} AS builder - -ENV NEXT_TELEMETRY_DISABLED=1 \ - npm_config_fund=false \ - npm_config_audit=false - -WORKDIR /app - -# Layer the lockfile-only install above the source copy so source-only -# edits don't bust the install cache. -COPY ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json ./ -COPY ui/litellm-dashboard/vendor/ ./vendor/ -RUN --mount=type=cache,target=/root/.npm \ - npm ci --prefer-offline - -COPY ui/litellm-dashboard/ ./ -RUN npm run build - -# ---------- runtime ---------- -FROM nginx:${NGINX_VERSION} AS runtime - -RUN apk upgrade --no-cache brotli-libs - -# Drop the upstream default :80 server; we own the config. -RUN rm -f /etc/nginx/conf.d/default.conf - -# Static export → web root. -COPY --from=builder /app/out /usr/share/nginx/html - -# Runtime backend discovery config. -COPY ui/docker-entrypoint.d/40-litellm-runtime-config.sh /docker-entrypoint.d/ - -# Routing rules — see ui/nginx.conf for the full description. -COPY ui/nginx.conf /etc/nginx/nginx.conf - -EXPOSE 3000/tcp - -# nginx as PID 1 in foreground; respects SIGTERM out of the box, so -# no tini/dumb-init wrapper needed. -CMD ["nginx", "-g", "daemon off;"] diff --git a/ui/nginx.conf b/ui/nginx.conf index dd43defcde9..e695686c670 100644 --- a/ui/nginx.conf +++ b/ui/nginx.conf @@ -5,6 +5,7 @@ worker_processes auto; # nginx image's /var/cache/nginx and /run are root-owned 755) and works # with readOnlyRootFilesystem when /tmp is an emptyDir. pid /tmp/nginx.pid; +error_log /dev/stderr warn; events { worker_connections 1024; } @@ -15,6 +16,7 @@ http { uwsgi_temp_path /tmp/nginx-uwsgi-temp; scgi_temp_path /tmp/nginx-scgi-temp; + access_log /dev/stdout; include /etc/nginx/mime.types; default_type application/octet-stream; sendfile on; @@ -29,7 +31,6 @@ http { application/javascript application/json text/css - text/html image/svg+xml font/woff font/woff2; @@ -37,16 +38,16 @@ http { server { listen 3000 default_server; server_name _; - root /usr/share/nginx/html; + root /var/lib/litellm/ui; # next.config.mjs sets assetPrefix=/litellm-asset-prefix, which makes # the built HTML reference /litellm-asset-prefix/_next/... — but the # static export only emits files under /_next/. Map the prefix to # the real tree at request time instead of duplicating the directory # at build time. NB: alias rewrites the location prefix, so - # /litellm-asset-prefix/_next/foo.js → /usr/share/nginx/html/_next/foo.js. + # /litellm-asset-prefix/_next/foo.js -> /var/lib/litellm/ui/_next/foo.js. location /litellm-asset-prefix/_next/ { - alias /usr/share/nginx/html/_next/; + alias /var/lib/litellm/ui/_next/; expires 1y; add_header Cache-Control "public, immutable"; } @@ -63,7 +64,7 @@ http { add_header Cache-Control "public, immutable"; } location ^~ /ui/assets/ { - alias /usr/share/nginx/html/assets/; + alias /var/lib/litellm/ui/assets/; } location = /favicon.ico { try_files $uri =404; From 9ebbcc7710821fc3d245417fb3d00a06b906c114 Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:33:54 -0800 Subject: [PATCH 065/189] feat(anthropic): forward message threads on native /v1/messages (#50976) * feat(anthropic): forward message threads on native /v1/messages Keep the thread body field on native Anthropic legs, add the message-threads-2026-08-12 beta for it, reject it with a 400 on legs that cannot continue a stored thread, and relay error.details of an upstream Anthropic error so thread_not_found reaches SDK thread helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(anthropic): keep a stale thread 404 from cooling the deployment and type the thread beta ignores Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style: sort the cooldown handler imports Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): dedupe the message-threads beta map rows and annotate test finals Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): bind message threads to the key that created them and answer unsupported continues with thread_not_found Every virtual key on a proxy shares one Anthropic credential, so forwarding thread alone let any key continue another key's thread by its message id. A new default proxy hook binds each thread reply id to its virtual key, or to the signed-in identity when there is no key, and answers a continue naming an id it did not bind with the 404 thread_not_found the SDK replays on. Legs without message threads now serve a create without the thread and answer a continue with that 404 before any upstream call, instead of answering it with no history. * fix(proxy): keep a cached message thread reply with the key that got it first A response cache hit replays the original reply id, and binding it again handed the thread to the second key, so the creator's continue got thread_not_found. * fix(anthropic): keep message threads bound on every route and stable across turns Check the thread binding on every route that carries thread except pass-through, keep bindings in their own cache so they never evict rate-limit counters, add no tool or message derived betas on thread turns, answer a create on a leg without threads with a 400 thread_unsupported_request unless drop_params is set, and forward threads on Claude Platform on AWS * fix(anthropic): reject message threads on /v1/messages passthrough deployments A deployment that forwards /v1/messages untranslated to an OpenAI-compatible server inherited the native config's thread support, so `thread` reached a server that keeps no thread state. It now answers the same 400 thread_unsupported_request on create and 404 thread_not_found on continue as a translated leg * fix(anthropic): resend a thread's create betas on continues and check bound ids on pass-through A thread create now gets the betas LiteLLM derives from its tools and messages again, so an advisor tool create that answered 400 on the previous commit answers 200. Anthropic fingerprints the beta set, so the proxy keeps the anthropic-beta header the create sent upstream in the thread binding and adds it to every continue on /v1/messages. It reads that header from the upstream request, since success logging overwrites the logged request details before the success hook runs A continue on the /anthropic pass-through route of a thread the proxy bound to another key now gets the 404 thread_not_found. Ids the proxy never bound still pass through, as before * fix(anthropic): continue unbound threads off /v1/messages and keep every anthropic-beta on chat and Responses * fix(anthropic): skip non-string anthropic-beta list entries instead of raising * test(anthropic): keep the header merge regression test clear of main's appended tests * fix(anthropic): add a thread's create betas to the client's own on continues and keep base header merging on chat and Responses * fix(anthropic): answer an unbound thread id on chat and Responses with the 404 and drop the stale beta-join tests * fix(proxy): check a thread continue sent inside extra_body against its owner Azure AI promotes extra_body into the request body, so another key could continue a thread by putting thread there, past the ownership hook that read only the top-level field * fix(proxy): write a thread's first owner atomically and bind creates sent inside extra_body Two workers that both miss a reply's binding now agree on the first key stored in Redis, since the claim uses SET NX and each worker caches the stored winner. A create sent inside extra_body now binds its reply like a top-level create. The redundant Claude Platform supports_message_threads override is removed, since the provider already resolves to the native Anthropic beta list * fix(proxy): include agent_id in the key-less message thread owner * fix(proxy): tag message thread ownership Redis calls with their key family --------- Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/anthropic_beta_headers_config.json | 10 +- .../exceptions/exception_mapping_utils.py | 36 +- .../exceptions/exceptions.py | 1 + litellm/constants.py | 1 + .../llms/anthropic/messages/native/handler.py | 39 +- .../messages/native/transformation.py | 29 +- .../llms/anthropic/messages/native/utils.py | 37 ++ .../anthropic_messages/transformation.py | 3 + .../openai_like/messages/transformation.py | 3 + litellm/proxy/hooks/__init__.py | 2 + .../proxy/hooks/message_thread_ownership.py | 319 ++++++++++ litellm/proxy/utils.py | 6 + litellm/router_rust/surface/health.py | 2 + litellm/router_utils/cooldown_handlers.py | 7 + litellm/types/llms/anthropic.py | 13 + .../test_message_threads_wire.py | 413 +++++++++++++ .../test_exception_mapping_utils.py | 45 ++ .../native/test_anthropic_messages_handler.py | 79 +++ .../test_anthropic_messages_transformation.py | 130 ++++ .../test_request_optional_param_utils.py | 8 + .../bedrock/test_claude_platform_provider.py | 27 + .../hooks/test_message_thread_ownership.py | 558 ++++++++++++++++++ .../utils/proxy_logging/test_pre_call_hook.py | 1 + .../router_utils/test_cooldown_handlers.py | 69 +++ 24 files changed, 1826 insertions(+), 12 deletions(-) create mode 100644 litellm/proxy/hooks/message_thread_ownership.py create mode 100644 tests/integration/messages_endpoint/test_message_threads_wire.py create mode 100644 tests/unit/proxy/hooks/test_message_thread_ownership.py diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index bdfddb48522..1b659770181 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -21,6 +21,7 @@ "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", "mcp-client-2025-11-20": "mcp-client-2025-11-20", "mcp-client-2025-04-04": "mcp-client-2025-04-04", + "message-threads-2026-08-12": "message-threads-2026-08-12", "mcp-servers-2025-12-04": null, "oauth-2025-04-20": "oauth-2025-04-20", "output-128k-2025-02-19": "output-128k-2025-02-19", @@ -56,7 +57,6 @@ "mcp-client-2026-09-15": "mcp-client-2026-09-15", "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22", "message-batches-2024-09-24": "message-batches-2024-09-24", - "message-threads-2026-08-12": "message-threads-2026-08-12", "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21", "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26", "output-300k-2026-03-24": "output-300k-2026-03-24", @@ -98,6 +98,7 @@ "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", "mcp-client-2025-11-20": "mcp-client-2025-11-20", "mcp-client-2025-04-04": "mcp-client-2025-04-04", + "message-threads-2026-08-12": "message-threads-2026-08-12", "mcp-servers-2025-12-04": null, "output-128k-2025-02-19": "output-128k-2025-02-19", "structured-output-2024-03-01": null, @@ -132,7 +133,6 @@ "mcp-client-2026-09-15": "mcp-client-2026-09-15", "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22", "message-batches-2024-09-24": "message-batches-2024-09-24", - "message-threads-2026-08-12": "message-threads-2026-08-12", "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01", "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21", "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", @@ -177,6 +177,7 @@ "interleaved-thinking-2025-05-14": null, "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, + "message-threads-2026-08-12": null, "mcp-servers-2025-12-04": null, "output-128k-2025-02-19": null, "structured-output-2024-03-01": null, @@ -212,6 +213,7 @@ "interleaved-thinking-2025-05-14": null, "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, + "message-threads-2026-08-12": null, "mcp-servers-2025-12-04": null, "output-128k-2025-02-19": null, "structured-output-2024-03-01": null, @@ -249,6 +251,7 @@ "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14", "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", "mcp-client-2025-04-04": null, + "message-threads-2026-08-12": null, "mcp-client-2025-11-20": null, "mcp-servers-2025-12-04": null, "output-128k-2025-02-19": "output-128k-2025-02-19", @@ -286,6 +289,7 @@ "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, + "message-threads-2026-08-12": null, "mcp-servers-2025-12-04": null, "output-128k-2025-02-19": "output-128k-2025-02-19", "structured-output-2024-03-01": null, @@ -321,7 +325,6 @@ "mcp-client-2026-09-15": null, "mcp-tunnels-2026-06-22": null, "message-batches-2024-09-24": null, - "message-threads-2026-08-12": null, "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01", "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21", "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", @@ -386,6 +389,7 @@ "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, + "message-threads-2026-08-12": null, "mcp-servers-2025-12-04": null, "oauth-2025-04-20": null, "output-128k-2025-02-19": "output-128k-2025-02-19", diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index b4c95fafff3..058ac4dcf4c 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -5,15 +5,18 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop """ import json +from collections.abc import Mapping from typing import Final -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType _PARSED_ERROR_BODY: Final = TypeAdapter(object) +_ERROR_DETAILS: Final = TypeAdapter(Mapping[str, object]) +_JSON_DECODER: Final = json.JSONDecoder() # HTTP status code -> Anthropic error type # Source: https://platform.claude.com/docs/en/api/errors @@ -166,12 +169,41 @@ class AnthropicExceptionMapping: else: message = raw_message - return AnthropicExceptionMapping.create_error_response( + response: Final = AnthropicExceptionMapping.create_error_response( status_code=status_code, message=message, request_id=request_id, error_type=error_type, ) + details: Final = AnthropicExceptionMapping.embedded_error_details(raw_message) + if details is None: + return response + with_details: Final[AnthropicErrorResponse] = {**response, "error": {**response["error"], "details": details}} + return with_details + + @staticmethod + def embedded_error_details(raw_message: str) -> Mapping[str, object] | None: + """ + ``error.details`` of the Anthropic error body quoted inside ``raw_message``, when there is one. + + litellm wraps an upstream error as ``" - {upstream json}\n\nLiteLLM: ..."``. Anthropic + puts machine-readable codes such as ``thread_not_found`` under ``error.details``, and the SDKs + branch on them, so the code is lifted next to the wrapped message instead of staying only inside it. + """ + start: Final = raw_message.find("{") + if start < 0: + return None + try: + embedded, _ = _JSON_DECODER.raw_decode(raw_message, start) + except ValueError: + return None + try: + envelope: Final = _ERROR_DETAILS.validate_python(embedded, strict=True) + error: Final = _ERROR_DETAILS.validate_python(envelope.get("error"), strict=True) + details: Final = _ERROR_DETAILS.validate_python(error.get("details"), strict=True) + except ValidationError: + return None + return details if envelope.get("type") == "error" else None class AnthropicErrorSseFrame(str): diff --git a/litellm/anthropic_interface/exceptions/exceptions.py b/litellm/anthropic_interface/exceptions/exceptions.py index 18b3ca37a3b..71d4de0029d 100644 --- a/litellm/anthropic_interface/exceptions/exceptions.py +++ b/litellm/anthropic_interface/exceptions/exceptions.py @@ -27,6 +27,7 @@ class AnthropicErrorDetail(TypedDict): type: AnthropicErrorType message: str + details: NotRequired[ReadOnly[Mapping[str, object]]] provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]] litellm_error_type: NotRequired[ReadOnly[str]] litellm_call_id: NotRequired[ReadOnly[str]] diff --git a/litellm/constants.py b/litellm/constants.py index 931ac4f2b81..5d0eab30b90 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2374,6 +2374,7 @@ ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS: Final = ( MAX_FILE_DOWNLOADS_PER_MINUTE_KEY, ) FILE_USAGE_MAX_TRACKED_COUNTERS: Final = 20_000 +MESSAGE_THREAD_MAX_TRACKED_OWNERS: Final = 20_000 # Shared read-only empty mapping, for defaulting optional Mapping parameters without # constructing a fresh mutable dict at each call site. diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 59ffb6fef04..08c3e0b48a8 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -15,10 +15,11 @@ from typing import Final, cast from pydantic import TypeAdapter, ValidationError import litellm +from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import aload_beta_headers_config from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.reasoning_effort_capability import ( @@ -29,6 +30,7 @@ from litellm.litellm_core_utils.reasoning_effort_capability import ( resolve_deployment_reasoning_effort, ) from litellm.llms.anthropic.common_utils import ( + AnthropicError, AnthropicModelInfo, strip_provider_specific_fields_from_anthropic_messages, tools_marked_for_a_tool_config_point, @@ -58,7 +60,14 @@ from ..safeguards import ( with_safeguard_results, ) from .interceptors import get_messages_interceptors -from .utils import AnthropicMessagesRequestUtils, mock_response, prepare_native_messages +from .utils import ( + AnthropicMessagesRequestUtils, + continues_message_thread, + message_thread_not_found_body, + message_thread_unsupported_body, + mock_response, + prepare_native_messages, +) __all__ = ("anthropic_messages", "anthropic_messages_handler") @@ -769,6 +778,32 @@ def anthropic_messages_handler( anthropic_messages_provider_config: Final = _native_messages_config( model, custom_llm_provider, kwargs.get("model_info"), api_base ) + thread: Final[object] = kwargs.get("thread") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # **kwargs of the handler is untyped + if thread is not None and ( + anthropic_messages_provider_config is None or not anthropic_messages_provider_config.supports_message_threads() + ): + if continues_message_thread(thread): + raise AnthropicError( + status_code=404, + message=message_thread_not_found_body( + f"message threads are not supported for provider {custom_llm_provider}, " + "so this thread cannot be continued. Send the full conversation to start over" + ), + ) + if not drops_unsupported_params(litellm_params.drop_params): + raise AnthropicError( + status_code=400, + message=message_thread_unsupported_body( + f"message threads are not supported for provider {custom_llm_provider}. " + "Send the request without thread, or set drop_params to serve it without a server-side thread" + ), + ) + verbose_logger.warning( + "Dropping thread for provider %s: message threads are not supported there, " + "so the request is served without a server-side thread", + custom_llm_provider, + ) + kwargs.pop("thread") # pyright: ignore[reportUnknownMemberType] # **kwargs of the handler is untyped if anthropic_messages_provider_config is None: # Route to Responses API for OpenAI / Azure, chat/completions for everything else. if kwargs.get("compaction") is None and _should_route_to_responses_api( diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index aa0693d6f03..b0558292271 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -59,6 +59,7 @@ from .mid_conversation_system import ( convert_mid_conversation_system_turns, is_system_role_message, ) +from .utils import continues_message_thread DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" @@ -137,6 +138,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "output_config", "reasoning_effort", "safeguards", + "thread", # TODO: Add Anthropic `metadata` support # "metadata", ] @@ -144,6 +146,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): def should_filter_anthropic_beta_headers(self) -> bool: return self._resolved_provider != "anthropic" + def supports_message_threads(self) -> bool: + return is_beta_header_supported_by_bundled_config( + ANTHROPIC_BETA_HEADER_VALUES.MESSAGE_THREADS_2026_08_12.value, self._resolved_provider + ) + def supports_safeguards(self) -> bool: """ Whether the upstream platform answers the ``safeguards`` body field itself. A platform @@ -771,10 +778,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): # Auto-strip advisor blocks from history if advisor tool is absent. # Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. + # A thread continue turn omits tools by contract and the stored tools still apply, so it keeps them. tools: Final = anthropic_messages_optional_request_params.get("tools") _tools: Final = tools or [] _has_advisor: Final = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _tools) - if not _has_advisor: + if not _has_advisor and not continues_message_thread( + anthropic_messages_optional_request_params.get("thread") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the request params arrive as a bare dict + ): messages = strip_advisor_blocks_from_messages(messages) gated: Final = ( @@ -797,7 +807,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ), max_tokens=max_tokens, model=model, - **anthropic_messages_optional_request_params, + **anthropic_messages_optional_request_params, # pyright: ignore[reportUnknownArgumentType] # the request params arrive as a bare dict ) request: Final = ( {**dict(anthropic_messages_request), "thinking": gated.thinking} @@ -877,6 +887,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): drop_params: Whether the deployment or request set drop_params """ beta_values: Final[set[str]] = set() + threaded: Final = optional_params.get("thread") is not None # pyright: ignore[reportUnknownMemberType] # the optional params arrive as a bare dict + scanned_messages: Final = ( + () + if continues_message_thread(optional_params.get("thread")) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the optional params arrive as a bare dict + else messages + ) existing_beta: Final = tuple( piece.strip() @@ -887,7 +903,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ) beta_values.update(existing_beta) - if requires_native_compaction_beta(custom_llm_provider, optional_params, messages): + if requires_native_compaction_beta(custom_llm_provider, optional_params, scanned_messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) beta_values.update(context_management_betas(optional_params.get("context_management"))) @@ -911,9 +927,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value) - if messages_carry_output_config(messages): + if messages_carry_output_config(scanned_messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) + if threaded: + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.MESSAGE_THREADS_2026_08_12.value) + tools: Final = optional_params.get("tools") if any(isinstance(tool, dict) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for tool in tools or ()): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value) @@ -924,7 +943,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): thinking_betas: Final = AnthropicModelInfo().thinking_betas(optional_params.get("thinking")) tool_change_betas: Final = ( (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) - if AnthropicModelInfo().is_mid_conversation_tool_change_used(messages) + if AnthropicModelInfo().is_mid_conversation_tool_change_used(scanned_messages) else () ) all_beta_values: Final = beta_values.union(thinking_betas, tool_change_betas) diff --git a/litellm/llms/anthropic/messages/native/utils.py b/litellm/llms/anthropic/messages/native/utils.py index 8f33d000c7c..dfeedf1c191 100644 --- a/litellm/llms/anthropic/messages/native/utils.py +++ b/litellm/llms/anthropic/messages/native/utils.py @@ -1,3 +1,4 @@ +import json from collections.abc import Iterable, Mapping, Sequence from functools import lru_cache from typing import TYPE_CHECKING, Final, cast, get_type_hints @@ -13,7 +14,10 @@ from litellm.llms.anthropic.common_utils import ( ) from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled from litellm.types.llms.anthropic import ( + MESSAGE_THREAD_NOT_FOUND_ERROR_CODE, + MESSAGE_THREAD_UNSUPPORTED_ERROR_CODE, AnthropicMessagesRequestOptionalParams, + AnthropicMessageThread, AnthropicStopDetails, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -279,6 +283,39 @@ def prepare_native_messages( ) +_THREAD_ADAPTER: Final = TypeAdapter(AnthropicMessageThread) + + +def continues_message_thread(thread: object) -> bool: + """True when ``thread`` is a well-formed ``{"type": "continue", ...}`` body field.""" + try: + return _THREAD_ADAPTER.validate_python(thread)["type"] == "continue" + except ValidationError: + return False + + +def continued_message_id(thread: object) -> str | None: + try: + validated: Final = _THREAD_ADAPTER.validate_python(thread) + except ValidationError: + return None + return validated.get("previous_message_id") if validated["type"] == "continue" else None + + +def _message_thread_error_body(error_type: str, error_code: str, message: str) -> str: + return json.dumps( + {"type": "error", "error": {"type": error_type, "message": message, "details": {"error_code": error_code}}} + ) + + +def message_thread_not_found_body(message: str) -> str: + return _message_thread_error_body("not_found_error", MESSAGE_THREAD_NOT_FOUND_ERROR_CODE, message) + + +def message_thread_unsupported_body(message: str) -> str: + return _message_thread_error_body("invalid_request_error", MESSAGE_THREAD_UNSUPPORTED_ERROR_CODE, message) + + @lru_cache(maxsize=1) def anthropic_messages_optional_param_keys() -> frozenset[str]: """ diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index 487e7aca663..3e999467ad5 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -145,6 +145,9 @@ class BaseAnthropicMessagesConfig(ABC): def supports_safeguards(self) -> bool: return True + def supports_message_threads(self) -> bool: + return False + def serves_api_base(self, model: str, api_base: str | None) -> bool: return True diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index a5d825a39d7..cea57e3505d 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -37,6 +37,9 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): def is_native_messages_route(self) -> bool: return False + def supports_message_threads(self) -> bool: + return False + def hoists_wrapped_defer_loading(self, model: str) -> bool: return False diff --git a/litellm/proxy/hooks/__init__.py b/litellm/proxy/hooks/__init__.py index 138050ccc9a..49cf8760033 100644 --- a/litellm/proxy/hooks/__init__.py +++ b/litellm/proxy/hooks/__init__.py @@ -16,6 +16,7 @@ from .max_iterations_limiter import ( # noqa: F401 # backwards-compatible pack PROXY_MaxIterationsHandler, _PROXY_MaxIterationsHandler, # pyright: ignore[reportPrivateUsage] # backwards-compatible package export ) +from .message_thread_ownership import MessageThreadOwnership from .parallel_request_limiter import ( # noqa: F401 # backwards-compatible package export PROXY_MaxParallelRequestsHandler, _PROXY_MaxParallelRequestsHandler, # pyright: ignore[reportPrivateUsage] # backwards-compatible package export @@ -39,6 +40,7 @@ PROXY_HOOKS: Final = { "parallel_request_limiter": PROXY_MaxParallelRequestsHandler_v3, "cache_control_check": PROXY_CacheControlCheck, "responses_id_security": ResponsesIDSecurity, + "message_thread_ownership": MessageThreadOwnership, "litellm_skills": SkillsInjectionHook, "max_iterations_limiter": PROXY_MaxIterationsHandler, "max_budget_per_session_limiter": PROXY_MaxBudgetPerSessionHandler, diff --git a/litellm/proxy/hooks/message_thread_ownership.py b/litellm/proxy/hooks/message_thread_ownership.py new file mode 100644 index 00000000000..0f2d9fd1933 --- /dev/null +++ b/litellm/proxy/hooks/message_thread_ownership.py @@ -0,0 +1,319 @@ +import json +import re +from collections.abc import AsyncGenerator, AsyncIterable, Mapping +from typing import TYPE_CHECKING, Final, Protocol + +import httpx +from fastapi import HTTPException +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +from litellm._internal_context import with_service_target +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers +from litellm.llms.anthropic.messages.native.utils import continued_message_id, message_thread_not_found_body +from litellm.llms.custom_httpx.sse_lines import split_sse_lines_keepends +from litellm.types.utils import CallTypesLiteral, LLMResponseTypes + +if TYPE_CHECKING: + from litellm.caching.caching import DualCache + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import InternalUsageCache + +_PASS_THROUGH_CALL_TYPE: Final = "pass_through_endpoint" +_ANTHROPIC_BETA_HEADER: Final = "anthropic-beta" +_OWNER_KEY_PREFIX: Final = "anthropic_message_thread_owner:" +_OWNER_TTL_SECONDS: Final = 3600 +_SSE_DATA_FIELD: Final = "data:" +_SSE_FRAME_END_PATTERN: Final = re.compile(rb"\r\n\r\n|\r\r|\n\n") +_MAX_SCANNED_BYTES: Final = 65536 +_UNOWNED_THREAD_MESSAGE: Final = ( + "message thread not found for this key. Send the full conversation to start a new thread" +) + +_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_MAPPINGS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +class _FirstClaimStore(Protocol): + async def async_set_cache(self, key: str, value: object, *, nx: bool, ttl: int) -> object: ... + + async def async_get_cache(self, key: str) -> object: ... + + +class _ThreadBinding(BaseModel): + model_config = ConfigDict(frozen=True) + + owner: str + anthropic_beta: str | None = None + + +_BINDING_ADAPTER: Final = TypeAdapter(_ThreadBinding) + + +def _as_mapping(value: object) -> Mapping[str, object] | None: + try: + return _MAPPING_ADAPTER.validate_python(value, strict=True) + except ValidationError: + return None + + +def _as_binding(value: object) -> _ThreadBinding | None: + try: + return _BINDING_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _anthropic_beta(headers: Mapping[str, object]) -> str | None: + return next( + ( + value + for name, value in headers.items() + if name.lower() == _ANTHROPIC_BETA_HEADER and isinstance(value, str) and value + ), + None, + ) + + +def _sent_anthropic_beta(request_data: Mapping[str, object]) -> str | None: + logging_obj: Final = request_data.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging): + return None + response: Final[object] = logging_obj.model_call_details.get("httpx_response") + if not isinstance(response, httpx.Response): + return None + try: + sent_headers: Final = response.request.headers + except RuntimeError: + return None + return sent_headers.get(_ANTHROPIC_BETA_HEADER) or None + + +def _with_anthropic_beta(extra_headers: object, anthropic_beta: str) -> Mapping[str, object]: + headers: Final = _as_mapping(extra_headers) or {} + return { + **{name: value for name, value in headers.items() if name.lower() != _ANTHROPIC_BETA_HEADER}, + _ANTHROPIC_BETA_HEADER: merge_anthropic_beta_headers(_anthropic_beta(headers), anthropic_beta), + } + + +def _as_mappings(value: object) -> tuple[Mapping[str, object], ...]: + try: + return _MAPPINGS_ADAPTER.validate_python(value) + except ValidationError: + return () + + +def _client_headers_with_anthropic_beta(scoped: Mapping[str, object], anthropic_beta: str) -> Mapping[str, object]: + client_headers: Final = _as_mapping(scoped.get("extra_headers")) + if client_headers is None or _anthropic_beta(client_headers) is None: + return scoped + return {**scoped, "extra_headers": dict(_with_anthropic_beta(client_headers, anthropic_beta))} + + +def _provider_headers_with_anthropic_beta( + provider_specific_header: object, anthropic_beta: str +) -> Mapping[str, object]: + single: Final = _as_mapping(provider_specific_header) + if single is not None: + return {"provider_specific_header": _client_headers_with_anthropic_beta(single, anthropic_beta)} + scoped_headers: Final = _as_mappings(provider_specific_header) + if not scoped_headers: + return {} + return { + "provider_specific_header": tuple( + _client_headers_with_anthropic_beta(scoped, anthropic_beta) for scoped in scoped_headers + ) + } + + +def _owner_key(message_id: str) -> str: + return f"{_OWNER_KEY_PREFIX}{message_id}" + + +def _owner(user_api_key_dict: "UserAPIKeyAuth") -> str: + if user_api_key_dict.token: + return f"key:{user_api_key_dict.token}" + return "identity:" + json.dumps( + [ + user_api_key_dict.user_id, + user_api_key_dict.team_id, + user_api_key_dict.end_user_id, + user_api_key_dict.org_id, + user_api_key_dict.agent_id, + ] + ) + + +def _threads(data: Mapping[str, object]) -> tuple[object, object]: + extra_body: Final = _as_mapping(data.get("extra_body")) or {} + return data.get("thread"), extra_body.get("thread") + + +def _sends_thread(data: Mapping[str, object]) -> bool: + return any(thread is not None for thread in _threads(data)) + + +def _continued_message_ids(data: Mapping[str, object]) -> tuple[str, ...]: + continued: Final = (continued_message_id(thread) for thread in _threads(data)) + return tuple(message_id for message_id in continued if message_id is not None) + + +def _may_continue(binding: _ThreadBinding | None, owner: str, pass_through: bool) -> bool: + if binding is None: + return pass_through + return binding.owner == owner + + +def _response_message_id(response: object) -> str | None: + message: Final = _as_mapping(response) + if message is None or message.get("type") != "message": + return None + message_id: Final = message.get("id") + return message_id if isinstance(message_id, str) and message_id else None + + +def _message_start_id(event: Mapping[str, object]) -> str | None: + return _response_message_id(event.get("message")) if event.get("type") == "message_start" else None + + +def _frame_event(frame: bytes) -> Mapping[str, object] | None: + data_lines: Final = tuple( + line.strip()[len(_SSE_DATA_FIELD) :].strip() + for line in split_sse_lines_keepends(frame.decode("utf-8", errors="ignore")) + if line.strip().startswith(_SSE_DATA_FIELD) + ) + payload: Final = "".join(data_lines) + if not payload or payload == "[DONE]": + return None + try: + return _MAPPING_ADAPTER.validate_json(payload) + except ValidationError: + return None + + +class _StreamedMessageIdScanner: + def __init__(self, finished: bool) -> None: + self._head = b"" + self.finished = finished + + def scan(self, chunk: object) -> str | None: + if isinstance(chunk, (bytes, bytearray)): + return self._scan_encoded(bytes(chunk)) + if isinstance(chunk, str): + return self._scan_encoded(chunk.encode("utf-8")) + event: Final = _as_mapping(chunk) + if event is None: + self.finished = True + return None + return self._settled(event) + + def _settled(self, event: Mapping[str, object]) -> str | None: + if event.get("type") == "ping": + return None + self.finished = True + return _message_start_id(event) + + def _scan_encoded(self, data: bytes) -> str | None: + head: Final = self._head + data + boundaries: Final = tuple(match.end() for match in _SSE_FRAME_END_PATTERN.finditer(head)) + frames: Final = tuple(head[start:end] for start, end in zip((0, *boundaries[:-1]), boundaries)) + events: Final = (event for frame in frames if (event := _frame_event(frame)) is not None) + decider: Final = next((event for event in events if event.get("type") != "ping"), None) + if decider is not None: + return self._settled(decider) + self._head = head[boundaries[-1] :] if boundaries else head + if len(self._head) > _MAX_SCANNED_BYTES: + self.finished = True + return None + + +class MessageThreadOwnership(CustomLogger): + def __init__(self, message_thread_owner_cache: "InternalUsageCache") -> None: + self._owners: Final = message_thread_owner_cache + + async def async_pre_call_hook( + self, + user_api_key_dict: "UserAPIKeyAuth", + cache: "DualCache", + data: Mapping[str, object], + call_type: CallTypesLiteral, + ) -> dict[str, object] | None: # mutable-ok: the proxy swaps in a pre-call hook's result only when it is a dict + previous_message_ids: Final = _continued_message_ids(data) + if not previous_message_ids: + return None + bindings: Final = tuple( + [await self._binding(message_id, user_api_key_dict) for message_id in previous_message_ids] + ) + owner: Final = _owner(user_api_key_dict) + pass_through: Final = call_type == _PASS_THROUGH_CALL_TYPE + if not all(_may_continue(binding, owner, pass_through) for binding in bindings): + raise HTTPException(status_code=404, detail=message_thread_not_found_body(_UNOWNED_THREAD_MESSAGE)) + anthropic_beta: Final = next( + (binding.anthropic_beta for binding in bindings if binding is not None and binding.anthropic_beta), None + ) + if anthropic_beta is None or pass_through: + return None + return { + **data, + "extra_headers": _with_anthropic_beta(data.get("extra_headers"), anthropic_beta), + **_provider_headers_with_anthropic_beta(data.get("provider_specific_header"), anthropic_beta), + } + + async def async_post_call_success_hook( + self, + data: Mapping[str, object], + user_api_key_dict: "UserAPIKeyAuth", + response: LLMResponseTypes, + ) -> LLMResponseTypes: + message_id: Final = _response_message_id(response) if _sends_thread(data) else None + if message_id is not None: + await self._record_owner(message_id, user_api_key_dict, _sent_anthropic_beta(data)) + return response + + async def async_post_call_streaming_iterator_hook( # pyright: ignore[reportIncompatibleMethodOverride] # the proxy streams bytes, str and dict chunks here, not the base's ModelResponseStream + self, + user_api_key_dict: "UserAPIKeyAuth", + response: AsyncIterable[object], + request_data: Mapping[str, object], + ) -> AsyncGenerator[object, None]: + scanner: Final = _StreamedMessageIdScanner(finished=not _sends_thread(request_data)) + async for chunk in response: + yield chunk + if not scanner.finished and (message_id := scanner.scan(chunk)) is not None: + await self._record_owner(message_id, user_api_key_dict, _sent_anthropic_beta(request_data)) + + @with_service_target("message_thread_owners") + async def _binding(self, message_id: str, user_api_key_dict: "UserAPIKeyAuth") -> _ThreadBinding | None: + cached: Final[object] = await self._owners.async_get_cache( + key=_owner_key(message_id), litellm_parent_otel_span=user_api_key_dict.parent_otel_span + ) + return _as_binding(cached) + + @with_service_target("message_thread_owners") + async def _record_owner( + self, message_id: str, user_api_key_dict: "UserAPIKeyAuth", anthropic_beta: str | None + ) -> None: + if await self._binding(message_id, user_api_key_dict) is not None: + return + key: Final = _owner_key(message_id) + claimed: Final = _ThreadBinding(owner=_owner(user_api_key_dict), anthropic_beta=anthropic_beta).model_dump() + await self._owners.async_set_cache( + key=key, + value=await _first_claim(self._owners.dual_cache.redis_cache, key, claimed), + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + local_only=True, + ttl=_OWNER_TTL_SECONDS, + ) + + +async def _first_claim(store: _FirstClaimStore | None, key: str, claimed: Mapping[str, object]) -> object: + if store is None: + return claimed + try: + await store.async_set_cache(key, claimed, nx=True, ttl=_OWNER_TTL_SECONDS) + stored: Final = await store.async_get_cache(key) + except Exception: # noqa: BLE001 # a Redis outage keeps the binding in this worker, as DualCache's writes do + return claimed + return claimed if stored is None else stored diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 4468a724d2e..bf00bdfb5fb 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -65,6 +65,7 @@ from litellm.constants import ( LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, MAX_TEAM_LIST_LIMIT, + MESSAGE_THREAD_MAX_TRACKED_OWNERS, PROXY_REJECTED_BEFORE_ROUTING_KEY, REDACTED_BY_LITELLM, REDIS_SPEND_LOGS_BUFFER_DEQUEUE_COUNT, @@ -1963,6 +1964,9 @@ class ProxyLogging: self.file_usage_cache: Final = InternalUsageCache( dual_cache=DualCache(in_memory_cache=InMemoryCache(max_size_in_memory=FILE_USAGE_MAX_TRACKED_COUNTERS)) ) + self.message_thread_owner_cache: Final = InternalUsageCache( + dual_cache=DualCache(in_memory_cache=InMemoryCache(max_size_in_memory=MESSAGE_THREAD_MAX_TRACKED_OWNERS)) + ) self.max_parallel_request_limiter = PROXY_MaxParallelRequestsHandler(self.internal_usage_cache) self.cache_control_check = PROXY_CacheControlCheck() self.alerting: list[str] | None = None @@ -2122,6 +2126,7 @@ class ProxyLogging: if redis_cache is not None: self.internal_usage_cache.dual_cache.redis_cache = redis_cache self.file_usage_cache.dual_cache.redis_cache = redis_cache + self.message_thread_owner_cache.dual_cache.redis_cache = redis_cache self.db_spend_update_writer.redis_update_buffer.redis_cache = redis_cache self.db_spend_update_writer.pod_lock_manager.redis_cache = redis_cache @@ -2141,6 +2146,7 @@ class ProxyLogging: continue available_args: Mapping[str, object] = { "internal_usage_cache": self.internal_usage_cache, + "message_thread_owner_cache": self.message_thread_owner_cache, "prisma_client": prisma_client, } passed_in_args = {name: value for name, value in available_args.items() if name in expected_args} diff --git a/litellm/router_rust/surface/health.py b/litellm/router_rust/surface/health.py index 80f7e148185..f2ac72a262a 100644 --- a/litellm/router_rust/surface/health.py +++ b/litellm/router_rust/surface/health.py @@ -279,6 +279,7 @@ class RustHealthCallbacks: is_caller_scoped_auth_failure, is_caller_timeout_408, is_failure_outside_selected_deployment, + is_message_thread_not_found, is_response_lookup_not_found, is_video_lookup_not_found, ) @@ -290,6 +291,7 @@ class RustHealthCallbacks: or (isinstance(exception, Exception) and is_background_response_poll_not_found(exception, litellm_params)) or is_video_lookup_not_found(exception, logged) or is_response_lookup_not_found(exception, logged) + or is_message_thread_not_found(exception) or is_caller_timeout_408(logged, status) # pyright: ignore[reportArgumentType] # the status as raised or is_caller_scoped_auth_failure(self._deployment_of(deployment_id), status) # pyright: ignore[reportArgumentType] # see above ) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 897f31840d3..190ef43a3e3 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -30,6 +30,7 @@ from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials from litellm.router_deprecated import renamed_router_param from litellm.router_utils.cooldown_cache import CooldownCacheValue from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback +from litellm.types.llms.anthropic import MESSAGE_THREAD_NOT_FOUND_ERROR_CODE from litellm.types.router import Deployment from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, @@ -106,6 +107,12 @@ def is_video_lookup_not_found(exception: object, failure_kwargs: Mapping[str, ob return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in VIDEO_ID_CALL_TYPES +def is_message_thread_not_found(exception: object) -> bool: + """Whether a message-thread continue failed with Anthropic's 404 ``thread_not_found``: the caller's + ``previous_message_id`` is stale and the SDK replays the conversation, the deployment is healthy.""" + return getattr(exception, "status_code", None) == 404 and MESSAGE_THREAD_NOT_FOUND_ERROR_CODE in str(exception) + + # The calls on an existing response id. ``compact_responses`` is left out: it sends a model and input, # so its 404 can mean a missing model rather than a missing response. RESPONSE_ID_CALL_TYPES: Final = frozenset( diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 3abe64df3c6..244883859f0 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -448,6 +448,14 @@ class AnthropicCompaction(TypedDict, total=False): instructions: ReadOnly[str] +class AnthropicMessageThread(TypedDict, total=False): + """``thread`` body field of the message-threads beta: ``{"type": "create"}`` opens a server-side + thread, ``{"type": "continue", "previous_message_id": "msg_..."}`` appends only the new messages to it.""" + + type: Required[ReadOnly[Literal["create", "continue"]]] + previous_message_id: ReadOnly[str] + + class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): max_tokens: int | None metadata: AnthropicMetadata | dict | None @@ -472,6 +480,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): reasoning_effort: str | None safeguards: ReadOnly[list[dict[str, object]] | None] service_tier: ReadOnly[str | None] + thread: ReadOnly[AnthropicMessageThread | None] class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False): @@ -883,6 +892,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): PER_TURN_CONTROL_2026_07_01 = "per-turn-control-2026-07-01" DANGEROUS_TOOL_USE_2026_09_03 = "dangerous-tool-use-2026-09-03" MCP_CLIENT_2025_04_04 = "mcp-client-2025-04-04" + MESSAGE_THREADS_2026_08_12 = "message-threads-2026-08-12" # Tool search beta header constant (for Anthropic direct API and Microsoft Foundry) @@ -910,6 +920,9 @@ ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01" ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: Final = "thinking-display-updates-2026-08-18" + +MESSAGE_THREAD_NOT_FOUND_ERROR_CODE: Final = "thread_not_found" +MESSAGE_THREAD_UNSUPPORTED_ERROR_CODE: Final = "thread_unsupported_request" ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: Final = "thinking-binding-controls-2026-08-01" ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: Final = "mid-conversation-tool-changes-2026-07-01" diff --git a/tests/integration/messages_endpoint/test_message_threads_wire.py b/tests/integration/messages_endpoint/test_message_threads_wire.py new file mode 100644 index 00000000000..2ba3e627ea0 --- /dev/null +++ b/tests/integration/messages_endpoint/test_message_threads_wire.py @@ -0,0 +1,413 @@ +import json +import re +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.shares_worker_state + +ANTHROPIC_MODEL: Final = "threads-anthropic" +BRIDGED_MODEL: Final = "threads-bridged" +BRIDGED_DROP_PARAMS_MODEL: Final = "threads-bridged-drop-params" +PASSTHROUGH_MODEL: Final = "threads-messages-passthrough" +MESSAGE_THREADS_BETA: Final = "message-threads-2026-08-12" +EXPIRED_THREAD_MARKER: Final = "thread-expired" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_NONCE: Final = re.compile(rb"nonce-([0-9a-f]{32})") + + +def nonce() -> str: + return f"nonce-{uuid.uuid4().hex}" + + +def thread_not_found_reply() -> Reply: + return Reply( + status=404, + body=json.dumps( + { + "type": "error", + "error": { + "type": "not_found_error", + "message": "No thread state was found for the requested `previous_message_id`.", + "details": {"error_code": "thread_not_found"}, + }, + "request_id": "req_upstream", + } + ).encode(), + ) + + +def anthropic_message(identity: str) -> dict[str, JsonValue]: + return { + "id": f"msg_{identity}", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": f"answered {identity}"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 3}, + } + + +def sse_event(event: str, data: Mapping[str, JsonValue]) -> bytes: + return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() + + +def streamed_anthropic_reply(identity: str) -> Reply: + message_start: Final = sse_event( + "message_start", {"type": "message_start", "message": {**anthropic_message(identity), "content": []}} + ) + return Reply( + chunks=( + message_start[:40], + message_start[40:], + sse_event( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + sse_event( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}, + ), + sse_event("content_block_stop", {"type": "content_block_stop", "index": 0}), + sse_event( + "message_delta", + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}, + ), + sse_event("message_stop", {"type": "message_stop"}), + ), + content_type="text/event-stream", + ) + + +def responses_reply(identity: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.5", + "output": [ + { + "type": "message", + "id": f"msg_{identity}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": f"answered {identity}", "annotations": []}], + } + ], + "usage": {"input_tokens": 12, "output_tokens": 3, "total_tokens": 15}, + } + ).encode() + ) + + +def respond(request: Request) -> Reply: + if request.method != "POST": + return Reply(body=b'{"object":"list","data":[]}') + match: Final = _NONCE.search(request.body) + assert match is not None, request.body[:200] + identity: Final = match.group(1).decode() + if request.target.startswith("/openai"): + return responses_reply(identity) + if EXPIRED_THREAD_MARKER.encode() in request.body: + return thread_not_found_reply() + if b'"stream": true' in request.body or b'"stream":true' in request.body: + return streamed_anthropic_reply(identity) + return Reply(body=json.dumps(anthropic_message(identity)).encode()) + + +def write_config(directory: Path, wire_url: str) -> Path: + path: Final = directory / f"threads-{uuid.uuid4().hex}.json" + path.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": ANTHROPIC_MODEL, + "litellm_params": { + "model": "anthropic/claude-opus-4-8", + "api_base": f"{wire_url}/anthropic", + "api_key": "synthetic-anthropic-key", + }, + }, + { + "model_name": BRIDGED_MODEL, + "litellm_params": { + "model": "openai/gpt-5.5", + "api_base": f"{wire_url}/openai", + "api_key": "synthetic-openai-key", + }, + }, + { + "model_name": BRIDGED_DROP_PARAMS_MODEL, + "litellm_params": { + "model": "openai/gpt-5.5", + "api_base": f"{wire_url}/openai", + "api_key": "synthetic-openai-key", + "drop_params": True, + }, + }, + { + "model_name": PASSTHROUGH_MODEL, + "litellm_params": { + "model": "hosted_vllm/qwen3-coder", + "api_base": f"{wire_url}/vllm", + "api_key": "synthetic-vllm-key", + }, + "model_info": {"supported_endpoints": ["/v1/messages"]}, + }, + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + }, + } + ) + ) + return path + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + wire: Wire + + def send( + self, + model: str, + request_nonce: str, + thread: Mapping[str, str], + *, + key: str | None = None, + text: str = "hi", + **fields: JsonValue, + ) -> tuple[int, bytes]: + response: Final = self.gateway.client.post( + "/v1/messages", + json={ + "model": model, + "max_tokens": 64, + "messages": [{"role": "user", "content": f"{text} {request_nonce}"}], + "thread": dict(thread), + **fields, + }, + headers={"Authorization": f"Bearer {self.gateway.key if key is None else key}"}, + timeout=120, + ) + return response.status_code, response.content + + def create(self, *, key: str | None = None, **fields: JsonValue) -> str: + request_nonce: Final = nonce() + status, raw = self.send(ANTHROPIC_MODEL, request_nonce, {"type": "create"}, key=key, **fields) + assert status == 200, raw[:500] + self.received(request_nonce) + return f"msg_{request_nonce.removeprefix('nonce-')}" + + def received(self, request_nonce: str) -> tuple[Request, ...]: + return tuple(request for request in self.wire.drain() if request_nonce.encode() in request.body) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("threads") + with ( + gateway_from_environment() as shared, + wire_server(respond) as wire, + owned_proxy( + shared, + directory, + { + "LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS": "true", + "ANTHROPIC_API_BASE": f"{wire.url}/anthropic", + "ANTHROPIC_API_KEY": "synthetic-anthropic-key", + }, + config=write_config(directory, wire.url), + workers=1, + ) as gateway, + ): + yield Rig(gateway, wire) + + +def beta_header(headers: Mapping[str, str]) -> frozenset[str]: + return frozenset( + piece for key, value in headers.items() if key.lower() == "anthropic-beta" for piece in value.split(",") + ) # comprehension-ok: one header line may carry several comma-separated betas + + +def test_a_create_turn_reaches_anthropic_with_the_thread_field_and_its_beta(rig: Rig) -> None: + request_nonce: Final = nonce() + status, raw = rig.send(ANTHROPIC_MODEL, request_nonce, {"type": "create"}, system="be terse") + assert status == 200, raw[:500] + assert _JSON_OBJECT.validate_json(raw)["id"] == f"msg_{request_nonce.removeprefix('nonce-')}" + (upstream,) = rig.received(request_nonce) + body: Final = _JSON_OBJECT.validate_json(upstream.body) + assert body["thread"] == {"type": "create"} + assert body["system"] == "be terse" + assert MESSAGE_THREADS_BETA in beta_header(upstream.headers), upstream.headers + + +def test_a_continue_turn_reaches_anthropic_with_only_the_new_messages_and_the_same_beta(rig: Rig) -> None: + request_nonce: Final = nonce() + thread: Final = {"type": "continue", "previous_message_id": rig.create()} + status, raw = rig.send(ANTHROPIC_MODEL, request_nonce, thread) + assert status == 200, raw[:500] + (upstream,) = rig.received(request_nonce) + body: Final = _JSON_OBJECT.validate_json(upstream.body) + assert body["thread"] == thread + assert body["messages"] == [{"role": "user", "content": f"hi {request_nonce}"}] + assert "system" not in body and "tools" not in body + assert MESSAGE_THREADS_BETA in beta_header(upstream.headers), upstream.headers + + +def assert_thread_not_found(status: int, raw: bytes) -> None: + assert status == 404, raw[:500] + error: Final = _JSON_OBJECT.validate_json(raw) + detail: Final = error["error"] + assert isinstance(detail, dict), error + assert detail["type"] == "not_found_error" + assert detail["details"] == {"error_code": "thread_not_found"} + + +@pytest.mark.parametrize("model", [BRIDGED_MODEL, PASSTHROUGH_MODEL], ids=["translated", "messages-passthrough"]) +def test_a_create_turn_on_a_leg_without_threads_is_thread_unsupported_and_never_reaches_the_provider( + rig: Rig, model: str +) -> None: + request_nonce: Final = nonce() + status, raw = rig.send(model, request_nonce, {"type": "create"}) + assert status == 400, raw[:500] + detail: Final = _JSON_OBJECT.validate_json(raw)["error"] + assert isinstance(detail, dict), raw[:500] + assert detail["type"] == "invalid_request_error" + assert detail["details"] == {"error_code": "thread_unsupported_request"} + assert rig.received(request_nonce) == () + + +def test_a_create_turn_on_a_translated_leg_is_served_without_the_thread_under_drop_params(rig: Rig) -> None: + request_nonce: Final = nonce() + status, raw = rig.send(BRIDGED_DROP_PARAMS_MODEL, request_nonce, {"type": "create"}) + assert status == 200, raw[:500] + (upstream,) = rig.received(request_nonce) + assert upstream.target.startswith("/openai"), upstream.target + assert "thread" not in _JSON_OBJECT.validate_json(upstream.body) + + +@pytest.mark.parametrize("model", [BRIDGED_MODEL, PASSTHROUGH_MODEL], ids=["translated", "messages-passthrough"]) +def test_a_continue_turn_on_a_leg_without_threads_is_thread_not_found_and_never_reaches_the_provider( + rig: Rig, model: str +) -> None: + request_nonce: Final = nonce() + status, raw = rig.send(model, request_nonce, {"type": "continue", "previous_message_id": rig.create()}) + assert_thread_not_found(status, raw) + assert rig.received(request_nonce) == () + + +def test_another_key_cannot_continue_a_thread_and_never_reaches_anthropic(rig: Rig) -> None: + with rig.gateway.scenario() as scenario: + owner: Final = scenario.key() + intruder: Final = scenario.key() + thread: Final = {"type": "continue", "previous_message_id": rig.create(key=owner)} + intruder_nonce: Final = nonce() + assert_thread_not_found(*rig.send(ANTHROPIC_MODEL, intruder_nonce, thread, key=intruder)) + assert rig.received(intruder_nonce) == () + owner_nonce: Final = nonce() + status, raw = rig.send(ANTHROPIC_MODEL, owner_nonce, thread, key=owner) + assert status == 200, raw[:500] + assert len(rig.received(owner_nonce)) == 1 + + +def test_another_key_cannot_continue_a_thread_through_chat_completions(rig: Rig) -> None: + with rig.gateway.scenario() as scenario: + owner: Final = scenario.key() + intruder: Final = scenario.key() + thread: Final = {"type": "continue", "previous_message_id": rig.create(key=owner)} + intruder_nonce: Final = nonce() + response: Final = rig.gateway.client.post( + "/v1/chat/completions", + json={ + "model": ANTHROPIC_MODEL, + "max_tokens": 64, + "messages": [{"role": "user", "content": f"hi {intruder_nonce}"}], + "thread": thread, + }, + headers={"Authorization": f"Bearer {intruder}"}, + timeout=120, + ) + assert response.status_code == 404, response.content[:500] + assert rig.received(intruder_nonce) == () + + +ADVISOR_TOOL: Final = {"type": "advisor_20260301", "name": "advisor", "model": "claude-opus-4-8"} +ADVISOR_BETA: Final = "advisor-tool-2026-03-01" + + +@pytest.mark.parametrize("stream", [False, True], ids=["json", "streamed"]) +def test_a_continue_turn_resends_the_betas_its_create_turn_took_from_its_tools(rig: Rig, stream: bool) -> None: + create_nonce: Final = nonce() + status, raw = rig.send(ANTHROPIC_MODEL, create_nonce, {"type": "create"}, tools=[ADVISOR_TOOL], stream=stream) + assert status == 200, raw[:500] + (created,) = rig.received(create_nonce) + create_betas: Final = beta_header(created.headers) + assert {ADVISOR_BETA, MESSAGE_THREADS_BETA} <= create_betas, created.headers + continue_nonce: Final = nonce() + thread: Final = {"type": "continue", "previous_message_id": f"msg_{create_nonce.removeprefix('nonce-')}"} + status, raw = rig.send(ANTHROPIC_MODEL, continue_nonce, thread) + assert status == 200, raw[:500] + (continued,) = rig.received(continue_nonce) + assert "tools" not in _JSON_OBJECT.validate_json(continued.body) + assert beta_header(continued.headers) == create_betas, continued.headers + + +def pass_through_continue(rig: Rig, request_nonce: str, previous_message_id: str, key: str) -> int: + return rig.gateway.client.post( + "/anthropic/v1/messages", + json={ + "model": "claude-opus-4-8", + "max_tokens": 64, + "messages": [{"role": "user", "content": f"hi {request_nonce}"}], + "thread": {"type": "continue", "previous_message_id": previous_message_id}, + }, + headers={"Authorization": f"Bearer {key}"}, + timeout=120, + ).status_code + + +def test_another_key_cannot_continue_a_thread_through_the_anthropic_pass_through_route(rig: Rig) -> None: + with rig.gateway.scenario() as scenario: + owner: Final = scenario.key() + intruder: Final = scenario.key() + previous_message_id: Final = rig.create(key=owner) + intruder_nonce: Final = nonce() + assert pass_through_continue(rig, intruder_nonce, previous_message_id, intruder) == 404 + assert rig.received(intruder_nonce) == () + owner_nonce: Final = nonce() + assert pass_through_continue(rig, owner_nonce, previous_message_id, owner) == 200 + assert len(rig.received(owner_nonce)) == 1 + + +def test_a_streamed_create_turn_can_be_continued_by_its_key(rig: Rig) -> None: + previous_message_id: Final = rig.create(stream=True) + request_nonce: Final = nonce() + status, raw = rig.send( + ANTHROPIC_MODEL, request_nonce, {"type": "continue", "previous_message_id": previous_message_id} + ) + assert status == 200, raw[:500] + assert len(rig.received(request_nonce)) == 1 + + +def test_a_thread_not_found_from_anthropic_keeps_its_error_code_for_the_sdk_to_replay_on(rig: Rig) -> None: + request_nonce: Final = nonce() + thread: Final = {"type": "continue", "previous_message_id": rig.create()} + assert_thread_not_found(*rig.send(ANTHROPIC_MODEL, request_nonce, thread, text=EXPIRED_THREAD_MARKER)) + assert len(rig.received(request_nonce)) == 1 diff --git a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py index 210ab6717aa..0e0e1c8c53e 100644 --- a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py +++ b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py @@ -258,3 +258,48 @@ class TestAnthropicErrorSseFrame: "type": "error", "error": {"type": "api_error", "message": "upstream unavailable"}, } + + +THREAD_NOT_FOUND_UPSTREAM: Final = json.dumps( + { + "type": "error", + "error": { + "type": "not_found_error", + "message": "No thread state was found for the requested `previous_message_id`.", + "details": {"error_code": "thread_not_found"}, + }, + "request_id": "req_upstream", + } +) + + +class TestEmbeddedErrorDetails: + def test_a_wrapped_upstream_error_keeps_its_details_next_to_the_wrapped_message(self) -> None: + wrapped: Final = ( + f"litellm.NotFoundError: AnthropicException - {THREAD_NOT_FOUND_UPSTREAM}\n\n" + "LiteLLM: model group 'haiku' failed with the error above. No fallback was attempted." + ) + response: Final = AnthropicExceptionMapping.transform_to_anthropic_error(status_code=404, raw_message=wrapped) + assert response["error"]["details"] == {"error_code": "thread_not_found"} + assert response["error"]["type"] == "not_found_error" + assert response["error"]["message"] == wrapped + + def test_the_error_event_of_a_failed_stream_carries_the_same_details(self) -> None: + wrapped: Final = f"litellm.NotFoundError: AnthropicException - {THREAD_NOT_FOUND_UPSTREAM}" + frame: Final = anthropic_error_sse_frame(404, wrapped) + assert frame.error_response["error"]["details"] == {"error_code": "thread_not_found"} + + @pytest.mark.parametrize( + "raw_message", + ( + "plain text with a { brace and no json", + 'litellm.BadRequestError: OpenAIException - {"error": {"message": "bad", "type": "invalid_request_error"}}', + 'AnthropicException - {"type": "error", "error": {"type": "api_error", "message": "no details here"}}', + ), + ids=("no_json", "openai_shape", "anthropic_without_details"), + ) + def test_a_message_without_anthropic_details_gets_no_details_key(self, raw_message: str) -> None: + response: Final = AnthropicExceptionMapping.transform_to_anthropic_error( + status_code=400, raw_message=raw_message + ) + assert "details" not in response["error"] diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index f6300b2e70c..ff150af324d 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -2801,6 +2801,85 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry" +@pytest.mark.parametrize( + "model", + ("openai/gpt-5.5", "vertex_ai/claude-opus-4-8", "bedrock/anthropic.claude-opus-4-8-v1:0"), + ids=("chat_bridge", "vertex_native", "bedrock_native"), +) +def test_a_thread_continue_on_a_leg_without_message_threads_is_thread_not_found_before_any_upstream_call( + model: str, +) -> None: + from litellm.llms.anthropic.common_utils import AnthropicError + from litellm.llms.anthropic.messages.native.handler import anthropic_messages_handler + + with pytest.raises(AnthropicError) as raised: + anthropic_messages_handler( + max_tokens=64, + messages=[{"role": "user", "content": "hi"}], + model=model, + api_key="synthetic-key", + thread={"type": "continue", "previous_message_id": "msg_01previous"}, + ) + assert raised.value.status_code == 404 + error: Final = _JSON_BODY.validate_json(raised.value.message)["error"] + assert isinstance(error, dict) + assert error["type"] == "not_found_error" + assert error["details"] == {"error_code": "thread_not_found"} + + +@pytest.mark.parametrize( + "model", + ("openai/gpt-5.5", "vertex_ai/claude-opus-4-8", "bedrock/anthropic.claude-opus-4-8-v1:0"), + ids=("chat_bridge", "vertex_native", "bedrock_native"), +) +def test_a_thread_create_on_a_leg_without_message_threads_is_thread_unsupported_before_any_upstream_call( + model: str, +) -> None: + from litellm.llms.anthropic.common_utils import AnthropicError + from litellm.llms.anthropic.messages.native.handler import anthropic_messages_handler + + with pytest.raises(AnthropicError) as raised: + anthropic_messages_handler( + max_tokens=64, + messages=[{"role": "user", "content": "hi"}], + model=model, + api_key="synthetic-key", + thread={"type": "create"}, + ) + assert raised.value.status_code == 400 + error: Final = _JSON_BODY.validate_json(raised.value.message)["error"] + assert isinstance(error, dict) + assert error["type"] == "invalid_request_error" + assert error["details"] == {"error_code": "thread_unsupported_request"} + + +@pytest.mark.asyncio +async def test_a_thread_create_on_a_native_leg_without_message_threads_is_served_without_the_thread_under_drop_params( + local_beta_headers_config: None, +) -> None: + from litellm.llms.anthropic.messages.native import handler + + captured: Final[dict[str, object]] = {} + response: Final = await handler.anthropic_messages( + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + model="bedrock/us.anthropic.claude-sonnet-5", + custom_llm_provider="bedrock", + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + client=_upstream_answering_with([], captured), + thread={"type": "create"}, + drop_params=True, + ) + + body: Final = captured["body"] + assert isinstance(body, dict) + assert "thread" not in body + assert body["messages"] == [{"role": "user", "content": "hi"}] + assert _JSON_BODY.validate_python(response)["id"] == "msg_1" + + @pytest.mark.asyncio @pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize("stream", [False, True], ids=["complete", "stream"]) diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 8f86888728f..97d5b0550d4 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -17,6 +17,7 @@ from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.anthropic.wif import get_anthropic_wif_token +from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig from litellm.llms.minimax.messages.transformation import MinimaxMessagesConfig from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig @@ -723,6 +724,135 @@ def test_a_first_party_messages_request_never_fetches_the_remote_beta_headers_co assert not remote_config.called +MESSAGE_THREADS_BETA: Final = "message-threads-2026-08-12" + + +@pytest.mark.parametrize( + "thread", + ({"type": "create"}, {"type": "continue", "previous_message_id": "msg_01previous"}), + ids=("create", "continue"), +) +def test_messages_request_forwards_the_thread_field_to_anthropic(thread: dict[str, str]) -> None: + request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + model="claude-opus-4-8", + messages=[{"role": "user", "content": "what did I just say"}], + anthropic_messages_optional_request_params={"max_tokens": 64, "thread": dict(thread)}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["thread"] == thread + assert request["messages"] == [{"role": "user", "content": "what did I just say"}] + + +def test_messages_headers_add_the_message_threads_beta_next_to_the_callers_betas() -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + headers={"anthropic-beta": "context-1m-2025-08-07"}, + optional_params={"thread": {"type": "continue", "previous_message_id": "msg_01previous"}}, + messages=[], + ) + assert headers["anthropic-beta"] == f"context-1m-2025-08-07,{MESSAGE_THREADS_BETA}" + + +def test_messages_headers_add_no_message_threads_beta_without_a_thread() -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + headers={}, optional_params={"max_tokens": 64}, messages=[] + ) + assert MESSAGE_THREADS_BETA not in headers.get("anthropic-beta", "") # pyright: ignore[reportUnknownMemberType] # bare-dict result + + +TOOL_SEARCH_TOOLS: Final = ( + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + {"type": "advisor_20260301", "name": "advisor", "model": "claude-opus-4-8"}, +) +TURN_WITH_OUTPUT_CONFIG: Final = ( + {"role": "user", "content": "be brief", "output_config": {"effort": "low"}}, + { + "role": "system", + "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "tool_name": "lookup_record"}}], + }, +) + + +def test_a_thread_create_turn_takes_betas_from_its_tools_and_messages() -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + headers={"anthropic-beta": "context-1m-2025-08-07"}, + optional_params={"thread": {"type": "create"}, "system": "be terse", "tools": list(TOOL_SEARCH_TOOLS)}, + messages=list(TURN_WITH_OUTPUT_CONFIG), + ) + assert set(headers["anthropic-beta"].split(",")) == { # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # bare-dict result + "context-1m-2025-08-07", + MESSAGE_THREADS_BETA, + get_tool_search_beta_header("anthropic"), + "advisor-tool-2026-03-01", + "per-turn-control-2026-07-01", + "mid-conversation-tool-changes-2026-07-01", + } + + +def test_a_thread_continue_turn_takes_no_betas_from_its_new_messages() -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + headers={"anthropic-beta": "context-1m-2025-08-07"}, + optional_params={"thread": {"type": "continue", "previous_message_id": "msg_01p"}}, + messages=list(TURN_WITH_OUTPUT_CONFIG), + ) + assert headers["anthropic-beta"] == f"context-1m-2025-08-07,{MESSAGE_THREADS_BETA}" + + +def test_a_request_without_a_thread_still_takes_betas_from_its_tools_and_messages() -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + headers={}, optional_params={"tools": list(TOOL_SEARCH_TOOLS)}, messages=list(TURN_WITH_OUTPUT_CONFIG) + ) + assert set(headers["anthropic-beta"].split(",")) >= { # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # bare-dict result + get_tool_search_beta_header("anthropic"), + "advisor-tool-2026-03-01", + "per-turn-control-2026-07-01", + "mid-conversation-tool-changes-2026-07-01", + } + + +def _advisor_turn() -> list[dict[str, object]]: + return [ + { + "role": "assistant", + "content": [ + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "advisor", "input": {}}, + { + "type": "advisor_tool_result", + "tool_use_id": "srvtoolu_01", + "content": [{"type": "text", "text": "ok"}], + }, + {"type": "text", "text": "done"}, + ], + }, + {"role": "user", "content": "and now?"}, + ] + + +def test_a_thread_continue_turn_keeps_advisor_blocks_the_stored_tools_still_explain() -> None: + request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + model="claude-opus-4-8", + messages=_advisor_turn(), + anthropic_messages_optional_request_params={ + "max_tokens": 64, + "thread": {"type": "continue", "previous_message_id": "msg_01previous"}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["messages"][0]["content"] == _advisor_turn()[0]["content"] + + +def test_a_request_without_a_thread_still_strips_advisor_blocks_when_the_tool_is_absent() -> None: + request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType] # bare-dict signature + model="claude-opus-4-8", + messages=_advisor_turn(), + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["messages"][0]["content"] == [{"type": "text", "text": "done"}] + + @pytest.mark.parametrize( "model, expected", [ diff --git a/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py b/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py index 8104c0cef56..910234fa3f1 100644 --- a/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py +++ b/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py @@ -137,6 +137,14 @@ def test_sampling_param_raises_clean_400_without_drop_params(monkeypatch): ) +def test_the_thread_field_survives_optional_param_filtering(): + thread: Final = {"type": "continue", "previous_message_id": "msg_01previous"} + result: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + {"max_tokens": 64, "thread": thread} + ) + assert result == {"max_tokens": 64, "thread": thread} + + @pytest.mark.parametrize( ("model", "param", "value"), [ diff --git a/tests/unit/llms/bedrock/test_claude_platform_provider.py b/tests/unit/llms/bedrock/test_claude_platform_provider.py index 9142fe45752..b1c170a6f6f 100644 --- a/tests/unit/llms/bedrock/test_claude_platform_provider.py +++ b/tests/unit/llms/bedrock/test_claude_platform_provider.py @@ -927,3 +927,30 @@ def test_sigv4_no_duplicate_content_type_when_caller_sets_lowercase(): "Duplicate keys produce 'application/json, application/json' in the " "SigV4 canonical string and cause a 401." ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "thread", + ({"type": "create"}, {"type": "continue", "previous_message_id": "msg_01previous"}), + ids=("create", "continue"), +) +async def test_anthropic_messages_bedrock_claude_platform_forwards_a_message_thread(thread: dict[str, str]) -> None: + import litellm + + requests: list[dict] = [] + + await litellm.anthropic_messages( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "what did I just say"}], + max_tokens=10, + api_base="https://aws-external-anthropic.us-west-2.api.aws", + api_key="fake-platform-key", + workspace_id="wrkspc_test", + thread=dict(thread), + client=_async_gateway_client(requests, lambda url, headers, data: _anthropic_response(url)), + ) + + assert len(requests) == 1 + assert requests[0]["body"]["thread"] == thread + assert requests[0]["headers"]["anthropic-beta"] == "message-threads-2026-08-12" diff --git a/tests/unit/proxy/hooks/test_message_thread_ownership.py b/tests/unit/proxy/hooks/test_message_thread_ownership.py new file mode 100644 index 00000000000..e596e2958b9 --- /dev/null +++ b/tests/unit/proxy/hooks/test_message_thread_ownership.py @@ -0,0 +1,558 @@ +import asyncio +import datetime +import json +from collections.abc import AsyncIterator, Sequence +from typing import Final + +import httpx +import pytest +from fastapi import HTTPException +from pydantic import JsonValue, TypeAdapter + +from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCache +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.hooks.message_thread_ownership import MessageThreadOwnership +from litellm.types.utils import CallTypesLiteral +from litellm.proxy.utils import InternalUsageCache, ProxyLogging + +OWNER: Final = UserAPIKeyAuth(token="hashed-owner-key") +INTRUDER: Final = UserAPIKeyAuth(token="hashed-intruder-key") +CREATE: Final = {"type": "create"} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def continue_from(message_id: str) -> dict[str, object]: + return {"thread": {"type": "continue", "previous_message_id": message_id}, "messages": []} + + +def anthropic_message(message_id: str) -> dict[str, object]: + return {"id": message_id, "type": "message", "role": "assistant", "content": []} + + +def sse_frame(event: dict[str, object]) -> bytes: + return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() + + +def message_start(message_id: str) -> dict[str, object]: + return {"type": "message_start", "message": anthropic_message(message_id)} + + +def sent_with(anthropic_beta: str, **data: object) -> dict[str, object]: + logging_obj: Final = Logging( + model="claude-opus-4-8", + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="call", + function_id="function", + ) + sent: Final = httpx.Request("POST", "https://api.anthropic.com/v1/messages", headers={"anthropic-beta": anthropic_beta}) + logging_obj.model_call_details["httpx_response"] = httpx.Response(200, request=sent) + logging_obj.post_call(original_response="{}", additional_args={"complete_input_dict": {}}) + return {**data, "litellm_logging_obj": logging_obj} + + +def hook() -> MessageThreadOwnership: + return MessageThreadOwnership(message_thread_owner_cache=InternalUsageCache(DualCache())) + + +async def chunks(items: Sequence[object]) -> AsyncIterator[object]: + for item in items: + yield item + + +async def served( + ownership: MessageThreadOwnership, items: Sequence[object], request_data: dict[str, object] +) -> list[object]: + return [ + chunk + async for chunk in ownership.async_post_call_streaming_iterator_hook( + user_api_key_dict=OWNER, response=chunks(items), request_data=request_data + ) + ] + + +async def rejection( + ownership: MessageThreadOwnership, key: UserAPIKeyAuth, message_id: str, call_type: CallTypesLiteral +) -> HTTPException | None: + try: + await ownership.async_pre_call_hook( + user_api_key_dict=key, cache=DualCache(), data=continue_from(message_id), call_type=call_type + ) + except HTTPException as rejected: + return rejected + return None + + +async def allows( + ownership: MessageThreadOwnership, + key: UserAPIKeyAuth, + message_id: str, + call_type: CallTypesLiteral = "anthropic_messages", +) -> bool: + rejected: Final = await rejection(ownership, key, message_id, call_type) + if rejected is None: + return True + assert rejected.status_code == 404 + error: Final = _JSON_OBJECT.validate_json(rejected.detail)["error"] + assert isinstance(error, dict) + assert error["type"] == "not_found_error" + assert error["details"] == {"error_code": "thread_not_found"} + return False + + +@pytest.mark.asyncio +async def test_the_proxy_starts_with_a_hook_that_keeps_each_thread_with_its_key() -> None: + proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging._add_proxy_hooks() + await proxy_logging.post_call_success_hook( + data={"thread": CREATE}, response=anthropic_message("msg_started"), user_api_key_dict=OWNER + ) + + with pytest.raises(HTTPException) as rejected: + await proxy_logging.pre_call_hook( + user_api_key_dict=INTRUDER, data=continue_from("msg_started"), call_type="anthropic_messages" + ) + assert rejected.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_only_the_key_that_created_a_thread_can_continue_it() -> None: + ownership: Final = hook() + await ownership.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=OWNER, response=anthropic_message("msg_created") + ) + + assert await allows(ownership, OWNER, "msg_created") + assert not await allows(ownership, INTRUDER, "msg_created") + + +@pytest.mark.asyncio +async def test_without_a_key_only_the_signed_in_identity_that_created_a_thread_can_continue_it() -> None: + ownership: Final = hook() + alice: Final = UserAPIKeyAuth(user_id="alice", team_id="team-1") + bob: Final = UserAPIKeyAuth(user_id="bob", team_id="team-1") + await ownership.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=alice, response=anthropic_message("msg_signed_in") + ) + + assert await allows(ownership, alice, "msg_signed_in") + assert not await allows(ownership, bob, "msg_signed_in") + + +@pytest.mark.asyncio +async def test_without_a_key_one_managed_agent_cannot_continue_another_agents_thread() -> None: + ownership: Final = hook() + planner: Final = UserAPIKeyAuth(agent_id="agent-planner") + coder: Final = UserAPIKeyAuth(agent_id="agent-coder") + await ownership.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=planner, response=anthropic_message("msg_agent") + ) + + assert await allows(ownership, planner, "msg_agent") + assert not await allows(ownership, coder, "msg_agent") + + +@pytest.mark.parametrize("call_type", ("anthropic_messages", "acompletion", "aresponses")) +@pytest.mark.asyncio +async def test_a_continue_naming_a_message_this_proxy_never_served_in_a_thread_is_thread_not_found( + call_type: CallTypesLiteral, +) -> None: + ownership: Final = hook() + await ownership.async_post_call_success_hook( + data={}, user_api_key_dict=OWNER, response=anthropic_message("msg_unthreaded") + ) + + assert not await allows(ownership, OWNER, "msg_unthreaded", call_type) + assert not await allows(ownership, OWNER, "msg_unknown", call_type) + assert not await allows(ownership, INTRUDER, "msg_unknown", call_type) + + +@pytest.mark.asyncio +async def test_each_continued_turn_hands_its_own_message_to_the_same_key() -> None: + ownership: Final = hook() + await ownership.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=OWNER, response=anthropic_message("msg_first") + ) + await ownership.async_post_call_success_hook( + data=continue_from("msg_first"), user_api_key_dict=OWNER, response=anthropic_message("msg_second") + ) + + assert await allows(ownership, OWNER, "msg_second") + assert not await allows(ownership, INTRUDER, "msg_second") + + +@pytest.mark.asyncio +async def test_a_reply_served_again_to_another_key_stays_with_the_key_that_got_it_first() -> None: + ownership: Final = hook() + for key in (OWNER, INTRUDER): + await ownership.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=key, response=anthropic_message("msg_cached") + ) + + assert await allows(ownership, OWNER, "msg_cached") + assert not await allows(ownership, INTRUDER, "msg_cached") + + +@pytest.mark.asyncio +async def test_a_worker_that_writes_after_another_worker_claimed_the_reply_keeps_the_first_owner( + router_redis: tuple[RedisCache, object], +) -> None: + redis_cache, _ = router_redis + late_write_started: Final = asyncio.Event() + owner_stored: Final = asyncio.Event() + + class WritesAfterTheOwner(RedisCache): + async def async_set_cache( + self, + key: str | None, + value: object, + **kwargs: object, # kwargs-ok: RedisCache write signature + ) -> None: + late_write_started.set() + await owner_stored.wait() + await super().async_set_cache(key, value, **kwargs) + + first: Final = MessageThreadOwnership( + message_thread_owner_cache=InternalUsageCache(DualCache(redis_cache=redis_cache)) + ) + late: Final = MessageThreadOwnership( + message_thread_owner_cache=InternalUsageCache( + DualCache(redis_cache=WritesAfterTheOwner(url="redis://router-unit.example:6379/0")) + ) + ) + + async def owner_claims() -> None: + await late_write_started.wait() + await first.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=OWNER, response=anthropic_message("msg_raced") + ) + owner_stored.set() + + await asyncio.gather( + late.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=INTRUDER, response=anthropic_message("msg_raced") + ), + owner_claims(), + ) + + for worker in (first, late): + assert await allows(worker, OWNER, "msg_raced") + assert not await allows(worker, INTRUDER, "msg_raced") + + +@pytest.mark.asyncio +async def test_every_redis_command_for_a_thread_binding_is_traced_under_the_message_thread_owners_family( + router_redis: tuple[RedisCache, asyncio.Queue[tuple[tuple[object, ...], str | None, str | None]]], +) -> None: + redis_cache, commands = router_redis + creating_worker: Final = MessageThreadOwnership( + message_thread_owner_cache=InternalUsageCache(DualCache(redis_cache=redis_cache)) + ) + continuing_worker: Final = MessageThreadOwnership( + message_thread_owner_cache=InternalUsageCache(DualCache(redis_cache=redis_cache)) + ) + await creating_worker.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=OWNER, response=anthropic_message("msg_traced") + ) + + assert await allows(continuing_worker, OWNER, "msg_traced") + sent: Final = tuple(commands.get_nowait() for _ in range(commands.qsize())) + thread_commands: Final = tuple((str(args[0]), target) for args, target, _ in sent if args[0] != "CLIENT") + assert {command for command, _ in thread_commands} == {"GET", "SET"} + assert {target for _, target in thread_commands} == {"message_thread_owners"} + + +@pytest.mark.parametrize("call_type", ("acompletion", "aresponses")) +@pytest.mark.asyncio +async def test_a_thread_continue_on_another_route_stays_with_the_key_that_created_the_thread( + call_type: CallTypesLiteral, +) -> None: + ownership: Final = hook() + await ownership.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=OWNER, response=anthropic_message("msg_cross_route") + ) + + assert await allows(ownership, OWNER, "msg_cross_route", call_type) + assert not await allows(ownership, INTRUDER, "msg_cross_route", call_type) + + +@pytest.mark.parametrize("call_type", ("anthropic_messages", "acompletion", "aresponses")) +@pytest.mark.asyncio +async def test_a_continue_sent_in_extra_body_stays_with_the_key_that_created_the_thread( + call_type: CallTypesLiteral, +) -> None: + ownership: Final = hook() + await ownership.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=OWNER, response=anthropic_message("msg_extra_body") + ) + in_extra_body: Final = {"messages": [], "extra_body": {"thread": continue_from("msg_extra_body")["thread"]}} + + assert ( + await ownership.async_pre_call_hook( + user_api_key_dict=OWNER, cache=DualCache(), data=in_extra_body, call_type=call_type + ) + is None + ) + with pytest.raises(HTTPException) as rejected: + await ownership.async_pre_call_hook( + user_api_key_dict=INTRUDER, cache=DualCache(), data=in_extra_body, call_type=call_type + ) + assert rejected.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_an_owned_top_level_continue_does_not_let_another_keys_extra_body_continue_through() -> None: + ownership: Final = hook() + for key, message_id in ((OWNER, "msg_owner"), (INTRUDER, "msg_intruder")): + await ownership.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=key, response=anthropic_message(message_id) + ) + mixed: Final = {**continue_from("msg_intruder"), "extra_body": {"thread": continue_from("msg_owner")["thread"]}} + + with pytest.raises(HTTPException) as rejected: + await ownership.async_pre_call_hook( + user_api_key_dict=INTRUDER, cache=DualCache(), data=mixed, call_type="acompletion" + ) + assert rejected.value.status_code == 404 + + +@pytest.mark.parametrize( + "create", + ( + pytest.param( + lambda ownership, data: ownership.async_post_call_success_hook( + data=data, user_api_key_dict=OWNER, response=anthropic_message("msg_body_create") + ), + id="reply", + ), + pytest.param( + lambda ownership, data: served( + ownership, (sse_frame(message_start("msg_body_create")), sse_frame({"type": "message_stop"})), data + ), + id="stream", + ), + ), +) +@pytest.mark.asyncio +async def test_a_thread_created_in_extra_body_stays_with_the_key_that_created_it(create) -> None: + ownership: Final = hook() + await create(ownership, {"messages": [], "extra_body": {"thread": CREATE}}) + + assert await allows(ownership, OWNER, "msg_body_create", "acompletion") + assert not await allows(ownership, INTRUDER, "msg_body_create", "acompletion") + + +@pytest.mark.asyncio +async def test_a_pass_through_continue_of_a_message_the_proxy_never_bound_is_left_to_the_upstream() -> None: + assert await allows(hook(), OWNER, "msg_unknown", "pass_through_endpoint") + assert await allows(hook(), INTRUDER, "msg_unknown", "pass_through_endpoint") + + +@pytest.mark.asyncio +async def test_a_pass_through_continue_of_another_keys_thread_is_thread_not_found() -> None: + ownership: Final = hook() + await ownership.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=OWNER, response=anthropic_message("msg_bound") + ) + + assert await allows(ownership, OWNER, "msg_bound", "pass_through_endpoint") + assert not await allows(ownership, INTRUDER, "msg_bound", "pass_through_endpoint") + + +@pytest.mark.asyncio +async def test_a_continue_resends_the_betas_its_create_turn_sent_next_to_its_own_extra_headers() -> None: + ownership: Final = hook() + await ownership.async_post_call_success_hook( + data=sent_with("advisor-tool-2026-03-01,message-threads-2026-08-12", thread=CREATE), + user_api_key_dict=OWNER, + response=anthropic_message("msg_advisor"), + ) + continued: Final = await ownership.async_pre_call_hook( + user_api_key_dict=OWNER, + cache=DualCache(), + data={ + **continue_from("msg_advisor"), + "extra_headers": {"Anthropic-Beta": "context-1m-2025-08-07", "x-trace": "abc"}, + }, + call_type="anthropic_messages", + ) + + assert continued is not None + assert continued["extra_headers"] == { + "x-trace": "abc", + "anthropic-beta": "advisor-tool-2026-03-01,context-1m-2025-08-07,message-threads-2026-08-12", + } + assert continued["thread"] == continue_from("msg_advisor")["thread"] + + +@pytest.mark.asyncio +async def test_the_create_turns_betas_reach_every_later_turn_of_the_thread() -> None: + ownership: Final = hook() + items: Final = (sse_frame(message_start("msg_streamed_first")), sse_frame({"type": "message_stop"})) + await served(ownership, items, sent_with("advisor-tool-2026-03-01,message-threads-2026-08-12", thread=CREATE)) + second: Final = await ownership.async_pre_call_hook( + user_api_key_dict=OWNER, cache=DualCache(), data=continue_from("msg_streamed_first"), call_type="anthropic_messages" + ) + assert second is not None + second_headers: Final = second["extra_headers"] + assert isinstance(second_headers, dict) + await ownership.async_post_call_success_hook( + data=sent_with(str(second_headers["anthropic-beta"]), **continue_from("msg_streamed_first")), + user_api_key_dict=OWNER, + response=anthropic_message("msg_second_turn"), + ) + third: Final = await ownership.async_pre_call_hook( + user_api_key_dict=OWNER, cache=DualCache(), data=continue_from("msg_second_turn"), call_type="anthropic_messages" + ) + + assert third is not None + assert third["extra_headers"] == {"anthropic-beta": "advisor-tool-2026-03-01,message-threads-2026-08-12"} + + +@pytest.mark.parametrize("call_type", ("acompletion", "aresponses")) +@pytest.mark.asyncio +async def test_a_continue_on_chat_or_responses_resends_the_betas_its_create_turn_sent( + call_type: CallTypesLiteral, +) -> None: + ownership: Final = hook() + await ownership.async_post_call_success_hook( + data=sent_with("advisor-tool-2026-03-01,message-threads-2026-08-12", thread=CREATE), + user_api_key_dict=OWNER, + response=anthropic_message("msg_other_route"), + ) + continued: Final = await ownership.async_pre_call_hook( + user_api_key_dict=OWNER, cache=DualCache(), data=continue_from("msg_other_route"), call_type=call_type + ) + + assert continued is not None + assert continued["extra_headers"] == {"anthropic-beta": "advisor-tool-2026-03-01,message-threads-2026-08-12"} + + +CLIENT_BETAS: Final = {"custom_llm_provider": "anthropic,azure_ai", "extra_headers": {"anthropic-beta": "context-1m-2025-08-07"}} +CLIENT_VERSION: Final = {"custom_llm_provider": "anthropic", "extra_headers": {"anthropic-version": "2023-06-01"}} +CLIENT_BETAS_WITH_THE_CREATES: Final = { + "custom_llm_provider": "anthropic,azure_ai", + "extra_headers": {"anthropic-beta": "advisor-tool-2026-03-01,context-1m-2025-08-07,message-threads-2026-08-12"}, +} + + +@pytest.mark.parametrize( + ("client_headers", "expected"), + ( + (CLIENT_BETAS, CLIENT_BETAS_WITH_THE_CREATES), + ([CLIENT_BETAS, CLIENT_VERSION], (CLIENT_BETAS_WITH_THE_CREATES, CLIENT_VERSION)), + (CLIENT_VERSION, CLIENT_VERSION), + ), +) +@pytest.mark.parametrize("call_type", ("acompletion", "aresponses", "anthropic_messages")) +@pytest.mark.asyncio +async def test_a_continue_that_sends_its_own_betas_keeps_the_create_turns_betas_next_to_them( + call_type: CallTypesLiteral, client_headers: object, expected: object +) -> None: + ownership: Final = hook() + await ownership.async_post_call_success_hook( + data=sent_with("advisor-tool-2026-03-01,message-threads-2026-08-12", thread=CREATE), + user_api_key_dict=OWNER, + response=anthropic_message("msg_client_betas"), + ) + continued: Final = await ownership.async_pre_call_hook( + user_api_key_dict=OWNER, + cache=DualCache(), + data={**continue_from("msg_client_betas"), "provider_specific_header": client_headers}, + call_type=call_type, + ) + + assert continued is not None + assert continued["provider_specific_header"] == expected + assert continued["extra_headers"] == {"anthropic-beta": "advisor-tool-2026-03-01,message-threads-2026-08-12"} + + +@pytest.mark.asyncio +async def test_a_pass_through_continue_is_checked_but_left_unchanged() -> None: + ownership: Final = hook() + await ownership.async_post_call_success_hook( + data=sent_with("advisor-tool-2026-03-01", thread=CREATE), + user_api_key_dict=OWNER, + response=anthropic_message("msg_other_route"), + ) + + assert ( + await ownership.async_pre_call_hook( + user_api_key_dict=OWNER, + cache=DualCache(), + data=continue_from("msg_other_route"), + call_type="pass_through_endpoint", + ) + is None + ) + + +@pytest.mark.asyncio +async def test_thread_bindings_never_push_the_proxys_rate_limit_counters_out_of_memory() -> None: + proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache()) + ownership: Final = MessageThreadOwnership(message_thread_owner_cache=proxy_logging.message_thread_owner_cache) + await proxy_logging.internal_usage_cache.async_set_cache( + key="rpm-counter", value=1, litellm_parent_otel_span=None, ttl=60 + ) + for turn in range(250): + await ownership.async_post_call_success_hook( + data={"thread": CREATE}, user_api_key_dict=OWNER, response=anthropic_message(f"msg_{turn}") + ) + + assert await proxy_logging.internal_usage_cache.async_get_cache(key="rpm-counter", litellm_parent_otel_span=None) == 1 + assert await allows(ownership, OWNER, "msg_0") + + +@pytest.mark.parametrize( + "stream", + ( + pytest.param(lambda frame: (frame,), id="whole_frame"), + pytest.param(lambda frame: (frame[:25], frame[25:60], frame[60:]), id="torn_frame"), + pytest.param(lambda frame: (frame.decode()[:30], frame.decode()[30:]), id="torn_text_frame"), + pytest.param( + lambda frame: (sse_frame({"type": "ping"}) + frame[:30], frame[30:]), id="after_a_ping_in_one_chunk" + ), + ), +) +@pytest.mark.asyncio +async def test_a_streamed_thread_turn_hands_its_message_to_its_key_and_streams_unchanged(stream) -> None: + ownership: Final = hook() + items: Final = (*stream(sse_frame(message_start("msg_streamed"))), sse_frame({"type": "message_stop"})) + + assert await served(ownership, items, {"thread": CREATE}) == list(items) + assert await allows(ownership, OWNER, "msg_streamed") + assert not await allows(ownership, INTRUDER, "msg_streamed") + + +@pytest.mark.asyncio +async def test_a_streamed_thread_turn_of_event_dicts_hands_its_message_to_its_key() -> None: + ownership: Final = hook() + items: Final = (message_start("msg_dict_stream"), {"type": "message_stop"}) + + assert await served(ownership, items, {"thread": CREATE}) == list(items) + assert await allows(ownership, OWNER, "msg_dict_stream") + + +@pytest.mark.asyncio +async def test_a_stream_without_a_thread_hands_out_nothing() -> None: + ownership: Final = hook() + items: Final = (sse_frame(message_start("msg_plain_stream")), sse_frame({"type": "message_stop"})) + + assert await served(ownership, items, {}) == list(items) + assert not await allows(ownership, OWNER, "msg_plain_stream") + + +@pytest.mark.asyncio +async def test_a_message_start_after_another_event_is_not_read_as_the_threads_message() -> None: + ownership: Final = hook() + items: Final = ( + sse_frame({"type": "content_block_start", "index": 0}), + sse_frame(message_start("msg_late")), + ) + + assert await served(ownership, items, {"thread": CREATE}) == list(items) + assert not await allows(ownership, OWNER, "msg_late") diff --git a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py index 26ee9f04d01..ff800b24912 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py @@ -454,6 +454,7 @@ def test_every_pre_call_customlogger_is_deliberately_classified(): "_PROXY_BatchRedisRequests", "_PROXY_SensitiveDataRoutingHandler", "ResponsesIDSecurity", + "MessageThreadOwnership", "SkillsInjectionHook", "_PROXY_LiteLLMManagedFiles", "_PROXY_LiteLLMManagedVectorStores", diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index e16a8e01d81..3a61ca48f26 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -3411,6 +3411,75 @@ def test_the_refusal_names_each_cooled_deployment_some_cooldown_state_routes_the assert {deployment["model_id"] for deployment in refusal.value.cooldown_deployments or ()} == reachable +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +class TestMessageThreadNotFoundCooldown: + THREAD_NOT_FOUND_BODY: Final = ( + '{"type":"error","error":{"type":"not_found_error","message":"No thread state was found for the ' + 'requested `previous_message_id`.","details":{"error_code":"thread_not_found"}}}' + ) + + @staticmethod + def _not_found(message: str) -> Exception: + return litellm.NotFoundError( + message=f"AnthropicException - {message}", + model="claude-opus-4-8", + llm_provider="anthropic", + ) + + @pytest.mark.asyncio + async def test_a_stale_thread_404_leaves_the_deployment_out_of_cooldown(self) -> None: + model_id: Final = "anthropic-threads" + router: Final = _make_router( + model_list=[ + { + "model_name": "claude", + "litellm_params": {"model": "anthropic/claude-opus-4-8", "api_key": "synthetic-key"}, + "model_info": {"id": model_id}, + } + ], + ) + + result: Final = router.deployment_callback_on_failure( + kwargs={ + "exception": self._not_found(self.THREAD_NOT_FOUND_BODY), + "litellm_params": {"model_info": {"id": model_id}}, + }, + completion_response=None, + start_time=0, + end_time=1, + ) + + assert result is False + assert get_cooldown_deployments(RoutingContext(router), parent_otel_span=None) == [] + + @pytest.mark.asyncio + async def test_a_404_without_the_thread_code_still_cools_the_deployment_down(self) -> None: + model_id: Final = "anthropic-missing-model" + router: Final = _make_router( + model_list=[ + { + "model_name": "claude", + "litellm_params": {"model": "anthropic/claude-opus-4-8", "api_key": "synthetic-key"}, + "model_info": {"id": model_id}, + } + ], + ) + + router.deployment_callback_on_failure( + kwargs={ + "exception": self._not_found( + '{"type":"error","error":{"type":"not_found_error","message":"model: x"}}' + ), + "litellm_params": {"model_info": {"id": model_id}}, + }, + completion_response=None, + start_time=0, + end_time=1, + ) + + assert get_cooldown_deployments(RoutingContext(router), parent_otel_span=None) == [model_id] + + @pytest.mark.parametrize( ("status_code", "call_type", "expected"), [ From 373fe799e723ed350c724ea2908d878527de2936 Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:35:19 -0800 Subject: [PATCH 066/189] fix(router): send managed batch uploads to batch deployments when a chat deployment has a lower order (#50971) * fix(router): send managed batch uploads to batch deployments when a chat deployment has a lower order A purpose=batch upload on a mixed Azure chat and batch group was order-filtered before batch routing narrowed it, so the file landed only on the order-1 chat deployment and the batch create that followed got Azure's 400 invalid_deployment_type. The upload now runs under the same batch routing as the batch create. * fix(router): keep a failed batch upload's cooldown and order fallback on the batch deployments * fix(router): keep a batch upload off chat deployments while its batch deployment cools down * fix(router): keep a refused batch upload's retry off the chat deployment * fix(router): keep a refused batch upload's error and skip chat while the batch deployment is paused * test(router): pin the order fallback for a batch create on a group with no batch deployment * test(integration): expect an Azure batch upload on the batch deployment alone --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../crates/python-bridge/src/router/config.rs | 2 + litellm-rust/crates/router-core/src/call.rs | 10 +- litellm-rust/crates/router-core/src/host.rs | 4 + litellm-rust/crates/router-core/src/select.rs | 33 +++- .../crates/router-core/tests/call_loop.rs | 163 +++++++++++++++++ litellm/router_rust/core.py | 24 ++- litellm/router_rust/invoke/endpoints.py | 6 + .../test_batch_mode_deployments_wire.py | 3 +- tests/unit/test_router/test_router.py | 166 ++++++++++++++++++ 9 files changed, 398 insertions(+), 13 deletions(-) diff --git a/litellm-rust/crates/python-bridge/src/router/config.rs b/litellm-rust/crates/python-bridge/src/router/config.rs index de383a0ffa1..5e099b8841c 100644 --- a/litellm-rust/crates/python-bridge/src/router/config.rs +++ b/litellm-rust/crates/python-bridge/src/router/config.rs @@ -189,6 +189,7 @@ struct RequestSpec { route_handler: Option, decisions_media: Option, batch_create: Option, + batch_upload: Option, embedding_batch: Option, wildcard_request: Option, unrouted: Option, @@ -258,6 +259,7 @@ pub(crate) fn parse_request(value: Value, model: Option<&str>) -> Result = group_members(snapshot, order_group, listing_team) - .iter() + let members: Vec<_> = group_members(snapshot, order_group, listing_team) + .into_iter() .filter(|d| d.listed_for_team(listing_team)) + .collect(); + let batch_levels_only = + request.batch_create && members.iter().any(|d| d.is_batch_capable()); + let mut orders: Vec = members + .iter() + .filter(|d| !batch_levels_only || !d.cannot_run_batches()) .filter_map(|d| d.order()) .collect(); orders.sort_unstable(); diff --git a/litellm-rust/crates/router-core/src/host.rs b/litellm-rust/crates/router-core/src/host.rs index ae93288b0fa..66b99abbeda 100644 --- a/litellm-rust/crates/router-core/src/host.rs +++ b/litellm-rust/crates/router-core/src/host.rs @@ -87,6 +87,10 @@ pub struct RouterRequest { /// `model_info.mode: batch`, the Azure deployments that are not are skipped /// (`without_deployments_that_cannot_batch`). pub batch_create: bool, + /// A batch file upload: the batch create's narrowing also holds while those batch deployments + /// cool down, are paused, or fail their health check, so the file never lands on a deployment + /// that cannot run its batch. + pub batch_upload: bool, /// An embedding pick: the host keeps the candidates whose `max_embedding_batch_size` admits /// the input (`deployments_admitting_embedding_batch`). pub embedding_batch: bool, diff --git a/litellm-rust/crates/router-core/src/select.rs b/litellm-rust/crates/router-core/src/select.rs index 1cc32d23b57..43e42371248 100644 --- a/litellm-rust/crates/router-core/src/select.rs +++ b/litellm-rust/crates/router-core/src/select.rs @@ -232,6 +232,25 @@ fn filter_team_based( .collect()) } +/// `without_deployments_that_cannot_batch`: a batch create on a group the caller may use a healthy +/// batch deployment of skips the Azure deployments that cannot run batches. A batch upload skips +/// them while the caller may use any batch deployment of the group, healthy or not, and the retry +/// count reads the same list. +fn without_deployments_that_cannot_batch( + request: &RouterRequest, + all: &[Arc], + healthy: Vec>, +) -> Vec> { + let has_batch = |pool: &[Arc]| pool.iter().any(|d| d.is_batch_capable()); + if !request.batch_create || !(has_batch(&healthy) || (request.batch_upload && has_batch(all))) { + return healthy; + } + healthy + .into_iter() + .filter(|d| !d.cannot_run_batches()) + .collect() +} + impl Core { fn context<'a>( &'a self, @@ -482,6 +501,11 @@ impl Core { None }; if inputs.count_only { + let healthy = if resolution.kind.is_single() { + healthy + } else { + without_deployments_that_cannot_batch(request, &all, healthy) + }; return Ok(Candidates { name: name.to_owned(), model_group: resolution.group, @@ -646,13 +670,8 @@ impl Core { .await?; } - // `without_deployments_that_cannot_batch`: a batch create on a group the caller may use a - // batch deployment of skips the Azure deployments that cannot run batches. - if request.batch_create - && !resolution.kind.is_single() - && healthy.iter().any(|d| d.is_batch_capable()) - { - healthy.retain(|d| !d.cannot_run_batches()); + if !resolution.kind.is_single() { + healthy = without_deployments_that_cannot_batch(request, &all, healthy); } if let Some(pinned) = [ diff --git a/litellm-rust/crates/router-core/tests/call_loop.rs b/litellm-rust/crates/router-core/tests/call_loop.rs index fce612ebdc5..732eb13371b 100644 --- a/litellm-rust/crates/router-core/tests/call_loop.rs +++ b/litellm-rust/crates/router-core/tests/call_loop.rs @@ -1245,6 +1245,169 @@ async fn a_served_fallback_and_a_disabled_walk_report_no_failure(config: RouterC assert_eq!(fallbacks_failed(&host), vec![]); } +fn azure_chat_and_batch_tiers() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "azure/chat", "order": 1}, "model_info": {"id": "c1", "mode": "chat"}}, + {"model_name": "primary", "litellm_params": {"model": "azure/batch-a", "order": 2}, "model_info": {"id": "b2", "mode": "batch"}}, + {"model_name": "primary", "litellm_params": {"model": "azure/batch-b", "order": 3}, "model_info": {"id": "b3", "mode": "batch"}}, + ] + })) + .unwrap() +} + +fn azure_chat_tiers() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "azure/chat-a", "order": 1}, "model_info": {"id": "c1", "mode": "chat"}}, + {"model_name": "primary", "litellm_params": {"model": "azure/chat-b", "order": 2}, "model_info": {"id": "c2", "mode": "chat"}}, + ] + })) + .unwrap() +} + +#[rstest] +#[case::batch_create_moves_past_the_level_it_used(azure_chat_and_batch_tiers(), true, vec!["b2", "b3"])] +#[case::other_calls_walk_every_level(azure_chat_and_batch_tiers(), false, vec!["c1", "b2"])] +#[case::batch_create_without_a_batch_deployment_walks_every_level(azure_chat_tiers(), true, vec!["c1", "c2"])] +#[tokio::test] +async fn a_failed_order_level_falls_back_to_the_next_level_its_pick_can_use( + #[case] config: RouterConfig, + #[case] batch_create: bool, + #[case] expected: Vec<&str>, +) { + let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); + let router = router(config, json!({"num_retries": 0}), host.clone()); + let request = RouterRequest { + batch_create, + ..request() + }; + router + .call(Op::from("acreate_batch"), request) + .await + .unwrap(); + assert_eq!(host.selected(), expected); +} + +fn azure_chat_then_batch() -> RouterConfig { + serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "azure/chat", "order": 1}, "model_info": {"id": "c1", "mode": "chat"}}, + {"model_name": "primary", "litellm_params": {"model": "azure/batch", "order": 2}, "model_info": {"id": "b2", "mode": "batch"}}, + ] + })) + .unwrap() +} + +#[rstest] +#[case::a_batch_upload_finds_no_deployment(true, vec![])] +#[case::a_batch_create_falls_back_to_chat(false, vec!["c1"])] +#[tokio::test] +async fn while_the_batch_deployment_cools_down_a_batch_upload_finds_no_healthy_deployment( + #[case] batch_upload: bool, + #[case] expected: Vec<&str>, +) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![failure(FailureKind::RateLimit, 429)], + )); + let router = router( + azure_chat_then_batch(), + json!({"num_retries": 0}), + host.clone(), + ); + let create = RouterRequest { + batch_create: true, + ..request() + }; + router + .call(Op::from("acreate_file"), create.clone()) + .await + .unwrap_err(); + let upload = RouterRequest { + batch_upload, + ..create + }; + let healthy: Vec = router + .healthy(&Op::from("acreate_file"), &upload) + .await + .map(|candidates| { + candidates + .healthy + .iter() + .map(|d| d.id.to_string()) + .collect() + }) + .unwrap_or_default(); + assert_eq!(host.selected(), vec!["b2"]); + assert_eq!(healthy, expected); +} + +#[rstest] +#[case::a_batch_upload_finds_no_deployment(true, vec![])] +#[case::a_batch_create_falls_back_to_chat(false, vec!["c1"])] +#[tokio::test] +async fn while_the_batch_deployment_is_paused_a_batch_upload_finds_no_healthy_deployment( + #[case] batch_upload: bool, + #[case] expected: Vec<&str>, +) { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "primary", "litellm_params": {"model": "azure/chat", "order": 1}, "model_info": {"id": "c1", "mode": "chat"}}, + {"model_name": "primary", "litellm_params": {"model": "azure/batch", "order": 2}, "model_info": {"id": "b2", "mode": "batch", "blocked": true}}, + ] + })) + .unwrap(); + let router = router(config, json!({}), Arc::new(ScriptedHost::default())); + let upload = RouterRequest { + batch_create: true, + batch_upload, + ..request() + }; + let healthy: Vec = router + .healthy(&Op::from("acreate_file"), &upload) + .await + .map(|candidates| { + candidates + .healthy + .iter() + .map(|d| d.id.to_string()) + .collect() + }) + .unwrap_or_default(); + assert_eq!(healthy, expected); +} + +#[rstest] +#[case::a_rate_limit(FailureKind::RateLimit, 429)] +#[case::a_wrong_key(FailureKind::Authentication, 401)] +#[tokio::test] +async fn a_batch_upload_the_batch_deployment_refuses_keeps_its_error_with_retries_on( + #[case] kind: FailureKind, + #[case] status: u16, +) { + let host = Arc::new(ScriptedHost::failing( + "primary", + (0..3).map(|_| failure(kind, status)).collect(), + )); + let router = router( + azure_chat_then_batch(), + json!({"num_retries": 2}), + host.clone(), + ); + let upload = RouterRequest { + batch_create: true, + batch_upload: true, + ..request() + }; + let error = router + .call(Op::from("acreate_file"), upload) + .await + .unwrap_err(); + assert_eq!(host.selected(), vec!["b2"]); + assert!(matches!(error, Error::Attempt(refused) if refused.kind == kind)); +} + #[tokio::test] async fn the_order_path_reports_no_failed_fallbacks() { let host = Arc::new(ScriptedHost::failing("primary", vec![server_error()])); diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index aef4db05ca9..69724f0b9fe 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -102,6 +102,7 @@ from litellm.router_rust.invoke.endpoints import ( endpoint_names, failover_fields, generic_plan, + is_batch_upload, log_refused_call, log_refused_call_sync, pinned_vector_store_file_deployment, @@ -1375,9 +1376,15 @@ class _EndpointPort: kind: str = "pick", specific_deployment: bool | None = None, messages: object = None, + batch_upload: bool = False, ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: return await self._core.select( - model, request_kwargs, kind=kind, specific_deployment=specific_deployment, messages=messages + model, + request_kwargs, + kind=kind, + specific_deployment=specific_deployment, + messages=messages, + batch_upload=batch_upload, ) async def acompletion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload @@ -2437,6 +2444,7 @@ class RustCore( messages: object = None, input: object = None, health_check_probe: bool = False, + batch_upload: bool = False, **_: object, # kwargs-ok: the RouterCore select options this core ignores ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: """Pick a deployment for ``model`` without calling it (``pick``), list the deployments a @@ -2455,6 +2463,7 @@ class RustCore( messages, input, probe=kind == "healthy" and health_check_probe, + batch_upload=batch_upload, ) # A pick that makes no call holds no least-busy reservation (`select` on the Python core). with picks_reserve(False): @@ -2758,8 +2767,9 @@ class RustCore( **({"web_search": True} if web_search else {}), **({"route_handler": handler} if serves_route_capability(handler) else {}), **({"decisions_media": True} if decisions_media else {}), - # `batch_create_routing`: a batch create keeps to the deployments that can run batches. - **({"batch_create": True} if plan.kind == "batch" else {}), + # `batch_create_routing`: a batch create and its file upload keep to the deployments that can run batches. + **({"batch_create": True} if _routes_as_batch_create(plan, request) else {}), + **({"batch_upload": True} if plan.kind == "create_file" and is_batch_upload(request) else {}), # `deployments_admitting_embedding_batch`: an embedding keeps to the deployments admitting its input. **({"embedding_batch": True} if plan.kind == "embedding" else {}), } @@ -4045,6 +4055,7 @@ class _Selection: input: object, *, probe: bool = False, + batch_upload: bool = False, ) -> None: self._core: Final = core # `_async_get_available_deployment` reads the alias map first, every other pick its names. @@ -4076,6 +4087,9 @@ class _Selection: fields["endpoint"] = endpoint if probe: fields["health_check_probe"] = True + if batch_upload: + fields["batch_create"] = True + fields["batch_upload"] = True if kind in ("pick", "async_pass_through") or (kind == "sync_pick" and core._custom_strategy is not None): # pyright: ignore[reportPrivateUsage] # the selection serves its own core fields["pre_routes"] = True self._specific_deployment: Final = specific_deployment @@ -4289,6 +4303,10 @@ def _narrowing_fields(request: dict[str, object], kind: str) -> Mapping[str, obj return fields +def _routes_as_batch_create(plan: Plan, request: Mapping[str, object]) -> bool: + return plan.kind == "batch" or (plan.kind == "create_file" and is_batch_upload(request)) + + def _request_narrowing(request: Mapping[str, object]) -> Mapping[str, object]: """The narrowing keys a request carries, as the fields router-core reads them.""" from litellm.litellm_core_utils.routing_common_utils import ( diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index a5307ff8af9..fd3d055f707 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -147,6 +147,7 @@ class EndpointCore(Protocol): kind: str = "pick", specific_deployment: bool | None = None, messages: object = None, + batch_upload: bool = False, ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: ... async def acompletion(self, *args: object, **kwargs: object) -> object: ... # kwargs-ok: endpoint payload @@ -677,6 +678,10 @@ _DIRECT_ENDPOINTS: Final = frozenset( ) +def is_batch_upload(request: Mapping[str, object]) -> bool: + return request.get("purpose") == "batch" + + def endpoint_names() -> frozenset[str]: """Every endpoint name ``RustEndpoints.member`` may serve: what ``dir()`` of a router lists.""" return frozenset( @@ -2313,6 +2318,7 @@ class RustEndpoints: kind="healthy", specific_deployment=bool(specific) or None, messages=_user_message(_PICK_TEXTS["create_file"]), + batch_upload=is_batch_upload(kwargs), ) deployments: Final = list(healthy) if isinstance(healthy, Sequence) else [healthy] diff --git a/tests/integration/routing/test_batch_mode_deployments_wire.py b/tests/integration/routing/test_batch_mode_deployments_wire.py index 7d702f499f5..648d3f2635f 100644 --- a/tests/integration/routing/test_batch_mode_deployments_wire.py +++ b/tests/integration/routing/test_batch_mode_deployments_wire.py @@ -249,7 +249,8 @@ def test_a_batch_create_on_a_mixed_group_reaches_the_deployment_its_provider_all with wire_server(_vendor) as wire, gateway.scenario() as scenario: group: Final = _group(scenario, wire, (chat, batch)) managed_file: Final = _managed_upload(gateway, group, gateway.key) - assert set(_keys_posted_to(wire.drain(), "/files")) == {chat.key, batch.key} + uploads_reach: Final = {batch.key} if creates_reach == "batch" else {chat.key, batch.key} + assert set(_keys_posted_to(wire.drain(), "/files")) == uploads_reach creates: Final = tuple(_create(gateway, managed_file, gateway.key) for _ in range(3)) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index e84d187f937..ba928bfb8c3 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -7968,6 +7968,172 @@ async def test_acreate_batch_on_a_mixed_group_only_reaches_batch_deployments( assert batch_route.call_count == 20 +def _ordered(deployment: dict[str, object], order: int) -> dict[str, object]: + return {**deployment, "litellm_params": {**deployment["litellm_params"], "order": order}} + + +_AZURE_FILE_CREATED: Final = { + "id": "file-batch-input", + "object": "file", + "bytes": 1, + "created_at": 0, + "filename": "input.jsonl", + "purpose": "batch", + "status": "processed", +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("purpose", "uploaded_to"), [("batch", "batch-deployment"), ("fine-tune", "chat-deployment")]) +async def test_batch_file_upload_reaches_the_batch_deployment_when_the_chat_deployment_has_a_lower_order( + monkeypatch: pytest.MonkeyPatch, purpose: str, uploaded_to: str +): + from io import BytesIO + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + _ordered(_azure_sku_deployment("gpt-mixed", "chat-sku", "chat", id="chat-deployment"), 1), + _ordered(_azure_sku_deployment("gpt-mixed", "batch-sku", "batch", id="batch-deployment"), 2), + ], + num_retries=0, + ) + batch_row: Final = {"custom_id": "1", "method": "POST", "url": "/chat/completions", "body": {"model": "gpt-mixed"}} + + with respx.mock(assert_all_called=False) as respx_mock: + routes: Final = { + deployment_id: respx_mock.post(host=f"{sku_host}.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_FILE_CREATED) + ) + for deployment_id, sku_host in (("chat-deployment", "chat-sku"), ("batch-deployment", "batch-sku")) + } + uploaded: Final = await router.acreate_file( + model="gpt-mixed", purpose=purpose, file=BytesIO(json.dumps(batch_row).encode()) + ) + + assert {deployment_id: route.call_count for deployment_id, route in routes.items()} == { + "chat-deployment": int(uploaded_to == "chat-deployment"), + "batch-deployment": int(uploaded_to == "batch-deployment"), + } + assert uploaded._hidden_params["model_file_id_mapping"] == {uploaded_to: "file-batch-input"} + + +_AZURE_CHAT_COMPLETION: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "gpt-mixed", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "ok"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("status", "raised"), [(429, openai.RateLimitError), (500, openai.InternalServerError)]) +async def test_a_failed_batch_file_upload_tries_the_batch_deployment_once_and_leaves_chat_serving( + monkeypatch: pytest.MonkeyPatch, status: int, raised: type[Exception] +): + from io import BytesIO + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + _ordered(_azure_sku_deployment("gpt-mixed", "chat-sku", "chat", id="chat-deployment"), 1), + _ordered(_azure_sku_deployment("gpt-mixed", "batch-sku", "batch", id="batch-deployment"), 2), + ], + num_retries=0, + ) + batch_row: Final = {"custom_id": "1", "method": "POST", "url": "/chat/completions", "body": {"model": "gpt-mixed"}} + + with respx.mock(assert_all_called=False) as respx_mock: + batch_upload: Final = respx_mock.post(host="batch-sku.openai.azure.com").mock( + return_value=httpx.Response(status, json={"error": {"code": str(status), "message": "upload refused"}}) + ) + chat_completions: Final = respx_mock.post(host="chat-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_CHAT_COMPLETION) + ) + with pytest.raises(raised): + await router.acreate_file(model="gpt-mixed", purpose="batch", file=BytesIO(json.dumps(batch_row).encode())) + for _ in range(3): + await router.acompletion(model="gpt-mixed", messages=[{"role": "user", "content": "hi"}]) + + assert batch_upload.call_count == 1 + assert chat_completions.call_count == 3 + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("status", "raised"), [(429, openai.RateLimitError), (401, openai.AuthenticationError)]) +async def test_a_batch_file_upload_the_batch_deployment_refuses_stays_off_chat_and_leaves_chat_serving( + monkeypatch: pytest.MonkeyPatch, status: int, raised: type[Exception] +): + from io import BytesIO + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + _ordered(_azure_sku_deployment("gpt-mixed", "chat-sku", "chat", id="chat-deployment"), 1), + _ordered(_azure_sku_deployment("gpt-mixed", "batch-sku", "batch", id="batch-deployment"), 2), + ], + num_retries=2, + ) + batch_row: Final = {"custom_id": "1", "method": "POST", "url": "/chat/completions", "body": {"model": "gpt-mixed"}} + + with respx.mock(assert_all_called=False) as respx_mock: + batch_upload: Final = respx_mock.post(host="batch-sku.openai.azure.com").mock( + return_value=httpx.Response(status, json={"error": {"code": str(status), "message": "upload refused"}}) + ) + chat_upload: Final = respx_mock.post(host="chat-sku.openai.azure.com", path__regex=r"/files").mock( + return_value=httpx.Response(200, json=_AZURE_FILE_CREATED) + ) + chat_completions: Final = respx_mock.post(host="chat-sku.openai.azure.com", path__regex=r"/chat/completions").mock( + return_value=httpx.Response(200, json=_AZURE_CHAT_COMPLETION) + ) + with pytest.raises(raised, match="upload refused"): + await router.acreate_file(model="gpt-mixed", purpose="batch", file=BytesIO(json.dumps(batch_row).encode())) + await asyncio.gather(*(task for task in asyncio.all_tasks() if task is not asyncio.current_task())) + for _ in range(3): + await router.acompletion(model="gpt-mixed", messages=[{"role": "user", "content": "hi"}]) + + assert (batch_upload.call_count, chat_upload.call_count, chat_completions.call_count) == (1, 0, 3) + + +@pytest.mark.asyncio +async def test_a_batch_file_upload_whose_batch_deployment_cools_down_after_the_pick_stays_off_chat( + monkeypatch: pytest.MonkeyPatch, +): + from io import BytesIO + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + _ordered(_azure_sku_deployment("gpt-mixed", "chat-sku", "chat", id="chat-deployment"), 1), + _ordered(_azure_sku_deployment("gpt-mixed", "batch-sku", "batch", id="batch-deployment"), 2), + ], + num_retries=0, + ) + + class _ConcurrentBatchFailure(CustomLogger): + async def async_pre_call_check(self, deployment, parent_otel_span): + _cool_down(router, "batch-deployment") + + batch_row: Final = {"custom_id": "1", "method": "POST", "url": "/chat/completions", "body": {"model": "gpt-mixed"}} + + with ( + patch.object(litellm, "callbacks", [_ConcurrentBatchFailure()]), # test-quality-ok: router reads this global + respx.mock(assert_all_called=False) as respx_mock, + ): + batch_upload: Final = respx_mock.post(host="batch-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_FILE_CREATED) + ) + chat_upload: Final = respx_mock.post(host="chat-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_FILE_CREATED) + ) + with pytest.raises(RouterRateLimitError, match="No deployments available"): + await router.acreate_file(model="gpt-mixed", purpose="batch", file=BytesIO(json.dumps(batch_row).encode())) + + assert (batch_upload.call_count, chat_upload.call_count) == (0, 0) + + def _azure_wildcard_deployment(sku_host: str, mode: str) -> dict[str, object]: return { "model_name": "mixed-wild/*", From 917fd1196680b812f7ead048bf8effa300ebcbb7 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 18:36:55 -0800 Subject: [PATCH 067/189] fix(rate-limit): settle a fallback's tokens on the model group that served it (#51082) * fix(rate-limit): settle a fallback's tokens on the model group that served it The v3 limiter pinned post-call TPM settlement to the model reserved at admission, so when the router fell back to another model group the requested model's counter took the usage and the serving model's counter stayed empty. When the served model group differs from the admitted one, charge the served group and refund the abandoned group's per-model reservation. Same-model retries and alias reloads keep the admission target. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(rate-limit): refund an abandoned fallback reservation only on counters the served model does not settle Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(rate-limit): wait for the primary refund and the fallback charge before reading counters Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(rate-limit): only treat a limiter-admitted fallback target as the served model at settlement Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(rate-limit): drop unrelated formatting from the v3 limiter unit test file Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(rate-limit): remove helper copies duplicated while appending the fallback settlement tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../hooks/parallel_request_limiter_v3.py | 54 +++++-- .../test_fallback_settles_serving_model.py | 91 +++++++++++ .../hooks/test_parallel_request_limiter_v3.py | 141 ++++++++++++++++++ 3 files changed, 277 insertions(+), 9 deletions(-) create mode 100644 tests/integration/routing/test_fallback_settles_serving_model.py diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a3227bca235..14b9c48b0d6 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -500,6 +500,7 @@ class RequestRateLimiterStash: reserved_tokens: int = 0 reserved_model: RateLimitedModel | None = None reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset) + fallback_models: frozenset[str] = field(default_factory=frozenset) ptu_ceiling: PTUTeamCeiling | None = None ptu_reserved_tokens: int = 0 ptu_reserved_window_identities: frozenset[ReservationWindowIdentity] = field(default_factory=frozenset) @@ -3786,6 +3787,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) -> bool: """Reserve the caller's per-model limits on a router fallback target. False when any is exhausted.""" data: Final = MappingProxyType({**request_data, "model": model}) + stash: Final = _request_stash.get() hidden_stash: Final = _request_stash.set(None) try: built: Final = await self._build_request_rate_limit_descriptors(user_api_key_dict, data, None) @@ -3794,10 +3796,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptors: Final = [ descriptor for descriptor in built if descriptor["key"] in PER_MODEL_RATE_LIMIT_DESCRIPTOR_KEYS ] - if not descriptors: - return True - response: Final = await self.should_rate_limit(descriptors, parent_otel_span=user_api_key_dict.parent_otel_span) - return response["overall_code"] == "OK" + allowed: Final = ( + not descriptors + or (await self.should_rate_limit(descriptors, parent_otel_span=user_api_key_dict.parent_otel_span))[ + "overall_code" + ] + == "OK" + ) + if allowed and stash is not None: + stash.fallback_models = stash.fallback_models | {model} + return allowed @asynccontextmanager async def request_capacity( @@ -4720,12 +4728,21 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): reserved_tokens: Final = open_reservation.reserved_tokens if open_reservation is not None else 0 reserved_model: Final = stash.reserved_model if stash is not None else None reserved_scopes: Final[frozenset[tuple[str, str]]] = stash.reserved_scopes if stash is not None else frozenset() - # Reconciliation must target the same model-scoped counter that the - # pre-call reservation incremented. If a reservation was made, + served_model: Final = self._rate_limited_model(model_group) # ``reserved_model`` (resolved at admission, so an alias map reload - # mid-flight cannot move the charge) is authoritative; otherwise fall - # back to the router's ``model_group`` (the no-reservation charge path). - reconcile_model: Final = reserved_model if reserved_model is not None else self._rate_limited_model(model_group) + # mid-flight cannot move the charge) settles the request unless the + # router fell back to a group this limiter admitted as a fallback + # target: then the served group is charged and the abandoned group's + # per-model reservation is refunded. Client-supplied ``model_group`` + # metadata alone never moves the charge. + fell_back: Final = ( + stash is not None + and reserved_model is not None + and served_model is not None + and served_model.requested != reserved_model.requested + and served_model.requested in stash.fallback_models + ) + reconcile_model: Final = served_model if fell_back or reserved_model is None else reserved_model reconcile_models: Final = self._rate_limited_models(reconcile_model) pipeline_operations: Final[list[ReservationOperation]] = [] @@ -4764,6 +4781,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) ) charged_targets: Final = [target for target in targets if target not in key_owned_team_scopes] + abandoned_scopes: Final = ( + tuple( + sorted( + scope + for scope in reserved_scopes + if scope[0] in PER_MODEL_RATE_LIMIT_DESCRIPTOR_KEYS and scope not in charged_targets + ) + ) + if fell_back + else () + ) if reserved_tokens > 0 and total_tokens < reserved_tokens: verbose_proxy_logger.debug( "Releasing unused TPM budget on success: reserved=%s, actual=%s, release=%s", @@ -4779,6 +4807,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): reserved_tokens=reserved_tokens, ) ) + pipeline_operations.extend( + self._build_reservation_aware_tpm_ops( + targets=abandoned_scopes, + reserved_scopes=reserved_scopes, + actual_tokens=0, + reserved_tokens=reserved_tokens, + ) + ) call_type: Final[object] = kwargs.get("call_type") ptu_skips_socket_usage: Final = call_type == CallTypes.aresponses_websocket.value pipeline_operations.extend( diff --git a/tests/integration/routing/test_fallback_settles_serving_model.py b/tests/integration/routing/test_fallback_settles_serving_model.py new file mode 100644 index 00000000000..b9c32f7e91f --- /dev/null +++ b/tests/integration/routing/test_fallback_settles_serving_model.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import os +import time +import uuid +from collections.abc import Iterator +from contextlib import ExitStack +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import OwnedState, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import wire_server +from integration.routing.test_fallback_rate_limit_check import provider_error_reply +from integration.routing.test_per_model_rate_limit_fallbacks import ( + assert_served, + chat, + healthy_reply, + model_entry, + proxy_config, +) +from redis import Redis + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +ADMISSION_PROBE_TOKENS: Final = 1 +SETTLEMENT_DEADLINE_SECONDS: Final = 15.0 + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + primary: str + fallback: str + + +def key_model_tokens(group: str) -> int: + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), socket_timeout=2) as cache: + counters: Final = tuple(cache.scan_iter(match=f"*model_per_key:*:{group}}}:tokens")) + return sum(int(cache.get(counter) or 0) for counter in counters) + + +def settled_key_model_tokens(primary: str, fallback: str, total_tokens: int) -> tuple[int, int]: + deadline: Final = time.monotonic() + SETTLEMENT_DEADLINE_SECONDS + while True: + charged: Final = (key_model_tokens(primary), key_model_tokens(fallback)) + if (charged[0] == 0 and charged[1] >= total_tokens) or time.monotonic() >= deadline: + return charged + time.sleep(0.5) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + suffix: Final = uuid.uuid4().hex[:10] + primary: Final = f"settle-primary-{suffix}" + fallback: Final = f"settle-fallback-{suffix}" + directory: Final = tmp_path_factory.mktemp("fallback-settles-serving-model") + with ExitStack() as stack: + rig_gateway: Final = stack.enter_context(gateway_from_environment()) + primary_wire: Final = stack.enter_context(wire_server(provider_error_reply)) + fallback_wire: Final = stack.enter_context(wire_server(healthy_reply)) + config: Final = proxy_config( + directory, + model_list=(model_entry(primary, primary_wire), model_entry(fallback, fallback_wire)), + fallbacks=({primary: [fallback]},), + general_settings={}, + ) + owned: Final = stack.enter_context( + owned_proxy_process( + rig_gateway, directory, {}, config=config, workers=2, state=OwnedState.SHARED_WITH_WORKER + ) + ) + yield Rig(owned.gateway, primary, fallback) + + +def test_provider_error_fallback_charges_its_tokens_to_the_model_that_served(rig: Rig) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(model_tpm_limit={rig.primary: 100_000, rig.fallback: 1_000}) + + response: Final = chat(rig.gateway, key, rig.primary, f"pmrl-{uuid.uuid4().hex}") + assert_served(response, rig.fallback, "served pmrl-") + total_tokens: Final = int(response.json()["usage"]["total_tokens"]) + assert total_tokens > 0, response.text + + primary_tokens, fallback_tokens = settled_key_model_tokens(rig.primary, rig.fallback, total_tokens) + assert primary_tokens == 0, (primary_tokens, fallback_tokens) + assert total_tokens <= fallback_tokens <= total_tokens + ADMISSION_PROBE_TOKENS, ( + primary_tokens, + fallback_tokens, + ) diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index a39d02a2788..e3a63d097ec 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -9941,3 +9941,144 @@ async def test_end_user_model_rpm_limit_is_shared_across_keys_and_isolated_per_c await _rpm_request(handler, cache, globex, "target") await _rpm_request(handler, cache, acme_via_key_one, "other-model") + + + +@pytest.mark.asyncio +async def test_success_tpm_accounting_moves_the_charge_to_the_model_group_that_served_a_fallback() -> None: + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + key_metadata: Final = {"model_tpm_limit": {"primary": 1000, "fallback": 1000}} + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-fallback-settle"), metadata=key_metadata) + + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": "primary", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}, + call_type="acompletion", + ) + stash: Final = get_request_stash() + assert stash is not None + assert stash.reserved_model == RateLimitedModel(requested="primary", group="primary") + assert stash.reserved_tokens > 0 + assert await handler.async_fallback_target_allowed( + auth, "fallback", {"messages": [{"role": "user", "content": "hello"}]} + ) + + response: Final = ModelResponse( + id="fallback-settle", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="fallback", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + choices=[], + ) + kwargs: Final = { + "standard_logging_object": {"metadata": {"user_api_key_hash": auth.api_key}}, + "litellm_params": { + "metadata": { + "model_group": "fallback", + "original_model_group": "primary", + "user_api_key_metadata": key_metadata, + } + }, + "model": "fallback", + } + + ops: Final = handler._build_success_event_pipeline_operations( + kwargs=kwargs, response_obj=response, rate_limit_type="total" + ) + + charged: Final = {_increment_of(op).counter: _increment_of(op).amount for op in ops} + assert charged[_counter("model_per_key", f"{auth.api_key}:fallback")] == 150 + assert charged[_counter("model_per_key", f"{auth.api_key}:primary")] == -stash.reserved_tokens + + +@pytest.mark.asyncio +async def test_success_tpm_accounting_settles_an_alias_to_target_fallback_once_per_counter( + collision_limiter: tuple[DualCache, PROXY_MaxParallelRequestsHandler_v3, str, str], +) -> None: + cache, handler, alias, target = collision_limiter + key_metadata: Final = {"model_tpm_limit": {alias: 1000, target: 1000}} + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-alias-fallback-settle"), metadata=key_metadata) + + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": alias, "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}, + call_type="acompletion", + ) + stash: Final = get_request_stash() + assert stash is not None + assert stash.reserved_model == RateLimitedModel(requested=alias, group=target) + assert stash.reserved_tokens > 0 + assert await handler.async_fallback_target_allowed( + auth, target, {"messages": [{"role": "user", "content": "hello"}]} + ) + + response: Final = ModelResponse( + id="alias-fallback-settle", + object="chat.completion", + created=int(datetime.now().timestamp()), + model=target, + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + choices=[], + ) + kwargs: Final = { + "standard_logging_object": {"metadata": {"user_api_key_hash": auth.api_key}}, + "litellm_params": { + "metadata": {"model_group": target, "original_model_group": alias, "user_api_key_metadata": key_metadata} + }, + "model": target, + } + + ops: Final = handler._build_success_event_pipeline_operations( + kwargs=kwargs, response_obj=response, rate_limit_type="total" + ) + + increments: Final = tuple(_increment_of(op) for op in ops) + target_counter: Final = _counter("model_per_key", f"{auth.api_key}:{target}") + alias_counter: Final = _counter("model_per_key", f"{auth.api_key}:{alias}") + assert [inc.amount for inc in increments if inc.counter == target_counter] == [150 - stash.reserved_tokens] + assert [inc.amount for inc in increments if inc.counter == alias_counter] == [] + + + +@pytest.mark.asyncio +async def test_success_tpm_accounting_ignores_a_client_supplied_model_group_without_a_router_fallback() -> None: + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + key_metadata: Final = {"model_tpm_limit": {"primary": 1000}} + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-client-model-group"), metadata=key_metadata) + + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": "primary", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}, + call_type="acompletion", + ) + stash: Final = get_request_stash() + assert stash is not None + assert stash.reserved_tokens > 0 + + response: Final = ModelResponse( + id="client-model-group", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="primary", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + choices=[], + ) + kwargs: Final = { + "standard_logging_object": {"metadata": {"user_api_key_hash": auth.api_key}}, + "litellm_params": {"metadata": {"model_group": "uncapped", "user_api_key_metadata": key_metadata}}, + "model": "primary", + } + + ops: Final = handler._build_success_event_pipeline_operations( + kwargs=kwargs, response_obj=response, rate_limit_type="total" + ) + + charged: Final = {_increment_of(op).counter: _increment_of(op).amount for op in ops} + assert charged[_counter("model_per_key", f"{auth.api_key}:primary")] == 150 - stash.reserved_tokens + assert not any(":uncapped" in counter.scope.descriptor_value for counter in charged) From bb89296adf5dbf16b4fbc7f99fc5730b4933dd76 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 10 Nov 2026 18:43:06 -0800 Subject: [PATCH 068/189] fix(proxy): keep the upgrade notice off the request path and count area-labelled release notes (#51027) * fix(proxy): keep the upgrade notice off the request path and count area-labelled release notes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(proxy): regenerate dashboard API types and read the release cache through one typed helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): respond 502 from /get/latest_release_info when no release note entry is countable Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): respond 502 when the release fetch failed and announce a release with no countable notes without counts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): re-ask for the latest release while the proxy is still fetching it and keep fetch diagnostics out of the 502 body Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): keep the known release on screen while the proxy refetches it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): answer 202 while the release is being fetched so the dashboard polls only then Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): name the release query options to stay inside the lint budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): return the 202 as a response value instead of mutating the injected response Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): fetch the release on the background client so its 502 never reaches the error handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../latest_release_endpoints.py | 56 ++++++-- .../test_latest_release_endpoints.py | 126 ++++++++++++++++-- .../useLatestReleaseInfo.test.ts | 76 +++++++++++ .../latestRelease/useLatestReleaseInfo.ts | 49 +++++-- .../NotificationsBell.test.tsx | 15 +++ .../NotificationsBell/NotificationsBell.tsx | 27 ++-- ui/litellm-dashboard/src/i18n/locales/en.json | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 ++- 8 files changed, 323 insertions(+), 47 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo.test.ts diff --git a/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py b/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py index 4bb17fd290e..49f90997879 100644 --- a/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py @@ -9,7 +9,8 @@ from types import MappingProxyType from typing import Annotated, Final, Literal, Protocol, TypeAlias import httpx -from fastapi import APIRouter, Depends +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status +from fastapi.responses import JSONResponse from pydantic import ValidationError from litellm._logging import verbose_proxy_logger @@ -29,11 +30,13 @@ LATEST_RELEASE_UNAVAILABLE_CACHE_TTL_SECONDS: Final = 5 * 60 LATEST_RELEASE_CACHE_KEY: Final = "latest_release_info" _RELEASE_BULLET_PATTERN: Final = re.compile(r"^\*\s+(?:([A-Za-z]+)(?:\([^)]*\))?!?:\s)?\S") +_AREA_BULLET_PATTERN: Final = re.compile(r"^-\s+\*\*[^*]+\*\*\s+([A-Za-z]+)") _NEW_CONTRIBUTOR_PATTERN: Final = re.compile(r"^\*\s+@\S+ made their first contribution\b") _CHANGELOG_PAGE_PATTERN: Final = re.compile(rf"{re.escape(CHANGELOG_URL)}v(\d+)\.(\d+)\.(\d+)/") _Bucket: TypeAlias = Literal["new_features", "bug_fixes", "other_updates"] _PREFIX_BUCKETS: Final[Mapping[str, _Bucket]] = MappingProxyType({"feat": "new_features", "fix": "bug_fixes"}) +_VERB_BUCKETS: Final[Mapping[str, _Bucket]] = MappingProxyType({"added": "new_features", "fixed": "bug_fixes"}) class LatestReleaseInfo(LiteLLMBaseModel): @@ -97,6 +100,9 @@ def _default_release_check_enabled() -> bool: def _bucket_for(line: str) -> _Bucket | None: if _NEW_CONTRIBUTOR_PATTERN.match(line) is not None: return None + area_match: Final = _AREA_BULLET_PATTERN.match(line) + if area_match is not None: + return _VERB_BUCKETS.get(area_match.group(1).lower(), "other_updates") match: Final = _RELEASE_BULLET_PATTERN.match(line) if match is None: return None @@ -105,7 +111,8 @@ def _bucket_for(line: str) -> _Bucket | None: def count_release_bullets(body: str) -> Mapping[_Bucket, int]: - """Bucket release-note bullets by conventional-commit type or ``other_updates``.""" + """Bucket release-note bullets by conventional-commit type (``* fix(x): ...``) or by the verb after the area label + (``- **Area:** Fixed ...``), falling back to ``other_updates``.""" return MappingProxyType(Counter(bucket for line in body.splitlines() if (bucket := _bucket_for(line)) is not None)) @@ -200,15 +207,20 @@ async def fetch_latest_release(client: _AsyncGetClient) -> LatestReleaseInfo | L ) +def _cached_release(cache: InMemoryCache) -> LatestReleaseInfo | LatestReleaseUnavailable | None: + cached: Final = cache.get_cache(LATEST_RELEASE_CACHE_KEY) + return cached if isinstance(cached, (LatestReleaseInfo, LatestReleaseUnavailable)) else None + + async def get_latest_release_info( client: _AsyncGetClient, cache: InMemoryCache, fetch_lock: asyncio.Lock ) -> LatestReleaseInfo | LatestReleaseUnavailable: - cached: Final = cache.get_cache(LATEST_RELEASE_CACHE_KEY) - if isinstance(cached, (LatestReleaseInfo, LatestReleaseUnavailable)): + cached: Final = _cached_release(cache) + if cached is not None: return cached async with fetch_lock: - cached_after_lock: Final = cache.get_cache(LATEST_RELEASE_CACHE_KEY) - if isinstance(cached_after_lock, (LatestReleaseInfo, LatestReleaseUnavailable)): + cached_after_lock: Final = _cached_release(cache) + if cached_after_lock is not None: return cached_after_lock result: Final = await fetch_latest_release(client) ttl: Final = ( @@ -220,28 +232,46 @@ async def get_latest_release_info( return result +async def refresh_latest_release_info(client: _AsyncGetClient, cache: InMemoryCache, fetch_lock: asyncio.Lock) -> None: + result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock) + if isinstance(result, LatestReleaseUnavailable): + verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason) + + @router.get( "/get/latest_release_info", tags=["UI Settings"], dependencies=[Depends(user_api_key_auth)], response_model=LatestReleaseInfo | None, + responses={ + 202: {"description": "The release is being fetched in the background, so ask again shortly", "model": None}, + 502: {"description": "The changelog or GitHub could not be reached or answered with something unusable"}, + }, ) async def latest_release_info( + background_tasks: BackgroundTasks, client: Annotated[_AsyncGetClient, Depends(_default_client)], cache: Annotated[InMemoryCache, Depends(_default_cache)], fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)], release_check_enabled: Annotated[bool, Depends(_default_release_check_enabled)], -) -> LatestReleaseInfo | None: +) -> LatestReleaseInfo | JSONResponse | None: """ Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates. - Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render. + Never waits on the changelog or GitHub. It answers from the cache and, when the cache is empty or expired, responds + 202 with null and fetches after the response is sent, so a later call returns the release. Responds 502 once that + fetch has failed because the changelog or GitHub couldn't be reached or answered with something unusable, so the dashboard + upgrade notice simply doesn't render. The three counts are all zero when the release notes list no change in a + format this proxy can count. Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request. """ if not release_check_enabled: return None - result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock) - if isinstance(result, LatestReleaseUnavailable): - verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason) - return None - return result + cached: Final = _cached_release(cache) + if isinstance(cached, LatestReleaseInfo): + return cached + if isinstance(cached, LatestReleaseUnavailable): + raise HTTPException(status_code=502, detail="Latest release info unavailable") + if not fetch_lock.locked(): + background_tasks.add_task(refresh_latest_release_info, client, cache, fetch_lock) + return JSONResponse(status_code=status.HTTP_202_ACCEPTED, content=None) diff --git a/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py index b5a41c31810..e3d9e66c208 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py @@ -121,6 +121,53 @@ def _override_dependencies(client: _ByUrl, cache: InMemoryCache, role: LitellmUs app.dependency_overrides[_default_fetch_lock] = lambda: asyncio.Lock() +class _AsgiGet: + """One GET sent straight to the ASGI app, so the response body is readable while the app call is still running.""" + + def __init__(self, path: str) -> None: + self._chunks: list[bytes] = [] + self._sent = asyncio.Event() + self.task = asyncio.create_task( + app( + { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "query_string": b"", + "headers": [(b"host", b"proxy")], + "client": ("127.0.0.1", 1), + "server": ("proxy", 80), + }, + self._receive, + self._send, + ) + ) + + async def _receive(self) -> dict[str, object]: + return {"type": "http.request", "body": b"", "more_body": False} + + async def _send(self, message: dict[str, object]) -> None: + if message["type"] != "http.response.body": + return + body = message.get("body", b"") + assert isinstance(body, bytes) + self._chunks.append(body) + if not message.get("more_body", False): + self._sent.set() + + @property + def finished(self) -> bool: + return self.task.done() + + async def body(self) -> bytes: + await self._sent.wait() + return b"".join(self._chunks) + + @pytest.fixture def http_client(): yield TestClient(app) @@ -152,6 +199,18 @@ class TestCountReleaseBullets: == 0 ) + def test_buckets_area_labelled_bullets_by_their_leading_verb(self): + counts = count_release_bullets( + "### Changes\n\n" + "- **OpenAI:** Fixed empty `api_key` sending the admin key (#1)\n" + "- **AWS Bedrock:** fixed Converse `service_tier` (#2)\n" + "- **MCP:** Added per-server timeouts (#3)\n" + "- **Routing and fallbacks:** Changed the default cooldown (#4)\n" + "- **UI:** Moved the upgrade notice into the bell (#5)\n" + "- plain dash line that is not a change entry\n" + ) + assert (counts["new_features"], counts["bug_fixes"], counts["other_updates"]) == (1, 2, 2) + def test_empty_body_yields_zero_counts(self): counts = count_release_bullets("") assert (counts["new_features"], counts["bug_fixes"], counts["other_updates"]) == (0, 0, 0) @@ -387,19 +446,62 @@ class TestPatchReleaseAggregatesMinorLine: class TestLatestReleaseInfoEndpoint: - def test_returns_release_stats_for_authenticated_user(self, http_client): + def test_first_request_returns_null_and_the_next_returns_the_fetched_release(self, http_client): _override_dependencies(_minor_release(), _fresh_cache(), LitellmUserRoles.INTERNAL_USER) - response = http_client.get("/get/latest_release_info") - assert response.status_code == 200 - assert response.json() == EXPECTED_INFO + first = http_client.get("/get/latest_release_info") + assert (first.status_code, first.json()) == (202, None) + second = http_client.get("/get/latest_release_info") + assert (second.status_code, second.json()) == (200, EXPECTED_INFO) - def test_returns_null_when_the_changelog_is_unreachable(self, http_client): + @pytest.mark.asyncio + async def test_responds_while_the_changelog_fetch_is_still_hanging(self): + release = asyncio.Event() + + class _HangingClient(_ByUrl): + async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: + self.calls.append((url, timeout)) + await release.wait() + raise httpx.ConnectTimeout("blackholed") + + client: Final = _HangingClient({}) + _override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN) + lock: Final = asyncio.Lock() + app.dependency_overrides[_default_fetch_lock] = lambda: lock + try: + requests: Final = [_AsgiGet("/get/latest_release_info") for _ in range(3)] + bodies: Final = [await asyncio.wait_for(request.body(), timeout=5) for request in requests] + still_fetching: Final = [request for request in requests if not request.finished] + release.set() + await asyncio.gather(*(request.task for request in requests)) + finally: + release.set() + for dependency in (user_api_key_auth, _default_client, _default_cache, _default_fetch_lock): + app.dependency_overrides.pop(dependency, None) + assert bodies == [b"null", b"null", b"null"] + assert len(still_fetching) == 1, "exactly one request should still be running the hanging fetch" + assert client.urls == [CHANGELOG_FEED_URL] + + def test_responds_502_once_the_changelog_fetch_has_failed(self, http_client): + client: Final = _ByUrl({CHANGELOG_FEED_URL: httpx.ConnectError("boom")}) + _override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN) + first: Final = http_client.get("/get/latest_release_info") + assert (first.status_code, first.json()) == (202, None) + later: Final = [http_client.get("/get/latest_release_info") for _ in range(2)] + assert [response.status_code for response in later] == [502, 502] + assert later[0].json() == {"detail": "Latest release info unavailable"} + assert len(client.calls) == 1 + + def test_release_with_no_countable_notes_is_still_announced_with_zero_counts(self, http_client): + uncountable: Final = _github_response(payload=_release("v1.102.0", "## Changes\n\n1. Reworked the router\n")) _override_dependencies( - _ByUrl({CHANGELOG_FEED_URL: httpx.ConnectError("boom")}), _fresh_cache(), LitellmUserRoles.PROXY_ADMIN + _minor_release(**{_tag_url("v1.102.0"): uncountable}), _fresh_cache(), LitellmUserRoles.PROXY_ADMIN + ) + http_client.get("/get/latest_release_info") + response: Final = http_client.get("/get/latest_release_info") + assert (response.status_code, response.json()) == ( + 200, + {**EXPECTED_INFO, "new_features": 0, "bug_fixes": 0, "other_updates": 0}, ) - response = http_client.get("/get/latest_release_info") - assert response.status_code == 200 - assert response.json() is None @pytest.mark.parametrize("name", ["LITELLM_DISABLE_LATEST_RELEASE_CHECK", "LITELLM_OFFLINE_MODE"]) @pytest.mark.parametrize( @@ -416,6 +518,8 @@ class TestLatestReleaseInfoEndpoint: monkeypatch.setenv(name, flag) client = _minor_release() _override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN) + first = http_client.get("/get/latest_release_info") + assert first.status_code == (200 if expected_body is None else 202) response = http_client.get("/get/latest_release_info") assert response.status_code == 200 assert response.json() == expected_body @@ -424,8 +528,8 @@ class TestLatestReleaseInfoEndpoint: def test_repeated_requests_reuse_cache(self, http_client): client = _minor_release() _override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN) - assert http_client.get("/get/latest_release_info").json() == EXPECTED_INFO - assert http_client.get("/get/latest_release_info").json() == EXPECTED_INFO + bodies = [http_client.get("/get/latest_release_info").json() for _ in range(3)] + assert bodies == [None, EXPECTED_INFO, EXPECTED_INFO] assert len(client.calls) == 2 def test_rejects_unauthenticated_requests(self, monkeypatch): diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo.test.ts new file mode 100644 index 00000000000..ed76b5524be --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo.test.ts @@ -0,0 +1,76 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { registerErrorHandler } from "@/lib/http/runtime"; +import { + PENDING_RELEASE_POLL_MS, + fetchLatestRelease, + keepKnownRelease, + pendingReleasePollMs, +} from "./useLatestReleaseInfo"; + +const RELEASE = { version: "1.103.0", new_features: 1, bug_fixes: 2, other_updates: 3, release_url: "https://x" }; + +describe("pendingReleasePollMs", () => { + it("asks again soon while the proxy is still fetching the release", () => { + const state = { status: "success", data: { release: null, pending: true } }; + expect(pendingReleasePollMs(state)).toBe(PENDING_RELEASE_POLL_MS); + }); + + it("keeps asking while a known release is shown and the proxy is refetching it", () => { + const state = { status: "success", data: { release: RELEASE, pending: true } }; + expect(pendingReleasePollMs(state)).toBe(PENDING_RELEASE_POLL_MS); + }); + + it("stops once the release arrives", () => { + expect(pendingReleasePollMs({ status: "success", data: { release: RELEASE, pending: false } })).toBe(false); + }); + + it("stops once the proxy reports the fetch failed", () => { + expect(pendingReleasePollMs({ status: "error", data: { release: null, pending: true } })).toBe(false); + }); + + it("does not ask again when the release check is turned off", () => { + expect(pendingReleasePollMs({ status: "success", data: { release: null, pending: false } })).toBe(false); + }); +}); + +describe("keepKnownRelease", () => { + it("keeps showing the known release while the proxy refetches it", () => { + const kept = keepKnownRelease({ release: RELEASE, pending: false }, { release: null, pending: true }); + expect(kept).toEqual({ release: RELEASE, pending: true }); + }); + + it("takes a newer release over the known one", () => { + const newer = { release: { ...RELEASE, version: "1.104.0" }, pending: false }; + expect(keepKnownRelease({ release: RELEASE, pending: true }, newer)).toBe(newer); + }); + + it("drops the known release when the release check has been turned off", () => { + const off = { release: null, pending: false }; + expect(keepKnownRelease({ release: RELEASE, pending: false }, off)).toBe(off); + }); + + it("records a first answer as is", () => { + const first = { release: null, pending: true }; + expect(keepKnownRelease(undefined, first)).toBe(first); + }); +}); + +describe("fetchLatestRelease", () => { + afterEach(() => { + vi.unstubAllGlobals(); + registerErrorHandler(() => {}); + }); + + it("keeps a failed release fetch away from the dashboard error handler", async () => { + const onError = vi.fn(); + registerErrorHandler(onError); + const unavailable = new Response(JSON.stringify({ detail: "Latest release info unavailable" }), { + status: 502, + headers: { "content-type": "application/json" }, + }); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(unavailable)); + + await expect(fetchLatestRelease()).rejects.toMatchObject({ status: 502 }); + expect(onError).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo.ts index 5186baa605c..7a3dcf92dac 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/latestRelease/useLatestReleaseInfo.ts @@ -1,16 +1,41 @@ -import { $api } from "@/lib/http/api"; +import { useQuery } from "@tanstack/react-query"; +import { backgroundFetchClient } from "@/lib/http/api"; import type { components } from "@/lib/http/schema"; +import { createQueryKeys } from "../common/queryKeysFactory"; export type LatestReleaseInfo = components["schemas"]["LatestReleaseInfo"]; -export const useLatestReleaseInfo = (accessToken: string | null | undefined) => - $api.useQuery( - "get", - "/get/latest_release_info", - {}, - { - enabled: Boolean(accessToken), - staleTime: 60 * 60 * 1000, - retry: false, - }, - ); +export type LatestReleaseAnswer = { release: LatestReleaseInfo | null; pending: boolean }; + +export const PENDING_RELEASE_POLL_MS = 5 * 1000; + +export const latestReleaseKeys = createQueryKeys("latestRelease"); + +export const pendingReleasePollMs = (state: { status: string; data?: LatestReleaseAnswer }) => + state.status === "success" && state.data?.pending ? PENDING_RELEASE_POLL_MS : false; + +export const keepKnownRelease = (known: unknown, answer: unknown) => { + const previous = known as LatestReleaseAnswer | undefined; + const next = answer as LatestReleaseAnswer; + return next.pending && previous?.release ? { ...next, release: previous.release } : next; +}; + +export const fetchLatestRelease = async (): Promise => { + const { data, response } = await backgroundFetchClient.GET("/get/latest_release_info"); + return { release: data ?? null, pending: response.status === 202 }; +}; + +export const useLatestReleaseInfo = (accessToken: string | null | undefined) => { + const options = { + queryKey: latestReleaseKeys.all, + queryFn: fetchLatestRelease, + enabled: Boolean(accessToken), + staleTime: 60 * 60 * 1000, + retry: false, + refetchInterval: (query: { state: { status: string; data?: LatestReleaseAnswer } }) => + pendingReleasePollMs(query.state), + structuralSharing: keepKnownRelease, + }; + const { data, isError } = useQuery(options); + return { data: isError ? undefined : data?.release }; +}; diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx index 3b0eec6feec..3110d39a75d 100644 --- a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx @@ -75,6 +75,21 @@ describe("NotificationsBellView", () => { expect(announcement("LiteLLM v1.103.0 is available")).toHaveTextContent("1 new feature, 1 fix, and 1 other update"); }); + it("leaves the counts out when the release notes yielded none", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + await user.click(bell(2)); + const upgrade = announcement("LiteLLM v1.103.0 is available"); + expect(upgrade).toHaveTextContent("Your current version is v1.102.0"); + expect(upgrade).not.toHaveTextContent("0 new features"); + expect(upgrade).not.toHaveTextContent("other update"); + }); + it.each([ ["the running version is unknown", undefined, RELEASE], ["the latest release is unavailable", "1.102.0", null], diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx index 05e3881373e..20cb15a552d 100644 --- a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx @@ -45,12 +45,22 @@ const autoRouterAnnouncement = (): Announcement => ({ href: AUTO_ROUTER_DOCS_URL, }); -const describeRelease = ({ new_features, bug_fixes, other_updates }: LatestReleaseInfo): string => - [ - i18n.t("shell.notifications.release.newFeatures", { count: new_features }), - i18n.t("shell.notifications.release.fixes", { count: bug_fixes }), - i18n.t("shell.notifications.release.otherUpdates", { count: other_updates }), - ].join(", "); +const describeUpgrade = ( + currentVersion: string, + { new_features, bug_fixes, other_updates }: LatestReleaseInfo, +): string => { + if (new_features + bug_fixes + other_updates === 0) { + return i18n.t("shell.notifications.upgrade.descriptionWithoutCounts", { current: currentVersion }); + } + return i18n.t("shell.notifications.upgrade.description", { + summary: [ + i18n.t("shell.notifications.release.newFeatures", { count: new_features }), + i18n.t("shell.notifications.release.fixes", { count: bug_fixes }), + i18n.t("shell.notifications.release.otherUpdates", { count: other_updates }), + ].join(", "), + current: currentVersion, + }); +}; const upgradeAnnouncement = ( currentVersion: string | null | undefined, @@ -62,10 +72,7 @@ const upgradeAnnouncement = ( return { readKey: `litellm:upgradeBannerDismissed:${latestRelease.version}`, title: i18n.t("shell.notifications.upgrade.title", { version: latestRelease.version }), - description: i18n.t("shell.notifications.upgrade.description", { - summary: describeRelease(latestRelease), - current: currentVersion, - }), + description: describeUpgrade(currentVersion, latestRelease), linkLabel: i18n.t("shell.notifications.upgrade.link"), href: latestRelease.release_url, }; diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index 113c77e1916..25bf2817e27 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -8208,6 +8208,7 @@ "upgrade": { "title": "LiteLLM v{{version}} is available", "description": "{{summary}}. Your current version is v{{current}}", + "descriptionWithoutCounts": "Your current version is v{{current}}", "link": "View release notes" }, "announcement": { diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 4f946a4be89..f5f75e65c83 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -6077,7 +6077,11 @@ export interface paths { * Latest Release Info * @description Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole * minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates. - * Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render. + * Never waits on the changelog or GitHub. It answers from the cache and, when the cache is empty or expired, responds + * 202 with null and fetches after the response is sent, so a later call returns the release. Responds 502 once that + * fetch has failed because the changelog or GitHub couldn't be reached or answered with something unusable, so the dashboard + * upgrade notice simply doesn't render. The three counts are all zero when the release notes list no change in a + * format this proxy can count. * Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request. */ get: operations["latest_release_info_get_latest_release_info_get"]; @@ -63074,6 +63078,20 @@ export interface operations { "application/json": components["schemas"]["LatestReleaseInfo"] | null; }; }; + /** @description The release is being fetched in the background, so ask again shortly */ + 202: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description The changelog or GitHub could not be reached or answered with something unusable */ + 502: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; }; }; get_mcp_semantic_filter_settings_get_mcp_semantic_filter_settings_get: { From 461b56b14b2a024ff73e59f56c01db169dfb902f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:50:55 +0000 Subject: [PATCH 069/189] fix(vertex_ai): bill gemini image text, thinking and priority tokens at their own rates (#51017) * fix(vertex_ai): bill gemini image text, thinking and priority tokens at their own rates Co-authored-by: kerry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(images): bill unitemized output as text when no images return Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost_calculator): price image calls only by the tier the provider served --------- Co-authored-by: kenneth Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 9 + .../litellm_core_utils/llm_cost_calc/utils.py | 42 +++-- .../llms/gemini/image_usage_transformation.py | 138 +++++++++++---- .../vertex_and_google_ai_studio_gemini.py | 17 +- .../image_generation/cost_calculator.py | 2 + .../vertex_gemini_transformation.py | 42 ++--- .../test_vertex_gemini_image_token_spend.py | 158 ++++++++++++++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 44 +++++ .../test_gemini_image_edit_transformation.py | 4 +- ..._gemini_image_generation_transformation.py | 6 +- .../gemini/test_image_usage_transformation.py | 67 +++++++- ...tex_ai_image_generation_cost_calculator.py | 72 ++++++++ ...rtex_ai_image_generation_transformation.py | 92 ++++++++++ tests/unit/test_cost_calculator.py | 30 ++++ 14 files changed, 628 insertions(+), 95 deletions(-) create mode 100644 tests/integration/spend/test_vertex_gemini_image_token_spend.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 53137b73f8b..5b4a1d30ff0 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1237,6 +1237,14 @@ def _normalize_service_tier(service_tier: object) -> str | None: _BASE_PRICING_SERVICE_TIERS: Final[frozenset[str]] = frozenset({"default", "standard"}) +def _served_image_service_tier(completion_response: object, usage_object: Usage | None) -> str | None: + """Image requests never send a tier upstream, so only the tier the provider reports can price them.""" + return _resolve_billable_service_tier( + requested=None, + served=_served_service_tier(completion_response, usage_object), + ) + + def _resolve_billable_service_tier(requested: object, served: object) -> str | None: """Served tier wins when it names a priced tier or explicitly says base pricing; otherwise the request decides.""" served_lower: Final = served.lower() if isinstance(served, str) else None @@ -2043,6 +2051,7 @@ def completion_cost( call_type=call_type, model_info=_deployment_model_info(litellm_logging_obj, custom_pricing, router_model_id), vertex_location=vertex_location, + service_tier=_served_image_service_tier(completion_response, cost_per_token_usage_object), ) elif call_type in _VIDEO_CALL_TYPES: ### VIDEO GENERATION COST CALCULATION ### diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 2eb204d151b..1f6a66f699a 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -2107,6 +2107,7 @@ def calculate_image_response_cost_from_usage( custom_llm_provider: str, model_info: ModelInfo | None = None, vertex_location: str | None = None, + service_tier: str | None = None, ) -> float | None: """ Calculate image generation cost from usage metadata when available. @@ -2154,28 +2155,22 @@ def calculate_image_response_cost_from_usage( else completion_tokens_details_attr ) - if output_tokens_details is None: - completion_tokens_details = CompletionTokensDetailsWrapper( - text_tokens=0, - image_tokens=completion_tokens, - reasoning_tokens=0, - audio_tokens=0, - ) - else: - text_tokens = _get_token_detail_value(output_tokens_details, "text_tokens") or 0 - image_tokens: Final = _get_token_detail_value(output_tokens_details, "image_tokens") or 0 - audio_tokens: Final = _get_token_detail_value(output_tokens_details, "audio_tokens") or 0 - reasoning_tokens: Final = _get_token_detail_value(output_tokens_details, "reasoning_tokens") or 0 - known_output_tokens: Final = text_tokens + image_tokens + audio_tokens + reasoning_tokens - if completion_tokens > known_output_tokens: - text_tokens += completion_tokens - known_output_tokens - - completion_tokens_details = CompletionTokensDetailsWrapper( - text_tokens=text_tokens, - image_tokens=image_tokens, - reasoning_tokens=reasoning_tokens, - audio_tokens=audio_tokens, - ) + text_tokens: Final = _get_token_detail_value(output_tokens_details, "text_tokens") or 0 + image_tokens: Final = _get_token_detail_value(output_tokens_details, "image_tokens") or 0 + audio_tokens: Final = _get_token_detail_value(output_tokens_details, "audio_tokens") or 0 + reasoning_tokens: Final = _get_token_detail_value(output_tokens_details, "reasoning_tokens") or 0 + has_output_modalities: Final = any( + _get_token_detail_value(output_tokens_details, modality) is not None + for modality in ("text_tokens", "image_tokens", "audio_tokens") + ) + unitemized_tokens: Final = max(completion_tokens - text_tokens - image_tokens - audio_tokens - reasoning_tokens, 0) + unitemized_image_tokens: Final = unitemized_tokens if not has_output_modalities and image_response.data else 0 + completion_tokens_details: Final = CompletionTokensDetailsWrapper( + text_tokens=text_tokens + unitemized_tokens - unitemized_image_tokens, + image_tokens=image_tokens + unitemized_image_tokens, + reasoning_tokens=reasoning_tokens, + audio_tokens=audio_tokens, + ) normalized_usage: Final = Usage( prompt_tokens=prompt_tokens, @@ -2191,6 +2186,7 @@ def calculate_image_response_cost_from_usage( custom_llm_provider=custom_llm_provider, model_info=model_info, vertex_location=vertex_location, + service_tier=service_tier, ) return prompt_cost + completion_cost @@ -2298,6 +2294,7 @@ class CostCalculatorUtils: call_type: str | None = None, model_info: ModelInfo | None = None, vertex_location: str | None = None, + service_tier: str | None = None, ) -> float: """ Route the image generation cost calculator based on the custom_llm_provider @@ -2354,6 +2351,7 @@ class CostCalculatorUtils: image_response=completion_response, model_info=pricing, vertex_location=vertex_location, + service_tier=service_tier, ) elif custom_llm_provider == litellm.LlmProviders.BEDROCK.value: if isinstance(completion_response, ImageResponse): diff --git a/litellm/llms/gemini/image_usage_transformation.py b/litellm/llms/gemini/image_usage_transformation.py index 3ef403f79ee..3ea8e445a47 100644 --- a/litellm/llms/gemini/image_usage_transformation.py +++ b/litellm/llms/gemini/image_usage_transformation.py @@ -1,67 +1,131 @@ +from collections.abc import Mapping from typing import Final from pydantic import TypeAdapter +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig from litellm.types.utils import ImageUsage, ImageUsageInputTokensDetails +_TOKEN_COUNT_ADAPTER: Final = TypeAdapter(int) +_TOKEN_DETAILS_LIST_ADAPTER: Final[TypeAdapter[list[object]]] = TypeAdapter(list[object]) +_TOKEN_DETAILS_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) -def _get_token_count(details: dict) -> int: + +def _get_token_count(details: Mapping[str, object]) -> int: raw_token_count: Final = details.get("tokenCount", details.get("token_count", 0)) return raw_token_count if isinstance(raw_token_count, int) else 0 -def _get_modality_token_details(usage_metadata: dict, *details_keys: str) -> list: - for details_key in details_keys: - details = usage_metadata.get(details_key) - if isinstance(details, list): - return details - return [] +def _get_usage_token_count(usage_metadata: Mapping[str, object], key: str) -> int: + return _TOKEN_COUNT_ADAPTER.validate_python(usage_metadata.get(key, 0)) -def _sum_modality_token_details(usage_metadata: dict, *details_keys: str) -> ImageUsageInputTokensDetails: - tokens_details: Final = ImageUsageInputTokensDetails( - image_tokens=0, - text_tokens=0, +def _get_modality_token_details( + usage_metadata: Mapping[str, object], *details_keys: str +) -> tuple[Mapping[str, object], ...]: + details: Final[list[object]] = next( + ( + _TOKEN_DETAILS_LIST_ADAPTER.validate_python(usage_metadata.get(details_key)) + for details_key in details_keys + if isinstance(usage_metadata.get(details_key), list) + ), + [], + ) + return tuple( + _TOKEN_DETAILS_MAPPING_ADAPTER.validate_python(detail) for detail in details if isinstance(detail, Mapping) ) - for details in _get_modality_token_details(usage_metadata, *details_keys): - if isinstance(details, dict): - modality = str(details.get("modality", "")).upper() - token_count = _get_token_count(details) - if modality == "TEXT": - tokens_details.text_tokens += token_count - elif modality == "IMAGE": - tokens_details.image_tokens += token_count - return tokens_details +def _sum_modality_token_details( + token_details: tuple[Mapping[str, object], ...], +) -> ImageUsageInputTokensDetails: + return ImageUsageInputTokensDetails( + image_tokens=sum( + _get_token_count(details) + for details in token_details + if str(details.get("modality", "")).upper() == "IMAGE" + ), + text_tokens=sum( + _get_token_count(details) for details in token_details if str(details.get("modality", "")).upper() == "TEXT" + ), + ) -def transform_gemini_image_usage(usage_metadata: dict[str, object]) -> ImageUsage: +def _output_modality_token_details( + candidate_tokens: int, + reasoning_tokens: int, + candidate_tokens_are_inclusive: bool, + modality_tokens: ImageUsageInputTokensDetails, +) -> ImageUsageInputTokensDetails: + non_reasoning_candidate_tokens: Final = max( + candidate_tokens - (reasoning_tokens if candidate_tokens_are_inclusive else 0), + 0, + ) + known_modality_tokens: Final = modality_tokens.text_tokens + modality_tokens.image_tokens + text_tokens: Final = ( + max( + modality_tokens.text_tokens - (known_modality_tokens - non_reasoning_candidate_tokens), + 0, + ) + if known_modality_tokens > non_reasoning_candidate_tokens + else modality_tokens.text_tokens + non_reasoning_candidate_tokens - known_modality_tokens + ) + return ImageUsageInputTokensDetails(image_tokens=modality_tokens.image_tokens, text_tokens=text_tokens) + + +def transform_gemini_image_usage(usage_metadata: Mapping[str, object]) -> ImageUsage: """ Transform Gemini usageMetadata to ImageUsage format. """ - input_tokens_details = _sum_modality_token_details(usage_metadata, "promptTokensDetails", "prompt_tokens_details") - output_tokens: Final = TypeAdapter(int).validate_python(usage_metadata.get("candidatesTokenCount", 0)) - output_tokens_details: Final = _sum_modality_token_details( + validated_counts: Final = ImageUsage.model_validate( + { + "input_tokens": usage_metadata.get("promptTokenCount", 0), + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": usage_metadata.get("candidatesTokenCount", 0), + "total_tokens": usage_metadata.get("totalTokenCount", 0), + } + ) + prompt_tokens: Final = validated_counts.input_tokens + candidate_tokens: Final = validated_counts.output_tokens + reasoning_tokens: Final = _get_usage_token_count(usage_metadata, "thoughtsTokenCount") + candidate_tokens_are_inclusive: Final = VertexGeminiConfig.is_candidate_token_count_inclusive(usage_metadata) + completion_tokens: Final = candidate_tokens + (0 if candidate_tokens_are_inclusive else reasoning_tokens) + input_tokens_details: Final = _sum_modality_token_details( + _get_modality_token_details(usage_metadata, "promptTokensDetails", "prompt_tokens_details") + ) + candidate_token_details: Final = _get_modality_token_details( usage_metadata, "candidatesTokensDetails", "candidates_tokens_details" ) + modality_tokens: Final = _sum_modality_token_details(candidate_token_details) + output_modality_tokens: Final = ( + _output_modality_token_details( + candidate_tokens, + reasoning_tokens, + candidate_tokens_are_inclusive, + modality_tokens, + ) + if candidate_token_details + else None + ) - if not _get_modality_token_details(usage_metadata, "candidatesTokensDetails", "candidates_tokens_details"): - output_tokens_details.image_tokens = output_tokens - else: - known_output_tokens: Final = output_tokens_details.text_tokens + output_tokens_details.image_tokens - if output_tokens > known_output_tokens: - output_tokens_details.text_tokens += output_tokens - known_output_tokens + output_tokens_details: Final[dict[str, int]] = { + **( + {"image_tokens": output_modality_tokens.image_tokens, "text_tokens": output_modality_tokens.text_tokens} + if output_modality_tokens is not None + else {} + ), + **({"reasoning_tokens": reasoning_tokens} if "thoughtsTokenCount" in usage_metadata else {}), + } usage_payload: Final[dict[str, object]] = { - "input_tokens": usage_metadata.get("promptTokenCount", 0), + "input_tokens": prompt_tokens, "input_tokens_details": input_tokens_details, - "output_tokens": output_tokens, - "total_tokens": usage_metadata.get("totalTokenCount", 0), - "prompt_tokens": usage_metadata.get("promptTokenCount", 0), + "output_tokens": completion_tokens, + "total_tokens": validated_counts.total_tokens, + "prompt_tokens": prompt_tokens, "prompt_tokens_details": input_tokens_details.model_dump(), - "completion_tokens": output_tokens, - "completion_tokens_details": output_tokens_details.model_dump(), - "output_tokens_details": output_tokens_details.model_dump(), + "completion_tokens": completion_tokens, + "completion_tokens_details": {**output_tokens_details}, + "output_tokens_details": {**output_tokens_details}, } return ImageUsage.model_validate(usage_payload) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 80244cfc3eb..b5267f9681a 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2442,7 +2442,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return model_response @staticmethod - def is_candidate_token_count_inclusive(usage_metadata: UsageMetadata) -> bool: + def is_candidate_token_count_inclusive(usage_metadata: Mapping[str, object]) -> bool: """ Check if the candidate token count is inclusive of the thinking token count @@ -2452,12 +2452,17 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Addresses - https://github.com/BerriAI/litellm/pull/10141#discussion_r2052272035 """ - non_thinking_tokens: Final = ( - usage_metadata.get("promptTokenCount", 0) - + usage_metadata.get("candidatesTokenCount", 0) - + usage_metadata.get("toolUsePromptTokenCount", 0) + prompt_tokens: Final = usage_metadata.get("promptTokenCount", 0) + candidate_tokens: Final = usage_metadata.get("candidatesTokenCount", 0) + tool_use_prompt_tokens: Final = usage_metadata.get("toolUsePromptTokenCount", 0) + total_tokens: Final = usage_metadata.get("totalTokenCount", 0) + return ( + isinstance(prompt_tokens, int) + and isinstance(candidate_tokens, int) + and isinstance(tool_use_prompt_tokens, int) + and isinstance(total_tokens, int) + and prompt_tokens + candidate_tokens + tool_use_prompt_tokens == total_tokens ) - return non_thinking_tokens == usage_metadata.get("totalTokenCount", 0) @staticmethod def _response_has_search_grounding( diff --git a/litellm/llms/vertex_ai/image_generation/cost_calculator.py b/litellm/llms/vertex_ai/image_generation/cost_calculator.py index df8629aac46..6334c8646b6 100644 --- a/litellm/llms/vertex_ai/image_generation/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_generation/cost_calculator.py @@ -18,6 +18,7 @@ def cost_calculator( image_response: ImageResponse, model_info: ModelInfo | None = None, vertex_location: str | None = None, + service_tier: str | None = None, ) -> float: """ Vertex AI Image Generation Cost Calculator @@ -40,6 +41,7 @@ def cost_calculator( custom_llm_provider="vertex_ai", model_info=_model_info, vertex_location=vertex_location, + service_tier=service_tier, ) if token_based_cost is not None: return token_based_cost + web_search_cost diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index e1049ef8ece..252054d7330 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -14,6 +14,7 @@ from litellm.llms.gemini.common_utils import ( get_gemini_image_web_search_requests, map_gemini_image_tools_params, ) +from litellm.llms.gemini.image_usage_transformation import transform_gemini_image_usage from litellm.llms.vertex_ai.common_utils import get_vertex_base_url, map_vertex_image_size from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.secret_managers.main import get_secret_str @@ -21,8 +22,6 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ( ImageObject, ImageResponse, - ImageUsage, - ImageUsageInputTokensDetails, ) if TYPE_CHECKING: @@ -262,27 +261,6 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): return request_body - def _transform_image_usage(self, usage: dict) -> ImageUsage: - input_tokens_details: Final = ImageUsageInputTokensDetails( - image_tokens=0, - text_tokens=0, - ) - tokens_details: Final = usage.get("promptTokensDetails", []) - for details in tokens_details: - if isinstance(details, dict) and (modality := details.get("modality")): - token_count = details.get("tokenCount", 0) - if modality == "TEXT": - input_tokens_details.text_tokens += token_count - elif modality == "IMAGE": - input_tokens_details.image_tokens += token_count - - return ImageUsage( - input_tokens=usage.get("promptTokenCount", 0), - input_tokens_details=input_tokens_details, - output_tokens=usage.get("candidatesTokenCount", 0), - total_tokens=usage.get("totalTokenCount", 0), - ) - def transform_image_generation_response( self, model: str, @@ -334,7 +312,23 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): ) if usage_metadata := response_object.get("usageMetadata", None): - model_response.usage = self._transform_image_usage(_JSON_DICT.validate_python(usage_metadata)) + normalized_usage_metadata: Final = _JSON_DICT.validate_python(usage_metadata) + model_response.usage = transform_gemini_image_usage(normalized_usage_metadata) + traffic_type: Final = normalized_usage_metadata.get("trafficType") + if isinstance(traffic_type, str) and traffic_type: + hidden_params: Final = model_response.hidden_params + provider_specific_fields: Final[dict[str, object]] = ( + _JSON_DICT.validate_python(hidden_params.get("provider_specific_fields")) + if isinstance(hidden_params.get("provider_specific_fields"), Mapping) + else {} + ) + model_response.hidden_params = { # rebind-ok: transformation fills the caller-owned response metadata + **hidden_params, + "provider_specific_fields": { + **provider_specific_fields, + "traffic_type": traffic_type, + }, + } web_search_requests: Final = get_gemini_image_web_search_requests(response_object) if web_search_requests and model_response.usage is not None: diff --git a/tests/integration/spend/test_vertex_gemini_image_token_spend.py b/tests/integration/spend/test_vertex_gemini_image_token_spend.py new file mode 100644 index 00000000000..65864fcba61 --- /dev/null +++ b/tests/integration/spend/test_vertex_gemini_image_token_spend.py @@ -0,0 +1,158 @@ +import json +from collections.abc import Callable +from hashlib import sha256 +from typing import Final + +import pytest +from integration._support.client import Gateway, Scenario, eventually +from integration._support.database import read_rows +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server + +_PROJECT: Final = "scripted-project" +_LOCATION: Final = "global" +_BACKEND: Final = "gemini-scripted-image-tokens" +_MODEL_PATH: Final = f"/v1/projects/{_PROJECT}/locations/{_LOCATION}/publishers/google/models" +_PROMPT: Final = "scripted image token spend" +_PNG_B64: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" +_PROMPT_TOKENS: Final = 12 +_TEXT_OUTPUT_TOKENS: Final = 10 +_IMAGE_OUTPUT_TOKENS: Final = 1290 +_THOUGHTS_TOKENS: Final = 40 +_INPUT_RATE: Final = 2e-6 +_TEXT_OUTPUT_RATE: Final = 1e-5 +_IMAGE_OUTPUT_RATE: Final = 4e-5 +_PRIORITY_INPUT_RATE: Final = 4e-6 +_PRIORITY_OUTPUT_RATE: Final = 2e-5 +_PRIORITY_IMAGE_RATE: Final = 8e-5 +_EXPECTED_STANDARD_SPEND: Final = ( + _PROMPT_TOKENS * _INPUT_RATE + + (_TEXT_OUTPUT_TOKENS + _THOUGHTS_TOKENS) * _TEXT_OUTPUT_RATE + + _IMAGE_OUTPUT_TOKENS * _IMAGE_OUTPUT_RATE +) +_EXPECTED_PRIORITY_SPEND: Final = ( + _PROMPT_TOKENS * _PRIORITY_INPUT_RATE + + (_TEXT_OUTPUT_TOKENS + _THOUGHTS_TOKENS) * _PRIORITY_OUTPUT_RATE + + _IMAGE_OUTPUT_TOKENS * _PRIORITY_IMAGE_RATE +) + + +def _image_reply(traffic_type: str | None, text_only: bool, breakdown: bool) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert (request.method, request.target) == ( + "POST", + f"{_MODEL_PATH}/{_BACKEND}:generateContent", + ), (request.method, request.target) + assert _PROMPT in request.body.decode(), request.body + usage_metadata: Final[dict[str, object]] = { + "promptTokenCount": _PROMPT_TOKENS, + "candidatesTokenCount": 25 if text_only else _TEXT_OUTPUT_TOKENS + _IMAGE_OUTPUT_TOKENS, + "thoughtsTokenCount": 0 if text_only else _THOUGHTS_TOKENS, + "totalTokenCount": _PROMPT_TOKENS + + (25 if text_only else _TEXT_OUTPUT_TOKENS + _IMAGE_OUTPUT_TOKENS + _THOUGHTS_TOKENS), + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": _PROMPT_TOKENS}], + **( + { + "candidatesTokensDetails": [ + {"modality": "TEXT", "tokenCount": 25 if text_only else _TEXT_OUTPUT_TOKENS}, + *([] if text_only else [{"modality": "IMAGE", "tokenCount": _IMAGE_OUTPUT_TOKENS}]), + ] + } + if breakdown + else {} + ), + **({"trafficType": traffic_type} if traffic_type is not None else {}), + } + return Reply( + body=json.dumps( + { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + {"text": "No image generated"} + if text_only + else {"inlineData": {"mimeType": "image/png", "data": _PNG_B64}}, + ], + }, + "finishReason": "NO_IMAGE" if text_only else "STOP", + } + ], + "usageMetadata": usage_metadata, + } + ).encode() + ) + + return respond + + +def _deployment(gateway: Gateway, scenario: Scenario, wire: Wire) -> str: + return scenario.model( + model=f"vertex_ai/{_BACKEND}", + api_base=f"{wire.url}{_MODEL_PATH}/{_BACKEND}:generateContent", + api_key=None, + vertex_project=_PROJECT, + vertex_location=_LOCATION, + vertex_credentials=service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")), + input_cost_per_token=_INPUT_RATE, + output_cost_per_token=_TEXT_OUTPUT_RATE, + output_cost_per_image_token=_IMAGE_OUTPUT_RATE, + input_cost_per_token_priority=_PRIORITY_INPUT_RATE, + output_cost_per_token_priority=_PRIORITY_OUTPUT_RATE, + output_cost_per_image_token_priority=_PRIORITY_IMAGE_RATE, + ) + + +def _spend_for_key(key: str) -> float: + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE api_key=%s', + (sha256(key.encode()).hexdigest(),), + ), + lambda found: len(found) == 1, + seconds=70, + ) + return float(str(rows[0]["spend"])) + + +@pytest.mark.timeout(120) +@pytest.mark.parametrize( + ("traffic_type", "text_only", "breakdown", "expected_spend"), + [ + pytest.param(None, False, True, _EXPECTED_STANDARD_SPEND, id="standard"), + pytest.param("ON_DEMAND_PRIORITY", False, True, _EXPECTED_PRIORITY_SPEND, id="on-demand-priority"), + pytest.param(None, True, True, _PROMPT_TOKENS * _INPUT_RATE + 25 * _TEXT_OUTPUT_RATE, id="text-only"), + pytest.param( + None, True, False, _PROMPT_TOKENS * _INPUT_RATE + 25 * _TEXT_OUTPUT_RATE, id="text-only-no-breakdown" + ), + pytest.param( + None, + False, + False, + _PROMPT_TOKENS * _INPUT_RATE + + _THOUGHTS_TOKENS * _TEXT_OUTPUT_RATE + + (_TEXT_OUTPUT_TOKENS + _IMAGE_OUTPUT_TOKENS) * _IMAGE_OUTPUT_RATE, + id="image-no-breakdown", + ), + ], +) +def test_vertex_gemini_image_spend_prices_text_thinking_and_served_tier( + gateway: Gateway, + traffic_type: str | None, + text_only: bool, + breakdown: bool, + expected_spend: float, +) -> None: + with wire_server(_image_reply(traffic_type, text_only, breakdown)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(gateway, scenario, wire) + key: Final = scenario.key(models=[model]) + response: Final = gateway.request( + "POST", + "/v1/images/generations", + {"model": model, "prompt": _PROMPT}, + key=key, + ) + + assert response.status_code == 200, response.text + assert _spend_for_key(key) == pytest.approx(expected_spend) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0aa76deb9d4..cb9d87622e2 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( apply_off_peak_pricing, apply_provider_cache_read_default, calculate_cache_writing_cost, + calculate_image_response_cost_from_usage, generic_cost_per_token, get_billed_token_rates, get_regional_uplift_multiplier, @@ -27,6 +28,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( from litellm.llms.gemini.image_generation.cost_calculator import ( cost_calculator as gemini_image_generation_cost_calculator, ) +from litellm.llms.gemini.image_usage_transformation import transform_gemini_image_usage from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_image_generation_cost_calculator, ) @@ -3511,6 +3513,48 @@ def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(_local_model assert text_output_cost + geo.reasoning_cost == pytest.approx(completion_cost) +@pytest.mark.parametrize("has_image", [False, True]) +@pytest.mark.parametrize("thoughts", [0, 40]) +@pytest.mark.parametrize("gemini_usage", [False, True]) +@pytest.mark.parametrize("explicit_breakdown", [False, True]) +def test_image_response_unitemized_output_uses_returned_images( + has_image: bool, thoughts: int, gemini_usage: bool, explicit_breakdown: bool +) -> None: + prices: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 3e-6, + "output_cost_per_image_token": 7e-5, + } + response: Final = ImageResponse( + data=[ImageObject(b64_json="x")] if has_image else [], + usage=transform_gemini_image_usage( + { + "promptTokenCount": 5, + "candidatesTokenCount": 25, + "thoughtsTokenCount": thoughts, + "totalTokenCount": 30 + thoughts, + **({"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 25}]} if explicit_breakdown else {}), + } + ) + if gemini_usage + else ImageUsage( + input_tokens=5, + input_tokens_details=ImageUsageInputTokensDetails(text_tokens=5, image_tokens=0), + output_tokens=25 + thoughts, + total_tokens=30 + thoughts, + **( + {"output_tokens_details": {"text_tokens": 25, "image_tokens": 0, "reasoning_tokens": thoughts}} + if explicit_breakdown + else {"output_tokens_details": {"reasoning_tokens": thoughts}} + ), + ), + ) + expected_output_rate: Final = 7e-5 if has_image and not explicit_breakdown else 3e-6 + assert calculate_image_response_cost_from_usage( + model="gemini-image-test", image_response=response, custom_llm_provider="gemini", model_info=prices + ) == pytest.approx(5 * 2e-6 + 25 * expected_output_rate + thoughts * 3e-6) + + @pytest.mark.parametrize("details_as_dict", [True, False]) def test_image_response_input_image_tokens_priced_at_image_rate(details_as_dict): """ diff --git a/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py b/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py index c21dec9b6bb..34826fa0571 100644 --- a/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py +++ b/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py @@ -234,12 +234,12 @@ class TestGeminiImageEditTransformation: assert usage["prompt_tokens"] == 35 assert usage["completion_tokens"] == 1716 assert usage["prompt_tokens_details"]["image_tokens"] == 5 - assert usage["completion_tokens_details"]["image_tokens"] == 1716 + assert usage["completion_tokens_details"] == {} logging_usage = StandardLoggingPayloadSetup.get_usage_as_dict( response_obj=image_response.model_dump() ) - assert logging_usage["completion_tokens_details"]["image_tokens"] == 1716 + assert all(value is None for value in logging_usage["completion_tokens_details"].values()) @pytest.mark.parametrize( ("body", "error", "message"), diff --git a/tests/unit/llms/gemini/test_gemini_image_generation_transformation.py b/tests/unit/llms/gemini/test_gemini_image_generation_transformation.py index 138aafcbfcd..5d5edc2a49c 100644 --- a/tests/unit/llms/gemini/test_gemini_image_generation_transformation.py +++ b/tests/unit/llms/gemini/test_gemini_image_generation_transformation.py @@ -294,7 +294,7 @@ def test_gemini_image_generation_preserves_tool_config_side_effect(): } -def test_gemini_image_generation_usage_without_output_details_treats_output_as_image(): +def test_gemini_image_generation_preserves_unspecified_output_modality(): config = GoogleImageGenConfig() raw_response = httpx.Response( status_code=200, @@ -334,8 +334,8 @@ def test_gemini_image_generation_usage_without_output_details_treats_output_as_i ) usage = result.model_dump()["usage"] - assert usage["completion_tokens_details"]["text_tokens"] == 0 - assert usage["completion_tokens_details"]["image_tokens"] == 1716 + assert usage["completion_tokens_details"] == {} + assert usage["output_tokens"] == 1716 def test_gemini_image_generation_response_tracks_web_search_requests(): diff --git a/tests/unit/llms/gemini/test_image_usage_transformation.py b/tests/unit/llms/gemini/test_image_usage_transformation.py index f2f1c3d999f..cb64ddb61ac 100644 --- a/tests/unit/llms/gemini/test_image_usage_transformation.py +++ b/tests/unit/llms/gemini/test_image_usage_transformation.py @@ -11,7 +11,6 @@ PROMPT_DETAILS: Final = [{"modality": "TEXT", "tokenCount": 30}, {"modality": "I @pytest.mark.parametrize( ("candidates_details", "expected_output_details"), [ - ({}, {"image_tokens": 1716, "text_tokens": 0}), ( {"candidatesTokensDetails": [{"modality": "IMAGE", "tokenCount": 1120}]}, {"image_tokens": 1120, "text_tokens": 596}, @@ -48,6 +47,72 @@ def test_transform_gemini_image_usage_reports_image_and_chat_style_counts( } +def test_transform_gemini_image_usage_adds_exclusive_thoughts_to_output_tokens() -> None: + usage: Final = transform_gemini_image_usage( + { + "promptTokenCount": 12, + "candidatesTokenCount": 1300, + "thoughtsTokenCount": 40, + "totalTokenCount": 1352, + "candidatesTokensDetails": [ + {"modality": "TEXT", "tokenCount": 10}, + {"modality": "IMAGE", "tokenCount": 1290}, + ], + } + ) + + assert usage.output_tokens == 1340 + assert usage.completion_tokens == 1340 + assert usage.completion_tokens_details == { + "text_tokens": 10, + "image_tokens": 1290, + "reasoning_tokens": 40, + } + assert usage.output_tokens_details == usage.completion_tokens_details + + +def test_transform_gemini_image_usage_does_not_double_count_inclusive_thoughts() -> None: + usage: Final = transform_gemini_image_usage( + { + "promptTokenCount": 12, + "candidatesTokenCount": 1300, + "thoughtsTokenCount": 40, + "totalTokenCount": 1312, + "candidatesTokensDetails": [ + {"modality": "TEXT", "tokenCount": 10}, + {"modality": "IMAGE", "tokenCount": 1250}, + ], + } + ) + + assert usage.output_tokens == 1300 + assert usage.completion_tokens == 1300 + assert usage.completion_tokens_details == { + "text_tokens": 10, + "image_tokens": 1250, + "reasoning_tokens": 40, + } + assert usage.output_tokens_details == usage.completion_tokens_details + + +@pytest.mark.parametrize("thoughts", [0, 40]) +def test_transform_gemini_image_usage_preserves_unspecified_output_modality(thoughts: int) -> None: + usage: Final = transform_gemini_image_usage( + { + "promptTokenCount": 12, + "candidatesTokenCount": 1300, + "thoughtsTokenCount": thoughts, + "totalTokenCount": 1312 + thoughts, + } + ) + + assert usage.output_tokens == 1300 + thoughts + assert usage.completion_tokens_details == { + "reasoning_tokens": thoughts, + } + assert usage.output_tokens_details == usage.completion_tokens_details + + @pytest.mark.parametrize("count_field", ["promptTokenCount", "totalTokenCount"]) def test_transform_gemini_image_usage_rejects_a_count_that_is_not_a_number(count_field: str): with pytest.raises(ValidationError): diff --git a/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py b/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py index 65d6e97f8e5..c1c4c5010ed 100644 --- a/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py +++ b/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py @@ -19,6 +19,7 @@ from litellm.types.utils import ( ImageResponse, ImageUsage, ImageUsageInputTokensDetails, + ModelInfo, PromptTokensDetailsWrapper, Usage, ) @@ -148,3 +149,74 @@ def test_vertex_image_generation_cost_no_web_search_when_absent(monkeypatch): ) assert cost_zero == cost_none + + +@pytest.mark.parametrize("priority_image_rate", [None, 8e-5]) +def test_vertex_image_generation_cost_prices_token_details_by_service_tier(priority_image_rate: float | None) -> None: + model: Final = "gemini-scripted-image-tokens" + model_info: Final[ModelInfo] = { + "key": model, + "max_tokens": 1000, + "max_input_tokens": 1000, + "max_output_tokens": 1000, + "input_cost_per_token": 2e-6, + "output_cost_per_token": 1e-5, + "output_cost_per_image_token": 4e-5, + "input_cost_per_token_priority": 4e-6, + "output_cost_per_token_priority": 2e-5, + "output_cost_per_image_token_priority": priority_image_rate, + "litellm_provider": "vertex_ai", + "mode": "image_generation", + "supported_openai_params": None, + } + usage: Final = ImageUsage.model_validate( + { + "input_tokens": 12, + "input_tokens_details": {"text_tokens": 12, "image_tokens": 0}, + "output_tokens": 1340, + "total_tokens": 1352, + "completion_tokens_details": { + "text_tokens": 10, + "image_tokens": 1290, + "reasoning_tokens": 40, + }, + "output_tokens_details": { + "text_tokens": 10, + "image_tokens": 1290, + "reasoning_tokens": 40, + }, + } + ) + image_response: Final = ImageResponse( + data=[ImageObject(b64_json="synthetic-image")], + usage=usage, + ) + standard_cost: Final = vertex_image_generation_cost_calculator( + model=model, + image_response=image_response, + model_info=model_info, + vertex_location="global", + ) + priority_cost: Final = vertex_image_generation_cost_calculator( + model=model, + image_response=image_response, + model_info=model_info, + vertex_location="global", + service_tier="priority", + ) + + expected_standard_cost: Final = 12 * 2e-6 + (10 + 40) * 1e-5 + 1290 * 4e-5 + expected_priority_cost: Final = ( + 12 * 4e-6 + (10 + 40) * 2e-5 + 1290 * (4e-5 if priority_image_rate is None else priority_image_rate) + ) + assert standard_cost == pytest.approx(expected_standard_cost) + assert priority_cost == pytest.approx(expected_priority_cost) + + +def test_vertex_image_generation_without_usage_keeps_flat_image_price() -> None: + cost: Final = vertex_image_generation_cost_calculator( + model="gemini-scripted-image-tokens", + image_response=ImageResponse(data=[ImageObject(b64_json="image")]), + model_info={"output_cost_per_image": 0.07}, + ) + assert cost == pytest.approx(0.07) diff --git a/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py b/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py index 10ce24e969c..e1eadb89328 100644 --- a/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py +++ b/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py @@ -15,6 +15,7 @@ from litellm.llms.gemini.image_generation.transformation import GoogleImageGenCo from litellm.llms.vertex_ai.image_generation import ( get_vertex_ai_image_generation_config, ) +from litellm.llms.vertex_ai.image_generation.cost_calculator import cost_calculator from litellm.llms.vertex_ai.image_generation.vertex_gemini_transformation import ( VertexAIGeminiImageGenerationConfig, ) @@ -410,6 +411,97 @@ class TestVertexAIGeminiImageGenerationConfig: assert result.usage.output_tokens == 17 assert result.usage.total_tokens == 110 + def test_transform_text_only_no_image_response_bills_text_tokens(self) -> None: + result: Final = self.config.transform_image_generation_response( + model="gemini-3.1-flash-image", + raw_response=httpx.Response( + 200, + json={ + "candidates": [ + {"content": {"parts": [{"text": "No image generated"}]}, "finishReason": "NO_IMAGE"} + ], + "usageMetadata": { + "promptTokenCount": 5, + "candidatesTokenCount": 25, + "totalTokenCount": 30, + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 25}], + }, + }, + ), + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert result.data == [] + assert cost_calculator( + model="gemini-3.1-flash-image", + image_response=result, + model_info={ + "input_cost_per_token": 5e-7, + "output_cost_per_token": 3e-6, + "output_cost_per_image_token": 6e-5, + }, + vertex_location="global", + ) == pytest.approx(5 * 5e-7 + 25 * 3e-6) + + def test_transform_image_generation_response_preserves_usage_details_and_traffic_type(self): + mock_response: Final = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "base64_encoded_image_data", + } + } + ] + } + } + ], + "usageMetadata": { + "promptTokenCount": 12, + "candidatesTokenCount": 1300, + "thoughtsTokenCount": 40, + "totalTokenCount": 1352, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 12}], + "candidatesTokensDetails": [ + {"modality": "TEXT", "tokenCount": 10}, + {"modality": "IMAGE", "tokenCount": 1290}, + ], + "trafficType": "ON_DEMAND_PRIORITY", + }, + } + mock_response.headers = {} + + model_response: Final = ImageResponse() + result: Final = self.config.transform_image_generation_response( + model="gemini-3.1-flash-image", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert result.usage.output_tokens == 1340 + assert result.usage.input_tokens_details.model_dump() == {"text_tokens": 12, "image_tokens": 0} + assert result.usage.completion_tokens_details == { + "text_tokens": 10, + "image_tokens": 1290, + "reasoning_tokens": 40, + } + assert result.usage.output_tokens_details == result.usage.completion_tokens_details + assert result._hidden_params["provider_specific_fields"] == {"traffic_type": "ON_DEMAND_PRIORITY"} + def test_transform_image_generation_response_multiple_images(self): """Test response transformation with multiple images""" mock_response = MagicMock(spec=httpx.Response) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 07b6111aa67..b991f492bdd 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -2503,6 +2503,36 @@ def test_vertex_gemini_flash_image_generation_regional_costs_uplift_over_global( assert image_cost("us-central1") == pytest.approx(global_cost * 1.10, rel=1e-9) +@pytest.mark.parametrize("requested_tier", ["flex", "priority"]) +def test_vertex_gemini_image_generation_bills_only_the_tier_vertex_served( + _local_model_cost_map: None, requested_tier: str +) -> None: + """Image requests never send service_tier upstream, so a request tier must not change the bill + when Vertex reports no trafficType, while a served ON_DEMAND_PRIORITY still bills priority rates.""" + usage: Final = ImageUsage( + input_tokens=1000, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=1000), + output_tokens=1120, + total_tokens=2120, + ) + + def image_cost(traffic_type: str | None, service_tier: str | None) -> float: + response: Final = ImageResponse(data=[ImageObject(b64_json="img")], usage=usage) + response.hidden_params["provider_specific_fields"] = {"traffic_type": traffic_type} if traffic_type else {} + return completion_cost( + completion_response=response, + model="vertex_ai/gemini-3.1-flash-image", + call_type="image_generation", + optional_params={"service_tier": service_tier} if service_tier else {}, + ) + + standard_cost: Final = image_cost(None, None) + assert standard_cost > 0 + assert image_cost(None, requested_tier) == standard_cost + assert image_cost("ON_DEMAND", requested_tier) == standard_cost + assert image_cost("ON_DEMAND_PRIORITY", requested_tier) > standard_cost + + def test_vertex_uplift_composes_with_above_128k_pricing(monkeypatch): """The regional-endpoint uplift multiplies whatever rate the request priced at, including the above-128k dynamic rates, so a synthetic model carrying both keys From cb024047ac189b26fcf4a1f47a9f588c40eebef9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:32:19 +0000 Subject: [PATCH 070/189] test(spend): expect token billing for Vertex image edits with usage (#51109) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integration/spend/test_vertex_image_regional_uplift.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/spend/test_vertex_image_regional_uplift.py b/tests/integration/spend/test_vertex_image_regional_uplift.py index dfd52d28f1f..8cb116bd347 100644 --- a/tests/integration/spend/test_vertex_image_regional_uplift.py +++ b/tests/integration/spend/test_vertex_image_regional_uplift.py @@ -242,7 +242,7 @@ def test_image_generation_bills_the_deployment_location( [ pytest.param("global", False, _GLOBAL_IMAGE_COST, id="global-edit"), pytest.param(_REGION, False, _REGIONAL_IMAGE_COST, id="regional-edit"), - pytest.param(_REGION, True, _REGIONAL_IMAGE_COST, id="regional-edit-usage-is-per-image"), + pytest.param(_REGION, True, _REGIONAL_TOKEN_COST, id="regional-edit-usage-bills-tokens"), ], ) def test_image_edit_bills_the_deployment_location( @@ -489,8 +489,8 @@ def _burst_calls(global_model: str, regional_model: str) -> tuple[_Call, ...]: kinds: Final = ( ("/v1/images/generations", global_model, _GLOBAL_TOKEN_COST), ("/v1/images/generations", regional_model, _REGIONAL_TOKEN_COST), - ("/v1/images/edits", global_model, _GLOBAL_IMAGE_COST), - ("/v1/images/edits", regional_model, _REGIONAL_IMAGE_COST), + ("/v1/images/edits", global_model, _GLOBAL_TOKEN_COST), + ("/v1/images/edits", regional_model, _REGIONAL_TOKEN_COST), ) return tuple( _Call(call_id=uuid.uuid4().hex, route=route, model=model, expected=expected) From 054ed65cdc4b1b1252eb5a4d0bf12885f4f664d7 Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:38:52 -0800 Subject: [PATCH 071/189] fix(reasoning): log the GPT-5 reasoning_effort drop warning only when the provider gets no effort (#50995) * fix(reasoning): log the GPT-5 reasoning_effort drop warning only when the provider gets no effort * fix(proxy): count an extra_body effort as sent and pass the drop holder through ChatGPT and Copilot * fix(proxy): pass the drop holder to get_optional_params under the internal _litellm_ prefix * fix(openai): keep the reasoning effort drop warning when extra_body sends another level The held warning now stays quiet only when the request sends the level the gate dropped, so a top-level xhigh dropped while extra_body sends none still logs. The Copilot test case writes the access-token file the current authenticator reads. * fix(openai): keep the reasoning effort drop warning when a request fails before reaching the provider The drop warning holder now lives for the whole completion call, and completion's exception handler prints whatever it still holds, since a request that fails after mapping never reached the provider. Each flush empties the holder, so a provider error after dispatch does not print the warning a second time * fix(openai): warn when extra_body's reasoning_effort carries no effort level on the Responses bridge --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/_logging.py | 7 +- .../chat/gpt_reasoning_transformation.py | 3 + litellm/llms/azure_ai/chat/transformation.py | 4 +- litellm/llms/chatgpt/chat/transformation.py | 13 +- .../chat/gpt_reasoning_transformation.py | 46 +++- litellm/llms/openai/openai.py | 5 +- litellm/main.py | 124 ++++++--- litellm/utils.py | 70 +++-- tests/unit/test_main.py | 255 ++++++++++++++++++ 9 files changed, 457 insertions(+), 70 deletions(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index 6a38eea2b12..e5c8d7b6b77 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -11,7 +11,7 @@ from collections.abc import Iterator, Mapping from datetime import datetime from logging import Formatter from types import FrameType -from typing import Final, TextIO +from typing import Final, Protocol, TextIO from urllib.parse import unquote from pydantic import TypeAdapter @@ -936,6 +936,11 @@ else: handler.setFormatter(formatter) + +class WarningLogger(Protocol): + def warning(self, msg: str, /, *args: object) -> None: ... + + verbose_proxy_logger = logging.getLogger("LiteLLM Proxy") # Malformed virtual key rejections log through this child; LevelRoutingStreamHandler # writes its WARNING records to stdout. It has no handler or level of its own. diff --git a/litellm/llms/azure/chat/gpt_reasoning_transformation.py b/litellm/llms/azure/chat/gpt_reasoning_transformation.py index 29305866e23..428813f771c 100644 --- a/litellm/llms/azure/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/azure/chat/gpt_reasoning_transformation.py @@ -4,6 +4,7 @@ from collections.abc import Collection, Mapping from typing import ClassVar, Final import litellm +from litellm._logging import WarningLogger, verbose_logger from litellm.llms.openai.chat.gpt_reasoning_transformation import ( OpenAIGPT5Config, is_gpt_reasoning_series_name, @@ -90,6 +91,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): api_version: str = "", *, declared_reasoning_efforts: Collection[str] | None = None, + reasoning_drop_logger: WarningLogger = verbose_logger, ) -> dict: return OpenAIGPT5Config.map_openai_params( self, @@ -98,6 +100,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): model=model, drop_params=drop_params, declared_reasoning_efforts=declared_reasoning_efforts, + reasoning_drop_logger=reasoning_drop_logger, ) def transform_request( diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 93ff6d8170a..9c87050fde4 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -9,7 +9,7 @@ from httpx import Response from pydantic import TypeAdapter, ValidationError import litellm -from litellm._logging import verbose_logger +from litellm._logging import WarningLogger, verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, declared_prompt_cache_breakpoint, @@ -203,6 +203,7 @@ class AzureAIStudioConfig(OpenAIConfig): drop_params: bool, *, declared_reasoning_efforts: Collection[str] | None = None, + reasoning_drop_logger: WarningLogger = verbose_logger, ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if not azureAIGPT5Config.is_model_gpt_5_model(model): return super().map_openai_params( @@ -217,6 +218,7 @@ class AzureAIStudioConfig(OpenAIConfig): model=model, drop_params=drop_params, declared_reasoning_efforts=declared_reasoning_efforts, + reasoning_drop_logger=reasoning_drop_logger, ) def _supports_stop_reason(self, model: str) -> bool: diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.py index e8f057b4d3d..9e1e752d8b8 100644 --- a/litellm/llms/chatgpt/chat/transformation.py +++ b/litellm/llms/chatgpt/chat/transformation.py @@ -1,6 +1,7 @@ from collections.abc import Mapping from typing import Any, Final +from litellm._logging import WarningLogger, verbose_logger from litellm.exceptions import AuthenticationError from litellm.llms.openai.openai import OpenAIConfig from litellm.types.llms.openai import AllMessageValues @@ -118,11 +119,15 @@ class ChatGPTConfig(OpenAIConfig): def map_openai_params( self, - non_default_params: dict, - optional_params: dict, + non_default_params: dict[str, object], + optional_params: dict[str, object], model: str, drop_params: bool, - ) -> dict: - optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params) + *, + reasoning_drop_logger: WarningLogger = verbose_logger, + ) -> dict[str, object]: + optional_params = super().map_openai_params( + non_default_params, optional_params, model, drop_params, reasoning_drop_logger=reasoning_drop_logger + ) optional_params.setdefault("stream", False) return optional_params diff --git a/litellm/llms/openai/chat/gpt_reasoning_transformation.py b/litellm/llms/openai/chat/gpt_reasoning_transformation.py index 8c9551d01c0..ad90dcf9655 100644 --- a/litellm/llms/openai/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/openai/chat/gpt_reasoning_transformation.py @@ -1,11 +1,12 @@ """Support for OpenAI gpt-5 model family.""" import re -from collections.abc import Collection +from collections.abc import Collection, Mapping +from dataclasses import dataclass from typing import ClassVar, Final import litellm -from litellm._logging import verbose_logger +from litellm._logging import WarningLogger, verbose_logger from litellm.utils import ( declared_value_factory, has_catalog_entry, @@ -37,6 +38,44 @@ def refused_reasoning_effort_warning(*, suggest_declared_levels: bool, suggest_b ) +def _effort_level(sent_effort: object) -> str | None: + match sent_effort: + case str(level) | {"effort": str(level)}: + return level + case _: + return None + + +def _sent_effort_level(sent_params: Mapping[str, object]) -> str | None: + match sent_params: + case {"extra_body": {"reasoning_effort": sent_effort}}: + return _effort_level(sent_effort) + case {"reasoning_effort": sent_effort}: + return _effort_level(sent_effort) + case _: + return None + + +@dataclass(slots=True) +class DeferredReasoningDropWarnings: + logger: WarningLogger + held: tuple[tuple[str, tuple[object, ...]], ...] = () + + def warning(self, msg: str, /, *args: object) -> None: + self.held = (*self.held, (msg, args)) + + def log_unless_dropped_effort_sent(self, sent_params: Mapping[str, object]) -> None: + sent_effort: Final = _sent_effort_level(sent_params) + held: Final = self.held + self.held = () + for msg, args in held: + match args: + case ("reasoning_effort", dropped_effort, *_) if dropped_effort == sent_effort: + continue + case _: + self.logger.warning(msg, *args) + + def _catalogue_declares_default_effort() -> bool: """Whether the loaded cost map carries default_reasoning_effort for ANY entry. @@ -330,6 +369,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): drop_params: bool, *, declared_reasoning_efforts: Collection[str] | None = None, + reasoning_drop_logger: WarningLogger = verbose_logger, ) -> dict: if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: @@ -364,7 +404,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): and self.refuses_reasoning_effort_level(model, requested_effort) ): if litellm.drop_params or drop_params: - verbose_logger.warning( + reasoning_drop_logger.warning( refused_reasoning_effort_warning( suggest_declared_levels=declared_reasoning_efforts is not None, suggest_base_model=self.LOOKS_UP_BASE_MODEL diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 495ebf594d8..55442dfb1cf 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -27,7 +27,7 @@ from typing_extensions import overload import litellm from litellm import LlmProviders -from litellm._logging import verbose_logger +from litellm._logging import WarningLogger, verbose_logger from litellm.constants import DEFAULT_MAX_RETRIES from litellm.exceptions import UnsupportedParamsError from litellm.files.types import FileContentStreamingResult @@ -260,6 +260,8 @@ class OpenAIConfig(BaseConfig): optional_params: dict, model: str, drop_params: bool, + *, + reasoning_drop_logger: WarningLogger = verbose_logger, ) -> dict: """ """ if openaiOSeriesConfig.is_model_o_series_model(model=model): @@ -275,6 +277,7 @@ class OpenAIConfig(BaseConfig): optional_params=optional_params, model=model, drop_params=drop_params, + reasoning_drop_logger=reasoning_drop_logger, ) elif litellm.openAIGPTAudioConfig.is_model_gpt_audio_model(model=model): return litellm.openAIGPTAudioConfig.map_openai_params( diff --git a/litellm/main.py b/litellm/main.py index ce71a41d516..2d6cd18162b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -128,6 +128,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.reasoning_effort_capability import ( + RequestedReasoningEffort, model_group_from_metadata, reasoning_effort_needs_allowing, resolve_chat_reasoning, @@ -154,7 +155,10 @@ from litellm.llms.bedrock.common_utils import ( from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled -from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_reasoning_transformation import ( + DeferredReasoningDropWarnings, + OpenAIGPT5Config, +) from litellm.llms.openai.common_utils import get_openai_credentials from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( @@ -225,7 +229,7 @@ from litellm.utils import ( token_counter as token_counter, # noqa: PLC0414 # public re-export ) -from ._logging import verbose_logger +from ._logging import WarningLogger, verbose_logger from .caching.caching import disable_cache, enable_cache, update_cache from .litellm_core_utils.core_helpers import ( as_str_mapping, @@ -5478,6 +5482,51 @@ def _complete_langflow(ctx: CompletionDispatchContext) -> _CompletionDispatchRes ) +def _responses_bridge_optional_params( + optional_params: dict[str, object], # mutable-ok: completion's optional_params, which the Responses bridge takes + summary_alias: object, + requested: RequestedReasoningEffort, + *, + custom_llm_provider: str, + model: str, + base_model: str | None, + api_base: str | None, + deployment_model_info: Mapping[str, object] | None, + drop_params: bool, + additional_drop_params: object, +) -> dict[str, object]: # mutable-ok: completion's optional_params, which the Responses bridge takes + if isinstance(requested, dict): + from litellm.responses.utils import ( + RefusedReasoningFields, + bridge_reasoning, + raise_refused_reasoning_fields, + ) + + bridged_reasoning: Final = bridge_reasoning( + requested=requested, + effort_kept="reasoning_effort" in optional_params, + summary_alias=summary_alias, + model=model, + base_model=base_model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + deployment_model_info=deployment_model_info, + drop_params=drop_params, + additional_drop_params=additional_drop_params, + ) + if isinstance(bridged_reasoning, RefusedReasoningFields): + raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider) + return {**optional_params, "reasoning_effort": bridged_reasoning} + if summary_alias is None: + return optional_params + kept_effort: Final = optional_params.get("reasoning_effort", requested) + if isinstance(kept_effort, dict): + return {**optional_params, "reasoning_effort": {**kept_effort, "summary": summary_alias}} + if kept_effort is None: + return {**optional_params, "reasoning_effort": {"summary": summary_alias}} + return {**optional_params, "reasoning_effort": {"effort": kept_effort, "summary": summary_alias}} + + def _complete_databricks_agent(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: from litellm.llms.databricks.agent.transformation import DatabricksAgentConfig @@ -5830,6 +5879,7 @@ def completion( merge_with_first_system=True, ) + reasoning_drop_warnings: Final = DeferredReasoningDropWarnings(logger=verbose_logger) try: if base_url is not None: api_base = base_url @@ -6085,14 +6135,22 @@ def completion( mapping_args: Final = MappingProxyType({**optional_param_args, "reasoning_effort": provider_reasoning_effort}) mapping_special_params: Final = MappingProxyType(dict(provider_params)) - def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]: - return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params) + def optional_params_for_model_info( + deployment_info: Mapping[str, object] | None, + reasoning_drop_logger: WarningLogger = verbose_logger, + ) -> dict[str, object]: + return get_optional_params( + **mapping_args, + model_info=deployment_info, + _litellm_reasoning_drop_logger=reasoning_drop_logger, + **mapping_special_params, + ) validated_model_info: Final = ( _MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None ) deployment_base_model: Final = base_model if isinstance(base_model, str) else None - optional_params = optional_params_for_model_info(validated_model_info) + optional_params = optional_params_for_model_info(validated_model_info, reasoning_drop_warnings) fakes_stream: Final = partial( _completion_fakes_stream, model, @@ -6289,6 +6347,7 @@ def completion( ) if mock_response is not None or mock_tool_calls or mock_timeout: kwargs.pop("mock_timeout", None) # remove for any fallbacks triggered + reasoning_drop_warnings.log_unless_dropped_effort_sent(optional_params) return mock_completion( model, messages, @@ -6337,41 +6396,19 @@ def completion( from litellm.completion_extras import responses_api_bridge optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) - - if isinstance(resolved_reasoning_effort, dict): - from litellm.responses.utils import ( - RefusedReasoningFields, - bridge_reasoning, - raise_refused_reasoning_fields, - ) - - bridged_reasoning: Final = bridge_reasoning( - requested=resolved_reasoning_effort, - effort_kept="reasoning_effort" in optional_params, - summary_alias=rs_val, - model=model, - base_model=base_model if isinstance(base_model, str) else None, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - deployment_model_info=typed_deployment_model_info, - drop_params=drop_unsupported_reasoning, - additional_drop_params=routing_kwargs.get("additional_drop_params"), - ) - if isinstance(bridged_reasoning, RefusedReasoningFields): - raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider) - optional_params["reasoning_effort"] = bridged_reasoning - elif rs_val is not None: - eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort) - if isinstance(eff, dict): - optional_params["reasoning_effort"] = {**eff, "summary": rs_val} - elif eff is not None: - optional_params["reasoning_effort"] = { - "effort": eff, - "summary": rs_val, - } - else: - optional_params["reasoning_effort"] = {"summary": rs_val} - + bridge_optional_params: Final = _responses_bridge_optional_params( + optional_params, + rs_val, + resolved_reasoning_effort, + custom_llm_provider=custom_llm_provider, + model=model, + base_model=deployment_base_model, + api_base=api_base, + deployment_model_info=typed_deployment_model_info, + drop_params=drop_unsupported_reasoning, + additional_drop_params=routing_kwargs.get("additional_drop_params"), + ) + reasoning_drop_warnings.log_unless_dropped_effort_sent(bridge_optional_params) return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller model=model, messages=messages, @@ -6381,7 +6418,7 @@ def completion( api_base=api_base, acompletion=acompletion, logging_obj=logging, - optional_params=optional_params, + optional_params=bridge_optional_params, litellm_params=litellm_params, timeout=timeout, client=client, # pass AsyncOpenAI, OpenAI client @@ -6390,7 +6427,9 @@ def completion( stream=stream, cache=kwargs.get("cache"), ) - elif (custom_llm_provider == "openai" and OpenAIGPT5Config.is_model_gpt_5_model(model)) or ( + + reasoning_drop_warnings.log_unless_dropped_effort_sent(optional_params) + if (custom_llm_provider == "openai" and OpenAIGPT5Config.is_model_gpt_5_model(model)) or ( custom_llm_provider == "azure" and litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(_azure_detection_model) ): @@ -6656,6 +6695,7 @@ def completion( raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) return response except Exception as e: + reasoning_drop_warnings.log_unless_dropped_effort_sent(MappingProxyType({})) ## Map to OpenAI Exception raise exception_type( model=model, diff --git a/litellm/utils.py b/litellm/utils.py index c6ade2603c4..156247897c3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -516,7 +516,7 @@ from litellm.secret_managers.main import ( resolve_api_key_oidc_reference, ) -from ._logging import is_debugging_on, verbose_logger +from ._logging import WarningLogger, is_debugging_on, verbose_logger from .caching.caching import ( AzureBlobCache, Cache, @@ -5441,6 +5441,42 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl return deployment_reasoning_effort_levels(model_info) or () +def _map_params_with_provider_config( + provider_config: BaseConfig, + *, + custom_llm_provider: str, + non_default_params: dict[str, object], # mutable-ok: map_openai_params takes the caller's dicts + optional_params: dict[str, object], # mutable-ok: map_openai_params fills and returns the caller's dict + model: str, + drop_params: bool, + model_info: Mapping[str, object] | None, + reasoning_drop_logger: WarningLogger, +) -> dict[str, object]: # mutable-ok: get_optional_params keeps building on the mapped params + if custom_llm_provider == "azure_ai" and isinstance(provider_config, litellm.AzureAIStudioConfig): + return provider_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + declared_reasoning_efforts=_declared_reasoning_efforts(model_info), + reasoning_drop_logger=reasoning_drop_logger, + ) + if isinstance(provider_config, (litellm.ChatGPTConfig, litellm.GithubCopilotConfig)): + return provider_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + reasoning_drop_logger=reasoning_drop_logger, + ) + return provider_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) + + _DROPPED_REASONING_PARAMS: Final = ("reasoning_effort", "thinking") _DROP_UNREAD_OUTPUT_CONFIG_WARNING: Final = ( @@ -5530,6 +5566,7 @@ def get_optional_params( prompt_cache_key: str | None = None, base_model: str | None = None, model_info: Mapping[str, object] | None = None, + _litellm_reasoning_drop_logger: WarningLogger = verbose_logger, **kwargs: object, ): drop_params = normalize_drop_params(drop_params) # rebind-ok: config and DB deployments pass "true" as a string @@ -5541,6 +5578,7 @@ def get_optional_params( # OpenAI param. passed_params.pop("base_model", None) passed_params.pop("model_info", None) + passed_params.pop("_litellm_reasoning_drop_logger", None) provider_config: BaseConfig | None = None if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]: provider_config = ProviderConfigManager.get_provider_chat_config( @@ -6035,6 +6073,7 @@ def get_optional_params( model=model, drop_params=bool(drop_params), declared_reasoning_efforts=_declared_reasoning_efforts(model_info), + reasoning_drop_logger=_litellm_reasoning_drop_logger, ) if litellm.OpenAIGPT5Config.is_model_gpt_5_model(model=model) else litellm.OpenAIConfig().map_openai_params( @@ -6067,6 +6106,7 @@ def get_optional_params( model=_azure_detection_model, drop_params=bool(drop_params), declared_reasoning_efforts=_declared_reasoning_efforts(model_info), + reasoning_drop_logger=_litellm_reasoning_drop_logger, ) else: verbose_logger.debug( @@ -6088,21 +6128,15 @@ def get_optional_params( drop_params=bool(drop_params), ) elif provider_config is not None: - optional_params = ( - provider_config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model=model, - drop_params=bool(drop_params), - declared_reasoning_efforts=_declared_reasoning_efforts(model_info), - ) - if custom_llm_provider == "azure_ai" and isinstance(provider_config, litellm.AzureAIStudioConfig) - else provider_config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model=model, - drop_params=bool(drop_params), - ) + optional_params = _map_params_with_provider_config( + provider_config, + custom_llm_provider=custom_llm_provider, + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=bool(drop_params), + model_info=model_info, + reasoning_drop_logger=_litellm_reasoning_drop_logger, ) else: # assume passing in params for openai-like api optional_params = litellm.OpenAILikeChatConfig().map_openai_params( @@ -11734,8 +11768,8 @@ def reasoning_effort_with_summary_alias( def strip_reasoning_summary_aliases_from_optional_params( - optional_params: dict, -) -> tuple[dict, object | None]: + optional_params: Mapping[str, object], +) -> tuple[dict[str, object], object | None]: """Copy optional_params; remove reasoningSummary aliases from top-level and extra_body.""" op: Final = dict(optional_params) rs_val = op.pop("reasoningSummary", None) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index c4879587038..577d4b4b11d 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -2451,6 +2451,261 @@ def test_openai_responses_bridge_drops_a_reasoning_effort_dict_for_a_model_witho assert "reasoning" not in request_body +_AZURE_TEST_API_BASE: Final = "https://example.openai.azure.com" +_UNLISTED_AZURE_GPT_5_MODEL: Final = "azure/gpt-5.6-litellm-unlisted" +_FUNCTION_TOOL: Final = {"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}} + + +@dataclass(frozen=True, slots=True) +class AzureUpstreamRoutes: + chat: respx.Route + responses: respx.Route + + def sent_reasoning_effort(self) -> object: + if self.responses.called: + return json.loads(self.responses.calls[0].request.content).get("reasoning", {}).get("effort") + return json.loads(self.chat.calls[0].request.content).get("reasoning_effort") + + +@pytest.fixture +def azure_upstream_routes( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + openai_api_response: dict[str, object], +) -> AzureUpstreamRoutes: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("LITELLM_REASONING_AUTO_SUMMARY", raising=False) + return AzureUpstreamRoutes( + chat=respx_mock.post(url__regex=r".*/chat/completions.*").respond(json=openai_api_response), + responses=respx_mock.post(url__regex=r".*/responses.*").respond( + json={ + "id": "resp_test", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "gpt-5.6-litellm-unlisted", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + ), + ) + + +@pytest.mark.parametrize( + ("request_params", "expected_sent_effort"), + [ + pytest.param({"reasoning_effort": "none", "reasoningSummary": "auto"}, "none", id="summary-alias"), + pytest.param( + {"reasoning_effort": "none", "extra_body": {"reasoningSummary": "auto"}}, + "none", + id="summary-alias-in-extra-body", + ), + pytest.param( + {"reasoning_effort": {"effort": "none", "summary": "auto"}, "tools": [_FUNCTION_TOOL]}, + None, + id="dict-with-summary-and-tool", + ), + pytest.param( + {"reasoning_effort": {"effort": "none"}, "reasoningSummary": "auto"}, None, id="dict-with-summary-alias" + ), + pytest.param({"reasoning_effort": "none"}, None, id="chat"), + pytest.param({"reasoning_effort": "none", "tools": [_FUNCTION_TOOL]}, None, id="bridged-without-summary"), + pytest.param({"reasoning_effort": {"effort": "none", "summary": "auto"}}, None, id="dict-with-summary-on-chat"), + pytest.param( + {"reasoning_effort": "none", "extra_body": {"reasoning_effort": "none"}}, + "none", + id="effort-in-extra-body", + ), + pytest.param( + {"reasoning_effort": "none", "tools": [_FUNCTION_TOOL], "extra_body": {"reasoning_effort": "none"}}, + "none", + id="effort-in-extra-body-bridged", + ), + pytest.param( + { + "reasoning_effort": "none", + "tools": [_FUNCTION_TOOL], + "extra_body": {"reasoning_effort": {"effort": "none", "summary": "auto"}}, + }, + "none", + id="effort-dict-in-extra-body-bridged", + ), + pytest.param( + { + "reasoning_effort": "none", + "reasoningSummary": "auto", + "extra_body": {"reasoning_effort": {"summary": "detailed"}}, + }, + None, + id="summary-only-dict-in-extra-body-bridged", + ), + pytest.param( + {"reasoning_effort": "none", "extra_body": {"reasoning_effort": "low"}}, + "low", + id="other-effort-in-extra-body", + ), + pytest.param( + {"reasoning_effort": "none", "tools": [_FUNCTION_TOOL], "extra_body": {"reasoning_effort": "low"}}, + "low", + id="other-effort-in-extra-body-bridged", + ), + ], +) +def test_reasoning_effort_drop_warning_fires_unless_the_provider_gets_the_dropped_effort( + request_params: dict[str, object], + expected_sent_effort: str | None, + azure_upstream_routes: AzureUpstreamRoutes, + caplog: pytest.LogCaptureFixture, +): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + litellm.completion( + model=_UNLISTED_AZURE_GPT_5_MODEL, + messages=[{"role": "user", "content": "hello"}], + api_base=_AZURE_TEST_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + drop_params=True, + **request_params, + ) + + sent_effort: Final = azure_upstream_routes.sent_reasoning_effort() + drop_warnings: Final = [r.getMessage() for r in caplog.records if "Dropping reasoning_effort" in r.getMessage()] + assert sent_effort == expected_sent_effort + assert len(drop_warnings) == (0 if sent_effort == "none" else 1) + + +@pytest.mark.parametrize( + "request_params", + [ + pytest.param( + {"provider_affinity_header": "x-session-affinity", "litellm_session_id": "a\nb"}, + id="rejected-before-dispatch", + ), + pytest.param({}, id="rejected-by-the-provider"), + ], +) +def test_reasoning_effort_drop_warning_fires_once_when_the_request_fails( + request_params: dict[str, object], + azure_upstream_routes: AzureUpstreamRoutes, + caplog: pytest.LogCaptureFixture, +): + azure_upstream_routes.chat.respond(400, json={"error": {"message": "refused", "type": "invalid_request_error"}}) + with caplog.at_level(logging.WARNING, logger="LiteLLM"), pytest.raises(litellm.BadRequestError): + litellm.completion( + model=_UNLISTED_AZURE_GPT_5_MODEL, + messages=[{"role": "user", "content": "hello"}], + api_base=_AZURE_TEST_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + drop_params=True, + reasoning_effort="none", + **request_params, + ) + + assert len([r for r in caplog.records if "Dropping reasoning_effort" in r.getMessage()]) == 1 + + +@pytest.fixture +def github_copilot_access_token(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "access-token").write_text("token") + + +@pytest.mark.parametrize( + ("model", "api_base", "refused_effort", "model_info"), + [ + pytest.param("openai/gpt-5.6-litellm-unlisted", "https://api.openai.com/v1", "xhigh", None, id="openai"), + pytest.param( + "azure_ai/gpt-5.6-litellm-unlisted", + "https://example.services.ai.azure.com/openai/v1", + "xhigh", + None, + id="azure_ai", + ), + pytest.param( + "chatgpt/gpt-5.5", + "https://chatgpt.example.com/backend-api/codex", + "minimal", + {"reasoning_effort_levels": ["minimal", "low", "medium", "high"]}, + id="chatgpt-declared-level", + ), + pytest.param( + "github_copilot/gpt-5.5", + "https://api.githubcopilot.com", + "minimal", + {"reasoning_effort_levels": ["minimal", "low", "medium", "high"]}, + id="github-copilot-declared-level", + ), + ], +) +def test_reasoning_effort_drop_warning_stays_quiet_when_the_summary_alias_restores_the_effort( + model: str, + api_base: str, + refused_effort: str, + model_info: dict[str, object] | None, + azure_upstream_routes: AzureUpstreamRoutes, + github_copilot_access_token: None, + caplog: pytest.LogCaptureFixture, +): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_base=api_base, + api_key="fake-key", + drop_params=True, + reasoning_effort=refused_effort, + reasoningSummary="auto", + model_info=model_info, + ) + + assert azure_upstream_routes.sent_reasoning_effort() == refused_effort + assert not [r for r in caplog.records if "Dropping reasoning_effort" in r.getMessage()] + + +def test_a_request_param_named_like_the_drop_holder_still_reaches_the_provider( + azure_upstream_routes: AzureUpstreamRoutes, +): + litellm.completion( + model=_UNLISTED_AZURE_GPT_5_MODEL, + messages=[{"role": "user", "content": "hello"}], + api_base=_AZURE_TEST_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + drop_params=True, + reasoning_drop_logger="sent by the caller", + ) + + sent_body: Final = json.loads(azure_upstream_routes.chat.calls[0].request.content) + assert sent_body["reasoning_drop_logger"] == "sent by the caller" + + +def test_mock_response_keeps_the_reasoning_effort_drop_warning( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + litellm.completion( + model=_UNLISTED_AZURE_GPT_5_MODEL, + messages=[{"role": "user", "content": "hello"}], + api_base=_AZURE_TEST_API_BASE, + api_key="fake-key", + reasoning_effort="none", + drop_params=True, + mock_response="hi", + ) + + assert any("Dropping reasoning_effort='none'" in r.getMessage() for r in caplog.records) + + @pytest.mark.parametrize( "drop_params, expected_reasoning", [ From 478d9bf5d230419884efb9abb64d94ab071337dd Mon Sep 17 00:00:00 2001 From: shrey Date: Tue, 10 Nov 2026 19:50:44 -0800 Subject: [PATCH 072/189] test(mcp): read the comma grant through real credential storage (#51065) --- .../mcp_server/test_discoverable_endpoints.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/unit/proxy/mcp_server/test_discoverable_endpoints.py b/tests/unit/proxy/mcp_server/test_discoverable_endpoints.py index 47d24a9626f..64413c4a74a 100644 --- a/tests/unit/proxy/mcp_server/test_discoverable_endpoints.py +++ b/tests/unit/proxy/mcp_server/test_discoverable_endpoints.py @@ -5,6 +5,7 @@ import json import time from base64 import urlsafe_b64encode from datetime import datetime, timedelta, timezone +from types import SimpleNamespace from typing import TYPE_CHECKING, Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -15375,6 +15376,25 @@ _PUBLISHED_SLACK_METADATA: Final = { } +def _in_memory_credentials_table(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + from litellm.proxy import proxy_server + + monkeypatch.setenv("LITELLM_SALT_KEY", "connect-check-test-salt") + prisma: Final = MagicMock() + rows: Final = prisma.db.litellm_mcpusercredentials + + async def upsert(where: "Mapping[str, Mapping[str, str]]", data: "Mapping[str, Mapping[str, str]]") -> None: + key: Final = where["user_id_server_id"] + rows.find_unique.return_value = SimpleNamespace( + credential_b64=data["create"]["credential_b64"], user_id=key["user_id"], server_id=key["server_id"] + ) + + rows.find_unique = AsyncMock(return_value=None) + rows.upsert = AsyncMock(side_effect=upsert) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + return prisma + + @pytest.mark.asyncio @pytest.mark.parametrize( "server_update", @@ -15391,9 +15411,10 @@ async def test_connect_reads_a_comma_grant_against_the_server_scopes( from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server: Final = _cimd_oauth_server().model_copy(update=server_update) - credential: Final = {**_stored_grant(), "scopes": ["channels:read,chat:write,files:read"]} - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) - monkeypatch.setattr(db, "get_user_oauth_credential", AsyncMock(return_value=credential)) + prisma: Final = _in_memory_credentials_table(monkeypatch) + await db.store_user_oauth_credential( + prisma, "owner", server.server_id, "vendor-token", scopes=["channels:read,chat:write,files:read"] + ) assert await endpoints._vendor_credential_state("owner", server.server_id, ("channels:read",)) == "absent" monkeypatch.setitem(global_mcp_server_manager.registry, server.server_id, server) assert await endpoints._vendor_credential_state("owner", server.server_id, ("channels:read",)) == "present" From 63d39b41fd4cfa3b9aa0c47f3ea04756e4054e98 Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:21:39 -0800 Subject: [PATCH 073/189] fix(proxy): keep a request's cache opt-in when its key has cache metadata (#50999) * fix(proxy): merge a key's cache controls into the request's cache settings instead of replacing them * fix(proxy): keep a key with no-cache out of opt-in caching A key whose cache metadata sets no-cache now drops the caller's use-cache, so on a default_off proxy its responses stay out of the shared cache as before the merge change. default_on proxies ignore use-cache, so they are unchanged * fix(proxy): keep base handling of non-dict key cache metadata Key cache metadata that is not a dict, such as false or a string, wipes the request's cache settings as before the merge change, so those keys stay out of opt-in caching * fix(proxy): keep a key with use-cache false out of opt-in caching * fix(proxy): only keep request cache settings on an allowed opt-in --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/models/verification_token.py | 2 +- litellm/proxy/litellm_pre_call_utils.py | 41 ++++- .../unit/proxy/test_litellm_pre_call_utils.py | 155 ++++++++++++++++++ 3 files changed, 190 insertions(+), 8 deletions(-) diff --git a/litellm/models/verification_token.py b/litellm/models/verification_token.py index 89048c56a9f..4b804ebde1c 100644 --- a/litellm/models/verification_token.py +++ b/litellm/models/verification_token.py @@ -35,7 +35,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase): tpd_limit: int | None = None budget_duration: str | None = None budget_reset_at: datetime | None = None - allowed_cache_controls: list | None = [] + allowed_cache_controls: list[object] | None = [] allowed_routes: list | None = [] key_type: str | None = None permissions: dict = {} diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 447b41f9cfd..0b2ba93af71 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2217,15 +2217,41 @@ class LiteLLMProxyRequestSetup: return data @staticmethod - def add_key_level_controls(key_metadata: dict | None, data: dict, _metadata_variable_name: str): + def _request_cache_kept_for_key( + request_cache: Mapping[str, object], + key_cache: Mapping[str, object], + allowed_cache_controls: Sequence[object] | None, + ) -> Mapping[str, object]: + key_opts_out: Final = ( + key_cache.get("no-cache", False) is not False or key_cache.get("use-cache", True) is not True + ) + request_is_allowed: Final = not allowed_cache_controls or all( + control in allowed_cache_controls for control in request_cache + ) + if key_opts_out or request_cache.get("use-cache") is not True or not request_is_allowed: + return MappingProxyType({}) + return {k: v for k, v in request_cache.items() if not (k == "s-max-age" and "s-maxage" in key_cache)} + + @staticmethod + def add_key_level_controls( + key_metadata: dict | None, + data: dict, + _metadata_variable_name: str, + allowed_cache_controls: Sequence[object] | None = None, + ): if key_metadata is None: return data - if "cache" in key_metadata: - data["cache"] = {} - if isinstance(key_metadata["cache"], dict): - for k, v in key_metadata["cache"].items(): - if k in SupportedCacheControls: - data["cache"][k] = v + if "cache" in key_metadata and not isinstance(key_metadata["cache"], dict): + data["cache"] = {} # rebind-ok: data is an out-param + elif "cache" in key_metadata: + request_cache: Final = _DECODED_JSON.validate_python(data["cache"]) if "cache" in data else None + key_cache: Final = _request_fields(_DECODED_JSON.validate_python(key_metadata["cache"])) + data["cache"] = { # rebind-ok: data is an out-param + **LiteLLMProxyRequestSetup._request_cache_kept_for_key( + _request_fields(request_cache), key_cache, allowed_cache_controls + ), + **{k: v for k, v in key_cache.items() if k in SupportedCacheControls}, + } ## KEY-LEVEL SPEND LOGS / TAGS if "tags" in key_metadata and key_metadata["tags"] is not None: @@ -2967,6 +2993,7 @@ async def add_litellm_data_to_request( key_metadata=key_metadata, data=data, _metadata_variable_name=_metadata_variable_name, + allowed_cache_controls=user_api_key_dict.allowed_cache_controls, ) data[_metadata_variable_name] = LiteLLMProxyRequestSetup.add_team_and_project_level_controls( user_api_key_dict=user_api_key_dict, diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index f4a836b4b4b..f5757a0ff18 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -17,6 +17,7 @@ from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers import litellm +from litellm.caching.caching import Cache, CacheMode from litellm.constants import ( CALLER_SESSION_ID_METADATA_KEY, COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY, @@ -64,6 +65,7 @@ from litellm.proxy.litellm_pre_call_utils import ( resolve_tenant_otel_destinations, ) from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.types.caching import LiteLLMCacheType from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.types.utils import CredentialItem from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stubs @@ -8379,6 +8381,159 @@ async def test_key_level_callback_vars_survive_the_strip(): assert updated["dd_site"] == "us5.datadoghq.com" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_cache, request_cache, expected_cache, uses_cache_when_default_off", + [ + pytest.param( + {"ttl": 600}, + {"use-cache": True, "namespace": "team-a"}, + {"use-cache": True, "namespace": "team-a", "ttl": 600}, + True, + id="key-control-added-to-request-opt-in", + ), + pytest.param( + {"no-store": True}, + {"use-cache": True, "no-store": False}, + {"use-cache": True, "no-store": True}, + True, + id="key-control-wins-over-request", + ), + pytest.param( + {"use-cache": True, "namespace": "key-ns"}, + None, + {}, + False, + id="unsupported-key-controls-ignored", + ), + pytest.param(None, {"use-cache": True}, {}, False, id="non-dict-key-cache-still-wipes-request"), + pytest.param( + {"no-cache": True}, + {"use-cache": True, "ttl": 60}, + {"no-cache": True}, + False, + id="key-no-cache-keeps-the-key-out-of-opt-in-caching", + ), + pytest.param( + {"use-cache": False}, + {"use-cache": True, "ttl": 60}, + {}, + False, + id="key-use-cache-false-keeps-the-key-out-of-opt-in-caching", + ), + pytest.param( + {"use-cache": True}, + {"use-cache": True}, + {"use-cache": True}, + True, + id="key-use-cache-true-leaves-the-request-opt-in", + ), + pytest.param( + {"no-cache": "true"}, + {"use-cache": True}, + {"no-cache": "true"}, + False, + id="key-non-boolean-no-cache-keeps-the-key-out-of-opt-in-caching", + ), + pytest.param( + {"no-cache": False}, + {"use-cache": True}, + {"use-cache": True, "no-cache": False}, + True, + id="key-no-cache-false-leaves-the-request-opt-in", + ), + pytest.param( + {"ttl": 600}, + {"no-cache": True, "namespace": "team-a"}, + {"ttl": 600}, + False, + id="request-without-opt-in-is-still-replaced", + ), + pytest.param( + {"ttl": 600}, + {"use-cache": "true", "no-store": True}, + {"ttl": 600}, + False, + id="request-non-boolean-use-cache-is-still-replaced", + ), + pytest.param( + {"s-maxage": 1}, + {"use-cache": True, "s-max-age": 100000}, + {"use-cache": True, "s-maxage": 1}, + True, + id="key-s-maxage-overrides-the-request-s-max-age-alias", + ), + pytest.param( + {"ttl": 600}, + {"use-cache": True, "s-max-age": 5}, + {"use-cache": True, "s-max-age": 5, "ttl": 600}, + True, + id="request-s-max-age-kept-when-the-key-has-no-s-maxage", + ), + ], +) +async def test_key_cache_controls_merge_into_request_cache_controls( + key_cache: dict[str, object] | None, + request_cache: dict[str, object] | None, + expected_cache: dict[str, object], + uses_cache_when_default_off: bool, +): + key: Final = UserAPIKeyAuth(api_key="hashed-key", metadata={"cache": key_cache}) + data: Final = { + "model": "gpt-4.1-mini", + "messages": [{"role": "user", "content": "hello"}], + **({} if request_cache is None else {"cache": request_cache}), + } + + updated: Final = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=key, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated["cache"] == expected_cache + opt_in_cache: Final = Cache(type=LiteLLMCacheType.LOCAL, mode=CacheMode.default_off) + assert opt_in_cache.should_use_cache(**updated) is uses_cache_when_default_off + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "allowed_cache_controls, request_cache, expected_cache", + [ + pytest.param(["no-store"], {"use-cache": True}, {"ttl": 600}, id="opt-in-not-allowed-is-replaced"), + pytest.param( + ["use-cache"], {"use-cache": True, "no-store": True}, {"ttl": 600}, id="any-disallowed-control-is-replaced" + ), + pytest.param(["use-cache"], {"use-cache": True}, {"use-cache": True, "ttl": 600}, id="allowed-opt-in-is-kept"), + pytest.param([], {"use-cache": True}, {"use-cache": True, "ttl": 600}, id="empty-list-allows-the-opt-in"), + ], +) +async def test_key_cache_metadata_only_keeps_request_controls_the_key_allows( + allowed_cache_controls: list[str], + request_cache: dict[str, object], + expected_cache: dict[str, object], +): + key: Final = UserAPIKeyAuth( + api_key="hashed-key", + metadata={"cache": {"ttl": 600}}, + allowed_cache_controls=allowed_cache_controls, + ) + + updated: Final = await add_litellm_data_to_request( + data={"model": "gpt-4.1-mini", "messages": [{"role": "user", "content": "hello"}], "cache": request_cache}, + request=_callback_credential_request_mock(), + user_api_key_dict=key, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated["cache"] == expected_cache + + class TestPromotedTraceControlFields: """LIT-5137: caller metadata trace fields must reach litellm_metadata.""" From 83c47061e142178c41a956efafa7f38cb61d5272 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:33:52 +0000 Subject: [PATCH 074/189] feat(replicate): add google/gemini-4-carbon from the Replicate model page (#51125) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 19 +++++++++++++++++++ model_prices_and_context_window.json | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 092987ce69e..ced7e002ba4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -100030,5 +100030,24 @@ "supports_tool_choice": true, "supports_vision": true, "thinking_always_on": true + }, + "replicate/google/gemini-4-carbon": { + "input_cost_per_token": 2e-06, + "litellm_provider": "replicate", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "mode": "chat", + "output_cost_per_reasoning_token": 1e-05, + "output_cost_per_token": 1e-05, + "source": "https://replicate.com/google/gemini-4-carbon", + "supports_audio_input": true, + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": false, + "supports_video_input": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 092987ce69e..ced7e002ba4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -100030,5 +100030,24 @@ "supports_tool_choice": true, "supports_vision": true, "thinking_always_on": true + }, + "replicate/google/gemini-4-carbon": { + "input_cost_per_token": 2e-06, + "litellm_provider": "replicate", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "mode": "chat", + "output_cost_per_reasoning_token": 1e-05, + "output_cost_per_token": 1e-05, + "source": "https://replicate.com/google/gemini-4-carbon", + "supports_audio_input": true, + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": false, + "supports_video_input": true, + "supports_vision": true } } From 27e5c3ba75bb9bb4e1ba05d772a5e226112771cf Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 10 Nov 2026 20:51:34 -0800 Subject: [PATCH 075/189] fix(mcp)!: forward the LiteLLM key when the admin configures it (#50604) * fix(mcp)!: forward the LiteLLM key when the admin configures it Restores forwarding of the caller's LiteLLM key through explicit channels (Extra Headers listing X-LiteLLM-API-Key or Authorization, x-mcp-{alias}-authorization and x-mcp-auth) that #45401 and #46932 started stripping. Implicit Authorization forwarding stays scrubbed for true_passthrough and unlisted servers, gateway session tokens and bridge envelopes stay scrubbed, and guardrail and JWT signer bearer reads now skip the admission key * test(mcp): serve signer introspection from an in-process transport * fix(mcp): copy oauth2 headers so a listed key never reaches a later server * test(mcp): drive the shared-header regression through the real MCP client --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../mcp_jwt_signer/mcp_jwt_signer.py | 10 +- .../mcp_server/auth/user_api_key_auth_mcp.py | 47 +- .../proxy/mcp_server/mcp_server_manager.py | 16 +- litellm/proxy/mcp_server/operations.py | 2 +- litellm/proxy/mcp_server/rest_endpoints.py | 1 - tests/integration/mcp/test_mcp_credentials.py | 47 +- .../proxy/guardrails/test_mcp_jwt_signer.py | 48 +- .../auth/test_user_api_key_auth_mcp.py | 476 +++--------------- .../mcp_server/test_mcp_server_manager.py | 137 ++++- .../unit/proxy/mcp_server/test_operations.py | 7 +- .../proxy/mcp_server/test_rest_endpoints.py | 16 +- .../mcp/test_aresponses_api_with_mcp.py | 7 +- 12 files changed, 304 insertions(+), 510 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py index 9cb929dec92..9463023854e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py @@ -941,15 +941,13 @@ async def inject_mcp_jwt_headers_for_upstream( if signer is None or user_api_key_dict is None: return merged - normalized_raw: Final = {k.lower(): v for k, v in (raw_headers or {}).items()} - incoming_bearer_token: str | None = None - auth_hdr: Final = normalized_raw.get("authorization", "") - if auth_hdr.lower().startswith("bearer "): - incoming_bearer_token = auth_hdr[len("bearer ") :] + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + + incoming_bearer_token: Final = MCPRequestHandler.guardrail_bearer_from_raw_headers(raw_headers, user_api_key_dict) hook_data: Final = { "mcp_tool_name": "" if for_list_tools else mcp_tool_name, - "incoming_bearer_token": incoming_bearer_token, + "incoming_bearer_token": incoming_bearer_token.get_secret_value() if incoming_bearer_token else None, "extra_headers": merged, } call_type: Final[CallTypesLiteral] = "list_mcp_tools" if for_list_tools else "call_mcp_tool" diff --git a/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py index be24b1630b0..bcb979d720b 100644 --- a/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py @@ -919,7 +919,6 @@ class MCPRequestHandler: mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, admitted_credential=admitted_credential, - preserve_jwt_subject=bool(validated_user_api_key_auth.jwt_claims), ) return ( @@ -959,49 +958,29 @@ class MCPRequestHandler: mcp_server_auth_headers: dict[str, dict[str, str]] | None, *, admitted_credential: str | None, - preserve_jwt_subject: bool = False, ) -> tuple[dict[str, str] | None, dict[str, str], str | None, dict[str, dict[str, str]] | None]: - """Remove gateway-shaped credentials and the caller's admitted credential from EVERY egress - header context: top-level ``Authorization`` (oauth2 + raw), the deprecated ``x-mcp-auth``, and - per-server ``x-mcp-{alias}-authorization``. Preserve the raw ``x-litellm-api-key`` admission - header; an admitted subject's top-level Authorization is otherwise dropped unconditionally.""" + """Remove gateway-shaped credentials from every egress header context, and the caller's admitted + credential from the implicitly forwarded top-level ``Authorization``. Raw headers (only forwarded + when a server lists them in ``extra_headers``), ``x-mcp-auth`` and ``x-mcp-{alias}-authorization`` + keep the caller's LiteLLM key, since the admin or caller chose to send it there.""" + gateway_shaped: Final = MCPRequestHandler._is_gateway_admission_credential - def cred(value: str | None) -> bool: - return MCPRequestHandler._is_gateway_admission_credential( - value - ) or MCPRequestHandler._is_caller_admission_key(value, admitted_credential) - - # 1. Top-level Authorization → oauth2_headers. authz: Final = oauth2_headers.get("Authorization") if oauth2_headers else None - preserve_subject: Final = bool( - preserve_jwt_subject - and not admitted - and authz - and not _normalize_caller_admission_credential(authz).startswith("sk-") - and not MCPRequestHandler._is_gateway_admission_credential(authz) - and MCPRequestHandler._is_caller_admission_key(authz, admitted_credential) - ) - if admitted or cred(authz): + if admitted or gateway_shaped(authz) or MCPRequestHandler._is_caller_admission_key(authz, admitted_credential): oauth2_headers = None - # 2. raw_headers: drop the admitted subject's Authorization, and ANY header whose value is a - # gateway credential (covers x-mcp-auth and x-mcp-{alias}-authorization in their raw form). raw_headers = { k: v for k, v in raw_headers.items() - if k.lower() == "x-litellm-api-key" - or (preserve_subject and k.lower() == "authorization") - or not ((admitted and k.lower() == "authorization") or cred(v)) + if not ((admitted and k.lower() == "authorization") or gateway_shaped(v)) } - # 3. Deprecated x-mcp-auth value. - if cred(mcp_auth_header): + if gateway_shaped(mcp_auth_header): mcp_auth_header = None - # 4. Per-server x-mcp-{alias}-authorization values (drop the value, then any now-empty server dict). if mcp_server_auth_headers: stripped: Final = { - alias: {h: val for h, val in hdrs.items() if not cred(val)} + alias: {h: val for h, val in hdrs.items() if not gateway_shaped(val)} for alias, hdrs in mcp_server_auth_headers.items() } mcp_server_auth_headers = {alias: hdrs for alias, hdrs in stripped.items() if hdrs} @@ -1892,6 +1871,14 @@ class MCPRequestHandler: return None return bearer + @staticmethod + def guardrail_bearer_from_raw_headers( + raw_headers: Mapping[str, str] | None, auth: UserAPIKeyAuth | None + ) -> SecretStr | None: + if auth is None: + return MCPRequestHandler.get_incoming_bearer_token(raw_headers or {}) + return MCPRequestHandler.get_guardrail_bearer_token(Headers(headers=dict(raw_headers or {})), auth) + @staticmethod def get_oauth2_headers_from_headers(headers: Headers) -> dict[str, str]: """ diff --git a/litellm/proxy/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py index 013336e1d5d..cc02fc43ef3 100644 --- a/litellm/proxy/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -1206,8 +1206,10 @@ def should_strip_caller_authorization( admission credential — a virtual key, an IdP JWT, or an SSO / OIDC / session token whose ``api_key`` is ``None`` — and must never reach the upstream, so it is stripped regardless of the ``api_key`` value. - - **Every other auth-enabled mode**: strip when ``Authorization`` is the - credential admission validated. + - **true_passthrough**: strip when ``Authorization`` is the credential + admission validated. + - **Every other mode**: forward. The caller's ``Authorization`` only reaches + the upstream because the admin listed it in ``extra_headers``. """ if ( user_api_key_auth is not None @@ -1233,6 +1235,8 @@ def should_strip_caller_authorization( mcp_server.auth_type == MCPAuth.oauth2 and mcp_server.delegate_auth_to_upstream ) if not (mcp_server.is_oauth_passthrough or is_delegated_oauth): + if not mcp_server.is_client_forwarded_token: + return False from litellm.proxy.auth.user_api_key_auth import is_no_auth_dev_mode from litellm.proxy.proxy_server import ( general_settings, # pyright: ignore[reportUnknownVariableType] # legacy global is declared as a bare dict @@ -1403,7 +1407,7 @@ def _openapi_forwarded_extra_headers( ) forwarded: Final[dict[str, str]] = {} for header_name in mcp_server.extra_headers: - if not isinstance(header_name, str) or header_name.lower() == "x-litellm-api-key": + if not isinstance(header_name, str): continue if skip_caller_authorization and header_name.lower() == "authorization": continue @@ -6241,7 +6245,7 @@ class MCPServerManager: ) caller_bearer: Final = ( - MCPRequestHandler.get_incoming_bearer_token(raw_headers or {}) + MCPRequestHandler.guardrail_bearer_from_raw_headers(raw_headers, user_api_key_auth) if incoming_bearer_token is ... else incoming_bearer_token ) @@ -6640,7 +6644,7 @@ class MCPServerManager: # For M2M OAuth servers, Authorization must come from token fetch. extra_headers = None else: - extra_headers = oauth2_headers + extra_headers = dict(oauth2_headers) if oauth2_headers else None # Migrated authorization_code: the v2 resolver injects the stored per-user # token, so drop the caller-forwarded Authorization (apply-if-absent would # otherwise let it shadow the resolved token). Delegate keeps it. Centralized @@ -6673,7 +6677,7 @@ class MCPServerManager: ) for header in mcp_server.extra_headers: - if not isinstance(header, str) or header.lower() == "x-litellm-api-key": + if not isinstance(header, str): continue if header.lower() == "authorization" and strip_caller_authorization: continue diff --git a/litellm/proxy/mcp_server/operations.py b/litellm/proxy/mcp_server/operations.py index dfc45417814..8422d516d8c 100644 --- a/litellm/proxy/mcp_server/operations.py +++ b/litellm/proxy/mcp_server/operations.py @@ -1080,7 +1080,7 @@ def _prepare_mcp_server_headers( ) for header in server.extra_headers: - if not isinstance(header, str) or header.lower() == "x-litellm-api-key": + if not isinstance(header, str): continue if header.lower() == "authorization" and (strip_caller_authorization or withhold_forwarded_authorization): continue diff --git a/litellm/proxy/mcp_server/rest_endpoints.py b/litellm/proxy/mcp_server/rest_endpoints.py index 2737b567985..815c1d6270c 100644 --- a/litellm/proxy/mcp_server/rest_endpoints.py +++ b/litellm/proxy/mcp_server/rest_endpoints.py @@ -627,7 +627,6 @@ if MCP_AVAILABLE: ) = MCPRequestHandler.scrub_gateway_admission_credentials( admitted=False, admitted_credential=admitted_credential, - preserve_jwt_subject=bool(user_api_key_dict.jwt_claims), oauth2_headers=oauth2_headers_from_request, raw_headers=dict(headers), mcp_auth_header=MCPRequestHandler.get_mcp_auth_header_from_headers(headers), diff --git a/tests/integration/mcp/test_mcp_credentials.py b/tests/integration/mcp/test_mcp_credentials.py index 08dd5234b68..d21e6fea28e 100644 --- a/tests/integration/mcp/test_mcp_credentials.py +++ b/tests/integration/mcp/test_mcp_credentials.py @@ -155,11 +155,11 @@ def test_caller_headers_for_other_servers_and_unknown_headers_never_reach_the_pe @pytest.mark.parametrize( - ("auth_type", "per_server_header"), - (("none", True), ("true_passthrough", False)), + ("auth_type", "per_server_header", "forwarded"), + (("none", True, True), ("true_passthrough", False, False)), ) -def test_callers_own_litellm_key_never_reaches_the_peer_over_rest( - gateway: Gateway, auth_type: str, per_server_header: bool +def test_callers_litellm_key_reaches_the_peer_only_through_a_per_server_header( + gateway: Gateway, auth_type: str, per_server_header: bool, forwarded: bool ) -> None: with mcp_peer() as peer, gateway.scenario() as scenario: alias: Final = "cred" + uuid.uuid4().hex[:8] @@ -178,19 +178,14 @@ def test_callers_own_litellm_key_never_reaches_the_peer_over_rest( observed: Final = peer.drain() calls: Final = tool_calls(observed) assert len(calls) == 1, calls - header_sets: Final = tuple( - TypeAdapter(dict[bytes, bytes]).validate_python(request["headers"]) for request in observed - ) - assert all(all(key.encode() not in value for value in headers.values()) for headers in header_sets), ( - "caller key appeared in the recorded MCP header set" - ) + assert (_header(calls[0], b"authorization") == f"Bearer {key}".encode()) is forwarded @pytest.mark.parametrize("entry", ("server_mcp", "rest")) def test_empty_primary_header_does_not_expose_authorization_key(gateway: Gateway, entry: EntryPoint) -> None: with mcp_peer() as peer, gateway.scenario() as scenario: alias: Final = "cred" + uuid.uuid4().hex[:8] - identity: Final = register_mcp(scenario, peer, alias, extra_headers=["x-upstream-token", "x-tenant"]) + identity: Final = register_mcp(scenario, peer, alias, extra_headers=["x-tenant"]) key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) caller: Final = McpCaller( gateway, @@ -200,8 +195,6 @@ def test_empty_primary_header_does_not_expose_authorization_key(gateway: Gateway headers={ "x-litellm-api-key": "", "Authorization": f"Bearer {key}", - f"x-mcp-{alias}-authorization": f"Bearer {key}", - "x-upstream-token": key, "x-tenant": "tenant-control", }, ) @@ -216,40 +209,40 @@ def test_empty_primary_header_does_not_expose_authorization_key(gateway: Gateway TypeAdapter(dict[bytes, bytes]).validate_python(request["headers"]) for request in observed ) assert all(all(key.encode() not in value for value in headers.values()) for headers in header_sets), ( - "empty primary header prevented scrubbing the admitted Authorization key" + "admission key reached the peer through a header the server does not list" ) @pytest.mark.parametrize("entry", ENTRY_POINTS) -def test_extra_headers_cannot_forward_gateway_admission_key(gateway: Gateway, entry: EntryPoint) -> None: +def test_extra_headers_forward_listed_admission_key(gateway: Gateway, entry: EntryPoint) -> None: with mcp_peer() as peer, gateway.scenario() as scenario: alias: Final = "cred" + uuid.uuid4().hex[:8] identity: Final = register_mcp( - scenario, peer, alias, extra_headers=["X-LiteLLM-API-Key", "x-tenant"], + scenario, + peer, + alias, + extra_headers=["X-LiteLLM-API-Key", "x-tenant"], ) key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) caller: Final = McpCaller(gateway, key, entry, alias, headers={"x-tenant": "tenant-control"}) peer.drain() outcome: Final = caller.call(f"{alias}-add", ADD, identity if entry != "server_mcp" else None) assert outcome.ok, outcome.raw - observed: Final = peer.drain() - calls: Final = tool_calls(observed) + calls: Final = tool_calls(peer.drain()) assert len(calls) == 1, calls assert _header(calls[0], b"x-tenant") == b"tenant-control" - header_sets: Final = tuple( - TypeAdapter(dict[bytes, bytes]).validate_python(request["headers"]) for request in observed - ) - assert all(all(key.encode() not in value for value in headers.values()) for headers in header_sets), ( - "gateway admission key reached the upstream through Extra Headers" - ) + assert key.encode() in (_header(calls[0], b"x-litellm-api-key") or b"") @pytest.mark.parametrize("entry", ("server_mcp", "rest")) -def test_openapi_extra_headers_cannot_forward_gateway_admission_key(gateway: Gateway, entry: EntryPoint) -> None: +def test_openapi_extra_headers_forward_listed_admission_key(gateway: Gateway, entry: EntryPoint) -> None: with openapi_peer() as peer, gateway.scenario() as scenario: alias: Final = "cred" + uuid.uuid4().hex[:8] identity: Final = register_mcp( - scenario, peer, alias, extra_headers=["X-LiteLLM-API-Key", "x-tenant"], + scenario, + peer, + alias, + extra_headers=["X-LiteLLM-API-Key", "x-tenant"], ) key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) caller: Final = McpCaller(gateway, key, entry, alias, headers={"x-tenant": "tenant-control"}) @@ -260,7 +253,7 @@ def test_openapi_extra_headers_cannot_forward_gateway_admission_key(gateway: Gat assert [(request["method"], request["path"]) for request in observed] == [("GET", "/pets/7")] headers: Final = TypeAdapter(dict[bytes, bytes]).validate_python(observed[0]["headers"]) assert headers.get(b"x-tenant") == b"tenant-control" - assert all(key.encode() not in value for value in headers.values()), "gateway admission key reached OpenAPI" + assert key.encode() in headers.get(b"x-litellm-api-key", b"") def test_server_scoped_caller_header_reaches_only_its_server(gateway: Gateway) -> None: diff --git a/tests/unit/proxy/guardrails/test_mcp_jwt_signer.py b/tests/unit/proxy/guardrails/test_mcp_jwt_signer.py index a7b24169398..8b7a420c739 100644 --- a/tests/unit/proxy/guardrails/test_mcp_jwt_signer.py +++ b/tests/unit/proxy/guardrails/test_mcp_jwt_signer.py @@ -11,7 +11,7 @@ Tests cover: import base64 import time -from typing import Any, Dict, Optional +from typing import Any, Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch import jwt @@ -1292,3 +1292,49 @@ async def test_inject_mcp_jwt_signs_for_tool_call_path(): scopes = set(decoded["scope"].split()) assert "mcp:tools/call" in scopes assert "mcp:tools/search_web:call" in scopes + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("raw_headers", "introspected"), + [ + ({"Authorization": "Bearer sk-admission-key"}, False), + ({"x-litellm-api-key": "sk-admission-key", "Authorization": "Bearer upstream-opaque"}, True), + ], +) +async def test_inject_mcp_jwt_never_introspects_the_admission_key( + raw_headers: dict[str, str], introspected: bool +) -> None: + import httpx + from fastapi import HTTPException + + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy._types import UserAPIKeyAuth, hash_token + from litellm.proxy.guardrails.guardrail_hooks.mcp_jwt_signer.mcp_jwt_signer import ( + inject_mcp_jwt_headers_for_upstream, + ) + + _make_signer( + access_token_discovery_uri="https://idp.example.com/.well-known/openid-configuration", + token_introspection_endpoint="https://idp.example.com/introspect", + ) + inactive_introspection: Final = AsyncHTTPHandler( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"active": False})) + ) + + async def sign() -> dict[str, str]: + with patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + return_value=inactive_introspection, + ): + return await inject_mcp_jwt_headers_for_upstream( + user_api_key_dict=UserAPIKeyAuth(api_key=hash_token("sk-admission-key"), user_id="alice"), + raw_headers=raw_headers, + for_list_tools=True, + ) + + if introspected: + with pytest.raises(HTTPException): + await sign() + else: + assert (await sign())["Authorization"].startswith("Bearer ") diff --git a/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py index 7e74e62894f..ea54d456307 100644 --- a/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -10141,444 +10141,100 @@ class TestSessionBearerEgressScrub: assert "authorization" not in {k.lower() for k in raw} assert per_server == {"github": {"Authorization": "Bearer gh_injected_upstream"}} - @pytest.mark.parametrize( - "context", - ["per_server", "mcp_auth", "oauth2_authorization", "raw_authorization"], - ) @pytest.mark.parametrize( "caller_value", - ["sk-caller-admission-key-123", "Bearer sk-caller-admission-key-123"], + [ + "sk-caller-admission-key-123", + "Bearer sk-caller-admission-key-123", + "BEARER sk-caller-admission-key-123", + "Bearer sk-caller-admission-key-123", + "Basic sk-caller-admission-key-123", + ], ) - async def test_scrub_removes_caller_admission_key_from_each_egress_context( - self, - context: Literal["per_server", "mcp_auth", "oauth2_authorization", "raw_authorization"], - caller_value: str, - ) -> None: + async def test_scrub_drops_caller_key_only_from_implicit_authorization(self, caller_value: str) -> None: caller_key: Final = "sk-caller-admission-key-123" - upstream_token: Final = "Bearer real-upstream-token" - oauth2_headers: Final = {"Authorization": caller_value if context == "oauth2_authorization" else upstream_token} raw_headers: Final = { "x-litellm-api-key": f"Bearer {caller_key}", - "authorization": caller_value if context == "raw_authorization" else upstream_token, - "x-upstream-token": upstream_token, - } - mcp_auth_header: Final = caller_value if context == "mcp_auth" else upstream_token - mcp_server_auth_headers: Final = { - "echo_srv": { - "Authorization": caller_value if context == "per_server" else upstream_token, - } + "authorization": caller_value, + "x-tenant": caller_value, } oauth2, raw, mcp_auth, per_server = MCPRequestHandler.scrub_gateway_admission_credentials( admitted=False, - oauth2_headers=oauth2_headers, + oauth2_headers={"Authorization": caller_value}, raw_headers=raw_headers, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=caller_value, + mcp_server_auth_headers={"echo_srv": {"Authorization": caller_value}}, admitted_credential=caller_key, ) - assert raw["x-litellm-api-key"] == f"Bearer {caller_key}" - assert raw["x-upstream-token"] == upstream_token - if context == "raw_authorization": - assert "authorization" not in raw - else: - assert raw["authorization"] == upstream_token - if context == "oauth2_authorization": - assert oauth2 is None - else: - assert oauth2 == {"Authorization": upstream_token} - if context == "mcp_auth": - assert mcp_auth is None - else: - assert mcp_auth == upstream_token - if context == "per_server": - assert not per_server - else: - assert per_server == {"echo_srv": {"Authorization": upstream_token}} + assert oauth2 is None + assert raw == raw_headers + assert mcp_auth == caller_value + assert per_server == {"echo_srv": {"Authorization": caller_value}} - async def test_scrub_keeps_non_ascii_per_server_token(self) -> None: - upstream_token: Final = "Bearer t\u00f6ken" - _oauth2, _raw, _mcp_auth, per_server = MCPRequestHandler.scrub_gateway_admission_credentials( + async def test_scrub_keeps_distinct_upstream_authorization(self) -> None: + upstream_token: Final = "Bearer real-upstream-token" + oauth2, _raw, _mcp_auth, _per_server = MCPRequestHandler.scrub_gateway_admission_credentials( admitted=False, - admitted_credential="sk-caller-admission-key-123", - oauth2_headers=None, + oauth2_headers={"Authorization": upstream_token}, raw_headers={}, mcp_auth_header=None, - mcp_server_auth_headers={"echo_srv": {"Authorization": upstream_token}}, + mcp_server_auth_headers=None, + admitted_credential="sk-caller-admission-key-123", ) - assert per_server == {"echo_srv": {"Authorization": upstream_token}} - - async def test_scrub_keeps_caller_key_when_admission_credential_is_missing(self) -> None: - caller_key: Final = "Bearer sk-caller-admission-key-123" - oauth2, raw, mcp_auth, per_server = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers={"Authorization": caller_key}, - raw_headers={"x-litellm-api-key": caller_key, "authorization": caller_key}, - mcp_auth_header=caller_key, - mcp_server_auth_headers={"echo_srv": {"Authorization": caller_key}}, - admitted_credential=None, - ) - - assert oauth2 == {"Authorization": caller_key} - assert raw == {"x-litellm-api-key": caller_key, "authorization": caller_key} - assert mcp_auth == caller_key - assert per_server == {"echo_srv": {"Authorization": caller_key}} + assert oauth2 == {"Authorization": upstream_token} @pytest.mark.parametrize( - "admission_value,per_server_value", - ( - ("Bearer sk-caller-admission-key-123", "BEARER sk-caller-admission-key-123"), - ("Bearer sk-caller-admission-key-123", "Bearer sk-caller-admission-key-123"), - ("Bearer sk-caller-admission-key-123", "Basic sk-caller-admission-key-123"), - ("Basic sk-caller-admission-key-123", "sk-caller-admission-key-123"), - ("Bearer sk-caller-admission-key-123", "bearer sk-caller-admission-key-123"), - ), + ("headers", "custom_key_header_name", "expected"), + [ + ( + { + "x-litellm-api-key": "Bearer sk-caller", + "authorization": "Bearer upstream", + SpecialHeaders.azure_authorization.value: "provider", + }, + None, + "sk-caller", + ), + ( + {"authorization": "Bearer sk-caller", SpecialHeaders.azure_authorization.value: "provider"}, + None, + "sk-caller", + ), + ( + {"x-custom-api-key": "Bearer sk-caller", "x-litellm-api-key": "Bearer standard"}, + "x-custom-api-key", + "sk-caller", + ), + ({"x-litellm-api-key": "", "authorization": "sk-caller"}, None, "sk-caller"), + ({"authorization": "", "api-key": "sk-caller"}, None, "sk-caller"), + ( + { + SpecialHeaders.azure_authorization.value: "azure-key", + SpecialHeaders.anthropic_authorization.value: "anthropic-key", + SpecialHeaders.google_ai_studio_authorization.value: "google-key", + SpecialHeaders.azure_apim_authorization.value: "apim-key", + }, + None, + "azure-key", + ), + ({SpecialHeaders.anthropic_authorization.value: "anthropic-key"}, None, "anthropic-key"), + ({SpecialHeaders.google_ai_studio_authorization.value: "google-key"}, None, "google-key"), + ({SpecialHeaders.azure_apim_authorization.value: "apim-key"}, None, "apim-key"), + ], ) - async def test_caller_admission_credential_scrubs_bearer_variants_and_basic( - self, - admission_value: str, - per_server_value: str, + async def test_caller_admission_credential_matches_admission_header_precedence( + self, headers: dict[str, str], custom_key_header_name: str | None, expected: str ) -> None: - upstream_authorization: Final = "Bearer unrelated-upstream-token" - headers: Final = Headers( - { - "x-litellm-api-key": admission_value, - "authorization": upstream_authorization, - "x-mcp-auth": per_server_value, - "x-mcp-echo_srv-authorization": per_server_value, - "x-upstream-token": upstream_authorization, - } - ) - user_api_key_auth: Final = UserAPIKeyAuth(api_key="stored-key-hash") admitted_credential: Final = MCPRequestHandler.caller_admission_credential( - headers, - user_api_key_auth, - custom_key_header_name=None, - ) - result: Final = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers={"Authorization": upstream_authorization}, - raw_headers=dict(headers), - mcp_auth_header=per_server_value, - mcp_server_auth_headers={ - "echo_srv": {"Authorization": per_server_value}, - "other_srv": {"Authorization": upstream_authorization}, - }, - admitted_credential=admitted_credential, - ) - - assert result == ( - {"Authorization": upstream_authorization}, - { - "x-litellm-api-key": admission_value, - "authorization": upstream_authorization, - "x-upstream-token": upstream_authorization, - }, - None, - {"other_srv": {"Authorization": upstream_authorization}}, - ) - - @pytest.mark.parametrize( - ("empty_header", "admission_header"), - (("x-litellm-api-key", "authorization"), ("authorization", "api-key")), - ) - async def test_empty_admission_header_falls_back_to_authenticated_key( - self, empty_header: str, admission_header: str - ) -> None: - caller_key: Final = "sk-caller-admission-key-123" - headers: Final = Headers({empty_header: "", admission_header: caller_key}) - credential: Final = MCPRequestHandler.caller_admission_credential( - headers, UserAPIKeyAuth(api_key="stored-key-hash"), custom_key_header_name=None - ) - assert credential == caller_key - result: Final = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers={"Authorization": f"Bearer {caller_key}"}, - raw_headers={**dict(headers), "x-upstream-token": caller_key, "x-tenant": "tenant-control"}, - mcp_auth_header=caller_key, - mcp_server_auth_headers={"echo": {"Authorization": caller_key}}, - admitted_credential=credential, - ) - assert result == (None, {empty_header: "", "x-tenant": "tenant-control"}, None, {}) - - @pytest.mark.parametrize("claims", [None, {"sub": "caller"}]) - @pytest.mark.parametrize("bearer", ["validated-jwt", "sk-admission-key"]) - async def test_scrubbing_preserves_only_validated_jwt_admission( - self, claims: dict[str, str] | None, bearer: str - ) -> None: - headers: Final = Headers({"Authorization": f"Bearer {bearer}"}) - auth: Final = UserAPIKeyAuth(api_key="stored-key-hash", jwt_claims=claims) - credential: Final = MCPRequestHandler.caller_admission_credential(headers, auth, custom_key_header_name=None) - result: Final = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers={"Authorization": f"Bearer {bearer}"}, - raw_headers={**dict(headers), "x-mcp-echo-authorization": f"Bearer {bearer}"}, - mcp_auth_header=bearer, - mcp_server_auth_headers={"echo": {"Authorization": f"Bearer {bearer}"}}, - admitted_credential=credential, - preserve_jwt_subject=bool(claims), - ) - expected: Final = {"Authorization": f"Bearer {bearer}"} if claims and not bearer.startswith("sk-") else None - assert result == (None, dict(headers) if expected else {}, None, {}) - - async def test_caller_admission_credential_prefers_x_litellm_header(self) -> None: - caller_key: Final = "sk-caller-admission-key-123" - upstream_authorization: Final = "Bearer unrelated-upstream-token" - provider_key: Final = "provider-key-not-admitted" - headers: Final = Headers( - { - "x-litellm-api-key": f"Bearer {caller_key}", - "authorization": upstream_authorization, - SpecialHeaders.azure_authorization.value: provider_key, - "x-mcp-auth": f"Bearer {caller_key}", - "x-mcp-echo_srv-authorization": f"Bearer {caller_key}", - } - ) - admitted_credential: Final = MCPRequestHandler.caller_admission_credential( - headers, + Headers(headers), UserAPIKeyAuth(api_key="stored-key-hash"), - custom_key_header_name=None, - ) - result: Final = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers={"Authorization": upstream_authorization}, - raw_headers=dict(headers), - mcp_auth_header=f"Bearer {caller_key}", - mcp_server_auth_headers={ - "echo_srv": {"Authorization": f"Bearer {caller_key}"}, - "other_srv": {"Authorization": upstream_authorization}, - }, - admitted_credential=admitted_credential, + custom_key_header_name=custom_key_header_name, ) - assert result == ( - {"Authorization": upstream_authorization}, - { - "x-litellm-api-key": f"Bearer {caller_key}", - "authorization": upstream_authorization, - "api-key": provider_key, - }, - None, - {"other_srv": {"Authorization": upstream_authorization}}, - ) - - async def test_caller_admission_credential_prefers_authorization_before_provider_headers(self) -> None: - caller_key: Final = "sk-caller-admission-key-123" - provider_key: Final = "provider-key-not-admitted" - headers: Final = Headers( - { - "authorization": f"Bearer {caller_key}", - SpecialHeaders.azure_authorization.value: provider_key, - "x-mcp-auth": f"Bearer {caller_key}", - "x-mcp-echo_srv-authorization": f"Bearer {caller_key}", - } - ) - admitted_credential: Final = MCPRequestHandler.caller_admission_credential( - headers, - UserAPIKeyAuth(api_key="stored-key-hash"), - custom_key_header_name=None, - ) - result: Final = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers={"Authorization": f"Bearer {caller_key}"}, - raw_headers=dict(headers), - mcp_auth_header=f"Bearer {caller_key}", - mcp_server_auth_headers={"echo_srv": {"Authorization": f"Bearer {caller_key}"}}, - admitted_credential=admitted_credential, - ) - - assert result == ( - None, - {"api-key": provider_key}, - None, - {}, - ) - - async def test_caller_admission_credential_uses_configured_custom_header(self) -> None: - caller_key: Final = "sk-caller-admission-key-123" - standard_key: Final = "Bearer standard-key-not-admitted" - upstream_authorization: Final = "Bearer unrelated-upstream-token" - headers: Final = Headers( - { - "x-custom-api-key": f"Bearer {caller_key}", - "x-litellm-api-key": standard_key, - "authorization": upstream_authorization, - "x-mcp-auth": f"Bearer {caller_key}", - "x-mcp-echo_srv-authorization": f"Bearer {caller_key}", - } - ) - admitted_credential: Final = MCPRequestHandler.caller_admission_credential( - headers, - UserAPIKeyAuth(api_key="stored-key-hash"), - custom_key_header_name="x-custom-api-key", - ) - result: Final = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers={"Authorization": upstream_authorization}, - raw_headers=dict(headers), - mcp_auth_header=f"Bearer {caller_key}", - mcp_server_auth_headers={"echo_srv": {"Authorization": f"Bearer {caller_key}"}}, - admitted_credential=admitted_credential, - ) - - assert result == ( - {"Authorization": upstream_authorization}, - { - "x-litellm-api-key": standard_key, - "authorization": upstream_authorization, - }, - None, - {}, - ) - - async def test_caller_admission_credential_falls_back_to_standard_headers_when_custom_header_is_absent( - self, - ) -> None: - caller_value: Final = "Bearer sk-caller-admission-key-123" - headers: Final = Headers( - { - "x-litellm-api-key": caller_value, - "authorization": caller_value, - "x-mcp-auth": caller_value, - "x-mcp-echo_srv-authorization": caller_value, - } - ) - admitted_credential: Final = MCPRequestHandler.caller_admission_credential( - headers, - UserAPIKeyAuth(api_key="stored-key-hash"), - custom_key_header_name="x-custom-api-key", - ) - result: Final = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers={"Authorization": caller_value}, - raw_headers=dict(headers), - mcp_auth_header=caller_value, - mcp_server_auth_headers={"echo_srv": {"Authorization": caller_value}}, - admitted_credential=admitted_credential, - ) - - assert admitted_credential == "sk-caller-admission-key-123" - assert result == ( - None, - {"x-litellm-api-key": caller_value}, - None, - {}, - ) - - async def test_caller_admission_credential_uses_master_key_alias_as_admission_gate(self) -> None: - master_key: Final = "sk-" + "1234" - headers: Final = Headers( - { - "x-litellm-api-key": f"Bearer {master_key}", - "authorization": f"Bearer {master_key}", - "x-mcp-auth": f"Bearer {master_key}", - "x-mcp-echo_srv-authorization": f"Bearer {master_key}", - } - ) - admitted_credential: Final = MCPRequestHandler.caller_admission_credential( - headers, - UserAPIKeyAuth(api_key="litellm_proxy_master_key"), - custom_key_header_name=None, - ) - result: Final = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers={"Authorization": f"Bearer {master_key}"}, - raw_headers=dict(headers), - mcp_auth_header=f"Bearer {master_key}", - mcp_server_auth_headers={"echo_srv": {"Authorization": f"Bearer {master_key}"}}, - admitted_credential=admitted_credential, - ) - - assert result == ( - None, - {"x-litellm-api-key": f"Bearer {master_key}"}, - None, - {}, - ) - - @pytest.mark.parametrize( - "provider_header_name", - ( - SpecialHeaders.azure_authorization.value, - SpecialHeaders.anthropic_authorization.value, - SpecialHeaders.google_ai_studio_authorization.value, - SpecialHeaders.azure_apim_authorization.value, - ), - ) - async def test_caller_admission_credential_scrubs_provider_header( - self, - provider_header_name: str, - ) -> None: - provider_key: Final = "provider-admission-key" - headers: Final = Headers( - { - provider_header_name: provider_key, - "x-mcp-auth": f"Bearer {provider_key}", - "x-mcp-echo_srv-authorization": f"Bearer {provider_key}", - "x-upstream-token": "Bearer unrelated-upstream-token", - } - ) - admitted_credential: Final = MCPRequestHandler.caller_admission_credential( - headers, - UserAPIKeyAuth(api_key="stored-key-hash"), - custom_key_header_name=None, - ) - result: Final = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers=None, - raw_headers=dict(headers), - mcp_auth_header=f"Bearer {provider_key}", - mcp_server_auth_headers={ - "echo_srv": {"Authorization": f"Bearer {provider_key}"}, - "other_srv": {"Authorization": "Bearer unrelated-upstream-token"}, - }, - admitted_credential=admitted_credential, - ) - - assert result == ( - None, - {"x-upstream-token": "Bearer unrelated-upstream-token"}, - None, - {"other_srv": {"Authorization": "Bearer unrelated-upstream-token"}}, - ) - - async def test_caller_admission_credential_follows_provider_header_precedence(self) -> None: - azure_key: Final = "azure-admission-key" - headers: Final = Headers( - { - SpecialHeaders.azure_authorization.value: azure_key, - SpecialHeaders.anthropic_authorization.value: "anthropic-not-admitted", - SpecialHeaders.google_ai_studio_authorization.value: "google-not-admitted", - SpecialHeaders.azure_apim_authorization.value: "apim-not-admitted", - "x-mcp-echo_srv-authorization": f"Bearer {azure_key}", - } - ) - admitted_credential: Final = MCPRequestHandler.caller_admission_credential( - headers, - UserAPIKeyAuth(api_key="stored-key-hash"), - custom_key_header_name=None, - ) - result: Final = MCPRequestHandler.scrub_gateway_admission_credentials( - admitted=False, - oauth2_headers=None, - raw_headers=dict(headers), - mcp_auth_header=f"Bearer {azure_key}", - mcp_server_auth_headers={"echo_srv": {"Authorization": f"Bearer {azure_key}"}}, - admitted_credential=admitted_credential, - ) - - assert result == ( - None, - { - "x-api-key": "anthropic-not-admitted", - "x-goog-api-key": "google-not-admitted", - "ocp-apim-subscription-key": "apim-not-admitted", - }, - None, - {}, - ) + assert admitted_credential == expected async def test_caller_admission_credential_returns_none_without_admitted_api_key(self) -> None: caller_value: Final = "Bearer sk-caller-admission-key-123" diff --git a/tests/unit/proxy/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/mcp_server/test_mcp_server_manager.py index 13e4704bd90..f7cbf7be8f7 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_manager.py @@ -321,7 +321,7 @@ def test_validated_admission_jwt_is_not_forwarded_as_an_extra_header(auth_type: "auth_type", [MCPAuth.none, MCPAuth.api_key, MCPAuth.bearer_token, MCPAuth.basic, MCPAuth.true_passthrough], ) -def test_admission_authorization_is_not_forwarded_for_any_auth_type( +def test_listed_admission_authorization_forwards_except_for_true_passthrough( auth_type: MCPAuth, monkeypatch: pytest.MonkeyPatch ) -> None: from litellm.proxy import proxy_server @@ -341,7 +341,11 @@ def test_admission_authorization_is_not_forwarded_for_any_auth_type( "Authorization": "Bearer sk-" + "1234", "x-tenant": "tenant-control", } - expected_without_admission_key: Final = {"x-tenant": "tenant-control"} + expected_without_admission_key: Final = ( + {"x-tenant": "tenant-control"} + if auth_type == MCPAuth.true_passthrough + else {"x-tenant": "tenant-control", "Authorization": "Bearer sk-" + "1234"} + ) admission_upstream_headers: Final = _prepare_mcp_server_headers( server, mcp_auth_header=None, @@ -414,12 +418,13 @@ def test_no_master_key_preserves_caller_authorization_unless_auth_is_enabled( raw_headers=raw_headers, user_api_key_auth=user, )[1] - assert auth_enabled_headers == {"x-tenant": "t"} - assert _openapi_forwarded_extra_headers(server, raw_headers, user) == {"x-tenant": "t"} + expected_auth_enabled: Final = {"x-tenant": "t"} if auth_type == MCPAuth.true_passthrough else expected_forwarded + assert auth_enabled_headers == expected_auth_enabled + assert _openapi_forwarded_extra_headers(server, raw_headers, user) == expected_auth_enabled @pytest.mark.parametrize("auth_type", [MCPAuth.none, MCPAuth.bearer_token, MCPAuth.true_passthrough]) -def test_custom_auth_is_stripped_in_no_auth_dev_mode( +def test_custom_auth_is_stripped_in_no_auth_dev_mode_for_true_passthrough( auth_type: MCPAuth, monkeypatch: pytest.MonkeyPatch ) -> None: from litellm.proxy import proxy_server @@ -438,7 +443,11 @@ def test_custom_auth_is_stripped_in_no_auth_dev_mode( raw_headers: Final = {"Authorization": "Bearer custom-tok", "x-tenant": "t"} user: Final = UserAPIKeyAuth(api_key="custom-tok") user.authenticated_by_custom_auth = True - expected_headers: Final = {"x-tenant": "t"} + expected_headers: Final = ( + {"x-tenant": "t"} + if auth_type == MCPAuth.true_passthrough + else {"x-tenant": "t", "Authorization": "Bearer custom-tok"} + ) forwarded_headers: Final = _prepare_mcp_server_headers( server, mcp_auth_header=None, @@ -502,6 +511,11 @@ def test_custom_admission_header_controls_authorization_forwarding( extra_headers=["Authorization", "x-tenant"], ) user: Final = UserAPIKeyAuth(api_key="hashed-key") + expected_headers: Final = ( + expected + if auth_type == MCPAuth.true_passthrough + else {"x-tenant": "t", "Authorization": raw_headers["Authorization"]} + ) upstream_headers: Final = _prepare_mcp_server_headers( server, @@ -511,8 +525,8 @@ def test_custom_admission_header_controls_authorization_forwarding( raw_headers=raw_headers, user_api_key_auth=user, )[1] - assert upstream_headers == expected - assert _openapi_forwarded_extra_headers(server, raw_headers, user) == expected + assert upstream_headers == expected_headers + assert _openapi_forwarded_extra_headers(server, raw_headers, user) == expected_headers class TestMCPServerManager: @@ -3856,7 +3870,7 @@ class TestMCPServerManager: assert captured_extra_headers == {"x-request-id": "req-123"} @pytest.mark.asyncio - async def test_call_regular_mcp_tool_never_forwards_admission_key_for_none_auth( + async def test_call_regular_mcp_tool_forwards_listed_admission_key_for_none_auth( self, monkeypatch: pytest.MonkeyPatch ) -> None: from litellm.proxy import proxy_server @@ -3907,7 +3921,76 @@ class TestMCPServerManager: assert capture_extra_headers.await_args is not None captured_extra_headers: Final = capture_extra_headers.await_args.args[0] - assert captured_extra_headers == {"x-tenant": "tenant-control"} + assert captured_extra_headers == {"Authorization": "Bearer sk-" + "1234", "x-tenant": "tenant-control"} + + @pytest.mark.asyncio + async def test_call_regular_mcp_tool_listed_key_does_not_reach_a_later_server( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from mcp.types import JSONRPCRequest + + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "master_key", "test-master-key") + tool_call_headers: Final[dict[str, httpx2.Headers]] = {} + + async def respond(request: httpx2.Request) -> httpx2.Response: + if request.method == "DELETE": + return httpx2.Response(200) + payload: Final = _JSONRPC_ADAPTER.validate_json(request.content) + if not isinstance(payload, JSONRPCRequest): + return httpx2.Response(202) + if payload.method == "tools/call": + tool_call_headers[request.url.host] = request.headers + result: Final[dict[str, object]] = {"content": [], "isError": False} + else: + result = { + "protocolVersion": "2025-03-26", + "serverInfo": {"name": "upstream", "version": "1"}, + "capabilities": {"tools": {}}, + } + return httpx2.Response(200, json={"jsonrpc": "2.0", "id": payload.id, "result": result}) + + manager: Final = MCPServerManager(upstream_transport=httpx2.MockTransport(respond)) + listing_server: Final = MCPServer( + server_id="lists-key", + name="lists-key", + url="https://lists-key.example/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=True, + extra_headers=["X-LiteLLM-API-Key"], + ) + later_server: Final = listing_server.model_copy( + update={ + "server_id": "no-key", + "name": "no-key", + "url": "https://no-key.example/mcp", + "extra_headers": None, + } + ) + oauth2_headers: Final = {"Authorization": "Bearer upstream-token"} + raw_headers: Final = { + "x-litellm-api-key": "sk-" + "1234", + "Authorization": "Bearer upstream-token", + } + + for server in (listing_server, later_server): + await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="tool", + arguments={}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + proxy_logging_obj=None, + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key"), + ) + + assert tool_call_headers["lists-key.example"].get("x-litellm-api-key") == "sk-" + "1234" + assert "x-litellm-api-key" not in tool_call_headers["no-key.example"] @pytest.mark.asyncio async def test_call_regular_mcp_tool_v2_authz_code_drops_caller_authorization( @@ -21576,7 +21659,7 @@ async def test_repeated_stale_discovery_uses_current_callers_endpoint(endpoint: @pytest.mark.parametrize("header_name", ("x-litellm-api-key", "X-LiteLLM-API-Key")) -def test_openapi_extra_headers_exclude_gateway_admission_key(header_name: str) -> None: +def test_openapi_extra_headers_forward_listed_admission_key(header_name: str) -> None: caller_key: Final = "Bearer sk-admission-only" raw_headers: Final = {"x-litellm-api-key": caller_key, "x-tenant": "tenant-control"} server: Final = MCPServer( @@ -21593,8 +21676,7 @@ def test_openapi_extra_headers_exclude_gateway_admission_key(header_name: str) - None, raw_headers, UserAPIKeyAuth(api_key="sk-admission-only"), - ) == (None, {"X-Tenant": "tenant-control"}, None) - assert raw_headers == {"x-litellm-api-key": caller_key, "x-tenant": "tenant-control"} + ) == (None, {header_name: caller_key, "X-Tenant": "tenant-control"}, None) @pytest.mark.asyncio @@ -21900,3 +21982,32 @@ def test_id_jag_listed_tool_metadata_is_isolated_between_identity_assertions() - manager.record_listed_tools(server, (first,), callers[0]) manager.record_listed_tools(server, (second,), callers[1]) assert tuple(manager.get_listed_tool(server, "list_notes", caller) for caller in callers) == (first, second, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("raw_headers", "expected_bearer"), + [ + ({"authorization": "Bearer sk-admission-key"}, None), + ({"x-litellm-api-key": "sk-admission-key", "authorization": "Bearer upstream-token"}, "upstream-token"), + ], +) +async def test_pre_call_guardrail_bearer_never_carries_the_admission_key( + raw_headers: dict[str, str], expected_bearer: str | None +) -> None: + from litellm.proxy._types import hash_token + + proxy_logging_obj: Final = _permissive_proxy_logging() + + await MCPServerManager().pre_call_tool_check( + name="tool", + arguments={}, + server_name="guarded", + user_api_key_auth=UserAPIKeyAuth(api_key=hash_token("sk-admission-key")), + proxy_logging_obj=proxy_logging_obj, + server=MCPServer(server_id="guarded", name="guarded", transport=MCPTransport.stdio), + raw_headers=raw_headers, + ) + + hook_kwargs: Final = proxy_logging_obj.create_mcp_request_object_from_kwargs.call_args.args[0] + assert hook_kwargs["incoming_bearer_token"] == expected_bearer diff --git a/tests/unit/proxy/mcp_server/test_operations.py b/tests/unit/proxy/mcp_server/test_operations.py index c482f588521..df2502c6be7 100644 --- a/tests/unit/proxy/mcp_server/test_operations.py +++ b/tests/unit/proxy/mcp_server/test_operations.py @@ -140,7 +140,7 @@ async def test_mapped_caller_bearer_reaches_agent_evaluation( mcp_auth_header=None, mcp_server_auth_headers=None, ) - assert "authorization" not in raw + assert oauth is None provider_requests: Final[asyncio.Queue[str]] = asyncio.Queue() def provider(exchange: httpx.Request) -> httpx.Response: @@ -1917,7 +1917,7 @@ async def test_initial_tool_listing_preserves_legacy_error_fallback( @pytest.mark.parametrize("header_name", ("x-litellm-api-key", "X-LiteLLM-API-Key")) -def test_discovery_extra_headers_exclude_gateway_admission_key(header_name: str) -> None: +def test_discovery_extra_headers_forward_listed_admission_key(header_name: str) -> None: caller_key: Final = "Bearer sk-admission-only" raw_headers: Final = {"x-litellm-api-key": caller_key, "x-tenant": "tenant-control"} server: Final = MCPServer( @@ -1937,8 +1937,7 @@ def test_discovery_extra_headers_exclude_gateway_admission_key(header_name: str) UserAPIKeyAuth(api_key="sk-admission-only"), ) assert auth_header is None - assert extra_headers == {"X-Tenant": "tenant-control"} - assert raw_headers == {"x-litellm-api-key": caller_key, "x-tenant": "tenant-control"} + assert extra_headers == {header_name: caller_key, "X-Tenant": "tenant-control"} @pytest.mark.asyncio diff --git a/tests/unit/proxy/mcp_server/test_rest_endpoints.py b/tests/unit/proxy/mcp_server/test_rest_endpoints.py index fdb1ff3cd09..b7eb68cae03 100644 --- a/tests/unit/proxy/mcp_server/test_rest_endpoints.py +++ b/tests/unit/proxy/mcp_server/test_rest_endpoints.py @@ -2764,7 +2764,7 @@ class TestCallToolRestAPI: assert resolved_server is target_server assert billed.team_id == "team-b" - async def test_extract_mcp_headers_scrubs_admitted_caller_credential(self) -> None: + async def test_extract_mcp_headers_scrubs_caller_key_only_from_implicit_authorization(self) -> None: caller_key: Final = "sk-rest-caller-admission-key-123" upstream_token: Final = "Bearer unrelated-upstream-token" request: Final = _build_request( @@ -2783,9 +2783,14 @@ class TestCallToolRestAPI: assert extracted_headers == ( None, - {"upstream": {"Authorization": upstream_token}}, + { + "echo_srv": {"Authorization": f"Bearer {caller_key}"}, + "upstream": {"Authorization": upstream_token}, + }, { "x-litellm-api-key": f"Bearer {caller_key}", + "authorization": f"Bearer {caller_key}", + "x-mcp-echo_srv-authorization": f"Bearer {caller_key}", "x-mcp-upstream-authorization": upstream_token, }, None, @@ -5097,12 +5102,5 @@ async def test_rest_guardrail_bearer_excludes_admission_key( assert result.is_error is False assert observed.qsize() == 1 assert observed.get_nowait() == expected - if claims is None: - list_headers: Final = rest_endpoints._extract_mcp_headers_from_request( - _build_request(headers), - auth, - )[2] - authorization: Final = list_headers.get("authorization", "") - assert (authorization.removeprefix("Bearer ") or None) == expected finally: global_mcp_tool_registry.unregister_tools_with_prefix("guarded-") diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index 24f5173d6a3..d9d61b86590 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -1066,7 +1066,7 @@ async def test_mcp_tool_execution_events_creation(): @pytest.mark.asyncio -async def test_aresponses_mcp_call_does_not_forward_admission_authorization( +async def test_aresponses_mcp_call_forwards_listed_admission_authorization( monkeypatch: pytest.MonkeyPatch, ) -> None: import importlib @@ -1165,7 +1165,10 @@ async def test_aresponses_mcp_call_does_not_forward_admission_authorization( assert mcp_client.call_tool.await_args is not None assert manager.create_mcp_client.await_args is not None - assert manager.create_mcp_client.await_args.kwargs["extra_headers"] == {"x-tenant": "tenant-a"} + assert manager.create_mcp_client.await_args.kwargs["extra_headers"] == { + "Authorization": "Bearer sk-" + "1234", + "x-tenant": "tenant-a", + } @pytest.mark.asyncio From e35eec2bbe10138fd76e5b629590967e995ec934 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:51:57 +0000 Subject: [PATCH 076/189] feat(azure): add azure_ai/mistral-large-4-latest from the Azure Models API, priced from Mistral's model page (#51047) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...model_prices_and_context_window_backup.json | 18 ++++++++++++++++++ model_prices_and_context_window.json | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ced7e002ba4..b5829c7cade 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -100049,5 +100049,23 @@ "supports_tool_choice": false, "supports_video_input": true, "supports_vision": true + }, + "azure_ai/mistral-large-4-latest": { + "deprecation_date": "2027-10-27", + "deprecation_source": "https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01", + "input_cost_per_token": 7e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 2.1e-06, + "source": "https://docs.mistral.ai/models/mistral-large-4-0", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_pdf_input": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ced7e002ba4..b5829c7cade 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -100049,5 +100049,23 @@ "supports_tool_choice": false, "supports_video_input": true, "supports_vision": true + }, + "azure_ai/mistral-large-4-latest": { + "deprecation_date": "2027-10-27", + "deprecation_source": "https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01", + "input_cost_per_token": 7e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 2.1e-06, + "source": "https://docs.mistral.ai/models/mistral-large-4-0", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_pdf_input": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } From f8a60328b47b430c8f2e49fb652e88a20e429f64 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:05:29 -0800 Subject: [PATCH 077/189] ci(image-scan): build Rust with a fast ci cargo profile instead of fat-LTO release (#51075) * ci: speed up image scans with an opt-in Rust profile Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: separate cached Rust wheels by maturin profile Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/image-scan.yml | 4 ++-- Dockerfile | 1 + litellm-rust/Cargo.toml | 6 ++++++ packaging/litellm-proxy/pyproject.toml | 9 +++++++++ pyproject.toml | 1 + 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml index 6e13b49eb15..54051104cb9 100644 --- a/.github/workflows/image-scan.yml +++ b/.github/workflows/image-scan.yml @@ -60,7 +60,7 @@ jobs: persist-credentials: false - name: Build the image - run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -t "$IMAGE" . + run: docker build --build-arg MATURIN_PEP517_ARGS="--profile ci" --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -t "$IMAGE" . - name: Save the image for the verify matrix run: docker save "$IMAGE" | zstd -T0 -3 -o "$RUNNER_TEMP/litellm-image.tar.zst" @@ -73,7 +73,7 @@ jobs: compression-level: 0 - name: Build the Admin MCP schema image - run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci --target builder -t "$ADMIN_MCP_SCHEMA_IMAGE" . + run: docker build --build-arg MATURIN_PEP517_ARGS="--profile ci" --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci --target builder -t "$ADMIN_MCP_SCHEMA_IMAGE" . - name: Save the Admin MCP schema image for the verify matrix run: docker save "$ADMIN_MCP_SCHEMA_IMAGE" | zstd -T0 -3 -o "$RUNNER_TEMP/litellm-admin-mcp-schema.tar.zst" diff --git a/Dockerfile b/Dockerfile index 205f0cc981d..0e9337e658b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -102,6 +102,7 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh # Install project and workspace packages (fast - deps already cached) +ARG MATURIN_PEP517_ARGS="" RUN uv build --package litellm-proxy --wheel --out-dir /wheels RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \ --extra proxy \ diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index cc72cd01dc0..a00a8ec2446 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -161,3 +161,9 @@ panic = "unwind" debug = false incremental = false strip = "symbols" + +[profile.ci] +inherits = "release" +opt-level = 0 +lto = false +codegen-units = 256 diff --git a/packaging/litellm-proxy/pyproject.toml b/packaging/litellm-proxy/pyproject.toml index 172f40a7228..6b99c77df7a 100644 --- a/packaging/litellm-proxy/pyproject.toml +++ b/packaging/litellm-proxy/pyproject.toml @@ -20,3 +20,12 @@ include = [ { path = ".cargo/config.toml", format = "sdist" }, { path = "model_prices_and_context_window.json", format = "sdist" }, ] + +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "setup.py" }, + { file = "setup.cfg" }, + { dir = "src" }, + { env = "MATURIN_PEP517_ARGS" }, +] diff --git a/pyproject.toml b/pyproject.toml index 6145db57902..e80ba2d5d44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -368,6 +368,7 @@ exclude = [ concurrent-builds = 1 cache-keys = [ { file = "pyproject.toml" }, + { env = "MATURIN_PEP517_ARGS" }, { file = "rust-toolchain.toml" }, { file = ".cargo/config.toml" }, { file = "litellm-rust/Cargo.lock" }, From 19c48218ae78b81349bc68d10b96f34cdf5e35da Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:05:41 -0800 Subject: [PATCH 078/189] test: deflake proxy metrics and shutdown drain (#51078) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-unit.yml | 19 ++++++++++++++----- .../db/test_log_db_metrics_service_spans.py | 12 ++++++------ .../unit/responses/test_streaming_iterator.py | 17 ++++++++++------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index d3f4821a05b..f0b51c6f1fb 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -633,23 +633,32 @@ jobs: job-timeout-minutes: 60 dist: loadscope - - shard: proxy-infra + - shard: proxy-infra-db-spend test-path: |- tests/unit/proxy/db --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py - tests/unit/proxy/middleware - --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py - tests/unit/proxy/pass_through_endpoints - tests/unit/proxy/mcp_server/test_tool_registry.py tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py + python-version: "3.12" + workers: 4 + reruns: 0 + timeout-minutes: 20 + job-timeout-minutes: 60 + dist: loadscope + + - shard: proxy-infra-routing + test-path: |- + tests/unit/proxy/middleware + --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py + tests/unit/proxy/pass_through_endpoints + tests/unit/proxy/mcp_server/test_tool_registry.py tests/unit/proxy/enterprise_billing tests/unit/proxy/types_utils tests/unit/proxy/logging_endpoints diff --git a/tests/unit/proxy/db/test_log_db_metrics_service_spans.py b/tests/unit/proxy/db/test_log_db_metrics_service_spans.py index a7fd830b948..7de9b6fac43 100644 --- a/tests/unit/proxy/db/test_log_db_metrics_service_spans.py +++ b/tests/unit/proxy/db/test_log_db_metrics_service_spans.py @@ -18,12 +18,12 @@ from prisma.errors import ClientNotConnectedError from pydantic import TypeAdapter import litellm -from litellm._service_logger import ServiceTypes +from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.integrations.datadog.datadog import DataDogLogger from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig from litellm.proxy.db.log_db_metrics import log_db_metrics from litellm.proxy.db.prisma_client import _PrismaDrainTracker, _TrackedPrismaEngine -from litellm.proxy.proxy_server import proxy_logging_obj +from litellm.proxy import proxy_server from litellm.integrations.prometheus_services import PrometheusServicesLogger from prometheus_client import REGISTRY @@ -80,10 +80,10 @@ def rig(monkeypatch: pytest.MonkeyPatch) -> _Rig: with patch("asyncio.create_task", side_effect=_discard_periodic_flush): datadog: Final = DataDogLogger() monkeypatch.setattr(litellm, "service_callback", [otel, datadog, "prometheus_system"]) - monkeypatch.setattr(proxy_logging_obj.service_logging_obj, "dd_logger", datadog, raising=False) - monkeypatch.setattr( - proxy_logging_obj.service_logging_obj, "prometheusServicesLogger", PrometheusServicesLogger(), raising=False - ) + service_logging: Final = ServiceLogging() + service_logging.dd_logger = datadog + service_logging.prometheusServicesLogger = PrometheusServicesLogger() + monkeypatch.setattr(proxy_server.proxy_logging_obj, "service_logging_obj", service_logging) return _Rig(exporter, provider, datadog) diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 3fb9edc9459..11f4e78619e 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -2743,14 +2743,17 @@ async def test_shutdown_drain_waits_for_a_pending_logging_task(): @pytest.mark.asyncio async def test_shutdown_drain_gives_up_on_a_stuck_logging_task_at_its_timeout(): - stuck: Final = streaming_module.spawn_logging_task( - asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck" - ) + loop: Final = asyncio.get_running_loop() + with patch.object(loop, "time", return_value=0.0): + stuck: Final = streaming_module.spawn_logging_task(loop, asyncio.Event().wait(), task_name="stuck") - await asyncio.wait_for(streaming_module.drain_logging_tasks(timeout=0.05), timeout=1) - - assert not stuck.done() - stuck.cancel() + try: + await streaming_module.drain_logging_tasks(timeout=0) + assert not stuck.done() + finally: + stuck.cancel() + with suppress(asyncio.CancelledError): + await stuck async def _drain_held_until_released(release: asyncio.Event) -> bool: From 407ce3da2f05eed8fdf560271b4c7baeec30aee0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 05:18:02 +0000 Subject: [PATCH 079/189] chore(cost-map): add 3 fal_ai rows priced from the fal pricing API and model page (third fal-ai sync) (#51126) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 38 +++++++++++++++++++ model_prices_and_context_window.json | 38 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b5829c7cade..733ff3bb965 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -100067,5 +100067,43 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "fal_ai/fal-ai/recraft-20b": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default realistic_image style, from the fal pricing API; vector styles cost 2x and have no catalog field; the fal.ai page for this id redirects to fal-ai/recraft/v2/text-to-image, which rounds the rate to $0.022"}, + "mode": "image_generation", + "output_cost_per_image": 0.0219, + "source": "https://api.fal.ai/v1/models/pricing?endpoint_id=fal-ai/recraft-20b", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/stable-diffusion-v3-medium/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://api.fal.ai/v1/models/pricing?endpoint_id=fal-ai/stable-diffusion-v3-medium/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/pixverse/lipsync": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video; when audio_url is not provided the text is spoken with TTS at $0.24 per 100 characters, which is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://fal.ai/models/fal-ai/pixverse/lipsync", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b5829c7cade..733ff3bb965 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -100067,5 +100067,43 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "fal_ai/fal-ai/recraft-20b": { + "litellm_provider": "fal_ai", + "metadata": {"notes":"per image at the default realistic_image style, from the fal pricing API; vector styles cost 2x and have no catalog field; the fal.ai page for this id redirects to fal-ai/recraft/v2/text-to-image, which rounds the rate to $0.022"}, + "mode": "image_generation", + "output_cost_per_image": 0.0219, + "source": "https://api.fal.ai/v1/models/pricing?endpoint_id=fal-ai/recraft-20b", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/stable-diffusion-v3-medium/image-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.035, + "source": "https://api.fal.ai/v1/models/pricing?endpoint_id=fal-ai/stable-diffusion-v3-medium/image-to-image", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/pixverse/lipsync": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of output video; when audio_url is not provided the text is spoken with TTS at $0.24 per 100 characters, which is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.04, + "source": "https://fal.ai/models/fal-ai/pixverse/lipsync", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] } } From 7ecb9ca70494815ab8579ef3665735f1f32aabc1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 05:36:28 +0000 Subject: [PATCH 080/189] fix(bedrock): govcloud eol dates from the lifecycle table, gpt-5.6 cyber govcloud mantle row and daybreak blue input limit (#51095) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 63 ++++++++++++++----- model_prices_and_context_window.json | 63 ++++++++++++++----- 2 files changed, 96 insertions(+), 30 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 733ff3bb965..3210f3e080b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15776,8 +15776,8 @@ "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-08", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "deprecation_date": "2027-04-27", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "successor_model": "bedrock/us-gov-east-1/anthropic.claude-sonnet-5-5", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", @@ -15808,8 +15808,8 @@ "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-08", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "deprecation_date": "2027-04-27", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -15969,8 +15969,8 @@ "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-08", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "deprecation_date": "2027-04-27", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "successor_model": "bedrock/us-gov-west-1/anthropic.claude-sonnet-5-5", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", @@ -16001,8 +16001,8 @@ "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-08", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "deprecation_date": "2027-04-27", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -16026,8 +16026,8 @@ "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { - "deprecation_date": "2027-03-30", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "deprecation_date": "2027-05-09", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -16038,8 +16038,8 @@ "supports_pdf_input": true }, "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": { - "deprecation_date": "2027-03-30", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "deprecation_date": "2027-05-09", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -50861,8 +50861,8 @@ "cache_creation_input_token_cost_above_200k_tokens": 9e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, - "deprecation_date": "2027-04-08", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "deprecation_date": "2027-04-27", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "successor_model": "us-gov.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, @@ -61526,7 +61526,7 @@ "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012}, @@ -100105,5 +100105,38 @@ "supported_output_modalities": [ "video" ] + }, + "bedrock_mantle/us-gov-east-1/openai.gpt-5.6-cyber": { + "cache_creation_input_token_cost": 2.109375e-05, + "cache_read_input_token_cost": 1.6875e-06, + "input_cost_per_token": 1.6875e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 0.00010125, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-cyber.html", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": false, + "supports_xhigh_reasoning_effort": true, + "use_openai_responses_path": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 733ff3bb965..3210f3e080b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15776,8 +15776,8 @@ "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-08", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "deprecation_date": "2027-04-27", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "successor_model": "bedrock/us-gov-east-1/anthropic.claude-sonnet-5-5", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", @@ -15808,8 +15808,8 @@ "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-08", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "deprecation_date": "2027-04-27", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -15969,8 +15969,8 @@ "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-08", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "deprecation_date": "2027-04-27", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "successor_model": "bedrock/us-gov-west-1/anthropic.claude-sonnet-5-5", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", @@ -16001,8 +16001,8 @@ "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2027-04-08", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "deprecation_date": "2027-04-27", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "supports_tool_search": true, @@ -16026,8 +16026,8 @@ "token_count_model": {"model":"anthropic.claude-sonnet-4-5-20250929-v1:0","litellm_provider":"bedrock"} }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { - "deprecation_date": "2027-03-30", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "deprecation_date": "2027-05-09", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -16038,8 +16038,8 @@ "supports_pdf_input": true }, "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": { - "deprecation_date": "2027-03-30", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "deprecation_date": "2027-05-09", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -50861,8 +50861,8 @@ "cache_creation_input_token_cost_above_200k_tokens": 9e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, - "deprecation_date": "2027-04-08", - "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md", + "deprecation_date": "2027-04-27", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", "successor_model": "us-gov.anthropic.claude-sonnet-5-5", "litellm_provider": "bedrock_converse", "supports_tool_search": true, @@ -61526,7 +61526,7 @@ "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012}, @@ -100105,5 +100105,38 @@ "supported_output_modalities": [ "video" ] + }, + "bedrock_mantle/us-gov-east-1/openai.gpt-5.6-cyber": { + "cache_creation_input_token_cost": 2.109375e-05, + "cache_read_input_token_cost": 1.6875e-06, + "input_cost_per_token": 1.6875e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 0.00010125, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-cyber.html", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": false, + "supports_xhigh_reasoning_effort": true, + "use_openai_responses_path": true } } From 60103ca01785e3d26d699f4b514c56c8a981f2db Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:03:33 -0800 Subject: [PATCH 081/189] feat(helm): single chart with monolith mode and shared image, remove litellm-helm (#43330) helm/litellm becomes the only chart. Every container runs the one shared image with args only, monolith.enabled renders a single proxy Deployment, the retired chart's KEDA, extraResources, generated master key, helm test, minReadySeconds and rollout strategy settings are ported, and the bundled PostgreSQL and Redis subcharts are removed Resolves LIT-8880 Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/config.yml | 31 +- .github/workflows/test-unit.yml | 21 +- .gitignore | 2 +- Makefile | 2 +- docker-entrypoint.sh | 37 +- helm/litellm-helm/.helmignore | 23 - helm/litellm-helm/Chart.lock | 9 - helm/litellm-helm/Chart.yaml | 41 -- helm/litellm-helm/README.md | 266 ------- .../litellm-helm/charts/postgresql-14.3.1.tgz | Bin 72928 -> 0 bytes helm/litellm-helm/charts/redis-18.19.1.tgz | Bin 87594 -> 0 bytes helm/litellm-helm/ci/test-values.yaml | 15 - helm/litellm-helm/templates/NOTES.txt | 23 - helm/litellm-helm/templates/_helpers.tpl | 399 ----------- .../templates/configmap-litellm.yaml | 22 - helm/litellm-helm/templates/deployment.yaml | 258 ------- .../templates/extra-resources.yaml | 6 - helm/litellm-helm/templates/hpa.yaml | 64 -- helm/litellm-helm/templates/ingress.yaml | 78 -- helm/litellm-helm/templates/keda.yaml | 58 -- .../templates/migrations-job.yaml | 122 ---- .../templates/poddisruptionbudget.yaml | 33 - .../templates/secret-dbcredentials.yaml | 13 - .../templates/secret-masterkey.yaml | 12 - .../templates/service-metrics.yaml | 17 - helm/litellm-helm/templates/service.yaml | 22 - .../templates/serviceaccount.yaml | 13 - .../templates/servicemonitor.yaml | 39 - .../templates/tests/test-connection.yaml | 25 - .../templates/tests/test-env-vars.yaml | 43 -- .../templates/tests/test-servicemonitor.yaml | 152 ---- .../tests/billing_metrics_tests.yaml | 100 --- .../tests/bundled_db_images_tests.yaml | 94 --- helm/litellm-helm/tests/collector_tests.yaml | 272 ------- .../tests/connection_pool_tests.yaml | 112 --- .../tests/coordination_redis_tests.yaml | 143 ---- .../tests/database_auth_tests.yaml | 288 -------- .../deployment_command_args_labels_tests.yaml | 68 -- helm/litellm-helm/tests/deployment_tests.yaml | 520 -------------- helm/litellm-helm/tests/hpa_tests.yaml | 144 ---- helm/litellm-helm/tests/ingress_tests.yaml | 45 -- helm/litellm-helm/tests/keda_tests.yaml | 106 --- .../tests/lens_connection_tests.yaml | 128 ---- .../tests/masterkey-secret_tests.yaml | 71 -- .../tests/metrics_server_tests.yaml | 106 --- .../tests/migrations-job_tests.yaml | 432 ----------- helm/litellm-helm/tests/pdb_tests.yaml | 45 -- helm/litellm-helm/tests/service_tests.yaml | 116 --- helm/litellm-helm/values.yaml | 676 ------------------ helm/litellm/.helmignore | 20 + helm/litellm/Chart.yaml | 6 +- helm/litellm/README.md | 174 ++++- helm/litellm/ci/postgres.yaml | 58 ++ helm/litellm/ci/test-values.yaml | 34 + helm/litellm/templates/NOTES.txt | 43 +- helm/litellm/templates/_helpers.tpl | 172 ++++- helm/litellm/templates/_workload.tpl | 515 +++++++++++++ .../litellm/templates/backend/deployment.yaml | 31 +- helm/litellm/templates/backend/hpa.yaml | 2 +- .../backend/poddisruptionbudget.yaml | 1 + helm/litellm/templates/backend/service.yaml | 9 +- helm/litellm/templates/extra-resources.yaml | 4 + helm/litellm/templates/gateway/configmap.yaml | 5 +- .../litellm/templates/gateway/deployment.yaml | 247 +------ helm/litellm/templates/gateway/hpa.yaml | 66 +- helm/litellm/templates/gateway/keda.yaml | 3 + .../gateway/poddisruptionbudget.yaml | 1 + .../templates/gateway/service-metrics.yaml | 19 +- helm/litellm/templates/gateway/service.yaml | 19 +- .../templates/gateway/servicemonitor.yaml | 29 +- helm/litellm/templates/ingress.yaml | 42 +- .../templates/liteadmin.yaml | 2 +- helm/litellm/templates/migrations-job.yaml | 41 +- .../templates/monolith/deployment.yaml | 3 + helm/litellm/templates/monolith/hpa.yaml | 3 + helm/litellm/templates/monolith/keda.yaml | 3 + .../monolith/poddisruptionbudget.yaml | 7 + .../templates/monolith/service-metrics.yaml | 3 + helm/litellm/templates/monolith/service.yaml | 3 + .../templates/monolith/servicemonitor.yaml | 3 + helm/litellm/templates/secret-masterkey.yaml | 22 + helm/litellm/templates/serviceaccount.yaml | 6 +- .../templates/tests/test-connection.yaml | 53 ++ helm/litellm/templates/ui/deployment.yaml | 23 +- helm/litellm/templates/ui/hpa.yaml | 2 +- .../templates/ui/poddisruptionbudget.yaml | 1 + helm/litellm/templates/ui/service.yaml | 9 +- helm/litellm/tests/collector_tests.yaml | 10 +- helm/litellm/tests/component_hosts_tests.yaml | 12 +- helm/litellm/tests/extra_resources_tests.yaml | 45 ++ .../tests/hpa_workload_metrics_tests.yaml | 44 ++ .../litellm/tests/ingress_monolith_tests.yaml | 164 +++++ helm/litellm/tests/keda_tests.yaml | 266 +++++++ helm/litellm/tests/liteadmin_tests.yaml | 149 ++++ .../litellm/tests/masterkey_secret_tests.yaml | 156 ++++ helm/litellm/tests/metrics_server_tests.yaml | 10 +- helm/litellm/tests/migration_job_tests.yaml | 75 +- helm/litellm/tests/monolith_tests.yaml | 539 ++++++++++++++ .../tests/no_bundled_datastores_tests.yaml | 54 ++ helm/litellm/tests/redis_env_tests.yaml | 4 +- .../litellm/tests/rollout_strategy_tests.yaml | 36 + helm/litellm/tests/shared_image_tests.yaml | 264 +++++++ helm/litellm/values.schema.json | 183 +++++ helm/litellm/values.yaml | 229 ++++-- scripts/tpm_headline_test.sh | 2 +- tests/unit/proxy/test_proxy_cli.py | 24 +- tests/unit/test_docker_entrypoint.py | 144 ++++ 107 files changed, 3637 insertions(+), 5769 deletions(-) delete mode 100644 helm/litellm-helm/.helmignore delete mode 100644 helm/litellm-helm/Chart.lock delete mode 100644 helm/litellm-helm/Chart.yaml delete mode 100644 helm/litellm-helm/README.md delete mode 100644 helm/litellm-helm/charts/postgresql-14.3.1.tgz delete mode 100644 helm/litellm-helm/charts/redis-18.19.1.tgz delete mode 100644 helm/litellm-helm/ci/test-values.yaml delete mode 100644 helm/litellm-helm/templates/NOTES.txt delete mode 100644 helm/litellm-helm/templates/_helpers.tpl delete mode 100644 helm/litellm-helm/templates/configmap-litellm.yaml delete mode 100644 helm/litellm-helm/templates/deployment.yaml delete mode 100644 helm/litellm-helm/templates/extra-resources.yaml delete mode 100644 helm/litellm-helm/templates/hpa.yaml delete mode 100644 helm/litellm-helm/templates/ingress.yaml delete mode 100644 helm/litellm-helm/templates/keda.yaml delete mode 100644 helm/litellm-helm/templates/migrations-job.yaml delete mode 100644 helm/litellm-helm/templates/poddisruptionbudget.yaml delete mode 100644 helm/litellm-helm/templates/secret-dbcredentials.yaml delete mode 100644 helm/litellm-helm/templates/secret-masterkey.yaml delete mode 100644 helm/litellm-helm/templates/service-metrics.yaml delete mode 100644 helm/litellm-helm/templates/service.yaml delete mode 100644 helm/litellm-helm/templates/serviceaccount.yaml delete mode 100644 helm/litellm-helm/templates/servicemonitor.yaml delete mode 100644 helm/litellm-helm/templates/tests/test-connection.yaml delete mode 100644 helm/litellm-helm/templates/tests/test-env-vars.yaml delete mode 100644 helm/litellm-helm/templates/tests/test-servicemonitor.yaml delete mode 100644 helm/litellm-helm/tests/billing_metrics_tests.yaml delete mode 100644 helm/litellm-helm/tests/bundled_db_images_tests.yaml delete mode 100644 helm/litellm-helm/tests/collector_tests.yaml delete mode 100644 helm/litellm-helm/tests/connection_pool_tests.yaml delete mode 100644 helm/litellm-helm/tests/coordination_redis_tests.yaml delete mode 100644 helm/litellm-helm/tests/database_auth_tests.yaml delete mode 100644 helm/litellm-helm/tests/deployment_command_args_labels_tests.yaml delete mode 100644 helm/litellm-helm/tests/deployment_tests.yaml delete mode 100644 helm/litellm-helm/tests/hpa_tests.yaml delete mode 100644 helm/litellm-helm/tests/ingress_tests.yaml delete mode 100644 helm/litellm-helm/tests/keda_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_connection_tests.yaml delete mode 100644 helm/litellm-helm/tests/masterkey-secret_tests.yaml delete mode 100644 helm/litellm-helm/tests/metrics_server_tests.yaml delete mode 100644 helm/litellm-helm/tests/migrations-job_tests.yaml delete mode 100644 helm/litellm-helm/tests/pdb_tests.yaml delete mode 100644 helm/litellm-helm/tests/service_tests.yaml delete mode 100644 helm/litellm-helm/values.yaml create mode 100644 helm/litellm/.helmignore create mode 100644 helm/litellm/ci/postgres.yaml create mode 100644 helm/litellm/ci/test-values.yaml create mode 100644 helm/litellm/templates/_workload.tpl create mode 100644 helm/litellm/templates/extra-resources.yaml create mode 100644 helm/litellm/templates/gateway/keda.yaml rename helm/{litellm-helm => litellm}/templates/liteadmin.yaml (96%) create mode 100644 helm/litellm/templates/monolith/deployment.yaml create mode 100644 helm/litellm/templates/monolith/hpa.yaml create mode 100644 helm/litellm/templates/monolith/keda.yaml create mode 100644 helm/litellm/templates/monolith/poddisruptionbudget.yaml create mode 100644 helm/litellm/templates/monolith/service-metrics.yaml create mode 100644 helm/litellm/templates/monolith/service.yaml create mode 100644 helm/litellm/templates/monolith/servicemonitor.yaml create mode 100644 helm/litellm/templates/secret-masterkey.yaml create mode 100644 helm/litellm/templates/tests/test-connection.yaml create mode 100644 helm/litellm/tests/extra_resources_tests.yaml create mode 100644 helm/litellm/tests/ingress_monolith_tests.yaml create mode 100644 helm/litellm/tests/keda_tests.yaml create mode 100644 helm/litellm/tests/liteadmin_tests.yaml create mode 100644 helm/litellm/tests/masterkey_secret_tests.yaml create mode 100644 helm/litellm/tests/monolith_tests.yaml create mode 100644 helm/litellm/tests/no_bundled_datastores_tests.yaml create mode 100644 helm/litellm/tests/shared_image_tests.yaml create mode 100644 helm/litellm/values.schema.json diff --git a/.circleci/config.yml b/.circleci/config.yml index cf1a7b94fe5..15129f1e5e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -531,31 +531,36 @@ jobs: IMAGE_TAG=${CIRCLE_SHA1:-ci} kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test - # Run helm lint - run: name: Run helm lint command: | - helm lint ./helm/litellm-helm + helm lint ./helm/litellm -f ./helm/litellm/ci/test-values.yaml + + - run: + name: Start PostgreSQL for the chart + command: | + kubectl apply -f ./helm/litellm/ci/postgres.yaml + kubectl rollout status deployment/litellm-ci-postgres --timeout=3m - # Run helm tests - run: name: Run helm tests command: | IMAGE_TAG=${CIRCLE_SHA1:-ci} - helm install litellm ./helm/litellm-helm -f ./helm/litellm-helm/ci/test-values.yaml \ + helm install litellm ./helm/litellm -f ./helm/litellm/ci/test-values.yaml \ --set image.repository=litellm-ci \ --set image.tag=${IMAGE_TAG} \ - --set image.pullPolicy=Never - if ! kubectl rollout status deployment/litellm --timeout=180s; then - kubectl describe pods -l app.kubernetes.io/name=litellm - kubectl logs -l app.kubernetes.io/name=litellm --tail=200 - exit 1 - fi + --set image.pullPolicy=Never \ + --wait --wait-for-jobs --timeout 10m || { + kubectl get pods -o wide + kubectl describe pods -l app.kubernetes.io/instance=litellm + kubectl logs -l app.kubernetes.io/component=proxy --tail=200 || true + kubectl logs -l app.kubernetes.io/component=migrations --tail=200 || true + exit 1 + } - echo "Printing pod logs..." - kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") + echo "Printing proxy pod logs..." + kubectl logs -l app.kubernetes.io/component=proxy --tail=200 - # Run the helm tests helm test litellm --logs # Cleanup diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f0b51c6f1fb..f8b93b5cc11 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -1353,17 +1353,16 @@ jobs: - name: Run unit tests run: | - for chart in helm/litellm-helm helm/litellm; do - declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')" - output="$(mktemp)" - helm unittest -f 'tests/*.yaml' "$chart" | tee "$output" - executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")" - if [ "$declared" != "$executed" ]; then - echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute." - exit 1 - fi - echo "$chart: all $declared declared test suites ran" - done + chart=helm/litellm + declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')" + output="$(mktemp)" + helm unittest -f 'tests/*.yaml' "$chart" | tee "$output" + executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")" + if [ "$declared" != "$executed" ]; then + echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute." + exit 1 + fi + echo "$chart: all $declared declared test suites ran" openai-sdk-compat: name: OpenAI ${{ matrix.openai-version }} compatibility needs: rust-bridge diff --git a/.gitignore b/.gitignore index b08ee92d6af..1b4785d193a 100644 --- a/.gitignore +++ b/.gitignore @@ -61,7 +61,7 @@ ui/litellm-dashboard/next-env.d.ts *.tsbuildinfo ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json -helm/litellm-helm/*.tgz +helm/litellm/charts/*.tgz helm/*.tgz litellm/proxy/vertex_key.json **/.vim/ diff --git a/Makefile b/Makefile index 48df3c8e215..c256d273d8a 100644 --- a/Makefile +++ b/Makefile @@ -341,7 +341,7 @@ test-integration: install-test-deps $(UV_RUN) pytest tests/ test-unit-helm: install-helm-unittest - helm unittest -f 'tests/*.yaml' helm/litellm-helm + helm unittest -f 'tests/*.yaml' helm/litellm # LLM Translation testing targets test-llm-translation: install-test-deps diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 7bcb62253a7..a48aad33324 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -16,8 +16,11 @@ # --admin-agent the bundled LiteAdmin Slack agent (litellm-admin-agent --web ARGS) # # gateway and backend get their host and port defaults first, so ARGS such as -# --port 8080 override them. An unknown first word is executed as-is (docker run -# sh), through ddtrace-run like every component when USE_DDTRACE=true. PgBouncer is not a component: LITELLM_PGBOUNCER_ENABLED=true +# --port 8080 override them. With no DATABASE_URL, proxy assembles DATABASE_URL +# and DATABASE_URL_READ_REPLICA from DATABASE_HOST and the other discrete +# variables the way gateway and backend do. An unknown first word is executed +# as-is (docker run sh), through ddtrace-run like every component when +# USE_DDTRACE=true. PgBouncer is not a component: LITELLM_PGBOUNCER_ENABLED=true # starts it inside proxy and gateway. Components that write Prometheus samples # start with an empty PROMETHEUS_MULTIPROC_DIR; metrics, collector and raw # readers keep the workers' files, other raw commands clear stale samples. @@ -60,7 +63,35 @@ case "$component" in raw) ;; proxy) - set -- litellm "$@" + if { [ -z "${DATABASE_URL:-}" ] && [ -n "${DATABASE_HOST:-}" ]; } || + { [ -z "${DATABASE_URL_READ_REPLICA:-}" ] && [ -n "${DATABASE_HOST_READ_REPLICA:-}" ]; }; then + set -- python -c 'import os, sys +from litellm.proxy.db.db_url_settings import ( + DatabaseURLSettings, + add_missing_query_params, + connection_params_from_url, + idle_lifetime_params, + translate_libpq_ssl_params, +) +if os.environ.get("USE_AWS_KMS") == "True": + from litellm.secret_managers.aws_secret_manager import decrypt_env_var + os.environ.update(decrypt_env_var()) +settings = DatabaseURLSettings.from_env() +settings.apply_writer_url_to_env() +reader = settings.build_reader_url() +if reader: + os.environ["DATABASE_URL_READ_REPLICA"] = translate_libpq_ssl_params( + add_missing_query_params(reader, connection_params_from_url(os.environ.get("DATABASE_URL", ""))) + ) +if settings.max_idle_connection_lifetime is not None: + lifetime = idle_lifetime_params(settings.max_idle_connection_lifetime) + for name in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA"): + if os.environ.get(name): + os.environ[name] = add_missing_query_params(os.environ[name], lifetime) +os.execvp("litellm", ["litellm", *sys.argv[1:]])' "$@" + else + set -- litellm "$@" + fi ;; gateway) set -- python -m gateway.launch --workers "${NUM_WORKERS:-1}" --host 0.0.0.0 --port 4000 "$@" diff --git a/helm/litellm-helm/.helmignore b/helm/litellm-helm/.helmignore deleted file mode 100644 index 0e8a0eb36f4..00000000000 --- a/helm/litellm-helm/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/litellm-helm/Chart.lock b/helm/litellm-helm/Chart.lock deleted file mode 100644 index 1e63f4f9eca..00000000000 --- a/helm/litellm-helm/Chart.lock +++ /dev/null @@ -1,9 +0,0 @@ -dependencies: -- name: postgresql - repository: oci://registry-1.docker.io/bitnamicharts - version: 14.3.1 -- name: redis - repository: oci://registry-1.docker.io/bitnamicharts - version: 18.19.1 -digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009 -generated: '2026-10-30T02:13:11.505726000Z' diff --git a/helm/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml deleted file mode 100644 index 059889cb667..00000000000 --- a/helm/litellm-helm/Chart.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v2 - -# We can't call ourselves just "litellm" because then we couldn't publish to the -# same OCI repository as the "litellm" OCI image -name: litellm-helm -description: Call all LLM APIs using the OpenAI format - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.4 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: v1.85.1 - -annotations: - org.opencontainers.image.source: "https://github.com/BerriAI/litellm" - org.opencontainers.image.url: "https://docs.litellm.ai/" - -dependencies: - - name: "postgresql" - version: "14.3.1" - repository: oci://registry-1.docker.io/bitnamicharts - condition: db.deployStandalone - - name: redis - version: "18.19.1" - repository: oci://registry-1.docker.io/bitnamicharts - condition: redis.enabled diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md deleted file mode 100644 index bc6e29632f1..00000000000 --- a/helm/litellm-helm/README.md +++ /dev/null @@ -1,266 +0,0 @@ -# Helm Chart for LiteLLM - -> [!IMPORTANT] -> This is community maintained, Please make an issue if you run into a bug -> We recommend using [Docker or Kubernetes for production deployments](https://docs.litellm.ai/docs/proxy/prod) - -## Prerequisites - -- Kubernetes 1.21+ -- Helm 3.8.0+ - -If `db.deployStandalone` is used: - -- PV provisioner support in the underlying infrastructure - -If `db.useStackgresOperator` is used (not yet implemented): - -- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing. - -## Parameters - -### LiteLLM Proxy Deployment Settings - -| Name | Description | Value | -| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | -| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` | -| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A | -| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A | -| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated on first install and reused on upgrades. | N/A | -| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` | -| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` | -| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` | -| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` | -| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` | -| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` | -| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` | -| `livenessProbe.*` | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` | -| `readinessProbe.*` | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` | -| `startupProbe.*` | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` | -| `resources.*` | CPU/memory requests and limits for the LiteLLM container. Unset by default; production deployments should set 1 CPU and 4Gi of memory per worker. | `{}` | -| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` | -| `ingress.labels` | Additional labels for the Ingress resource | `{}` | -| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A | -| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` | -| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` | -| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` | -| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` | -| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | -| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` | -| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | -| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | -| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` | -| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` | - -| `billingMetrics.enabled` | Enable enterprise billable-request metering. Requires an enterprise license. | `false` | -| `billingMetrics.endpoint` | Collector that the billable-request counter is pushed to. | `https://telemetry.litellm.ai` | -| `billingMetrics.tokenUrl` | Cognito OAuth2 client-credentials token endpoint. | `https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token` | -| `billingMetrics.secretName` | Name of an existing Secret with `client-id` and `client-secret` keys. | `litellm-billing-metrics` | -| `billingMetrics.exportIntervalMs` | How often the counter is pushed, in milliseconds. The proxy defaults to `60000` when unset. | `""` | - -#### Example `proxy_config` ConfigMap from values (default): - -``` -proxyConfigMap: - create: true - key: "config.yaml" - -proxy_config: - general_settings: - master_key: os.environ/PROXY_MASTER_KEY - model_list: [] -``` - -The chart ships no models, so add them under `proxy_config.model_list` or from the Admin UI with `store_model_in_db: true` - -``` -proxy_config: - model_list: - - model_name: gpt-5.6 - litellm_params: - model: openai/gpt-5.6 - api_key: os.environ/OPENAI_API_KEY -``` - -#### Example using existing `proxyConfigMap` instead of creating it: - -``` -proxyConfigMap: - create: false - name: my-litellm-config - key: config.yaml - -# proxy_config is ignored in this mode -``` - -#### Example `environmentSecrets` Secret - -``` -apiVersion: v1 -kind: Secret -metadata: - name: litellm-envsecrets -data: - AZURE_OPENAI_API_KEY: TXlTZWN1cmVLM3k= -type: Opaque -``` - -#### Enterprise billable-request metering - -Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector with Cognito bearer authentication. Create a Secret with `client-id` and `client-secret` keys, then enable the chart values: - -``` -kubectl create secret generic litellm-billing-metrics --from-literal=client-id=... --from-literal=client-secret=... -``` - -``` -billingMetrics: - enabled: true -``` - -The endpoint and token URL must use HTTPS. The chart requires both URLs and -the Secret name when billing metrics are enabled - -### Database Settings - -| Name | Description | Value | -| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | -| `db.useExisting` | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below. | `false` | -| `db.endpoint` | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to. | `localhost` | -| `db.database` | If `db.useExisting` is `true`, the name of the existing database to connect to. | `litellm` | -| `db.url` | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value. | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` | -| `db.secret.name` | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials. | `postgres` | -| `db.secret.usernameKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance. | `username` | -| `db.secret.passwordKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user. | `password` | -| `db.username` | If `db.useExisting` is `true`, the database user as a plain value. When set, `db.secret.usernameKey` is not read. | `""` | -| `db.port` | If `db.useExisting` is `true` with token auth, the Postgres port used to build the connection URL. Password auth takes the port from `db.url`. | `5432` | -| `db.schema` | With token auth, the Postgres schema holding the LiteLLM tables, rendered as `DATABASE_SCHEMA` when the tables are not in `public`. Ignored with password auth, where the schema stays in `db.url` as `?schema=`. | `""` | -| `db.useIAMAuth` | If `db.useExisting` is `true` and `db.deployStandalone` is `false`, authenticate with an AWS RDS IAM token instead of a password. See [Token auth](#token-auth-to-an-existing-postgres-rds-iam-or-microsoft-entra-id). | `false` | -| `db.useAzureEntraAuth` | If `db.useExisting` is `true` and `db.deployStandalone` is `false`, authenticate with a Microsoft Entra ID token instead of a password. Mutually exclusive with `db.useIAMAuth`. | `false` | -| `db.useStackgresOperator` | Not yet implemented. | `false` | -| `db.deployStandalone` | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups. | `true` | -| `postgresql.*` | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml) | -| `postgresql.auth.*` | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used. | `NoTaGrEaTpAsSwOrD` | -| `postgresql.image.*` | If `db.deployStandalone` is `true`, the image for the bundled Postgres. Pinned to a `docker.io/bitnamilegacy` build because Bitnami retired the versioned tags under `docker.io/bitnami`. | `bitnamilegacy/postgresql:16.2.0-debian-12-r6` | -| `redis.image.*` | If `redis.enabled` is `true`, the image for the bundled Redis. Pinned to a `docker.io/bitnamilegacy` build for the same reason. | `bitnamilegacy/redis:7.2.4-debian-12-r9` | - -#### Bundled Postgres image - -Bitnami removed the versioned tags from `docker.io/bitnami` and republished the archived builds under `docker.io/bitnamilegacy`, so the image defaults that ship inside the `postgresql` and `redis` subcharts no longer pull. The chart pins both to the `bitnamilegacy` copies of the exact builds those subchart versions were released with, which keeps the on-disk data directory layout unchanged for existing installs. - -Keep `postgresql.image.tag` pinned. `docker.io/bitnami/postgresql` still publishes a floating `latest`, and pointing the bundled Postgres at a different major version starts the server against a data directory it cannot read (`database files are incompatible with server`). There is no in-place way back, so crossing a major version means dumping the database with the old image and restoring it into the new one. The chart refuses to render when the tag is empty or `latest`. - -Those images no longer receive updates. For anything beyond getting started, run Postgres outside the chart and point at it with `db.useExisting`. - -#### Example Postgres `db.useExisting` Secret - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: postgres -data: - # Password for the "postgres" user - postgres-password: - username: litellm - password: -type: Opaque -``` - -#### Token auth to an existing Postgres (RDS IAM or Microsoft Entra ID) - -When the database accepts short-lived tokens instead of a password (AWS RDS or Aurora with IAM database authentication, or Azure Database for PostgreSQL with Microsoft Entra ID), there is no password to put in a Secret. Turn on `db.useIAMAuth` or `db.useAzureEntraAuth` and the chart stops referencing `db.secret.passwordKey` and `db.url`. It sets `IAM_TOKEN_DB_AUTH` (or `AZURE_POSTGRESQL_AUTH`) and emits `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_USERNAME` and `DATABASE_NAME` for the proxy, the migrations job and the collector sidecar. Each of them mints its own token at startup and builds `DATABASE_URL` from those parts, and the in-container pgbouncer (`db.connectionPool`) refreshes the token before it expires. - -```yaml -db: - useExisting: true - deployStandalone: false - useIAMAuth: true - endpoint: litellm.cluster-abc123.us-west-1.rds.amazonaws.com - database: litellm - username: litellm_iam # the role granted rds_iam; no db.secret needed -``` - -The pod needs cloud credentials for the token: on EKS use IRSA or Pod Identity on the chart's `serviceAccount` (and `migrationJob.serviceAccountName` when the Helm hook runs before the ServiceAccount exists), on AKS use workload identity. Set `AWS_RDS_REGION` under `envVars` when the RDS hostname is not a standard regional endpoint (the token must be signed for the region the database lives in). TLS options that would otherwise go in `db.url` are set as `DATABASE_SSLMODE` and `DATABASE_SSLROOTCERT` under `envVars`. Leaving `db.username` empty reads the user from `db.secret.usernameKey` as before. - -#### Examples for `environmentSecrets` and `environemntConfigMaps` - -```yaml -# Use config map for not-secret configuration data -apiVersion: v1 -kind: ConfigMap -metadata: - name: litellm-env-configmap -data: - SOME_KEY: someValue - ANOTHER_KEY: anotherValue -``` - -```yaml -# Use secrets for things which are actually secret like API keys, credentials, etc -# Base64 encode the values stored in a Kubernetes Secret: $ pbpaste | base64 | pbcopy -# The --decode flag is convenient: $ pbpaste | base64 --decode - -apiVersion: v1 -kind: Secret -metadata: - name: litellm-env-secret -type: Opaque -data: - SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded - ANOTHER_PASSWORD: AAZbUGVXeU5e0ZB # base64 encoded -``` - -Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472732157221e04c6b15e3b3f094) - -### Migration Job Settings - -The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments. - -| Name | Description | Value | -| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------- | -| `migrationJob.enabled` | Enable or disable the schema migration Job | `true` | -| `migrationJob.backoffLimit` | Backoff limit for Job restarts | `4` | -| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs | `120` | -| `migrationJob.annotations` | Additional annotations for the migration job pod | `{}` | -| `migrationJob.extraContainers` | Additional containers to run alongside the migration job | `[]` | -| `migrationJob.hooks.argocd.enabled` | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy) | `true` | -| `migrationJob.hooks.helm.enabled` | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` | -| `migrationJob.hooks.helm.weight` | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified. | N/A | - -## Accessing the Admin UI - -When browsing to the URL published per the settings in `ingress.*`, you will -be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal -(from the `litellm` pod's perspective) URL published by the `-litellm` -Kubernetes Service. If the deployment uses the default settings for this -service, the **Proxy Endpoint** should be set to `http://-litellm:4000`. - -The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey` -was not provided to the helm command line, the `masterkey` is a randomly -generated string in the `sk-...` format stored in the `-litellm-masterkey` Kubernetes Secret. -The key is generated once on the first install; later `helm upgrade` runs reuse the -value already in that Secret, so upgrading never rotates the master key. - -```bash -kubectl -n litellm get secret -litellm-masterkey -o jsonpath="{.data.masterkey}" -``` - -## Admin UI Limitations - -At the time of writing, the Admin UI is unable to add models. This is because -it would need to update the `config.yaml` file which is a exposed ConfigMap, and -therefore, read-only. This is a limitation of this helm chart, not the Admin UI -itself. - -## Connect Lens - -This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render - -## Shutdown accounting - -`GRACEFUL_SHUTDOWN_TIMEOUT` defaults to 30 seconds. Pending success callbacks share the time remaining since SIGTERM or the first `/health/drain` call. Callback flushing reserves the existing 10-second teardown headroom for cancellation, tracked accounting tasks and persistence. Exhausting this budget can discard callbacks, which is logged - -Set `terminationGracePeriodSeconds` to at least `GRACEFUL_SHUTDOWN_TIMEOUT + 10`, allowing additional time for preStop hooks and database writes. This is not a hard shutdown bound or a guarantee of persistence. Database retries retain their own limits, and cancellation-resistant callbacks can remain alive. The existing preStop hook can also be followed by Uvicorn's separate connection drain, as described in `values.yaml` diff --git a/helm/litellm-helm/charts/postgresql-14.3.1.tgz b/helm/litellm-helm/charts/postgresql-14.3.1.tgz deleted file mode 100644 index e8e2fac0fdaf8d5614921bfa3b7cddeeb088b3f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 72928 zcmV)4K+3-#iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0POwyb{jXcD2&hFdJ6og%&|0EOB82QQ!h>G1f)tD~bA#|N*D|8#Kp{P^J2 zpTNO}=_vgqj6?dTgSBzx8~2rbU=$G!IVKV7Z2{mzj;4ezdf)_dIED;u0f6UlidYZ) zz6AgP9*E!^^#EZWo1rk24|Sm)2#9}&Xcv?HG3F7R<9$=EcX;FlXpCXx9Ugh~@TCLI z$f1Wm#DsE0o4^(-^bWd*FS-W~j^k1szctdFmFyk9>>jyL1>qs~QN#pO`Z4rp$UEvD zY{3}+j3~n->Vf;CEf~k?PX%QwKrxB}6!{pj-j<=uC-XUpwg8|gCJb}Qf8yib{ys%h z%s5?mhus2|z8J4;3jhwMvhzHk-*_p+dqcoc_e1x?R)CmKam?kTp=#vl|DJ;#{hjRs z%m9o4iBSX?NvMwi8UKoW?okv-+>9TW8to<`sfE!ZCTmEef&3F z9UUEfIC_5Ejic$-9a=mP8p!^$Br5x{<)sYT@_>3rQ5<4l@^8xnewZ*J;&X^24snF2 zB@I7cJV1)}z**#Xw*W|Jn6+Sv`79aB1{hJ*Qc#d^)p-(bW9@xT5!x5jl#GC_RE7?Z zyU)9aTc2BHefw+#J5>9Ub+KUUgqSe|d25 z{qf81|Ks4WcW}`8rn9O(*7(1NVS?BrEC6c9|IzaohsQ7S5#Te(0iR(89&i`}a*rs*fq0IrYI^`jewYL}ngUxt?E#-e6Fg0X z0munRn5sEdc%}SqZEtU>-@+mm)JeFyOjla`*II=cf={F#Q>y`B^qZ`07C)v&4|F<} zV24Qkl*ii+YzJ?{)85+D@Dif{U@|Ysk=i9fTK+iS%dBUZWmld z;X=R+z{3nh;C(A5z88yehWPRg7P9nY?ad+k49&Gx*H6~!a-17$y!<1|^|Y@lui~`M z6(6IDtVeONlo`<0BbNm0Mb=J(9)P@9Tg4R#Rqx67VjvD7pAb4{ zX3q5On&})_5AP~Q5u4!&w}~d^p`oN zrX3T*NW2eF9FoOcSOlDaIlPk=*gS1nh`F_==myC?YXBd`-ADlv;y#ExxiG&3h$=_4r*fM%vK~u8c)(iJ@MMi&|Du zDbE=Yp+C~fBTfZZI1xyNDhwByt*(3nU1`I1CrKDe1B#=$=ph7lq-AeBo04Y~$u)q) zu`9^1k>%(x083~QuUgX~(Bxs|Q0Y!TA%a+&VFN7<>WzTp6z!Rs-H`Y&v>>7noWc`1 zL_lBI^~wU`6ly0r6YWrANR1PlJv&Hibk5^WIh+)O#v-z z0gBRfk4Do26Bsg7N00$F(0@6}gjW#b6XX0ZYkW3kh{`>^k;OuU;yR>|P8sNne^x># zsvBH5n&1fYg`|o z#BRE>{=1$8df@u>&DrSY`fskk**zI<^+7Dyg zFqmciwkM`VhWK8IQ-;H7fr2;*!yyT=zbH~6-ho)YEUjn!BcmAkc(Sl-DVviO8;r)u zG^;QH>VcZx0h^H|48&N~8bj^rO4qgV*`$8zoI|^0uS;&Gn(f;Brmyj08$;001T(LT z;){BHqGU*p#AkhJEIbqRXPu83cqTMFhd=zu}MGy3SwR69}9aH29CZoVpkIZ<_`mzUXoC@VttTueZ1z@JF+G8bV zw!E)NJFiS1&lAQ)$IM}rNEpi15EvPiGj|Mq)NR*54%wY-opJ$sF$ui1t^1-Ss&QUS z@jc|oGZL_`;Gceg^Efn0;lP(s9khTJFBeuMvTfZ=1nRbSw;AGM{pax+@+cwPGwVi= zMBW5b#yy%8_4OJUr3?s^&@&6tC&eWes(FN4W}Tg>9<@T3Q@sg|5GxhXXADmx0MiAw z?D;B3WfxrW8KMuEp*^|k;RrzAM=Ucz#-UJ7#uz z6nIkZZ4){wrsN(6C`d;^xho<^QfuF;0h@DUvzoR|`?NPm+9Ua& zSDe<{IZ-kn zWw2%YLRQ$i_<_&!Y{j_1fz{pLNuush@PaQn@>fj9cw<>2l0)#cT@ zkvF)yxH!8>M_RrMdZ0rl6QN?u+0g?Bl?)VJpubDKVVscGbGc&bf_Dr}l2F+n!99$8 z>3YjbX`?<}S>z!1zWZX^JBas2#9EocJFGvc1V90!hl4P+3)BFfvFRcsfT4qG|il! zVrmb`RBEg;SSOhI)!<0N*)jm*spOt$h=AnQ6t$F|l=~#As912Kh`Ifn2R48lirSJv z`n!Up%M|LWHeBH70x<@sLX)d6l8O-BqfoARXON=I!!2#n5+>miR zmauSyK&)Qwm>nKGk^3tQ@!c|bH+YWzMIr>!FUGVL5XA^*9L{5=)_eh$+h>-fWKLaI z8l2~D&t|VOwwiF5JfAc5P+(&ZAW8{U7&5kKNLJI#9)`3Z3)75dh%_a0{%~1A2i%LI zC#S>f(?S2{^h6a;Xb8@%+-114AFZ;!PyYN zAfRHwOQ=*|W?`eqk)^u%%5o=H=n|nu<@6Ed>&SJM>IJ|_0e})4Ya5HoFfudFs2QPA)|uVTKo)luTTpFkT5}p7COFcyhTU??t5`a@b#hi=$w9xHt-0z)hDZ zD}aw5Dy2rX3S1My4b#xnY*rv)H>CqwvKtqKUePZqOXE=qig+mg3i&Ck5I+>x!c}$i z+)!lQLeA-uyihg!oKrkbII88j#xlX7FbP({^IBG|8Ltj8Ap0h_`MEy>a4OccJSXf< z@03GTsP$S|b)tHr7eI2n1VpWj#QXn{OoxsGv;f+lp*b?XSGjm*?+rxJS)(#Oa<$N4 zY7Aor0u)o^Lym&~_y76-7g)zbX4GgU=d3bdgi=Q^CwgfQZq7#lMLu1`x*c1hUaxE` zQR%icnv$9Ydu&ezN)PEDYQ&fyQuwVO@q?-3cCYhPb&q)@*{Xl=P%j~C_B_o@pp^5w z>vJ_^WmJWh4H*g{d+6`%pS}HkVT1j>Cz|)qeMSHKE;yT{j-p%af!FGe2ENQk)AB$Y7j64o|SNQkMtZ0!+b#gn=}0whHBYplmNw)GWm^ z5_VHCq$ngXILBiODMk#a$3hIma}cI4cI5W3$B+QTk$AWl3vG~oSyfbRF%Y%T}%`6!%FDFywSR))lAk>Tcw_6N|aMLnP5LPVw(dp1R$7j^2Qt&0_K~H3Y$YQ zyU5*56iz&brx6MijfiqQ5mQgLRq6n$!X`8|q9dut2RNPyef7%>NrAUcEGR|)j45QY z98gtJ2uuoBWwXi*C64lPOnTIDNY4*D6$el>~ip+(db-v z7z5)4py;;*DK5)UNoezL318J;yD!zeHAj3#0_IFzZfnXLOvALb;*05L>0osle~s>m z_9dNTmW3i^0Mco)BrIr1@f^}cE-olmPu+p8A9f4dUiwA0t+s=v=HVWgLngykLIeZx zx7+Qe`ct2SRz*vZIr^uJa2XEhmXa$GfTLek*eGORAm3lWcm;1G6Emrwb2_@+X=5++ zrfEq$eLIGOrsin3lWR`rwD8JpkOCP`z1bMbc4V!Plf$--wI~qU1HXGF>O%UMrOhYp zKALEe13MCL&{ zP3B^LOiwM!=lN5Erp@ZD3T?F`4i)7pvdRXTou8poR&vfH2R9@27Z?hoJ&Wtp(H)_g z4vlxnqJmQxF{oz2kW6j#dBDx2sO&5ZoL~u1$Of61hU%9Ff}t{S${Gl2#-d3CAO|wo z35*Fbhe(X&u55QzCH)P+d~qvq-l{iUHZ!?@_( z(X`ufj@oEM;Nz!OY{&*qZV0t(q#8Om#pA~B%kjN0AvVR_y7RP^t^c&47wiawL(E;+ zvc4IL80PpM*-?YLC2jLz1Re;z1281fw6bw>N4tU1WgN&I4y$awoOF#)<_1X`3ntd7 zACNB{i!)R%*=aOFi=k`WwiS5oD7|)gixFT((;}d4A9MWU8B6WdY*S*4OlW-+jMEol za#T(aeR4#B&O}qrX~3t1vGg#Ky{9YvCu#|jD>cK@8G!c?hdTL^hW$5^D9{m7Q*;_Q zTVnsDajQ@r?9#pj=~|c|D8nP9Br993j6xI0R61d5cBHF%Ye}q8Rr6N*YaMRN`PJd4 z++8BP7PtQWR+x)7GX&(9EHFVu*=n5HbLrq{16*!e-AEav%LA9E>Lt7iw<`cvBht}f z;jT0GC_Ihse}RP_X^W8{uokKw@BNp2BCc48k@4)45~~4 z>!WmWb~?6vm7>kkXCTD)C_;?w0g7ON#aEp~4pN@P`aIYsjvr*t4Nfmb>1lMHLbtzd z0kH_kvDCmBiXwS`sPht3Ec{bOw16*WL=3|d6vD*_`6LRMTpC~)fZ!}T9(W<2=G)%`DnnO)`7m9)Y?^>VK++ba^PA>>v8~FO6qbzTS{pGsIs=xXO$-R`ug>-LPif?EG4AM zRyucP{(*K-IS=J7UX}<&ozYho8Hut@vLZO`nW;j3?douiJEz%~LGA)Ec!KR{ zHj`(;Y%MCAR_@EUv}NtH!WoJCa~Q!Xl1Z8qd45*sbFl2(ig`m9l&6PPSb`PaVcr7h z`-BEMe3C>cO(CGF?ZDh2g@{CG4@58`dw|Vh818{N3UD&t10keSvkw2ILdx2oIUX3@aO`k`-GC_e68ojT89Tm z#}~N3J9|t8z~tTmI{D5f7J(*vl6-_d@bb|T%NfH?{8KPF3~jzmTQeA)H)$|~(G=+v z4MGUyhS-8}GzU|eSjm?qOrnOC(Ak3dJXB(JCm0i^r#Y5|pigK?z(t6te6JI4(&fL**ST&sYT(U3xh@F~#>d zL{k~u+<+x3yJTAs46dSZA*g$u4fp0&mDXDhx3oBhhVWD7Lv_hTGxn;Q%i;H7thQ}1 zV%>m}SVmj{G6D7*y|EYLrR1y?eay8nj5DpiCJErM@zlH=IQyW|y(KZgMLMw)eo_JN@(Xj@IiA zPfa<&+8*d!tFSJe{iobUlS>4WM0}*OPxgHocY8zbP^8aB>o-8Z0$jeskv((Ofhmhn zHRxxA@qUOQLry@2*`SI^z)a|sETn+6Sx@L}n#_w37j_|zru#4mJQQ(?kqp)m(-Pvo zClE020rQ#s>S)Y%%#F)XNZ!tYYMCkQDzP+24I67>#VB=zH1Yrku)*& zY&a;;A)al7hO^Vx1L@wmvaq3T}m&I-Fz%*Z|Fj4 zB8cKye?}%;#ed8o4c;&5BVhgv1xbjbsfX@Wu+cu0d648-j!1y^x23%=e;ax{7zG}T zI8L9~Y;xLP9pNe%%5h6>*|`3IEGsag2LzRsZX$stFG7#e4)Je+VDT_VZAB z)y+;o$@Dx;e7K_jv@kRp2DUzlRXH#0)hoh19TryAmxc6EGVlWPXR#*Ov;3FJX6mbmQujEIzYi>^v3<|HN| znJz}LSl$ds#5ff&MN8Bhpk4v{o%GL&2{CO2V5$^Uv4abeWTA6#PNJ#Vo$>;i;+l2A zX?ZegMdo!|ex$zPeZg1pk7wHCDVjV#qkMn6_{^ISIZ3jNZxuYL8|24{)A@9LFK@QcT$b%b~5Z1su^ij-;Hvq0mP|XOB7EYc4m*B7_2!-9|$} zU@CyfRPIX-t)eLHZ(GE9Wa5fR9o?0!P2vD@G~yI;G+h)75o21}3n``xOM{^eP)b6f z-rmWlP0)2=pkU;-6s;q6tMbw(5f?awU|4wfw-rK9w+?5Kh;D0omaTMcH%vKY>?f5P zS~~XC+2#v%{J13Xu=4(g+B*h=0PD+W@9m0h_Gs|HNj>+youLiafvcLqfvkXOteI@5A?L>6S%DQE92%jb z7fA12F^5U4Tbn6&@kmFVopY=Q4v(H6r+Up0Ud23N?hG;p1k-#-Ucp~ z0v7|~rN?9nEF33nF(x0Zt3SPA^Wf<$ z=Jr~o==p9!EPc_ZAEX@+f)f%w<8ogdQk?NoBHy(L&!hZh-9MX$65iMz5znLiw(HS(eSxRlDMJDwCB-KY{W2 zN!}_32r8-EIwuo#;gnc`!7#LGW7T9S@JkRUN7Y%RsQSCiy#G(BuuMLA^@b+;z=j~% z7F<+z;Lir?th}wuuPikj+Zr}~BYLf*XppYQ+;{4|^cHBZZ1KG+LrI8w0(`_#6s&~d z^g}G#CYM%KXryCA;3_Wwbdp7}usn;Z$`rg8`H$x|#NoUN1!yG$rz?xVqEZA-*OLHF zzV-i-{cq+Ai~DV*K=t-;sDXW)yIbX3( z*?Tr@fZFypL+4zoeJ^1ly{^V5opWk-mIq#A!jOAl*q;%?^j(Q##2*lf!2f=JaBy(o zJwJN&^3{JfkRFi$6_Nv|XmdU_1FOC7uf*Kb&-S+kkEQ!B&zh8`GWJxuzDxNVbH zJ4ssgEk24QPm$!|I5+JS+mfoSC4^)opj?e3phQ=zW6C(}=i z5$Fhw*AY6iGmtWPHcZFX*~EG%Z}DfyPh@TjJ%rNW^z3Rzs&Z|a&=u*nC&6YF#yzGy zfuV>_~U5!mONDlnZG{DyO@lN{YqofsoAN+Rg-y(Jj4?A~@hajxJ4{J;g3Gv}4}-(pl$TDu4N?!R+jYyL zaLQa*h%)KCsIn8S0#&x6RU{wBUxHC4jkVFL$ob*VgJm5`4qlV>D};DnnuKcj^FVKN zxf;T%S2G*hxMaZL5b~LHLd8%!X>_JjGk#?$6abiI!7S4lcDmf|eUr`v*P zmW)%9Su$4Ig{YKC7WHENqN6PhP<`75jzEC!?MxZ!AdRpOhd)a>HG6G&6H$#eML5uy%m-4oR7)()8=wh}P_PGVplZwSf7_4#)2gH0haIm|R;|<7 z0}nIo&&&n73v1L!(%? z%Y&~$;=4Or8Iqppd5EKRNj^a#;HvsvAIUIGV6`)smBWZx3Y=2^L!Tx}VrpE_GxWqUT1vaPIT%}LuT zsoGTLwiaYnt%ndgS>tUxH{;6{5)_@>sL$$ zx13x*MozeD1gj^8tA@EwcDQP&>!ygShWv-m6X)8HPoFHV1piZKj4L7G>C(pKjDx1! zaV7fVQM1R{>+UZke_U?T`J$7^mD8GLlB<6Gm8O%cXwbKua$i(VIj5J_Of1&~YXjNk znow*Y#attb)$`1?WYD){bISk1@&jRfhtd@Cfh zYr==kCvbwb}7pnOrtSo@>^9HA(WEMmJ>2%bM6O zS6&_L%7l64NdCwv^NJy?kCr#DWPf_8%z0I$ZY+JCGm6?K&~x?AI$88e(f?cI(Q|mX zZZf@cHh%f(^oruWPENgAS^jdf>bW-ZQ>WIe!Tv43-e<_K*S0@CUy{8#dHt4Y?~jye zts&E1lOTLcxA#X7f>!DFYDD1)a_+Uk{*@%&vsPm(q~3E4^Nr@;D+T=(CE+Wj|Lf1h zSK6#EARS*RT}y8mFP)XI6z#Xvd<_ixx6RMjC@$ZU^ld0fUzOY~o2jo>H`M0pE2ZFD z!oKwr_O)T`zgyP6Ql4xubzk`~0PU0aS-!G0()YPrVXbobWr)|$;#ZCSky80pBK+d> z`BhT>1t;~Zrg^E%emRNzmfP>?a{E>G;L|4fs}`h9` zDqFY7%zxLZviU>i{wwF<<`V#BI(wBAfXh?)W#j=|jqcBv4Dd(j-zBjf4kCB`WW=T_ zf#>=-r5x@Q-(whpkW8@;Ll7kM82FS#zml=xqDkA=O+o4Ci8*pM1|vOs7hsw|3L}mX zOJOAHSksikc?JhF5D^X_U=fVjjBp_E1Tt1c$lL>@!TuQfcjAjYaDI>I0*qxFC73nG zsA#G8D;cx>Ed|s~Zm1FE#*1i=!>=5lK!>7f5<(g-#N=BT7Mi{JajcQ)w`mMn>ftiI zl6Gwx%B8fd{?P-S{}Vv0aM!>W>#_e424OXPQ)J7m-^7m+lOhLM@I0YZNP=~mx*SEJ ztny;!pZ3(IYCYZw#32g)i;US@ER3MVIi6!)gS@f5H)cjU&uv`nYR_{2pnGA|#3mRq8DJMZXPkP`g(#d=3@inI{7#2#%nRY-)^j~DWuXOp+NLJ2gupF9| za~dp(X4$+3D>18>+dy}M-XAMZZcye5X-;rZOmR?~RwLU|sAvP=q2go)b^!H;4HeSWQsDcIT5VUeWPNQQ@7#%7)SeTmj3qq^lK-c-ijp`3`5{~ zeu!WMJP%Ji>F;8|Q@hmn7<#cVR^^5*6-vqTeoF{P9t=Yd2Vylo!H9|)5%FTk*aM*f z@MJ3VAHMrId~@>dVt9LUcKvBT!|8puBadD@`;R~W&HMA*`*Yy^`K|Zoi)Xvtcq+;? zFh{!r^QNbIT<-_WL9$(!d2rUd7Jkbd;I;<>b>C_$DR&5MGloZYE|AHn92vXq#N>=h zJc~x?G&72s_A0UiF6Z)FB|l*O(&S6d5m-fSnkBHVKtrBD$(5(f6qrg-b+*7ZJ+;w{ zfd;3Q4zQSvPfMtkB2atY?IeMzGhQbAJG7tV69wuQS*k#{?onCYG+ki2&s5xKFE}A& zx!eZuP({l)dlm18W&YLo`r?w}oh3i04YH6#=>2X+hP{6{{CNQ83FFo&3^0dLB2TgC z+@rZ2;W0^cn60&UU%m_7qSdCqvURv@Dpq@ycL>ED8QXeKZ~!QplY6}t&One*@g0I0 z3g_Avtiy>hiMrq%!Fvr0&11e$Ezsp81eH=%&JAMBIR0d-aW0L^w64XjufVQ#a(=IO zwcXy$@Aqzc$5-#0wb}8_cSY&uZHta@H*f8|+dPJno!auf+BV%^w^!TH@utDfrfr*+ zT9%JMows&s18!Pj$O1>?KUtVX7J8whMJ!xSB0p>o_zc*$&%>ybclZjMggUhOCyV`wyrtG%_2|nB-njV?uL~S+& zh)P<=IFcsOjyz0nMeKyc&raIz+LS#m^btaJMkGbjBJk1v6h%UtqrfWW!5Ht~A9jxp z_P3Skty$vTA9{1dq3UKaaRrD&95M@a`qkNUxAzD2j^q&0IaWto+`YmrQTi-*x-~1& z6R%a9Ef77RoSjlw$%DL7CknKeS<7&o-(}umyGop6f+H?|aRQ!YOKa`*d*}pjbXRZr0ATPGMO+o?&QcJWkVOVMw8}aaJbar#4QSTGxu^9@(WoR&9H5Kd* z$CX5IIBgACl-saRpxBzShk6yg&LD=nf zyI>e1$PkDK7c!5n5wfR_F7Q2NjhX!W5)OycR>a2XIr!EDiz;P_sq0XvE4$coIGC!G z9wwisO8_H@JrEH84$&?q`ni#g9xy94BV%eq4^DgF@W>0$7{karJo4zFy52-}LpoE% zR(^o9$tB@(;g-1@N)D*Q2*{(3Vi2b}26V&I)u&;EStz6n1hZn$a*&=g>u$!uA=;x8 zCBzipL+LtDOx!PB5pHh-tK*y#m-EH7uKE*Nr~;SJjgPadrlduFWg5` zLvZRQX(;oR-l$O5s&CgSR9vgzj=QSN?pc$i(Jd|`{TUeDc@I(pfSD_rpf2_TJ*CJdCenm9F1~HgzR_DiN@s#bml|8G$ z+L*SyO_FjeA9+oN@@&(qEp)vX>W(da zG$~y(o_>fC#VV|0cPpREMi8+kU!pI_1h@&+OHy;E0wzJE+p3kQN~EkiNy1PTcCsp_7hLBO(dFo_ z%~US~BAHA>uf0MA)~SLd(r5Xc_GT%Gwzt7Rot(6j{|J*tX8th^!z>RXAd*WFQimfG zhQLqXGteKNt(qf0-~Oe}i%LQ;#Frn< zSl4NobT0YPQ1C?9CF%n8G4y+u$d=@d7kE_m6@B92`p5nt%g2&tfX*(umhY^LVKBxz zcFfI_W~2-_n@Vn(G<%|Q?IAkOmzWv?%!3fKl*Nh^v8u>l8KuK0msvTeb*xMwnueI^ zXI96=EH?e}i_X9$Y#{Xn+Od^7FdzKVzrH-XeA6qwQ^EEzMZQ_@SqF^o54%T)!VDGT zm+pZig7*-I+Lt6>zerz;&W>4J>}z|%xj>v^pXH{Uxq9kqP5m%SJH{d901D%GE*hOM zc`%#uY~HQ$M;T~sy-=3k9cBnmHNrX6r$##$V=satYvY+78 w!%% zb{ypoi9V4-$)&||BeiOJl@F;1)i_MjyEQiLA1a7K<|sPo9=_-vIL@NuZ_c8paTetS z!Y85HWngmEN=caBr;@!Y@fumaicF1pVtgeL=8LrVZ%yxSPeM3lWlQ@EPdZyYz|cBt zlfBQUoK0IHXIrC^Pvj1;o$6DY8@l*Lnj@e`Q!hdfo(3yJ)@11P1W#oUnwQN-x~Vpu zX=~%@tvPpO&DU~Ys|Pxx)AQ584G6jclsQsCehkh*aPw>z+~g`-l-x4>FRR)vI6WVof&*}Rc_JtfZ*Rv7F}J?H zzPhLY0%QsoD!m<&c42Jd2<$vFw)DZXJ&^u56h92BpY3*RpNz{gkfm33bH)OYFqv1i zbKd{)^t_XQ8zAOW94p6L=h9kC7@9jSj;x`TnBMf?y*X{bG{XNXU=o~=kUQyW$dQjK zX;(SpqWhmAH5p#4Gy&iqQmmpSvKw{EClAN!01TmZ282a|my%GGco^7iFwacB+~J$s z)Bhe`UEiEu-=3U}`ahnZ-kzMk?!P;~xxF~Oxjq|=Qf5mJa1V6Es;=W?rZvD*%~#KZ z7I!#)#xodYihI*{Q#L~0tkBc2UhLD}4v$Gd&95%(is zJlTkU?K>x146kq3O@99CCI?1+>obW)kL6sLO*2S{?@@#p+mp@3kxCvNn7i>se~nSg z?bx>7T!Yg~QFKO>cL(!q~spt-;ZVTwKbN7k5Ph>5CZay?{qkgUhQs3-zGoy&jNEiTpOd0U>$tqy= z0rU*)DA8%%8^CS>_hFR+De1W$P@(#Znu=rxDQ^_xt zUH9$!!kfdkob-{nEJkl^Kkwv^%J0k@Rm62My^_w);wW39JKrPzMAz6?8< zmXlz+YBTIc>0J(73&~v$U`wf84roh>EdbpD&a<4fW>!$CxN6C4C$EPU@~Y2SwG~+9 zxj1)>1HA|4S`0E&*Lc90bErb4sj(a(4!8#A-?gjTHSV0oUk14gumStX9+MikGftPm zYy+#tYhRAUx^uG~Rj&+8u`HfT=Sg>04OXIb<9P#rJ^cPvD@V!~6MvY^B42dWVS&sv z*0A)=R~lhhY|FQ}!iu=U(F#{oq_dB6g+#qpNcRp~-6@e*1a;Wgy(wWc3AxS5Jx|_N zM;;;%#~xJK=Iv(IUAT~MSw6G5*)&tz9Qg#~Q?lpe9FIvL7(hu91s)~iELGvpTBn%I z=7vuj<%r$rHYv#Sjkm*f(wkliH%?>%#)%9c@&e3c+CRw%BaC_BnWi(R(+ zAcO0C3S-;fUo4_-DC&It2)c@?-TJZd>62De7U5U=!XGmfFGz&Bx{a-JMCvjP15=mE z2O#F~T$?FW8i%6IVpcK~>xAA#a&1=DX7a2cD&6mvxxLl5Tv%3t!X0hKq!&(IBNYdc zf^ae$rzo1onxRVO9lM7iI5S@$*XM!_N3v`Blz;@~8}`=p=M#zuv-h^lB~NB5lov#t z5I>QjpX!$D7shi~x*}iQ$--s3P zXYPWM2lZ5_V!}`W7|Bakc4>oB8Ylqcg(dAad6lVFs(5aWrdopsM~uVBN9lP>JC{oh zt8)evrwF+QR7odnJA0|FE&(laEmt&)B1y7L+jp(LD>pVv?}n7&htlw2CTD`tD>iNm zQpsH(73kIDYm0~r`$-E@a5P0M&xZoo_7u%XM5xZzZ33L^XI7Rrmy$p6G1q43;bw+2q?!%U>rUGJ8`h zKY}VK=xRzMz6TiDQ#Y+E(|J!FXqVN*SPoE(e_@Q6Fnlt**A!QrK9> zl4xu)1nB$R?Z#D2yKbrcv#Ppmn#brWljg2$X#b5t#*rE_F%R7>aRQt{F` zgsudrbuPR^fJqmfI`Gda#5&1?a|fnMM9i%#-^^gk>;w~s4bn*L5D7!_Z8AkVdALA+ z4yRcm0MLh+_CP$nrCE{N5Kcd8_&*3cG^RM>6VUlHGofM~u!AE%OacTtrDqK~pt~z9 z(@$U@Jb!s`0RGd>12hrS!{OY16qenk$xYCnDi zjnsorpPUF<20}QU3cGc@5C8;UzEdjX)2EIoJtGO-VOc&Y-GF31hho+2+&q%evpitE zVQpgF|MczirGB#Ey02VrGNJk$c7J6g+7OO|gM)*Y$H(&T!NEc9@8Q9#=gLnk z(b0?JgIC9YIyiiBc=+;9;9x`eYd;C&kpAglZCv@reI=idTL5&vQ+?j)fsQaBdi(pL z_dP8MgiiO>*X_=pD9#r#5@nTlLOsiF8SC`GM>+mGb`G-isa2nGileElubyKb$`_U% za$uw-Cxp&BYDY1BW~oyFqn~pmzqgX03DfDlrQzPI$^6rvraR%YGJ37{y7cxFK0^^V z`qEi-)DvC8Hu+o`K$;ISHpqEz5j7?xL@-0+Ql_^v|Isny{2v_vgE@|b%|YEtc?Qa~ zk^q7<>!+USD}Z&?`OpW<(4NfD#*w+vJxv)G68}!S)aB`oA`upo(#M~&5-IATCaQ`p zIoZm|muZo)4_D#OiDf!MKw%V+IXFD<#99^lV%-D7cm|_njwtqlA_(C0$mDYEP9vvR zYHv24bd-JyuC9f4#D(Q%PufolI3G%73|{%%8(VjI4##YHkm zuHxk7Vv#;>X8?68Tje~T;UGX!t^oja`!=UfSLNlS zNzP~B@s|D9m=)H&ZEdWJD${`Bf2}CwgtWd& z_@N_Xn@irki!c{(-{4LI|I%IDrEIfnbKLwotzKm^W?{*N@QT%}xU_afcjbt#gC*ao z-R;8jdI6~G*XF);nR%fC0d)G(VnIg`4ys;DX*fDK{=TAs5s3@jaSD9O4M;yuJ|iy% z!*?5E#6eX7!-#o~4@)Q=O9i(XHf7Rn4W&$-pQxOg&FV_Ka@ShEScsRk53*Yct0=$` zN7KAUODT}IIi?`K^eLv0yJ($>W|f|+;z_O68Z}#M7!-1*x`{?j*552gs<0rg z^Z8MmW*6f%eqx4Y>Li_f(%=mpI@Chs-4<<$|39_!x?29fEDG)N>gIIRaUGwSFmU{d1>GkOB>e4Rovlx3M z5@q_sVOfcO99xC9{_+=-yJIXbnU^tK4>PRq2~~ObF#}^Htoi`OAz7dR{N*oOTOU7q z!pt|3MgfM?h%k=*3le0zjh{Yk-K2oTB72NblmWsK$SQ)ls0sREsLzOj8N5fjhDw7U z$QVhJr%HTJVUl}9A1LM=MeLM%D)?_f67T@kG!US1 zGIdC^DH9J{QpGX&-(r-2tJmP>?I{>uo%|>8Jg`qkh>&K#Z^6Sx1Za&LyJtZ zVZvt`>Q89Fy3)04DEJ92gsmaH4nM0AvBL%7bm0OhknX-UPr(lQ&0$c=mCo+2JD@zn z6#>B{5$OFOb3%dKmoYNP{w6DyYQ}v+i|XXD_(nYCnOBXgP6rmE>#v81}2ouWlL}* z)Dtz~(0cGzqlqb=L%PWA`gg1JRF~cRCNR9KE?!Vqmh8(rO9Wpdm0K;HR?%;n7+9+g zcjYJv>bW_{K`lq){;MTK6V4h zVfW+5irRUN*a6*7pFVZK@A8WIm&d>hb{+0lvg6}Nhv5Q9HDA79j)=*t6+`sAxCv4E z_%S`HW{K6;#ZF-6&TJ3Lnz0+DZWpXG-GEx5&B?D;EW+}V$~fKBq)A?dJC|9s+@N<-6-Fi9`sOj|3hc^ zCKH^3EOo#?v|GXRgq>AT9AVq0kr0AIaEIVdaCdiicM0w~KyV1|?(XjHZiBnKyUp@_ z`|rutR_z@2VRv=c`#jfm-!maKa3?U~3+I0{`J+6YVEl+~yA#qz1mE4?&=u<4*-53h zl6-*9;@=vJ!f#Z>?Lq0u=Te0N^4lc}nCO{dea$V=6{1gh`~T|L=BTZqfh%oH+VQ1- z1&t7zp$|3f&*2?4hiPZIXz1a&!L;FoT{d!wz!a%%5FfRd=hdx7Wz7v$0_sr>@CTVb z=6;jU%5jo}U{U#qC0V~u znwTzyt2k<|eCp(Ow)(RzvW5#+H0lZJoF(k0Q zX0rWI4Jh7-*y^b<81GoK(PBAIEvJ*4W^H!WF!iLls{~6QuWLZ@3IdNj+edrDg?u=5%C6K&n)b}znnT?^!MoBQ18JGIixtonEKZI zdRG*nc?w}V+E4WQ8Di3nog4=!F)7^-mouMiHms1O$Df$R@gwD`0erX#7J-YTwEW#3 zX89tk7rOvz_7Ycnv>c=!i@2Fx3rukcdPwMYCFBO4<1UQxvnjJ-n;}e+ZRu8~uE*X^ z+q-T91V%>a6YZ&VGlkGY4^$z-oR049!4ZR}&(FDhV{E>sq|*dY<0!bJ`+b?$4DT_4 z?@nIVV;m_zOGPli6oH!dz~+qifqZKs=7LvlgO>c;#(p3%O6bgAt`xje+F}lfE=I?L zgT~)qc{b9Cf95LMDM&G?+v;q&7Z%TaScfc zaeh9jTuPnoIw5EZ`4G)ASCQMZq!MMj^@imLB+#w5jP*=in>Ur)|Ngm2(~ifSlvBkv zIL^II5X|9+M>YAYt>=&G{!%ScjM!}&g;XE1_&hGhBuku@f0at(F1g&y4;qB4`lf&T zwe5(@vABtrq0q+`2f$I$ABj*i`>wWpjgjTgT6~2!gw4!ou(_wV9P40Fug}p(!_|mL zjyHr1NQhmTAkVnz;pjKO3Qml>;EnT8?=&V-OdhX2#>+Pjf%|o(JBUwEN(?0r$H&xj>K$!GT4B=rw>nHq!fZ87#UJTnq9YaCd z9y=I#sAsSq4z=PNxD&sF94KnM2%C=puwqHlr%B?k7L4(2r0+u`Auz&zv7B{R6iST0 zVIf^q|H+UC-Pe*&^h~)E0}D2rP}5Xtr`_7iVujvHoIWOtVv;Z58rJmhDpIGo#@>&T zic{{Aa{ioNh4Tjd-lfTkjvM_30K;b>QIK0;yEStwG?(psxw3w&Q5ecvf195$)Ymgl zzH-75tC)3rlxYxwPIB0s@_v2=W$9pCeqSJ=%EqmT9p}1b8g5vG4BU9r{ac@4USH!m zU1HK(`{qtdjWSrgn?z@5?NDJP?TcQeOOmC~$XS%9AEE9Bb1W-6&y55tF1R$(Jb^Z+ zmhtW)`(s)=86O>Uespm3Jm3J=;Fs+h`#|fN^~HD2nMGTTMc5i;-@88t8xQTW7gaS= zONRo9z;1d5xqWo2Z5asWWRH{fQI)u4wO`h{_KOQD=vF5xHR+jEVmgNO$${T5(UQp< zX(i2dG<>V-!%#IB@1`|iUXmx;nN5XM zrHL8$tE;L5ViQl}lCs2kcVU6S3aBF*jY9Q^Qs5+i>>RU@HAy)_s?DHtm^((9dS8gA zkV2;UJhj7yP+@&%ltU_9<%wY!;(+~T7qpPG%#ymIhx-OgOIB>!a*td<3z!n4j0geSD+1$?ZD0gDC6FT1@{2 zJAyZqxCyR2R}$)9I%}(|r`x%4x;{zWy&pUXTkkOzGX~RX1rYxpa;N#3l-`6->|eZo zg4WPcxV}Ec{4)lCE58-B(?efZ?(Af|v_K^~uS^p`_u2zonf*6X?a!v66(xVisvdvz zD<#R%h7w=`&Ts;gYAP;YhI@)Az5%=+{r;@bI{oMj*^yyB{_j3u9Zk$zarYdjDBnul z8GQ7W#0&D>dEZX9h$~mydt^x(l{qoB$pSmX3-? zwOXcsazmL`m3U;-bm0(dm`u{b0fZFe<>@LX!T!lKy4-I%)?_tsOV0T4)2yoZgD3DS$n=t4)x$&UPX=O`~>2*9tTrOx6Cq`GF+#5pGijnt?KVVk!3`}Gml&}DONT}(*twk>*OOll<`RpC*|oRPaHbY=lX zqV0F^-G8@!_Qmoo&Ae#3(E5%ECaAgBYELULuq*vu7mayTeq zqkz93rLr?4v<$s)s%c&GsuZoF^TIVW;uZcDL6}LE8NG-d;gS-EUu8dqpPE0>%2+`! z`r9xCwN0Voo%@>H)|ZNVz?lCbArlR6|gvf(Q-Ie;X+Kb&- zt*O*f2ITD6o>Wq{07^3lR)u(9Dz9ZKM*+0=#O)u>pC~RBd0B~76m(|?0azfEk9B^4 z8y5us)AY35E4#URawkB<$V{&Mou0caNDZXfEVK1$Sh$jCF7Wy6vZ>Jy(;h=SI^%blIxN1vZuK+K!6_MykG$snu1eCTNBweB|_ zf{pU&7e47>I+-+<8;KJl64wA~r45>i?$c(RV79%K4on|z;Fn%zi#MpmL;}ijc z$BQ*)P8E|`L;V))4~%E06jI^(HU|GvrYSSi^9TjM6|Ap;tPtk(2G{>SC*#-F&q0;6aw|je`-YS!+rblVnc{ z<8e#yGPuuqd4G+LOSVO#GUL;D){tx5aK-G`|ZbD55Mx1Qim@qN?ZrrKBZ2Q zI@(ErK2aq+-=Hc;&t=|Srywn-QfrIB%`b7mRkmtup$%R9AEApd1(Wd(Rj=AkKT%S% zmFM$bMg1=dDr<#01SEGo6yi!YDDF~3D0grB4kHWWBo|djqI;-t+s3Q99*PpoheQqk zDzrXSj;F5`Eknyi`9Pcb`~r-34jS#l8JhhkTv75(aa2e2Ojfd35Z!c8=tG}$&UCOE zL>!Cp7>oloDzA(g=8~h6^cIn;^8oCnK~rNSM-HKUW?(Y2`@I)PKXrq5$0cyUipG*O z^8PQLROh>qOC?)H;tQ7p3$GyK4~Iap5emhe!qQz+I#!o!yibtS5W6RY`ij**_{2cF zBd_ET9NYh+%xLP7fl#*N3~wnjrT+zvlZ*V4q0pT5Vb0?Anzlh|a4i>eUw$#)x_5p;lqKoLfKD$;ZuKnOifOo`b}H&fsHF z+>u&{H_kOvFhE1`%u06WeH0<@nXro^JGRN5zWE|@TPScjlZPk&b~e_ zF3cQsJA(QBMV=kwye4bBny?*;mKp?I@CI<=rc`y*jY_vhBu|0p6(_4+eTdO_A|>8G zG4sYXeJQKw*5bUN77Km>A>T~NaKuY%J|JYyz^ciXjlvjh;lwd3i_~m;JrV9<4aZEiV!`fVIPwe;AHvavb3Q-5bllIzu{0$w!Ah2z%cdbS>C^e- zYE6%BZS(vIo|DLlGKrvGI;N^17KL}136luqLfAjxrtsPLN0!h)vCo=Mr&!%%>w_tJ zx4KEyG7(2aE=WT)HC`j!-gVI!)$^05O$}~MpF-mT2xIhbie)UI4|>}5`p7w8fs7($ zb$;WOq_3oorH}nfViq-Qy`5B=ETh>c3(aFaju=iJ!mEgEvJusv9ji>+CiK6Tt=Ol2 zEwkJlG1<;XO>Nc$0X9D6y|1b}pYDo>KC=HFg@}i|yjM@_cRn|P?$ELku0YFcW39YI z=JKtRBkF5akB{&-E>>|v(*7K475?R{tbV8S46AnG z?t8xJFlY}Tb-{n{uY3$+dXM(F)jL~`>G5{;{QScHc16XP-T>0kr3yO!Hb@*BzH!%4 zSJC>9etHbAdCPf{oUd?CA9wier1$V)L7@McpNqpYF&gCcaC*G<3<#eNUINI9XTous zCb$(l$0ldB(d#eUP!dP8q$Fvd+8EfVw#8^WCw{#_9GIr+1Eic~_w!bdfF`spB6}x^;lL=`7 zku6d{NYQsUAn_1}u1S#m-3#p^=g+CAtldD`g}^wK-K!~Ow6%NAb;(M%VrW)al<+FB$h4`Ow6@ZnYN zRWIII3I|lg&{`jj7i}LSRrV~sFPs8=a97d}0r{W&)T^N3>Fy2C8)>%q0_c#Q8x<@L zP}Q&*)aB*iBbu#u^&vY9z~=IWJ8plFS(11l*35uwqr-%WIN4fwBJt5W0W|3m8O(o9 zkN0XbRZ)?+AE z#nsc1UAdjutlWGl#KsJ$l-M}(l+~iSRbi>-MQLTj#F@YN5*nQh_{+Sy1MdEb4istY zP|ZBf)>J(5v=S^ijeMkH51_G%XpgVzMv3qt;Ccz^9bq$3qyTai)k{AsBe9!q7`a}s1Xt4 z$JxTW1#5;OR$RzD((UGif~qPh@ykZebBe23lHC?{4Z?!^_JAd6QVQsLd)gSG2yPRRu0XTu$3c*fccS#XR*(HBue|o z{p}~iQVL0pC^g7U&fo*OkGt+z1-q#McK1U(aAs|C5p?}+B-MyXMAL#Q4mClwHO>snudYEJ%C#XhSxSXr`|wdD}gLk_BW03O+}9sw73H$ zM!MlLVi;;F?aJY$ePw*X=-#sR{<&FQANXo9Su5e~+QwSJ`be(N^N4-^>tffB>J5IU zpN4YBV~Bpi51<=|UJ$_j`V`pw@Kt@txA$3{R0T(;i>?i&^=!{_@f!H+*TvGu`}ctF zw^(PsPH-BdQ-Z=@4K|}_3jXhBlUNGN=g+4)Xg~trdG0UW*CY+X*cbS+J#Ni4&+Ak{ zNM&}uxs_2}5b*1-D~#sa`7V+5ep^+X%)J8Iz&G)A&4)zWtcEjtJO`1h3zr0sxuW@q z#_}W=S)T)CtbAM^anc=4(<2Am0P5`u33SoA<^x`9OVzWTwEi}z#qiGFWNUU=-Y$A; zgB{==OT9#IKC7$PFQGdc;xj|sQR<+b8Rc5)q%@(aNwe4^R=a1=8Ux+w1ptvVjg(l} znk=BC5rl*k9i`tjaRtIB^d1LYlV;1Vf~@tg<{D1S$Bhlg<*T8cV^MaaYq8vSX$^wQ zs+Uq2&e@y7b(k)E<3bCWO!PGi;#P7_{e@c*W0msr)my8r^= zN8b-K!_WCMb+NaZa!u%iAL);Ogq`|rO=^JfYp{7dZ{o=AR`J8haJ4Eq;S61 zuFf}Nn-)}UIuCPEvub%4*lw~>Il0?}1ZS+UTy^Q}s$WT5zhN{)tpHx6*7k=&{?1zG zeo^#`mh3N=YMc9H=--Fai`F4tQWnHX1u7*mgVHUU!KwS&^N5Z|%15;mx`ZPp1Z))b z&?~JB*K(D7w^k1?wP*(GTVtmrLoIy88VGEI-EqgsEu$qRiXs$cP|#fA_t17jx#>zN z>q=c!6%`yze1p`6&`>=J#eQI;-WjRoUza&!l!MPCeRqPPb}8qMmUd`Ihg0ON^uqa1 zL=~*_KM_@t+KxY@?e)r!uDZ2KuUZhc8|rknS{`xf0c9{+rtdH$!r7FR2YC;hmCkH< zDry|Kh_LTy<>T*AuZ2r^C2K~(c~AR>H4V#`RdBg=M9Z~!W>oEU4I4qnux#n&+$gjv zrXI}SPsoxUlL|gMYPfXkM+cXO2(BeNu5XvtSoC}}2NZp=%{Ek1&>3+UhAXg?(3@p} zTdb-T>-MOYH{qoap=s@fY)F^_Z1OQxB zJ*n*JWvrePrf2-`-?ZLwVK#w;6w&DR>ep~+8 z_S;Z7kX$|Kc%2NokeZd2EwRn-!P-G>-y9<_d|nr5NJ=l823*z@T;I;#>3h)JZNe?h_4T?|GwB} zGHVx#W|bU|dL;+Bu$*X+ax=8k2K-J(*>oTX3A_QN#E`;rZM$7ZPnV;BJ%gPjfJdLA zoa(ZlhAbL)!9T}a`ZeXZJq8sn!LdmIf815|maA9y{O_u!u3v4dXO&A#LHy$x^FkF# zcA^_HcL{k-5C=e5&jVH|vU7^_>p4-H)IBZ+nTJ(o(W7%C_SLFMv`PUPZHTdI(@z1B`ip0awkEAxrq>xJ?M*v@QJ`rg<|( zF@Kj={QP4aR^mL??2oyiDJTf8ugc5?hmIPlt#1|?GPti!&D89Hxt8v%Z5~cOT(8|n z&Hd$P>1*R|nEY9K>OYgAxYP@aETgZE%J1ATi<>FTCy!6*vS1VF&a^T!{ntD7$=@^@EBKKkG-Mx`!bqX@dUW%veOI(`iO= zK6y$A{50SN8f4$Tf8|ZyXlz^)=+)WXm+g2fLsxfGPbX95j<_*Q^T${e%1!C(T3_ag}!5x zp-3+KO&q2CMNE}}ZD)n5iu!crbocUc@p!Ox=KgQhU@4*Mg?%%=Q1TT$9;`EAbdnGu z*`1=8|DwttNpy808~aaNXD3geH9|x?hBwS{hi^kCZtMgdNx?SuH%)+#qSRW`>*PHX-rkjzXQ*JK*|oT5{nw*$+Sl?Ln>s4VFj)<$PYZh= zwhqPdShc(bh*_(NB~NpzvNLrX)(-ZOq()XTD?Z!#D${=<5nXum{~5q5(~a$qx9ni{ zY;I`x@XH0*NgI?h)f}(i3;6%s7G&x@0(kg4>Cos{b1_0)1So!`-k^PFw5>~V%sf<~ zVz7FlF+B+ket~WYN>*SwecX8PFzxq-5FV&t39qbeWL1Yp|Jo{KYu|T?3?p~isZo8e zL#}eQ-^0N!EAYC0YzQh+a(-i0;Tr|gdYZ^uLpM(D4kNoc>HUs%LDc4qq1N>Dd42r~ zg)t@O!F<-RXm54e{_`h`M`sLe+_@M%h|6QZ2&PCLG#6aJxmr0Gg>`p8L%ZIk{E;UG zl{@urvE-7wTbb(W>UIVwTQ@I z$==`eL!!2%os&@VpPOf;dFfj3%Js#`!`J|9V(78Nlo2G}G;*a5w@M5%M`u^5o7{r< zWMX7ItPFqVrwzUckmgCp?K$*T5X6e8s8WYqVI&MNU42FWV?j8(KX0$;SUs&?^z`a& zQ+&pDOaXrQ&-38)AIrOTWlxUZGQy}z%9{&c8?4CR^6~Nc-pLi9>0KQ6AZyJfNE2~m zbj5-!@~c#GQUcCbLiY_%+@fS6$JHb&TQ{qTwQj7YqR~?xd~=)g6*Fx0_VMt1_a0D1 zpB$5l*8BbVU$Iil^k{=D%TW4fdwe~vpsTbM-SyeaxXyX=-%nWpI`#LM{+hHD8~ee* zhB=Y3?f3Kh+r!7*^V`Q=XO`1$P8)8@g-R|=jy8eFUMej!(x}bHYI*rUzij;ndE4n} z-;PWD>_&mi(!p7HB{X;{SQz-Jn-rn&a?6tI@RIi8{|L@>9MJ427yCVS*bgd%25OF4 z3t~(yB|epPTIdjE;bImI)Bn75JtmBxIaGB_4SKaiQ4C0tM4*g+)BBdN{i9=Az#d%_DZ6=2i9PCX!mdA59yWtF+GZ`R zNFAokkKbP4_(kksT8%xJ1-sWK45o6mjuckRc~dvPI|ETnf6A6A^l>Cv-Vj3C$x1`5 zid)%86ulU)BWm4rfwivM73Ei_$b@j zFH E*5LoGc%dF%b{Xp0#klrfi-BO9{4`CUkm*#kbq4vX)bWBtccVm71$1eDD$IM ziIupX-#@mR@=`5M*J@}`*=D`7l11`4-7sA2%QZkv?5i>D+9jlScucgj1fHmydTGDv z{HZldPN=l=evvFn6zdbgrS^!h7wp@SnF}kc(n0};g&`?xUu98?f9)O=tB!4M112l+ ze=iP8crIkHc*-#501SY^Yel>we~&jtrX7NF-adoww?Ia}(nipmvjAROsYh5mYZJE8 zQOaV(sa6w?Yw_@D;!Nag);SVt4rM%4-^vt{)f4&iWlv%1{Q5`Q$QI2JN3+6mE1~3$ zKoUJ+y~>W|VPX?tuK}mcB_k!;J_n-49uP3Uri=h8Hrpns^Vjb;M%FESnn$h_pc5-b zLR+H|R!WdJ%FS@t>rP<7hG2EYGrxm%No(vsQyg`jx{i?v;Iej)FVhy>x_#_4xWDOx z=CZZdUAa*KB3%!E<>W(i;uGTH+9hV*Uu3~?iFsHy+GGq zpZuIiLkq+FFM;E>?1XYmQc_@Bj0 zzZZR7pzy9;WFimr_0`EWklLrn39q;=5uC^a9ZT(+>jz|j_jt{DnQ$vM=5XvyYf~yW z$F6JP3~W3_sQ`@RwxXYeUdJd9=*8#^7r-K<$dGM9m6xY zW{%X$J+$+^p@%4`SIAV6KK8VBaWr?5&iU!t3Q!QIXqwinDwB7-#p`&_g}67wj9WT< z)+U=ew^Pv5+<@*WI+dIMXg3l6lILk+=FG8qA1!8kG^vLCc!{V_`ypd3-7>k6qun~y z_Akd&N2{}ARJAK*!k>v@I&XGNGbpW`Z!%qBlxpjVhf2q1IX#Cl^e@qZaL3l4G$76d zP2>R;|2`Mv5MS8vuzk>Q0#J-@tZQ_X@HuXTPErs?mT+G#0S|U+K3pnzm5jmoggcB2 znTUx#h|X?FejIiKMwLD~mjY0x8(xNct-IK(Qdn4jS>W{klX_K81LAgg=41Sn+oY>OL*L!x&5~rgKSN)U5cAg0y5EYfLUZ;VVnsxz{N8daCqZ-;p~Q84AcCOfqoU4m~QeM8Y?2W(qdmWDILdtNn1PVlue>W z6TN6}iPj8-IvabN~*g{1S7J;|AEwN1J!mZCl_;r36hG#*H$ zzAE}R?rz_vI(-E)vffW~gW9B(#Z0#IH9DIPI?$A0td}*QH;$TNn`D?sEKs{2ob=oN zo5%+pWhlz`{V*&jseNGcC43p)GhNACnuvDi0|K^`R*MA2y>!&(aOhab)O(% zAgESJj*WjJ778I7%G*M66cXPX6HD)(lHk2YZW;m^<%Y2?A9VHvps$k?=A&dwW06t# z`FQYt{tkcp`OzPVf58uP^!qU6>ht~c;|!nf?ZG0C2qzZ;2@b~~_?zuuZrk)~Uv9On z)isNqOrCJKWMK3TOK5A)d0ssUd-CCxaZ6F|`6k>mO)xYXtbiaG%0ugU=C^X1n4Z5J z$411ngt=Z&zYHA}Sk^~53<$p`l&+dcMqSnfihRlr{$M^fagzm7T=OF4Mnfl*a#1i~ zt9{QO1-Jk80REC+CJ*=;I{rFV$9ww)a39!*bi|fsYxuhRoR8Kj0viva>H^f2`V$WB zm2eBlq&l>JqVNun(w0N9{>l(5Kyw1qCDEu%i4E;kefzxc`5y=b0(-BO-eJq0Y5-m;zg)|YKo zM5z>7O=cwW2KhDzYBi4#Z}#D26>c3g#?~={itKKK)-4^06O~Th+LO^<`18Sn2+YQG zz4ZL8PmQU<-4N)XBW`~q*<$0shpIbjmuSdahVksFW8*JSjTNPPSMlC-@Xs z`pWX~%GW*RVCd$}?Yi273dWnt1k&w7jXDNn=XVl^lz=2?BVz~-S z;O4ROz6YS?GW%!&~a1QtSKCEcKKRY8nk_tCMB~TxyqX+H z5=2q*Hw|5fxGtI8ldzSO?do-Q>d9{!W}JhVUj+ZrB6>p;N%GDn&;iYGjS-F`L2&VJ!)j9wh$r_u*c zR32Eg1Es=sc_BfHdbJV#!dcsj3}NIYX;qx@PlEpPg*%(bF1-FXx2w&LHE=9@)vdBJ zX?mGW1?q*M0vJs941t>p>_5zjV}McQkf3yD&jHU(IJd32N&sLex$c7CX2i+xL)2O z(|A1A3T#zDy~bL8zIJz@9(UYz6GrQyUjVN3I=%G1o@AE&hrVl2^#l3G(4rf=wiZqG z3pZzakswy~+)Be`KJLjMEB!BVBY?)WV-BW@R=~VRmre=Av|SvvhmWCf=nR+ZFT!-^MJTLeeOFzUO&HX zoZHWvusGXF%T^C74m&o9h(*Y%i(yXH+=2Dvdfr2tDJdKpgLrFdbS{xpSVdcZnXMJN z-~kQ!L?cKf6x7&`U?Z|TU-np9?p31X@E8hYR<$VUWWgn-ZP%ZsY-u`d=d0^wu2?z@mC2V-gWx!;1qQP*TPl5%D>f`{8Vm2R_-!YNpiD0Z_$W^ zJ)Pag;>&W;(eLEF*zb}vo-KAb{wv;(?3E@D)mY^PWGU9n(hLWVIYT(Am!m2t)8iP{=x zVqi&GYZyXTDxP{{XB6DKxF{x5(K0N!uw=B2&PKvqXI8&V!ex|5TmL?l-vx z)R&;*2|sfv`cT!q%wk8(YSkZ4yE@vOR9v|MHAkusEi|*jwD)_&2mgFf;@+IkS%f>bjxH>r0_n(#w2)3dU{M0T^@3UiXB5&ok09}l?F-#$kQqN@{MmMnL3Yt1w8;w z5f3NX@!=@_=5Q@fOhfN*ar5n6}eJlcp5+APnB7>-`{lRaQQoBX>Wrk1txkR-Y4Gd(}wgpG%#jfgQjU)%8K? zvU{=R_Pv`HcbbLulsWpAf%b8}wqhQ~op#qa%aRY~-Il)tKV0X*`qABF+@2EUC;Vn> zGnU?JXkIMTq5M=(ePF!<^}4;L_Z%u~^L$-OAzTt&!}{1Uq<^YE_f9P+FY%*3OmvUJ-#*UgtsvUK61Zu3tgCGB} zaw9`WE&Vu>jo4_|?n<;M7Z>JASl4k4`4Jh%mtx;q+t6R;n7bG!pTX*X!=OihjKzd- zr=XzwhX;2}!nstdHZRmZnvtYJjc@^Jna;3A?ido7D|9|%iB2_aj8d8uoW}a`yiVk! zDzyUtItgZ$2{h`BC9A|{pGxUE%b7DKd2z3G_R5uZntKVI9;@MV=AAOBzpj4zb_I`N zs@`8X-doLmn>JD2tX`Ip>UBxu@NrygvHNEN-KLEMwnppA7!bUvTB46oc*NeBpqhTQkKu_ z4PqH+Z=9r-d%btyR&r^fGzgl$oIVqs1t}DMyR&>&Dv2x!G!#$+lMc1 zj*`=Izs!4f!5AugRJB^54;>56o0)DeYTB-m?-S;ueqZg=+ex(Q9zV;~-JUuY~jxvL#P-VW)S)^hjBKVoI1lXx*DKJGh-EjHp8{K`6 zjnzcUQW;37JXMNnSg>e@OqFw=jirIa0GVZ5R{=2 zTIMJ$Z-wZh!L@O}aoULx3LT>mYB^kKd_1YA1Yp8hywt$8&){@is@fG-&=*<@dlcV_ z>%Cm5t6t4C13+&rbq@7!@TfUxpC4C=&ZQ$VaYJr=hBM(ffNg~;$-6(KWdih+>nDk| zVJx5=@qR%U@dk}|#fZUwm!p1xzI$A`tVw4Qq+-{wTE_QeFKcbaO%hvLbq3O~q=f|ly-Cn?+1D0ciBT`e>+fv(8II0m$69GYq zeJ;C8d^IM0kyT2p4_fTkd(r%TxzSfhFBTo!N9XuA=OIb6>D}Qajs`V{oh9+I`XMJ6 z$HEUdUDu06b(a))uX6|t`0;J|nWuEtudNxgzEzvt-J)Q~Rr;+c14+fXDYst+*n2~bZw9-i z5!MfHLJ`uM11CC?o}t3RpRDnTwwun=M+>%#$^uV?InbWxJD3)O$3|=FOXoeZuu?*E zNLdU)qq}vAtM{5pK>*&Gt3mZ{6;eliu4+nSdvy+-}#W>Io`(!oEWjxK_VCAT{RoKiElU6)C_@U9!B4T$^qDqI) zCk~UsIWLK(2r2JWUvoY7K3kpUEFK9wc4UMg)GZ~rhSB)nyU0}-Z-+@` zaEhC43k4D`{?MfBF7E0(_Qw32lYNKFTHo$HpMRl$XnT*WuaI?fS5g;-0gKY%KuU@q zA4dXwc>4ldKeXwXm~Uc|p3S(>2-{iV5_qZBlLB4L_fm`>>6~uYqG zf;AmioIYQm)=e;t&u%(6uXOY>vexyyTRy!Xb}9l;T|N9R;UY|JR2yjwqyARXbP!Hv zu8ZpuT&VGF;MI7xmZd5z#>%F_C&q9vy}%O{Clj_eAF*f4b7f9{JscopTIsqa+$yEi z*PjqUse7NFiujy0BWnPK7|I+|*CjV3TkflK92at0(i!p3J)^Xbbz9iAx zyMvdb-Sk4QL72n#8r->!xxezwsh9RDR*jdBb#JXsPM_m1lSh{?lf`%G)5VyXTT^xz zwtoh$!RqPFo>DU7ECR9ZYEly>IeEKVEA>rIC%F-6jk@lW@4D&ka!;`}un>`$9Xx*} zug@5^kuqk`t~bTQVy#cF@?x;qTuvVJU39kUXtj(ZG|LZkBN;b-?qct3zDG|TqZZwh!8nBMfWPc6 zPmD*LIW_LZ7W7A#wk%UpwfBcnA*hsmmW1`-kr`OwR`)@%7WmnEYEA>pC3XT-q4zBm*^D zh8#H~-K0LjSf#eY8?z-*83daEp+90P`YV!Nmn2aXmk(x9)ZG?u(#7g`TeHoR=1~vk ziQek)^3`UXM3Wi%ZJa->?S{X)3z|e&PCNh9d@V9pR&U2>pj{4!h3_GU9f=ahr9!R`pNiVxZ5bm51b| zLntSMCOzca?AX%ydx(0Y!QrY-mr+%YHmkH*%+=yuT55L9bi^SEUD~wu;e2K4*E*}f zXx60FoJOA&E?W;+mulzo($*TD1*FZWn5k#g=G30)I~ccgG3l+~F8(@_*jra`ayuH6 zVEA-pweLTybJ;^O^=uBnUE3vW-sI^SW>wnHVRkh~;vT4X_U$OSB;N5MK&xhpC>HdY z@(sCJds6eiNpvC{*SPl*KWV;r7af}Fz*k6P#Cmjll%tvlpl76M)~ZkPs(sBVM#XAw z2D#c@r=Zo(U!6`h;QYIEm~{30fxY`!-y&`x@Sk7Vqtb?$-Md(uu00+oq(ykD5`R4v zn#!PqsO8I*(jHXrMERbOTh{HQtZaOzsnqLEe2P;G8qS=6mU{DDz1nitZj%)FB#jOo zX|kpn4KHrXHrObu{{uySuk!lol6FTg?nhK<#!!4)z^`pKZYc0ZSu^$hw)#??n}0Dg zh|GHQ|Kc4gX-TN&6ZfQh|4-iWOv%xIai#57HFL6Xsd7}Xw(%qR-6(&2wY`e35YB5q zxy~>Xss4`G5TCh_=`q$DN9s&}Dl>OzL^m#)G1m&U*{khYb*1$#;zc>7#k#8nBKQ72 z_@a)1!Fi0bqeQxsn>M^$)HGmRs;dHjzqob+=;H*n_s)9mP`IUMQF5U1sl_zoDgV~e z4^j~eUahY3gY1g3%zZyQ8~b+VpwT`$9EzBZPqW}^E37;}6<2>$L}7(~p)(@Itld)U zZ&nm8-aM(|UAt4-^PZpGv4?Y08$v)sFR%4xgyM0(pM^l(y&UVFzjwVL&emPRA|bB+ zXBJD8%q#n)pViI9L0j`aF7`^ciA39Ci>>PKX`on*v@ER2*@28t^@ z$nm7+f<%c&%SF%QsH*KQg^APXIg29btg0f3TyEWtt*22f8a6^4)}{*8_&na-8(!UA zW@TVD@HbXVaHI|s*LI3R|sqAZIV9^hg)>!9!^m>TU|A9%0 zm-Rg~QabfZs*w8P5Gbnj(}G#=RJ!;gDmx8|8{t=VpGC0Xl~Q9`cKYSVqW+w><}PwY zOzm&%8lM6dC9-5Mt}O{O&%BZL?!l0>HCG0v<<@?x$`HZtrDwSYWG-jn6^c?C_k1+T zCWx$B!f@y7TTj&!2FE;`|0NJ-ihL+^3I9sUU7X0P8;V4kIJixaDxYhV<2r*+^FQd~ zd)KHnlfB$dswTc_v=aA0pbwg?DQD%xOX#)%X|`w8D?Zqau=fG#w4p zFObjYR;r@|n;%w0IjPnt4%p;Dk-r%(H(PUPf3r=qzs2)Q*;R6J6Hb`OH85H5Bc^%F zeGWL%YCGLS{?%H8L!PeF*V@Y5Vp;|!^=t%~)`+Orh%LX~$xk<2m~ey0b{`d(7Kpo_ zv?S|%O+k9kxL@Q!{A#O96~xQMMg93S+5U6|x<9>WC)DrueR@0|ZU%69KJ2}@ReydH zr@63%8q^_rEY2_NRy|)2j>)(P;Xluqv*;VKC*7V0LK;1T(F;{~&R=AFsGr5TbL_Xc z5M#nK5vAbwOCIgtH{UIuJW7<#WP#bRX8gL zQsGy*oB9^FQ;H_9I$itfP8|Fz{KQ9?A&$R8V>zsM$@xx6r*rX&^IJRl3laY2U#mTc zCpK4SH7Q+b7-{WXT@zu;p|si>^qCN%Wwb-}Ofl2tQ-iMICR_OgsRB%S9kL{fYb%8h z(>3FM=zCNzlSKi_P_m3Cf?I0B5*^_n`pRur=@)s{B4>kr+FSpzuWWHz0PQXqG^}Re zo=0lsgj0YJ|6eAd^6>v;5~XsYGsC2KQe?D!S^tZww+xCaXuCymcX#*T?(Xgc3+_I+ zy99T42=4BX;6AuRaEIUyci#6q-#K;v?WyYCHG8^idaq~6vsvsSFL5#|o<6oD!+W14 zFQp>>*DZGUhl&7-8(5$m{?%$K0X+K6{@@@vdPxV%oXAwi{AY?YiCMt&)$H|7|8_8A zULYa#931xdwF9e>T3*eSc0JyE$|?=2oRj>Y1S~Y@!VwE(M-a_1GblkB@BTBEWwt5Qd|iVW7%x27 zlv9N%Tea#115dnR!X&C?XSqG*-<78RLr5$!{a-@j_9FSegv5FHe+da@?q_$APG5#k=mqW`j`JwjHI^fX_sCX-r~1+OMFtw z-MD0~Za-=k6}$eE*WUDK_u9|)-1UrkqL1s3IW@H}-@yB;>E_QTtQM`;9mU)cruh#` ztzO;f3hl{kz79Y34@dY`$0W0*a4tc+;2AqgEm$2n*hBr!&_GPXz<{DHh3NMJwIFd+ z4s6ez%0;e+My%0p7@Bt}=n7{29Iox@A$@xS$GTai}VJ3DyC zYe96R{0<6}RK!4LecFH5PF-%Y*KcW;wTZbe!OoXm;rZ>ot6!c6eUQigxT&LM=Wk$r z#GTny^pZ*(W3~h6mCc{E`rVY>G!1q*;Cy6Rd96uWklK=sI&Uj@9TQMfA{%CZ$13>xK33`{mqy&@^t}RBVql;=_FVbd=de2`c{Ac43(D? zC>xJUw(>J53}Pz$hp|ggf;)KLvh-&mr@VevHSu$lj01^`y5UeCz(Q%Ap}^eFBKMCb z_x0j=DKraZYl?rT##v=?{+rww*aO3FgMPmR-JJloBVMDJz0?b6&zb5i1zBMoI^6`^ zN8D+KYJ!m!_d#pNlG)h3=B|b~O?r@p`%w z+We)UOLfSDXcJSp?_EFcg5|-pR76)wsu{#t zo={XgOlC{y1~v!piaCE3c|R*+%)blGXX8GZtVs60nD0SRdEe#cBpC=y2f7}uRLx7d z-L+!nU-4LLFGQzJ8*^uPD_z}vWU<1BB;ClN4O|6YU;V*)z213URp6yxn~l(%0xSje z`5$J5PQKjG8q?jYgjk2w?KgHxsyZ)x)lqSjn;&c$R=dwI2|9lm`kdEEPBz-Huzzk{vyTHNRp$WG%vsq;SDP$R(P(1OwNvZjE_@3f# z{^4^G$=Ju*an?(wa5Tx|WdAefRN;lTk*-m3SHLO_r|<3M;E$+k%N_fr^C|yuI?FVB z9k2TAouitO?g+|~!!;m^!bxf}1^zmN>VspwUExam?6;ntgKx8M?e6RSHX`My=&x3> zp2=!##F=SIBPya_+0qrNW^8Xwrs70CcOTJBeDxxU^`6166Jc3CzkO7#19Z_NSr`)B z3L;pb3%>)G(IO{yiJ51W`E67dv2DrrsCC-sQn{9!m;aRNBk6GJV26CR#?E0nUmiJ$aNpeo&lScB7i55HcuRata zIjOqVY;pMG&~f8Ie8P&2BA@wR_1A}AqFnBRnn#&e~z^;M+^FBisOkAj`80G zLveF|zrYQkV7l}%la-4LbGkczTFnwI{#=Lu=;cc5XOZNXA+S|Bcy$ul`S^;c+wZf6ew}UbRn@rXknCuD`4<7m z)zS2Yhuei?JFRDUnEd8~y~90g(SDB-`H~qDUbmdOL87bM^%)b?u5AsOkNb7_(apmF zDz_JW1m=fn_k6C9qU_Mz7%W(bG3l6q}Uj z{&)1zecxN^yoR9bb6{_8CLdxBaqwKu-k8iSz0=QEVkRm6;Min*L{bWa>OlN84JE-= zJ>%Y(c1ZJ(JjPQtKIWvopXT=$E2_Hi^HLopkpI}wFdUT%5} zUg$-*$v$TnGXEw0&>6g!&rr#25pKPAm-pyuXgP8XsM@(z>=|uZ*=Aajt6iQD3$dL) zPTi`Bd2eK)^o*FF+A<*VZAer{`_^9*Wx&iXEB$kYWY<_Rg51KZbtkjri@4YK&J4t? z-+ewbf9<3I5l_g-SrPf*5Pa@ivzka#255KDC=RxY$d4}baGeDxh^D5AQ2fF|!9dEb z(5_zjKfBlr z=$2@SXziZsR$H@%V))zM;N`YgzsnbxpTlnQ9uuP^9=^I>1TH^GFH}DQZwIZt=)PE9 zX!}u7o1}u#o3E^DpA2;$xHUSH!AreFs1`Kq)_qEQ@C?; zf8@3b^2bAjh`WGEnpe*Sv4hrQa~EDDC{PvS7P5o(IhZua*qnDLwpzZ^K-Mn`@1cQ| zqqFJq{d4!n+3+*?$-vq6&J#$THX0d;ii+B%gJUuHk(Z|*;U_445l@QalR`Ok!!FYyo95jBRo{6pN}@KkvKFHfwz%`g`6@sZ2Iwvc9t#K2a&eHOO;ve zj$@iX#7-2d(&)iMzE77}p$vu$Lq+C8T&8%2o9e>Yzq!5UeFNdh^7UoKBOuWGW4466 z9DF;e3Oq3p5etenzh$Zqyh0`(6&0TZgL$>|q3TlO8ik`$hCri`M_kc)BVhIX!NY@@ zzBeTgLz8RSlmcTS>BdZj`#SIsQ_jHkt15(}+OrH2t=d^JWVvi%5Xu89=q2=^n!%YG z1=M|21giNQe>3WV+tUZt70Qr7u)v>xZ%({;R@alk z_IGmu{CTbCRCd=F@8bCH9ssZ;61{k~_qyU7cA~jg=2N!1b7i}*M^6uo3U}~-s&M{{ z_ygA9RWWqY2*j!VE&hIi9rZZyhXU%p*(fwK~{a76!_clo|BuH=yTXki@0!EVyL9q&D0>1h$~9)a}ql!V;aD<}&= znuSi$dWy5JYh}Evs4CH5AK-KOt^tc=^ zoGPG1xt^g@C0kJD$oLFEXf0+dJ1n7BzZ^bj&hm63cvHw$MO_J5|H}N-yvP|cnxW?d zWB92IQT17au%Ek_8t$Xs6%T+$4&ipY4@)16GcDpKrFm>nSxO|-&1+0~KSq4k@=nyR zX>R6#c86%hQ`$vxO-PajIfjK$l*EXkvH~v@i&|NKvxgUqCjscH(a*EG3C&|8o=4yc z^7l`2bbZU9uqznYPoEXP717^&c&317;WsK1L9OHcCWfiyH9h@)ifQCfnHa-|0L~VX z59@-SEq?vfpAnZS7w<+ucEzq|obY{xp&FHAe$l1Sb@cM)WAtK)TjP)@$1|zj9IHIh z=hvi3@@vYWaQ&6wxAGsWB~O`bduAxTw-W`U&B?hkj5h^!Sx(Y(sU_ixXZk|oAt5f{ zx%kyB>&I*WswX~eq4uUO%cIQw4yjq!$t=)#K6EGE(fQE(>>K&&01_5h07)a&WrH@c z!sv7?9)Y6cWTueEoMWVr5M}(}^LLD35sS1-M8*Ntog|Kdhg1KP~;L zdSQL#3~6tRW}GquDI2+OsE&&)G_kQpPEhQgf)AR(m${Qs$0LJHnDK6)9`&EvoIq3T` z637}8GMg$F-$y??XJ}`k@#$Kpm8P)y>yGdq)` z)eHOiNHVXx+9AVu?74uRN2hmrn&0rkq_c(2mvt{ly+dA7=gNBh&qxxT`eeF?7}LA@K%Lf+E&^b)zS&NhflUg7-v0kmZNkbB5OPZY3?g%9e+02T zuw7JM^u<0lHfBA!v{5L)Q!tW-96)nG`Vn1~*Wwor2y4*W#Z^kT9W5@>S##%^E3l8& z4o!#V*bfrLz^uOaO+v<&7!C?s)QZ_xjLHLU5=E3Xl9v-z%Mmn?*>5M8I|zR^t+5Dp zqt+x_T6llDK&}K^$pkOW6A3;@>a-nk=bV!yZrBUGK{6^W;I|LIoB7&fhzwQD(G%yC z0yW$$C$0JSlf119#J`*Er;r~VmS5Nya+sAMQit4vD+MJ0&podb&~4i{D$5OJ)0_%# zS9yxBL><5J;lS#3%StuW$Q15kir+6;d!Qx;=Ef;u!3NKeZ5nTni5E6=3F+asrI!J_ z@eDNPOfvVKE%mTQ%C{`&5^A}SpY}~^F;lp>Ayw;h zC|Unf62SItK5{O+^*dy5p&yY>HdaQhh-jE2(B`I?z|qCrezR_ERwopWLbpT>+lO>F zU|=Pi9TeQydxSjO+8rVfb-s+gi34K!gbb&lpslUe0E26ganDQ}SdMinvkbTP6^gk6 zn-7W+Yk*o8SpBvyTNW2HK*{Bp}4m=KE<@|gFwU|pzrw>+UBU9?yjXlj>W9}w;z5v+( zLbpsTU0}G?{DE7(<*$6Vww;4pt*ZyOmYJZ44%pN~yuG#VmXNZ2tnE!h$Fd(a3%^X4?xWjgC@@e-IR{bxGb&-ZYGgjvgu;+~u4|)7VM# z(}@6bZZk}vNlb-x0plGl4kaTg-7iqXgtnJOYx^0vC))@sOM7d^=erX|#Va#RWR-(J z7MouZkt8|q-^X0WAUv}9!2;2F4W_9xm}uIYXWmyc|JG68NskkLEXK;R6M(*y!6T24 z{k@*M)Squtd<5ya>x#bx)tjk76+~4Rd6}r)3yRt1yy>(u;9%?=dpE(bEPE+#c3SxOU`eZL>}5W{ z4+`Im8OZCSU7nLPxw<<3az4vHJP*|3y+*OTOGJcIuaiyu%Nb~M1jh!TApI{!xba?CMJ{?u5x4BpgL zgLsQ1!>jmFlm;j}N@BM{p(CXA&2P4QQ7;JF>C&G@;!8kIfK?iSk(!lgX5>pyB6Ycl zbR9X0P){n1iyUutjLqbc*W%$)u7)pfI6M&Mrn*0lpI&QygaI~5DZz-fj(L}jfL5S5 zlFy7|f8|9`Q6aIrNA26fPqf&3GqoyL6>|R?GkJI?DC;QMF)6W&gN$w~hF=k8cZl$k zhs;*C?>ensYECrFHuBIwIqCbmFI5isVbd^)B)qZ$dYLP;{W#~#>58FQX;s6Jr{v%y zMBOJgg8st;L2in_cQXzLt8}$l!)N`k?j5}s5GX?2-tn#rvAMJMrqV~95lXEUUtX@2 zTqmmKBkIXgQkNQBA2VM8eqOxA2JZnxYBZk#){A%C21xVg#IQDkj11D5r`ffB2GG~< zwSusKES@a0X?%>6Sj|bo$ysTOu3LDHN&sm+?1qBINt(H?VCvYTf($p9?^al~e2L#kOVMF;|aVu|n4NF5k`-qUf)0v#Oh$c)nz%`4AP?+G{oWm7L z*_Lu(UsiDjF4R{`@zMhY1Ovuo_V<`VqF!hD>?g*l`$W09bKiqm#yf1uTJyOnHs_(x z#>COa)W_Fah$z_NVP;3#^Z-dbC=A(^=Yk7UP!gfwH~C8TF)14cp?FbXn3N;PAD243GvV121!k<;X9)nrIkMFImkfk!^gpFD(_Yrz z-NVvG?6~t@zf0qIk3SSvB%}Sx^5*gYT@gZ`R)Z~Hl)>Z z2u5l2*}mmi|8%Maqcpy>0L7UQ0m&Y#JC40{=A&TQFJB1KM8Cm-!+#Z}5R;>npS@s5 zlT?m=E37(Xfz{)DuT4-UB-cU6?)=bZ)jL+(gaSY?ts&ZOj?iV#Ivo0r$HpwCWwRUb zfQ!ihp-}uUc#PZVb92P+iq;lL(pOjNSg4>T=KU-?bymzP?S^cQ`Tp7RIjSiuj9As` z&cJhGVC+6MmkKLCtQXyCd8dpZG@1yG?9#|sI6>%H4!(awsG6QE+on?XAt2zA<)-`T zS_oZjOm`r=Riy1%yquFARL14UW0~^@Ukz`JPX-pgIg5d$h*CwaIcsd{da3XiqJ~g8 z_HsQGb8TiLFBFxrC>}1b+^xB71kqVf$yufQ$454ysjvEJngFyZv7IXsfMTq7(H}N7 zZbc@58@!ex-$AN(JX`yFZ!+Hb#Qi(8@5H!A7?15IHwM(*e!BjHc93Pc^{OpUbE^Gw zXk0UY*-rN}mLn9GgQi`L=H^Ht-|+w)_Ac&VK2QB$OS3_;TL!!Pt}k-Mgrpuq_REt~ zi|`UwspI0U(zk!v^K=-6v0o-6nbU>Rxt5<>S%LlwL1kCY0VRd>TAzYFUt#i!|Fj;? z{MzKb&T2(=g?RWJR&35l;>=?4Ox7js(YqW-f!fSL z*Rz%L+1UIl#2PM7LUDwkC;A_`#5L=B!7)PavYjBIt3_}GwpFrR@kv}EgJ&fHu=Gk< z)j0x^^VBpUA^@IHc~`t2rq2(xBS3BR+r(~1FlYF7PLpvfSD{oU`kBt3w8agzdIwy+ z2++~EgvKd0k~vaL!bR--bHtQTEl2#?@KkBX76=ms#WvVZ8SxSYa~QLM6);yHEW5oy zvI?5CgT_fxotk(jdVaRjhSj3uDrN@KXJkDq{<7-3@Q6~OJ=F!9mhxMcn}vwakx{nO z1LwrO(Pa?gApuE&=9|H?W1bl^^7TX3DRPe}gsV%BwFi!ZgX}|6_34lj{Y~hH$mZ$Z*%lz_@Z%pkHIy^j;d~ho| zJShv#1dd42%5~ZVEjhW-TsnuWh1gilwW77))`oVLe3}1{tUF&iuoL-0a1y~EX?_F+ zmfX)QVKxoPxk=|}J-QROg5esZym^V@!%To7Ln-49$^;ei@db~tmb2ZlUb9XSGY%&X ziH-y1s=*GtQ(tH{8xB|!;zuV&C6}ob*M6-!4PHh$Cw@(0w9noTn~#O}&eI}ITH46C^1Y*%1k}8nJZVu}HZZn|^I2bw z%<2L^L1%u4d`Kn5sP*(TEhM&?*!KpQ1C&9V;}Y{`HWCf)V4!@p8uLu#cWPYl6AIhUhp$T*b_+K{f^a;Qs`W;9q` zh#5wS3}#d+tW0E34~=9@gDs2+(HIcVV@8C4MVJ5WFVCROx`gb_Mp-VH)_iec)~W8G z%^9CT?yoKx$#!aBAO9h5_Ldwr%eXE12{Q_Y*k65_SQc`6)N(>5>|7`@YHK1RkPGIP zJX;Q_vfS#)KnS+V2+Y56)tWp3V*68k@8C)&WoJ7qk=DIv27N9k#_h2h(%1PLL&nh;E>c_++gOI^`49z@V<~UobuG z=hLok0zi|r7AVL2GmmUOW0GHKd)N`4Uxahg{~`e3lp`# zEB}3ksdrFom#@s3+6Y_&LJ=LUDOMA}m<;$~b|boGcz3>PUg(104m}{liQ?DUl9gvCr|Pjf`A3&3v3?F2$4a^T zB$vCJPz&m^*tH8eDEON-$Xl8XIeP0;uSykO&TMCRt|^lU0WTN@>9G3UGuCo!Tsr7a zA_AvtvYOGgoVImD!3CEi=ON@&;_xER4;YmxKDHqOj8-M#PrVvdC~5{6h6=tAu&U@tC7R@yhkP;~nAH{;uVvsAO+Wj*|=rHCj&DR+YaVNc+06Db@ zw(t;efTs#ahn5+h3^}id0@fbsuzDEM)V<}7{u6l&o-=qYGtAciHH{qA^T;ScJ(zRp zAwN1#EUK~onWd6wP<2@-D%5yu>mye*hLRM16B`L$Fn4MTSyW9$p{MK8jMLFpLm*`I zY>RRZ%ZG;(PL}t*B2jLj0x~WQJG0Ug#kvdw=#Klo3u@7MF00HCd2713CYlyEE_Ra{ z^4LF5k32oz{uqH2D~9zVJXfFTR@dcNjtFPO(_HOaDvX@%>os^(xU|=*%;%gZx<53U z?<**4c>VKwqa&05qI>0t6)=s?Ku!xhAi3qdtl1xT0O!;l)h1iE_VK9LiEC(ozvsku zm@r@@Z%6KKyE_xWsY`lo*-9}+L8WP953%c}m_#^Zt9g_l6jxUE_XeiMEWY6WI?3=l zr+LVq`}a%LvlvbRFq=KdIc`RREOBxA@m@C9RG2qeYi8pm;m(=&gA5HTu`fUF4?L^; zS|o}nb-#2GDjnRws6l2NtkL=O5=3ET+9^ei=(k!O1c=@!^bh8S)iBj`*>w(gNKdFc zXfb$eEu!?L4X_sX9XlKwa3B)Q%GNiK%(&hVcE!+J=|5f>@Y?1T)6D>(C0}3pa zPvGWlZk4eEOKbMEUn>Z}T+4~ddeYPMMKw-(ecz7Ud+!xR5i1h}90&02QbQ_@wb%Rr z0$1FH-WGzoT24A-VuG@hU>43w5)L$YCI_(U#o1LWfyB*~=+?-&Z@Iaz$u`|Ki?V1? z|6^s{Ht@*#LoFqIKfU2R2hJ|5rj8Npd}64_C0yIR$i5i0(6A=3WgA{ZB^#nu56@=O zGb zvy8iKiS!XW1|cxiaNuaeH>vZw)W+^|TX`cl57Opi1yG4;#nlX1E2zwsK&#eXt)E_fxWYHNWZYsw}6;R`VQ$^ zJ0%i224!oMjUB6CB3WWgUlr6_5Vz}>aQL=KK@?itcu_B6JyhK2$4er0;Z4UiX4$5) zR+kl`h!~gKZGuwCNbU%_U~`Hzn|VEKQZHO|OLs!pJF2%nB8lpwUsL0)IE-t!z- zm}d={>emgoCS9aS=6>@9L-|!73-?Zb-+%(HI&E`+nz67USiW@f-!ED#A#e3v$Ez()Cpfo>Z;#cmTTI{mRl#Pxd5ICR>3fFDp|7cIrci}dNW5dL z5(~Go5*Xw>aENkkw~M`i$4P&F-w4c=kj0G917s#A97#G5R^VHUyN0hLZyJNL5KJmL z)Y{@EzE9M=G16yBI%@6xJ|Q)nWb$+vr`HR?v__>Q@(?n2&Y@{$4y{O#ImuTon(!GKlO=nppb;G8{&|9d3DT zjw%*^Z-2B;9}kSM)i^$29Wn}==yJ9-!q4`+@N`6&4rUERwn3FjV*bVHi2-5Z(f4w( z^kBTSZk9z6fs_~dRIpher??x&#u*GtAw3d@8Tk~Z=AZOL!bXv%B9N{`zfAtHvL^Iv z8A|u(?rGXSa4KYGTXowY15{7^nF#E6~ znNc$=?r3t3a8#PCl-}Gd>#H*1)tJu3)z-(VTE~$>xe@EAm1jL=@heySMw4bTUsu-D z2M6$=IMylJV$Ysqb7jU`@5@^oljgN9m!BrcOW8qotZow+`WaLn-!NY7s+4f`LyI)# zSLbsZ^`Y{M)-M{l@c0iCnN-NT-N}~AOzVFG>3`X4Nv|-BKqz4_+Me=XGiH1q7bbD|R|MAP;fb}aP%tdF4l$}n8&b9{1;6;?G41&OstfDnOh8l1CK(iav{quK2 zA(|t8loH#8-U9G9w=L_G(l(Ps1AGKHzOmCD^fQnAwLyts`Vb|fT2)$s1mlSpx1R$k zD&3$ceqr#ysroCNETi%j9(#>{eWzU$3Efd~SPOz^zD$$?~;9G98S_*tL}w$>U?CH ztchMimHt`bI2&b%_#2>afm+|=ZTO?SJINbI!o4xFn$-^+9~1Lo1(Y;k$%}l{0ps$`7n8ZXg$J^lnqLf4H*ciSH?w4R4b*~;Rg_)wskf02=*S+OmB{9I4= zYEq6<7NwF z9f2N#1V@J5v#L;JOta?r+5yz4R_TmGyk)zuMV~9ScUv{J0)>X(x!=Q~ech!Jt!I{c zO7(kk*bOf6mjF-BgTYmP^|9y!qNupS)*4LmD>*%U=qi{5P9*I+CCkr7Z6NBT7c0}d zCUF3i-fshK)>P)$I=nM2S2NH9G?`tD;Qd(m7oo03ld%xO;td^9_AkZVXSgNgfz|XN zAIgjOn>G>a6oQChfsWFZN3X5$(AXD9#4;}5U)p6nn}n)>7)Z^EQ;Ka|G4q`tUL?;I z75WGz4KhM2ydQ%IaSnm?UN!8MiY&x<`w@RyauSPKX`TFy7!B(c?;vN6MrAiulQRe8 zjN0nYfP!{kLKRAgFF`2*=vcvi^ZYBJfk$EZdLGn3_5Ywpf=ZW&uLw1cX80<5(wd4O zcJ$x$?cJ*C1P#^7-}B$E%HZI1rO(zk8t+^SCMhR|X#}}CmRQVn1&?TGlDIl5F(N|-B{{}Nt)?h&2xuHP-H9yNx4_{&Qos=h& zV6Bva{b^D2$7m!%SDdA&L>#N2%~&*jgopLvC1ewR1Pw-eby}ZZJ9sM8!?N}Zw}E;3 zJ*PdE{C=I?Hy?eg1Q$G3LN25a9gqRUW6p__3fdjHz3ZLKXW693; z1%q5&Rke)i>=@XKCxn}d$hz)K;3GUcxO>HzBKW@!iSx~OX}7=(L_phZ8g_|wW>j@0 z@z2Rb)VUP0sVOhja%QDqhEa(gpCkxspb)hLA24T7Ke=t~4Oylcl{50FjBXp_ZLwi- z4%D0qA98egjlPKATy6&}LW2+hZ1 zJ(70^$JfL**3|ylhdc;fXt3p+u_0MphVsI35BDz+5Axn74H*7T92oqmk-v%X0Tkvv zDC<+2$q@h@bHHC2{G>}i$5ploO{F^veyy}X(HCbwGNo$=E z8_WvvsE`2s0GQzb`x=7T1WOp9*LtB-Bn z2QJ+Z%~*qM!@fRYCTZiC9IkG3e83WuC?S8U#?BTDq}fQM42#*EGmW=<)9AuNj!|#~ z+JVJ54t~|nge5LnEU3R`%u^mwPGcy}YDh~}9_ZRZ8QRPi(Lca`2H<|ThiFzXzO?DB z>`tX&{{=P3@Hxrg6tr7*5pQLqhJ;M@e~<;FWHuu+m%#X1O?kU=1uBTj6isV^pToO@ z^Cmo2Xs<9(X1h@B$B^zcdSsGGVj7+!9;BjdpA1&g>r6Wma*5db3z>}oQh8_=v>CYp zbS@d#Qq@!2FC~9-rRi6>CKnsX?ba64-*5N&->r)=)8a)ejuP__#Tlv;X*b9^U~)A; z<0193x7Akld`l!FDYXbVphb%)qtGQB=ohh^rX}Du>(JX|Q z;eW93udl^yIc>H!sZSi-_V}t>_*G~eZjZ$|c{46x?qt}OBAkaqkwuWh7nRP>PzlQM zu5LcNeHcM=@s7?sCz4ZV5}TIgRr}|wN0vIq^-HFjk(oSX?ZPm}PqQ6lx{42C!tylG zyFE;Lf3YasaHo+3MT(!8t)_|l>d0SjX>fh9@OLMyeWd*dRyf}DF|aO#QqV;uS7J^7D}*L+rX;%lqnz^d$d;S zD1%2qC2b4N`SXHQ7ESi0h~c+zp^`{QrucP(ie%Kt0Z>??Vl)v1E8uAGZvO)-{1R#9 zm@Og83kl?2MvvxA{l@-_E6glE)>i%pR}fkq*6A`Oz-&wx53?>VcYZrt>E5K6Owi;|a$)uRbIOyJRcQAvFq<6#0s035TTFQ7w>y5r>SQ{DqG& zWOS?W7I|D=u60D{C1gaSkI2!37~~)_E)uO7R4uTRQ8UiP9Av-nGhAAM7)`2T+w{KK zgMN?g>E>->=DWDa3=Ojo@r~#lm2JU>Nz=>zLrYD;6MH~q%cRs0Ezcdw-BRt-dQt6t zJ3OeP1r#@9Z?c#@P6YRPf=R53W zd4>_8VN()>a(qQ4RSL=})9UEO2hoFBE_y3HGniSE^7RU2oF!XPnKk1UKS|Q)n&r^x!>jsNc7v0 zU5l@+mX8nR-nDH$wTeBx?!VXaOQ}@`Pb;|wu&;dAcH$dCKt!r?F4GsA?mH;Wv_-&l znc^uNd20v z@gkZqN@sttQwyuNl0xXDk&(N>xZPpV6}uQKe39}mF~I}XLx20~wyIT5Zv!oi5r6Y;qdS+zgzQ?gZ`g0ygZ9@Q^|&KOzK>oopO~+(9bp z4tB!Ny7gf3`LsLGJwWKqYw5+$!{zRJ@FJ)7`cZQtDdX$rV@Gc%L4W71_BzncKkxz2 zPmFvh0JxziJPA)$!zD&lOa!IkRUyXyXK(bXj$G`CZrnt*MYmZO>g2V})tvm~t4`!~ zTCk3R$u{w~$?dkB&Q}{DtA$8BRy+Cz3x&WrByYRGD)*x&5&{3G1i>C8oSH0ZP=Xp7 z(E#2X6$z`Oo6gS}RUB81**wQsP=nk>=l>Ixq*{)1Us zscC<0EWvbLV-55YN{g?RC9Gb!!7js{r0B?y`Qq)kaZpz77=F|LzzVq3-*^f1`cCB~ zOxbfZRVj9*+`uGvyk1)2nc=@D?h{vyPsS?4BC5D^zER$k6b-bd5rKOU;lc<9VYj;lA>I#$JloN~OXm0aFJ%aMHYZIdG7o zlquRB$QgKBpyocj7?{L4Gd<zpy*%#T`Y53XVUs@ zOE}~uo+iqa=<(i!1B(`PhQ4c6-kf97bs_ib*7Ede6`Yryaw|udoDTvYP~TV9bOFb9 zE56ZR47DZtGDo^_x_I8vPjv^P60aYjLBc<|2S4;jw_7(f8BDFa02JAFG~152#f9+3 z+jcfxN)N@`TsEqLP-kB*x0m<>o#R^{fPm}G z=pJ4>re~)am$>-ME9B1(dYw3WUx~T8oZb;ODlo`9cA{L7AW%~-Rpz|*S|^?_9T~lP zC`k@!xhwK@dHWsYBkLWL;GB3cToJNJD=|e}<8_sIMv!7|jA{+>--MsQSk>+=M+w@2 zC^NXqfkJV&@{oJcawsg=SA*{i60y(x<|U$ov|{sT&T*!Ch&CXtn}RH|r>sSszB6VZ zLT}7(F6;>a5l+3zRA@_#P{gr$xg8w+ik7fPKWUQfwTO=B`cz22$H-eY%CE;azH3Iu zKLxOrDFm&i94{nC0Dg2VEPhDT)GcVN7b9Pd@7+(;K`m>CYfdNIHD&7?%}V4)6A-hMF+pRzErVANc=HD(?`Gw$`q6>MavYf6x$L&b~JI&AcpQDdS5+4Q=15{DT&Gk~9M-@xFi)JMoqZvWP}F1�iln-XiG^4d>+(XgJugsGmsVxGO>|&4 zV$H+BaAsIoG_IR>UP*F`9j7kjx%5WZVg8dDzfz{yUZ6;W{vnCS%45}HI<~(ET$_;j zWpsAjAVZ(+&s1B(!lkTbN_T}!_ER{UnvjXh-XNzEc+~+V`1W;~bO0ns7Ady+r>~ac9Sx#l+%SbJnpd zpWJ2$1x_nk_3gp!(h!`}SB@n`&oe1EZ$YABWKT_n=ul}si z@W2iThS`Ri-8-$kV_A8LiX&ND4%*rpN*VQGFo6{zeE9z-l48b>H{2DQ1(W~-s{ieN zg6e;K>JDs4^?#~T>oe`@M43603ii1+4sbr^kpDPwVl?SrnDM~%&*2R^s)ZAZK3_^* zyT`0-fcVtE>+DM;Iq^hU2oR#qv%=>Dx7YJ0*sf z=u2ceBLLFM=HQb(V2`Ci6U=PPTlfIYz#9aa$dc~Rk`v)VLO6&DEK?GACeo?lZOkGd0oqyX<)4|v^*l}i%P|SM6)=JrAEa{U zYNHKlq>Q6K!OB}c0;ZV>)~bDU%D>IMpp$yqg~*hPo&xCMl-M7=(L5gP^bv|Aj*5xG z0Hk8sKMyo1jwc->Wq*iL;F7takGo@Z2Av(vLD-pqi{mC4FeLGi2uL8x9zcXKQ*;_o zIKf9ux)#OA)+LYWgYQ9Lbb|fQhYbi5jh|D{rWEl6_DLK+?K8GeynBEqIw<1hOn_R+ znWC%eG>=+7-hMv_6UQS*=%5&}%#&{72`HxoVn42Gzgi7qewW25!QO+=w8#57YlPmf zoobP<)r$tZJC3uddurrijaPGMKDP;SxBjn|B{46Y%&{Y3X8}O|=pBm^mCh-T)vIO8 zfTS|neMlza6f?ha1`QO(P-si+L8Tp^3pHUJ>n&s@7l;;mQ8h`&>F7!k9cx4|1hC^J zCPu9j3a`sfgo5rrpzna2D7ZQ<=)^4}MZ zkOUf&zGC{i7$EBX$W1K|p$iy3x?Qc3=9c%itA2LEr1L4qcxLbKx4ewU_}al`)$yPNUJfMYGvO}7?M0OlA@D%n#?POm6GmiS8QDv|Dvb_s7J)_ zpw5eYZ9@)4G)CK!)7@)U+5R$t6{|@n+y-It5WqW{3oqAWHjqM(TmgyF8;(Hv%I)?r!~+BPS|zg zj6gHP4hbFgXO4;rkDtYY9vh@2c+R}Z`fEscZ&j@J@HaRLoy_<@AQ&T@b zyeC1QqIMGW-b85HQ1&eqog0`4CVDX&Ut;U8%d})ZkeeuZLce}p_~DAx4pVP?wcxc& zeu-`yWX6*11$#K{Z3B$66CpA^Gfhgeh&7ehn7PoA;Uf)$zuY7}e9{^Ob_j_?pduuq z@4~O7bP?de?nl*GacN&Bc%Hrq8~~z79k}5^9QI z0aA{($@%Q(hrxy5O&8yiK5Op&Qj5f8Fc=JG27?)zjGsKQ$>nh|BZk|X!$p>$gj|`G zr^ZVYMP`r#G38`NSuWfuH;eNM3!6edn{bwol_>`nv!W#D zAYFs4`Xk+QFvrvQfnnYX!<38foTg8nD5S^2&2U1(OINd(46hql9%o^p0xfqI0EiW*FAVoM!fB=m z14vG<)DUJ-(vjQ~8JbGwqoO);2D=Wyf)c*vj8KnWR7PW5P}IzAY3nzHIma0daU9Q! zJ=R5nAZh?5|C7RZLb;!a0389A)WGyhxe(q#PI^OH%4G!rU4;gRF&ma?P_@v@O#0p4 zg#|Up(s<+LV6PjXAK4X|LBz|Xc@EV6Ylmr&BgFuzi{u=SNhCLLnxQG4gD^14C5~XG z>hmO*wFuA;G>H^brpbu$sSenmGG4@nHzQvPYtdCwdP)qXNDN~&N?HM|rPIDk}l;N=`dX#+x zdLCf1vOHs4iYC20k$g78aPAHb`LeFlQ@7JO2C+(HoF(IUUh)*u{E(JbY6HqD5TlyDu%k^UQCFg;4vT<*|m7IyjSShINw z1B~d5M!B4AO(|IAYmhr|#>%m8wH&0^pdEH~sDddWsXjfaeZ;4R-^ zh*C-=*}`sVi%2|RkUU}XG^`r3z#@*9=es(U0368EDJP*+bmT|lh|)F@n&f|m^;bae z=m>hHsU4F=bTqH3Hg++Bb_C!XRd7r5^m(cE%HmaVa2>0JViOr2e{kdkoJ2h-l9W6R zZLsTFZ8x~KZA6du<@3XnPDlQWM4p1zohCUGPu9>2Fr(qzjKcO2M>q{WxKh57>Qz=5 z(9|?rQ&T-+a>W2cDXJ6#i{TlHvQo}c!sEGy26`+3>p~ZFIzD>x1OU>|OSx6w%5$qp z1*;5oylm?_i^T;ezlDTn zw%+;6Go%D*mcWXZOrP)!_B<)Yl-ue!3e`U7l^OY%bzGu3@G_nguVYBzz>|ON_j=R$ zh5YM6^`Z+%j2(DUc%U17zZHcCjE~J{n8O1vOb)=-0iWB8rdJ0Zlr!)21RZ!RKLCvT z>~cXH?UQrh$ys=>ra_kmWFL4UALK4x1eW@EVmSk+*r@$Rsm zRQwiyFrJliOEG7bCx@@V>M6AuB__Z$DpDy_@FH~zqDdI%pdf(#MWAUeGGI|6#HkXe z#e`iYW_2n9pGJ^obcK!?NEU;7C>>u+hFJX?(O7P!LbIP&IPG*`d!H04I+nYqW+1>f zjA%is*eJ<8&~{+8XNd+U&~$4phdH`f)jJ{in1O&>(vCm$K9sh_=0a}E7$mQZ(_}2A zgOZUaV<5`mPz_H`7cZ3lGO%ECFtC+AeF!=oJ9LRa7{7_H?M^|%DAVzeL)->^(CHi} z7L}SN1u=1dm6&3?1zUT9Buco%`;nPvEtyM!VL}>aP863HB9;)?0kwKwnGq`FTh;Is zr!J^K(?-gpqA5?p6)Vh=dI*|w^_6HzXBdq*5fkg)qwLx5j1Ex{(_$UCP}3IKV!B!y zDNDGy5*v+RjZ{)1SP!%vP_URX9x3A_`A9-zPt)~_=`vDMrTqr-ko`r&K$j>9NDf_> z$mDudVF#wjLU*_#VU(e?J=_9$pyQF6ie~tX5{Lvz1O@ZDYdbI}3GPWi`7B{xG5EC% zNu_%zuS_^*(f5Xc4XH%xQnddNh{=a7`5(DKfDuyu8I1q^K@+@+&M5K7@krUK7GosQ``f!>}@{Q>9TvH5Fa?Mx%dUr59r(MbL^FnIx6! ztirY5Es&6$K7ODvfz*rp=%}awd{Mq`ulN(0x758P^S1gG(ac9bP%g49liy7l0bo3^ zI@K`jC=|Ee@4#uPb6g;QUgkp*X0d5Ww=yOP;WRAa!p~yi>+ill;%w^6b`OG{EER;j z@wFUOjn=M$1_*|pR}UCpr9bMqQ8RZ6K)#G$3~%Q~GLcrBy#N|nsysp4`d zLQn9?iIsDNXc9eE0@A6a`F$WLarZP>J=1gDF96(=x9zWr;4B zBQj+-3()f%4qI@7SmKYFM7KH^Yuq0T%cuLNJ{sG5_Sfy5F5+a2m9e@gQi4_+;bEfL z2z^)~2*2XAAk=F1QBTPH0rpL>N+L2Ne4b4t5Lb*}f*40WrX&Fb)t-+;+@e0mh|rFqS}>Te;n03yByK49zH3 ztXOF9K4>=T?uBP`n#OZ&`qQF@wQ-lu3S?7`J-a9+Ie!4U-5gKj&IZ^6D>Ay?%R2hM z`yGS~RPt{hd;s^Me;?!^U%mhehoA1J;_ZpeAhnMU<10KD@C{UkedYF{8P!b zdLIq8cXpe>*49o? z$1RhxuG+3^*UL6x-2PA zZfV~+;h9h4t`^rU+O@o2Bv!2?Ljqhl96*xgZ#m&qN)l&8A?QfX^kacOI0w2P)In-u z_IQ{Fxz$O9Qe#HJ(0VxKVB(6ioQQ%agoLd^av;(G70@hBtnv4PMO$3R|HP;!1trPy-N)X%=aXf%6Jk|*_1Im zHM*;PQ_kA?O>?PFRUfy;>fa!w<^m0>-->oWEo!&1k5VDqVA1808Vg>2?KNgiH5y*d zI^Q&)<(oRLYOgv?Gf;);Q^PA+(b+0&#j||qhlU0|RbFlDir4G0N~)+%zNIl*(u4g$ zvYl)q7l`)B7?PM>>7`F<*kq zsPX?h$cV_~kB$L+5UgHydxf&q62?FPCFG9m_lCHZtnxXZjVNglqY51PpRQ4q1!=)tC@w&wg6GPqg-RVnx@M07NwVIF4z zI!q8Qtgz_H5RG%j>Y8hmCGvvVMw;aZ+;-mV-3nOChldq=m>24Gn%kDb6NlSQd()(>W=8 zJe8V^j$%xwN(P04WXXL9NtHYaU5A_r;drJv)=0^xz#cA@Ly66Q9KF~=hrgUDFVrbb zsFVPMj*->g6=yB~+T188wrgFq6em^oE`^G>M^fX1V}1d%mKO68SgjMsQcclv5dcmT za16FC1S3piAn$n+YTZ4FhI4Qwa2@ZaQ<{k_BdNh!;>On%#t^bdyN4BSPp)s2IRd{G z+AyZqlG9~2VM2{^3#H1B2QHrgNlamIzr{-Tn1XJi{I1K_3DAfpR7|WQ|2+;b^Hlp} z04<-Vmxt?j>a-v?+(Ml$JC&M-VMDIFB!fx-t5tcRIUZAR^p|B= z-G~P?D7Pi2jN)VO-jbMbR!qkt_qunk6l#HaJAhcY7 z*N*0l?wDL;fw{rm-?fe>XBAA6$J*~$!D?g_J#5)hL_|gt?PRZNgK1Q16}nOdpT+Wj z+v;HNAgkP^?xNm<#m$63ZzS^b6rH>krhx`PWh?g9UuBNkYZWpKn2ofvi&ItZFI(js z2U5JtCWKcGBV{9u(=knOtQlGb3_X`>TzZTV3`@2-QSQ}FD=>3Gr(+%Cl|!H8A|yiZ zjHjU-p2U)*!xBkAu$9IIS6(hFc{5JwnB$E25++|4@fDT#9Cg$=rO{mAHOVA1SH>Aw z>Do}3s1GK=I9$!N&|0|!y~Cy?2$*pWr}z_`NmdjSHJ*w&t<)!&)E4@PW>{Q`p1ICJ zxUZ?B7m+Z=ocJ05UxB)zZh!l8miypc)3tXb(1*nBy11>HF`X7d9~C&j&IZ9A`iuO} zQ^%SLXM9=C!BMt6p2}&RK5KbSk~y+hJ5do2DbQXC#qk`GlLUdvEaoq89ABW6Mb_uf zHnm^6!C;Ofbo)*@d%o@^BnyTl!$BF>Wh@^2{@$aB+?2$7Z(sfpOgYBC_x9g=&Zypd zpFe9)j$h<*En`EYc*~x+8L12$;MErJtHdbmTYI}Z+jw)&SQVU!ib|&#$d*lQDEK7ki8s$u?TuIbL?l#pfP?Dx9~Y`a4gT#9CGSm zq=WH}^4P2vz2=^4-?|V&6=YY0Bu#s%J|j;e^zJ#_tw3+IuXV9pq?n;EUPy*;VMPfo zWa{G~RB52zjdedZ=ZePJ7D!afBvLqst6R>;#?0JXnw*W6p}n@0Jr!o8IiI)gyx5(p zaBG}MS>?^y`9DryoL&5U_TqG_`1$;M=f}z6+1W2|PM^Jcwt zHCfvuEnu^9UU;qz8caLveh}p&A$1zAJjR6)wTfdfr8p>&OZz7z=mSk?c4WlX4*@!a z6V@R~$cSdD5)~D{7AhQ%V-^{05gB;UkjS;pahQR}HOQ<&uJaV@18+H)(grA3S;pxw zSBen8H^DqA8kO8Ha$}Zv#pkvhzo>;t z;dSx$^Ir8`Or}asEWfXza5oqz3)X~=CxjPxEQ-L2S&pl~Ucw@lWVG&U4%{kJU<6Q3 zLoj66;09Gh*XL7$n5PA9^$}E5MVf~6Aj#v{qPU@4Lt4;-a>C5z6J&8z?rrUaxT_L` zo7@@>^p{t;jbYZ3&>p;deE9O^<9}V7Cbc_{lCw@&^jILQ<8x}8ka4YaUCbnrx-Mp> zit7XR-~avJ0k)w&troEh-Zu14M7mR_4q7fP&_fE1%HY;AfIcvrp! zcfDJjs;(I=mC}vA%6nc|*EJu9`Ab=3qP6OKa$CiRB%`&DjH~Z*G?kBzA_?*vI_E5x zTmFPGNqwEpBC41ui$FwhtadJapOA3QxZKNQtfgQr3_KQ&5yf0bl-p{#S2WIPL>B0V zO1zDG6L#fitmjtufrHpp`LdQocV@3JNjfy%fl7*jH?EJuhb=TD z5+yNHUf*izQI0~!IT0yKBJHteyc^+!9&|Y4;#WvnvbwS$fGPkkBUWhu@x|^*#V|S@ zdzaL47Bxmrb6*|q)WHQ^k?}mta&COcC8~E?QLQi6A@^={O=bj3*H&hW!vS|!{Q0f)2Jr#r4zKucb>Ot(? z)*$wl4Pvh}h=Cf!vA$$wM$mR6>k11r%%`bZ#}^X(3naAL!eK1AQ&!-^+{ENWMcEK! zCKqbyPG7Nosp|~_!S$eFZM5N9W_cO1>2DIxGF^hF|{|HnBZNxKb3BXO7s+zeo~3@ zbFt-q{aGKxmESg(eXft#x0qY0&~qhj9&7?x3Y9fK!?wAqnCH?hla&!3As> z@&(Sb$pxC?R2c}0-%?Hy+G3m%F7<02%*8o1a<2TqJPKwGN%TEePSibkHNpANFUV85 zP^HQs>dP@?9KT3r7cj{HaT~=u7iOT!Tg-(e%<^5C;(KM{N;gnoCTjOmO9t`B;?E zq2B+3u4E~m#;V|axahN75;dn_=Mcn zD$7;>RTOwhr!=dqUI1@lk@GP9iI5b>bVh`U&%LNf6q5y?k64~mdN`9DRH`{TbhXBo zGnDaUh)>Cs&BzZ_DkqEC_0NK!3n`nI7n+m+2#ypA6>1N~ncP#ppRcic^{KETtBSQn zL*Ayub8ii(g^4@ckt z)|4^FA^%2B)R<+bU`YD)p$bn|$bkS(?ksAsAy!7=#y}nRd#>Jo?e7u0rdBxo;xNpZvVGcTZd$egz%rk}{ zS7MXL4oj3_$zG)^ZhTo2#i-C^T>VA+CT^B}^{gke5O8vD&FV&fP?3a%%ULhF`$fFK z6DVfyIAQJDAZJ3wp3$P7iI&ZC6^8&Pb8Q+D1mV!!7aauX%@A(TRQ;^Xy%ZfZaAfc% zR>xYHa~*2}fG(eDD&K2+R`Xvk?Xk|mqfK+*siH3H%T@J5&jLrtXXsbKk`!l?1Mhcm zDsz-Z-$=EeJ@g4;7ChZWz8{g0MWovV%iE(080joTek?V`$_cBvvU*T-x5ZgZ4vd~r z0gYy4D&F<~6--Hn0Zuuv&t6xZwhnYKzuG17k`9xwj2h1cO zxWr!bak7ru=rk+>oTdV~h;vZfFawoNYfY=UTil2p_s@D!1M!#zQ7l^U7O6*9DUa>w z=hM>{ug@>uzBoNQe)HOyMDme@%_H;s$oC~xOpqU<$0VFE^yvHJ^Vf&3jxUa19sc8m zY3^}XMH4ja#~E1N>kHzxE{fXvsDb#MN)20EA%n-5PRk9vJ#y;WFfa#?Dj2cY{-^PD zQkM{znKQV&Z|;M-7ohYhjxl+;^hgC#x8;gUAWFSs30XvXvdxw$v2gxC;p&gFa%-M0 zv?pQ=(Vi5irAZ|?f<9}C+1T}KePl0p$`7cr$XO&-U*+{u;2FEHgcK`SLXvEK_7DVt zmS}Aba3z;dTu?)9b_oaVogSoa&Za*XwZbQgw3V4hpE6A z5D;Y15>#Gugqzh$%?6-DRq8$SO*SF?iVEWTbJ8S&D?XevZts`_UrUIdS%|@EO?^mE zb46TM0VOAi^>%78RdnX`9bqoF5*`M$e6- zbwoMj&zXb$xQK9}0?OA{&R}37VCp-ERY*#<>cMU>2#ULs=L`+{N+y2yCPA;+jKB?q z!Oj-i?r- zwJzTmni|wCggKtZ7lsZl00-(_$47_Y?`|}h3y{2=))ksAwh%TIV-{Xss4&w~A}L$} zl`v2eB<)$96MBynie4E?Xjr>w6m(*I%`(EL=tAA(Eg1+P5R>*BGd2jLl*f2RF7zkY5OHCSEX*wH zk+8$A$YMYW3cC1=~Y+qP88%bm)8tcric;AXV(N~rAbKnw-b126iK6MC|w{g>Ed z5zHRVw)NUFmu?MzEu0ZS&TwkioFxq>z{y-Lg`faLFG3QPiN+osQnS!!YNboS#UjX^ z8avB`3@FwjUNo)v+4jY3=R*0CQL*L$e(-dp`>) z_Dk9OQpm}%7pl&+4ocW0K-D&Ak){gy(jGdOu09M-QkKjYA* z6QZ?jq0rXFkcjNUdh+*FjoyZ7!dEZ4pc$R1tLY|GzKy0$I8ouQwTg!$mTGCL0t%u@ z24VFjX5^l6CBGHwxN$CCx(ENYaSQ&dy90mwumeBrz<0R=*I0?tj2?E{hn@Cer)}72 z*IgcWxzVDRG$e^2oz4+U=bVlw8Tw1e()r)ec7J;pea{(=LN*NwtBYsec}h_S{ocGo z$4MBNybClFkQIzZ6mzwm>F!(yj>8m(6Jh|_LfX8Bwu8RfEj<0n>;A0+SC(};AayuH zRX|fjK87R(V-$F>P{`_T&7Zo70Ag&*2AJaV7|FpJcCJeyoOL>Xm%@TRxWE7+99qPx zi0HjMK6>%`?1jG_^uOzLeonw`)*5_=b94G4C4?AXA;u9NbA7%-6LiI?TuWO>uu*n} zIqBGofhF`9n5c7eQ04B!3Gxol(D9jvzCS!WKHKX2a(w>do1f3oFNde6hp*3%U!0*g zr|9U->*vSk$8TPrp*KIE!`J^rKOMh*zJ&;ddjaHQ$|dY_!lq!JE+U?o&W+2zy%HMZ zWSrwML1Q)}kaDmZ+_M8$qP3-~Hk{y3xBEDm(X^pg=cy`;k~i+7-|zRI?(V{W`~80P zzk~kv&YuRmd(XDF_jdcwcK_5LJRR&k{S)fn$_SS{xyUg8Q~$<&RR{M6d5Q=G5P>tq z85uJ^-$%#Eh+}K>?jU3q@V;g|ashYBrwtvPruOG{F!(m;cO2L1eRPOk(jk}Hl5>I~ z*9f?$nFI_hfiw`Iy%j$!T~`P?2%VP_ph~8u0%w?04s}oybS7*{_K{Iv9K&LVh7zEL zFTIFeC6H^a7qc-F=9@po{4z+Bap#iEA%OXQ$Jg+7d^90(N_fXdCc#_BN25Fe0lDLw zA#|pgng~F$x{7xC_LbygTd)Uo5XyO6K-PLeuA}N|0Ca{l+2-<^**qotC{|O^nN?=& zOOxcobB|9e|98Tw-n5S;{J+2b^l6p(Q(9cO4zlzb++gwgA8haK^~>vjd$7Ip>|y=i$MgH| zy(gU(7O)Bfc+&g)xzmxYpc$R+!~J`yRziG;sYGr#A^WH();T2*FbjkJhTNO>DkMKd z{qFq!yUxl*kf$gzaL5pXHwLl+4{b&?%#e3U<{t8Xc=N_Xp4z)Sv>8*8A@BRcQ&0Z; zf4puN`JX>4pnw0}H#q>7^dPIGpt5rkJ%`LOXW@iIc}$b>v5IM=UcZ36Ltvvq$VaKo z$i1(_J5O;IPF^U*(a7BD@zEuj@1x&;N098}^Jgb0H%ke}8RJqacV6o|w590Xa#S2L z#|HW)aj7NQkL&_{LcirKBh|06I!Pk)`ie-Z*hhno`QX>*5rXjCWFBP$cRP=&2U*2z zN~=$!Y~hT=;G}m>_|#@9_yf)5g4w&l#MwEMi`_$>S}k`ngKF~BnCJ+R-w;HgF@5Th z!`IcgM&L9^VnNESDtcL`RaK+Zs>b&M+-EFhF&oc+B5R=DsZ>mW`wpGKkf3q2IPReDUyBJ~P+ zaDnudT)fpn)h|tm)%8z`;Q&*pG*bO60foDl9(uI$)HtIx7=C2Oi}nwSiSbZSd??B~ zm~e{&>bfW>eYdr0NNA{8D^spr8@=0v302nZqE9HHD!lukUuM&qp(QLsn`D}1b7zXX)uJfn zG|5KDTi{B+kNylmS#qd#=j`v*edf_6nQx&-DmcylK?5;VA$p`XIN0oTy~gdR*0Hm) zpz=*Po?D_@mt9IDY^FP?^EFb@qyGtHROc6eJTj8oqWas}5eMt-_aG_X-bPrGoBzje zgoO(cN~9>a0qt&52!?cI1yG&0%w5aX zk8T@o!3Cv!rx>sg=eLpVf+H8spNiUnX~lHV+Oo?-bT7Bl!%7>4j)K5h$|-oET0b8X z5;~%S=it^e%%d^Mg4O0M)#*BHd!?BZWI7}Kh)q+>3G)6v2(|})->b_f?Q{@yZG#{( zAWwDKn+=8}!vn93*0!>BqvmySrDKNafv<+8N^X-?k@f7xxPqvRaGJ3izKWzj!R5Cg zjol{dLe7$3*>FXqnbP-3TSF-$u7)a8^!CsbVKQZjSb6OgKT{PL0 zTC3uzsPG?!DCY{oikNb1+$XB^(7yzdbT+IN5oAiYAjj;{y4ZqGP7Aiw3Pp^5^JNIO zx#PdyxZ|c>=Tl>)^60;tr?OR9GE_{o6nxCu1ry zzCQi6*1bJ6{wPVCB^kuDi_l@7F%jaJ@D;H-N%0zj`_8)P)VAnuX@`}V+pghCLr#-% zZ?+9ZmLRuQp$)A|E8pI9yd@lhMO|+)W)G9G&SVIb@f`N;lOiy?&VodpY?dopNJJXg zD$8)ccRjwMY)k<4`hz@-^mWCfYR<Qd|_Y_hl!3 zMVMBy;I7otb-25v4TxEyqTCe$Om~8vZ-UMtgkj7W3T+O!dg&{yuB)0jHgIJZp z6YKs0huq|H`ns$Hiwd`IED%PVjLAnh z?KbID5F5b}^{5>N^!YOyk`U*DsFooHpuX;YNz@HrQoVcj`Av0085W18Q~)w z62QoaHOCT zt~9^K0q~OhpWDy2pH|O*pFP`oIRCwm=ibkMk=t{Rbygc*sDB)JjeQ@M`cq$Cj& zI?6gf3p^(KYEf13OIVcL6#!8BI+^Crvs4kFE^Ewaw*% z@E$~GFqtVK(pbcRXJ0XXiCAvqb`ROda_UJ=<+S><0XjQ6T74clprewu%6c3L653bB z2HuwIYIT3pl^`&)_eh7&GrjB|pr8P&L_xa$ibp}IH%BX3u|`2~x~{FwINBt?t=UKb z!>OTe#CS-&F51LNbbGBzHCr~i&QdlegRUK}9B}UmQVHbB0r^*;S(-RhUm;*1kivzp zI@Je=O=(tp?dg2+9`bB9ck{RY;5Ybpw0n%nzngo~skl6lH~xvI@eL%3RrY_Cr22Yy zzyGv+|KHi(eOB53`@1`P5BvXpJomZ(Ltp1Q$*s!YQiuU=wLkloqjU5|;6h zsB%Ha-BupWa7-i6sb9zr`hCfotDL4NB=pT%ome=4ChbN8-_(tV6laqlV>a7uzwg0D z#7SoVcny~!J&5uTRa8;G-}efIw+7+ht{@z&0pY+xSOZ0`ejQ@bHZSUsxWlq;zYQ}| zM?tr|O-qtGHPJ&%hblR!P}dx)y&)fI#mh&gW`JH&mHfIWEtO)LjD=44!9f(=lKZBT zn-u9$xl1IEGe~c6WmHnxukmU3k=^Q@{!TE2DK(p9tEjOIrNzcjMHSp2`IXTGuDaB* z;K~tl)2`RITllGtKzl4q7*2P8dIT_PobDeW?|ovTVxFNFr>AdD_aPw|tjCnI88vY{ zO|aHt{Z`X86j5peuf5X)ZP6*c^xh|^`n+ZkW)o+K%t(T;N&-Sm-V>D#n#EBmD(w`{ zB>^OaL?*KXnVYwlIv}DA`kA2_i}NW#VT|ci+zQw~EV}>V*IhFxvL!a(`B8)BY24DM5j@ zZy^ZGC%16Q1kG4War3dLAcz!hGR9%6-FN!U2%pkK?GEL3GCXz-&aTO!`Qn3C)>R7z zW~1uX2YJv^qe+Xc{w=#JAdTplh|C4Y8Snw}{&!`samRen&``suE1j1|Qs0uiKa1X< zh5VmaDO;U82htK;wIir{532d~TU_OcQZJq7EW9Kf9Rs=B89Gye?q5T|{mmi`_dY~^ zF4MWPbDiXItZE5!1k_!y7}KJ7Wl37Gzjaj(w%WgKciYzNDoeHml_Nqs@8uE#@FY&F zE5oG|9d^lG04|RFl7)zl&?QZxV}ZOWPK&#Mjt$9$;Q-+@l`~s<3sK7}4VQ>(owlm& z6t_^@ZBM03#Tut2%GO`fKQlW6g4y%G4vNS!f`yr;iXVcKbU8z2!OvxewrCqll9aY z=)yyR3Y)(Qx13ddS{}iNb39%}ndm)H4lt=oh}95R{uN|onbIT@J>?hH8^&zd+uj-N z4I;AhE$)B&^l7+9M*ZO^+}$4TjGpc6h0k^d{b$2x{eI76S0A&Dm%9W1ck_Lq|N3920U1jf5q!9amZwR>pkZL$8)s{gNK|Izo$u4@2V zV*lCQdsegmJncWkf4GBOJ|Sc*bNYIcyes>s=G&j zbylm1R`=pKkHFm>eG;a>>RwH4&XXEl|_=Qz97CA-NCa3VLDg3SX50>~8H(jo6 zyj-GvV%)}?u3>iTrP}XIst%l)SMrEU3dMIhk)`i2vsDhSzd_LHwgwSOWm~RcPcAhn zzpAP9YJJF}wf5YPu6MUYm1&SBS-G`t@of|{oITyGemC6#sn$FEBuT76WLs?aI#Hh8 z8JB{p_863WK#EA`^OUH(`sE8K%Nc|5{XNmGcuJUDMzZy44xw!}T>blYZRhx!cRXZG zTa;Wc4%-? zZUOf^e|d%k9VaB}$O?2sLn(J5IYhag>YSdwFDH#=+qTJpjr-CHCxtzD2Msjb% zPN?rTvwU)8+pkp9avLBbE0hpy&5kS#i}M){6yxc)U;PcL>T8% zO;JbTT(bFtK*~b^g0M*xjj}|L#4U|K7`U z|MI^h%vX|oH!&bhd!}R&?zl0yv8q&$MY`DYyVi%ZGsn}<_|Wek z7*5%k=eA+{f)vVu-aV~E>zVZH+7Q%I&6tUbdhJOlzS|JdxLXs{ zo9D4Hs%vQ`WkaPoTu{Z#B5Lf<%%2jmOHmn4Kr`0X{$|FW0N**WHkrPBFBKT zxTKK}wyrW7g ze@Zfyb7GTf>(Xjw+BN$o4BrMjMX;99W=g}$Yn*N@30mGopTNfbPdtqq`K4OGix_Fw zba_)Izdp0>78gNolT)qBb-r2|_u*-OR+0Y|Wn5eQUn>7U>kq2(|MqtOLH@sw=idE4 zUwlmECLp(*L>tKAT0h7ft=o#rocyAQ4uuSLfeo2`Tb6t?Bb?KSNbbnX| z%SR*n5qW-F>artFq(rICbOB5WvpM<0T4Wi{3((bWx@-Vis4H2Qz*o@!v+&9TQGXi= z>=WW7jd4hj_l<|V3-9`a!WMC(JdWq+w;acEULz@|z+eL@+Cb}2K9hY&$b}8~AlQ_+ zr!s}rP?@b_I;Hv$BAZZwHZ`iAzdn;?;Upv_VK?X;kIY>V)rbvg;ozPY2ONNR2wEY5 z*U&D;xhz;SBpNy~A1$qzGOYzx!nvBn9R1n#XK8h^L>3ca^0)-!MwI*w*r&@Sr zr@93&xr@c=bis;QSn3v*jYUtnt5ugyqv%`!-}LpyOR2Hd+g?(Kam=pVTXjH+B9*&^ zy5`6fdLp8NSWWXdqkd7yQ7aKmP{^i3xP7Js!c{h9Xs2OkcE_#yav2`Hx%n=QTQ@QY zillEGy4L|8ivmTfZXZqKtW0-{hF7eqpdso(4T(W_EmYi4{#!-8L;m;sdA?x&cSBZiGD<|&gPj_&;jbjj&=6~@AVy<4BMC}5fxz_V6C#NIrFMA9 z(}q~k5M@NU*+^+UVHuhdZ2~jfHI?nKC=c7RTotsgdZ?zkN^ruKBv-CCVcjXuCt4m+ zfGeI2gO{8)>CuB-Ej(w3aj>n{HdwyilV2&6%NQ0=s*3P@0 z>7|f0(%eJdpx>W*$eWTW<8%3Y``cI4Ltd>*4|#k2SJdm82GuC4&Fxj1`|EaVbA$f& zu4z#I<4tMESA~1K($V(b)0@D(^K7rw+|IYVtHUj(IF46@d#{4;y+J#IR)G6i1r5)h zwohLj-z1{>^qPR{)@}{*!QM{$4Awx8v2(7ojKqU}iBkHzHzV<2uS6;RXKRr7$8`YT zE(5;X|Le_YyxiQ=!R{I~zP%RYyXCIFUBm`ffc$B>xxx1Kw=0u*p$_Py1GM*6(V0g{S9yH^4Rjw<|Ua6 z3jyTJv~dwyY_@Yl3ZnO|6JtYU1`ckFG??ZHulh)RK?&`KGG5z zbKL8=fSY^P@Lr!=f`_&ebhUj74rCD|j&;OW8=hB%%~X0eONiz(%A=4PnU^W=U=tf@ z^VeKt`taEzzpLfv@bU`tsBpknVfX-oZ)h@N`%M>sbU+%zJ2Fm4Z;EMBe1J*ZFr!!f z0#0kw;l^+(@OTW4>e@vho=aAKMbHG#9$|JMcLykoTlcUrgJY?!g8Qx*#)&8S)ki5 z@-AI$ritZi%?9ve9C4CyN@he-7Wf0joZx7VLNMhsoSK}uR8e%GLdtKzkBpGKD4-NV-hfK~P+#x8R;FSJeczH}O{Tjt0zeKW+ z)l5Dv3};spqOC|4$5bGx3`tZjSPPYw8y17CTt*b~A)eyjRb$_kTdOi96Xngsn`mH4 zTZjbXeJO&8-(rhlFF?Q0I7Z;-HKJT(D()l62-g<^_4?Erl>$}G3G(57QNb-G%<79C z=~&Zdgg8zjHnoUT&8X=_KKh=qnBb(`)GreP5&JTRa1>8FpxX1;*J)9H5Q<_|K{-s;RgO7`vRzPh%Rq^Zp^|qr2AOt zn#EExr{xw_<+OvdhHi>9B)~~*0C?3G7Q#l@M2|1Yd|@}4$@8Ju1}q3Fsw7nKJ2MFi zl_ctR5ZVsVi47WShB~9T%7F^x$=#^ng~9-B&{JlF3*m=LwZQ!>Kg2G7b)_|e^r~IIg z0zLfVq(#wyp{$>j{8trH@{GnJ7?bQLGXH_IsoXNHfFfzjLqTOByCr zjnUTm46>^mkHeCOk$h{VKcV5%-H3!va2S2tS;eRM4F$y}2yDOG6f>|P0`hsW`Mmf3 zEZ%!4Vnii)(oE6HSL*xw#4|`RuvMeqYS3C{zGdVsrJay2&Ea)d=(%ex2t}>&ZmUa5DmI$Gf&fcsZTmg`i0lFW}v(XDXr}UZ57Cvd^}ZtxQB84l4f9j z^xnV!#lZ!{GyNpxa*=Vgw`${V+)&SAqA204uTMx!^3f@d}(XK znp*Ty>DJ7?H6G546z4fF?^f8VT*7mE+iVfD+-$iTPlbnu9d0 zXDO@9Y?C#fQ2w+$3d)tX=wOs`GNUXPb=?WOO1M~Mv8$5H7W)@ly=zkUGU7I)tby5g>mAYDDt(*dJv)k;x* zOJ5HbS6X;T4otINOMluVzkwq_sYo^Qui0hYGAiiZd+)vHRqYom)>WELGi0r!dEn}| z`K-eKi|7h>a{ps{Z?Ag)doXy&|9C&o7vTR%M7TZlP-jZ^1j+hcrG!s#DwL8YLYCTS zh5Cn3S3r_%(VfeNxFDz;7^Gp>sA<EYg&X>y${#t2em>^lGF-le&0T0?gm6Lj`O3)e%OV5!g0;idm zI-7&Wb*X6G%Dz^{14|-yEgDiQ0t2B>ItUSTSXVgJ`&uZTuUlMMW&c-4i+2|PWv^e) z|F`|1|J~1X@B6mPjss;QbD^;y{*AF`z%oefpE}rt%>xTy-n_KBsYn zNHRk+%$3uR@;AmR49C)mtRO|rqM4cL+_%>J%CS4c7x0NsV+vb~kbgjmXsQ0{ z0XsF|Sn&u8@#W1I_?-8}wuNS)6sYw$kFNn-!8c z9ERy#UCrRqnOo>?V!db8ueFv~t>MDDEJvfAZ=4wnyz)6j^AQVdXxW?LukypAiaSC@ z%qV)Qxo|t}_zjLCsfxnE$&3;em_xEouFtgCNL>(Ac5_4%WJIn6lx+E=N>>4g&a*tcq#HJ!j2}Sr^ho|5`=3&Pm}}9 zlj0m}^N1xQI(~)IEpsno#5iOyr3p$|DUSXYA|EB5Ot2RdwG?d%LeR$G+nr}myUQht zPy>9fp|yBb5PQ84V4C6-&Yag}YW3zV!7#;=0i79Q<_uVyCdrd#P^L!kJr?9dauHCw zy6*mN)fBCbuqn+{!=9TU#k-^l?5i3@UZJskLSllq_nyir`(tP?56?}XRrJ3r%;hq@ zwhFjJ|9iUktQP?TZ$H2PrcoCDfbg0XpyY71` z{pBx*r>~D+|6?DW7NbWI$w-)y2yIFo%t;mh`SZSfBg=gL+}%Q2#!OVKZWSpU{5&{I z#_=3+B8N#5eai#S7>(-K2JsU+A(5mSM>$K>#c8;nX>VsnEEIvp0^HHe@{DncW6^{A zB@kHL*RA`qbzftTaIx5$S|$pKQ9-iI?B-S`(4nCu%yI07ty_(1Ok5_{QlQh;ID(^* z7^SFac2~8#zxar!X$-y8ooLtTCKU8Uxg>LQ8G!5bbMVKE#rc$+5I&_ssG44b;CgPw z=tSQH2KW#bBA*&#G+ZNLam=oq$Z6k~l7WMbRTX8o^3B)5tHW?T#_oOio$LBfm*8+H(Zt-V-FXYRA9I zf{k<=aQR1JoJ;1j$r6I5m?jRXnz@3_wyXG};~4-Q;`NSCNs!4kz_PRm~%P<7Q@P1M#CB)v)&O7hCx`6X$3Ab`}+rW&is1H zaH}A?9v-@;>s1Laf|e<0q!UxAkx(>hj!n);Rdj*{(w8p|)UtuQ*=bBee8lo3JLqqr zh(R}o(b23Jw}r#a=THEmn=XN2t5l&jg-c~M*ak+RN~ia0ErHEKZ*^Ht(5>~nWLY%L zOZoCnGg@2(dtFp%PZ4lJNQ5v7vGPaH7~E(!v5r+4Eg8RZB=CMX54)?%l5;+(qNXZhYM$cBt7!_lK^>?g=TUX3CKKGt3H zQ6oepC9bH1)vLdvdRo2CwdvY$TklLa5U;lkutHv{e`~$~T_`}yd4SPDD|eVpyA5}& zoAbgsbGIM^L;jJ*6!WT(wdXpnM1M{`c(JPBs7gPJj17|GSsx%jtiwBqcrn{?@fWL!n=d z?q>$ts`)|8ikoSDrjNVP`5I8Vmd0o5>*jEtO|Gr>n~Sd7uP>khX9g9 z+J;jvQE&AtRf(&!)T$Pj3bw1pnnrv%#kZ^-mmnyQ27q)4Y}C7+3}FYZ!A(vkmQC$B?sPGq-HUF@soR-mhuGX=~+%1{~AuS8l#> z+37lKtR&Rkq4DMpms@EFH(6)yjb2hB>)@*O=Kt%>F~pY61R|8Fjg2(~PF+&#u>ai0^9Aicl9HZ({})cif4hAnkS5cw#|Y$( zw%rn>z`mg^$nENm3_^AEu4fR^^>4)>RBo%qAhg(wQ*NTgjI-3JR2fveQR!9MPe{DbQKYOtM-N*9H@wqKs)8nfI%(eZ!+*TgEy&k;1>KZ<*=>Gtt)|dA6 zDsj%Q%Bu8*J-rkGKKOZk8C%oRJ@57`O}Dl2FNgU$8WuGA$F?!S5LTY;>)MwZ+qkK9 z2^#&%ZA&W~G1^79yEI|k&GNdjnY#*Wa?MRh(FAbVYd!b0L)X$J1r zys&s=Uz4HXdg60LLK+d9e4a?a!z}J1Z8o$qQtc1RBkNg^y84HqLheyT%nfE z$>SOu%Cr8(Y)#8|(Df`4%iD7{I)x~e%XB4f*h-+)PL?~&Hfzcz?CoZCP90d9ZAjhn za2Dv(t$nQ9@GYCFoFT4BG1oI%EokV5mMeF%zHalCF?Q5M!ZoL$MXbC-W615Q2P=>0 zPPj)=^k;h@x}1Z6tXyB33+NWPYb>pajK!`{4mQ20WP4!!+Ah@G;~j7kS6ICsD=CY+ ze_$$Z9MA1>Hy6=sa=L1(OPac6e&_ab8Gia%mmY5|U{liL|oJU3yqlZe9EF zy7pgABD!uOux0tbcKc7O{(pPB5Bb0D<+&aE@3joTr;?QJ=K8Oq(yzw$3*fYyec|)X zEWWCTyD|3Gkhz*!R#jWgBHQY4S8C*v%teqw*-Ar@dJJ>=ZxZQ5mrEOSssuBW91}7^ zwA-*VDPi)to6iGspjO7=H91jqm)5FRAip&Z&6oz;%2k|v*6Z_*Y=Zhd;jAbu9umi6oXx~S2u48>lvS# z+qj`-rW*Y!O-^dZQ_16vBBn;aGN!>Iqc_ew)Uj?&3TfZ=+oxI8ap!C?>Q#1fdkUjc z0803w#mB;6R494QFmS9>dTE&feRqHcyWN#%qEYO)0sS|&=NP-(oyRcs^7gg4hRoGf zd!g4Nd&v%Tmd4tN}bK#jI!CtaXukR6w$dd4b&z* zqe&L2M=;0JxOCgGQ1)8vDTP?kYENnISD-OTApLLDTm70POwscGRWUs_4NQbH+}uUG z-@^)WeX&*P6xG%`R#KRjQ;(p@xTY=?G#uE8Zq9)amFc{9i&y*dF}04x ztEMPex+JG_-z{56!_J=%i{^e8O-0qHwT}Gq@Jvyg=-%+Y!jLRSn1Ti}3f5Va<+W3_ zSzc4Tbyi&4hP(V5=xiuX8)IvXe!sQ5#FKGaqp|OFto1!(-()&U%d#X(arlk`B3)H;yDVbUKIV-H#-mmS6vC(x?s5;bC#!)r6?LhF&onm4M}!INMZp|^@2`kV*{PP{M+->PUpji4^k%XL?k6i zM3RsaEqBXZabLFwJG3@}?xXGCWAM@V%P>JR#^?Kpg>=8y<77-l#^?SZh*)?@ctBZi zsQOpk%PyhckR^h}q$5XxZ2xnVr&7?g#Q;2<(zmAj*`RYtlV~4F6`YP=;j}X)8J6O& zgj~Vhp!3KNA{gaytehF)NZQnz&XsI~ydOy%GxQ7NapWl=UNUwmahoxe3xZhA5l*e6 zkdR_{%2|}dDZ~Fl=X{RdDRU1x$H~8QbXb%|M>xYV8y6F;krRY$+9OFuc*?0D-7OGg ziWi7P6dGV7JD&~hs>wZ0G7C_r16zv{d{WM4G$I0FLn+1@i6$gY2^Z*ULc<9PaUw-7 zIBtN86*CgE6eZ+JQxVFgAg^e+MzvkDwV^3a@R;zR(PS75$Tr~TSxQ2RV?>iQ&yWZwWQyU-20EVNF*#){D`?g% z1xa4nJSF=`fZEkTC?X={6m+kB^c>a+GQ}OGlFCMRug$$VWU)W3rFz>gsfy zRk)<9Q(F?Sr`ngR!EIR+WG@-hhf4bA@ zq@0v{~Hr!NM*; z(G5DChzxTF-9vRGSVN5(3mm`D5t_3cU9milP)sk0{2~ZJ~}USS-KRrQDXdMdfAZpZ<+lKvV7! z5+Po(h%5v55K5vciwcM1R=7(Yoe~`V!f8g{zyt{T@1>oc(SMUQdieGqw9}Ca&9Zh+ zw3TmIOk`AQ$?T*H$=Q&On)Hp*bSS)TAP2D(1Z*I{006xKYM9flrP&uHKca$cK{8n~ zO4kZ!{E@VQuQb@{T(^En=C5!nWxxGuTl&2f167+T2_@&bBy-^Wuo1{FiPN+ENRu`o z6#Q%mn%W{{D^bjXx`A9^HxRP6L|MqV*8D`~7L^j0FklH-E;-dNQh~nGex68>L{LzP zVJ$-J8&^c23BnriEhN~Kpo~t5kPTen1Qz93)>e`oR}Kj&6M_zDOtZP_7G%E(p%_yBh^3?`d4Yd*An)#aE4$>&vbP#dt_!y+h$NUA?NB!Z#`YEhkn6E@tER-;upo zVa&as(D3OlNkV&N^YiCUY0yy`7b6aWK${s!GTXP8myKF+a(H(3%bU~Z_5)J3#UD7E z?mI7$Ml?8_jmlpZpv%mT8nHGM*PrBk6B44yTJi*zkk0+#QniF4vANddArOp z@W#i*FgM2|$&p5khjQm+!XDx4i{WqvA$b;!W038DmEo93b-JcX11pYWB&A<8o$E6n zFqBs&>cI21D!4rGJfQf;m7*(J;;S7zG7J4eHc_$8x1TU{?5ZOlQ^qsoG;{E1Q|g6b z7DHuoV&?;&OL0i-UE+ckkk8Psf+Z5X1MhdpzZ5uB_}P=}4GZ#g7x_`Q(}aS-wNNl{ zQ82h26b$YF1%oOIG}Ayor(=#IDX9PpIcI1P^n-q<^X^T8Ub7jQ4hcttoh`K8-|lZI z6?(R9OxfrP3j`{NRF6h9p<+U!Ld74`Y?2Sb_#uf`n@TrChO=DwY~*JX;%log*hHv6 zQ~WFA#vCP8J>_92q~!{WvKMb&2AV7X zGQpXJ{fcmcs5a>kee7Gs!~g1Ze00Le46Nz;rBL{w!X0of2>MW#NIsxV%{2fI!#NtV zY@#Y{!}xX#@|}j{)GA{gjKAFGcD+qyT|q2~=iLCk`;l<6*xqUEoRS7`O7)<8VC!8_ zdqbQ^qKeRz0geh~7Y|vk)|CU-2X$~BbbR!)#DWrov|05-4W`*nPy%a&f9q}N(mnv^ zYcgK~jqLEeOidCFfl{UZ8e#hhAQra!Cva?nCRm`M6y6z+h$LtnrxQFR84Y1YfrL9@ zQ#K|Eku!gV=SmKiJHUikTcKkc(=>J@i8UfzTQT8iDmSZLEfi`gNmAFbHZ^F;S(PQ# zl4!))6k$0uMT;yUW=}GDns%(LnFz$a56_!h@6PkyHql|UPqcxMMx6N zDU%FB@@zd*+UVleju;WLd&9EKdc?_GPGkW_F$&NVDvQk>np!jxY_SOLGBWgD!pVqm z!>Hb|(TIZ45sF1D%%x@nbGCT#%(gFPI~P+rRukOYuz&iwAbyAi5na7Z^rvL3CeMr^ zV`#RcspV%uMtO{4np}eQ^zWu`*=OJyv9mm5oZ?vYI1wzK5nqD04gc==DHEArfTCu) z_nq6~Dx+zvFpSx_r^*mtjs*JjXWM?p^q)Ju?_gWg&X*m>L-Q{oOXq)++t@Dpo--VU zY^ofLGQy|GGw(b#Lol)C9k_MMIqxwpnLHBtP|%2CP9fls?k+-k5^0zC!xV=TVgT7f z=JwKd&_|oHfTurs-M@7t^-b{{2wC!Ppdw%pMk6G7*nx&b!K zFz^gHShHK|QV3_A4j^^E*Sos93NU~eFh1_Zs)*>lJU)8y`V3rDzw2~Dc zVQyr3R8em|NM&qo0PMZ{TiiIZKfFKlSLk%IC$L}KG~_nZXS3gz0cOH8;qn5RIs4Ae zlL%Ygt;KD5Eg9&|*zm86nX^-+c*Otygm{wFZF3t`GXIb|^W z(_m>{<;r~}KQKuMgA9{|4%Yw(Awy%5&4yqfGI$9oS_1%2;TX{&_+t$K00NM}DH?*S z{1SyM7SB|fA&5wLg|a>-TbGz6aEiB7vEg=q=llLn5TQ#9lVE!%$eugo2o2~2#jW&o zx#6}$Z2Of%O_+5XDN$(ny1&zZVUzm4N-03MDajbhI*~M`hui&^-}MJJ@fRNA6eSEN zsI}^D>8jwb*8qrdh!V= zI$YDzg=9J<$r=C|N(seG*aZpkaBC|=V@z2#3%2_eIYT~}aOE>N7Dg2S^(HVBuQmUj z{%!wuEkZQRaLUA^GhWU0vkibLfC(TeN&qE!7NSi6qidLiC<0e#7F@$PM}RVtp*}cd zygV5(lza0~9AA;qh2}gPonh?auQT{WKY`U7^_x$s*A_(z*jKxX{5`AnGT&>l9*936xna z`ODD_%+Mw{Ov3&e0C^S{LX9z-v9)IllKPKVTUGQj`{|C=sY**(0!S>_)e;+@0GxA%*`}X(!!CHjJh_WG| z6S(v8)ewd+Uw)4+U%r05J&3;he((bAyn6Lw=LJH~U%d#USKqygUZb5CFSlQR_j-FY z2w#S;UXNafFJ2A?;m&0ku8mQGGRROg1ihWX&Wm91JQ#d;vHf!R^40Lgpg(xAvpv|^ ze)<3P9*3CbAKCxKEJE*U0nph0cV0gq?7S%V|Cg^G{r`LUc>?xGI?M2Q!dx%m2{=F9 z|KH%9^e2PEh`XI5jItrvJ>A`Vdmuc^wI@%&|3gv8JvZ=EOfF#zQkcOhVko2ht$MgG z#tP}%mP(w)2vW2O62cH*6UYFYUD#utWLeOpSn>^3T&ej?Yn;Ax5QV z$VdjqXfKA8S5K-)5K-j&4i&G0^dMXFtr>z$}9vinhF(+PVE~ zL^5#72XxA~wtgXTK1J(vqe@B)CjAsr%3W1WO!)KvouTpn0P|{HH!KAoVa_JzqmP{I znxr?>Yfr7KA?Wqi0C@5Q9Q4P1`CC~bL;0AmI;J!I;m>F$UmwFM5(PAA`%CcK?`yzm zKgCck6vgsP%UOuBN2V!c_!7sM%>ayGs@Bs&aX;smC`%AS6r?d^Ba%(279h00l;m6K zounwC6Fg$7xcGm6{ux5pP4Q*w-^?9D?Lp=$zeH+z4^H&(X_=w&&G=VVQDM&o5k791 zkI%(&n742WGlpRt&x(Orx{7@<(DX-n9P{5eahpVmx-RD7z+ZKtUMAs@pQbRJmxcLG z*T5lCVj{nM2QN`ff%=CL03$K$z&KX@14V#b{u6~vG?o3;5PbSvA-G#?$pQQKuB5GR zO7!KV&`w{L;}k1jlqHruWuNfXzHMc?3q|scZZ@aPZ(Qd(Jjm1zNjJC<eF?y_*M`g`fHTyileiLa&W0TXa|%GH`7=?&QIhk#|$6S1uJ zjDMYTMi-L|(Futo@FT==p7C)`KjBk>7T2mlWmfG;FY9^YDIz%o`d^Adl0+0- zB6fpN0@93JBDxTJNl`IBNQA!?VK`0*WjF*=5+Q(-(zL?q6ormRgw*`5q<7!-Q6fj) z8uIJBA<>R(2RfrksDb{tVg zJ#{bK1pe(%?Htn>p|pu7%*GV#^8YNtjeo6K;bepp%x2E~W4UvORc~I*L_f&w0K2Gg zw)ndYjf%a;Nq(t!5kfNE%G~|JEe6MQ%d%BLq;GuO-_%S$1ZM|79iCsD{r$u7?$N>T z6}~=yTGP_|8xFP8lk>xi6Gcx7hGQ5TP z>LuE!2qs>IZD)r>cAX7*NKzu)hxMU`GtVOq4(CU<)%Vw;{77K7u81Zf_}rzFPV ztSSY62U79U`<-5-_&=1B3$4}}UFKumhgEgOBi@O*jmZ-d;h5`%f!T{HZCbe-(Be!& z9eR|HeDofb_Uq466ynhgz+#~w)gWK*OKTCG?AfmH{ho2n7Hy3wcosv3D0{XEP)xZy zE&SwXhofV{PBTPN!k$TTPPe!+^JpKFt%!tlD?L_Xb?mq%*7rJHn`Q^tqKDNJ%fBg_!wPP8&f<2?1d zRggh?C2A)Z?mNHcs5847gUZ%O*7KUPyiy2xTMt*isBS*kETRa!jJ z!+*`-D4j906(JW1?#@TkNlMYoHK!?09Ll9d0L;P(=Bpa+HdsR0&--9kt(f%M=p$t? ziC|0;^bwE@e6-x2kDP=3`OtQXIBSc|Zq6nsVd`b6){Z4CR&|OP!^t>c6BNj!yg)C^ z1Cj(IoKY5JxzGyC*~E1k#D+u3RWLgf&=I{d;|q_iLQJ2@won5k0lFw3IEW^1&U9<& z8+x9-l}@|0J$kt1A}3Y;psKcgAYzk@3@<4P2cJj*cRp8O~y({lCfDkw=np2Uc1 zsa;+r*)5gV%H*#?PLVG2BaRUTM}ty};p7>QRnp$x%@w@*&F=)9GZYn+ipY}#DI z40D6kqLgX-JY1g;KW1dAY(h$743^6w)n{p~%BdhwSro}}bk7+^f0Oq6yWR|O2U*#A zFp9)%#E38{N#F>kl_5y(T4J)rNQy%}2NmmShxHKr5%4ba2~1Oz z@WCmb1zacG(-x2T6EKr=HrSVo5pcHu1F*i{6l|;q0LB@D(M*p^e2%5?8oB!u+OT8= zrZBDTutqtu&(27mfiS0xOiQZe3qPwzCnQ2tuD469%vOnKW}HH?HPA3KuE`?quq~hU zLt$XY`ScQHydG7;-0EQxB72iy9o?oV!`$j&ya_V&uN-HHax1~o4RYn#C{|y2!iHem zFN$4*y(Ud5(WJtv4pEH_8W|i5ER70E7`2d9D^05QO)6use$7>*tc@fd@eCck4i3pQ zjS)lftXMN&&OnCvyqt^yg5gAMd7x>^W@?V{xqs!_>TTvHk}#jy$taLcDGmcIv+=&% zn(Ec+kMG`}zuo`A{%iN$JAb>$)wtOl)LX{J^)R~t>p4ZEJeC_matu`+L%P}<&EYlx%5+4sKDnH|6$e_q@U=vpfN5hDjziA{orY09ENp z#$PEY)+*$dW$?!sj-5UME9^mx69h)0pttvy7a9d&z0L?gF9@dac8aDXn+0-Z46aEG z8IDoFSlqKqatAuNgyB_8#sS6uMqqoe^J040D_jG^p{a5Q$@MQVbE+pdOMUHTZ}c_} zP^F40Cf1Tl#)|xSa-tsrxsn)y-X+Z3O6adURAcW59IH9XGeM~?BSsuuM_#v@HnS>J z5@qkGZX~r!DX*p?kb1r{X=wWjMID~ZO6TMh_24Z}T67K)<&cyu_EdG)kuB+GuD4uIpfngI! zHd{}uk1fWoKE=3_r>DF?c_nDSYg1+uS)5?*jqjrv&d${~`k{OVBL2{9u5*>zpSF_b zo(zdJ(Npo);?$Vjnb+jXTmN&@8_Cm(w&RPrXwp0k5v2vqbNQf+rZ~?po=ID;Y~KUm zX{;gG8OS4lTiZkMQap5(rk|F@lzt-sWtGVCl`3ni_S&WMa!$i^3D0Ml&bRHvTWxbr zdL68DPKq7vb55!qEwoBi1npZ7OE#)psfS{sX1w*2|bM4_$G-Vn4kzd);1zy~_2|xjm=v$t9ID&x^3bA|_29jmk9|NleA7 z#6&nmE!%g_kgfQ-(qTzL60`{tn2=3Cr!bB;!4yR}pKgK}W@EGoZq>g#@|TT1xR_vS?OX#qsz|9e zhrzn_Uc`e&MvC>Q%{8)~2Bh-AoK=Xc2S*}9C$rF+UEtG>n1G!-Y9TOW)VeU`>g z-fh1u1_MXEL(t=j@vdTus+8NHVkn|^5gv_@h|n#*jPVr$d#CRK$$(sCgLOoy-1#o) z1l*A9Dkd;e8z_c}-Q>JXH?ntoO_ zARS9!x#$@`pgiJ;-@l>Q8O(@OisjW%Ejr;@x4eUAFsTSK}cYzo9sP7qE9s zx^#qERusL%Nq&2xHfDa4k&&B89*bm#--ThEABaL``vxb40CRCO;n_9z)>N$`wtIe- zO*Y=nvoQ9#m>!elj1aakTlM{1jD>%3n&E34qcJ+5A&lit{UHUIfm8>wGXju&=|py< zBwD8%Ak_rCqey+N5sXffc*dpvu^5${dD=Ie@bAQ-CdjzHiUDm-OAfhLcbXASpwDJ0 zT153OnYkngVG2cZ1V*$UWn>|Xvx$iFwTz^dJ=c!It=qYWfcI*-QLDR>89sq!>O(Ln zo+vA{sMgt-y%pv+1-pc+XPtAxTWtu=@`SJBR5*%wWvfLO=BgQj-+H_6-u1i(0#`VR zM0hsB;^=DM1gdU=a|Rh2s89HYfWN18-ow6#kr6fRASz!u_qzH=jQ{;Na$@sl^y7Cr# zBDn=hC|n$WxMG;REB)X?3NPIfn1D+Yw|Qm@ya`T{eH0^xz`DhEV-uVq84Ag5QycAu zovx*NO11iK159zkSKVgw>n*tfQ#uwWpgG0`pC z1W_)czVd?Q8UQ6Was=_5++reztr$`&?7Ggmuaqo~YD+KbEDz?4Oi7+Fb*Q`>iXe;& za)lBVU!>ju^$IZY4ktkSYij^Bk?{pKDeeMYN`lBI5x-Sw@oRVzc_JJxCDa z_KTD87L1|*B`m{;28;-26|pej3}|qJ*+l#rC|;%mTQb(2sHP-RDM<%nX%WjmmK}La zZOApR4!YS=Rgu!F?3^vhZOd9_SP4&lY3tIG?nz<`2+0<^ociP&?k6xqrx^*6G;0y{ z49@ve00Cb*#t5YHeGwLIQfgov>vT#tKV$g`qIMO;Web!@ff1>gAAB`x=rUYSBbh3q zHrY-|1fY7PcF_9Zu-NtbsJpr<^HFVk`C^jEh{UeXbYb>^fwjiZ= zoCuvHBtlzH#ByHzt>p<|5(O|}*gUaq!l`TBC?<@DVV`1by5F95DwQ4rE~OOhm_!Ja zs;5+N7qavJR)pjaOA@ZRS<_0uYC-7+iAt2D7qj~6TCwCT2ovij;T4)m0mT@s!?&v-*M#NiM${E) zm~McyE|V;`=s8BkNL#vs}r7)7-h=ixUhb|dvoA(G80W-$Y4ab#Hur4 zMp6=!@$5Y1-v1s+_=*Q7i~{)t*pF(fvv_Tar5M)=|4*Pr3aF5PdmdT>bwpR-chrh+ z=WfB<^DA^?qOirdsJ;_<5{b^u60f-RzE3D*n2toq;}GnVW5UkWvbRuf9qSrZM+sHO z4*SRFK>b>)(s?b~vpk3rD$H5Et0>P1Gz5Ec2jEAXQPX3s*EKQY36(qVqC#B)*zc`g zQW#?dYX7A1mAOr!RWtg3=EU)Dd5hA(EoRQk3FXGd#q2gu+=Eqql@y$!^3Q z_J$3@b-Ez}ZTTV;GE~ZTP)l@@0XS0IC>bk`Wn6Z3LWwgY3CBZy*_`rBDaMqEe2*gF z`C5HbUk$mNti`bzHAe8tGGd9oh(zkcZrb$-C%>zX zoj@Dw57vpt^anol)?qS@2jT zDhaZdko9qW>ZrT|7Y6oXh^N$c%&TV}jpE4I8gQDCYXN7Wf^EcL@lhlk?c?mD9@_dg zCt0$7A!D8NoPQ%3?$L#@A=Zs|3fV-Y=!su=SCT}HdoFe} zQ3N3KH$l-oG&8w$cwb2?Dne_@&=%Lr$}{`K9 z2)`B$LlI<@w6%aH!2S$L-5^C zxIQS)4e!m9Y&(_i)!5WRS8 zlP(H(BW@C|`MfF)%3+b5#`(x@s!>`rIEz7RQ45euk|&Wn(wfxhdY{0Vv*l!Ni&LyN z3ui={wRnAo_ybH58%}4HC%*+(IY@_`YmjNS-p%?Bf*Ox#Pco(ekHpT}gxqkIOEjlQ zoC!zztd-lNSPh0drCsFuRejY+DjT$ga^}r>Xz-vr0Ip0fQJOR`qbJ_I-W=Bzo z(Bdc$kw4P2t3>;%?)4&xkaL)(F-EkIuOy06Va082U|*ZqRVZo*&JdrB&0{64_1qiJ zuJ$O(p1DA65pQaIY4x@?a6*w~MYw^r->-Ia4o{nDr&c;>gXAQ0cJ+k|>CzFvfg6C}*7jr|!t@shKgrR;^@Es8anqq zpi36~Ry0CrJ-J=MaW@@7=c|}pdNo!^tiB|+VllGw>Fnb>i_k}~PSn2WCJ1AaM?mA8 ztkk5`kjd-Nri+$kl20AdQ$2&p*c&9}SCC3}(Oi}q{!CztP+^eDU7;)(;SAltINo}q zfyM(xW<)Ya8~W_QS}B;r=DMmn$Ish?zWm={Yv)CsZk?}Nhci7EAD%X8^+3-pho@bX z+09zaOoCJDQ;8*-UklDT{LX-no&D# zH&VcwF4T9M;wG_61bW9LK|MZy^!CL3dwANjHb#nv;<^fo!kpo21ms*@FEPDPT~Y6p#LoUvS5fjE-Z6I7a!TkLLrl=L$M$1-WOyGqs3qt&=3<4P~(OJ5XA zc@H7WE<>eTXHj3yFMd84G}Nd0JdmMmiW6ZjPr%pEcVZzW@n4qYRgw`#-0Av1% zN`f|3l^4I2BmLF@XynDKAq8}!T;21=(uzfuXpsZ%>_ zqEDVx;U=~VRD+yo2_@E?M=DHBo#BW%S}&xP)6H3562d6UV4BKv>{fyz7wn)!Njzkv zpe7)&yO0yx*r(2}dUaFT;$msUB2#-D!ap&tYbxTw3dqOm)V(0+q`%SrgJCDNN*BbP z^o=x4lve~j>7~6m_N15GGT@V5g3F>$dWqf*{A4p!+O`)eO+8%}+iz#bTj_-XC_7l_ z-C$7qsV@OS=_j}p3ZKfivECIHWexphz$j}-E{8^0Lv(pK z${NDoJ|3mfNeRd(arZ0(C+Rd(ZB16DZ)XCGW;EALvo$~pUc9i2Hfai;*5 z9jv?+!LnO>G+AdT3a03mCL zJ|aTaAVO+=Jq$?5;)J>Z6SCx1FM$eaxtK>{wawM1JZ#@Hkq^k90X{JB+L2U&=m0VT zfBy5)?&0x=y_4gM-NWO9GjYfNpZ~lFqF|R?G!g|Bsc818T6VPVw$H;8pJ|{?Kiwt*) zl3C_m3zO-30bdv>vv$b+xaq#Eh03gNy=Ad7ZOUE1GFPP7mBD3d`8(oe-ici-jF?#n zzc^&3U86_P%qGxG6LQ}HHB%#0Ea0}@D{iJ$r>?-6_g}RK0-agJC$(W`7WgFh0G{bY zE3X(m(;ghU;%9o~xdQ;r>elaE5SnHww+x1+PdncuvJqKl);hiCBdiA#yq-NJzEDWmIAo87{ zYSvfC7PJOe(<|&Fu;wa(HGL|+CuGgK(Ps&0O>b@P7F*NDa@F9PUQIo4bWNYktAp3{ z&e?a5ui4-#d_w@6m0Es5gw45RxgUtlxkdW6FgA;@k$F)z&5E=JX0t+hKA_Dq`5L6n zs{D&#ZI;<}2HVsEbcKd_pm3Yz`nTe3Ruun@B5sxy(~5XmQOviFdFe5odj=&fOW=jw zthCGgz?)@0^~jqQW)B{Fv#D-!dDc>=Zw`L5Nh5Xmn=LweMBr@gy9=XEmQ_FhqVExS za&^FyEjs-gktfaJ^<6R*`Btzg%5eMNEr%_rZNI zgFZeY(pgpe2q$=&Pm7+c)gw;&i;tY$@Zx)&2;RaF-WCku?feXj?}uQ!3r7aCF=BhC z@4+d`LXX{vhLV|lF8(g1OP zV5c>VzdE$jntD9KowmZA)@knk@lMZC!f=A(M!eHCs!F+tr~0XbsOF>XvgC7%ic~o~ zIz2hN*gd`&%9BCiaA>qmUOxqMCSko??@FZPOESvtO(40&tk@e!eb5JVsK2i z{CXBF`WYHy%CZ^wO&RSFoE`jhcz$vAcag~QdxLoT+h+P)RpLm_=5Ts)et2=Bc$9>0 zW~s^9>0c$D42~N}@VD!DI?b6h3^XH)s~Q4j;FzfAy6K0&Z|!P=9}mIn{!afz5TQ#9 zlVE!%$X?4+8__cMEvQn_|D`nT#}f##^bdB}~CH0UpAhZ2}Zilg#(* zaCA(V2=h(Yv$>ZyR!EIJ_7|Ii0arVU0#77>6%u1}){H&TUoZhs0G5(bQGeC)#x2m6 zppT}^k(558Qig7FTKcP*XL)^8WKf#nYbelgq-kvV4h$`;byoG$*_rz6XEc)qRKB32 zI{m4&q*pfUSZi?t$e;QWbd32gy!1C6P2;ARMNdO=lwm7pxpTz8k-oTsfyBy?GzErW zIy0ARc_{rWCs{r%#QA^4-dgIJU)`DgazdP;mmzB{X*pF#^Up%wWQhLdU}5{EpKh z2~zv>`n$`Ssfo;F;5CO_C_ajCNl;O5vGJV{aW>w!6DUO0MhLPzL&b451igsdB)cPq zvLhVFSmhk|l~jd6iC}j1g$eie2{#0<1_QCD zRbns%+ux}-9%=ZWag5^gSrSs~@&V)Q3wu|)n!LZOiAPLf`eU^!Rw?Z_X3-!b!(l~! zho{MnC`&fStp%oiyWU!0c|RTQEwDtoKPdiUG`@Q*{?a<~fr0TCwz>i|{_|4HH=*$t zYFQOrz0XkZ1+Ly(l@;ODdn(nASML>QRe<&F64e0K+fuCtvA&MtqLB4<^d10ay*+Z< zp!NEBH{5!E+xc+o&7K(8`PQ-PjcOMJulJ2RjigtEUhgNqIDEaI-ZJ?0eu~Qi*!!t| z>j?Iy)(b<}cd^sE!LY9(zXXVV4aKEU>}zN)4P#$J_aWifugGLy)h%-uMzZf>vv&n$ zU&mk>Ec-f|%YoU~QC%L*zK-&@2xnhoz4wi0Ut)Z(fc7Qc_l9V1=e4F^Ca<+v2rrV@ zN}X}4C9xH(AKY>amdMN-o#T7FB0gsLtu({WtK)l0@$bgTYZb%gHd^1 z7T8=&sNz4&K#btETEJ7J5z#MWm|Si0XET$JTx7Fd&SZN8a&Vc# z)ZTmoYEM=~@#rAvyg?L$Or)|2GQ7 z;?vAMBaKu<@IsAL8l(%-thZIMb>unL(V%FT zFx1Jn36fv~#Wh=HFf>~h?GClcUk{d2QP+b9OKGf}JnEv&mHOt;QW{lK2ba>)@{e#S z?E}#ANmFyRz4yt}NH$4im z)WhQ3tk9VAsvJ=x936FU<*{9`*zI@ISt3ski_1Z5JelDCeUa{{a{Tm&CNHJpjTDk3 zL3Xa%uM@El5?-0m=Oz#vqT+fVBtZ@OcwYH4uU!r8zx9( zUXY|OC6EXwRxv@G(@U8xv`oPi7o--6xnR8_XoM)!eTU0Yrr0(??;wf*1O0nqN3M8>E+K_h_IRPckTUjC_F&c8h2}2qQWDeQ~w|tdI zoqKUy2I5*b&c*i{s4HbjHUx)>zeI5to989x;0Uib^0bo6JB^-(Jt(&;TBunM7Hw5V z9^152n;vv&Qq?Js&yNZ{fp?|QER%$dWJ zBi&?JnGH3FBK~Qxt!C?b@`OvJ5Biq&_Y_pIynamuHfZ?bCr{SWGAq~9tGw15UKT8x z0kf)z-3Pz!o*f?^|1|VeNd$1E845MPO9|3>n=fu-C?JdS1YScNOL(6|ynbn3n?q6s zU{^(FsMez7vsR>gM#0$+yL)n8s<^AhD;H(TZL^kL!q8Qty;WGn>64sHfR7KQucd;5 zzMMl6CB_3|E}X6eqi8Oh3X2NDP*W@L|MBp}+&9Ol?vYbAXJ%;7Zmv18b%>*)mj$U<+^G zTa(EVy!A3xyR7!S)4Szg72k*;JR2;{E?7(HXEgH?`x(u=1g-M5H1~E3lDn-U7NxxeqlwsIrG_N` z#l|4ne=7rcsKBoxStO?xnx|qXY5%PV42VF8(+SFoV4*T$-NDTM?vZlC-5p+QjaCS% z*8{rCiL_0T$0F~7O*1m(gI!L=d3dJa%+%Sq4cjFNHFZ!A_UeJAX=fJ0q@Yk()XC5LaE6CBW4Zg}DJ&UE)=|)e`;t!CQ5O^24o`1gnEv zt&;Mft@>!$XsgSDrs}qr5p{4p1l#?W-}MJJYN|ZdEup49;*72kXS8Y*CBPwGEi;I| zy|_5FGE>whP%Wqs|8@wz-`-KK))II^n--YRf9vqt`+^lx>@>WPS8WS|3Ar~Uw^Lpb zmXMeB;!r|fa?2nIc?m8HB;+M}H#kB?MEm^%3BB}IdSMu$4i~>!4 zpyz(q>7Jp3T)y{;9OSaTH{hU(uUdm4tPkG`UzMB4>HvicQ;#sc5LY_AC6s|(vHWAi zS$V|bR8)jmS|injP-b>fzIOVmrTK0~&&UcUH*`r27)j-_^+ac)1B!n?1+4FB{ zaDRI@8|%P+n2qx((%D&-RC-mmDPABaBOLX|eLK*C(=$go;RJ8;#Wo0Hzkt~MpTQ`Krx z(z|i>W~9%-wwjUNjc-jt`W&2n3F)o8yXJ#;lcQ5Uc)LFCG)KOJt+!^Qchg2gHhQZN zt;y)!M0iX_UqLduGvlo|fpkez3MW*J@VW^;&;1b4JqHoIv)&{QsD+ZXegzM zmpR{_O0y09q z_x<+HcCUUYoE!bZY=Y~YZ#SVy#U&Y|S*IIv|C@^|fCtnFt{`uX$R{e+TvIqA31%cS zIf)7+HnP{;B4$VxbW+AEZbH5MW$=!QX66I$D3ec%L$F5mm%^bPSPkbGtg$WHkO?Y$ zzfoiYk7Di_n4l~{eYSn2O&)xg1UOknIEMlf+AS113$mFqTVj< zw0NBlA0&T`SSBRMM61fbaH1gct&ndmYzYv_7Qql`AF8wjVp_O{u@J!g0h!`&*4-u^ zTQ*SSW+lnbhT!?2V|z#{=Jw4A(kE*wBD$!(G6*aZqFlZXk?vK_kXSdw0<%F!D4ofN za!X#G4Wwx-k7UHEjB2btZ5$2=xlOZu1k;kp$BOlW;$S{#^8Q)EV=qzeF4Tl1BvZ>e zeapE0%&Zv90ZP$IzMM7d+qwEiu{-pxav^}uuCt@+x&KBjv|ijggn2W)OmqDc4Snz9noR2uvx;$C<{M<~k*uoN9U=nivg5Oi2{Z(3GC-Rt>n^`bb-Kbcng1{Q@d(Q_{jk??} zA)5?APi;|H&*hGUlFcrk!Kro^LD8l|@LO>%Jrn?vz28l#>dUKT(J;fQ40Tw5BBo`cuGWvl z!Tno2o;ZA^^8e<_q8dY<^?%?~^@qzDLv*RO`3u+{3`%^qn)puP?ec6(4E_ebcV+Nc zh~Gq-Y^1?Rma-D82Q@|rnC4;;FqK)#HVeme{wTc!OEK2v#L=dD`)85j%BY2hA;s#~ z8sE=oc2Iy;l&!BUvOw&I6s=2h5O5bL`f!D2^aCRd#>*JQZ!f36Z-Bo6j~Q&!hiL{A zNP$uU;7Yjj=C>L8EI>%1bCqLA8F&e%ILTS*zD);M$-pHXCYV`a>{VRcXEMa^6lGH^ zu?e5)+zOKxY<~pyCi3W%i$iYsdzeIEEFz*6q(^Z%qY5BYXhOm-QO8B$M$|I8A6$t#=`4haS_$g_CYoq!n1yDdt3jI3hM^X{<007IvGFCg zUx_!5026l#FmdP%i_hr8iQlf;=>w1CZV=eSbhniGN~61*YV{cBT@&Z+_PmFdp6Jix zt2XpKB@oWR*i9wGw>|Bb8ojYKRfN(-? z#MGfpB!Ucr5s9M$pfiY&yck1(g`(`gg&Vr5v!7+2GaZ7DI08ZbaVY-%3H$-VoPi+f zfgT7(J0CR%DU~>KQsThCx&nV8&O0t=#gRNCAWtL6q{oD*x+P2=wyTAP zSh}rW9Bs&qf+tVbt|WccaW%r5<|>_=YrfPWJ~-4`?_81fIOpn3XP4X+#p{QcNp|qQ z)hAW*K9Ad_&s}Nvt1iy=(rD@>hdUHx2p zmHPV?y3dN4 z7#$+H1BH13OIv_(^tMRK%m68O*r+zwK>sa=@IXf31Y+>+(}~W*EsxI+W>KOQQ@G2@ ziM^O0$8|NHEw_?FecHNPh?;6jvf>XESqxezuazL*jo+$|u8jN3AMk0GdoTf=QHNM* zw@^&QjDQqneEk7bae2haop1p)9`R20GuOX$ZQFW+P$U8urikIGLI$}eI082?OL&2E%fRZD?3ToFvBFL zfX)&IZ`Ec0BKA-gG@0K)ln9rMtE&&dv6({EePm#-xk7oau?32r@)I0Y4jmin!#JJ5zB{6?*)epxr5xZtJ^uX1{D`omEjfGkX*fYs z*#9Rb$trma27|$?7ca!WgTbKm@Amf2_RgQSU%Y(1^XmD_*E`SuG}wMI*nagVFj%D! zbw4>}F#FSBX=L;0vsuX*{81!a`vobqqIqIho;^sK@v@3BPK;7R`x@Jj=p<^X!R^g)2r`&NWD2$i0r#X~$WR8v9GK)&l;IFaak!hWt_^gV zPH+^Vq*NCGdcvG~Y69ML-=$EPf-s3K-6&R(&uiu1K0Br)H(lp4CUwYW5Zv|d+G2)u zg_Fga?^or`gP%EtMOv@YUH4HMli3s{?Bgc*c+Mc>i|up7WbfUFXi5_Pr_0Xt-{O1T zf35HNpXO`TnL*i>b#P^f>hw`+5?3Fm#rN{kVYivOX{e@^Kz)}7ZQZ_!pIZa9P_Y$J zDO_s8>L)sbr3h7(c$Hu;hImSQ4#Cf*-bC@g%gP5W23gYxhV3~gIhQ^%zM#(pyBPD| zDkbFeBg|E>UH2B=_5>tr&HMKWW}L@QxJtifAObCe5wHg&l}~N8jBj`X&Jb1pt8MU7 zwmsMAB}e@^2QQC8j3S&_d#$1e$C=Qz__{_#Gwg{`&>j}+#z8aGm4-q8$fp`(kK_rf zt29m+8khP)yF+noZUuFly~dkZ@d#Yv`8WVS4#43424H1(cRK=CV)43+#rL|VPzLJe z%al4}$)@=-1SXzSWr~>I1da6hNLtigN1TbMK9wM&5Dvt7iSYf+ndJy4;^rdCa8h34 z_+53Cebqt3zB#PuB1Q74&iaiIgE*#l(?o``1}CZV*GPVi&0_MJDUMJO!pvGsft)sS4WeL5D5x_DSUe%_98;;Uh3N8^J=_0X%gCg*kzpSl&`V;^>0{vg)Aw`7YI3bkb@Q6eO+{Wk6 zYZr#l1ae9!DG1>N6d!^q{}SxRvAPl$OyD(AUufWUsLt(l%!i>#u2FWAVa8A*PVz{qz9gEfImQjlZb2Y1+OyzP+kzW zI<}l(BhR&;kc?B&H=3j*qTq6-0#I=>7Anz!kYmJTi6AHdM?QV3h%0m5O%a*w4uTlK zG%X!p>Ll+s+}_UAxv4hVwOxH*qJ(;zksRDWnJpZl%Y5vJYD>x=E+{3#;IF(Ffs-G> z#oGgLdb0mF5Cq^7(#hJ|p#_7e&l8m48{bJ^(yYR9bu_O$)7Iv-w)Uf5(Fz>{;0^7S zqdH1hHWP``eL86>nB_^Ii&^Sv8j8p6V+DVc{?RH(pb2%BsQAJb4JASZ@oP+FTPu;3 z8mW<8?t8h{9`tOu^`0iBjsf(CFlQ6H!rTN#VlY^@iz!@n8JVZcZ-90Gdl99i5A-*L z;K6Et*1)g3XUB)fKgq`U!9)Y{;k4(`#v4}|Vw|z-{zf>0J<)sk=qShlpg>>0P6=fq z*e9MTM?WfGSyaAiA(mGX3LL&X2AudH<*PJ-n)9{riva=h!(wL5}{L)v0fkSQu}_%P4RIkE}f*rrs8PA zxpL$hNAfO?YZ&9GI42{Cbq-d9x%OAz(&esD`Bj@WZ&SytbBa!th~f$_*Q9+#e$2>J zhEQ&Rb&`Q~G)>v8+{>_?Ax1aKrPPR5F7^fbL8CTgJIkKXO(&|42u^ZL#9|SRLKTe! z72v2g5=`Y51a~hefZ}>Q?wTMNPC%-#?$(QRE&k3KCFOXn;*r%@si|+sDjy_NUPG+= zhKdT7Ax2aPcP)eUi^c$0_QgxWCe5wuSe3tJ1tlB#HzjEv+cTcHN>zkCVJehxx7|c{ z2=uSQ*sb#b_l)0~7aAu*Ys^y!kMc4J6oG3juUrxCaG8dCRgT0Q{`^^QT8#*wD!BFR zAD_$cFad|i$55Q`2F>(M3Toc%3LDh#>df@%b=bEgSM2tQ07?G**<3#j#66ED*Mv&v zfWO(|{xPy%ZGsWbC{rpb&lx6>RI%DrS+Ll~Wx`2Uof?Gc&!4SyN6K{>$w~;l!b@_^ z4bSl>8d|lsg=mK+N;90W5$OGyssyY4pQ%vvpJ@-Q``g88zm2*wPTOs?w^^lSzGmciM<{^9wD)7|s)Ur)~V-#lHn7m0FOTWkU5=t!u%Pfb#YUZfH5 zPfC&$vdNp?r+x$(?7LIw=N|Y&;A6dd0fNXi3|_;KBnb+|N>Ws`1|r$~AL@(wVmB&~}qPZm>xtc)ZHf*?a;P5I{8F}gYH$0DwJ`&ZMd@-ksZ5y;BWZ zoRY}ZUYIVXye)1QTa0W;{q;FBN29|nHcjofcfs`v&FYz6iM!!f$8&GDu9oVJ+W`Oo z?{Wes{G{UX#ZvXOj96Ky>xJcZVN#-(S3a z(<^oTUeWhW(}7^pd2#|lI)^*#3hE9F+=gPxM;30oQC^vMF`7LA zXFiAaYfzcx?cTrEid$KPU!*NjJ5YC~zluBE&Evdx72S&lOlZ?X)`8c-{<8f`na+Ky zqh_aWm2=+O+A(2h2+na5B5-UMwzAbW<#;E~W~|$h7MmtDO1wlFa9m~S6_ys-9&egE zmiaessRDi%EUSo@wgy#=<2i6^tsVEWtLADiml}OQCS={D0fLOdyY@`4?Y~8 zmexB7iO`~_V(wA~Gc={Y4Sw%a23bz~Fp4t1Ocl@6AD??06_HL)&MrzqtwJ1{qEOCG zoPVNp_PM7I2WV#-GusWT9wVNrCgF4JQ=^Nr8+G@CLD4B3J97_by4aco$vb3V3TJ@9 zD+C}Ip&N~5ZQyBa%HZ%+CG(nXw!Vqd`4uvKUy?8?dDab{IV<+-MbArbI-9^Vrzf2s zoc(gR=jlufi_+a23yN|INj!te81Y#*LbpJ7KKqK_K2FBG^MQeyc>D4n+4|gh@7u{^HWpr2VaE?~^@4GUL?VZ>CL4VNS z{yNN}>Na+mmVM{{kCSriq`dID>T#mBC+d3Zdz_{hpQaZ!&TnYyZa!jqD80fnaW@Uf zE1$L(x0J4`9(Hb%d!NQ1q>1>qKKx-!rlsV@K{!F-)fo}Dp;GWe@55bezEmfep)}dRj|P z`wP~7Gym?GfCNbh(p=Kp6unr63Gt8QUqyU1MNIDRs4IzVzwXI@IsXR^kN8}_dwc;( zZ`S^P@*W%?9PERO6L5NP_T$Of5!gKj?@xa^+uc6^Kc1Y4C|os+o`c=veemna+0XAz zcEvY89Du#EgWZdR{VsM`u?P(jW+JK;L>PY{f}%B`Ly<04q)A(MT!Nl7q@FK@Z^I43 z{(L7XGUfb_4-*upC=-BktK!j?|FQj|g8%dS5&!3Yem;HL`peoLNoN@zPZ;>+=mutJ z6C5UCf9?Eq|9^vbI7A6W!C}OWc7#zj1iPoZdv6bdo&I3$FI#d{+xiQ*#?ztf-Z(}g z#CXb;8&1#=6vfug5X-YfU+|eC9ZoLCQa%?;qPz#n8(vFs1%ZmnF7-}{yr~PRekuMs zBg9%6N@kOK$1v6!L?0y#&$8hvMBvUA^=@ ztE;J&L+ws#RZ1t8!){=w4>#Ah#*0|G=5^l5LOQszFGr}WiE}r?((*gh7L!`R?KTQ+ zJ6_9oBHN2w*4Xw105muh#56Xsf>{Q~e6i6uu2~q(x?WWeL{&_tb2(qwl{8+6JLJ_2 z9saJ@W|lq`mK>I#O>r>gs>`s*fljeb7vw!voo`AA?|xNrrSnhov8B2ZplEuHvOO|Q zA@{2P_GY{PA{cCfzi#(m2ZJ8y@4*ya;utfGX#eN@5-BSEpD@$XXnJBv33}+3p+qQR z>w5bVF}Pi@6mD#(yb=^Ao_N>lBq%%4ia8s@i_B zsE%Z-HS49&<2>5PaMKqua7nK+Q)&>6hp7NUBCSPB~RQ4T54R&sWy)w3+!JLtw zP=YdnlD=Na3=`koV#;tbHmNh^0(K;$Dzu>jBE$+^)O0Q@QQA;58&1Z`MJdNk)~SVQ zg~)(e&F^UuMGJ;+KuBPWXownW+fq+k-8S*Ss(rD2(*x@)jh%w_$f6Ca`e6`etlOr~ zTv|>cGXaib&qF4!)a!H)Wv+>>mv=YH=GX@t-a2{mfVc#7i)*PTlx?%B=*A$X^!AEI z5%LQMsQ{K%AjjNraH3;HdX;AVQGfYIHxZZ9^A(zP()G`1c3men6fv^G)y{W zm-(QEXrj;sJeFt3_tY=FPDJL_ph6OzPlNKK^KN={PYyb$bFp)ljTVLts-C0SE16UE zCe^oHFNb4m12@+ISfQ^A_Z$l(bDox0pLJ~dQi|MsZEtqA=W6^KmS0orw^U)i=C|er z2N_)bHHB&82BRre+s4{Y-!t?Yo$mV2y1L*FzWIusakCUGw6oYs&{WIOnTudhRl)pg zH4U0$k|X7srgoyIp4C|82-#hEm8bbiQ~EygHD-m#N}8^s0q)exz?KZL45Mol$+JNx zud;)dG(1Du`>!Bi)GSp!?<|rF>DB(eO~q~!YKsqPcLSM8U*yQkE3ii-6TUmd@){kS z0GI$Rp57@~$0LB5d`ves*9e=S>;_ZBpN`cA2QBdQ1#M3|j1d!BC}1&qcJFu^qXZ1! zfOS2JZodNkcPJUN$p-iX;DkBD$5V3oPfBpY(hfBuNxwXw>(1# z8qN?N02${a-t>8s`~fasy@*igeCrwf15y3mIGw=bd@5@bGz{(Kamr2tR_$A7%GLJA zBS12L(1^Q9-FGPfS04#sWk?q9XB%auH8ZJb&d*ASU9c#t*ITT;Rm_0oGe5N%eq00x zb$}Y!kRS9a%U36ld;|2P@%F&>0QAIw~;WKro3yl#kOT@~!=M9UHaL*?E&NG>&I7R@*vAMsNk1j+Y)h6Q@ zfhnRCj?t!=7D8Td1aVwDwUh&R%$BI4L-{x*&RM)~U@ETuDxQ_i#HHV7BqcE!&(6~f z!Dx>plw}Ynj5@RHQY;)0{FPCy9x+JAYIPh~yh0^N;f~aDwT`OBEGr)NsusWt{*&MY z^!}0btfGMx>1E35kAa!?|B?K%`|kb0IrwGw-QoW3#o@{E@E-|xF3d-N2or7ht-vp; zk}9mqVC~DPCjiv7D#|GM`8%rb{;wN5x|?Xp=~^!xDE)Q2zx}G~c~W(WSj$-O6|xRN zZ6Pt}o{|V$s1iUH0X88nrbve@yeX0h9f7~S0bG0uPKT2SOiR~!0?0buoN8{q+)9c3 zPoRYkWCQUvlNM=f$tD*!`%1(*Wbo>oK*MgW?v5JehE7mT5TkV;$3{ajH`dW=PP>>; z{bAy%0!1+j8H)N+9aC39F~5e^gFw8qp1tYa-rnB!HY%-CT5nXt zHKlXZDdqHS(rv%@*dgWZ) zIb8Rgn=f*(ZZ(6&yP|n@{qDuPbC0goFe*Ostfj3}ciwF(SVannRI%Wd48|sez%JQe zcFcFr2bK|0v^Qow@5;cmV1^PU$iw66{R|SqQ3C0}-TE-{n;nDl3W0S?f!8B?V7U?xK}8 zBiWaYY+X8Rt}h;u%)MZ`#xhXlMr0FgS8RfH2Kv+O=f7&%TL*hNXWu=&R81^Gwf0?e zdw0!i_X=yNppHc+gfgz7p_AQIuYQyCA~V;XUZL40czO+Ez6z=1>AlfgiutsooV@6j z_)SPEIQ{|tl@o^QA&9MCzjTEA<4@C1Ise0)ru1iUx}pq_=KPPr%h!X~rTmYVJ3Ej0 zANTUZJL+mOKpH}StRps&Ww}-a><+>8_SzLrq9Kr_j$pbrMGQue!QmPJW{6zwU{P$n zn(f|U$Ba^IoC6XasphH;`K~WlU|G}y>p<>7hdE_r`cAwq_A}*I@~Is(4LtXIl+OL+ zr={T>*mGp(%RqqlHjpZV4g#vNUN>5CB0U6u1j4a-0^U=U1$;3YWA60|HttFH(09%+ z3+_25;`aH*a`klSw$#+AtE|*P5^*QoSDQ7UTu4_D5zai>y4Ll8eGwIAIJjz4PHDT^ zEY_>lo0sssit`Jr?%vr`cwTm4>GX6fpNUX413arz_d%~$CyB2`eTrTy{PU(J)n&T2 zsVd*0QEl$4ZB;+zC7V?f?cA;{AFQNdz1pshvwj3s^R&|by)8CP)h!`)F!d(w?6i#B zDZVp*Pt`quJ{mfpkIjQ!6(l4=!Jyx{`~<-$MwG7Z(tx>ehN?yXm6I%=Rx}lnnCWiS2S)M>eT4%Mrvexkg#Q@D!2U-_otTG?fcQq|7CS8H{5T z2XvN%3)V-|I2R=9qUahnOw5@(EiP!lC}BZ{WbX;&txfIXd9QjB=;VR^E6wRdMUhIx z=02VHJF+`^7P7StpHA;x?f6+kdSQpcU#qn$lIoI+?DF~#-Leb@F{W&F3&583-(cr; z`Tob3&tL95uK(`iXO-(ecP^9nEWhF?`c7`cF|Nuo`UdirWKlfDD!u%|y;Kc{ZZSM>VV z$z5BnamS6ODC7;vW+@s1gS)o|G-mqY=@1-~1a)2Eruk)z>ExKOGX$gAt_CeNclA3} z5i2)oNP+6wEKCDYCAD_oMTRB(I}hu*?gmkSVO>G*oT~@`01Reh#7>1^g{Ym>VZ_us*P|98z*kiDnte+9j#pFH1v?rro8=>$~q zajSIe!F1I@0!>Lm(VM61dIDDaumJ_zFTvAoc_keH#XWWA2>`(a2t7;)W#At^1cs8& zAU;a9QZC+|OAvF5qQKp?|5li~o*ZTibVD3}{KHL*8kGYoCwpSkmfFGqjqzJU-$xxN z{NMUdWU5nDCjZS(C9NoplsskOR^enU{*ZXvcm#d}y{G@T2Lkjj(A(a5(F4E#4`36N zs83KhA)t4Grzip>XFdJ+7BjFd{up7Kls1PR_^&shcY1RClb>Qyu!f@OEHS=DkA3B_ zuY6;DMccuTIHT-Wl0|Qxu8WUNuK<4lz8(ibvJL(K@a76U`y}Dso^Joo=Vu$OU5;C( z{nD_KrQV(F?jLq3c(i+dad5VO^6T+J{kYpYnRoSKeOJ@TKY5b&%d5YKJ_wYo?8}Y^ z=Lg3ZhsOu+%)s#B`2Eoj2WMTzhn|nd-mXwtZ@HntPx|2OqC#Ac4Bjt(KKQ#=nofg) zq4Z76q%fn%^e@_nH&>>IZQ*naeERX^{qg>X!_zn0vVL3Kn9Xx>iH?4g#WRLRe- zd6iZPdN;1C>mvv3_Q9`It~3VvsMHLFrJ|ii#PT;Q8Vz0QZ%@u0UnecA$QklFj-LN9 z4%1gV+Bxfn$KLbUd%oq~^N>~&3-{L2me>cFM4pZ-gWEntIODx7!kK`E>pdOlj>Tzz zkhhP$8Hgvnr^QEGHM?&;-ZTFA!}h-IaFG}Y>uLrLm1|tYFq8={5v^-Qkt`c+V%4y&EnXGw=Uir0+dSv0 zhg%J#0&$L46NOl5$7)Vno<@+NbCyAd#xtj6y;+H<=_yJIvDNpo6&U$DoW?Dj{5EcO z58cvB(=92%MpRH_rZ^E1W#=ldrVJ4#P_}lS2Lq4(opJlB01cDCih^;8ysrb6CGbmc z{?Ja5>I&9?&wo}h;&$i~-`Oj&Zk@teZ#wOuKorZ67W#Q1O&!+e+^M1!%AM+2S&UYK zR;?O#%sAIy2{cTaPYa*}ISkNBx{?3!WHK#NHZ26prrt+<=kzpdL9z#lxt;-?A@tyn1$GiHyv-q3C zF5qZ{6U=6o9@LYS*s>wtv2mTVY$Hjr1x=#eM)A}qSi&$C*I=?)>K{iHwbKj<088UP z^{gAayEtifrM3sowmBcv7+$g57R5W1Js}b5EI2RRBDqaZimk4t7gwjX7Z!06dX3v& zT$Ze>UdJB7>7J`aa8cHNx{{vauvyu#&q$22@@#EAT#4v5lNCx{hwyCGqo&A;YE>t| z%IYYtKs(Jyh~y|PFS00RH{WaDbkIVlt2E&RMR|-ecS3R=d$~K`EjkfJ2~`-rrl5J` zrXn<4+k3(J9KX@V4r< zK^i|O=o+vZYt5~5-|YN8{v-TsuP>(V;O@I8+{Q2WG`M)zPtD7!iPg7tj7KP(g)w?d z$W`f7-Mm{#qn9|eL0E3Jc913cm+>RoninP4Wv+4p_~G#4c=zb=!~Ve!?|*Vi00~X5CW9=2 z*$ngq{6FZ8U`$a@fP}lFeS|Laaf=mg_$PuwCK>5)Z<3#rP zYvj8aSZj-l+%*(7E(ux$vuY$LC;IOe79w>6*qyYtiE zt&!hjER{>Y+a=n6`{8u=Z1?DVZo!Jn%;)N5?Ssu78FGVARi8p&pdBRFzrd_AIIqC3 zaL9M`FPa@WYi9QF*gI-^tL{`SI}p{8y#BqPb`sq7+tZNxz#TDLY=W=56uuIc@~y;ip=AtFyP|bZXGJA zT-RYJ=n#fPT+P}fN_lWFuluQ>(HwtJVmSAq#5IZYDLNu~Qs-4WyQ?K@5Ya-XkWGdi zc4ocQr71O#^%PiUpU~an>sTFkmvEl_Fv%{04A!&iep61Z3OwJ@{PNWMsd~}IxAyyn z8O8Jc!EKt=%D`WCl*3P_Mf-9+Jy#=(wQ- zTxC)h%iC!;s$7#&l(JP)H-IW)5iTMUN+H3T+WdhD&X;osD5#Gg;<0{T26u;)*ZmW`7;=51mi>Ws~i`JKSwLpzCm- zSOfKg|2;i9yEr)e@aw@3AKo3FUmP5N*xlbhJ2*C#oz~O&OC-F4!JwH%=RR47pJy9`pX^L zHXQg>ePYmj^DtZa-j05nxUQg~7dZS`VN++N`Yh1ae9->M)6#MWYT5ShY7~o|HdH>I z|G1${;_lBI{2cG#z+uV#PMD@ahds}`IW?H)jNlvDgta}syW54W<+AiPY|GB9iaWv) z3QNB73QqP7+4i)$nqEU1cJ0E)#WUq zYB=+sD%8+uJzto2e;2iF@G7ODe}Q$t$T`b!G6w4na@HDfmM6RPJw+MV*Z@74Gt#T> z3WO7KlYrn1e8dq5@{dFD?@!}7d`^^#fV#c8#Z_9E9Bi2^7Q!; z{V5}PTDqBkKJjb3tovvn@41uXtaj%HdWM~YuQcCrhF78mgy||ki;G+3xRA5kL}qTc zw}IH5JHV~HAD%1hbAulvQ@&JWO1raJ=e-oI#+`>i7V9#5D}vppY<0#1dcCqx-KOmF z80Kgt&G0pj(HKRZ)rJ{ChhR`YC@Ep??tr?)V1;o3it<8(Xyx0vN6s>ZJ0C9n3|FK? zF-E;5l%3D$0~%8&go@tTK-p8I%TQ5yAnWPDEvXB+G`VH2R(!zT>f<>;f{PzDfEHzNH+H!x?KdDcVnn^Q^>st-be4pj7)ura0>R*5xh9Bp8q%-Gc>Jq zgJn*)^#*H0-F>dD=yUs+PpK_E-t(w7nzPMt%CNZ_qt&CV`G%Q>{>~qXuD(`P+Lm3k zp`yMKs&<3K_DH5E!uhn(XTkqQ?=WBG_}iy(r>~Y_$w=hWHup(=yNVp-4mAY8ylD07 z425}QTZQ3wxT&ejPH=}&q|`N7tnzYe#t~=m+h`Kt(s8Y zl62~0yrzd1_u33lZ!Y4*PME{$Z^E(qKs@i2IP#~%eIH$J!LjRCdpbLC$D{@B{b;B^ z^dtILE?@Fzf!0x~lfE+a+KBxCK#};%U8mByqYUS7VqSddYAX<3!#5Rprh#HR$Jyig zSB_|Ti-a7B%u$*7Kb%S2c~owlxHTE*e2msuxZ@XnsB&4PFCbD>@PyW;7}>tr!p zs#Yb5RKxef7}5gp++y4$6zGj(atY(2jQrc<#D4N#$7=n07x(|w-2a%;boqN9^-uHt zkAt1pFJG4Ke|-7;)!^~|$NTvCwA%Y0VG_06-o-78=BY^BiiF3eJBD?PuACKlmlG)6 z>zIm{Ti5?Tdw<{6IF2lil+=^R4O-aRw(0bwhN z7sv8gGIXZl_u9Wl54I&)eo7n~dgI+QO>C)DDwRs5s#0n98d10_Z)zNxmDe{k+H~J% zaq}YdeS9}e{9(t$fj)<|4|oFE4gfN|LM+fx?4y0~-G`Xa^Eeog@5EcCK%`4}56nF4 zwJxQx8|ktzgP#|vJjH~b3=cEB*^4to?{IXZzZ{+%WiM1$VDc^rMLfrta-;j4P&8A& z{~mi0a*rY$9xC%KQQrrrrybRgxIE=*MaxuCur3gSPQ4)htf9?O9t3lSHIpblV!Ko5 z^8p~wt%sV=st3MH?f2g$Z-kXH{h3%Q(ihhK-**)XMA7pa@}|7z!*dH?g9SFd;1 z>;Gdst62XFuX7d)oR0(4Ewi?&mZr^GhTAF&uWI(!mxCd>JdxYsg_eIxWUAK zQnr#dZbkK1dG&N1gGg(pkP;O~w|gvV@vzxCv|X)X-!Ls33)?iBS~I?y9i!xq;s&OR zuWGa??80N)DfS-09l07sy0Sg zodEfYvNoYiL85P8=eJ66K)1lomTB?1O{_ff2DG(pXAu=BABvF7a#ee&uoQ8fQ!U?~ z;@J!vtNOXT9ZTz5?FBhe>*zeR->Ri@9p(=EYnc4 z2r~wjQ=LsniM1Cr#M)Y+7+Z6vT8s2dvTbt1OHn;vsc|w)I-|cW>xEvjUG(d;$*-8s zBh6pe#an^#l_)P1>T_=9vyFlJU4;8ucX(Bx>+LhN8~1ge*U3vF~@RK?{|P7D0}hR=zdnN`RO1mSt`zH`BLhxlCc!H&~V% zrWyoxS+Ck(Tx)+UGfVmYA3A(bwxk)LV*kJUZs)DB|9|)9?Rx+JD9;M^|8{XPkgOzQ zIyRHQ{950eSd zIt1--$+I*^RGAr45&13XK<)O>0~0+O}qM_FIk3^^J$sm~9mnwi#Q?Otm$tw}x}AR_j}9>j8OG ziffkCN)v5iqSz&kv=&>}l+v)L=fh3ST@HpmB2GTptZw+?S<}mkWLBOq!W28$-v<|m zLyMtIMWo;y_KM6s#n2nsd)hLwG+!*&u_~31ss9HeI$fNxXlVyfuK(}uyxuY6f8XxD z-COJbkMXpu|0i=)vQhpNwNUBL)X$pMuUSiUC|NOSC#kTtl;216EOr0qc$$YFl94Mw zM;w$$Mg5~L32&1HpK;U0Jj@PToCK2y5h8ybCqGJgnjca+mddNBB2{4icLMs$LoOYG zS|G1#7u!IS#q|Xku(p945FX#_$LzCg&$tk#CpDKhuhgszsmAf?XsCh2F}=od-~tB+ z0dmBUQw%VjqA1x}un{HY%{9L`$)L4q-&KoR8cB;A%_Q&5mb6p96zd%@*fl%f^L5yE z*REAnu&20VNMXw*v+@)}uQXTWHWld)FHojJ!h>FrW5I8T*}mj2Pb!fsS-r7#&s$A4 z)jojQt7FgM7nSXUE{$T;eIAnoSaUAvw3YmJHu{jEQB`z^UA45v8mWG;z)+OTT`QVO z!zz&SYcCgtD{jTiG#<|X5`009B`dcp5bfF6fu3fm5J)|G6_1rg4XHUw3lX>sktHZx zNScNjQVYxohWJYaav0$mqEi$TK%5AMh=4JUG`mDC*<9y-xk~YvK&d4rtK>m?qjP<^ zkgM_)0QMtidl_r{iZ=#B>ZG-$V?SG+r z-QK}}`BcY$y)*28uXpxdukC-2@vO-H$Jof8y~ko~)AIur82i#RKSv&k;*jkNe~8@) zqH8nYvoHf%7-Z%JgInnNOzRZ&)Hj;)XfIwCa?fkP&z}NlG@-rAH)5$`_hx;4TV@+e zSWf<9cZLIx;^@)xzr5Xfo6G<5`pw$^^C-_(jQjYo_t(k>|Gd zvCa!R90zWQy&#J?DKwKP1k>Y1s>hL^guPYGBFUSomhv@=4A><=gpugNn;{%4cx$h# z5GjDM05($`;K}U&ug)`UvdXzn zbrH?Otb^^Bd@7_e5t2@*q`o?(>=UPIdNh~>`YnH9dOx53zOZ^d-fYH!4f$*dZd#mY z7UPVp<@yq)sT>Jm1ZPAYQuf=HnmlLOGplA+x`&5-!`>!Ah{~EZt~h)7@|L>t=)r8Y zaW8oIp~`Z`%eo@KgjIcL`p%=%(uD!eLSNqY(Gwf8?2Q=n1tE)=OBJ&6rVw(0iJ$NH z&frYVGP;0$;Tzj*uAe9O`Z+uPmUGxLAETA%-ToM#2=f6a^@Re3@i`i%*N6pdq_AX?-+ zRt$X-6OnPg=qQPDwmg{@|K^9wB^jRSlKaKTQL8el#t`a-uLhqc+gd^O#`z}`I5RIeN6?**kb?u4SaP?5-TvOI zMU%H0;H^`^M4>hbL+pNl0i5vbe$;#cmJimJZ2*e1+a|h^oCU^xrfpn!Q9T>9Yp!d~ z2hFhE&@Z2lJ8fr|(X696k-m_6JyZfzIQXCiaw42Ib(6^}DT&JBrY+fGtK2msHy z0h?)0H@QFn8~x-u-2M-cpdqAF(9LRc{kXfum+*0fXME;vfiB~dF1SzT@;x9^xcBBQ z(J$b(SoKrnrU?4dr_=;9QLWb!f+;q{iM)`K#^}g;?c*b^@^6d|ueSW4*+o-M2Wnyd z6o6myHvfh)VHF)K0KZ7b)O&?BoP;>=W^jUrvG0%g=o6uE7dRa)BRj=}4tx(1q`7L- zm&>K?wY5@U0sF7|etv~*%**F;Cmo4iz}_9U&Oqw%#eXilWI#@PNRX4hszzv$-jZDY zL~F7|*=R|&MZUDuO$(`R0Ll_UmXAo}VcvTm62fp%JkI7Py9$#7kn;vqg+NiU_Rk*7F1>!P|)D4=~Dt&|I*_O9Pc~P{ehjph413b#z z(yqR3XUb^pw6_jW?4!tB!IhWoAg{L4Chj5j@nk*a*2Ek3vi6k*rc*?>IFA;t4h9@GC@!@^8!9Ke~O@khA8s5CRS*p zDXozKPLFhJC%}arUA^alJzNpDDr)?Trj(`Snc{94b^fg%rXP#L=8QAtt_LRpCe(92 zU>Di)eiX8a?~Rc&cYO37;~V2>Zu+eqeleOVTB`RdqXg?6l1`c}ys_F!rbpd46rr{> zZ4@%p;SJU;C|pX553|vbgxRPo@^XHPTUWOR9cJUl%)99*1yJiGkx@#xr&oN1=d!}%~6T^yfZzW+G7&`K2u?Iexz z2ay5G%KSQUDj-ZTp+!Ab89DLzmtN_)TXNmI#VuDfnsstl3D~`ze(f5Gr>|enP5-)sO*WI!y^xf{H?ItxAoeF+oE} z$Q_Q{A3Y!Ca?I!>pLFK($CJ|niPqJ`zX&T6o)#j0^+MTRw+4%?uDv1-v3JqnGYss^V)6wPe+2DuM z;~KM+Pf$KrlX*ma(r}WhH7YS#U&o4@5xrHmJPYzK;OITe2|M%m6E(E=NE~)qrn!_Iy;BVm;cRJ+93Ym-!G#gwY zMn~^2hlBIMhf!n1j^2yL8cwrnVY(_#F2TyKh(z!>xcwPMx#g~nZI%YV=60)mXRGW} zY>ga#DE%xv-MI}_UcT3$*Wri!5Q}h*4=}VY2|L;nc*urzK}Z^cr}9K5j7&7y-0?M@cS2_lt&3JtpMZUsoamI;%@j)Mrf zm_~?9v2Q;qp#wClm>|c{^Wz9L0cln-L88xDfLHQZEyzHK{i)@pVY3u3Ww_*a=mnF@ z1gA?MJJ7#`fqR);7&@))Lm8c$Ps(!yU9VtPM%uSFr=`(q3-eOP)jlze{wy>zjdog@ zng+~P=B5F&g~@3^Y-M&DFrUWsOp&TPKO4|`X~$`V2)}oZn5->dm!9m%D%YE20V)QO z#DWDlK;2SgOp%d(?uQ&kMI4vWVq@3qE%xIX`her0NJZBCM+SHB@XH}jx>W6ZvO*WE zRRY#hU>R$wZkVju9o@Fs#LOj{b0LK#O<6t{MXjQUc!Z+BVIt;ws3o~d9{Ir@wOR*2 z*-ByoaQr!p3b9}x>tbRDR6${}2W~Ww{3~*Mh?)*m0TRtZI`6q&)Cj5xO&JInhWzM7 z4rY@U=`1XQPiJASa&yC{;8h1j)Ej#aq^Ry}7yV}y%^FPhe+>NjKA=&ImNhf{tjMYb z%}3>@JMyg*-g;BViu9${B_HY#=vSTsEm+GqPLbYN&^TsGt64X8%dGqaLzJpeTZbqa zWLi~-lG(yzg(z8LV(tx*Qh94hDnpp08i?U6sv_ve|2O=2esO$$`H$ltE>BNJ7sqFpgQKJK<56*&$zk{XzucCHG21Oo z)=?{qK+uu>G%_MbQmkaoEn72M76z-*oeLiNR$cndeBO)tpc(tvM~P!f_P4h!%R2ui za(=$Ygo1~M{e@f@1(>T&=6Vx?J88LRdqvLR;`Z9~ZPyHG(9swyh(eizx|+kZRbmDH zRxAAF{H-NEy1Y^hO1*To1!y}?y0p1RbJCTx*n(d8_)a=o%d2qGt&jt;a<@W3iz{#|tls3g8oxjjFTl59gVnUXmDym6TUpu$TW*$3LxemRIfkO# zhEG3*TYcoX;HdOQfRIZBO@=6%d4%w@{rd-62BBNCKrWY%&G%ARrZ{EI;>T5sqbj|? zqlf7s=E5!~)ySMU)Xz7dgUO{Rt7x_5kvE_(H2x{-MHI1k5d4S; zIk@s%(2KA3`M*Dbd*H+r^jy#dUCUMp5iXqZ9|r+KPEp#IA;6j zC3;YxZ9T>ySshTC(ptYY&`q1u($qEcYgjQ_0GYZ7RH(Z778g=_%fK{Ne@lR^ToJBp z`+2CuP2x}%p4jU^m7-Cv);^0AnQZ8o7SR*M>MY|DVEwf_xn-r>HIb^fcAJ7sqe>Wg zx1NtC$hB-gQZc#@c8VpFL}qpFYNLS`qXZ=6g(PXNZe#IjLRh)Xv^Uri5xJmxj|m;k z$rBQ+Q%v~mEP-q(Qb$zeUWk5L5V0 z5X{AJHo#2OZ5gqqmZC~45tGB^@P3T&%yy7Xrcl-p&%D>oo!cHov)nXT#@C%#TTA+G z({K5lJM!3?S~24-uL^@Zs~E>jL7wEhwh*hCm?@d>>=mEtojlRD)uJUGwMnRAV~MJg z%#iEFvvRG)`$OK>X&<37<6lXW2A_zxl`531+{$_48l40n{ZOslr68JM*9QAn8;NTf zh-=6aV}ej$o~|vBCYlJ{d-HYyyQ*T)Z9gBo27JkE>?M}y_VcYTiFhU(tasyB47^)a zH|}Uzptg#6&)7t+rBoB7o}rYgyU9kMJwm-;a_B>ER-1iN2vkf4URxh)hrO~~5Yz>m zG|V64D#l5>B`Bpu)u^e6og?w>yY4bii^DGH8|k0`Pq{HxWr0wkd}do^;P zzbse*Lm^he3%6OlOf9lsW*uL0Qpf` zrh*s_JrOEj+Im)1r&#eN{}$<)Sw}RJ$?q{Drp19!93_UiDj{Cfdest=OMkE26W5C0 z(%M?BYYOUXg=!mYl8-or)r#5M_IstYxKzIkl;r659UqeH0tThwQmjCC;^S-RXNAOX zi$i$WjI zXDG1TGM#NX6l+EY6u1@^U?+ZMs;E_XCs{Jnl(Moi{9M0sR#sLPh9g0*0;Ng?KskjE zho2I<^Z=cQXdj$QZMLH;x90xK31d@-Gj6E;?}hHCRdpzVeD4p=R+ZGr zbq*AmI7C5JM;Am;8dLXCZDNwCit&X{4jE+9z_m~Bb44p$Rnd$oUDmL42!A~#DmfY+Qrev%hI;30G7Z&y|>f)sT~`BdF!>;2rb=2;-02X z*i+MZheg@n6C^@g<`l7$80M0KtFx@uLZhr3O{e8nVd+2um1w6}C+|!9TU1#M5NI%0 zfTi^y_We%b)~kdac{mM6POH!oyD8AidCW40l$*X=RM0Y@_1)CvPQnH*YU|zQODM+w z-Y%@F{Xca06@E*Z0V?+YyYF^(^!@+d&d%F+>;3;@Jdfc2k(@M;oa96ZQuIhx0@-K> z#uW45YW@8MzCm{ToPl&%eTIY87-7b{OV~OHE%fzS8Xzj|3MmnvlzE{UR!_>x(BhJn zlmFOn#pbI%74m=Y)f?0Q|9Wrt^;-Tv#?!L=Pv_b-bP`G5(3#Jx=6tI{s-I)O!0=l) zf$DRXu+PHL9>u;0RaEz!r2`(4uutNUCn0vZF+RaVNd;jGdFD~SJTpWZ2iXJPY0Qv^ zkW>*I@m&YlLM(gWyAZp{+2WAZ^biq+Z&7ru7K{nfp|BO5?1Nu)cZ3M_t?P{Vt%lLa zREF{&Qd9Ob|--R^G&8Ne@$azwrtJqEvY*>c)_3A!QMrgc>udseFro5`FX zls+tc<|YGH#Eg6E@WjTCZT-LY0@n*Bt*HT3`@h$@_5aPA*X#Y?qdd=V{r^F5=UNST z+G+sykqod)mYRJahBl^C9Ld<2{>^`pzK1(W;2c-*8jmN5um6P{VN&k_s{sI%=W8$6 zk$?N^hbjMMDu!NqcD;pFpj7_v?(DtW-7({T-tD|v%m2rC+LHfPBivzmS5T#&G&@1`lI=q`x4@S)40DPJtBel18@t_(8UuRJ-Pr4P(pRkJa3Ha3wn z#h|;kAF^h= zG57`a{s7$#_OT0W_XRnSAi&!Io_UiH0Nv5?*~Q7(@oCq*B1k+Od&MZ8j-+v^LGm6L zXTX!^uYb)q&F`tG*{(buTkp0IYm ztrekqEo^*we>^xkJsypcIb*^S5~_sa?BXB@SR-fz{c)h0s8Dw@bDeD2a^bep#N9kVxaM)xFKvqX&3Z-s?G`q z{TvbO$4q#s7xEU}9R>jL5eiFyh--1pNtK&0{ACp32y{<|0J?632mxap0kik~UXwA7 z1Gk%peEnmPBwygI3%5PZYTGuaZQHi(Y1^2#ZClf}ZQHhW`*;55yzjknBW^`T)Q-%k z{8CX_d$0AZwR^&9D>~V#$<{Z1>U?v)Ta&_J(IHDI_;nq@QW&I1@T{%LKyaPew|sGQ zGJo;n{9(o%h|K74ovCWSK9NCk_7)cJ&W5=ApS3$CIA)N`e{1)b#8W@zKFHkmxQ#QX zp9we{N9V`?NCfrO2Zvo08Bh6@w&d(>qK9oH=5@#Mzh1gW-wLbJ*6h)mEB2Qst6Usc z#ZtP5IU>x``Jab+h*Op^-k~5;oP_6){&C>y;N|@C;o#vt(=1N)b2$Y!@}gw^&4hI% z-Moftu_k3gc}V-Stk#C-)#>0n`D)AC)>|@}SGVt!9zO8QWe3MzLk!S`VyiR;>%@D6 zKDlL))!&y|>V2p$g`7Fx%x!&QY~_yhcGB;<;`fo^B0{sPz#`<0c4KO@y}kRWENye^ zMqM3lgK4`-A2o~+06<19*9#Y5qs3Ezbp&@ciPvvD_xhHFhHe~sq`ya~c}+Pj~c3)NPQte>+L zNGOf+OK~-$UBlsbV=spnncub#==Uu?_%i<=cDC<~OMco%yf&>(Zu3^mQtMs{GZy2( zzI7B)o*n5pr~Z&AS%n}!F`ysomp&F|Z9Itkswgm!(Tdc}UEz;?y;wajd)%tuiMS7U z_do9UhuzTq`|4SbK5vQk<|KZ$QOFmZ*_=Gz+FL%{^<6(_vem2K@%`DlpxNCb6)6kb zq6@34w08R+PhC*~8&9=7{r+MOjla6Yi3iR@uhW~_7Fjc>N!%1N?2^ru#53?6aQ+cQ z%~XXN#hqS7_)aLj!7`+2^sh*fLgI)kn~-K0{GlE#6k74A zh|dAm!Y)jkg!aQE5L#*n>dzj&poUcEM@&|Ts=n(iv>n2Tp8hdBp!-1r9oEIT@QKI z+X?R)(H-<1G2tvqqMq;qC~)YkF5-=evD1pxa}*u%Gd^-NJ|FHC8l-)#5dGf^>8td9 z&x3bwDOuZix@+ARUr)KxHrUayJFDj>3$*672^kBmuIs6jHBY@gefRMhJY#6?oR4Ap zizRFpdMQS7FB1#1z6YaaDpas<2jq$bB6s5Zza3YOT;bWg!Pr*IGb(oX+VvTVDc*Ko zyTty}=dJxav$C=_4pGb3g<2M1J2z|dElIUA8(v@o9LR&vrF8?iP839&8UObDUQRni zl%IG)f*LT`vLW!oXg@L`&)og4Y=`U7>$$pGQuSvh=|3S#zn|^rPwo0}w=5oMK~Qd& z3KGAV4~(7!SzlEh*3>J9=Ob%I1Qu=54P$>@TEtbqJF$%q+^nKCA^QnD%#mw68W4GT0%Z-hED8h!m14A zd>l5Sh-TS7PZ+fm^>OkV7E>hn1eeQo8QhUth5vtKz_=w`EnU{ZOFMWgOVyv9K)Mbk zjni7Y{`3NCeeTd2f*@^dtD(t0_;=7HKk6p;?%Pfg^}Br$erqCSNr=U_ehxa4Ck1aN zB>Tp6LIrbAJ5OCC;=9!?F{GkpMrk(BBFgtcDIG9Svh~wpp8~BzvQ!Z14Cu^MFZuYL7e>*Wngr8E&gF=myf8b$n^lL%Yt-EtqnJcv zs*&eUkJN3WX9A39`rbaqzrV;*>=Vd2q6WB;8F|KZNkwG06jC7gtEg^f!YFCzCk|Aw z++P~=<$DsspmxnGg_wZVLyfD2n3f6CM%!J)i=od(LyeiTDWgYN3iUn_Woz5a1>m_@ ziI{1x;|aWR@9IFkXCixkr8ht~w14w`3_a=hZBcm9z}(GaR&H$hTniw{(f2?>;5~ln zO9^36FP{{%zTmH~cYn2gp4W7LmGiSneGEVSjKoHt^}fyCm-W8ID0^Cn?lwk%*SBEV zYb&2CKAx=9dQ(oVX9QAo?SCKXewhAcnfdgk@A^*i>b?y|f_~^eZU^~l5x&oZ9b(vh zfE?%NUtWX|q`}_)#PVAH=?U$cwhK#`L(X7G$p+`uQtIL9A%uat1y1YRRL&K5uP0}Q z;V!gVL^cqi)T`N#bi5V8vVj4OZd4>i660)iIhjR)<&i8ITaMt?R8WTx?UV*Y*=&EN z-jZ|o9M26;{nCSCX-%H3@=*qTxom0dD!UmqU`|(t__I`5)wo9tIhzR42-v*~%2&A) zASbx(MXmZ{9a2d)bpX!M4}JU3@*Vjl44FVlj%3$#gFOR!-(%{av18C*&%dm*{Hk@V z13zm6W_CsE{e`4vLi&+z__1q7*1Sn=K_PBCQfg2Wa%>^ZzPx|=Y_VaZf5 zIp?8zS9C%*DOqcl(_i)V&ujd)9XtMsW6G%5cG|Rj{jYMVzWdu_P28+B7A2nSHf1Xk zP@4?{r@thva6Z54N8(duJqby`K7v2H$6N9aCFLYPq#YXCXsn1f)T3G2@?YGAA=mn= zUjnFLNzI(5lc21}HZ^dEZ87nvroRmkk|lXuUO_Aw9P%455p_n6a@Jq zzRakFW(xMRg3twGV}pLfz3_{kl!@e1`SKi@TtqMpNyUp-lTVyRE|#J=br zvbV!>eFk&&?RUmjm%iEA&=URVBZZ|xv7d0|k5Q$0vPNgJpOQ{V5(9wv`NhZ(s*F$I-C zj%AYgHnMb!bME%ng*)`_u58#z*r7O^T(%)OYb|}du7V4%Js15580Jy>9!(2dhSjh7 z0ZmJa6xN&l9&J7EmocMB`&~Eb-))h16Uds&rFkR?1eEB`9%7}!zcT*2+idy4Q|p#O zrem^fpbl}2e5b0jr-OXxFJX!*uGFkUWo0{!gVnMa^6OGKpzL(DtxpF79(fEQ+-t|b zKlsxFL6Mmm&xOOxc7l0<_o1f$o$@<<1ZrWpTVI}wGf^FVOqgaotDE_Qn(hK-I3UIA zy(t^dDgkKh136L{tB3%@*5APP#dxR)b>lMIPB@?#ggt=j8CWv?kT-o{g8^K%Q^W(g z4|k)@W9_II2DkVZPCk5!2vgmjq{bYglU%|GZXaxDcx@2ZKmR&?mx*45OA58jVW9ts zQUH;-;s}p$B%1-^U|*mGBY?rG+3|?4o?z?N+2yi502csQlB!k~i3IaL5R2tRo{)Ve znbHt2|4_nSDFOt~JT`fD5}4BiZPA?->|s6X@+w`@?w%*FoVxH#*48L_-gGVM zHs@Me^xpn#z!tUVD4!eVsLBk*P&MpR&ygT@Sru(OfM%SJ68W4h5r|sz5ASRrYIegp_9Hye&&~+0W9!mp`UB z&B*a=f-fMV$_P=PHxdZD#gFisTm>Xyip0ef zet>vG!pIvfvo#Hd`KkNKJfq$(B&Hd!%LQi9yHp65Bn?}9PmNsJy|ljk$ipm3(PHkx ze#KJG-2N?jk)bs0vSyU2Lbt^>205Mlgq_KFR*x(Ma6cE`#Gj+dgwwsHSvn&(yThbS z!zY0pqv<(%LkYO+RZFC2e%fmtZqQ|K{pUEY!~I@D*$#MWnEhHe&C}Tc=#T2Z2i5;< z)um($Z=N04Tv%3<9l&Onf$sa{(J3qR4F=blvsxb$=xyJ=-g*o22S2QWg14uZSqKo3 z@eZtCZ}s6eXc#cCQj%xJ1XrwU{VsHMp;2oBrdQrPuf&_4w$GDd=;{}D(z*7 z8~{NVwL!N6R(&i#<6YC0xknq=t5QL?0;J^*&lpBs4j%xdgrqJBgAaxa3&OS;gRpV6fMB8DDfOT*Xzk?ZC~5d*b~Lv zFNb&U5Ah}^UWftOB<(UeaQgkCNN@p;!Z?y)T(PZ&GZMdV9FTz6p1X|gIr7IdNT>%% zJhhpgeNFmLOJ2vK%zY7#vqs9#<^C3mJzk6cH!{n) z?$6ej#q(pvA5S~IPWP7QGdZqyyFc5PU(gpcn~u@sWB^g$HH|jf2B+x*1DeK9?_!r+y*w z?ugsLb4YsoPW^lv0(A)=ar_JX1sM7>ijpFyibB*I|F%6+fgx6OPNTe|VzrPmhFDC~ zZR2QomkPVtcw3@zgN$4!7(eiTAp|4`@M6`XvUv(*IV8AM<^5rDZtYw0X3oe8bxVk< zHH0(nVNVIt`pbs4MR8vn%M?fg&AynEA2b+A>^=^RX% z#xrP#r~|U`m%%v&Hw*}I%>FugJsF0Y7S@1eD4_YNmEMrg@zLA#gVT=cOp#-_hBc_OOg@P*sGW|TyCH}IH4n?ydG zZbl=eeds!SYw+p1O3?c@dCIVWTAu@= zG0Q+@2ClNPs-#;>9$fS%?0E8u@d$gsOffx78dOYbMnG^p^o=3DM!=CjL8#-3HP0)Z zGoNC{^Vv^!1k%TNvfHx)fQJ$J1oZYcAVN~)zDtOXnKc`b5jw8q-Z^yn&4SH3v23Mm zR!y7I!FY4$JWu*JLIo zB_Ty~i&?sZ`>>({9WH$OnR29)8=p$__x&c~_&XP<6!+1ZquB#qWZF@W$s{&&fhH2_ zhSmq=f8~4q^sM`XbF(anMlm|F$*z4Kfz0>PM(luj=VxDk_F`7Vi#F4(c?D_^zei*U z#0px-y+UY-7MIbpA-veXSoA75_q~)Mn-;b zcLRzNg8Qu9^uPH-19LnJDXK;XRu5Z$v6!&5VpLgZM*;@O{ zi0Hh)8h9uqHTNv!(uw7g$AeDmuWYWJ-QTZ32S5`dhc3aVQ0WIM3HVY@o=LscC0l}?VbM#^h}hIC!XsL#Ok4+|b@AOl&8K;-%R9F6k5z+pnKtaE6zK#; zExl%G*GQTTtH>QKcNx=cu0opMiD+k+zCl;T{R$~orsGRBdK1`I41@4b&%J}YwFwq68Yq*pWu!*& zxZvgt1Z%SmgAhV~>W(Bd{9~bnD7USQBi=vtH;&{ij>633!qcz=25g{*1W(?elf`C8 zYGdrMJ32C%t`b>b%i%g`aZ4xxrF8*8E0?rQnPdjda-+9j!r6VFIeD^l4WKmJ-660t0@Mlk9x^x~GPa}?izeFhG3%7p zcurihZUuhmTqSW45HS1WSxUEFr6g#P)NPyL2B+QB5JvDnN`M<@(+}PC6hHm1aev?& zB*IKkXhW}IgwHw3bQhlxB#E98)!wY*_Z>|Oh+*ZxA$lxRl@4+35QLKj;vM*@_bigr zdBAG}Z<=lU3gP+`K=k1b3&{ghbS5u5PaaK&;$84wG4iHFrCnSJp;%MuOAZqQU@oUNe?TSm@cLOwlnDeHevW2i?r0XOp z5n^kfiy^RAd-s;1y$$d$b+VWX<7Sl1bX&@gmy4osR3F%9ilfYd{V|jB2FmXXHxq^b z+CJnD;&in_(6E;Wg2I4tA4LY9CzUB2XPbyZh-Sc;)Q%Aj2+(m#uq{N|Qz^&|j!1FfW_Y&R9jrB!Y&Mawb)Y#DMX0JU+99*o9mBTsf$9qIB7$ zF$3*Hk0Yik`c)=6E~I!6u!1m{_;Agx7wHwaBX24#F5OZWnWnQ{f7YpHyGR_*$`qph zl@spem*^t{%(RP7j}lz1hs+1`+sp@QZgN+>uoWJXL#5xn#Ob3_#m7+Y;uQ05yFq0v z+jisax`|22yEn+zP?zzvZI;Y_B~qKQ&_HePdnFXqOz3H6O?y={H|?T(T_}0R(Vbxu zBz7T{9HiLM!Wj%l1?5sulfekdpbLT)~l*=v9I+b)fQb0(}5yEPjn#o`>3d9%t}Y#qZWOG+qqjcTp0WDhHu ze@nqp&<+5ltbaebKI?WfPJef~Kib-Sdq$O*yyx(s88K6Y5079zSgzlBtb%MtY$~jw zX4muHoy6g|fKn!ht(I;IyEqR~^)YAPhu0L}Vl)NdQ#E7Qh#Jv^$6!Uy`fAo;G%qxz z3$w^xEZ&4?8lHRsd1G);oY}HIrR-Z)DY_kKv^=t2BX=|4 z|Da*?)u+@hvLT~%!oFHt*S*6QYc?i&>1yg#Y)sw61&DAMNQ+O6jp_ZqkU~)NX;F;hK%TMPPoGh zXLOTWY=djz;00;x4T*RclN-RveXUsav=FIh?KCFfTnRKFYZSU~8)w#m@DnZ)igPohs|NrgpDc z)9&VOq)n0q z)~g=YU|EbtY@frK3n(L$;m!nU!qTlQ4ra-1otR<{Tu37m%?fB>*Vw1UaIx{_u`8R{EXKh^Js}(H0_^-LS>SxkqwZ692{Qn<50iu}y*SV)(n(JLde@R>M4DiFa zaA5j|mXM5NZUMu-l*0}wQNYyE-)ozmkFI9Z6F=>&k8dBx1>ABBU*{DNeypP~rLu9_ znmt~UJubi^pVY?FImK)l>>BTyN&|6otuq<{|)H-$q*mzZ^QQp zO5iWDlw!J3AE#3&+cu)IeE@-S2rvWWEff>5mp34-#@T=d*;<8{<8dl!U@k0(ozz&# zztx6DAKiYpR7jZ<%nuxWK#jh`p%)Dn!Dk&3E}xm$`dYP)5pY!+AIzc4#`d!it&59| zDS^kjHD?W3M?zo!W|y3JF78+?_9gtC`wdEB4s{A}Zg~`@mX-j1*2me!%xTuj9ZxNU zy-Eo8LKp#Db-GLKX;{#s{)g&2uVMTjQvVKQ?0-1@{_7O;zp}1+Q;LD6DN;)rl*yTV z;5+CU!jWS^D$bQaA4IrP%0)P`f;eu>90L6-&x0yV!sg3ttc1m>+x&E$1)#$Q-E=p< zsHPCLl?lysn5pF$^FX%)!6=1Qbm`16nwu1JJ-^h#HR@7VC&%xiBXiOOi%@I-gjVUq zdT8gRj?-br&f_5;6xNg@X%@B!V-GcaB;2piZ#>O%q=Gy}~=3GQb z!TS|AvtaU-zOr*%Q(wg$w$cUbIUzQ_Gy{^HyO=Q*B62GGl;;rJNkn3%`av|PdVdiM zBOarc3Efinz5d#ASzu|BYv(Et+cS*zuqmkW)5bi56SwKm*rgg!@dL545N#8<*DSJ| z;a<;PqM(mJd#t@8fp)XRcQ2?lt;bJV+s%+gtIB#ZW*Ov-m=Mu&UxmN3q_)|^GsKrW zDqs+t;TZtVLwW!y|G(k;cNO`eNUpK5#~V*2m7>klOG$Yk@-|+Xl#WLDB5!Afoy{#Z zD%x%;j-dIrC#wlv_03jgGr6>jtn}-a{iHc|2Ec!j))JMe z3_g(=0U?09w#*o&iK2ax3e!j#yT)qd@jO9fsMWSAYlQDCOG)F;-T}1xQ{9jK4@TLG ziAZWFGHrO7V5#!cuYNj-%PLYBgU>@FXF}J^FD+;&Qb}*lH)8B%b2+ z41%BrnRW_m{97)}q;xfhm=4#+CSlz!en}%)M*qtVfO9bwU^Xi zX;|%3Y?$VBx^OD4@(-LfmRKA%4+ge28sW%HR#?=hXH~>OH%f)uRgtm|*ON}FA#9Mh zw`!(DoBn@N{)7U*p$`PHz!}oV{cvUuN5R_Hk|qp`4#GdF{+W+=g_&|>;p1mT(ec6k zaI(iK{V~VrDP}pmANF3lVY*c-4H2S0;bQ!62lo#Hx7+FQU453f;o~b^c0PhdB)R;!x-|C;U_#7R#y~4zCL@I=EumOc2)>$m&;XhWra{1 zn{SL=4c&Ic|H%6Jas0>OcsR1U@w$lLo|$hOx$Y8+#yHw99RMR94V@=z@r;T@E5IrgIVxD$*-W% z6aEG}fT8olmm>!Uz840o=*3B=dF3uk1zK^%-hpO=m$R3vgQKh0TTn@Rh#`te(H)RO zXLxDTw~MiVNFPjaz|xdtQQ%eqIVl;-LUwkC_-q9!2U-?rQ_%42j-kD~v!d@h;{Sl$ z0-uO;7K1yai}6A!dkca|3$Y`(-`j(?ud6#Ovu8i-#bQc>Xrz*~`DB?T+q!rM_T-%r z@gziPY5)>9yzMIwO*7T_P`&vt0JvA``m>`{c>QBjljBR?Ljx4t=;iGFo9C|xT%Qqy zwgh9MrqU0AIHe+-lmGC38X?6Cq{@tVO^&tQrIuAHNCf7>3tsTtHv%;7dkD>E094I; zSYO!CE<@k)5I4zKO0sp*E1DL=N|_s)`cTT%C|S1qE22~WqcGd%X8Ag0JAT1TQsm#9+!$ih7bOs zQigVM%PL65DOmko+7_mby4wfzLrg4!)c(YyDov9#8Vqb_p=)g3rU(_D_lgxi^*#!a z_ytE;Y@|id2;klJm!Byr$^nLl4^AE#!6empal#Fbz%q4eaNJYXq=k9oS|9grkLqE=co#6$@ru2P}ou@NW;{pCdr;_j(j*yz}u%c(>~ zrDLZ=5l-5qzc>ghC{C^|B~x*jIN!4nd;!taoOzwYzn=de%a63|l?@?v!~a59IRAn19=r>aS{i{Y5|Ked@Ma`$&tPPcGjoS#s@r5HAtX>=A!dM4djsar8YR)qEAKp*K zM@HaoSe|_@5}lhqlj!efFf#3WsLy5*H`Pb<;*54g#s2&Y%G6a}xU=q#};$akqXG08C3cT}_?>E6U z6ko!DQ~KpClysIqJ14rnQ@;oS zj>^)YOd$w7AtHo?wv!wo9|RI66-)%^@xsc3ieW&ABr|ItA2JfbNMc_fYSeX!Qcw`O zoVZBNkBZo7Vr*HDVx|Il<2HRZ3c{d_rbRR|F5Bc7p+GP?R1g)BsV0~(2NwXHrVl9F zkrdNvV83N`EZyFB?-?@;M09DU4W_DWq=t#r6cel>&O7@ha4%jObsr&^-xW0E9z4<~2tN6}V z>d6+y#~tiRbnhSV?9=NWyY5fa8A=#O7>4=E)?mG7F??%l&e#ADH!XUu$f80EnKY1`6_`iRrV#Ee*$XeDfz7Ly) zIiZAJ6X)#15fO1$;)~zZi+L;XG)#9-NGBpe_g>B8`O_(Mj3ny|QI1)vqS?Lj-O|be@ zu(pShcpRuZ@V7&F3uBFWfFiq*&F<>oG7g)oS)3?RS~>$F*<9W(&)KCr!7Y*holL4m z*|R<7CSIHkbhhOC`=_RI5-sFkon6hKz|-Rt_oRXkjjxTS(90QNWfT@v;?nj?Ygl_rJxt-Bi<-eBV9fc>Y9IvcBvok_^#o@#Ql zRdNQW)FwAuzgb>P$;uaH1&#jqf~D#jA1ie>VGOc(xJq?uY%#EMfdc>%^e#Nt9RGqP z{bsyv$k86{4hL~2x)3<_kHjs8;w*zu1exU>Ebh1Be%|+L{y!Z75dF~0P zJZa5^ZOdifYS-K-v*qf}XVErIpt*|QOtnaRGrCg>+DGK(rkM>4pzSk)i(f=4yKFpb zfoQXES{odj%ydwVLgO!{13`Un@QQ@3S~Ahcx?Yv^tMYnQU46P=8Lry*?y~HVi>6ib zCrbE*J5S`1ubyS14N}?dDuqsZ;wn>k_LE}J%4E2+<-wPUwOVCHzwo^PXc}0;b^gRR zL`V(>K`bZRCi^XZK~*3WH?jSH=QnK0X<>F{5X{e%e@Y}JItKK_6+zQ@c8}(K}=$Uy>WA*#b z45Ws<9fIBPW@u#HkT@ahe&SNi(X%rO!s|H?)fSxT{m zHavcD;sg;wh@cz>B8+tE>9+pAJqv;!y7D;&Wr$z_Kgz=5KWgqe9^+z2r9>A_`Tjwc zW1u4mbhR!6_9c-CDn(UhDlU$X9QbR-Ng}c)%@ULPubxq17T3f8v2cwJNo}+yUDC?9 z7@P>!d*8e2X3REi!bUbV<0LwK$@VUi(S<)=;^LO$D(SR?(9Yz%T_HZI;vdcBvM$ZA zA+EF3neR`RTrsh`=l+?E?~tTM3_`PDm_j`!0oQRb{QOyHEb64pw5^7ChRm z>h({)0R!sq;TOUfX(!uXrSox#6e_ZOzIy#GjltfBawDBF7Dp(s@XidVY%J2`}l`@9r)j!IPOzyjm*rc|6hjT^H%2@%8Er%D5{(^MbuYggR_?DarS6GhQs^&6e0X^9DADL&3z~rMqbJe ziTzvF?9IuyM@e=dNPq`<)OcrIXudZC20Z*W{`1otSJrh4J{~77rOZOC`lux6sAON2 z86%_#R#=@55_VD?L@}sQMC-kfudlzIyuP?w)oZ;+rH#rpFK5ji_1`+zCp_ zXf>lMJ8-G_bEdTKrjoWt%Ghsa95;Yi4AJ-YRi}^9*tc4Z>+;<arTFY~kFRcfPt z6GHqV%o6ZuxGuvf7l&Z#^oxy?ODunRp_@A=KqrVi&mEi*ubgJP{7g&pMMC^jFeb^L z)mbMdoa;pDpXDW~9E=*P3Z4FCcH$msUJ+^TKYy(A$^o`jv4Wmobt$5gj|XLXQeTjP z%G?eKXr-z=@w)s9&ag=n52G2Jv`1E2`-kPT1hQa4&VrCb&_R3}^%7f?ikey1BYGWdAj83^$JZnRvarTF?xh-@{eF@n0#S1l`-88&zTF zP>D=BFo83lo3TM(&R1Fzv8GZzk_#!h)(p=~sy7oV(e=7UGA}Vz_G>O7Xat>}LVbEm zJVH%{!T5bPr-?aX-cFQr%PX1ouCssiAPdK%*Cc;B=j9&PlQ!GiIEpmWCDlZw$OQRA z*H{~%@?CLrR%!F@&^)Nrbl(X^y&(6L^{|6hcTj+3pM@3-7*2@!wUbTdBLCUa8u-V5 z$1jJ!ZBsGPqqr;{|9nt(o!K2dyz#A+^B{bq#O3+Tr*i$R*XjK}wetO+0c-Q^omg`C z`CW{;G8Ns0F43XwND#0N(}2O%3_&$alNq+et4WA66DyFKyJJs)ef7H*R=3kT0WCZ> zhCsmVRMhJ`cWa8*rauN+WELq~fL?xnRbdgp6(dHb=Cv+!OC7lw_g1C97c z|KtN#;>cbKHvHPyCGbGd6Ha!jF`sWOE<`a)4Kl zuF=|apLA<-A2T^=I1P^iltGjmLVuH zpMt>8kU-rJk~Z*syEibH_65CmnYk7#>qISUm?zM&BwE z=Ft*{EuZ+zV9!O*??-U!RHGASH z>=49B-$R&Vw{zT&WZxO){Du-^d;bmd!1Z~~mv8g6eK6=;KlzpLLDJIe3UPwP$?gC( zlpP~}{Lb47%eDPnvn%s6%nHuB{{GHF-!a2*=OEiTI7xMWGYI<0+^n5?(gm#NYX^NV zx&wEtf<@I`=a=qJlC8`69#b8CzuxYg^gH+-RToCjy?(ozUF*<&nLS^wFQ31DV@&&g zw=SOpV}$UeePrGC`wE-ism+{!R?WKu`yi%4qA&staUd75O}B7n9nkL}0fG7n0f3NF zUy=Er(IDU*dw-=r2X)5R{@#EjPfQv%iU2{(V~ij=V0s0G_T%nAM8q2W$;W}M-~oyF zx%mfTkbvM|kXX!X{H9NhP0@!L1;nLzi3#X}1W@d5nFkO@Xn(bDpMKokCYKGJ!0*(} zEpM}xB!;}N6MdR*_iJTM#fiDC(63+J#dky!2%^MLPQ&d&wV6W{WB@QYifr-W0)Fpg zSb&@WL`GnNq7cInuVJoB6*$SDw5OF6J%g0eltM0B^AP$tU>F6!^vOsxhP9)h__&Y2 z>G%&dkg2?Z$Y#(zaCFI z=3Cp^95D#+T~aN9I{f5M-u2n8M=!y8{tiPAjUY{WwtN3LR}_Yp1=6ag(gumz#Qg-W zoFy3BwWPs@TP9m<$E@@~20@!E2LC=0{)HXwpc{k(p)w@0x1t&e8@-j}{>cCXtEz$B8-3H( z&Vfv&)>NCaCsH;FxE~w`=0!JFUy37G&##|yDC$;47LZR~G3-yJ2X7MEfSfhO=Rji| z;5S?hjOS!H!<7k2-JY>KMFF{Jh&<_2(y*>3rf$*bRSUk24*&4u_tx70`PY&^92mqD z-liDBy@c(>GVUVFbj{e+o`*Z5I1H?I4QI)PMUSHuV8`?uupn_4Ir}CCS-UTq@xFClCjy8p9A5*FLQ}n?zlK>qil22}50DI2Sv@Gq?7Ma}9OHTzBZ# zZ=Y^d&@Q-N*-)^$UvWKmg69NN9|5^s^-|0po7Ks6&WL$IcZFEDQeOIMB9W(iD%K(R z@Jok6ZedI~z!l%10Yg=HLemQ(E}jC3>N4CS{fp#|0D2w^lp)#*hpt!g>kkDpE28?T z{ZSLPF|4WHzcEWHp(nBXY#s%;_ohMrD8>5q;88HpkSkP@$!x$k!KN5_P@j070|jBu zoIJ7N-nHx7o{u?M*{SL^Z{LgC2Hhci4TE<}#CY(jrX-^N(CS0;4mG>jPA5%k-i|N7)0LvBp!jFUBU;JgFIM@4c9_=oo5p58le8o5$N55?~nQUl5&Eg zB#jBco#70})sy)Nr|c3C0SxyPa32J=YgBtOKaw9_dV|X+SiZ5geuq4$I6a{3@?|%NDCaIF&+iZHJQosV*&qy#$ z1fFj{LPKHrP%OU)o&CP}4Xzo;EFAXib$9no-*xv-H6>o2-)*QdJds)&^%Qqy8TC|g z4oFRy!v4X0^LUKr{WO&t6;*h!fJsqh#|~%G08;v{mM}jPu5urlk0l5J;mR$6%tDd% zUYNn1JGjXgR3}qGi+q63-fcc*wzvTD2nJGUbjfLr~Tu&U2|_dxebE1I28m;X&|d&`R$~8axPdH z9BD?99YVCugLrnCnR7s?mBZA7ar^A?7rT)Ys+Cm zn2`nSd0uA&@#qTZ>K-19+GlN0*0~_^HXex!;|Qb}Lxw^Pv`L=ZKE5vdx#jHe?6Y zhX-K~y@)7%w}}3T??6LO!9(Iu%J58+Jlb!SHQMsn2pJ}v9octSik}>oJqZ#L{86 zyI&C&_9%M9m?F+5&L;(lJZHL>ssNo|;7OF}iZ>A!XwYPDY6SG^d8u{akGa>=0z?e{ ztfsc8P-L}E;^k2Fg480_V7)8>;{&yH0jm<@iv*>jzL-KZLVXkvGpJh=3trLMe*Yy} zH7(_BqIOb3iNKN&Yoh`zwO5GsHT-gC`mhNo7N4GSh+xWR2&Qi(CZIN)*Eri=Kb)v4r$^x?b%g2dR%=y8OXHyZwWpK62xb6_$p zg(DKe31AJDGsq53{H_2nzo5@U#Zf-3!od3kW#c5`DpnTdJewpYiGaK&AGb_=eDcX6UB_$&j8ACrR*^QlTkE zRvZM&0L1?2R|30a0kl?8?A4M;iGIHWLhF$t8bpE!Q+}T^_q&&)(|^@U%#rd68mLX7 zVh1p!kF@rg-Ue}_xk3l%GCYCAO#S|=7^l+gPQJ0HQZ>h0&Xl2MJWuZ${G_0=x;F#{ z8it)svR<X!E0rC)kO)W6JN7|t{>(mKN^gvY>CnV=3M>c_* z3Ca}gX}5JHv~vCYu1~>}k@_CT9q;fW)m%Xm)&;1i$co^u1e2wC%M12cebjM1tGgxu zJbO0Db|&0n7Jp!s`lZ9_hv-@=(hABAHMRWB&f`#@ahi4E9&HqAtse0=*&gVt4N@G<0B+R3*uO=5vbj3`X;4!Nq1e6)pq_1S+S$#;3BAGTK zJxV;riJs-D>M}k7*eqt6xZAZeBSO~0On+}^Lbn|g8R?;vV1a#==*?|8SM-X96}n13P#5TjqBG3s}+qEU4M2B4xni|Y+AM#3Fh6O8V#lT8XZ`l1!iHO$=qe65YcvN4f<5cT;Vy6F*oi4?YK71n*u)fZP6zQLQRak{ppe4nTrW#Ab0#ak4yq+gTPS42x1%bVg|y|#Da<) zJewtSCEmqIAG4vq$jX`#dvHEq1Y8+R31EMUh~8NU*BtEMnWN6)Wr)*nG}ekU&ftCM zutq_Ixjy70tsN1DIy+ju^R$sy#uKxn{)%WnH-@Px$ti`KG6Jm<9*2?D{Px`fRLC_d zYhdh!#dkRuEA#V8v%>idMHxB_;6qOPGnh>C{PTD}+x)Jvd_0KM#(^jHiXM~h$1=7^ zRXO(o1yDam0#GJlxz7ZC`m^kLkr`oCe&#T_WE!oG1g)p@I zk$zjoR^$;O-7Bs9WUo5|^3@Wed<7&-C4nX{ut|2ICXp~Uv|Cw8xJ2A~9=bDZf^{Nc zV#eAy;=$yHsk*Lf{%)Se4hGkrxkA095yHW`T^PNv3e11(x2(+U#FuW~&i9w1Rt@E}!B5k$rtQ%pgid}rRMdZNo0pS2)X6c{Bu_h8UmbC0d*?|{h=MA~q z7r|USXpF6>xBN;Nt5U9ycBE}DMr=Reisc8BkW4zhTZqOQLy;-A`<>aRA26q6LuEW< z@j=u3Q!5UNl6^V2d#L6#c3Jl`;bGmeFtCQ~lMEL<8p!iWgN;GDgb)F~OS{PZ*qWos zYAL0cb&k;zx4I3Jn*_qZp50@3h=UqD;uf3WhJ+{y|MzRkZhAP^z#EM`Ju@3M(=^F! zJFbIUDxQ@mH1rb6)wMDM#0p6I^Dx(%a0LEu;`TACdb+1O4pXXk;m`E3vN+{6yAt_Ha#!M*xiJgArPuI^gOA(s|sArnX>Hgac ztN`h^0nN|a-N7-k)Z#%WOb2`B08>BEP%Xao-uoZzf99veG$#`Qk{rwp5F~;36KF!- zj&ps&Y|k);4T6p`%l*3mcSe200=D_%m6}!F8kHyL96o2Wu?DJK-JTN@4QWKkJ~!r= zgp0=R?`UvDF`PT*ObI2LGHU31obBLcVX@|T+PmOEE@b2p8!Rv>PAMZz`V;8!*8>eopq;cVTqq zl_;F*+{G0HkbrE1yz+K~GW%8LJ)jE@D}AJ5C7S&Tyu#HfVvI#Jq~Ezsb1n^EhYa$nY+i4f0upHyZ4InUzF51 zXuOG00&74O2u0<tm z1{4*+G{PQNy611J*L_GGADISf((NlRWJHO&DXY4*O_t=RmXhS<&sYhwNwfsIwgF;o?_d}cHJV`b|%n~`4w29RfJ=HwXD6)V28MjypBl~w? zw$6^RrMMS?oP>^;d_jNh%LW~x#PgpiD<2tTvkCDzQA~56M-iB3t5uZp^0zOMVE@Eg zVOz8ik?>VMOd(fKKgiI9;)NmyT@fNc#pGio>pFWNiWuf6)0J#B66wkowgK0Ng0_Mf)|ElFdb&5S9A%a{_zrwo~bn zz%9KIEBo?WwlwgtMh>&QvoxyP691KBfjGd)*8p{PcOtA;W=BbP5*JnjT+p;cyiafW zz#;hlo9gb~Q}jK{C02+ai(Osh2xb3sL)-Htl{>!Ia0c@e+RA@dSSj-ItiI5Q#^)Ln z4>K3>49WFqrdN#YG-$v0_#%ROF=o5tPvAIy@;dJTTo@WJXDz|%I0HgGDVjzg>Gh23 zUPy!?V3pSpb>Z#xa{;fr5r(Srfut<#pBuurLl-8ztUnUj{mvktMv>%?0zb{gUqa8w z4YC-?8&IM~Mr__cwf^009QWIk5m5G(J=O*_m)!9sUFK%532mynTi0HBzF3h59s%*{ z_=PAVPjS-3Gbf_zr@hWxk`WQYBR=~neyI4s=%9d&qS@lLtOZ4rJ)Amy^1T^c`TaIl z$x~nGsMYlX#w*>)F^v!-08Ta%#*bf9<6Y@_Hta+pa(^z`zZy}(0hdeJ_0kA5C&3FD zEylv%WtP)%lgqh=PJpd^F7^@P7uu*%l5P-U=o-MW+*6Uuv;Wg}o_b|fu6?rpF0#&7 zIg>aS`P%M`Q<)(e+H?biTqxca){G&dZA~`Qs@4&g#}w8>IVCvCH6G;xw;lrzA>?_G z!9RlL7WoPKW|*gT5Os%f92tlhOy`hTi~%cp^4Q8s`2<4hva@f&>nu!biEw=j6`UAS zs(p*xODq``bj1S3o)F0M7<~QTz z!opsxL#D0ry%nXLdy&PQc`o<94<*qG&{eq~M#2IbIKAJz;;8Vjs>y|(n>F_no9y|Y z=e|q}U`(@91AtX}Jz|*lZ8x-XUp}gOK=g~8WH5svhxphc0UtBvD=`g#woh{N_mI^Y}$1dj@^>s!`j zzDaG|L@M~x(WOtOn?>ndiFtdANHBrvX=v#;fi&p&q1&YCSl<2-E~-CX%U>fKn;X-O zIhUJAWT8c4dCe_hiN+veM{HsbA{$(4^qbuRY65?|I)9ZY7|ueh=)_U~`%Pw1B+nMo z47En&NEJ1h8G7?_zwT58gKfnnXUS%>=nc@fq$}H7PbNhK8KK6_*^FrOz0`;;N@DFF zPa_qyE{=fSiWh`zTMbK(R(@A3XX*-igy7|>IPy0)mcrxj#epF+H z0b^?*xlsl`$K~2)mz{Fuum2{I$vNEY@!Fo6T4o&d>BDAo)Rj**Kx_RCnSt{d%ccsS zG-hpVKLl*$kYd3GN*e(Qv@R%WbaqjM0T3vTVzR#Ndv}#hxAOP9N*1Iqs>eUX>nv1F zcoa8(91f&UK?xog$z;T>;@Xvh_Hw~0-;_2;CLdVFtZ^(Tu^uKJUF=NOa6P$2QqelUd6#E>yKZH9hby@aQt)IfiJXw$v{q1M zN}Q-PF*d_Y$(4((>wdh;b4Z1TQ$mg%YuT4(>>rW+J<9ue+u4 z?l@t^M}d{lJVZ~svVr0$b71Q0d3Lty39n`7B}mfwhHVa#RszKF;>yCpT?;OYot`3f zpjrrQ1EM=E| z)NwL@=j+i$srrsP;D#Q@yrE^=pl%|*)%tGwxbF7UdVF=Kg|9)8LYVlb29H;tCT?KF zW%R0gVwMOkX&`2Nk4ewc1lg{nyBXXTdULjQ^sdR?W4ZsdC7I5H4aw<`O01qi32&JM zxD(CH*N_qM=$#B~7EqvRnhuOhV$a{7SAV)OOcO}X9VSSq*O2?D!(l_$ij8nD#?zTPk@wQo#-DW_;Y6q~mlZU*{30vi7cRgKU_}RV! zkO0i+6X?K~*bW2qD|k&u4#7BG0Jd>~N|t$oy!J|gN<8*a9jJnoRF)Zd7i&%Q({r<3 zwqI6zK^=6p^0a11aB9)N*qP$@yUE!t=%{M|I-}^VIVv^~rSQ8#94l2c4FI@9ufqpz zEp4v}Z&bwIGbLCer_?1M(QA76EcNw$bh2|>tn|JS(=w7bU#NUqLV6Vk>m>5L$*E-h zK&RiZf6J`|cc@yMB~#=xla6Y64O*%~w6knpO~krCAVD?@53gf>D%0i7N+$i0Ms<_E zwznJ3t7WHsn3%1n{> zrrH#Ss+R)SJ#uS)CTQZtsAz5GQplQF_?f}#zmMiQrHCMHOAi0_;*!UQv8$d|9<%6A z<5u|e)cSlf#4WR(S-`flMOgGav4kIgr2fC+@V(0=aAPs8Z{^w@OKiu%KsB^>iJEB}Eo&zdyoNltssZ*9GQHK( zKx*}vaiYg}VYahcY;`gwHqj<5DB2=1mYH^I8uQvD#ZIFwqvG#7ydTEQL?^^G5wG7& zcJ(xd5^2tqiA2jfCRqu6v{@$%9ID3N2Yr^CUli%{QTdGX=^Gy3 z(D-oaCK(9vvagV02=ab}LWztxu$+WAGL)&n_`1Kr+kV{%|AbIbUcznE0B~D3rTSk) z8&Z=R6UFLb?jhuhWAxMJ*Xb2g*cLQ%V3I5)}Vk@Ge(VaKk&2HvgF z|C;?U7JdnOiIU5{t%J9gM@BtnQ{+c%K$0<|dps3l(R=jwJ!Aec6}ax$@ThKT_P6e@ z9=%VF(JZlqp{|A4Tl5mnqSbIcFM3H=vF>b3szz&d-?l`> z%Dl8%&qKJM31vEd)5=A#oe9lorF;rAANm1$Dr@Df+tYdj&Ow0|1W8U~nN{|TA2{+^ zkDJvFNuqA~eu}b;6=qipZPrujFOtJOR&BR@p@wX~+Ou}uuA#SBPiGXK>4G=vN996i zWlv76TGwFs;UOxMaTeI)N##j;4k>XOR;WtKADeaFcyd&i%mYjfEv+$ly}KAQJ16EN zHSzd8^s4!3^VkSY`K}^qa*!;4!n@D6o}G+0QKO9+r{#axuMvdbYg*+i(fphQv{J3- z`DvM^MlR6KdZmCi_Rtry^=^`_r&W?A6!8$L_tDjU97al;11KOLZq-G0pNd0Yw0QM@ zB{F#IkQawledB}ry37Xs`4)(ebHk)FRoCC>Q9XrNxfc?x*s`0VntZn+;u;i@P!~kj z&XyVWTYju6vYu=o;PKGCWxH(0Ym`M}xc9eV75=}}O=KGHDsqL0_Wm%sh;#|yK;@b4 zfrjS>w2&C%Kc-4k2Q<2+anNY5-`Ve)TaRirx&_d&w=Q_*BQ(gUL;3l%p8eYw6XC*?)oj_#f@l~Ep zCy`=$2!4-DGC37SNzCbopoq8U?bP>xW}K1mPj_{M;F-*xZL$+X0#Iyk*0q-3U~7L- zI~FMc8KNqvO<}zT#>_e?P@?6(yIuo;*yO8wt*RY@k5#@4l=XQU zbSv9him)%;1Y*|#Yb>bT4WsAB{iij;XpMa0TX81U`kWz7tQv*L_%!pD+~O0VX>We{ zD?3#+?c5glyTxy|c}O;TQEEqeRP*FW91#hi_O24G^Vf)jbAq&JjPNj{lqV}q_B?ZU zg2clc&&N7cPkS>_rg;G`RJDhTk66uj@z2WV?x#9s^+CCrl^%NZaRt1ct#sO9K#@0F$pzKYqSzx^1JMEC9zWmKbq2h) zL5W}6GCE5Zt4#(@KcW;{$oL#eRaC2LVBd?xUud}*bDFLqxAxsE8#eXB39-K3(~%1exM_- z=eXu>-X|=xzYvnO|2?#Q!>C^d;4#kphdh$q(fUNes9%&d$2HIjW5>BvgpP`!ES8ZL z_=k|{FshWY%^)2*k~HK@)0;g%U#7b)d2SQ|AM}2K}i6d;Evze)o7@A)uA_TJIRc6pT8r zxqsK4b^kWv4{Y&(PN#7uT7F66+ZL>%$@2!fvn8L%!5CC}4%0m-29{p;A}T2+qJESL zGGD+h2C<`QuX2c#*soNk;0#tQ_N-;<1Hn>F$sUg#DQoT;wpQI34L5XU*sW$`zQ5iv zI(5ihheAdrj4~;gS2zMZb>(rOy#zo~N`ucd!Vj#cc}&EOG*KtB=m7QBKX3Pr!5aK5 zGe4Od+}8aQ)uZnk>-)z{n*hh;TUGYA361l+N9Tm{Ep)sLCxm-h5OOm0P6=DM`wQ;|=ckFbQEzN~_^YGgu;;smE-SD+De3OZ%E4tTEgWtr}r0R%{I za%*qyV}(l?FWoO^<JF-^ed$IS~1{QvJ zA#0S;oCH(JkD#6QKx>^Q!5d!6v`Az}5!DZq@qBeY5mFz7aV02c!sDPFPgJpE7_0E*`l)#CGQh)TNtML%TC`MRq*=t>a9uvBxGVPq}oS_jCBX> zT@aP0nS5R5yBZUAs(y}uW0pq+vqP47hnA*2pInjh!!lL-ec-l}rEU1kVY+$UfL}6; z8F@4cv`$iyOk+WT9u)R2nuXWy_~_{Tne(t_Uit2k*b;k)U5ecD=zRBaS zaPKrKGp%@``ep{{9I78Jo6XR_X3X9#MU$V%8a6_Qeq5>P*SmR)ls4brSZ!Fw=4xIg zTeKSS3~G9pTprI~QQPJ;3p~b_Q?E4t2gX(o#T3VC#;8k-#_r#-#UIU%GC!mijOcjG znxE_xMyL3Cu8W6m90sU6)Hb9nmn`2z>xV=2o3q#&F0k>Koh^#}i$MtS7YL(CaGZ)y z8iEXQWN!pT2UT67h2Iu`B!s;Fw}*?H-y32CCdc_|Mi2u=)cNa&%i1sxl&+{dQ>VNl zP=AuUx%kO7O~ghSl~ej74x!9Te7V|d3A#W88ggt2x4MhaV#Jm2r%&EIMHG=!B2xwC zScus_)T6jI+Rpb`?SF>{N`3`gfBb~@%n%y?>01Uc(9buyDuj3Gf% z@Zv4IXCJ*IbM;Tk^L`M})dY6;OA(SLXG#@KJzWRmHz9g}sb4~43K4%hw9=*)~x55O?58x;#s98*5U+P?bS9CK6RSIE?i-FJgv6N7XW z@j_y6gG)0hHi*`CGZ2yHkq}50xebf6+`Ifv{?&d`qd`q^AV6e}0eEEFtdhtIWEc=n zwKL2Zbt)GeE;0{Rfw3Fta56uY(p!lxrc0Q^euSOF~=>&@a~RLbr_tfh%3D`!S4Xuf$hds zo8Rho2gX25Uit=k&U07#EAVYVpAJH+YayT3TWyPOUOFObt*%$~)Kx0GN=n7XZNN2gnoQ8jmev z`f0{GaxcI-_pW9Mqv?#@U+B{;3!pXEX@Y3FjTQG&RHUgm&1!8XHLxfrNk6?2MKY-0`Iysqcx?o}ORatvvNX)Q$xvy9DJjV5Zkw zssdNA1WE{;yR2_Q^$!4OYx82OeIuS5Mu^{Nr+%xl@O!zyS2VgBJj0I{%O4^MEouYu zLzR*C(-47r&gpvNOs=~}%T|sx|=M;2EQ+`n+9Y_C8&#vae7!w|tPR$Z5Q%q?G8c;fV zBVg<#`>aiy$511{Rmeq8%!wmWCQg_ph*~*>^og^0-WLX$!zG8c6Fd0ESF3awj8*z_Rc z2fDD)d=se=T0N7n<4eGY3zN%0pMi(6 z*6OiDt}78g64K9#tdUwc6U}o*5h43YX*+tCk7#C*np(k{8Lz0l!0Te2?et;bcC&|K zMwg*yi*v}3`q>Gk0WWW3LK>!E_%LDALLORc!LZkIG$GA7?*a@u*!>-nC1n*pk3l9; zIJNxPW>!CU(87!z2jD^V!}zPpOY@EDi!nv5Yf`f0NI#e_A-ppn`wo(o-IWbdKoBOv_Tz&xP^7jiC!D^shJSg zhB8g~U8b+1+|3Z+aEkVix>Fg*PESFZ`5D3r3(7yaaP&N73rwi&CYG_)@JHJKABK|_ z^q^@_CbT+vkR@nB+opwLrv`|_{$QpKnw)gx?_T8+GO0#K|2L5@CDz^{i+L1HF2Z>r zwwoP)Aqq8GX=!q1LNL55kvr+!214&@qs(;*BI=rcO*B6#kM-n@V1;@P^n(u(n8vXF zrU7apjp~~u!Uh+{dC3GzKC{{z7(m~8=kE~4sZ((IwqqI5NYtb92BHwdP%$q?S3fI{ zb<|crE{}V&vw2F7VzVQw6bgRmIU+8omOYgPS@sdUVfg0L1{#CpHYpzhoKpE#1Igvq zL$yAQY8u)c0U#h1oF+v3+v(#_8FDt#!$F8&5^W&OaJ2RiRSk8}Pw!X^Ce1hKPXQoH>!!yNXcs)So}qixMJv4k)Pj&PM!z|K@CjGDD)g7)V7n_p>7Z>^lM$z)b=KmIWKN@7T!Bytf+V$3g1@>#~Cdv z?q-iep&=9G@9Pj9^2fc66Z4swB_c1dsrw(Q8nPihVb+dEs+dBpy{m8Sizr%V6UZDP9>4UzlZJC`)fitaA8lPsKo za_vtDrs5RdF^0wnxr+~tDH^8q@Iy*GJ~n(APESh^E|kAnP9!iV!Z{QiyN$XnDH`-xB;zeALw3v|r+^jVO86~=A*s0ysl7=24X=OC$${|o<3f5UVY=oY@ zE`cQ*;e6e_S&0CaWMjxI3n{endg7Y48%{bmx`N(sQ}|$tkwlWG{g?x95p*_|9xMab zt}RDO(I8D@%gnfv`nnn3 zJL)HOtf^}m@4u=ME0;Qd3hY|O0KVLH4X547K9aFm$fY)B)8BdN3cG7FP^<@)szQF; zKdca3Uq3Pf>#gB1aR{~XJ>>4~$gT-sJ$RoiM0Z0}l$KnYp2HD5qW?bPif86U z(ijWq!?ggmW6djAoR*rX!ynu|YH~9evF%wLX%GNi zV%rz7ZMm|eNx3I&(+K)AwMtJ;aw@i*5 zdM|2^}k9v81EGsD`+!@<9##E@nTiq2FedQJ!B zUMYR_c1b>d#k-0CqbDBvE~jEt5c&O#+(|Hn(3Ja<#weDfl)e0|Wj z-J$(RXNZ_z}0G5iVrcfJquJi}I_`ki46gA#2 z_<#8M+&sA^F>0guwJGRCt3geV%&5AM7j?XYyySFpb94EdBX5c|!UF2wWM4>qM$u35 z9%Uzo5vhrAg^6_MpcRSRpC9jirHUw`MkU9~j7U&24U?NTe_Lx?^p*ogQ;?f^9ki_{sr)<>O73}xtbWFrQE75RK>qIvF6yhFMIxiJ)=^@<5kCE!GH?R&pzGt4%HLCFOPki#d6RYjIUVr)W7#6Rw9T0-NA4N<)5Sh;&8TaCE+T@^(=oo(Q3L6) zfoV4N)8tI|>BD1kUYqv67NXY;CnLsw+g5E@esN=-X}tc|G|bnTc)b(3J#m(vm8NT- z@99YZtyErre3U!23HZ|;|96|tA6R$+@$w0fAL_Vx^PU{|S;f{$<{}JSA(1PxRNDM=b**Yt6~-j$sKTElxlF%pVXm$r{q;QGEck>E*W#)V$Pi@kxqF zo~_Jj@6<$r}Rwd2PP8O`VroB}eK(bj?>FNlAiP;T^eF>ae`x_uW?d$_$)r(7Dl;Y&!8 zWBSa!MVP1QL`~*P(v=^I@rZ4Ffn~qp^mgU!ZF_01eeH%}W$@5g=O$Qce)bu2l~`G3 z{jntUlRh-E0z54fV<>OP$Xwo7r$tyoe_Z(zm9$uF4j^ZyO z^gNE@H&uNu8N*aswXuGrN^~#UeO8O#6}Ob9q3;7kj*H!1s>P3sTRhXK?ncbVu@0LM zrr}Tcl1-1|W^x(+hR2onI;)QBs`oo@x#kc))(7cJbNk& zyUJ`1d|3nktzONw6IprKp^(CWc7h}AUqtU*hftUXfjvY+*NbRCnA*Z_YSNBfrtNes z05Z|G#Yn zbhhDZf%*tk3uv9|1hgIgBUKWj323}o8lvg!tE)psKpAT1yn*Vw)5@ru(q);4aVP41 zh3Tmmf=ZhL)8 zf@>XyS0_!2!FEj%q{#8)t{73eGo!S^03#wT1{xL{NeatHjWv<`&2KE z1JO87H{GBf!XIKpC{BSwJK*qn@w=ByNa^fZH0<{|COmCfP6PLVPLFJOBhfDhPpSUZ z?I==_NY4w|a`V_B(j*!}jHH12Ht7KLWVEBWM1Q&2bf0PLhcT-FakdGXQ46C+2VI6j z(y7G&$>7)tXWO=PZ``oEuy>5H*va+W zwdwF}rQ;)g5WZ8*c`a_2^wrhE(lh4ImG6ILp%4hy*;iOFgXPz=G^P9_4YJ;)`_$S& z_a7m{W_cO5fCWE}%Mfl-uwI(GK%U-$cZHq={~?URytDV###{x^%;STl*W_FUR1x%m zzckPNju-#zllfTxX;SoMK;~!b0(q$D&w$dy)((1C#-ARkkFy2rxrjeG_&>c1_|yBg zH}$uGHK+WXu>QA(&tLoh-+MObYijwwfd%3DY`5|Xohk0E1>>QT`)k=cHHzv^Ptj-P%> zB@r_SKs3_7PL=r zazOZJooDVa9DuMWrDei#x{EIQmjMSZlL`1KoLepID2=d!j64LIW?&`pXMJgg3=;_ZLE7`2yaTz zOXvCgB9RmnKNg=*`$!70g?##q&@Py_f#irFC+mJivBu2GcNiFf}*cuL%D@YEm?kRg_{MOO;Pgio?5&TdgvJ`cHaHERJ2AdT9>I%X;nK z?Xg8M7s#x7d`u?O3l?8VxGKcOBsSTNH4OlsOXKT)Bej`C4^8t|Z z>P0XQ#yRrU^4e9pzT#OsVcYnJI^?a#-~I1n{Y&&*38b^Vuz8Rl_fPqYM)Ba__bHWY zzN&q}sx?mE>(cF-m0H*HXaFF`($@Q6&D(^0qlUUQqoR>Z`d~K&=Eq8?NKr_ZQG!2* z`v`+ubR&&_*pX(k`(n%{Br;lW{x+3*Fdxt9VK$08lU!0hg+&JO7hi22Amt;31`SBJf#SBUvo7S*CY?JB@)@b%C(E0i8eLhHqRh57)m=e@}uWfUr6o3IBL{ z*AdU*SVcJO!(i)rgX7JZDMRX}!1vqNl|&YNM2Zl%dzkl)I(9hNRN){#jD_p4nc0M+ zZbC^*CTheQ-7+}UoKrPx&8o&AHSjUT@Dp*hH>}14B4SF0mxaVSuUxXdS&MEdZYssP z(TLigy^be8d&%hcJp6l(zCX-&PSC=}Fza0|TIBeN_>-YSf6})7dhnxVB9mSFpGQ6B z8-)qJ*9gJ1i$d{4IaSZ3Mh{8HS+DYLunCEdf8IetOPO)g#Q=sbm7U0$b_crUdkv?M z7@n0?N5hbE*0o4JzfT^v0mCiHR3ghbXstQ_chICjgq1zT$lNR^LO3RG7I7OPrCYHf zgI0`P78SRkI0ymBNU(FzHJDmH^$bv`TCPp2x!bl&j(7o0LdKZN@wKew%2%$ycjc-> zrXM7|FN|iIH>N#FF}h2X+@E7d(ak|pUU5Hs*?K+qy+aUS=ss>VHEJX#DXI;tec+{%VygL5=*OgDFP%zDp-&HmDz+XXbHeIO} z~Y+^SIW)+mpG3EE<@_KYT>FC+rFK< zhgd!@_x|Hl8Ej}WM_Wn`{+18uBM*nT+PS%>i}%n=MMtC_^L;|L%0*jxxMd{S{0n%p z*G*xn#IOo_n1YhT4sBrzhCi7>*0UlW^7*R=K1jOMI8dkxt;E!Ig-0(wgYrI|6n%5x zR)!OeW?Hw+H0A|L8@IpoW#uhU1fqxp=Qd&pzaNOmEXg)CJM=o5GZ$lAY1a5JT`@bK ztVNBZY_FAb2AKiK)Gwwc^?1h5-KJ?JpkiYcM>0SV^7>rmx<$4^95cXFH_EbLDBTLr zxItKaLKq2k09Xa=O{q!5Q&05pht?}_4Sx#H!~X2P62=v1c8+c0ng|VJ=_s7l#+q$f zxgulztE+Q<$@A@S>S`qm`BhSD-d+F zP7x)${M3_)TTAp1GMCik53(nOP;50zJZqW6&(-fT8i3N!wBLmZ=V5~`kHkoV%vdL_ zpbD*(av~#A)nc45M6+@u-i8IJ&&eGrSHFt8oPq)zzAFble5^1#vEX6U4Mvs-tF~CI zM_lmqnw$#3TE-4s5;2pgEzq1wR+X7|!)vWyxWw*KsB1;pq{~r_V@Shz7Mljcz?a7T z#2rRW+UgmEsT%%eNRM{1P7+}-!M-rm(>=y*Gce^ak_|6Ww`fe@xYMVzUu;WLf&L5O z9-x?5Pu+2u4-pUAT%nL3=zBTu08W9Oxja=tx3fVMg~;5PXmPAd9EcS`8t#`gK{S<4 zqrBM=r(?TXm31-$ek&j9$M70@7+L+Gs(Je3KNF)1!845WK%a+n&c@AXWQ)$Yo4l&kISVQ| zq>T~hF&-T|Qn7Ll>K$f^D7C3*R;uJOvgD&sx^LNUuag45_%n(*I|M5>>_cH000DNG zVU=sH1=h74bTD@(1%W1*Z_d*8H-qgQ8XW`j2^aA=kRcoBFP91j^bKPEm;c)T^QVqz69+ty~$8|!a|BHWL`=86jC97*_L-Wj5x+CZ7P3j>jHZ!f3^`S8 z?2OqjRAJNrX<7IM!SLAR(5PdbDF1p5@xv5`xRhSA>TX6KW!OXv3`%v5as02DO5;r$ z7Cb`boS|ZmKpBXCIN!O^f6c)*uM2xNP{u!*$5arTg?1i`rme{f!B%3HezRH2tGa5{ zAzMY`X|IJFb346jj~a%@i%pT*{jOf{ZRiNb+0QAE^K%N=w7@|c`OQi2r%gGXdcO3N z*8lcIFXT_z@o9WUurjDHK>lsfPNb33i#y@dT;|@D#jBaN@xw zC7;TMAtXbEQ+#R{Oo6LqW7c{x(gG1iIi^#X(<#awO!1*_kV%(<PT345b*<|3X% z?3>CkagAmb?3kWbx@?l~tU*BqmMc$FA}-!Kp?Rhi8^t_x7Nh>}Xmf`+B- z1WfsDxbQ}q7Z$IkHWGMnrGwro&%XmDXiL0j@H?lYuSF;{o!IfJ|C;n~iJsSlZSS2> zDmvb}YV(>KGahF~Bvs2%;mO94sVL8(M)Ti5GN55hxdI%7Uwb!0s1K3{y9TJiCGv(_ zUY7_yY0`}7>(G^MAv~)@ED&m2fBXeJD*|+hg(Z!DQpjauJ&#ZQWS8vECqx@WDSYVZ z>fQUynBa_tS@Gh+dMy9_N^B}2OyS^IW5JNDbI(~{XRiTAcGQu;z+HWL^zk&=ufI5_kaNdvqp&UG+U6pc$L4Q8-Y!2=AA$NFeoTr zX+E+G>}N?I6?x?=sOgp)f;)PdchEd`rV?aET3l-7xO8Lr{4O$>*SpH6Ib8N1GwRT8 zgf%-1h4%@#u*h~IB2o02RQ`Evm0F#`yf$toE>^#XP`D(=99j0)T1Priocz~StDZtA z;Trh*VBsDgmi$fDb^JifA$W(;5WcQNGKEaD*`OP&diXv?H0d)`Y;->k)dq6vx}i>R zu@i;7y}Pu4=D3k=v0m7yXpYn)Quxczb# zBH{#5H?7jt{h}g+DBJ2twj=_sRlB8pjTYpeGR5;?)qJe5#X4%azsnDx!M5n{66HkB z?qN}!Ac{95MM;SP5Z{8+LKA&ljUoJ-4E$fk23anR7o%@Oxo@k4R#gr8 zMq6`5+9VK|rVuZB-Wg`h#F))SyO>;B7(j0Wz&}Jm1}<5f0{ z;fkx@MaDKM*`6~b|4wG~Krv01j(moML9dUI{bLTwa)u^V(-yI-3XY$3AH>y6mwkd8 zbhXTP^UhUbIsS_{11Aa(n;WcYvjf&h-iBU4vd>uQv&lT~$t$e?N#>ru(!TdMP2yv& zYDo%J;sA_TH!TOX#cq30c#IjoYpKG;Al6)f9Qn&03wnkXRMBMz zLJMksA;1B%V7Kxz{*LbJ+j%{l=PuJb*~qGg4pG{L#?-ffO9q3+=iBZ69jbpSqMr)) zL8)*OqK1s#MP>Jz!j8;Uj4oMkGdSBUi~Hx{rsDVZv-8h< z-I%jw++ISmS>Ar`3BJ4vW-W6E_6_-!-h-w|*|+&FH4O+us~aC_u1eyh5Wcc;lv z`E(1(Mv0%6^wbb7e447gR#Gom$+($O77?ti(HNKlK}>=FzGeQ-Za)0Z=V`!{=h_EnTbs`!`XspC;q1yE#Nc@cjnE=`1Y1(5K93 zjDFm0oBN73qR5sRN~I|&W_;QydoOPpuiwSIqmW^%#WvLb83ZM8jAILIZN<y%pAVEoDa-CGP5M7iR26Fe9I6>G@O|Vcc9`aAn6;K_q+&K5g`4G11nzm zt6&nkFIj}ReQ_)=eG9Sba<1l|yFPbZHytbmkayV0lU__JU|uxw)8mnET)T-x|5=& z$-<=5CLvhedJ6UgjEf3lS#~Modbn-1fUq6I8ecb+nvQoK3yCC;iDo3^} zdinm8+4%;o_xu1PNNW1;Z3 z{IycLQ1)&`U*Bz(>o+c>d>Ni9^$wo7*So!yx0x%vs-#$wyC&DIdKMvzJ1+tc!S=jQ z&jH-EHcPV%wV$amtcD=7)xp?xYlok^G$E}8HwCXgijs@kQBOgB8+*w2^6af z(wG3hhHl?X$qBEJ^A|NX$z{@nF6QJ92zp;8d8D1%128VI6l8J<*kj2F^^fBqZ;-F} zF?sXe9jAdhxp{cGILRUGba~18dHKG1Mb}wi+Pt6)`t>D$ z{hlJ}*(B1x@_SX6 zw=%$YY(O1oma1ZfhY?2DP+>#wP03h-Z=7H{UAP>N_Rv`$q*vkuli8=p$U@?3L6Yoj zC}&4HYl6KhS|XVctVr-*J_fdCmbuP5%kR&`6#!nUxG|TBZ%0Ar<9WW*S$i{@&z#8y zk{9Va{$;@xRR0tBOv}>fS0%;M5@W8YP?H46 z>!Yh5a=}>NgdyTmtG8GWz_D#M;PTdk8|cv;w4%>t0|5#{YD%=1;1`2u$K2I+k9xp$ zr5CE^zpAtFG+8|cw;S!@KHVx)XvOk95a)!lh|b*HAS+@BHErZ8x6PJ?e?W8$9q)f( zB{T&OIqIDD)r#S~=+9qSo8NkY&}6|!BkvC*G)wQhdW;dKstpw>gvQ>J<}X3zBi)AE zrC=?|N*}OKHERINsofy07#^I&{-K-Nh(a}PTo|J}hP>mx&UL^R6LjEgD3x)|&X9GLQ z6}&Of;41u(y}$hN85$=#!q<@g(2Bqo(!st-h_RTfpr7m>cXPefg)t}BgmQVs@Bg&F zaV;*6C^B zoAq=xC!1*>FYJ|KxJsYWJZD)VvUKvanppX1CZ3l16i`CIv1__X_`tezx}zpgqf-Un z`e_Gd03{*AcvsxZ_Dgn(JaM}7gLgj=H&!(0tHj0DhFFpIV6&a zvi4V&O*nn^JqfDWg7$(R%Hz}eXS*uQHXz+@x%SK--;t-s73=GMIf_2P8$b=aJR5p5 zEFt-jm$r>KC2*N)Z~wlO&X^~i;4PqaXtInpcgFqpLJ`B1nGs)@r4~D@2WPyZ**F)H zmPQV?4S#83PLOg9!xDaKmg1SGXCaL(_W>OCzaFXx99O38A}R*{&7`1m=x6K-jd)4j zr!;cj;m#l-R4>+_0cUz>__>$QLXve1yUxy(*}+##`jnenL}`(mg^fn*mb$qOo!bp< ziHch>36hQr=lMdx+X78glBkKW!+@=mu^(@zM)l1>(d$`zXCYt8BJ1TM*}mK*Jx+Q( zIVQcH))PZia*>c;&qb+IFHws9)61USQn2pslkVkix0Fk}M#46~o`@v;*$LU*{h%Uv z0^;&M=|Z=$^{#*8ub>LUc20j@s;s*EV zvm#Bq_2>Vb`l1VtOJ@dImk>oQQBH4Yoz@+vP=;r2Mw&4a1V z08G-mHX zGd1`8KJh$4!iElA&4+x%wal4=+2 zd)7G98X8;arSx>Mw=@ua+*^NB7e1quM*v347F1tOxXM#HDep5yq4Pr9&=-`b0vHDL z5Cr^Pe`^Ed*-$d2j?dmw9gkezX6QKHIl@(!&v%XYvC~~Vn}*ESd?J!cs)@{CbsA*ehA+K6&r9cU(zn*CiPe`l_>z=DXSNhJ|K(}Di0 zGQcJ!WD}aGe1rx|cf;`ADn7yMwtu$vzz7E|=T1cOr6SJgt>OR&HuB(k4T=w@k)+sb zIQL2D*!v&<@gLrPF1R|{fLqW;zj=Bg3SH}jzg@pg7kR1_9|EfcUSZo$Tx&g=B|VyH z-gCcfMxD+9@Tas?pq47?LV5Y#Mmnstg!jj0Rl54N@LypQNFu-((~sH*cX2Ag_tl69 zs|EAGZ{27)Q=2ZBOR0Q!LLbauE)r+*Xk#X9l@9`YNdHhsd~QZldl>3jfc&3JQCy@< zjf60(uhW^gDn^DW7bIacMfzuBOE4*gR7A4Wc^g>G*Ow~Lp}Ei$ToGMul?)4>usKl= zkjAFlBXQN!Vmti2XHLr=G|=Np6-;4#1@cQilr(`c9gs2omymWBIahisk!?VD#>?Gk zFM39L-;v9`XeZj)Bs@#3(UOxxBJ{xQu8ni}CS zaDy`O1yT#N*D$YHFH?6*L}MwGs@n9Y=)8c|!&eLpdM{}F;fl(n+6*`@ zWWrQB{jf%nFJDAEu&+SMq#1VFYP>Hdm>%1*0V>C;;-jlv#Sgq>Sxq9yPp%E9ZV!)-+c#67q01+(!x&K*ySg&ZePp(Av zr4HEwXN?8%;>;ke7gpJq3x@Fc(JBqb2zJcSHi*I0RPF2^LA6(|qDpto{KA?pU4=4n zg~vwJu^#cNV2+g?bIpft_Eny87u?T)Vm2AG_sUl=o*`5F=-Fgr1If?|oo2{;z{6}( zI>UepW5~!Z*(lv9Gl-<6aQA&_`sq6DLQp__9Ga35AC8=<^|#+{?sqWEo$Fxhi-zhI z22T2!HVocy64@2=3;Y_v^(&YFefw?S_kWLU&}@R{YAjV&;bb>Qkplsx+Fce{4>n{h zvC-B0ZaIl_L3Y0}M9eY|Nc?=n5$ODscMV=V*uWx{=p?Xq`Y)m#WRqV+d)DzgV^?Or zA-X4i0LpWT@tt6*EY%I~xowaWLq4upX?sr~;*(of?}MJbi1xhSc_uUqEE6TYity|$ z6q81-XPJf*1J$Yczb~S_?VX+8>FYPI56)g6zdL_@w&|S03U^9GoeqY^dsQ$Oa>N{K zrrU~szw0!dzxO+2?QrCziK1Q z10IhQxzdu+u?t@VA0{l9%t#}N z3qvskv#+2<-w--Br$rK)-`n1(m}OLQfw5T&npOq^VE4$GC77Y-4n`rsS%fh6ism_^ z65ANiO97sP=9Cc`Q+W4MKhDsc=zN5}*yJfFC7+TJRpf%P3>ZZkm+@NhExTZz+Vjes z1Y2R>a>)k#qYEy2;qn{=7a|Jwh#U zwyoVfoRwRpo7u?;C-mdz?qOv(^*mv!?{Ol zWL010Wi%VjC5<(YncBuLwx75vFkPkEG<^y6`|e^72tad}taN9EKHNC(PZ>?djQr?s zBtg_NeVBz#rgWA+9@f^NmoIzye=9|g-|U|YVoItEo!GFTyJSWrnb&45@5FUA;t z6p2i~ygooH4#%=h$GWf#s9BB}^e!+00bUn0pt)q1kX^t#6#m1xzU+lMLs&?lV+;LK zsWX)dOq(u1UlH0)aqm?S_83iaQ?tBEC!}eEtbcAi!8Sm-X@A^wab6?1cX9{7$*+gV z%WFYKBDLNzDD^sHnHi%lHHfaV{vunvig!iT4b%BjQKcZDw!e?}3_eA3NYuOc_h*|3 zqG-LW5MB*0|Ni9Q?CiI9r?39m*39tz>;E`9J3l)9@%NKo5C8dSLO8>O@Y+H$X$1*+ zT7~s#V8|EvgO0l|hW>id;}-!{}0?%$W&+qYAU|G4cd-7Z*}pb-zV&Ck_vgc@D+(=Qq`@ zJa5pg1oN^NJ@fP8PzK*Zo@od&%oOQYZ$|oCgp&6uzmQZ;>S3!Zr>nIzv#(mmjO{0I zXn{SG;XGKDuq>WL4Z*FIz18qh{tBC_FLu|215sqj9IwN+cRlu>J^W|Sb{H=$ zHeGtlEsQj-3Tah_-h0z&o^=>e9mmC)a>@}*1P*HOY~l{x%2dD>dH*gW$KsNWFPJ2| z&$h_k&fd-z(y7ZmuRk_C2dp6oF32;kMl5Lr!N6@1Ff5HOR8I_gdc=BGfPv^uQ~T0} zRFO;-sV`n@nuk%vqtKjgz3?Fjh2%Fa_ z+DN?Bh!RHps5oBr$%gX}fSafZ1}uh;@KWHJ13#$9Os!v1qE_eLbe-_&Gtf|PMurOx zcSF;LbQ{sE0h)6=s2<$HZ~9x?!JLam6t#T6`gntH*rfC7$e9$$bjrL}OgE z1oRRnDD+rDAcxQe^U=vOLdi&sg<*fqL|oB{zrG07dL@i5ex+fJdE#SgVR%=kr*u)h z%81}SV*RqOC}!gcxZ{dS0FEJAo_QkkIY68+YZo%|2IaXjpfR2-cpcil$ucPmx!kks z#w!dgZ(oWUa4Qphod~AF8z{FuXf`pkTy~=zg}t$_to?9kxQ0!))x3UJ!Kfd1x)S~r zUYlXN(6p8>_kO>8_PdR@Ewb$={?A^aSkJI&YSXUTGAcJro*RP}>5&4O^*w|oN?EKSQh+*XQ%v8W>nv~bJ(nx7ZRNqk_7|X4BAJw$M z8NtTrEJaQ4kU$%q8$3_qI8BSMkM{*@zEm<+?9(V2N`c}m`nv?#oqQy28_@zi1Eoko zkQ>n~2_pMIH6T^eOTm+pRkh~hrRcw9)r{!(YOUF=UkT2|kY9pDfE2~hIg7$%eEj(F zV?|}eGtv+AAb~n~2Upq;4!etpW z++YB@3>P0PNECtAW@RTO7nqax>u4w1?FT!jHdf0s)i-CSM}D*6XpM~S;#|5le!)Vv z!W{#?`Dz;&pu`#CR~DnY7yv`2hUvw)Oj}0cLgN}zM$b-4hWFd@?qDjki0228S4$jm zQOvAE%_`|71SD>#g>l#-N#R|FAS;fpOMXqd`B)?@NZS(}OgSAc7usYS<8RkFsA_ng zAdV|Yx@JZzW1dwzOlatu3+1&34A9!L^e=a#kI~1z5A^D#&9UK(%9)DO0IzG{6iF9LR?wP zF)3y(*Al8m7lO)WS!m{5g+YXX?C^{2p`-(v(cjDHLDtjHw-NnIiEPsh$O(x)X5;(X zbYUl!Uqw&+0gFfd&PQ{~?|uvxm(3$qPF`rP;Em~M0daJl72Nb5csm^gI$>x7X$YYb z6{n<%0SxAeH77J!q#It#3XB)xV?-KELj#x3FvEhC>l2YN8%x(me%vp$!EqXVK0N}% zGHeV&ot+JIi^8JK_l(O$WX;pT62&HCQONE5GVXyt{D8?#46c=tlonY$B9sgyQ=`D5 zA;?Pry-ZARqE@ci%PL=It5yGg{Q5W3?stzG)fpPq9tk2nkPWkPgCL_@BoTyK<85)3 z5y{nu&3&7g)q1D&dU+v-tY7BU!pmdNwW9rWsT-NQ)o_!5qQl*wqu%L)uU z;)SkK??1iVs(mJZx9Y&}s*BP8{OI<7;}teTW#mKoGf+vpIHaM|tTPEPyq2td7pxYoW~DQrp~)BPJn>5-=H705h`&txm%SR$NvhP?2Qj zL}E&vNO8fUoJkH>)FMk%e~a(|VdLBg9an)v2XMCbN@#8cmM5TjE(MK8B&G2OGepJ7ferz5gKR!Tv}N2o1buK#6z!W4&wnGm5tnT;78 zbvkEa>=VR-2H3G|HllsSat!UFFgF7TRQLmYSK)CDKCr3KR%ipVRiS`nPBTy))r=B&pUwy*lAN$WI(slg&&ZTLkRRiW*EchL_(8;&6Y= zQyz1TX*#C^G*|t`EO%rL65DK0EB<6)rcfqstYurynFQm+pbqQq>C`3EE~PNT#1zyd z;Sv+zITGL!dO`&TLY9GWj;uA+H7*A|K3G2b;cT6Rd)c&oJ{g4JV4VRxeo>3U-FLFPykxT9v9>ROm*t@@%iaLPTn0IpKk?rig4=pJC0VlB-M$#()hl1JsFZ73PCCZDqw$* zthB%TMX{;Yz(WbY@IRT!WHhN*-8x5k7z_@AWm--k#3}kY*_t6j-47`pq9bBXl?pRc zVHisyCXAKs;JCNxrz&_T#(9C&PYaOXkL zb(aHHal<0?E6K6)Ty0QH$< zpgfJp8DpjHx%obE+v>Gnyx4pG9YpI#HQa!IWNN98t*(oPiXF(Nmca(M5!J|XoBp;46`5&>!C;yY|?d*L;zL$a~u^30Cm2TJl) zTdqe4zOZ27dV=MFhc}O$FitGlF5X7|T}Jw+Pjp4(!wY?d_|ptB67uBO!;|DZZk%-y9vj zK0bTh+lzL->vVp}Ql?6+v)Q;dyEAUtD|0x}p|rvu&mj8>&xS_e8t5x3Stl{VAHOJc z1s>NYWS9I4kl>$m56;NZS(ki&aCUUI)%oq{{HJ%noRi-UPEQYx&yQZ8k$0!$@ZIsN zqw}M8$7kf-59HwZALQqw<5ycQzBT)pW3C~P=2AH7U7ASOxKpnBRO>0thJ`tQhvrPq ztW4?IQko?l$eU|#!Ka23{aaH(6Cz6pZI=1+?!NJprOVFF&d&3%zJmYm?CjM4ySuaZ z?B903`ufG*^Jibb*n9SGJG)=+JpcON$j;-y_kLyfk8l6MFxWLoxYrmb5CzqK8}_>WNM;xEm)woj}LT<5YjW1o9T zK9)~dn&^LU^M5OCETI28d(WTO>Ho85&mQUjrzlTI$%R(P&=YcY^6Ec&mU-wMfsfXJ zGr3O=P7V%#0yYfz*{Y?j`33Wcg`C;p(e~jh2<~cgUZM-PVNjnuA^#g9W?C)RuV(di zo;>M<|8{f^4R#+%)UB1K{qIgNwf%b;A}7-Ms?=cq{@>mE`sf#{p$JS{{Ix^ z&p)@Hc4m-(HU!}5_FsQ>I;IwK$;bPkq&LDLh|XgJZaiZ9q3)B3+BN_Cf4iHT zr1#ff*y%t2>|uioTQHyGAb3lbyn>+cXPD@cXTzgmCj8H@(E-4@+2;OY(`U$_Iv%}7 z@z3$xSbF3Go9vT6|0Iwu?XNIEBc|#V(s1R$&n|{R7|GF$O2WNGWUqAos)l;(K@;Sf z{80$a>QBw*EKA(aSIn%6eX`qe4|>z{IHEGS(L9<8?sgur1@nnJl-f_Dso~6OcFvg` z`y@DPG?#N`?*lr#INyX1@3QxlBVA^e3OFWG-tPE40X<%9Cw0EzH*fNqmSmFNDe zDG(aIoO-AF_0;IYDhwrXRZP9O`cYf0$q`bxjB(Vvq^^O2ejhgLRK_d3#R)I#x@3dc zHCI_-RhY|pj_hA#bCX{BD*_PGHnT3CN$Xqi<)4HH|wwZoLw}}qv~e#AMKOB zL8tt3m})LK-|Oeh7a!PUi+q6=Df=%Q7DK&9U*G`;hn@Yd@i>}lIM`Y6@l7aR_{Qcm z?@}3I^SXm??&B)@;vp=fX?n4K%g7vy_;+w3?k;!U1E+j@2VsS8e((oj$;1D5IS7}R zReH^&>hm3U--O$@HIceGiE1EFE1q(#iBqk~J0?tRPqfdr9%dX|Yq22ao>%vHL9{r6 zZte8>G{Qf&bPKY7AWPkz=pUR*izNTgt>>Lp7U=(Dnp1EuzR3;10{#Ej&v(A6>;J!c zzWb>E{}ko3>Hop{oysB8VOkm&q+=FosH{hG47gNXe#7yBXF!LkxS(leFV`Q2b%Za@ zd8-$^2IM37x8v=i!?2$}mtRM&7T3VcnnN05Te+PUS{)SNk)G#L(z#lB|U@r`Y(C$wM*Dsbr-4_uXi3x3|-fkbL$!h&FwNz%r07 zHtFn(yH!|cy*sVcToo&u+QtsN9ZJ=FKdr%fabv8e?XluUHn~_!fZW?Nz;S1^eQnO8pa%a|i&va{^LZ9*bHm@>xZ$Q# z=T;7@N7O#`5r6vq!6A{%nlHE9Rq}$yOVFE}vqjiop+`dU|H2qQ%Ro5q^V@^+@bpz% z0Araff#1!S9?f~V?3aRW#+SE;+QG@unN2s+4m+B2_d$CIcWwhh8?0ja^*8g|+k^2Z zv$RE~X-ZB+LJkToR7_JQTcSG3=@N|lvt{A5y6DZy2@C$TQ&`!a^K7_%xd*Q-0C(zx zS+Fkr`1YpbS0*8_>S~j5`8XNNOooE^Xl4{AWf~8gwwS9jUTaGYiOM7Y$s+8(UX8Ax zjtP3b{2&h!>vVV+#!*~hq_nZ3c7y-!e-=Ia_o#CKAmToXfNN zK>qs6=S=gdD4o;#2ZkQL?p>adzdwtf*;E4xgS~elybFX%H;6!{hc0IPj5<%Bo(qij zjOp60FqttUG1W8`!!H#X^Nf#+tZ6o$K5ePL!lOn7eXZ=j0GaNe8<+2Qdf)zY^XdEl z_t!t4{`J=vU9wSuhg=ga&M*c|a4Wy?p!uBle~a#aD@m$XyZ~O{|M~3M-qii?^B0f) z|DU9M^7~)nPuhU0CQM^mR(^frjv-gEf2IyMZe@UXUTa^`L+pk#^ zRI~Q=LIbBZUz7{Ze(S#$l5DU)maHU#;Y>j{5He=nO$_&RYqc6N+f=&DR(2|bO+Q>S z;O(d66qhT{f{oK+&GIZ*^|b{I3#2sRt55YyBF0=#{oEa>A5D{8(v?Mapng$Iy1(s% zUZq}Z_!umkNU=9OD?Yx0q$y9F{H_dCD~`U}_4-&Hv77$)EV5|+Zf?oO8tVeL^$$8u?-f6_Ze;=g2i*8g zeE7A)r(sEj~CCr`s$JY`xNC<@_*3QxlKODT|;=&uG9i^mxgs9z|OCRH!>UU0_5c z)BcZRaE9$dMl|>YU+wMebW4TDR1bIWs)xHv^l;braH=bo1h{O5ep;q`5)R9oeK*X= zGz6PfHq9)lQx`qPX9I$;-U{7Z>aCF;v4VGxxGy8}hGYEV;FNQ5W3|;T7*Pbmnd8xl zOOI=tZc2}&xN<59)2=&7d&IMPL3<|@QsOx#X(p?=2-5OstV0DJME?>z16QYM6hc1I zFm2P{dlv50W^g@Hzu362;R*c(^wDCv|B`h7GjmyD&&cc3(|4zs3jp?GF2yBJ7>0z7 z3kfKc`?sEuuS{bQb!RvF(rWZ7KX(5!BlU7Dv`yE^E?GwCKr=}s3qwQ# zaL^fJiA$u%Chpe{Hms$O5$pm|lQ^Y(tZt>-KNJ$<`3I>NZQ=)vlxg~<$;|9@m?k{k z9*L`-7Ta2i;v!{gB!tEkad4=+ymU{>!S`_a^{z%dWxX=_48HGD0j)5^ztujnE<2Ui zrWfQj1S4L+4u0-q^1+q`{2xU8zGMFH#ft|1Z|5=o(WjzpEWF_W7foe z*cAx>DSZ&(pQl0u8X(92=n=!WNf*@<-MT{vFc`DW=4yMMYjcTW3pXq58BmCj(LWsL$w}Fhv9`osB&s{e6h+Jtiko!iRsgWd^zKb zCON+Tw`s2IX~Kt0>99M&fM1gC|E`@i;h0}GRMar)TI1D`OdmyVBVoB@G1V;Dhxpyqb&h(@ zlOj#=E1ywebIxYW%hzjL(zE?DcjfNZ>}PwMv-WJ8EnDeTz(NPVn=J(0Gnm$QMz~IF z*sX8_gcwa}@;>k^IZ~uMrg`ZL8fekNKD;C}&&|wMevJ$PtgFq!#kEXb4eGar+P8{R zxy7`yvh9<6y>VEqhAbDxM(CLyRJ;kxvA!zyYe-ePa!rbi|KYsyF?2HoZ>x2u(wp0( z0_>8e1t!LJIl`2WRITQ0^(p#r>l?l%_%GPx4gFcX)$P+bxU6x1SLXYR#_Nq~y3qXI zCm|D4wZ|9P^Tw**M2=YJ3fTrmMjXx-xElxtFCTk%ukAJ#!rbf$&Gnh(-M3^?A z4YjX!{+W_;c zf8X5r&&dAuzrsIz@cf|ne`)W({EnO=2zqPHh#{S3fkHk(n=4I)22hwCI8DO{I(M`|Ly*GV7k%f7S1&PJ-L0piX}g6#?fBc?mP(l6Si| z-PpsgO*{VU?W0Ub1bzZmVM9og72^vgH=U`9{S49pLI&0gh6!|2$ats)WX@%97c-&b zG_W7@*Ii}GM2R_2H5;<2tFBiDxbU9qe8l`?XY%VanL1Hpo>BbN?;t=T@a*g!Dz$d80ep?q#8vho~ewoc*gGYC39t3pC@Q&*_i zGh5B@uQ-+6sW!~3Q}u-p{|v1A4OFf|p6P0Bo8_}XDyV+`RsFeZ4wmY4gV$z>HLz^+ z^z0B+h}n8{AOwA zh$-&~$k#aE?E%4A&8NW*?J%WE0kydio=3IXmc(`X^SS;oE5Li_=z4HtCUtL2GdhGD zX=~8%|Fie?&u!bvqQC2}z@y!3E61WFf2G&wx_8nfJ;$%j$Bo@&&NOdYh=e566u|=}(_;TxINv|eKMTx2@7C({&azJ}!Lg-rL>L{E`F~8DhEzR8|lxZY-IG7Qy zYm2uBef0x=E&9LC1A3?DzlX!aa{b4r`^S&xzxVRHfBoN5=0%F*5c5zMSgsR5ngLDA zBHeXua%=TbH5cvj$p1)YY~xCeQuC%$qZ9mFcPlnU@VM*C1LJ7~rZ)n)q-_ zmc>>Ld{988`WZ|S7CM8Q-&jXqssdsjxDA#_6F05(GJ#@c%P&il&If`*h!VZRaVqWY zh_M8*IpIhFGj-#YgIE`K+{ua-dGUB<+jD|1?Xu;2#{L(3 zh@^qv`6s&~bHeu&AUcz3p2ka}bj7JIE?*M5*2QMtQ7u(GC7H=NDV4R?Gt+F@*HL&A z;uOJJM!N|OE^lzU(UP>ji#~ym`(Jn#)yhk?fR_oc`p z7EZmQiB5$L-+~x2`?hTPW==S#A(7INXK{dz58%IacAkyL^cs1+rqXrKIguKrI@1L- zCC=vL4{McWv?x$lyJ_11v{L7ImmpTK_l@w}fvEo*4eS%*B#Cf9koPwadB1u$9~90B zH_oDHf&R#FBc#7G*%nSh5)yR%&e_=91yMlqlqMGKX>q^-bcduR5_yg7a+=GA6-T1s6Z10X71O4% z!Af+uU5MMUjsaeoUoAXyawsV7o}1(9JY5^XG8a0~RKM#m+j5n^l?l};6^VnzSILXb zu2jX1#qcb~G$x4bMju5M?ex7It%1TH@3kdsxToBB<)FGHIN6KE>U7D9S=wp?%O)p} z+tsQ|r&e_?kgt1t>!nl+b<<1gSroA=`&J#WqDbU!p{_YHMV^qTBxbWLN@=fXWSNzS z#wcL35!^mgTHz|2GQg?XneB0_zHEU9Pj0_UA9Z`^I)$OBhoTce* zu<(jC<=14rs1VWbu0@Jl>VGZzf03E_7B)cJ{r}^qmHbbG$NK;G^ZSDJ{|#@!$tV$N zA9iY;)C7&N_6lM&p>qMsUNy{-FUV|M6xt;H%=jU+QT8==e5xA3QyB zz#V*hxH{fqhNEajypKxsJ{mR?Xa&5VmdNn*xOw`j^d=$AW;aA!L)#eU!=r=d8LXfl zVe4F{jm76um9RApzgm}ohpA_}R|d6|k1Ho1{@f6qj!51(`5ceVZ;TwY-w76JGw4I^OiHyV%G zN!AtdV+%LM$ZNDl+ z+2bpmB<7x`buU=ra;kONC0}jlNH<^PZCz{zMEhFH6ZkQXI7vArb0Rql;(;PgaJWDL zxN^gl)?|?`Perxad7dT}_GhBIPuu0!d>?l5)vfC_%1S4Dw@_ zP^bs_6#q*B`_6?{7D^_|o5?p(Bb4?K@h2xz1rvWnIfXqReV|c=$}7kmKYT@zL2B)clkww%v*?OSDH(=x=T*`%CtyVcDCxg@ zHb%HcKFGcRs|>0OzgVUOBiZV1HN%8^!sxK@=o?#R1 zT$07oZc|sE~JN5)>&(*zF*+@1wVQ(pWRpImKlOR1i<@MujXC25Cbc zR}e16A1>8`^vmTTw&km>l}VCLvHn2vR%zTs3&5PHx?%004}A0v^Op1AyjlM$5x=%s#{g}^jz!M&M`YVMV^Q9oOxRC zl^z#z)`}hQJUe_KKRnTZ^%h~Z9}bYOhhLnuC>}7J^`p}N%1TO>(n$CdlKw~*KX5ja zTV}4HNZ#_0U)onK?VxhypwcaTv$SJqA-n$3vRyz2svX19-qrNHz$pepHe&2DOOQ90 zx{)!1Ryfhl;p+MB`k@4AVl?b_)t|4k8R0ZQ!!Fv*lBC+|lTMTVtJkz< zpuGqvt?UF%ACL*Tey8GapGDD6GzIsg_wnNgix!a2^pn)fMat39s=)2MpT1lqE3O1iwxX7$)K82x|QRcZ=ty}W&bOeoMCIjzcBVJ!{35=3Pau_e6D zbGrP)0l4gXJ`}wv)GWwr<>qIRvS3fu$-1HfZr}?2pR+&<5xVJ`1&TuIs7L z{k+$gn6%WKEtJIlO1%p9pcY=sR&=X&Yi2)p9?p#x=UET$R@o|D!n1qZZBVo9 zYRMYqMjflHQTXmOSK+t~9Oh$-Qhyq2H7F9xFTW8}>ZgyyMe?<1SglA5gg)scM9^W? z;8f2msdyf?y3(@$t5e0h&Hr*VsMh}*KH7ix^Sk%`UnO&GxT@CUdHqHn45%#EBrpAQHV%9M{~?o(2t5Y#wra*>qM4_qGr*|%xuxiz4`4^ zQvzlAo#73{#3vDjtwqTHK#6E+{^|i+U%;{Ab1cXYl}}-B?3D`;mHV)b^G0Yz{hp>4 zyGR{y`lwrDAH$!;S!W}ZhEUzK(8S>|Oz-My23u!tX}C!qJ*)m&Yl&3=7v5z#8XbIN z&7kkO=MeQrEU>GkZ$`h%7u%(9gp!(3^i&Dqw&3(?0!0#)g@cnhB`Pt86rEh3X|b`o zAgKK2h{njMT-yy1Ax?01et@!Tj$#UATn@2boGv>W$xl$`a{h{lsco(9_db6{OU_W# zv6`Ww6|3ePtRcpAOMP+yO9&#sBNVf^r(3VWa?PPtrBLomvKYF22m##da-!2;`2v6* zK7LzgF36Aa;mIk|f9h5)m)T9rlYFxP{$$9 zrzE%(*$iRJ64KM-87m3WHJ~Rd0Om2bGRqX;y@7aH4~t_owX9{^4>oWPm$xBg}Gdzw=)4-?7!oqrN%3pqX_U`rB>wlb}cg5&YNKz7{Bt*N?2D3`VfBt+TpU5_! zKX><#)-hw1t6OCXhd2+;;z_hXoXBC4Sl{x%b4J4|wL$&FR!Ssk#!=2$F!YfN4y+f$&+)>w+8(HNzwXm(e%yT81~vm}CEszJ2tbQ1{rR)r*Ua~Xi^@^kQ? zIg7Fxc}w_=3ZcIAY82OtT#b(PO<=$eVIi`aaYn;65*9`5$}0X0wucHes>7uiLtdb+ z^00zq>3#86`vNQ|=9+K9F4bJN(zZcIdPRhsdC0A@vG21O?vY8y$ z$|HYBHQKWj7;yPVQJhQRv&&+FW|+nntD3oj9k;9Gq7@kc!bID3 zngm`!9`|Z+b%=#8q4LTr5xEEywyAhp1XDF!4R))MR*8Z+QTZh$BUUIR(C&;uOB|y$ zsk42lttqg}mBsI(@_`@-$p(?gSvFDkn2Iup5t&21i8!SZ!kp7Nh#2Nd8I7xt%z8)I zANfJSrX{@0?4O+GbJnYt49^WjJHkVMX-8GU1HWMkYWc)`RLdxuG-sw}q-r|F0_n>a zr)t^2-RvZy0e;T1I6WQgp^!m0M$pl;7aJV*1d1sJEDlHlhH^ZnlZ*q#y0}s=CxoT;_8)ps@^bs% zx~bOxHNZmPIOKR;XF%Ke?{L4I|9Agj@Ob`vFTZ=Y|MgWtA{9r)PDN$Z{N^EuA}K`R z2IA5iRyCI?3r5WWHXRO|!r%FAPm{>SwX{~do8e{{GD#^wv=)Q4(m<+98Rq6vj&*+C zDS8@P)vvjRT9>-Q&y$lBJ(;v=SQOtfF3IYf-S0$=x6wpfmAKZSQz`vX%@IZcL4&L$ z)^gzKJHjggGODBlAsqcI2!G4Tm|m|0=8`OY^-&3uxl(LjHnz6*;+)#;)4lf!x*_Ar zakO2|{R9=r%UQ%A#=2`>YJ#Yu#3hrkeDzn9Ppj9tHeDHRp5bs;XdrMtO|tmby$>CvP8cQ3y$ zXaBvD?DXRMjaz?)O&^TyX9n76`9aQ#+gW|4kGrw?YDl`4#b>_P;Bb~sZ*26HVAY#^ zmAQkG7eO~(=(xzWlG8qCIbjHL~xBe!08POmj^sCT%M&yv)F0aY#M7G>B%YHuyyQU;Eo26v>D|l z`2s5pS7#yD987}TF5ysf*0E_PgrRhm$QyM-ln-9);R8E5(#NybLd}6_F_-m=s+s$0 z+ONQb#aDMgraGLKQ|#Zxk+aGUxG`VO29<$T@VieNVE#xa9fW8QD* z!fAqXO9zet`@r2dwm4m9jX6r)9XfCBaJe}{xXn7VccgU|R>OB2cFf83yT;5GYW7z- zs|ETkGipo8x*4;d-_#Aaj{k?n6SjH(&v0;bSkC`4e0qHJ=>NHo-xu`%NOpSh{eN&W z`TORbK$=${juXfpZL=pxp?ynVklocCIfSa@UC$w;zu$;M$c5G55L)iWaREmXxa~nS zxN%yYN~J+HJCz=qL+Axg@d%6Z_o04#Uk;^nVlH-9T}kR)E2L*xI(=F2iUlX{+X;JrY;1*LK*o*8oz26OV{^VYP`*YRF&kIF z`zAgC4fgByz3D-$G4D6^y){6&t?x~Pec-;gl|3{~8uVRyXf|x;G>)OoO=B&n8@Ori z)FryiB>8%$=JtM3Gi7)07yZ%>o|fM_{y!+Ka)U%5ZSh~nhn4uRqv7$R|L;D2U(o-z zIRBrYHt+az>i>Lsw;v4hP9wjdhdXlfm5I5YlMlY$h>OpLbi;@*2bmkkd)ZJPqrD!Z zy{Z;ItmuCOr`DH_^>QTVgYqhU;YcsVfRAxrU&hzey60`q)3m9Lza8hBSXcn`>-I6h z5LTY;oBEe(Vcgcc1VA6SZ)s&GMziX6moALESzfpHT&Q*RmB(|{tI=QGY_9Gi%%$O7 z#c|#Aw}$^GVIrMyvc40jCH`ypwBrBSKYG0XcR#-`=l^-D*y%2uKo+My7*9}vQ?om$ zXmdNCkk!}SIELy8yq0&!0&&YUFg1{BT|~KcnYXviuXd4^y(!gZwUli(^Q+C9?b@2Pv7r<*RV-Zhil&e?k!x|GJ&Rr`v#0SR)O1mv zk0r!j*P*bKIaiv2yLB%tAK624G~7&m4oN^mk{6#R5;3_W>SDnf&t-dFmDw^ud{g1r zDZ+C}7CkoZnRmbjeNkU!dvK5&1 zi9a{RF`*(vvkyC!8YW-Z^*o>kYH1u^T@XdL^{eU!@2R2 z0;>iIhE&+j$F)e6yAa!nWH()zJPwvW3-T;Wr|ZvWUeu_qc_zr)wa|PJ-u$9nR#50R zk%SJln?0!c*d_j^YpwUWE;}p!ti)|gV(09Z&Ax3{1UGiPcAT8u``7QdZgWE^0lRFk zuHO{ab3WC>xTR&LfIdiLPg;6B{CH??$M;M9arbMYC zx>lya+LUKB#Uk|x7I+ppw;c;(ufd;Es1=R=l=^-JnvfXE|Azh5uUWzrwfC=@>9J{K zLY(6IE}G*WR?zE<*{`jqEEN$#R4VSRiA6 zoki)cowCpJhU%@g;+ig3DwWy{%<@rro>NBHD~zA6xrAvwsh)PvE9czXbDZB8mCHeI zWloa`NuOt&lelmydS2OagX1ul99m+O{3oWTCaS?l7wly`e~-BxB-wyQVZ)${!dwni z6UEvmALEa)iAMZJe2i->9o4zQT+W=PfZuHh^m07_3jEGDVy7zB@*>7K4AD-nC)FZp zZk=w+=2{9!EN~q9(h4hBqnLf`nIT|>61TN$>hynqN}?H^#Eg@T=$I|~|KR9o;OhUw zqy49k`v1NBw$NLgN*N=d);vtQ+%RG^%4h^dwi6s&;t3IcXA50SsX!u262?=3#FRu4 zM5~|~PJ=0pCwqvK2&Z(eH@xCGj>FCtipfO9H}5K^f{r6a|EKGtH*vH;EQT*7N+=;5 zMKmUU$A5AD>$%*XI$P*Da4CBK{2Yapi;h2`X&?So=y&|lpS%zMng`QKU;bnM6!W-W z02<-oGE2Z}5uGQ#xJo)t{1LwFJn_?6(s}a#J6q^I=9FauoxOM|I)1|0?<7b&J`D-( zt9G3I-tp%mU?GvP>1=|=ED8zl(AlK_U_A_V>;K)$XD?p8^k?CQ`e?cTbv)RAT3Y{y zhX=#Q^?x6~E%ZH2V?3i~HT*93lpwq% zoOEU+#Zvv1m@B+%Y##YS_~R^!R4^kPNt>@`3#Hp2@1G=!82Z3?6nYAYpBTH8v`rbx z1VJq02q$@wZz(;T5r2euZa_#~XoX-EXZhEt475=}{z5H8Tw zlm=52;8?0$aNGbFE9N9(35v;;<|33yMPBi6g=@QJYeO>};|bw@r{glA(*f*Lzx1FK)cjW{E*E1iqS)oL~JpW%BWP16?piQjS;M;=L$ZJQ4nP!C0sN7TTZ~-7c?cJ z)9E4EACE~&1RDDL2Y>7I6o3Q&TYvC3^^LSciG%@vo+l)rI6^c|vJ{D6N@f_|Y@xFm zo{)Epr3KHLr6AcWT_ofL2{5}l2!%uhoPzCjf?mKnL8iH7R8k34+F20`r0vx~h+N}Y z5|I;>udYtVT7@lLy~|qy^%Q)$8f?g#A^Y)!#@C%riYIMg6`T}LY%oKAc1Qs_lp7{%sF0k5TYqjUWnzG8Y4PdAYqV>6%fglIr_RE zv|HTt!6ILPqU(1$AsJ;Bxku_qutFL&7C3&PW3*ryx?))rqKIA+`9=_e$aTswqVbsV znGz9dUl8^rsl!6a+s|S|W0B%GAR5?+oOHHwwzN5J%Mtu8zidhRH_ZMPvZf!|Gtt>X zYUmmxkh^kSy_Hg5q$CbVv9k0*a4Vh6>8WZjm0knOw}~L>*y)IraXcZK4Z~kM- zgmSnP2&U*gi?SI(=L!%4NPFmPYd3(t2<~3|po~LDQNQJ~xpl|`(rIOZi)1&dk z!Px70oemBHBE%~ek~VaoK}$4aVG(fLh_}KQTNjvzQ!Oj*M`XyPs!im)VPRg+K^IQ#7FcT6;$#Y2-Ao*b2((JejNJy9vbVMVXE>yRm z`%MYuqD&2`TA110DkOPd(+ZyVdcAt}Ra52Wx+_8v9+61zP&i3e|5Q!kn*{%nkz5BZ zX5+8FP=2w(xO+dL(eWXPgZ#?o=g%Ey&|wl4Bldk?yBSG5KglmI8?)l=v-9&0Z{EGg zUm)dM{DHICiS-a^LPNmW*nP8vTxNFEh_$1*k_DmuhGF?^!tL^sESC0oX?dns?NHyh zDgqlne+K$>w(MgIAZuwwxr@JTHKF(R0j6GHuisp}JonS<)UnBKaX>s6-0u>`nVY8=4>n_(wOl;?wm~INBH_;IGjO9nMI=rbUP4bI3m)Vu9(swilYcg z?HA1!`pgF$<)w)_^}M|jEl)iUIDV(pbVW~mxr1%9(0`R6O5XYA6NXN{>d1A%c#14A zr`x;IE)3ELJ~k(IJ&>`C#41vx%My|CM< zBf;=SBpBKx7;c6H!#g0suuK9iG|NI3Y;=VM0uw}Ig#Uqxfb!Fl8n>sl6J3aKXq=Hg|v|IJJ3QqF_-$84P z|LAY&);&P3r`fy(8`k-EbIgte##VEi|C@nU7Xll_&w8b*K zU1I3Hgp)DhMo|62#$yUjM`#wZAd{92%-QnAGvEJpe(>vzPSgbVxAOn?GC_I)7DROQ zGSRQ%iJCk!hLoZCf##N<1sP`%ifDWZ-qZgweak+5+lifKDdQAJqR)w7(VX-odi(J2 zK`&tn~neKmQLtJGvwI7TkHtDN2q$fuLefH-2y_D(y9`wJ1ZA}MXcB}}^{|H#J z_`BT34$=3V;V@t`6=0MSK0}^)=BXKii8aq4tW(Z;pK&SVk;q1Zh7@xO35RrdAPf$Ub#>)qK+$J>(vMUV(f{e}`ODYm5Tg2B zr}J|h5h050cC;`@BU0u^_zFo;G~q;T5i~|uoThMUE!a4{!klzMDpF2IS?Z9-;6w$o z8%W6>Bk$QcIy?8!_s`DH&i6VW&MyA>=I0Ca;n}-)&t6}gy*x*6-l6AjUcWfIID7N@ z9KHDgJ$wBx^yAs<7kh|M-redis", not "-redis-master". -*/}} -{{- define "litellm.redis.serviceName" -}} -{{- if .Values.redis.sentinel.enabled -}} -{{- printf "%s-%s" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}} -{{- else -}} -{{- printf "%s-%s-master" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}} -{{- end -}} -{{- end -}} - -{{/* -Get redis service port -*/}} -{{- define "litellm.redis.port" -}} -{{- if .Values.redis.sentinel.enabled -}} -{{ .Values.redis.sentinel.service.ports.sentinel }} -{{- else -}} -{{ .Values.redis.master.service.ports.redis }} -{{- end -}} -{{- end -}} - -{{/* -Reject an unpinned image tag for the bundled PostgreSQL. -A floating tag lets a chart upgrade start a newer PostgreSQL major against the -existing PersistentVolumeClaim. The server then refuses to start on a data -directory written by another major version, and the only way back is a dump -taken before the change, which by that point no longer exists. -*/}} -{{- define "litellm.validateBundledPostgresImageTag" -}} -{{- $tag := .Values.postgresql.image.tag | default "" | toString -}} -{{- $digest := .Values.postgresql.image.digest | default "" | toString -}} -{{- if and (eq $digest "") (or (eq $tag "") (eq $tag "latest")) -}} -{{- fail (printf "postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got %q). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore." $tag) -}} -{{- end -}} -{{- end -}} - -{{/* -Whether the existing Postgres is reached with a short-lived token (AWS RDS IAM -or Microsoft Entra ID) instead of a password. -*/}} -{{- define "litellm.db.tokenAuth" -}} -{{- if or .Values.db.useIAMAuth .Values.db.useAzureEntraAuth -}}true{{- end -}} -{{- end -}} - -{{- define "litellm.validateDbTokenAuth" -}} -{{- if and .Values.db.useIAMAuth .Values.db.useAzureEntraAuth -}} -{{- fail "db.useIAMAuth and db.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source" -}} -{{- end -}} -{{- if and (include "litellm.db.tokenAuth" .) (or (not .Values.db.useExisting) .Values.db.deployStandalone) -}} -{{- fail "db.useIAMAuth and db.useAzureEntraAuth need db.useExisting: true and db.deployStandalone: false (token auth targets an existing RDS or Azure Database for PostgreSQL; the bundled postgresql subchart only supports password auth)" -}} -{{- end -}} -{{- end -}} - -{{/* -Database env for an existing Postgres, shared by the proxy container, the -collector sidecar and the migrations Job. Under token auth every entrypoint -mints its own token and assembles DATABASE_URL itself, so db.url is left out. -*/}} -{{- define "litellm.existingDbEnv" -}} -{{- $tokenAuth := include "litellm.db.tokenAuth" . -}} -- name: DATABASE_USERNAME - {{- if .Values.db.username }} - value: {{ .Values.db.username | quote }} - {{- else }} - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.usernameKey }} - {{- end }} -{{- if .Values.db.useIAMAuth }} -- name: IAM_TOKEN_DB_AUTH - value: "true" -{{- else if .Values.db.useAzureEntraAuth }} -- name: AZURE_POSTGRESQL_AUTH - value: "true" -{{- else }} -- name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.passwordKey }} -{{- end }} -- name: DATABASE_HOST - {{- if .Values.db.secret.endpointKey }} - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.endpointKey }} - {{- else }} - value: {{ .Values.db.endpoint }} - {{- end }} -{{- if $tokenAuth }} -- name: DATABASE_PORT - value: {{ .Values.db.port | quote }} -{{- end }} -- name: DATABASE_NAME - value: {{ .Values.db.database }} -{{- if and $tokenAuth .Values.db.schema }} -- name: DATABASE_SCHEMA - value: {{ .Values.db.schema | quote }} -{{- end }} -{{- if not $tokenAuth }} -- name: DATABASE_URL - value: {{ .Values.db.url | quote }} -{{- end }} -{{- end -}} - -{{/* -Environment shared by the proxy container and the opt-in collector sidecar: -database, pgbouncer, master key, redis, user envVars. Both containers must see -the same DATABASE_URL and REDIS_* so the sidecar reaches the pod's pgbouncer -and the same spend transaction buffer. -*/}} -{{- define "litellm.proxyEnv" -}} -{{- include "litellm.validateDbTokenAuth" . -}} -- name: HOST - value: "{{ .Values.listen | default "0.0.0.0" }}" -- name: PORT - value: {{ .Values.service.port | quote}} -{{- if .Values.db.deployStandalone }} -- name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: {{ include "litellm.fullname" . }}-dbcredentials - key: username -- name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "litellm.fullname" . }}-dbcredentials - key: password -- name: DATABASE_HOST - value: {{ .Release.Name }}-postgresql -- name: DATABASE_NAME - value: litellm -{{- else if .Values.db.useExisting }} -{{ include "litellm.existingDbEnv" . }} -{{- end }} -{{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }} -- name: DATABASE_READER_HOST - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.readReplicaEndpointKey }} -{{- end }} -{{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }} -- name: DATABASE_URL_READ_REPLICA - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.readReplicaUrlKey }} -{{- else if .Values.db.readReplicaUrl }} -- name: DATABASE_URL_READ_REPLICA - value: {{ .Values.db.readReplicaUrl | quote }} -{{- end }} -{{- if .Values.db.connectionPool.enabled }} -- name: LITELLM_PGBOUNCER_ENABLED - value: "true" -- name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: {{ .Values.db.connectionPool.maxDbConnections | quote }} -- name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - value: {{ .Values.db.connectionPool.maxClientConn | quote }} -{{- end }} -- name: PROXY_MASTER_KEY - valueFrom: - secretKeyRef: - name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }} - key: {{ .Values.masterkeySecretKey | default "masterkey" }} -{{- if .Values.redis.enabled }} -- name: REDIS_HOST - value: {{ include "litellm.redis.serviceName" . }} -- name: REDIS_PORT - value: {{ include "litellm.redis.port" . | quote }} -- name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "redis.secretName" .Subcharts.redis }} - key: {{include "redis.secretPasswordKey" .Subcharts.redis }} -{{- end }} -{{- /* - Inject LITELLM_LOG only when envVars does not already define it. -*/}} -{{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }} -- name: LITELLM_LOG - value: {{ .Values.logLevel | quote }} -{{- end }} -{{- if .Values.envVars }} -{{- range $key, $val := .Values.envVars }} -- name: {{ $key }} - value: {{ $val | quote }} -{{- end }} -{{- end }} -{{- with .Values.extraEnvVars }} -{{ toYaml . }} -{{- end }} -{{- if .Values.migrationJob.enabled }} -# Schema updates are owned by the dedicated migrations Job; skip -# the proxy's startup `prisma db push` so N replicas don't race -# one DB on every rollout. Placed last (after envVars and -# extraEnvVars) so this override can't be silently shadowed by a -# user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env -# semantics — same pattern the migrations Job uses. -- name: DISABLE_SCHEMA_UPDATE - value: "true" -{{- end }} -{{- end -}} - -{{/* -Proxy-only metering and metrics env. The collector sidecar serves no HTTP -traffic, so it gets neither. -*/}} -{{- define "litellm.proxyMetricsEnv" -}} -{{- if .Values.billingMetrics.enabled }} -{{ include "litellm.billingMetricsEnv" . }} -{{- end }} -{{- if .Values.metricsServer.enabled }} -{{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }} -{{- fail "metricsServer.port must differ from service.port" }} -{{- end }} -- name: PROMETHEUS_METRICS_PORT - value: {{ .Values.metricsServer.port | quote }} -{{- end }} -{{- end -}} - -{{/* -Directory of the collector's unix socket, shared between the two containers -through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP. -*/}} -{{- define "litellm.collector.socketDir" -}} -{{- if and .Values.collector.enabled (hasPrefix "unix://" .Values.collector.address) -}} -{{- dir (trimPrefix "unix://" .Values.collector.address) -}} -{{- end -}} -{{- end -}} - -{{- define "litellm.collectorEnv" -}} -- name: LITELLM_COLLECTOR_ENABLED - value: "true" -- name: LITELLM_COLLECTOR_ADDRESS - value: {{ .Values.collector.address | quote }} -- name: LITELLM_COLLECTOR_BUFFER_SIZE - value: {{ .Values.collector.bufferSize | quote }} -- name: LITELLM_COLLECTOR_ON_UNAVAILABLE - value: {{ .Values.collector.onUnavailable | quote }} -- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS - value: {{ .Values.collector.drainTimeoutSeconds | quote }} -{{- end -}} - -{{- define "litellm.gateway.collectorSocketDir" -}} -{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}} -{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}} -{{- end -}} -{{- end -}} - -{{/* -LITELLM_COLLECTOR_* env shared by the producer (gateway container) and the -consumer (collector container), so both agree on the transport and the -shutdown drain window. -*/}} -{{- define "litellm.gateway.collectorEnv" -}} -{{- with .Values.gateway.collector }} -- name: LITELLM_COLLECTOR_ENABLED - value: "true" -- name: LITELLM_COLLECTOR_ADDRESS - value: {{ .address | quote }} -- name: LITELLM_COLLECTOR_BUFFER_SIZE - value: {{ .bufferSize | quote }} -- name: LITELLM_COLLECTOR_ON_UNAVAILABLE - value: {{ .onUnavailable | quote }} -- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS - value: {{ .drainTimeoutSeconds | quote }} -{{- end }} -{{- end -}} - -{{- define "litellm.lensConnectionEnv" -}} -{{- with .Values.lensWorker }} -{{- if or .enabled (and .mode (ne .mode "disabled")) }} -{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }} -{{- end }} -{{- end }} -{{- with .Values.lens }} -{{- if .enabled }} -- name: LITELLM_LENS_URL - value: {{ required "lens.url is required when lens.enabled is true" .url | quote }} -- name: LITELLM_LENS_PUBLIC_URL - value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }} -- name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }} - key: {{ .gatewaySecret.key | quote }} -- name: LITELLM_LENS_SERVICE_TOKEN - valueFrom: - secretKeyRef: - name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }} - key: {{ .serviceTokenSecret.key | quote }} -{{- end }} -{{- end }} -{{- end -}} diff --git a/helm/litellm-helm/templates/configmap-litellm.yaml b/helm/litellm-helm/templates/configmap-litellm.yaml deleted file mode 100644 index 03e4f620206..00000000000 --- a/helm/litellm-helm/templates/configmap-litellm.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if .Values.proxyConfigMap.create }} -{{- $config := deepCopy .Values.proxy_config }} -{{- if and .Values.redis.enabled (dig "coordination" "enabled" true .Values.redis) }} -{{- $generalSettings := (get $config "general_settings") | default dict }} -{{- if not (hasKey $generalSettings "coordination_redis") }} -{{- $coordinationRedis := dict "host" "os.environ/REDIS_HOST" "port" "os.environ/REDIS_PORT" "password" "os.environ/REDIS_PASSWORD" }} -{{- if .Values.redis.sentinel.enabled }} -{{- $sentinelNode := list (include "litellm.redis.serviceName" .) (include "litellm.redis.port" . | int) }} -{{- $coordinationRedis = dict "sentinel_nodes" (list $sentinelNode) "service_name" (default "mymaster" .Values.redis.sentinel.masterSet) "password" "os.environ/REDIS_PASSWORD" }} -{{- end }} -{{- $_ := set $generalSettings "coordination_redis" $coordinationRedis }} -{{- $_ := set $config "general_settings" $generalSettings }} -{{- end }} -{{- end }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "litellm.fullname" . }}-config -data: - config.yaml: | -{{ $config | toYaml | indent 6 }} -{{- end }} diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml deleted file mode 100644 index a37193f0efa..00000000000 --- a/helm/litellm-helm/templates/deployment.yaml +++ /dev/null @@ -1,258 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - {{- toYaml .Values.deploymentAnnotations | nindent 4 }} - name: {{ include "litellm.fullname" . }} - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- if .Values.deploymentLabels }} - {{- toYaml .Values.deploymentLabels | nindent 4 }} - {{- end }} -spec: - {{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }} - replicas: {{ .Values.replicaCount }} - {{- end }} - {{- with .Values.strategy }} - strategy: - {{- toYaml . | nindent 4 }} - {{- end }} - selector: - matchLabels: - {{- include "litellm.selectorLabels" . | nindent 6 }} - {{- if .Values.deploymentMinReadySeconds }} - minReadySeconds: {{ .Values.deploymentMinReadySeconds }} - {{- end }} - template: - metadata: - annotations: - {{- if .Values.proxyConfigMap.create }} - checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }} - {{- end }} - {{- with .Values.podAnnotations }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - labels: - {{- include "litellm.labels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "litellm.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - {{- with .Values.extraInitContainers }} - initContainers: - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - containers: - - name: {{ include "litellm.name" . }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - {{- include "litellm.lensConnectionEnv" . | nindent 12 }} - {{- include "litellm.proxyEnv" . | nindent 12 }} - {{- if .Values.liteadmin.enabled }} - - name: LITELLM_ADMIN_AGENT_URL - value: {{ printf "http://%s-liteadmin:10000" (include "litellm.fullname" . | trunc 53 | trimSuffix "-") | quote }} - - name: ADMIN_AGENT_SERVICE_TOKEN - valueFrom: - secretKeyRef: - name: {{ required "liteadmin.existingSecret is required" .Values.liteadmin.existingSecret }} - key: ADMIN_AGENT_SERVICE_TOKEN - {{- if not (hasKey (default dict .Values.envVars) "PROXY_BASE_URL") }} - - name: PROXY_BASE_URL - value: {{ required "liteadmin.gatewayUrl is required" .Values.liteadmin.gatewayUrl | quote }} - {{- end }} - {{- end }} - {{- include "litellm.proxyMetricsEnv" . | nindent 12 }} - {{- if .Values.collector.enabled }} - {{- include "litellm.collectorEnv" . | nindent 12 }} - {{- end }} - envFrom: - {{- range .Values.environmentSecrets }} - - secretRef: - name: {{ . }} - {{- end }} - {{- range .Values.environmentConfigMaps }} - - configMapRef: - name: {{ . }} - {{- end }} - {{- if .Values.command }} - command: {{ toYaml .Values.command | nindent 12 }} - {{- end }} - {{- if .Values.args }} - args: {{ toYaml .Values.args | nindent 12 }} - {{- else }} - args: - - --config - - /etc/litellm/config.yaml - {{ if .Values.numWorkers }} - - --num_workers - - {{ .Values.numWorkers | quote }} - {{- end }} - {{- end }} - ports: - - name: http - containerPort: {{ .Values.service.port }} - protocol: TCP - {{- if .Values.metricsServer.enabled }} - - name: metrics - containerPort: {{ .Values.metricsServer.port }} - protocol: TCP - {{- end }} - livenessProbe: - httpGet: - path: {{ .Values.livenessProbe.path | quote }} - port: "http" - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.livenessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} - successThreshold: {{ .Values.livenessProbe.successThreshold }} - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} - readinessProbe: - httpGet: - path: {{ .Values.readinessProbe.path | quote }} - port: "http" - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.readinessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} - successThreshold: {{ .Values.readinessProbe.successThreshold }} - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - startupProbe: - httpGet: - path: {{ .Values.startupProbe.path | quote }} - port: "http" - initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.startupProbe.periodSeconds }} - timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }} - successThreshold: {{ .Values.startupProbe.successThreshold }} - failureThreshold: {{ .Values.startupProbe.failureThreshold }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: litellm-config - mountPath: /etc/litellm/config.yaml - subPath: config.yaml - {{ if .Values.securityContext.readOnlyRootFilesystem }} - - name: tmp - mountPath: /tmp - - name: cache - mountPath: /.cache - - name: npm - mountPath: /.npm - {{- end }} - {{- if include "litellm.collector.socketDir" . }} - - name: collector-socket - mountPath: {{ include "litellm.collector.socketDir" . }} - {{- end }} - {{- with .Values.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.lifecycle }} - lifecycle: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.collector.enabled }} - - name: {{ include "litellm.name" . }}-collector - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: {{ toYaml .Values.collector.command | nindent 12 }} - env: - {{- include "litellm.proxyEnv" . | nindent 12 }} - {{- include "litellm.collectorEnv" . | nindent 12 }} - - name: LITELLM_JOB_ROLE - value: collector - {{- if not (hasKey (default dict .Values.envVars) "CONFIG_FILE_PATH") }} - - name: CONFIG_FILE_PATH - value: /etc/litellm/config.yaml - {{- end }} - envFrom: - {{- range .Values.environmentSecrets }} - - secretRef: - name: {{ . }} - {{- end }} - {{- range .Values.environmentConfigMaps }} - - configMapRef: - name: {{ . }} - {{- end }} - resources: - {{- toYaml .Values.collector.resources | nindent 12 }} - volumeMounts: - - name: litellm-config - mountPath: /etc/litellm/config.yaml - subPath: config.yaml - {{- if include "litellm.collector.socketDir" . }} - - name: collector-socket - mountPath: {{ include "litellm.collector.socketDir" . }} - {{- end }} - {{ if .Values.securityContext.readOnlyRootFilesystem }} - - name: tmp - mountPath: /tmp - - name: cache - mountPath: /.cache - - name: npm - mountPath: /.npm - {{- end }} - {{- with .Values.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- end }} - {{- with .Values.extraContainers }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - volumes: - {{ if .Values.securityContext.readOnlyRootFilesystem }} - - name: tmp - emptyDir: - sizeLimit: 500Mi - - name: cache - emptyDir: - sizeLimit: 500Mi - - name: npm - emptyDir: - sizeLimit: 500Mi - {{- end }} - - name: litellm-config - configMap: - {{- if .Values.proxyConfigMap.create }} - name: {{ include "litellm.fullname" . }}-config - {{- else }} - name: {{ .Values.proxyConfigMap.name }} - {{- end }} - items: - - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }} - path: "config.yaml" - {{- if include "litellm.collector.socketDir" . }} - - name: collector-socket - emptyDir: - sizeLimit: 1Mi - {{- end }} - {{- with .Values.volumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 90 }} - {{- if .Values.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} - {{- end }} diff --git a/helm/litellm-helm/templates/extra-resources.yaml b/helm/litellm-helm/templates/extra-resources.yaml deleted file mode 100644 index 33190d96fc0..00000000000 --- a/helm/litellm-helm/templates/extra-resources.yaml +++ /dev/null @@ -1,6 +0,0 @@ -{{- if .Values.extraResources }} -{{- range .Values.extraResources }} ---- -{{ toYaml . | nindent 0 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/helm/litellm-helm/templates/hpa.yaml b/helm/litellm-helm/templates/hpa.yaml deleted file mode 100644 index a651f916d21..00000000000 --- a/helm/litellm-helm/templates/hpa.yaml +++ /dev/null @@ -1,64 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "litellm.fullname" . }} - labels: - {{- include "litellm.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "litellm.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - {{- if .Values.autoscaling.behavior }} - behavior: - {{- toYaml .Values.autoscaling.behavior | nindent 4 }} - {{- end }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- if and .Values.collector.enabled .Values.collector.scaleOnProxyContainerCpu }} - - type: ContainerResource - containerResource: - name: cpu - container: {{ include "litellm.name" . }} - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- else }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} - {{- with .Values.autoscaling.targetRequestsPerSecond }} - - type: Pods - pods: - metric: - name: litellm_requests_per_second - target: - type: AverageValue - averageValue: {{ toJson . | trimAll "\"" | quote }} - {{- end }} - {{- with .Values.autoscaling.targetTokensPerSecond }} - - type: Pods - pods: - metric: - name: litellm_tokens_per_second - target: - type: AverageValue - averageValue: {{ toJson . | trimAll "\"" | quote }} - {{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/ingress.yaml b/helm/litellm-helm/templates/ingress.yaml deleted file mode 100644 index 3d6e3f4af7a..00000000000 --- a/helm/litellm-helm/templates/ingress.yaml +++ /dev/null @@ -1,78 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "litellm.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- with .Values.ingress.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- if and $.Values.lens.enabled $.Values.lens.service.name }} - - path: /lens-ingest - pathType: Prefix - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $.Values.lens.service.name }} - port: - number: {{ $.Values.lens.service.port }} - {{- else }} - serviceName: {{ $.Values.lens.service.name }} - servicePort: {{ $.Values.lens.service.port }} - {{- end }} - {{- end }} - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/keda.yaml b/helm/litellm-helm/templates/keda.yaml deleted file mode 100644 index bf585d0d4be..00000000000 --- a/helm/litellm-helm/templates/keda.yaml +++ /dev/null @@ -1,58 +0,0 @@ -{{- if and .Values.keda.enabled (not .Values.autoscaling.enabled) }} -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: {{ include "litellm.fullname" . }} - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- if .Values.keda.scaledObject.annotations }} - annotations: {{ toYaml .Values.keda.scaledObject.annotations | nindent 4 }} - {{- end }} -spec: - scaleTargetRef: - name: {{ include "litellm.fullname" . }} - pollingInterval: {{ .Values.keda.pollingInterval }} - cooldownPeriod: {{ .Values.keda.cooldownPeriod }} - minReplicaCount: {{ .Values.keda.minReplicas }} - maxReplicaCount: {{ .Values.keda.maxReplicas }} -{{- with .Values.keda.fallback }} - fallback: - failureThreshold: {{ .failureThreshold | default 3 }} - replicas: {{ .replicas | default $.Values.keda.maxReplicas }} -{{- end }} - triggers: -{{- with .Values.keda.triggers }} - {{- toYaml . | nindent 2 }} -{{- end }} -{{- $prom := .Values.keda.prometheus }} -{{- if or $prom.requestsPerSecond $prom.tokensPerSecond }} -{{- if not $prom.serverAddress }} -{{- fail "keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set" }} -{{- end }} -{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s%s" (include "litellm.fullname" .) (ternary "-metrics" "" .Values.metricsServer.enabled)) }} -{{- with $prom.requestsPerSecond }} - - type: prometheus - metadata: - serverAddress: {{ $prom.serverAddress | quote }} - threshold: {{ toJson . | trimAll "\"" | quote }} - query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }} -{{- end }} -{{- with $prom.tokensPerSecond }} - - type: prometheus - metadata: - serverAddress: {{ $prom.serverAddress | quote }} - threshold: {{ toJson . | trimAll "\"" | quote }} - query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }} -{{- end }} -{{- end }} - advanced: - restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }} -{{- if .Values.keda.behavior }} - horizontalPodAutoscalerConfig: - behavior: -{{- with .Values.keda.behavior }} -{{- toYaml . | nindent 8 }} -{{- end }} - -{{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/migrations-job.yaml b/helm/litellm-helm/templates/migrations-job.yaml deleted file mode 100644 index 7aa96a59102..00000000000 --- a/helm/litellm-helm/templates/migrations-job.yaml +++ /dev/null @@ -1,122 +0,0 @@ -{{- if .Values.migrationJob.enabled }} -# This job runs the Prisma migrations for the LiteLLM DB. -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "litellm.fullname" . }}-migrations - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- with .Values.migrationJob.jobLabels }} - {{- toYaml . | nindent 4 }} - {{- end }} - annotations: - {{- if .Values.migrationJob.hooks.argocd.enabled }} - argocd.argoproj.io/hook: PreSync - argocd.argoproj.io/hook-delete-policy: BeforeHookCreation - {{- end }} - {{- if .Values.migrationJob.hooks.helm.enabled }} - helm.sh/hook: "pre-install,pre-upgrade" - helm.sh/hook-delete-policy: "before-hook-creation" - helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "1" | quote }} - {{- end }} - checksum/config: {{ toYaml .Values | sha256sum }} - {{- with .Values.migrationJob.jobAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - template: - metadata: - labels: - {{- include "litellm.labels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.migrationJob.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - annotations: - {{- with .Values.migrationJob.annotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - {{- with .Values.migrationJob.extraInitContainers }} - initContainers: - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - containers: - - name: prisma-migrations - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - {{- if .Values.migrationJob.command }} - command: {{ toYaml .Values.migrationJob.command | nindent 12 }} - {{- else }} - command: ["python", "litellm/proxy/prisma_migration.py"] - {{- end }} - {{- if .Values.migrationJob.args }} - args: {{ toYaml .Values.migrationJob.args | nindent 12 }} - {{- else if .Values.migrationJob.command }} - args: [] - {{- end }} - workingDir: "/app" - env: - {{- include "litellm.validateDbTokenAuth" . -}} - {{- if .Values.db.useExisting }} - {{- include "litellm.existingDbEnv" . | nindent 12 }} - {{- else if .Values.db.deployStandalone }} - - name: DATABASE_URL - value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} - {{- end }} - {{- if .Values.envVars }} - {{- range $key, $val := .Values.envVars }} - - name: {{ $key }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with .Values.extraEnvVars }} - {{- toYaml . | nindent 12 }} - {{- end }} - - name: DISABLE_SCHEMA_UPDATE - value: "false" # always run the migration from the Helm PreSync hook, override the value set - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.migrationJob.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.migrationJob.extraContainers }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: OnFailure - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }} - backoffLimit: {{ .Values.migrationJob.backoffLimit }} - {{- with .Values.migrationJob.activeDeadlineSeconds }} - activeDeadlineSeconds: {{ . }} - {{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/poddisruptionbudget.yaml b/helm/litellm-helm/templates/poddisruptionbudget.yaml deleted file mode 100644 index 1715b94c1f6..00000000000 --- a/helm/litellm-helm/templates/poddisruptionbudget.yaml +++ /dev/null @@ -1,33 +0,0 @@ -{{- /* -PodDisruptionBudget for LiteLLM proxy -Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable} -Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins. -*/ -}} -{{- if .Values.pdb.enabled }} -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ include "litellm.fullname" . }} - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- with .Values.pdb.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.pdb.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - {{- /* Match the Deployment selector to target the same pod set */ -}} - {{- include "litellm.selectorLabels" . | nindent 6 }} - {{- if .Values.pdb.minAvailable }} - minAvailable: {{ .Values.pdb.minAvailable }} - {{- else if .Values.pdb.maxUnavailable }} - maxUnavailable: {{ .Values.pdb.maxUnavailable }} - {{- else }} - # Safe default if enabled but not configured - maxUnavailable: 1 - {{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/secret-dbcredentials.yaml b/helm/litellm-helm/templates/secret-dbcredentials.yaml deleted file mode 100644 index 8ab89a4579e..00000000000 --- a/helm/litellm-helm/templates/secret-dbcredentials.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.db.deployStandalone -}} -{{- include "litellm.validateBundledPostgresImageTag" . -}} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "litellm.fullname" . }}-dbcredentials -data: - # Password for the "postgres" user - postgres-password: {{ ( index .Values.postgresql.auth "postgres-password") | default "litellm" | b64enc }} - username: {{ .Values.postgresql.auth.username | default "litellm" | b64enc }} - password: {{ .Values.postgresql.auth.password | default "litellm" | b64enc }} -type: Opaque -{{- end -}} \ No newline at end of file diff --git a/helm/litellm-helm/templates/secret-masterkey.yaml b/helm/litellm-helm/templates/secret-masterkey.yaml deleted file mode 100644 index 60ab4e74c6b..00000000000 --- a/helm/litellm-helm/templates/secret-masterkey.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if not .Values.masterkeySecretName }} -{{- $secretName := printf "%s-masterkey" (include "litellm.fullname" .) }} -{{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName }} -{{- $masterkey := .Values.masterkey | default (dig "data" "masterkey" "" $existing | b64dec) | default (printf "sk-%s" (randAlphaNum 18)) }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ $secretName }} -data: - masterkey: {{ $masterkey | b64enc }} -type: Opaque -{{- end }} diff --git a/helm/litellm-helm/templates/service-metrics.yaml b/helm/litellm-helm/templates/service-metrics.yaml deleted file mode 100644 index 1d23fe39606..00000000000 --- a/helm/litellm-helm/templates/service-metrics.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.metricsServer.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "litellm.fullname" . }}-metrics - labels: - {{- include "litellm.labels" . | nindent 4 }} -spec: - type: ClusterIP - ports: - - port: {{ .Values.metricsServer.port }} - targetPort: metrics - protocol: TCP - name: metrics - selector: - {{- include "litellm.selectorLabels" . | nindent 4 }} -{{- end }} diff --git a/helm/litellm-helm/templates/service.yaml b/helm/litellm-helm/templates/service.yaml deleted file mode 100644 index 11812208929..00000000000 --- a/helm/litellm-helm/templates/service.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "litellm.fullname" . }} - {{- with .Values.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} - labels: - {{- include "litellm.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - {{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerClass }} - loadBalancerClass: {{ .Values.service.loadBalancerClass }} - {{- end }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "litellm.selectorLabels" . | nindent 4 }} diff --git a/helm/litellm-helm/templates/serviceaccount.yaml b/helm/litellm-helm/templates/serviceaccount.yaml deleted file mode 100644 index 7655470fa42..00000000000 --- a/helm/litellm-helm/templates/serviceaccount.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "litellm.serviceAccountName" . }} - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -automountServiceAccountToken: {{ .Values.serviceAccount.automount }} -{{- end }} diff --git a/helm/litellm-helm/templates/servicemonitor.yaml b/helm/litellm-helm/templates/servicemonitor.yaml deleted file mode 100644 index 68083d0da61..00000000000 --- a/helm/litellm-helm/templates/servicemonitor.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- with .Values.serviceMonitor }} -{{- if and (eq .enabled true) }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ include "litellm.fullname" $ }} - labels: - {{- include "litellm.labels" $ | nindent 4 }} - {{- if .labels }} - {{- toYaml .labels | nindent 4 }} - {{- end }} - {{- if .annotations }} - annotations: - {{- toYaml .annotations | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - {{- include "litellm.selectorLabels" $ | nindent 6 }} - namespaceSelector: - matchNames: - # if not set, use the release namespace - {{- if not .namespaceSelector.matchNames }} - - {{ $.Release.Namespace | quote }} - {{- else }} - {{- toYaml .namespaceSelector.matchNames | nindent 4 }} - {{- end }} - endpoints: - - port: {{ ternary "metrics" "http" $.Values.metricsServer.enabled }} - path: /metrics/ - interval: {{ .interval }} - scrapeTimeout: {{ .scrapeTimeout }} - scheme: http - {{- if .relabelings }} - relabelings: -{{- toYaml .relabelings | nindent 4 }} - {{- end }} -{{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/tests/test-connection.yaml b/helm/litellm-helm/templates/tests/test-connection.yaml deleted file mode 100644 index 86a8f66b10b..00000000000 --- a/helm/litellm-helm/templates/tests/test-connection.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "litellm.fullname" . }}-test-connection" - labels: - {{- include "litellm.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['sh', '-c'] - args: - - | - # Wait for a bit to allow the service to be ready - sleep 10 - # Try multiple times with a delay between attempts - for i in $(seq 1 30); do - wget -T 5 "{{ include "litellm.fullname" . }}:{{ .Values.service.port }}/health/readiness" && exit 0 - echo "Attempt $i failed, waiting..." - sleep 2 - done - exit 1 - restartPolicy: Never \ No newline at end of file diff --git a/helm/litellm-helm/templates/tests/test-env-vars.yaml b/helm/litellm-helm/templates/tests/test-env-vars.yaml deleted file mode 100644 index 9f0277557a4..00000000000 --- a/helm/litellm-helm/templates/tests/test-env-vars.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "litellm.fullname" . }}-env-test" - labels: - {{- include "litellm.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: test - image: busybox - command: ['sh', '-c'] - args: - - | - # Test DD_ENV - if [ "$DD_ENV" != "dev_helm" ]; then - echo "❌ Environment variable DD_ENV mismatch. Expected: dev_helm, Got: $DD_ENV" - exit 1 - fi - echo "✅ Environment variable DD_ENV matches expected value: $DD_ENV" - - # Test DD_SERVICE - if [ "$DD_SERVICE" != "litellm" ]; then - echo "❌ Environment variable DD_SERVICE mismatch. Expected: litellm, Got: $DD_SERVICE" - exit 1 - fi - echo "✅ Environment variable DD_SERVICE matches expected value: $DD_SERVICE" - - # Test USE_DDTRACE - if [ "$USE_DDTRACE" != "true" ]; then - echo "❌ Environment variable USE_DDTRACE mismatch. Expected: true, Got: $USE_DDTRACE" - exit 1 - fi - echo "✅ Environment variable USE_DDTRACE matches expected value: $USE_DDTRACE" - env: - - name: DD_ENV - value: {{ .Values.envVars.DD_ENV | quote }} - - name: DD_SERVICE - value: {{ .Values.envVars.DD_SERVICE | quote }} - - name: USE_DDTRACE - value: {{ .Values.envVars.USE_DDTRACE | quote }} - restartPolicy: Never \ No newline at end of file diff --git a/helm/litellm-helm/templates/tests/test-servicemonitor.yaml b/helm/litellm-helm/templates/tests/test-servicemonitor.yaml deleted file mode 100644 index ef8475339c3..00000000000 --- a/helm/litellm-helm/templates/tests/test-servicemonitor.yaml +++ /dev/null @@ -1,152 +0,0 @@ -{{- if .Values.serviceMonitor.enabled }} -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "litellm.fullname" . }}-test-servicemonitor" - labels: - {{- include "litellm.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: test - image: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3 - command: ['sh', '-c'] - args: - - | - set -e - echo "🔍 Testing ServiceMonitor configuration..." - - # Check if ServiceMonitor exists - if ! kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} &>/dev/null; then - echo "❌ ServiceMonitor not found" - exit 1 - fi - echo "✅ ServiceMonitor exists" - - # Get ServiceMonitor YAML - SM=$(kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o yaml) - - # Test endpoint configuration - ENDPOINT_PORT=$(echo "$SM" | grep -A 5 "endpoints:" | grep "port:" | awk '{print $2}') - if [ "$ENDPOINT_PORT" != "http" ]; then - echo "❌ Endpoint port mismatch. Expected: http, Got: $ENDPOINT_PORT" - exit 1 - fi - echo "✅ Endpoint port is correctly set to: $ENDPOINT_PORT" - - # Test endpoint path - ENDPOINT_PATH=$(echo "$SM" | grep -A 5 "endpoints:" | grep "path:" | awk '{print $2}') - if [ "$ENDPOINT_PATH" != "/metrics/" ]; then - echo "❌ Endpoint path mismatch. Expected: /metrics/, Got: $ENDPOINT_PATH" - exit 1 - fi - echo "✅ Endpoint path is correctly set to: $ENDPOINT_PATH" - - # Test interval - INTERVAL=$(echo "$SM" | grep "interval:" | awk '{print $2}') - if [ "$INTERVAL" != "{{ .Values.serviceMonitor.interval }}" ]; then - echo "❌ Interval mismatch. Expected: {{ .Values.serviceMonitor.interval }}, Got: $INTERVAL" - exit 1 - fi - echo "✅ Interval is correctly set to: $INTERVAL" - - # Test scrapeTimeout - TIMEOUT=$(echo "$SM" | grep "scrapeTimeout:" | awk '{print $2}') - if [ "$TIMEOUT" != "{{ .Values.serviceMonitor.scrapeTimeout }}" ]; then - echo "❌ ScrapeTimeout mismatch. Expected: {{ .Values.serviceMonitor.scrapeTimeout }}, Got: $TIMEOUT" - exit 1 - fi - echo "✅ ScrapeTimeout is correctly set to: $TIMEOUT" - - # Test scheme - SCHEME=$(echo "$SM" | grep "scheme:" | awk '{print $2}') - if [ "$SCHEME" != "http" ]; then - echo "❌ Scheme mismatch. Expected: http, Got: $SCHEME" - exit 1 - fi - echo "✅ Scheme is correctly set to: $SCHEME" - - {{- if .Values.serviceMonitor.labels }} - # Test custom labels - echo "🔍 Checking custom labels..." - {{- range $key, $value := .Values.serviceMonitor.labels }} - LABEL_VALUE=$(echo "$SM" | grep -A 20 "metadata:" | grep "{{ $key }}:" | awk '{print $2}') - if [ "$LABEL_VALUE" != "{{ $value }}" ]; then - echo "❌ Label {{ $key }} mismatch. Expected: {{ $value }}, Got: $LABEL_VALUE" - exit 1 - fi - echo "✅ Label {{ $key }} is correctly set to: {{ $value }}" - {{- end }} - {{- end }} - - {{- if .Values.serviceMonitor.annotations }} - # Test annotations - echo "🔍 Checking annotations..." - {{- range $key, $value := .Values.serviceMonitor.annotations }} - ANNOTATION_VALUE=$(echo "$SM" | grep -A 10 "annotations:" | grep "{{ $key }}:" | awk '{print $2}') - if [ "$ANNOTATION_VALUE" != "{{ $value }}" ]; then - echo "❌ Annotation {{ $key }} mismatch. Expected: {{ $value }}, Got: $ANNOTATION_VALUE" - exit 1 - fi - echo "✅ Annotation {{ $key }} is correctly set to: {{ $value }}" - {{- end }} - {{- end }} - - {{- if .Values.serviceMonitor.namespaceSelector.matchNames }} - # Test namespace selector - echo "🔍 Checking namespace selector..." - {{- range .Values.serviceMonitor.namespaceSelector.matchNames }} - if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ . }}"; then - echo "❌ Namespace {{ . }} not found in namespaceSelector" - exit 1 - fi - echo "✅ Namespace {{ . }} found in namespaceSelector" - {{- end }} - {{- else }} - # Test default namespace selector (should be release namespace) - if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ .Release.Namespace }}"; then - echo "❌ Release namespace {{ .Release.Namespace }} not found in namespaceSelector" - exit 1 - fi - echo "✅ Default namespace selector set to release namespace: {{ .Release.Namespace }}" - {{- end }} - - {{- if .Values.serviceMonitor.relabelings }} - # Test relabelings - echo "🔍 Checking relabelings configuration..." - if ! echo "$SM" | grep -q "relabelings:"; then - echo "❌ Relabelings section not found" - exit 1 - fi - echo "✅ Relabelings section exists" - {{- range .Values.serviceMonitor.relabelings }} - {{- if .targetLabel }} - if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "targetLabel: {{ .targetLabel }}"; then - echo "❌ Relabeling targetLabel {{ .targetLabel }} not found" - exit 1 - fi - echo "✅ Relabeling targetLabel {{ .targetLabel }} found" - {{- end }} - {{- if .action }} - if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "action: {{ .action }}"; then - echo "❌ Relabeling action {{ .action }} not found" - exit 1 - fi - echo "✅ Relabeling action {{ .action }} found" - {{- end }} - {{- end }} - {{- end }} - - # Test selector labels match the service - echo "🔍 Checking selector labels match service..." - SVC_LABELS=$(kubectl get svc {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.metadata.labels}') - echo "Service labels: $SVC_LABELS" - echo "✅ Selector labels validation passed" - - echo "" - echo "🎉 All ServiceMonitor tests passed successfully!" - serviceAccountName: {{ include "litellm.serviceAccountName" . }} - restartPolicy: Never -{{- end }} - diff --git a/helm/litellm-helm/tests/billing_metrics_tests.yaml b/helm/litellm-helm/tests/billing_metrics_tests.yaml deleted file mode 100644 index 52185c8dd9d..00000000000 --- a/helm/litellm-helm/tests/billing_metrics_tests.yaml +++ /dev/null @@ -1,100 +0,0 @@ -suite: test bearer-only billingMetrics wiring -templates: - - deployment.yaml - - configmap-litellm.yaml -tests: - - it: is disabled by default - template: deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - - notContains: - path: spec.template.spec.volumes - content: - name: billing-metrics - - - it: renders required URLs and Cognito secret references when enabled - template: deployment.yaml - set: - billingMetrics: - enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_TOKEN_URL - value: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_CLIENT_ID - valueFrom: - secretKeyRef: - name: litellm-billing-metrics - key: client-id - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: litellm-billing-metrics - key: client-secret - - notContains: - path: spec.template.spec.volumes - content: - name: billing-metrics - - notContains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics - - - it: renders the export interval only when set - template: deployment.yaml - set: - billingMetrics: - enabled: true - exportIntervalMs: 5000 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS - value: "5000" - - - it: requires the endpoint when enabled - template: deployment.yaml - set: - billingMetrics: - enabled: true - endpoint: "" - asserts: - - failedTemplate: - errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true - - - it: requires the token URL when enabled - template: deployment.yaml - set: - billingMetrics: - enabled: true - tokenUrl: "" - asserts: - - failedTemplate: - errorMessage: billingMetrics.tokenUrl is required when billingMetrics.enabled is true - - - it: requires the secret name when enabled - template: deployment.yaml - set: - billingMetrics: - enabled: true - secretName: "" - asserts: - - failedTemplate: - errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true diff --git a/helm/litellm-helm/tests/bundled_db_images_tests.yaml b/helm/litellm-helm/tests/bundled_db_images_tests.yaml deleted file mode 100644 index 8f0860c2721..00000000000 --- a/helm/litellm-helm/tests/bundled_db_images_tests.yaml +++ /dev/null @@ -1,94 +0,0 @@ -suite: test bundled database images -templates: - - charts/postgresql/templates/primary/statefulset.yaml - - charts/redis/templates/master/application.yaml - - charts/redis/templates/configmap.yaml - - charts/redis/templates/health-configmap.yaml - - charts/redis/templates/scripts-configmap.yaml - - charts/redis/templates/secret.yaml - - secret-dbcredentials.yaml - - templates/tests/test-servicemonitor.yaml -tests: - - it: should pull the bundled postgres from a repository that still publishes the pinned tag - template: charts/postgresql/templates/primary/statefulset.yaml - set: - db.deployStandalone: true - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: docker.io/bitnamilegacy/postgresql:16.2.0-debian-12-r6 - - - it: should pull the bundled postgres metrics exporter from the same repository - template: charts/postgresql/templates/primary/statefulset.yaml - set: - db.deployStandalone: true - postgresql.metrics.enabled: true - asserts: - - equal: - path: spec.template.spec.containers[1].image - value: docker.io/bitnamilegacy/postgres-exporter:0.15.0-debian-12-r14 - - - it: should run the bundled postgres init container from the same repository - template: charts/postgresql/templates/primary/statefulset.yaml - set: - db.deployStandalone: true - postgresql.volumePermissions.enabled: true - asserts: - - equal: - path: spec.template.spec.initContainers[0].image - value: docker.io/bitnamilegacy/os-shell:12-debian-12-r16 - - - it: should pull the bundled redis from a repository that still publishes the pinned tag - template: charts/redis/templates/master/application.yaml - set: - redis.enabled: true - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: docker.io/bitnamilegacy/redis:7.2.4-debian-12-r9 - - - it: should reject a floating postgres tag that could cross a major version on an existing volume - template: secret-dbcredentials.yaml - set: - db.deployStandalone: true - postgresql.image.tag: latest - asserts: - - failedTemplate: - errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got "latest"). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.' - - - it: should reject an empty postgres tag - template: secret-dbcredentials.yaml - set: - db.deployStandalone: true - postgresql.image.tag: "" - asserts: - - failedTemplate: - errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got ""). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.' - - - it: should accept an empty postgres tag when the image is pinned by digest - template: secret-dbcredentials.yaml - set: - db.deployStandalone: true - postgresql.image.tag: "" - postgresql.image.digest: sha256:0d0e2f1a5b3c4d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6 - asserts: - - hasDocuments: - count: 1 - - - it: should run the servicemonitor test pod from a pinned image - template: templates/tests/test-servicemonitor.yaml - set: - serviceMonitor.enabled: true - asserts: - - equal: - path: spec.containers[0].image - value: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3 - - - it: should not constrain the postgres tag when the bundled database is not deployed - template: secret-dbcredentials.yaml - set: - db.deployStandalone: false - postgresql.image.tag: latest - asserts: - - hasDocuments: - count: 0 diff --git a/helm/litellm-helm/tests/collector_tests.yaml b/helm/litellm-helm/tests/collector_tests.yaml deleted file mode 100644 index 1365290e16c..00000000000 --- a/helm/litellm-helm/tests/collector_tests.yaml +++ /dev/null @@ -1,272 +0,0 @@ -suite: test collector sidecar -templates: - - deployment.yaml - - hpa.yaml - - configmap-litellm.yaml -tests: - - it: should run the proxy alone with no collector env by default - template: deployment.yaml - asserts: - - lengthEqual: - path: spec.template.spec.containers - count: 1 - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ENABLED - value: "true" - - notContains: - path: spec.template.spec.volumes - content: - name: collector-socket - any: true - - - it: should add the sidecar on the same image and point both containers at the unix socket - template: deployment.yaml - set: - image.tag: test - db.connectionPool.enabled: true - collector.enabled: true - collector.resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: "1" - memory: 2Gi - asserts: - - lengthEqual: - path: spec.template.spec.containers - count: 2 - - equal: - path: spec.template.spec.containers[1].name - value: litellm-collector - - equal: - path: spec.template.spec.containers[1].image - value: ghcr.io/berriai/litellm:test - - equal: - path: spec.template.spec.containers[1].command - value: [python, -m, litellm.proxy.collector] - - equal: - path: spec.template.spec.containers[1].resources.requests.cpu - value: 500m - - equal: - path: spec.template.spec.containers[1].resources.limits.memory - value: 2Gi - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ENABLED - value: "true" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ADDRESS - value: unix:///var/run/litellm/collector.sock - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_BUFFER_SIZE - value: "1000" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ON_UNAVAILABLE - value: fallback - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_JOB_ROLE - value: collector - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_JOB_ROLE - value: collector - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_COLLECTOR_ADDRESS - value: unix:///var/run/litellm/collector.sock - - contains: - path: spec.template.spec.containers[1].env - content: - name: CONFIG_FILE_PATH - value: /etc/litellm/config.yaml - - contains: - path: spec.template.spec.containers[1].env - content: - name: DATABASE_HOST - value: RELEASE-NAME-postgresql - - contains: - path: spec.template.spec.containers[1].env - content: - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: RELEASE-NAME-litellm-dbcredentials - key: password - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_ENABLED - value: "true" - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: "20" - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: collector-socket - mountPath: /var/run/litellm - - contains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: collector-socket - mountPath: /var/run/litellm - - contains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: litellm-config - mountPath: /etc/litellm/config.yaml - subPath: config.yaml - - contains: - path: spec.template.spec.volumes - content: - name: collector-socket - emptyDir: - sizeLimit: 1Mi - - - it: should skip the socket volume and pass the policy through on tcp transport - template: deployment.yaml - set: - collector.enabled: true - collector.address: tcp://127.0.0.1:4100 - collector.onUnavailable: drop - collector.bufferSize: 50 - envVars: - CONFIG_FILE_PATH: /custom/config.yaml - asserts: - - lengthEqual: - path: spec.template.spec.containers - count: 2 - - notContains: - path: spec.template.spec.containers[1].env - content: - name: CONFIG_FILE_PATH - value: /etc/litellm/config.yaml - - contains: - path: spec.template.spec.containers[1].env - content: - name: CONFIG_FILE_PATH - value: /custom/config.yaml - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ADDRESS - value: tcp://127.0.0.1:4100 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ON_UNAVAILABLE - value: drop - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_BUFFER_SIZE - value: "50" - - notContains: - path: spec.template.spec.volumes - content: - name: collector-socket - any: true - - - it: should keep metrics and billing env on the proxy container only - template: deployment.yaml - set: - collector.enabled: true - metricsServer.enabled: true - metricsServer.port: 9090 - billingMetrics.enabled: true - billingMetrics.endpoint: https://metering.example.com - billingMetrics.secretName: billing-metrics - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PROMETHEUS_METRICS_PORT - value: "9090" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://metering.example.com - - notContains: - path: spec.template.spec.containers[1].env - content: - name: PROMETHEUS_METRICS_PORT - any: true - - notContains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - any: true - - notContains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_BILLING_METRICS_CLIENT_SECRET - any: true - - - it: should give the sidecar the same scratch mounts as the proxy on a read-only root - template: deployment.yaml - set: - collector.enabled: true - securityContext.readOnlyRootFilesystem: true - asserts: - - contains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: npm - mountPath: /.npm - - contains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: cache - mountPath: /.cache - - contains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: tmp - mountPath: /tmp - - - it: should keep the pod-wide cpu metric unless asked to scale on the proxy container - template: hpa.yaml - set: - autoscaling.enabled: true - collector.enabled: true - asserts: - - equal: { path: "spec.metrics[0].type", value: Resource } - - equal: { path: "spec.metrics[0].resource.name", value: cpu } - - - it: should scale on the proxy container's cpu only when opted in - template: hpa.yaml - set: - autoscaling.enabled: true - collector.enabled: true - collector.scaleOnProxyContainerCpu: true - asserts: - - equal: { path: "spec.metrics[0].type", value: ContainerResource } - - equal: { path: "spec.metrics[0].containerResource.name", value: cpu } - - equal: { path: "spec.metrics[0].containerResource.container", value: litellm } - - equal: { path: "spec.metrics[0].containerResource.target.averageUtilization", value: 60 } - - isNull: { path: "spec.metrics[0].resource" } - - - it: should not switch to the container metric while the sidecar is off - template: hpa.yaml - set: - autoscaling.enabled: true - collector.scaleOnProxyContainerCpu: true - asserts: - - equal: { path: "spec.metrics[0].type", value: Resource } diff --git a/helm/litellm-helm/tests/connection_pool_tests.yaml b/helm/litellm-helm/tests/connection_pool_tests.yaml deleted file mode 100644 index 203082f27ba..00000000000 --- a/helm/litellm-helm/tests/connection_pool_tests.yaml +++ /dev/null @@ -1,112 +0,0 @@ -suite: test in-container connection pool -templates: - - deployment.yaml - - configmap-litellm.yaml -tests: - - it: should not emit pgbouncer env vars by default - template: deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_ENABLED - value: "true" - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: "20" - - - it: should enable the pool with the default sizing when connectionPool.enabled is set - template: deployment.yaml - set: - db.connectionPool.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_ENABLED - value: "true" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: "20" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - value: "1000" - - - it: should pass custom sizing through as strings next to the worker count - template: deployment.yaml - set: - numWorkers: 4 - db.connectionPool.enabled: true - db.connectionPool.maxDbConnections: 8 - db.connectionPool.maxClientConn: 400 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: "8" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - value: "400" - - contains: - path: spec.template.spec.containers[0].args - content: "4" - - - it: should give the collector sidecar the same pool env as the proxy container - template: deployment.yaml - set: - collector.enabled: true - db.connectionPool.enabled: true - db.connectionPool.maxDbConnections: 8 - db.connectionPool.maxClientConn: 400 - asserts: - - equal: - path: spec.template.spec.containers[1].name - value: litellm-collector - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_ENABLED - value: "true" - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: "8" - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - value: "400" - - - it: should give the collector sidecar no pool env when the pool is off - template: deployment.yaml - set: - collector.enabled: true - asserts: - - equal: - path: spec.template.spec.containers[1].name - value: litellm-collector - - notContains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_ENABLED - any: true - - notContains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - any: true - - notContains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - any: true diff --git a/helm/litellm-helm/tests/coordination_redis_tests.yaml b/helm/litellm-helm/tests/coordination_redis_tests.yaml deleted file mode 100644 index 0b58b1e6bc8..00000000000 --- a/helm/litellm-helm/tests/coordination_redis_tests.yaml +++ /dev/null @@ -1,143 +0,0 @@ -suite: test coordination redis -templates: - - configmap-litellm.yaml - - deployment.yaml -tests: - - it: should not render coordination_redis when redis is disabled - template: configmap-litellm.yaml - set: - redis.enabled: false - asserts: - - notMatchRegex: - path: data["config.yaml"] - pattern: coordination_redis - - - it: should not emit redis env vars when redis is disabled - template: deployment.yaml - set: - redis.enabled: false - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_HOST - value: RELEASE-NAME-redis-master - any: true - - - it: should render coordination_redis pointing at the bundled redis when enabled - template: configmap-litellm.yaml - set: - redis.enabled: true - asserts: - - matchRegex: - path: data["config.yaml"] - pattern: "coordination_redis:\n host: os.environ/REDIS_HOST\n password: os.environ/REDIS_PASSWORD\n port: os.environ/REDIS_PORT\n" - - matchRegex: - path: data["config.yaml"] - pattern: "master_key: os.environ/PROXY_MASTER_KEY" - - - it: should emit redis env vars backing the coordination_redis os.environ refs - template: deployment.yaml - set: - redis.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_HOST - value: RELEASE-NAME-redis-master - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_PORT - value: "6379" - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: RELEASE-NAME-redis - key: redis-password - - - it: should not render coordination_redis when coordination is opted out - template: configmap-litellm.yaml - set: - redis.enabled: true - redis.coordination.enabled: false - asserts: - - notMatchRegex: - path: data["config.yaml"] - pattern: coordination_redis - - - it: should keep emitting redis env vars when coordination is opted out - template: deployment.yaml - set: - redis.enabled: true - redis.coordination.enabled: false - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_HOST - value: RELEASE-NAME-redis-master - - - it: should not clobber a user supplied coordination_redis block - template: configmap-litellm.yaml - set: - redis.enabled: true - proxy_config.general_settings.coordination_redis: - url: os.environ/COORDINATION_REDIS_URL - asserts: - - matchRegex: - path: data["config.yaml"] - pattern: "coordination_redis:\n url: os.environ/COORDINATION_REDIS_URL\n" - - notMatchRegex: - path: data["config.yaml"] - pattern: "host: os.environ/REDIS_HOST" - - - it: should render sentinel_nodes and service_name in sentinel mode - template: configmap-litellm.yaml - set: - redis.enabled: true - redis.architecture: replication - redis.sentinel.enabled: true - asserts: - # The sentinel Service the redis subchart renders is "-redis", and a - # plain client cannot speak the sentinel protocol, so host/port must not appear - - matchRegex: - path: data["config.yaml"] - pattern: "coordination_redis:\n password: os.environ/REDIS_PASSWORD\n sentinel_nodes:\n - - RELEASE-NAME-redis\n - 26379\n service_name: mymaster\n" - - notMatchRegex: - path: data["config.yaml"] - pattern: "host: os.environ/REDIS_HOST" - - - it: should carry a custom sentinel masterSet into service_name - template: configmap-litellm.yaml - set: - redis.enabled: true - redis.architecture: replication - redis.sentinel.enabled: true - redis.sentinel.masterSet: litellm-master - asserts: - - matchRegex: - path: data["config.yaml"] - pattern: "service_name: litellm-master" - - - it: should point REDIS_HOST at the sentinel service in sentinel mode - template: deployment.yaml - set: - redis.enabled: true - redis.architecture: replication - redis.sentinel.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_HOST - value: RELEASE-NAME-redis - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_PORT - value: "26379" diff --git a/helm/litellm-helm/tests/database_auth_tests.yaml b/helm/litellm-helm/tests/database_auth_tests.yaml deleted file mode 100644 index 2c480196dfb..00000000000 --- a/helm/litellm-helm/tests/database_auth_tests.yaml +++ /dev/null @@ -1,288 +0,0 @@ -suite: test existing database token auth -templates: - - deployment.yaml - - migrations-job.yaml - - configmap-litellm.yaml -tests: - - it: should keep password auth from the secret when no token auth is set - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: postgres - key: username - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: postgres - key: password - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME) - - notContains: - path: spec.template.spec.containers[0].env - content: - name: IAM_TOKEN_DB_AUTH - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: AZURE_POSTGRESQL_AUTH - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PORT - any: true - - - it: should render RDS IAM auth without a password secret on the proxy and the collector sidecar - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - db.username: litellm_iam - db.endpoint: writer.cluster-abc.us-west-1.rds.amazonaws.com - db.port: 5433 - collector.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: IAM_TOKEN_DB_AUTH - value: "true" - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_USERNAME - value: litellm_iam - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_HOST - value: writer.cluster-abc.us-west-1.rds.amazonaws.com - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PORT - value: "5433" - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_NAME - value: litellm - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PASSWORD - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: AZURE_POSTGRESQL_AUTH - any: true - - contains: - path: spec.template.spec.containers[1].env - content: - name: IAM_TOKEN_DB_AUTH - value: "true" - - notContains: - path: spec.template.spec.containers[1].env - content: - name: DATABASE_PASSWORD - any: true - - - it: should give the migrations job the same token auth env as the proxy - template: migrations-job.yaml - set: - migrationJob.enabled: true - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - db.username: litellm_iam - db.endpoint: writer.cluster-abc.us-west-1.rds.amazonaws.com - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: IAM_TOKEN_DB_AUTH - value: "true" - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_USERNAME - value: litellm_iam - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_HOST - value: writer.cluster-abc.us-west-1.rds.amazonaws.com - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PORT - value: "5432" - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PASSWORD - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - any: true - - - it: should carry db.schema as DATABASE_SCHEMA under token auth and omit it when empty - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - db.username: litellm_iam - db.schema: tenant1 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_SCHEMA - value: tenant1 - - - it: should leave the schema to db.url under password auth even when db.schema is set - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.schema: tenant1 - db.url: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)?schema=tenant1 - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_SCHEMA - any: true - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)?schema=tenant1 - - - it: should not emit DATABASE_SCHEMA when db.schema is empty - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_SCHEMA - any: true - - - it: should read the user from the secret when db.username is empty under token auth - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - db.secret.name: rds-identity - db.secret.usernameKey: user - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: rds-identity - key: user - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PASSWORD - any: true - - - it: should render Entra auth with AZURE_POSTGRESQL_AUTH and no password - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.useAzureEntraAuth: true - db.username: litellm@example.onmicrosoft.com - db.endpoint: litellm.postgres.database.azure.com - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: AZURE_POSTGRESQL_AUTH - value: "true" - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_USERNAME - value: litellm@example.onmicrosoft.com - - notContains: - path: spec.template.spec.containers[0].env - content: - name: IAM_TOKEN_DB_AUTH - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PASSWORD - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - any: true - - - it: should fail when both token sources are enabled - template: deployment.yaml - set: - migrationJob.enabled: false - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - db.useAzureEntraAuth: true - asserts: - - failedTemplate: - errorMessage: "db.useIAMAuth and db.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source" - - - it: should fail when token auth is set while the bundled postgres is still deployed - template: deployment.yaml - set: - migrationJob.enabled: false - db.useExisting: true - db.useIAMAuth: true - asserts: - - failedTemplate: - errorPattern: "need db.useExisting: true and db.deployStandalone: false" - - - it: should fail when token auth is set without useExisting - template: migrations-job.yaml - set: - migrationJob.enabled: true - db.deployStandalone: false - db.useAzureEntraAuth: true - asserts: - - failedTemplate: - errorPattern: "need db.useExisting: true and db.deployStandalone: false" diff --git a/helm/litellm-helm/tests/deployment_command_args_labels_tests.yaml b/helm/litellm-helm/tests/deployment_command_args_labels_tests.yaml deleted file mode 100644 index 6b0d45ebf48..00000000000 --- a/helm/litellm-helm/tests/deployment_command_args_labels_tests.yaml +++ /dev/null @@ -1,68 +0,0 @@ -suite: test deployment command, args, and deploymentLabels -templates: - - deployment.yaml - - configmap-litellm.yaml -tests: - - it: should override args when custom args specified - template: deployment.yaml - set: - args: - - --custom-arg1 - - value1 - - --custom-arg2 - asserts: - - equal: - path: spec.template.spec.containers[0].args - value: - - --custom-arg1 - - value1 - - --custom-arg2 - - it: should set custom command when specified - template: deployment.yaml - set: - command: - - /bin/sh - - -c - asserts: - - equal: - path: spec.template.spec.containers[0].command - value: - - /bin/sh - - -c - - it: should set custom command and args together - template: deployment.yaml - set: - command: - - python - - -u - args: - - my_script.py - - --verbose - asserts: - - equal: - path: spec.template.spec.containers[0].command - value: - - python - - -u - - equal: - path: spec.template.spec.containers[0].args - value: - - my_script.py - - --verbose - - it: should add deploymentLabels to deployment metadata - template: deployment.yaml - set: - deploymentLabels: - environment: production - team: platform - version: v1.2.3 - asserts: - - equal: - path: metadata.labels.environment - value: production - - equal: - path: metadata.labels.team - value: platform - - equal: - path: metadata.labels.version - value: v1.2.3 diff --git a/helm/litellm-helm/tests/deployment_tests.yaml b/helm/litellm-helm/tests/deployment_tests.yaml deleted file mode 100644 index 292eeadfd13..00000000000 --- a/helm/litellm-helm/tests/deployment_tests.yaml +++ /dev/null @@ -1,520 +0,0 @@ -suite: test deployment -templates: - - deployment.yaml - - configmap-litellm.yaml -tests: - - it: should work - template: deployment.yaml - set: - image.tag: test - asserts: - - isKind: - of: Deployment - - matchRegex: - path: metadata.name - pattern: -litellm$ - - equal: - path: spec.template.spec.containers[0].image - value: ghcr.io/berriai/litellm:test - - it: should work with tolerations - template: deployment.yaml - set: - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - asserts: - - equal: - path: spec.template.spec.tolerations[0].key - value: node-role.kubernetes.io/master - - equal: - path: spec.template.spec.tolerations[0].operator - value: Exists - - it: should work with affinity - template: deployment.yaml - set: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: topology.kubernetes.io/zone - operator: In - values: - - antarctica-east1 - asserts: - - equal: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key - value: topology.kubernetes.io/zone - - equal: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator - value: In - - equal: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0] - value: antarctica-east1 - - it: should work without masterkeySecretName or masterkeySecretKey - template: deployment.yaml - set: - masterkeySecretName: "" - masterkeySecretKey: "" - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PROXY_MASTER_KEY - valueFrom: - secretKeyRef: - name: RELEASE-NAME-litellm-masterkey - key: masterkey - - it: should work with masterkeySecretName and masterkeySecretKey - template: deployment.yaml - set: - masterkeySecretName: my-secret - masterkeySecretKey: my-key - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PROXY_MASTER_KEY - valueFrom: - secretKeyRef: - name: my-secret - key: my-key - - it: should inject DATABASE_READER_HOST from readReplicaEndpointKey before DATABASE_URL_READ_REPLICA - template: deployment.yaml - set: - db: - deployStandalone: false - useExisting: true - secret: - name: postgres - usernameKey: username - passwordKey: password - readReplicaEndpointKey: reader-host - readReplicaUrl: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)?sslmode=require - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_READER_HOST - valueFrom: - secretKeyRef: - name: postgres - key: reader-host - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL_READ_REPLICA - value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)?sslmode=require - # $(VAR) interpolation only resolves vars defined EARLIER in the env - # array, so the reader host must precede the composed URL - - equal: - path: spec.template.spec.containers[0].env[7].name - value: DATABASE_READER_HOST - - equal: - path: spec.template.spec.containers[0].env[8].name - value: DATABASE_URL_READ_REPLICA - - it: should omit reader host when readReplicaUrl is unset - template: deployment.yaml - set: - db: - deployStandalone: false - useExisting: true - secret: - name: postgres - usernameKey: username - passwordKey: password - readReplicaEndpointKey: reader-host - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_READER_HOST - valueFrom: - secretKeyRef: - name: postgres - key: reader-host - - it: should prefer readReplicaUrlKey over readReplicaEndpointKey composition - template: deployment.yaml - set: - db: - useExisting: true - secret: - name: postgres - usernameKey: username - passwordKey: password - readReplicaUrlKey: reader-url - readReplicaEndpointKey: reader-host - readReplicaUrl: postgresql://ignored - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL_READ_REPLICA - valueFrom: - secretKeyRef: - name: postgres - key: reader-url - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL_READ_REPLICA - value: postgresql://ignored - # the unused reader-host secret ref must be suppressed so a missing - # key can't fail pod creation - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_READER_HOST - valueFrom: - secretKeyRef: - name: postgres - key: reader-host - - it: should work with extraEnvVars - template: deployment.yaml - set: - extraEnvVars: - - name: EXTRA_ENV_VAR - valueFrom: - fieldRef: - fieldPath: metadata.labels['env'] - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: EXTRA_ENV_VAR - valueFrom: - fieldRef: - fieldPath: metadata.labels['env'] - - it: should work with both extraEnvVars and envVars - template: deployment.yaml - set: - envVars: - ENV_VAR: ENV_VAR_VALUE - extraEnvVars: - - name: EXTRA_ENV_VAR - value: EXTRA_ENV_VAR_VALUE - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: ENV_VAR - value: ENV_VAR_VALUE - - contains: - path: spec.template.spec.containers[0].env - content: - name: EXTRA_ENV_VAR - value: EXTRA_ENV_VAR_VALUE - - it: should mount existing configmap when create=false - template: deployment.yaml - set: - proxyConfigMap: - create: false - name: my-litellm-config - key: custom.yaml - asserts: - - contains: - path: spec.template.spec.volumes - content: - name: litellm-config - configMap: - name: my-litellm-config - items: - - key: custom.yaml - path: config.yaml - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: litellm-config - mountPath: /etc/litellm/config.yaml - subPath: config.yaml - - it: should work with lifecycle hooks - template: deployment.yaml - set: - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - echo "Container stopping" - asserts: - - exists: - path: spec.template.spec.containers[0].lifecycle - - equal: - path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[0] - value: /bin/sh - - equal: - path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[1] - value: -c - - equal: - path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2] - value: echo "Container stopping" - - it: should render background health check settings from proxy_config.general_settings - template: configmap-litellm.yaml - set: - proxy_config.general_settings.background_health_checks: true - proxy_config.general_settings.health_check_interval: 240 - proxy_config.general_settings.health_check_concurrency: 16 - proxy_config.general_settings.health_check_details: false - asserts: - - matchRegex: - path: data["config.yaml"] - pattern: '(?m)^\s*background_health_checks:\s*true$' - - matchRegex: - path: data["config.yaml"] - pattern: '(?m)^\s*health_check_interval:\s*240$' - - matchRegex: - path: data["config.yaml"] - pattern: '(?m)^\s*health_check_concurrency:\s*16$' - - matchRegex: - path: data["config.yaml"] - pattern: '(?m)^\s*health_check_details:\s*false$' - - it: should not render any placeholder models when the user sets proxy_config without model_list - template: configmap-litellm.yaml - set: - proxy_config.general_settings.store_model_in_db: true - asserts: - - notMatchRegex: - path: data["config.yaml"] - pattern: 'model_name:' - - matchRegex: - path: data["config.yaml"] - pattern: 'store_model_in_db: true' - - matchRegex: - path: data["config.yaml"] - pattern: 'master_key: os.environ/PROXY_MASTER_KEY' - - it: should render a user-supplied model_list as the only models - template: configmap-litellm.yaml - set: - proxy_config.model_list: - - model_name: my-model - litellm_params: - model: openai/my-model - api_key: os.environ/OPENAI_API_KEY - asserts: - - matchRegex: - path: data["config.yaml"] - pattern: 'model_list:\n- litellm_params:\n api_key: os.environ/OPENAI_API_KEY\n model: openai/my-model\n model_name: my-model\n$' - - it: should allow overriding liveness, readiness, and startup probes - template: deployment.yaml - set: - livenessProbe: - path: /custom/livez - initialDelaySeconds: 5 - periodSeconds: 15 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - path: /custom/readyz - initialDelaySeconds: 10 - periodSeconds: 20 - timeoutSeconds: 6 - successThreshold: 1 - failureThreshold: 6 - startupProbe: - path: /custom/startupz - initialDelaySeconds: 15 - periodSeconds: 25 - timeoutSeconds: 7 - successThreshold: 1 - failureThreshold: 40 - asserts: - - equal: - path: spec.template.spec.containers[0].livenessProbe.httpGet.path - value: /custom/livez - - equal: - path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds - value: 5 - - equal: - path: spec.template.spec.containers[0].readinessProbe.httpGet.path - value: /custom/readyz - - equal: - path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds - value: 6 - - equal: - path: spec.template.spec.containers[0].startupProbe.httpGet.path - value: /custom/startupz - - equal: - path: spec.template.spec.containers[0].startupProbe.failureThreshold - value: 40 - - it: should render container resources from values - template: deployment.yaml - set: - resources: - limits: - cpu: 500m - memory: 2Gi - requests: - cpu: 250m - memory: 1Gi - asserts: - - equal: - path: spec.template.spec.containers[0].resources.limits.cpu - value: 500m - - equal: - path: spec.template.spec.containers[0].resources.limits.memory - value: 2Gi - - equal: - path: spec.template.spec.containers[0].resources.requests.cpu - value: 250m - - equal: - path: spec.template.spec.containers[0].resources.requests.memory - value: 1Gi - - it: should keep default probes and empty resources unchanged - template: deployment.yaml - asserts: - - equal: - path: spec.template.spec.containers[0].livenessProbe.httpGet.path - value: /health/liveliness - - equal: - path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds - value: 0 - - equal: - path: spec.template.spec.containers[0].livenessProbe.periodSeconds - value: 15 - - equal: - path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds - value: 5 - - equal: - path: spec.template.spec.containers[0].livenessProbe.successThreshold - value: 1 - - equal: - path: spec.template.spec.containers[0].livenessProbe.failureThreshold - value: 5 - - equal: - path: spec.template.spec.containers[0].readinessProbe.httpGet.path - value: /health/readiness - - equal: - path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds - value: 0 - - equal: - path: spec.template.spec.containers[0].readinessProbe.periodSeconds - value: 10 - - equal: - path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds - value: 5 - - equal: - path: spec.template.spec.containers[0].readinessProbe.successThreshold - value: 1 - - equal: - path: spec.template.spec.containers[0].readinessProbe.failureThreshold - value: 3 - - equal: - path: spec.template.spec.containers[0].startupProbe.httpGet.path - value: /health/readiness - - equal: - path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds - value: 0 - - equal: - path: spec.template.spec.containers[0].startupProbe.periodSeconds - value: 10 - - equal: - path: spec.template.spec.containers[0].startupProbe.timeoutSeconds - value: 5 - - equal: - path: spec.template.spec.containers[0].startupProbe.successThreshold - value: 1 - - equal: - path: spec.template.spec.containers[0].startupProbe.failureThreshold - value: 30 - - equal: - path: spec.template.spec.containers[0].resources - value: {} - - it: should be able to set minReadySeconds - template: deployment.yaml - set: - deploymentMinReadySeconds: 5 - asserts: - - equal: - path: spec.minReadySeconds - value: 5 - - it: should have minReadySeconds absent when deploymentMinReadySeconds is not set - template: deployment.yaml - asserts: - - notExists: - path: spec.minReadySeconds - - it: should work with extraInitContainers - template: deployment.yaml - set: - extraInitContainers: - - name: init-test - image: busybox:latest - command: ["echo", "hello"] - asserts: - - contains: - path: spec.template.spec.initContainers - content: - name: init-test - image: busybox:latest - command: ["echo", "hello"] - - it: should support tpl in extraInitContainers - template: deployment.yaml - set: - image: - repository: ghcr.io/berriai/litellm - tag: test - extraInitContainers: - - name: init-tpl - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - command: ["echo", "hello"] - asserts: - - contains: - path: spec.template.spec.initContainers - content: - name: init-tpl - image: "ghcr.io/berriai/litellm:test" - command: ["echo", "hello"] - - it: should work with extraContainers - template: deployment.yaml - set: - extraContainers: - - name: sidecar - image: busybox:latest - asserts: - - contains: - path: spec.template.spec.containers - content: - name: sidecar - image: busybox:latest - - it: should support tpl in extraContainers - template: deployment.yaml - set: - image: - repository: ghcr.io/berriai/litellm - tag: test - extraContainers: - - name: sidecar-tpl - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - asserts: - - contains: - path: spec.template.spec.containers - content: - name: sidecar-tpl - image: "ghcr.io/berriai/litellm:test" - - it: should support tpl in podAnnotations - template: deployment.yaml - set: - image: - repository: ghcr.io/berriai/litellm - tag: test - # Mirrors the real-world scenario this feature unblocks: - # user disables the built-in ConfigMap (and its built-in checksum/config - # annotation) and re-implements checksum/config themselves via tpl. - proxyConfigMap: - create: false - podAnnotations: - checksum/config: "{{ .Values.image.tag }}" - example.com/some-key: "{{ .Values.image.repository }}" - example.com/literal: "plain-string-value" - asserts: - - equal: - path: spec.template.metadata.annotations["checksum/config"] - value: "test" - - equal: - path: spec.template.metadata.annotations["example.com/some-key"] - value: "ghcr.io/berriai/litellm" - - equal: - path: spec.template.metadata.annotations["example.com/literal"] - value: "plain-string-value" diff --git a/helm/litellm-helm/tests/hpa_tests.yaml b/helm/litellm-helm/tests/hpa_tests.yaml deleted file mode 100644 index e446f58c8fe..00000000000 --- a/helm/litellm-helm/tests/hpa_tests.yaml +++ /dev/null @@ -1,144 +0,0 @@ -suite: "hpa" -templates: - - hpa.yaml -tests: - - it: "renders behavior when set" - set: - autoscaling.enabled: true - autoscaling.behavior: - scaleUp: - stabilizationWindowSeconds: 60 - policies: - - type: Pods - value: 2 - periodSeconds: 60 - scaleDown: - stabilizationWindowSeconds: 90 - policies: - - type: Pods - value: 1 - periodSeconds: 60 - asserts: - - isKind: { of: HorizontalPodAutoscaler } - - equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 } - - equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 } - - - it: "does not render behavior when not set" - set: - autoscaling.enabled: true - asserts: - - isKind: { of: HorizontalPodAutoscaler } - - isNull: { path: spec.behavior } - - - it: "scales on cpu at the documented 60 percent by default" - set: - autoscaling.enabled: true - asserts: - - isKind: { of: HorizontalPodAutoscaler } - - equal: { path: "spec.metrics[0].resource.name", value: cpu } - - equal: { path: "spec.metrics[0].resource.target.type", value: Utilization } - - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 } - - - it: "does not scale on memory by default" - set: - autoscaling.enabled: true - asserts: - - lengthEqual: { path: spec.metrics, count: 1 } - - - it: "honours an explicit cpu target override" - set: - autoscaling.enabled: true - autoscaling.targetCPUUtilizationPercentage: 75 - asserts: - - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 } - - - it: "renders a memory metric only when a memory target is set" - set: - autoscaling.enabled: true - autoscaling.targetMemoryUtilizationPercentage: 80 - asserts: - - lengthEqual: { path: spec.metrics, count: 2 } - - equal: { path: "spec.metrics[1].resource.name", value: memory } - - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 } - - - it: "renders no workload metrics by default" - set: - autoscaling.enabled: true - autoscaling.targetMemoryUtilizationPercentage: 80 - asserts: - - lengthEqual: { path: spec.metrics, count: 2 } - - notContains: { path: spec.metrics, content: { type: Pods }, any: true } - - - it: "adds a requests-per-second Pods metric after the cpu metric" - set: - autoscaling.enabled: true - autoscaling.targetRequestsPerSecond: 90 - asserts: - - lengthEqual: { path: spec.metrics, count: 2 } - - equal: { path: "spec.metrics[0].resource.name", value: cpu } - - equal: - path: "spec.metrics[1]" - value: - type: Pods - pods: - metric: { name: litellm_requests_per_second } - target: { type: AverageValue, averageValue: "90" } - - - it: "adds a tokens-per-second Pods metric on its own" - set: - autoscaling.enabled: true - autoscaling.targetTokensPerSecond: 6M - asserts: - - lengthEqual: { path: spec.metrics, count: 2 } - - equal: - path: "spec.metrics[1]" - value: - type: Pods - pods: - metric: { name: litellm_tokens_per_second } - target: { type: AverageValue, averageValue: "6M" } - - notContains: - path: spec.metrics - content: { type: Pods, pods: { metric: { name: litellm_requests_per_second } } } - any: true - - - it: "renders requests, tokens, cpu and memory metrics together" - set: - autoscaling.enabled: true - autoscaling.targetMemoryUtilizationPercentage: 80 - autoscaling.targetRequestsPerSecond: 90 - autoscaling.targetTokensPerSecond: 6000000 - asserts: - - lengthEqual: { path: spec.metrics, count: 4 } - - equal: { path: "spec.metrics[0].resource.name", value: cpu } - - equal: { path: "spec.metrics[1].resource.name", value: memory } - - equal: { path: "spec.metrics[2].pods.metric.name", value: litellm_requests_per_second } - - equal: { path: "spec.metrics[2].pods.target.averageValue", value: "90" } - - equal: { path: "spec.metrics[3].pods.metric.name", value: litellm_tokens_per_second } - - equal: { path: "spec.metrics[3].pods.target.averageValue", value: "6000000" } - - - it: "scales on workload metrics alone when the cpu target is cleared" - set: - autoscaling.enabled: true - autoscaling.targetCPUUtilizationPercentage: null - autoscaling.targetRequestsPerSecond: 90 - autoscaling.targetTokensPerSecond: 6000000 - asserts: - - lengthEqual: { path: spec.metrics, count: 2 } - - notContains: { path: spec.metrics, content: { type: Resource }, any: true } - - equal: { path: "spec.metrics[0].pods.metric.name", value: litellm_requests_per_second } - - equal: { path: "spec.metrics[1].pods.metric.name", value: litellm_tokens_per_second } - - notMatchRegexRaw: { pattern: per_minute } - - - it: "ignores the per-minute keys, which the chart never shipped" - set: - autoscaling.enabled: true - autoscaling.targetRequestsPerMinute: 5400 - autoscaling.targetTokensPerMinute: 360000000 - asserts: - - lengthEqual: { path: spec.metrics, count: 1 } - - notContains: { path: spec.metrics, content: { type: Pods }, any: true } - - - it: "renders no hpa when autoscaling is disabled" - asserts: - - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/tests/ingress_tests.yaml b/helm/litellm-helm/tests/ingress_tests.yaml deleted file mode 100644 index aad6ecfcee8..00000000000 --- a/helm/litellm-helm/tests/ingress_tests.yaml +++ /dev/null @@ -1,45 +0,0 @@ -suite: Ingress Configuration Tests -templates: - - ingress.yaml -tests: - - it: should not create Ingress by default - asserts: - - hasDocuments: - count: 0 - - - it: should create Ingress when enabled - set: - ingress.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Ingress - - - it: should add custom labels - set: - ingress.enabled: true - ingress.labels: - custom-label: "true" - another-label: "value" - asserts: - - isKind: - of: Ingress - - equal: - path: metadata.labels.custom-label - value: "true" - - equal: - path: metadata.labels.another-label - value: "value" - - - it: should add annotations - set: - ingress.enabled: true - ingress.annotations: - kubernetes.io/ingress.class: "nginx" - asserts: - - isKind: - of: Ingress - - equal: - path: metadata.annotations["kubernetes.io/ingress.class"] - value: "nginx" diff --git a/helm/litellm-helm/tests/keda_tests.yaml b/helm/litellm-helm/tests/keda_tests.yaml deleted file mode 100644 index c9598646223..00000000000 --- a/helm/litellm-helm/tests/keda_tests.yaml +++ /dev/null @@ -1,106 +0,0 @@ -suite: "keda" -templates: - - keda.yaml -release: - name: rel - namespace: llm -tests: - - it: "renders no scaled object by default" - asserts: - - hasDocuments: { count: 0 } - - - it: "passes user triggers through and adds no prometheus triggers by default" - set: - keda.enabled: true - keda.triggers: - - type: cpu - metricType: Utilization - metadata: { value: "60" } - asserts: - - isKind: { of: ScaledObject } - - equal: - path: spec.triggers - value: - - type: cpu - metricType: Utilization - metadata: { value: "60" } - - - it: "scales on release-wide requests per second divided by the per-replica target" - set: - keda.enabled: true - keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090 - keda.prometheus.requestsPerSecond: 90 - asserts: - - lengthEqual: { path: spec.triggers, count: 1 } - - equal: - path: "spec.triggers[0]" - value: - type: prometheus - metadata: - serverAddress: http://prometheus-operated.monitoring.svc:9090 - threshold: "90" - query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm"}[1m])) - - - it: "scales on tokens per second on its own" - set: - keda.enabled: true - keda.prometheus.serverAddress: http://prom:9090 - keda.prometheus.tokensPerSecond: 6000000 - asserts: - - lengthEqual: { path: spec.triggers, count: 1 } - - equal: { path: "spec.triggers[0].type", value: prometheus } - - equal: { path: "spec.triggers[0].metadata.threshold", value: "6000000" } - - equal: - path: "spec.triggers[0].metadata.query" - value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm"}[1m])) - - - it: "appends requests and tokens triggers after user triggers and selects the metrics service job" - set: - keda.enabled: true - metricsServer.enabled: true - keda.triggers: - - type: cpu - metricType: Utilization - metadata: { value: "60" } - keda.prometheus.serverAddress: http://prom:9090 - keda.prometheus.requestsPerSecond: 90 - keda.prometheus.tokensPerSecond: 6000000 - asserts: - - lengthEqual: { path: spec.triggers, count: 3 } - - equal: { path: "spec.triggers[0].type", value: cpu } - - equal: { path: "spec.triggers[1].metadata.threshold", value: "90" } - - equal: - path: "spec.triggers[1].metadata.query" - value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) - - equal: { path: "spec.triggers[2].metadata.threshold", value: "6000000" } - - equal: - path: "spec.triggers[2].metadata.query" - value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) - - notMatchRegexRaw: { pattern: "\\* *60|per_minute|PerMinute" } - - - it: "ignores the per-minute keys, which the chart never shipped" - set: - keda.enabled: true - keda.prometheus.serverAddress: http://prom:9090 - keda.prometheus.requestsPerMinute: 5400 - keda.prometheus.tokensPerMinute: 360000000 - asserts: - - isKind: { of: ScaledObject } - - isNullOrEmpty: { path: spec.triggers } - - - it: "refuses a workload target without a prometheus server address" - set: - keda.enabled: true - keda.prometheus.requestsPerSecond: 90 - asserts: - - failedTemplate: - errorMessage: keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set - - - it: "yields to the hpa when both autoscalers are enabled" - set: - autoscaling.enabled: true - keda.enabled: true - keda.prometheus.serverAddress: http://prom:9090 - keda.prometheus.requestsPerSecond: 90 - asserts: - - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/tests/lens_connection_tests.yaml b/helm/litellm-helm/tests/lens_connection_tests.yaml deleted file mode 100644 index e01e49d0ffc..00000000000 --- a/helm/litellm-helm/tests/lens_connection_tests.yaml +++ /dev/null @@ -1,128 +0,0 @@ -suite: lens connection -templates: - - deployment.yaml - - configmap-litellm.yaml - - ingress.yaml -tests: - - it: connects the proxy to an external Lens - template: deployment.yaml - set: - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.publicUrl: https://lens.example.com/lens-ingest - lens.gatewaySecret.name: lens-gateway - lens.serviceTokenSecret.name: lens-service - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens.lens.svc:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://lens.example.com/lens-ingest - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: lens-gateway - key: gateway-secret - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_SERVICE_TOKEN - valueFrom: - secretKeyRef: - name: lens-service - key: service-token - - - it: requires lens.url when Lens is enabled - template: deployment.yaml - set: - lens.enabled: true - lens.publicUrl: https://lens.example.com/lens-ingest - lens.gatewaySecret.name: lens-gateway - lens.serviceTokenSecret.name: lens-service - asserts: - - failedTemplate: - errorMessage: lens.url is required when lens.enabled is true - - - it: requires lens.publicUrl when Lens is enabled - template: deployment.yaml - set: - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.gatewaySecret.name: lens-gateway - lens.serviceTokenSecret.name: lens-service - asserts: - - failedTemplate: - errorMessage: lens.publicUrl is required when lens.enabled is true - - - it: requires lens.gatewaySecret.name when Lens is enabled - template: deployment.yaml - set: - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.publicUrl: https://lens.example.com/lens-ingest - lens.serviceTokenSecret.name: lens-service - asserts: - - failedTemplate: - errorMessage: lens.gatewaySecret.name is required when lens.enabled is true - - - it: requires lens.serviceTokenSecret.name when Lens is enabled - template: deployment.yaml - set: - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.publicUrl: https://lens.example.com/lens-ingest - lens.gatewaySecret.name: lens-gateway - asserts: - - failedTemplate: - errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true - - - it: routes /lens-ingest to the named Lens service ahead of the host paths - template: ingress.yaml - set: - ingress.enabled: true - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.publicUrl: https://lens.example.com/lens-ingest - lens.gatewaySecret.name: lens-gateway - lens.serviceTokenSecret.name: lens-service - lens.service.name: existing-lens - asserts: - - equal: - path: spec.rules[0].http.paths[0].path - value: /lens-ingest - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: existing-lens - port: - number: 4318 - - - it: leaves /lens-ingest off the ingress without a Lens service name - template: ingress.yaml - set: - ingress.enabled: true - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.publicUrl: https://lens.example.com/lens-ingest - lens.gatewaySecret.name: lens-gateway - lens.serviceTokenSecret.name: lens-service - asserts: - - exists: - path: spec.rules[0].http.paths - - notContains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - any: true diff --git a/helm/litellm-helm/tests/masterkey-secret_tests.yaml b/helm/litellm-helm/tests/masterkey-secret_tests.yaml deleted file mode 100644 index 296f26755b8..00000000000 --- a/helm/litellm-helm/tests/masterkey-secret_tests.yaml +++ /dev/null @@ -1,71 +0,0 @@ -suite: test masterkey secret -templates: - - secret-masterkey.yaml -tests: - - it: should create a secret if masterkeySecretName is not set. should start with sk-xxxx (base64 encoded as c2st*) - template: secret-masterkey.yaml - set: - masterkeySecretName: "" - asserts: - - isKind: - of: Secret - - matchRegex: - path: data.masterkey - pattern: ^c2st - # Note: The masterkey is generated as "sk-<18-random-chars>" in plain text, - # but stored as base64 encoded in Kubernetes secret (requirement). - # "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern. - - it: should reuse the master key already stored in the cluster instead of generating a new one on upgrade - template: secret-masterkey.yaml - set: - masterkeySecretName: "" - kubernetesProvider: - scheme: - "v1/Secret": - gvr: - version: "v1" - resource: "secrets" - namespaced: true - objects: - - kind: Secret - apiVersion: v1 - metadata: - name: RELEASE-NAME-litellm-masterkey - namespace: NAMESPACE - data: - masterkey: c2stZXhpc3Rpbmcta2V5 - asserts: - - equal: - path: data.masterkey - value: c2stZXhpc3Rpbmcta2V5 - - it: should let an explicit masterkey value override the one already stored in the cluster - template: secret-masterkey.yaml - set: - masterkeySecretName: "" - masterkey: sk-explicit - kubernetesProvider: - scheme: - "v1/Secret": - gvr: - version: "v1" - resource: "secrets" - namespaced: true - objects: - - kind: Secret - apiVersion: v1 - metadata: - name: RELEASE-NAME-litellm-masterkey - namespace: NAMESPACE - data: - masterkey: c2stZXhpc3Rpbmcta2V5 - asserts: - - equal: - path: data.masterkey - value: c2stZXhwbGljaXQ= - - it: should not create a secret if masterkeySecretName is set - template: secret-masterkey.yaml - set: - masterkeySecretName: my-secret - asserts: - - hasDocuments: - count: 0 diff --git a/helm/litellm-helm/tests/metrics_server_tests.yaml b/helm/litellm-helm/tests/metrics_server_tests.yaml deleted file mode 100644 index 085d69ac640..00000000000 --- a/helm/litellm-helm/tests/metrics_server_tests.yaml +++ /dev/null @@ -1,106 +0,0 @@ -suite: separate metrics server -templates: - - configmap-litellm.yaml - - deployment.yaml - - service.yaml - - service-metrics.yaml - - servicemonitor.yaml -tests: - - it: should not expose a metrics port or PROMETHEUS_METRICS_PORT by default - asserts: - - notContains: - path: spec.template.spec.containers[0].ports - content: - name: metrics - any: true - template: deployment.yaml - - notContains: - path: spec.template.spec.containers[0].env - content: - name: PROMETHEUS_METRICS_PORT - any: true - template: deployment.yaml - - lengthEqual: - path: spec.ports - count: 1 - template: service.yaml - - hasDocuments: - count: 0 - template: service-metrics.yaml - - - it: should scrape the proxy port when the metrics server is disabled - template: servicemonitor.yaml - set: - serviceMonitor.enabled: true - asserts: - - equal: - path: spec.endpoints[0].port - value: http - - - it: should wire the separate metrics server through container, a ClusterIP metrics service and servicemonitor - set: - metricsServer.enabled: true - metricsServer.port: 4101 - serviceMonitor.enabled: true - service.type: LoadBalancer - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PROMETHEUS_METRICS_PORT - value: "4101" - template: deployment.yaml - - contains: - path: spec.template.spec.containers[0].ports - content: - name: metrics - containerPort: 4101 - protocol: TCP - template: deployment.yaml - - lengthEqual: - path: spec.ports - count: 1 - template: service.yaml - - equal: - path: spec.type - value: LoadBalancer - template: service.yaml - - equal: - path: metadata.name - value: RELEASE-NAME-litellm-metrics - template: service-metrics.yaml - - equal: - path: spec.type - value: ClusterIP - template: service-metrics.yaml - - equal: - path: spec.ports - value: - - port: 4101 - targetPort: metrics - protocol: TCP - name: metrics - template: service-metrics.yaml - - equal: - path: spec.selector - value: - app.kubernetes.io/name: litellm - app.kubernetes.io/instance: RELEASE-NAME - template: service-metrics.yaml - - equal: - path: spec.endpoints[0].port - value: metrics - template: servicemonitor.yaml - - equal: - path: spec.endpoints[0].path - value: /metrics/ - template: servicemonitor.yaml - - - it: should reject a metrics port equal to the proxy port - template: deployment.yaml - set: - metricsServer.enabled: true - metricsServer.port: 4000 - asserts: - - failedTemplate: - errorMessage: metricsServer.port must differ from service.port diff --git a/helm/litellm-helm/tests/migrations-job_tests.yaml b/helm/litellm-helm/tests/migrations-job_tests.yaml deleted file mode 100644 index bd5dd457f95..00000000000 --- a/helm/litellm-helm/tests/migrations-job_tests.yaml +++ /dev/null @@ -1,432 +0,0 @@ -suite: test migrations job -templates: - - migrations-job.yaml -tests: - - it: should work with envVars - template: migrations-job.yaml - set: - envVars: - TEST_ENV_VAR: "test_value" - ANOTHER_VAR: "another_value" - migrationJob: - enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: TEST_ENV_VAR - value: "test_value" - - contains: - path: spec.template.spec.containers[0].env - content: - name: ANOTHER_VAR - value: "another_value" - - - it: should work with extraEnvVars - template: migrations-job.yaml - set: - extraEnvVars: - - name: EXTRA_ENV_VAR - valueFrom: - fieldRef: - fieldPath: metadata.labels['env'] - - name: SIMPLE_EXTRA_VAR - value: "simple_value" - migrationJob: - enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: EXTRA_ENV_VAR - valueFrom: - fieldRef: - fieldPath: metadata.labels['env'] - - contains: - path: spec.template.spec.containers[0].env - content: - name: SIMPLE_EXTRA_VAR - value: "simple_value" - - - it: should work with both envVars and extraEnvVars - template: migrations-job.yaml - set: - envVars: - ENV_VAR: "env_var_value" - extraEnvVars: - - name: EXTRA_ENV_VAR - value: "extra_env_var_value" - migrationJob: - enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: ENV_VAR - value: "env_var_value" - - contains: - path: spec.template.spec.containers[0].env - content: - name: EXTRA_ENV_VAR - value: "extra_env_var_value" - - - it: should not render when migrations job is disabled - template: migrations-job.yaml - set: - migrationJob: - enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should still include default env vars - template: migrations-job.yaml - set: - envVars: - CUSTOM_VAR: "custom_value" - migrationJob: - enabled: true - db: - useExisting: true - endpoint: "test-db" - database: "testdb" - url: "postgresql://user:pass@test-db:5432/testdb" - secret: - name: "test-secret" - usernameKey: "username" - passwordKey: "password" - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DISABLE_SCHEMA_UPDATE - value: "false" - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_HOST - value: "test-db" - - contains: - path: spec.template.spec.containers[0].env - content: - name: CUSTOM_VAR - value: "custom_value" - - - it: should override a user-supplied DISABLE_SCHEMA_UPDATE so the Job always migrates - template: migrations-job.yaml - set: - envVars: - DISABLE_SCHEMA_UPDATE: "true" - migrationJob: - enabled: true - asserts: - # The Job is what owns the schema, so it renders its own - # DISABLE_SCHEMA_UPDATE=false after envVars and extraEnvVars. Kubernetes - # takes the last value for a duplicated name, so the user's "true" cannot - # leave the schema unmigrated. Skipping migrations is migrationJob.enabled. - - equal: - path: spec.template.spec.containers[0].env[-1] - value: - name: DISABLE_SCHEMA_UPDATE - value: "false" - - - it: should not include DATABASE_URL when deployStandalone is false - template: migrations-job.yaml - set: - migrationJob: - enabled: true - db: - deployStandalone: false - useExisting: false - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - - - it: should use default service account for helm hooks when serviceAccount.create is true - template: migrations-job.yaml - set: - migrationJob: - enabled: true - hooks: - helm: - enabled: true - serviceAccount: - create: true - asserts: - - equal: - path: spec.template.spec.serviceAccountName - value: default - - - it: should use migrationJob.serviceAccountName override for helm hooks when serviceAccount.create is true - template: migrations-job.yaml - set: - migrationJob: - enabled: true - serviceAccountName: migration-sa - hooks: - helm: - enabled: true - serviceAccount: - create: true - asserts: - - equal: - path: spec.template.spec.serviceAccountName - value: migration-sa - - - it: should use chart service account when helm hooks are disabled - template: migrations-job.yaml - set: - migrationJob: - enabled: true - hooks: - helm: - enabled: false - serviceAccount: - create: true - name: my-custom-sa - asserts: - - equal: - path: spec.template.spec.serviceAccountName - value: my-custom-sa - - - it: should use pre-existing service account when helm hooks are enabled but serviceAccount.create is false - template: migrations-job.yaml - set: - migrationJob: - enabled: true - hooks: - helm: - enabled: true - serviceAccount: - create: false - name: pre-existing-sa - asserts: - - equal: - path: spec.template.spec.serviceAccountName - value: pre-existing-sa - - it: should work with extraInitContainers - template: migrations-job.yaml - set: - migrationJob: - enabled: true - extraInitContainers: - - name: init-test - image: busybox:latest - command: ["echo", "hello"] - asserts: - - contains: - path: spec.template.spec.initContainers - content: - name: init-test - image: busybox:latest - command: ["echo", "hello"] - - it: should support tpl in extraInitContainers - template: migrations-job.yaml - set: - image: - repository: ghcr.io/berriai/litellm - tag: test - migrationJob: - enabled: true - extraInitContainers: - - name: init-tpl - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - command: ["echo", "hello"] - asserts: - - contains: - path: spec.template.spec.initContainers - content: - name: init-tpl - image: "ghcr.io/berriai/litellm:test" - command: ["echo", "hello"] - - it: should work with extraContainers - template: migrations-job.yaml - set: - migrationJob: - enabled: true - extraContainers: - - name: sidecar - image: busybox:latest - asserts: - - contains: - path: spec.template.spec.containers - content: - name: sidecar - image: busybox:latest - - it: should support tpl in extraContainers - template: migrations-job.yaml - set: - image: - repository: ghcr.io/berriai/litellm - tag: test - migrationJob: - enabled: true - extraContainers: - - name: sidecar-tpl - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - asserts: - - contains: - path: spec.template.spec.containers - content: - name: sidecar-tpl - image: "ghcr.io/berriai/litellm:test" - - it: should render the pod-level securityContext from podSecurityContext - template: migrations-job.yaml - set: - migrationJob: - enabled: true - podSecurityContext: - fsGroup: 10000 - runAsUser: 10000 - runAsNonRoot: true - asserts: - - equal: - path: spec.template.spec.securityContext - value: - fsGroup: 10000 - runAsUser: 10000 - runAsNonRoot: true - - it: should keep the pod-level and container-level securityContext separate - template: migrations-job.yaml - set: - migrationJob: - enabled: true - podSecurityContext: - fsGroup: 10000 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - asserts: - - equal: - path: spec.template.spec.securityContext - value: - fsGroup: 10000 - - equal: - path: spec.template.spec.containers[0].securityContext - value: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - - it: should schedule onto the same nodes as the gateway - template: migrations-job.yaml - set: - migrationJob: - enabled: true - nodeSelector: - karpenter.sh/nodepool: litellm-e2e - tolerations: - - key: workload - operator: Equal - value: litellm-e2e - effect: NoSchedule - asserts: - - equal: - path: spec.template.spec.nodeSelector - value: - karpenter.sh/nodepool: litellm-e2e - - equal: - path: spec.template.spec.tolerations - value: - - key: workload - operator: Equal - value: litellm-e2e - effect: NoSchedule - - - it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever - set: - migrationJob: - enabled: true - asserts: - - equal: - path: spec.activeDeadlineSeconds - value: 1800 - - - it: honours an operator-supplied deadline - set: - migrationJob: - enabled: true - activeDeadlineSeconds: 600 - asserts: - - equal: - path: spec.activeDeadlineSeconds - value: 600 - - - it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour - set: - migrationJob: - enabled: true - activeDeadlineSeconds: null - asserts: - - notExists: - path: spec.activeDeadlineSeconds - - - it: should set custom jobLabels and jobAnnotations on Job metadata - template: migrations-job.yaml - set: - migrationJob: - enabled: true - jobLabels: - environment: production - team: platform - jobAnnotations: - example.com/cost-center: "1234" - asserts: - - equal: - path: metadata.labels.environment - value: production - - equal: - path: metadata.labels.team - value: platform - - equal: - path: metadata.annotations['example.com/cost-center'] - value: "1234" - - - it: should set custom podLabels on Pod template - template: migrations-job.yaml - set: - migrationJob: - enabled: true - podLabels: - custom.io/pod-role: migration - asserts: - - equal: - path: spec.template.metadata.labels['custom.io/pod-role'] - value: migration - - - it: should override container command and args - template: migrations-job.yaml - set: - migrationJob: - enabled: true - command: - - sh - args: - - -c - - echo migrating - asserts: - - equal: - path: spec.template.spec.containers[0].command - value: - - sh - - equal: - path: spec.template.spec.containers[0].args - value: - - -c - - echo migrating - - - it: should clear container args when only command is specified - template: migrations-job.yaml - set: - migrationJob: - enabled: true - command: - - sh - asserts: - - equal: - path: spec.template.spec.containers[0].command - value: - - sh - - equal: - path: spec.template.spec.containers[0].args - value: [] diff --git a/helm/litellm-helm/tests/pdb_tests.yaml b/helm/litellm-helm/tests/pdb_tests.yaml deleted file mode 100644 index 5e042e80bd3..00000000000 --- a/helm/litellm-helm/tests/pdb_tests.yaml +++ /dev/null @@ -1,45 +0,0 @@ -suite: "pdb enabled" -templates: - - poddisruptionbudget.yaml -tests: - - it: "renders a PDB with maxUnavailable=1" - set: - pdb.enabled: true - pdb.maxUnavailable: 1 - asserts: - - hasDocuments: { count: 1 } - - isKind: { of: PodDisruptionBudget } - - equal: { path: apiVersion, value: policy/v1 } - - equal: { path: spec.maxUnavailable, value: 1 } - - equal: - path: spec.selector.matchLabels - value: - app.kubernetes.io/name: litellm - app.kubernetes.io/instance: RELEASE-NAME - ---- -suite: "pdb disabled" -templates: - - poddisruptionbudget.yaml -tests: - - it: "does not render when disabled" - set: - pdb.enabled: false - asserts: - - hasDocuments: { count: 0 } - ---- -suite: "pdb minAvailable precedence" -templates: - - poddisruptionbudget.yaml -tests: - - it: "uses minAvailable when both are set" - set: - pdb.enabled: true - pdb.minAvailable: "50%" - pdb.maxUnavailable: 1 - asserts: - - isKind: { of: PodDisruptionBudget } - - equal: { path: apiVersion, value: policy/v1 } - - equal: { path: spec.minAvailable, value: "50%" } - - isNull: { path: spec.maxUnavailable } diff --git a/helm/litellm-helm/tests/service_tests.yaml b/helm/litellm-helm/tests/service_tests.yaml deleted file mode 100644 index 43ed0180bc8..00000000000 --- a/helm/litellm-helm/tests/service_tests.yaml +++ /dev/null @@ -1,116 +0,0 @@ -suite: Service Configuration Tests -templates: - - service.yaml -tests: - - it: should create a default ClusterIP service - template: service.yaml - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: ClusterIP - - equal: - path: spec.ports[0].port - value: 4000 - - equal: - path: spec.ports[0].targetPort - value: http - - equal: - path: spec.ports[0].protocol - value: TCP - - equal: - path: spec.ports[0].name - value: http - - isNull: - path: spec.loadBalancerClass - - - it: should create a NodePort service when specified - template: service.yaml - set: - service.type: NodePort - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: NodePort - - isNull: - path: spec.loadBalancerClass - - - it: should create a LoadBalancer service when specified - template: service.yaml - set: - service.type: LoadBalancer - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: LoadBalancer - - isNull: - path: spec.loadBalancerClass - - - it: should add loadBalancerClass when specified with LoadBalancer type - template: service.yaml - set: - service.type: LoadBalancer - service.loadBalancerClass: tailscale - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: LoadBalancer - - equal: - path: spec.loadBalancerClass - value: tailscale - - - it: should not add loadBalancerClass when specified with ClusterIP type - template: service.yaml - set: - service.type: ClusterIP - service.loadBalancerClass: tailscale - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: ClusterIP - - isNull: - path: spec.loadBalancerClass - - - it: should use custom port when specified - template: service.yaml - set: - service.port: 8080 - asserts: - - equal: - path: spec.ports[0].port - value: 8080 - - - it: should add service annotations when specified - template: service.yaml - set: - service.annotations: - cloud.google.com/load-balancer-type: "Internal" - service.beta.kubernetes.io/aws-load-balancer-internal: "true" - asserts: - - isKind: - of: Service - - equal: - path: metadata.annotations - value: - cloud.google.com/load-balancer-type: "Internal" - service.beta.kubernetes.io/aws-load-balancer-internal: "true" - - - it: should use the correct selector labels - template: service.yaml - asserts: - - isNotNull: - path: spec.selector - - equal: - path: spec.selector - value: - app.kubernetes.io/name: litellm - app.kubernetes.io/instance: RELEASE-NAME diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml deleted file mode 100644 index 9afce050261..00000000000 --- a/helm/litellm-helm/values.yaml +++ /dev/null @@ -1,676 +0,0 @@ -# Default values for litellm. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 -liteadmin: - enabled: false - existingSecret: "" - gatewayUrl: "" - model: "" - readOnly: false - storageSize: 1Gi - storageClassName: "" - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 1Gi -# numWorkers: 2 - -image: - # Bundles the prisma CLI and engines, which is what lets the migrations job - # and the proxy's own schema check run without network access. - repository: ghcr.io/berriai/litellm - pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - # tag: "latest" - tag: "" - -imagePullSecrets: [] -nameOverride: "litellm" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -# annotations for litellm deployment -deploymentAnnotations: {} -deploymentLabels: {} -deploymentMinReadySeconds: 0 - -# annotations for litellm pods -podAnnotations: {} -podLabels: {} - -# -- Deployment strategy configuration -# Example: -# type: RollingUpdate -# rollingUpdate: -# maxUnavailable: 0 -# maxSurge: 1 -strategy: {} - -terminationGracePeriodSeconds: 90 -topologySpreadConstraints: - [] - # - maxSkew: 1 - # topologyKey: kubernetes.io/hostname - # whenUnsatisfiable: DoNotSchedule - # labelSelector: - # matchLabels: - # app: litellm - -# At the time of writing, the litellm docker image requires write access to the -# filesystem on startup so that prisma can install some dependencies. -podSecurityContext: {} -securityContext: - {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: false - # runAsNonRoot: true - # runAsUser: 1000 - -# A list of Kubernetes Secret objects that will be exported to the LiteLLM proxy -# pod as environment variables. These secrets can then be referenced in the -# configuration file (or "litellm" ConfigMap) with `os.environ/` -environmentSecrets: - [] - # - litellm-env-secret - -# A list of Kubernetes ConfigMap objects that will be exported to the LiteLLM proxy -# pod as environment variables. The ConfigMap kv-pairs can then be referenced in the -# configuration file (or "litellm" ConfigMap) with `os.environ/` -environmentConfigMaps: - [] - # - litellm-env-configmap - -service: - type: ClusterIP - port: 4000 - # If service type is `LoadBalancer` you can - # optionally specify loadBalancerClass - # loadBalancerClass: tailscale - -# Probes for LiteLLM gateway container -livenessProbe: - path: /health/liveliness - initialDelaySeconds: 0 - periodSeconds: 15 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - -readinessProbe: - path: /health/readiness - initialDelaySeconds: 0 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - -startupProbe: - path: /health/readiness - initialDelaySeconds: 0 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 30 - -ingress: - enabled: false - className: "nginx" - labels: {} - annotations: - {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: api.example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -# masterkey: changeit - -# if set, use this secret for the master key; otherwise, autogenerate a new one -masterkeySecretName: "" - -# if set, use this secret key for the master key; otherwise, use the default key -masterkeySecretKey: "" - -# Optional: enterprise billable-request metering. Requires an enterprise license. -billingMetrics: - enabled: false - endpoint: https://telemetry.litellm.ai - tokenUrl: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token - secretName: litellm-billing-metrics - exportIntervalMs: "" - -proxyConfigMap: - # when true, creates a new configmap - create: true - # if create is false and name is set, use existing ConfigMap - # create: false - # name: "" - # key: "config.yaml" - -# The elements within proxy_config are rendered as config.yaml for the proxy -# Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml -# Reference: https://docs.litellm.ai/docs/proxy/configs -proxy_config: - model_list: [] - general_settings: - master_key: os.environ/PROXY_MASTER_KEY - -# Serve Prometheus /metrics from a separate process (PROMETHEUS_METRICS_PORT) -# so a scrape never runs on an inference worker. Adds a `metrics` port to the -# container and a dedicated ClusterIP `-metrics` Service, and the -# ServiceMonitor scrapes it instead of the proxy port. The separate port has -# no virtual-key auth: keep it off public ingress. Needs the proxy image -# v1.101.0 or newer. -metricsServer: - enabled: false - port: 4001 - -# Opt-in sidecar that runs the post-response spend pipeline (cost calculation, -# spend logs, spend counters, budget reservation reconciliation) so the proxy's -# uvicorn workers only serialise a compact typed event and go back to serving -# inference. Same image and tag as the proxy, second container in the same pod, -# fed over loopback (a unix socket on a shared emptyDir, or 127.0.0.1 TCP). It -# reuses the pod's in-container pgbouncer (db.connectionPool) and the same Redis -# spend transaction buffer, so the per-pod DB connection budget is unchanged. -# Delivery is at-most-once inside the pod: events already handed to the sidecar -# are lost if it crashes before writing them; events the workers could not hand -# over follow onUnavailable. Both containers drain on SIGTERM within -# terminationGracePeriodSeconds -collector: - enabled: false - # unix:////.sock (the becomes a shared emptyDir) or tcp://127.0.0.1: - address: unix:///var/run/litellm/collector.sock - # Events each uvicorn worker holds in memory while the sidecar is slow or restarting - bufferSize: 1000 - # fallback: run the pipeline in the worker when the sidecar is unreachable or the - # buffer is full (spend stays exact, that request costs proxy CPU again) - # drop: count and discard the event instead (spend under-reports) - onUnavailable: fallback - # How long the workers keep pushing buffered events on shutdown, and how long the - # sidecar keeps serving its open connections after SIGTERM - drainTimeoutSeconds: 10 - command: - - python - - -m - - litellm.proxy.collector - # Sized independently of the proxy container; the pipeline is CPU bound - resources: {} - # requests: - # cpu: 500m - # memory: 1Gi - # limits: - # cpu: "1" - # memory: 2Gi - # When autoscaling.enabled, swap the pod-wide cpu Resource metric for an - # autoscaling/v2 ContainerResource metric on the proxy container only, so the - # sidecar's CPU never scales inference replicas. Needs Kubernetes 1.30+ (or the - # HPAContainerMetrics feature gate on 1.27 to 1.29) - scaleOnProxyContainerCpu: false - -resources: - {} - # Unset by default so the chart installs on small clusters such as Minikube, and so an - # upgrade never leaves a running pod Pending. Production deployments should set these. - # A proxy at DB-connected steady state needs about 1 CPU and 4Gi of memory per worker; - # sizing below that gets the pod OOMKilled once traffic and DB connections ramp up. - # Scale both figures with --num_workers, then uncomment the lines below and remove the - # curly braces after 'resources:'. See "Recommended Machine Specifications" in - # https://docs.litellm.ai/docs/proxy/prod. - # requests: - # cpu: "1" - # memory: 4Gi - # limits: - # cpu: "1" - # memory: 4Gi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - # 60 is the documented recommendation. See "Recommended Machine Specifications" - # in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe - # above only after up to failureThreshold x periodSeconds = 300 seconds, so a target - # high enough to trip near saturation adds capacity minutes after it was needed. - targetCPUUtilizationPercentage: 60 - # Deliberately left unset rather than given a value. The prisma query engine's - # resident memory is a high-water mark that ratchets to the pod's worst-ever write - # and is never returned, so a memory target reads the largest write a pod ever did - # rather than what it is doing now, and replicas ratchet up without scaling back in. - # Memory is a floor to provision under 'resources', not a signal to scale on. - # targetMemoryUtilizationPercentage: 80 - # behavior: {} - # Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics - # named `litellm_requests_per_second` and `litellm_tokens_per_second` with an - # AverageValue target, alongside whichever resource targets are set (the HPA - # follows the metric asking for the most replicas). A Prometheus Adapter must - # serve those two names on custom.metrics.k8s.io from the proxy's counters, - # grouped by the scrape target's `pod` label (enable serviceMonitor below so - # every pod is scraped on its own): - # litellm_requests_per_second: - # sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) - # litellm_tokens_per_second: - # sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) - # rate() over [1m] is already per second, so no `* 60`. How fast the HPA - # reacts is set by that window, the scrape interval and the HPA sync period - # (15s by default), not by the unit: keep serviceMonitor.interval at 15s or - # faster so a 1m window holds at least 4 samples. averageValue takes SI - # suffixes, so "6M" is six million tokens per second per pod. Tokens are - # counted when a response completes, so TPS trails long streams. - targetRequestsPerSecond: "" - targetTokensPerSecond: "" - -# Autoscaling with keda is mutually exclusive with hpa -keda: - enabled: false - minReplicas: 1 - maxReplicas: 100 - pollingInterval: 30 - cooldownPeriod: 300 - # fallback: - # failureThreshold: 3 - # replicas: 11 - restoreToOriginalReplicaCount: false - scaledObject: - annotations: {} - triggers: [] - # - type: prometheus - # metadata: - # serverAddress: http://:9090 - # metricName: http_requests_total - # threshold: '100' - # query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) - # First-class Prometheus triggers on the proxy's own request and token - # counters, appended to `triggers`. Each target is the per-second load one - # replica should carry: KEDA divides the release-wide - # `sum(rate([1m]))` by it to pick the replica count. Thresholds - # are plain numbers (KEDA parses them as floats, no SI suffixes). The - # queries select samples by the release namespace and the `job` label the - # chart's ServiceMonitor produces (the metrics Service name), so enable - # serviceMonitor below together with metricsServer: the http port serves - # /metrics/ behind virtual-key auth and answers an unauthenticated scrape - # with 401. Reaction time comes from the [1m] window, the scrape interval - # and pollingInterval above, so keep both at 15s or faster. Tokens are - # counted at completion, so TPS trails long streams. serverAddress is - # required once either target is set. - prometheus: - serverAddress: "" - requestsPerSecond: "" - tokensPerSecond: "" - behavior: {} - # scaleDown: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 1 - # periodSeconds: 180 - # scaleUp: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 2 - # periodSeconds: 60 - -# Additional volumes on the output Deployment definition. -volumes: [] -# - name: foo -# secret: -# secretName: mysecret -# optional: false - -# Additional volumeMounts on the output Deployment definition. -volumeMounts: [] -# - name: foo -# mountPath: "/etc/foo" -# readOnly: true - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -db: - # Use an existing postgres server/cluster - useExisting: false - - # How to connect to the existing postgres server/cluster - endpoint: localhost - database: litellm - url: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME) - # Authenticate with a short-lived token instead of a password. useIAMAuth is - # AWS RDS / Aurora IAM database authentication (the pod needs AWS credentials, - # e.g. IRSA or EKS Pod Identity, and the database user needs GRANT rds_iam); - # useAzureEntraAuth is Azure Database for PostgreSQL with a Microsoft Entra ID - # identity (workload identity). Mutually exclusive, and both need - # useExisting: true with deployStandalone: false. The chart then sets - # IAM_TOKEN_DB_AUTH or AZURE_POSTGRESQL_AUTH, emits DATABASE_HOST, DATABASE_PORT, - # DATABASE_USERNAME and DATABASE_NAME and skips DATABASE_PASSWORD and db.url: - # the proxy, the migrations job and the collector sidecar each mint their own - # token and build DATABASE_URL from those parts. Set DATABASE_SSLMODE / - # DATABASE_SSLROOTCERT under envVars for TLS options you would otherwise put - # in db.url. - useIAMAuth: false - useAzureEntraAuth: false - # Postgres port, used to build the token-auth URL (password auth takes the - # port from db.url) - port: 5432 - # Postgres schema for the LiteLLM tables, rendered as DATABASE_SCHEMA under - # token auth only (password auth carries it as ?schema= in db.url) - schema: "" - # Database user as a plain value. Empty means read it from - # db.secret.usernameKey. With token auth the user name is the only credential - # the chart needs, so setting it here removes the need for db.secret entirely. - username: "" - secret: - name: postgres - usernameKey: username - passwordKey: password - # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint - endpointKey: "" - # Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this - # secret key instead of db.readReplicaUrl. Prefer this over the plain - # value: read-replica URLs typically embed credentials, and a value - # written to db.readReplicaUrl ends up visible in the rendered pod spec - # and the Helm release secret. - readReplicaUrlKey: "" - # Optional: when set, a DATABASE_READER_HOST env var is sourced from this - # secret key, so db.readReplicaUrl can compose the reader URL from - # individual secret components, e.g. - # postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME) - # Use this when your secret store holds the bare reader hostname rather - # than a full connection URL. Only takes effect when readReplicaUrl is - # set; ignored when readReplicaUrlKey is set. - readReplicaEndpointKey: "" - - # Optional read-replica routing. When set, the proxy sends read-only - # queries (find_*, count, group_by, query_raw/_first) to this URL while - # writes continue to go to db.url. Useful for Aurora-style clusters with - # separate reader/writer endpoints. Leave empty to keep single-DB behavior. - # When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed - # alongside the writer (host/port/user/db are parsed from this URL once - # at startup; only the IAM token rotates). - # - # If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over - # this field — the plain value is rendered into the pod spec and the - # Helm release secret. This field is intended for credential-less URLs - # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime). - readReplicaUrl: "" - - # In-container connection pool (PgBouncer, transaction mode) shared by every - # worker in the pod. Without it each --num_workers worker opens its own - # connection_limit connections to Postgres, so a pod's footprint against the - # database's connection ceiling is workers x connection_limit and grows with - # every replica. With it, the pod holds at most maxDbConnections upstream - # connections no matter how many workers run; the workers connect to the pool - # over loopback, with no extra network hop. Migrations still go straight to - # Postgres. Starting profile for numWorkers: 4 is maxDbConnections: 20, so - # a database with a 5000-connection ceiling fits roughly 200 replicas. - connectionPool: - enabled: false - maxDbConnections: 20 - maxClientConn: 1000 - - # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster. - # The Stackgres Operator must already be installed within the target - # Kubernetes cluster. - # TODO: Stackgres deployment currently unsupported - useStackgresOperator: false - - # Use the Postgres Helm chart to create a single node, stand alone postgres - # instance. See the "postgresql" top level key for additional configuration. - deployStandalone: true - -# Lifecycle hooks for the LiteLLM container -# -# Prefer the native /health/drain preStop hook over a fixed `sleep`: it marks -# the pod NotReady and blocks only until in-flight requests actually finish -# (bounded by GRACEFUL_SHUTDOWN_TIMEOUT, default 30s), instead of always -# waiting the worst-case duration. After SIGTERM the server waits up to -# GRACEFUL_SHUTDOWN_TIMEOUT again for connections still open and then cancels -# them, so set terminationGracePeriodSeconds a few seconds above twice -# GRACEFUL_SHUTDOWN_TIMEOUT to leave room for teardown before SIGKILL. -# -# /health/drain is off by default; enable it with -# general_settings.enable_drain_endpoint: true. The kubelet calls preStop -# hooks without proxy credentials, so when the health port is reachable from -# other pods (the common case) also set -# general_settings.drain_endpoint_token (or the DRAIN_ENDPOINT_TOKEN env -# var) and send the same value on the X-Drain-Token header from the hook. -# Calls missing/wrong the token get a 401 and have no side effect. -# Example: -# lifecycle: -# preStop: -# httpGet: -# path: /health/drain -# port: 4000 -# httpHeaders: -# - name: X-Drain-Token -# value: -lifecycle: {} - -# Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored -# otherwise) -# -# Bitnami retired the versioned tags under docker.io/bitnami and republished the -# archived builds under docker.io/bitnamilegacy, so the subchart's own image -# defaults no longer resolve. The repository below points at the same build the -# subchart was released with, which keeps the on-disk data directory layout -# identical for existing installs. -# -# Keep the tag pinned. docker.io/bitnami still publishes a floating `latest`, -# and starting a newer PostgreSQL major against an existing data directory -# leaves the server refusing to boot ("database files are incompatible with -# server") with no way back other than a dump taken beforehand. Crossing a major -# version is a dump-and-restore, not an image bump. The chart refuses to render -# an unpinned tag for this reason -postgresql: - architecture: standalone - image: - repository: bitnamilegacy/postgresql - tag: 16.2.0-debian-12-r6 - volumePermissions: - image: - repository: bitnamilegacy/os-shell - tag: 12-debian-12-r16 - metrics: - image: - repository: bitnamilegacy/postgres-exporter - tag: 0.15.0-debian-12-r14 - auth: - username: litellm - database: litellm - - # You should override these on the helm command line with - # `--set postgresql.auth.postgres-password=,postgresql.auth.password=` - password: NoTaGrEaTpAsSwOrD - postgres-password: NoTaGrEaTpAsSwOrD - - # A secret is created by this chart (litellm-helm) with the credentials that - # the new Postgres instance should use. - # existingSecret: "" - # secretKeys: - # userPasswordKey: password - -# Redis is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend -# tracking, and the pod lock manager. Enabling this deploys the bundled Redis -# subchart, wires REDIS_HOST / REDIS_PORT / REDIS_PASSWORD into the proxy, and -# renders a `general_settings.coordination_redis` block into the proxy config. -# -# To point at an existing Redis instead, leave `enabled: false` and pass a -# secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL; the proxy -# falls back to those env vars for coordination. Set `cache: true` in the proxy -# config only if you also want LLM response caching, which is independent of -# coordination -# -# When `redis.sentinel.enabled` is set, the coordination block is rendered with -# `sentinel_nodes` and `service_name` (from `redis.sentinel.masterSet`) instead -# of host/port, because a plain Redis client cannot talk to the sentinel port -# -# The image repositories carry the same bitnamilegacy repoint as postgresql -# above; the versioned tags the subchart ships with are gone from -# docker.io/bitnami -redis: - enabled: false - architecture: standalone - image: - repository: bitnamilegacy/redis - tag: 7.2.4-debian-12-r9 - sentinel: - image: - repository: bitnamilegacy/redis-sentinel - tag: 7.2.4-debian-12-r7 - metrics: - image: - repository: bitnamilegacy/redis-exporter - tag: 1.58.0-debian-12-r4 - volumePermissions: - image: - repository: bitnamilegacy/os-shell - tag: 12-debian-12-r16 - sysctl: - image: - repository: bitnamilegacy/os-shell - tag: 12-debian-12-r16 - kubectl: - image: - repository: bitnamilegacy/kubectl - tag: 1.29.2-debian-12-r3 - coordination: - # Set to false to keep the bundled Redis for response caching only and leave - # `general_settings.coordination_redis` out of the rendered config. A - # `coordination_redis` block you define yourself in `proxy_config` always wins - enabled: true - -# Prisma migration job settings -migrationJob: - enabled: true # Enable or disable the schema migration Job - backoffLimit: 4 # Backoff limit for Job restarts - # Wall-clock budget for the whole Job, shared across every `backoffLimit` - # retry rather than granted per attempt. Without it a migration that blocks - # on the database never fails, and when the Helm hook is enabled the release - # waits on it forever: `helm upgrade` and any GitOps controller driving it - # stop reconciling the whole chart until someone deletes the Job by hand. - # Set to null to opt out and restore the unbounded behaviour. - activeDeadlineSeconds: 1800 - # Optional service account for the migration job. - # Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true. - # In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default". - serviceAccountName: "" - annotations: {} - jobLabels: {} # Custom labels for the Job metadata - jobAnnotations: {} # Custom annotations for the Job metadata - podLabels: {} # Custom labels for the Job pod template - command: [] # Override container command (defaults to ["python", "litellm/proxy/prisma_migration.py"]) - args: [] # Override container args - ttlSecondsAfterFinished: 120 - resources: {} - # Unset by default. This job runs the database migration and exits, so it does not - # need the steady-state headroom the proxy does; size it from your own migration - # runs rather than from the proxy figures above. - extraContainers: [] - extraInitContainers: [] - - # Hook configuration - hooks: - argocd: - enabled: true - helm: - enabled: false - -# Log level for the litellm proxy (sets LITELLM_LOG in the deployment env). -# Rendered as a direct `env:` entry, which in Kubernetes takes precedence over -# any `envFrom:` source. If you currently source LITELLM_LOG from an -# environmentSecret or environmentConfigMap, set `logLevel: ""` here to -# disable injection — otherwise this value silently overrides your secret / -# configmap entry. -# -# Setting LITELLM_LOG inside `envVars:` below also wins: the template skips -# this injection entirely when envVars already defines LITELLM_LOG. -logLevel: INFO - -# Additional environment variables to be added to the deployment as a map of key-value pairs -envVars: {} - -# USE_DDTRACE: "true" -# Additional environment variables to be added to the deployment as a list of k8s env vars -extraEnvVars: {} - -# if you want to override the container command, you can do so here -command: {} -# if you want to override the container args, you can do so here -args: {} - -# - name: EXTRA_ENV_VAR -# value: EXTRA_ENV_VAR_VALUE -# Additional Kubernetes resources to deploy with litellm -extraResources: [] - -# - apiVersion: v1 -# kind: ConfigMap -# metadata: -# name: my-extra-config -# data: -# foo: bar -# Pod Disruption Budget -pdb: - enabled: false - # Set exactly one of the following. If both are set, minAvailable takes precedence. - minAvailable: null # e.g. "50%" or 1 - maxUnavailable: null # e.g. 1 or "20%" - annotations: {} - labels: {} - -serviceMonitor: - enabled: false - labels: - {} - # test: test - annotations: - {} - # kubernetes.io/test: test - interval: 15s - scrapeTimeout: 10s - relabelings: [] - # - targetLabel: __meta_kubernetes_pod_node_name - # replacement: $1 - # action: replace - namespaceSelector: - matchNames: [] - # - test-namespace - -lens: - enabled: false - url: "" - publicUrl: "" - service: - name: "" - port: 4318 - gatewaySecret: - name: "" - key: gateway-secret - serviceTokenSecret: - name: "" - key: service-token diff --git a/helm/litellm/.helmignore b/helm/litellm/.helmignore new file mode 100644 index 00000000000..6b8d3002acd --- /dev/null +++ b/helm/litellm/.helmignore @@ -0,0 +1,20 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +*.swp +*.bak +*.tmp +*.orig +*~ +.project +.idea/ +*.tmproj +.vscode/ +/tests/ +/ci/ diff --git a/helm/litellm/Chart.yaml b/helm/litellm/Chart.yaml index 341172da925..7f308a454f5 100644 --- a/helm/litellm/Chart.yaml +++ b/helm/litellm/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: litellm -description: LiteLLM componentized — gateway, UI backend, and UI as separate services +description: Deploys LiteLLM from one image, either componentized (gateway, UI backend, and UI as separate Deployments) or as a single monolith Deployment type: application -version: 0.1.1 -appVersion: "0.1.0" +version: 1.0.0 +appVersion: "1.104.0" annotations: org.opencontainers.image.source: "https://github.com/BerriAI/litellm" diff --git a/helm/litellm/README.md b/helm/litellm/README.md index d66bb3284bf..5ca63d90bcd 100644 --- a/helm/litellm/README.md +++ b/helm/litellm/README.md @@ -1,6 +1,51 @@ -# LiteLLM componentized chart +# litellm Helm chart -This chart deploys separate gateway, backend and UI services +Deploys [LiteLLM](https://github.com/BerriAI/litellm) from one image, `ghcr.io/berriai/litellm` (mirrored at `docker.litellm.ai/berriai/litellm`), in one of two layouts: + +- componentized (default): the `gateway` (LLM data plane, port 4000), the `backend` (management API, port 4001) and the `ui` (static dashboard, port 3000) each get their own Deployment, Service, HPA and PDB so they scale independently +- monolith: `monolith.enabled: true` renders one Deployment and Service running the full proxy (`args: [proxy, ...]`), which serves the gateway routes, the management routes and the Admin UI from a single process + +Both layouts run the same image. The image entrypoint dispatches on the first container argument (`proxy`, `gateway`, `backend`, `ui`, `migrations`, `metrics`, `collector`), so the chart only ever sets `args` and never `command` + +## Requirements + +Kubernetes 1.25+ and Helm 3.8+. Bring your own PostgreSQL (`database.writer.*`) and, optionally, Redis (`redis.*`). The proxy also needs a master key, either an existing Secret named by `masterKey.secretName` or one the chart generates with `masterKey.generate: true` and `masterKey.secretName: ""` + +## Install + +```bash +kubectl create secret generic litellm-master-key-secret --from-literal=master-key=sk-change-me +helm install litellm helm/litellm \ + --set database.writer.host=postgres.example.com \ + --set database.writer.dbname=litellm \ + --set database.writer.passwordSecret.name=litellm-db-secret +``` + +The image tag defaults to the chart's `appVersion`. Override it with `image.tag`, or pin the bytes with `image.digest`, which renders as `repository:tag@digest` + +## Monolith quickstart + +```bash +helm install litellm helm/litellm \ + --set monolith.enabled=true \ + --set masterKey.secretName="" \ + --set masterKey.generate=true \ + --set database.writer.host=postgres.example.com \ + --set database.writer.dbname=litellm \ + --set database.writer.passwordSecret.name=litellm-db-secret +kubectl port-forward svc/litellm-litellm 4000:4000 +``` + +With `monolith.enabled: true`: + +- one Deployment and one Service named `-litellm` render, running `args: [proxy, --port, 4000, --config, /app/config/config.yaml]` plus `monolith.extraArgs` +- the gateway, backend and ui Deployments, Services, HPAs, PDBs and ServiceMonitors are not rendered, whatever `gateway.enabled`, `backend.enabled` and `ui.enabled` say +- every `gateway.*` value configures the monolith pod: `config`, `numWorkers`, `resources`, probes, `securityContext`, `hpa`, `keda`, `pdb`, `metricsServer`, `collector`, `volumes`, `extraEnv`, scheduling. The monolith runs as `serviceAccounts.gateway` +- every `backend.*` and `ui.*` value is ignored +- the Ingress sends every path, built in or from `ingress.extraPaths`, to the monolith Service +- the migrations Job renders exactly as in componentized mode + +## Lens This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render @@ -22,39 +67,114 @@ backend: gatewayUrl: https://gateway.example.com ``` -## Enterprise billing metrics - -Billing metrics use Cognito bearer authentication. Create the Secret with -`client-id` and `client-secret` keys, then enable billing metrics: +## Testing ```bash -kubectl create secret generic litellm-billing-metrics --from-literal=client-id=... --from-literal=client-secret=... +helm lint helm/litellm +helm unittest -f 'tests/*.yaml' helm/litellm +helm test --logs ``` +## Migrating from the litellm-helm chart + +The `litellm-helm` chart (`oci://ghcr.io/berriai/litellm-helm`) is retired; its published packages stay available for a grace period. Its flat values described one monolith Deployment, so the equivalent install here is `monolith.enabled: true` with the values moved under `gateway.*`. The per-component `gateway.image`, `backend.image`, `ui.image` and `migrations.image` blocks of earlier `helm/litellm` versions are gone too: the chart is a major bump to `1.0.0` and every container uses the top-level `image` + +| litellm-helm value | litellm value | +|---|---| +| (implicit single Deployment) | `monolith.enabled: true` | +| `image.repository` / `image.tag` / `image.pullPolicy` | `image.repository` / `image.tag` / `image.pullPolicy` (`image.digest` is new) | +| `replicaCount` | `gateway.replicaCount` | +| `args` | `monolith.extraArgs` (appended after the chart's proxy arguments). Drop any `--port` argument and its value so the proxy keeps listening on port 4000, matching the container port and health checks. Set `gateway.service.port` to change the external Service port | +| `command` | removed: the image entrypoint dispatcher must stay in place | +| `proxy_config` | `gateway.config.proxy_config` | +| `proxyConfigMap.create: false` + `proxyConfigMap.name` | Set `gateway.config.create: false`, mount your ConfigMap with `gateway.volumes` / `gateway.volumeMounts`, and pass `--config` followed by the mounted file path in `monolith.extraArgs` | +| `masterkeySecretName` / `masterkeySecretKey` | `masterKey.secretName` / `masterKey.secretKey`. The default Secret field changes from `masterkey` to `master-key`. When reusing a Secret with the old default field, explicitly set `masterKey.secretKey: masterkey` | +| `masterkeySecretName: ""` (auto generated Secret) | `masterKey.generate: true` with `masterKey.secretName: ""` (creates a new key; copy the old key first, see below, to keep existing credentials valid) | +| `db.useExisting`, `db.endpoint`, `db.database` | `database.writer.host`, `database.writer.port`, `database.writer.dbname` | +| `db.useIAMAuth` / `db.useAzureEntraAuth` | `database.writer.useIAMAuth` / `database.writer.useAzureEntraAuth` (mutually exclusive). Both token-auth modes still require a username in the Secret referenced by `database.writer.passwordSecret.name` and `usernameKey` | +| `db.schema` | `database.writer.schema` | +| `db.username` | Store the username in the Secret referenced by `database.writer.passwordSecret.name`, under its `usernameKey`. There is no literal username value in the new chart | +| `db.secret.name` / `db.secret.usernameKey` / `db.secret.passwordKey` | `database.writer.passwordSecret.name` / `usernameKey` / `passwordKey` | +| `db.secret.endpointKey` (writer host read from a Secret) | `database.writer.passwordSecret.hostKey` with `database.writer.host` left empty | +| `db.url` (custom URL template) | `database.writer.passwordSecret.urlKey` pointing at a Secret key that holds the complete writer URL, with the other `database.writer.*` connection values left empty | +| `db.readReplicaUrl` (credential-less reader URL) | `database.reader.host`, `database.reader.port`, `database.reader.dbname`, with `database.reader.passwordSecret.*` for its credentials (or `database.reader.useIAMAuth: true`) | +| `db.secret.readReplicaEndpointKey` (reader host read from a Secret) | `database.reader.passwordSecret.hostKey` with `database.reader.host` left empty | +| `db.secret.readReplicaUrlKey` (full reader URL in a Secret) | `database.reader.passwordSecret.urlKey` with the other `database.reader.*` connection values left empty | +| `db.connectionPool.*` | `database.connectionPool.*` | +| `db.deployStandalone: true` / `postgresql.*` | removed: the chart ships no PostgreSQL, point `database.writer.*` at your own | +| `redis.enabled: true` (bundled) | removed: the chart ships no Redis, point `redis.host` at your own | +| external Redis via `envVars` | `redis.host`, `redis.port`, `redis.passwordSecret.*`, `redis.cluster` | +| `envVars` / `extraEnvVars` | `gateway.extraEnv` (list of `name` / `value` or `valueFrom` entries). The retired chart also passed these to the migration Job, so copy any it needs, such as the AWS region and credentials for IAM database auth, into `migrationJob.extraEnv` too | +| `environmentSecrets` | `gateway.envSecrets` | +| `environmentConfigMaps` | `gateway.envConfigMaps` | +| `logLevel` | `gateway.logLevel` | +| `resources` | `gateway.resources` | +| `livenessProbe` / `readinessProbe` / `startupProbe` | `gateway.livenessProbe` / `gateway.readinessProbe` / `gateway.startupProbe`, written as full Kubernetes probes: the old `path: /health/readiness` becomes `httpGet: { path: /health/readiness, port: http }`, the timing fields (`initialDelaySeconds`, `periodSeconds`, `timeoutSeconds`, `successThreshold`, `failureThreshold`) carry over unchanged | +| `securityContext` / `podSecurityContext` | `gateway.securityContext` / `gateway.podSecurityContext`. With `readOnlyRootFilesystem: true` the retired chart added `emptyDir` mounts at `/tmp`, `/.cache` and `/.npm`; this chart does not, so add them through `gateway.volumes` / `gateway.volumeMounts`. The retired chart applied both to the migration Job as well; set `migrationJob.securityContext` / `migrationJob.podSecurityContext` to keep the Job under the same policy | +| `service.*` | `gateway.service.*` | +| `ingress.className`, `ingress.annotations`, `ingress.tls` | same keys (routes to the monolith Service in monolith mode) | +| `ingress.hosts[].host` | `ingress.host`, one host per release; the built-in paths are rendered for that host, so `ingress.hosts[].paths` has no equivalent. For more than one hostname, add a second Ingress through `extraResources` or point a DNS alias at the one host | +| `autoscaling.*` | `gateway.hpa.*`. Defaults change from 100 to 10 maximum replicas and from 60% to 70% CPU utilization, with an 80% memory target now enabled. To keep the old scaling behavior, set `gateway.hpa.maxReplicas: 100`, `gateway.hpa.targetCPUUtilizationPercentage: 60` and `gateway.hpa.targetMemoryUtilizationPercentage: 0` | +| `keda.*` | `gateway.keda.*` with these renames: `minReplicas` / `maxReplicas` are `minReplicaCount` / `maxReplicaCount` (the new default maximum is 10, the old one was 100), `behavior` moves to `advanced.horizontalPodAutoscalerConfig.behavior`, `restoreToOriginalReplicaCount` moves to `advanced.restoreToOriginalReplicaCount`, and `keda.prometheus.requestsPerSecond` / `tokensPerSecond` are `gateway.keda.prometheus.targetRequestsPerSecond` / `targetTokensPerSecond`. `pollingInterval`, `cooldownPeriod`, `fallback`, `triggers` and `scaledObject.annotations` (for example KEDA's pause annotations) keep their names | +| `numWorkers` | `gateway.numWorkers` | +| `pdb.*` | `gateway.pdb.*`, except `labels` and `annotations` are no longer supported. To retain custom PDB metadata, set `gateway.pdb.enabled: false` and define the PDB through `extraResources` | +| `metricsServer.*` | `gateway.metricsServer.*` | +| `serviceMonitor.*` | `gateway.serviceMonitor.*` (`labels`, `annotations`, `interval`, `scrapeTimeout`, `relabelings`, `namespaceSelector.matchNames`); it now requires `gateway.metricsServer.enabled` | +| `collector.*` | `gateway.collector.*`, except `command` is no longer supported and `scaleOnProxyContainerCpu` becomes `scaleOnGatewayContainerCpu`. Its default changes from `false` to `true`. Set `gateway.collector.scaleOnGatewayContainerCpu: false` to keep pod-wide CPU scaling. For a custom collector command, disable the built-in sidecar with `gateway.collector.enabled: false` and define it through `gateway.extraContainers`, with its environment, volumes and volume mounts configured explicitly | +| `billingMetrics.*` | `billingMetrics.*` | +| `migrationJob.*` | `migrationJob.*` | +| `volumes` / `volumeMounts` | `gateway.volumes` / `gateway.volumeMounts`. The retired chart also mounted these in the migration Job; copy any mounts the Job needs, such as a database CA certificate, to `migrationJob.volumes` / `migrationJob.volumeMounts` too | +| `extraContainers` / `extraInitContainers` | `gateway.extraContainers` / `gateway.extraInitContainers` | +| `lifecycle` | `gateway.lifecycle` | +| `strategy` | `gateway.strategy` | +| `deploymentAnnotations` / `deploymentLabels` / `deploymentMinReadySeconds` | `gateway.deploymentAnnotations` / `gateway.deploymentLabels` / `gateway.minReadySeconds` | +| `podAnnotations` / `podLabels` | `gateway.podAnnotations` / `gateway.podLabels`. The retired chart ran `podAnnotations` through `tpl`; this chart renders them verbatim, so a copied template such as `checksum/config: {{ ... }}` stays literal text. Resolve it before install instead, e.g. `--set-string gateway.podAnnotations.checksum/config=$(kubectl get configmap my-config -o yaml \| sha256sum \| cut -d' ' -f1)`. The chart's own proxy ConfigMap already rolls the pods when `gateway.config` changes. The retired chart also put `podLabels` on the migration Job pod; copy any the Job needs, such as a network-policy selector, to `migrationJob.podLabels` | +| `nodeSelector` / `tolerations` / `affinity` / `topologySpreadConstraints` | `gateway.nodeSelector` / `gateway.tolerations` / `gateway.affinity` / `gateway.topologySpreadConstraints`. The retired chart also applied `nodeSelector`, `tolerations` and `affinity` to the migration Job; copy them to `migrationJob.nodeSelector` / `migrationJob.tolerations` / `migrationJob.affinity` too, or a Job that needs tainted or zoned nodes to reach the database stays Pending and blocks the install | +| `terminationGracePeriodSeconds` | `gateway.terminationGracePeriodSeconds` | +| `serviceAccount.*` | `serviceAccounts.gateway.*`. The retired chart also shared an existing ServiceAccount with migrations; set `migrationJob.serviceAccountName` to that existing account too when the Job needs its permissions, such as for IAM database auth | +| `lens.*` | `lens.*`. Deploy Lens from its own chart. The removed `lensWorker.*` values fail the render when enabled | +| `extraResources` | `extraResources` | +| `nameOverride: "litellm"` | `nameOverride: ""` (the chart name is already `litellm`) | +| `fullnameOverride` | `fullnameOverride`. The new default always includes the chart-name suffix, even when the release name already contains it. Set this to the existing Service name to retain its DNS name | + +A minimal migration: + ```yaml -billingMetrics: +monolith: enabled: true +masterKey: + secretName: litellm-master-key-secret +database: + writer: + host: postgres.example.com + dbname: litellm + passwordSecret: + name: litellm-db-secret + usernameKey: username + passwordKey: password +gateway: + replicaCount: 2 + config: + proxy_config: + model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + envSecrets: + - litellm-provider-keys ``` -The chart requires an HTTPS endpoint, an HTTPS token URL, and the Secret name -when billing metrics are enabled +The monolith Service is named `-litellm`. To retain a different old Service name, set `fullnameOverride` to that existing name. The generated Secret in this chart has `helm.sh/resource-policy: keep` and is reused across upgrades of this chart. The retired chart stored its generated key under a different Secret and data key, and `helm uninstall` deletes that Secret, so copy it into a Secret you own before uninstalling the old release to keep existing credentials valid. Run the following in the old release's namespace, replacing `OLD_MASTER_KEY_SECRET` with the actual existing Secret name. For a release named `litellm` without a `fullnameOverride`, that name is `litellm-masterkey`: -## Routes to another Service - -For a shared hostname, `ingress.extraPaths` can route to a Service in the same namespace using `serviceName` and `servicePort`. The port can be a number or a named Service port. Omit `service` when using these fields - -```yaml -ingress: - enabled: true - className: alb - host: gateway.example.com - extraPaths: - - path: /moyai - pathType: Prefix - serviceName: moyai-ingress - servicePort: 8080 +```bash +set -euo pipefail +OLD_MASTER_KEY_SECRET=litellm-masterkey +old_master_key="$(kubectl get secret "$OLD_MASTER_KEY_SECRET" -o jsonpath='{.data.masterkey}' | base64 -d)" +test -n "$old_master_key" +kubectl create secret generic litellm-master-key-secret \ + --from-literal=master-key="$old_master_key" +unset old_master_key ``` -The Service and application must already support the path prefix. The chart adds the path to the existing Ingress and leaves its built-in routes in place. It does not create the target Service or rewrite requests - -For AWS ALB, configure the target Service's `alb.ingress.kubernetes.io/healthcheck-path` annotation when the application needs a different health check from LiteLLM. Existing `service` aliases (`gateway`, `backend`, `ui`) remain supported +Then set `masterKey.secretName: litellm-master-key-secret`, as in the minimal example above. Releases that set `masterkeySecretName` already point `masterKey.secretName` and `masterKey.secretKey` at that Secret diff --git a/helm/litellm/ci/postgres.yaml b/helm/litellm/ci/postgres.yaml new file mode 100644 index 00000000000..aa210e932b0 --- /dev/null +++ b/helm/litellm/ci/postgres.yaml @@ -0,0 +1,58 @@ +# Throwaway PostgreSQL for the CircleCI helm_chart_testing job. Applied with +# kubectl before `helm install`; the chart itself ships no database. +apiVersion: v1 +kind: Secret +metadata: + name: litellm-ci-postgres +type: Opaque +stringData: + username: litellm + password: litellm-ci +--- +apiVersion: v1 +kind: Service +metadata: + name: litellm-ci-postgres +spec: + selector: + app: litellm-ci-postgres + ports: + - port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm-ci-postgres +spec: + replicas: 1 + selector: + matchLabels: + app: litellm-ci-postgres + template: + metadata: + labels: + app: litellm-ci-postgres + spec: + containers: + - name: postgres + image: postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297 + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: litellm + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: litellm-ci-postgres + key: username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: litellm-ci-postgres + key: password + readinessProbe: + exec: + command: [pg_isready, -U, litellm, -d, litellm] + periodSeconds: 2 diff --git a/helm/litellm/ci/test-values.yaml b/helm/litellm/ci/test-values.yaml new file mode 100644 index 00000000000..40c68489ed3 --- /dev/null +++ b/helm/litellm/ci/test-values.yaml @@ -0,0 +1,34 @@ +# Values for the CircleCI helm_chart_testing job: a monolith install on a +# kind cluster from the image built by the pipeline (image.repository / tag / +# pullPolicy are passed with --set), backed by the throwaway PostgreSQL in +# ci/postgres.yaml so the migrations Job and the proxy have a database. +monolith: + enabled: true + +masterKey: + secretName: "" + generate: true + +gateway: + replicaCount: 1 + hpa: + enabled: false + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + memory: 2Gi + config: + create: true + proxy_config: + general_settings: {} + +database: + writer: + host: litellm-ci-postgres + dbname: litellm + passwordSecret: + name: litellm-ci-postgres + usernameKey: username + passwordKey: password diff --git a/helm/litellm/templates/NOTES.txt b/helm/litellm/templates/NOTES.txt index 135dcb876e0..a046e4e5280 100644 --- a/helm/litellm/templates/NOTES.txt +++ b/helm/litellm/templates/NOTES.txt @@ -1,4 +1,16 @@ -LiteLLM componentized — release {{ .Release.Name }} in namespace {{ .Release.Namespace }}. +{{- if .Values.monolith.enabled }} +LiteLLM monolith: release {{ .Release.Name }} in namespace {{ .Release.Namespace }}, image {{ include "litellm.image" . }} + +One Deployment runs the full proxy (gateway routes, management routes and the Admin UI): + - proxy : Service {{ include "litellm.workload.fullname" . }} on port {{ .Values.gateway.service.port }} + +Port-forward: + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.workload.fullname" . }} {{ .Values.gateway.service.port }} + +The monolith takes its configuration from the gateway.* values (config, resources, probes, hpa, pdb, keda, +metricsServer, collector, volumes, env, scheduling). backend.* and ui.* are ignored while monolith.enabled is true +{{- else }} +LiteLLM componentized: release {{ .Release.Name }} in namespace {{ .Release.Namespace }}, image {{ include "litellm.image" . }} Components: {{- if .Values.gateway.enabled }} @@ -16,43 +28,48 @@ Port-forward examples: kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.backend.fullname" . }} {{ .Values.backend.service.port }} kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.ui.fullname" . }} {{ .Values.ui.service.port }} +Set monolith.enabled=true to run everything as one Deployment instead +{{- end }} + Reminders: - Sensitive values come from Secret references only. Before installing, set: - - masterKey.secretName (Secret with the proxy master key) + - masterKey.secretName (Secret with the proxy master key, or + masterKey.generate: true with secretName "" to let the chart create one) - database.writer.{host,port,dbname} (writer connection pieces) - database.writer.passwordSecret.{name,usernameKey,passwordKey} (Secret holding the writer DB username + password) - database.writer.passwordSecret.{hostKey,urlKey} (optional, read writer host or full URL from that Secret) - - database.writer.useIAMAuth: true (optional — chart sets IAM_TOKEN_DB_AUTH=true and + - database.writer.useIAMAuth: true (optional: chart sets IAM_TOKEN_DB_AUTH=true and omits DATABASE_PASSWORD / DATABASE_URL so the proxy mints the URL from an IAM token at startup) - database.reader.host / database.reader.passwordSecret.{hostKey,urlKey} (optional, enables read-replica routing. The Secret name is required for hostKey, urlKey, and for host with password auth) - - database.reader.useIAMAuth: true (optional, requires database.writer.useIAMAuth: true — + - database.reader.useIAMAuth: true (optional, requires database.writer.useIAMAuth: true; chart emits DATABASE_*_READ_REPLICA env vars and omits DATABASE_PASSWORD_READ_REPLICA / DATABASE_URL_READ_REPLICA so the proxy mints the reader URL from an IAM token at startup) - - redis.passwordSecret.name (optional — set when redis.host is provided and the + - redis.host (optional: the proxy's coordination Redis) + - redis.passwordSecret.name (optional: set when redis.host is provided and the cache requires auth) - redis.passwordSecret.{hostKey,urlKey} (optional, read Redis host or full URL from that Secret) (the Secret name is required when either key is set) - - redis.cluster: true (optional — chart sets REDIS_CLUSTER_NODES from + - redis.cluster: true (optional: chart sets REDIS_CLUSTER_NODES from redis.host / redis.port so the proxy's Cache() constructs a RedisClusterCache; the cluster client discovers remaining nodes from CLUSTER SLOTS) - - Per-component extras (gateway / backend / ui): - - {component}.extraEnv / envConfigMaps / envSecrets (the latter two are lists of resource names → - envFrom configMapRef / secretRef) + - Per-component extras (gateway / backend / ui; gateway.* also drives the monolith): + - {component}.extraEnv / envConfigMaps / envSecrets (the latter two are lists of resource names rendered + as envFrom configMapRef / secretRef) - {component}.logLevel (renders as LITELLM_LOG) - gateway.config.proxy_config (rendered into a ConfigMap and mounted at - /app/config/config.yaml; gateway reads it via - CONFIG_FILE_PATH) + /app/config/config.yaml; read via CONFIG_FILE_PATH) - {component}.pdb.{enabled,minAvailable,maxUnavailable} (per-component PodDisruptionBudget; disabled by - default — with hpa.minReplicas of 1, minAvailable: 1 + default: with hpa.minReplicas of 1, minAvailable: 1 would block node drains) - {component}.topologySpreadConstraints (standard k8s list, e.g. spread replicas across topology.kubernetes.io/zone) - - Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend. + - Enable ingress.enabled=true to dispatch / to ui, gateway data-plane prefixes to gateway and the catch-all to + backend; in monolith mode every path goes to the monolith Service diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index b704782f5d1..13222d8284e 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -1,11 +1,116 @@ {{/* -Common naming + label helpers shared by gateway, backend, and ui templates. +Common naming + label helpers shared by gateway, backend, ui, and monolith templates. */}} {{- define "litellm.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +The one image reference every container in the chart uses. `repository:tag`, +or `repository:tag@digest` when image.digest is set; the tag falls back to +the chart appVersion. +*/}} +{{- define "litellm.image" -}} +{{- $tag := .Values.image.tag | default .Chart.AppVersion -}} +{{- if .Values.image.digest -}} +{{- printf "%s:%s@%s" .Values.image.repository $tag .Values.image.digest -}} +{{- else -}} +{{- printf "%s:%s" .Values.image.repository $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Componentized mode renders the gateway / backend / ui Deployments only when +monolith mode is off. +*/}} +{{- define "litellm.gateway.render" -}} +{{- if and .Values.gateway.enabled (not .Values.monolith.enabled) -}}true{{- end -}} +{{- end -}} + +{{- define "litellm.backend.render" -}} +{{- if and .Values.backend.enabled (not .Values.monolith.enabled) -}}true{{- end -}} +{{- end -}} + +{{- define "litellm.ui.render" -}} +{{- if and .Values.ui.enabled (not .Values.monolith.enabled) -}}true{{- end -}} +{{- end -}} + +{{/* +The proxy Deployment (monolith mode) and the gateway Deployment +(componentized mode) are the same pod spec fed by .Values.gateway, so the +templates that serve both (HPA, KEDA, PDB, metrics Service, ServiceMonitor) +resolve their name, component label and selector through these. +*/}} +{{- define "litellm.proxy.fullname" -}} +{{- include "litellm.fullname" . -}} +{{- end -}} + +{{- define "litellm.proxy.selectorLabels" -}} +app.kubernetes.io/name: {{ include "litellm.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: proxy +{{- end -}} + +{{- define "litellm.workload.componentName" -}} +{{- if .Values.monolith.enabled -}}proxy{{- else -}}gateway{{- end -}} +{{- end -}} + +{{- define "litellm.workload.fullname" -}} +{{- if .Values.monolith.enabled -}} +{{- include "litellm.proxy.fullname" . -}} +{{- else -}} +{{- include "litellm.gateway.fullname" . -}} +{{- end -}} +{{- end -}} + +{{- define "litellm.workload.selectorLabels" -}} +{{- if .Values.monolith.enabled -}} +{{- include "litellm.proxy.selectorLabels" . -}} +{{- else -}} +{{- include "litellm.gateway.selectorLabels" . -}} +{{- end -}} +{{- end -}} + +{{- define "litellm.workload.render" -}} +{{- if or .Values.monolith.enabled .Values.gateway.enabled -}}true{{- end -}} +{{- end -}} + +{{/* +Container arguments for the proxy (monolith) container. The image entrypoint +dispatches on the first argument. +*/}} +{{- define "litellm.proxy.args" -}} +- proxy +- --port +- "4000" +{{- if .Values.gateway.config.create }} +- --config +- /app/config/config.yaml +{{- end }} +{{- with .Values.monolith.extraArgs }} +{{ toYaml . }} +{{- end }} +{{- end -}} + +{{/* +Master key Secret reference. `masterKey.secretName` when set, otherwise the +Secret the chart generates when `masterKey.generate` is true. +*/}} +{{- define "litellm.masterKey.generatedSecretName" -}} +{{- printf "%s-masterkey" (include "litellm.fullname" .) -}} +{{- end -}} + +{{- define "litellm.masterKey.secretName" -}} +{{- if .Values.masterKey.secretName -}} +{{- .Values.masterKey.secretName -}} +{{- else if .Values.masterKey.generate -}} +{{- include "litellm.masterKey.generatedSecretName" . -}} +{{- else -}} +{{- fail "masterKey.secretName is required (the chart never accepts an inline master key); set it to an existing Secret or set masterKey.generate: true" -}} +{{- end -}} +{{- end -}} + {{- define "litellm.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} @@ -108,6 +213,7 @@ Each component (gateway, backend, ui) has its own SA config under .Values.serviceAccounts.. When `create` is true and `name` is empty the chart defaults to "-litellm-". When `create` is false the chart uses the provided name, or the namespace `default` SA. +The monolith pod runs as the gateway ServiceAccount. */}} {{- define "litellm.gateway.serviceAccountName" -}} {{- if .Values.serviceAccounts.gateway.create -}} @@ -186,6 +292,22 @@ by the controller rather than declared, so nothing there can collide. {{- toYaml .podLabels }} {{- end -}} +{{/* +LITELLM_MASTER_KEY, and PROXY_MASTER_KEY for configs copied from litellm-helm, +for the app containers only. The migrations Job runs as a +pre-install hook, before the chart's generated master key Secret exists, and +migrations/run.py never reads the key, so the Job must not reference it. +*/}} +{{- define "litellm.masterKeyEnv" -}} +{{- range list "LITELLM_MASTER_KEY" "PROXY_MASTER_KEY" }} +- name: {{ . }} + valueFrom: + secretKeyRef: + name: {{ include "litellm.masterKey.secretName" $ }} + key: {{ $.Values.masterKey.secretKey | default "master-key" }} +{{- end }} +{{- end -}} + {{/* Master-key + database + redis env block — shared by gateway, backend, and the migrations Job. @@ -229,11 +351,6 @@ IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets. {{- define "litellm.serverEnv" -}} {{- $root := .root -}} {{- $component := .component -}} -- name: LITELLM_MASTER_KEY - valueFrom: - secretKeyRef: - name: {{ required "masterKey.secretName is required (the chart no longer accepts an inline master key)" $root.Values.masterKey.secretName }} - key: {{ $root.Values.masterKey.secretKey | default "master-key" }} {{- if $component.logLevel }} - name: LITELLM_LOG value: {{ $component.logLevel | quote }} @@ -276,6 +393,11 @@ IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets. secretKeyRef: name: {{ required "database.writer.passwordSecret.name is required" .passwordSecret.name }} key: {{ .passwordSecret.usernameKey | default "username" }} +- name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .passwordSecret.name }} + key: {{ .passwordSecret.usernameKey | default "username" }} - name: DATABASE_NAME value: {{ required "database.writer.dbname is required" .dbname | quote }} {{- if .schema }} @@ -473,7 +595,7 @@ harmless no-op for the Job and authoritative for the app pods. {{- end -}} {{/* -In-container PgBouncer env for the gateway container. Under IAM or Entra auth the pooler mints and renews the database token itself. +In-container PgBouncer env for the gateway and proxy containers. Under IAM or Entra auth the pooler mints and renews the database token itself. */}} {{- define "litellm.connectionPoolEnv" -}} {{- with .Values.database.connectionPool -}} @@ -510,7 +632,7 @@ than silently replaced by the fallback. {{- $max := $component.pdb.maxUnavailable -}} {{- $minSet := not (or (kindIs "invalid" $min) (eq (printf "%v" $min) "")) -}} {{- $maxSet := not (or (kindIs "invalid" $max) (eq (printf "%v" $max) "")) -}} -{{- if and $component.enabled $component.pdb $component.pdb.enabled }} +{{- if and .enabled $component.pdb $component.pdb.enabled }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -568,6 +690,16 @@ ImplementationSpecific {{- end -}} {{- end -}} +{{/* +Service spec fields shared by the component and monolith Services. +Invoke with the component's `service` dict. +*/}} +{{- define "litellm.service.extras" -}} +{{- if and (eq .type "LoadBalancer") .loadBalancerClass -}} +loadBalancerClass: {{ .loadBalancerClass | quote }} +{{- end }} +{{- end -}} + {{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}} {{/* @@ -626,3 +758,27 @@ shutdown drain window. {{- end }} {{- end }} {{- end -}} +{{- define "litellm.liteadminEnv" -}} +{{- $root := .root -}} +{{- with $root.Values.liteadmin }} +{{- if .enabled }} +- name: LITELLM_ADMIN_AGENT_URL + value: {{ printf "http://%s-liteadmin:10000" (include "litellm.fullname" $root | trunc 53 | trimSuffix "-") | quote }} +- name: ADMIN_AGENT_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: {{ required "liteadmin.existingSecret is required" .existingSecret }} + key: ADMIN_AGENT_SERVICE_TOKEN +{{- $hasProxyBaseUrl := false }} +{{- range $.component.extraEnv }} +{{- if eq .name "PROXY_BASE_URL" }} +{{- $hasProxyBaseUrl = true }} +{{- end }} +{{- end }} +{{- if not $hasProxyBaseUrl }} +- name: PROXY_BASE_URL + value: {{ required "liteadmin.gatewayUrl is required" .gatewayUrl | quote }} +{{- end }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/helm/litellm/templates/_workload.tpl b/helm/litellm/templates/_workload.tpl new file mode 100644 index 00000000000..8f52cd52f2a --- /dev/null +++ b/helm/litellm/templates/_workload.tpl @@ -0,0 +1,515 @@ +{{/* +The pod that runs the LLM data plane. In componentized mode it is the gateway +Deployment (`args: [gateway, ...]`), in monolith mode the proxy Deployment +(`args: [proxy, ...]`) that also serves the management API and the Admin UI. +Both are configured by .Values.gateway; the name, component label, selector +and args come from the litellm.workload.* helpers. +*/}} +{{- define "litellm.workload.deployment" -}} +{{- $component := include "litellm.workload.componentName" . -}} +{{- $fullname := include "litellm.workload.fullname" . -}} +{{- if and .Values.gateway.hpa.enabled .Values.gateway.keda.enabled }} +{{- fail "gateway.hpa.enabled and gateway.keda.enabled are mutually exclusive: two autoscalers on one Deployment fight over the replica count, so set gateway.hpa.enabled: false when using KEDA" }} +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullname }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} + {{- with .Values.gateway.deploymentLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.gateway.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.gateway.hpa.enabled) (not .Values.gateway.keda.enabled) (not (kindIs "invalid" .Values.gateway.replicaCount)) }} + replicas: {{ .Values.gateway.replicaCount }} + {{- end }} + {{- $minReady := .Values.gateway.minReadySeconds }} + {{- if not (or (kindIs "invalid" $minReady) (eq (printf "%v" $minReady) "")) }} + minReadySeconds: {{ $minReady }} + {{- end }} + {{- with .Values.gateway.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "litellm.workload.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- if .Values.gateway.config.create }} + checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.gateway.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "litellm.workload.selectorLabels" . | nindent 8 }} + {{- with .Values.gateway.podLabels }} + {{- include "litellm.podLabels" (dict "podLabels" . "componentName" "gateway") | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }} + {{- with .Values.gateway.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.extraInitContainers }} + initContainers: + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + containers: + - name: {{ $component }} + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + args: + {{- if .Values.monolith.enabled }} + {{- include "litellm.proxy.args" . | nindent 12 }} + {{- else }} + - gateway + - --host + - 0.0.0.0 + - --port + - "4000" + {{- end }} + ports: + - name: http + containerPort: 4000 + protocol: TCP + env: + {{- include "litellm.lensConnectionEnv" . | nindent 12 }} + {{- if and (not .Values.monolith.enabled) .Values.ui.publicUrl }} + - name: LITELLM_CORS_ORIGINS + value: {{ include "litellm.ui.publicOrigin" . | quote }} + {{- end }} + {{- include "litellm.masterKeyEnv" $ | nindent 12 }} + {{- if .Values.monolith.enabled }} + {{- include "litellm.liteadminEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} + {{- end }} + {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} + {{- if .Values.gateway.config.create }} + - name: CONFIG_FILE_PATH + value: /app/config/config.yaml + {{- end }} + {{- if .Values.gateway.numWorkers }} + - name: NUM_WORKERS + value: {{ .Values.gateway.numWorkers | quote }} + {{- end }} + {{- if .Values.database.connectionPool.enabled }} + {{- include "litellm.connectionPoolEnv" $ | nindent 12 }} + {{- end }} + {{- if .Values.billingMetrics.enabled }} + {{- include "litellm.billingMetricsEnv" . | nindent 12 }} + {{- end }} + {{- if .Values.gateway.metricsServer.enabled }} + {{- if eq (int .Values.gateway.metricsServer.port) 4000 }} + {{- fail (printf "gateway.metricsServer.port must differ from the %s port 4000" $component) }} + {{- end }} + - name: PROMETHEUS_MULTIPROC_DIR + value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + {{- end }} + {{- if .Values.gateway.collector.enabled }} + {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} + {{- end }} + {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} + volumeMounts: + {{- if .Values.gateway.config.create }} + - name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + {{- end }} + {{- if .Values.gateway.metricsServer.enabled }} + - name: prometheus-multiproc + mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} + {{- end }} + {{- with .Values.gateway.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .Values.gateway.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.gateway.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.gateway.startupProbe }} + startupProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.gateway.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.gateway.resources | nindent 12 }} + {{- if .Values.gateway.metricsServer.enabled }} + - name: metrics + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + args: + - metrics + - --port + - {{ .Values.gateway.metricsServer.port | quote }} + env: + - name: PROMETHEUS_MULTIPROC_DIR + value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + ports: + - name: metrics + containerPort: {{ .Values.gateway.metricsServer.port }} + protocol: TCP + volumeMounts: + - name: prometheus-multiproc + mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + readinessProbe: + tcpSocket: { port: metrics } + periodSeconds: 10 + livenessProbe: + tcpSocket: { port: metrics } + periodSeconds: 15 + failureThreshold: 6 + resources: + {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }} + {{- end }} + {{- if .Values.gateway.collector.enabled }} + - name: collector + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + args: + - collector + env: + {{- include "litellm.masterKeyEnv" $ | nindent 12 }} + {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} + {{- if .Values.gateway.config.create }} + - name: CONFIG_FILE_PATH + value: /app/config/config.yaml + {{- end }} + {{- if .Values.database.connectionPool.enabled }} + {{- include "litellm.connectionPoolEnv" $ | nindent 12 }} + {{- end }} + {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} + - name: LITELLM_JOB_ROLE + value: collector + {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts (include "litellm.gateway.collectorSocketDir" .) }} + volumeMounts: + {{- if .Values.gateway.config.create }} + - name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} + {{- end }} + {{- with .Values.gateway.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + resources: + {{- toYaml .Values.gateway.collector.resources | nindent 12 }} + {{- end }} + {{- with .Values.gateway.extraContainers }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} + volumes: + {{- if .Values.gateway.config.create }} + - name: gateway-config + configMap: + name: {{ include "litellm.workload.fullname" . }}-config + {{- end }} + {{- if .Values.gateway.metricsServer.enabled }} + - name: prometheus-multiproc + emptyDir: {} + {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + emptyDir: + sizeLimit: 1Mi + {{- end }} + {{- with .Values.gateway.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.gateway.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }} + {{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }} + terminationGracePeriodSeconds: {{ $gracePeriod }} + {{- end }} +{{- end -}} + +{{- define "litellm.workload.service" -}} +{{- $component := include "litellm.workload.componentName" . -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "litellm.workload.fullname" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} + {{- with .Values.gateway.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.gateway.service.type }} + {{- with include "litellm.service.extras" .Values.gateway.service }} + {{- . | nindent 2 }} + {{- end }} + ports: + - port: {{ .Values.gateway.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "litellm.workload.selectorLabels" . | nindent 4 }} +{{- end -}} + +{{- define "litellm.workload.hpa" -}} +{{- $component := include "litellm.workload.componentName" . -}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "litellm.workload.fullname" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "litellm.workload.fullname" . }} + minReplicas: {{ .Values.gateway.hpa.minReplicas }} + maxReplicas: {{ .Values.gateway.hpa.maxReplicas }} + metrics: + {{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- if and .Values.gateway.collector.enabled .Values.gateway.collector.scaleOnGatewayContainerCpu }} + - type: ContainerResource + containerResource: + name: cpu + container: {{ $component }} + target: + type: Utilization + averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- else }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- end }} + {{- end }} + {{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }} + {{- end }} + {{- with .Values.gateway.hpa.targetRequestsPerSecond }} + - type: Pods + pods: + metric: + name: litellm_requests_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} + {{- with .Values.gateway.hpa.targetTokensPerSecond }} + - type: Pods + pods: + metric: + name: litellm_tokens_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} + {{- with .Values.gateway.hpa.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} + +{{/* +KEDA ScaledObject targeting the workload Deployment. The optional Prometheus +triggers query the same per pod counters the HPA workload metrics use, +scoped to this release's metrics Service through the `job` label the +ServiceMonitor gives every scrape. +*/}} +{{- define "litellm.workload.keda" -}} +{{- $component := include "litellm.workload.componentName" . -}} +{{- $keda := .Values.gateway.keda -}} +{{- $prom := $keda.prometheus -}} +{{- $wantsPrometheus := or $prom.targetRequestsPerSecond $prom.targetTokensPerSecond -}} +{{- if and $wantsPrometheus (not $prom.serverAddress) }} +{{- fail "gateway.keda.prometheus.serverAddress is required when gateway.keda.prometheus.targetRequestsPerSecond or targetTokensPerSecond is set" }} +{{- end }} +{{- if and $wantsPrometheus (not .Values.gateway.metricsServer.enabled) }} +{{- fail "gateway.metricsServer.enabled is required when gateway.keda.prometheus.targetRequestsPerSecond or targetTokensPerSecond is set" }} +{{- end }} +{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s-metrics" (include "litellm.workload.fullname" .)) -}} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ include "litellm.workload.fullname" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} + {{- with (dig "scaledObject" "annotations" (dict) $keda) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "litellm.workload.fullname" . }} + minReplicaCount: {{ $keda.minReplicaCount }} + maxReplicaCount: {{ $keda.maxReplicaCount }} + pollingInterval: {{ $keda.pollingInterval }} + cooldownPeriod: {{ $keda.cooldownPeriod }} + {{- with $keda.advanced }} + advanced: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $keda.fallback }} + {{- $fallback := deepCopy . }} + {{- if not (hasKey $fallback "failureThreshold") }} + {{- $_ := set $fallback "failureThreshold" 3 }} + {{- end }} + {{- if not (hasKey $fallback "replicas") }} + {{- $_ := set $fallback "replicas" $keda.maxReplicaCount }} + {{- end }} + fallback: + {{- toYaml $fallback | nindent 4 }} + {{- end }} + triggers: + {{- with $keda.triggers }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $prom.targetRequestsPerSecond }} + - type: prometheus + metricType: AverageValue + metadata: + serverAddress: {{ $prom.serverAddress | quote }} + query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }} + threshold: {{ toJson . | trimAll "\"" | quote }} + {{- end }} + {{- with $prom.targetTokensPerSecond }} + - type: prometheus + metricType: AverageValue + metadata: + serverAddress: {{ $prom.serverAddress | quote }} + query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }} + threshold: {{ toJson . | trimAll "\"" | quote }} + {{- end }} +{{- end -}} + +{{- define "litellm.workload.serviceMetrics" -}} +{{- $component := include "litellm.workload.componentName" . -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "litellm.workload.fullname" . }}-metrics + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.gateway.metricsServer.port }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "litellm.workload.selectorLabels" . | nindent 4 }} +{{- end -}} + +{{- define "litellm.workload.serviceMonitor" -}} +{{- $component := include "litellm.workload.componentName" . -}} +{{- if not .Values.gateway.metricsServer.enabled }} +{{- fail "gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled: the http port serves /metrics/ behind virtual-key auth, so an unauthenticated scrape gets 401" }} +{{- end }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "litellm.workload.fullname" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} + {{- with .Values.gateway.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.gateway.serviceMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "litellm.workload.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + {{- with .Values.gateway.serviceMonitor.namespaceSelector.matchNames }} + {{- toYaml . | nindent 6 }} + {{- else }} + - {{ .Release.Namespace | quote }} + {{- end }} + endpoints: + - port: metrics + path: /metrics/ + interval: {{ .Values.gateway.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }} + scheme: http + {{- with .Values.gateway.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index 7db14ad87a5..9118c6f3d88 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -1,4 +1,4 @@ -{{- if .Values.backend.enabled }} +{{- if include "litellm.backend.render" . }} {{- $componentHosts := .Values.ingress.componentHosts | default dict -}} {{- $hasComponentHosts := or $componentHosts.ui $componentHosts.backend $componentHosts.gateway -}} apiVersion: apps/v1 @@ -8,10 +8,21 @@ metadata: labels: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: backend + {{- with .Values.backend.deploymentLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.backend.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- if and (not .Values.backend.hpa.enabled) (not (kindIs "invalid" .Values.backend.replicaCount)) }} replicas: {{ .Values.backend.replicaCount }} {{- end }} + {{- $minReady := .Values.backend.minReadySeconds }} + {{- if not (or (kindIs "invalid" $minReady) (eq (printf "%v" $minReady) "")) }} + minReadySeconds: {{ $minReady }} + {{- end }} {{- with .Values.backend.strategy }} strategy: {{- toYaml . | nindent 4 }} @@ -46,14 +57,24 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.backend.extraInitContainers }} + initContainers: + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} containers: - name: backend - image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.backend.image.pullPolicy }} + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.backend.securityContext }} securityContext: {{- toYaml . | nindent 12 }} {{- end }} + args: + - backend + - --host + - 0.0.0.0 + - --port + - "4001" ports: - name: http containerPort: 4001 @@ -66,6 +87,8 @@ spec: - name: LITELLM_CORS_ALLOW_CREDENTIALS value: "true" {{- end }} + {{- include "litellm.masterKeyEnv" $ | nindent 12 }} + {{- include "litellm.liteadminEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }} {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }} {{- if .Values.backend.gatewayUrl }} - name: LITELLM_GATEWAY_URL @@ -116,7 +139,7 @@ spec: {{- if .Values.gateway.config.create }} - name: gateway-config configMap: - name: {{ include "litellm.gateway.fullname" . }}-config + name: {{ include "litellm.workload.fullname" . }}-config {{- end }} {{- with .Values.backend.volumes }} {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/templates/backend/hpa.yaml b/helm/litellm/templates/backend/hpa.yaml index a414092fb39..62efaacb830 100644 --- a/helm/litellm/templates/backend/hpa.yaml +++ b/helm/litellm/templates/backend/hpa.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.backend.enabled .Values.backend.hpa.enabled }} +{{- if and (include "litellm.backend.render" .) .Values.backend.hpa.enabled }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: diff --git a/helm/litellm/templates/backend/poddisruptionbudget.yaml b/helm/litellm/templates/backend/poddisruptionbudget.yaml index 02853ac879c..ed40bda0d7c 100644 --- a/helm/litellm/templates/backend/poddisruptionbudget.yaml +++ b/helm/litellm/templates/backend/poddisruptionbudget.yaml @@ -1,5 +1,6 @@ {{- include "litellm.pdb" (dict "root" $ + "enabled" (include "litellm.backend.render" .) "component" .Values.backend "componentName" "backend" "fullname" (include "litellm.backend.fullname" .) diff --git a/helm/litellm/templates/backend/service.yaml b/helm/litellm/templates/backend/service.yaml index d480c654784..cd11ad95ccf 100644 --- a/helm/litellm/templates/backend/service.yaml +++ b/helm/litellm/templates/backend/service.yaml @@ -1,4 +1,4 @@ -{{- if .Values.backend.enabled }} +{{- if include "litellm.backend.render" . }} apiVersion: v1 kind: Service metadata: @@ -6,8 +6,15 @@ metadata: labels: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: backend + {{- with .Values.backend.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.backend.service.type }} + {{- with include "litellm.service.extras" .Values.backend.service }} + {{- . | nindent 2 }} + {{- end }} ports: - port: {{ .Values.backend.service.port }} targetPort: http diff --git a/helm/litellm/templates/extra-resources.yaml b/helm/litellm/templates/extra-resources.yaml new file mode 100644 index 00000000000..bd809ffb696 --- /dev/null +++ b/helm/litellm/templates/extra-resources.yaml @@ -0,0 +1,4 @@ +{{- range .Values.extraResources }} +--- +{{ tpl (toYaml .) $ }} +{{- end }} diff --git a/helm/litellm/templates/gateway/configmap.yaml b/helm/litellm/templates/gateway/configmap.yaml index d262bf25b87..8944aaa0a2f 100644 --- a/helm/litellm/templates/gateway/configmap.yaml +++ b/helm/litellm/templates/gateway/configmap.yaml @@ -2,7 +2,10 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "litellm.gateway.fullname" . }}-config + name: {{ include "litellm.workload.fullname" . }}-config + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ include "litellm.workload.componentName" . }} data: config.yaml: | {{ .Values.gateway.config.proxy_config | toYaml | indent 6 }} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index 987109e361e..9db08850e26 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -1,246 +1,3 @@ -{{- if .Values.gateway.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "litellm.gateway.fullname" . }} - labels: - {{- include "litellm.commonLabels" . | nindent 4 }} - app.kubernetes.io/component: gateway -spec: - {{- if and (not .Values.gateway.hpa.enabled) (not (kindIs "invalid" .Values.gateway.replicaCount)) }} - replicas: {{ .Values.gateway.replicaCount }} - {{- end }} - {{- with .Values.gateway.strategy }} - strategy: - {{- toYaml . | nindent 4 }} - {{- end }} - selector: - matchLabels: - {{- include "litellm.gateway.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - {{- if .Values.gateway.config.create }} - checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }} - {{- end }} - {{- with .Values.gateway.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "litellm.gateway.selectorLabels" . | nindent 8 }} - {{- with .Values.gateway.podLabels }} - {{- include "litellm.podLabels" (dict "podLabels" . "componentName" "gateway") | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }} - automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }} - {{- with .Values.gateway.podSecurityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: gateway - image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} - {{- with .Values.gateway.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 4000 - protocol: TCP - env: - {{- include "litellm.lensConnectionEnv" . | nindent 12 }} - {{- if .Values.ui.publicUrl }} - - name: LITELLM_CORS_ORIGINS - value: {{ include "litellm.ui.publicOrigin" . | quote }} - {{- end }} - {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} - {{- if .Values.gateway.config.create }} - - name: CONFIG_FILE_PATH - value: /app/config/config.yaml - {{- end }} - {{- if .Values.gateway.numWorkers }} - - name: NUM_WORKERS - value: {{ .Values.gateway.numWorkers | quote }} - {{- end }} - {{- if .Values.database.connectionPool.enabled }} - {{- include "litellm.connectionPoolEnv" $ | nindent 12 }} - {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsEnv" . | nindent 12 }} - {{- end }} - {{- if .Values.gateway.metricsServer.enabled }} - {{- if eq (int .Values.gateway.metricsServer.port) 4000 }} - {{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }} - {{- end }} - - name: PROMETHEUS_MULTIPROC_DIR - value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} - {{- end }} - {{- if .Values.gateway.collector.enabled }} - {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} - {{- end }} - {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} - volumeMounts: - {{- if .Values.gateway.config.create }} - - name: gateway-config - mountPath: /app/config/config.yaml - subPath: config.yaml - {{- end }} - {{- if .Values.gateway.metricsServer.enabled }} - - name: prometheus-multiproc - mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} - {{- end }} - {{- if include "litellm.gateway.collectorSocketDir" . }} - - name: collector-socket - mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} - {{- end }} - {{- with .Values.gateway.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- end }} - {{- with .Values.gateway.livenessProbe }} - livenessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.gateway.readinessProbe }} - readinessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.gateway.startupProbe }} - startupProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.gateway.lifecycle }} - lifecycle: - {{- toYaml . | nindent 12 }} - {{- end }} - resources: - {{- toYaml .Values.gateway.resources | nindent 12 }} - {{- if .Values.gateway.metricsServer.enabled }} - - name: metrics - image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} - {{- with .Values.gateway.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - command: - - python - - -m - - litellm.proxy.prometheus_metrics_server - - --port - - {{ .Values.gateway.metricsServer.port | quote }} - env: - - name: PROMETHEUS_MULTIPROC_DIR - value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} - ports: - - name: metrics - containerPort: {{ .Values.gateway.metricsServer.port }} - protocol: TCP - volumeMounts: - - name: prometheus-multiproc - mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} - readinessProbe: - tcpSocket: { port: metrics } - periodSeconds: 10 - livenessProbe: - tcpSocket: { port: metrics } - periodSeconds: 15 - failureThreshold: 6 - resources: - {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }} - {{- end }} - {{- if .Values.gateway.collector.enabled }} - - name: collector - image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} - {{- with .Values.gateway.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - command: - - python - - -m - - litellm.proxy.collector - env: - {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} - {{- if .Values.gateway.config.create }} - - name: CONFIG_FILE_PATH - value: /app/config/config.yaml - {{- end }} - {{- if .Values.database.connectionPool.enabled }} - {{- include "litellm.connectionPoolEnv" $ | nindent 12 }} - {{- end }} - {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} - - name: LITELLM_JOB_ROLE - value: collector - {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts (include "litellm.gateway.collectorSocketDir" .) }} - volumeMounts: - {{- if .Values.gateway.config.create }} - - name: gateway-config - mountPath: /app/config/config.yaml - subPath: config.yaml - {{- end }} - {{- if include "litellm.gateway.collectorSocketDir" . }} - - name: collector-socket - mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} - {{- end }} - {{- with .Values.gateway.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- end }} - resources: - {{- toYaml .Values.gateway.collector.resources | nindent 12 }} - {{- end }} - {{- with .Values.gateway.extraContainers }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} - volumes: - {{- if .Values.gateway.config.create }} - - name: gateway-config - configMap: - name: {{ include "litellm.gateway.fullname" . }}-config - {{- end }} - {{- if .Values.gateway.metricsServer.enabled }} - - name: prometheus-multiproc - emptyDir: {} - {{- end }} - {{- if include "litellm.gateway.collectorSocketDir" . }} - - name: collector-socket - emptyDir: - sizeLimit: 1Mi - {{- end }} - {{- with .Values.gateway.volumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- end }} - {{- with .Values.gateway.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }} - {{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }} - terminationGracePeriodSeconds: {{ $gracePeriod }} - {{- end }} +{{- if include "litellm.gateway.render" . }} +{{- include "litellm.workload.deployment" . }} {{- end }} diff --git a/helm/litellm/templates/gateway/hpa.yaml b/helm/litellm/templates/gateway/hpa.yaml index e7094e96106..d416a6056bd 100644 --- a/helm/litellm/templates/gateway/hpa.yaml +++ b/helm/litellm/templates/gateway/hpa.yaml @@ -1,65 +1,3 @@ -{{- if and .Values.gateway.enabled .Values.gateway.hpa.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "litellm.gateway.fullname" . }} - labels: - {{- include "litellm.commonLabels" . | nindent 4 }} - app.kubernetes.io/component: gateway -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "litellm.gateway.fullname" . }} - minReplicas: {{ .Values.gateway.hpa.minReplicas }} - maxReplicas: {{ .Values.gateway.hpa.maxReplicas }} - metrics: - {{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }} - {{- if and .Values.gateway.collector.enabled .Values.gateway.collector.scaleOnGatewayContainerCpu }} - - type: ContainerResource - containerResource: - name: cpu - container: gateway - target: - type: Utilization - averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} - {{- else }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} - {{- end }} - {{- end }} - {{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }} - {{- end }} - {{- with .Values.gateway.hpa.targetRequestsPerSecond }} - - type: Pods - pods: - metric: - name: litellm_requests_per_second - target: - type: AverageValue - averageValue: {{ toJson . | trimAll "\"" | quote }} - {{- end }} - {{- with .Values.gateway.hpa.targetTokensPerSecond }} - - type: Pods - pods: - metric: - name: litellm_tokens_per_second - target: - type: AverageValue - averageValue: {{ toJson . | trimAll "\"" | quote }} - {{- end }} - {{- with .Values.gateway.hpa.behavior }} - behavior: - {{- toYaml . | nindent 4 }} - {{- end }} +{{- if and (include "litellm.gateway.render" .) .Values.gateway.hpa.enabled }} +{{- include "litellm.workload.hpa" . }} {{- end }} diff --git a/helm/litellm/templates/gateway/keda.yaml b/helm/litellm/templates/gateway/keda.yaml new file mode 100644 index 00000000000..d46085ffef1 --- /dev/null +++ b/helm/litellm/templates/gateway/keda.yaml @@ -0,0 +1,3 @@ +{{- if and (include "litellm.gateway.render" .) .Values.gateway.keda.enabled }} +{{- include "litellm.workload.keda" . }} +{{- end }} diff --git a/helm/litellm/templates/gateway/poddisruptionbudget.yaml b/helm/litellm/templates/gateway/poddisruptionbudget.yaml index 15e89af17d7..313e3cc7535 100644 --- a/helm/litellm/templates/gateway/poddisruptionbudget.yaml +++ b/helm/litellm/templates/gateway/poddisruptionbudget.yaml @@ -1,5 +1,6 @@ {{- include "litellm.pdb" (dict "root" $ + "enabled" (include "litellm.gateway.render" .) "component" .Values.gateway "componentName" "gateway" "fullname" (include "litellm.gateway.fullname" .) diff --git a/helm/litellm/templates/gateway/service-metrics.yaml b/helm/litellm/templates/gateway/service-metrics.yaml index ad9bc05a9fd..e33ae8c63e3 100644 --- a/helm/litellm/templates/gateway/service-metrics.yaml +++ b/helm/litellm/templates/gateway/service-metrics.yaml @@ -1,18 +1,3 @@ -{{- if and .Values.gateway.enabled .Values.gateway.metricsServer.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "litellm.gateway.fullname" . }}-metrics - labels: - {{- include "litellm.commonLabels" . | nindent 4 }} - app.kubernetes.io/component: gateway -spec: - type: ClusterIP - ports: - - port: {{ .Values.gateway.metricsServer.port }} - targetPort: metrics - protocol: TCP - name: metrics - selector: - {{- include "litellm.gateway.selectorLabels" . | nindent 4 }} +{{- if and (include "litellm.gateway.render" .) .Values.gateway.metricsServer.enabled }} +{{- include "litellm.workload.serviceMetrics" . }} {{- end }} diff --git a/helm/litellm/templates/gateway/service.yaml b/helm/litellm/templates/gateway/service.yaml index 03a4167a0ab..8f3e7ad0673 100644 --- a/helm/litellm/templates/gateway/service.yaml +++ b/helm/litellm/templates/gateway/service.yaml @@ -1,18 +1,3 @@ -{{- if .Values.gateway.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "litellm.gateway.fullname" . }} - labels: - {{- include "litellm.commonLabels" . | nindent 4 }} - app.kubernetes.io/component: gateway -spec: - type: {{ .Values.gateway.service.type }} - ports: - - port: {{ .Values.gateway.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "litellm.gateway.selectorLabels" . | nindent 4 }} +{{- if include "litellm.gateway.render" . }} +{{- include "litellm.workload.service" . }} {{- end }} diff --git a/helm/litellm/templates/gateway/servicemonitor.yaml b/helm/litellm/templates/gateway/servicemonitor.yaml index e1bafa6e388..3e5d538b4e0 100644 --- a/helm/litellm/templates/gateway/servicemonitor.yaml +++ b/helm/litellm/templates/gateway/servicemonitor.yaml @@ -1,28 +1,3 @@ -{{- if and .Values.gateway.enabled .Values.gateway.serviceMonitor.enabled }} -{{- if not .Values.gateway.metricsServer.enabled }} -{{- fail "gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled: the http port serves /metrics/ behind virtual-key auth, so an unauthenticated scrape gets 401" }} -{{- end }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ include "litellm.gateway.fullname" . }} - labels: - {{- include "litellm.commonLabels" . | nindent 4 }} - app.kubernetes.io/component: gateway - {{- with .Values.gateway.serviceMonitor.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - {{- include "litellm.gateway.selectorLabels" . | nindent 6 }} - namespaceSelector: - matchNames: - - {{ .Release.Namespace | quote }} - endpoints: - - port: metrics - path: /metrics/ - interval: {{ .Values.gateway.serviceMonitor.interval }} - scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }} - scheme: http +{{- if and (include "litellm.gateway.render" .) .Values.gateway.serviceMonitor.enabled }} +{{- include "litellm.workload.serviceMonitor" . }} {{- end }} diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index 9f778e78087..3fb138aacdb 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -4,6 +4,7 @@ {{- if not (and $componentHosts.ui $componentHosts.backend $componentHosts.gateway) }} {{- fail "ingress.componentHosts.ui, backend, and gateway must all be set together" }} {{- end }} +{{- if not .Values.monolith.enabled }} {{- if not .Values.gateway.enabled }} {{- fail "ingress.componentHosts requires gateway.enabled=true" }} {{- end }} @@ -13,12 +14,14 @@ {{- if not .Values.ui.enabled }} {{- fail "ingress.componentHosts requires ui.enabled=true" }} {{- end }} +{{- end }} {{- if .Values.ingress.host }} {{- fail "ingress.host must be empty when ingress.componentHosts is configured" }} {{- end }} {{- if .Values.ingress.extraPaths }} {{- fail "ingress.extraPaths must be empty when ingress.componentHosts is configured" }} {{- end }} +{{- if not .Values.monolith.enabled }} {{- if not (trim .Values.ui.backendUrl) }} {{- fail "ui.backendUrl is required when ingress.componentHosts is configured" }} {{- end }} @@ -31,24 +34,30 @@ {{- fail "ui.publicUrl is required when ingress.componentHosts is configured" }} {{- end }} {{- end }} +{{- end }} {{- if .Values.ingress.enabled -}} +{{- $monolith := .Values.monolith.enabled -}} {{- $gatewayName := include "litellm.gateway.fullname" . -}} {{- $backendName := include "litellm.backend.fullname" . -}} {{- $uiName := include "litellm.ui.fullname" . -}} {{- $gatewayPort := .Values.gateway.service.port -}} {{- $backendPort := .Values.backend.service.port -}} {{- $uiPort := .Values.ui.service.port -}} +{{- $proxyName := include "litellm.proxy.fullname" . -}} {{- $controller := .Values.ingress.controller | default "alb" -}} {{- if not (has $controller (list "alb" "nginx")) }} {{- fail (printf "ingress.controller: unknown controller %q, expected one of alb, nginx" $controller) }} {{- end }} {{/* Backends addressable from ingress.extraPaths, keyed by the `service` field. + In monolith mode the one proxy Service serves every component, so every + key resolves to it and an entry's `service` only has to be a known name. */}} +{{- $proxyBackend := dict "name" $proxyName "port" $gatewayPort -}} {{- $extraPathBackends := dict - "gateway" (dict "name" $gatewayName "port" $gatewayPort) - "backend" (dict "name" $backendName "port" $backendPort) - "ui" (dict "name" $uiName "port" $uiPort) + "gateway" (ternary $proxyBackend (dict "name" $gatewayName "port" $gatewayPort) $monolith) + "backend" (ternary $proxyBackend (dict "name" $backendName "port" $backendPort) $monolith) + "ui" (ternary $proxyBackend (dict "name" $uiName "port" $uiPort) $monolith) -}} {{/* UI paths (Next.js static export). @@ -122,7 +131,7 @@ at "/" Prefix would swallow the whole backend management API) instead of adding to it. */}} -{{- $builtinPathKeys := list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix" -}} +{{- $builtinPathKeys := ternary (list "/|Prefix") (list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix") $monolith -}} {{- if $hasComponentHosts }} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -150,9 +159,9 @@ spec: pathType: Prefix backend: service: - name: {{ $uiName }} + name: {{ $extraPathBackends.ui.name }} port: - number: {{ $uiPort }} + number: {{ $extraPathBackends.ui.port }} - host: {{ $componentHosts.backend | quote }} http: paths: @@ -160,9 +169,9 @@ spec: pathType: Prefix backend: service: - name: {{ $backendName }} + name: {{ $extraPathBackends.backend.name }} port: - number: {{ $backendPort }} + number: {{ $extraPathBackends.backend.port }} - host: {{ $componentHosts.gateway | quote }} http: paths: @@ -179,9 +188,9 @@ spec: pathType: Prefix backend: service: - name: {{ $gatewayName }} + name: {{ $extraPathBackends.gateway.name }} port: - number: {{ $gatewayPort }} + number: {{ $extraPathBackends.gateway.port }} {{- else }} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -207,6 +216,7 @@ spec: {{- end }} http: paths: + {{- if not $monolith }} # --- UI (Next.js static export) --- {{- range $uiPaths }} {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" .path "pathType" .pathType) }} @@ -249,6 +259,7 @@ spec: port: number: {{ $gatewayPort }} {{- end }} + {{- end }} {{- if and .Values.lens.enabled .Values.lens.service.name }} {{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }} - path: /lens-ingest @@ -324,6 +335,16 @@ spec: port: {{- toYaml $port | nindent 18 }} {{- end }} + {{- if $monolith }} + # --- Monolith: the proxy serves the UI, the data plane and the management API --- + - path: / + pathType: Prefix + backend: + service: + name: {{ $proxyName }} + port: + number: {{ $gatewayPort }} + {{- else }} # --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) --- - path: / pathType: Prefix @@ -332,5 +353,6 @@ spec: name: {{ $backendName }} port: number: {{ $backendPort }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/litellm-helm/templates/liteadmin.yaml b/helm/litellm/templates/liteadmin.yaml similarity index 96% rename from helm/litellm-helm/templates/liteadmin.yaml rename to helm/litellm/templates/liteadmin.yaml index 711edaf6913..bc5f2c9510b 100644 --- a/helm/litellm-helm/templates/liteadmin.yaml +++ b/helm/litellm/templates/liteadmin.yaml @@ -32,7 +32,7 @@ spec: runAsNonRoot: true containers: - name: liteadmin - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: {{ include "litellm.image" . | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: ["--admin-agent"] securityContext: diff --git a/helm/litellm/templates/migrations-job.yaml b/helm/litellm/templates/migrations-job.yaml index de1cc2b103b..8179c26cbd7 100644 --- a/helm/litellm/templates/migrations-job.yaml +++ b/helm/litellm/templates/migrations-job.yaml @@ -1,7 +1,7 @@ {{- if .Values.migrationJob.enabled -}} # Pre-install / pre-upgrade hook that runs `prisma migrate deploy` against -# the writer database before the gateway and backend Deployments are rolled -# out. Required because the gateway and backend both spin up Prisma at +# the writer database before the application Deployments are rolled out. +# Required because the proxy, gateway and backend all spin up Prisma at # startup and assume the LiteLLM schema (LiteLLM_Config, # LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...) already exists. # @@ -16,9 +16,13 @@ metadata: labels: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: migrations - {{- if or .Values.migrationJob.hooks.helm.enabled .Values.migrationJob.hooks.argocd.enabled }} + {{- with .Values.migrationJob.jobLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- $helmHook := .Values.migrationJob.hooks.helm.enabled }} + {{- if or $helmHook .Values.migrationJob.hooks.argocd.enabled .Values.migrationJob.jobAnnotations }} annotations: - {{- if .Values.migrationJob.hooks.helm.enabled }} + {{- if $helmHook }} helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-delete-policy: before-hook-creation helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "0" | quote }} @@ -27,6 +31,9 @@ metadata: argocd.argoproj.io/hook: PreSync argocd.argoproj.io/hook-delete-policy: BeforeHookCreation {{- end }} + {{- with .Values.migrationJob.jobAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} spec: backoffLimit: {{ .Values.migrationJob.backoffLimit }} @@ -43,6 +50,10 @@ spec: {{- $chartLabels := merge (dict "app.kubernetes.io/component" "migrations") (fromYaml (include "litellm.commonLabels" .)) }} labels: {{- toYaml (merge (deepCopy .Values.migrationJob.podLabels) $chartLabels) | nindent 8 }} + {{- with .Values.migrationJob.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: restartPolicy: Never serviceAccountName: {{ include "litellm.migrations.serviceAccountName" . }} @@ -55,14 +66,29 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.migrationJob.extraInitContainers }} + initContainers: + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} containers: - name: prisma-migrations - image: "{{ .Values.migrationJob.image.repository }}:{{ .Values.migrationJob.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.migrationJob.image.pullPolicy }} + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.migrationJob.securityContext }} securityContext: {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.migrationJob.command }} + command: {{ toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.migrationJob.args }} + args: {{ toYaml .Values.migrationJob.args | nindent 12 }} + {{- else if .Values.migrationJob.command }} + args: [] + {{- else }} + args: + - migrations + {{- end }} env: {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.migrationJob) | nindent 12 }} {{- with .Values.migrationJob.volumeMounts }} @@ -73,6 +99,9 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.migrationJob.extraContainers }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} {{- with .Values.migrationJob.volumes }} volumes: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/templates/monolith/deployment.yaml b/helm/litellm/templates/monolith/deployment.yaml new file mode 100644 index 00000000000..7bad16386e1 --- /dev/null +++ b/helm/litellm/templates/monolith/deployment.yaml @@ -0,0 +1,3 @@ +{{- if .Values.monolith.enabled }} +{{- include "litellm.workload.deployment" . }} +{{- end }} diff --git a/helm/litellm/templates/monolith/hpa.yaml b/helm/litellm/templates/monolith/hpa.yaml new file mode 100644 index 00000000000..ee9180e71bb --- /dev/null +++ b/helm/litellm/templates/monolith/hpa.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.monolith.enabled .Values.gateway.hpa.enabled }} +{{- include "litellm.workload.hpa" . }} +{{- end }} diff --git a/helm/litellm/templates/monolith/keda.yaml b/helm/litellm/templates/monolith/keda.yaml new file mode 100644 index 00000000000..d4054432e38 --- /dev/null +++ b/helm/litellm/templates/monolith/keda.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.monolith.enabled .Values.gateway.keda.enabled }} +{{- include "litellm.workload.keda" . }} +{{- end }} diff --git a/helm/litellm/templates/monolith/poddisruptionbudget.yaml b/helm/litellm/templates/monolith/poddisruptionbudget.yaml new file mode 100644 index 00000000000..2be83e57dfc --- /dev/null +++ b/helm/litellm/templates/monolith/poddisruptionbudget.yaml @@ -0,0 +1,7 @@ +{{- include "litellm.pdb" (dict + "root" $ + "enabled" .Values.monolith.enabled + "component" .Values.gateway + "componentName" "proxy" + "fullname" (include "litellm.proxy.fullname" .) + "selectorLabels" (include "litellm.proxy.selectorLabels" .)) }} diff --git a/helm/litellm/templates/monolith/service-metrics.yaml b/helm/litellm/templates/monolith/service-metrics.yaml new file mode 100644 index 00000000000..07a68c453df --- /dev/null +++ b/helm/litellm/templates/monolith/service-metrics.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.monolith.enabled .Values.gateway.metricsServer.enabled }} +{{- include "litellm.workload.serviceMetrics" . }} +{{- end }} diff --git a/helm/litellm/templates/monolith/service.yaml b/helm/litellm/templates/monolith/service.yaml new file mode 100644 index 00000000000..4f87259983a --- /dev/null +++ b/helm/litellm/templates/monolith/service.yaml @@ -0,0 +1,3 @@ +{{- if .Values.monolith.enabled }} +{{- include "litellm.workload.service" . }} +{{- end }} diff --git a/helm/litellm/templates/monolith/servicemonitor.yaml b/helm/litellm/templates/monolith/servicemonitor.yaml new file mode 100644 index 00000000000..29a51a379f2 --- /dev/null +++ b/helm/litellm/templates/monolith/servicemonitor.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.monolith.enabled .Values.gateway.serviceMonitor.enabled }} +{{- include "litellm.workload.serviceMonitor" . }} +{{- end }} diff --git a/helm/litellm/templates/secret-masterkey.yaml b/helm/litellm/templates/secret-masterkey.yaml new file mode 100644 index 00000000000..3af2ee6f07d --- /dev/null +++ b/helm/litellm/templates/secret-masterkey.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.masterKey.generate (not .Values.masterKey.secretName) }} +{{- $name := include "litellm.masterKey.generatedSecretName" . }} +{{- $key := .Values.masterKey.secretKey | default "master-key" }} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }} +{{- $value := "" }} +{{- if and $existing (index $existing.data $key) }} +{{- $value = index $existing.data $key | b64dec }} +{{- else }} +{{- $value = printf "sk-%s" (randAlphaNum 32) }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + {{ $key }}: {{ $value | b64enc }} +{{- end }} diff --git a/helm/litellm/templates/serviceaccount.yaml b/helm/litellm/templates/serviceaccount.yaml index a2fc52f47c0..9b1171a3e90 100644 --- a/helm/litellm/templates/serviceaccount.yaml +++ b/helm/litellm/templates/serviceaccount.yaml @@ -1,5 +1,5 @@ {{- $prev := false -}} -{{- if .Values.serviceAccounts.gateway.create -}} +{{- if and .Values.serviceAccounts.gateway.create (include "litellm.workload.render" .) -}} {{- $prev = true }} apiVersion: v1 kind: ServiceAccount @@ -14,7 +14,7 @@ metadata: {{- end }} automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }} {{- end }} -{{- if .Values.serviceAccounts.backend.create }} +{{- if and .Values.serviceAccounts.backend.create (include "litellm.backend.render" .) }} {{- if $prev }} --- {{- end }} @@ -32,7 +32,7 @@ metadata: {{- end }} automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }} {{- end }} -{{- if .Values.serviceAccounts.ui.create }} +{{- if and .Values.serviceAccounts.ui.create (include "litellm.ui.render" .) }} {{- if $prev }} --- {{- end }} diff --git a/helm/litellm/templates/tests/test-connection.yaml b/helm/litellm/templates/tests/test-connection.yaml new file mode 100644 index 00000000000..bc258c92790 --- /dev/null +++ b/helm/litellm/templates/tests/test-connection.yaml @@ -0,0 +1,53 @@ +{{- if include "litellm.workload.render" . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "litellm.fullname" . }}-test-connection + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + helm.sh/hook: test + helm.sh/hook-delete-policy: before-hook-creation +spec: + restartPolicy: Never + {{- with .Values.gateway.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: health + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + env: + - name: LITELLM_READINESS_URL + value: http://{{ include "litellm.workload.fullname" . }}:{{ .Values.gateway.service.port }}/health/readiness + args: + - python + - -c + - | + import json, os, sys, time, urllib.error, urllib.request + url = os.environ["LITELLM_READINESS_URL"] + for attempt in range(1, 31): + try: + with urllib.request.urlopen(url, timeout=5) as response: + body = response.read().decode() + print(url, response.status, body) + db = json.loads(body).get("db") + if db != "connected": + print(f"readiness reports db={db!r}: the proxy is not using the configured database") + sys.exit(1) + sys.exit(0) + except (urllib.error.URLError, OSError) as error: + print(f"attempt {attempt}: {url}: {error}") + time.sleep(2) + sys.exit(1) +{{- end }} diff --git a/helm/litellm/templates/ui/deployment.yaml b/helm/litellm/templates/ui/deployment.yaml index 394eea17451..486e1dae5c8 100644 --- a/helm/litellm/templates/ui/deployment.yaml +++ b/helm/litellm/templates/ui/deployment.yaml @@ -1,6 +1,6 @@ +{{- if include "litellm.ui.render" . }} {{- $componentHosts := .Values.ingress.componentHosts | default dict -}} {{- $hasComponentHosts := or $componentHosts.ui $componentHosts.backend $componentHosts.gateway -}} -{{- if .Values.ui.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -8,10 +8,21 @@ metadata: labels: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: ui + {{- with .Values.ui.deploymentLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ui.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- if and (not .Values.ui.hpa.enabled) (not (kindIs "invalid" .Values.ui.replicaCount)) }} replicas: {{ .Values.ui.replicaCount }} {{- end }} + {{- $minReady := .Values.ui.minReadySeconds }} + {{- if not (or (kindIs "invalid" $minReady) (eq (printf "%v" $minReady) "")) }} + minReadySeconds: {{ $minReady }} + {{- end }} {{- with .Values.ui.strategy }} strategy: {{- toYaml . | nindent 4 }} @@ -41,14 +52,20 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.ui.extraInitContainers }} + initContainers: + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} containers: - name: ui - image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.ui.image.pullPolicy }} + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.ui.securityContext }} securityContext: {{- toYaml . | nindent 12 }} {{- end }} + args: + - ui ports: - name: http containerPort: 3000 diff --git a/helm/litellm/templates/ui/hpa.yaml b/helm/litellm/templates/ui/hpa.yaml index a9b0b51129e..67e17b269c7 100644 --- a/helm/litellm/templates/ui/hpa.yaml +++ b/helm/litellm/templates/ui/hpa.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.ui.enabled .Values.ui.hpa.enabled }} +{{- if and (include "litellm.ui.render" .) .Values.ui.hpa.enabled }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: diff --git a/helm/litellm/templates/ui/poddisruptionbudget.yaml b/helm/litellm/templates/ui/poddisruptionbudget.yaml index f7a3a694e9c..fa4bb44612c 100644 --- a/helm/litellm/templates/ui/poddisruptionbudget.yaml +++ b/helm/litellm/templates/ui/poddisruptionbudget.yaml @@ -1,5 +1,6 @@ {{- include "litellm.pdb" (dict "root" $ + "enabled" (include "litellm.ui.render" .) "component" .Values.ui "componentName" "ui" "fullname" (include "litellm.ui.fullname" .) diff --git a/helm/litellm/templates/ui/service.yaml b/helm/litellm/templates/ui/service.yaml index 52b539fa00c..d544d9d6148 100644 --- a/helm/litellm/templates/ui/service.yaml +++ b/helm/litellm/templates/ui/service.yaml @@ -1,4 +1,4 @@ -{{- if .Values.ui.enabled }} +{{- if include "litellm.ui.render" . }} apiVersion: v1 kind: Service metadata: @@ -6,8 +6,15 @@ metadata: labels: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: ui + {{- with .Values.ui.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.ui.service.type }} + {{- with include "litellm.service.extras" .Values.ui.service }} + {{- . | nindent 2 }} + {{- end }} ports: - port: {{ .Values.ui.service.port }} targetPort: http diff --git a/helm/litellm/tests/collector_tests.yaml b/helm/litellm/tests/collector_tests.yaml index 4ef7e3c8ca4..8e9804dece8 100644 --- a/helm/litellm/tests/collector_tests.yaml +++ b/helm/litellm/tests/collector_tests.yaml @@ -34,7 +34,7 @@ tests: gateway.collector.enabled: true gateway.collector.bufferSize: 250 gateway.collector.onUnavailable: drop - gateway.image.tag: v1.102.0 + image.tag: v1.102.0 gateway.numWorkers: 4 database.connectionPool.enabled: true database.connectionPool.maxDbConnections: 8 @@ -80,14 +80,12 @@ tests: template: gateway/deployment.yaml - equal: path: spec.template.spec.containers[1].image - value: ghcr.io/berriai/litellm-gateway:v1.102.0 + value: ghcr.io/berriai/litellm:v1.102.0 template: gateway/deployment.yaml - equal: - path: spec.template.spec.containers[1].command + path: spec.template.spec.containers[1].args value: - - python - - -m - - litellm.proxy.collector + - collector template: gateway/deployment.yaml - contains: path: spec.template.spec.containers[1].env diff --git a/helm/litellm/tests/component_hosts_tests.yaml b/helm/litellm/tests/component_hosts_tests.yaml index b55b3b694a0..047f1e7006d 100644 --- a/helm/litellm/tests/component_hosts_tests.yaml +++ b/helm/litellm/tests/component_hosts_tests.yaml @@ -316,16 +316,16 @@ tests: path: spec.template.spec.containers[0].env[1].value value: "true" - equal: - path: spec.template.spec.containers[0].env[10].name + path: spec.template.spec.containers[0].env[12].name value: LITELLM_CORS_ORIGINS - equal: - path: spec.template.spec.containers[0].env[10].value + path: spec.template.spec.containers[0].env[12].value value: https://backend-override.example.com - equal: - path: spec.template.spec.containers[0].env[11].name + path: spec.template.spec.containers[0].env[13].name value: LITELLM_CORS_ALLOW_CREDENTIALS - equal: - path: spec.template.spec.containers[0].env[11].value + path: spec.template.spec.containers[0].env[13].value value: "false" - it: exposes the public UI origin to the gateway without enabling credentials @@ -389,10 +389,10 @@ tests: path: spec.template.spec.containers[0].env[0].value value: https://ui.example.com - equal: - path: spec.template.spec.containers[0].env[9].name + path: spec.template.spec.containers[0].env[11].name value: LITELLM_CORS_ORIGINS - equal: - path: spec.template.spec.containers[0].env[9].value + path: spec.template.spec.containers[0].env[11].value value: https://gateway-override.example.com --- diff --git a/helm/litellm/tests/extra_resources_tests.yaml b/helm/litellm/tests/extra_resources_tests.yaml new file mode 100644 index 00000000000..4935b3a5401 --- /dev/null +++ b/helm/litellm/tests/extra_resources_tests.yaml @@ -0,0 +1,45 @@ +suite: test extraResources +templates: + - extra-resources.yaml +values: + - ./values/required.yaml +tests: + - it: renders nothing by default + asserts: + - hasDocuments: + count: 0 + + - it: renders every entry as its own document and templates it against the release + set: + extraResources: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: "{{ .Release.Name }}-extra" + data: + chart: "{{ .Chart.Name }}" + - apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: deny-all + spec: + podSelector: {} + policyTypes: + - Ingress + asserts: + - hasDocuments: + count: 2 + - isKind: + of: ConfigMap + documentIndex: 0 + - equal: + path: metadata.name + value: RELEASE-NAME-extra + documentIndex: 0 + - equal: + path: data.chart + value: litellm + documentIndex: 0 + - isKind: + of: NetworkPolicy + documentIndex: 1 diff --git a/helm/litellm/tests/hpa_workload_metrics_tests.yaml b/helm/litellm/tests/hpa_workload_metrics_tests.yaml index a29c53c5ed9..a6c17942a11 100644 --- a/helm/litellm/tests/hpa_workload_metrics_tests.yaml +++ b/helm/litellm/tests/hpa_workload_metrics_tests.yaml @@ -201,6 +201,50 @@ tests: scrapeTimeout: 10s scheme: http + - it: carries litellm-helm annotations, relabelings and namespaceSelector onto the ServiceMonitor + template: gateway/servicemonitor.yaml + release: + namespace: llm + set: + gateway.serviceMonitor.enabled: true + gateway.metricsServer.enabled: true + gateway.serviceMonitor.annotations: + team: platform + gateway.serviceMonitor.relabelings: + - targetLabel: node + sourceLabels: [__meta_kubernetes_pod_node_name] + action: replace + gateway.serviceMonitor.namespaceSelector.matchNames: + - llm + - shared + asserts: + - equal: + path: metadata.annotations + value: + team: platform + - equal: + path: spec.endpoints[0].relabelings + value: + - targetLabel: node + sourceLabels: [__meta_kubernetes_pod_node_name] + action: replace + - equal: + path: spec.namespaceSelector.matchNames + value: + - llm + - shared + + - it: renders no ServiceMonitor annotations or relabelings by default + template: gateway/servicemonitor.yaml + set: + gateway.serviceMonitor.enabled: true + gateway.metricsServer.enabled: true + asserts: + - notExists: + path: metadata.annotations + - notExists: + path: spec.endpoints[0].relabelings + - it: honours a custom scrape interval template: gateway/servicemonitor.yaml set: diff --git a/helm/litellm/tests/ingress_monolith_tests.yaml b/helm/litellm/tests/ingress_monolith_tests.yaml new file mode 100644 index 00000000000..08f7c598260 --- /dev/null +++ b/helm/litellm/tests/ingress_monolith_tests.yaml @@ -0,0 +1,164 @@ +suite: test ingress routing in monolith mode +templates: + - ingress.yaml +values: + - ./values/required.yaml +tests: + - it: monolith mode routes all component hosts to the proxy Service + set: + monolith.enabled: true + ingress.enabled: true + gateway.enabled: false + backend.enabled: false + ui.enabled: false + ingress.componentHosts.ui: ui.example.com + ingress.componentHosts.backend: backend.example.com + ingress.componentHosts.gateway: gateway.example.com + gateway.service.port: 8080 + asserts: + - lengthEqual: + path: spec.rules + count: 3 + - equal: + path: spec.rules[0].http.paths[0].backend.service + value: + name: RELEASE-NAME-litellm + port: + number: 8080 + - equal: + path: spec.rules[1].http.paths[0].backend.service + value: + name: RELEASE-NAME-litellm + port: + number: 8080 + - equal: + path: spec.rules[2].http.paths[0].backend.service + value: + name: RELEASE-NAME-litellm + port: + number: 8080 + + - it: componentized mode still splits traffic across ui, gateway and backend Services + set: + ingress.enabled: true + asserts: + - equal: + path: spec.rules[0].http.paths[0].backend.service.name + value: RELEASE-NAME-litellm-ui + - contains: + path: spec.rules[0].http.paths + content: + path: /v1/chat + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 + - equal: + path: spec.rules[0].http.paths[-1].backend.service.name + value: RELEASE-NAME-litellm-backend + + - it: monolith mode routes a single catch-all to the monolith Service and nothing else + set: + monolith.enabled: true + ingress.enabled: true + asserts: + - lengthEqual: + path: spec.rules[0].http.paths + count: 1 + - equal: + path: spec.rules[0].http.paths[0] + value: + path: / + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm + port: + number: 4000 + - notMatchRegexRaw: + pattern: RELEASE-NAME-litellm-(gateway|backend|ui) + + - it: monolith mode uses the gateway service port for the catch-all + set: + monolith.enabled: true + ingress.enabled: true + gateway.service.port: 8080 + asserts: + - equal: + path: spec.rules[0].http.paths[0].backend.service.port.number + value: 8080 + + - it: monolith mode sends every extraPaths entry to the monolith Service regardless of its service field + set: + monolith.enabled: true + ingress.enabled: true + ingress.extraPaths: + - path: /watsonx + - path: /admin + service: backend + pathType: Exact + - path: /dashboard + service: ui + asserts: + - lengthEqual: + path: spec.rules[0].http.paths + count: 4 + - equal: + path: spec.rules[0].http.paths[0] + value: + path: /watsonx + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm + port: + number: 4000 + - equal: + path: spec.rules[0].http.paths[1].backend.service.name + value: RELEASE-NAME-litellm + - equal: + path: spec.rules[0].http.paths[1].pathType + value: Exact + - equal: + path: spec.rules[0].http.paths[2].backend.service.name + value: RELEASE-NAME-litellm + - equal: + path: spec.rules[0].http.paths[3].path + value: / + + - it: monolith mode keeps rejecting an extra path at the root + set: + monolith.enabled: true + ingress.enabled: true + ingress.extraPaths: + - path: / + asserts: + - failedTemplate: + errorPattern: "ingress.extraPaths\\[0\\]: path / is already routed" + + - it: monolith mode keeps the host, TLS and class settings + set: + monolith.enabled: true + ingress.enabled: true + ingress.className: nginx + ingress.controller: nginx + ingress.host: llm.example.com + ingress.tls: + - hosts: + - llm.example.com + secretName: llm-tls + asserts: + - equal: + path: spec.ingressClassName + value: nginx + - equal: + path: spec.rules[0].host + value: llm.example.com + - equal: + path: spec.tls[0].secretName + value: llm-tls + - equal: + path: spec.rules[0].http.paths[0].backend.service.name + value: RELEASE-NAME-litellm diff --git a/helm/litellm/tests/keda_tests.yaml b/helm/litellm/tests/keda_tests.yaml new file mode 100644 index 00000000000..1f48654fa73 --- /dev/null +++ b/helm/litellm/tests/keda_tests.yaml @@ -0,0 +1,266 @@ +suite: test KEDA ScaledObject +templates: + - gateway/keda.yaml + - gateway/hpa.yaml + - gateway/deployment.yaml + - gateway/configmap.yaml +values: + - ./values/required.yaml +release: + name: rel + namespace: llm +tests: + - it: renders no ScaledObject by default + template: gateway/keda.yaml + asserts: + - hasDocuments: + count: 0 + + - it: passes user triggers through and adds no prometheus triggers by default + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.triggers: + - type: cpu + metricType: Utilization + metadata: + value: "60" + templates: + - gateway/keda.yaml + - gateway/hpa.yaml + asserts: + - hasDocuments: + count: 0 + template: gateway/hpa.yaml + - isKind: + of: ScaledObject + template: gateway/keda.yaml + - equal: + path: metadata.name + value: rel-litellm-gateway + template: gateway/keda.yaml + - equal: + path: spec.scaleTargetRef + value: + apiVersion: apps/v1 + kind: Deployment + name: rel-litellm-gateway + template: gateway/keda.yaml + - equal: + path: spec.minReplicaCount + value: 1 + template: gateway/keda.yaml + - equal: + path: spec.maxReplicaCount + value: 10 + template: gateway/keda.yaml + - equal: + path: spec.triggers + value: + - type: cpu + metricType: Utilization + metadata: + value: "60" + template: gateway/keda.yaml + + - it: leaves spec.replicas off the Deployment so KEDA owns the replica count + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.replicaCount: 3 + template: gateway/deployment.yaml + asserts: + - notExists: + path: spec.replicas + + - it: scales on requests per second against the metrics Service job + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090 + gateway.metricsServer.enabled: true + gateway.keda.prometheus.targetRequestsPerSecond: 90 + template: gateway/keda.yaml + asserts: + - lengthEqual: + path: spec.triggers + count: 1 + - equal: + path: spec.triggers[0] + value: + type: prometheus + metricType: AverageValue + metadata: + serverAddress: http://prometheus-operated.monitoring.svc:9090 + threshold: "90" + query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-gateway-metrics"}[1m])) + + - it: scales on tokens per second on its own + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.prometheus.serverAddress: http://prom:9090 + gateway.metricsServer.enabled: true + gateway.keda.prometheus.targetTokensPerSecond: 6000000 + template: gateway/keda.yaml + asserts: + - lengthEqual: + path: spec.triggers + count: 1 + - equal: + path: spec.triggers[0].type + value: prometheus + - equal: + path: spec.triggers[0].metadata.threshold + value: "6000000" + - equal: + path: spec.triggers[0].metadata.query + value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-gateway-metrics"}[1m])) + + - it: appends the requests and tokens triggers after user triggers + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.metricsServer.enabled: true + gateway.keda.triggers: + - type: cpu + metricType: Utilization + metadata: + value: "60" + gateway.keda.prometheus.serverAddress: http://prom:9090 + gateway.keda.prometheus.targetRequestsPerSecond: 90 + gateway.keda.prometheus.targetTokensPerSecond: 6000000 + template: gateway/keda.yaml + asserts: + - lengthEqual: + path: spec.triggers + count: 3 + - equal: + path: spec.triggers[0].type + value: cpu + - equal: + path: spec.triggers[1].metadata.threshold + value: "90" + - equal: + path: spec.triggers[2].metadata.threshold + value: "6000000" + - notMatchRegexRaw: + pattern: "\\* *60|per_minute|PerMinute" + + - it: fills a partial fallback with the litellm-helm defaults + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.maxReplicaCount: 20 + gateway.keda.fallback: + replicas: 4 + template: gateway/keda.yaml + asserts: + - equal: + path: spec.fallback + value: + failureThreshold: 3 + replicas: 4 + + - it: defaults fallback replicas to maxReplicaCount and keeps an explicit zero threshold + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.maxReplicaCount: 20 + gateway.keda.fallback: + failureThreshold: 0 + template: gateway/keda.yaml + asserts: + - equal: + path: spec.fallback + value: + failureThreshold: 0 + replicas: 20 + + - it: renders advanced, fallback, polling and cooldown settings + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.minReplicaCount: 2 + gateway.keda.maxReplicaCount: 20 + gateway.keda.pollingInterval: 15 + gateway.keda.cooldownPeriod: 120 + gateway.keda.fallback: + failureThreshold: 3 + replicas: 4 + gateway.keda.advanced: + restoreToOriginalReplicaCount: true + template: gateway/keda.yaml + asserts: + - equal: + path: spec.minReplicaCount + value: 2 + - equal: + path: spec.maxReplicaCount + value: 20 + - equal: + path: spec.pollingInterval + value: 15 + - equal: + path: spec.cooldownPeriod + value: 120 + - equal: + path: spec.fallback + value: + failureThreshold: 3 + replicas: 4 + - equal: + path: spec.advanced.restoreToOriginalReplicaCount + value: true + + - it: refuses a workload target without a prometheus server address + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.prometheus.targetRequestsPerSecond: 90 + template: gateway/keda.yaml + asserts: + - failedTemplate: + errorMessage: gateway.keda.prometheus.serverAddress is required when gateway.keda.prometheus.targetRequestsPerSecond or targetTokensPerSecond is set + + - it: refuses a workload target without the metrics sidecar the trigger queries + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.prometheus.serverAddress: http://prom:9090 + gateway.keda.prometheus.targetRequestsPerSecond: 90 + template: gateway/keda.yaml + asserts: + - failedTemplate: + errorMessage: gateway.metricsServer.enabled is required when gateway.keda.prometheus.targetRequestsPerSecond or targetTokensPerSecond is set + + - it: refuses to run the HPA and KEDA against the same Deployment + set: + gateway.hpa.enabled: true + gateway.keda.enabled: true + template: gateway/deployment.yaml + asserts: + - failedTemplate: + errorPattern: gateway.hpa.enabled and gateway.keda.enabled are mutually exclusive + + - it: renders scaledObject annotations, such as the KEDA pause annotation, on the ScaledObject + template: gateway/keda.yaml + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.scaledObject.annotations: + autoscaling.keda.sh/paused-replicas: "2" + asserts: + - equal: + path: metadata.annotations + value: + autoscaling.keda.sh/paused-replicas: "2" + + - it: renders no ScaledObject annotations by default + template: gateway/keda.yaml + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + asserts: + - notExists: + path: metadata.annotations diff --git a/helm/litellm/tests/liteadmin_tests.yaml b/helm/litellm/tests/liteadmin_tests.yaml new file mode 100644 index 00000000000..429896ba9b4 --- /dev/null +++ b/helm/litellm/tests/liteadmin_tests.yaml @@ -0,0 +1,149 @@ +suite: test the LiteAdmin worker and its proxy wiring +templates: + - liteadmin.yaml + - gateway/configmap.yaml + - gateway/deployment.yaml + - backend/deployment.yaml + - monolith/deployment.yaml +values: + - ./values/required.yaml +tests: + - it: renders nothing when liteadmin is disabled + template: liteadmin.yaml + asserts: + - hasDocuments: + count: 0 + + - it: enabled renders a Deployment, Service and PVC on the shared image + set: + liteadmin.enabled: true + liteadmin.existingSecret: liteadmin-secret + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + template: liteadmin.yaml + asserts: + - hasDocuments: + count: 3 + - isKind: + of: Deployment + documentIndex: 0 + - isKind: + of: Service + documentIndex: 1 + - isKind: + of: PersistentVolumeClaim + documentIndex: 2 + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + documentIndex: 0 + - equal: + path: spec.template.spec.containers[0].args + value: ["--admin-agent"] + documentIndex: 0 + + - it: refuses to render without liteadmin.existingSecret + set: + liteadmin.enabled: true + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + templates: + - backend/deployment.yaml + template: backend/deployment.yaml + asserts: + - failedTemplate: + errorPattern: liteadmin.existingSecret is required + + - it: wires the liteadmin env onto the backend pod in componentized mode + set: + liteadmin.enabled: true + liteadmin.existingSecret: liteadmin-secret + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + template: backend/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_ADMIN_AGENT_URL + value: http://RELEASE-NAME-litellm-liteadmin:10000 + - contains: + path: spec.template.spec.containers[0].env + content: + name: ADMIN_AGENT_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: liteadmin-secret + key: ADMIN_AGENT_SERVICE_TOKEN + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_BASE_URL + value: http://gateway.example.com + + - it: leaves the liteadmin env off the gateway pod in componentized mode + set: + liteadmin.enabled: true + liteadmin.existingSecret: liteadmin-secret + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + template: gateway/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_ADMIN_AGENT_URL + - notContains: + path: spec.template.spec.containers[0].env + content: + name: ADMIN_AGENT_SERVICE_TOKEN + + - it: wires the liteadmin env onto the monolith pod in monolith mode + set: + monolith.enabled: true + liteadmin.enabled: true + liteadmin.existingSecret: liteadmin-secret + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + template: monolith/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_ADMIN_AGENT_URL + value: http://RELEASE-NAME-litellm-liteadmin:10000 + - contains: + path: spec.template.spec.containers[0].env + content: + name: ADMIN_AGENT_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: liteadmin-secret + key: ADMIN_AGENT_SERVICE_TOKEN + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_BASE_URL + value: http://gateway.example.com + + - it: a user-supplied PROXY_BASE_URL in extraEnv wins over liteadmin.gatewayUrl + set: + liteadmin.enabled: true + liteadmin.existingSecret: liteadmin-secret + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + backend.extraEnv: + - name: PROXY_BASE_URL + value: https://custom.example.com + template: backend/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_BASE_URL + value: http://gateway.example.com + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_BASE_URL + value: https://custom.example.com diff --git a/helm/litellm/tests/masterkey_secret_tests.yaml b/helm/litellm/tests/masterkey_secret_tests.yaml new file mode 100644 index 00000000000..8dd261053e4 --- /dev/null +++ b/helm/litellm/tests/masterkey_secret_tests.yaml @@ -0,0 +1,156 @@ +suite: test the generated master key Secret +templates: + - secret-masterkey.yaml + - gateway/deployment.yaml + - backend/deployment.yaml + - gateway/configmap.yaml + - monolith/deployment.yaml + - migrations-job.yaml +values: + - ./values/required.yaml +tests: + - it: renders no Secret when the operator supplies masterKey.secretName + template: secret-masterkey.yaml + asserts: + - hasDocuments: + count: 0 + + - it: refuses to render the workload without a Secret name when masterKey.generate is off + set: + masterKey.secretName: "" + template: gateway/deployment.yaml + asserts: + - failedTemplate: + errorPattern: masterKey.secretName is required + + - it: generates a Secret whose key starts with sk- (base64 c2st) and keeps it across uninstall + set: + masterKey.secretName: "" + masterKey.generate: true + template: secret-masterkey.yaml + asserts: + - isKind: + of: Secret + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-masterkey + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - matchRegex: + path: data["master-key"] + pattern: ^c2st + + - it: stores the value under masterKey.secretKey + set: + masterKey.secretName: "" + masterKey.generate: true + masterKey.secretKey: LITELLM_MASTER_KEY + template: secret-masterkey.yaml + asserts: + - matchRegex: + path: data.LITELLM_MASTER_KEY + pattern: ^c2st + + - it: reuses the master key already stored in the cluster instead of generating a new one on upgrade + set: + masterKey.secretName: "" + masterKey.generate: true + template: secret-masterkey.yaml + kubernetesProvider: + scheme: + "v1/Secret": + gvr: + version: "v1" + resource: "secrets" + namespaced: true + objects: + - kind: Secret + apiVersion: v1 + metadata: + name: RELEASE-NAME-litellm-masterkey + namespace: NAMESPACE + data: + master-key: c2stZXhpc3Rpbmcta2V5 + asserts: + - equal: + path: data["master-key"] + value: c2stZXhpc3Rpbmcta2V5 + + - it: points the app containers at the generated Secret and keeps it out of the pre-install migrations Job + set: + masterKey.secretName: "" + masterKey.generate: true + monolith.enabled: true + templates: + - monolith/deployment.yaml + - migrations-job.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: RELEASE-NAME-litellm-masterkey + key: master-key + template: monolith/deployment.yaml + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-install,pre-upgrade + template: migrations-job.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_MASTER_KEY + any: true + template: migrations-job.yaml + + - it: an explicit masterKey.secretName wins over masterKey.generate + set: + masterKey.secretName: operator-secret + masterKey.generate: true + templates: + - secret-masterkey.yaml + - gateway/deployment.yaml + asserts: + - hasDocuments: + count: 0 + template: secret-masterkey.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: operator-secret + key: master-key + template: gateway/deployment.yaml + + - it: exposes the key as PROXY_MASTER_KEY too, so a proxy_config copied from litellm-helm keeps resolving + set: + monolith.enabled: true + templates: + - monolith/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_MASTER_KEY + valueFrom: + secretKeyRef: + name: litellm-master-key-secret + key: master-key + - it: exposes PROXY_MASTER_KEY on the componentized gateway and backend + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_MASTER_KEY + valueFrom: + secretKeyRef: + name: litellm-master-key-secret + key: master-key diff --git a/helm/litellm/tests/metrics_server_tests.yaml b/helm/litellm/tests/metrics_server_tests.yaml index 0e7d9d9e9ee..1a48b7be671 100644 --- a/helm/litellm/tests/metrics_server_tests.yaml +++ b/helm/litellm/tests/metrics_server_tests.yaml @@ -38,7 +38,7 @@ tests: gateway.metricsServer.enabled: true gateway.metricsServer.port: 4101 gateway.service.type: LoadBalancer - gateway.image.tag: v1.101.0 + image.tag: v1.101.0 asserts: - contains: path: spec.template.spec.containers[0].env @@ -58,14 +58,12 @@ tests: template: gateway/deployment.yaml - equal: path: spec.template.spec.containers[1].image - value: ghcr.io/berriai/litellm-gateway:v1.101.0 + value: ghcr.io/berriai/litellm:v1.101.0 template: gateway/deployment.yaml - equal: - path: spec.template.spec.containers[1].command + path: spec.template.spec.containers[1].args value: - - python - - -m - - litellm.proxy.prometheus_metrics_server + - metrics - --port - "4101" template: gateway/deployment.yaml diff --git a/helm/litellm/tests/migration_job_tests.yaml b/helm/litellm/tests/migration_job_tests.yaml index b9e52ba7123..86251e8a1d2 100644 --- a/helm/litellm/tests/migration_job_tests.yaml +++ b/helm/litellm/tests/migration_job_tests.yaml @@ -91,7 +91,7 @@ tests: app.kubernetes.io/name: litellm app.kubernetes.io/instance: RELEASE-NAME app.kubernetes.io/managed-by: Helm - helm.sh/chart: litellm-0.1.1 + helm.sh/chart: litellm-1.0.0 app.kubernetes.io/component: migrations - it: renders pod-level and container-level securityContext in their own scopes @@ -254,3 +254,76 @@ tests: path: spec.template.spec.nodeSelector - isNull: path: spec.template.spec.tolerations + + - it: sets jobLabels and jobAnnotations on the Job metadata + set: + migrationJob.jobLabels: + environment: production + migrationJob.jobAnnotations: + example.com/cost-center: "1234" + asserts: + - equal: + path: metadata.labels.environment + value: production + - equal: + path: metadata.annotations['example.com/cost-center'] + value: "1234" + - equal: + path: metadata.annotations['helm.sh/hook'] + value: pre-install,pre-upgrade + + - it: renders jobAnnotations when both hooks are off + set: + migrationJob.hooks.helm.enabled: false + migrationJob.jobAnnotations: + example.com/cost-center: "1234" + asserts: + - equal: + path: metadata.annotations + value: + example.com/cost-center: "1234" + + - it: runs the image entrypoint with the migrations component by default + asserts: + - notExists: + path: spec.template.spec.containers[0].command + - equal: + path: spec.template.spec.containers[0].args + value: [migrations] + + - it: overrides the container command and args + set: + migrationJob.command: [sh] + migrationJob.args: [-c, echo migrating] + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: [sh] + - equal: + path: spec.template.spec.containers[0].args + value: [-c, echo migrating] + + - it: clears the container args when only command is set + set: + migrationJob.command: [sh] + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: [sh] + - equal: + path: spec.template.spec.containers[0].args + value: [] + + - it: puts migrationJob.annotations on the Job pod, as litellm-helm did + set: + migrationJob.annotations: + sidecar.istio.io/inject: "false" + asserts: + - equal: + path: spec.template.metadata.annotations + value: + sidecar.istio.io/inject: "false" + - it: renders no pod annotations by default + asserts: + - notExists: + path: spec.template.metadata.annotations diff --git a/helm/litellm/tests/monolith_tests.yaml b/helm/litellm/tests/monolith_tests.yaml new file mode 100644 index 00000000000..cb8aca608ec --- /dev/null +++ b/helm/litellm/tests/monolith_tests.yaml @@ -0,0 +1,539 @@ +suite: test monolith mode +templates: + - monolith/deployment.yaml + - monolith/service.yaml + - monolith/hpa.yaml + - monolith/keda.yaml + - monolith/poddisruptionbudget.yaml + - monolith/service-metrics.yaml + - monolith/servicemonitor.yaml + - gateway/deployment.yaml + - gateway/service.yaml + - gateway/hpa.yaml + - gateway/keda.yaml + - gateway/poddisruptionbudget.yaml + - gateway/service-metrics.yaml + - gateway/servicemonitor.yaml + - gateway/configmap.yaml + - backend/deployment.yaml + - backend/service.yaml + - backend/hpa.yaml + - backend/poddisruptionbudget.yaml + - ui/deployment.yaml + - ui/service.yaml + - ui/hpa.yaml + - ui/poddisruptionbudget.yaml + - serviceaccount.yaml + - migrations-job.yaml +values: + - ./values/required.yaml +tests: + - it: monolith ignores a leftover invalid UI public URL + set: + monolith.enabled: true + ui.publicUrl: not-an-origin + template: monolith/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_CORS_ORIGINS + any: true + + - it: renders no monolith resources by default + templates: + - monolith/deployment.yaml + - monolith/service.yaml + - monolith/hpa.yaml + - monolith/poddisruptionbudget.yaml + - monolith/service-metrics.yaml + - monolith/servicemonitor.yaml + asserts: + - hasDocuments: + count: 0 + + - it: componentized mode renders one Deployment per component + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Deployment + + - it: monolith mode renders exactly one Deployment, running the proxy dispatcher + set: + monolith.enabled: true + templates: + - monolith/deployment.yaml + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + asserts: + - hasDocuments: + count: 1 + template: monolith/deployment.yaml + - hasDocuments: + count: 0 + template: gateway/deployment.yaml + - hasDocuments: + count: 0 + template: backend/deployment.yaml + - hasDocuments: + count: 0 + template: ui/deployment.yaml + - isKind: + of: Deployment + template: monolith/deployment.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-litellm + template: monolith/deployment.yaml + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: proxy + template: monolith/deployment.yaml + - equal: + path: spec.template.spec.containers[0].name + value: proxy + template: monolith/deployment.yaml + - equal: + path: spec.template.spec.containers[0].args[0] + value: proxy + template: monolith/deployment.yaml + - equal: + path: spec.template.spec.containers[0].args + value: + - proxy + - --port + - "4000" + - --config + - /app/config/config.yaml + template: monolith/deployment.yaml + - notExists: + path: spec.template.spec.containers[0].command + template: monolith/deployment.yaml + + - it: monolith appends monolith.extraArgs after the chart's own proxy arguments + set: + monolith.enabled: true + monolith.extraArgs: + - --detailed_debug + - --run_granian + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].args + value: + - proxy + - --port + - "4000" + - --config + - /app/config/config.yaml + - --detailed_debug + - --run_granian + + - it: monolith drops the --config pair when the chart does not create the ConfigMap + set: + monolith.enabled: true + gateway.config.create: false + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].args + value: + - proxy + - --port + - "4000" + - notExists: + path: spec.template.spec.volumes + + - it: monolith mounts the proxy config ConfigMap the chart renders under the monolith name + set: + monolith.enabled: true + gateway.config.proxy_config: + model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + templates: + - monolith/deployment.yaml + - gateway/configmap.yaml + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-config + template: gateway/configmap.yaml + - contains: + path: spec.template.spec.volumes + content: + name: gateway-config + configMap: + name: RELEASE-NAME-litellm-config + template: monolith/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: CONFIG_FILE_PATH + value: /app/config/config.yaml + template: monolith/deployment.yaml + + - it: monolith mode renders exactly one Service and no component Services + set: + monolith.enabled: true + templates: + - monolith/service.yaml + - gateway/service.yaml + - backend/service.yaml + - ui/service.yaml + asserts: + - hasDocuments: + count: 1 + template: monolith/service.yaml + - hasDocuments: + count: 0 + template: gateway/service.yaml + - hasDocuments: + count: 0 + template: backend/service.yaml + - hasDocuments: + count: 0 + template: ui/service.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-litellm + template: monolith/service.yaml + - equal: + path: spec.selector + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: proxy + template: monolith/service.yaml + - equal: + path: spec.ports[0].port + value: 4000 + template: monolith/service.yaml + + - it: monolith Service selector matches the monolith pod labels + set: + monolith.enabled: true + templates: + - monolith/deployment.yaml + asserts: + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: proxy + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/component"] + value: proxy + + - it: monolith reuses gateway resources, replicas, numWorkers and probes + set: + monolith.enabled: true + gateway.replicaCount: 3 + gateway.hpa.enabled: false + gateway.numWorkers: 6 + gateway.resources: + requests: + cpu: "2" + memory: 4Gi + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 3 + - equal: + path: spec.template.spec.containers[0].resources.requests.cpu + value: "2" + - contains: + path: spec.template.spec.containers[0].env + content: + name: NUM_WORKERS + value: "6" + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /health/readiness + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /health/liveliness + + - it: monolith takes the rollout settings a litellm-helm install carried from gateway.minReadySeconds and gateway.strategy + set: + monolith.enabled: true + gateway.minReadySeconds: 60 + gateway.strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + backend.minReadySeconds: 5 + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.minReadySeconds + value: 60 + - equal: + path: spec.strategy + value: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + + - it: monolith ignores backend and ui replica overrides + set: + monolith.enabled: true + gateway.hpa.enabled: false + gateway.replicaCount: 2 + backend.replicaCount: 9 + ui.replicaCount: 9 + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 2 + + - it: monolith HPA targets the monolith Deployment and the gateway HPA is not rendered + set: + monolith.enabled: true + gateway.hpa.enabled: true + templates: + - monolith/hpa.yaml + - gateway/hpa.yaml + - backend/hpa.yaml + - ui/hpa.yaml + asserts: + - hasDocuments: + count: 1 + template: monolith/hpa.yaml + - hasDocuments: + count: 0 + template: gateway/hpa.yaml + - hasDocuments: + count: 0 + template: backend/hpa.yaml + - hasDocuments: + count: 0 + template: ui/hpa.yaml + - equal: + path: spec.scaleTargetRef + value: + apiVersion: apps/v1 + kind: Deployment + name: RELEASE-NAME-litellm + template: monolith/hpa.yaml + + - it: monolith KEDA ScaledObject targets the monolith Deployment + set: + monolith.enabled: true + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.prometheus.serverAddress: http://prometheus:9090 + gateway.metricsServer.enabled: true + gateway.keda.prometheus.targetRequestsPerSecond: 50 + templates: + - monolith/keda.yaml + - gateway/keda.yaml + asserts: + - hasDocuments: + count: 1 + template: monolith/keda.yaml + - hasDocuments: + count: 0 + template: gateway/keda.yaml + - equal: + path: spec.scaleTargetRef.name + value: RELEASE-NAME-litellm + template: monolith/keda.yaml + - equal: + path: spec.triggers[0].metadata.query + value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="NAMESPACE",job="RELEASE-NAME-litellm-metrics"}[1m])) + template: monolith/keda.yaml + + - it: monolith PDB selects the monolith pods + set: + monolith.enabled: true + gateway.pdb.enabled: true + gateway.pdb.maxUnavailable: 1 + templates: + - monolith/poddisruptionbudget.yaml + - gateway/poddisruptionbudget.yaml + - backend/poddisruptionbudget.yaml + - ui/poddisruptionbudget.yaml + asserts: + - hasDocuments: + count: 1 + template: monolith/poddisruptionbudget.yaml + - hasDocuments: + count: 0 + template: gateway/poddisruptionbudget.yaml + - hasDocuments: + count: 0 + template: backend/poddisruptionbudget.yaml + - hasDocuments: + count: 0 + template: ui/poddisruptionbudget.yaml + - equal: + path: spec.maxUnavailable + value: 1 + template: monolith/poddisruptionbudget.yaml + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/component"] + value: proxy + template: monolith/poddisruptionbudget.yaml + + - it: monolith metrics sidecar, metrics Service and ServiceMonitor follow the monolith name + set: + monolith.enabled: true + gateway.metricsServer.enabled: true + gateway.serviceMonitor.enabled: true + templates: + - monolith/deployment.yaml + - monolith/service-metrics.yaml + - monolith/servicemonitor.yaml + - gateway/service-metrics.yaml + - gateway/servicemonitor.yaml + asserts: + - hasDocuments: + count: 0 + template: gateway/service-metrics.yaml + - hasDocuments: + count: 0 + template: gateway/servicemonitor.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-metrics + template: monolith/service-metrics.yaml + - equal: + path: spec.selector["app.kubernetes.io/component"] + value: proxy + template: monolith/service-metrics.yaml + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/component"] + value: proxy + template: monolith/servicemonitor.yaml + - equal: + path: spec.template.spec.containers[1].name + value: metrics + template: monolith/deployment.yaml + - equal: + path: spec.template.spec.containers[1].args[0] + value: metrics + template: monolith/deployment.yaml + + - it: monolith collector sidecar runs the collector dispatcher from the same image + set: + monolith.enabled: true + gateway.collector.enabled: true + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[1].name + value: collector + - equal: + path: spec.template.spec.containers[1].args + value: + - collector + - equal: + path: spec.template.spec.containers[1].image + value: ghcr.io/berriai/litellm:1.104.0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ENABLED + value: "true" + + - it: monolith renders only the gateway ServiceAccount and runs the pod with it + set: + monolith.enabled: true + serviceAccounts.gateway.create: true + serviceAccounts.backend.create: true + serviceAccounts.ui.create: true + templates: + - serviceaccount.yaml + - monolith/deployment.yaml + asserts: + - hasDocuments: + count: 1 + template: serviceaccount.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-gateway + template: serviceaccount.yaml + - equal: + path: spec.template.spec.serviceAccountName + value: RELEASE-NAME-litellm-gateway + template: monolith/deployment.yaml + + - it: the migrations Job renders the same way in monolith mode + set: + monolith.enabled: true + template: migrations-job.yaml + asserts: + - isKind: + of: Job + - equal: + path: spec.template.spec.containers[0].args + value: + - migrations + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + + - it: monolith gets the writer user under both names the proxy CLI and the componentized entrypoints read + set: + monolith.enabled: true + template: monolith/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_USER + valueFrom: + secretKeyRef: + name: litellm-writer-secret + key: username + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: litellm-writer-secret + key: username + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_NAME + value: litellm + + - it: disabling every component without monolith renders no application workload + set: + gateway.enabled: false + backend.enabled: false + ui.enabled: false + templates: + - monolith/deployment.yaml + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + asserts: + - hasDocuments: + count: 0 + + - it: monolith renders even when gateway.enabled is false + set: + monolith.enabled: true + gateway.enabled: false + templates: + - monolith/deployment.yaml + - monolith/service.yaml + asserts: + - hasDocuments: + count: 1 diff --git a/helm/litellm/tests/no_bundled_datastores_tests.yaml b/helm/litellm/tests/no_bundled_datastores_tests.yaml new file mode 100644 index 00000000000..dbafa5a9a11 --- /dev/null +++ b/helm/litellm/tests/no_bundled_datastores_tests.yaml @@ -0,0 +1,54 @@ +suite: the chart ships no PostgreSQL or Redis of its own +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - migrations-job.yaml +tests: + - it: rejects the retired postgresql block instead of silently ignoring it + set: + postgresql.enabled: true + postgresql.auth.password: super-secret + asserts: + - failedTemplate: {} + + - it: rejects the retired redis.enabled flag instead of silently ignoring it + set: + redis.enabled: true + asserts: + - failedTemplate: {} + + - it: wires the writer straight from database.writer with no subchart in between + template: gateway/deployment.yaml + set: + database.writer.host: postgres.example.com + database.writer.dbname: proxydb + database.writer.passwordSecret.name: pg-secret + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_HOST + value: postgres.example.com + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_NAME + value: proxydb + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: pg-secret + key: password + + - it: keeps the migrations Job a pre-install hook whenever the Helm hook is enabled + template: migrations-job.yaml + set: + database.writer.host: postgres.example.com + database.writer.dbname: litellm + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-install,pre-upgrade diff --git a/helm/litellm/tests/redis_env_tests.yaml b/helm/litellm/tests/redis_env_tests.yaml index 79a14dfc6e5..849f13e5438 100644 --- a/helm/litellm/tests/redis_env_tests.yaml +++ b/helm/litellm/tests/redis_env_tests.yaml @@ -225,10 +225,10 @@ tests: name: REDIS_CLUSTER_NODES value: '[{"host":"$(REDIS_HOST)","port":6379}]' - equal: - path: spec.template.spec.containers[0].env[8].name + path: spec.template.spec.containers[0].env[10].name value: REDIS_HOST - equal: - path: spec.template.spec.containers[0].env[11].name + path: spec.template.spec.containers[0].env[13].name value: REDIS_CLUSTER_NODES - it: gateway gets REDIS_URL from urlKey without discrete Redis env diff --git a/helm/litellm/tests/rollout_strategy_tests.yaml b/helm/litellm/tests/rollout_strategy_tests.yaml index b12e2073c7c..af6e41d038e 100644 --- a/helm/litellm/tests/rollout_strategy_tests.yaml +++ b/helm/litellm/tests/rollout_strategy_tests.yaml @@ -64,3 +64,39 @@ tests: - notExists: path: spec.strategy template: ui/deployment.yaml + + - it: leaves minReadySeconds to the Kubernetes default when unset + asserts: + - notExists: + path: spec.minReadySeconds + + - it: renders the configured minReadySeconds on each deployment + set: + gateway.minReadySeconds: 60 + backend.minReadySeconds: 30 + ui.minReadySeconds: 10 + asserts: + - equal: + path: spec.minReadySeconds + value: 60 + template: gateway/deployment.yaml + - equal: + path: spec.minReadySeconds + value: 30 + template: backend/deployment.yaml + - equal: + path: spec.minReadySeconds + value: 10 + template: ui/deployment.yaml + + - it: renders an explicit minReadySeconds of 0 instead of dropping it + set: + gateway.minReadySeconds: 0 + asserts: + - equal: + path: spec.minReadySeconds + value: 0 + template: gateway/deployment.yaml + - notExists: + path: spec.minReadySeconds + template: backend/deployment.yaml diff --git a/helm/litellm/tests/shared_image_tests.yaml b/helm/litellm/tests/shared_image_tests.yaml new file mode 100644 index 00000000000..1e7837b1973 --- /dev/null +++ b/helm/litellm/tests/shared_image_tests.yaml @@ -0,0 +1,264 @@ +suite: test the shared image and the entrypoint dispatcher args +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - backend/deployment.yaml + - ui/deployment.yaml + - monolith/deployment.yaml + - migrations-job.yaml + - templates/tests/test-connection.yaml +values: + - ./values/required.yaml +tests: + - it: every componentized container runs the shared image at the chart appVersion + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + - migrations-job.yaml + - templates/tests/test-connection.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + template: backend/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + template: ui/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + template: migrations-job.yaml + - equal: + path: spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + template: templates/tests/test-connection.yaml + + - it: image.repository and image.tag are applied to every container, sidecars included + set: + image.repository: registry.example.com/litellm + image.tag: v9.9.9 + image.pullPolicy: Always + gateway.metricsServer.enabled: true + gateway.collector.enabled: true + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + - migrations-job.yaml + - templates/tests/test-connection.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example.com/litellm:v9.9.9 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].image + value: registry.example.com/litellm:v9.9.9 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[2].image + value: registry.example.com/litellm:v9.9.9 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: registry.example.com/litellm:v9.9.9 + template: backend/deployment.yaml + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + template: backend/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: registry.example.com/litellm:v9.9.9 + template: ui/deployment.yaml + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + template: ui/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: registry.example.com/litellm:v9.9.9 + template: migrations-job.yaml + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + template: migrations-job.yaml + - equal: + path: spec.containers[0].image + value: registry.example.com/litellm:v9.9.9 + template: templates/tests/test-connection.yaml + + - it: the monolith and its sidecars run the shared image + set: + monolith.enabled: true + image.tag: v9.9.9 + gateway.metricsServer.enabled: true + gateway.collector.enabled: true + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9 + - equal: + path: spec.template.spec.containers[1].image + value: ghcr.io/berriai/litellm:v9.9.9 + - equal: + path: spec.template.spec.containers[2].image + value: ghcr.io/berriai/litellm:v9.9.9 + + - it: image.digest renders as repository:tag@digest everywhere + set: + image.tag: v9.9.9 + image.digest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + - migrations-job.yaml + - templates/tests/test-connection.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + template: backend/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + template: ui/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + template: migrations-job.yaml + - equal: + path: spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + template: templates/tests/test-connection.yaml + + - it: image.digest with an empty tag falls back to the appVersion tag + set: + image.digest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + monolith.enabled: true + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + + - it: componentized containers select their component through args and never set command + set: + gateway.metricsServer.enabled: true + gateway.collector.enabled: true + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].args + value: + - gateway + - --host + - 0.0.0.0 + - --port + - "4000" + template: gateway/deployment.yaml + - notExists: + path: spec.template.spec.containers[0].command + template: gateway/deployment.yaml + - notExists: + path: spec.template.spec.containers[1].command + template: gateway/deployment.yaml + - notExists: + path: spec.template.spec.containers[2].command + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].args + value: + - backend + - --host + - 0.0.0.0 + - --port + - "4001" + template: backend/deployment.yaml + - notExists: + path: spec.template.spec.containers[0].command + template: backend/deployment.yaml + - equal: + path: spec.template.spec.containers[0].args + value: + - ui + template: ui/deployment.yaml + - notExists: + path: spec.template.spec.containers[0].command + template: ui/deployment.yaml + + - it: the migrations Job runs the migrations dispatcher and nothing else + template: migrations-job.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].args + value: + - migrations + - notExists: + path: spec.template.spec.containers[0].command + + - it: the Helm test pod polls the workload Service readiness endpoint through the image entrypoint + template: templates/tests/test-connection.yaml + asserts: + - isKind: + of: Pod + - equal: + path: metadata.annotations["helm.sh/hook"] + value: test + - equal: + path: spec.containers[0].env[0] + value: + name: LITELLM_READINESS_URL + value: http://RELEASE-NAME-litellm-gateway:4000/health/readiness + - equal: + path: spec.containers[0].args[0] + value: python + - equal: + path: spec.containers[0].args[1] + value: -c + - matchRegex: + path: spec.containers[0].args[2] + pattern: LITELLM_READINESS_URL + - matchRegex: + path: spec.containers[0].args[2] + pattern: 'db != "connected"' + - matchRegex: + path: spec.containers[0].args[2] + pattern: sys\.exit\(1\) + - notExists: + path: spec.containers[0].command + + - it: the Helm test pod targets the monolith Service in monolith mode + set: + monolith.enabled: true + template: templates/tests/test-connection.yaml + asserts: + - equal: + path: spec.containers[0].env[0].value + value: http://RELEASE-NAME-litellm:4000/health/readiness + + - it: the removed per-component image blocks are rejected by the values schema + set: + gateway.image.tag: v1 + template: gateway/deployment.yaml + asserts: + - failedTemplate: {} diff --git a/helm/litellm/values.schema.json b/helm/litellm/values.schema.json new file mode 100644 index 00000000000..3aafbe68fe6 --- /dev/null +++ b/helm/litellm/values.schema.json @@ -0,0 +1,183 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "LiteLLM chart values", + "type": "object", + "additionalProperties": true, + "properties": { + "image": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository" + ], + "properties": { + "repository": { + "type": "string", + "minLength": 1 + }, + "tag": { + "type": "string" + }, + "pullPolicy": { + "type": "string", + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + }, + "digest": { + "type": "string", + "pattern": "^(sha256:[a-f0-9]{64})?$" + } + } + }, + "imagePullSecrets": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + } + }, + "monolith": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "extraArgs": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "gateway": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + }, + "replicaCount": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "numWorkers": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "image": false + } + }, + "backend": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + }, + "replicaCount": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "image": false + } + }, + "ui": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + }, + "replicaCount": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "image": false + } + }, + "migrationJob": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + }, + "image": false + } + }, + "masterKey": { + "type": "object", + "additionalProperties": true, + "properties": { + "secretName": { + "type": "string" + }, + "secretKey": { + "type": "string" + }, + "generate": { + "type": "boolean" + } + } + }, + "postgresql": false, + "redis": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": false, + "cluster": { + "type": "boolean" + }, + "host": { + "type": "string" + }, + "port": { + "type": [ + "integer", + "string" + ] + } + } + }, + "ingress": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "extraResources": { + "type": "array", + "items": { + "type": "object" + } + } + } +} diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index cb37656cb77..9bfc3ca32e9 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -1,10 +1,64 @@ # LiteLLM helm chart values +# +# Two deployment modes, both from the single image below: +# +# componentized (default): `gateway.enabled`, `backend.enabled` and +# `ui.enabled` each render their own Deployment, Service, HPA and PDB, so +# the LLM data plane, the management API and the static dashboard scale +# independently. Their containers run `args: [gateway|backend|ui, ...]`. +# +# monolith: `monolith.enabled: true` renders ONE Deployment and Service +# running the full proxy (`args: [proxy, ...]`): gateway routes, +# management routes and the Admin UI served by the proxy itself. The +# gateway, backend and ui Deployments and Services are not rendered and +# the Ingress sends every path to the monolith Service. The monolith is +# configured through the `gateway.*` values (config, resources, probes, +# securityContext, hpa, pdb, keda, collector, metricsServer, volumes, +# env), so a monolith install only flips this flag. See `monolith` below +# for the precedence rules. nameOverride: "" fullnameOverride: "" +# The one image every container in this chart runs: gateway, backend, ui, +# monolith, the migrations Job, the metrics and collector sidecars and the +# Helm test pod. Its entrypoint dispatches on the first container argument +# (proxy, gateway, backend, ui, migrations, metrics, collector), which is why +# the templates only ever set `args` and never `command`: overriding the +# entrypoint would drop the wrappers it applies (ddtrace, Prometheus +# multiprocess dir cleanup). `tag` defaults to the chart's appVersion. When +# `digest` is set the image renders as `repository:tag@digest`, so the tag is +# informational and the digest pins the bytes. Also mirrored at +# docker.litellm.ai/berriai/litellm. +image: + repository: ghcr.io/berriai/litellm + tag: "" + pullPolicy: IfNotPresent + digest: "" + imagePullSecrets: [] +# Monolith mode. When enabled the chart renders a single Deployment named +# `-litellm` running `args: [proxy, --port, 4000, --config, +# /app/config/config.yaml]` (the --config pair only when gateway.config.create +# is true) plus `extraArgs`, and a Service of the same name. Precedence when +# this is on: +# - gateway.enabled / backend.enabled / ui.enabled are ignored: no +# component Deployment, Service, HPA, PDB or ServiceMonitor is rendered +# - every `backend.*` and `ui.*` value is ignored, including their +# ServiceAccounts; the monolith runs as serviceAccounts.gateway +# - every `gateway.*` value applies to the monolith pod instead of a gateway +# pod: config, numWorkers, resources, probes, securityContext, +# hpa / keda / pdb, metricsServer, collector, volumes, env, scheduling +# - ingress.extraPaths entries keep their `service` field but every path, +# built in or extra, targets the monolith Service +# - the migrations Job renders exactly as in componentized mode +monolith: + enabled: false + # Extra CLI arguments appended after the chart's own proxy arguments, e.g. + # ["--detailed_debug"] or ["--run_granian"]. + extraArgs: [] + # Optional Ingress wiring the three component Services behind a single L7 # entrypoint. Required when serving the static UI bundle over the network. ingress: @@ -86,10 +140,8 @@ serviceAccounts: # LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...). Disable if your # pipeline runs migrations out-of-band. # -# Uses a dedicated `litellm-migrations` image (prisma CLI + the migration -# files from `litellm-proxy-extras`) instead of the backend image, so the -# Job doesn't drag in the rest of the proxy and doesn't run `prisma -# generate` — the migration engine doesn't need the generated client. +# Runs the shared image with `args: [migrations]`, which the entrypoint +# dispatches to the prisma migration runner without importing the proxy. migrationJob: enabled: true # Which controller is responsible for running the Job. @@ -150,8 +202,18 @@ migrationJob: # pod. Same shape as gateway.podSecurityContext / gateway.securityContext. podSecurityContext: {} securityContext: {} + # Extra labels and annotations on the Job object itself. + jobLabels: {} + jobAnnotations: {} # Extra pod labels on the Job pod, merged into the chart's common labels. podLabels: {} + # Annotations on the Job pod, e.g. sidecar.istio.io/inject: "false" so an + # injected sidecar cannot keep the pod running after the migration exits. + annotations: {} + # Override the container command and args. The default runs the image + # entrypoint with `migrations`; a command with no args clears the args. + command: [] + args: [] # Additional volumes on the Job pod and volumeMounts on its container, e.g. # the writable scratch space a read-only root filesystem needs. volumes: [] @@ -163,21 +225,26 @@ migrationJob: nodeSelector: {} tolerations: [] affinity: {} - image: - repository: ghcr.io/berriai/litellm-migrations - tag: "" # defaults to .Chart.AppVersion - pullPolicy: IfNotPresent + # Extra init containers and sidecars on the Job pod, rendered through `tpl`. + extraInitContainers: [] + extraContainers: [] # Extra env appended to the migration container. The migration entrypoint # uses the v2 resolver by default (no diff-and-force recovery — avoids the # schema thrashing seen during rolling deploys). To opt back into the v1 # resolver, append `- name: USE_V2_MIGRATION_RESOLVER` / `value: "false"`. extraEnv: [] -# Required: a master key used by gateway + backend to mint/verify proxy tokens. -# Must reference an existing Secret. +# Master key used by the proxy to mint and verify virtual keys. Reference an +# existing Secret through `secretName` / `secretKey`, or set `generate: true` +# and leave `secretName` empty to have the chart create +# `-litellm-masterkey` with a random `sk-...` key under `secretKey`. +# The generated Secret is looked up on upgrade so the key never rotates by +# accident (`helm template` cannot look anything up and renders a fresh key +# each time; that is expected). The chart never accepts an inline key. masterKey: secretName: litellm-master-key-secret # name of a Secret containing the master key secretKey: master-key + generate: false # Optional: enterprise billable-request metering. When enabled, the gateway and # backend count successful requests to inference, MCP, and A2A endpoints. Requires @@ -189,7 +256,8 @@ billingMetrics: secretName: litellm-billing-metrics exportIntervalMs: "" -# External Postgres connection. +# Postgres connection. The chart ships no database; point `writer` at your +# own PostgreSQL (managed or self hosted) and its credentials Secret. database: writer: host: "" @@ -249,24 +317,23 @@ database: maxDbConnections: 20 maxClientConn: 1000 -# Optional Redis. Leave host empty to disable. +# Redis is the proxy's coordination store: cross-pod tpm/rpm rate limits, +# spend tracking, and the pod lock manager. The chart emits REDIS_HOST / +# REDIS_PORT / REDIS_PASSWORD, which the proxy picks up through its +# coordination Redis env fallback. Response caching is separate and off unless +# you enable it in `proxy_config.litellm_settings.cache`. # -# This is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend -# tracking, and the pod lock manager. The chart emits REDIS_HOST / REDIS_PORT / -# REDIS_PASSWORD, which the proxy picks up through its coordination Redis env -# fallback. Response caching is separate and off unless you enable it in -# `proxy_config.litellm_settings.cache`. +# Set `host` (and `passwordSecret.name` when auth is required). Set +# `cluster: true` for Redis Cluster mode (AWS ElastiCache Cluster, self-hosted +# Redis Cluster): the chart emits REDIS_CLUSTER_NODES from `host` / `port` as +# the single seed and the client discovers the rest from CLUSTER SLOTS. The +# chart ships no Redis of its own. # # For full control, define `general_settings.coordination_redis` in # `proxy_config` (host/port/password/username/url/ssl/startup_nodes/ # sentinel_nodes/sentinel_password/service_name, each accepting os.environ/VAR # refs). An explicit block overrides these env vars. # -# Set `cluster: true` for Redis Cluster mode (e.g. AWS ElastiCache Cluster, -# self-hosted Redis Cluster). The chart emits REDIS_CLUSTER_NODES from -# `host` / `port` as the single seed; the cluster client discovers the -# remaining nodes from CLUSTER SLOTS at startup. -# # For a TLS Redis endpoint set `ssl: true`, which the chart emits as REDIS_SSL. # `sslCertReqs` (none / optional / required) becomes REDIS_SSL_CERT_REQS and # `sslCaCerts` (path to a CA bundle mounted in the pod, e.g. via @@ -287,12 +354,17 @@ redis: hostKey: "" # Secret key containing the Redis host urlKey: "" # Secret key containing the complete Redis URL +# Arbitrary extra Kubernetes manifests rendered verbatim with the release, +# e.g. a NetworkPolicy or an ExternalSecret the proxy needs. +extraResources: [] + # ---------- gateway (LLM data plane) ---------- +# In monolith mode this whole block configures the monolith pod instead. gateway: enabled: true logLevel: INFO - # Number of uvicorn worker processes per gateway pod. Sets NUM_WORKERS, - # consumed by the gateway image entrypoint. Default is 1. + # Number of worker processes per pod. Sets NUM_WORKERS, read by the gateway + # launcher and by the proxy's --num_workers default. Default is 1. numWorkers: 1 extraEnv: [] # Add extra environment variables to the gateway envConfigMaps: [] # Add extra environment variables to the gateway from config maps @@ -311,7 +383,6 @@ gateway: # scrape never runs on an inference worker. Adds a `metrics` port to the pod # and a dedicated ClusterIP `-metrics` Service; point your scrape # config at it. The port has no virtual-key auth: keep it off public ingress. - # Needs the gateway image v1.101.0 or newer. metricsServer: enabled: false port: 4001 @@ -329,8 +400,13 @@ gateway: serviceMonitor: enabled: false labels: {} + annotations: {} interval: 15s scrapeTimeout: 10s + relabelings: [] + # Empty scrapes the release namespace. + namespaceSelector: + matchNames: [] # Opt-in `collector` sidecar (same image, `python -m litellm.proxy.collector`) # that runs the post-response spend pipeline (cost calculation, spend logs, # spend counters, budget reservation reconciliation) so the uvicorn workers @@ -367,13 +443,13 @@ gateway: # ContainerResource metric of the `gateway` container only, so the # sidecar's CPU never drives inference replicas. Needs Kubernetes 1.30+. scaleOnGatewayContainerCpu: true - image: - repository: ghcr.io/berriai/litellm-gateway - tag: "" # defaults to .Chart.AppVersion - pullPolicy: IfNotPresent service: type: ClusterIP port: 4000 + annotations: {} + # For LoadBalancer Services on clusters with several load balancer + # implementations. + loadBalancerClass: "" resources: requests: cpu: "1" @@ -447,6 +523,41 @@ gateway: # counted when a response completes, so TPS trails long streams. targetRequestsPerSecond: "" targetTokensPerSecond: "" + # KEDA ScaledObject as an alternative to the HPA above. Mutually exclusive + # with hpa.enabled (two autoscalers on one Deployment fight each other), so + # set `hpa.enabled: false` when turning this on. Needs KEDA installed in the + # cluster. `triggers` are rendered verbatim; the two optional Prometheus + # workload targets below add prometheus triggers on the same counters the + # HPA workload metrics use, scraped through the metrics Service (enable + # metricsServer and serviceMonitor). + keda: + enabled: false + minReplicaCount: 1 + maxReplicaCount: 10 + pollingInterval: 30 + cooldownPeriod: 300 + # Rendered on the ScaledObject, e.g. autoscaling.keda.sh/paused-replicas: "2" + # to pause scaling at a fixed replica count + scaledObject: + annotations: {} + # Optional; rendered verbatim under spec.advanced + advanced: {} + # Optional; rendered under spec.fallback. A missing `failureThreshold` + # defaults to 3 and a missing `replicas` to maxReplicaCount, as in litellm-helm. + fallback: {} + # Custom triggers, rendered verbatim, e.g. + # - type: cpu + # metricType: Utilization + # metadata: + # value: "70" + triggers: [] + prometheus: + # Prometheus server URL, required when either workload target is set. + serverAddress: "" + # Per pod averages, same units as hpa.targetRequestsPerSecond / + # hpa.targetTokensPerSecond. Empty leaves the trigger out. + targetRequestsPerSecond: "" + targetTokensPerSecond: "" # PodDisruptionBudget for the gateway pods. Set exactly one of # `minAvailable` / `maxUnavailable` (minAvailable wins if both are set; # enabling without either falls back to `maxUnavailable: 1`). Disabled by @@ -456,6 +567,13 @@ gateway: enabled: false minAvailable: "" maxUnavailable: "" + # Extra annotations and labels on the Deployment object itself (not the + # pods), e.g. for a GitOps controller or a reloader. + deploymentAnnotations: {} + deploymentLabels: {} + # Seconds a new pod must be ready before the rollout treats it as + # available. Empty inherits the Kubernetes default of 0. + minReadySeconds: "" podAnnotations: {} # Extra pod labels, merged into the chart's selector labels. Do not # re-declare `app.kubernetes.io/name` / `instance` / `component` here: they @@ -482,7 +600,16 @@ gateway: # egress proxy. Rendered through `tpl`, so entries may reference chart # values and release metadata. extraContainers: [] + # Init containers, rendered through `tpl` the same way. + extraInitContainers: [] # Container lifecycle hooks (postStart / preStop) for the gateway container. + # + # Prefer the proxy's /health/drain preStop hook over a fixed `sleep`: it + # marks the pod NotReady and blocks only until in-flight requests finish + # (bounded by GRACEFUL_SHUTDOWN_TIMEOUT). It is off by default; enable it + # with general_settings.enable_drain_endpoint: true and, when the health + # port is reachable from other pods, set general_settings.drain_endpoint_token + # and send the same value on the X-Drain-Token header from the hook. lifecycle: {} # Grace period the kubelet allows between SIGTERM and SIGKILL. Leave empty # to inherit the Kubernetes default of 30s. Set it a few seconds above the @@ -513,13 +640,11 @@ backend: volumes: [] # Additional volumeMounts on the backend container. volumeMounts: [] - image: - repository: ghcr.io/berriai/litellm-backend - tag: "" - pullPolicy: IfNotPresent service: type: ClusterIP port: 4001 + annotations: {} + loadBalancerClass: "" resources: requests: cpu: "1" @@ -556,12 +681,16 @@ backend: enabled: false minAvailable: "" maxUnavailable: "" + deploymentAnnotations: {} + deploymentLabels: {} + minReadySeconds: "" podAnnotations: {} # Same shape as the gateway blocks of the same name. podLabels: {} podSecurityContext: {} securityContext: {} extraContainers: [] + extraInitContainers: [] lifecycle: {} terminationGracePeriodSeconds: "" nodeSelector: {} @@ -581,13 +710,13 @@ ui: volumes: [] # Additional volumeMounts on the ui container. volumeMounts: [] - image: - repository: ghcr.io/berriai/litellm-ui - tag: "" - pullPolicy: IfNotPresent service: type: ClusterIP port: 3000 + annotations: {} + loadBalancerClass: "" + # The dashboard expects to know where to reach the backend API. Set this to + # the externally-routable URL (typically the ingress host + /api or similar). # The backend's public origin, passed to the UI only when ingress.componentHosts is configured. backendUrl: "" # Public origin of the UI, used for backend and gateway CORS. @@ -625,17 +754,19 @@ ui: enabled: false minAvailable: "" maxUnavailable: "" + deploymentAnnotations: {} + deploymentLabels: {} + minReadySeconds: "" podAnnotations: {} # Same shape as the gateway blocks of the same name. The nginx runtime # writes its pid, cache, and proxy temp files under /tmp, so it boots as # any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem: - # true` here needs an emptyDir volume mounted over /tmp. Images before - # the /tmp move instead need emptyDirs over /var/cache/nginx and /run to - # run as a non-root uid at all. + # true` here needs an emptyDir volume mounted over /tmp. podLabels: {} podSecurityContext: {} securityContext: {} extraContainers: [] + extraInitContainers: [] lifecycle: {} terminationGracePeriodSeconds: "" nodeSelector: {} @@ -644,6 +775,24 @@ ui: # Same shape as gateway.topologySpreadConstraints. topologySpreadConstraints: [] +# Optional: bundled LiteAdmin worker. Renders one Deployment, Service and PVC +# running the shared image with `args: ["--admin-agent"]`, and points the pod +# that serves the /liteadmin routes (the backend in componentized mode, the +# monolith in monolith mode) at it via LITELLM_ADMIN_AGENT_URL. +liteadmin: + enabled: false + existingSecret: "" + gatewayUrl: "" + model: "" + readOnly: false + storageSize: 1Gi + storageClassName: "" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi lens: enabled: false url: "" diff --git a/scripts/tpm_headline_test.sh b/scripts/tpm_headline_test.sh index a2f4063d311..ef6750e2433 100755 --- a/scripts/tpm_headline_test.sh +++ b/scripts/tpm_headline_test.sh @@ -12,7 +12,7 @@ # Post-PR: only ~1–2 fit under tpm_limit=100, rest return 429. # # Setup (separate terminal): -# kubectl port-forward -n litellm svc/yassin-veks-litellm-helm 4000:4000 +# kubectl port-forward -n litellm svc/litellm-gateway 4000:4000 # # Run: # bash scripts/tpm_headline_test.sh diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 6347c165865..9a091959631 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -2492,17 +2492,15 @@ class TestProxyInitializationHelpers: class TestQueryEngineReaperWiring: - def _invoke_run_server(self, args): + def _invoke_run_server(self, args, env: dict[str, str] | None = None): from click.testing import CliRunner from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL", "NUM_WORKERS")} + if env: + clean_env.update(env) with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -2548,6 +2546,20 @@ class TestQueryEngineReaperWiring: mock_uvicorn_run.assert_called_once() mock_start_reaper.assert_not_called() + def test_num_workers_env_var_sets_uvicorn_workers(self): + result, mock_uvicorn_run, _ = self._invoke_run_server(["--local"], env={"NUM_WORKERS": "3"}) + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" + mock_uvicorn_run.assert_called_once() + assert mock_uvicorn_run.call_args.kwargs["workers"] == 3 + + def test_num_workers_flag_beats_env_var(self): + result, mock_uvicorn_run, _ = self._invoke_run_server( + ["--local", "--num_workers", "2"], env={"NUM_WORKERS": "3"} + ) + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" + mock_uvicorn_run.assert_called_once() + assert mock_uvicorn_run.call_args.kwargs["workers"] == 2 + @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") def test_gunicorn_arbiter_starts_reaper(self): pytest.importorskip("gunicorn") diff --git a/tests/unit/test_docker_entrypoint.py b/tests/unit/test_docker_entrypoint.py index 1d8da4a9b62..bd0b9b0fd4b 100644 --- a/tests/unit/test_docker_entrypoint.py +++ b/tests/unit/test_docker_entrypoint.py @@ -4,8 +4,10 @@ import re import shlex import stat import subprocess +import sys from pathlib import Path from typing import Final +from urllib.parse import parse_qs, urlencode, urlsplit import pytest @@ -555,3 +557,145 @@ def test_entrypoint_script_is_executable() -> None: def test_entrypoint_script_has_no_carriage_returns() -> None: assert b"\r" not in DOCKER_ENTRYPOINT.read_bytes() + + +def _run_proxy_with_discrete_database( + tmp_path: Path, extra_env: dict[str, str | None], extra_pythonpath: Path | None = None +) -> list[str]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + (bin_dir / "litellm").write_text( + '#!/bin/sh\n{ echo "args=$*"; echo "DATABASE_URL=$DATABASE_URL"; ' + 'echo "DATABASE_URL_READ_REPLICA=${DATABASE_URL_READ_REPLICA-}"; } >> "$RECORD"\n' + ) + (bin_dir / "litellm").chmod(0o755) + (bin_dir / "python").write_text(f'#!/bin/sh\nexec "{sys.executable}" "$@"\n') + (bin_dir / "python").chmod(0o755) + record = tmp_path / "record.txt" + env = _entrypoint_env( + bin_dir, + record, + { + "DATABASE_URL": None, + "DATABASE_URL_READ_REPLICA": None, + "DIRECT_URL": None, + "DATABASE_MAX_IDLE_CONNECTION_LIFETIME": None, + "USE_AWS_KMS": None, + "DATABASE_HOST": "writer.db", + "DATABASE_PORT": "6543", + "DATABASE_USER": "app", + "DATABASE_PASSWORD": "p@ss", + "DATABASE_NAME": "litellm", + "DATABASE_HOST_READ_REPLICA": "reader.db", + "DATABASE_PORT_READ_REPLICA": "6544", + **extra_env, + }, + ) + env["PYTHONPATH"] = os.pathsep.join( + [str(extra_pythonpath), str(REPO_ROOT)] if extra_pythonpath else [str(REPO_ROOT)] + ) + + result = subprocess.run( + ["sh", str(DOCKER_ENTRYPOINT), "proxy", "--port", "4000"], + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" + return record.read_text().splitlines() + + +def test_proxy_assembles_database_urls_from_discrete_variables(tmp_path: Path) -> None: + lines = _run_proxy_with_discrete_database(tmp_path, {}) + + assert lines[0] == "args=--port 4000" + assert lines[1].startswith("DATABASE_URL=postgresql://app:p%40ss@writer.db:6543/litellm") + assert lines[2].startswith("DATABASE_URL_READ_REPLICA=postgresql://app:p%40ss@reader.db:6544/litellm") + assert "max_idle_connection_lifetime" not in parse_qs(urlsplit(lines[1]).query) + assert "max_idle_connection_lifetime" not in parse_qs(urlsplit(lines[2]).query) + + +def test_proxy_decrypts_a_kms_database_password_before_assembling(tmp_path: Path) -> None: + site_dir = tmp_path / "site" + site_dir.mkdir() + (site_dir / "sitecustomize.py").write_text( + "import os\n" + "import litellm.secret_managers.aws_secret_manager as kms\n" + "kms.decrypt_env_var = lambda: {'DATABASE_PASSWORD': 'p@ss'} " + "if os.environ.get('DATABASE_PASSWORD') == 'aws_kms/ciphertext' else {}\n" + ) + lines = _run_proxy_with_discrete_database( + tmp_path, + {"USE_AWS_KMS": "True", "DATABASE_PASSWORD": "aws_kms/ciphertext"}, + extra_pythonpath=site_dir, + ) + + assert lines[1].startswith("DATABASE_URL=postgresql://app:p%40ss@writer.db:6543/litellm") + assert lines[2].startswith("DATABASE_URL_READ_REPLICA=postgresql://app:p%40ss@reader.db:6544/litellm") + + +def test_proxy_pins_an_env_configured_idle_lifetime(tmp_path: Path) -> None: + lines = _run_proxy_with_discrete_database(tmp_path, {"DATABASE_MAX_IDLE_CONNECTION_LIFETIME": "120"}) + + assert "max_idle_connection_lifetime=120" in lines[1] + assert "max_idle_connection_lifetime=120" in lines[2] + + +def test_proxy_keeps_a_reader_url_pinned_idle_lifetime(tmp_path: Path) -> None: + lines: Final = _run_proxy_with_discrete_database( + tmp_path, {"DATABASE_URL_READ_REPLICA": "postgresql://app:pass@reader.db/litellm?max_idle_connection_lifetime=300"} + ) + assert "max_idle_connection_lifetime" not in parse_qs(urlsplit(lines[1]).query) + assert "max_idle_connection_lifetime=300" in lines[2] + assert "max_idle_connection_lifetime=60" not in lines[2] + + +@pytest.mark.parametrize("pinned_writer", (False, True)) +def test_proxy_reader_uses_prisma_tls_and_writer_pool_settings(tmp_path: Path, pinned_writer: bool) -> None: + root_cert: Final = str(tmp_path / "absent-ca.pem") + writer_query: Final = urlencode( + { + "sslmode": "verify-full", + "sslrootcert": root_cert, + "connection_limit": "7", + "socket_timeout": "9", + "max_idle_connection_lifetime": "300", + } + ) + lines: Final = _run_proxy_with_discrete_database( + tmp_path, + { + "DATABASE_URL": f"postgresql://app:pass@writer.db/litellm?{writer_query}" if pinned_writer else None, + "DATABASE_HOST": None if pinned_writer else "writer.db", + "DATABASE_USER_READ_REPLICA": "reader", + "DATABASE_PASSWORD_READ_REPLICA": "rpass", + "DATABASE_NAME_READ_REPLICA": "replica", + "DATABASE_SSLMODE": "verify-full", + "DATABASE_SSLROOTCERT": root_cert, + }, + ) + reader: Final = urlsplit(lines[2].removeprefix("DATABASE_URL_READ_REPLICA=")) + assert (reader.hostname, reader.username, reader.path) == ("reader.db", "reader", "/replica") + assert parse_qs(reader.query) == { + "sslmode": ["require"], + "sslcert": [root_cert], + "sslaccept": ["strict"], + **( + {"connection_limit": ["7"], "socket_timeout": ["9"], "max_idle_connection_lifetime": ["300"]} + if pinned_writer + else {} + ), + } + + +def test_proxy_keeps_an_operator_database_url(tmp_path: Path) -> None: + record = _run_entrypoint( + ("proxy",), + tmp_path, + DATABASE_URL="postgresql://pinned/db", + DATABASE_HOST="writer.db", + ) + + assert record[0] == "exec=litellm" From ec8d96d7d4016815f6ed90435f616e7489c72f97 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:03:33 -0800 Subject: [PATCH 082/189] feat(terraform): run every AWS and GCP workload from the one litellm image (#43358) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- terraform/litellm/README.md | 31 ++-- terraform/litellm/aws/README.md | 43 +++--- terraform/litellm/aws/ecs.tf | 72 ++++------ terraform/litellm/aws/locals.tf | 5 +- terraform/litellm/aws/migrations.tf | 22 +-- .../litellm/aws/tests/collector.tftest.hcl | 11 +- .../aws/tests/connection_pool.tftest.hcl | 8 +- terraform/litellm/aws/tests/images.tftest.hcl | 50 +++---- .../aws/tests/metrics_sidecar.tftest.hcl | 7 +- .../aws/tests/unified_image.tftest.hcl | 73 ++++++++++ terraform/litellm/aws/variables.tf | 55 +++---- terraform/litellm/gcp/README.md | 42 +++--- terraform/litellm/gcp/bootstrap.tf | 6 +- terraform/litellm/gcp/cloudrun.tf | 41 +++--- .../litellm/gcp/examples/default/TUTORIAL.md | 2 +- .../gcp/examples/default/deploystack.json | 4 +- .../litellm/gcp/examples/default/main.tf | 2 +- .../examples/default/terraform.tfvars.example | 11 +- .../litellm/gcp/examples/default/variables.tf | 8 +- terraform/litellm/gcp/locals.tf | 8 +- .../litellm/gcp/tests/collector.tftest.hcl | 6 +- .../gcp/tests/connection_pool.tftest.hcl | 12 +- .../gcp/tests/metrics_sidecar.tftest.hcl | 8 +- .../gcp/tests/unified_image.tftest.hcl | 72 ++++++++++ terraform/litellm/gcp/variables.tf | 71 ++++----- tests/unit/test_docker_entrypoint.py | 136 +++++++----------- 26 files changed, 416 insertions(+), 390 deletions(-) create mode 100644 terraform/litellm/aws/tests/unified_image.tftest.hcl create mode 100644 terraform/litellm/gcp/tests/unified_image.tftest.hcl diff --git a/terraform/litellm/README.md b/terraform/litellm/README.md index 7cc6e4c08ba..1c2ad1d9b25 100644 --- a/terraform/litellm/README.md +++ b/terraform/litellm/README.md @@ -34,11 +34,16 @@ secret-manager refs. The proxy is split into three deployables: -| Component | Default image | Port | Role | -| --------- | ---------------------------------------- | ---- | -------------------------------------------------------------------- | -| `gateway` | `ghcr.io/berriai/litellm-gateway:main-stable` | 4000 | LLM data plane (`/v1/chat/completions`, `/v1/embeddings`, …) | -| `backend` | `ghcr.io/berriai/litellm-backend:main-stable` | 4001 | Management API (`/key/*`, `/user/*`, `/team/*`, `/model/*`, …) | -| `ui` | `ghcr.io/berriai/litellm-ui:main-stable` | 3000 | Static Next.js dashboard served by nginx | +| Component | Entrypoint argument | Port | Role | +| --------- | ------------------- | ---- | -------------------------------------------------------------------- | +| `gateway` | `gateway` | 4000 | LLM data plane (`/v1/chat/completions`, `/v1/embeddings`, …) | +| `backend` | `backend` | 4001 | Management API (`/key/*`, `/user/*`, `/team/*`, `/model/*`, …) | +| `ui` | `ui` | 3000 | Static Next.js dashboard served by nginx | + +All of them run the same `ghcr.io/berriai/litellm` image (`image` on AWS, +`image_registry` + `image_tag` or `image` on GCP). The image entrypoint +starts the process named by its first argument, so each workload only +differs in the argument the module passes The load balancer routes gateway path prefixes (mirrored verbatim from `gateway/routes/allowlist.py`) to the gateway, UI asset paths (`/`, @@ -152,8 +157,8 @@ pin to a specific tag for production: LiteLLM's proxy runs `prisma migrate deploy` at startup, but on first apply the gateway/backend can race the empty database. Both stacks expose a -one-off migration task that runs `python litellm/proxy/prisma_migration.py` -against the backend image: +one-off migration task that runs the `migrations` component of the same +image: - AWS: an `aws_ecs_task_definition` (`litellm-migrations`). Run with `aws ecs run-task` — the command is printed in `terraform output`. @@ -234,19 +239,17 @@ dynamic-credentials OIDC). Required overrides the launcher must supply per stack: - **AWS** (`terraform/litellm/aws`): `region`, `azs`, `tenant`, `env`. - The image vars (`gateway_image`, `backend_image`, `ui_image`, - `migrations_image`) can be left at their defaults — the GHCR images - are anonymous-readable and ECS Fargate pulls them without extra - credentials. + `image` can be left at its default: the GHCR image is + anonymous-readable and ECS Fargate pulls it without extra credentials. - **GCP** (`terraform/litellm/gcp`): `project`, `tenant`, `env`, **and one of**: - `image_registry` pointed at an Artifact Registry **remote** repository backed by `https://ghcr.io` (e.g. `us-central1-docker.pkg.dev//litellm/berriai`), so Cloud Run - pulls the four upstream `litellm-*` images through it; or - - all four per-component `*_image` URIs pointing at images mirrored - into a regular Artifact Registry repo. + pulls the upstream `litellm` image through it; or + - `image` pointing at a copy mirrored into a regular Artifact Registry + repo. The defaults (`ghcr.io/berriai`) cause Cloud Run admission to reject the service spec — Cloud Run only authenticates against Artifact diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 6f0450c0091..7aebfe2acc9 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -12,7 +12,7 @@ Deploys the componentized LiteLLM proxy on AWS: - LLM data-plane prefixes (`/v1/chat/*`, `/v1/embeddings`, …) → `gateway` - UI assets (`/`, `/_next/*`, `/litellm-asset-prefix/*`, …) → `ui` - Everything else (management API: `/key/*`, `/user/*`, …) → `backend` -- **One-off migration task** (`litellm-migrations`) that runs `prisma migrate deploy` from the dedicated `ghcr.io/berriai/litellm-migrations` image +- **One-off migration task** (`litellm-migrations`) that runs `prisma migrate deploy` through the `migrations` component of the same image ## Bring your own networking, database, and Redis @@ -238,8 +238,7 @@ Metering also requires an enterprise license. To tune the export cadence, set (`python -m litellm.proxy.prometheus_metrics_server`) to the gateway task that aggregates the workers' samples over a shared task volume, so a scrape never runs on an inference worker. The ALB never routes to that port and the tasks -security group only opens it to `gateway_metrics_scrape_cidrs`. Needs -`gateway_image` v1.101.0 or newer. See +security group only opens it to `gateway_metrics_scrape_cidrs`. See [Prometheus metrics](https://docs.litellm.ai/docs/proxy/prometheus) for the metrics themselves. @@ -275,10 +274,10 @@ tokens the workers used to (see [Aurora + IAM auth](#aurora--iam-auth)): the pooler mints a token from the task role, renews it before it expires and hands the workers a loopback URL with a static password instead -The componentized `gateway_image` starts through `python -m gateway.launch`, +The image's `gateway` component starts through `python -m gateway.launch`, which reads these variables, starts the pooler once per task and hands the -workers its loopback URL; the classic `litellm` image honours them the same -way. +workers its loopback URL; the monolithic `proxy` component honours them the +same way. ### Scaling the gateway on requests and tokens @@ -492,20 +491,26 @@ top — set org-wide tags there, per-deployment tags via the `tags` input. ## Image pulls -The defaults pull `ghcr.io/berriai/litellm-:${image_tag}`, -which is anonymous-readable. There are four images: `litellm-gateway`, -`litellm-backend`, `litellm-ui`, and `litellm-migrations` (slim image used -only by the one-off migration task — runs `prisma migrate deploy` against -the writer DB and exits). A published module version (Terraform Registry -`BerriAI/litellm/aws`) ships with `image_tag` stamped to its own release -tag, so bumping the module version bumps all four images with it. A git -ref of this monorepo has no published images of its own, so there +The default pulls `ghcr.io/berriai/litellm:${image_tag}`, which is +anonymous-readable. The gateway, backend and UI services, the metrics and +collector sidecars and the one-off migration task all run that one image +and pick their process through the entrypoint's first argument, so bumping +LiteLLM is a single variable change. A published module version (Terraform +Registry `BerriAI/litellm/aws`) ships with `image_tag` stamped to its own +release tag, so bumping the module version bumps every workload with it. A +git ref of this monorepo has no published images of its own, so there `image_tag` is required. Set `image_tag` to pin a different LiteLLM -release, or -`image_registry` to pull the same four images from a mirror. Set -`gateway_image` / `backend_image` / `ui_image` / `migrations_image` only -for a per-component override (custom build, different tag). To pull from a -private registry: +release, `image_registry` to pull from a mirror, or `image` for a full +override (custom build, digest pin). A tag older than the one this module +version ships with has no component entrypoint and only knows how to run +the monolithic proxy, so pinning one fails at container start. + +Migrating from the retired `gateway_image`, `backend_image`, `ui_image` +and `migrations_image` inputs: drop them and set `image_tag` (or `image`) +once. Terraform +rejects the old names at plan time (`An argument named "gateway_image" +is not expected here`), so a stale `.tfvars` cannot silently keep pulling +the per-component images. To pull from a private registry: - **ECR (same account)**: the execution role already has `AmazonECSTaskExecutionRolePolicy`, which grants ECR pull for repos in diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index dec7f599ced..bc40d78413e 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -194,11 +194,18 @@ locals { proxy_config_fetch_cmd = "python -c \"import os, boto3; boto3.client('s3', region_name=os.environ['AWS_REGION']).download_file(os.environ['LITELLM_PROXY_CONFIG_S3_BUCKET'], os.environ['LITELLM_PROXY_CONFIG_S3_KEY'], os.environ['CONFIG_FILE_PATH'])\"" - # Gateway always needs --workers wired in (no NUM_WORKERS env var support - # in the image entrypoint). When proxy_config is enabled we also have to - # pull the config from S3 first, so the command goes through `sh -c`; - # otherwise we keep the image's ENTRYPOINT and only override `command`. - gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}" + image_entrypoint = "/app/docker-entrypoint.sh" + component_args = { + gateway = ["gateway", "--workers", tostring(var.gateway_num_workers)] + backend = ["backend"] + collector = ["collector"] + } + component_overrides = { + for component, args in local.component_args : component => local.proxy_config_enabled ? { + entryPoint = ["sh", "-c"] + command = ["${local.proxy_config_fetch_cmd} && exec ${local.image_entrypoint} ${join(" ", args)}"] + } : { command = args } + } gateway_pool_env = var.gateway_connection_pool_enabled ? [ { name = "LITELLM_PGBOUNCER_ENABLED", value = "true" }, @@ -215,11 +222,10 @@ locals { gateway_metrics_container = local.metrics_enabled ? [ { - name = "metrics" - image = local.gateway_image - essential = false - entryPoint = ["python", "-m", "litellm.proxy.prometheus_metrics_server"] - command = ["--port", tostring(var.gateway_metrics_port)] + name = "metrics" + image = local.image + essential = false + command = ["metrics", "--port", tostring(var.gateway_metrics_port)] portMappings = [{ containerPort = var.gateway_metrics_port, protocol = "tcp" }] environment = local.metrics_env @@ -244,28 +250,6 @@ locals { } ] : [] - backend_uvicorn_args = "--host 0.0.0.0 --port 4001" - - gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run python -m gateway.launch ${local.gateway_uvicorn_args};; *) exec python -m gateway.launch ${local.gateway_uvicorn_args};; esac" - backend_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn backend.main:app ${local.backend_uvicorn_args};; *) exec uvicorn backend.main:app ${local.backend_uvicorn_args};; esac" - - gateway_proxy_overrides = local.proxy_config_enabled ? { - entryPoint = ["sh", "-c"] - command = [ - "${local.proxy_config_fetch_cmd} && ${local.gateway_launch_cmd}" - ] - } : { - entryPoint = ["sh", "-c"] - command = [local.gateway_launch_cmd] - } - - backend_proxy_overrides = local.proxy_config_enabled ? { - entryPoint = ["sh", "-c"] - command = [ - "${local.proxy_config_fetch_cmd} && ${local.backend_launch_cmd}" - ] - } : {} - collector_address = "tcp://127.0.0.1:${var.collector_port}" collector_env = var.collector_enabled ? [ { name = "LITELLM_COLLECTOR_ENABLED", value = "true" }, @@ -286,22 +270,15 @@ locals { local.collector_env, ) - collector_launch_cmd = "exec python -m litellm.proxy.collector" - collector_command = [ - local.proxy_config_enabled ? "${local.proxy_config_fetch_cmd} && ${local.collector_launch_cmd}" : local.collector_launch_cmd - ] - - collector_container = var.collector_enabled ? [{ + collector_container = var.collector_enabled ? [merge({ name = "collector" - image = local.gateway_image + image = local.image essential = false cpu = var.collector_cpu memory = var.collector_memory restartPolicy = { enabled = true } - entryPoint = ["sh", "-c"] - command = local.collector_command environment = concat( local.shared_env, local.gateway_extra_env_list, @@ -320,7 +297,7 @@ locals { awslogs-stream-prefix = "collector" } } - }] : [] + }, local.component_overrides.collector)] : [] } # ---------- Gateway ---------- @@ -361,7 +338,7 @@ resource "aws_ecs_task_definition" "gateway" { merge( { name = "gateway" - image = local.gateway_image + image = local.image essential = true portMappings = [{ containerPort = 4000, protocol = "tcp" }] @@ -382,7 +359,7 @@ resource "aws_ecs_task_definition" "gateway" { } } }, - local.gateway_proxy_overrides, + local.component_overrides.gateway, ) ], local.gateway_metrics_container, local.collector_container)) @@ -469,7 +446,7 @@ resource "aws_ecs_task_definition" "backend" { merge( { name = "backend" - image = local.backend_image + image = local.image essential = true portMappings = [{ containerPort = 4001, protocol = "tcp" }] @@ -492,7 +469,7 @@ resource "aws_ecs_task_definition" "backend" { } } }, - local.backend_proxy_overrides, + local.component_overrides.backend, ) ]) @@ -559,8 +536,9 @@ resource "aws_ecs_task_definition" "ui" { container_definitions = jsonencode([ { name = "ui" - image = local.ui_image + image = local.image essential = true + command = ["ui"] portMappings = [{ containerPort = 3000, protocol = "tcp" }] logConfiguration = { diff --git a/terraform/litellm/aws/locals.tf b/terraform/litellm/aws/locals.tf index c06b171d6bd..e31e2b3c11b 100644 --- a/terraform/litellm/aws/locals.tf +++ b/terraform/litellm/aws/locals.tf @@ -11,10 +11,7 @@ locals { # the stack can reference local.name. name = "${var.tenant}-litellm-${var.env}" - gateway_image = var.gateway_image != "" ? var.gateway_image : "${var.image_registry}/litellm-gateway:${var.image_tag}" - backend_image = var.backend_image != "" ? var.backend_image : "${var.image_registry}/litellm-backend:${var.image_tag}" - ui_image = var.ui_image != "" ? var.ui_image : "${var.image_registry}/litellm-ui:${var.image_tag}" - migrations_image = var.migrations_image != "" ? var.migrations_image : "${var.image_registry}/litellm-migrations:${var.image_tag}" + image = var.image != "" ? var.image : "${var.image_registry}/litellm:${var.image_tag}" # This is a reusable module — it declares no `provider` block, so the AWS # provider's `default_tags` is the caller's concern, not ours. To keep the diff --git a/terraform/litellm/aws/migrations.tf b/terraform/litellm/aws/migrations.tf index 57c33f69a8b..43ae33b5b83 100644 --- a/terraform/litellm/aws/migrations.tf +++ b/terraform/litellm/aws/migrations.tf @@ -1,16 +1,8 @@ -# Task definition for the dedicated litellm-migrations image. Mirrors the -# pre-install/pre-upgrade Helm hook in helm/litellm/templates/migrations-job.yaml. -# -# The image (built from migrations/Dockerfile) ships with -# `ENTRYPOINT ["python3", "/app/run.py"]`. run.py assembles DATABASE_URL from -# the discrete DATABASE_* env vars (IAM auth here) via DatabaseURLSettings, -# then calls ProxyExtrasDBManager.setup_database() — i.e. `prisma migrate -# deploy` with the v2 resolver and P3005/P3009/P3018 recovery. It does NOT -# read CONFIG_FILE_PATH, the master key, or DISABLE_SCHEMA_UPDATE, so we -# don't pass them. -# -# Invoked automatically by `terraform_data.migration` in bootstrap.tf during -# every apply (after the IAM-authed user has been created). The +# Runs the `migrations` component of local.image: run.py assembles DATABASE_URL +# from the discrete DATABASE_* env vars (IAM auth here) and runs `prisma +# migrate deploy`. It does not read CONFIG_FILE_PATH, the master key, or +# DISABLE_SCHEMA_UPDATE, so we don't pass them. Invoked automatically by +# `terraform_data.migration` in bootstrap.tf during every apply; the # `migration_run_command` output is preserved for break-glass manual re-runs. resource "aws_ecs_task_definition" "migrations" { count = local.database_enabled ? 1 : 0 @@ -28,10 +20,10 @@ resource "aws_ecs_task_definition" "migrations" { container_definitions = jsonencode([{ name = "migrations" - image = local.migrations_image + image = local.image essential = true + command = ["migrations"] - # No entryPoint/command override — the image's ENTRYPOINT runs run.py. environment = local.shared_env secrets = local.byo_database_secrets diff --git a/terraform/litellm/aws/tests/collector.tftest.hcl b/terraform/litellm/aws/tests/collector.tftest.hcl index 082f17b2c08..d38c3882a1b 100644 --- a/terraform/litellm/aws/tests/collector.tftest.hcl +++ b/terraform/litellm/aws/tests/collector.tftest.hcl @@ -67,14 +67,14 @@ run "enabled_adds_a_sidecar_that_shares_the_gateway_transport" { assert { condition = ( - local.collector_container[0].image == local.gateway_image && - local.collector_container[0].entryPoint == ["sh", "-c"] && - local.collector_container[0].command == ["exec python -m litellm.proxy.collector"] && + local.collector_container[0].image == local.image && + !can(local.collector_container[0].entryPoint) && + local.collector_container[0].command == tolist(["collector"]) && local.collector_container[0].essential == false && local.collector_container[0].restartPolicy.enabled == true && { for e in local.collector_container[0].environment : e.name => e.value }["LITELLM_JOB_ROLE"] == "collector" ) - error_message = "The sidecar must run litellm.proxy.collector from the gateway image as a restartable, non-essential collector." + error_message = "The sidecar must run the collector component of the shared image as a restartable, non-essential collector." } assert { @@ -109,8 +109,9 @@ run "proxy_config_is_fetched_by_the_sidecar_too" { assert { condition = ( + local.collector_container[0].entryPoint == tolist(["sh", "-c"]) && startswith(local.collector_container[0].command[0], local.proxy_config_fetch_cmd) && - endswith(local.collector_container[0].command[0], "exec python -m litellm.proxy.collector") && + endswith(local.collector_container[0].command[0], "exec /app/docker-entrypoint.sh collector") && contains([for e in local.collector_container[0].environment : e.name], "CONFIG_FILE_PATH") ) error_message = "The sidecar must pull the proxy config from S3 before starting, like the gateway does." diff --git a/terraform/litellm/aws/tests/connection_pool.tftest.hcl b/terraform/litellm/aws/tests/connection_pool.tftest.hcl index c618bd11768..c22f0a11fc1 100644 --- a/terraform/litellm/aws/tests/connection_pool.tftest.hcl +++ b/terraform/litellm/aws/tests/connection_pool.tftest.hcl @@ -117,12 +117,10 @@ run "gateway_starts_through_the_pool_aware_launcher" { assert { condition = alltrue([ - strcontains(local.gateway_launch_cmd, "exec python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), - strcontains(local.gateway_launch_cmd, "exec ddtrace-run python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), - !strcontains(local.gateway_launch_cmd, "uvicorn gateway.main:app"), - local.gateway_proxy_overrides.command[0] == local.gateway_launch_cmd, + local.component_overrides.gateway.command == tolist(["gateway", "--workers", "4"]), + !can(local.component_overrides.gateway.entryPoint), ]) - error_message = "The gateway must start through gateway.launch (with and without ddtrace) so the pooler starts once before uvicorn forks the workers." + error_message = "The gateway must start through the image's gateway component with the configured worker count so the pooler starts once before uvicorn forks the workers." } } diff --git a/terraform/litellm/aws/tests/images.tftest.hcl b/terraform/litellm/aws/tests/images.tftest.hcl index 2d03a46b7a3..800b989af8a 100644 --- a/terraform/litellm/aws/tests/images.tftest.hcl +++ b/terraform/litellm/aws/tests/images.tftest.hcl @@ -16,7 +16,7 @@ variables { azs = ["us-east-1a", "us-east-1b"] } -run "every_component_follows_image_registry_and_image_tag" { +run "the_image_follows_image_registry_and_image_tag" { command = plan variables { @@ -25,17 +25,12 @@ run "every_component_follows_image_registry_and_image_tag" { } assert { - condition = ( - local.gateway_image == "123456789012.dkr.ecr.us-east-1.amazonaws.com/mirror/litellm-gateway:v9.9.9" && - local.backend_image == "123456789012.dkr.ecr.us-east-1.amazonaws.com/mirror/litellm-backend:v9.9.9" && - local.ui_image == "123456789012.dkr.ecr.us-east-1.amazonaws.com/mirror/litellm-ui:v9.9.9" && - local.migrations_image == "123456789012.dkr.ecr.us-east-1.amazonaws.com/mirror/litellm-migrations:v9.9.9" - ) - error_message = "All four images must be composed as /litellm-:." + condition = local.image == "123456789012.dkr.ecr.us-east-1.amazonaws.com/mirror/litellm:v9.9.9" + error_message = "The image must be composed as /litellm:." } } -run "an_empty_image_tag_fails_closed_unless_every_image_is_overridden" { +run "an_empty_image_tag_fails_closed_unless_image_is_set" { command = plan variables { @@ -55,50 +50,39 @@ run "a_null_image_tag_takes_the_module_default" { expect_failures = [var.image_tag] } -run "four_explicit_uris_need_no_image_tag" { +run "an_explicit_uri_needs_no_image_tag" { command = plan variables { - image_tag = "" - gateway_image = "example.com/custom/gateway:nightly" - backend_image = "example.com/custom/backend:nightly" - ui_image = "example.com/custom/ui:nightly" - migrations_image = "example.com/custom/migrations:nightly" + image_tag = "" + image = "example.com/custom/litellm:nightly" } assert { - condition = local.migrations_image == "example.com/custom/migrations:nightly" - error_message = "Explicit per-component URIs must satisfy the module without an image_tag." + condition = local.image == "example.com/custom/litellm:nightly" + error_message = "An explicit image URI must satisfy the module without an image_tag." } } -run "composed_defaults_pull_from_ghcr_at_the_image_tag" { +run "the_composed_default_pulls_from_ghcr_at_the_image_tag" { command = plan assert { - condition = alltrue([ - for image in [local.gateway_image, local.backend_image, local.ui_image, local.migrations_image] : - startswith(image, "ghcr.io/berriai/litellm-") && endswith(image, ":${var.image_tag}") - ]) - error_message = "With no overrides every component must come from ghcr.io/berriai at image_tag." + condition = local.image == "ghcr.io/berriai/litellm:${var.image_tag}" + error_message = "With no override the image must come from ghcr.io/berriai at image_tag." } } -run "a_per_component_uri_wins_over_the_composed_image" { +run "an_explicit_uri_wins_over_the_composed_image" { command = plan variables { - image_tag = "v9.9.9" - gateway_image = "example.com/custom/gateway:nightly" + image_tag = "v9.9.9" + image = "example.com/custom/litellm:nightly" } assert { - condition = ( - local.gateway_image == "example.com/custom/gateway:nightly" && - local.backend_image == "ghcr.io/berriai/litellm-backend:v9.9.9" && - local.ui_image == "ghcr.io/berriai/litellm-ui:v9.9.9" && - local.migrations_image == "ghcr.io/berriai/litellm-migrations:v9.9.9" - ) - error_message = "A per-component override must replace only that component's image." + condition = local.image == "example.com/custom/litellm:nightly" + error_message = "A full image URI must replace the composed image." } } diff --git a/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl b/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl index b330d4b91c3..ab1a94247f2 100644 --- a/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl +++ b/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl @@ -56,14 +56,15 @@ run "metrics_port_adds_a_sidecar_volume_and_scrape_rule" { length(local.gateway_metrics_container) == 1, local.gateway_metrics_container[0].name == "metrics", local.gateway_metrics_container[0].essential == false, - join(" ", local.gateway_metrics_container[0].entryPoint) == "python -m litellm.proxy.prometheus_metrics_server", - join(" ", local.gateway_metrics_container[0].command) == "--port 9464", + local.gateway_metrics_container[0].image == local.image, + !can(local.gateway_metrics_container[0].entryPoint), + join(" ", local.gateway_metrics_container[0].command) == "metrics --port 9464", one(local.gateway_metrics_container[0].portMappings).containerPort == 9464, one(local.gateway_metrics_container[0].environment).value == "/tmp/litellm_prometheus_multiproc", one(local.gateway_metrics_container[0].mountPoints).sourceVolume == "prometheus-multiproc", strcontains(local.gateway_metrics_container[0].healthCheck.command[3], "9464"), ]) - error_message = "The metrics sidecar must run prometheus_metrics_server on the configured port, share the multiproc volume, and health-check that port." + error_message = "The metrics sidecar must run the shared image's metrics component on the configured port, share the multiproc volume, and health-check that port." } assert { diff --git a/terraform/litellm/aws/tests/unified_image.tftest.hcl b/terraform/litellm/aws/tests/unified_image.tftest.hcl new file mode 100644 index 00000000000..2aead6b474c --- /dev/null +++ b/terraform/litellm/aws/tests/unified_image.tftest.hcl @@ -0,0 +1,73 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} +mock_provider "random" {} + +variables { + region = "us-east-1" + tenant = "acme" + env = "test" + allow_plaintext_alb = true + azs = ["us-east-1a", "us-east-1b"] + image = "123456789012.dkr.ecr.us-east-1.amazonaws.com/litellm:test" + gateway_num_workers = 3 + gateway_metrics_port = 9464 + collector_enabled = true +} + +run "every_container_runs_a_component_of_the_one_image" { + command = plan + + assert { + condition = alltrue([ + jsondecode(aws_ecs_task_definition.ui.container_definitions)[0].image == var.image, + join(" ", jsondecode(aws_ecs_task_definition.ui.container_definitions)[0].command) == "ui", + !can(jsondecode(aws_ecs_task_definition.ui.container_definitions)[0].entryPoint), + ]) + error_message = "The UI task must run the ui component of var.image through the image entrypoint." + } + + assert { + condition = alltrue([ + join(" ", local.component_overrides.gateway.command) == "gateway --workers 3", + join(" ", local.component_overrides.backend.command) == "backend", + join(" ", local.component_overrides.collector.command) == "collector", + alltrue([for component in keys(local.component_args) : !can(local.component_overrides[component].entryPoint)]), + local.gateway_metrics_container[0].image == var.image, + join(" ", local.gateway_metrics_container[0].command) == "metrics --port 9464", + !can(local.gateway_metrics_container[0].entryPoint), + local.collector_container[0].image == var.image, + ]) + error_message = "Gateway, backend and the sidecars must keep the image entrypoint and select their component (plus the gateway worker count) through command." + } +} + +run "proxy_config_wraps_the_fetch_around_the_same_component" { + command = plan + + variables { + proxy_config = { model_list = [] } + } + + assert { + condition = alltrue([ + for component, args in local.component_args : + join(" ", local.component_overrides[component].entryPoint) == "sh -c" && + startswith(local.component_overrides[component].command[0], local.proxy_config_fetch_cmd) && + endswith(local.component_overrides[component].command[0], " && exec /app/docker-entrypoint.sh ${join(" ", args)}") + ]) + error_message = "With proxy_config, gateway, backend and collector must download the config and then exec the image entrypoint with their component." + } + + assert { + condition = alltrue([ + !can(local.gateway_metrics_container[0].entryPoint), + !can(jsondecode(aws_ecs_task_definition.ui.container_definitions)[0].entryPoint), + ]) + error_message = "The metrics sidecar and the UI never read the proxy config and must keep the plain image entrypoint." + } +} diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index be7656c2d48..fb5460ff56d 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -133,52 +133,33 @@ variable "azs" { default = [] } -# ---------- Component images ---------- +# ---------- Image ---------- # variable "image_registry" { - description = "Registry path prefix used to compose the four LiteLLM image URIs as `/litellm-:`. Per-component overrides (`gateway_image`, `backend_image`, `ui_image`, `migrations_image`) bypass this entirely when set." + description = "Registry path prefix used to compose the LiteLLM image URI as `/litellm:`. `image` bypasses this entirely when set." type = string default = "ghcr.io/berriai" } variable "image_tag" { - description = "Tag applied to all four litellm-* images when composed from `image_registry`. A published module version ships with this stamped to its own release tag, which matches the `v`-prefixed tag published to GHCR for the same LiteLLM release. Set it to pin a different LiteLLM release. When sourcing the module from a git ref of the monorepo it has no default and must be set." + description = "Tag of the LiteLLM image when composed from `image_registry`. A published module version ships with this stamped to its own release tag, which matches the `v`-prefixed tag published to GHCR for the same LiteLLM release. Set it to pin a different LiteLLM release. When sourcing the module from a git ref of the monorepo it has no default and must be set. Tags older than the one a module version ships with have no component entrypoint and only run the monolithic proxy." type = string default = "" nullable = false validation { - condition = var.image_tag != "" || (var.gateway_image != "" && var.backend_image != "" && var.ui_image != "" && var.migrations_image != "") - error_message = "image_tag is empty. Published module versions (registry.terraform.io/modules/BerriAI/litellm) ship with it stamped to their own release. A git ref of the monorepo has no published images of its own, so set image_tag to a published LiteLLM release tag or set all four *_image URIs." + condition = var.image_tag != "" || var.image != "" + error_message = "image_tag is empty. Published module versions (registry.terraform.io/modules/BerriAI/litellm) ship with it stamped to their own release. A git ref of the monorepo has no published images of its own, so set image_tag to a published LiteLLM release tag or set image to a full URI." } } -variable "gateway_image" { - description = "Full image URI for the gateway (data plane, port 4000). Empty (default) composes from `image_registry` + `image_tag`." - type = string - default = "" -} - -variable "backend_image" { - description = "Full image URI for the backend (management API, port 4001). Empty (default) composes from `image_registry` + `image_tag`." - type = string - default = "" -} - -variable "ui_image" { - description = "Full image URI for the UI (nginx static export, port 3000). Empty (default) composes from `image_registry` + `image_tag`." - type = string - default = "" -} - -variable "migrations_image" { +variable "image" { description = <<-EOT - Full image URI for the one-off prisma migration task. Empty (default) - composes from `image_registry` + `image_tag`. Built from - `migrations/Dockerfile` — slim image whose ENTRYPOINT runs - `python3 /app/run.py` (assembles DATABASE_URL from DATABASE_* env vars - via DatabaseURLSettings, then runs `prisma migrate deploy`). Should track - the same release tag as gateway/backend/ui. + Full URI of the one LiteLLM image every task runs. Its entrypoint picks + the process from the first argument (`gateway`, `backend`, `ui`, + `migrations`, `metrics`, `collector`), so the gateway, backend, UI and + migration task definitions and the sidecars all pull this URI. Empty + (default) composes from `image_registry` + `image_tag`. EOT type = string default = "" @@ -649,13 +630,11 @@ variable "gateway_metrics_port" { description = <<-EOT Serve Prometheus /metrics from a `metrics` sidecar container in the gateway task on this port (1-65535, not 4000), so a scrape never runs on - an inference worker. The sidecar runs the gateway image with - `python -m litellm.proxy.prometheus_metrics_server` and aggregates the - workers' PROMETHEUS_MULTIPROC_DIR samples over a task volume. Null (the - default) leaves /metrics on the gateway port only. The sidecar port has - no virtual-key auth and is not routed through the ALB; open it to your - scrapers with gateway_metrics_scrape_cidrs. Needs gateway_image v1.101.0 - or newer. + an inference worker. The sidecar runs the `metrics` component of `image` + and aggregates the workers' PROMETHEUS_MULTIPROC_DIR samples over a task + volume. Null (the default) leaves /metrics on the gateway port only. The + sidecar port has no virtual-key auth and is not routed through the ALB; + open it to your scrapers with gateway_metrics_scrape_cidrs. EOT type = number default = null @@ -811,7 +790,7 @@ variable "billing_metrics_token_url" { # ---------- Collector sidecar ---------- # # Opt-in offload of spend tracking from the gateway's uvicorn workers to a -# `python -m litellm.proxy.collector` sidecar in the same Fargate task (helm's +# `collector` component sidecar in the same Fargate task (helm's # `gateway.collector`). Fargate awsvpc tasks share one network namespace, # so the sidecar listens on loopback TCP. Disabled (the default) adds nothing # to the task definition. diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 9e21ba14543..32bbd1533f9 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -15,7 +15,7 @@ Deploys the componentized LiteLLM proxy on GCP: - **Secret Manager** entries for `LITELLM_MASTER_KEY` and `DATABASE_PASSWORD` - **Cloud Run v2** services for `gateway` (port 4000), `backend` (port 4001), and `ui` (port 3000), all using a shared runtime service account -- **Cloud Run Job** (`litellm-migrations`) that runs `prisma migrate deploy` from the dedicated `ghcr.io/berriai/litellm-migrations` image +- **Cloud Run Job** (`litellm-migrations`) that runs `prisma migrate deploy` through the `migrations` component of the same image - **External global HTTP(S) load balancer** with serverless NEGs and a URL map mirroring the helm-chart ingress path routing: - LLM data-plane prefixes → `gateway` @@ -24,18 +24,18 @@ Deploys the componentized LiteLLM proxy on GCP: ## Image pulls -There are four images: `litellm-gateway`, `litellm-backend`, `litellm-ui`, -and `litellm-migrations` (slim image used only by the one-off Cloud Run -Job — runs `prisma migrate deploy` against the writer DB and exits). -Bump them together when bumping LiteLLM. +Every Cloud Run service, sidecar and the migrations Job runs the one +`litellm` image and picks its process through the entrypoint's first +argument (`gateway`, `backend`, `ui`, `migrations`, `metrics`, +`collector`), so bumping LiteLLM is a single `image_tag` change. **Required override.** The `image_registry` default (`ghcr.io/berriai`) does **not** work as-is — Cloud Run only accepts images from Artifact Registry, `[region.]gcr.io`, or `docker.io`, and rejects `ghcr.io` URIs at apply time. Every deploy (including HCP Terraform 1-click) must supply either `image_registry` pointed at an Artifact Registry remote -repo backed by GHCR, or full per-component `*_image` URIs against -images you've already mirrored. The default is present only so +repo backed by GHCR, or a full `image` URI for a copy you've already +mirrored. The default is present only so `terraform plan` succeeds during local iteration. **One-time setup (per project):** create a remote repo and let Cloud Run @@ -56,15 +56,22 @@ Then point the stack at it via `image_registry`: image_registry = "us-central1-docker.pkg.dev/my-gcp-project/litellm/berriai" ``` -The four `litellm-:${image_tag}` URIs are composed from -`image_registry` and `image_tag`. A published module version (Terraform -Registry `BerriAI/litellm/google`) ships with `image_tag` stamped to its -own release tag, so bumping the module version bumps all four images with -it. A git ref of this monorepo has no published images of its own, so -there `image_tag` is required. Set `image_tag` to pin a different LiteLLM -release. Set `gateway_image` / `backend_image` / `ui_image` / -`migrations_image` only if you need a per-component override (custom -build, different tag). +The `/litellm:` URI is composed from those two +vars. A published module version (Terraform Registry `BerriAI/litellm/google`) +ships with `image_tag` stamped to its own release tag, so bumping the module +version bumps every workload with it. A git ref of this monorepo has no +published images of its own, so there `image_tag` is required. Set `image` +only if you need a full override (custom build, digest pin). + +A tag older than the one this module version ships with has no component +entrypoint and only knows how to run the monolithic proxy, so pinning one +fails at container start. + +Migrating from the retired `gateway_image`, `backend_image`, `ui_image` +and `migrations_image` inputs: drop them and set `image_registry` + +`image_tag` (or `image`) once. Terraform rejects the old names at plan +time, so a stale `.tfvars` cannot silently keep pulling the per-component +images. Two further notes: @@ -248,8 +255,7 @@ stack also adds Google's Secret Manager that scrapes `localhost:/metrics` every 30s and writes to Cloud Monitoring as `prometheus.googleapis.com/...` metrics. Enabling it grants the runtime service account `roles/monitoring.metricWriter` and -`roles/logging.logWriter` on the project. Needs `gateway_image` v1.101.0 or -newer. See [Prometheus metrics](https://docs.litellm.ai/docs/proxy/prometheus) +`roles/logging.logWriter` on the project. See [Prometheus metrics](https://docs.litellm.ai/docs/proxy/prometheus) for the metrics themselves ```hcl diff --git a/terraform/litellm/gcp/bootstrap.tf b/terraform/litellm/gcp/bootstrap.tf index dead5c41f6b..deaf32ad153 100644 --- a/terraform/litellm/gcp/bootstrap.tf +++ b/terraform/litellm/gcp/bootstrap.tf @@ -7,8 +7,8 @@ # so they don't go live until the schema is in place. # # Triggers: -# - re-runs if the migrations image changes (new release ships new prisma -# migration files). +# - re-runs if the image changes (new release ships new prisma migration +# files). # - re-runs if the migration job is recreated. # # Requires `gcloud` on the machine running terraform, with user creds live @@ -19,7 +19,7 @@ resource "terraform_data" "migration" { triggers_replace = { job_id = google_cloud_run_v2_job.migrations[0].id - job_image = local.migrations_image + job_image = local.image } provisioner "local-exec" { diff --git a/terraform/litellm/gcp/cloudrun.tf b/terraform/litellm/gcp/cloudrun.tf index 07bf68b3a62..27877c7f928 100644 --- a/terraform/litellm/gcp/cloudrun.tf +++ b/terraform/litellm/gcp/cloudrun.tf @@ -131,16 +131,12 @@ locals { { name = "LITELLM_PGBOUNCER_MAX_CLIENT_CONN", value = tostring(var.gateway_pool_max_client_conn) }, ] : [] - gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}" - backend_uvicorn_args = "--host 0.0.0.0 --port 4001" - - gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run python -m gateway.launch ${local.gateway_uvicorn_args};; *) exec python -m gateway.launch ${local.gateway_uvicorn_args};; esac" - backend_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn backend.main:app ${local.backend_uvicorn_args};; *) exec uvicorn backend.main:app ${local.backend_uvicorn_args};; esac" + image_entrypoint = "/app/docker-entrypoint.sh" gateway_args = join(" && ", concat( local.redis_ca_fragment, local.database_url_fragment, - [local.gateway_launch_cmd], + ["exec ${local.image_entrypoint} gateway --workers ${var.gateway_num_workers}"], )) metrics_enabled = var.create_runtime && var.gateway_metrics_port != null @@ -161,7 +157,7 @@ locals { backend_args = join(" && ", concat( local.redis_ca_fragment, local.database_url_fragment, - [local.backend_launch_cmd], + ["exec ${local.image_entrypoint} backend"], )) collector_address = "tcp://127.0.0.1:${var.collector_port}" @@ -189,13 +185,12 @@ locals { collector_args = join(" && ", concat( local.redis_ca_fragment, local.database_url_fragment, - ["exec python -m litellm.proxy.collector"], + ["exec ${local.image_entrypoint} collector"], )) - # Env shipped to the migrations Job. The migrations image runs run.py - # which assembles DATABASE_URL from these discrete vars itself, so we - # only need writer-side DB env (no read replica, no proxy_config, no - # master key). + # run.py assembles DATABASE_URL from these discrete vars itself, so the + # migrations Job only needs writer-side DB env (no read replica, no + # proxy_config, no master key). migrations_env_kv = [ { name = "DATABASE_HOST", value = google_sql_database_instance.writer.private_ip_address }, { name = "DATABASE_PORT", value = "5432" }, @@ -241,7 +236,7 @@ resource "google_cloud_run_v2_service" "gateway" { containers { name = "gateway" - image = local.gateway_image + image = local.image command = ["sh", "-c"] args = [local.gateway_args] @@ -317,10 +312,9 @@ resource "google_cloud_run_v2_service" "gateway" { dynamic "containers" { for_each = local.metrics_enabled ? [1] : [] content { - name = "metrics" - image = local.gateway_image - command = ["python", "-m", "litellm.proxy.prometheus_metrics_server"] - args = ["--port", tostring(var.gateway_metrics_port)] + name = "metrics" + image = local.image + args = ["metrics", "--port", tostring(var.gateway_metrics_port)] dynamic "env" { for_each = local.metrics_env_kv @@ -383,7 +377,7 @@ resource "google_cloud_run_v2_service" "gateway" { for_each = var.collector_enabled ? [1] : [] content { name = "spend-collector" - image = local.gateway_image + image = local.image command = ["sh", "-c"] args = [local.collector_args] @@ -504,7 +498,7 @@ resource "google_cloud_run_v2_service" "backend" { } containers { - image = local.backend_image + image = local.image command = ["sh", "-c"] args = [local.backend_args] @@ -618,7 +612,8 @@ resource "google_cloud_run_v2_service" "ui" { } containers { - image = local.ui_image + image = local.image + args = ["ui"] ports { container_port = 3000 @@ -680,9 +675,6 @@ resource "google_cloud_run_v2_service_iam_member" "ui_allusers" { } # ---------- Migrations job ---------- -# Dedicated litellm-migrations image — slim, ENTRYPOINT runs run.py which -# assembles DATABASE_URL from the DATABASE_* env vars and runs `prisma -# migrate deploy`. No proxy_config, no master key, no shell wrapper. resource "google_cloud_run_v2_job" "migrations" { count = var.create_runtime ? 1 : 0 @@ -701,7 +693,8 @@ resource "google_cloud_run_v2_job" "migrations" { } containers { - image = local.migrations_image + image = local.image + args = ["migrations"] # Prisma's Node + Rust engine plus the v2 migration resolver # routinely peaks above 1 GiB while applying the schema, so 2 GiB diff --git a/terraform/litellm/gcp/examples/default/TUTORIAL.md b/terraform/litellm/gcp/examples/default/TUTORIAL.md index ec82feb739a..6e816167aeb 100644 --- a/terraform/litellm/gcp/examples/default/TUTORIAL.md +++ b/terraform/litellm/gcp/examples/default/TUTORIAL.md @@ -31,7 +31,7 @@ gcloud services enable \ ## Create the Artifact Registry passthrough to GHCR -Cloud Run only pulls from Artifact Registry, `gcr.io`, or `docker.io`; it rejects `ghcr.io` URIs at apply time. The four LiteLLM images live on GHCR, so the stack needs a remote Artifact Registry repo pointed at GHCR. This is a one-time setup per project. +Cloud Run only pulls from Artifact Registry, `gcr.io`, or `docker.io`; it rejects `ghcr.io` URIs at apply time. The LiteLLM image lives on GHCR, so the stack needs a remote Artifact Registry repo pointed at GHCR. This is a one-time setup per project. ```bash gcloud artifacts repositories create litellm \ diff --git a/terraform/litellm/gcp/examples/default/deploystack.json b/terraform/litellm/gcp/examples/default/deploystack.json index 732d4a187b1..2b80084ffa3 100644 --- a/terraform/litellm/gcp/examples/default/deploystack.json +++ b/terraform/litellm/gcp/examples/default/deploystack.json @@ -24,12 +24,12 @@ }, { "name": "image_tag", - "description": "Tag for the four litellm-* images (gateway, backend, ui, migrations). Bump together when bumping LiteLLM", + "description": "Tag of the litellm image every workload (gateway, backend, ui, migrations) runs", "default": "" }, { "name": "image_registry", - "description": "Artifact Registry path prefix for the four litellm-* images. Format: -docker.pkg.dev//litellm/berriai, pointing at the remote repo you created above. Substitute BOTH REGION and PROJECT_ID in the default to match the AR repo you just created (REGION must match the region you picked above). The ghcr.io/berriai default in the module does NOT work; Cloud Run rejects ghcr.io URIs at apply time", + "description": "Artifact Registry path prefix for the litellm image. Format: -docker.pkg.dev//litellm/berriai, pointing at the remote repo you created above. Substitute BOTH REGION and PROJECT_ID in the default to match the AR repo you just created (REGION must match the region you picked above). The ghcr.io/berriai default in the module does NOT work; Cloud Run rejects ghcr.io URIs at apply time", "default": "REGION-docker.pkg.dev/PROJECT_ID/litellm/berriai" }, { diff --git a/terraform/litellm/gcp/examples/default/main.tf b/terraform/litellm/gcp/examples/default/main.tf index 9b18a6df863..b1462fd2f95 100644 --- a/terraform/litellm/gcp/examples/default/main.tf +++ b/terraform/litellm/gcp/examples/default/main.tf @@ -21,7 +21,7 @@ # "Using as a module" section. # # Knobs not surfaced as variables here (per-component sizing/instances, -# Cloud SQL tier/edition, Memorystore tier, per-component image overrides) +# Cloud SQL tier/edition, Memorystore tier, a full `image` override) # can be set directly on this block — see ../../variables.tf. module "litellm" { source = "../../" diff --git a/terraform/litellm/gcp/examples/default/terraform.tfvars.example b/terraform/litellm/gcp/examples/default/terraform.tfvars.example index 6456b525d08..07901d10186 100644 --- a/terraform/litellm/gcp/examples/default/terraform.tfvars.example +++ b/terraform/litellm/gcp/examples/default/terraform.tfvars.example @@ -38,11 +38,12 @@ env = "stage" # Images. Cloud Run rejects ghcr.io, so a real deploy must point # image_registry at an Artifact Registry remote repo (see README "Image -# pulls"); image_tag is applied to all four litellm-* images. A published -# module version ships with it stamped to its own release tag, a checkout of -# the monorepo has no default and needs it set. Per-component *_image overrides are -# NOT exposed here. Set them directly on the `module "litellm"` block in -# main.tf (see ../../variables.tf) if you need to mix-and-match versions. +# pulls"); every workload runs /litellm:. A +# published module version ships with image_tag stamped to its own release +# tag, a checkout of the monorepo has no default and needs it set. The full +# `image` override is NOT exposed here: set it directly on the +# `module "litellm"` block in main.tf (see ../../variables.tf) if you need +# a custom build. # image_registry = "us-central1-docker.pkg.dev/my-gcp-project/litellm/berriai" # image_tag = "" diff --git a/terraform/litellm/gcp/examples/default/variables.tf b/terraform/litellm/gcp/examples/default/variables.tf index dcf84572fd7..d77a29b9899 100644 --- a/terraform/litellm/gcp/examples/default/variables.tf +++ b/terraform/litellm/gcp/examples/default/variables.tf @@ -1,6 +1,6 @@ # Curated surface for the one-command deploy path. The module (../../) # exposes far more knobs (per-component CPU/memory/instances, Cloud SQL -# tier/edition, Memorystore tier, per-component image overrides, …). To +# tier/edition, Memorystore tier, a full `image` override, …). To # tune those, set them directly on the `module "litellm"` block in # main.tf, or call the module from your own root config. Full per-variable # docs live in ../../variables.tf — the module is the source of truth. @@ -75,15 +75,15 @@ variable "ui_password" { # Image source. Cloud Run rejects ghcr.io, so a real deploy must point # image_registry at an Artifact Registry remote repo (see README "Image -# pulls"). Per-component overrides live in ../../variables.tf. +# pulls"). The full `image` override lives in ../../variables.tf. variable "image_registry" { - description = "Registry path prefix; images composed as /litellm-:." + description = "Registry path prefix; the image is composed as /litellm:." type = string default = "ghcr.io/berriai" } variable "image_tag" { - description = "Tag applied to all four litellm-* images. Bump in lockstep." + description = "Tag of the litellm image every workload runs." type = string default = "" } diff --git a/terraform/litellm/gcp/locals.tf b/terraform/litellm/gcp/locals.tf index e82892b27cb..270036590c5 100644 --- a/terraform/litellm/gcp/locals.tf +++ b/terraform/litellm/gcp/locals.tf @@ -92,11 +92,5 @@ locals { { name = "PROXY_CONFIG_HASH", value = md5(local.proxy_config_yaml) }, ] : [] - # Resolved image URIs: per-component override wins, otherwise compose - # from image_registry + image_tag. Cloud Run only accepts AR / gcr.io / - # docker.io paths — see variables.tf for the full constraint list. - gateway_image = var.gateway_image != "" ? var.gateway_image : "${var.image_registry}/litellm-gateway:${var.image_tag}" - backend_image = var.backend_image != "" ? var.backend_image : "${var.image_registry}/litellm-backend:${var.image_tag}" - ui_image = var.ui_image != "" ? var.ui_image : "${var.image_registry}/litellm-ui:${var.image_tag}" - migrations_image = var.migrations_image != "" ? var.migrations_image : "${var.image_registry}/litellm-migrations:${var.image_tag}" + image = var.image != "" ? var.image : "${var.image_registry}/litellm:${var.image_tag}" } diff --git a/terraform/litellm/gcp/tests/collector.tftest.hcl b/terraform/litellm/gcp/tests/collector.tftest.hcl index f16d02d9c7e..6c7810454df 100644 --- a/terraform/litellm/gcp/tests/collector.tftest.hcl +++ b/terraform/litellm/gcp/tests/collector.tftest.hcl @@ -65,14 +65,14 @@ run "enabled_adds_a_sidecar_that_shares_the_gateway_transport" { assert { condition = ( - google_cloud_run_v2_service.gateway[0].template[0].containers[1].image == local.gateway_image && + google_cloud_run_v2_service.gateway[0].template[0].containers[1].image == google_cloud_run_v2_service.gateway[0].template[0].containers[0].image && google_cloud_run_v2_service.gateway[0].template[0].containers[1].command == tolist(["sh", "-c"]) && - endswith(google_cloud_run_v2_service.gateway[0].template[0].containers[1].args[0], " && exec python -m litellm.proxy.collector") && + endswith(google_cloud_run_v2_service.gateway[0].template[0].containers[1].args[0], " && exec /app/docker-entrypoint.sh collector") && strcontains(google_cloud_run_v2_service.gateway[0].template[0].containers[1].args[0], "export DATABASE_URL=") && strcontains(google_cloud_run_v2_service.gateway[0].template[0].containers[1].args[0], "REDIS_SSL_CA_CERTS") && { for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e.name => e.value }["LITELLM_JOB_ROLE"] == "collector" ) - error_message = "The sidecar must run litellm.proxy.collector from the gateway image with the same Redis CA + DATABASE_URL bootstrap as the gateway." + error_message = "The sidecar must run the collector component of the gateway's image with the same Redis CA + DATABASE_URL bootstrap as the gateway." } assert { diff --git a/terraform/litellm/gcp/tests/connection_pool.tftest.hcl b/terraform/litellm/gcp/tests/connection_pool.tftest.hcl index c0ad842ff95..98ef6f7fe5d 100644 --- a/terraform/litellm/gcp/tests/connection_pool.tftest.hcl +++ b/terraform/litellm/gcp/tests/connection_pool.tftest.hcl @@ -145,17 +145,15 @@ run "gateway_starts_through_the_pool_aware_launcher" { assert { condition = alltrue([ - strcontains(local.gateway_launch_cmd, "exec python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), - strcontains(local.gateway_launch_cmd, "exec ddtrace-run python -m gateway.launch --host 0.0.0.0 --port 4000 --workers 4"), - !strcontains(local.gateway_launch_cmd, "uvicorn gateway.main:app"), - endswith(google_cloud_run_v2_service.gateway[0].template[0].containers[0].args[0], local.gateway_launch_cmd), + google_cloud_run_v2_service.gateway[0].template[0].containers[0].command == tolist(["sh", "-c"]), + endswith(google_cloud_run_v2_service.gateway[0].template[0].containers[0].args[0], " && exec /app/docker-entrypoint.sh gateway --workers 4"), ]) - error_message = "The gateway must start through gateway.launch (with and without ddtrace) so the pooler starts once before uvicorn forks the workers." + error_message = "The gateway must start through the image's gateway component with the configured worker count so the pooler starts once before uvicorn forks the workers." } assert { - condition = strcontains(local.backend_launch_cmd, "uvicorn backend.main:app") - error_message = "The backend has no workers to share a pooler and keeps starting uvicorn directly." + condition = endswith(google_cloud_run_v2_service.backend[0].template[0].containers[0].args[0], " && exec /app/docker-entrypoint.sh backend") + error_message = "The backend has no workers to share a pooler and starts the plain backend component." } } diff --git a/terraform/litellm/gcp/tests/metrics_sidecar.tftest.hcl b/terraform/litellm/gcp/tests/metrics_sidecar.tftest.hcl index fb5446b4c56..449a048428a 100644 --- a/terraform/litellm/gcp/tests/metrics_sidecar.tftest.hcl +++ b/terraform/litellm/gcp/tests/metrics_sidecar.tftest.hcl @@ -66,11 +66,11 @@ run "metrics_sidecar_enabled" { assert { condition = alltrue([ - google_cloud_run_v2_service.gateway[0].template[0].containers[1].image == local.gateway_image, - join(" ", google_cloud_run_v2_service.gateway[0].template[0].containers[1].command) == "python -m litellm.proxy.prometheus_metrics_server", - join(" ", google_cloud_run_v2_service.gateway[0].template[0].containers[1].args) == "--port 4001", + google_cloud_run_v2_service.gateway[0].template[0].containers[1].image == google_cloud_run_v2_service.gateway[0].template[0].containers[0].image, + google_cloud_run_v2_service.gateway[0].template[0].containers[1].command == null, + join(" ", google_cloud_run_v2_service.gateway[0].template[0].containers[1].args) == "metrics --port 4001", ]) - error_message = "The metrics sidecar must run the gateway image's prometheus_metrics_server on the configured port." + error_message = "The metrics sidecar must run the gateway image's metrics component on the configured port." } assert { diff --git a/terraform/litellm/gcp/tests/unified_image.tftest.hcl b/terraform/litellm/gcp/tests/unified_image.tftest.hcl new file mode 100644 index 00000000000..1c7652a7110 --- /dev/null +++ b/terraform/litellm/gcp/tests/unified_image.tftest.hcl @@ -0,0 +1,72 @@ +mock_provider "google" { + mock_resource "google_redis_instance" { + defaults = { + host = "10.0.0.4" + port = 6379 + server_ca_certs = [{ + cert = "-----BEGIN CERTIFICATE-----\nmock\n-----END CERTIFICATE-----" + }] + } + } +} + +mock_provider "google-beta" {} +mock_provider "random" {} + +variables { + project_id = "test-project" + tenant = "tenant" + env = "test" + allow_plaintext_lb = true + image_registry = "us-central1-docker.pkg.dev/test-project/litellm/berriai" + image_tag = "v9.9.9-stable" + gateway_num_workers = 3 +} + +run "registry_and_tag_compose_one_image_for_every_workload" { + command = plan + + assert { + condition = alltrue([ + google_cloud_run_v2_service.gateway[0].template[0].containers[0].image == "us-central1-docker.pkg.dev/test-project/litellm/berriai/litellm:v9.9.9-stable", + google_cloud_run_v2_service.backend[0].template[0].containers[0].image == "us-central1-docker.pkg.dev/test-project/litellm/berriai/litellm:v9.9.9-stable", + google_cloud_run_v2_service.ui[0].template[0].containers[0].image == "us-central1-docker.pkg.dev/test-project/litellm/berriai/litellm:v9.9.9-stable", + google_cloud_run_v2_job.migrations[0].template[0].template[0].containers[0].image == "us-central1-docker.pkg.dev/test-project/litellm/berriai/litellm:v9.9.9-stable", + terraform_data.migration[0].triggers_replace.job_image == "us-central1-docker.pkg.dev/test-project/litellm/berriai/litellm:v9.9.9-stable", + ]) + error_message = "All workloads must pull /litellm: and the migration re-run trigger must follow it." + } + + assert { + condition = alltrue([ + google_cloud_run_v2_service.ui[0].template[0].containers[0].command == null, + google_cloud_run_v2_service.ui[0].template[0].containers[0].args == tolist(["ui"]), + google_cloud_run_v2_job.migrations[0].template[0].template[0].containers[0].command == null, + google_cloud_run_v2_job.migrations[0].template[0].template[0].containers[0].args == tolist(["migrations"]), + google_cloud_run_v2_service.gateway[0].template[0].containers[0].command == tolist(["sh", "-c"]), + endswith(google_cloud_run_v2_service.gateway[0].template[0].containers[0].args[0], " && exec /app/docker-entrypoint.sh gateway --workers 3"), + google_cloud_run_v2_service.backend[0].template[0].containers[0].command == tolist(["sh", "-c"]), + endswith(google_cloud_run_v2_service.backend[0].template[0].containers[0].args[0], " && exec /app/docker-entrypoint.sh backend"), + ]) + error_message = "UI and migrations must keep the image entrypoint with their component as args; gateway and backend must exec it with their component after the DATABASE_URL bootstrap." + } +} + +run "image_overrides_the_composed_uri_everywhere" { + command = plan + + variables { + image = "us-central1-docker.pkg.dev/test-project/mirror/litellm:custom" + } + + assert { + condition = alltrue([ + google_cloud_run_v2_service.gateway[0].template[0].containers[0].image == var.image, + google_cloud_run_v2_service.backend[0].template[0].containers[0].image == var.image, + google_cloud_run_v2_service.ui[0].template[0].containers[0].image == var.image, + google_cloud_run_v2_job.migrations[0].template[0].template[0].containers[0].image == var.image, + terraform_data.migration[0].triggers_replace.job_image == var.image, + ]) + error_message = "A full image URI must win over image_registry + image_tag for every workload." + } +} diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index e40ee36fa18..21d80eb2083 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -119,69 +119,49 @@ variable "vpc_connector_cidr" { default = "10.41.0.0/28" } -# ---------- Component images ---------- +# ---------- Image ---------- # # Cloud Run only pulls from Artifact Registry, [region.]gcr.io, or -# docker.io — it rejects arbitrary registries (notably ghcr.io) at apply -# time. The four images live on GHCR upstream, so any real deploy must -# either set `image_registry` to an Artifact Registry remote repository -# pointed at ghcr.io (e.g. `us-central1-docker.pkg.dev/my-proj/litellm/berriai`) -# or override the per-component `*_image` vars individually with full URIs. +# docker.io and rejects arbitrary registries (notably ghcr.io) at apply +# time. The image lives on GHCR upstream, so any real deploy must either +# set `image_registry` to an Artifact Registry remote repository pointed at +# ghcr.io (e.g. `us-central1-docker.pkg.dev/my-proj/litellm/berriai`) or +# set `image` to the full URI of a copy you mirrored yourself. variable "image_registry" { description = <<-EOT - Registry path prefix used to compose the four LiteLLM image URIs as - `/litellm-:`. The default - (`ghcr.io/berriai`) only works on registries Cloud Run accepts — for - GHCR-backed deploys, create an Artifact Registry remote repository - pointed at `https://ghcr.io` and set this to that repo's path + Registry path prefix used to compose the LiteLLM image URI as + `/litellm:`. The default (`ghcr.io/berriai`) + only works on registries Cloud Run accepts: for GHCR-backed deploys, + create an Artifact Registry remote repository pointed at + `https://ghcr.io` and set this to that repo's path (e.g. `us-central1-docker.pkg.dev///berriai`). - Per-component overrides (`gateway_image`, `backend_image`, `ui_image`, - `migrations_image`) bypass this entirely when set. + `image` bypasses this entirely when set. EOT type = string default = "ghcr.io/berriai" } variable "image_tag" { - description = "Tag applied to all four litellm-* images when composed from `image_registry`. A published module version ships with this stamped to its own release tag, which matches the `v`-prefixed tag published to GHCR for the same LiteLLM release. Set it to pin a different LiteLLM release. When sourcing the module from a git ref of the monorepo it has no default and must be set." + description = "Tag of the LiteLLM image when composed from `image_registry`. A published module version ships with this stamped to its own release tag, which matches the `v`-prefixed tag published to GHCR for the same LiteLLM release. Set it to pin a different LiteLLM release. When sourcing the module from a git ref of the monorepo it has no default and must be set. Tags older than the one a module version ships with have no component entrypoint and only run the monolithic proxy." type = string default = "" nullable = false validation { - condition = !var.create_runtime || var.image_tag != "" || (var.gateway_image != "" && var.backend_image != "" && var.ui_image != "" && var.migrations_image != "") - error_message = "image_tag is empty. Published module versions (registry.terraform.io/modules/BerriAI/litellm) ship with it stamped to their own release. A git ref of the monorepo has no published images of its own, so set image_tag to a published LiteLLM release tag or set all four *_image URIs." + condition = !var.create_runtime || var.image_tag != "" || var.image != "" + error_message = "image_tag is empty. Published module versions (registry.terraform.io/modules/BerriAI/litellm) ship with it stamped to their own release. A git ref of the monorepo has no published images of its own, so set image_tag to a published LiteLLM release tag or set image to a full URI." } } -variable "gateway_image" { - description = "Full image URI for the gateway. Empty (default) composes from `image_registry` + `image_tag`. Public images or Artifact Registry only — Cloud Run won't authenticate against arbitrary private registries." - type = string - default = "" -} - -variable "backend_image" { - description = "Full image URI for the backend. Empty (default) composes from `image_registry` + `image_tag`." - type = string - default = "" -} - -variable "ui_image" { - description = "Full image URI for the UI. Empty (default) composes from `image_registry` + `image_tag`." - type = string - default = "" -} - -variable "migrations_image" { +variable "image" { description = <<-EOT - Full image URI for the one-off prisma migration Cloud Run Job. Empty - (default) composes from `image_registry` + `image_tag` as - `litellm-migrations`. Built from `migrations/Dockerfile` — slim image - whose ENTRYPOINT runs `python3 /app/run.py` (assembles DATABASE_URL - from DATABASE_* env vars via DatabaseURLSettings, then runs - `prisma migrate deploy`). Should track the same release tag as - gateway/backend/ui. + Full URI of the one LiteLLM image every Cloud Run service and the + migrations Job run. Its entrypoint picks the process from the first + argument (`gateway`, `backend`, `ui`, `migrations`, `metrics`, + `collector`). Empty (default) composes from `image_registry` + + `image_tag`. Public images or Artifact Registry only: Cloud Run won't + authenticate against arbitrary private registries. EOT type = string default = "" @@ -569,8 +549,7 @@ variable "gateway_metrics_port" { Serve Prometheus /metrics from a `metrics` sidecar container in the gateway Cloud Run service on this port (a whole number 1-65535, not 4000 or 13133), so the collector's scrape never runs on an inference worker. - The sidecar runs the gateway image with - `python -m litellm.proxy.prometheus_metrics_server` and aggregates the + The sidecar runs the `metrics` component of the image and aggregates the workers' PROMETHEUS_MULTIPROC_DIR samples over an in-memory volume shared with the gateway container. Cloud Run only routes ingress to the gateway container, so the sidecar port is reachable on localhost inside the @@ -578,7 +557,7 @@ variable "gateway_metrics_port" { (gateway_metrics_collector_image) scrapes it and writes the series to Cloud Monitoring. The load balancer keeps serving the authenticated /metrics on the gateway port as before. Null (the default) leaves /metrics - on the gateway port only. Needs gateway_image v1.101.0 or newer. + on the gateway port only. EOT type = number default = null @@ -649,7 +628,7 @@ variable "billing_metrics_token_url" { # ---------- Collector sidecar ---------- # # Opt-in offload of spend tracking from the gateway's uvicorn workers to a -# `python -m litellm.proxy.collector` sidecar container in the same Cloud Run +# `collector` component sidecar container in the same Cloud Run # instance (helm's `gateway.collector`, mirrors the AWS stack). Containers # in one instance share localhost, so the sidecar listens on loopback TCP. # Disabled (the default) adds nothing to the service. diff --git a/tests/unit/test_docker_entrypoint.py b/tests/unit/test_docker_entrypoint.py index bd0b9b0fd4b..4364fcc6203 100644 --- a/tests/unit/test_docker_entrypoint.py +++ b/tests/unit/test_docker_entrypoint.py @@ -16,8 +16,10 @@ from litellm.constants import PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX REPO_ROOT: Final = Path(__file__).resolve().parents[2] DOCKER_ENTRYPOINT: Final = REPO_ROOT / "docker-entrypoint.sh" DOCKERFILE: Final = REPO_ROOT / "Dockerfile" -TERRAFORM_ECS: Final = REPO_ROOT / "terraform" / "litellm" / "aws" / "ecs.tf" -TERRAFORM_CLOUDRUN: Final = REPO_ROOT / "terraform" / "litellm" / "gcp" / "cloudrun.tf" +TERRAFORM_MODULES: Final = { + "aws": REPO_ROOT / "terraform" / "litellm" / "aws", + "gcp": REPO_ROOT / "terraform" / "litellm" / "gcp", +} IMAGE_ENTRYPOINT_PATH: Final = "/app/docker-entrypoint.sh" STUBBED_EXECUTABLES: Final = ("ddtrace-run", "uvicorn", "python", "litellm", "nginx") @@ -38,17 +40,25 @@ _STUB_TEMPLATE: Final = """#!/bin/sh _ENTRYPOINT_RE: Final = re.compile(r"^ENTRYPOINT\s+(\[.*\])\s*$", re.MULTILINE) _CMD_RE: Final = re.compile(r"^CMD\s+(\[.*\])\s*$", re.MULTILINE) -_APP_TARGET_RE: Final = re.compile(r"(?:gateway|backend)\.main:app|gateway\.launch") -_TF_STRING_LOCAL_RE: Final = re.compile(r'^\s*(\w+)\s*=\s*"((?:[^"\\]|\\.)*)"\s*$', re.MULTILINE) -_TF_INTERPOLATION_RE: Final = re.compile(r"\$\{(local|var)\.(\w+)\}") +_TF_ENTRYPOINT_LOCAL_RE: Final = re.compile(r'^\s*image_entrypoint\s*=\s*"([^"]*)"\s*$', re.MULTILINE) +_TF_WORD: Final = r'(?:"[^"\s]*"|tostring\(var\.\w+\)|\$\{var\.\w+\}|[\w.-]+)' +_TF_ENTRYPOINT_CALL_RE: Final = re.compile(rf"exec \$\{{local\.image_entrypoint\}}((?: {_TF_WORD})+)") +_TF_COMPONENT_LIST_RE: Final = re.compile( + rf"(?:command|args|gateway|backend|ui|migrations|metrics|collector)\s*=\s*" + rf'\[\s*("(?:proxy|gateway|backend|ui|migrations|metrics|collector)"(?:\s*,\s*{_TF_WORD})*)\s*\]' +) +_TF_VAR_RE: Final = re.compile(r"tostring\(var\.(\w+)\)|\$\{var\.(\w+)\}") -TERRAFORM_LAUNCH_SITES: Final = {TERRAFORM_ECS: 2, TERRAFORM_CLOUDRUN: 2} -TERRAFORM_VAR_STUBS: Final = {"gateway_num_workers": "2"} -COMPONENT_LAUNCHERS: Final = { - "gateway": ("python", "-m", "gateway.launch"), - "backend": ("uvicorn", "backend.main:app"), +TERRAFORM_VAR_STUBS: Final = {"gateway_num_workers": "2", "gateway_metrics_port": "9464"} +TERRAFORM_WORKLOADS: Final = frozenset({"gateway", "backend", "ui", "migrations", "metrics", "collector"}) +COMPONENT_BINARIES: Final = { + "gateway": "python", + "backend": "uvicorn", + "ui": "nginx", + "migrations": "python", + "metrics": "python", + "collector": "python", } -_MAX_INTERPOLATION_PASSES: Final = 5 def _write_stubs(bin_dir: Path, names: tuple[str, ...]) -> None: @@ -124,28 +134,22 @@ def test_backend_leaves_ui_runtime_config_unchanged(tmp_path: Path) -> None: assert json.loads(runtime_file.read_text()) == {"backend_url": "https://stale.example.com"} -def _run_shell_command(command: str, bin_dir: Path, record: Path, use_ddtrace: str | None) -> tuple[str, ...]: - env = _entrypoint_env(bin_dir, record, {"USE_DDTRACE": use_ddtrace}) - result = subprocess.run(["sh", "-c", command], env=env, capture_output=True, text=True, check=False) - assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}" - return tuple(record.read_text().splitlines()) if record.exists() else () +def _tf_word(word: str) -> str: + var: Final = _TF_VAR_RE.fullmatch(word) + if var is not None: + return TERRAFORM_VAR_STUBS[var.group(1) or var.group(2)] + return word.strip('"') -def _resolve_tf_local(terraform_file: Path, name: str) -> str: - values = { - m.group(1): m.group(2).replace('\\"', '"') for m in _TF_STRING_LOCAL_RE.finditer(terraform_file.read_text()) - } - - assert name in values, f"{terraform_file} defines no {name} local" - resolved = values[name] - for _ in range(_MAX_INTERPOLATION_PASSES): - if "${" not in resolved: - return resolved - resolved = _TF_INTERPOLATION_RE.sub( - lambda m: values[m.group(2)] if m.group(1) == "local" else TERRAFORM_VAR_STUBS[m.group(2)], - resolved, +def _terraform_component_argvs(module_dir: Path) -> tuple[tuple[str, tuple[str, ...]], ...]: + return tuple( + (tf.name, tuple(_tf_word(word) for word in words)) + for tf in sorted(module_dir.glob("*.tf")) + for words in ( + *(m.group(1).split() for m in _TF_ENTRYPOINT_CALL_RE.finditer(tf.read_text())), + *(re.split(r"\s*,\s*", m.group(1)) for m in _TF_COMPONENT_LIST_RE.finditer(tf.read_text())), ) - raise AssertionError(f"{terraform_file}:{name} still has unresolved interpolations: {resolved}") + ) # comprehension-ok: the two regex scans over one file read are clearer inline than as a generator helper def _entrypoint_argv(dockerfile: Path) -> tuple[str, ...]: @@ -474,62 +478,30 @@ def test_creates_a_missing_prometheus_multiproc_dir(tmp_path: Path) -> None: assert missing.is_dir() -@pytest.mark.parametrize("terraform_file", TERRAFORM_LAUNCH_SITES, ids=lambda p: p.parent.name) -@pytest.mark.parametrize("component", ["gateway", "backend"]) -@pytest.mark.parametrize("use_ddtrace", [*TRUTHY_USE_DDTRACE, *FALSY_USE_DDTRACE]) -def test_terraform_launch_command_matches_the_script_contract( - terraform_file: Path, component: str, use_ddtrace: str | None, tmp_path: Path -) -> None: - launcher = COMPONENT_LAUNCHERS[component] - app_target = " ".join(launcher[1:]) - command = _resolve_tf_local(terraform_file, f"{component}_launch_cmd") +@pytest.mark.parametrize("module", TERRAFORM_MODULES, ids=str) +def test_terraform_hands_every_workload_to_the_image_entrypoint(module: str) -> None: + argvs: Final = _terraform_component_argvs(TERRAFORM_MODULES[module]) - bin_dir = tmp_path / "bin" - bin_dir.mkdir(parents=True) - _write_stubs(bin_dir, ("ddtrace-run", "uvicorn", "python")) - from_terraform = _run_shell_command(command, bin_dir, tmp_path / "terraform.txt", use_ddtrace) - - from_script = _run_entrypoint((component,), tmp_path / "script", use_ddtrace=use_ddtrace) - - assert from_terraform[0] == from_script[0], ( - f"{terraform_file} disagrees with the script on USE_DDTRACE={use_ddtrace}" - ) - assert from_terraform[2] == from_script[2], f"{terraform_file} disagrees with the script on the openai integration" - assert app_target in from_terraform[1] - assert app_target in from_script[1] - assert "gateway.main:app" not in from_terraform[1], f"{terraform_file} bypasses the gateway.launch supervisor" - - if use_ddtrace in TRUTHY_USE_DDTRACE: - assert from_terraform[0] == "exec=ddtrace-run" - assert from_terraform[1].startswith(f"args={launcher[0]} ") - assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=False" - else: - assert from_terraform[0] == f"exec={launcher[0]}" - assert from_terraform[2] == "DD_TRACE_OPENAI_ENABLED=" + assert {argv[0] for _, argv in argvs} == TERRAFORM_WORKLOADS, f"{module} workloads: {argvs}" + for tf in TERRAFORM_MODULES[module].glob("*.tf"): + for entrypoint_path in _TF_ENTRYPOINT_LOCAL_RE.findall(tf.read_text()): + assert entrypoint_path == _entrypoint_argv(DOCKERFILE)[0], f"{tf} execs a path the image does not ship" -@pytest.mark.parametrize("terraform_file", TERRAFORM_LAUNCH_SITES, ids=lambda p: p.parent.name) -def test_terraform_routes_every_launch_site_through_a_traced_command(terraform_file: Path) -> None: - launch_sites = tuple(line for line in terraform_file.read_text().splitlines() if _APP_TARGET_RE.search(line)) +@pytest.mark.parametrize("module", TERRAFORM_MODULES, ids=str) +@pytest.mark.parametrize("use_ddtrace", ["true", None]) +def test_the_entrypoint_accepts_every_terraform_argv(module: str, use_ddtrace: str | None, tmp_path: Path) -> None: + for index, (tf, argv) in enumerate(_terraform_component_argvs(TERRAFORM_MODULES[module])): + component, *extra = argv + recorded: Final = _run_entrypoint(argv, tmp_path / str(index), use_ddtrace=use_ddtrace) - assert len(launch_sites) == TERRAFORM_LAUNCH_SITES[terraform_file], ( - f"{terraform_file} launch-site count changed; re-check each one honors USE_DDTRACE" - ) - for line in launch_sites: - assert "ddtrace-run" in line, f"{terraform_file} launches uvicorn without honoring USE_DDTRACE: {line.strip()}" - - body = terraform_file.read_text() - for component in ("gateway", "backend"): - assert f"local.{component}_launch_cmd" in body, ( - f"{terraform_file} defines a {component} launch command but never uses it" - ) - - -@pytest.mark.parametrize("terraform_file", TERRAFORM_LAUNCH_SITES, ids=lambda p: p.parent.name) -def test_terraform_does_not_depend_on_the_entrypoint_script(terraform_file: Path) -> None: - assert IMAGE_ENTRYPOINT_PATH not in terraform_file.read_text(), ( - f"{terraform_file} depends on a script that images predating it do not ship" - ) + expected_exec: Final = "ddtrace-run" if use_ddtrace and component != "ui" else COMPONENT_BINARIES[component] + assert recorded[0] == f"exec={expected_exec}", f"{module}/{tf} {argv}: {recorded}" + assert recorded[1].endswith(" ".join(extra)), f"{module}/{tf} {argv} drops its flags: {recorded[1]}" + if component == "gateway": + assert extra == ["--workers", TERRAFORM_VAR_STUBS["gateway_num_workers"]], f"{module}/{tf}: {argv}" + if component == "metrics": + assert extra == ["--port", TERRAFORM_VAR_STUBS["gateway_metrics_port"]], f"{module}/{tf}: {argv}" def test_help_prints_the_header_comment_block_without_shell_source(tmp_path: Path) -> None: From 3622831fbab8fb7f14af7aff673305e1b52ab25f Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:06:50 -0800 Subject: [PATCH 083/189] fix(spend): keep empty-key rows in owner-scoped spend log reads (#51127) * fix(spend): preserve empty-key rows in owner-scoped log reads Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): isolate passthrough tests without reloading shared modules Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../spend_tracking/spend_log_api_key_scope.py | 4 +- .../test_team_endpoints.py | 6 +-- .../test_llm_passthrough_endpoints.py | 27 ++---------- .../test_spend_management_endpoints.py | 43 ++++++++++++++++++- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_log_api_key_scope.py b/litellm/proxy/spend_tracking/spend_log_api_key_scope.py index 6507011a977..8b5c899553a 100644 --- a/litellm/proxy/spend_tracking/spend_log_api_key_scope.py +++ b/litellm/proxy/spend_tracking/spend_log_api_key_scope.py @@ -47,7 +47,9 @@ _API_KEY_ROWS: Final = TypeAdapter(tuple[_ApiKeyRow, ...]) def _narrowing_keys(rows: Sequence[Mapping[str, object]]) -> tuple[str, ...] | None: """None when the list cannot narrow the read: nothing known about the owner, or too many keys to help.""" keys: Final = tuple(row.api_key for row in _API_KEY_ROWS.validate_python(rows)) - return None if not keys or len(keys) > SPEND_LOG_API_KEY_SCOPE_MAX_KEYS else keys + if not keys or len(keys) > SPEND_LOG_API_KEY_SCOPE_MAX_KEYS: + return None + return keys if "" in keys else (*keys, "") def _daily_dates_around(start: date, end: date) -> tuple[str, str]: diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index ac46ccc1cfc..29e9be69776 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -17400,7 +17400,7 @@ async def test_get_team_spend_by_user_admin_groups_spend_logs_by_team_and_user(m assert any("statement_timeout = 30000" in statement for statement in transaction.statements) mock_db_client.db.tx.assert_not_called() sql, *params = transaction.query_raw.call_args.args - assert params == ["2026-09-01", "2026-09-04", "team-alpha", "team-beta", ["sk-alpha", "sk-beta"]] + assert params == ["2026-09-01", "2026-09-04", "team-alpha", "team-beta", ["sk-alpha", "sk-beta", ""]] assert 'FROM "LiteLLM_SpendLogs" sl' in sql assert 'sl."startTime" >= $1::timestamp' in sql assert "sl.\"startTime\" < $2::timestamp + INTERVAL '1 day'" in sql @@ -17440,7 +17440,7 @@ async def test_get_team_spend_by_user_team_admin_sees_every_member(mock_db_clien ) sql, *params = transaction.query_raw.call_args.args - assert params == ["2026-09-01", "2026-09-04", "team-alpha", ["sk-alpha"]] + assert params == ["2026-09-01", "2026-09-04", "team-alpha", ["sk-alpha", ""]] assert 'sl."user" = $' not in sql assert "sl.api_key = ANY($4::text[])" in sql @@ -17465,7 +17465,7 @@ async def test_get_team_spend_by_user_plain_member_only_sees_own_row(mock_db_cli ) sql, *params = transaction.query_raw.call_args.args - assert params == ["2026-09-01", "2026-09-04", "team-alpha", "bob", ["sk-bob"]] + assert params == ["2026-09-01", "2026-09-04", "team-alpha", "bob", ["sk-bob", ""]] assert "sl.team_id IN ($3)" in sql assert 'AND sl."user" = $4' in sql assert "sl.api_key = ANY($5::text[])" in sql diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py index 60b263948b6..2e196d6a8c2 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py @@ -1,5 +1,3 @@ -import asyncio -import importlib import json from collections.abc import Iterator from typing import Final @@ -18,6 +16,9 @@ from litellm.router import Deployment, LiteLLM_Params from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome +pytestmark: Final = pytest.mark.usefixtures("drained_logging_worker") + + @pytest.fixture def isolate_passthrough_endpoint_router_state(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( @@ -177,28 +178,6 @@ def _vcr_outcome_gate(request, vcr): record_vcr_outcome(request, vcr) -@pytest.fixture(scope="function", autouse=True) -def setup_and_teardown(): - """ - This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. - """ - from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER - - asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) - importlib.reload(litellm) - try: - if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - importlib.reload(litellm.proxy.proxy_server) - except Exception as e: - print(f"Error reloading litellm.proxy.proxy_server: {e}") - loop = asyncio.get_event_loop_policy().new_event_loop() - asyncio.set_event_loop(loop) - print(litellm) - yield - loop.close() - asyncio.set_event_loop(None) - - @pytest.mark.asyncio async def test_gemini_pass_through_endpoint_forwards_key_and_body( respx_mock: "respx.MockRouter", httpx_transport: None, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 29e2f97d11c..337aa2939dd 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -944,10 +944,49 @@ async def test_ui_view_spend_logs_user_filter_reads_spend_logs_through_the_users assert '"LiteLLM_VerificationToken"' in lookup_sql assert lookup_params == ("alice", "2026-08-31", "2026-09-04", SPEND_LOG_API_KEY_SCOPE_MAX_KEYS + 1) page_sql, _, scoped_api_keys = _spend_logs_page_query(queries) - assert scoped_api_keys == ["sk-deleted", "sk-live"] + assert scoped_api_keys == ["sk-deleted", "sk-live", ""] assert '"user" = $' in page_sql +@pytest.mark.asyncio +@pytest.mark.parametrize("owner_filter", ["user_id", "team_id"]) +async def test_ui_view_spend_logs_owner_filter_preserves_empty_key_rows( + client: TestClient, monkeypatch: pytest.MonkeyPatch, owner_filter: str +) -> None: + logs: Final = [ + _owned_log("known", "sk-live", "alice", "team-alpha"), + _owned_log("unknown-key", "", "alice", "team-alpha"), + _owned_log("foreign", "", "bob", "team-beta"), + ] + monkeypatch.setattr( + ps, + "prisma_client", + make_ui_spend_logs_mock_prisma( + logs, + _owner_filtered_logs(logs), + owner_api_keys_fn=_owner_api_keys( + daily_rows={}, live_keys={"alice": {"sk-live"}, "team-alpha": {"sk-live"}} + ), + ), + ) + monkeypatch.setitem( + app.dependency_overrides, + ps.user_api_key_auth, + lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user"), + ) + response: Final = client.get( + "/spend/logs/ui", + params={ + owner_filter: "alice" if owner_filter == "user_id" else "team-alpha", + "start_date": "2026-09-01 00:00:00", + "end_date": "2026-09-03 23:59:59", + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200, response.text + assert {row["request_id"] for row in response.json()["data"]} == {"known", "unknown-key"} + + @pytest.mark.asyncio async def test_ui_view_spend_logs_team_filter_reads_spend_logs_through_the_teams_api_keys(client, monkeypatch): logs = [ @@ -989,7 +1028,7 @@ async def test_ui_view_spend_logs_team_filter_reads_spend_logs_through_the_teams _, lookup_params = next((sql, params) for sql, params in queries if '"LiteLLM_DailyTeamSpend"' in sql) assert lookup_params[:3] == (["team-alpha"], "2026-08-31", "2026-09-04") page_sql, _, scoped_api_keys = _spend_logs_page_query(queries) - assert scoped_api_keys == ["sk-alpha-1", "sk-alpha-2"] + assert scoped_api_keys == ["sk-alpha-1", "sk-alpha-2", ""] assert "team_id = $" in page_sql From ac6e78cb1f1202b45c51854315b88c111f63e7ad Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:20:06 -0800 Subject: [PATCH 084/189] fix(proxy): list callable ids for partial aliases over a catch-all deployment (#50963) * fix(proxy): list ids that route back for a prefixed partial alias over a catch-all deployment * test(proxy): call the router registry in the catch-all alias listing test * fix(proxy): keep main's listing for whole-name aliases over a catch-all Only partial aliases like team1/gpt-* over a bare * run the per-id route check. A whole-name alias like team1/* keeps the string-only listing it had, so it lists the same ids at the same cost. Adds a test that a more specific sibling alias's ids are left out of the partial alias's list. * fix(proxy): list catch-all aliases fast and skip ids a sibling alias takes * fix(proxy): check team wildcard aliases without a router call per id and skip suffix ids that name another catalog model Team keys now read the team's own wildcard patterns the way the router does (any team pattern first, then the provider form when its prefix can be a provider), so ids no rival can take skip the router call. A suffix alias such as team1/*-mini no longer lists team1/gpt-5.4-mini for gpt-5.4. * fix(proxy): keep team key listings fast when aliases overlap and honor a pinned provider on partial aliases A team key whose team defines the same alias as the global config, or a team pattern that can take a global alias's ids, now checks only the team's own patterns before calling the router, so the listing no longer pays a router call per catalog id. The router also tries the provider-prefixed form of an id only when the bare form matches nothing. A partial alias over `*` with `custom_llm_provider` set now lists only that provider's catalog, so `team1/gpt-*` pinned to OpenAI no longer lists `team1/gpt-claude-*` ids that answer 404. * fix(proxy): ask a pinned catch-all's provider for its models once per listing A partial alias over a `*` deployment pinned with custom_llm_provider fetched the whole `*` catalog before taking the pinned branch, so with check_provider_endpoint on each listing called the provider's /v1/models one extra time. The pinned branch now runs before that fetch. Tests also cover a provider-less team alias that takes ids under their provider and a wildcard model_group_alias --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/auth/model_checks.py | 161 ++++++++- litellm/router_rust/surface/indexes.py | 5 +- tests/unit/proxy/auth/test_model_checks.py | 370 ++++++++++++++++++++- 3 files changed, 520 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 7f2aa33246d..07a0b3e9377 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -6,18 +6,19 @@ import copy import functools import itertools import re -from collections.abc import Callable, Collection, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from collections.abc import Set as AbstractSet from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, replace from types import MappingProxyType -from typing import Final, TypeVar +from typing import Final, TypeAlias, TypeVar import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model -from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter +from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.proxy.common_utils.model_listing_utils import CallerAliases, alias_map, dispatched_model @@ -429,6 +430,71 @@ def _ids_routing_to_catalog_models( ) +def _names_whole_models(name_pattern: str) -> bool: + return name_pattern.count("*") == 1 and name_pattern.endswith("/*") + + +@dataclass(frozen=True, slots=True) +class _Rivals: + direct: tuple[re.Pattern[str], ...] + under_a_provider: tuple[re.Pattern[str], ...] + + +RivalPatterns: TypeAlias = Callable[[str], _Rivals | None] + + +def _rival_may_take(listed_id: str, rivals: _Rivals) -> bool: + if any(rival.match(listed_id) for rival in rivals.direct): + return True + if not rivals.under_a_provider: + return False + provider_request: Final = f"{inferred_provider(listed_id)}/{listed_id}" + return any(rival.match(provider_request) for rival in rivals.under_a_provider) + + +def _routes_through_alias( + name_pattern: str, + rivals: _Rivals | None, + routes_to_model: Callable[[str, str], bool], +) -> Callable[[str], bool]: + alias_regex: Final = _wildcard_name_regex(name_pattern) + + def routes_through(listed_id: str) -> bool: + alias_match: Final = alias_regex.match(listed_id) + if alias_match is None or alias_match.group(1) == AZURE_MODEL_PLACEHOLDER: + return False + no_rival_matches: Final = rivals is not None and not _rival_may_take(listed_id, rivals) + return no_rival_matches or routes_to_model(listed_id, alias_match.group(1)) + + return routes_through + + +def _reads_as_another_catalog_id(listed_id: str, catalog_id: str, name_scope: str, catalog: AbstractSet[str]) -> bool: + read_as: Final = listed_id.removeprefix(name_scope) + return read_as != catalog_id and read_as in catalog + + +def _ids_routing_through_catch_all( + name_pattern: str, catalog_ids: Sequence[str], routes_through: Callable[[str], bool] +) -> tuple[str, ...]: + if name_pattern.count("*") != 1: + return () + name_scope: Final = "".join(name_pattern.partition("*")[0].rpartition("/")[:2]) + catalog: Final = frozenset(catalog_ids) + named_ids: Final = ((name_pattern.replace("*", catalog_id), catalog_id) for catalog_id in catalog_ids) + listed_ids: Final = ( + listed_id + for listed_id, catalog_id in named_ids + if not _reads_as_another_catalog_id(listed_id, catalog_id, name_scope, catalog) and routes_through(listed_id) + ) + return tuple(dict.fromkeys(listed_ids)) + + +def _catch_all_catalog(litellm_params: LiteLLM_Params | None) -> Sequence[str]: + pinned_provider: Final = litellm_params.custom_llm_provider if litellm_params is not None else None + return get_provider_models(provider=pinned_provider or "*", litellm_params=litellm_params) or () + + def _catalog_ids_a_wider_name_matches( name_pattern: str, deployment_pattern: str, @@ -477,6 +543,71 @@ def _deployed_by_name(llm_router: Router | None) -> Callable[[str, str | None], return lambda model, team_id: model in deployed_names or (team_id, model) in team_deployed_names +def _literal_start(regex: str) -> str: + return regex.partition("(.*)")[0] + + +def _prefixes_overlap(first: str, second: str) -> bool: + return first.startswith(second) or second.startswith(first) + + +def _may_match_directly(own_regex: str, regex: str) -> bool: + return regex != own_regex and _prefixes_overlap(_literal_start(regex), _literal_start(own_regex)) + + +_UNINFERRED_PROVIDER: Final = str(None) + + +def _may_name_a_provider(escaped_provider: str) -> bool: + provider: Final = re.sub(r"\\(.)", r"\1", escaped_provider) + return provider == _UNINFERRED_PROVIDER or inferred_provider(f"{provider}/{provider}") == provider + + +def _may_match_under_a_provider(own_regex: str, regex: str) -> bool: + provider, has_provider, rest = _literal_start(regex).partition("/") + if not has_provider: + return True + return _prefixes_overlap(rest, _literal_start(own_regex)) and _may_name_a_provider(provider) + + +def _can_take_ids_of(own_regex: str, regex: str) -> bool: + return _may_match_directly(own_regex, regex) and PatternUtils.calculate_pattern_specificity( + regex + ) >= PatternUtils.calculate_pattern_specificity(own_regex) + + +def _compiled(regexes: Iterable[str]) -> tuple[re.Pattern[str], ...]: + return tuple(re.compile(regex) for regex in regexes) + + +def _pattern_regexes(pattern_router: PatternMatchRouter | None) -> tuple[str, ...]: + if pattern_router is None: + return () + return tuple(pattern_router.patterns) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped deployment lists + + +def _rival_patterns(llm_router: Router | None, team_id: str | None) -> RivalPatterns: + if llm_router is None: + return lambda _name_pattern: None + registry: Final = llm_router.registry() + router_regexes: Final = _pattern_regexes(registry.patterns()) + team_regexes: Final = _pattern_regexes(registry.patterns(team=True).get(team_id) if team_id is not None else None) + + def rivals(name_pattern: str) -> _Rivals | None: + own_regex: Final = PatternMatchRouter().pattern_to_regex(name_pattern) + if own_regex in team_regexes: + return _Rivals(_compiled(regex for regex in team_regexes if _can_take_ids_of(own_regex, regex)), ()) + if own_regex not in router_regexes: + return None + return _Rivals( + _compiled(regex for regex in router_regexes if _can_take_ids_of(own_regex, regex)) + + _compiled(regex for regex in team_regexes if _may_match_directly(own_regex, regex)), + _compiled(regex for regex in team_regexes if _may_match_under_a_provider(own_regex, regex)), + ) + + return functools.cache(rivals) + + def _routes_to_model( routed_models: Callable[[str, str | None], frozenset[str]] | None, team_id: str | None ) -> Callable[[str, str], bool]: @@ -495,10 +626,11 @@ def get_known_models_from_wildcard( wildcard_model: str, litellm_params: LiteLLM_Params | None = None, routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True, + rival_patterns: RivalPatterns = lambda _name_pattern: None, ) -> list[str]: provider_id_prefix: Final = _custom_llm_provider_prefix(wildcard_model, litellm_params) if litellm_params is None or provider_id_prefix is None: - return list(_known_models_from_wildcard(wildcard_model, litellm_params, routes_to_model)) + return list(_known_models_from_wildcard(wildcard_model, litellm_params, routes_to_model, rival_patterns)) return [ listed_id.removeprefix(provider_id_prefix) for listed_id in _known_models_from_wildcard( @@ -507,6 +639,7 @@ def get_known_models_from_wildcard( lambda listed_id, model: routes_to_model( listed_id.removeprefix(provider_id_prefix), model.removeprefix(provider_id_prefix) ), + rival_patterns, ) ] @@ -532,6 +665,7 @@ def _known_models_from_wildcard( wildcard_model: str, litellm_params: LiteLLM_Params | None, routes_to_model: Callable[[str, str], bool], + rival_patterns: RivalPatterns, ) -> tuple[str, ...]: wildcard_model_to_expand: Final = ( litellm_params.model @@ -562,6 +696,16 @@ def _known_models_from_wildcard( litellm_params = _hydrate_litellm_credential_name(litellm_params) + routes_through_catch_all: Final = ( + _routes_through_alias(wildcard_model, rival_patterns(wildcard_model), routes_to_model) + if litellm_params is not None and litellm_params.model == "*" + else None + ) + if routes_through_catch_all is not None and not _names_whole_models(wildcard_model): + return _ids_routing_through_catch_all( + wildcard_model, _catch_all_catalog(litellm_params), routes_through_catch_all + ) + wildcard_models = get_provider_models(provider=provider, litellm_params=litellm_params) if wildcard_models is None: @@ -588,9 +732,12 @@ def _known_models_from_wildcard( # add model prefix to wildcard models wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models] - return tuple( + listed_ids: Final = tuple( dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models) ) + if routes_through_catch_all is None: + return listed_ids + return tuple(listed_id for listed_id in listed_ids if routes_through_catch_all(listed_id)) def _under_wildcard_prefix(model: str, wildcard_provider_prefix: str, listed_provider: str) -> str: @@ -647,6 +794,7 @@ def expand_wildcard_deployments_for_model_info( """ expanded: Final[list[dict[str, object]]] = [] routed_models: Final = _cached_wildcard_routing(llm_router) + rival_patterns_of_team: Final = functools.cache(functools.partial(_rival_patterns, llm_router)) deployed_by_name: Final = _deployed_by_name(llm_router) for deployment in deployments: model_name = str(deployment.get("model_name") or "") @@ -668,6 +816,7 @@ def expand_wildcard_deployments_for_model_info( wildcard_model=model_name, litellm_params=litellm_params, routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)), + rival_patterns=rival_patterns_of_team(_deployment_team_id(deployment)), ) if not expanded_names: expanded.append(deployment) @@ -826,6 +975,7 @@ def _get_wildcard_models( ) -> list[str]: models_to_remove: Final = set() routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) + rival_patterns: Final = _rival_patterns(llm_router, team_id) all_wildcard_models: Final = [] for model in unique_models: if _check_wildcard_routing(model=model): @@ -844,6 +994,7 @@ def _get_wildcard_models( wildcard_model=model, litellm_params=LiteLLM_Params(**router_model["litellm_params"]), routes_to_model=routes_to_model, + rival_patterns=rival_patterns, ) ) else: diff --git a/litellm/router_rust/surface/indexes.py b/litellm/router_rust/surface/indexes.py index f5aca3e1bad..32f63bc593b 100644 --- a/litellm/router_rust/surface/indexes.py +++ b/litellm/router_rust/surface/indexes.py @@ -308,8 +308,9 @@ class HostedPatternRouter(PatternMatchRouter): if not isinstance(model, str): # pyright: ignore[reportUnnecessaryIsInstance] # a caller's non-string model goes to the base class return super().get_models_by_pattern(model, custom_llm_provider) - provider: Final = custom_llm_provider or inferred_provider(model) - return self._owner.models_by_pattern(self._team, [model, f"{provider}/{model}"]) + return self._owner.models_by_pattern(self._team, [model]) or self._owner.models_by_pattern( + self._team, [f"{custom_llm_provider or inferred_provider(model)}/{model}"] + ) class _TeamRouters(dict[str, HostedPatternRouter]): diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 1c576b1ee24..83fc81815d8 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -2463,27 +2463,379 @@ def test_wildcard_lists_an_id_a_more_specific_sibling_serves_with_the_same_model assert "openai/gpt-partial-route-9*" not in model_info_names -def test_prefixed_alias_over_a_bare_catch_all_lists_ids_that_route_through_it(monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize( + ("model_name", "expected_id"), + [ + pytest.param("team-partial-route/*", f"team-partial-route/{_ROUTE_OPENAI_ID}", id="whole-name"), + pytest.param("team-partial-route/gpt-*", f"team-partial-route/gpt-{_ROUTE_OPENAI_ID}", id="partial-name"), + pytest.param("team-partial-route/*-*", None, id="more-wildcards-than-the-deployment"), + ], +) +def test_prefixed_alias_over_a_bare_catch_all_lists_ids_that_route_to_catalog_models( + monkeypatch: pytest.MonkeyPatch, model_name: str, expected_id: str | None +) -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + from litellm.utils import get_valid_models + + monkeypatch.setenv("OPENAI_API_KEY", "sk-partial-route-test") + monkeypatch.setenv("AZURE_API_KEY", "azure-partial-route-test") + router: Final = Router(model_list=[{"model_name": model_name, "litellm_params": {"model": "*"}}]) + + def routed_model(listed_id: str) -> object: + routed: Final = router.registry().deployments(listed_id, source="wildcard") + return routed[0]["litellm_params"]["model"] if routed else None + + with _registered_chat_models({_ROUTE_OPENAI_ID: "openai"}): + catalog_ids: Final = frozenset(get_valid_models()) - {AZURE_MODEL_PLACEHOLDER} + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[model_name], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.registry().deployments() or []], router + ) + if row["model_name"] != model_name + ] + + listed_ids: Final = [*v1_models, *model_info_names] + assert [listed_id for listed_id in listed_ids if routed_model(listed_id) not in catalog_ids] == [] + if expected_id is None: + assert listed_ids == [] + else: + assert expected_id in v1_models + assert expected_id in model_info_names + assert routed_model(expected_id) == _ROUTE_OPENAI_ID + + +_ROUTE_OPENAI_CHAT_ID: Final = "partial-route-9-chat" + + +@pytest.mark.parametrize( + ("alias_names", "stolen_id", "kept_ids"), + [ + pytest.param( + ("team-partial-route/gpt-*", "team-partial-route/gpt-gpt-partial-route-*"), + f"team-partial-route/gpt-{_ROUTE_OPENAI_ID}", + ( + f"team-partial-route/gpt-gpt-partial-route-{_ROUTE_OPENAI_ID}", + f"team-partial-route/gpt-{_ROUTE_OPENAI_CHAT_ID}", + ), + id="partial-name-next-to-a-longer-partial-name", + ), + pytest.param( + ("team-partial-route/*", "team-partial-route/gpt-*"), + f"team-partial-route/{_ROUTE_OPENAI_ID}", + (f"team-partial-route/gpt-{_ROUTE_OPENAI_ID}", f"team-partial-route/{_ROUTE_OPENAI_CHAT_ID}"), + id="whole-name-next-to-a-partial-name", + ), + ], +) +def test_alias_over_a_bare_catch_all_skips_ids_a_more_specific_sibling_routes_elsewhere( + monkeypatch: pytest.MonkeyPatch, alias_names: tuple[str, ...], stolen_id: str, kept_ids: tuple[str, ...] +) -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + from litellm.utils import get_valid_models + + monkeypatch.setenv("OPENAI_API_KEY", "sk-partial-route-test") + router: Final = Router(model_list=[{"model_name": name, "litellm_params": {"model": "*"}} for name in alias_names]) + + def routed_model(listed_id: str) -> object: + routed: Final = router.registry().deployments(listed_id, source="wildcard") + return routed[0]["litellm_params"]["model"] if routed else None + + with _registered_chat_models({_ROUTE_OPENAI_ID: "openai", _ROUTE_OPENAI_CHAT_ID: "openai"}): + catalog_ids: Final = frozenset(get_valid_models()) + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=list(alias_names), + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.registry().deployments() or []], router + ) + if row["model_name"] not in alias_names + ] + + listed_ids: Final = [*v1_models, *model_info_names] + assert routed_model(stolen_id) not in catalog_ids + assert [listed_id for listed_id in listed_ids if routed_model(listed_id) not in catalog_ids] == [] + assert [kept_id for kept_id in kept_ids if kept_id not in v1_models or kept_id not in model_info_names] == [] + + +def test_suffix_alias_over_a_bare_catch_all_lists_no_id_that_names_another_catalog_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + from litellm.utils import get_valid_models + + monkeypatch.setenv("OPENAI_API_KEY", "sk-partial-route-test") + alias: Final = "team-partial-route/*-mini" + full_size_id: Final = _ROUTE_OPENAI_ID.removesuffix("-mini") + router: Final = Router(model_list=[{"model_name": alias, "litellm_params": {"model": "*"}}]) + + def routed_model(listed_id: str) -> object: + routed: Final = router.registry().deployments(listed_id, source="wildcard") + return routed[0]["litellm_params"]["model"] if routed else None + + with _registered_chat_models({_ROUTE_OPENAI_ID: "openai", full_size_id: "openai"}): + catalog_ids: Final = frozenset(get_valid_models()) + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[alias], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.registry().deployments() or []], router + ) + if row["model_name"] != alias + ] + + listed_ids: Final = [*v1_models, *model_info_names] + named_ids: Final = {listed_id: listed_id.removeprefix("team-partial-route/") for listed_id in listed_ids} + assert routed_model(f"team-partial-route/{_ROUTE_OPENAI_ID}") == full_size_id + assert { + listed_id: routed_model(listed_id) + for listed_id, named_id in named_ids.items() + if named_id in catalog_ids and routed_model(listed_id) != named_id + } == {} + assert f"team-partial-route/{_ROUTE_OPENAI_ID}-mini" in v1_models + assert f"team-partial-route/{_ROUTE_OPENAI_ID}-mini" in model_info_names + + +@pytest.mark.parametrize( + ("global_alias", "team_alias", "team_alias_model", "stolen_id", "named_model", "kept_ids"), + [ + pytest.param( + "team-partial-route/*", + "team-partial-route/gpt-*", + "*", + f"team-partial-route/{_ROUTE_OPENAI_ID}", + _ROUTE_OPENAI_ID, + (f"team-partial-route/{_ROUTE_OPENAI_CHAT_ID}",), + id="team-alias-matching-the-id", + ), + pytest.param( + "team-partial-route/*", + "openai/*", + "openai/*", + f"team-partial-route/{_ROUTE_OPENAI_CHAT_ID}", + _ROUTE_OPENAI_CHAT_ID, + (f"team-partial-route/{_ROUTE_OPENAI_ID}",), + id="team-alias-matching-the-id-under-its-provider", + ), + pytest.param( + "team-partial-route/gpt-*", + "team-partial-route/*", + "*", + f"team-partial-route/gpt-{_ROUTE_OPENAI_ID}", + _ROUTE_OPENAI_ID, + (), + id="team-alias-broader-than-the-global-alias", + ), + pytest.param( + "team-partial-route/*", + "o*", + "*", + f"team-partial-route/{_ROUTE_OPENAI_ID}", + _ROUTE_OPENAI_ID, + (), + id="provider-less-team-alias-matching-the-id-under-its-provider", + ), + ], +) +def test_team_key_listing_skips_ids_the_team_wildcard_routes_elsewhere( + monkeypatch: pytest.MonkeyPatch, + global_alias: str, + team_alias: str, + team_alias_model: str, + stolen_id: str, + named_model: str, + kept_ids: tuple[str, ...], +) -> None: from litellm import Router from litellm.proxy.auth.model_checks import get_complete_model_list monkeypatch.setenv("OPENAI_API_KEY", "sk-partial-route-test") - router: Final = Router(model_list=[{"model_name": "team-partial-route/*", "litellm_params": {"model": "*"}}]) - with _registered_chat_models({_ROUTE_OPENAI_ID: "openai"}): - v1_models: Final = get_complete_model_list( + router: Final = Router( + model_list=[ + {"model_name": global_alias, "litellm_params": {"model": "*"}}, + { + "model_name": "model_name_team-1_partial-route", + "litellm_params": {"model": team_alias_model}, + "model_info": {"id": "team-partial-route", "team_id": "team-1", "team_public_model_name": team_alias}, + }, + ] + ) + + def listed_for(team_id: str | None) -> list[str]: + return get_complete_model_list( key_models=[], team_models=[], + proxy_model_list=[global_alias], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + team_id=team_id, + ) + + with _registered_chat_models({_ROUTE_OPENAI_ID: "openai", _ROUTE_OPENAI_CHAT_ID: "openai", stolen_id: "openai"}): + listed_without_team: Final = listed_for(None) + listed_for_team: Final = listed_for("team-1") + stolen_route: Final = router.registry().names(stolen_id, kind="wildcard", team_id="team-1") + + assert named_model not in stolen_route + assert stolen_id in listed_without_team + assert stolen_id not in listed_for_team + assert set(kept_ids) <= set(listed_for_team) + + +def test_wildcard_model_group_alias_skips_ids_its_target_sends_to_another_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setenv("OPENAI_API_KEY", "sk-partial-route-test") + router: Final = Router( + model_list=[{"model_name": "team-partial-route/*", "litellm_params": {"model": "*"}}], + model_group_alias={"team-partial-route/gpt-*": "team-partial-route/*"}, + ) + misnamed_id: Final = f"team-partial-route/gpt-{_ROUTE_OPENAI_ID}" + + with _registered_chat_models({_ROUTE_OPENAI_ID: "openai"}): + listed: Final = get_complete_model_list( + key_models=["team-partial-route/gpt-*"], + team_models=[], proxy_model_list=["team-partial-route/*"], user_model=None, infer_model_from_keys=False, llm_router=router, ) + misnamed_route: Final = router.registry().names(misnamed_id, kind="wildcard") - unrouted_ids: Final = [ - listed_id for listed_id in v1_models if not router.wildcard_route_deployments(listed_id, None) - ] - assert f"team-partial-route/{_ROUTE_OPENAI_ID}" in v1_models - assert unrouted_ids == [] + assert _ROUTE_OPENAI_ID not in misnamed_route + assert misnamed_id not in listed + + +def test_team_copy_of_a_partial_alias_lists_ids_a_global_sibling_would_take(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setenv("OPENAI_API_KEY", "sk-partial-route-test") + alias: Final = "team-partial-route/gpt-*" + sibling_taken_id: Final = f"team-partial-route/gpt-{_ROUTE_OPENAI_CHAT_ID}" + router: Final = Router( + model_list=[ + {"model_name": alias, "litellm_params": {"model": "*"}}, + {"model_name": "team-partial-route/gpt-partial-*", "litellm_params": {"model": "openai/*"}}, + { + "model_name": "model_name_team-1_partial-route", + "litellm_params": {"model": "*"}, + "model_info": {"id": "team-partial-route", "team_id": "team-1", "team_public_model_name": alias}, + }, + ] + ) + + def listed_for(team_id: str | None) -> list[str]: + return get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[alias], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + team_id=team_id, + ) + + with _registered_chat_models({_ROUTE_OPENAI_ID: "openai", _ROUTE_OPENAI_CHAT_ID: "openai"}): + listed_without_team: Final = listed_for(None) + listed_for_team: Final = listed_for("team-1") + team_route: Final = router.registry().names(sibling_taken_id, kind="wildcard", team_id="team-1") + + assert team_route == {_ROUTE_OPENAI_CHAT_ID} + assert sibling_taken_id not in listed_without_team + assert sibling_taken_id in listed_for_team + assert f"team-partial-route/gpt-{_ROUTE_OPENAI_ID}" in listed_for_team + + +def test_partial_alias_over_a_catch_all_pinned_to_a_provider_lists_only_that_providers_models( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + + monkeypatch.setenv("OPENAI_API_KEY", "sk-partial-route-test") + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-partial-route-test") + alias: Final = "team-partial-route/gpt-*" + anthropic_id: Final = "claude-partial-route-9" + router: Final = Router( + model_list=[{"model_name": alias, "litellm_params": {"model": "*", "custom_llm_provider": "openai"}}] + ) + + with _registered_chat_models({_ROUTE_OPENAI_ID: "openai", anthropic_id: "anthropic"}): + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[alias], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.registry().deployments() or []], router + ) + ] + + assert f"team-partial-route/gpt-{_ROUTE_OPENAI_ID}" in v1_models + assert f"team-partial-route/gpt-{_ROUTE_OPENAI_ID}" in model_info_names + assert f"team-partial-route/gpt-{anthropic_id}" not in v1_models + assert f"team-partial-route/gpt-{anthropic_id}" not in model_info_names + + +@respx.mock +def test_partial_alias_over_a_pinned_catch_all_asks_the_provider_for_its_models_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("OPENAI_API_KEY", "sk-pinned") + models_route: Final = respx.get("http://pinned-catch-all.test/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "pinned-route-model"}]}) + ) + + listed: Final = get_known_models_from_wildcard( + "team-pinned-route/gpt-*", + LiteLLM_Params( + model="*", custom_llm_provider="openai", api_base="http://pinned-catch-all.test/v1", api_key="sk-pinned" + ), + ) + + assert "team-pinned-route/gpt-pinned-route-model" in listed + assert models_route.call_count == 1 @pytest.mark.asyncio From 97f07af044c854d861347181c7a118f9dd78e285 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:27:12 +0000 Subject: [PATCH 085/189] test(providers): preserve unspecified Gemini image output modality (#51149) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../providers/test_gemini_image_config_and_usage_wire.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/providers/test_gemini_image_config_and_usage_wire.py b/tests/integration/providers/test_gemini_image_config_and_usage_wire.py index 27b408b27aa..c0957c5c3cb 100644 --- a/tests/integration/providers/test_gemini_image_config_and_usage_wire.py +++ b/tests/integration/providers/test_gemini_image_config_and_usage_wire.py @@ -34,7 +34,7 @@ _COUNTS_ONLY_REPLY: Final[dict[str, JsonValue]] = { "input_tokens": 10, "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, "output_tokens": 20, - "output_tokens_details": {"image_tokens": 20, "text_tokens": 0}, + "output_tokens_details": {}, } @@ -167,7 +167,7 @@ def test_gemini_image_edit_rejects_an_image_config_string_that_is_not_json( "input_tokens": 0, "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, "output_tokens": 0, - "output_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens_details": {}, }, id="empty-usage-object", ), From 525df3f9a113d750e49e1ec865104c5c5d40bc67 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:27:58 +0000 Subject: [PATCH 086/189] test(spend): expect token billing for Vertex image edits (#51148) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/spend/test_vertex_regional_uplift_spend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/spend/test_vertex_regional_uplift_spend.py b/tests/integration/spend/test_vertex_regional_uplift_spend.py index fc592ba2a99..40b6aaeb96b 100644 --- a/tests/integration/spend/test_vertex_regional_uplift_spend.py +++ b/tests/integration/spend/test_vertex_regional_uplift_spend.py @@ -393,7 +393,7 @@ def test_image_edit_spend_follows_the_vertex_location(gateway: Gateway, location images: Final = raw.parse() assert images.data is not None and [image.b64_json for image in images.data] == ["aW1n"], images prices: Final = _published_prices(gateway, model) - expected: Final = _rate(prices, "output_cost_per_image") * _uplift(prices, placed) + expected: Final = _standard_cost(prices) * _uplift(prices, placed) assert float(raw.headers["x-litellm-response-cost"]) == pytest.approx(expected), raw.headers (sent,) = _only_generate_calls(wire, 1) forwarded: Final = sent.body.decode() From 606490a52dbafa822a24f943d0f8a14a1bc29940 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:31:49 +0000 Subject: [PATCH 087/189] fix(gemini): batch and flex tier fields the -latest alias rows were missing (#51150) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...model_prices_and_context_window_backup.json | 18 ++++++++++++++++++ model_prices_and_context_window.json | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3210f3e080b..ead85584c33 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -32280,6 +32280,7 @@ }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -32338,6 +32339,7 @@ }, "gemini/gemini-flash-lite-latest": { "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -60939,6 +60941,7 @@ }, "gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -60997,6 +61000,7 @@ }, "gemini-flash-lite-latest": { "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -61055,6 +61059,8 @@ "gemini-pro-latest": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_flex": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "gemini", @@ -61098,6 +61104,7 @@ }, "google_maps_grounding_cost_per_query": 0.014, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_priority": 3.6e-07, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, @@ -61110,12 +61117,18 @@ "supports_url_context": true, "web_search_billing_unit": "per_query", "cache_read_input_token_cost_flex": 2e-07, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_above_200k_tokens_flex": 2e-06, "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_above_200k_tokens_batches": 9e-06, + "output_cost_per_token_above_200k_tokens_flex": 9e-06, "output_cost_per_token_flex": 6e-06 }, "gemini/gemini-pro-latest": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_flex": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "gemini", @@ -61159,6 +61172,7 @@ }, "google_maps_grounding_cost_per_query": 0.014, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_priority": 3.6e-07, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, @@ -61171,7 +61185,11 @@ "supports_url_context": true, "web_search_billing_unit": "per_query", "cache_read_input_token_cost_flex": 2e-07, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_above_200k_tokens_flex": 2e-06, "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_above_200k_tokens_batches": 9e-06, + "output_cost_per_token_above_200k_tokens_flex": 9e-06, "output_cost_per_token_flex": 6e-06 }, "gemini-exp-1206": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3210f3e080b..ead85584c33 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -32280,6 +32280,7 @@ }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -32338,6 +32339,7 @@ }, "gemini/gemini-flash-lite-latest": { "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -60939,6 +60941,7 @@ }, "gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -60997,6 +61000,7 @@ }, "gemini-flash-lite-latest": { "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -61055,6 +61059,8 @@ "gemini-pro-latest": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_flex": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "gemini", @@ -61098,6 +61104,7 @@ }, "google_maps_grounding_cost_per_query": 0.014, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_priority": 3.6e-07, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, @@ -61110,12 +61117,18 @@ "supports_url_context": true, "web_search_billing_unit": "per_query", "cache_read_input_token_cost_flex": 2e-07, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_above_200k_tokens_flex": 2e-06, "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_above_200k_tokens_batches": 9e-06, + "output_cost_per_token_above_200k_tokens_flex": 9e-06, "output_cost_per_token_flex": 6e-06 }, "gemini/gemini-pro-latest": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_batches": 4e-07, + "cache_read_input_token_cost_above_200k_tokens_flex": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "gemini", @@ -61159,6 +61172,7 @@ }, "google_maps_grounding_cost_per_query": 0.014, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_priority": 3.6e-07, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, @@ -61171,7 +61185,11 @@ "supports_url_context": true, "web_search_billing_unit": "per_query", "cache_read_input_token_cost_flex": 2e-07, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "input_cost_per_token_above_200k_tokens_flex": 2e-06, "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_above_200k_tokens_batches": 9e-06, + "output_cost_per_token_above_200k_tokens_flex": 9e-06, "output_cost_per_token_flex": 6e-06 }, "gemini-exp-1206": { From c2afb07f93b47e982ae57724bbf94100c1aefd4e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 07:02:00 +0000 Subject: [PATCH 088/189] fix(spend): keep free response reads and cancels off the creation spend row (#51110) * fix(spend): track self-heal for test_with_id_security_off_reads_and_cancels_add_no_rows_and_no_replayed_turns Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend): preserve response turn rows when id security is disabled Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/db/db_spend_update_writer.py | 7 +++ .../test_responses_stream_spend_row_id.py | 6 +++ .../proxy/db/test_db_spend_update_writer.py | 51 +++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index fd47b2fbb6b..12267120965 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -771,6 +771,13 @@ class DBSpendUpdateWriter: llm_router=llm_router, ) billable_cost: Final = _billable_cost(response_cost, payload) + if ( + proxy_settings.get("disable_responses_id_security") is True + and payload["call_type"] in {"aget_responses", "acancel_responses"} + and payload["status"] == "success" + and not billable_cost + ): + return True payload["spend"] = billable_cost or 0.0 payload["startTime"] = _iso_timestamp(payload["startTime"]) payload["endTime"] = _iso_timestamp(payload["endTime"]) diff --git a/tests/integration/spend/test_responses_stream_spend_row_id.py b/tests/integration/spend/test_responses_stream_spend_row_id.py index 58ed9f7785f..c13ef59fee3 100644 --- a/tests/integration/spend/test_responses_stream_spend_row_id.py +++ b/tests/integration/spend/test_responses_stream_spend_row_id.py @@ -448,6 +448,12 @@ def test_with_id_security_off_reads_and_cancels_add_no_rows_and_no_replayed_turn assert [string_value(JSON_OBJECT.validate_json(read.content)["id"]) for read in reads] == [first_id] * 3 later_id: Final = _created_response_id(insecure, key, {"model": native, "input": "a later request"}) assert _request_ids_for_key(insecure, key, 2) == [first_id, later_id] + (first_turn,) = insecure.rows( + 'SELECT request_id, proxy_server_request, messages, call_type FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (first_id,), + ) + assert first_turn["call_type"] == "aresponses", first_turn + assert object_value(first_turn["proxy_server_request"])["input"] == first_prompt, first_turn wire.drain() _terminal_response_id( diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index 8bf7cd36a8c..36aeacd8476 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -5890,6 +5890,57 @@ async def test_request_error_counts_survive_the_ledger_path_switch(monkeypatch: ): 1} +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["aresponses", "aget_responses", "acancel_responses"]) +@pytest.mark.parametrize( + ("security_disabled", "status", "cost", "keep_read"), + [ + (False, "success", 0.0, True), + (True, "success", 0.0, False), + (True, "success", None, False), + (True, "success", 0.23, True), + (True, "failure", 0.0, True), + ], +) +async def test_unsecured_free_response_reads_leave_the_creation_spend_row_alone( + monkeypatch: pytest.MonkeyPatch, + call_type: str, + security_disabled: bool, + status: str, + cost: float | None, + keep_read: bool, +) -> None: + from litellm.proxy import proxy_server + + writer: Final = DBSpendUpdateWriter() + prisma: Final = _tool_usage_prisma() + prisma.get_request_status.return_value = status + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "disable_spend_logs", False) + monkeypatch.setitem(proxy_server.general_settings, "disable_responses_id_security", security_disabled) + now: Final = datetime(2026, 11, 11, tzinfo=timezone.utc) + assert await writer.update_database( + token="test-token", + user_id="test-user", + end_user_id=None, + team_id=None, + org_id=None, + kwargs={ + "model": "response-model", + "call_type": call_type, + "litellm_call_id": "read-call", + "litellm_params": {"metadata": {"user_api_key": "test-token", "status": status}}, + }, + completion_response={"id": "resp-shared"}, + start_time=now, + end_time=now, + response_cost=cost, + ) + await drain_logging_tasks() + expected: Final = [call_type] if call_type == "aresponses" or keep_read else [] + assert [row["call_type"] for row in prisma.spend_log_transactions] == expected + + class _SessionTotalsRedis: def __init__(self) -> None: self.values: dict[str, float] = {} # mutable-ok: stands in for Redis state From 3616014548d40020f508566620c5cd8745721f3c Mon Sep 17 00:00:00 2001 From: Satwik Bankapur Date: Wed, 11 Nov 2026 12:39:43 +0530 Subject: [PATCH 089/189] fix(proxy): add tzdata to the proxy extra so a legacy TZ doesn't stop startup (#49914) * fix(proxy): add tzdata to the proxy extra so a legacy TZ doesn't stop startup * style(proxy): drop the inline comment on the tzdata requirement * test(proxy): check tzdata is installed with the proxy extra on every platform * fix(proxy): set the first spend report run in UTC so a scheduler zone that differs from the system clock can't skip it --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 2 +- pyproject.toml | 1 + .../unit/proxy/proxy_server/test_lifecycle.py | 20 +++++++++++++ tests/unit/test_package_metadata.py | 29 +++++++++++++++++++ uv.lock | 2 ++ 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f7a27e4896c..de98a4d571b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12453,7 +12453,7 @@ class ProxyStartupEvent: _scheduled_weekly_spend_report, "interval", days=days, - next_run_time=datetime.now() + timedelta(seconds=10 + random.randint(0, 300)), + next_run_time=datetime.now(timezone.utc) + timedelta(seconds=10 + random.randint(0, 300)), id=WEEKLY_SPEND_REPORT_JOB_ID, replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, diff --git a/pyproject.toml b/pyproject.toml index e80ba2d5d44..76bb3a54223 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ proxy = [ "hiredis>=3.0.0,<4.0; python_version < '3.15'", "hiredis>=3.5.0,<4.0; python_version >= '3.15'", "apscheduler>=3.11.2,<4.0", + "tzdata>=2024.1", "fastapi-sso>=0.19.0,<1.0", "PyJWT>=2.13.0,<3.0", "python-multipart>=0.0.27,<1.0", diff --git a/tests/unit/proxy/proxy_server/test_lifecycle.py b/tests/unit/proxy/proxy_server/test_lifecycle.py index db5be07fb6f..7f99d12c868 100644 --- a/tests/unit/proxy/proxy_server/test_lifecycle.py +++ b/tests/unit/proxy/proxy_server/test_lifecycle.py @@ -25,6 +25,7 @@ import os import subprocess from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -1393,6 +1394,25 @@ async def test_monthly_spend_report_sent_when_the_lock_is_free_or_absent(acquire proxy_logging_obj.slack_alerting_instance.send_monthly_spend_report.assert_awaited_once_with() +@pytest.mark.asyncio +async def test_weekly_spend_report_first_run_is_minutes_ahead_when_the_scheduler_zone_differs_from_the_clock(): + """TZ=Asia/Calcutta on a host without the legacy alias puts the scheduler on IST through tzdata while + datetime.now() stays on UTC, so a naive first run landed hours in the past, past the misfire grace.""" + local_offset: Final = datetime.now().astimezone().utcoffset() or timedelta(0) + scheduler: Final = AsyncIOScheduler(timezone=timezone(local_offset + timedelta(hours=5, minutes=30))) + registered_after: Final = datetime.now(timezone.utc) + + await ProxyStartupEvent._initialize_slack_alerting_jobs( + scheduler=scheduler, + general_settings={}, + proxy_logging_obj=_make_slack_alerting_proxy_logging(acquire_lock_result=True), + prisma_client=MagicMock(), + ) + + first_run: Final = scheduler.get_job("weekly_spend_report_job").next_run_time + assert registered_after < first_run < datetime.now(timezone.utc) + timedelta(seconds=311) + + @pytest.mark.asyncio async def test_spend_report_locks_are_never_released(): """The lock is a per-window marker, not a mutex: releasing it lets the next pod re-send.""" diff --git a/tests/unit/test_package_metadata.py b/tests/unit/test_package_metadata.py index dc6e3dca64e..d7dce33bd74 100644 --- a/tests/unit/test_package_metadata.py +++ b/tests/unit/test_package_metadata.py @@ -2,6 +2,7 @@ import sys from pathlib import Path from typing import Final +import pytest from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet from packaging.utils import NormalizedName, canonicalize_name @@ -22,6 +23,11 @@ AIOHTTP_POOL_POISONING_SOURCE: Final = "aio-libs/aiohttp#12953, fixed in aiohttp ANYIO_STRANDED_LOCK_WAITER_FIX: Final = Version("4.14.0") ANYIO_STRANDED_LOCK_WAITER_SOURCE: Final = "agronholm/anyio#1145, fixed in anyio 4.14.0 released 2026-06-15" CLI_RUNTIME_DEPENDENCIES: Final = frozenset({"rich", "pyyaml", "requests"}) +PROXY_INSTALL_PLATFORMS: Final = ( + ("linux", "Linux", "posix"), + ("darwin", "Darwin", "posix"), + ("win32", "Windows", "nt"), +) PROXY_SERVER_ONLY_DEPENDENCIES: Final = frozenset( { "fastapi", @@ -50,6 +56,20 @@ def _requirement_names(requirements: list[str]) -> frozenset[NormalizedName]: return frozenset(canonicalize_name(Requirement(requirement).name) for requirement in requirements) +def _proxy_requirements_installed_on(sys_platform: str, platform_system: str, os_name: str) -> frozenset[NormalizedName]: + environment: Final = { + "sys_platform": sys_platform, + "platform_system": platform_system, + "os_name": os_name, + "extra": "proxy", + } + return frozenset( + canonicalize_name(requirement.name) + for requirement in map(Requirement, OPTIONAL_DEPENDENCIES["proxy"]) + if requirement.marker is None or requirement.marker.evaluate(environment) + ) + + def test_importing_litellm_reports_the_declared_release_version() -> None: import litellm @@ -79,6 +99,15 @@ def test_cli_extra_is_a_thin_client_install() -> None: assert cli & PROXY_SERVER_ONLY_DEPENDENCIES == frozenset() +@pytest.mark.parametrize(("sys_platform", "platform_system", "os_name"), PROXY_INSTALL_PLATFORMS) +def test_proxy_extra_installs_tzdata_beside_its_scheduler_on_every_platform( + sys_platform: str, platform_system: str, os_name: str +) -> None: + installed: Final = _proxy_requirements_installed_on(sys_platform, platform_system, os_name) + assert canonicalize_name("apscheduler") in installed + assert canonicalize_name("tzdata") in installed + + def test_declared_aiohttp_floor_excludes_pool_poisoning_releases() -> None: declared: Final = tuple( Requirement(requirement) diff --git a/uv.lock b/uv.lock index 2dec937800c..386f4339b3f 100644 --- a/uv.lock +++ b/uv.lock @@ -5138,6 +5138,7 @@ proxy = [ { name = "soundfile" }, { name = "starlette" }, { name = "tomlkit" }, + { name = "tzdata" }, { name = "uvicorn" }, { name = "uvloop", marker = "sys_platform != 'win32'" }, { name = "websockets" }, @@ -5423,6 +5424,7 @@ requires-dist = [ { name = "tomlkit", marker = "extra == 'cli'", specifier = ">=0.13.3,<1.0" }, { name = "tomlkit", marker = "extra == 'proxy'", specifier = ">=0.13.3,<1.0" }, { name = "typing-extensions", specifier = ">=4.13.0,<5.0" }, + { name = "tzdata", marker = "extra == 'proxy'", specifier = ">=2024.1" }, { name = "uvicorn", marker = "extra == 'proxy'", specifier = ">=0.33.0,<1.0" }, { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.22.1,<1.0" }, { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" }, From eba880e108109fec9a9a131c6e26c71b9c2b5828 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 11 Nov 2026 00:48:11 -0800 Subject: [PATCH 090/189] fix(tls)!: exclude RSA key exchange and ChaCha20 from the outbound cipher list in FIPS mode (#50671) * fix(tls): exclude RSA key exchange from outbound cipher list in FIPS mode * fix(tls): apply the FIPS cipher filter to SSL contexts clients already hold enforce_fips_mode runs after load_config, so guardrail and logger clients built during config load held an SSL context without the RSA key exchange exclusion. enforce_fips_mode now updates every cached context in place, and the SDK no longer imports the proxy's FIPS module. * fix(tls): refuse to start in FIPS mode when SSL_SECURITY_LEVEL has only RSA key exchange Excluding RSA key exchange from an SSL_SECURITY_LEVEL that lists only RSA key-exchange suites leaves no TLS 1.2 suite, so set_ciphers raised on every client build. enforce_fips_mode now checks the configured list first and refuses to start with a FipsModeError that names the setting. The cipher list test no longer assumes the host has RSA key-exchange suites. * fix(tls): offer only TLS 1.3 instead of refusing to start when no TLS 1.2 suite without RSA key exchange is left * chore(tls): warn when fips_mode leaves a cipher list with only TLS 1.3 * style(tls): split the TLS 1.3 fallback warning to fit the line limit * fix(proxy): exclude RSA key exchange before config-load secret reads in FIPS mode get_config reads secrets (AWS, GCP, HashiCorp, CyberArk secret managers) over the shared outbound SSL context before enforce_fips_mode runs, so with general_settings.fips_mode true those first TLS handshakes still offered RSA key exchange. Turn the exclusion on in get_config from the raw general_settings, right before the secret manager starts, reusing the existing fips_mode_setting (now public) and exclude_rsa_key_exchange. enforce_fips_mode still owns the refusal and the final state; fips_mode given as an os.environ/ reference is still handled there. * test(proxy): skip the config-load RSA key exchange test on an OpenSSL without RSA suites On an OpenSSL whose default cipher list offers no RSA key exchange suite, fips and standard mode read secrets the same way, so the test has nothing to tell apart and skips, as the http_handler fallback test already does. * fix(tls): exclude TLS 1.2 ChaCha20 suites from the outbound cipher list in FIPS mode ChaCha20-Poly1305 is not a FIPS approved algorithm. FIPS providers already refuse it, so on the Chainguard image this changes nothing, but fips_mode now drops it from TLS 1.2 explicitly next to RSA key exchange. Python's set_ciphers cannot touch TLS 1.3 suites, which still come from OpenSSL's config. exclude_rsa_key_exchange is renamed to exclude_rsa_key_exchange_and_chacha20 to match. --- litellm/llms/custom_httpx/http_handler.py | 55 +++++++-- litellm/proxy/common_utils/fips.py | 9 +- litellm/proxy/proxy_server.py | 9 +- .../llms/custom_httpx/test_http_handler.py | 108 ++++++++++++++++++ .../proxy/proxy_server/test_proxy_config.py | 70 ++++++++++++ 5 files changed, 234 insertions(+), 17 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 307d67bab21..011551faf0c 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -316,6 +316,48 @@ def _prepare_request_data_and_content( # Cache for SSL contexts to avoid creating duplicate contexts with the same configuration _ssl_context_cache: Final[dict[tuple[str | None, str | None, str | None, bool], ssl.SSLContext]] = {} +_rsa_key_exchange_and_chacha20_excluded: Final = threading.Event() + + +def _set_ciphers( + context: ssl.SSLContext, ssl_security_level: str | None, rsa_key_exchange_and_chacha20_excluded: bool +) -> None: + """ + Offer TLS 1.2 and up with SSL_SECURITY_LEVEL's cipher string if set, else DEFAULT_SSL_CIPHERS. + With RSA key exchange and ChaCha20 excluded, drop their TLS 1.2 suites, or offer only TLS 1.3 and log a warning + when the string has no other TLS 1.2 suite. TLS 1.3 suites come from OpenSSL's config, which set_ciphers can't + change. + """ + ciphers: Final = ( + ssl_security_level if ssl_security_level and isinstance(ssl_security_level, str) else DEFAULT_SSL_CIPHERS + ) + context.minimum_version = ssl.TLSVersion.TLSv1_2 # rebind-ok: sets up the caller's context in place + context.set_ciphers(ciphers) + if rsa_key_exchange_and_chacha20_excluded: + try: + context.set_ciphers(f"{ciphers}:!kRSA:!CHACHA20") + except ssl.SSLError: + context.minimum_version = ssl.TLSVersion.TLSv1_3 # rebind-ok: sets up the caller's context in place + verbose_logger.warning( + "fips_mode: cipher list '%s' has no TLS 1.2 suite without RSA key exchange or ChaCha20, " + "so outbound TLS offers only TLS 1.3 and TLS 1.2-only servers will fail.", + ciphers, + ) + + +def exclude_rsa_key_exchange_and_chacha20(excluded: bool) -> None: + """ + Turn RSA key exchange and TLS 1.2 ChaCha20 off or back on for new SSL contexts and every cached one, which clients + already hold. + """ + if excluded == _rsa_key_exchange_and_chacha20_excluded.is_set(): + return + if excluded: + _rsa_key_exchange_and_chacha20_excluded.set() + else: + _rsa_key_exchange_and_chacha20_excluded.clear() + for (_, ssl_security_level, _, _), context in list(_ssl_context_cache.items()): + _set_ciphers(context, ssl_security_level, excluded) def _create_ssl_context( @@ -329,18 +371,7 @@ def _create_ssl_context( """ custom_ssl_context: Final = ssl.create_default_context(cafile=cafile) - # Optimize SSL handshake performance - # Set minimum TLS version to 1.2 for better performance - custom_ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 - - # Configure cipher suites for optimal performance - if ssl_security_level and isinstance(ssl_security_level, str): - # User provided custom cipher configuration (e.g., via SSL_SECURITY_LEVEL env var) - custom_ssl_context.set_ciphers(ssl_security_level) - else: - # Use optimized cipher list that strongly prefers fast ciphers - # but falls back to widely compatible ones - custom_ssl_context.set_ciphers(DEFAULT_SSL_CIPHERS) + _set_ciphers(custom_ssl_context, ssl_security_level, _rsa_key_exchange_and_chacha20_excluded.is_set()) # Configure ECDH curve for key exchange (e.g., to disable PQC and improve performance) # Set SSL_ECDH_CURVE env var or litellm.ssl_ecdh_curve to 'X25519' to disable PQC diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 6cd23e5c0ff..efa57817482 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -2,6 +2,7 @@ import threading from collections.abc import Mapping from typing import Final +from litellm.llms.custom_httpx.http_handler import exclude_rsa_key_exchange_and_chacha20 from litellm.secret_managers.main import str_to_bool FIPS_MODE_SETTING: Final = "fips_mode" @@ -13,7 +14,7 @@ class FipsModeError(Exception): pass -def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: +def fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: """general_settings.fips_mode as a boolean: False when the key is absent, None when it is not a boolean, including a key with no value.""" if FIPS_MODE_SETTING not in general_settings: @@ -60,7 +61,7 @@ def cryptography_enforces_fips() -> bool: def fips_mode_refusal(general_settings: Mapping[str, object]) -> str | None: """Why the proxy must not start: general_settings.fips_mode is not a boolean, or is on but Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" - setting: Final = _fips_mode_setting(general_settings) + setting: Final = fips_mode_setting(general_settings) if setting is None: value: Final = general_settings.get(FIPS_MODE_SETTING) if value is None: @@ -89,7 +90,9 @@ def enforce_fips_mode(general_settings: Mapping[str, object]) -> None: refusal: Final = fips_mode_refusal(general_settings) if refusal is not None: raise FipsModeError(refusal) - if _fips_mode_setting(general_settings): + active: Final = fips_mode_setting(general_settings) is True + exclude_rsa_key_exchange_and_chacha20(active) + if active: _FIPS_MODE_ACTIVE.set() else: _FIPS_MODE_ACTIVE.clear() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index de98a4d571b..9f721e3fdcb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -419,7 +419,7 @@ from litellm.llms.anthropic.messages.convertible_blocks import ( malformed_tool_result_count_error, ) from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, exclude_rsa_key_exchange_and_chacha20 from litellm.llms.openai.live import parse_live_session_start from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -547,7 +547,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id -from litellm.proxy.common_utils.fips import enforce_fips_mode +from litellm.proxy.common_utils.fips import enforce_fips_mode, fips_mode_setting from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -6680,6 +6680,11 @@ class ProxyConfig: printed_yaml: Final = copy.deepcopy(config) printed_yaml.pop("environment_variables", None) + # Secret manager reads and the rest of config load make outbound TLS calls before enforce_fips_mode runs + raw_general_settings: Final = as_str_mapping(config.get("general_settings")) + if raw_general_settings is not None and fips_mode_setting(raw_general_settings) is True: + exclude_rsa_key_exchange_and_chacha20(True) + self._initialize_secret_manager_from_raw_config(config=config, config_file_path=config_file_path) config = self._check_for_os_environ_vars(config=config) diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index 4e031531180..adb6410d360 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -339,6 +339,114 @@ def test_get_ssl_configuration_integration(): assert ssl_context.verify_mode is not None +def _offered_ciphers(context: bool | str | ssl.SSLContext) -> tuple[tuple[str, str, str, str | None], ...]: + assert isinstance(context, ssl.SSLContext) + return tuple( + (cipher["name"], cipher["protocol"], cipher["kea"], cipher["symmetric"]) for cipher in context.get_ciphers() + ) + + +def _kept_in_fips_mode(cipher: tuple[str, str, str, str | None]) -> bool: + _, protocol, kea, symmetric = cipher + return kea != "kx-rsa" and (protocol, symmetric) != ("TLSv1.2", "chacha20-poly1305") + + +def _use_ssl_security_level(monkeypatch: pytest.MonkeyPatch, ssl_security_level: str | None) -> None: + from litellm.llms.custom_httpx.http_handler import _ssl_context_cache + + _ssl_context_cache.clear() + for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE", "SSL_CERT_FILE", "SSL_VERIFY"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "ssl_security_level", ssl_security_level) + + +def _fake_fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None: + import _hashlib + + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) + monkeypatch.setattr( + _hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported")) + ) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) + + +@pytest.mark.parametrize( + "ssl_security_level", + [None, "AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES256-GCM-SHA384"], + ids=["default-cipher-list", "ssl-security-level"], +) +def test_fips_mode_drops_rsa_key_exchange_and_tls_1_2_chacha20_from_the_outbound_cipher_list( + monkeypatch: pytest.MonkeyPatch, ssl_security_level: str | None +) -> None: + from litellm.constants import DEFAULT_SSL_CIPHERS + from litellm.llms.custom_httpx.http_handler import _ssl_context_cache + from litellm.proxy.common_utils.fips import enforce_fips_mode + + _use_ssl_security_level(monkeypatch, ssl_security_level) + configured: Final = ssl.create_default_context() + configured.set_ciphers(ssl_security_level or DEFAULT_SSL_CIPHERS) + + # A client built while the config loads holds its context from before FIPS mode is enforced + held: Final = get_ssl_configuration() + standard: Final = _offered_ciphers(held) + _fake_fips_openssl(monkeypatch) + enforce_fips_mode({"fips_mode": True}) + try: + held_in_fips: Final = _offered_ciphers(held) + new_in_fips: Final = _offered_ciphers(get_ssl_configuration(http2=True)) + finally: + enforce_fips_mode({}) + held_after_fips: Final = _offered_ciphers(held) + _ssl_context_cache.clear() + + assert standard == _offered_ciphers(configured) + kept_in_fips: Final = tuple(filter(_kept_in_fips_mode, standard)) + assert held_in_fips == kept_in_fips + assert new_in_fips == kept_in_fips + assert held_after_fips == standard + + +@pytest.mark.parametrize("excluded_suites", ["kRSA", "CHACHA20"]) +def test_fips_mode_offers_only_tls_1_3_when_the_cipher_list_has_only_excluded_tls_1_2_suites( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + excluded_suites: str, +) -> None: + from litellm.llms.custom_httpx.http_handler import _ssl_context_cache + from litellm.proxy.common_utils.fips import enforce_fips_mode, fips_mode_active + + excluded: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + try: + excluded.set_ciphers(excluded_suites) + except ssl.SSLError: + pytest.skip(f"this OpenSSL has no {excluded_suites} suites") + _use_ssl_security_level( + monkeypatch, + ":".join(cipher["name"] for cipher in excluded.get_ciphers() if cipher["protocol"] != "TLSv1.3"), + ) + held: Final = get_ssl_configuration() + assert isinstance(held, ssl.SSLContext) + _fake_fips_openssl(monkeypatch) + caplog.set_level(logging.WARNING, logger="LiteLLM") + enforce_fips_mode({"fips_mode": True}) + try: + started_in_fips: Final = fips_mode_active() + new: Final = get_ssl_configuration(http2=True) + assert isinstance(new, ssl.SSLContext) + in_fips: Final = (held.minimum_version, new.minimum_version) + finally: + enforce_fips_mode({}) + after_fips: Final = held.minimum_version + _ssl_context_cache.clear() + + assert started_in_fips is True + assert in_fips == (ssl.TLSVersion.TLSv1_3, ssl.TLSVersion.TLSv1_3) + assert after_fips == ssl.TLSVersion.TLSv1_2 + assert any("offers only TLS 1.3" in record.getMessage() for record in caplog.records) + + # Session Reuse Tests class MockClientSession: """Mock ClientSession that is not callable""" diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 712b4134c05..91ff818da4c 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -14,6 +14,7 @@ import json import logging import os import re +import ssl import sys from collections.abc import Awaitable, Callable, Mapping from contextlib import nullcontext @@ -34,9 +35,11 @@ from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import turn_on_json +from litellm.constants import DEFAULT_SSL_CIPHERS from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.proxy._types import CommonProxyErrors, ConfigGeneralSettings from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.common_utils.fips import enforce_fips_mode from litellm.proxy.config_resolvers.settings_store import SettingsStore from litellm.router_api import ApplyResult, RouterSettings from litellm.types.router import Deployment, LiteLLM_Params @@ -1901,6 +1904,73 @@ async def test_ProxyConfig_get_config_reuses_an_already_initialized_secret_manag } == {"client_reused": True, "master_key": "master-from-vault"} +TLS_RECORDING_SECRET_MANAGER_MODULE = """ +import os + +from litellm.integrations.custom_secret_manager import CustomSecretManager +from litellm.llms.custom_httpx.http_handler import get_ssl_configuration + + +class TlsRecordingSecretManager(CustomSecretManager): + def sync_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + offers_rsa = any(c["kea"] == "kx-rsa" for c in get_ssl_configuration().get_ciphers()) + with open(os.environ["RSA_OFFER_LOG"], "a") as f: + f.write(f"{secret_name} {offers_rsa}\\n") + return "provider-from-vault" + + async def async_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + return self.sync_read_secret(secret_name) +""" + +TLS_RECORDING_CONFIG = """ +model_list: + - model_name: my-model + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/MY_PROVIDER_KEY + +general_settings: + fips_mode: {fips_mode} + key_management_system: custom + key_management_settings: + custom_secret_manager: tls_recording_secret_manager.TlsRecordingSecretManager + hosted_keys: + - MY_PROVIDER_KEY +""" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, rsa_offered", + [("true", False), ('"true"', False), ("false", True)], + ids=["fips-mode", "fips-mode-from-a-string", "standard-mode"], +) +async def test_ProxyConfig_get_config_reads_secrets_without_rsa_key_exchange_in_fips_mode( + tmp_path, monkeypatch, fips_mode: str, rsa_offered: bool +): + """Secret reads during get_config run before enforce_fips_mode, so fips_mode already excludes RSA key exchange.""" + default_cipher_list: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + default_cipher_list.set_ciphers(DEFAULT_SSL_CIPHERS) + if not any(cipher["kea"] == "kx-rsa" for cipher in default_cipher_list.get_ciphers()): + pytest.skip("this OpenSSL offers no RSA key exchange suite from DEFAULT_SSL_CIPHERS") + config_file_path = _write_vault_backed_config( + tmp_path, monkeypatch, TLS_RECORDING_CONFIG.format(fips_mode=fips_mode) + ) + (tmp_path / "tls_recording_secret_manager.py").write_text(TLS_RECORDING_SECRET_MANAGER_MODULE) + rsa_offer_log: Final = tmp_path / "rsa_offers.log" + monkeypatch.setenv("RSA_OFFER_LOG", str(rsa_offer_log)) + + try: + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + finally: + enforce_fips_mode({}) + + assert { + "reads": set(rsa_offer_log.read_text().splitlines()), + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + } == {"reads": {f"MY_PROVIDER_KEY {rsa_offered}"}, "api_key": "provider-from-vault"} + + @pytest.mark.asyncio async def test_ProxyConfig_get_config_without_key_management_system_leaves_secret_manager_unset(tmp_path, monkeypatch): """No ``key_management_system`` means no manager, an unresolvable reference stays None, and From 8b8ec4e3a64facae7af657ab06bc214dd5bf9060 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:06:57 +0000 Subject: [PATCH 091/189] fix(vertex): drop floor-only retirement dates, veo 3.1 successor and gemini token limits from the Vertex model pages (#51180) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 232 +++++++----------- model_prices_and_context_window.json | 232 +++++++----------- 2 files changed, 182 insertions(+), 282 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ead85584c33..f3cf258a2bb 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29732,12 +29732,12 @@ }, "gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_above_200k_tokens": 3e-07, @@ -29753,7 +29753,7 @@ "input_cost_per_token_priority": 5.4e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29845,13 +29845,11 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, - "deprecation_date": "2027-05-28", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -29873,6 +29871,8 @@ "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -29894,12 +29894,6 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "web_search_billing_unit": "per_query" }, "gemini-3-pro-image-preview": { @@ -29961,15 +29955,13 @@ "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, - "deprecation_date": "2027-05-28", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "input_cost_per_token_priority": 9e-07, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, + "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, @@ -29980,6 +29972,8 @@ "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, "output_cost_per_token_priority": 5.4e-06, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -30001,12 +29995,6 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-image-preview": { @@ -30152,7 +30140,6 @@ }, "gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_audio_token_cost_batches": 2.5e-08, "cache_read_input_audio_token_cost_flex": 2.5e-08, "cache_read_input_audio_token_cost_priority": 9e-08, @@ -30160,7 +30147,6 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2027-05-07", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, "input_cost_per_audio_token_above_200k_tokens": 5e-07, @@ -30219,12 +30205,11 @@ "web_search_billing_unit": "per_query" }, "gemini-3.5-flash-lite": { - "deprecation_date": "2027-07-21", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -30238,6 +30223,8 @@ "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, "output_cost_per_token_priority": 4.5e-06, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -30257,6 +30244,7 @@ "supports_audio_output": false, "supports_computer_use": true, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -30269,15 +30257,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "web_search_billing_unit": "per_query" }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -30316,12 +30296,12 @@ }, "gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 3e-07, "input_cost_per_audio_token_above_200k_tokens": 3e-07, @@ -30337,7 +30317,7 @@ "input_cost_per_token_priority": 1.8e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -30604,23 +30584,39 @@ "gemini_native_audio": true }, "gemini-2.5-pro": { - "deprecation_date": "2026-10-20", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", - "successor_model": "gemini-3.8-flash", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, + "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, + "input_cost_per_token_above_200k_tokens_flex": 1.25e-06, + "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_flex": 6.25e-07, + "input_cost_per_token_priority": 2.25e-06, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, + "output_cost_per_token_above_200k_tokens_batches": 7.5e-06, + "output_cost_per_token_above_200k_tokens_flex": 7.5e-06, + "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 1.8e-05, + "prompt_cache_min_tokens": 2048.0, + "search_context_cost_per_query": {"search_context_size_low":0.035,"search_context_size_medium":0.035,"search_context_size_high":0.035}, + "search_free_requests": {"count":10000,"period":"day","shared_group":"gemini-2.5-pro"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "gemini-3.8-flash", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -30638,33 +30634,13 @@ "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, - "prompt_cache_min_tokens": 2048, "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"}, - "google_maps_grounding_cost_per_query": 0.025, - "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, - "input_cost_per_token_above_200k_tokens_flex": 1.25e-06, - "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, - "input_cost_per_token_batches": 6.25e-07, - "input_cost_per_token_flex": 6.25e-07, - "input_cost_per_token_priority": 2.25e-06, - "output_cost_per_token_above_200k_tokens_batches": 7.5e-06, - "output_cost_per_token_above_200k_tokens_flex": 7.5e-06, - "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, - "output_cost_per_token_batches": 5e-06, - "output_cost_per_token_flex": 5e-06, - "output_cost_per_token_priority": 1.8e-05 + "supports_web_search": true }, "gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -30852,7 +30828,7 @@ "input_cost_per_token_priority": 9e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -30891,22 +30867,32 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash": { - "prompt_cache_min_tokens": 4096, - "deprecation_date": "2027-05-19", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", - "successor_model": "vertex_ai/gemini-3.8-flash", "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_priority": 2.7e-07, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1.5e-06, + "input_cost_per_audio_token_priority": 2.7e-06, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "input_cost_per_token_priority": 2.7e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, + "output_cost_per_token_batches": 4.5e-06, + "output_cost_per_token_flex": 4.5e-06, + "output_cost_per_token_priority": 1.62e-05, + "prompt_cache_min_tokens": 4096.0, "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "vertex_ai/gemini-3.8-flash", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -30924,6 +30910,7 @@ "supports_audio_input": true, "supports_computer_use": true, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -30935,24 +30922,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 2.7e-06, - "input_cost_per_audio_token_priority": 2.7e-06, - "output_cost_per_token_priority": 1.62e-05, - "cache_read_input_token_cost_priority": 2.7e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_token_batches": 7.5e-07, - "output_cost_per_token_batches": 4.5e-06, - "input_cost_per_token_flex": 7.5e-07, - "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -33162,7 +33132,7 @@ "input_cost_per_token_priority": 9e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -33234,21 +33204,31 @@ "deprecation_date": "2026-11-06" }, "gemini-3.5-flash": { - "prompt_cache_min_tokens": 4096, - "deprecation_date": "2027-05-19", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", - "successor_model": "gemini-3.8-flash", "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_priority": 2.7e-07, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1.5e-06, + "input_cost_per_audio_token_priority": 2.7e-06, "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "input_cost_per_token_priority": 2.7e-06, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, + "output_cost_per_token_batches": 4.5e-06, + "output_cost_per_token_flex": 4.5e-06, + "output_cost_per_token_priority": 1.62e-05, + "prompt_cache_min_tokens": 4096.0, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "gemini-3.8-flash", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -33263,10 +33243,11 @@ "supported_output_modalities": [ "text" ], - "supports_audio_output": false, "supports_audio_input": true, + "supports_audio_output": false, "supports_computer_use": true, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -33278,24 +33259,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 2.7e-06, - "input_cost_per_audio_token_priority": 2.7e-06, - "output_cost_per_token_priority": 1.62e-05, - "cache_read_input_token_cost_priority": 2.7e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_token_batches": 7.5e-07, - "output_cost_per_token_batches": 4.5e-06, - "input_cost_per_token_flex": 7.5e-07, - "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "web_search_billing_unit": "per_query" }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -54132,13 +54096,11 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, - "deprecation_date": "2027-05-28", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -54160,15 +54122,14 @@ "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, - "supports_reasoning": false, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", "/v1/images/edits", "/v1/batch" - ] - + ], + "supports_reasoning": false }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -54210,15 +54171,13 @@ "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, - "deprecation_date": "2027-05-28", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "input_cost_per_token_priority": 9e-07, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, + "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, @@ -54230,15 +54189,14 @@ "output_cost_per_token_flex": 1.5e-06, "output_cost_per_token_priority": 5.4e-06, "regional_endpoint_uplift_multiplier": 1.1, - "supports_reasoning": false, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", "/v1/images/edits", "/v1/batch" - ] - + ], + "supports_reasoning": false }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -54364,7 +54322,6 @@ }, "vertex_ai/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_audio_token_cost_batches": 2.5e-08, "cache_read_input_audio_token_cost_flex": 2.5e-08, "cache_read_input_audio_token_cost_priority": 9e-08, @@ -54372,7 +54329,6 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2027-05-07", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, "input_cost_per_audio_token_above_200k_tokens": 5e-07, @@ -54432,12 +54388,11 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash-lite": { - "deprecation_date": "2027-07-21", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -54452,6 +54407,8 @@ "output_cost_per_token_flex": 1.25e-06, "output_cost_per_token_priority": 4.5e-06, "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -54471,6 +54428,7 @@ "supports_audio_output": false, "supports_computer_use": true, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -54483,15 +54441,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "web_search_billing_unit": "per_query" }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -55287,14 +55237,13 @@ ] }, "vertex_ai/veo-3.1-generate-001": { - "deprecation_date": "2026-11-17", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "successor_model": "vertex_ai/gemini-omni-1.2-flash", "supported_modalities": [ "text" ], @@ -55303,8 +55252,6 @@ ] }, "vertex_ai/veo-3.1-fast-generate-001": { - "deprecation_date": "2026-11-17", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -55312,6 +55259,7 @@ "output_cost_per_second_1080p": 0.12, "output_cost_per_second_4k": 0.3, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "successor_model": "vertex_ai/gemini-omni-1.2-flash", "supported_modalities": [ "text" ], @@ -72086,18 +72034,20 @@ "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "vertex_ai/gemini-omni-1.2-flash" }, "vertex_ai/gemini-omni-1.1-flash-preview": { "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", - "max_output_tokens": 57920, "max_input_tokens": 131072, + "max_output_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-1-1-flash", + "successor_model": "vertex_ai/gemini-omni-1.2-flash", "supported_endpoints": [ "/v1beta/interactions" ], diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ead85584c33..f3cf258a2bb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29732,12 +29732,12 @@ }, "gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_above_200k_tokens": 3e-07, @@ -29753,7 +29753,7 @@ "input_cost_per_token_priority": 5.4e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29845,13 +29845,11 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, - "deprecation_date": "2027-05-28", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -29873,6 +29871,8 @@ "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -29894,12 +29894,6 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "web_search_billing_unit": "per_query" }, "gemini-3-pro-image-preview": { @@ -29961,15 +29955,13 @@ "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, - "deprecation_date": "2027-05-28", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "input_cost_per_token_priority": 9e-07, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, + "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, @@ -29980,6 +29972,8 @@ "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, "output_cost_per_token_priority": 5.4e-06, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -30001,12 +29995,6 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-image-preview": { @@ -30152,7 +30140,6 @@ }, "gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_audio_token_cost_batches": 2.5e-08, "cache_read_input_audio_token_cost_flex": 2.5e-08, "cache_read_input_audio_token_cost_priority": 9e-08, @@ -30160,7 +30147,6 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2027-05-07", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, "input_cost_per_audio_token_above_200k_tokens": 5e-07, @@ -30219,12 +30205,11 @@ "web_search_billing_unit": "per_query" }, "gemini-3.5-flash-lite": { - "deprecation_date": "2027-07-21", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -30238,6 +30223,8 @@ "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, "output_cost_per_token_priority": 4.5e-06, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -30257,6 +30244,7 @@ "supports_audio_output": false, "supports_computer_use": true, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -30269,15 +30257,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "web_search_billing_unit": "per_query" }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -30316,12 +30296,12 @@ }, "gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 3e-07, "input_cost_per_audio_token_above_200k_tokens": 3e-07, @@ -30337,7 +30317,7 @@ "input_cost_per_token_priority": 1.8e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -30604,23 +30584,39 @@ "gemini_native_audio": true }, "gemini-2.5-pro": { - "deprecation_date": "2026-10-20", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", - "successor_model": "gemini-3.8-flash", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, + "deprecation_date": "2026-10-20", + "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", + "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, + "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, + "input_cost_per_token_above_200k_tokens_flex": 1.25e-06, + "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_flex": 6.25e-07, + "input_cost_per_token_priority": 2.25e-06, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, + "output_cost_per_token_above_200k_tokens_batches": 7.5e-06, + "output_cost_per_token_above_200k_tokens_flex": 7.5e-06, + "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 1.8e-05, + "prompt_cache_min_tokens": 2048.0, + "search_context_cost_per_query": {"search_context_size_low":0.035,"search_context_size_medium":0.035,"search_context_size_high":0.035}, + "search_free_requests": {"count":10000,"period":"day","shared_group":"gemini-2.5-pro"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "gemini-3.8-flash", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -30638,33 +30634,13 @@ "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, - "prompt_cache_min_tokens": 2048, "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"}, - "google_maps_grounding_cost_per_query": 0.025, - "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, - "input_cost_per_token_above_200k_tokens_flex": 1.25e-06, - "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, - "input_cost_per_token_batches": 6.25e-07, - "input_cost_per_token_flex": 6.25e-07, - "input_cost_per_token_priority": 2.25e-06, - "output_cost_per_token_above_200k_tokens_batches": 7.5e-06, - "output_cost_per_token_above_200k_tokens_flex": 7.5e-06, - "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, - "output_cost_per_token_batches": 5e-06, - "output_cost_per_token_flex": 5e-06, - "output_cost_per_token_priority": 1.8e-05 + "supports_web_search": true }, "gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, @@ -30852,7 +30828,7 @@ "input_cost_per_token_priority": 9e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -30891,22 +30867,32 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash": { - "prompt_cache_min_tokens": 4096, - "deprecation_date": "2027-05-19", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", - "successor_model": "vertex_ai/gemini-3.8-flash", "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_priority": 2.7e-07, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1.5e-06, + "input_cost_per_audio_token_priority": 2.7e-06, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "input_cost_per_token_priority": 2.7e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, + "output_cost_per_token_batches": 4.5e-06, + "output_cost_per_token_flex": 4.5e-06, + "output_cost_per_token_priority": 1.62e-05, + "prompt_cache_min_tokens": 4096.0, "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "vertex_ai/gemini-3.8-flash", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -30924,6 +30910,7 @@ "supports_audio_input": true, "supports_computer_use": true, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -30935,24 +30922,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 2.7e-06, - "input_cost_per_audio_token_priority": 2.7e-06, - "output_cost_per_token_priority": 1.62e-05, - "cache_read_input_token_cost_priority": 2.7e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_token_batches": 7.5e-07, - "output_cost_per_token_batches": 4.5e-06, - "input_cost_per_token_flex": 7.5e-07, - "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -33162,7 +33132,7 @@ "input_cost_per_token_priority": 9e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -33234,21 +33204,31 @@ "deprecation_date": "2026-11-06" }, "gemini-3.5-flash": { - "prompt_cache_min_tokens": 4096, - "deprecation_date": "2027-05-19", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", - "successor_model": "gemini-3.8-flash", "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_priority": 2.7e-07, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1.5e-06, + "input_cost_per_audio_token_priority": 2.7e-06, "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "input_cost_per_token_priority": 2.7e-06, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, + "output_cost_per_token_batches": 4.5e-06, + "output_cost_per_token_flex": 4.5e-06, + "output_cost_per_token_priority": 1.62e-05, + "prompt_cache_min_tokens": 4096.0, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "gemini-3.8-flash", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -33263,10 +33243,11 @@ "supported_output_modalities": [ "text" ], - "supports_audio_output": false, "supports_audio_input": true, + "supports_audio_output": false, "supports_computer_use": true, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -33278,24 +33259,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 2.7e-06, - "input_cost_per_audio_token_priority": 2.7e-06, - "output_cost_per_token_priority": 1.62e-05, - "cache_read_input_token_cost_priority": 2.7e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_token_batches": 7.5e-07, - "output_cost_per_token_batches": 4.5e-06, - "input_cost_per_token_flex": 7.5e-07, - "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "web_search_billing_unit": "per_query" }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -54132,13 +54096,11 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, - "deprecation_date": "2027-05-28", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -54160,15 +54122,14 @@ "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, - "supports_reasoning": false, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", "/v1/images/edits", "/v1/batch" - ] - + ], + "supports_reasoning": false }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -54210,15 +54171,13 @@ "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, - "deprecation_date": "2027-05-28", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "input_cost_per_token_priority": 9e-07, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, + "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, @@ -54230,15 +54189,14 @@ "output_cost_per_token_flex": 1.5e-06, "output_cost_per_token_priority": 5.4e-06, "regional_endpoint_uplift_multiplier": 1.1, - "supports_reasoning": false, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/images/generations", "/v1/images/edits", "/v1/batch" - ] - + ], + "supports_reasoning": false }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -54364,7 +54322,6 @@ }, "vertex_ai/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_audio_token_cost_batches": 2.5e-08, "cache_read_input_audio_token_cost_flex": 2.5e-08, "cache_read_input_audio_token_cost_priority": 9e-08, @@ -54372,7 +54329,6 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2027-05-07", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, "input_cost_per_audio_token_above_200k_tokens": 5e-07, @@ -54432,12 +54388,11 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash-lite": { - "deprecation_date": "2027-07-21", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -54452,6 +54407,8 @@ "output_cost_per_token_flex": 1.25e-06, "output_cost_per_token_priority": 4.5e-06, "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -54471,6 +54428,7 @@ "supports_audio_output": false, "supports_computer_use": true, "supports_function_calling": true, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -54483,15 +54441,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "web_search_billing_unit": "per_query" }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -55287,14 +55237,13 @@ ] }, "vertex_ai/veo-3.1-generate-001": { - "deprecation_date": "2026-11-17", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "successor_model": "vertex_ai/gemini-omni-1.2-flash", "supported_modalities": [ "text" ], @@ -55303,8 +55252,6 @@ ] }, "vertex_ai/veo-3.1-fast-generate-001": { - "deprecation_date": "2026-11-17", - "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "mode": "video_generation", @@ -55312,6 +55259,7 @@ "output_cost_per_second_1080p": 0.12, "output_cost_per_second_4k": 0.3, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "successor_model": "vertex_ai/gemini-omni-1.2-flash", "supported_modalities": [ "text" ], @@ -72086,18 +72034,20 @@ "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "successor_model": "vertex_ai/gemini-omni-1.2-flash" }, "vertex_ai/gemini-omni-1.1-flash-preview": { "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", - "max_output_tokens": 57920, "max_input_tokens": 131072, + "max_output_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-1-1-flash", + "successor_model": "vertex_ai/gemini-omni-1.2-flash", "supported_endpoints": [ "/v1beta/interactions" ], From 25b1de636985aaa483ccd679ce0963f5cc30ae46 Mon Sep 17 00:00:00 2001 From: Oliver Jensen Date: Wed, 11 Nov 2026 16:23:45 +0100 Subject: [PATCH 092/189] docs(security): updated mailinglist and vdp in security.md (#51199) * tweak wording on ghsa again --- SECURITY.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index d3603448d46..6d0a9a6414d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,23 +4,19 @@ LiteLLM maintains a security announcements mailing list that is open to anyone. Subscribers receive advance notice, typically one to two days, before we release a fix for a particularly severe vulnerability or for any vulnerability exploitable by an unauthenticated attacker. This notice is provided on a best-effort basis -To subscribe, visit [https://berriai.github.io/security-announce-signup/](https://berriai.github.io/security-announce-signup/) +To subscribe, visit [https://security.litellm.ai/mailinglist/](https://security.litellm.ai/mailinglist/) ## Security Vulnerability Reporting Guidelines -> [!WARNING] -> Reports that do not include a video demonstrating the exploit will be closed without review. See [Reproduction Video Requirement](#reproduction-video-requirement) below. -We value the security community's role in protecting our systems and users. To report a security vulnerability: +We value the security community's role in protecting our systems and users. +To report a security vulnerability, please submit a report through our vulnerability disclosure form: -- File a private vulnerability report on GitHub: [Report a vulnerability](https://github.com/BerriAI/litellm/security/advisories/new) -- Include steps to reproduce the issue -- Include a video or screen recording demonstrating the full exploit against a live LiteLLM instance, from initial access through to impact. A terminal recording (for example asciinema) is fine for CLI-only exploits. -- Provide any relevant additional information +[LiteLLM VDP](https://security.litellm.ai/vdp/) -### Reproduction Video Requirement +These reports are submitted to HackerOne and are triaged there, though no account is needed. -A video demonstrating the exploit is required for every report. AI tools have made it easy to produce plausible-sounding vulnerability reports that do not reproduce in practice, and triaging them takes time away from real issues. Reports submitted without a working reproduction video will be closed without review. If you add a video to a closed report, we will reopen and triage it. +**Please do not submit vulnerabilities as draft GHSAs. As of 2026-11-11 new GHSA submissions will be closed.** ### Vulnerability Categories @@ -55,7 +51,9 @@ To qualify for a bounty, reports must include clear reproduction steps, a reprod ### Known Non-Issues -- Attacks that require a misconfiguration on setup (e.g not setting a `master_key` on the proxy configuration), are **explicitly not in scope** and are not considered vulnerable. +- **Misconfigurations:** attacks that require misconfiguration of the application (e.g. not setting a `master_key` on the proxy configuration). +- **Insecure configurations:** attacks that require deliberate configuration changes to disable security functionality. (e.g. IDOR on response IDs requiring configuration of `disable_responses_id_security` to `true`) +- **Social engineering:** attacks that rely on social engineering as their primary mechanism (e.g. convincing a user's agent to install a malicious package we did not publish). ## Security Measures From 1d97931c84703453368852d786a33bda644524e7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 07:44:09 -0800 Subject: [PATCH 093/189] chore(cost-map): rolling registry PR, fal_ai rows, databricks gemini-4-carbon, deepinfra and bedrock tier prices (#51193) --- ...odel_prices_and_context_window_backup.json | 347 +++++++++++++++++- model_prices_and_context_window.json | 347 +++++++++++++++++- tests/unit/test_cost_calculator.py | 31 ++ 3 files changed, 715 insertions(+), 10 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f3cf258a2bb..66e3a094879 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14379,12 +14379,14 @@ "bedrock/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, "supports_reasoning": true, @@ -24828,7 +24830,7 @@ "deepinfra/mistralai/Mistral-Nemo-Instruct-2407": { "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 1.9e-08, + "input_cost_per_token": 2.9e-08, "output_cost_per_token": 3e-08, "litellm_provider": "deepinfra", "mode": "chat", @@ -24909,8 +24911,8 @@ }, "deepinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning": { "max_input_tokens": 262144, - "input_cost_per_token": 8e-08, - "output_cost_per_token": 2e-07, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.6e-07, "litellm_provider": "deepinfra", "mode": "chat", "source": "https://deepinfra.com/pricing", @@ -56200,6 +56202,7 @@ "zai.glm-5": { "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, "input_cost_per_token_priority": 1.75e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock_converse", @@ -56207,6 +56210,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token_batches": 1.6e-06, + "output_cost_per_token_flex": 1.6e-06, "output_cost_per_token_priority": 5.6e-06, "supports_function_calling": true, "supports_reasoning": true, @@ -67981,12 +67985,12 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-sonnet-5": { - "input_cost_per_token": 2e-06, + "input_cost_per_token": 3e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 1e-05, + "output_cost_per_token": 1.5e-05, "prompt_cache_min_tokens": 1024, "source": "https://deepinfra.com/pricing", "supports_adaptive_thinking": true, @@ -100106,5 +100110,338 @@ "supports_web_search": false, "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true + }, + "fal_ai/fal-ai/kling-video/v2.6/pro/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video with the default generate_audio=true ($0.14); videos without audio are $0.07 per second and videos with audio and voice control are $0.168 per second, which have no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.6/pro/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/vidu/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the fixed 4s clip price of $0.20 per video"}, + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/vidu/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/vidu/start-end-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the fixed 4s clip price of $0.20 per video"}, + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/vidu/start-end-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/vidu/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the fixed 4s clip price of $0.40 per video"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/vidu/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/luma/agent/ray/v3.2/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the 5s clip prices by resolution ($0.15 at 540p, $0.30 at 720p, $1.20 at 1080p); output_cost_per_second is the default resolution 540p rate, which has no tier field; HDR ($0.60 at 720p, $2.40 at 1080p per 5s) and HDR+EXR ($0.90, $3.60) are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.24, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/luma/agent/ray/v3.2/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/luma/agent/ray/v3.2/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the 5s and 10s clip prices by resolution ($0.50/$1.00 at 540p, $1.00/$2.00 at 720p, $2.00/$4.00 at 1080p); output_cost_per_second is the default resolution 540p rate, which has no tier field"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.4, + "output_cost_per_second_720p": 0.2, + "source": "https://fal.ai/models/luma/agent/ray/v3.2/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/luma/agent/ray/v3.2/video-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the 5s and 10s clip prices by resolution ($0.72/$1.44 at 540p, $1.08/$2.16 at 720p, $2.16/$4.32 at 1080p); output_cost_per_second is the default resolution 540p rate, which has no tier field; HDR and HDR+EXR prices are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.144, + "output_cost_per_second_1080p": 0.432, + "output_cost_per_second_720p": 0.216, + "source": "https://fal.ai/models/luma/agent/ray/v3.2/video-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/pixverse/v6/max/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution without audio (the default generate_audio_switch=false); output_cost_per_second is the default resolution 720p rate; 360p $0.054 and 540p $0.081 have no catalog field; with audio the rates are $0.081/$0.099/$0.135/$0.252 per second, which is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.099, + "output_cost_per_second_1080p": 0.198, + "output_cost_per_second_720p": 0.099, + "source": "https://fal.ai/models/pixverse/v6/max/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/pixverse/v6/max/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution without audio (the default generate_audio_switch=false); output_cost_per_second is the default resolution 720p rate; 360p $0.054 and 540p $0.081 have no catalog field; with audio the rates are $0.081/$0.099/$0.135/$0.252 per second, which is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.099, + "output_cost_per_second_1080p": 0.198, + "output_cost_per_second_720p": 0.099, + "source": "https://fal.ai/models/pixverse/v6/max/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/flux-2-pro": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the default 1 MP output; fal bills $0.03 for the first output megapixel plus $0.015 per additional input or output megapixel, rounded up, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/flux-2-pro", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-pro/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the default 1 MP output; fal bills $0.03 for the first output megapixel plus $0.015 per additional input or output megapixel, rounded up, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/flux-2-pro/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-pro/outpaint": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the default 1 MP output; fal bills $0.03 for the first output megapixel plus $0.015 per additional input or output megapixel, rounded up, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/flux-2-pro/outpaint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-max": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the default 1 MP output; fal bills $0.07 for the first processed megapixel plus $0.03 per additional megapixel, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.07, + "source": "https://fal.ai/models/fal-ai/flux-2-max", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-max/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the default 1 MP output; fal bills $0.07 for the first processed megapixel plus $0.03 per additional megapixel, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.07, + "source": "https://fal.ai/models/fal-ai/flux-2-max/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/v1/erase": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the first generated megapixel; fal bills $0.004 for each subsequent generated megapixel and $0.004 per reference megapixel with a 3 MP minimum, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1/erase", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/smart-resize": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per single-image request at standard resolution: $0.15 per output image plus the $0.05 minimum vision analysis fee per request; additional output images in one request ($0.15 each) and 4K outputs (charged double) are not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.2, + "source": "https://fal.ai/models/fal-ai/smart-resize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/pro/layerize": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per generated layer for total pixel area under 1536x1536; layers for a base image over 1536x1536 are $0.0675 each, which has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.03375, + "source": "https://fal.ai/models/bytedance/seedream/v5/pro/layerize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/luma/agent/uni-1/v1/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per edit with one source image ($0.042 per edit plus $0.003 per reference or source image); each additional reference image adds $0.003, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.045, + "source": "https://fal.ai/models/luma/agent/uni-1/v1/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/luma/agent/uni-1/v1/max/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per edit with one source image ($0.102 per edit plus $0.003 per reference or source image); each additional reference image adds $0.003, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.105, + "source": "https://fal.ai/models/luma/agent/uni-1/v1/max/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "databricks/databricks-gemini-4-carbon": { + "cache_creation_input_token_cost": 2e-06, + "cache_read_input_token_cost": 2e-06, + "input_cost_per_token": 2e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "metadata": {"notes":"Gemini 4 Carbon on Databricks is available only through Unity Gateway with the model service name system.ai.gemini-4-carbon on a global endpoint. The Databricks pricing pages list no row for it yet, so input and output are the Global rates on the Vertex AI pricing page for Gemini 4 Carbon (listed there through November 30, 2026), no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true + }, + "databricks/system.ai.gemini-4-carbon": { + "cache_creation_input_token_cost": 2e-06, + "cache_read_input_token_cost": 2e-06, + "input_cost_per_token": 2e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "metadata": {"notes":"Gemini 4 Carbon on Databricks is available only through Unity Gateway with the model service name system.ai.gemini-4-carbon on a global endpoint. The Databricks pricing pages list no row for it yet, so input and output are the Global rates on the Vertex AI pricing page for Gemini 4 Carbon (listed there through November 30, 2026), no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f3cf258a2bb..66e3a094879 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14379,12 +14379,14 @@ "bedrock/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, "supports_reasoning": true, @@ -24828,7 +24830,7 @@ "deepinfra/mistralai/Mistral-Nemo-Instruct-2407": { "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 1.9e-08, + "input_cost_per_token": 2.9e-08, "output_cost_per_token": 3e-08, "litellm_provider": "deepinfra", "mode": "chat", @@ -24909,8 +24911,8 @@ }, "deepinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning": { "max_input_tokens": 262144, - "input_cost_per_token": 8e-08, - "output_cost_per_token": 2e-07, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.6e-07, "litellm_provider": "deepinfra", "mode": "chat", "source": "https://deepinfra.com/pricing", @@ -56200,6 +56202,7 @@ "zai.glm-5": { "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, "input_cost_per_token_priority": 1.75e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock_converse", @@ -56207,6 +56210,7 @@ "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token_batches": 1.6e-06, + "output_cost_per_token_flex": 1.6e-06, "output_cost_per_token_priority": 5.6e-06, "supports_function_calling": true, "supports_reasoning": true, @@ -67981,12 +67985,12 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-sonnet-5": { - "input_cost_per_token": 2e-06, + "input_cost_per_token": 3e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 1e-05, + "output_cost_per_token": 1.5e-05, "prompt_cache_min_tokens": 1024, "source": "https://deepinfra.com/pricing", "supports_adaptive_thinking": true, @@ -100106,5 +100110,338 @@ "supports_web_search": false, "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true + }, + "fal_ai/fal-ai/kling-video/v2.6/pro/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video with the default generate_audio=true ($0.14); videos without audio are $0.07 per second and videos with audio and voice control are $0.168 per second, which have no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.14, + "source": "https://fal.ai/models/fal-ai/kling-video/v2.6/pro/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/vidu/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the fixed 4s clip price of $0.20 per video"}, + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/vidu/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/vidu/start-end-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the fixed 4s clip price of $0.20 per video"}, + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/vidu/start-end-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/vidu/reference-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the fixed 4s clip price of $0.40 per video"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "source": "https://fal.ai/models/fal-ai/vidu/reference-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/luma/agent/ray/v3.2/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the 5s clip prices by resolution ($0.15 at 540p, $0.30 at 720p, $1.20 at 1080p); output_cost_per_second is the default resolution 540p rate, which has no tier field; HDR ($0.60 at 720p, $2.40 at 1080p per 5s) and HDR+EXR ($0.90, $3.60) are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.24, + "output_cost_per_second_720p": 0.06, + "source": "https://fal.ai/models/luma/agent/ray/v3.2/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/luma/agent/ray/v3.2/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the 5s and 10s clip prices by resolution ($0.50/$1.00 at 540p, $1.00/$2.00 at 720p, $2.00/$4.00 at 1080p); output_cost_per_second is the default resolution 540p rate, which has no tier field"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.4, + "output_cost_per_second_720p": 0.2, + "source": "https://fal.ai/models/luma/agent/ray/v3.2/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/luma/agent/ray/v3.2/video-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the 5s and 10s clip prices by resolution ($0.72/$1.44 at 540p, $1.08/$2.16 at 720p, $2.16/$4.32 at 1080p); output_cost_per_second is the default resolution 540p rate, which has no tier field; HDR and HDR+EXR prices are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.144, + "output_cost_per_second_1080p": 0.432, + "output_cost_per_second_720p": 0.216, + "source": "https://fal.ai/models/luma/agent/ray/v3.2/video-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/pixverse/v6/max/text-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution without audio (the default generate_audio_switch=false); output_cost_per_second is the default resolution 720p rate; 360p $0.054 and 540p $0.081 have no catalog field; with audio the rates are $0.081/$0.099/$0.135/$0.252 per second, which is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.099, + "output_cost_per_second_1080p": 0.198, + "output_cost_per_second_720p": 0.099, + "source": "https://fal.ai/models/pixverse/v6/max/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/pixverse/v6/max/image-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of generated video by resolution without audio (the default generate_audio_switch=false); output_cost_per_second is the default resolution 720p rate; 360p $0.054 and 540p $0.081 have no catalog field; with audio the rates are $0.081/$0.099/$0.135/$0.252 per second, which is not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.099, + "output_cost_per_second_1080p": 0.198, + "output_cost_per_second_720p": 0.099, + "source": "https://fal.ai/models/pixverse/v6/max/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/flux-2-pro": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the default 1 MP output; fal bills $0.03 for the first output megapixel plus $0.015 per additional input or output megapixel, rounded up, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/flux-2-pro", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-pro/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the default 1 MP output; fal bills $0.03 for the first output megapixel plus $0.015 per additional input or output megapixel, rounded up, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/flux-2-pro/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-pro/outpaint": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the default 1 MP output; fal bills $0.03 for the first output megapixel plus $0.015 per additional input or output megapixel, rounded up, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/flux-2-pro/outpaint", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-2-max": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the default 1 MP output; fal bills $0.07 for the first processed megapixel plus $0.03 per additional megapixel, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.07, + "source": "https://fal.ai/models/fal-ai/flux-2-max", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/flux-2-max/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the default 1 MP output; fal bills $0.07 for the first processed megapixel plus $0.03 per additional megapixel, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.07, + "source": "https://fal.ai/models/fal-ai/flux-2-max/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/v1/erase": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image for the first generated megapixel; fal bills $0.004 for each subsequent generated megapixel and $0.004 per reference megapixel with a 3 MP minimum, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.03, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1/erase", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/smart-resize": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per single-image request at standard resolution: $0.15 per output image plus the $0.05 minimum vision analysis fee per request; additional output images in one request ($0.15 each) and 4K outputs (charged double) are not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.2, + "source": "https://fal.ai/models/fal-ai/smart-resize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/pro/layerize": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per generated layer for total pixel area under 1536x1536; layers for a base image over 1536x1536 are $0.0675 each, which has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.03375, + "source": "https://fal.ai/models/bytedance/seedream/v5/pro/layerize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/luma/agent/uni-1/v1/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per edit with one source image ($0.042 per edit plus $0.003 per reference or source image); each additional reference image adds $0.003, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.045, + "source": "https://fal.ai/models/luma/agent/uni-1/v1/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/luma/agent/uni-1/v1/max/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per edit with one source image ($0.102 per edit plus $0.003 per reference or source image); each additional reference image adds $0.003, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.105, + "source": "https://fal.ai/models/luma/agent/uni-1/v1/max/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "databricks/databricks-gemini-4-carbon": { + "cache_creation_input_token_cost": 2e-06, + "cache_read_input_token_cost": 2e-06, + "input_cost_per_token": 2e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "metadata": {"notes":"Gemini 4 Carbon on Databricks is available only through Unity Gateway with the model service name system.ai.gemini-4-carbon on a global endpoint. The Databricks pricing pages list no row for it yet, so input and output are the Global rates on the Vertex AI pricing page for Gemini 4 Carbon (listed there through November 30, 2026), no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true + }, + "databricks/system.ai.gemini-4-carbon": { + "cache_creation_input_token_cost": 2e-06, + "cache_read_input_token_cost": 2e-06, + "input_cost_per_token": 2e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "metadata": {"notes":"Gemini 4 Carbon on Databricks is available only through Unity Gateway with the model service name system.ai.gemini-4-carbon on a global endpoint. The Databricks pricing pages list no row for it yet, so input and output are the Global rates on the Vertex AI pricing page for Gemini 4 Carbon (listed there through November 30, 2026), no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true } } diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index b991f492bdd..ad1e15330d2 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -3259,6 +3259,37 @@ def test_completion_cost_service_tier_for_bedrock(_local_model_cost_map): assert priority_cost > default_cost > flex_cost > 0 +@pytest.mark.parametrize( + ("model", "service_tier", "regional_row"), + [ + pytest.param( + "bedrock/moonshotai.kimi-k2.5", + "priority", + "bedrock/us-west-2/moonshotai.kimi-k2.5", + id="kimi-k2.5-priority", + ), + pytest.param("zai.glm-5", "flex", "bedrock/us-west-2/zai.glm-5", id="glm-5-flex"), + ], +) +def test_cost_per_token_bedrock_service_tier_without_a_region_matches_the_regional_row( + _local_model_cost_map, model: str, service_tier: str, regional_row: str +) -> None: + """Regression for #46880: a Bedrock priority or flex call priced without a region fell back to the + standard rate because the region-less row lacked the tier prices its regional rows carry.""" + regional: Final = litellm.model_cost[regional_row] + tier_input: Final = regional[f"input_cost_per_token_{service_tier}"] + tier_output: Final = regional[f"output_cost_per_token_{service_tier}"] + assert tier_input != regional["input_cost_per_token"] + + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + prompt_usd, completion_usd = cost_per_token( + model=model, custom_llm_provider="bedrock", usage_object=usage, service_tier=service_tier + ) + + assert prompt_usd == pytest.approx(1000 * tier_input) + assert completion_usd == pytest.approx(500 * tier_output) + + def test_completion_cost_service_tier_for_anthropic(_local_model_cost_map): """ Anthropic priority-tier requests must be priced at the priority rate. From 0a4ccf95544127d7b1e9b9121ee0c41deec4f0c9 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 11 Nov 2026 07:44:45 -0800 Subject: [PATCH 094/189] fix(router): preserve held tiers for OpenCode subagent results (#51176) * fix(router): preserve held tiers for OpenCode task results * fix(router): handle OpenCode subagent completion envelopes --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../complexity_router/README.md | 4 +- .../complexity_router/complexity_router.py | 21 +++- .../router_strategy/test_complexity_router.py | 117 ++++++++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index d06248a555f..394052c1dd4 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -690,7 +690,7 @@ generic behavior; use the real client when checking Claude Code routing. Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead -By default the router strips complete `` blocks. For requests with a Codex user agent, it also strips complete ``, ``, ``, and `` blocks, plus repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path. Other clients keep those tags and their contents +By default the router strips complete `` blocks. For requests with a Codex user agent, it also strips complete ``, ``, ``, and `` blocks, plus repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path. For requests with an `opencode/` user agent, it also strips complete `` blocks and `` blocks through ``, attributes included, which OpenCode uses to return subagent output as a user turn. Other clients keep those tags and their contents The proxy records the incoming user agent in request metadata. SDK callers can supply `metadata.user_agent` (or `litellm_metadata.user_agent` on Responses requests), or configure `reminder_markers` explicitly when their client identity is unavailable @@ -713,7 +713,7 @@ model_list: close: "[[SUBAGENT_CONTEXT_END]]" ``` -Setting `reminder_markers` replaces all built-in pairs, including the Codex heading pair, so include every default your harness still needs. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten +Setting `reminder_markers` replaces all built-in pairs, including the Codex and OpenCode pairs, so include every default your harness still needs. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten ### Code Detection diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index acf1445ac36..2ce1bda173a 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -520,6 +520,11 @@ _CODEX_REMINDER_MARKERS: Final = _DEFAULT_REMINDER_MARKERS + ( ("", ""), ("# agents.md instructions for ", ""), ) +_OPENCODE_REMINDER_MARKERS: Final = _DEFAULT_REMINDER_MARKERS + ( + ("", ""), + (""), +) +_OPENCODE_USER_AGENT_PREFIX: Final = "opencode/" _TRUNCATION_MARKER: Final = "..." _TRUNCATION_HEAD_FRACTION: Final = 0.3 @@ -530,6 +535,10 @@ _CLASSIFIER_CIRCUIT_OPEN_SIGNAL: Final = "classifier-circuit-open" _CJK_CHARACTER: Final = re.compile("[぀-ヿㇰ-ㇿ㐀-䶿一-鿿豈-﫿ｦ-ﾝ\U00020000-\U0003ffff]") +def _is_opencode_user_agent(user_agent: str) -> bool: + return user_agent.lower().startswith(_OPENCODE_USER_AGENT_PREFIX) + + def _message_text(content: object) -> str: """Flatten message content to plain text, joining multi-part text blocks. @@ -4418,12 +4427,20 @@ class ComplexityRouter(CustomLogger): def _reminder_markers_for_request(self, request_kwargs: Mapping[str, object]) -> tuple[tuple[str, str], ...]: if self.config.reminder_markers is not None: return self._reminder_markers - return _CODEX_REMINDER_MARKERS if self._is_codex_request(request_kwargs) else _DEFAULT_REMINDER_MARKERS + if self._is_codex_request(request_kwargs): + return _CODEX_REMINDER_MARKERS + if self._request_user_agent_matches(request_kwargs, _is_opencode_user_agent): + return _OPENCODE_REMINDER_MARKERS + return _DEFAULT_REMINDER_MARKERS @staticmethod def _is_codex_request(request_kwargs: Mapping[str, object]) -> bool: + return ComplexityRouter._request_user_agent_matches(request_kwargs, is_codex_user_agent) + + @staticmethod + def _request_user_agent_matches(request_kwargs: Mapping[str, object], matches: Callable[[str], bool]) -> bool: return any( - is_codex_user_agent(user_agent) + matches(user_agent) for metadata_key in ("litellm_metadata", "metadata") if isinstance(metadata := request_kwargs.get(metadata_key), Mapping) if isinstance(user_agent := metadata.get("user_agent"), str) diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 59f637bb294..054c48bbb5e 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -10250,6 +10250,12 @@ _CODEX_ENVELOPES: Final = ( "LITELLM ESCALATE environment", "# AGENTS.md instructions for /repo with spaces/中文\nLITELLM ESCALATE instructions", ) +_OPENCODE_USER_AGENT: Final = "opencode/latest/2.0.29/cli" +_OPENCODE_SUBAGENT_RESULTS: Final = ( + "\nLITELLM ESCALATE subagent found the race in cache.py", + '\n' + "LITELLM ESCALATE subagent found the race in cache.py\n", +) class TestContextAwareClassifier: @@ -10490,6 +10496,117 @@ class TestContextAwareClassifier: completion.assert_awaited_once() assert completion.call_args.kwargs["messages"][1]["content"].strip() == f"Classify this message:\n{envelope}" + @pytest.mark.asyncio + @pytest.mark.parametrize("subagent_result", _OPENCODE_SUBAGENT_RESULTS) + @pytest.mark.parametrize("user_agent", (_OPENCODE_USER_AGENT, "OpenCode/latest/2.0.29/cli")) + async def test_opencode_subagent_result_turn_keeps_held_tier_until_a_new_ask( + self, user_agent: str, subagent_result: str + ) -> None: + completion: Final = AsyncMock( + side_effect=[_llm_response('{"tier":"SIMPLE"}'), _llm_response('{"tier":"COMPLEX"}')] + ) + router: Final = ComplexityRouter( + model_name="router", + ctx=RoutingContext(MagicMock(acompletion=completion, **{"health.return_value.store": DualCache()})), + complexity_router_config={ + "tiers": {"SIMPLE": "simple-model", "COMPLEX": "task-model"}, + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "classification_mode": "user_turn", + }, + ) + metadata: Final = {"user_agent": user_agent, "session_id": "opencode-task-session"} + asked: Final = [{"role": "user", "content": "Find the race in cache.py"}] + task_returned: Final = [ + *asked, + {"role": "assistant", "content": "Delegating to a subagent"}, + {"role": "user", "content": subagent_result}, + ] + new_ask: Final = [ + *task_returned, + {"role": "assistant", "content": "Found it"}, + {"role": "user", "content": "Now design cache invalidation"}, + {"role": "user", "content": subagent_result}, + ] + original: Final = deepcopy((asked, task_returned, new_ask)) + + results: Final = [ + await router.async_pre_routing_hook( + model="router", request_kwargs={"metadata": metadata}, messages=messages + ) + for messages in (asked, task_returned, new_ask) + ] + + assert [(result.model, result.routing_decision["cause"]) for result in results] == [ + ("simple-model", "llm_classifier"), + ("simple-model", "user_turn_continuation"), + ("task-model", "llm_classifier"), + ] + classified: Final = [call.kwargs["messages"][1]["content"] for call in completion.await_args_list] + assert [payload.rsplit("Classify this message:\n", 1)[1] for payload in classified] == [ + "Find the race in cache.py", + "Now design cache invalidation", + ] + assert all("subagent found the race" not in payload for payload in classified) + assert tuple(result.messages for result in results) == original + + @pytest.mark.asyncio + @pytest.mark.parametrize("subagent_result", _OPENCODE_SUBAGENT_RESULTS) + @pytest.mark.parametrize( + "request_kwargs,content,custom_markers,expected_ask", + ( + ({"metadata": {"user_agent": _OPENCODE_USER_AGENT}}, "{result}\nAdd retries", False, "Add retries"), + ({"litellm_metadata": {"user_agent": _OPENCODE_USER_AGENT}}, "Add retries {result}", False, "Add retries"), + ( + {"metadata": {"user_agent": _OPENCODE_USER_AGENT}}, + "{unclosed}\nAdd retries", + False, + "{unclosed}\nAdd retries", + ), + ({"metadata": {"user_agent": _OPENCODE_USER_AGENT}}, "{result}", True, "{result}"), + ({}, "{result}", False, "{result}"), + ({"metadata": {"user_agent": "curl/8.7.1"}}, "{result}", False, "{result}"), + ({"metadata": {"user_agent": "codex-tui"}}, "{result}", False, "{result}"), + ({"metadata": {"user_agent": "opencode_go/1.0"}}, "{result}", False, "{result}"), + ({"metadata": {"user_agent": "opencodex/1.0"}}, "{result}", False, "{result}"), + ), + ) + async def test_classifier_strips_only_complete_opencode_subagent_results( + self, + request_kwargs: dict[str, object], + content: str, + custom_markers: bool, + expected_ask: str, + subagent_result: str, + ) -> None: + unclosed: Final = subagent_result[: subagent_result.rindex("", "close": ""}]} if custom_markers else {}), + }, + ) + messages: Final = [{"role": "user", "content": message}] + + response: Final = await router.async_pre_routing_hook( + model="router", request_kwargs=request_kwargs, messages=messages + ) + + assert response is not None and response.model == "task-model" + assert response.messages == [{"role": "user", "content": message}] + completion.assert_awaited_once() + classified: Final = completion.call_args.kwargs["messages"][1]["content"] + expected: Final = expected_ask.format(result=subagent_result, unclosed=unclosed) + assert classified.strip() == f"Classify this message:\n{expected}" + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) def test_codex_envelopes_preserve_delegated_task_and_prior_context(self, envelope: str) -> None: from litellm.router_strategy.complexity_router.complexity_router import ( From 20d53f2951c79bb9164af9032588c8b0a561727b Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:18:18 +0000 Subject: [PATCH 095/189] refactor: remove redundant comments from fresh changes (#51163) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/rubrik.py | 2 -- litellm/passthrough/utils.py | 1 - ui/litellm-dashboard/src/contexts/PluginModeContext.tsx | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/litellm/integrations/rubrik.py b/litellm/integrations/rubrik.py index 7d04b1df95e..131ab765b2c 100644 --- a/litellm/integrations/rubrik.py +++ b/litellm/integrations/rubrik.py @@ -675,8 +675,6 @@ class RubrikLogger(CustomGuardrail, CustomBatchLogger): else: tools = optional_params.get("tools") - # The response moderation service consumes only messages/model/tools - # (plus metadata.rac_agent_id when set). # Don't forward proxy_server_request -- in litellm >=1.83 its ``body`` # snapshot carries a UserAPIKeyAuth instance that breaks json.dumps, # silently fail-opening the guardrail. diff --git a/litellm/passthrough/utils.py b/litellm/passthrough/utils.py index 1bf5f08b1c1..7ef426898d8 100644 --- a/litellm/passthrough/utils.py +++ b/litellm/passthrough/utils.py @@ -96,7 +96,6 @@ class BasePassthroughUtils: header_name, ) continue - # Strip the 'x-pass-' prefix and normalize to lowercase actual_header_name = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower() headers[actual_header_name] = header_value diff --git a/ui/litellm-dashboard/src/contexts/PluginModeContext.tsx b/ui/litellm-dashboard/src/contexts/PluginModeContext.tsx index 247cae44235..617ed455fe3 100644 --- a/ui/litellm-dashboard/src/contexts/PluginModeContext.tsx +++ b/ui/litellm-dashboard/src/contexts/PluginModeContext.tsx @@ -4,7 +4,7 @@ import React, { createContext, useContext, useState, useEffect } from "react"; import { createApiClient } from "@/lib/http/client"; import { getProxyBaseUrl } from "@/components/networking"; -export type PluginMode = "ai-gateway" | "customer-portal" | string; // built-in modes or a registered plugin name +export type PluginMode = "ai-gateway" | "customer-portal" | string; export const BUILT_IN_MODES: ReadonlySet = new Set(["ai-gateway", "customer-portal"]); export interface PluginNavItem { From 3cd48a6ec74da40e946642ab4b8deb862860d889 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:15:42 +0000 Subject: [PATCH 096/189] test(security): expect the async release check on offline mode (#51213) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/security/test_offline_mode.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/integration/security/test_offline_mode.py b/tests/integration/security/test_offline_mode.py index 11012eb14bc..2488d05bc2f 100644 --- a/tests/integration/security/test_offline_mode.py +++ b/tests/integration/security/test_offline_mode.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Final import pytest -from integration._support.client import Gateway, object_value +from integration._support.client import Gateway, eventually, object_value from integration._support.forward_proxy import refusing_forward_proxy from integration._support.process import owned_proxy from pydantic import JsonValue @@ -120,7 +120,19 @@ def test_offline_mode_stops_every_self_initiated_request( source: Final = object_value(_json(isolated, "GET", "/model/cost_map/source")) reloaded: Final = isolated.request("POST", "/reload/model_cost_map") beta_headers: Final = object_value(_json(isolated, "POST", "/reload/anthropic_beta_headers")) - release: Final = _json(isolated, "GET", "/get/latest_release_info") + release: Final = isolated.request("GET", "/get/latest_release_info") + assert (release.status_code, release.json()) == (202 if offline_mode is None else 200, None), release.text + if offline_mode is None: + settled_release: Final = eventually( + lambda: isolated.request("GET", "/get/latest_release_info"), + lambda response: response.status_code != 202, + ) + assert (settled_release.status_code, settled_release.json()) == ( + 502, + {"detail": "Latest release info unavailable"}, + ), settled_release.text + else: + assert _json(isolated, "GET", "/get/latest_release_info") is None presets: Final = object_value(_json(isolated, "GET", "/public/autorouter_presets")) templates: Final = _json(isolated, "GET", "/policy/templates") blog: Final = object_value(_json(isolated, "GET", "/public/litellm_blog_posts")) @@ -164,7 +176,6 @@ def test_offline_mode_stops_every_self_initiated_request( assert source["is_env_forced"] is (offline_mode is not None), source assert reloaded.status_code == reload_status, reloaded.text assert beta_headers["status"] == "success", beta_headers - assert release is None assert sorted(presets) == sorted(object_value(_bundled("proxy/public_endpoints/autorouter_presets.json"))) assert templates == _bundled("policy_templates_backup.json") assert [object_value(post)["url"] for post in _list(blog["posts"])] == [ From 3a91fdfc077166ef4251bc3f1eb5dc360111d1eb Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:48:06 +0000 Subject: [PATCH 097/189] fix(management): keep classifier deployments serving during reloads (#51111) * fix(management): track self-heal for test_moving_the_workspace_under_a_burst_moves_the_classifier_and_logs_every_request_once Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(management): keep classifier deployments serving during reloads Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/router-core/src/lib.rs | 7 +- .../crates/router-core/src/registry.rs | 75 ++++++++++++++++ .../model_management_endpoints.py | 59 +----------- litellm/proxy/proxy_server.py | 5 +- litellm/router_rust/steps/prerouting.py | 81 ++++++++++------- litellm/router_rust/surface/registry.py | 89 ++++++++----------- .../test_model_management_endpoints.py | 66 ++++---------- tests/unit/router_rust/test_atomic_upsert.py | 89 +++++++++++++++++++ .../core/test_gaps_l11100_strategy.py | 6 +- .../test_registry_characterization_writes.py | 24 ++--- 10 files changed, 293 insertions(+), 208 deletions(-) create mode 100644 tests/unit/router_rust/test_atomic_upsert.py diff --git a/litellm-rust/crates/router-core/src/lib.rs b/litellm-rust/crates/router-core/src/lib.rs index f9b71c689c9..e61d570c78f 100644 --- a/litellm-rust/crates/router-core/src/lib.rs +++ b/litellm-rust/crates/router-core/src/lib.rs @@ -688,7 +688,7 @@ impl Router { entries, resolve_secrets, |deployment| { - working.push_built(deployment.spec.clone(), add)?; + working.install_built(deployment.spec.clone(), add, install)?; installed.push(deployment.spec.clone()); core.validate_deployment(working.last().expect("just installed")) .map_err(Error::from) @@ -698,7 +698,7 @@ impl Router { if built.is_ok() && replace && path == BuildPath::Create { working.set_model_names(model_names); } - self.serve_built(&base, working, installed, add)?; + self.serve_built(&base, working, installed, add, install)?; built } @@ -710,6 +710,7 @@ impl Router { built: Registry, installed: Vec, add: bool, + install: build::Install, ) -> Result<(), ConfigError> { let _writer = self.core.writer.lock(); let current = self.core.snapshot.load_full(); @@ -718,7 +719,7 @@ impl Router { } else { let mut next = Registry::clone(¤t.registry); for spec in installed { - next.push_built(spec, add)?; + next.install_built(spec, add, install)?; } next }; diff --git a/litellm-rust/crates/router-core/src/registry.rs b/litellm-rust/crates/router-core/src/registry.rs index 594aff5f6dd..2aff7f243ff 100644 --- a/litellm-rust/crates/router-core/src/registry.rs +++ b/litellm-rust/crates/router-core/src/registry.rs @@ -626,6 +626,27 @@ impl Registry { } } + pub fn install_built( + &mut self, + spec: DeploymentSpec, + add: bool, + install: crate::build::Install, + ) -> Result<(), ConfigError> { + if install == crate::build::Install::Upsert { + let id = self.carried(spec.clone())?.id; + if self.by_id.contains_key(&id) { + *self = self.apply( + &ChangeSet { + remove: Some(id), + ..ChangeSet::default() + }, + None, + )?; + } + } + self.push_built(spec, add) + } + /// Add one deployment a build accepted at the end and index it in place (`add` is /// `add_deployment`, which also names its group). A build installs into a registry of its /// own this way, so a list of n deployments builds in O(n) rather than reindexing per @@ -1531,6 +1552,60 @@ mod tests { assert_eq!((resolution.kind, resolution.group.as_str()), (kind, group)); } + #[rstest] + #[case::empty(false)] + #[case::replacement(true)] + fn installing_an_upsert_replaces_only_the_working_snapshot(#[case] populated: bool) { + let registry = Registry::build( + if populated { + vec![spec( + "a", + json!({"model": "openai/old"}), + json!({"id": "1"}), + )] + } else { + Vec::new() + }, + BTreeMap::new(), + Vec::new(), + ) + .unwrap(); + let mut working = registry.clone(); + working + .install_built( + spec("a", json!({"model": "openai/new"}), json!({"id": "1"})), + true, + crate::build::Install::Upsert, + ) + .unwrap(); + assert_eq!(registry.all().len(), usize::from(populated)); + assert_eq!(working.all().len(), 1); + assert_eq!(working.all()[0].raw_params["model"], json!("openai/new")); + assert_eq!(working.host.position_of("1"), Some(0)); + } + + #[rstest] + fn an_invalid_upsert_keeps_the_previous_deployment() { + let mut registry = Registry::build( + vec![spec( + "a", + json!({"model": "openai/old"}), + json!({"id": "1"}), + )], + BTreeMap::new(), + Vec::new(), + ) + .unwrap(); + let invalid = spec("a", json!({"model": 42}), json!({"id": "1"})); + assert!( + registry + .install_built(invalid, true, crate::build::Install::Upsert) + .is_err() + ); + assert_eq!(registry.all().len(), 1); + assert_eq!(registry.all()[0].raw_params["model"], json!("openai/old")); + } + #[rstest] fn apply_moves_an_edited_deployment_to_the_end() { let registry = Registry::build( diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 38b814225ce..7846922bae2 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -4043,8 +4043,8 @@ async def clear_cache() -> ReconcileOutcome: raise_if_reload_degraded_serving. Runs under DB_RECONCILE_LOCK for its whole extent, not just the reload at the - end, so the auto-router reset and the reload that rebuilds those routers are atomic - to any other reconcile. The inner call is _add_deployment_locked because + end, so concurrent model writes cannot interleave their reconciles. + The inner call is _add_deployment_locked because add_deployment would re-acquire the same non-reentrant lock and deadlock. """ from litellm.proxy.proxy_server import ( @@ -4062,75 +4062,22 @@ async def clear_cache() -> ReconcileOutcome: async with DB_RECONCILE_LOCK: try: - # Only clear DB models, preserve config models - verbose_proxy_logger.debug("Clearing only DB models, preserving config models") + verbose_proxy_logger.debug("Reconciling DB models, preserving config models") # Get current models and filter out DB models current_models: Final = llm_router.registry().all().copy() config_models: Final = [] db_model_ids: Final = [] - db_router_names: Final = set() - for model in as_object_list(current_models) or (): deployment = decoded_json_object(model) model_info = decoded_json_object(deployment.get("model_info", {})) if model_info.get("db_model", False): db_model_ids.append(model_info.get("id")) - # Auto-router deployments (and only those) are wiped here, in the - # same pass, so the reload rebuilds them -- see the comment below. - deployment_name = deployment.get("model_name") - if isinstance(deployment_name, str) and str( - decoded_json_object(deployment.get("litellm_params", {})).get("model", "") - ).startswith("auto_router/"): - db_router_names.add(deployment_name) - router_model_id = model_info.get("id") - if isinstance(router_model_id, str): - llm_router.apply(ChangeSet(delete=[router_model_id])) else: # This is a config model, preserved by the reconcile below config_models.append(model) - # ORDINARY db deployments are deliberately NOT wiped. This used to - # delete_deployment() every db model before the reload put them back, which - # left the router serving ZERO db models for the whole width of the reload - # -- a real data-plane hole that every inference request landing in it fell - # into. It was also redundant for them: the reload's _delete_deployment - # evicts exactly the ids the db no longer lists, and upsert_deployment - # pops-and-re-adds a deployment whose params changed while no-opping one - # that did not, so the reconcile converges on its own. Every mutation is - # visible to that comparison -- `blocked` and (for premium) `updated_at` - # are written into model_info. - # - # AUTO-ROUTER db deployments are the exception and ARE wiped -- in the - # classification pass above, together with the strategy entries popped - # just below. Their strategy registries are keyed - # by model_name, which no deployment-id reconcile touches, so they have to - # be popped and rebuilt here. But the rebuild only happens on the ADD path: - # Router.upsert_deployment returns early when a deployment is unchanged and - # never reaches add_deployment -> _add_deployment -> - # init_auto_router_deployment, which is what repopulates the registries. - # Popping without deleting would therefore strip every db-backed auto, - # complexity, adaptive and quality router on this pod and never put it back, - # so ANY unrelated model write would leave them unroutable until a restart. - # Deleting the deployment forces upsert down the add path, which rebuilds - # both the deployment and its strategy entry. - # - # That pass restricts the wipe to deployments whose model is actually an - # auto_router/* so a config router that merely shares a model_name with a - # regular db model isn't evicted -- config routers are never re-added by the - # reload (it only reloads db models) and would be permanently unroutable. - # The auto_router/ prefix also covers quality_router/ and adaptive_router/, - # so pop the name from every registry (no-op where absent); a missing - # quality/adaptive entry would otherwise make init raise "already exists" - # on reload and abort it. - for model_name in db_router_names: - llm_router.registry().plugins("auto").pop(model_name, None) - llm_router.registry().plugins("complexity").pop(model_name, None) - llm_router.registry().plugins("adaptive").pop(model_name, None) - llm_router.registry().plugins("quality").pop(model_name, None) - llm_router.registry().plugins("switchyard").pop(model_name, None) - # Reload only DB models. _add_deployment_locked, not add_deployment: this # coroutine already holds DB_RECONCILE_LOCK and asyncio.Lock is not # reentrant, so the public wrapper would deadlock against itself. diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9f721e3fdcb..23d45e07dfa 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9024,9 +9024,8 @@ class ProxyConfig: proxy_logging_obj: ProxyLogging, ) -> ReconcileOutcome: """add_deployment's body, minus the locking. DB_RECONCILE_LOCK MUST already - be held. Split out for the one caller that has to hold the lock across more than - this reconcile -- clear_cache, which un-serves every db model before calling it - and would deadlock on a re-acquire.""" + be held. Split out for clear_cache, which already holds the lock and would + deadlock on a re-acquire.""" global llm_router, llm_model_list, master_key, general_settings still_desired_ids: frozenset[str] | None = None diff --git a/litellm/router_rust/steps/prerouting.py b/litellm/router_rust/steps/prerouting.py index 47bd1e69046..2953a87ed92 100644 --- a/litellm/router_rust/steps/prerouting.py +++ b/litellm/router_rust/steps/prerouting.py @@ -133,9 +133,13 @@ def _routing_definition_fingerprint(deployment: Deployment) -> str | None: return deployment_id(deployment.model_name, {"routing_definition": definition}) -def _unregister(registry: StrategyRegistry, model_name: str, tags: tuple[str, ...]) -> bool: +def _unregister( + registry: StrategyRegistry, model_name: str, tags: tuple[str, ...], previous: Deployment | None = None +) -> bool: existing: Final = registry.get(model_name, []) - remaining: Final = [entry for entry in existing if entry.tags != tags] + remaining: Final = [ + entry for entry in existing if entry.tags != tags or (previous is not None and entry.deployment != previous) + ] if len(remaining) == len(existing): return False if remaining: @@ -217,30 +221,31 @@ class PreRoutingStrategies: self._hosted._core_auto_routers = {} # pyright: ignore[reportPrivateUsage] # see above self._hosted._core_switchyard_routers = {} # pyright: ignore[reportPrivateUsage] # see above - def register(self, deployment: Deployment) -> None: + def register(self, deployment: Deployment, *, replace_existing: bool = False) -> None: """The end of ``_add_deployment``: an auto, complexity or quality marker registers its router (adaptive ones wait for ``finalize``).""" match classify_strategy_router_model(deployment.litellm_params.model): case "semantic": - self._init_auto_router(deployment) + self._init_auto_router(deployment, replace_existing=replace_existing) case "complexity": - self._init_complexity_router(deployment) + self._init_complexity_router(deployment, replace_existing=replace_existing) case "quality": - self._init_quality_router(deployment) + self._init_quality_router(deployment, replace_existing=replace_existing) case "switchyard": - self._init_switchyard_router(deployment) + self._init_switchyard_router(deployment, replace_existing=replace_existing) case _: pass - def release(self, deployment: Deployment) -> None: + def release(self, deployment: Deployment, *, replaced: bool = False) -> None: """``_unregister_pre_routing_strategy_for_deployment``.""" if not deployment.litellm_params.model.startswith("auto_router/"): return model_name: Final = deployment.model_name tags: Final = _deployment_tags(deployment) + previous: Final = deployment if replaced else None for registry in (self.auto, self.complexity, self.quality, self.switchyard): - _ = _unregister(registry, model_name, tags) - if _unregister(self.adaptive, model_name, tags): + _ = _unregister(registry, model_name, tags, previous) + if _unregister(self.adaptive, model_name, tags, previous): self._sync_adaptive_router_hooks() def release_entry(self, entry: Mapping[str, object]) -> None: @@ -261,15 +266,6 @@ class PreRoutingStrategies: ): self.finalize() - def without_capability_limit(self, action: Callable[[], object]) -> None: - """Run ``action`` without judging the capability ceiling: a rollback re-admits a serving router.""" - resolver: Final = self._capability_limit - self._capability_limit = None - try: - _ = action() - finally: - self._capability_limit = resolver - def finalize(self) -> None: """``_finalize_adaptive_router_if_configured``: an adaptive router for every adaptive marker, and the adaptive companion of every complexity router configured with ``adaptive``.""" @@ -328,16 +324,30 @@ class PreRoutingStrategies: # -- building -------------------------------------------------------------------------------- def _register( - self, registry: StrategyRegistry, deployment: Deployment, strategy: object, strategy_label: str + self, + registry: StrategyRegistry, + deployment: Deployment, + strategy: object, + strategy_label: str, + *, + replace_existing: bool = False, ) -> None: tags: Final = _deployment_tags(deployment) - if _has_registered_strategy(registry, deployment.model_name, tags): + existing: Final = registry.get(deployment.model_name, []) + remaining: Final = [ + entry + for entry in existing + if not replace_existing + or entry.deployment is None + or entry.deployment.model_info.id != deployment.model_info.id + ] + if any(entry.tags == tags for entry in remaining): raise ValueError( f"{strategy_label} deployment {deployment.model_name} with tags {list(tags)} already exists. " "Please use a different model name or set different tags." ) registry[deployment.model_name] = [ # rebind-ok: the live registry, edited in place as the Python router does - *registry.get(deployment.model_name, []), + *remaining, TaggedPreRoutingStrategy( tags=tags, strategy=strategy, @@ -346,17 +356,22 @@ class PreRoutingStrategies: ), ] - def _capability_violation(self, capability: GatedAutoRouterCapability) -> str | None: + def _capability_violation(self, capability: GatedAutoRouterCapability, deployment: Deployment) -> str | None: resolver: Final = self._capability_limit # `self.auto_router_capability_limit() if ... is not None`: a value that cannot be called raises. limit: Final = resolver() if resolver is not None else None # pyright: ignore[reportCallIssue] # called as given, as Python calls it others: Final = count_capability_routers( - (entry for entry in self._hosted._core_model_list if isinstance(entry, Mapping)), # pyright: ignore[reportPrivateUsage, reportUnnecessaryIsInstance] # the live list + ( + entry + for entry in self._hosted._core_model_list # pyright: ignore[reportPrivateUsage] # the live list + if isinstance(entry, Mapping) # pyright: ignore[reportUnnecessaryIsInstance] # stored entries are mappings + and as_dict(entry.get("model_info")).get("id") != deployment.model_info.id + ), capability=capability, ) return capability_limit_violation(capability=capability, held=others + 1, limit=limit) # pyright: ignore[reportArgumentType] # the resolver's limit - def _init_auto_router(self, deployment: Deployment) -> None: + def _init_auto_router(self, deployment: Deployment, *, replace_existing: bool = False) -> None: from litellm.router_strategy.auto_router.auto_router import AutoRouter params: Final = deployment.litellm_params @@ -382,15 +397,15 @@ class PreRoutingStrategies: ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext max_input_chars=max_input_chars if max_input_chars is not None else DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ) - self._register(self.auto, deployment, router, "Auto-router") + self._register(self.auto, deployment, router, "Auto-router", replace_existing=replace_existing) - def _init_complexity_router(self, deployment: Deployment) -> None: + def _init_complexity_router(self, deployment: Deployment, *, replace_existing: bool = False) -> None: from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig config: Final = deployment.litellm_params.complexity_router_config for capability in claimed_capabilities(config): - if (violation := self._capability_violation(capability)) is not None: + if (violation := self._capability_violation(capability, deployment)) is not None: raise ValueError(violation) default_model: Final = ( ComplexityRouterConfig.model_validate(config).resolve_default_model( @@ -410,7 +425,7 @@ class PreRoutingStrategies: ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext complexity_router_config=config, ) - self._register(self.complexity, deployment, router, "Complexity-router") + self._register(self.complexity, deployment, router, "Complexity-router", replace_existing=replace_existing) if router._uses_deployment_pin: # pyright: ignore[reportPrivateUsage] # the Python router reads the same flag self._ensure_deployment_affinity_callback() @@ -451,7 +466,7 @@ class PreRoutingStrategies: self._affinity = affinity self._host.callbacks.add_litellm_callback(affinity) - def _init_quality_router(self, deployment: Deployment) -> None: + def _init_quality_router(self, deployment: Deployment, *, replace_existing: bool = False) -> None: from litellm.router_strategy.quality_router.quality_router import QualityRouter config: Final = deployment.litellm_params.quality_router_config @@ -468,9 +483,9 @@ class PreRoutingStrategies: ctx=self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext quality_router_config=config, ) - self._register(self.quality, deployment, router, "Quality-router") + self._register(self.quality, deployment, router, "Quality-router", replace_existing=replace_existing) - def _init_switchyard_router(self, deployment: Deployment) -> None: + def _init_switchyard_router(self, deployment: Deployment, *, replace_existing: bool = False) -> None: from litellm.router_strategy.switchyard_router.config import SwitchyardRouterConfig, build_algorithm from litellm.router_strategy.switchyard_router.switchyard_router import SwitchyardRouter from litellm.types.decisions import OpenAIDecisionResponse @@ -494,7 +509,7 @@ class PreRoutingStrategies: return result router: Final = SwitchyardRouter(deployment.model_name, config, build_algorithm(config), complete_judge, decide) - self._register(self.switchyard, deployment, router, "Switchyard") + self._register(self.switchyard, deployment, router, "Switchyard", replace_existing=replace_existing) def _init_adaptive_router(self, deployment: Deployment) -> None: from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py index 83008bdbb94..59d3d7389be 100644 --- a/litellm/router_rust/surface/registry.py +++ b/litellm/router_rust/surface/registry.py @@ -215,6 +215,7 @@ class _Build: entries: dict[int, _BuildEntry] = dataclasses.field(default_factory=dict) # mutable-ok: filled in step order # `add`: the deployment to add, and the entry it is stored as. deployment: Deployment | None = None + replace_existing: bool = False stored: UntypedEntry | None = None # The deployment the build is on, its provider and its data-source values. pending: Deployment | None = None @@ -1007,12 +1008,13 @@ class HostRegistry(HostModelInfo): raise RuntimeError("the host registry builds deployments through its Rust core") for entry in entries: refuse_wide_numbers(entry) - self._core_model_list.begin_build(replacing=install == "replace") - try: - self.build_runner(entries, build.path, resolve_secrets, install, partial(self._answer_shown, build)) - finally: - self._core_model_list.end_build() - self._drop_removed_live_values() + with self._deployment_index_lock: + self._core_model_list.begin_build(replacing=install == "replace") + try: + self.build_runner(entries, build.path, resolve_secrets, install, partial(self._answer_shown, build)) + finally: + self._core_model_list.end_build() + self._drop_removed_live_values() if build.failure is not None: # router-core served what it built before the entry that raised (`set_model_list` # cleared the list first), so the router is left as the Python core leaves it. @@ -1220,7 +1222,11 @@ class HostRegistry(HostModelInfo): deployment: Final = build.deployment if deployment is None: raise RuntimeError("an add build holds its deployment") - if deployment.model_info.id and self._core_has_model_id(deployment.model_info.id): + if ( + not build.replace_existing + and deployment.model_info.id + and self._core_has_model_id(deployment.model_info.id) + ): return {"skip": True} declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True) warn_on_provider_credential_mismatch(model_name=deployment.model_name, litellm_params=declared_params) @@ -1273,7 +1279,9 @@ class HostRegistry(HostModelInfo): deployment: Final = build.pending if deployment is None: raise RuntimeError("an added step follows a validated deployment") - self._register_deployment(deployment, build.provider, build.data_source_values) + self._register_deployment( + deployment, build.provider, build.data_source_values, replace_existing=build.replace_existing + ) build.pending = None # rebind-ok: the build's state, which each step advances build.data_source_values = () # rebind-ok: the build's state, which each step advances self._invalidate_model_group_info_cache() @@ -1319,6 +1327,8 @@ class HostRegistry(HostModelInfo): deployment: Deployment, custom_llm_provider: str | None, data_source_values: Sequence[tuple[int, str, str]], + *, + replace_existing: bool, ) -> None: """``_add_deployment`` after the provider: its data-source values, pass-through credentials and pre-routing strategies (router-core registers its names and patterns).""" @@ -1366,7 +1376,7 @@ class HostRegistry(HostModelInfo): if custom_llm_provider is not None: self._initialize_deployment_for_pass_through(deployment, custom_llm_provider) if self.strategies is not None: - self.strategies.register(deployment) + self.strategies.register(deployment, replace_existing=replace_existing) def _initialize_deployment_for_pass_through(self, deployment: Deployment, custom_llm_provider: str) -> None: """Register vertex credentials for a pass-through deployment (``_initialize_deployment_for_pass_through``).""" @@ -1395,7 +1405,10 @@ class HostRegistry(HostModelInfo): def add_deployment(self, deployment: Deployment) -> Deployment | None: """Add one deployment unless its id exists (``add_deployment``), built by router-core.""" - build: Final = _Build(path="add", deployment=deployment) + return self._install_deployment(deployment, replace_existing=False) + + def _install_deployment(self, deployment: Deployment, *, replace_existing: bool) -> Deployment | None: + build: Final = _Build(path="add", deployment=deployment, replace_existing=replace_existing) self._run_build(build, [deployment.to_json(exclude_none=True)], resolve_secrets=False, install="upsert") if build.result is None: return None @@ -1410,62 +1423,38 @@ class HostRegistry(HostModelInfo): on_router: Final = self._core_get_deployment(model_id=model_id) served: Final = without_model_capability_flags(deployment) try: + if ( + on_router is not None + and deployment.model_name == on_router.model_name + and served.litellm_params == on_router.litellm_params + and deployment.model_info == on_router.model_info + ): + return None + self._install_deployment(deployment, replace_existing=True) if on_router is not None: - if ( - deployment.model_name == on_router.model_name - and served.litellm_params == on_router.litellm_params - and deployment.model_info == on_router.model_info - ): - return None - with self._deployment_index_lock: - removal_idx: Final = self.model_id_to_deployment_index_map.get(model_id) - if removal_idx is not None: - removed: Final = self._core_model_list[removal_idx] # pyright: ignore[reportArgumentType] # raises IndexError for a position past the list, as the pop does - self._remove(model_id) - withdraw_shared_capabilities( - self._host, removed, deployment.to_json(exclude_none=True), self._served_by_live_routers() - ) + withdraw_shared_capabilities( + self._host, + on_router.to_json(exclude_none=True), + deployment.to_json(exclude_none=True), + self._served_by_live_routers(), + ) if self.strategies is not None: - self.strategies.release(on_router) - self._on_change() - self.add_deployment(deployment=deployment) + self.strategies.release(on_router, replaced=True) if self.strategies is not None: self.strategies.upserted(deployment, on_router) except Exception as e: if not self.ignore_invalid_deployments: raise verbose_router_logger.warning( - "Error upserting deployment %s (id=%s): %s. Dropping it and continuing with other deployments.", + "Error upserting deployment %s (id=%s): %s. Keeping its previous deployment, if any.", deployment.model_name, deployment.model_info.id, e, ) - self._restore_deployment_after_failed_upsert(on_router, model_id) return None else: return deployment - def _restore_deployment_after_failed_upsert(self, previous: Deployment | None, model_id: str) -> None: - if previous is None or self._core_has_model_id(model_id): - return - try: - if self.strategies is None: - self.add_deployment(deployment=previous) - else: - self.strategies.without_capability_limit(lambda: self.add_deployment(deployment=previous)) - verbose_router_logger.info( - "Restored deployment %s (id=%s); it keeps serving its previous configuration.", - previous.model_name, - model_id, - ) - except Exception as restore_error: # noqa: BLE001 # best-effort restore: a second failure must not abort the reload - verbose_router_logger.warning( - "Could not restore previously served deployment %s (id=%s) after the failed upsert: %s", - previous.model_name, - model_id, - restore_error, - ) - def _live_handle(self) -> "RouterCoreHandle | None": return None if self.handle_lookup is None else self.handle_lookup() diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 2b269469f89..2f8456c6d0d 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -1113,7 +1113,7 @@ class TestClearCache: @pytest.mark.asyncio async def test_clear_cache_preserve_config_models(self): """ - clear_cache resets DB-backed auto-router entries and delegates every deployment + clear_cache preserves DB-backed auto-router entries and delegates every deployment change to the reload, leaving config models untouched. It must not wipe deployments itself -- see the delete_deployment assertion below. """ @@ -1122,8 +1122,7 @@ class TestClearCache: ) # Create mock router with mixed DB and config router deployments. The two DB - # entries are auto_router/* deployments (so their router-map entries should be - # cleared for reload); the config-defined router is preserved. + # entries and the config-defined router must remain available during reload. mock_router = MagicMock() plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ @@ -1165,20 +1164,9 @@ class TestClearCache: ): await clear_cache() - # clear_cache must wipe ONLY the db auto-router deployments -- the ones whose - # strategy entries are popped below and can only be rebuilt via the add path. - # Ordinary db models are left alone: wiping them un-served every db model for - # the width of the reload, and the reconcile converges without it. - assert mock_router.apply.call_count == 2 - mock_router.apply.assert_any_call(ChangeSet(delete=["db-model-1"])) - mock_router.apply.assert_any_call(ChangeSet(delete=["db-model-2"])) - - # DB-backed router entries are cleared so they can be re-populated by the - # reload below; the config-backed router must survive, since add_deployment() - # only reloads DB models and would otherwise leave it permanently unroutable - # (see TestClearCachePreservesConfigRouters). - assert "db-auto-router" not in plugins["auto"] - assert "db-complexity-router" not in plugins["complexity"] + mock_router.apply.assert_not_called() + assert "db-auto-router" in plugins["auto"] + assert "db-complexity-router" in plugins["complexity"] assert "config-router" in plugins["auto"] assert "config-router" in plugins["complexity"] @@ -1188,22 +1176,8 @@ class TestClearCache: ) @pytest.mark.asyncio - async def test_clear_cache_wipes_auto_routers_but_leaves_ordinary_db_models(self): - """An ordinary db model must survive clear_cache; a db auto-router must not. - - Two separate hazards meet here, and fixing one naively breaks the other: - - - Wiping ordinary db models un-serves EVERY db model for the width of the - reload. The reconcile converges without that, so the wipe is a pure - data-plane hole. - - NOT wiping a db auto-router strands it. Its strategy registries are keyed by - model_name and are popped here, but Router.upsert_deployment returns early - for an unchanged deployment and never reaches the add path that rebuilds - them. Any unrelated model write would then leave every db-backed auto, - complexity, adaptive and quality router unroutable until a restart. - - So the wipe is scoped to exactly the auto-router deployments. - """ + async def test_clear_cache_preserves_auto_routers_and_ordinary_db_models(self): + """Deployments and their strategies remain available while the reload runs.""" from litellm.proxy.management_endpoints.model_management_endpoints import ( clear_cache, ) @@ -1242,10 +1216,8 @@ class TestClearCache: ): await clear_cache() - # The auto-router deployment is wiped so the reload takes the add path and - # rebuilds its strategy entry; the ordinary db model is never touched. - mock_router.apply.assert_called_once_with(ChangeSet(delete=["db-auto-1"])) - assert "db-auto-router" not in plugins["auto"] + mock_router.apply.assert_not_called() + assert "db-auto-router" in plugins["auto"] class TestClearCachePreservesConfigRouters: @@ -1299,9 +1271,7 @@ class TestClearCachePreservesConfigRouters: ): await clear_cache() - # The DB-backed router for the model that was actually updated is cleared - # so the reload below can re-populate it. - assert "team-a-db-router" not in plugins["complexity"] + assert "team-a-db-router" in plugins["complexity"] # Config-defined routers for unrelated tenants must survive untouched. assert "config-defined-complexity-router" in plugins["complexity"] assert "config-semantic-router" in plugins["auto"] @@ -1346,11 +1316,8 @@ class TestClearCachePreservesConfigRouters: assert "shared-name" in plugins["complexity"] @pytest.mark.asyncio - async def test_db_quality_adaptive_and_switchyard_routers_are_evicted(self): - """The auto_router/ prefix also covers quality_router/, adaptive_router/ and switchyard. Their - registry entries must be popped too, or reload's init raises 'already exists' - (quality) or leaves a stale entry (adaptive). - """ + async def test_db_quality_adaptive_and_switchyard_routers_are_preserved(self): + """Every strategy remains available until the deployment reconcile replaces it.""" from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache mock_router = MagicMock() @@ -1393,9 +1360,9 @@ class TestClearCachePreservesConfigRouters: ): await clear_cache() - assert "q1" not in plugins["quality"] - assert "a1" not in plugins["adaptive"] - assert "s1" not in plugins["switchyard"] + assert "q1" in plugins["quality"] + assert "a1" in plugins["adaptive"] + assert "s1" in plugins["switchyard"] class TestDeleteModelClearsRouterRegistry: @@ -6136,8 +6103,7 @@ class TestConcurrentModelWritesDoNotEvictEachOther: @pytest.mark.asyncio async def test_clear_cache_reloads_under_the_lock_without_deadlocking(self, monkeypatch): - """clear_cache un-serves every db model before reloading, so it has to hold the - lock across the pair -- and therefore must call the already-locked reload. + """clear_cache holds the reconcile lock and must call the already-locked reload. asyncio.Lock is not reentrant: routing this back through the public add_deployment would block forever on a lock this coroutine already owns, taking diff --git a/tests/unit/router_rust/test_atomic_upsert.py b/tests/unit/router_rust/test_atomic_upsert.py new file mode 100644 index 00000000000..51050b0002d --- /dev/null +++ b/tests/unit/router_rust/test_atomic_upsert.py @@ -0,0 +1,89 @@ +from collections.abc import Callable +from types import SimpleNamespace +from typing import Final + +import pytest + +from litellm.proxy import proxy_server +from litellm.proxy._types import ReconcileOutcome +from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.router_python_host import Endpoint +from tests.unit.test_router.core.registry_helpers import typed +from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rust_core + + +class ObservingHost(ScriptedHost): + def __init__(self) -> None: + super().__init__() + self.observe: Callable[[], None] | None = None + + def invoke(self, op: str) -> Endpoint: + endpoint: Final = super().invoke(op) + if op != "get_llm_provider": + return endpoint + + def provider(*args: object, **kwargs: object) -> object: # kwargs-ok: provider resolver arguments + if self.observe is not None: + self.observe() + return endpoint(*args, **kwargs) + + return provider + + +@pytest.mark.asyncio +async def test_upsert_keeps_the_previous_deployment_serving_while_resolving_its_replacement() -> None: + host: Final = ObservingHost() + core: Final[RouterCore] = rust_core(host, model_list=[deployment("group", "only")]) + seen: Final[list[tuple[str, ...]]] = [] # mutable-ok: observations made by the injected provider resolver + + def observe() -> None: + seen.append(tuple(str(item["model_info"]["id"]) for item in core.registry().deployments("group"))) + + host.observe = observe + result: Final = core.apply(ChangeSet(upsert=[typed("group", "only", api_base="https://replacement.example")])) + + assert seen and set(seen) == {("only",)}, seen + assert list(result.upserted) == [typed("group", "only", api_base="https://replacement.example")] + await core.acompletion(model="group", messages=[{"role": "user", "content": "after replacement"}]) # pyright: ignore[reportAttributeAccessIssue] # the core's completion endpoint + assert host.deployments_called("acompletion") == ["only"] + + +@pytest.mark.asyncio +async def test_clear_cache_keeps_a_db_classifier_serving_while_the_reload_waits( + monkeypatch: pytest.MonkeyPatch, +) -> None: + host: Final = ScriptedHost() + core: Final = rust_core( + host, + model_list=[ + deployment("tier", "tier"), + deployment( + "smart", + "router", + model="auto_router/complexity_router", + complexity_router_config={ + "classifier_type": "heuristic_v2", + "tiers": {"SIMPLE": "tier", "MEDIUM": "tier", "COMPLEX": "tier", "REASONING": "tier"}, + }, + model_info={"db_model": True}, + ), + ], + ) + seen: Final[list[tuple[str, ...]]] = [] # mutable-ok: snapshots at the reload boundary + + async def reload(**kwargs: object) -> ReconcileOutcome: # kwargs-ok: the config reload's injected I/O boundary + seen.append(tuple(str(item["model_info"]["id"]) for item in core.registry().deployments("smart"))) + await core.acompletion(model="smart", messages=[{"role": "user", "content": "hello"}]) # pyright: ignore[reportAttributeAccessIssue] # the core's completion endpoint + return ReconcileOutcome(still_desired=frozenset({"router"}), live_after=frozenset({"router"})) + + monkeypatch.setattr(proxy_server, "llm_router", core) + monkeypatch.setattr(proxy_server, "prisma_client", object()) + monkeypatch.setattr(proxy_server, "proxy_config", SimpleNamespace(add_deployment_locked=reload)) + + outcome: Final = await clear_cache() + + assert seen == [("router",)] + assert outcome == ReconcileOutcome(still_desired=frozenset({"router"}), live_after=frozenset({"router"})) + assert host.deployments_called("acompletion") == ["tier"] diff --git a/tests/unit/test_router/core/test_gaps_l11100_strategy.py b/tests/unit/test_router/core/test_gaps_l11100_strategy.py index 2459c577b9c..c311459c4e5 100644 --- a/tests/unit/test_router/core/test_gaps_l11100_strategy.py +++ b/tests/unit/test_router/core/test_gaps_l11100_strategy.py @@ -300,7 +300,7 @@ def test_a_failed_edit_rolls_back_to_the_serving_router_even_above_a_tightened_l assert restored.litellm_params.tags is None -def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_leaves_the_router_removed( +def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_keeps_the_router_serving( make_core: MakeCore, ) -> None: limit: Final = {"value": 2} @@ -319,8 +319,8 @@ def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_leaves_the_ ChangeSet(upsert=[Deployment(**complexity_marker("two", "m2", config=HEURISTIC_V2, tags=["edited"]))]) # pyright: ignore[reportArgumentType] ) assert str(raised.value) == HEURISTIC_V2_LIMIT_MESSAGE - assert model_ids(core) == ["s", "b", "m1"] - assert registered_tags(core, "complexity") == {"one": [()]} + assert model_ids(core) == ["s", "b", "m1", "m2"] + assert registered_tags(core, "complexity") == {"one": [()], "two": [()]} # --- the adaptive router ------------------------------------------------------------------------------- diff --git a/tests/unit/test_router/core/test_registry_characterization_writes.py b/tests/unit/test_router/core/test_registry_characterization_writes.py index 23fa2a8e0e9..ecb6aa81832 100644 --- a/tests/unit/test_router/core/test_registry_characterization_writes.py +++ b/tests/unit/test_router/core/test_registry_characterization_writes.py @@ -158,14 +158,13 @@ def test_an_upsert_that_empties_a_group_drops_the_group_name(make_core: MakeCore assert core.registry().deployments("emb") == [] -def test_a_failing_upsert_of_an_existing_deployment_raises_and_leaves_it_removed(make_core: MakeCore) -> None: - """Odd but current: without ignore_invalid_deployments the old deployment is popped and not restored.""" +def test_a_failing_upsert_of_an_existing_deployment_raises_and_keeps_it_serving(make_core: MakeCore) -> None: core = _core(make_core) with pytest.raises(litellm.BadRequestError): core.apply(ChangeSet(upsert=[typed("gpt", "a", model="notaprovider/x")])) - assert core.registry().names(kind="ids") == ["b", "e", "w", "t1"] + assert core.registry().names(kind="ids") == ["a", "b", "e", "w", "t1"] def test_a_failing_upsert_under_ignore_invalid_deployments_restores_the_previous_deployment( @@ -179,7 +178,7 @@ def test_a_failing_upsert_under_ignore_invalid_deployments_restores_the_previous restored = core.registry().deployment("a") assert restored is not None assert restored.litellm_params.model == "openai/a" - assert core.registry().names(kind="ids") == ["b", "e", "w", "t1", "a"] + assert core.registry().names(kind="ids") == ["a", "b", "e", "w", "t1"] def test_the_deprecated_upsert_deployment_warns_and_returns_the_deployment(make_core: MakeCore) -> None: @@ -540,7 +539,7 @@ def test_a_read_on_another_thread_waits_for_an_upsert_to_read_the_list_back(make past the list nor, after a removal, the neighbouring group's entries.""" core: Final = _two_per_group(make_core) hosted: Final = core._hosted # pyright: ignore[reportAttributeAccessIssue] # the registry under test - write: Final[Callable[[Mapping[str, object]], None]] = hosted.changes_runner + write: Final = hosted.build_runner started: Final = threading.Event() pool: Final = ThreadPoolExecutor(max_workers=1) reads: Final[list[object]] = [] # mutable-ok: one per write @@ -549,25 +548,30 @@ def test_a_read_on_another_thread_waits_for_an_upsert_to_read_the_list_back(make started.set() return _group_ids(core) - def write_with_a_read_started(changes: Mapping[str, object]) -> None: - write(changes) + def write_with_a_read_started( + entries: Sequence[object], + path: str, + resolve_secrets: bool, + install: str, + answer: Callable[[Mapping[str, object]], object], + ) -> None: + write(entries, path, resolve_secrets, install, answer) started.clear() reads.append(pool.submit(read)) assert started.wait(timeout=5) # The reader runs while router-core holds the write and the mirror does not yet. _ = threading.Event().wait(0.02) - hosted.changes_runner = write_with_a_read_started + hosted.build_runner = write_with_a_read_started try: _ = core.apply(ChangeSet(upsert=[typed(f"m{upserted % _GROUPS}", f"d{upserted}", timeout=11)])) finally: - hosted.changes_runner = write + hosted.build_runner = write assert reads for future in reads: seen = future.result(timeout=5) # pyright: ignore[reportAttributeAccessIssue] # a Future for group in range(_GROUPS): - # Between the upsert's removal and its add, its group is served by the other deployment. assert seen[group], (group, seen) assert set(seen[group]) <= {f"d{group}", f"d{group + _GROUPS}"}, (group, seen) pool.shutdown() From 8fb9f2f0907596e689808c76b746494d93cc759d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:52:17 +0000 Subject: [PATCH 098/189] chore(cost-map): add 24 runwayml models and correct seedance2 and eleven_multilingual_v2 rates (#51216) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 369 +++++++++++++++++- model_prices_and_context_window.json | 369 +++++++++++++++++- 2 files changed, 706 insertions(+), 32 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 66e3a094879..b6eec1c9574 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -56675,7 +56675,7 @@ "runwayml/seedance2": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_second": 0.36, + "output_cost_per_second": 0.3, "output_cost_per_second_1080p": 0.4, "output_cost_per_second_4k": 1.5, "source": "https://docs.dev.runwayml.com/guides/pricing/", @@ -56687,14 +56687,13 @@ "supported_output_modalities": [ "video" ], - "metadata": { - "comment": "36 credits per second at 480p/720p, 40 at 1080p, 150 at 4K @ $0.01 per credit" - } + "metadata": {"comment":"14/30/40/150 credits per second at 480p/720p/1080p/4K @ $0.01 per credit; 56 credit minimum per generation not modeled"}, + "output_cost_per_second_480p": 0.14 }, "runwayml/seedance2_fast": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_second": 0.29, + "output_cost_per_second": 0.24, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -56704,14 +56703,13 @@ "supported_output_modalities": [ "video" ], - "metadata": { - "comment": "29 credits per second at 480p/720p @ $0.01 per credit = $0.29 per second" - } + "metadata": {"comment":"11 credits per second at 480p, 24 at 720p @ $0.01 per credit; 44 credit minimum per generation not modeled"}, + "output_cost_per_second_480p": 0.11 }, "runwayml/seedance2_mini": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_second": 0.16, + "output_cost_per_second": 0.15, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -56721,9 +56719,8 @@ "supported_output_modalities": [ "video" ], - "metadata": { - "comment": "16 credits per second @ $0.01 per credit = $0.16 per second; 64 credit minimum per task not modeled" - } + "metadata": {"comment":"7 credits per second at 480p, 15 at 720p @ $0.01 per credit; 28 credit minimum per generation not modeled"}, + "output_cost_per_second_480p": 0.07 }, "runwayml/seedance2_5": { "litellm_provider": "runwayml", @@ -56848,11 +56845,9 @@ "runwayml/eleven_multilingual_v2": { "litellm_provider": "runwayml", "mode": "audio_speech", - "input_cost_per_character": 3e-07, + "input_cost_per_character": 0.0002, "source": "https://docs.dev.runwayml.com/guides/pricing/", - "metadata": { - "comment": "Estimated cost based on standard TTS pricing. RunwayML uses ElevenLabs models." - } + "metadata": {"comment":"1 credit per 50 characters @ $0.01 per credit = $0.0002 per character"} }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-a35b-instruct": { "max_input_tokens": 262144, @@ -100443,5 +100438,347 @@ "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true + }, + "runwayml/kling2.5_turbo_pro": { + "litellm_provider": "runwayml", + "metadata": {"comment":"12 credits per second @ $0.01 per credit = $0.12 per second"}, + "mode": "video_generation", + "output_cost_per_second": 0.12, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/kling3.0_standard": { + "litellm_provider": "runwayml", + "metadata": {"comment":"13 credits per second with audio, 9 without @ $0.01 per credit; priced at the with-audio rate (audio defaults to true)"}, + "mode": "video_generation", + "output_cost_per_second": 0.13, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/kling3.0_pro": { + "litellm_provider": "runwayml", + "metadata": {"comment":"17 credits per second with audio, 12 without @ $0.01 per credit; priced at the with-audio rate (audio defaults to true)"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/kling3.0_4k": { + "litellm_provider": "runwayml", + "metadata": {"comment":"43 credits per second @ $0.01 per credit = $0.43 per second, with or without audio"}, + "mode": "video_generation", + "output_cost_per_second": 0.43, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/klingO3_standard": { + "litellm_provider": "runwayml", + "metadata": {"comment":"13 credits per second @ $0.01 per credit = $0.13 per second; video-to-video is billed per second of the input video, not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.13, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/klingO3_pro": { + "litellm_provider": "runwayml", + "metadata": {"comment":"17 credits per second @ $0.01 per credit = $0.17 per second; video-to-video is billed per second of the input video, not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/klingO3_4k": { + "litellm_provider": "runwayml", + "metadata": {"comment":"43 credits per second @ $0.01 per credit = $0.43 per second; video-to-video is billed per second of the input video, not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.43, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/kling_4_flash": { + "litellm_provider": "runwayml", + "metadata": {"comment":"9.4 credits per second (720p) without input video @ $0.01 per credit = $0.094 per second; the with-input-video rate (6.3 credits per output second plus 6.3 per second of the greater of input and output duration), rounding up to the next credit and the 29 credit minimum are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.094, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/wan3": { + "litellm_provider": "runwayml", + "metadata": {"comment":"5/10/20 credits per second at 480p/720p/1080p @ $0.01 per credit"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.2, + "output_cost_per_second_480p": 0.05, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/wan3_prime": { + "litellm_provider": "runwayml", + "metadata": {"comment":"6.8/14/28 credits per second at 480p/720p/1080p @ $0.01 per credit"}, + "mode": "video_generation", + "output_cost_per_second": 0.14, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_480p": 0.068, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/grok_imagine_1_5": { + "litellm_provider": "runwayml", + "metadata": {"comment":"10/16/29 credits per second at 480p/720p/1080p @ $0.01 per credit; 1 credit per image or audio reference not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.16, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_480p": 0.1, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/grok_imagine_1_5_lite": { + "litellm_provider": "runwayml", + "metadata": {"comment":"2/3/14 credits per second at 480p/720p/1080p @ $0.01 per credit; 1 credit for an image-to-video start frame not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.14, + "output_cost_per_second_480p": 0.02, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/h3_max": { + "litellm_provider": "runwayml", + "metadata": {"comment":"8 credits per second at 768p, 5 at 480p @ $0.01 per credit"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.05, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/happyhorse_1_0": { + "litellm_provider": "runwayml", + "metadata": {"comment":"15 credits per second at 720p, 30 at 1080p @ $0.01 per credit"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.3, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/gemini_omni_flash_1.2": { + "litellm_provider": "runwayml", + "metadata": {"comment":"7/10/21 credits per second at 720p/1080p/4K @ $0.01 per credit; the 360p rate (2 credits per second), 1 credit per reference image or keyframe, 1 credit per second of input video and the 6 credit minimum per request are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_1080p": 0.1, + "output_cost_per_second_4k": 0.21, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/gemini_omni_flash_1.1": { + "litellm_provider": "runwayml", + "metadata": {"comment":"10/15/30 credits per second at 720p/1080p/4K @ $0.01 per credit; the 360p rate (3.4 credits per second), 1 credit per reference image and 1 credit per second of input video are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_4k": 0.3, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/muse_image": { + "litellm_provider": "runwayml", + "metadata": {"comment":"1 credit per image (any resolution) @ $0.01 per credit = $0.01 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/seedream5_lite": { + "litellm_provider": "runwayml", + "metadata": {"comment":"4 credits per image (any resolution) @ $0.01 per credit = $0.04 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/seedream5_flash": { + "litellm_provider": "runwayml", + "metadata": {"comment":"2 credits per image (any resolution) @ $0.01 per credit = $0.02 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/gemini_image3.1_flash_lite": { + "litellm_provider": "runwayml", + "metadata": {"comment":"4 credits per 1K image @ $0.01 per credit = $0.04 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/bria_fibo_edit_1_5": { + "litellm_provider": "runwayml", + "metadata": {"comment":"5 credits per image @ $0.01 per credit = $0.05 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/gemini_2.5_flash": { + "litellm_provider": "runwayml", + "metadata": {"comment":"5 credits per image (any resolution) @ $0.01 per credit = $0.05 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/eleven_v3": { + "input_cost_per_character": 0.0002, + "litellm_provider": "runwayml", + "metadata": {"comment":"1 credit per 50 characters @ $0.01 per credit = $0.0002 per character; 1 credit minimum not modeled"}, + "mode": "audio_speech", + "source": "https://docs.dev.runwayml.com/guides/pricing/" + }, + "runwayml/eleven_v4": { + "input_cost_per_character": 5e-05, + "litellm_provider": "runwayml", + "metadata": {"comment":"5 credits per 1,000 characters @ $0.01 per credit = $0.00005 per character; 1 credit minimum not modeled"}, + "mode": "audio_speech", + "source": "https://docs.dev.runwayml.com/guides/pricing/" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 66e3a094879..b6eec1c9574 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -56675,7 +56675,7 @@ "runwayml/seedance2": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_second": 0.36, + "output_cost_per_second": 0.3, "output_cost_per_second_1080p": 0.4, "output_cost_per_second_4k": 1.5, "source": "https://docs.dev.runwayml.com/guides/pricing/", @@ -56687,14 +56687,13 @@ "supported_output_modalities": [ "video" ], - "metadata": { - "comment": "36 credits per second at 480p/720p, 40 at 1080p, 150 at 4K @ $0.01 per credit" - } + "metadata": {"comment":"14/30/40/150 credits per second at 480p/720p/1080p/4K @ $0.01 per credit; 56 credit minimum per generation not modeled"}, + "output_cost_per_second_480p": 0.14 }, "runwayml/seedance2_fast": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_second": 0.29, + "output_cost_per_second": 0.24, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -56704,14 +56703,13 @@ "supported_output_modalities": [ "video" ], - "metadata": { - "comment": "29 credits per second at 480p/720p @ $0.01 per credit = $0.29 per second" - } + "metadata": {"comment":"11 credits per second at 480p, 24 at 720p @ $0.01 per credit; 44 credit minimum per generation not modeled"}, + "output_cost_per_second_480p": 0.11 }, "runwayml/seedance2_mini": { "litellm_provider": "runwayml", "mode": "video_generation", - "output_cost_per_second": 0.16, + "output_cost_per_second": 0.15, "source": "https://docs.dev.runwayml.com/guides/pricing/", "supported_modalities": [ "text", @@ -56721,9 +56719,8 @@ "supported_output_modalities": [ "video" ], - "metadata": { - "comment": "16 credits per second @ $0.01 per credit = $0.16 per second; 64 credit minimum per task not modeled" - } + "metadata": {"comment":"7 credits per second at 480p, 15 at 720p @ $0.01 per credit; 28 credit minimum per generation not modeled"}, + "output_cost_per_second_480p": 0.07 }, "runwayml/seedance2_5": { "litellm_provider": "runwayml", @@ -56848,11 +56845,9 @@ "runwayml/eleven_multilingual_v2": { "litellm_provider": "runwayml", "mode": "audio_speech", - "input_cost_per_character": 3e-07, + "input_cost_per_character": 0.0002, "source": "https://docs.dev.runwayml.com/guides/pricing/", - "metadata": { - "comment": "Estimated cost based on standard TTS pricing. RunwayML uses ElevenLabs models." - } + "metadata": {"comment":"1 credit per 50 characters @ $0.01 per credit = $0.0002 per character"} }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-a35b-instruct": { "max_input_tokens": 262144, @@ -100443,5 +100438,347 @@ "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true + }, + "runwayml/kling2.5_turbo_pro": { + "litellm_provider": "runwayml", + "metadata": {"comment":"12 credits per second @ $0.01 per credit = $0.12 per second"}, + "mode": "video_generation", + "output_cost_per_second": 0.12, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/kling3.0_standard": { + "litellm_provider": "runwayml", + "metadata": {"comment":"13 credits per second with audio, 9 without @ $0.01 per credit; priced at the with-audio rate (audio defaults to true)"}, + "mode": "video_generation", + "output_cost_per_second": 0.13, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/kling3.0_pro": { + "litellm_provider": "runwayml", + "metadata": {"comment":"17 credits per second with audio, 12 without @ $0.01 per credit; priced at the with-audio rate (audio defaults to true)"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/kling3.0_4k": { + "litellm_provider": "runwayml", + "metadata": {"comment":"43 credits per second @ $0.01 per credit = $0.43 per second, with or without audio"}, + "mode": "video_generation", + "output_cost_per_second": 0.43, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/klingO3_standard": { + "litellm_provider": "runwayml", + "metadata": {"comment":"13 credits per second @ $0.01 per credit = $0.13 per second; video-to-video is billed per second of the input video, not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.13, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/klingO3_pro": { + "litellm_provider": "runwayml", + "metadata": {"comment":"17 credits per second @ $0.01 per credit = $0.17 per second; video-to-video is billed per second of the input video, not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.17, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/klingO3_4k": { + "litellm_provider": "runwayml", + "metadata": {"comment":"43 credits per second @ $0.01 per credit = $0.43 per second; video-to-video is billed per second of the input video, not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.43, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/kling_4_flash": { + "litellm_provider": "runwayml", + "metadata": {"comment":"9.4 credits per second (720p) without input video @ $0.01 per credit = $0.094 per second; the with-input-video rate (6.3 credits per output second plus 6.3 per second of the greater of input and output duration), rounding up to the next credit and the 29 credit minimum are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.094, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/wan3": { + "litellm_provider": "runwayml", + "metadata": {"comment":"5/10/20 credits per second at 480p/720p/1080p @ $0.01 per credit"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.2, + "output_cost_per_second_480p": 0.05, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/wan3_prime": { + "litellm_provider": "runwayml", + "metadata": {"comment":"6.8/14/28 credits per second at 480p/720p/1080p @ $0.01 per credit"}, + "mode": "video_generation", + "output_cost_per_second": 0.14, + "output_cost_per_second_1080p": 0.28, + "output_cost_per_second_480p": 0.068, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/grok_imagine_1_5": { + "litellm_provider": "runwayml", + "metadata": {"comment":"10/16/29 credits per second at 480p/720p/1080p @ $0.01 per credit; 1 credit per image or audio reference not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.16, + "output_cost_per_second_1080p": 0.29, + "output_cost_per_second_480p": 0.1, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/grok_imagine_1_5_lite": { + "litellm_provider": "runwayml", + "metadata": {"comment":"2/3/14 credits per second at 480p/720p/1080p @ $0.01 per credit; 1 credit for an image-to-video start frame not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.03, + "output_cost_per_second_1080p": 0.14, + "output_cost_per_second_480p": 0.02, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/h3_max": { + "litellm_provider": "runwayml", + "metadata": {"comment":"8 credits per second at 768p, 5 at 480p @ $0.01 per credit"}, + "mode": "video_generation", + "output_cost_per_second": 0.08, + "output_cost_per_second_480p": 0.05, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/happyhorse_1_0": { + "litellm_provider": "runwayml", + "metadata": {"comment":"15 credits per second at 720p, 30 at 1080p @ $0.01 per credit"}, + "mode": "video_generation", + "output_cost_per_second": 0.15, + "output_cost_per_second_1080p": 0.3, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/gemini_omni_flash_1.2": { + "litellm_provider": "runwayml", + "metadata": {"comment":"7/10/21 credits per second at 720p/1080p/4K @ $0.01 per credit; the 360p rate (2 credits per second), 1 credit per reference image or keyframe, 1 credit per second of input video and the 6 credit minimum per request are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.07, + "output_cost_per_second_1080p": 0.1, + "output_cost_per_second_4k": 0.21, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/gemini_omni_flash_1.1": { + "litellm_provider": "runwayml", + "metadata": {"comment":"10/15/30 credits per second at 720p/1080p/4K @ $0.01 per credit; the 360p rate (3.4 credits per second), 1 credit per reference image and 1 credit per second of input video are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.1, + "output_cost_per_second_1080p": 0.15, + "output_cost_per_second_4k": 0.3, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "runwayml/muse_image": { + "litellm_provider": "runwayml", + "metadata": {"comment":"1 credit per image (any resolution) @ $0.01 per credit = $0.01 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/seedream5_lite": { + "litellm_provider": "runwayml", + "metadata": {"comment":"4 credits per image (any resolution) @ $0.01 per credit = $0.04 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/seedream5_flash": { + "litellm_provider": "runwayml", + "metadata": {"comment":"2 credits per image (any resolution) @ $0.01 per credit = $0.02 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/gemini_image3.1_flash_lite": { + "litellm_provider": "runwayml", + "metadata": {"comment":"4 credits per 1K image @ $0.01 per credit = $0.04 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.04, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/bria_fibo_edit_1_5": { + "litellm_provider": "runwayml", + "metadata": {"comment":"5 credits per image @ $0.01 per credit = $0.05 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/gemini_2.5_flash": { + "litellm_provider": "runwayml", + "metadata": {"comment":"5 credits per image (any resolution) @ $0.01 per credit = $0.05 per image"}, + "mode": "image_generation", + "output_cost_per_image": 0.05, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] + }, + "runwayml/eleven_v3": { + "input_cost_per_character": 0.0002, + "litellm_provider": "runwayml", + "metadata": {"comment":"1 credit per 50 characters @ $0.01 per credit = $0.0002 per character; 1 credit minimum not modeled"}, + "mode": "audio_speech", + "source": "https://docs.dev.runwayml.com/guides/pricing/" + }, + "runwayml/eleven_v4": { + "input_cost_per_character": 5e-05, + "litellm_provider": "runwayml", + "metadata": {"comment":"5 credits per 1,000 characters @ $0.01 per credit = $0.00005 per character; 1 credit minimum not modeled"}, + "mode": "audio_speech", + "source": "https://docs.dev.runwayml.com/guides/pricing/" } } From 13089b191b0fc16af5e298c0d2b689610be928e0 Mon Sep 17 00:00:00 2001 From: kenneth-berri Date: Wed, 11 Nov 2026 10:53:54 -0800 Subject: [PATCH 099/189] fix(azure): use supported API versions for Responses requests (#51116) --- .../llms/azure/responses/transformation.py | 36 ++++++++++---- .../response/test_azure_transformation.py | 48 +++++++++++++++++-- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 6a0740b2f15..637d3cfc219 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -37,6 +37,16 @@ _AZURE_RESPONSES_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(AzureResponsesProm _DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}") +_DATED_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}(?:-preview)?") +_MIN_RESPONSES_API_DATE: Final = "2025-03-01" + + +def _predates_responses_api(api_version: str | None) -> bool: + return ( + api_version is not None + and _DATED_API_VERSION.fullmatch(api_version) is not None + and api_version[:10] < _MIN_RESPONSES_API_DATE + ) def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None: @@ -45,16 +55,14 @@ def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None: def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> str | None: - """ - Dated GA api_versions (e.g. 2024-10-21) predate the Responses API on Azure and 404 there, so a deployment - pinned to one for chat gets the default Responses version unless it sets responses_api_version. - """ - api_version: Final = _str_param(litellm_params, "api_version") - if api_base is not None and "api-version" in httpx.URL(api_base).params: - return api_version responses_api_version: Final = _str_param(litellm_params, "responses_api_version") if responses_api_version is not None: return responses_api_version + api_version: Final = _str_param(litellm_params, "api_version") + if api_base is not None and "api-version" in httpx.URL(api_base).params: + return api_version + if _predates_responses_api(api_version): + return None if api_version is not None and _DATED_GA_API_VERSION.fullmatch(api_version): return AZURE_DEFAULT_RESPONSES_API_VERSION return api_version @@ -188,7 +196,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def get_complete_url( self, api_base: str | None, - litellm_params: dict, + litellm_params: dict[str, object], ) -> str: """ Constructs a complete URL for the API request. @@ -206,7 +214,17 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): - A complete URL string, e.g., "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview" """ - resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base) + base: Final = BaseAzureLLM.resolve_azure_api_base(api_base) + parsed_url: Final = httpx.URL(base) if base is not None else None + resolved_api_base: Final = ( + str(parsed_url.copy_with(params=parsed_url.params.remove("api-version"))) + if parsed_url is not None + and ( + _predates_responses_api(parsed_url.params.get("api-version")) + or _str_param(litellm_params, "responses_api_version") is not None + ) + else base + ) responses_params: Final = _with_responses_api_version(resolved_api_base, litellm_params) parsed_base: Final = urlsplit(resolved_api_base or "") path: Final = parsed_base.path.rstrip("/") diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index 05c0817e19b..3a8c9d96da4 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -93,13 +93,13 @@ def test_get_complete_url(): """ azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig() api_base = "https://litellm8397336933.openai.azure.com" - litellm_params = {"api_version": "2024-05-01-preview"} + litellm_params = {"api_version": "2025-04-01-preview"} result = azure_openai_responses_apiconfig.get_complete_url( api_base=api_base, litellm_params=litellm_params ) - expected = "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview" + expected = "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-04-01-preview" assert result == expected @@ -467,7 +467,12 @@ class TestAzureResponsesAPIConfig: litellm_params={"api_version": "2024-10-21", "responses_api_version": responses_api_version}, ) - assert url == api_base + expected: Final = ( + "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=v1" + if responses_api_version is not None + else api_base + ) + assert url == expected def test_api_version_in_env_api_base_query_wins_over_a_dated_ga_api_version(self, monkeypatch: pytest.MonkeyPatch): env_api_base: Final = ( @@ -1015,6 +1020,43 @@ def _unset_api_version_responses_url() -> httpx.URL: return httpx.URL(AzureOpenAIResponsesAPIConfig().get_complete_url(api_base=_AZURE_RESOURCE, litellm_params={})) +@pytest.mark.parametrize("api_version", ["2025-02-01-preview", "2025-02-28", "2024-05-01-preview"]) +@pytest.mark.parametrize("source", ["params", "query", "both"]) +def test_pre_responses_api_version_uses_the_native_responses_default(api_version: str, source: str) -> None: + base: Final = ( + f"{_AZURE_RESOURCE}?api-version={api_version}&tenant=test" + if source in ("query", "both") + else f"{_AZURE_RESOURCE}?tenant=test" + ) + params: Final = {"api_version": api_version} if source in ("params", "both") else {} + url: Final = httpx.URL(AzureOpenAIResponsesAPIConfig().get_complete_url(api_base=base, litellm_params=params)) + expected: Final = _unset_api_version_responses_url() + + assert url.copy_with(query=None) == expected.copy_with(query=None) + assert dict(url.params) == {**dict(expected.params), "tenant": "test"} + + +@pytest.mark.parametrize("api_version", ["2025-03-01-preview", "2025-04-01-preview", "preview", "latest", "v1"]) +@pytest.mark.parametrize("source", ["params", "query"]) +def test_supported_responses_api_versions_are_preserved(api_version: str, source: str) -> None: + base: Final = f"{_AZURE_RESOURCE}?api-version={api_version}" if source == "query" else _AZURE_RESOURCE + params: Final = {"api_version": api_version} if source == "params" else {} + url: Final = httpx.URL(AzureOpenAIResponsesAPIConfig().get_complete_url(api_base=base, litellm_params=params)) + + assert url.params["api-version"] == api_version + + +@pytest.mark.parametrize("api_version", ["2025-02-01-preview", "2025-04-01-preview"]) +def test_responses_override_wins_over_query_and_chat_version(api_version: str) -> None: + config: Final = AzureOpenAIResponsesAPIConfig() + params: Final = {"api_version": api_version, "responses_api_version": "2025-03-01-preview"} + url: Final = config.get_complete_url( + api_base=f"{_AZURE_RESOURCE}?api-version={api_version}", litellm_params=params + ) + + assert url == f"{_AZURE_RESOURCE}/openai/responses?api-version=2025-03-01-preview" + + def _azure_responses_create(api_version: str) -> str | None: response: Final = litellm.responses( model="azure/gpt-5.4-nano", From fdafffb84285892c1e97bf8b31d5eba0cbd7420a Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 11 Nov 2026 11:16:21 -0800 Subject: [PATCH 100/189] feat(proxy): count a rate-limit fallback in x-litellm-attempted-fallbacks and name it in x-litellm-fallback-reason (#51138) * feat(proxy): count a rate-limit fallback in x-litellm-attempted-fallbacks and name it in x-litellm-fallback-reason Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): count every fallback candidate a local rate limit rejected in x-litellm-attempted-fallbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): count fallback attempts without a mutable list and keep self.data on the served request Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 45 +++++++-- .../common_utils/proxy_rate_limit_error.py | 92 ++++++++++++++++++- litellm/proxy/litellm_pre_call_utils.py | 1 + .../test_rate_limit_fallback_headers.py | 79 ++++++++++++++++ .../test_proxy_rate_limit_error.py | 58 ++++++++++++ .../proxy/test_common_request_processing.py | 35 +++++++ .../unit/proxy/test_litellm_pre_call_utils.py | 2 + 7 files changed, 304 insertions(+), 8 deletions(-) create mode 100644 tests/integration/routing/test_rate_limit_fallback_headers.py create mode 100644 tests/unit/proxy/common_utils/test_proxy_rate_limit_error.py diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 46a9f9910d7..c6981015417 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1,10 +1,11 @@ import asyncio import contextlib +import itertools import json import logging import math import time -from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from functools import lru_cache @@ -129,6 +130,11 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_type, relayed_provider_specific_fields, ) +from litellm.proxy.common_utils.proxy_rate_limit_error import ( + ATTEMPTED_FALLBACKS_HEADER, + rate_limit_fallback_from_request_data, + rate_limit_fallback_headers, +) from litellm.proxy.common_utils.responses_stream_errors import ( SSE_LINE_BREAK, ResponsesStreamErrorState, @@ -2384,12 +2390,12 @@ class ProxyBaseLLMRequestProcessing: response_cost: float | str | None = None, hidden_params: Mapping[str, object] | None = None, fastest_response_batch_completion: bool | None = None, - request_data: dict | None = {}, + request_data: Mapping[str, object] | None = None, timeout: float | httpx.Timeout | None = None, litellm_logging_obj: LiteLLMLoggingObj | None = None, read_timing_from_logging_obj: bool = True, - **kwargs, - ) -> dict: + **kwargs: object, + ) -> dict[str, str]: exclude_values: Final = {"", None, "None"} hidden_params = hidden_params or {} resolved_call_id: Final = ( @@ -2453,10 +2459,14 @@ class ProxyBaseLLMRequestProcessing: **{k: str(v) for k, v in kwargs.items()}, } if request_data: - remaining_tokens_header: Final = get_remaining_tokens_and_requests_from_request_data(request_data) + remaining_tokens_header: Final = get_remaining_tokens_and_requests_from_request_data(dict(request_data)) headers.update(remaining_tokens_header) - logging_caching_headers: Final = get_logging_caching_headers(request_data) + quota_fallback: Final = rate_limit_fallback_from_request_data(request_data) + if quota_fallback is not None: + headers.update(rate_limit_fallback_headers(quota_fallback, headers.get(ATTEMPTED_FALLBACKS_HEADER))) + + logging_caching_headers: Final = get_logging_caching_headers(dict(request_data)) if logging_caching_headers: headers.update(logging_caching_headers) @@ -2864,7 +2874,9 @@ class ProxyBaseLLMRequestProcessing: from litellm.proxy.common_utils.proxy_rate_limit_error import ( PER_MODEL_RATE_LIMIT_DESCRIPTOR_KEYS, ProxyRateLimitError, + RateLimitFallback, per_model_rate_limits_disable_fallbacks, + with_rate_limit_fallback, ) general_settings_view: Final = cast( # cast-ok: this method keeps its legacy bare-dict settings parameter @@ -2933,6 +2945,7 @@ class ProxyBaseLLMRequestProcessing: if isinstance(raw_router_settings, Mapping) else None ) + attempt_numbers: Final[Iterator[int]] = itertools.count(1) try: for fallback_model in fallback_models: if fallback_model == original_model: @@ -2955,7 +2968,10 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict.team_id, ): continue - return await self.common_processing_pre_call_logic( + attempt = next(attempt_numbers) + admitted: tuple[ + Mapping[str, object], LiteLLMLoggingObj + ] = await self.common_processing_pre_call_logic( request=request, general_settings=general_settings, proxy_logging_obj=proxy_logging_obj, @@ -2976,6 +2992,21 @@ class ProxyBaseLLMRequestProcessing: if is_hard_per_model_limit(fallback_exc): raise continue + else: + admitted_data, admitted_logging_obj = admitted + stamped: dict = dict( # mutable-ok: caller-defined request-payload shape + with_rate_limit_fallback( + admitted_data, + get_metadata_variable_name(request, admitted_data), + RateLimitFallback( + original_model_group=original_model, + descriptor_key=original_exc.descriptor_key or "unknown", + attempts=attempt, + ), + ) + ) + self.data = stamped + return stamped, admitted_logging_obj except BaseException: self.data = rate_limited_data raise diff --git a/litellm/proxy/common_utils/proxy_rate_limit_error.py b/litellm/proxy/common_utils/proxy_rate_limit_error.py index 28d28a5c9bd..6f3ef4e03d4 100644 --- a/litellm/proxy/common_utils/proxy_rate_limit_error.py +++ b/litellm/proxy/common_utils/proxy_rate_limit_error.py @@ -37,10 +37,12 @@ This module provides a single proxy-side error class that: import json from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType from typing import Final from fastapi import HTTPException -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.exceptions import RateLimitError, RateLimitErrorCategory, RateLimitType @@ -73,6 +75,94 @@ def per_model_rate_limits_disable_fallbacks(general_settings: Mapping[str, objec return False +RATE_LIMIT_FALLBACK_METADATA_KEY: Final = "rate_limit_fallback" +RATE_LIMIT_FALLBACK_REASON_HEADER: Final = "x-litellm-fallback-reason" +ATTEMPTED_FALLBACKS_HEADER: Final = "x-litellm-attempted-fallbacks" + + +@dataclass(frozen=True, slots=True) +class RateLimitFallback: + original_model_group: str + descriptor_key: str + attempts: int = 1 + + def stamp(self) -> Mapping[str, object]: + return MappingProxyType( + { + "original_model_group": self.original_model_group, + "descriptor_key": self.descriptor_key, + "attempts": self.attempts, + } + ) + + @property + def reason(self) -> str: + return f"rate_limit:{self.descriptor_key}" + + +class _StampedBucket(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + rate_limit_fallback: RateLimitFallback | None = None + + +class _StampedRequest(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + metadata: _StampedBucket | None = None + litellm_metadata: _StampedBucket | None = None + + +_STAMPED_REQUEST: Final[TypeAdapter[_StampedRequest]] = TypeAdapter(_StampedRequest) +_ROUTER_HOPS: Final[TypeAdapter[int]] = TypeAdapter(int) + + +def rate_limit_fallback_from_request_data(request_data: Mapping[str, object]) -> RateLimitFallback | None: + try: + stamped: Final = _STAMPED_REQUEST.validate_python(request_data) + except ValidationError: + return None + buckets: Final = (stamped.metadata, stamped.litellm_metadata) + return next( + (bucket.rate_limit_fallback for bucket in buckets if bucket is not None and bucket.rate_limit_fallback), None + ) + + +_METADATA_BUCKET: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def _metadata_bucket(value: object) -> Mapping[str, object]: + try: + return _METADATA_BUCKET.validate_python(value) + except ValidationError: + return MappingProxyType({}) + + +def with_rate_limit_fallback( + request_data: Mapping[str, object], bucket_name: str, fallback: RateLimitFallback +) -> Mapping[str, object]: + bucket: Final = _metadata_bucket(request_data.get(bucket_name)) + return MappingProxyType( + {**request_data, bucket_name: {**bucket, RATE_LIMIT_FALLBACK_METADATA_KEY: dict(fallback.stamp())}} + ) + + +def rate_limit_fallback_headers(fallback: RateLimitFallback, attempted_fallbacks: object) -> Mapping[str, str]: + try: + router_hops: Final = _ROUTER_HOPS.validate_python(attempted_fallbacks) + except ValidationError: + return MappingProxyType( + { + ATTEMPTED_FALLBACKS_HEADER: str(max(fallback.attempts, 1)), + RATE_LIMIT_FALLBACK_REASON_HEADER: fallback.reason, + } + ) + return MappingProxyType( + { + ATTEMPTED_FALLBACKS_HEADER: str(max(router_hops, 0) + max(fallback.attempts, 1)), + RATE_LIMIT_FALLBACK_REASON_HEADER: fallback.reason, + } + ) + + def map_v3_rate_limit_type( v3_value: str | None, ) -> RateLimitType | None: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0b2ba93af71..ff0883b6958 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -536,6 +536,7 @@ ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset( "attempted_fallbacks", "original_model_group", "previous_models", + "rate_limit_fallback", "request_retry_count", CLIENT_OUTPUT_CEILING_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, diff --git a/tests/integration/routing/test_rate_limit_fallback_headers.py b/tests/integration/routing/test_rate_limit_fallback_headers.py new file mode 100644 index 00000000000..18a42cb4d1d --- /dev/null +++ b/tests/integration/routing/test_rate_limit_fallback_headers.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from contextlib import ExitStack +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import OwnedState, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import wire_server +from integration.routing.test_per_model_rate_limit_fallbacks import ( + assert_served, + chat, + healthy_reply, + model_entry, + proxy_config, +) + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + primary: str + fallback: str + second_fallback: str + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + suffix: Final = uuid.uuid4().hex[:10] + primary: Final = f"quota-primary-{suffix}" + fallback: Final = f"quota-fallback-{suffix}" + second_fallback: Final = f"quota-second-fallback-{suffix}" + directory: Final = tmp_path_factory.mktemp("rate-limit-fallback-headers") + with ExitStack() as stack: + rig_gateway: Final = stack.enter_context(gateway_from_environment()) + primary_wire: Final = stack.enter_context(wire_server(healthy_reply)) + fallback_wire: Final = stack.enter_context(wire_server(healthy_reply)) + second_fallback_wire: Final = stack.enter_context(wire_server(healthy_reply)) + config: Final = proxy_config( + directory, + model_list=( + model_entry(primary, primary_wire), + model_entry(fallback, fallback_wire), + model_entry(second_fallback, second_fallback_wire), + ), + fallbacks=({primary: [fallback, second_fallback]},), + general_settings={}, + ) + owned: Final = stack.enter_context( + owned_proxy_process( + rig_gateway, directory, {}, config=config, workers=2, state=OwnedState.SHARED_WITH_WORKER + ) + ) + yield Rig(owned.gateway, primary, fallback, second_fallback) + + +def test_quota_fallback_response_names_the_hop_and_its_reason(rig: Rig) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(model_tpm_limit={rig.primary: 1}) + + response: Final = chat(rig.gateway, key, rig.primary, f"pmrl-{uuid.uuid4().hex}") + assert_served(response, rig.fallback, "served pmrl-") + assert response.headers.get("x-litellm-attempted-fallbacks") == "1", dict(response.headers) + assert response.headers.get("x-litellm-fallback-reason") == "rate_limit:model_per_key", dict(response.headers) + + +def test_quota_fallback_response_counts_every_candidate_a_limit_rejected(rig: Rig) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(model_tpm_limit={rig.primary: 1, rig.fallback: 1}) + + response: Final = chat(rig.gateway, key, rig.primary, f"pmrl-{uuid.uuid4().hex}") + assert_served(response, rig.second_fallback, "served pmrl-") + assert response.headers.get("x-litellm-attempted-fallbacks") == "2", dict(response.headers) + assert response.headers.get("x-litellm-fallback-reason") == "rate_limit:model_per_key", dict(response.headers) diff --git a/tests/unit/proxy/common_utils/test_proxy_rate_limit_error.py b/tests/unit/proxy/common_utils/test_proxy_rate_limit_error.py new file mode 100644 index 00000000000..a7628b72341 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_proxy_rate_limit_error.py @@ -0,0 +1,58 @@ +from typing import Final + +import pytest + +from litellm.proxy.common_utils.proxy_rate_limit_error import ( + RateLimitFallback, + rate_limit_fallback_from_request_data, + rate_limit_fallback_headers, +) + +STAMP: Final = {"original_model_group": "opus", "descriptor_key": "model_per_end_user"} + + +@pytest.mark.parametrize("bucket", ("metadata", "litellm_metadata")) +def test_rate_limit_fallback_is_read_from_either_metadata_bucket(bucket: str) -> None: + fallback: Final = rate_limit_fallback_from_request_data({"model": "luna", bucket: {"rate_limit_fallback": STAMP}}) + + assert fallback is not None + assert fallback == RateLimitFallback(original_model_group="opus", descriptor_key="model_per_end_user") + assert fallback.reason == "rate_limit:model_per_end_user" + + +@pytest.mark.parametrize( + "request_data", + ( + {"model": "luna"}, + {"model": "luna", "metadata": {}}, + {"model": "luna", "metadata": {"rate_limit_fallback": "opus"}}, + {"model": "luna", "metadata": {"rate_limit_fallback": {"original_model_group": "opus"}}}, + ), +) +def test_a_request_without_a_complete_stamp_has_no_rate_limit_fallback(request_data: dict[str, object]) -> None: + assert rate_limit_fallback_from_request_data(request_data) is None + + +@pytest.mark.parametrize(("router_hops", "expected"), ((None, "1"), (0, "1"), ("0", "1"), (2, "3"), ("junk", "1"))) +def test_rate_limit_fallback_headers_count_the_hop_on_top_of_the_router(router_hops: object, expected: str) -> None: + headers: Final = rate_limit_fallback_headers( + RateLimitFallback(original_model_group="opus", descriptor_key="model_per_key"), router_hops + ) + + assert headers == { + "x-litellm-attempted-fallbacks": expected, + "x-litellm-fallback-reason": "rate_limit:model_per_key", + } + + +@pytest.mark.parametrize( + ("attempts", "router_hops", "expected"), ((2, None, "2"), (2, 0, "2"), (3, 1, "4"), (0, 0, "1")) +) +def test_rate_limit_fallback_headers_count_every_candidate_a_limit_rejected( + attempts: int, router_hops: object, expected: str +) -> None: + headers: Final = rate_limit_fallback_headers( + RateLimitFallback(original_model_group="opus", descriptor_key="model_per_key", attempts=attempts), router_hops + ) + + assert headers["x-litellm-attempted-fallbacks"] == expected diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 23023560a41..bfe381cb848 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -13742,3 +13742,38 @@ class TestTotalTimeout: assert not prepared assert not called assert elapsed < 120 + + +@pytest.mark.parametrize("router_attempted_fallbacks", (0, 1)) +def test_get_custom_headers_count_a_quota_fallback_and_name_its_reason(router_attempted_fallbacks: int) -> None: + auth: Final = ProxyUserAPIKeyAuth(api_key="sk-quota-fallback", tpm_limit=None, rpm_limit=None, max_budget=None) + request_data: Final = { + "model": "luna", + "metadata": {"rate_limit_fallback": {"original_model_group": "opus", "descriptor_key": "model_per_key"}}, + } + + custom_headers: Callable[..., dict[str, str]] = ProxyBaseLLMRequestProcessing.get_custom_headers + headers: Final = custom_headers( + user_api_key_dict=auth, + call_id="quota-fallback", + request_data=request_data, + **{"x-litellm-attempted-fallbacks": router_attempted_fallbacks}, + ) + + assert headers["x-litellm-attempted-fallbacks"] == str(router_attempted_fallbacks + 1) + assert headers["x-litellm-fallback-reason"] == "rate_limit:model_per_key" + + +def test_get_custom_headers_leave_a_plain_request_without_a_fallback_reason() -> None: + auth: Final = ProxyUserAPIKeyAuth(api_key="sk-no-fallback", tpm_limit=None, rpm_limit=None, max_budget=None) + + custom_headers: Callable[..., dict[str, str]] = ProxyBaseLLMRequestProcessing.get_custom_headers + headers: Final = custom_headers( + user_api_key_dict=auth, + call_id="no-fallback", + request_data={"model": "opus", "metadata": {}}, + **{"x-litellm-attempted-fallbacks": 0}, + ) + + assert headers["x-litellm-attempted-fallbacks"] == "0" + assert "x-litellm-fallback-reason" not in headers diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index f5757a0ff18..7891a87b1dc 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -9369,6 +9369,7 @@ def _reserved_stamp_key(key_metadata: dict | None = None) -> UserAPIKeyAuth: _PLANTED_STAMPS = { "attempted_fallbacks": 99, "original_model_group": "spoofed-group", + "rate_limit_fallback": {"original_model_group": "spoofed-group", "descriptor_key": "model_per_key"}, "request_retry_count": -100, "previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}], "_client_output_ceiling": {"api_base": "https://attacker.example"}, @@ -9404,6 +9405,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo assert "litellm_metadata" not in updated assert "attempted_fallbacks" not in updated["metadata"] assert "original_model_group" not in updated["metadata"] + assert "rate_limit_fallback" not in updated["metadata"] assert "_client_output_ceiling" not in updated["metadata"] assert "request_retry_count" not in updated["metadata"] assert "previous_models" not in updated["metadata"] From b079bf5a51ddad9c7d5f89d515d3751ea230e5bf Mon Sep 17 00:00:00 2001 From: yujonglee Date: Thu, 12 Nov 2026 04:40:00 +0900 Subject: [PATCH 101/189] refactor(proxy): let the gunicorn server take a private listener (#50871) Extract serving_port() and configure_database() as they were inline, turn the LITELLM_RUST parser into a reusable RustSwitch type, and reformat tests/unit/proxy/test_proxy_cli.py to the 120 column limit. run_gunicorn_server takes private_listener, an inherited socket or an assigned loopback port. With one, gunicorn binds it instead of host:port and trusts no peer's forwarding headers, since every peer on it is the local process in front. Both are gunicorn's own settings, bind and forwarded_allow_ips, so the uvicorn worker is unchanged. The reload options take factory. No caller passes either yet, so every entry point serves as before. --- gateway/launch.py | 6 +- litellm/proxy/dev_reload.py | 7 +- litellm/proxy/proxy_cli.py | 16 +- litellm/proxy/server_listener.py | 38 ++ litellm/rust_bridge/configuration.py | 37 +- tests/unit/proxy/test_dev_reload.py | 41 +- tests/unit/proxy/test_proxy_cli.py | 667 +++++++------------ tests/unit/proxy/test_server_listener.py | 28 + tests/unit/rust_bridge/test_configuration.py | 13 +- 9 files changed, 394 insertions(+), 459 deletions(-) create mode 100644 litellm/proxy/server_listener.py create mode 100644 tests/unit/proxy/test_server_listener.py diff --git a/gateway/launch.py b/gateway/launch.py index 59f7121e513..b9303fd9447 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -75,7 +75,7 @@ def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: +def configure_database() -> None: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url @@ -86,6 +86,10 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) - sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") if pooled_url is not None: export_pooled_database_url(pooled_url) + + +def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + configure_database() serve(uvicorn_argv(argv, os.environ)) diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py index 500f581b34d..7b057271cb6 100644 --- a/litellm/proxy/dev_reload.py +++ b/litellm/proxy/dev_reload.py @@ -13,7 +13,7 @@ from socket import socket from typing import Final, Literal from pydantic import ConfigDict, ImportString, JsonValue, TypeAdapter, with_config -from typing_extensions import ReadOnly, Required, TypedDict +from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from uvicorn import Config, Server from uvicorn import config as uvicorn_config from uvicorn.supervisors import ChangeReload, Multiprocess @@ -42,6 +42,7 @@ class ServerOptions(TypedDict, total=False): @with_config(ConfigDict(extra="allow")) class ReloadOptions(ServerOptions): app: Required[ReadOnly[str]] + factory: NotRequired[ReadOnly[bool]] reload: ReadOnly[bool] reload_dirs: ReadOnly[tuple[str, ...]] reload_includes: ReadOnly[tuple[str, ...]] @@ -92,7 +93,9 @@ def load_worker_app(import_path: str, worker_app_factory: bool) -> Callable[..., def pool_config(options: ReloadOptions, workers: int, worker_overrides: Mapping[str, object] | None = None) -> Config: overrides: Final = worker_overrides or {} worker_app: Final = TypeAdapter(str).validate_python(overrides.get("app", options["app"])) - worker_app_factory: Final = TypeAdapter(bool).validate_python(overrides.get("factory", False)) + worker_app_factory: Final = TypeAdapter(bool).validate_python( + overrides.get("factory", options.get("factory", False)) + ) return Config( partial(load_worker_app, worker_app, worker_app_factory), factory=True, diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 41aedf4b5cd..b238cefd24f 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -31,6 +31,7 @@ from litellm.proxy.db.pgbouncer import ( start_in_container_pgbouncer, ) from litellm.proxy.db.query_engine_reaper import start_query_engine_reaper +from litellm.proxy.server_listener import PrivateListener, gunicorn_overrides from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: @@ -686,6 +687,7 @@ class ProxyInitializationHelpers: max_requests_before_restart: int | None = None, max_requests_before_restart_jitter: int | None = None, max_request_header_size: int | None = None, + private_listener: PrivateListener | None = None, ): """ Run litellm with `gunicorn` @@ -766,6 +768,7 @@ class ProxyInitializationHelpers: "accesslog": "-", # Log to stdout "timeout": 600, # default to very high number, bedrock/anthropic.claude-v2:1 can take 30+ seconds for the 1st chunk to come in "access_log_format": '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s', + **gunicorn_overrides(private_listener), } # Optional: recycle workers after N requests to mitigate memory growth @@ -821,7 +824,7 @@ class ProxyInitializationHelpers: _run_ollama_serve = run_ollama_serve @staticmethod - def is_port_in_use(port): + def is_port_in_use(port: int): import socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -829,6 +832,10 @@ class ProxyInitializationHelpers: _is_port_in_use = is_port_in_use + @staticmethod + def serving_port(port: int) -> int: + return random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port + @staticmethod def get_loop_type(): """Helper function to determine the event loop type based on platform""" @@ -1228,8 +1235,8 @@ class ProxyInitializationHelpers: ) def run_server( cli_args, - host, - port, + host: str, + port: int, api_base, api_version, model, @@ -1717,8 +1724,7 @@ def run_server( ) sys.exit(1) export_pooled_database_url(pooled_database_url) - if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port): - port = random.randint(1024, 49152) + port = ProxyInitializationHelpers.serving_port(port) if prometheus_metrics_port == port: raise click.UsageError("--prometheus_metrics_port must differ from --port") diff --git a/litellm/proxy/server_listener.py b/litellm/proxy/server_listener.py new file mode 100644 index 00000000000..bf57fce4ebd --- /dev/null +++ b/litellm/proxy/server_listener.py @@ -0,0 +1,38 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, TypeAlias + +from typing_extensions import assert_never + +LOOPBACK_HOST: Final = "127.0.0.1" +NO_TRUSTED_PROXIES: Final = "" + + +@dataclass(frozen=True, slots=True) +class InheritedFd: + fd: int + + +@dataclass(frozen=True, slots=True) +class AssignedPort: + port: int + + +PrivateListener: TypeAlias = InheritedFd | AssignedPort + + +def _gunicorn_bind(listener: PrivateListener) -> str: + match listener: + case InheritedFd(fd): + return f"fd://{fd}" + case AssignedPort(port): + return f"{LOOPBACK_HOST}:{port}" + case _: + assert_never(listener) + + +def gunicorn_overrides(listener: PrivateListener | None) -> Mapping[str, str]: + if listener is None: + return MappingProxyType({}) + return MappingProxyType({"bind": _gunicorn_bind(listener), "forwarded_allow_ips": NO_TRUSTED_PROXIES}) diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index 152ba632996..dedbf857a07 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -3,13 +3,29 @@ from __future__ import annotations import os from enum import Enum, auto from functools import lru_cache -from typing import Final +from typing import Annotated, Final, TypeAlias -from pydantic import TypeAdapter, ValidationError +from pydantic import BeforeValidator, TypeAdapter, ValidationError from typing_extensions import assert_never -_GLOBAL_ENV_NAME: Final = "LITELLM_RUST" -_ENV_BOOL: Final = TypeAdapter(bool) +ENV_NAME: Final = "LITELLM_RUST" + + +def _blank_as_unset(value: object) -> object: + stripped: Final = value.strip() if isinstance(value, str) else value + return None if stripped == "" else stripped + + +RustSwitch: TypeAlias = Annotated[bool | None, BeforeValidator(_blank_as_unset)] +_RUST_SWITCH: Final = TypeAdapter[bool | None](RustSwitch) + + +@lru_cache(maxsize=16) +def _switch_or_unset(raw: str | None) -> bool | None: + try: + return _RUST_SWITCH.validate_python(raw) + except ValidationError: + return None class Rollout(Enum): @@ -33,17 +49,6 @@ class _RustConfiguration: _CONFIGURATION: Final = _RustConfiguration() -@lru_cache(maxsize=16) -def _parse_env_bool(value: str | None) -> bool | None: - """`LITELLM_RUST` as a bool; cached by raw value because `decision` runs per tokenizer call.""" - if value is None: - return None - try: - return _ENV_BOOL.validate_python(value.strip()) - except ValidationError: - return None - - def decide( rollout: Rollout, *, @@ -72,7 +77,7 @@ def decision(rollout: Rollout) -> Decision: return decide( rollout, process_override=_CONFIGURATION.override, - environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)), + environment_override=_switch_or_unset(os.getenv(ENV_NAME)), ) diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index 60a7c53820b..6650c1cd3e3 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -86,6 +86,39 @@ def test_a_broken_factory_exits_the_worker_as_a_startup_failure( assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE +@pytest.mark.parametrize("worker_overrides", [None, {"factory": True}]) +def test_a_factory_flag_in_the_options_reaches_the_workers( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, worker_overrides: dict[str, object] | None +) -> None: + (tmp_path / "reload_factory_flag_app.py").write_text( + "async def served(scope, receive, send):\n" + " await send({'type': 'http.response.start', 'status': 204, 'headers': []})\n" + " await send({'type': 'http.response.body', 'body': b'', 'more_body': False})\n" + "\n" + "def factory():\n" + " return served\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + config: Final = pool_config( + RELOAD_OPTIONS.validate_python({**_options("reload_factory_flag_app:factory"), "factory": True}), + workers=POOL_WORKERS, + worker_overrides=worker_overrides, + ) + config.load() + loaded_app: Final = config.loaded_app + assert loaded_app is not None + sent: Final[list[Message]] = [] + + async def receive() -> Message: + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message: Message) -> None: + sent.append(message) + + asyncio.run(loaded_app({"type": "http", "method": "GET", "path": "/", "headers": []}, receive, send)) + assert sent[0] == {"type": "http.response.start", "status": 204, "headers": []} + + @REQUIRES_UVICORN_STARTUP_FAILURE @pytest.mark.parametrize( ("module", "source"), @@ -118,7 +151,9 @@ def test_the_reload_pool_defaults_the_worker_healthcheck_and_keeps_an_explicit_o explicit: Final = RELOAD_WORKER_HEALTHCHECK_SECONDS + 7 assert pool_config(_options("app:app"), workers=2).timeout_worker_healthcheck == RELOAD_WORKER_HEALTHCHECK_SECONDS assert ( - pool_config({**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2).timeout_worker_healthcheck + pool_config( + {**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2 + ).timeout_worker_healthcheck == explicit ) @@ -142,7 +177,9 @@ def test_the_reload_pool_accepts_and_keeps_every_uvicorn_arg_the_proxy_cli_build def test_the_reload_pool_passes_a_uvicorn_arg_it_does_not_list_through_to_the_workers() -> None: - config: Final = pool_config(RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS) + config: Final = pool_config( + RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS + ) assert (config.backlog, config.reload) == (123, False) diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 9a091959631..eec682f365f 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -34,6 +34,7 @@ from litellm._logging import turn_on_json from litellm.constants import DEFAULT_MAX_REQUEST_HEADER_SIZE, GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS from litellm.proxy import proxy_cli from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +from litellm.proxy.server_listener import AssignedPort, InheritedFd from litellm.proxy.shutdown.graceful_shutdown_manager import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT @@ -201,42 +202,30 @@ class TestProxyInitializationHelpers: # Execute with pytest.raises(ValueError, match="Invalid test value"): - ProxyInitializationHelpers._run_test_chat_completion( - "localhost", 8000, "gpt-3.5-turbo", True - ) + ProxyInitializationHelpers._run_test_chat_completion("localhost", 8000, "gpt-3.5-turbo", True) # Test with valid string test value - ProxyInitializationHelpers._run_test_chat_completion( - "localhost", 8000, "gpt-3.5-turbo", "http://test-url" - ) + ProxyInitializationHelpers._run_test_chat_completion("localhost", 8000, "gpt-3.5-turbo", "http://test-url") # Assert - mock_openai.assert_called_once_with( - api_key="My API Key", base_url="http://test-url" - ) + mock_openai.assert_called_once_with(api_key="My API Key", base_url="http://test-url") mock_client.chat.completions.create.assert_called() def test_get_default_unvicorn_init_args(self): # Test without log_config - args = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000 - ) + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000) assert args["app"] == "litellm.proxy.proxy_server:app" assert args["host"] == "localhost" assert args["port"] == 8000 assert args["server_header"] is False # Test with log_config - args = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000, "log_config.json" - ) + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000, "log_config.json") assert args["log_config"] == "log_config.json" # Test with json_logs=True with patch("litellm.json_logs", True): - args = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000 - ) + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000) # When json_logs is True, log_config should be set to the JSON log config dict assert args["log_config"] is not None assert isinstance(args["log_config"], dict) @@ -244,15 +233,11 @@ class TestProxyInitializationHelpers: assert "formatters" in args["log_config"] # Test with keepalive_timeout - args = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000, None, 60 - ) + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000, None, 60) assert args["timeout_keep_alive"] == 60 # Test with both log_config and keepalive_timeout - args = ProxyInitializationHelpers._get_default_unvicorn_init_args( - "localhost", 8000, "log_config.json", 120 - ) + args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000, "log_config.json", 120) assert args["log_config"] == "log_config.json" assert args["timeout_keep_alive"] == 120 @@ -383,14 +368,10 @@ class TestProxyInitializationHelpers: py_file = tmp_path / "module.py" py_file.write_text("x = 1\n") - applied = ProxyInitializationHelpers._patch_statreload_extra_paths( - [str(config_file)] - ) + applied = ProxyInitializationHelpers._patch_statreload_extra_paths([str(config_file)]) assert applied is True - fake_self = types.SimpleNamespace( - config=types.SimpleNamespace(reload_dirs=[tmp_path]) - ) + fake_self = types.SimpleNamespace(config=types.SimpleNamespace(reload_dirs=[tmp_path])) yielded_paths = {Path(p).resolve() for p in StatReload.iter_py_files(fake_self)} assert config_file.resolve() in yielded_paths @@ -407,14 +388,10 @@ class TestProxyInitializationHelpers: env_file = tmp_path / ".env" env_file.write_text("FOO=bar\n") - applied = ProxyInitializationHelpers._patch_statreload_extra_paths( - [str(env_file)] - ) + applied = ProxyInitializationHelpers._patch_statreload_extra_paths([str(env_file)]) assert applied is True - fake_self = types.SimpleNamespace( - config=types.SimpleNamespace(reload_dirs=[tmp_path]) - ) + fake_self = types.SimpleNamespace(config=types.SimpleNamespace(reload_dirs=[tmp_path])) yielded_paths = {Path(p).resolve() for p in StatReload.iter_py_files(fake_self)} assert env_file.resolve() in yielded_paths @@ -426,10 +403,7 @@ class TestProxyInitializationHelpers: StatReload._litellm_patched_config_paths.clear() assert ProxyInitializationHelpers._patch_statreload_extra_paths([]) is False - assert ( - ProxyInitializationHelpers._patch_statreload_extra_paths([None, ""]) - is False - ) + assert ProxyInitializationHelpers._patch_statreload_extra_paths([None, ""]) is False def test_patch_statreload_extra_paths_is_idempotent(self, tmp_path): from pathlib import Path @@ -447,18 +421,14 @@ class TestProxyInitializationHelpers: for _ in range(3): ProxyInitializationHelpers._patch_statreload_extra_paths([str(config_file)]) - fake_self = types.SimpleNamespace( - config=types.SimpleNamespace(reload_dirs=[tmp_path]) - ) + fake_self = types.SimpleNamespace(config=types.SimpleNamespace(reload_dirs=[tmp_path])) yielded = list(StatReload.iter_py_files(fake_self)) assert len(yielded) == len(set(map(str, yielded))) yielded_paths = {Path(p).resolve() for p in yielded} assert config_file.resolve() in yielded_paths assert py_file.resolve() in yielded_paths - def test_configure_dev_reload_watches_env_and_sets_override_flag( - self, tmp_path, monkeypatch - ): + def test_configure_dev_reload_watches_env_and_sets_override_flag(self, tmp_path, monkeypatch): from pathlib import Path from uvicorn.supervisors.statreload import StatReload @@ -475,9 +445,7 @@ class TestProxyInitializationHelpers: uvicorn_args: dict = {} with patch("litellm._logging.verbose_proxy_logger.warning") as mock_warning: - ProxyInitializationHelpers._configure_dev_reload( - uvicorn_args, str(config_file) - ) + ProxyInitializationHelpers._configure_dev_reload(uvicorn_args, str(config_file)) assert os.environ["LITELLM_DEV_ENV_HOT_RELOAD"] == "True" assert uvicorn_args["reload"] is True @@ -488,9 +456,7 @@ class TestProxyInitializationHelpers: assert "override" in warning_text assert ".env" in warning_text - fake_self = types.SimpleNamespace( - config=types.SimpleNamespace(reload_dirs=[tmp_path]) - ) + fake_self = types.SimpleNamespace(config=types.SimpleNamespace(reload_dirs=[tmp_path])) yielded_paths = {Path(p).resolve() for p in StatReload.iter_py_files(fake_self)} assert env_file.resolve() in yielded_paths assert config_file.resolve() in yielded_paths @@ -514,17 +480,13 @@ class TestProxyInitializationHelpers: mock_app = MagicMock() # Execute - ProxyInitializationHelpers._init_hypercorn_server( - mock_app, "localhost", 8000, None, None, None - ) + ProxyInitializationHelpers._init_hypercorn_server(mock_app, "localhost", 8000, None, None, None) # Assert mock_asyncio_run.assert_called_once() # Test with SSL - ProxyInitializationHelpers._init_hypercorn_server( - mock_app, "localhost", 8000, "cert.pem", "key.pem", "ECDHE" - ) + ProxyInitializationHelpers._init_hypercorn_server(mock_app, "localhost", 8000, "cert.pem", "key.pem", "ECDHE") @patch("granian.Granian") @patch("builtins.print") @@ -693,10 +655,14 @@ class TestProxyInitializationHelpers: database_name_enc = urllib.parse.quote_plus(database_name) # Construct DATABASE_URL from the provided variables - database_url = f"postgresql://{database_username_enc}:{database_password_enc}@{database_host}/{database_name_enc}" + database_url = ( + f"postgresql://{database_username_enc}:{database_password_enc}@{database_host}/{database_name_enc}" + ) # Assert the correct URL was constructed with properly escaped characters - expected_url = "postgresql://user%40with%2Bspecial:test-password-special-chars@localhost:5432/db_name%2Ftest" + expected_url = ( + "postgresql://user%40with%2Bspecial:test-password-special-chars@localhost:5432/db_name%2Ftest" + ) assert database_url == expected_url # Test appending query parameters @@ -714,9 +680,7 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("atexit.register") # critical @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_skip_server_startup( self, mock_should_update, mock_setup_db, mock_atexit_register, mock_uvicorn_run, tmp_path: Path ): @@ -734,11 +698,7 @@ class TestProxyInitializationHelpers: ) # Remove DATABASE_URL/DIRECT_URL so the CLI doesn't attempt # real prisma operations when these are set in CI. - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["PROMETHEUS_MULTIPROC_DIR"] = str(tmp_path) live_proxy_samples = tmp_path / "counter_123.db" live_proxy_samples.write_bytes(b"samples of a proxy that is still running") @@ -758,9 +718,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -771,18 +729,12 @@ class TestProxyInitializationHelpers: # --- skip startup --- result = runner.invoke(run_server, ["--local", "--skip_server_startup"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert "Skipping server startup" in result.output mock_uvicorn_run.assert_not_called() - result = runner.invoke( - run_server, ["--local", "--skip_server_startup", "--telemetry", "False"] - ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + result = runner.invoke(run_server, ["--local", "--skip_server_startup", "--telemetry", "False"]) + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert "Skipping server startup" in result.output assert "telemetry" not in runner.invoke(run_server, ["--help"]).output assert live_proxy_samples.exists() @@ -792,9 +744,7 @@ class TestProxyInitializationHelpers: result = runner.invoke(run_server, ["--local"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() assert not live_proxy_samples.exists() @@ -831,9 +781,7 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_limit_concurrency_passed_to_uvicorn( self, mock_should_update, mock_setup_db, mock_atexit_register, mock_uvicorn_run ): @@ -850,11 +798,7 @@ class TestProxyInitializationHelpers: KeyManagementSettings=MagicMock(), save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -864,9 +808,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.side_effect = lambda *a, **k: { "app": "litellm.proxy.proxy_server:app", @@ -874,20 +816,14 @@ class TestProxyInitializationHelpers: "port": 8000, } - result = runner.invoke( - run_server, ["--local", "--limit_concurrency", "250"] - ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + result = runner.invoke(run_server, ["--local", "--limit_concurrency", "250"]) + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() assert mock_uvicorn_run.call_args.kwargs.get("limit_concurrency") == 250 mock_uvicorn_run.reset_mock() result = runner.invoke(run_server, ["--local"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() assert "limit_concurrency" not in mock_uvicorn_run.call_args.kwargs @@ -905,7 +841,9 @@ class TestProxyInitializationHelpers: @patch("httpx.HTTPTransport.handle_request") @patch("atexit.register") @patch("subprocess.Popen") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.prisma_client.PrismaManager.setup_database" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above @patch( # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False ) @@ -943,9 +881,7 @@ class TestProxyInitializationHelpers: save_worker_config=MagicMock(), ) clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL", "PROMETHEUS_METRICS_PORT") + k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL", "PROMETHEUS_METRICS_PORT") } clean_env["PROMETHEUS_MULTIPROC_DIR"] = str(tmp_path) with ( @@ -971,9 +907,7 @@ class TestProxyInitializationHelpers: run_server, ["--local", "--host", "127.0.0.1", "--port", "4000", "--prometheus_metrics_port", "4001"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_popen.assert_called_once() spawned = list(mock_popen.call_args.args[0]) assert spawned[1:3] == ["-m", "litellm.proxy.prometheus_metrics_server"] @@ -996,9 +930,7 @@ class TestProxyInitializationHelpers: mock_popen.reset_mock() mock_uvicorn_run.reset_mock() result = runner.invoke(run_server, ["--local"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_popen.assert_not_called() mock_uvicorn_run.assert_called_once() @@ -1012,9 +944,7 @@ class TestProxyInitializationHelpers: mock_popen.assert_not_called() mock_uvicorn_run.assert_not_called() - result = runner.invoke( - run_server, ["--local", "--prometheus_metrics_port", "0"] - ) + result = runner.invoke(run_server, ["--local", "--prometheus_metrics_port", "0"]) assert result.exit_code == 2 assert "Invalid value for '--prometheus_metrics_port'" in result.output mock_popen.assert_not_called() @@ -1036,9 +966,7 @@ class TestProxyInitializationHelpers: @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_db_timeout_settings_are_forwarded_to_pool_timeout( self, mock_should_update, @@ -1071,11 +999,7 @@ class TestProxyInitializationHelpers: } ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), @@ -1086,9 +1010,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.append_query_params", side_effect=lambda url, params: ( @@ -1107,9 +1029,7 @@ class TestProxyInitializationHelpers: ["--local", "--config", "test-config.yaml", "--skip_server_startup"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_append_query_params.assert_called() appended_params = mock_append_query_params.call_args.args[1] assert appended_params["connection_limit"] == 5 @@ -1164,9 +1084,7 @@ class TestProxyInitializationHelpers: @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_db_connection_extra_params_forwarded_to_url( self, mock_should_update, @@ -1201,11 +1119,7 @@ class TestProxyInitializationHelpers: } ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), @@ -1216,9 +1130,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.append_query_params", side_effect=lambda url, params: str(url), @@ -1235,9 +1147,7 @@ class TestProxyInitializationHelpers: ["--local", "--config", "test-config.yaml", "--skip_server_startup"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_append_query_params.assert_called() appended_params = mock_append_query_params.call_args.args[1] assert appended_params["connect_timeout"] == 15 @@ -1288,9 +1198,7 @@ class TestProxyInitializationHelpers: @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_disable_prepared_statements_forwarded_to_url( self, mock_should_update, @@ -1322,11 +1230,7 @@ class TestProxyInitializationHelpers: } ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), @@ -1337,9 +1241,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.append_query_params", side_effect=lambda url, params: str(url), @@ -1356,9 +1258,7 @@ class TestProxyInitializationHelpers: ["--local", "--config", "test-config.yaml", "--skip_server_startup"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_append_query_params.assert_called() appended_params = mock_append_query_params.call_args.args[1] if expect_pgbouncer: @@ -1380,9 +1280,7 @@ class TestProxyInitializationHelpers: @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_disable_prepared_statements_env_var_forwarded_to_url( self, mock_should_update, @@ -1430,9 +1328,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.append_query_params", side_effect=lambda url, params: str(url), @@ -1449,9 +1345,7 @@ class TestProxyInitializationHelpers: ["--local", "--config", "test-config.yaml", "--skip_server_startup"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" appended_params = mock_append_query_params.call_args.args[1] if expect_pgbouncer: assert appended_params["pgbouncer"] == "true", appended_params @@ -1461,9 +1355,7 @@ class TestProxyInitializationHelpers: @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_malformed_disable_prepared_statements_env_var_is_rejected_even_when_config_enables_it( self, mock_should_update, @@ -1509,9 +1401,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.append_query_params", side_effect=lambda url, params: str(url), @@ -1535,9 +1425,7 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) def test_proxy_default_api_version_uses_azure_default( self, mock_should_update, mock_setup_db, mock_atexit_register, mock_uvicorn_run ): @@ -1554,11 +1442,7 @@ class TestProxyInitializationHelpers: KeyManagementSettings=MagicMock(), save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -1568,9 +1452,7 @@ class TestProxyInitializationHelpers: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1578,9 +1460,7 @@ class TestProxyInitializationHelpers: "port": 8000, } result = runner.invoke(run_server, ["--local", "--skip_server_startup"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_proxy_module.save_worker_config.assert_called_once() call_kwargs = mock_proxy_module.save_worker_config.call_args[1] assert call_kwargs["api_version"] == litellm.AZURE_DEFAULT_API_VERSION @@ -1588,12 +1468,8 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("builtins.print") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) - def test_keepalive_timeout_flag( - self, mock_should_update, mock_setup_db, mock_print, mock_uvicorn_run - ): + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) + def test_keepalive_timeout_flag(self, mock_should_update, mock_setup_db, mock_print, mock_uvicorn_run): """Test that the keepalive_timeout flag is properly passed to uvicorn""" from click.testing import CliRunner @@ -1610,11 +1486,7 @@ class TestProxyInitializationHelpers: # DB-setup block (un-timeout'd `subprocess.run(["prisma"])` + # migrate-deploy retry loop) — same isolation every other run_server # test in this file uses. - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), @@ -1629,9 +1501,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.ProxyInitializationHelpers.is_port_in_use", return_value=False, @@ -1664,12 +1534,8 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("builtins.print") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch( - "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False - ) - def test_timeout_worker_healthcheck_flag( - self, mock_should_update, mock_setup_db, mock_print, mock_uvicorn_run - ): + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) + def test_timeout_worker_healthcheck_flag(self, mock_should_update, mock_setup_db, mock_print, mock_uvicorn_run): """Test that the --timeout_worker_healthcheck flag is threaded through to the uvicorn init helper.""" from click.testing import CliRunner @@ -1686,11 +1552,7 @@ class TestProxyInitializationHelpers: # DB-setup block (un-timeout'd `subprocess.run(["prisma"])` + # migrate-deploy retry loop) — same isolation every other run_server # test in this file uses. - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), @@ -1705,9 +1567,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, patch( "litellm.proxy.proxy_cli.ProxyInitializationHelpers.is_port_in_use", return_value=False, @@ -1719,9 +1579,7 @@ class TestProxyInitializationHelpers: "port": 8000, } - result = runner.invoke( - run_server, ["--local", "--timeout_worker_healthcheck", "15"] - ) + result = runner.invoke(run_server, ["--local", "--timeout_worker_healthcheck", "15"]) assert result.exit_code == 0 mock_get_args.assert_called_once_with( @@ -1736,9 +1594,7 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("builtins.print") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - def test_max_requests_before_restart_flag( - self, mock_setup_db, mock_print, mock_uvicorn_run - ): + def test_max_requests_before_restart_flag(self, mock_setup_db, mock_print, mock_uvicorn_run): """Test that the max_requests_before_restart flag is passed to uvicorn as limit_max_requests""" from click.testing import CliRunner @@ -1751,11 +1607,7 @@ class TestProxyInitializationHelpers: mock_key_mgmt = MagicMock() mock_save_worker_config = MagicMock() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict( os.environ, @@ -1773,9 +1625,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1783,13 +1633,9 @@ class TestProxyInitializationHelpers: "port": 8000, } - result = runner.invoke( - run_server, ["--local", "--max_requests_before_restart", "123"] - ) + result = runner.invoke(run_server, ["--local", "--max_requests_before_restart", "123"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() # Check that uvicorn.run was called with limit_max_requests parameter @@ -1799,9 +1645,7 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("builtins.print") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - def test_max_requests_before_restart_jitter_flag( - self, mock_setup_db, mock_print, mock_uvicorn_run - ): + def test_max_requests_before_restart_jitter_flag(self, mock_setup_db, mock_print, mock_uvicorn_run): """--max_requests_before_restart_jitter maps to uvicorn limit_max_requests_jitter""" from click.testing import CliRunner @@ -1812,11 +1656,7 @@ class TestProxyInitializationHelpers: pass runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch("uvicorn.Config", _NewUvicornConfig), @@ -1831,9 +1671,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1852,9 +1690,7 @@ class TestProxyInitializationHelpers: ], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() call_args = mock_uvicorn_run.call_args assert call_args[1]["limit_max_requests"] == 1000 @@ -1873,11 +1709,7 @@ class TestProxyInitializationHelpers: from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -1891,9 +1723,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1913,9 +1743,7 @@ class TestProxyInitializationHelpers: ], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_not_called() mock_run_gunicorn.assert_called_once() g_kwargs = mock_run_gunicorn.call_args[1] @@ -1961,20 +1789,14 @@ class TestProxyInitializationHelpers: @patch("uvicorn.run") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - def test_single_worker_max_requests_warning_printed_on_uvicorn( - self, mock_setup_db, mock_uvicorn_run - ): + def test_single_worker_max_requests_warning_printed_on_uvicorn(self, mock_setup_db, mock_uvicorn_run): """uvicorn single-worker startup prints the does-not-recycle warning""" from click.testing import CliRunner from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -1988,9 +1810,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -1998,13 +1818,9 @@ class TestProxyInitializationHelpers: "port": 8000, } - result = runner.invoke( - run_server, ["--local", "--max_requests_before_restart", "123"] - ) + result = runner.invoke(run_server, ["--local", "--max_requests_before_restart", "123"]) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert "--max_requests_before_restart 123 with --num_workers 1" in result.output assert "uvicorn exits the process" in result.output @@ -2021,11 +1837,7 @@ class TestProxyInitializationHelpers: from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -2039,9 +1851,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2054,12 +1864,8 @@ class TestProxyInitializationHelpers: ["--local", "--num_workers", "2", "--max_requests_before_restart", "123"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" - assert not any( - "does not recycle a worker" in str(c) for c in mock_print.call_args_list - ) + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" + assert not any("does not recycle a worker" in str(c) for c in mock_print.call_args_list) @patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.run_gunicorn_server") @patch("uvicorn.run") @@ -2073,11 +1879,7 @@ class TestProxyInitializationHelpers: from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -2091,9 +1893,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2106,9 +1906,7 @@ class TestProxyInitializationHelpers: ["--local", "--run_gunicorn", "--max_requests_before_restart", "123"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert "--max_requests_before_restart 123 with --num_workers 1" in result.output assert "gunicorn accepts no connections" in result.output @@ -2125,11 +1923,7 @@ class TestProxyInitializationHelpers: from litellm.proxy.proxy_cli import run_server runner = CliRunner() - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} with ( patch.dict(os.environ, clean_env, clear=True), patch.dict( @@ -2143,9 +1937,7 @@ class TestProxyInitializationHelpers: ) }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2158,13 +1950,9 @@ class TestProxyInitializationHelpers: ["--local", "--run_hypercorn", "--max_requests_before_restart", "5"], ) - assert ( - result.exit_code == 0 - ), f"exit_code={result.exit_code}, output={result.output}" + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_init_hypercorn.assert_called_once() - assert not any( - "does not recycle a worker" in str(c) for c in mock_print.call_args_list - ) + assert not any("does not recycle a worker" in str(c) for c in mock_print.call_args_list) @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") def test_gunicorn_options_include_max_requests_jitter(self): @@ -2244,6 +2032,89 @@ class TestProxyInitializationHelpers: assert worker.config.h11_max_incomplete_event_size == max_request_header_size assert captured["options"]["graceful_timeout"] == 8 + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS + @staticmethod + def _gunicorn_application(**listener: object): + captured: Final[list] = [] + + def capture_run(application) -> None: + captured.append(application) + + with ( + patch("gunicorn.app.base.BaseApplication.run", capture_run), + patch( # test-quality-ok: option tests must not start a thread or change the pytest worker's child ownership + "litellm.proxy.proxy_cli.start_query_engine_reaper" + ), + ): + ProxyInitializationHelpers._run_gunicorn_server( + host="0.0.0.0", + port=4012, + app=MagicMock(), + num_workers=2, + ssl_certfile_path=None, + ssl_keyfile_path=None, + **listener, + ) + return captured[0] + + @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") + def test_gunicorn_serving_clients_directly_keeps_its_host_port_and_its_own_trusted_proxies(self): + pytest.importorskip("gunicorn") + import gunicorn.config + + config: Final = self._gunicorn_application().cfg + assert (config.address, config.forwarded_allow_ips) == ( + [("0.0.0.0", 4012)], + gunicorn.config.Config().forwarded_allow_ips, + ) + + @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") + def test_gunicorn_listens_on_the_socket_it_is_handed_instead_of_its_host_and_port(self): + pytest.importorskip("gunicorn") + import socket + + import gunicorn.glogging + import gunicorn.sock + + with socket.socket() as handed: + handed.bind(("127.0.0.1", 0)) + handed.listen() + config: Final = self._gunicorn_application(private_listener=InheritedFd(os.dup(handed.fileno()))).cfg + listeners: Final = gunicorn.sock.create_sockets(config, gunicorn.glogging.Logger(config)) + try: + assert [listener.getsockname() for listener in listeners] == [handed.getsockname()] + finally: + gunicorn.sock.close_sockets(listeners, unlink=False) + + @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") + def test_gunicorn_workers_on_a_private_listener_ignore_a_forwarded_client(self, monkeypatch): + pytest.importorskip("gunicorn") + import gunicorn.config + import gunicorn.glogging + + for name in ("uvicorn.error", "uvicorn.access"): + uvicorn_logger = logging.getLogger(name) + for attr in ("handlers", "level", "propagate"): + monkeypatch.setattr(uvicorn_logger, attr, getattr(uvicorn_logger, attr)) + application: Final = self._gunicorn_application(private_listener=AssignedPort(45556)) + worker: Final = application.options["worker_class"]( + 0, os.getpid(), [], MagicMock(), 30, application.cfg, gunicorn.glogging.Logger(gunicorn.config.Config()) + ) + seen: Final[list[str]] = [] + + async def probe(scope, receive, send) -> None: + seen.append(scope["client"][0]) + + worker.config.app = probe + worker.config.load() + request_from_loopback: Final = { + "type": "http", + "client": ("127.0.0.1", 51234), + "headers": [(b"x-forwarded-for", b"203.0.113.9")], + } + asyncio.run(worker.config.loaded_app(request_from_loopback, AsyncMock(), AsyncMock())) + + assert seen == ["127.0.0.1"] + @pytest.mark.skipif(os.name == "nt", reason="gunicorn server path skips Windows") def test_gunicorn_master_is_reserved_for_forking_before_it_runs(self, fork_reservation): """preload forks workers from the master, so native routes are forbidden there first""" @@ -2370,7 +2241,9 @@ class TestProxyInitializationHelpers: with patch.dict(os.environ, test_env_special): result = construct_database_url_from_env_vars() - expected_url = "postgresql://user%40with%2Bspecial:test-password-special-chars@localhost:5432/db_name%2Ftest" + expected_url = ( + "postgresql://user%40with%2Bspecial:test-password-special-chars@localhost:5432/db_name%2Ftest" + ) assert result == expected_url # Test without password (should still work) @@ -2515,12 +2388,8 @@ class TestQueryEngineReaperWiring: }, ), patch("uvicorn.run") as mock_uvicorn_run, - patch( - "litellm.proxy.proxy_cli.start_query_engine_reaper" - ) as mock_start_reaper, - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.start_query_engine_reaper") as mock_start_reaper, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2531,17 +2400,13 @@ class TestQueryEngineReaperWiring: return result, mock_uvicorn_run, mock_start_reaper def test_multi_worker_uvicorn_starts_reaper(self): - result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server( - ["--local", "--num_workers", "2"] - ) + result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server(["--local", "--num_workers", "2"]) assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() mock_start_reaper.assert_called_once() def test_single_worker_uvicorn_does_not_start_reaper(self): - result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server( - ["--local", "--num_workers", "1"] - ) + result, mock_uvicorn_run, mock_start_reaper = self._invoke_run_server(["--local", "--num_workers", "1"]) assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() mock_start_reaper.assert_not_called() @@ -2566,9 +2431,7 @@ class TestQueryEngineReaperWiring: with ( patch("gunicorn.app.base.BaseApplication.run"), - patch( - "litellm.proxy.proxy_cli.start_query_engine_reaper" - ) as mock_start_reaper, + patch("litellm.proxy.proxy_cli.start_query_engine_reaper") as mock_start_reaper, ): ProxyInitializationHelpers._run_gunicorn_server( host="127.0.0.1", @@ -2616,11 +2479,7 @@ class TestRunServerDbSetup: save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" with ( @@ -2632,9 +2491,7 @@ class TestRunServerDbSetup: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2665,10 +2522,18 @@ class TestRunServerDbSetup: mock_setup_database.assert_called_with(use_migrate=False, use_v2_resolver=True) @patch("atexit.register") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above - @patch("litellm.proxy.db.prisma_client.PrismaManager.build_request_log_indexes") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above - @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above - @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.prisma_client.PrismaManager.setup_database" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.prisma_client.PrismaManager.build_request_log_indexes" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.check_migration.check_prisma_schema_diff" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema" + ) # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above def test_migrations_run_when_the_prisma_cli_is_not_on_path( self, mock_should_update_schema, @@ -2692,11 +2557,7 @@ class TestRunServerDbSetup: save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" clean_env["PATH"] = str(empty_bin) @@ -2856,11 +2717,7 @@ class TestRunServerDbSetup: save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" with ( @@ -2872,9 +2729,7 @@ class TestRunServerDbSetup: "litellm.proxy.proxy_server": mock_proxy_module, }, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -2917,11 +2772,7 @@ class TestRunServerDbSetup: save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["DATABASE_URL"] = "sqlite:///data/litellm.db" with ( @@ -2935,9 +2786,7 @@ class TestRunServerDbSetup: ), ): with pytest.raises(SystemExit) as exc_info: - run_server.main( - ["--local", "--skip_server_startup"], standalone_mode=False - ) + run_server.main(["--local", "--skip_server_startup"], standalone_mode=False) assert exc_info.value.code == 1 mock_setup_database.assert_not_called() err: Final = capsys.readouterr().err @@ -2979,11 +2828,7 @@ class TestRunServerDbSetup: save_worker_config=MagicMock(), ) - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" clean_env["USE_V2_MIGRATION_RESOLVER"] = "true" @@ -2997,9 +2842,7 @@ class TestRunServerDbSetup: }, ), ): - run_server.main( - ["--local", "--skip_server_startup"], standalone_mode=False - ) + run_server.main(["--local", "--skip_server_startup"], standalone_mode=False) mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True) assert "--use_v2_migration_resolver is deprecated" not in capsys.readouterr().out @@ -3042,9 +2885,7 @@ class TestRunServerDbSetup: {"proxy_server": mock_proxy_module, "litellm.proxy.proxy_server": mock_proxy_module}, ), ): - run_server.main( - ["--local", "--skip_server_startup", "--use_v2_migration_resolver"], standalone_mode=False - ) + run_server.main(["--local", "--skip_server_startup", "--use_v2_migration_resolver"], standalone_mode=False) mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True) assert "--use_v2_migration_resolver is deprecated and has no effect" in capsys.readouterr().out @@ -3181,12 +3022,7 @@ class TestRunServerDbSetup: def test_resolve_v2_migration_resolver(self, use_legacy_flag, env_value, expected): from litellm.proxy.proxy_cli import resolve_v2_migration_resolver - assert ( - resolve_v2_migration_resolver( - use_legacy_flag=use_legacy_flag, env_value=env_value - ) - is expected - ) + assert resolve_v2_migration_resolver(use_legacy_flag=use_legacy_flag, env_value=env_value) is expected def test_deprecated_v2_flag_not_reported_outside_a_cli_invocation(self): from litellm.proxy.proxy_cli import deprecated_v2_flag_passed_on_cli @@ -3227,9 +3063,7 @@ class TestRunServerDbSetup: ) clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") + k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") } clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" @@ -3310,9 +3144,7 @@ class TestRunServerDbSetup: "sys.modules", {"proxy_server": mock_proxy_module, "litellm.proxy.proxy_server": mock_proxy_module}, ), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, outcome as exc_info, ): mock_get_args.return_value = {"app": "litellm.proxy.proxy_server:app", "host": "localhost", "port": 8000} @@ -3364,11 +3196,7 @@ class TestWorkerStartupHooks: "LITELLM_WORKER_STARTUP_HOOKS": "tests.unit.proxy.test_proxy_cli:_dummy_hook", } # Remove DATABASE_URL to avoid real DB setup - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env.update(env_overrides) with patch.dict(os.environ, clean_env, clear=True): @@ -3391,11 +3219,7 @@ class TestWorkerStartupHooks: env_overrides = { "LITELLM_WORKER_STARTUP_HOOKS": "tests.unit.proxy.test_proxy_cli:_dummy_async_hook", } - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env.update(env_overrides) with patch.dict(os.environ, clean_env, clear=True): @@ -3415,11 +3239,7 @@ class TestWorkerStartupHooks: env_overrides = { "LITELLM_WORKER_STARTUP_HOOKS": "tests.unit.proxy.test_proxy_cli:_failing_hook", } - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env.update(env_overrides) with patch.dict(os.environ, clean_env, clear=True): @@ -3448,18 +3268,11 @@ class TestWorkerStartupHooks: from litellm.proxy.proxy_server import proxy_startup_event - hooks = ( - "tests.unit.proxy.test_proxy_cli:_dummy_hook," - "tests.unit.proxy.test_proxy_cli:_dummy_async_hook" - ) + hooks = "tests.unit.proxy.test_proxy_cli:_dummy_hook,tests.unit.proxy.test_proxy_cli:_dummy_async_hook" env_overrides = { "LITELLM_WORKER_STARTUP_HOOKS": hooks, } - clean_env = { - k: v - for k, v in os.environ.items() - if k not in ("DATABASE_URL", "DIRECT_URL") - } + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} clean_env.update(env_overrides) with patch.dict(os.environ, clean_env, clear=True): @@ -3605,13 +3418,9 @@ class TestPostgresStatementTimeoutOptions: import yaml config_path = tmp_path / "config.yaml" - config_path.write_text( - yaml.dump({"model_list": [], "general_settings": {"database_statement_timeout": 60}}) - ) + config_path.write_text(yaml.dump({"model_list": [], "general_settings": {"database_statement_timeout": 60}})) - captured = _run_server_and_capture_urls( - str(config_path), direct_url="postgresql://t:t@localhost:5432/t" - ) + captured = _run_server_and_capture_urls(str(config_path), direct_url="postgresql://t:t@localhost:5432/t") assert "options" in urlparse.parse_qs(urlparse.urlparse(captured["DATABASE_URL"]).query) assert "options" not in urlparse.parse_qs(urlparse.urlparse(captured["DIRECT_URL"]).query) @@ -3783,9 +3592,7 @@ class TestReadReplicaConnectionParams: import yaml config_path: Final = tmp_path / "config.yaml" - config_path.write_text( - yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}}) - ) + config_path.write_text(yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}})) _run_server_and_capture_urls( str(config_path), @@ -3805,9 +3612,7 @@ class TestReadReplicaConnectionParams: import yaml config_path: Final = tmp_path / "config.yaml" - config_path.write_text( - yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}}) - ) + config_path.write_text(yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}})) _run_server_and_capture_urls( str(config_path), @@ -4002,9 +3807,7 @@ class TestTokenAuthCliFlags: patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False), patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database"), patch("uvicorn.run"), - patch( - "litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args" - ) as mock_get_args, + patch("litellm.proxy.proxy_cli.ProxyInitializationHelpers.get_default_unvicorn_init_args") as mock_get_args, ): mock_get_args.return_value = { "app": "litellm.proxy.proxy_server:app", @@ -4018,9 +3821,7 @@ class TestTokenAuthCliFlags: return result, database_url, toggle def test_azure_flag_assembles_a_token_bearing_database_url(self): - result, database_url, toggle = self._invoke_with_azure_host( - ["--local", "--azure_postgresql_auth"] - ) + result, database_url, toggle = self._invoke_with_azure_host(["--local", "--azure_postgresql_auth"]) assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert database_url is not None @@ -4248,7 +4049,11 @@ def probe_logger() -> Iterator[logging.Logger]: def _log_config_source(kind: str, tmp_path: Path) -> str | dict[str, object]: - config: Final[dict[str, object]] = {"version": 1, "incremental": True, "loggers": {_PROBE_LOGGER: {"level": "ERROR"}}} + config: Final[dict[str, object]] = { + "version": 1, + "incremental": True, + "loggers": {_PROBE_LOGGER: {"level": "ERROR"}}, + } if kind == "dict": return config written: Final = tmp_path / f"log_config.{kind}" diff --git a/tests/unit/proxy/test_server_listener.py b/tests/unit/proxy/test_server_listener.py new file mode 100644 index 00000000000..ec2f7a44ad4 --- /dev/null +++ b/tests/unit/proxy/test_server_listener.py @@ -0,0 +1,28 @@ +import os +from typing import Final + +import pytest + +from litellm.proxy.server_listener import AssignedPort, InheritedFd, PrivateListener, gunicorn_overrides + + +def test_gunicorn_keeps_its_own_listener_and_trust_without_a_private_listener() -> None: + assert dict(gunicorn_overrides(None)) == {} + + +@pytest.mark.skipif(os.name == "nt", reason="gunicorn does not run on Windows") +@pytest.mark.parametrize( + ("listener", "address"), + [ + (InheritedFd(17), 17), + (AssignedPort(45556), ("127.0.0.1", 45556)), + ], +) +def test_gunicorn_serves_a_private_listener_and_trusts_no_peer_to_name_the_client( + listener: PrivateListener, address: int | tuple[str, int] +) -> None: + gunicorn_config: Final = pytest.importorskip("gunicorn.config").Config() + gunicorn_config.set("forwarded_allow_ips", "*") + for name, value in gunicorn_overrides(listener).items(): + gunicorn_config.set(name, value) + assert (gunicorn_config.address, gunicorn_config.forwarded_allow_ips) == ([address], []) diff --git a/tests/unit/rust_bridge/test_configuration.py b/tests/unit/rust_bridge/test_configuration.py index 8116cf98076..70223421471 100644 --- a/tests/unit/rust_bridge/test_configuration.py +++ b/tests/unit/rust_bridge/test_configuration.py @@ -106,8 +106,10 @@ def test_process_override_applies_when_environment_is_unset() -> None: assert configuration.rust_enabled() is True -@pytest.mark.parametrize("value", ("", " ", "sometimes", "2")) -def test_invalid_environment_value_is_ignored(monkeypatch: pytest.MonkeyPatch, value: str) -> None: +@pytest.mark.parametrize("value", ("", " ", "sometimes", "2", "enabled")) +def test_blank_or_invalid_environment_value_leaves_the_switch_unset( + monkeypatch: pytest.MonkeyPatch, value: str +) -> None: monkeypatch.setenv("LITELLM_RUST", value) assert configuration.rust_enabled() is False @@ -116,6 +118,13 @@ def test_invalid_environment_value_is_ignored(monkeypatch: pytest.MonkeyPatch, v assert configuration.rust_enabled() is True +def test_environment_is_read_on_every_decision(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "1") + assert configuration.rust_enabled() is True + monkeypatch.setenv("LITELLM_RUST", "0") + assert configuration.rust_enabled() is False + + def test_process_override_and_reset_apply_to_existing_threads() -> None: with ThreadPoolExecutor(max_workers=1) as executor: assert executor.submit(configuration.rust_enabled).result() is False From 95ecfbbac60091dfb665e73c5f0eb7ef8ed899d7 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Thu, 12 Nov 2026 04:40:00 +0900 Subject: [PATCH 102/189] feat(proxy): supervise the opt-in Rust forwarding server (#49606) * feat(proxy): supervise the opt-in Rust forwarding server With LITELLM_RUST on, `litellm` and `python -m gateway.launch` start a supervisor that runs the Rust gateway on the public port and Python on a private loopback listener. Python still serves every route and sees each request as if the client had connected to it directly. Rust sends the client's TCP peer in `x-litellm-rust-peer`. The supervised server turns server-level proxy headers off, and a middleware restores the peer and then applies the operator's own trust, so client IPs, trusted proxies and every IP-based setting behave the same with Rust on or off. The supervisor is transparent to Python. Signals pass through, Python's exit status is the result, and both children shut down when the supervisor's end of their stdin closes, so nothing outlives it. Rust starts only once Python listens. When forwarding cannot start (no gateway binary, TLS, a Unix socket, Hypercorn or Granian), the entry point warns and serves from Python alone. Every forwarding decision lives in litellm/proxy/rust_forward, paired with the gateway-forward crate. OS differences sit behind a Platform protocol, with Windows supported for local CLI runs. AGENTS.md files in the touched folders state the boundary. The `--rust_gateway` flag is gone in favor of the environment variable. * refactor(proxy): switch Rust forwarding on with LITELLM_RUST_FIRST Forwarding gets its own switch, independent of LITELLM_RUST, which keeps selecting only the Rust paths inside the SDK. Neither turns the other on. --------- Co-authored-by: narmstrong-berri --- .github/workflows/test-unit.yml | 1 + gateway/AGENTS.md | 16 + gateway/launch.py | 81 +++- litellm-rust/crates/gateway-forward/AGENTS.md | 3 + .../crates/gateway-forward/Cargo.toml | 2 +- .../crates/gateway-forward/src/input.rs | 29 ++ .../crates/gateway-forward/src/lib.rs | 2 + .../crates/gateway-forward/tests/lifecycle.rs | 53 ++- .../gateway-forward/tests/support/mod.rs | 29 +- litellm-rust/crates/gateway/src/main.rs | 6 +- litellm/proxy/AGENTS.md | 5 + litellm/proxy/proxy_cli.py | 116 +++++- litellm/proxy/rust_forward/AGENTS.md | 48 +++ litellm/proxy/rust_forward/__init__.py | 0 litellm/proxy/rust_forward/mode.py | 142 +++++++ litellm/proxy/rust_forward/peer.py | 69 ++++ litellm/proxy/rust_forward/platform/AGENTS.md | 19 + .../proxy/rust_forward/platform/__init__.py | 58 +++ litellm/proxy/rust_forward/platform/posix.py | 64 +++ .../proxy/rust_forward/platform/windows.py | 68 +++ litellm/proxy/rust_forward/server.py | 75 ++++ .../proxy/rust_forward/supervisor/AGENTS.md | 21 + .../proxy/rust_forward/supervisor/__init__.py | 55 +++ .../proxy/rust_forward/supervisor/children.py | 43 ++ .../rust_forward/supervisor/lifecycle.py | 138 +++++++ .../rust_forward/supervisor/preflight.py | 28 ++ .../proxy/rust_forward/supervisor/runtime.py | 194 +++++++++ litellm/rust_bridge/AGENTS.md | 5 + litellm_cli/main.py | 8 +- tests/integration/_support/process.py | 5 + tests/integration/_support/rust_forward.py | 117 ++++++ .../configuration/test_rust_forward.py | 110 +++++ .../test_rust_forward_lifecycle.py | 269 ++++++++++++ .../test_rust_forward_traffic.py | 387 ++++++++++++++++++ tests/unit/gateway/test_launch.py | 160 +++++++- tests/unit/proxy/rust_forward/__init__.py | 0 .../proxy/rust_forward/supervisor/__init__.py | 0 .../proxy/rust_forward/supervisor/_support.py | 7 + .../rust_forward/supervisor/test_children.py | 54 +++ .../rust_forward/supervisor/test_lifecycle.py | 158 +++++++ .../rust_forward/supervisor/test_preflight.py | 77 ++++ .../rust_forward/supervisor/test_runtime.py | 189 +++++++++ tests/unit/proxy/rust_forward/test_mode.py | 127 ++++++ tests/unit/proxy/rust_forward/test_peer.py | 135 ++++++ tests/unit/proxy/test_proxy_cli.py | 155 ++++++- tests/windows_tests/AGENTS.md | 7 + .../test_rust_forward_platform.py | 35 ++ 47 files changed, 3344 insertions(+), 26 deletions(-) create mode 100644 gateway/AGENTS.md create mode 100644 litellm-rust/crates/gateway-forward/src/input.rs create mode 100644 litellm/proxy/AGENTS.md create mode 100644 litellm/proxy/rust_forward/AGENTS.md create mode 100644 litellm/proxy/rust_forward/__init__.py create mode 100644 litellm/proxy/rust_forward/mode.py create mode 100644 litellm/proxy/rust_forward/peer.py create mode 100644 litellm/proxy/rust_forward/platform/AGENTS.md create mode 100644 litellm/proxy/rust_forward/platform/__init__.py create mode 100644 litellm/proxy/rust_forward/platform/posix.py create mode 100644 litellm/proxy/rust_forward/platform/windows.py create mode 100644 litellm/proxy/rust_forward/server.py create mode 100644 litellm/proxy/rust_forward/supervisor/AGENTS.md create mode 100644 litellm/proxy/rust_forward/supervisor/__init__.py create mode 100644 litellm/proxy/rust_forward/supervisor/children.py create mode 100644 litellm/proxy/rust_forward/supervisor/lifecycle.py create mode 100644 litellm/proxy/rust_forward/supervisor/preflight.py create mode 100644 litellm/proxy/rust_forward/supervisor/runtime.py create mode 100644 tests/integration/_support/rust_forward.py create mode 100644 tests/integration/configuration/test_rust_forward.py create mode 100644 tests/integration/configuration/test_rust_forward_lifecycle.py create mode 100644 tests/integration/configuration/test_rust_forward_traffic.py create mode 100644 tests/unit/proxy/rust_forward/__init__.py create mode 100644 tests/unit/proxy/rust_forward/supervisor/__init__.py create mode 100644 tests/unit/proxy/rust_forward/supervisor/_support.py create mode 100644 tests/unit/proxy/rust_forward/supervisor/test_children.py create mode 100644 tests/unit/proxy/rust_forward/supervisor/test_lifecycle.py create mode 100644 tests/unit/proxy/rust_forward/supervisor/test_preflight.py create mode 100644 tests/unit/proxy/rust_forward/supervisor/test_runtime.py create mode 100644 tests/unit/proxy/rust_forward/test_mode.py create mode 100644 tests/unit/proxy/rust_forward/test_peer.py create mode 100644 tests/windows_tests/AGENTS.md create mode 100644 tests/windows_tests/test_rust_forward_platform.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f8b93b5cc11..d0726e0daa0 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -591,6 +591,7 @@ jobs: tests/unit/proxy/credential_endpoints tests/unit/proxy/discovery_endpoints tests/unit/proxy/shutdown + tests/unit/proxy/rust_forward tests/unit/proxy/public_endpoints tests/unit/proxy/prompts tests/unit/proxy/rag_endpoints diff --git a/gateway/AGENTS.md b/gateway/AGENTS.md new file mode 100644 index 00000000000..1bb4013179f --- /dev/null +++ b/gateway/AGENTS.md @@ -0,0 +1,16 @@ +# Gateway image entry point + +`python -m gateway.launch` is how the gateway image starts. `launch.py` hands its arguments to uvicorn's own CLI and serves `main.py`, which trims the proxy app to the LLM data plane + +- Importing `launch.py` stays cheap. Every uvicorn worker imports it again before it can answer its first health ping, so LiteLLM imports wait until `main` or `worker_app` runs +- PgBouncer starts exactly once per pod, before the workers start, and the workers inherit the loopback URL through `DATABASE_URL`. `configure_database` is the only place that does this + +## Rust forwarding + +`launch.py` is an entry point of Rust forwarding, and every forwarding decision lives in `litellm/proxy/rust_forward/`, whose `AGENTS.md` states the contract + +- `main` dispatches on `rust_forward_mode` and does nothing else about forwarding. With forwarding off or refused it serves exactly as before +- `python_server` reads the flags and environment through uvicorn's own CLI parser, so Rust gets the values uvicorn would run with. Never reparse a uvicorn flag by hand +- Under forwarding the supervisor process runs `configure_database`, as `before_start`, once forwarding is known to be possible. The supervised child never runs it and inherits the exported URL +- The supervised child appends `mode.uvicorn_flags` after the operator's flags, so the private listener and `--no-proxy-headers` win +- `worker_app` wraps the app with `peer.wrap`, which is the identity when forwarding is off diff --git a/gateway/launch.py b/gateway/launch.py index b9303fd9447..c8b5b53af92 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -22,15 +22,16 @@ import sys from collections.abc import Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final +from typing_extensions import assert_never from uvicorn.main import main as uvicorn_main from litellm_cli.main import without_gc_pauses if TYPE_CHECKING: - from fastapi import FastAPI - from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings + from litellm.proxy.rust_forward.peer import ASGIApp + from litellm.proxy.rust_forward.server import PythonServer GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" @@ -65,16 +66,39 @@ def pool_database_url( return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) -def worker_app() -> "FastAPI": +def worker_app() -> "ASGIApp": with without_gc_pauses(): from gateway.main import app - return app + from litellm.proxy.rust_forward.mode import rust_forward_mode + from litellm.proxy.rust_forward.peer import wrap + return wrap(app, rust_forward_mode(os.environ)) def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") +def python_server(argv: Sequence[str]) -> "PythonServer": + """What uvicorn would serve the public port with, read from the same flags and environment it reads.""" + from litellm.proxy.rust_forward.server import PythonServer, Server, UvicornCliParams + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager + + params: Final = UvicornCliParams.model_validate( + uvicorn_main.make_context("uvicorn", list(uvicorn_argv(argv, os.environ))).params + ) + return PythonServer( + server=Server.UVICORN, + listener=params.listener, + host=params.host, + port=params.port, + keepalive_seconds=params.timeout_keep_alive, + graceful_timeout_seconds=GracefulShutdownManager.get_timeout() + if params.timeout_graceful_shutdown is None + else params.timeout_graceful_shutdown, + trusted_proxies=params.trusted_proxies, + ) + + def configure_database() -> None: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url @@ -88,10 +112,57 @@ def configure_database() -> None: export_pooled_database_url(pooled_url) -def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: +def _serve_python_alone(argv: Sequence[str], serve: Callable[[Sequence[str]], None]) -> None: configure_database() serve(uvicorn_argv(argv, os.environ)) +def _python_server_or_exit(argv: Sequence[str]) -> "PythonServer": + """Report ``--help``, ``--version`` and a bad flag the way uvicorn's own CLI does, since it never gets to.""" + import click + + try: + return python_server(argv) + except click.exceptions.Exit as stopped: + sys.exit(stopped.exit_code) + except click.ClickException as error: + error.show() + sys.exit(error.exit_code) + + +def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + from pydantic import ValidationError + + from litellm.proxy.rust_forward.mode import ( + Mode, + Supervised, + environment_error, + rust_forward_mode, + stop_with_supervisor, + uvicorn_flags, + ) + from litellm.proxy.rust_forward.supervisor import supervise + from litellm.proxy.rust_forward.supervisor.preflight import StartupError + + try: + mode: Final = rust_forward_mode(os.environ) + except ValidationError as error: + sys.exit(environment_error(error)) + match mode: + case Mode.DISABLED: + _serve_python_alone(argv, serve) + case Mode.SUPERVISOR: + supervised: Final = supervise(_python_server_or_exit(argv), before_start=configure_database) + if isinstance(supervised, StartupError): + _serve_python_alone(argv, serve) + else: + sys.exit(supervised) + case Supervised(): + stop_with_supervisor(mode) + serve((*uvicorn_argv(argv, os.environ), *uvicorn_flags(mode))) + case _: + assert_never(mode) + + if __name__ == "__main__": main(sys.argv[1:]) diff --git a/litellm-rust/crates/gateway-forward/AGENTS.md b/litellm-rust/crates/gateway-forward/AGENTS.md index 2135980266f..3d07f9ee342 100644 --- a/litellm-rust/crates/gateway-forward/AGENTS.md +++ b/litellm-rust/crates/gateway-forward/AGENTS.md @@ -13,6 +13,7 @@ - Pooled upstream connections are retired well before Python's keep-alive timeout, so a request never lands on a connection Python is closing - Idle client connections close after Python's keep-alive timeout, as uvicorn closed them, so a client that vanishes cannot hold a socket forever. Only the wait for the next request is timed, never an exchange in progress - WebSocket is the one protocol switch Python serves. Upgrades are relayed as raw bytes after the 101 and drained on shutdown +- Drain on EOF of stdin exactly as on SIGTERM. The supervisor holds the only write end, so EOF means it is gone and nobody is left to stop the gateway. `input_closed` reads on a plain thread, since a read blocked on tokio's blocking pool would hold up the runtime's exit ## Layout @@ -22,6 +23,8 @@ Every forwarding decision has one home. `rewrite.rs` owns what Python and the cl `Settings` reads only these variables, and only when `LITELLM_FORWARD_UPSTREAM` is set. The Python supervisor in `litellm/proxy/rust_forward/` sets every one of them from the values Python actually runs with, so all of them are required and a missing or invalid one fails startup naming it. The crate never reads Python's own settings (`HOST`, `PORT`, `KEEPALIVE_TIMEOUT`, `GRACEFUL_SHUTDOWN_TIMEOUT`) and keeps no copy of Python's defaults, since Python can get those values from CLI flags the environment never shows +Only the Python supervisor starts the gateway in forwarding mode, from `litellm` and from `python -m gateway.launch`, on a laptop and in every image. `supervisor/children.py` is the only writer of these variables, so adding, renaming or removing one changes `settings.rs` and `children.rust_spec` in the same change. Without `LITELLM_FORWARD_UPSTREAM` the binary runs as the standalone Rust gateway and this crate does nothing + | Variable | Meaning | |---|---| | `LITELLM_FORWARD_UPSTREAM` | Python's loopback origin, e.g. `http://127.0.0.1:53211` | diff --git a/litellm-rust/crates/gateway-forward/Cargo.toml b/litellm-rust/crates/gateway-forward/Cargo.toml index 2fef19dca34..821ea91d269 100644 --- a/litellm-rust/crates/gateway-forward/Cargo.toml +++ b/litellm-rust/crates/gateway-forward/Cargo.toml @@ -13,7 +13,7 @@ hyper = { version = "1.12.0", features = ["http1", "server"] } hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } serde.workspace = true thiserror.workspace = true -tokio = { workspace = true, features = ["io-util"] } +tokio = { workspace = true, features = ["io-util", "sync"] } tokio-util = { version = "0.7", features = ["rt"] } tower-service = "0.3.3" tracing.workspace = true diff --git a/litellm-rust/crates/gateway-forward/src/input.rs b/litellm-rust/crates/gateway-forward/src/input.rs new file mode 100644 index 00000000000..772d4610683 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/input.rs @@ -0,0 +1,29 @@ +use std::io::{self, Read}; + +use tokio::sync::oneshot; + +/// Resolves once `input` reaches end of file or fails to read. The Python supervisor holds the only +/// write end of the gateway's stdin, so EOF means it is gone and the gateway drains as on SIGTERM. +/// +/// The read runs on a plain thread because a blocked read on tokio's blocking pool would hold up +/// runtime shutdown after a signal had already stopped the gateway. +pub fn input_closed(mut input: impl Read + Send + 'static) -> impl Future + Send { + let (closed, on_closed) = oneshot::channel::<()>(); + let reader = std::thread::Builder::new() + .name("stdin-eof".into()) + .spawn(move || { + let _ = io::copy(&mut input, &mut io::sink()); + let _ = closed.send(()); + }); + async move { + match reader { + Ok(_) => { + let _ = on_closed.await; + } + Err(error) => { + tracing::warn!(%error, "could not watch stdin, so only signals stop the gateway"); + std::future::pending::<()>().await; + } + } + } +} diff --git a/litellm-rust/crates/gateway-forward/src/lib.rs b/litellm-rust/crates/gateway-forward/src/lib.rs index 4ac17b50544..35f25d19c3c 100644 --- a/litellm-rust/crates/gateway-forward/src/lib.rs +++ b/litellm-rust/crates/gateway-forward/src/lib.rs @@ -2,6 +2,7 @@ mod connections; mod connector; mod error; mod forward; +mod input; mod rewrite; mod server; mod settings; @@ -10,6 +11,7 @@ mod websocket; pub use error::Error; pub(crate) use error::Rejection; pub use forward::Forward; +pub use input::input_closed; pub use rewrite::PEER_HEADER; pub use settings::{Settings, Upstream}; diff --git a/litellm-rust/crates/gateway-forward/tests/lifecycle.rs b/litellm-rust/crates/gateway-forward/tests/lifecycle.rs index 85d10a37be4..97907738ad6 100644 --- a/litellm-rust/crates/gateway-forward/tests/lifecycle.rs +++ b/litellm-rust/crates/gateway-forward/tests/lifecycle.rs @@ -7,9 +7,11 @@ use std::{ use axum::{Router, body::Body, extract::Request, http::StatusCode, response::Response}; use http_body_util::BodyExt; +use litellm_gateway_forward::input_closed; use rstest::rstest; use support::{ - DRAIN, Gateway, Raw, client, raw_upstream, recording, serve, unreachable_upstream, within, + DRAIN, Gateway, Raw, UVICORN_KEEPALIVE, client, raw_upstream, recording, serve, serve_until, + unreachable_upstream, within, }; use tokio::{ net::{TcpListener, TcpStream}, @@ -287,6 +289,55 @@ async fn shutdown_lets_an_in_flight_stream_finish() { within(server).await.unwrap(); } +#[rstest] +#[tokio::test] +async fn the_supervisor_closing_stdin_drains_like_a_stop_signal() { + let release = Arc::new(Notify::new()); + let released = release.clone(); + let upstream = raw_upstream(move |mut python: Raw| { + let released = released.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n5\r\nfirst\r\n") + .await; + released.notified().await; + python.send("4\r\nlast\r\n0\r\n\r\n").await; + } + }) + .await; + let (stdin, supervisor) = std::io::pipe().unwrap(); + let (address, server) = serve_until(upstream, UVICORN_KEEPALIVE, input_closed(stdin)).await; + let response = within( + client().request( + Request::get(format!("http://{address}/")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(TcpStream::connect(address).await.is_ok()); + + drop(supervisor); + + within(async { + while TcpStream::connect(address).await.is_ok() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + assert!(!server.is_finished()); + release.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, "firstlast"); + within(server).await.unwrap(); +} + #[rstest] #[tokio::test] async fn shutdown_stops_waiting_for_a_stream_still_open_after_the_drain() { diff --git a/litellm-rust/crates/gateway-forward/tests/support/mod.rs b/litellm-rust/crates/gateway-forward/tests/support/mod.rs index d39532a456e..7d92ca4a14a 100644 --- a/litellm-rust/crates/gateway-forward/tests/support/mod.rs +++ b/litellm-rust/crates/gateway-forward/tests/support/mod.rs @@ -32,6 +32,21 @@ pub async fn within(future: impl Future) -> T { tokio::time::timeout(WAIT, future).await.expect("timed out") } +/// A forwarder in front of `upstream` that drains once `shutdown` resolves, as `run` does. +pub async fn serve_until( + upstream: SocketAddr, + keepalive: Duration, + shutdown: impl Future + Send + 'static, +) -> (SocketAddr, JoinHandle<()>) { + let forward = Forward::new(&format!("http://{upstream}").parse().unwrap(), keepalive); + let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = public.local_addr().unwrap(); + ( + address, + tokio::spawn(forward.serve(public, shutdown, DRAIN)), + ) +} + /// A running forwarder in front of `upstream`, listening on a real socket like production. pub struct Gateway { pub address: SocketAddr, @@ -45,17 +60,11 @@ impl Gateway { } pub async fn with_keepalive(upstream: SocketAddr, keepalive: Duration) -> Self { - let forward = Forward::new(&format!("http://{upstream}").parse().unwrap(), keepalive); - let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = public.local_addr().unwrap(); let (stop, stopped) = oneshot::channel::<()>(); - let server = tokio::spawn(forward.serve( - public, - async { - stopped.await.ok(); - }, - DRAIN, - )); + let (address, server) = serve_until(upstream, keepalive, async { + stopped.await.ok(); + }) + .await; Self { address, stop, diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index dc77cfe28eb..176bc68d247 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -48,7 +48,11 @@ async fn main() -> Result<(), Box> { level: log.log_level(), }) .install_global()?; - litellm_gateway_forward::run(forward, shutdown_signal()).await?; + let stdin_closed = litellm_gateway_forward::input_closed(std::io::stdin()); + litellm_gateway_forward::run(forward, async { + tokio::select! { () = shutdown_signal() => (), () = stdin_closed => () } + }) + .await?; return Ok(()); } let settings: Settings = settings::from_iter(std::env::vars_os())?; diff --git a/litellm/proxy/AGENTS.md b/litellm/proxy/AGENTS.md new file mode 100644 index 00000000000..62726c30a10 --- /dev/null +++ b/litellm/proxy/AGENTS.md @@ -0,0 +1,5 @@ +# Proxy + +## Rust forwarding + +`proxy_server.py`, routes, auth and middleware never know whether Rust is in front of the proxy, and none of them may branch on it. `proxy_cli.py` only describes what it would serve with as a `rust_forward.server.PythonServer`, calls `rust_forward.supervisor.supervise`, and merges the options `rust_forward.mode` returns, which are identities when forwarding is off. Every forwarding decision lives in `rust_forward/`, and `rust_forward/AGENTS.md` states the contract diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index b238cefd24f..f252e69d739 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -36,6 +36,9 @@ from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: from fastapi import FastAPI + + from litellm.proxy.rust_forward.peer import ASGIApp + from litellm.proxy.rust_forward.server import PythonServer else: FastAPI = Any @@ -680,7 +683,7 @@ class ProxyInitializationHelpers: def run_gunicorn_server( host: str, port: int, - app: FastAPI, + app: "ASGIApp", num_workers: int, ssl_certfile_path: str, ssl_keyfile_path: str, @@ -836,6 +839,65 @@ class ProxyInitializationHelpers: def serving_port(port: int) -> int: return random.randint(1024, 49152) if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port) else port + @staticmethod + def python_server( + host: str, + port: int, + keepalive_timeout: int | None, + run_gunicorn: bool, + run_hypercorn: bool, + run_granian: bool, + tls: bool, + ) -> "PythonServer": + """``run_gunicorn_server`` never hands ``--keepalive_timeout`` or forwarded IPs to gunicorn, so its workers keep + gunicorn's keep-alive and gunicorn's trusted proxies, which include ``::1`` where uvicorn's do not.""" + import uvicorn + + from litellm.proxy.rust_forward.server import Listener, PythonServer, Server, uvicorn_trusted_proxies + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager + + server: Final = ( + Server.HYPERCORN + if run_hypercorn + else Server.GRANIAN + if run_granian + else Server.GUNICORN + if run_gunicorn + else Server.UVICORN + ) + keepalive, trusted_proxies = ( + ProxyInitializationHelpers.gunicorn_keepalive_and_trusted_proxies() + if server is Server.GUNICORN + else ( + uvicorn.Config(app="", log_config=None).timeout_keep_alive + if keepalive_timeout is None + else keepalive_timeout, + uvicorn_trusted_proxies(None), + ) + ) + return PythonServer( + server=server, + listener=Listener.TLS if tls else Listener.TCP, + host=host, + port=ProxyInitializationHelpers.serving_port(port), + keepalive_seconds=keepalive, + graceful_timeout_seconds=GracefulShutdownManager.get_timeout(), + trusted_proxies=trusted_proxies, + ) + + @staticmethod + def gunicorn_keepalive_and_trusted_proxies() -> tuple[int, str]: + """Loaded only when gunicorn is the server, because gunicorn imports ``fcntl``, which Windows lacks.""" + from gunicorn.config import Config as GunicornConfig + + config: Final = GunicornConfig() + keepalive: Final[object] = config.keepalive + forwarded_allow_ips: Final[object] = config.forwarded_allow_ips + return ( + TypeAdapter(int).validate_python(keepalive), + ",".join(TypeAdapter(list[str]).validate_python(forwarded_allow_ips)), + ) + @staticmethod def get_loop_type(): """Helper function to determine the event loop type based on platform""" @@ -1276,7 +1338,7 @@ def run_server( skip_server_startup, validate_config: bool, check_migrations_against_existing_data: bool, - keepalive_timeout, + keepalive_timeout: int | None, timeout_worker_healthcheck, max_request_header_size: int | None, max_requests_before_restart: int | None, @@ -1306,6 +1368,49 @@ def run_server( run_setup_wizard() return + from pydantic import ValidationError + + from litellm.proxy.rust_forward.mode import ( + Mode, + environment_error, + private_listener, + rust_forward_mode, + stop_with_supervisor, + uvicorn_options, + ) + + try: + forward_mode: Final = rust_forward_mode(os.environ) + except ValidationError as error: + raise click.ClickException(environment_error(error)) from None + stop_with_supervisor(forward_mode) + runs_one_command: Final = ( + version is True + or validate_config + or health is True + or test is True + or skip_server_startup is True + or check_migrations_against_existing_data + ) + if forward_mode is Mode.SUPERVISOR and not runs_one_command: + from litellm.proxy.rust_forward.supervisor import supervise + from litellm.proxy.rust_forward.supervisor.preflight import StartupError + + server: Final = ProxyInitializationHelpers.python_server( + host, + port, + keepalive_timeout, + run_gunicorn=run_gunicorn is True, + run_hypercorn=run_hypercorn is True, + run_granian=run_granian is True, + tls=None not in (ssl_keyfile_path, ssl_certfile_path), + ) + if prometheus_metrics_port == server.port: + raise click.UsageError("--prometheus_metrics_port must differ from --port") + supervised: Final = supervise(server) + if not isinstance(supervised, StartupError): + raise SystemExit(supervised) + args: Final = locals() try: from litellm.proxy.proxy_server import ( @@ -1773,6 +1878,8 @@ def run_server( timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None), max_request_header_size=max_request_header_size, ) + for option, value in uvicorn_options(forward_mode, WORKER_APP).items(): + uvicorn_args[option] = value # Optional: recycle uvicorn workers after N requests if max_requests_before_restart is not None: uvicorn_args["limit_max_requests"] = max_requests_before_restart @@ -1837,16 +1944,19 @@ def run_server( workers=num_workers, ) elif run_gunicorn is True: + from litellm.proxy.rust_forward.peer import wrap + ProxyInitializationHelpers.run_gunicorn_server( host=host, port=port, - app=app, + app=wrap(app, forward_mode), num_workers=num_workers, ssl_certfile_path=ssl_certfile_path, ssl_keyfile_path=ssl_keyfile_path, max_requests_before_restart=max_requests_before_restart, max_requests_before_restart_jitter=max_requests_before_restart_jitter, max_request_header_size=max_request_header_size, + private_listener=private_listener(forward_mode), ) elif run_hypercorn is True: ProxyInitializationHelpers.init_hypercorn_server( diff --git a/litellm/proxy/rust_forward/AGENTS.md b/litellm/proxy/rust_forward/AGENTS.md new file mode 100644 index 00000000000..935ff3f896e --- /dev/null +++ b/litellm/proxy/rust_forward/AGENTS.md @@ -0,0 +1,48 @@ +# Rust forwarding, Python half + +With `LITELLM_RUST_FIRST` on, a supervisor runs the Rust gateway on the public port and Python on a private loopback listener. Rust forwards every request to Python unchanged, so Python still serves every route. This package is the Python half of forwarding and pairs with the crate `litellm-rust/crates/gateway-forward`. Once Rust serves every route, both get deleted together, along with one dispatch branch in each entry point + +The same path runs everywhere. `litellm` (the main images and a local CLI run) and `python -m gateway.launch` (the gateway image) both call `supervisor.supervise`, and nothing else starts the Rust gateway in forwarding mode. In a container the supervisor is PID 1, so its signal and exit rules are deployment behavior and not CLI polish + +## Boundary + +A supervised Python server differs from Python alone in exactly two ways, and nothing else in the proxy may change because Rust is in front + +- It listens on the private loopback listener the supervisor gives it, never on the public address +- It restores the client address Rust saw, because its own TCP peer is always Rust + +`proxy_server.py`, routes, auth and middleware never know Rust is there. A change that needs one of them to branch on forwarding is in the wrong place + +## Modules + +- `server.py` describes what an entry point would serve with on the public port, as a `PythonServer`. The supervisor reads nothing else about the entry point +- `mode.py` is the Python child's side. `Supervised` is the whole contract the supervisor hands the child, and the `uvicorn_*` and `gunicorn_*` functions translate it into each server's own options +- `peer.py` restores the client address +- `supervisor/` starts, watches and stops both children. Its rules are in `supervisor/AGENTS.md` +- `platform/` holds every OS-specific operation. Its rules are in `platform/AGENTS.md` + +## Entry points + +Entry points (`litellm/proxy/proxy_cli.py`, `gateway/launch.py`, `litellm_cli/main.py`) only translate between their own flags and this package + +- In the supervisor they describe what they would serve with as a `server.PythonServer` and hand it to `supervisor.supervise`, which decides whether forwarding can start and what Rust and the Python child receive. A refusal (no gateway binary, or a server or listener Rust cannot front) is reported and returned, and the entry point then serves from Python alone exactly as with `LITELLM_RUST_FIRST` off +- In the supervised child they merge `mode.uvicorn_options` or `mode.uvicorn_flags` into their server settings, or hand `mode.private_listener` to `run_gunicorn_server`, and wrap the app with `peer.wrap` in their worker factory. These are identities when forwarding is off, so entry points never branch on it +- A server is started three ways here, through uvicorn keyword arguments, uvicorn's own CLI and gunicorn, and each one states the same two facts, the private listener and that the server itself trusts no peer's forwarding headers. The uvicorn translations are in `mode.py` and turn proxy headers off. `run_gunicorn_server` applies `server_listener.gunicorn_overrides` for the `PrivateListener` it is given, which sets gunicorn's `bind` and an empty `forwarded_allow_ips`. A new way to start a server gets its own translation, never a branch in the entry point + +## Client address + +Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and the proxy app itself never knows Rust is in front of it + +- Rust sends the client's TCP peer in `x-litellm-rust-peer` and leaves every forwarding header untouched. It never appends itself to `X-Forwarded-For`, because the proxy parses that header on its own (`use_x_forwarded_for`, `mcp_xff_num_trusted_hops`) and an extra hop would shift what those settings resolve +- Server-level proxy headers are off in a supervised server, through `proxy_headers: False` for uvicorn, `--no-proxy-headers` for its CLI and an empty `forwarded_allow_ips` for gunicorn. The server's TCP peer is always loopback, which uvicorn and gunicorn trust by default, so leaving them on would let any client set its own address and scheme +- `peer.RustPeerMiddleware` puts the real peer back into the scope and then runs uvicorn's own `ProxyHeadersMiddleware` with the operator's trust. It wraps the app from outside, so it runs before anything that reads the client address +- The operator's trust is one string, `trusted_proxies`. It is resolved by the server library the entry point uses (`server.uvicorn_trusted_proxies` or gunicorn's config), never defaulted in this package. `server.NO_TRUSTED_PROXIES` stands for proxy headers the operator turned off, since a trust list that matches nobody behaves the same +- `peer.py` honors the peer header only from the loopback hop and always removes it. Any process on the host can connect to the private listener and send that header, so loopback is the trust boundary of the private listener. That widens what Python alone allows only when the operator's trust list leaves loopback out + +## Environment contracts + +Each environment contract is written and read in one place + +- The Python child's contract is the `mode.Supervised` type itself. `mode.supervised_environment` serializes it into one variable and `mode.rust_forward_mode` validates it back, so a new field needs no parsing code +- `supervisor.children.rust_spec` writes the `LITELLM_FORWARD_*` variables that `gateway-forward`'s `settings.rs` reads. Python owns every value Rust needs, so a new one gets a new variable there and a matching field in `settings.rs` in the same change +- `mode.rust_forward_mode` reads `LITELLM_RUST_FIRST` strictly, so an invalid value stops the CLI at startup with one line naming it. It is independent of `LITELLM_RUST`, which only selects the Rust paths inside the SDK in `litellm/rust_bridge/configuration.py`, so neither turns the other on diff --git a/litellm/proxy/rust_forward/__init__.py b/litellm/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/proxy/rust_forward/mode.py b/litellm/proxy/rust_forward/mode.py new file mode 100644 index 00000000000..1c417db345f --- /dev/null +++ b/litellm/proxy/rust_forward/mode.py @@ -0,0 +1,142 @@ +import contextlib +import os +import signal +import threading +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from enum import Enum, auto +from types import MappingProxyType +from typing import Final, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError +from typing_extensions import assert_never + +from litellm.proxy.server_listener import LOOPBACK_HOST, AssignedPort, InheritedFd, PrivateListener +from litellm.rust_bridge.configuration import RustSwitch + +SWITCH_ENV: Final = "LITELLM_RUST_FIRST" +SUPERVISED_ENV: Final = "_LITELLM_GATEWAY_SUPERVISED" +RUST_HOP: Final = LOOPBACK_HOST + + +class _Switch(BaseModel): + model_config = ConfigDict(frozen=True) + + rust: RustSwitch = Field(None, validation_alias=SWITCH_ENV) + + +class Mode(Enum): + DISABLED = auto() + SUPERVISOR = auto() + + +@dataclass(frozen=True, slots=True) +class Supervised: + listener: PrivateListener + trusted_proxies: str + + +RustForwardMode: TypeAlias = Mode | Supervised +_SUPERVISED: Final = TypeAdapter(Supervised) + + +def supervised_environment(supervised: Supervised) -> Mapping[str, str]: + return MappingProxyType({SUPERVISED_ENV: _SUPERVISED.dump_json(supervised).decode()}) + + +def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode: + switch: Final = _Switch.model_validate(environ) + supervised: Final = environ.get(SUPERVISED_ENV) + if supervised is not None: + return _SUPERVISED.validate_json(supervised) + return Mode.SUPERVISOR if switch.rust else Mode.DISABLED + + +def _wait_for_eof(fd: int) -> None: + with contextlib.suppress(OSError): + while os.read(fd, 4096): + pass + + +def _stop_when_closed(fd: int, stop: Callable[[], None]) -> None: + _wait_for_eof(fd) + stop() + + +def _terminate_self() -> None: + signal.raise_signal(signal.SIGTERM) + + +def stop_with_supervisor(mode: RustForwardMode, fd: int = 0, stop: Callable[[], None] = _terminate_self) -> None: + """Shut the supervised server down gracefully once the supervisor's end of its stdin closes, so it never outlives + a supervisor that was killed. Call it only in the process the supervisor started, not in its workers.""" + match mode: + case Supervised(): + threading.Thread(target=_stop_when_closed, args=(fd, stop), name="supervisor-stdin", daemon=True).start() + case Mode.DISABLED | Mode.SUPERVISOR: + pass + case _: + assert_never(mode) + + +def environment_error(error: ValidationError) -> str: + return "; ".join( + f"{'.'.join(str(part) for part in detail['loc'])} is invalid: {detail['msg']}" + for detail in error.errors(include_url=False) + ) + + +# A supervised server's TCP peer is always Rust on loopback, so server-level proxy headers would trust any +# client's forwarding headers. They stay off there and ``peer.wrap`` applies the operator's trust instead. + + +def _uvicorn_listener(listener: PrivateListener) -> Mapping[str, object]: + match listener: + case InheritedFd(fd): + return MappingProxyType({"fd": fd}) + case AssignedPort(port): + return MappingProxyType({"host": RUST_HOP, "port": port}) + case _: + assert_never(listener) + + +def uvicorn_options(mode: RustForwardMode, factory: str) -> Mapping[str, object]: + match mode: + case Supervised(listener): + return MappingProxyType( + {"app": factory, "factory": True, **_uvicorn_listener(listener), "proxy_headers": False} + ) + case Mode.DISABLED | Mode.SUPERVISOR: + return MappingProxyType({}) + case _: + assert_never(mode) + + +def _uvicorn_listener_flags(listener: PrivateListener) -> tuple[str, ...]: + match listener: + case InheritedFd(fd): + return ("--fd", str(fd)) + case AssignedPort(port): + return ("--host", RUST_HOP, "--port", str(port)) + case _: + assert_never(listener) + + +def uvicorn_flags(mode: RustForwardMode) -> tuple[str, ...]: + match mode: + case Supervised(listener): + return (*_uvicorn_listener_flags(listener), "--no-proxy-headers") + case Mode.DISABLED | Mode.SUPERVISOR: + return () + case _: + assert_never(mode) + + +def private_listener(mode: RustForwardMode) -> PrivateListener | None: + match mode: + case Supervised(listener): + return listener + case Mode.DISABLED | Mode.SUPERVISOR: + return None + case _: + assert_never(mode) diff --git a/litellm/proxy/rust_forward/peer.py b/litellm/proxy/rust_forward/peer.py new file mode 100644 index 00000000000..1dd52e24564 --- /dev/null +++ b/litellm/proxy/rust_forward/peer.py @@ -0,0 +1,69 @@ +from typing import ( + TYPE_CHECKING, + Final, + TypeAlias, + cast, # noqa: TID251 # the one Starlette-to-uvicorn ASGI conversion in served +) + +from typing_extensions import assert_never + +from litellm.proxy.rust_forward.mode import RUST_HOP, Mode, RustForwardMode, Supervised + +if TYPE_CHECKING: + import starlette.types + from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope + +PEER_HEADER: Final = b"x-litellm-rust-peer" + +ASGIApp: TypeAlias = "ASGI3Application" + + +def served(app: "starlette.types.ASGIApp") -> ASGIApp: + return cast( # cast-ok: Starlette and uvicorn type the same ASGI 3 callable with unassignable scope/receive/send + "ASGI3Application", app + ) + + +def _restore_peer(scope: "WWWScope") -> None: + headers: Final = tuple(scope["headers"]) + peer: Final = next((value for name, value in headers if name == PEER_HEADER), None) + if peer is None: + return + remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER) + scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop + client: Final = scope["client"] + if client is None or client[0] != RUST_HOP: + return + host, _, port = peer.decode("latin-1").rpartition(":") + scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope + + +class RustPeerMiddleware: + """Make Python see each request as if the client had connected to it directly. + + Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real + TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the + real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust + settings Python would have used on the public port. + """ + + def __init__(self, app: ASGIApp, mode: Supervised) -> None: + from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + + self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) + + async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + if scope["type"] != "lifespan": + _restore_peer(scope) + await self.app(scope, receive, send) + + +def wrap(app: "starlette.types.ASGIApp", mode: RustForwardMode) -> ASGIApp: + converted: Final = served(app) + match mode: + case Supervised(): + return RustPeerMiddleware(converted, mode) + case Mode.DISABLED | Mode.SUPERVISOR: + return converted + case _: + assert_never(mode) diff --git a/litellm/proxy/rust_forward/platform/AGENTS.md b/litellm/proxy/rust_forward/platform/AGENTS.md new file mode 100644 index 00000000000..32d1e4ac5c9 --- /dev/null +++ b/litellm/proxy/rust_forward/platform/AGENTS.md @@ -0,0 +1,19 @@ +# Platforms + +Every OS-specific operation sits behind `Platform`, which `supervise` picks once with `current_platform()`. The supervisor, `mode.py` and the entry points never check the OS themselves, so a new OS difference becomes a `Platform` member and never an `os.name` check somewhere else + +A `Platform` answers these questions and nothing else. What the gateway binary is called, which servers it can supervise, how the private listener reaches Python, which spawn options the children get, which signals are relayed and which of them stop Rust, and how a child is signalled and stopped + +## POSIX + +`PosixPlatform` binds the private socket itself and hands Python the open descriptor (`server_listener.InheritedFd`). Nobody else can take the port between the supervisor picking it and Python serving it. The socket does not listen until Python does, so the readiness probe only succeeds against Python. Children run in their own session (`start_new_session`) only so that a terminal Ctrl-C reaches Python once, through the supervisor + +## Windows + +Windows is supported for running the proxy locally from the CLI. Nobody deploys the proxy on Windows, but `LITELLM_RUST_FIRST` is meant to become the default, and then this is the path every Windows laptop runs. Revisit the choices below before treating Windows as a deployment target + +- Python gets an assigned loopback port (`server_listener.AssignedPort`) instead of an open socket, which uvicorn on Windows cannot serve. Another process can take that port before Python binds it, and the readiness probe would then accept it as Python. Python's failed bind still ends the run +- Each child runs in its own process group, which can only be sent `CTRL_BREAK_EVENT`. Both relayed signals arrive as that, so a second Ctrl-C does not force Python's exit the way it does on POSIX +- gunicorn is refused, since it does not run on Windows +- The Windows-only names are spelled out as constants in `windows.py` so the module imports and type checks on every OS. `tests/windows_tests/test_rust_forward_platform.py` pins them to the stdlib on the Windows runner +- No CI job runs the supervisor itself on Windows, because the `windows-sdk` job installs no proxy dependencies and no `litellm-gateway.exe`. Run a change to `windows.py` by hand on Windows until one does diff --git a/litellm/proxy/rust_forward/platform/__init__.py b/litellm/proxy/rust_forward/platform/__init__.py new file mode 100644 index 00000000000..4043adacdfb --- /dev/null +++ b/litellm/proxy/rust_forward/platform/__init__.py @@ -0,0 +1,58 @@ +import asyncio +import os +import signal +from collections.abc import Callable +from contextlib import AbstractContextManager +from dataclasses import dataclass +from typing import Protocol + +from litellm.proxy.rust_forward.server import Server +from litellm.proxy.server_listener import PrivateListener + + +@dataclass(frozen=True, slots=True) +class PlatformListener: + listener: PrivateListener + port: int + pass_fds: tuple[int, ...] + + +@dataclass(frozen=True, slots=True) +class SpawnOptions: + pass_fds: tuple[int, ...] + start_new_session: bool + creationflags: int + + +class Platform(Protocol): + @property + def name(self) -> str: ... + + @property + def gateway_name(self) -> str: ... + + @property + def servers(self) -> frozenset[Server]: ... + + def private_listener(self) -> AbstractContextManager[PlatformListener]: ... + + def spawn_options(self, pass_fds: tuple[int, ...]) -> SpawnOptions: ... + + @property + def relayed_signals(self) -> tuple[signal.Signals, ...]: ... + + @property + def stop_signals(self) -> frozenset[signal.Signals]: ... + + def catch(self, loop: asyncio.AbstractEventLoop, on_signal: Callable[[signal.Signals], None]) -> None: ... + + def signal_child(self, process: asyncio.subprocess.Process, signum: signal.Signals) -> None: ... + + def stop_group(self, process: asyncio.subprocess.Process) -> None: ... + + +def current_platform() -> Platform: + from litellm.proxy.rust_forward.platform.posix import PosixPlatform + from litellm.proxy.rust_forward.platform.windows import WindowsPlatform + + return WindowsPlatform() if os.name == "nt" else PosixPlatform() diff --git a/litellm/proxy/rust_forward/platform/posix.py b/litellm/proxy/rust_forward/platform/posix.py new file mode 100644 index 00000000000..0d0c6c96649 --- /dev/null +++ b/litellm/proxy/rust_forward/platform/posix.py @@ -0,0 +1,64 @@ +import asyncio +import contextlib +import os +import signal +import socket +from collections.abc import Callable, Generator +from dataclasses import dataclass +from typing import Final + +from pydantic import TypeAdapter + +from litellm.proxy.rust_forward.mode import RUST_HOP +from litellm.proxy.rust_forward.platform import PlatformListener, SpawnOptions +from litellm.proxy.rust_forward.server import Server +from litellm.proxy.server_listener import InheritedFd + +_ADDRESS: Final = TypeAdapter(tuple[str, int]) + + +@dataclass(frozen=True, slots=True) +class PosixPlatform: + name: str = "POSIX" + gateway_name: str = "litellm-gateway" + servers: frozenset[Server] = frozenset({Server.UVICORN, Server.GUNICORN}) + + @contextlib.contextmanager + def private_listener(self) -> Generator[PlatformListener]: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as private: + private.bind((RUST_HOP, 0)) + _, port = _ADDRESS.validate_python(private.getsockname()) + yield PlatformListener(InheritedFd(private.fileno()), port, (private.fileno(),)) + + def spawn_options(self, pass_fds: tuple[int, ...]) -> SpawnOptions: + return SpawnOptions(pass_fds=pass_fds, start_new_session=True, creationflags=0) + + @property + def relayed_signals(self) -> tuple[signal.Signals, ...]: + return ( + signal.SIGTERM, + signal.SIGINT, + signal.SIGHUP, + signal.SIGQUIT, + signal.SIGUSR1, + signal.SIGUSR2, + signal.SIGTTIN, + signal.SIGTTOU, + signal.SIGWINCH, + ) + + @property + def stop_signals(self) -> frozenset[signal.Signals]: + return frozenset({signal.SIGTERM, signal.SIGINT, signal.SIGQUIT}) + + def catch(self, loop: asyncio.AbstractEventLoop, on_signal: Callable[[signal.Signals], None]) -> None: + for signum in self.relayed_signals: + loop.add_signal_handler(signum, on_signal, signum) + + def signal_child(self, process: asyncio.subprocess.Process, signum: signal.Signals) -> None: + with contextlib.suppress(ProcessLookupError): + process.send_signal(signum) + + def stop_group(self, process: asyncio.subprocess.Process) -> None: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGTERM) diff --git a/litellm/proxy/rust_forward/platform/windows.py b/litellm/proxy/rust_forward/platform/windows.py new file mode 100644 index 00000000000..163b96b9b20 --- /dev/null +++ b/litellm/proxy/rust_forward/platform/windows.py @@ -0,0 +1,68 @@ +import asyncio +import contextlib +import signal +import socket +from collections.abc import Callable, Generator +from dataclasses import dataclass +from types import FrameType +from typing import Final + +from pydantic import TypeAdapter + +from litellm.proxy.rust_forward.mode import RUST_HOP +from litellm.proxy.rust_forward.platform import PlatformListener, SpawnOptions +from litellm.proxy.rust_forward.server import Server +from litellm.proxy.server_listener import AssignedPort + +_ADDRESS: Final = TypeAdapter(tuple[str, int]) +# signal.CTRL_BREAK_EVENT, signal.SIGBREAK and subprocess.CREATE_NEW_PROCESS_GROUP exist only on Windows, so they are +# spelled out to keep this module importable and type checked everywhere. A Windows-only test pins them to the stdlib. +CTRL_BREAK_EVENT: Final = 1 +SIGBREAK: Final = 21 +CREATE_NEW_PROCESS_GROUP: Final = 0x00000200 + + +# Windows support exists only for running the proxy locally from the CLI, never for deploying it. That is why the +# supervisor picks a free loopback port and hands Python its number instead of an open socket, which Windows uvicorn +# cannot serve. Another process could take that port before Python binds it, and the readiness probe would then +# accept it as Python. Python's failed bind still ends the run, which is acceptable on a developer's machine. +@dataclass(frozen=True, slots=True) +class WindowsPlatform: + name: str = "Windows" + gateway_name: str = "litellm-gateway.exe" + servers: frozenset[Server] = frozenset({Server.UVICORN}) + + @contextlib.contextmanager + def private_listener(self) -> Generator[PlatformListener]: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind((RUST_HOP, 0)) + _, port = _ADDRESS.validate_python(probe.getsockname()) + yield PlatformListener(AssignedPort(port), port, ()) + + def spawn_options(self, pass_fds: tuple[int, ...]) -> SpawnOptions: + return SpawnOptions(pass_fds=pass_fds, start_new_session=False, creationflags=CREATE_NEW_PROCESS_GROUP) + + @property + def relayed_signals(self) -> tuple[signal.Signals, ...]: + return (signal.SIGINT, signal.Signals(SIGBREAK)) + + @property + def stop_signals(self) -> frozenset[signal.Signals]: + return frozenset(self.relayed_signals) + + def catch(self, loop: asyncio.AbstractEventLoop, on_signal: Callable[[signal.Signals], None]) -> None: + def handle(signum: int, _frame: FrameType | None) -> None: + loop.call_soon_threadsafe(on_signal, signal.Signals(signum)) + + for signum in self.relayed_signals: + signal.signal(signum, handle) + + # A child in its own process group can only be sent CTRL_BREAK_EVENT, since Windows never delivers CTRL_C_EVENT + # to a new group. Both children shut down on it as on Ctrl-C, but uvicorn force-exits only on a repeated SIGINT, + # so a second Ctrl-C does not cut a Windows shutdown short the way it does on POSIX. + def signal_child(self, process: asyncio.subprocess.Process, signum: signal.Signals) -> None: + self.stop_group(process) + + def stop_group(self, process: asyncio.subprocess.Process) -> None: + with contextlib.suppress(ProcessLookupError): + process.send_signal(CTRL_BREAK_EVENT) diff --git a/litellm/proxy/rust_forward/server.py b/litellm/proxy/rust_forward/server.py new file mode 100644 index 00000000000..95a693291bf --- /dev/null +++ b/litellm/proxy/rust_forward/server.py @@ -0,0 +1,75 @@ +from dataclasses import dataclass +from enum import Enum +from typing import Final + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.proxy.server_listener import NO_TRUSTED_PROXIES + +_TRUSTED_PROXIES: Final = TypeAdapter(str) + + +class Server(Enum): + UVICORN = "uvicorn" + GUNICORN = "gunicorn" + HYPERCORN = "Hypercorn" + GRANIAN = "Granian" + + +class Listener(Enum): + TCP = "plain TCP" + TLS = "TLS" + UNIX_SOCKET = "a Unix socket" + INHERITED_FD = "an inherited file descriptor" + + +@dataclass(frozen=True, slots=True) +class PythonServer: + """What an entry point would serve with on the public port if Rust were not in front of it.""" + + server: Server + listener: Listener + host: str + port: int + keepalive_seconds: int + graceful_timeout_seconds: float + trusted_proxies: str + + +class UvicornCliParams(BaseModel): + """The options uvicorn's own CLI parsed that decide how it serves the public port.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + host: str + port: int + uds: str | None + fd: int | None + ssl_keyfile: str | None + ssl_certfile: str | None + timeout_keep_alive: int + timeout_graceful_shutdown: int | None + proxy_headers: bool + forwarded_allow_ips: str | None + + @property + def listener(self) -> Listener: + if self.ssl_keyfile is not None or self.ssl_certfile is not None: + return Listener.TLS + if self.uds is not None: + return Listener.UNIX_SOCKET + if self.fd is not None: + return Listener.INHERITED_FD + return Listener.TCP + + @property + def trusted_proxies(self) -> str: + return uvicorn_trusted_proxies(self.forwarded_allow_ips) if self.proxy_headers else NO_TRUSTED_PROXIES + + +def uvicorn_trusted_proxies(configured: str | None) -> str: + import uvicorn + + return _TRUSTED_PROXIES.validate_python( + uvicorn.Config(app="", log_config=None, forwarded_allow_ips=configured).forwarded_allow_ips + ) diff --git a/litellm/proxy/rust_forward/supervisor/AGENTS.md b/litellm/proxy/rust_forward/supervisor/AGENTS.md new file mode 100644 index 00000000000..65af84481e5 --- /dev/null +++ b/litellm/proxy/rust_forward/supervisor/AGENTS.md @@ -0,0 +1,21 @@ +# Forwarding supervisor + +`supervise` serves the public port from Rust and runs the entry point's own command behind it, or says why that cannot start. That command is the supervisor's own interpreter command line (`OWN_COMMAND`, from `sys.orig_argv`), so the child has the same import path and interpreter flags as the process the operator started. An entry point never builds a second way to start itself. It is transparent to Python, so a supervised proxy behaves on signals and exits exactly as Python alone. `tests/integration/configuration/test_rust_forward_lifecycle.py` checks that equivalence against Python alone, and a change that breaks one of the rules below breaks it + +## Split by job + +- `preflight` decides whether forwarding can start. It is the only place that refuses, and a refusal is a returned `StartupError`, never an exit, so the entry point can serve from Python alone +- `children` says what each child runs with, and it is the only writer of both environment contracts. `python_spec` writes the `mode.Supervised` variable and hands the Python child the entry point's own arguments with the private port appended. `rust_spec` writes the `LITELLM_FORWARD_*` variables. Each child gets only those variables on top of the supervisor's own environment +- `lifecycle` decides. `step` is a pure function from a `Lifecycle` of facts and an event to the next facts and the actions to run, and `exit_status` reads the result from those facts. It touches no process, socket or clock +- `runtime` acts. It only executes the actions `step` returns and feeds events back. The `contextlib.AsyncExitStack` it enters each child into owns their cleanup, last spawned first + +A new behavior is a new fact, event or action in `lifecycle` with a property test in `test_lifecycle.py`, never a condition in `runtime` + +## Rules + +- Signals pass through. Every signal in `Platform.relayed_signals` goes to Python unchanged, and the ones in `Platform.stop_signals` go to Rust first. The supervisor never reads meaning into a signal, so a repeated SIGINT reaches Python again and forces its exit as it would alone +- Python's exit status is the result. When Python exits for any reason, Rust gets SIGTERM, and once Rust is gone the supervisor exits with Python's status, dying by the same signal when Python died by one. The one exception is Rust exiting while Python serves with no stop requested, which stops Python and exits 1 +- A container's PID 1 cannot die by a signal it sends itself, since the kernel drops it. uvicorn alone survives its own re-raise there and exits 0, so `runtime.exit_like` exits 0 when a stop signal killed Python and with 128 plus the signal number for any other signal, which is how a kill of Python alone is reported +- Children cannot outlive the supervisor. Each child's stdin is a pipe only the supervisor holds open. The Python process the supervisor started (`mode.stop_with_supervisor`, never its workers) and the Rust gateway (`litellm_gateway_forward::input_closed`) shut down gracefully on EOF, so a supervisor killed by SIGKILL or a closed terminal leaves nothing behind +- Rust starts once Python listens, never after a stop was requested or Python exited. Python alone takes no connections until it has booted, so the public port refuses them until then too +- Only a preflight refusal falls back to Python alone. `before_start` runs once forwarding is known to be possible, so a refused start leaves nothing behind, and a child that fails to spawn after that exits 1 diff --git a/litellm/proxy/rust_forward/supervisor/__init__.py b/litellm/proxy/rust_forward/supervisor/__init__.py new file mode 100644 index 00000000000..b6611e30ffa --- /dev/null +++ b/litellm/proxy/rust_forward/supervisor/__init__.py @@ -0,0 +1,55 @@ +import asyncio +import sys +import sysconfig +from collections.abc import Callable, Sequence +from typing import Final + +from typing_extensions import assert_never + +from litellm.proxy.rust_forward.platform import Platform, current_platform +from litellm.proxy.rust_forward.server import PythonServer +from litellm.proxy.rust_forward.supervisor.lifecycle import FAILURE +from litellm.proxy.rust_forward.supervisor.preflight import StartupError, gateway_binary, startup_check +from litellm.proxy.rust_forward.supervisor.runtime import SpawnFailed, exit_like, run + +OWN_COMMAND: Final = (sys.executable, *sys.orig_argv[1:]) + + +def _nothing_to_prepare() -> None: + pass + + +def supervise( + server: PythonServer, + command: Sequence[str] = OWN_COMMAND, + before_start: Callable[[], None] = _nothing_to_prepare, + scripts_dir: str = sysconfig.get_path("scripts"), + platform: Platform | None = None, +) -> int | StartupError: + """Serve ``server``'s public port from Rust and run ``command`` behind it, or say why that cannot start. + + A refusal is returned for the caller to serve from Python alone, and ``before_start`` runs only once forwarding + is known to be possible, so a refused start leaves nothing behind. + """ + chosen: Final = current_platform() if platform is None else platform + checked: Final = startup_check(chosen, server, gateway_binary(chosen, scripts_dir)) + if isinstance(checked, StartupError): + sys.stderr.write(f"{checked.reason}, so Python serves the public port without Rust forwarding\n") + return checked + before_start() + try: + outcome: Final = asyncio.run(run(chosen, checked, command, server)) + except OSError as error: + return _startup_failed(error) + match outcome: + case SpawnFailed(error): + return _startup_failed(error) + case int(): + return exit_like(outcome, chosen.stop_signals) + case _: + assert_never(outcome) + + +def _startup_failed(error: OSError) -> int: + sys.stderr.write(f"Rust gateway startup failed: {error}\n") + return FAILURE diff --git a/litellm/proxy/rust_forward/supervisor/children.py b/litellm/proxy/rust_forward/supervisor/children.py new file mode 100644 index 00000000000..9d8f074f0f2 --- /dev/null +++ b/litellm/proxy/rust_forward/supervisor/children.py @@ -0,0 +1,43 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType + +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS +from litellm.proxy.rust_forward.mode import RUST_HOP, Supervised, supervised_environment +from litellm.proxy.rust_forward.platform import Platform, PlatformListener, SpawnOptions +from litellm.proxy.rust_forward.server import PythonServer + + +@dataclass(frozen=True, slots=True) +class ChildSpec: + argv: tuple[str, ...] + env_overrides: Mapping[str, str] + spawn: SpawnOptions + + +def python_spec( + platform: Platform, command: Sequence[str], private: PlatformListener, server: PythonServer +) -> ChildSpec: + return ChildSpec( + argv=(*command, "--port", str(private.port)), + env_overrides=supervised_environment(Supervised(private.listener, server.trusted_proxies)), + spawn=platform.spawn_options(private.pass_fds), + ) + + +def rust_spec(platform: Platform, binary: str, upstream_port: int, server: PythonServer) -> ChildSpec: + return ChildSpec( + argv=(binary,), + env_overrides=MappingProxyType( + { + "LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}", + "LITELLM_FORWARD_HOST": server.host, + "LITELLM_FORWARD_PORT": str(server.port), + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(server.keepalive_seconds), + "LITELLM_FORWARD_SHUTDOWN_DRAIN": str( + server.graceful_timeout_seconds + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS + ), + } + ), + spawn=platform.spawn_options(()), + ) diff --git a/litellm/proxy/rust_forward/supervisor/lifecycle.py b/litellm/proxy/rust_forward/supervisor/lifecycle.py new file mode 100644 index 00000000000..e462c7df08a --- /dev/null +++ b/litellm/proxy/rust_forward/supervisor/lifecycle.py @@ -0,0 +1,138 @@ +import signal +from dataclasses import dataclass, replace +from enum import Enum, auto +from typing import Final, Literal, TypeAlias + +from typing_extensions import assert_never + + +class Child(Enum): + RUST = "rust" + PYTHON = "python" + + +FAILURE: Final = 1 + + +class Phase(Enum): + NOT_STARTED = auto() + SPAWNED = auto() + LISTENING = auto() + + +@dataclass(frozen=True, slots=True) +class Exited: + status: int + + +@dataclass(frozen=True, slots=True) +class Lost: + """Rust exited while Python was serving and nobody had asked either to stop.""" + + status: int + + +PythonState: TypeAlias = Literal[Phase.SPAWNED, Phase.LISTENING] | Exited +RustState: TypeAlias = Literal[Phase.NOT_STARTED, Phase.SPAWNED] | Exited | Lost + + +@dataclass(frozen=True, slots=True) +class Lifecycle: + python: PythonState + rust: RustState + stop_requested: bool + + +@dataclass(frozen=True, slots=True) +class Caught: + signum: signal.Signals + + +class Probe(Enum): + PYTHON_LISTENING = auto() + + +@dataclass(frozen=True, slots=True) +class ChildExited: + child: Child + status: int + + +@dataclass(frozen=True, slots=True) +class Spawn: + child: Child + + +@dataclass(frozen=True, slots=True) +class Send: + child: Child + signum: signal.Signals + + +Event: TypeAlias = Caught | Literal[Probe.PYTHON_LISTENING] | ChildExited +Action: TypeAlias = Spawn | Send +Transition: TypeAlias = tuple[Lifecycle, tuple[Action, ...]] + + +def boot() -> Transition: + return Lifecycle(Phase.SPAWNED, Phase.NOT_STARTED, stop_requested=False), (Spawn(Child.PYTHON),) + + +def _relay(state: Lifecycle, signum: signal.Signals, stops: bool) -> Transition: + to_rust: Final = (Send(Child.RUST, signum),) if stops and state.rust is Phase.SPAWNED else () + to_python: Final = (Send(Child.PYTHON, signum),) if not isinstance(state.python, Exited) else () + return replace(state, stop_requested=state.stop_requested or stops), (*to_rust, *to_python) + + +# Python alone takes no connections until it has booted, so Rust only opens the public port once Python listens. +def _python_listening(state: Lifecycle) -> Transition: + if state.python is not Phase.SPAWNED: + return state, () + if state.stop_requested: + return replace(state, python=Phase.LISTENING), () + return replace(state, python=Phase.LISTENING, rust=Phase.SPAWNED), (Spawn(Child.RUST),) + + +def _python_exited(state: Lifecycle, status: int) -> Transition: + if isinstance(state.python, Exited): + return state, () + exited: Final = replace(state, python=Exited(status)) + return exited, ((Send(Child.RUST, signal.SIGTERM),) if state.rust is Phase.SPAWNED else ()) + + +def _rust_exited(state: Lifecycle, status: int) -> Transition: + if state.rust is not Phase.SPAWNED: + return state, () + if state.stop_requested or isinstance(state.python, Exited): + return replace(state, rust=Exited(status)), () + return replace(state, rust=Lost(status)), (Send(Child.PYTHON, signal.SIGTERM),) + + +def step(state: Lifecycle, event: Event, stop_signals: frozenset[signal.Signals]) -> Transition: + """The supervisor never gives a signal or an exit a meaning of its own. Every signal reaches Python as sent, + Python's exit status is the supervisor's, and Rust only follows Python, except that losing Rust stops Python.""" + match event: + case Caught(signum): + return _relay(state, signum, signum in stop_signals) + case Probe.PYTHON_LISTENING: + return _python_listening(state) + case ChildExited(child, status): + match child: + case Child.PYTHON: + return _python_exited(state, status) + case Child.RUST: + return _rust_exited(state, status) + case _: + assert_never(child) + case _: + assert_never(event) + + +def exit_status(state: Lifecycle) -> int | None: + match state.python, state.rust: + case Exited(status), Phase.NOT_STARTED | Exited(): + return status + case Exited(), Lost(): + return FAILURE + case _: + return None diff --git a/litellm/proxy/rust_forward/supervisor/preflight.py b/litellm/proxy/rust_forward/supervisor/preflight.py new file mode 100644 index 00000000000..a561e77ba8d --- /dev/null +++ b/litellm/proxy/rust_forward/supervisor/preflight.py @@ -0,0 +1,28 @@ +import os +import shutil +from dataclasses import dataclass + +from litellm.proxy.rust_forward.mode import SWITCH_ENV +from litellm.proxy.rust_forward.platform import Platform +from litellm.proxy.rust_forward.server import Listener, PythonServer, Server + + +@dataclass(frozen=True, slots=True) +class StartupError: + reason: str + + +def gateway_binary(platform: Platform, scripts_dir: str) -> str | None: + return shutil.which(os.path.join(scripts_dir, platform.gateway_name)) + + +def startup_check(platform: Platform, server: PythonServer, binary: str | None) -> str | StartupError: + if server.server in (Server.HYPERCORN, Server.GRANIAN): + return StartupError(f"{SWITCH_ENV} does not support {server.server.value}") + if server.server not in platform.servers: + return StartupError(f"{SWITCH_ENV} does not support {server.server.value} on {platform.name}") + if server.listener is not Listener.TCP: + return StartupError(f"{SWITCH_ENV} does not support {server.listener.value} on the public port") + if binary is None: + return StartupError(f"{SWITCH_ENV} requires the litellm-gateway binary from the litellm-proxy package") + return binary diff --git a/litellm/proxy/rust_forward/supervisor/runtime.py b/litellm/proxy/rust_forward/supervisor/runtime.py new file mode 100644 index 00000000000..7b727fc0d80 --- /dev/null +++ b/litellm/proxy/rust_forward/supervisor/runtime.py @@ -0,0 +1,194 @@ +import asyncio +import contextlib +import math +import os +import signal +from collections.abc import AsyncGenerator, Callable, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Protocol + +import anyio +from anyio.abc import TaskGroup +from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream +from typing_extensions import assert_never + +from litellm.proxy.rust_forward.mode import RUST_HOP +from litellm.proxy.rust_forward.platform import Platform +from litellm.proxy.rust_forward.server import PythonServer +from litellm.proxy.rust_forward.supervisor.children import ChildSpec, python_spec, rust_spec +from litellm.proxy.rust_forward.supervisor.lifecycle import ( + Caught, + Child, + ChildExited, + Event, + Probe, + Send, + Spawn, + boot, + exit_status, + step, +) + +PROBE_INTERVAL_SECONDS: Final = 0.05 + + +@dataclass(frozen=True, slots=True) +class SpawnFailed: + error: OSError + + +class Executor(Protocol): + async def spawn(self, child: Child) -> SpawnFailed | None: ... + + def send(self, child: Child, signum: signal.Signals) -> None: ... + + async def next_event(self) -> Event: ... + + +async def run(platform: Platform, binary: str, command: Sequence[str], server: PythonServer) -> int | SpawnFailed: + with platform.private_listener() as private: + specs: Final = MappingProxyType( + { + Child.PYTHON: python_spec(platform, command, private, server), + Child.RUST: rust_spec(platform, binary, private.port, server), + } + ) + async with open_children(platform, specs, private.port) as children: + platform.catch(asyncio.get_running_loop(), children.caught) + return await drive(children, platform.stop_signals) + + +async def drive(executor: Executor, stop_signals: frozenset[signal.Signals]) -> int | SpawnFailed: + """Carry out what ``lifecycle.step`` decides until it settles the exit status.""" + state, actions = boot() + while True: + for action in actions: + match action: + case Spawn(child): + if (failed := await executor.spawn(child)) is not None: + return failed + case Send(child, signum): + executor.send(child, signum) + case _: + assert_never(action) + if (status := exit_status(state)) is not None: + return status + state, actions = step(state, await executor.next_event(), stop_signals) + + +def _die_by(signum: signal.Signals) -> None: + with contextlib.suppress(OSError): + signal.signal(signum, signal.SIG_DFL) + signal.raise_signal(signum) + + +def exit_like( + status: int, stop_signals: frozenset[signal.Signals], die_by: Callable[[signal.Signals], None] = _die_by +) -> int: + """A child killed by a signal kills the supervisor with the same signal, as uvicorn re-raises the one it caught. + + SIGKILL takes no handler and needs none to kill. The kernel drops the signal when the supervisor is a container's + PID 1. uvicorn alone survives its own re-raise there and exits 0, so a stop signal ends the supervisor with 0 too, + and any other signal with the status a shell reports for it.""" + if status >= 0: + return status + signum: Final = signal.Signals(-status) + die_by(signum) + return 0 if signum in stop_signals else 128 + signum + + +@dataclass(frozen=True, slots=True) +class Children: + platform: Platform + specs: Mapping[Child, ChildSpec] + private_port: int + stack: contextlib.AsyncExitStack + tasks: TaskGroup + events_in: MemoryObjectSendStream[Event] + events_out: MemoryObjectReceiveStream[Event] + processes: dict[Child, asyncio.subprocess.Process] # mutable-ok: the running children the stack owns + + def caught(self, signum: signal.Signals) -> None: + with contextlib.suppress(anyio.ClosedResourceError): + self.events_in.send_nowait(Caught(signum)) + + async def spawn(self, child: Child) -> SpawnFailed | None: + try: + process: Final = await self.stack.enter_async_context(_process(self.platform, self.specs[child])) + except OSError as error: + return SpawnFailed(error) + self.processes[child] = process + self.tasks.start_soon(_report_exit, child, process, self.events_in.clone()) + if child is Child.PYTHON: + probe: Final = self.tasks.start_soon(_report_once_listening, self.private_port, self.events_in.clone()) + self.stack.callback(probe.cancel) + return None + + def send(self, child: Child, signum: signal.Signals) -> None: + self.platform.signal_child(self.processes[child], signum) + + async def next_event(self) -> Event: + return await self.events_out.receive() + + +# The task group is entered before any child, so on the way out every child is stopped, last spawned first, before +# the task group waits for the tasks reporting their exits. Signal handlers cannot wait for room, hence the +# unbounded event buffer. +@contextlib.asynccontextmanager +async def open_children( + platform: Platform, specs: Mapping[Child, ChildSpec], private_port: int +) -> AsyncGenerator[Children]: + events_in, events_out = anyio.create_memory_object_stream[Event](math.inf) + with events_in, events_out: + async with contextlib.AsyncExitStack() as stack: + tasks: Final = await stack.enter_async_context(anyio.create_task_group()) + yield Children(platform, specs, private_port, stack, tasks, events_in, events_out, {}) + + +# A child's stdin is a pipe only the supervisor holds open, so the child sees EOF as soon as the supervisor is gone, +# however it went, and shuts down instead of outliving it. Stopping is shielded so a cancelled supervisor still waits +# for the child it is stopping. +@contextlib.asynccontextmanager +async def _process(platform: Platform, spec: ChildSpec) -> AsyncGenerator[asyncio.subprocess.Process]: + process: Final = await asyncio.create_subprocess_exec( + *spec.argv, + stdin=asyncio.subprocess.PIPE, + env={**os.environ, **spec.env_overrides}, + pass_fds=spec.spawn.pass_fds, + start_new_session=spec.spawn.start_new_session, + creationflags=spec.spawn.creationflags, + ) + try: + yield process + finally: + with anyio.CancelScope(shield=True): + platform.stop_group(process) + await process.wait() + if process.stdin is not None: + process.stdin.close() + + +async def _report_exit( + child: Child, process: asyncio.subprocess.Process, events: MemoryObjectSendStream[Event] +) -> None: + with events: + events.send_nowait(ChildExited(child, await process.wait())) + + +async def _report_once_listening(port: int, events: MemoryObjectSendStream[Event]) -> None: + with events: + while not await _accepts(port): + await anyio.sleep(PROBE_INTERVAL_SECONDS) + events.send_nowait(Probe.PYTHON_LISTENING) + + +async def _accepts(port: int) -> bool: + with anyio.move_on_after(PROBE_INTERVAL_SECONDS): + try: + stream: Final = await anyio.connect_tcp(RUST_HOP, port) + except OSError: + return False + await stream.aclose() + return True + return False diff --git a/litellm/rust_bridge/AGENTS.md b/litellm/rust_bridge/AGENTS.md index 167b91c9b89..a13f57fae4b 100644 --- a/litellm/rust_bridge/AGENTS.md +++ b/litellm/rust_bridge/AGENTS.md @@ -46,6 +46,11 @@ - A failure after execution starts is terminal and never triggers fallback - Creating an unstarted native coroutine acquires no clients or credentials and captures no execution context +# The `LITELLM_RUST` switch + +- `decision` treats a blank or invalid `LITELLM_RUST` as unset and caches the parse by raw value. It runs inside SDK calls, so it never fails one over a switch and never validates per call +- It does not put the Rust gateway in front of Python. That is `LITELLM_RUST_FIRST`, which `litellm/proxy/rust_forward/mode.py` reads strictly at startup with this same `RustSwitch` type, stopping the CLI with one line naming the variable + # Native extension - `_native` ships in every wheel and the sdist builds it, so it is always importable diff --git a/litellm_cli/main.py b/litellm_cli/main.py index a9dfa4be179..ea170603d08 100644 --- a/litellm_cli/main.py +++ b/litellm_cli/main.py @@ -33,7 +33,7 @@ from contextlib import contextmanager from typing import TYPE_CHECKING, Final if TYPE_CHECKING: - from fastapi import FastAPI + from litellm.proxy.rust_forward.peer import ASGIApp SDK_PACKAGE: Final = "litellm" CLI_PACKAGE: Final = "litellm.proxy.client" @@ -214,10 +214,12 @@ def without_gc_pauses() -> Generator[None, None, None]: gc.enable() -def proxy_worker_app() -> "FastAPI": +def proxy_worker_app() -> "ASGIApp": with without_gc_pauses(): from litellm.proxy.proxy_cli import ProxyInitializationHelpers ProxyInitializationHelpers.apply_worker_log_config(os.environ) from litellm.proxy.proxy_server import app - return app + from litellm.proxy.rust_forward.mode import rust_forward_mode + from litellm.proxy.rust_forward.peer import wrap + return wrap(app, rust_forward_mode(os.environ)) diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 491fc87dce2..7a1bd7208d5 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -145,6 +145,11 @@ def signal_group(group: int, action: int) -> None: pass +def send_signal(process: subprocess.Popen[bytes], signum: signal.Signals) -> None: + with suppress(ProcessLookupError): + process.send_signal(signum) + + def graceful_stop_seconds() -> float: return readiness_for(import_seconds(), 1) diff --git a/tests/integration/_support/rust_forward.py b/tests/integration/_support/rust_forward.py new file mode 100644 index 00000000000..14b557897c5 --- /dev/null +++ b/tests/integration/_support/rust_forward.py @@ -0,0 +1,117 @@ +import json +import os +import subprocess +import sys +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +from pydantic import JsonValue + +from tests.integration._support.client import eventually +from tests.integration._support.ports import allocate_port +from tests.integration._support.process import stop_root_process + +MASTER_KEY: Final = "sk-rust-forward-integration-key-not-a-secret" +WORKERS: Final = "2" +HEALTHCHECK_SECONDS: Final = "120" + + +class EntryPoint(Enum): + LITELLM = "litellm.proxy.proxy_cli" + GATEWAY = "gateway.launch" + + +@dataclass(frozen=True, slots=True) +class ForwardedProxy: + port: int + log: Path + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + +def _serving(url: str) -> bool: + try: + return httpx.get(f"{url}/health/liveliness", timeout=5, trust_env=False).status_code == 200 + except httpx.TransportError: + return False + + +def _launch(entry_point: EntryPoint, config: Path) -> tuple[tuple[str, ...], Mapping[str, str]]: + match entry_point: + case EntryPoint.LITELLM: + return ( + ( + "--config", + str(config), + "--num_workers", + WORKERS, + "--timeout_worker_healthcheck", + HEALTHCHECK_SECONDS, + ), + MappingProxyType({}), + ) + case EntryPoint.GATEWAY: + return ( + ("--workers", WORKERS, "--timeout-worker-healthcheck", HEALTHCHECK_SECONDS), + MappingProxyType({"CONFIG_FILE_PATH": str(config), "LITELLM_PGBOUNCER_ENABLED": "false"}), + ) + + +@contextmanager +def forwarded_proxy( + entry_point: EntryPoint, + config: Mapping[str, JsonValue], + directory: Path, + arguments: tuple[str, ...] = (), + environment: Mapping[str, str] = MappingProxyType({}), +) -> Iterator[ForwardedProxy]: + """``entry_point`` on 0.0.0.0 with two workers, ``LITELLM_RUST_FIRST=1`` and no database. It must exit 0 on SIGTERM.""" + proxy: Final = ForwardedProxy(allocate_port(), directory / f"{entry_point.name.lower()}.log") + config_file: Final = directory / "config.yaml" + config_file.write_text(json.dumps(config)) + launch_arguments, launch_environment = _launch(entry_point, config_file) + inherited: Final = { + name: value for name, value in os.environ.items() if not name.startswith("DATABASE_") and name != "DIRECT_URL" + } + with ( + proxy.log.open("wb") as output, + subprocess.Popen( + ( + sys.executable, + "-I", + "-m", + entry_point.value, + "--host", + "0.0.0.0", + "--port", + str(proxy.port), + *launch_arguments, + *arguments, + ), + env={ + **inherited, + "LITELLM_MASTER_KEY": MASTER_KEY, + "LITELLM_RUST_FIRST": "true", + "STORE_MODEL_IN_DB": "False", + "PYTHON_DOTENV_DISABLED": "1", + **launch_environment, + **environment, + }, + stdout=output, + stderr=output, + ) as process, + ): + try: + assert eventually(lambda: _serving(proxy.url), bool, seconds=180), proxy.log.read_text() + yield proxy + finally: + assert stop_root_process(process), proxy.log.read_text() + assert process.returncode == 0, proxy.log.read_text() diff --git a/tests/integration/configuration/test_rust_forward.py b/tests/integration/configuration/test_rust_forward.py new file mode 100644 index 00000000000..cc4e0dc2770 --- /dev/null +++ b/tests/integration/configuration/test_rust_forward.py @@ -0,0 +1,110 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import psutil +import pytest + +from tests.integration._support.client import eventually +from tests.integration._support.ports import allocate_port +from tests.integration._support.rust_forward import MASTER_KEY, EntryPoint, forwarded_proxy + +TRUSTED_BALANCER: Final = "127.0.0.3" +CLIENT: Final = "127.0.0.2" + + +@dataclass(frozen=True, slots=True) +class Caller: + connects_from: str + forwarded_for: str + resolved_ip: str + + +CALLERS: Final = ( + Caller(CLIENT, "1.2.3.4", resolved_ip=CLIENT), + Caller(TRUSTED_BALANCER, "203.0.113.9", resolved_ip="203.0.113.9"), + Caller(CLIENT, f"203.0.113.9, {TRUSTED_BALANCER}", resolved_ip=CLIENT), +) + + +def _client(url: str, local_address: str) -> httpx.Client: + return httpx.Client( + base_url=url, transport=httpx.HTTPTransport(local_address=local_address), trust_env=False, timeout=5 + ) + + +def _resolved_ip(url: str, caller: Caller) -> str: + with _client(url, caller.connects_from) as client: + response: Final = client.get( + "/v1/mcp/network/client-ip", + headers={"Authorization": f"Bearer {MASTER_KEY}", "X-Forwarded-For": caller.forwarded_for}, + ) + assert response.status_code == 200, response.text + return response.json()["ip"] + + +@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)]) +def test_only_the_trusted_balancer_can_name_the_client_ip_behind_rust( + tmp_path: Path, server_arguments: tuple[str, ...] +) -> None: + metrics_port: Final = allocate_port() + with forwarded_proxy( + EntryPoint.LITELLM, + { + "model_list": [], + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "use_x_forwarded_for": False}, + }, + tmp_path, + ("--prometheus_metrics_port", str(metrics_port), *server_arguments), + {"FORWARDED_ALLOW_IPS": TRUSTED_BALANCER}, + ) as proxy: + with _client(f"http://{CLIENT}:{metrics_port}", CLIENT) as external: + metrics: Final = external.get("/metrics", follow_redirects=True) + assert metrics.status_code == 200, metrics.text + assert tuple(_resolved_ip(proxy.url, caller) for caller in CALLERS) == tuple( + caller.resolved_ip for caller in CALLERS + ) + + +@pytest.mark.parametrize("server_arguments", [("--no-proxy-headers",), ("--forwarded-allow-ips", "127.0.0.1")]) +def test_gateway_launcher_logs_the_real_client_address_behind_rust( + tmp_path: Path, server_arguments: tuple[str, ...] +) -> None: + with forwarded_proxy( + EntryPoint.GATEWAY, + {"model_list": [], "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}}, + tmp_path, + server_arguments, + ) as proxy: + with _client(proxy.url, CLIENT) as external: + response: Final = external.get( + "/health/liveliness?client-ip-proof", headers={"X-Forwarded-For": "198.51.100.10"} + ) + assert response.status_code == 200, response.text + logged: Final = eventually( + lambda: tuple( + line for line in proxy.log.read_text().splitlines() if "GET /health/liveliness?client-ip-proof " in line + ), + bool, + seconds=10, + ) + assert len(logged) == 1 and f"{CLIENT}:" in logged[0], logged + + +@pytest.mark.parametrize("entry_point", list(EntryPoint), ids=[entry_point.name for entry_point in EntryPoint]) +def test_the_python_child_is_started_with_the_supervisors_own_command(tmp_path: Path, entry_point: EntryPoint) -> None: + with forwarded_proxy( + entry_point, + {"model_list": [], "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}}, + tmp_path, + ): + supervisor: Final = next( + started for started in psutil.Process().children() if entry_point.value in started.cmdline() + ) + own_command: Final = supervisor.cmdline()[1:] + python_children: Final = [ + child.cmdline()[1:] for child in supervisor.children() if entry_point.value in child.cmdline() + ] + assert "-I" in own_command + assert [child[: len(own_command)] for child in python_children] == [own_command] diff --git a/tests/integration/configuration/test_rust_forward_lifecycle.py b/tests/integration/configuration/test_rust_forward_lifecycle.py new file mode 100644 index 00000000000..b3ffb8b7169 --- /dev/null +++ b/tests/integration/configuration/test_rust_forward_lifecycle.py @@ -0,0 +1,269 @@ +import os +import signal +import socket +import subprocess +import sys +import time +from collections.abc import Callable +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Final, TypeVar + +import httpx +import psutil +import pytest + +from tests.integration._support.client import eventually +from tests.integration._support.ports import allocate_port +from tests.integration._support.process import kill_stray_process, send_signal + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="drives POSIX signals") + +APP: Final = """ +import asyncio + + +async def app(scope, receive, send): + if scope["type"] != "http": + return + await send({"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"text/plain")]}) + if scope["path"] != "/stream": + await send({"type": "http.response.body", "body": b"ok"}) + return + while True: + await send({"type": "http.response.body", "body": b"tick\\n", "more_body": True}) + await asyncio.sleep(0.1) +""" + +SERVE: Final = """ +import os +import pathlib +import sys + +import uvicorn + + + +def listener(): + from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode, stop_with_supervisor + + mode = rust_forward_mode(os.environ) + if not isinstance(mode, Supervised): + return {"host": "127.0.0.1", "port": int(sys.argv[2])} + stop_with_supervisor(mode) + pathlib.Path(os.environ["MARKERS"], "python.pid").write_text(str(os.getpid())) + return {"fd": mode.listener.fd} + + +if __name__ == "__main__": + uvicorn.run("app:app", app_dir=os.path.dirname(__file__), workers=int(sys.argv[1]), log_level="warning", **listener()) +""" + +GATEWAY: Final = """ +import asyncio +import os +import pathlib +import signal +import threading + + +def wait_for_eof(loop, stop): + while os.read(0, 4096): + pass + loop.call_soon_threadsafe(stop.set) + + +async def pipe(reader, writer): + try: + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + except OSError: + pass + finally: + writer.close() + + +async def main(): + pathlib.Path(os.environ["MARKERS"], "rust.pid").write_text(str(os.getpid())) + host, port = os.environ["LITELLM_FORWARD_UPSTREAM"].removeprefix("http://").rsplit(":", 1) + loop = asyncio.get_running_loop() + stop = asyncio.Event() + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop.set) + threading.Thread(target=wait_for_eof, args=(loop, stop), daemon=True).start() + relays = set() + + async def relay(client_reader, client_writer): + relays.add(asyncio.current_task()) + try: + upstream_reader, upstream_writer = await asyncio.open_connection(host, int(port)) + await asyncio.gather(pipe(client_reader, upstream_writer), pipe(upstream_reader, client_writer)) + finally: + relays.discard(asyncio.current_task()) + + server = await asyncio.start_server( + relay, os.environ["LITELLM_FORWARD_HOST"], int(os.environ["LITELLM_FORWARD_PORT"]) + ) + await stop.wait() + server.close() + if relays: + await asyncio.wait(relays, timeout=float(os.environ["LITELLM_FORWARD_SHUTDOWN_DRAIN"])) + + +asyncio.run(main()) +""" + +SUPERVISE: Final = """ +import sys +from pathlib import Path + +from litellm.proxy.rust_forward.server import Listener, PythonServer, Server +from litellm.proxy.rust_forward.supervisor import supervise + +workers, port, scripts = sys.argv[1], int(sys.argv[2]), Path(sys.argv[3]) +server = PythonServer(Server.UVICORN, Listener.TCP, "127.0.0.1", port, 5, 30.0, "127.0.0.1") +result = supervise(server, (sys.executable, str(scripts / "serve.py"), workers, str(port)), scripts_dir=str(scripts)) +raise SystemExit(result if isinstance(result, int) else 3) +""" + +T = TypeVar("T") +EXIT_SECONDS: Final = 15 +TIME_TOLERANCE_SECONDS: Final = 5 + + +class Scenario(Enum): + TERM = "SIGTERM" + INT = "SIGINT" + INT_TWICE_WHILE_STREAMING = "SIGINT twice during an open stream" + HUP = "SIGHUP" + + +@dataclass(frozen=True, slots=True) +class Outcome: + status: int | None + survived_hup: bool + + +@dataclass(frozen=True, slots=True) +class Run: + outcome: Outcome + seconds: float + + +def _scripts(directory: Path) -> Path: + (directory / "app.py").write_text(APP) + (directory / "serve.py").write_text(SERVE) + (directory / "supervise.py").write_text(SUPERVISE) + gateway: Final = directory / "litellm-gateway" + gateway.write_text(f"#!{sys.executable}\n{GATEWAY}") + gateway.chmod(0o755) + return directory + + +def _serving(port: int) -> bool: + try: + return httpx.get(f"http://127.0.0.1:{port}/", timeout=1, trust_env=False).text == "ok" + except httpx.TransportError: + return False + + +def _exit_status(process: subprocess.Popen[bytes], seconds: float) -> int | None: + try: + return process.wait(timeout=seconds) + except subprocess.TimeoutExpired: + return None + + +def _streaming(port: int) -> socket.socket: + stream: Final = socket.create_connection(("127.0.0.1", port), timeout=10) + stream.sendall(b"GET /stream HTTP/1.1\r\nHost: test\r\n\r\n") + assert eventually(lambda: stream.recv(4096), lambda received: b"tick" in received, seconds=10) + return stream + + +def _signal_after_ready(process: subprocess.Popen[bytes], port: int, scenario: Scenario) -> Run: + assert eventually(lambda: _serving(port), bool, seconds=120) + started: Final = time.monotonic() + match scenario: + case Scenario.TERM | Scenario.INT: + send_signal(process, signal.SIGTERM if scenario is Scenario.TERM else signal.SIGINT) + return Run(Outcome(_exit_status(process, EXIT_SECONDS), False), time.monotonic() - started) + case Scenario.INT_TWICE_WHILE_STREAMING: + with _streaming(port): + send_signal(process, signal.SIGINT) + time.sleep(0.5) + send_signal(process, signal.SIGINT) + status: Final = _exit_status(process, EXIT_SECONDS) + return Run(Outcome(status, False), time.monotonic() - started) + case Scenario.HUP: + send_signal(process, signal.SIGHUP) + after_hup: Final = _exit_status(process, 3) + if after_hup is not None: + return Run(Outcome(after_hup, False), time.monotonic() - started) + assert eventually(lambda: _serving(port), bool, seconds=30) + send_signal(process, signal.SIGTERM) + return Run(Outcome(_exit_status(process, EXIT_SECONDS), True), time.monotonic() - started) + + +def _run(argv: tuple[str, ...], environment: dict[str, str], act: Callable[[subprocess.Popen[bytes]], T]) -> T: + with subprocess.Popen(argv, env=environment, start_new_session=True) as process: + try: + return act(process) + finally: + if process.poll() is None: + for leftover in (*psutil.Process(process.pid).children(recursive=True), psutil.Process(process.pid)): + kill_stray_process(leftover.pid) + + +def _environment(markers: Path) -> dict[str, str]: + return {**os.environ, "MARKERS": str(markers), "PYTHONUNBUFFERED": "1"} + + +@pytest.mark.parametrize("workers", [1, 2]) +@pytest.mark.parametrize("scenario", list(Scenario), ids=[scenario.name for scenario in Scenario]) +def test_the_supervisor_ends_as_python_alone_would(tmp_path: Path, workers: int, scenario: Scenario) -> None: + scripts: Final = _scripts(tmp_path) + alone_port: Final = allocate_port() + alone: Final = _run( + (sys.executable, str(scripts / "serve.py"), str(workers), str(alone_port)), + _environment(tmp_path), + lambda process: _signal_after_ready(process, alone_port, scenario), + ) + supervised_port: Final = allocate_port() + supervised: Final = _run( + (sys.executable, str(scripts / "supervise.py"), str(workers), str(supervised_port), str(scripts)), + _environment(tmp_path), + lambda process: _signal_after_ready(process, supervised_port, scenario), + ) + assert supervised.outcome == alone.outcome + assert abs(supervised.seconds - alone.seconds) < TIME_TOLERANCE_SECONDS, (alone, supervised) + + +def _alive(pid: int) -> bool: + try: + return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE + except psutil.NoSuchProcess: + return False + + +@pytest.mark.parametrize("workers", [1, 2]) +def test_children_exit_when_the_supervisor_is_killed(tmp_path: Path, workers: int) -> None: + scripts: Final = _scripts(tmp_path) + port: Final = allocate_port() + + def survivors_of_killing_the_supervisor(process: subprocess.Popen[bytes]) -> tuple[int, ...]: + assert eventually(lambda: _serving(port), bool, seconds=120) + python: Final = int((tmp_path / "python.pid").read_text()) + rust: Final = int((tmp_path / "rust.pid").read_text()) + descendants: Final = (python, rust, *(child.pid for child in psutil.Process(python).children(recursive=True))) + kill_stray_process(process.pid) + return eventually(lambda: tuple(pid for pid in descendants if _alive(pid)), lambda alive: not alive) + + survivors: Final = _run( + (sys.executable, str(scripts / "supervise.py"), str(workers), str(port), str(scripts)), + _environment(tmp_path), + survivors_of_killing_the_supervisor, + ) + assert survivors == () diff --git a/tests/integration/configuration/test_rust_forward_traffic.py b/tests/integration/configuration/test_rust_forward_traffic.py new file mode 100644 index 00000000000..4cd2f09e6d7 --- /dev/null +++ b/tests/integration/configuration/test_rust_forward_traffic.py @@ -0,0 +1,387 @@ +import asyncio +import json +import os +import socket +import threading +import time +from collections.abc import AsyncIterator, Callable, Iterator, Mapping +from contextlib import AbstractContextManager, ExitStack +from dataclasses import dataclass +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final + +import anyio +import httpx +import openai +import pytest +from pydantic import BaseModel, JsonValue +from starlette.applications import Starlette +from starlette.requests import Request as StarletteRequest +from starlette.responses import StreamingResponse +from starlette.routing import Route, WebSocketRoute +from starlette.websockets import WebSocket, WebSocketDisconnect +from websockets.sync.client import connect + +from tests.integration._support.asgi import asgi_server +from tests.integration._support.mcp import mcp_peer +from tests.integration._support.rust_forward import MASTER_KEY, EntryPoint, forwarded_proxy +from tests.integration._support.wire import Reply, Request, wire_server + +KEEPALIVE_SECONDS: Final = 1 +FILES_MODEL: Final = "rust-forward-files" +FILE_ID: Final = "file-rust-forward" +CHAT_MODEL: Final = "rust-forward-chat" +REALTIME_MODEL: Final = "rust-forward-realtime" +MCP_HEADERS: Final = MappingProxyType( + { + "Authorization": f"Bearer {MASTER_KEY}", + "Accept": "application/json, text/event-stream", + "mcp-protocol-version": "2025-06-18", + } +) + + +@dataclass(frozen=True, slots=True) +class ScriptedProvider: + url: str + released: threading.Event + hung_up: SimpleQueue[str] + + +@dataclass(frozen=True, slots=True) +class RustProxy: + url: str + port: int + provider: ScriptedProvider + + +class _Message(BaseModel): + content: str + + +class _ChatRequest(BaseModel): + messages: tuple[_Message, ...] + + +class _InputText(BaseModel): + text: str + + +class _Item(BaseModel): + content: tuple[_InputText, ...] = () + + +class _RealtimeFrame(BaseModel): + type: str + item: _Item | None = None + + +def _sse(content: str | None, finish_reason: str | None = None) -> bytes: + chunk: Final = { + "id": "chatcmpl-rust-forward", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "delta": {} if content is None else {"content": content}, "finish_reason": finish_reason} + ], + } + return b"data: " + json.dumps(chunk).encode() + b"\n\n" + + +async def _gated_stream(released: threading.Event) -> AsyncIterator[bytes]: + yield _sse("before-gate") + await anyio.to_thread.run_sync(released.wait, 30) + yield _sse("after-gate") + yield _sse(None, "stop") + yield b"data: [DONE]\n\n" + + +async def _endless_stream(marker: str, hung_up: SimpleQueue[str]) -> AsyncIterator[bytes]: + try: + for index in range(600): + yield _sse(f"tick-{index} ") + await asyncio.sleep(0.1) + except (asyncio.CancelledError, GeneratorExit): + hung_up.put(marker) + raise + + +def _provider_app(released: threading.Event, hung_up: SimpleQueue[str]) -> Starlette: + async def chat(request: StarletteRequest) -> StreamingResponse: + prompt: Final = _ChatRequest.model_validate_json(await request.body()).messages[-1].content + return StreamingResponse( + _gated_stream(released) if prompt == "gated" else _endless_stream(prompt, hung_up), + media_type="text/event-stream", + ) + + async def realtime(websocket: WebSocket) -> None: + await websocket.accept() + await websocket.send_json({"type": "session.created", "session": {"id": "sess_rust_forward"}}) + try: + while True: + frame = _RealtimeFrame.model_validate_json(await websocket.receive_text()) # rebind-ok: per frame + if frame.item is not None: + text = "".join(part.text for part in frame.item.content) # rebind-ok: per frame + await websocket.send_json({"type": "response.output_text.delta", "delta": f"echo:{text}"}) + except WebSocketDisconnect: + return + + return Starlette( + routes=[ + Route("/v1/chat/completions", chat, methods=["POST"]), + WebSocketRoute("/v1/realtime", realtime), + ] + ) + + +def _routable_address() -> str: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe: + probe.connect(("192.0.2.1", 9)) + address: Final[str] = probe.getsockname()[0] + assert not address.startswith("127."), address + return address + + +def _files_peer(content: bytes) -> Callable[[Request], Reply]: + file_object: Final = json.dumps( + { + "id": FILE_ID, + "object": "file", + "purpose": "batch", + "bytes": len(content), + "created_at": 1, + "filename": "upload.jsonl", + "status": "processed", + } + ).encode() + + def respond(request: Request) -> Reply: + if request.method == "POST" and request.target.startswith("/v1/files"): + return Reply(body=file_object) + if request.method == "GET" and request.target.startswith(f"/v1/files/{FILE_ID}/content"): + return Reply( + content_type="application/octet-stream", body=content, chunks=(content[:65536], content[65536:]) + ) + return Reply(status=404, body=b'{"error": {"message": "not found"}}') + + return respond + + +UPLOAD: Final = b"".join( + json.dumps( + { + "custom_id": f"row-{index}", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": os.urandom(48).hex()}]}, + } + ).encode() + + b"\n" + for index in range(30_000) +) + + +@pytest.fixture(scope="module") +def files_upstream() -> Iterator[str]: + with wire_server(_files_peer(UPLOAD), keep_alive=True) as wire: + yield wire.url + + +@pytest.fixture(scope="module") +def rust_proxy(tmp_path_factory: pytest.TempPathFactory, files_upstream: str) -> Iterator[RustProxy]: + released: Final = threading.Event() + hung_up: Final[SimpleQueue[str]] = SimpleQueue() + with ExitStack() as stack: + mcp: Final = stack.enter_context(mcp_peer()) + provider: Final = ScriptedProvider( + stack.enter_context(asgi_server(_provider_app(released, hung_up))), released, hung_up + ) + proxy: Final = stack.enter_context( + forwarded_proxy( + EntryPoint.LITELLM, + { + "model_list": [ + _deployment(CHAT_MODEL, "openai/gpt-4o-mini", f"{provider.url}/v1"), + { + **_deployment(REALTIME_MODEL, "openai/gpt-realtime", provider.url), + "model_info": {"mode": "realtime"}, + }, + _deployment(FILES_MODEL, "openai/gpt-4o-mini", f"{files_upstream}/v1"), + ], + "mcp_servers": {"math": {**mcp.registration(), "available_on_public_internet": False}}, + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "mcp_internal_ip_ranges": ["127.0.0.0/8"], + }, + }, + tmp_path_factory.mktemp("rust-forward-traffic"), + ("--keepalive_timeout", str(KEEPALIVE_SECONDS)), + ) + ) + stack.callback(released.set) + yield RustProxy(proxy.url, proxy.port, provider) + + +def _deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": {"model": model, "api_base": api_base, "api_key": "integration-provider-key"}, + } + + +def _still_open(stream: httpx.Response) -> bool: + try: + next(stream.iter_raw()) + except httpx.ReadTimeout: + return True + except (StopIteration, httpx.TransportError): + return False + return True + + +def _initialize(client: httpx.Client, headers: Mapping[str, str]) -> httpx.Response: + return client.post( + "/mcp", + headers={**MCP_HEADERS, **headers}, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "rust-forward", "version": "1"}, + }, + }, + ) + + +def _mcp_post(client: httpx.Client, session: str, body: Mapping[str, JsonValue]) -> httpx.Response: + return client.post("/mcp", headers={**MCP_HEADERS, "mcp-session-id": session}, json=body) + + +def test_mcp_session_keeps_working_while_its_get_stream_sits_idle(rust_proxy: RustProxy) -> None: + idle_seconds: Final = 8 * KEEPALIVE_SECONDS + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=30) as client: + initialized: Final = _initialize(client, {}) + assert initialized.status_code == 200, initialized.text + session: Final = initialized.headers["mcp-session-id"] + assert _mcp_post(client, session, {"jsonrpc": "2.0", "method": "notifications/initialized"}).status_code == 202 + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=httpx.Timeout(5, read=2)) as listener: + with listener.stream( + "GET", "/mcp", headers={**MCP_HEADERS, "Accept": "text/event-stream", "mcp-session-id": session} + ) as notifications: + assert notifications.status_code == 200 + assert notifications.headers["content-type"].startswith("text/event-stream") + time.sleep(idle_seconds) + called: Final = _mcp_post( + client, + session, + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "math-add", "arguments": {"a": 2, "b": 3}}, + }, + ) + assert called.status_code == 200, called.text + assert '"text":"5"' in called.text.replace(" ", ""), called.text + assert _still_open(notifications) + + +def test_uploaded_file_and_its_download_cross_byte_for_byte(rust_proxy: RustProxy) -> None: + with openai.OpenAI(api_key=MASTER_KEY, base_url=rust_proxy.url, max_retries=0, timeout=60) as client: + created: Final = client.files.create( + file=("upload.jsonl", UPLOAD), purpose="batch", extra_body={"model": FILES_MODEL} + ) + downloaded: Final = client.files.content(created.id, extra_query={"model": FILES_MODEL}).content + assert len(UPLOAD) > 4 * 1024 * 1024 + assert downloaded == UPLOAD, (len(downloaded), len(UPLOAD)) + + +def test_internal_only_mcp_server_judges_the_connecting_peer_not_its_forwarded_for_claim( + rust_proxy: RustProxy, +) -> None: + outsider: Final = _routable_address() + with httpx.Client( + base_url=f"http://{outsider}:{rust_proxy.port}", + transport=httpx.HTTPTransport(local_address=outsider), + trust_env=False, + timeout=10, + ) as spoofing: + spoofed: Final = _initialize(spoofing, {"X-Forwarded-For": "127.0.0.1"}) + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=10) as internal: + admitted: Final = _initialize(internal, {}) + assert admitted.status_code == 200, admitted.text + assert spoofed.status_code == 403, spoofed.text + assert "client IP" in spoofed.text, spoofed.text + + +def test_oauth_metadata_names_the_origin_the_client_used(rust_proxy: RustProxy) -> None: + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=10) as client: + direct: Final = client.get("/.well-known/oauth-protected-resource") + balanced: Final = client.get( + "/.well-known/oauth-protected-resource", + headers={"Host": "gateway.example.test", "X-Forwarded-Proto": "https"}, + ) + assert direct.status_code == 200, direct.text + assert direct.json()["resource"] == rust_proxy.url + assert direct.json()["authorization_servers"] == [f"{rust_proxy.url}/mcp"] + assert balanced.status_code == 200, balanced.text + assert balanced.json()["resource"] == "https://gateway.example.test" + + +def _chat_stream(client: httpx.Client, prompt: str) -> AbstractContextManager[httpx.Response]: + return client.stream( + "POST", + "/v1/chat/completions", + headers={"Authorization": f"Bearer {MASTER_KEY}"}, + json={"model": CHAT_MODEL, "stream": True, "messages": [{"role": "user", "content": prompt}]}, + ) + + +def _content(lines: Iterator[str]) -> str: + return next(line for line in lines if '"content"' in line) + + +def test_stream_chunk_reaches_the_client_before_the_provider_sends_the_next(rust_proxy: RustProxy) -> None: + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=httpx.Timeout(10, read=5)) as client: + with _chat_stream(client, "gated") as response: + assert response.status_code == 200, response.read() + lines: Final = response.iter_lines() + first: Final = _content(lines) + rust_proxy.provider.released.set() + rest: Final = tuple(line for line in lines if line) + assert "before-gate" in first, first + assert any("after-gate" in line for line in rest), rest + assert rest[-1] == "data: [DONE]", rest + + +def test_client_hang_up_mid_stream_closes_the_provider_stream(rust_proxy: RustProxy) -> None: + marker: Final = f"endless-{time.monotonic_ns()}" + with httpx.Client(base_url=rust_proxy.url, trust_env=False, timeout=httpx.Timeout(10, read=5)) as client: + with _chat_stream(client, marker) as response: + assert response.status_code == 200, response.read() + assert "tick-0" in _content(response.iter_lines()) + assert rust_proxy.provider.hung_up.get(timeout=15) == marker + + +def test_realtime_websocket_frames_cross_both_ways(rust_proxy: RustProxy) -> None: + marker: Final = f"realtime-{time.monotonic_ns()}" + with connect( + f"ws://127.0.0.1:{rust_proxy.port}/v1/realtime?model={REALTIME_MODEL}", + additional_headers={"Authorization": f"Bearer {MASTER_KEY}"}, + open_timeout=10, + ) as session: + session.send( + json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": marker}]}, + } + ) + ) + replies: Final = tuple(session.recv(timeout=10) for _ in range(2)) + assert any(f"echo:{marker}" in reply for reply in replies), replies diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index d682fcbad2d..95b3a94c3bc 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -15,10 +15,19 @@ import pytest from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main -from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv +from gateway.launch import GATEWAY_APP, main, pool_database_url, python_server, uvicorn_argv from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_forward.mode import ( + Mode, + RustForwardMode, + Supervised, + uvicorn_flags, +) +from litellm.proxy.rust_forward.server import NO_TRUSTED_PROXIES, Listener +from litellm.proxy.server_listener import AssignedPort, InheritedFd +from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -114,6 +123,91 @@ def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]: class TestUvicornArgv: + @pytest.mark.parametrize( + ("arguments", "listener"), + [ + ((), Listener.TCP), + (("--uds", "/tmp/gateway.sock"), Listener.UNIX_SOCKET), + (("--fd", "0"), Listener.INHERITED_FD), + (("--fd=17",), Listener.INHERITED_FD), + (("--ssl-keyfile", "unused"), Listener.TLS), + (("--ssl-certfile", "unused"), Listener.TLS), + (("--ssl-ciphers", "TLSv1.2"), Listener.TCP), + ], + ) + def test_forwarding_sees_the_public_listener_uvicorn_would_open( + self, arguments: tuple[str, ...], listener: Listener + ) -> None: + assert python_server(arguments).listener is listener + + @pytest.mark.parametrize( + ("argv", "environ", "trust"), + [ + ((), {}, "127.0.0.1"), + ((), {"FORWARDED_ALLOW_IPS": "10.9.9.9"}, "10.9.9.9"), + (("--forwarded-allow-ips", "10.0.0.5"), {"FORWARDED_ALLOW_IPS": "10.9.9.9"}, "10.0.0.5"), + (("--forwarded-allow-ips=*",), {}, "*"), + (("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, NO_TRUSTED_PROXIES), + ], + ) + def test_forwarding_carries_the_trust_uvicorn_resolves_to_the_child( + self, + argv: tuple[str, ...], + environ: dict[str, str], + trust: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + server: Final = python_server(("--host", "0.0.0.0", "--port", "4000", *argv)) + assert (server.host, server.port) == ("0.0.0.0", 4000) + assert server.trusted_proxies == trust + + @pytest.mark.parametrize( + ("argv", "environ"), + [(("--timeout-keep-alive", "2"), {}), ((), {"KEEPALIVE_TIMEOUT": "7"}), ((), {})], + ) + def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with( + self, argv: tuple[str, ...], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + served: Final = _uvicorn_params(uvicorn_argv(argv, os.environ))["timeout_keep_alive"] + assert python_server(argv).keepalive_seconds == served + + @pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")]) + def test_rust_drains_for_pythons_graceful_timeout_plus_teardown( + self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5") + flag: Final = _uvicorn_params(uvicorn_argv(argv, {}))["timeout_graceful_shutdown"] + graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag)) + assert python_server(argv).graceful_timeout_seconds == graceful + + @pytest.mark.parametrize( + ("mode", "served"), + [ + (Mode.DISABLED, {"host": "0.0.0.0", "port": 4000, "fd": None, "proxy_headers": True}), + ( + Supervised(InheritedFd(17), "*"), + {"host": "0.0.0.0", "port": 4000, "fd": 17, "proxy_headers": False}, + ), + ( + Supervised(AssignedPort(45556), "*"), + {"host": "127.0.0.1", "port": 45556, "fd": None, "proxy_headers": False}, + ), + ], + ) + def test_supervised_uvicorn_serves_the_private_hop_and_never_trusts_it( + self, mode: RustForwardMode, served: dict[str, object] + ) -> None: + argv: Final = ("--host", "0.0.0.0", "--port", "4000", "--workers", "2", "--proxy-headers") + params: Final = _uvicorn_params((*uvicorn_argv((*argv, "--forwarded-allow-ips=*"), {}), *uvicorn_flags(mode))) + assert {name: params[name] for name in served} == served + assert (params["workers"], params["forwarded_allow_ips"]) == (2, "*") + def test_keepalive_env_reaches_uvicorn(self): params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"})) assert params["app"] == GATEWAY_APP @@ -229,6 +323,70 @@ class TestMain: DatabaseURLSettings.from_env().apply_to_env() assert os.environ["DATABASE_URL"] == pooled + @pytest.mark.parametrize( + ("rust", "warning"), + [ + ("false", ""), + ( + "true", + "LITELLM_RUST_FIRST does not support a Unix socket on the public port, " + "so Python serves the public port without Rust forwarding\n", + ), + ], + ) + def test_python_alone_keeps_uvicorn_socket_options( + self, + rust: str, + warning: str, + password_env: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + monkeypatch.setenv("LITELLM_RUST_FIRST", rust) + monkeypatch.delenv("_LITELLM_GATEWAY_SUPERVISED", raising=False) + monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False) + served: Final[list[tuple[str, ...]]] = [] + main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: served.append(tuple(argv))) + assert _uvicorn_params(served[0])["uds"] == "/tmp/gateway.sock" + assert capsys.readouterr().err == warning + + def test_an_invalid_rust_switch_stops_the_gateway_naming_the_variable( + self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LITELLM_RUST_FIRST", "maybe") + monkeypatch.delenv("_LITELLM_GATEWAY_SUPERVISED", raising=False) + served: Final[list[tuple[str, ...]]] = [] + with pytest.raises(SystemExit) as stopped: + main(("--workers", "4"), serve=lambda argv: served.append(tuple(argv))) + assert str(stopped.value).startswith("LITELLM_RUST_FIRST is invalid: ") + assert served == [] + + @pytest.mark.parametrize( + ("argv", "status", "said"), + [ + pytest.param(("--nope",), 2, "--nope", id="a bad flag is a usage error"), + pytest.param(("--help",), 0, "--timeout-keep-alive", id="help is printed once and exits cleanly"), + ], + ) + def test_the_supervisor_reports_uvicorn_arguments_the_way_uvicorn_does( + self, + argv: tuple[str, ...], + status: int, + said: str, + password_env: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + monkeypatch.setenv("LITELLM_RUST_FIRST", "true") + monkeypatch.delenv("_LITELLM_GATEWAY_SUPERVISED", raising=False) + served: Final[list[tuple[str, ...]]] = [] + with pytest.raises(SystemExit) as stopped: + main(argv, serve=lambda served_argv: served.append(tuple(served_argv))) + printed: Final = capsys.readouterr() + assert stopped.value.code == status + assert (printed.out + printed.err).count(said) == 1 + assert served == [] + def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn( self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path ): diff --git a/tests/unit/proxy/rust_forward/__init__.py b/tests/unit/proxy/rust_forward/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/rust_forward/supervisor/__init__.py b/tests/unit/proxy/rust_forward/supervisor/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/rust_forward/supervisor/_support.py b/tests/unit/proxy/rust_forward/supervisor/_support.py new file mode 100644 index 00000000000..bac5a68b7fc --- /dev/null +++ b/tests/unit/proxy/rust_forward/supervisor/_support.py @@ -0,0 +1,7 @@ +from litellm.proxy.rust_forward.server import Listener, PythonServer, Server + + +def python_server( + server: Server = Server.UVICORN, listener: Listener = Listener.TCP, trusted_proxies: str = "127.0.0.1" +) -> PythonServer: + return PythonServer(server, listener, "0.0.0.0", 4000, 2, 2.5, trusted_proxies) diff --git a/tests/unit/proxy/rust_forward/supervisor/test_children.py b/tests/unit/proxy/rust_forward/supervisor/test_children.py new file mode 100644 index 00000000000..f6ab9732b80 --- /dev/null +++ b/tests/unit/proxy/rust_forward/supervisor/test_children.py @@ -0,0 +1,54 @@ +import socket +from typing import Final + +from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS +from litellm.proxy.proxy_cli import run_server +from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode +from litellm.proxy.rust_forward.platform import PlatformListener +from litellm.proxy.rust_forward.platform.posix import PosixPlatform +from litellm.proxy.rust_forward.platform.windows import CREATE_NEW_PROCESS_GROUP, WindowsPlatform +from litellm.proxy.rust_forward.supervisor.children import python_spec, rust_spec +from litellm.proxy.server_listener import AssignedPort, InheritedFd +from tests.unit.proxy.rust_forward.supervisor._support import python_server + +POSIX: Final = PosixPlatform() +WINDOWS: Final = WindowsPlatform() + + +def test_python_child_serves_the_private_socket_with_the_operators_trust() -> None: + private: Final = PlatformListener(InheritedFd(17), 45555, (17,)) + spec: Final = python_spec(POSIX, ("python",), private, python_server(trusted_proxies="10.0.0.5")) + assert rust_forward_mode(spec.env_overrides) == Supervised(InheritedFd(17), "10.0.0.5") + assert (spec.spawn.pass_fds, spec.spawn.start_new_session) == ((17,), True) + + +def test_windows_python_child_binds_the_assigned_port_in_its_own_process_group() -> None: + with WINDOWS.private_listener() as private: + pass + spec: Final = python_spec(WINDOWS, ("python",), private, python_server()) + assert rust_forward_mode(spec.env_overrides) == Supervised(AssignedPort(private.port), "127.0.0.1") + assert spec.argv[-2:] == ("--port", str(private.port)) + assert (spec.spawn.pass_fds, spec.spawn.creationflags) == ((), CREATE_NEW_PROCESS_GROUP) + with socket.create_server(("127.0.0.1", private.port)): + pass + + +def test_python_child_keeps_the_public_host_and_its_options() -> None: + arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload") + spec: Final = python_spec(POSIX, arguments, PlatformListener(InheritedFd(17), 45555, (17,)), python_server()) + params: Final = run_server.make_context("litellm", list(spec.argv)).params + assert (params["host"], params["port"]) == ("0.0.0.0", 45555) + assert (params["num_workers"], params["reload"]) == (2, True) + + +def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None: + spec: Final = rust_spec(POSIX, "gateway", 45555, python_server()) + assert spec.argv == ("gateway",) + assert spec.spawn.pass_fds == () + assert dict(spec.env_overrides) == { + "LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555", + "LITELLM_FORWARD_HOST": "0.0.0.0", + "LITELLM_FORWARD_PORT": "4000", + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE": "2", + "LITELLM_FORWARD_SHUTDOWN_DRAIN": str(2.5 + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS), + } diff --git a/tests/unit/proxy/rust_forward/supervisor/test_lifecycle.py b/tests/unit/proxy/rust_forward/supervisor/test_lifecycle.py new file mode 100644 index 00000000000..4b479f3d22d --- /dev/null +++ b/tests/unit/proxy/rust_forward/supervisor/test_lifecycle.py @@ -0,0 +1,158 @@ +import functools +import itertools +import signal +from dataclasses import dataclass, replace +from typing import Final + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from litellm.proxy.rust_forward.platform.posix import PosixPlatform +from litellm.proxy.rust_forward.supervisor.lifecycle import ( + FAILURE, + Action, + Caught, + Child, + ChildExited, + Event, + Lifecycle, + Probe, + Send, + Spawn, + boot, + exit_status, + step, +) + +PY: Final = Child.PYTHON +RUST: Final = Child.RUST +LISTENING: Final = Probe.PYTHON_LISTENING +TERM: Final = signal.SIGTERM +INT: Final = signal.SIGINT +HUP: Final = signal.SIGHUP +STOPS: Final = PosixPlatform().stop_signals + + +def _replay(events: tuple[Event, ...]) -> tuple[tuple[tuple[Action, ...], ...], int | None]: + transitions: Final = tuple( + itertools.accumulate(events, lambda transition, event: step(transition[0], event, STOPS), initial=boot()) + ) + return tuple(actions for _, actions in transitions[1:]), exit_status(transitions[-1][0]) + + +@pytest.mark.parametrize( + ("trace", "status"), + [ + pytest.param( + ( + (LISTENING, (Spawn(RUST),)), + (Caught(TERM), (Send(RUST, TERM), Send(PY, TERM))), + (ChildExited(RUST, 0), ()), + (ChildExited(PY, -15), ()), + ), + -15, + id="TERM reaches Rust first, then Python, and the supervisor dies as Python did", + ), + pytest.param( + ( + (LISTENING, (Spawn(RUST),)), + (Caught(INT), (Send(RUST, INT), Send(PY, INT))), + (Caught(INT), (Send(RUST, INT), Send(PY, INT))), + (ChildExited(PY, 1), (Send(RUST, TERM),)), + (ChildExited(RUST, 0), ()), + ), + 1, + id="a second Ctrl-C reaches Python again to force the exit", + ), + pytest.param( + ( + (LISTENING, (Spawn(RUST),)), + (Caught(HUP), (Send(PY, HUP),)), + (Caught(signal.SIGTTIN), (Send(PY, signal.SIGTTIN),)), + (ChildExited(PY, 0), (Send(RUST, TERM),)), + (ChildExited(RUST, 0), ()), + ), + 0, + id="gunicorn's HUP and TTIN reach only Python and Rust keeps serving", + ), + pytest.param( + ( + (LISTENING, (Spawn(RUST),)), + (ChildExited(RUST, 0), (Send(PY, TERM),)), + (ChildExited(PY, -15), ()), + ), + FAILURE, + id="losing Rust while Python serves stops Python and fails", + ), + pytest.param( + ( + (Caught(TERM), (Send(PY, TERM),)), + (LISTENING, ()), + (ChildExited(PY, -15), ()), + ), + -15, + id="a stop before Python listens never starts Rust", + ), + pytest.param(((ChildExited(PY, 3), ()),), 3, id="Python failing to boot exits with its status"), + ], +) +def test_supervisor_relays_signals_and_exits_as_python_did( + trace: tuple[tuple[Event, tuple[Action, ...]], ...], status: int +) -> None: + events: Final = tuple(event for event, _ in trace) + assert _replay(events) == (tuple(actions for _, actions in trace), status) + + +@dataclass(frozen=True, slots=True) +class Observed: + """What an outside observer of the two processes knows, without looking inside ``Lifecycle``.""" + + state: Lifecycle + alive: frozenset[Child] = frozenset({PY}) + rust_spawns: int = 0 + stop_requested: bool = False + rust_lost: bool = False + python_status: int | None = None + + +def _observe(seen: Observed, event: Event) -> Observed: + state, actions = step(seen.state, event, STOPS) + death: Final = event if isinstance(event, ChildExited) and event.child in seen.alive else None + exited: Final = None if death is None else death.child + running: Final = seen.alive - {exited} + assert all(action.child in running for action in actions if isinstance(action, Send)), "signalled a dead child" + if isinstance(event, Caught): + to_rust: Final = (Send(RUST, event.signum),) if event.signum in STOPS and RUST in running else () + assert actions == (*to_rust, *((Send(PY, event.signum),) if PY in running else ())) + spawned: Final = Spawn(RUST) in actions + assert not spawned or (event is LISTENING and not seen.stop_requested), "started Rust out of turn" + stop_requested: Final = seen.stop_requested or (isinstance(event, Caught) and event.signum in STOPS) + after: Final = replace( + seen, + state=state, + alive=running | ({RUST} if spawned else frozenset()), + rust_spawns=seen.rust_spawns + int(spawned), + stop_requested=stop_requested, + rust_lost=seen.rust_lost or (exited is RUST and PY in running and not stop_requested), + python_status=death.status if death is not None and exited is PY else seen.python_status, + ) + settled: Final = None if after.alive or after.python_status is None else after.python_status + assert exit_status(state) == (FAILURE if after.rust_lost and settled is not None else settled) + return after + + +EVENTS: Final = st.lists( + st.one_of( + st.sampled_from(PosixPlatform().relayed_signals).map(Caught), + st.just(LISTENING), + st.builds(ChildExited, st.sampled_from(Child), st.integers(-15, 3)), + ), + max_size=40, +) + + +@given(EVENTS) +def test_any_event_order_keeps_the_supervisor_transparent_to_python(events: list[Event]) -> None: + final: Final = functools.reduce(_observe, events, Observed(boot()[0])) + assert final.rust_spawns <= 1 diff --git a/tests/unit/proxy/rust_forward/supervisor/test_preflight.py b/tests/unit/proxy/rust_forward/supervisor/test_preflight.py new file mode 100644 index 00000000000..82daf5b7ea0 --- /dev/null +++ b/tests/unit/proxy/rust_forward/supervisor/test_preflight.py @@ -0,0 +1,77 @@ +import sys +from pathlib import Path +from typing import Final + +import pytest + +from litellm.proxy.rust_forward.platform import Platform +from litellm.proxy.rust_forward.platform.posix import PosixPlatform +from litellm.proxy.rust_forward.platform.windows import WindowsPlatform +from litellm.proxy.rust_forward.server import Listener, PythonServer, Server +from litellm.proxy.rust_forward.supervisor import supervise +from litellm.proxy.rust_forward.supervisor.preflight import StartupError, gateway_binary, startup_check +from tests.unit.proxy.rust_forward.supervisor._support import python_server + +POSIX: Final = PosixPlatform() +WINDOWS: Final = WindowsPlatform() +POSIX_ONLY: Final = pytest.mark.skipif(sys.platform == "win32", reason="runs a POSIX shebang gateway") + + +@pytest.mark.parametrize( + ("platform", "server", "refusal"), + [ + (POSIX, python_server(), None), + (POSIX, python_server(Server.GUNICORN), None), + (WINDOWS, python_server(), None), + (WINDOWS, python_server(Server.GUNICORN), "does not support gunicorn on Windows"), + (POSIX, python_server(Server.HYPERCORN), "does not support Hypercorn"), + (POSIX, python_server(Server.GRANIAN), "does not support Granian"), + (POSIX, python_server(listener=Listener.TLS), "does not support TLS on the public port"), + (POSIX, python_server(listener=Listener.UNIX_SOCKET), "does not support a Unix socket on the public port"), + ( + POSIX, + python_server(Server.GUNICORN, Listener.INHERITED_FD), + "does not support an inherited file descriptor on the public port", + ), + ], +) +def test_only_plain_tcp_on_a_supported_server_can_be_forwarded( + platform: Platform, server: PythonServer, refusal: str | None +) -> None: + expected: Final = "gateway" if refusal is None else StartupError(f"LITELLM_RUST_FIRST {refusal}") + assert startup_check(platform, server, "gateway") == expected + + +@pytest.mark.parametrize( + ("server", "refusal"), + [ + (python_server(Server.HYPERCORN), "does not support Hypercorn"), + (python_server(), "requires the litellm-gateway binary from the litellm-proxy package"), + ], +) +def test_a_refused_start_says_why_in_one_line_and_starts_nothing( + server: PythonServer, refusal: str, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + prepared: Final[list[bool]] = [] + refused: Final = supervise( + server, ("python",), before_start=lambda: prepared.append(True), scripts_dir=str(tmp_path), platform=POSIX + ) + assert (refused, prepared) == (StartupError(f"LITELLM_RUST_FIRST {refusal}"), []) + assert ( + capsys.readouterr().err + == f"LITELLM_RUST_FIRST {refusal}, so Python serves the public port without Rust forwarding\n" + ) + + +@POSIX_ONLY +def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scripts: Final = tmp_path / "scripts" + elsewhere: Final = tmp_path / "elsewhere" + scripts.mkdir() + elsewhere.mkdir() + binary: Final = elsewhere / "litellm-gateway" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + monkeypatch.setenv("PATH", str(elsewhere)) + assert gateway_binary(POSIX, str(scripts)) is None + assert gateway_binary(POSIX, str(elsewhere)) == str(binary) diff --git a/tests/unit/proxy/rust_forward/supervisor/test_runtime.py b/tests/unit/proxy/rust_forward/supervisor/test_runtime.py new file mode 100644 index 00000000000..b84b1302e3c --- /dev/null +++ b/tests/unit/proxy/rust_forward/supervisor/test_runtime.py @@ -0,0 +1,189 @@ +import signal +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import anyio +import pytest + +from litellm.proxy.rust_forward.platform import Platform, current_platform +from litellm.proxy.rust_forward.platform.windows import WindowsPlatform +from litellm.proxy.rust_forward.supervisor import supervise +from litellm.proxy.rust_forward.supervisor.children import ChildSpec +from litellm.proxy.rust_forward.supervisor.lifecycle import ( + Action, + Caught, + Child, + ChildExited, + Event, + Probe, + Send, + Spawn, +) +from litellm.proxy.rust_forward.supervisor.runtime import SpawnFailed, drive, exit_like, open_children +from tests.unit.proxy.rust_forward.supervisor._support import python_server + + +PYTHON_CHILD: Final = """ +import json, os, pathlib, socket, time +time.sleep(0.5) +pathlib.Path(os.environ["MARKERS"], "listening").touch() +private = json.loads(os.environ["_LITELLM_GATEWAY_SUPERVISED"])["listener"] +listener = socket.socket(fileno=private["fd"]) if "fd" in private else socket.create_server(("127.0.0.1", private["port"])) +listener.listen() +time.sleep(10) +""" +RUST_CHILD: Final = """ +import os, pathlib +markers = pathlib.Path(os.environ["MARKERS"]) +(markers / "rust_saw_python_listening").write_text(str((markers / "listening").exists())) +""" + + +def _fake_gateway(scripts: Path) -> Platform: + if sys.platform != "win32": + gateway: Final = scripts / "litellm-gateway" + gateway.write_text(f"#!{sys.executable} -I\n{RUST_CHILD}") + gateway.chmod(0o755) + return current_platform() + script: Final = scripts / "rust_child.py" + script.write_text(RUST_CHILD) + (scripts / "litellm-gateway.cmd").write_text(f'@"{sys.executable}" -I "{script}"\r\n') + return WindowsPlatform(gateway_name="litellm-gateway.cmd") + + +def test_rust_opens_the_public_port_only_once_python_listens(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MARKERS", str(tmp_path)) + command: Final = (sys.executable, "-I", "-c", PYTHON_CHILD) + supervise(python_server(), command, scripts_dir=str(tmp_path), platform=_fake_gateway(tmp_path)) + assert (tmp_path / "rust_saw_python_listening").read_text() == "True" + + +TERM: Final = signal.SIGTERM +POSIX_ONLY: Final = pytest.mark.skipif(sys.platform == "win32", reason="the child traps SIGTERM") +STOPS: Final = frozenset({signal.SIGTERM, signal.SIGINT, signal.SIGQUIT}) +RUST_MISSING: Final = SpawnFailed(FileNotFoundError("litellm-gateway")) + + +@dataclass +class ScriptedExecutor: + """Replays a fixed event script and records every action the driver carries out.""" + + events: list[Event] + failing: frozenset[Child] + done: list[Action] = field(default_factory=list) + + async def spawn(self, child: Child) -> SpawnFailed | None: + self.done.append(Spawn(child)) + return RUST_MISSING if child in self.failing else None + + def send(self, child: Child, signum: signal.Signals) -> None: + self.done.append(Send(child, signum)) + + async def next_event(self) -> Event: + return self.events.pop(0) + + +@pytest.mark.parametrize( + ("events", "failing", "done", "outcome", "unread"), + [ + pytest.param( + [Probe.PYTHON_LISTENING, Caught(TERM), ChildExited(Child.RUST, 0), ChildExited(Child.PYTHON, -15)], + frozenset(), + [Spawn(Child.PYTHON), Spawn(Child.RUST), Send(Child.RUST, TERM), Send(Child.PYTHON, TERM)], + -15, + [], + id="carries out each action in order and ends with Python's status", + ), + pytest.param( + [ChildExited(Child.PYTHON, 3), Caught(TERM)], + frozenset(), + [Spawn(Child.PYTHON)], + 3, + [Caught(TERM)], + id="reads no event once the status is settled", + ), + pytest.param( + [Probe.PYTHON_LISTENING, Caught(TERM)], + frozenset({Child.RUST}), + [Spawn(Child.PYTHON), Spawn(Child.RUST)], + RUST_MISSING, + [Caught(TERM)], + id="returns a failed spawn without waiting for more events", + ), + ], +) +async def test_driver_executes_lifecycle_decisions( + events: list[Event], failing: frozenset[Child], done: list[Action], outcome: int | SpawnFailed, unread: list[Event] +) -> None: + executor: Final = ScriptedExecutor(events, failing) + assert await drive(executor, STOPS) == outcome + assert (executor.done, executor.events) == (done, unread) + + +def _children(argv: tuple[str, ...]) -> MappingProxyType[Child, ChildSpec]: + spec: Final = ChildSpec(argv, MappingProxyType({}), current_platform().spawn_options(())) + return MappingProxyType({Child.PYTHON: spec, Child.RUST: spec}) + + +SLOW_TO_STOP: Final = ( + sys.executable, + "-c", + "import signal, time; signal.signal(signal.SIGTERM, lambda *_: (time.sleep(0.5), exit(0))); time.sleep(30)", +) + + +@POSIX_ONLY +async def test_a_cancelled_supervisor_still_waits_for_the_child_it_stops() -> None: + with anyio.CancelScope() as scope: + async with open_children(current_platform(), _children(SLOW_TO_STOP), 1) as children: + assert await children.spawn(Child.RUST) is None + process: Final = children.processes[Child.RUST] + await anyio.sleep(0.3) + scope.cancel() + await anyio.sleep(1) + assert process.returncode == 0 + + +async def test_exiting_children_are_reported_as_events() -> None: + async with open_children(current_platform(), _children((sys.executable, "-c", "exit(7)")), 1) as children: + assert await children.spawn(Child.RUST) is None + with anyio.fail_after(5): + assert await children.next_event() == ChildExited(Child.RUST, 7) + + +async def test_a_child_that_cannot_start_is_returned_as_a_failure(tmp_path: Path) -> None: + async with open_children(current_platform(), _children((str(tmp_path / "missing"),)), 1) as children: + outcome: Final = await children.spawn(Child.RUST) + assert isinstance(outcome, SpawnFailed) + assert isinstance(outcome.error, FileNotFoundError) + + +def _dropped(_signum: signal.Signals) -> None: + pass + + +@POSIX_ONLY +@pytest.mark.parametrize( + ("status", "exits"), + [ + pytest.param(3, 3, id="an exit status is passed on"), + pytest.param(-signal.SIGTERM, 0, id="a stop signal ends it the way uvicorn alone ends as PID 1"), + pytest.param(-signal.SIGKILL, 137, id="a kill is reported the way a shell reports it"), + ], +) +def test_a_supervisor_the_kernel_will_not_signal_still_exits_as_python_alone_would(status: int, exits: int) -> None: + assert exit_like(status, STOPS, die_by=_dropped) == exits + + +@POSIX_ONLY +@pytest.mark.parametrize("signum", [signal.SIGTERM, signal.SIGKILL]) +def test_the_supervisor_dies_by_the_signal_that_killed_python(signum: signal.Signals) -> None: + script: Final = ( + "from litellm.proxy.rust_forward.supervisor.runtime import exit_like\n" + f"raise SystemExit(exit_like({-signum}, frozenset()))" + ) + assert subprocess.run((sys.executable, "-I", "-c", script), check=False, timeout=60).returncode == -signum diff --git a/tests/unit/proxy/rust_forward/test_mode.py b/tests/unit/proxy/rust_forward/test_mode.py new file mode 100644 index 00000000000..59f9feb1d3b --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_mode.py @@ -0,0 +1,127 @@ +import os +import threading +from typing import Final + +import pytest +import uvicorn +from pydantic import ValidationError + +from litellm.proxy.proxy_cli import WORKER_APP, ProxyInitializationHelpers +from litellm.proxy.rust_forward.mode import ( + SUPERVISED_ENV, + Mode, + RustForwardMode, + Supervised, + rust_forward_mode, + private_listener, + stop_with_supervisor, + supervised_environment, + uvicorn_options, +) +from litellm.proxy.rust_forward.server import NO_TRUSTED_PROXIES +from litellm.proxy.server_listener import AssignedPort, InheritedFd, PrivateListener + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + *((value, Mode.DISABLED) for value in (None, "", " ", "0", "false", "off")), + *((value, Mode.SUPERVISOR) for value in ("1", "true", "yes", "on", " 1 ")), + ], +) +def test_mode_is_resolved_from_environment(value: str | None, expected: RustForwardMode) -> None: + assert rust_forward_mode({} if value is None else {"LITELLM_RUST_FIRST": value}) == expected + + +@pytest.mark.parametrize( + ("environ", "expected"), + [ + ({"LITELLM_RUST": "true"}, Mode.DISABLED), + ({"LITELLM_RUST": "enabled"}, Mode.DISABLED), + ({"LITELLM_RUST": "false", "LITELLM_RUST_FIRST": "true"}, Mode.SUPERVISOR), + ], +) +def test_forwarding_is_independent_of_the_sdk_rust_switch(environ: dict[str, str], expected: RustForwardMode) -> None: + assert rust_forward_mode(environ) == expected + + +@pytest.mark.parametrize( + ("environ", "invalid"), + [ + ({"LITELLM_RUST_FIRST": "enabled"}, "LITELLM_RUST_FIRST"), + ({"LITELLM_RUST_FIRST": "2"}, "LITELLM_RUST_FIRST"), + ({SUPERVISED_ENV: '{"listener": {"fd": "seventeen"}, "trusted_proxies": "*"}'}, "fd"), + ({SUPERVISED_ENV: '{"listener": {"fd": 17}, "trusted_proxies": 7}'}, "trusted_proxies"), + ({SUPERVISED_ENV: '{"listener": {"fd": 17}}'}, "trusted_proxies"), + ], +) +def test_invalid_forwarding_value_stops_startup_naming_it(environ: dict[str, str], invalid: str) -> None: + with pytest.raises(ValidationError, match=invalid): + rust_forward_mode(environ) + + +@pytest.mark.parametrize( + "written", + [ + Supervised(InheritedFd(17), "127.0.0.1"), + Supervised(InheritedFd(17), "127.0.0.1,10.0.0.5"), + Supervised(InheritedFd(17), NO_TRUSTED_PROXIES), + Supervised(AssignedPort(45555), "*"), + ], +) +def test_child_reads_back_exactly_what_the_supervisor_wrote(written: Supervised) -> None: + inherited: Final = {"LITELLM_RUST_FIRST": "true", "FORWARDED_ALLOW_IPS": "10.9.9.9"} + assert rust_forward_mode({**inherited, **supervised_environment(written)}) == written + + +def _served_uvicorn_config(mode: RustForwardMode) -> uvicorn.Config: + options: Final = { + **ProxyInitializationHelpers.get_default_unvicorn_init_args("0.0.0.0", 45555), + **uvicorn_options(mode, WORKER_APP), + } + return uvicorn.Config(**options) + + +PUBLIC: Final = ("0.0.0.0", 45555, None) + + +@pytest.mark.parametrize( + ("mode", "app", "uvicorn_listener", "gunicorn_listener", "proxy_headers"), + [ + (Mode.DISABLED, "litellm.proxy.proxy_server:app", PUBLIC, None, True), + (Mode.SUPERVISOR, "litellm.proxy.proxy_server:app", PUBLIC, None, True), + (Supervised(InheritedFd(17), "*"), WORKER_APP, ("0.0.0.0", 45555, 17), InheritedFd(17), False), + (Supervised(AssignedPort(45556), "*"), WORKER_APP, ("127.0.0.1", 45556, None), AssignedPort(45556), False), + ], +) +def test_supervised_server_serves_the_private_hop_and_never_trusts_it( + mode: RustForwardMode, + app: str, + uvicorn_listener: tuple[str, int, int | None], + gunicorn_listener: PrivateListener | None, + proxy_headers: bool, +) -> None: + config: Final = _served_uvicorn_config(mode) + assert (config.app, (config.host, config.port, config.fd), config.proxy_headers) == ( + app, + uvicorn_listener, + proxy_headers, + ) + assert config.factory is (app == WORKER_APP) + assert private_listener(mode) == gunicorn_listener + + +@pytest.mark.parametrize( + ("mode", "stops"), + [(Supervised(InheritedFd(17), "127.0.0.1"), True), (Mode.DISABLED, False), (Mode.SUPERVISOR, False)], +) +def test_only_a_supervised_server_stops_once_its_supervisor_is_gone(mode: RustForwardMode, stops: bool) -> None: + read_end, write_end = os.pipe() + stopped: Final = threading.Event() + try: + stop_with_supervisor(mode, read_end, stopped.set) + assert not stopped.wait(0.2) + os.close(write_end) + assert stopped.wait(5 if stops else 0.5) is stops + finally: + os.close(read_end) diff --git a/tests/unit/proxy/rust_forward/test_peer.py b/tests/unit/proxy/rust_forward/test_peer.py new file mode 100644 index 00000000000..11a4737c6ce --- /dev/null +++ b/tests/unit/proxy/rust_forward/test_peer.py @@ -0,0 +1,135 @@ +import asyncio +import gc +from typing import TYPE_CHECKING, Final + +import httpx +import pytest +from fastapi import Request +from fastapi.routing import APIRoute +from uvicorn.importer import import_from_string +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware + +if TYPE_CHECKING: + from uvicorn._types import ( + ASGIReceiveCallable, + ASGISendCallable, + ASGISendEvent, + HTTPDisconnectEvent, + HTTPScope, + Scope, + ) + +from litellm.proxy.proxy_server import app as proxy_app +from litellm.proxy.rust_forward.mode import SUPERVISED_ENV, Supervised, supervised_environment +from litellm.proxy.rust_forward.peer import RustPeerMiddleware +from litellm.proxy.rust_forward.server import NO_TRUSTED_PROXIES +from litellm.proxy.server_listener import InheritedFd + +Headers = tuple[tuple[bytes, bytes], ...] + + +async def _receive() -> "HTTPDisconnectEvent": + return {"type": "http.disconnect"} + + +async def _send(message: "ASGISendEvent") -> None: + return None + + +async def _seen_by_app( + behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised +) -> tuple[tuple[str, int] | None, str, Headers]: + seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = ( + asyncio.get_running_loop().create_future() + ) + + async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: + assert scope["type"] == "http" + seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"]))) + + proxy_headers_off: Final = mode.trusted_proxies == NO_TRUSTED_PROXIES + direct: Final = probe if proxy_headers_off else ProxyHeadersMiddleware(probe, mode.trusted_proxies) + app: Final = RustPeerMiddleware(probe, mode) if behind_rust else direct + scope: Final[HTTPScope] = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "root_path": "", + "headers": headers, + "client": client, + "server": ("127.0.0.1", 4000), + } + await app(scope, _receive, _send) + return seen.result() + + +PEER: Final = ("203.0.113.7", 51234) +TRUSTED_PEER: Final = ("10.0.12.34", 40000) +FORWARDED: Final[Headers] = ( + (b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"), + (b"x-forwarded-proto", b"https"), +) + + +@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)]) +@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))]) +@pytest.mark.parametrize( + "mode", + [ + Supervised(InheritedFd(17), "127.0.0.1"), + Supervised(InheritedFd(17), "10.0.0.0/8"), + Supervised(InheritedFd(17), "*"), + Supervised(InheritedFd(17), NO_TRUSTED_PROXIES), + ], +) +async def test_python_behind_rust_sees_what_a_direct_connection_would( + peer: tuple[str, int], headers: Headers, mode: Supervised +) -> None: + rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}" + behind_rust: Final = await _seen_by_app( + True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode + ) + direct: Final = await _seen_by_app(False, peer, headers, mode) + assert behind_rust == direct + + +async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None: + client, _, headers = await _seen_by_app( + True, + ("192.0.2.9", 1000), + ((b"x-litellm-rust-peer", b"10.0.12.34:1"),), + Supervised(InheritedFd(17), "127.0.0.1"), + ) + assert (client, headers) == (("192.0.2.9", 1000), ()) + + +async def _peer_probe(request: Request) -> dict[str, object]: + assert request.client is not None + return {"client": [request.client.host, request.client.port], "peer": request.headers.get("x-litellm-rust-peer")} + + +@pytest.mark.parametrize("entry_point", ["litellm_cli.main:proxy_worker_app", "gateway.launch:worker_app"]) +async def test_every_entry_point_hands_routes_the_peer_rust_saw( + entry_point: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(SUPERVISED_ENV, supervised_environment(Supervised(InheritedFd(17), "127.0.0.1"))[SUPERVISED_ENV]) + monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context) + monkeypatch.setattr( + proxy_app.router, "routes", [APIRoute("/__rust_forward_peer_probe", _peer_probe), *proxy_app.router.routes] + ) + try: + served: Final = import_from_string(entry_point)() + finally: + gc.unfreeze() + transport: Final = httpx.ASGITransport(app=served, client=("127.0.0.1", 60000)) + async with httpx.AsyncClient(transport=transport, base_url="http://proxy") as client: + response: Final = await client.get( + "/__rust_forward_peer_probe", headers={"x-litellm-rust-peer": "203.0.113.7:51234"} + ) + assert response.status_code == 200, response.text + assert response.json() == {"client": ["203.0.113.7", 51234], "peer": None} diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index eec682f365f..d85d685c352 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -26,6 +26,7 @@ import urllib.parse as urlparse import uvicorn import yaml +from gunicorn.config import Config as GunicornConfig from uvicorn.config import LOOP_FACTORIES from uvicorn.importer import import_from_string @@ -34,8 +35,9 @@ from litellm._logging import turn_on_json from litellm.constants import DEFAULT_MAX_REQUEST_HEADER_SIZE, GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS from litellm.proxy import proxy_cli from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server +from litellm.proxy.rust_forward.server import Listener, Server from litellm.proxy.server_listener import AssignedPort, InheritedFd -from litellm.proxy.shutdown.graceful_shutdown_manager import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT +from litellm.proxy.shutdown.graceful_shutdown_manager import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT, GracefulShutdownManager @pytest.fixture(autouse=True) @@ -4382,3 +4384,154 @@ def test_migration_data_check_requires_database_url(monkeypatch): with pytest.raises(click.UsageError, match="requires DATABASE_URL"): ProxyInitializationHelpers.run_migration_data_check(use_migrate=True) + + +class TestRustForwardServer: + """The public listener ``run_server`` reports to the Rust supervisor matches what it would serve itself.""" + + @staticmethod + def _server(argv: list[str]): + params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params + return params, ProxyInitializationHelpers.python_server( + str(params["host"]), + int(str(params["port"])), + params["keepalive_timeout"], + run_gunicorn=params["run_gunicorn"] is True, + run_hypercorn=params["run_hypercorn"] is True, + run_granian=params["run_granian"] is True, + tls=None not in (params["ssl_keyfile_path"], params["ssl_certfile_path"]), + ) + + @pytest.mark.parametrize( + ("argv", "server", "listener"), + [ + ([], Server.UVICORN, Listener.TCP), + (["--run_gunicorn"], Server.GUNICORN, Listener.TCP), + (["--run_hypercorn"], Server.HYPERCORN, Listener.TCP), + (["--run_granian"], Server.GRANIAN, Listener.TCP), + (["--ssl_keyfile_path", "key.pem", "--ssl_certfile_path", "cert.pem"], Server.UVICORN, Listener.TLS), + ], + ) + def test_the_supervisor_sees_the_server_and_listener_python_would_use( + self, argv: list[str], server: Server, listener: Listener + ) -> None: + _, described = self._server(argv) + assert (described.server, described.listener) == (server, listener) + + def test_uvicorn_is_described_where_gunicorn_cannot_be_imported(self, monkeypatch: pytest.MonkeyPatch) -> None: + _, with_gunicorn = self._server([]) + monkeypatch.setitem(sys.modules, "gunicorn.config", None) + _, without_gunicorn = self._server([]) + assert without_gunicorn == with_gunicorn + + @patch("uvicorn.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) + @pytest.mark.parametrize( + ("argv", "reason", "served_keyfile"), + [ + ( + ["--ssl_keyfile_path", "key.pem", "--ssl_certfile_path", "cert.pem"], + "does not support TLS on the public port", + "key.pem", + ), + (["--ssl_keyfile_path", "key.pem"], "requires the litellm-gateway binary", None), + (["--ssl_certfile_path", "cert.pem"], "requires the litellm-gateway binary", None), + (["--ciphers", "ECDHE"], "requires the litellm-gateway binary", None), + ], + ) + def test_a_refused_forwarding_start_serves_from_python_alone( + self, + mock_should_update, + mock_setup_db, + mock_atexit_register, + mock_uvicorn_run, + argv: list[str], + reason: str, + served_keyfile: str | None, + ) -> None: + from click.testing import CliRunner + + proxy_module: Final = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + environ: Final = { + **{ + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL", "_LITELLM_GATEWAY_SUPERVISED") + }, + "LITELLM_RUST_FIRST": "true", + } + with ( + patch.dict(os.environ, environ, clear=True), + patch.dict("sys.modules", {"proxy_server": proxy_module, "litellm.proxy.proxy_server": proxy_module}), + patch("litellm.proxy.rust_forward.supervisor.gateway_binary", return_value=None), + ): + result: Final = CliRunner().invoke(run_server, ["--port", "45999", *argv]) + assert result.exit_code == 0, result.output + assert f"LITELLM_RUST_FIRST {reason}" in result.output + assert "so Python serves the public port without Rust forwarding" in result.output + mock_uvicorn_run.assert_called_once() + assert mock_uvicorn_run.call_args.kwargs.get("ssl_keyfile") == served_keyfile + assert mock_uvicorn_run.call_args.kwargs["port"] == 45999 + + def test_an_invalid_rust_switch_stops_startup_with_one_line_naming_it(self) -> None: + from click.testing import CliRunner + + with patch.dict(os.environ, {"LITELLM_RUST_FIRST": "sometimes"}): + result: Final = CliRunner().invoke(run_server, ["--port", "45999"]) + lines: Final = result.output.splitlines() + assert (result.exit_code, len(lines)) == (1, 1) + assert lines[0].startswith("Error: LITELLM_RUST_FIRST is invalid: ") + + def test_a_metrics_port_on_the_public_port_is_refused_before_forwarding_starts(self) -> None: + from click.testing import CliRunner + + with ( + patch.dict(os.environ, {"LITELLM_RUST_FIRST": "true"}), + patch("litellm.proxy.rust_forward.supervisor.supervise") as supervise, + ): + result: Final = CliRunner().invoke(run_server, ["--port", "45999", "--prometheus_metrics_port", "45999"]) + assert result.exit_code == 2 + assert "--prometheus_metrics_port must differ from --port" in result.output + supervise.assert_not_called() + + @staticmethod + def _served_by_python(params: dict[str, object]) -> tuple[object, ...]: + if params["run_gunicorn"]: + gunicorn: Final = GunicornConfig() + return gunicorn.keepalive, GracefulShutdownManager.get_timeout(), ",".join(gunicorn.forwarded_allow_ips) + served: Final = uvicorn.Config( + **ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"] + ) + ) + return served.timeout_keep_alive, served.timeout_graceful_shutdown, served.forwarded_allow_ips + + @pytest.mark.parametrize( + ("argv", "environ"), + [ + ([], {}), + (["--keepalive_timeout", "2"], {"GRACEFUL_SHUTDOWN_TIMEOUT": "12.5"}), + ([], {"KEEPALIVE_TIMEOUT": "7", "FORWARDED_ALLOW_IPS": "10.0.0.5"}), + (["--run_gunicorn", "--keepalive_timeout", "9"], {}), + ], + ) + def test_rust_follows_the_keepalive_drain_and_trust_python_serves_with( + self, argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch + ) -> None: + for name in ("KEEPALIVE_TIMEOUT", "GRACEFUL_SHUTDOWN_TIMEOUT", "FORWARDED_ALLOW_IPS"): + monkeypatch.delenv(name, raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + params, server = self._server(argv) + assert ( + server.keepalive_seconds, + server.graceful_timeout_seconds, + server.trusted_proxies, + ) == self._served_by_python(params) diff --git a/tests/windows_tests/AGENTS.md b/tests/windows_tests/AGENTS.md new file mode 100644 index 00000000000..565dba91ce6 --- /dev/null +++ b/tests/windows_tests/AGENTS.md @@ -0,0 +1,7 @@ +# Windows tests + +The `windows-sdk` job in `.github/workflows/test-merge-smoke.yml` is the only place these run, on a Windows runner with no proxy dependencies installed. They prove the SDK works on Windows without them, so a test here must not import uvicorn, fastapi, psutil or anything else outside the `dev` group, and must not need the `litellm-gateway.exe` binary + +A test that cannot run off Windows carries a `skipif` on `sys.platform`, so collecting this folder on another OS never fails + +Windows-only names such as `CTRL_BREAK_EVENT` come from `litellm/proxy/rust_forward/platform/windows.py`, which spells them out so the type checker accepts them on every OS. `test_rust_forward_platform.py` pins those constants to the stdlib diff --git a/tests/windows_tests/test_rust_forward_platform.py b/tests/windows_tests/test_rust_forward_platform.py new file mode 100644 index 00000000000..1415a11f57c --- /dev/null +++ b/tests/windows_tests/test_rust_forward_platform.py @@ -0,0 +1,35 @@ +import signal +import subprocess +import sys +from pathlib import Path +from typing import Final + +import pytest + +from litellm.proxy.rust_forward.platform.windows import ( + CREATE_NEW_PROCESS_GROUP, + CTRL_BREAK_EVENT, + SIGBREAK, + WindowsPlatform, +) +from litellm.proxy.rust_forward.supervisor.preflight import gateway_binary + + +@pytest.mark.skipif(sys.platform != "win32", reason="these names exist only on Windows") +def test_windows_constants_match_the_stdlib() -> None: + assert (CTRL_BREAK_EVENT, SIGBREAK) == (signal.CTRL_BREAK_EVENT, signal.SIGBREAK) + assert CREATE_NEW_PROCESS_GROUP == subprocess.CREATE_NEW_PROCESS_GROUP + + +@pytest.mark.skipif(sys.platform != "win32", reason="only Windows searches the working directory for an executable") +def test_gateway_binary_in_the_working_directory_is_never_chosen( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + scripts: Final = tmp_path / "scripts" + scripts.mkdir() + (tmp_path / "litellm-gateway.exe").write_bytes(b"") + monkeypatch.chdir(tmp_path) + assert gateway_binary(WindowsPlatform(), str(scripts)) is None + installed: Final = scripts / "litellm-gateway.exe" + installed.write_bytes(b"") + assert gateway_binary(WindowsPlatform(), str(scripts)) == str(installed) From c9d3c8b52f0c5ee471715da5e8cd84175725d367 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:00:11 -0800 Subject: [PATCH 103/189] fix(streaming): report zero cost for explicitly free calls (#51093) * fix(streaming): report zero cost for explicitly free calls Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(streaming): require a stored entry for free model pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(streaming): reuse declared pricing lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(streaming): preserve selected free pricing entries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(streaming): recognize explicitly free pricing tiers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(streaming): cover invalid pricing tier fallbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kenneth Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 33 +++++--- litellm/litellm_core_utils/litellm_logging.py | 28 +++++++ .../llm_cost_calc/zero_cost_diagnostic.py | 30 +++++++ .../streaming_chunk_builder_utils.py | 6 +- .../litellm_core_utils/streaming_handler.py | 7 +- litellm/responses/streaming_iterator.py | 6 +- .../test_zero_cost_diagnostic.py | 80 +++++++++++++++++++ .../test_litellm_logging.py | 58 ++++++++++++++ .../test_streaming_chunk_builder_utils.py | 79 ++++++++++++++++++ .../test_streaming_handler.py | 79 ++++++++++++++++++ .../unit/responses/test_streaming_iterator.py | 55 +++++++++++++ 11 files changed, 448 insertions(+), 13 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 5b4a1d30ff0..233ff48e8a3 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -964,6 +964,19 @@ def _bedrock_registration_has_prices(entry: Mapping[str, object]) -> bool: ) +def _bedrock_cost_registration(model: str | None) -> tuple[str, Mapping[str, object]] | None: + if model is None: + return None + return next( + ( + (key, entry) + for key, entry in runtime_model_cost_registrations(model, "bedrock") + if _bedrock_registration_has_prices(entry) + ), + None, + ) + + def _bedrock_pricing_info( model: str | None, invoked_model_id: str | None, @@ -972,14 +985,7 @@ def _bedrock_pricing_info( ) -> ModelInfo | None: if explicit_pricing or custom_llm_provider != "bedrock" or not model: return None - registration: Final = next( - ( - (key, entry) - for key, entry in runtime_model_cost_registrations(model, "bedrock") - if _bedrock_registration_has_prices(entry) - ), - None, - ) + registration: Final = _bedrock_cost_registration(model) profile: Final = _cost_map_model_info(invoked_model_id, "bedrock") if invoked_model_id else None if profile is None or not _declares_rates(profile): return _cost_map_model_info(registration[0], None) if registration is not None else None @@ -2770,7 +2776,14 @@ def pricing_entry_for_cost_calc( custom_pricing is True or base_model is not None, ) if bedrock_model_info is not None: - return bedrock_model_info["key"], bedrock_model_info + key: Final = bedrock_model_info["key"] + declared_pricing: Final = declared_cost_map_pricing(key, model or key, custom_llm_provider) + registration: Final = _bedrock_cost_registration(model) + declared_keys: Final = frozenset(declared_pricing) | ( + frozenset(registration[1]) if registration is not None else frozenset[str]() + ) + selected_entry: Final[Mapping[str, object]] = bedrock_model_info + return key, MappingProxyType({name: value for name, value in selected_entry.items() if name in declared_keys}) candidates: Final = (selected_model, _get_response_model(completion_response), model) resolved: Final = next( (info for info in (_cost_map_model_info(name, custom_llm_provider) for name in candidates if name) if info), @@ -2778,7 +2791,7 @@ def pricing_entry_for_cost_calc( ) if resolved is None: return None - return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved + return resolved["key"], declared_cost_map_pricing(resolved["key"], model or resolved["key"], custom_llm_provider) def ocr_cost( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d88c8876ae0..8f3101915be 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -135,6 +135,7 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ) from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( diagnose_zero_cost, + is_explicitly_free_usage, zero_cost_warning, ) from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import LiteLLMResponseObjectHandler @@ -2417,6 +2418,33 @@ class Logging(LiteLLMLoggingBaseClass): usage=usage, ) + def is_usage_explicitly_free(self, result: object) -> bool: + usage: Final = get_usage_object(completion_response=result) + if usage is None or self.model_call_details.get("response_cost_failure_debug_information") is not None: + return False + custom_pricing: Final = self.custom_pricing_for(result) + if custom_pricing and self.custom_model_info is not None: + return is_explicitly_free_usage(usage, self.custom_model_info) + if custom_pricing and self.custom_cost_per_token is not None: + return is_explicitly_free_usage(usage, self.custom_cost_per_token) + custom_llm_provider: Final = self.model_call_details.get("custom_llm_provider") + pricing: Final = pricing_entry_for_cost_calc( + model=self.model, + completion_response=result, + custom_llm_provider=self.get_router_pricing_provider() or custom_llm_provider, + custom_pricing=custom_pricing, + base_model=get_base_model_from_metadata(model_call_details=self.model_call_details), + router_model_id=self.get_router_model_id(), + region_name=_resolve_mantle_region_for_cost( + custom_llm_provider=custom_llm_provider, + litellm_params=self.model_call_details.get("litellm_params"), + ), + litellm_logging_obj=self, + ) + if pricing is None: + return False + return is_explicitly_free_usage(usage, pricing[1]) + def spilled_over_ptu_for(self, result: object) -> bool: result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({}) additional_headers: Final = ( diff --git a/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py index 3ed598b5e97..78912d0a938 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py +++ b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py @@ -1,10 +1,12 @@ from collections.abc import Mapping from functools import reduce +from types import MappingProxyType from typing import Final from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never +from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.types.utils import ( CompletionTokensDetailsWrapper, @@ -23,6 +25,7 @@ _TEXT_OUTPUT_RATE: Final = "output_cost_per_token" _AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token" _RATE_KEY_MARKERS: Final = ("cost", "pricing") _NESTED_PRICING: Final = TypeAdapter(Mapping[str, object] | tuple[object, ...]) +_PRICING_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _MAX_PRICING_DEPTH: Final = 4 @@ -105,6 +108,33 @@ def _is_explicit_zero(value: object) -> bool: return not isinstance(value, bool) and isinstance(value, (int, float)) and value == 0 +def _pricing_for_usage(usage: Usage, pricing_entry: Mapping[str, object]) -> Mapping[str, object]: + tiered_pricing: Final = pricing_entry.get("tiered_pricing") + if not isinstance(tiered_pricing, list): + return pricing_entry + try: + tiers: Final = _PRICING_TIERS.validate_python(tiered_pricing) + except ValidationError: + return pricing_entry + tier: Final = select_tier_for_input(tiers, usage.prompt_tokens) + if tier is None or _TEXT_INPUT_RATE not in tier: + return pricing_entry + cache_rate: Final = tier.get(_CACHE_READ_INPUT_RATE) + return MappingProxyType( + { + **pricing_entry, + **tier, + _CACHE_READ_INPUT_RATE: cache_rate if cache_rate is not None else tier[_TEXT_INPUT_RATE], + } + ) + + +def is_explicitly_free_usage(usage: Usage, pricing_entry: Mapping[str, object]) -> bool: + selected_pricing: Final = _pricing_for_usage(usage, pricing_entry) + used_keys: Final = tuple(_effective_pricing_key(key, selected_pricing) for key in used_pricing_keys(usage)) + return bool(used_keys) and all(_is_explicit_zero(selected_pricing.get(key)) for key in used_keys) + + def diagnose_zero_cost( usage: Usage, pricing_model: str, diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index c87357122ce..e2b651f8885 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1500,7 +1500,11 @@ def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_o computed_cost: Final = logging_obj.response_cost_calculator(result=response) if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0: return - if computed_cost > 0 or isinstance(getattr(usage, "cost", None), (int, float)): + if ( + computed_cost > 0 + or isinstance(getattr(usage, "cost", None), (int, float)) + or logging_obj.is_usage_explicitly_free(response) + ): stamp_litellm_bill_on_usage(response, usage, float(computed_cost)) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 361e576f3e1..956e202a65a 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -114,7 +114,12 @@ def _stamp_usage_cost( usage: Final[object] = getattr(usage_chunk, "usage", None) if usage is None or isinstance(bill, bool) or not isinstance(bill, (int, float)): return - if bill <= 0 and not isinstance(getattr(usage, "cost", None), (int, float)): + complete_usage: Final = getattr(complete_response, "usage", None) + if ( + bill <= 0 + and not isinstance(getattr(usage, "cost", None), (int, float)) + and not (bill == 0 and getattr(complete_usage, "cost", None) == 0) + ): return chunk_hidden: Final = get_hidden_params(usage_chunk) if chunk_hidden is not None and UPSTREAM_USAGE_COST in complete_hidden: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 28a82b500c9..c2191375cb4 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1931,7 +1931,11 @@ def stamp_responses_usage_cost( return if isinstance(bill, bool) or not isinstance(bill, (int, float)): return - if bill > 0 or isinstance(usage_obj.cost, (int, float)): + if ( + bill > 0 + or isinstance(usage_obj.cost, (int, float)) + or (bill == 0 and logging_obj.is_usage_explicitly_free(priced)) + ): stamp_litellm_bill_on_usage(response_obj, usage_obj, float(bill)) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py index bdc14c4934b..d325a4677f7 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py @@ -6,6 +6,7 @@ import pytest from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( ZERO_COST_COUNTER_NAME, diagnose_zero_cost, + is_explicitly_free_usage, used_pricing_keys, zero_cost_warning, ) @@ -18,6 +19,85 @@ PRICED_ENTRY: Final = {"input_cost_per_token": 1e-06, "output_cost_per_token": 2 TEXT_USAGE: Final = Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) +@pytest.mark.parametrize( + ("usage", "rates", "expected"), + [ + (TEXT_USAGE, FREE_ENTRY, True), + (TEXT_USAGE, PRICED_ENTRY, False), + (TEXT_USAGE, {}, False), + (TEXT_USAGE, {"input_cost_per_token": 0.0}, False), + (TEXT_USAGE, {"input_cost_per_token": False, "output_cost_per_token": 0.0}, False), + (Usage(prompt_tokens=10, completion_tokens=0), {"input_cost_per_token": 0.0}, True), + (Usage(prompt_tokens=0, completion_tokens=0), FREE_ENTRY, False), + ( + Usage(prompt_tokens=10, completion_tokens=0, prompt_tokens_details={"cached_tokens": 10}), + FREE_ENTRY, + False, + ), + ( + Usage(prompt_tokens=10, completion_tokens=0, prompt_tokens_details={"cached_tokens": 10}), + {"input_cost_per_token": 0.0}, + True, + ), + ], +) +def test_free_usage_requires_explicit_zero_rates_for_every_used_token_kind( + usage: Usage, rates: Mapping[str, object], expected: bool +) -> None: + assert is_explicitly_free_usage(usage, rates) is expected + + +@pytest.mark.parametrize( + ("input_tokens", "tier_output_rate", "expected"), + [(10, 0.0, True), (11, 0.0, False), (10, 0.001, False), (10, None, False)], +) +def test_free_usage_uses_the_selected_tier_and_preserves_missing_output_prices( + input_tokens: int, tier_output_rate: float | None, expected: bool +) -> None: + rates: Final = { + "input_cost_per_token": 0.01, + "tiered_pricing": [ + {"range": [0, 10], "input_cost_per_token": 0.0, "output_cost_per_token": tier_output_rate}, + {"range": [10, 20], "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, + ], + } + + assert is_explicitly_free_usage(Usage(prompt_tokens=input_tokens, completion_tokens=5), rates) is expected + + +@pytest.mark.parametrize("cache_rate", (None, 0.01)) +def test_free_cached_usage_uses_the_selected_tier_cache_rate_or_its_input_rate(cache_rate: float | None) -> None: + rates: Final = { + "input_cost_per_token": 0.01, + "cache_read_input_token_cost": 0.02, + "tiered_pricing": [ + {"range": [0, 20], "input_cost_per_token": 0.0, "cache_read_input_token_cost": cache_rate} + ], + } + usage: Final = Usage(prompt_tokens=10, completion_tokens=0, prompt_tokens_details={"cached_tokens": 10}) + + assert is_explicitly_free_usage(usage, rates) is (cache_rate is None) + + +@pytest.mark.parametrize( + "tiers", + [ + ["invalid tier"], + [], + [{"range": [0, 20], "output_cost_per_token": 0.0}], + [{"range": "invalid range", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0}], + ], +) +def test_invalid_or_inactive_tiers_do_not_make_paid_usage_free(tiers: object) -> None: + rates: Final = { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "tiered_pricing": tiers, + } + + assert is_explicitly_free_usage(Usage(prompt_tokens=10, completion_tokens=5), rates) is False + + def test_missing_pricing_key_names_every_rate_the_usage_needs() -> None: diagnostic = diagnose_zero_cost( usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index e8f90dc0146..c1d878d5b09 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -79,6 +79,64 @@ from typing import List, Optional from litellm.types.utils import LiteLLMCommonStrings +@pytest.mark.parametrize("metadata_key", ("metadata", "litellm_metadata")) +def test_free_usage_uses_deployment_metadata_prices(metadata_key: str) -> None: + obj: Final = Logging( + model="metadata-free-deployment", + messages=[], + stream=True, + call_type="completion", + start_time=1.0, + litellm_call_id="metadata-free", + function_id="test", + ) + obj.litellm_params = {metadata_key: {"model_info": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}}} + response: Final = ModelResponse( + model="metadata-free-deployment", usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + ) + + assert obj.is_usage_explicitly_free(response) is True + + +@pytest.mark.parametrize("pricing_source", ("model_info", "token_rates")) +@pytest.mark.parametrize("rate", (0.0, 0.001)) +@pytest.mark.parametrize("calculation_failed", (False, True)) +def test_per_call_free_prices_override_catalog_unless_pricing_failed( + monkeypatch: pytest.MonkeyPatch, pricing_source: str, rate: float, calculation_failed: bool +) -> None: + model: Final = "per-call-free-price-test" + monkeypatch.setitem( + litellm.model_cost, + model, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, + ) + obj: Final = Logging( + model=model, + messages=[], + stream=True, + call_type="completion", + start_time=1.0, + litellm_call_id="per-call-free", + function_id="test", + ) + obj.litellm_params = {"input_cost_per_token": rate, "output_cost_per_token": rate} + if pricing_source == "model_info": + obj.set_per_call_pricing( + { + **litellm.get_model_info(model=model, custom_llm_provider="openai"), + "input_cost_per_token": rate, + "output_cost_per_token": rate, + } + ) + else: + obj.custom_cost_per_token = {"input_cost_per_token": rate, "output_cost_per_token": rate} + if calculation_failed: + obj.model_call_details["response_cost_failure_debug_information"] = {"error": "pricing failed"} + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + + assert obj.is_usage_explicitly_free(response) is (rate == 0.0 and not calculation_failed) + + @pytest.fixture def logging_obj(): return LitellmLogging( diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index 527c83d5760..858c0f90930 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -10,6 +10,7 @@ import respx import litellm from litellm import ChatCompletionUsageBlock, stream_chunk_builder +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( ChunkProcessor, stream_chunk_builder_text_completion, @@ -31,6 +32,84 @@ from litellm.types.utils import ( ) +@pytest.mark.parametrize("routed_provider", (False, True)) +@pytest.mark.parametrize( + ("profile_rates", "registered_rates", "expected_cost"), + [ + ( + {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, + {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, + 0.0, + ), + ( + {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, + {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002}, + 0.02, + ), + ({"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, {"input_cost_per_token": 0.0}, 0.1), + ({"input_cost_per_token": 0.0}, {"input_cost_per_token": 0.0}, None), + ], +) +def test_bedrock_stream_cost_uses_registered_rates_over_profile_prices( + monkeypatch: pytest.MonkeyPatch, + profile_rates: dict[str, float], + registered_rates: dict[str, float], + expected_cost: float | None, + routed_provider: bool, +) -> None: + model: Final = "openai.stream-pricing-test" + profile: Final = f"us.{model}" + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + profile: {"litellm_provider": "bedrock", "mode": "chat", **profile_rates}, + }, + ) + litellm.register_model({model: registered_rates}, persist_across_reloads=False) + logging_obj: Final = Logging( + model=model, + messages=[], + stream=True, + call_type="completion", + start_time=1.0, + litellm_call_id="bedrock-free-stream", + function_id="test", + ) + logging_obj.optional_params = {} + logging_obj.litellm_params = ( + {"metadata": {"model_info": {"pricing_provider": "bedrock"}}} if routed_provider else {} + ) + logging_obj.model_call_details.update( + custom_llm_provider="openai" if routed_provider else "bedrock", model_id=profile + ) + chunks: Final = [ + ModelResponseStream( + id="bedrock-free-stream", + created=1, + model=model, + choices=[StreamingChoices(index=0, delta=Delta(content="hi"), finish_reason="stop")], + ), + ModelResponseStream( + id="bedrock-free-stream", + created=1, + model=model, + choices=[], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ), + ] + + response: Final = stream_chunk_builder(chunks, logging_obj=logging_obj) + + assert response is not None + assert response.usage.prompt_tokens == 10 + assert response.usage.completion_tokens == 5 + assert getattr(response.usage, "cost", None) == ( + pytest.approx(expected_cost) if expected_cost is not None else None + ) + + def test_get_combined_tool_content(): chunks = [ ModelResponseStream( diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 3b798b35096..cb0ef9a9a33 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -7649,6 +7649,85 @@ def test_chunk_creator_preserves_hidden_provider_specific_fields_from_parsed_chu assert assembled._hidden_params["provider_specific_fields"] == {"traffic_type": "ON_DEMAND_FLEX"} +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize( + ("rates", "expected_cost"), + [ + ({"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, 0.0), + ({"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, 20e-6), + ({"input_cost_per_token": 0.0}, None), + ({}, None), + ( + { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "tiered_pricing": [ + {"range": [0, 128000], "input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + ], + }, + 0.0, + ), + ( + { + "tiered_pricing": [ + {"range": [0, 128000], "input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + ] + }, + 0.0, + ), + ], +) +async def test_stream_usage_distinguishes_free_from_unpriced_calls( + monkeypatch: pytest.MonkeyPatch, sync_mode: bool, rates: dict[str, object], expected_cost: float | None +) -> None: + model: Final = "stream-zero-cost-regression" + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **rates}) + logging_obj: Final = Logging( + model=model, + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion" if sync_mode else "acompletion", + start_time=1.0, + litellm_call_id="free-stream", + function_id="test", + ) + logging_obj.optional_params = {} + logging_obj.litellm_params = {} + logging_obj.stream_options = {"include_usage": True} + logging_obj.model_call_details["custom_llm_provider"] = "openai" + chunks: Final = [ + ModelResponseStream( + id="free-stream", + created=1, + model=model, + choices=[StreamingChoices(index=0, delta=Delta(content="hi"), finish_reason="stop")], + ), + ModelResponseStream( + id="free-stream", + created=1, + model=model, + choices=[], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ), + ] + stream: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=chunks), + model=model, + custom_llm_provider="openai", + logging_obj=logging_obj, + stream_options={"include_usage": True}, + ) + emitted: Final = tuple(stream) if sync_mode else tuple([chunk async for chunk in stream]) + usage_chunks: Final = tuple(chunk for chunk in emitted if getattr(chunk, "usage", None) is not None) + + assert len(usage_chunks) == 1 + assert usage_chunks[0].usage.prompt_tokens == 10 + assert usage_chunks[0].usage.completion_tokens == 5 + assert getattr(usage_chunks[0].usage, "cost", None) == ( + pytest.approx(expected_cost) if expected_cost is not None else None + ) + + def test_the_usage_chunk_carries_the_bill_the_builder_priced_for_the_spend_row() -> None: from litellm.litellm_core_utils.streaming_handler import ( _stamp_usage_cost, # pyright: ignore[reportPrivateUsage] # the helper under test diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 11f4e78619e..17a5112ce6f 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -861,6 +861,61 @@ def _responses_api_response_with_usage() -> ResponsesAPIResponse: ) +@pytest.mark.parametrize( + ("rates", "expected_cost"), + [ + ({"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, 0.0), + ({"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, 140e-6), + ({"input_cost_per_token": 0.0}, None), + ({}, None), + ( + { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "tiered_pricing": [ + {"range": [0, 128000], "input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + ], + }, + 0.0, + ), + ( + { + "tiered_pricing": [ + {"range": [0, 128000], "input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + ] + }, + 0.0, + ), + ], +) +def test_responses_stream_usage_distinguishes_free_from_unpriced_calls( + monkeypatch: pytest.MonkeyPatch, rates: dict[str, object], expected_cost: float | None +) -> None: + from litellm.responses.streaming_iterator import stamp_responses_usage_cost + + model: Final = "responses-zero-cost-regression" + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **rates}) + response: Final = _responses_api_response_with_usage().model_copy(update={"model": model}) + logging_obj: Final = LiteLLMLoggingObj( + model=model, + messages=[], + stream=True, + call_type="responses", + start_time=1.0, + litellm_call_id="free-response", + function_id="test", + ) + logging_obj.optional_params = {} + logging_obj.litellm_params = {} + logging_obj.model_call_details["custom_llm_provider"] = "openai" + + stamp_responses_usage_cost(response, logging_obj) + + assert getattr(response.usage, "cost", None) == ( + pytest.approx(expected_cost) if expected_cost is not None else None + ) + + def test_stamp_responses_usage_cost_stamps_computed_cost(): from litellm.responses.streaming_iterator import stamp_responses_usage_cost From a0f2562088f44239e5f578c3bcb452c531ac9548 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 11 Nov 2026 12:00:32 -0800 Subject: [PATCH 104/189] feat(gemini): file-backed batch jobs on /v1/batches with output download and spend tracking (#51032) * feat(gemini): file-backed batch jobs on /v1/batches with output download Adds GeminiBatchesConfig (create + retrieve) behind custom_llm_provider=gemini, registers it in ProviderConfigManager, and implements file content retrieval on GoogleAIStudioFilesHandler so GET /v1/files/{output_file_id}/content returns the batch output. Gemini runs the OpenAI-shaped JSONL the proxy uploads today unchanged. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(gemini): type the file content request params so no pyright rule grows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(gemini): price completed batch output rows so spend lands on the key Gemini writes the usage of an OpenAI-shaped output row as camelCase (promptTokens, completionTokens, totalTokens) with thinking tokens only in totalTokens. Map it onto Usage with the thinking tokens folded into completion_tokens, so the shared batch cost path bills them at the gemini _batches rates. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(gemini): parse nanosecond file timestamps Gemini stamps files with nine fractional digits, which strptime's %f rejects, so every retrieve of a batch output file logged a ValueError. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(gemini): parse file timestamps on Python 3.10 too fromisoformat on 3.10 takes neither a Z suffix nor more than six fractional digits. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(gemini): price native batch output rows, stream output downloads, reject unsupported batch metadata Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(gemini): detect native batch rows by key, anchor the streaming file download handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(httpx): make anchor_handler_to public so the file content stream can use it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/batches/batch_utils.py | 10 + litellm/files/main.py | 12 +- litellm/llms/custom_httpx/http_handler.py | 20 +- litellm/llms/custom_httpx/llm_http_handler.py | 2 + litellm/llms/gemini/batches/__init__.py | 0 litellm/llms/gemini/batches/transformation.py | 301 +++++++++++++++++ litellm/llms/gemini/files/transformation.py | 96 +++--- litellm/types/utils.py | 2 +- litellm/utils.py | 4 + tests/unit/batches/test_batch_utils.py | 123 ++++++- tests/unit/batches/test_main.py | 25 ++ tests/unit/llms/gemini/batches/__init__.py | 0 .../test_gemini_batches_transformation.py | 310 ++++++++++++++++++ .../files/test_gemini_files_transformation.py | 61 +++- 14 files changed, 878 insertions(+), 88 deletions(-) create mode 100644 litellm/llms/gemini/batches/__init__.py create mode 100644 litellm/llms/gemini/batches/transformation.py create mode 100644 tests/unit/llms/gemini/batches/__init__.py create mode 100644 tests/unit/llms/gemini/batches/test_gemini_batches_transformation.py diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index cdae3de47c8..62ccb07f695 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -49,6 +49,10 @@ def _uses_native_vertex_output( model_name: str | None, first_row: Mapping[str, object] | None, ) -> bool: + if custom_llm_provider == "gemini": + from litellm.llms.gemini.batches.transformation import is_native_gemini_batch_output_row + + return first_row is not None and is_native_gemini_batch_output_row(first_row) if custom_llm_provider != "vertex_ai": return False if model_name and litellm.disable_vertex_batch_output_transformation: @@ -831,6 +835,12 @@ def _get_batch_job_usage_from_response_body( _usage_dict: Final = response_body.get("usage", None) or {} if ResponseAPILoggingUtils.is_response_api_usage(_usage_dict): return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_usage_dict) + if custom_llm_provider == "gemini": + from litellm.llms.gemini.batches.transformation import is_gemini_batch_usage_shape, usage_from_batch_output + + gemini_usage: Final = cast(Mapping[str, object], _usage_dict) # cast-ok: usage block comes from provider JSON + if is_gemini_batch_usage_shape(gemini_usage): + return usage_from_batch_output(gemini_usage) usage: Final[Usage] = Usage(**_usage_dict) if custom_llm_provider == "xai": from litellm.llms.xai.chat.transformation import XAIChatConfig diff --git a/litellm/files/main.py b/litellm/files/main.py index d1ddf6c4c4c..3c73f6d5389 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -1091,10 +1091,10 @@ def file_content_streaming( client=client if isinstance(client, (OpenAI, AsyncOpenAI)) else None, litellm_params=optional_params.model_dump(exclude_none=True), ) - elif custom_llm_provider == LlmProviders.VERTEX_AI.value: + elif custom_llm_provider in (LlmProviders.VERTEX_AI.value, LlmProviders.GEMINI.value): if not _is_async: raise litellm.exceptions.BadRequestError( - message="Streaming 'file_content' for vertex_ai is only supported through 'afile_content'.", + message=f"Streaming 'file_content' for {custom_llm_provider} is only supported through 'afile_content'.", model="n/a", llm_provider=custom_llm_provider, response=httpx.Response( @@ -1103,14 +1103,14 @@ def file_content_streaming( request=httpx.Request(method="file_content", url="https://github.com/BerriAI/litellm"), ), ) - vertex_files_config: Final = ProviderConfigManager.get_provider_files_config( + google_files_config: Final = ProviderConfigManager.get_provider_files_config( model="", - provider=LlmProviders.VERTEX_AI, + provider=LlmProviders(custom_llm_provider), ) - assert vertex_files_config is not None + assert google_files_config is not None response = base_llm_http_handler.async_retrieve_file_content_streaming( file_content_request=file_content_request, - provider_config=vertex_files_config, + provider_config=google_files_config, litellm_params=litellm_params, headers=extra_headers or {}, logging_obj=logging_obj, diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 011551faf0c..a5b15a24a7f 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -232,13 +232,13 @@ def _handler_may_close_client(client_refcount: int, owns_client: bool) -> bool: def _drop_streaming_anchor(_handler: object) -> None: - """Release a handler anchored to a streaming response. See ``_anchor_handler_to``. + """Release a handler anchored to a streaming response. See ``anchor_handler_to``. The work is the reference held until this point, so there is nothing to do here. """ -def _anchor_handler_to(response: httpx.Response, handler: object) -> None: +def anchor_handler_to(response: httpx.Response, handler: object) -> None: """Keep the handler alive for as long as a streaming response can still read. A body still arriving reads through the handler's connection pool, and closing @@ -927,7 +927,7 @@ class AsyncHTTPHandler: stream=True, follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects, ) - _anchor_handler_to(response, self) + anchor_handler_to(response, self) response.raise_for_status() except httpx.TimeoutException as e: _raise_timeout(e, start_time) @@ -1024,7 +1024,7 @@ class AsyncHTTPHandler: ) response: Final = await self.client.send(req, stream=stream) if stream: - _anchor_handler_to(response, self) + anchor_handler_to(response, self) response.raise_for_status() return response except httpx.ConnectError: @@ -1201,7 +1201,7 @@ class AsyncHTTPHandler: ) response: Final = await self.client.send(req, stream=stream) if stream: - _anchor_handler_to(response, self) + anchor_handler_to(response, self) response.raise_for_status() return response except (httpx.RemoteProtocolError, httpx.ConnectError): @@ -1706,7 +1706,7 @@ class HTTPHandler: stream=True, follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects, ) - _anchor_handler_to(response, self) + anchor_handler_to(response, self) response.raise_for_status() except httpx.TimeoutException as e: _raise_timeout(e, start_time) @@ -1771,7 +1771,7 @@ class HTTPHandler: ) response: Final = self.client.send(req, stream=stream) if stream: - _anchor_handler_to(response, self) + anchor_handler_to(response, self) response.raise_for_status() return response except httpx.TimeoutException as e: @@ -1820,7 +1820,7 @@ class HTTPHandler: ) response: Final = self.client.send(req, stream=stream) if stream: - _anchor_handler_to(response, self) + anchor_handler_to(response, self) response.raise_for_status() return response except httpx.TimeoutException as e: @@ -1869,7 +1869,7 @@ class HTTPHandler: ) response: Final = self.client.send(req, stream=stream) if stream: - _anchor_handler_to(response, self) + anchor_handler_to(response, self) return response except httpx.TimeoutException as e: _raise_timeout(e, start_time) @@ -1917,7 +1917,7 @@ class HTTPHandler: ) response: Final = self.client.send(req, stream=stream) if stream: - _anchor_handler_to(response, self) + anchor_handler_to(response, self) response.raise_for_status() return response except httpx.TimeoutException as e: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 5b6a64d2377..efb8b302664 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -133,6 +133,7 @@ from litellm.llms.custom_httpx.container_handler import raise_for_error_status from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, + anchor_handler_to, get_async_httpx_client, get_httpx_client, header_value, @@ -5890,6 +5891,7 @@ class BaseLLMHTTPHandler: except Exception as e: # noqa: BLE001 # _handle_error maps every failure kind, like the buffered fetch raise self._handle_error(e=e, provider_config=provider_config) + anchor_handler_to(response, async_httpx_client) if response.status_code >= 400: error_body: Final = await response.aread() await response.aclose() diff --git a/litellm/llms/gemini/batches/__init__.py b/litellm/llms/gemini/batches/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/gemini/batches/transformation.py b/litellm/llms/gemini/batches/transformation.py new file mode 100644 index 00000000000..10baec7d37d --- /dev/null +++ b/litellm/llms/gemini/batches/transformation.py @@ -0,0 +1,301 @@ +""" +Gemini Developer API batch jobs. Reference: https://ai.google.dev/gemini-api/docs/batch-api + +A file-backed job runs the uploaded JSONL unchanged. Gemini accepts OpenAI-shaped rows +(``{custom_id, method, url, body}``) and then writes the output file OpenAI-shaped too +(``{custom_id, response: {status_code, body}}``), so ``/v1/files purpose=batch`` uploads and the +shared batch cost accounting need no provider branch. The model is set on the job URL. +Ids cross the LiteLLM boundary bare: Gemini's ``batches/`` is the batch id ```` and its +``files/`` output file is the file id ````. +""" + +from collections.abc import Mapping, Sequence +from datetime import datetime +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +import httpx +from openai.types.batch import BatchRequestCounts +from openai.types.batch import Errors as BatchErrors +from openai.types.batch_error import BatchError +from pydantic import ConfigDict, Field +from typing_extensions import ReadOnly, TypedDict + +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base +from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo +from litellm.llms.gemini.files.transformation import gemini_file_name +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest +from litellm.types.utils import LiteLLMBatch, LlmProviders, Usage + +GeminiBatchState: TypeAlias = Literal[ + "BATCH_STATE_UNSPECIFIED", + "BATCH_STATE_PENDING", + "BATCH_STATE_RUNNING", + "BATCH_STATE_SUCCEEDED", + "BATCH_STATE_FAILED", + "BATCH_STATE_CANCELLED", + "BATCH_STATE_EXPIRED", +] +OpenAIBatchStatus: TypeAlias = Literal[ + "validating", "failed", "in_progress", "finalizing", "completed", "expired", "cancelling", "cancelled" +] + +GEMINI_BATCH_RESOURCE_PREFIX: Final = "batches/" +_GEMINI_FILE_RESOURCE_PREFIX: Final = "files/" +_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) +_STATUS_MAP: Final[MappingProxyType[GeminiBatchState, OpenAIBatchStatus]] = MappingProxyType( + { + "BATCH_STATE_UNSPECIFIED": "validating", + "BATCH_STATE_PENDING": "validating", + "BATCH_STATE_RUNNING": "in_progress", + "BATCH_STATE_SUCCEEDED": "completed", + "BATCH_STATE_FAILED": "failed", + "BATCH_STATE_CANCELLED": "cancelled", + "BATCH_STATE_EXPIRED": "expired", + } +) + + +class GeminiBatchInputConfig(TypedDict): + file_name: ReadOnly[str] + + +class GeminiBatchSpec(TypedDict): + display_name: ReadOnly[str] + input_config: ReadOnly[GeminiBatchInputConfig] + + +class GeminiCreateBatchRequest(TypedDict): + batch: ReadOnly[GeminiBatchSpec] + + +class GeminiPresignedRequest(TypedDict): + method: ReadOnly[Literal["GET"]] + url: ReadOnly[str] + headers: ReadOnly[Mapping[str, str]] + + +class _GeminiBatchStats(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + request_count: int = Field(default=0, alias="requestCount") + successful_request_count: int = Field(default=0, alias="successfulRequestCount") + failed_request_count: int = Field(default=0, alias="failedRequestCount") + + +class _GeminiBatchInputConfig(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + file_name: str | None = Field(default=None, alias="fileName") + + +class _GeminiBatchOutput(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + responses_file: str | None = Field(default=None, alias="responsesFile") + + +class _GeminiBatchMetadata(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + display_name: str | None = Field(default=None, alias="displayName") + input_config: _GeminiBatchInputConfig = Field(default_factory=_GeminiBatchInputConfig, alias="inputConfig") + output: _GeminiBatchOutput = Field(default_factory=_GeminiBatchOutput) + create_time: datetime = Field(alias="createTime") + end_time: datetime | None = Field(default=None, alias="endTime") + batch_stats: _GeminiBatchStats = Field(default_factory=_GeminiBatchStats, alias="batchStats") + state: GeminiBatchState = "BATCH_STATE_UNSPECIFIED" + + +class _GeminiOperationError(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + code: int | None = None + message: str = "" + + +class GeminiBatchOperation(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + name: str + metadata: _GeminiBatchMetadata + error: _GeminiOperationError | None = None + + +class _GeminiBatchOutputUsage(LiteLLMBaseModel): + """The ``usage`` block Gemini writes on an OpenAI-shaped batch output row. ``totalTokens`` + includes thinking tokens that ``completionTokens`` leaves out, and Gemini bills them as output.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + prompt_tokens: int = Field(default=0, alias="promptTokens") + completion_tokens: int = Field(default=0, alias="completionTokens") + total_tokens: int = Field(default=0, alias="totalTokens") + + +def is_native_gemini_batch_output_row(row: Mapping[str, object]) -> bool: + """Native rows carry Gemini's ``key`` (and a ``response`` or an error ``status``) where OpenAI-shaped rows + carry ``custom_id``, so the shape is known even when the first row failed.""" + return "key" in row and "custom_id" not in row + + +def is_gemini_batch_usage_shape(usage_object: Mapping[str, object]) -> bool: + return any(key in usage_object for key in ("promptTokens", "completionTokens", "totalTokens")) + + +def usage_from_batch_output(usage_object: Mapping[str, object]) -> Usage: + reported: Final = _GeminiBatchOutputUsage.model_validate(usage_object) + reasoning_tokens: Final = max(reported.total_tokens - reported.prompt_tokens - reported.completion_tokens, 0) + return Usage( + prompt_tokens=reported.prompt_tokens, + completion_tokens=reported.completion_tokens + reasoning_tokens, + total_tokens=reported.total_tokens, + reasoning_tokens=reasoning_tokens or None, + ) + + +def _bare_id(resource_name: str | None, prefix: str) -> str | None: + return None if resource_name is None else resource_name.removeprefix(prefix) + + +def _to_batch_errors(error: _GeminiOperationError | None) -> BatchErrors | None: + if error is None: + return None + return BatchErrors( + object="list", data=[BatchError(code=str(error.code) if error.code else None, message=error.message)] + ) + + +def _to_litellm_batch(operation: GeminiBatchOperation) -> LiteLLMBatch: + meta: Final = operation.metadata + status: Final = _STATUS_MAP[meta.state] + terminal_at: Final = int(meta.end_time.timestamp()) if meta.end_time is not None else None + return LiteLLMBatch( + id=_bare_id(operation.name, GEMINI_BATCH_RESOURCE_PREFIX) or operation.name, + object="batch", + endpoint="/v1/chat/completions", + input_file_id=_bare_id(meta.input_config.file_name, _GEMINI_FILE_RESOURCE_PREFIX) or "", + completion_window="24h", + status=status, + created_at=int(meta.create_time.timestamp()), + completed_at=terminal_at if status == "completed" else None, + failed_at=terminal_at if status == "failed" else None, + expired_at=terminal_at if status == "expired" else None, + cancelled_at=terminal_at if status == "cancelled" else None, + output_file_id=_bare_id(meta.output.responses_file, _GEMINI_FILE_RESOURCE_PREFIX), + errors=_to_batch_errors(operation.error), + request_counts=BatchRequestCounts( + total=meta.batch_stats.request_count, + completed=meta.batch_stats.successful_request_count, + failed=meta.batch_stats.failed_request_count, + ), + metadata={"display_name": meta.display_name} if meta.display_name else None, + ) + + +def _auth_headers(headers: Mapping[str, str], api_key: str | None) -> dict[str, str]: # mutable-ok: header contract + resolved_api_key: Final = GeminiModelInfo.get_api_key(api_key) + if not resolved_api_key: + raise ValueError("GEMINI_API_KEY is required for Gemini batch operations") + return {**headers, "x-goog-api-key": resolved_api_key} + + +class GeminiBatchesConfig(BaseBatchesConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.GEMINI + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: BaseBatchesConfig signature + return _auth_headers(headers, api_key) + + def get_complete_batch_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + data: CreateBatchRequest, + ) -> str: + model_segment: Final = encode_url_path_segment(model.removeprefix("models/"), field_name="model") + return join_api_base( + GeminiModelInfo.get_api_base(api_base), f"v1beta/models/{model_segment}:batchGenerateContent" + ) + + def transform_create_batch_request( + self, + model: str, + create_batch_data: CreateBatchRequest, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> dict[str, object]: # mutable-ok: BaseBatchesConfig signature + input_file_id: Final = create_batch_data.get("input_file_id") + if input_file_id is None: + raise ValueError("input_file_id is required to create a Gemini batch job") + file_name: Final = gemini_file_name(input_file_id) + metadata: Final = create_batch_data.get("metadata") or {} + unsupported_metadata: Final = sorted(set(metadata) - {"display_name"}) + if unsupported_metadata: + raise ValueError(f"Gemini batch jobs carry no metadata besides display_name, got {unsupported_metadata}") + display_name: Final = ( + metadata.get("display_name") or f"litellm-{file_name.removeprefix(_GEMINI_FILE_RESOURCE_PREFIX)}" + ) + body: Final = GeminiCreateBatchRequest( + batch=GeminiBatchSpec(display_name=display_name, input_config=GeminiBatchInputConfig(file_name=file_name)) + ) + return dict(body) + + def transform_create_batch_response( + self, + model: str | None, + raw_response: httpx.Response, + logging_obj: object, + litellm_params: Mapping[str, object], + ) -> LiteLLMBatch: + return _to_litellm_batch(GeminiBatchOperation.model_validate(raw_response.json())) + + def transform_retrieve_batch_request( + self, + batch_id: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> dict[str, object]: # mutable-ok: BaseBatchesConfig signature + bare_batch_id: Final = batch_id.removeprefix(GEMINI_BATCH_RESOURCE_PREFIX) + encoded_batch_id: Final = encode_url_path_segment(bare_batch_id, field_name="batch_id") + api_base: Final = litellm_params.get("api_base") + api_key: Final = litellm_params.get("api_key") + request: Final = GeminiPresignedRequest( + method="GET", + url=join_api_base( + GeminiModelInfo.get_api_base(api_base if isinstance(api_base, str) else None), + f"v1beta/{GEMINI_BATCH_RESOURCE_PREFIX}{encoded_batch_id}", + ), + headers=_auth_headers(_NO_HEADERS, api_key if isinstance(api_key, str) else None), + ) + return dict(request) + + def transform_retrieve_batch_response( + self, + model: str | None, + raw_response: httpx.Response, + logging_obj: object, + litellm_params: Mapping[str, object], + ) -> LiteLLMBatch: + return _to_litellm_batch(GeminiBatchOperation.model_validate(raw_response.json())) + + def get_error_class( + self, error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers + ) -> BaseLLMException: + return GeminiError(status_code=status_code, message=error_message, headers=dict(headers)) diff --git a/litellm/llms/gemini/files/transformation.py b/litellm/llms/gemini/files/transformation.py index 17460c0295a..4fb5a77653d 100644 --- a/litellm/llms/gemini/files/transformation.py +++ b/litellm/llms/gemini/files/transformation.py @@ -4,8 +4,10 @@ Supports writing files to Google AI Studio Files API. For vertex ai, check out the vertex_ai/files/handler.py file. """ +import re import time from collections.abc import Mapping +from datetime import datetime from typing import Final, Literal, TypedDict from urllib.parse import urlparse @@ -25,6 +27,7 @@ from litellm.llms.base_llm.files.transformation import ( from litellm.types.llms.openai import ( AllMessageValues, CreateFileRequest, + FileContentRequest, HttpxBinaryResponseContent, OpenAICreateFileRequestOptionalParams, OpenAIFileObject, @@ -53,6 +56,34 @@ class _GeminiCreateFileResponse(TypedDict): file: ReadOnly[_GeminiFileMetadata] +_RFC3339_PARTS: Final = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(\.\d+)?([+-]\d{2}:\d{2})") + + +def _gemini_timestamp(rfc3339: str) -> int: + """Gemini stamps files with up to nine fractional digits and a Z suffix; Python 3.10's + fromisoformat takes neither, so trim the fraction to microseconds and spell out UTC.""" + parts: Final = _RFC3339_PARTS.fullmatch(rfc3339.replace("Z", "+00:00")) + if parts is None: + raise ValueError(f"Unrecognized Gemini timestamp: {rfc3339}") + seconds, fraction, offset = parts.groups() + return int(datetime.fromisoformat(f"{seconds}{(fraction or '')[:7]}{offset}").timestamp()) + + +def gemini_file_name(file_id: str) -> str: + """ + Normalize a file identifier into Gemini's `files/{id}` resource name. + + Supports: + - `abc123` + - `files/abc123` + - `https://generativelanguage.googleapis.com/v1beta/files/abc123` + """ + path: Final = urlparse(file_id).path.lstrip("/") if file_id.startswith(("http://", "https://")) else file_id + files_index: Final = path.find("files/") + bare_id: Final = (path[files_index:] if files_index != -1 else path).strip("/").removeprefix("files/") + return f"files/{encode_url_path_segment(bare_id, field_name='file_id')}" + + class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): def __init__(self): pass @@ -194,14 +225,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): return OpenAIFileObject( id=response_object["uri"], # Gemini uses URI as identifier bytes=int(response_object["sizeBytes"]), # Gemini doesn't return file size - created_at=int( - time.mktime( - time.strptime( - response_object["createTime"].replace("Z", "+00:00"), - "%Y-%m-%dT%H:%M:%S.%f%z", - ) - ) - ), + created_at=_gemini_timestamp(response_object["createTime"]), filename=response_object["displayName"], object="file", purpose="user_data", # Default to assistants as that's the main use case @@ -246,32 +270,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): return url, {} def _normalize_gemini_file_id(self, file_id: str) -> str: - """ - Normalize file identifier into `files/{id}` form. - - Supports: - - `abc123` - - `files/abc123` - - `https://generativelanguage.googleapis.com/v1beta/files/abc123` - """ - if file_id.startswith(("http://", "https://")): - parsed: Final = urlparse(file_id) - path: Final = parsed.path.lstrip("/") - files_index: Final = path.find("files/") - if files_index != -1: - normalized_file_id = path[files_index:] - else: - normalized_file_id = path - else: - normalized_file_id = file_id - - normalized_file_id = normalized_file_id.strip("/") - if normalized_file_id.startswith("files/"): - normalized_file_id = normalized_file_id.removeprefix("files/") - - encoded_file_id: Final = encode_url_path_segment(normalized_file_id, field_name="file_id") - - return f"files/{encoded_file_id}" + return gemini_file_name(file_id) def transform_retrieve_file_response( self, @@ -299,14 +298,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): return OpenAIFileObject( id=response_json.get("uri", ""), bytes=int(response_json.get("sizeBytes", 0)), - created_at=int( - time.mktime( - time.strptime( - response_json["createTime"].replace("Z", "+00:00"), - "%Y-%m-%dT%H:%M:%S.%f%z", - ) - ) - ), + created_at=_gemini_timestamp(response_json["createTime"]), filename=response_json.get("displayName", ""), object="file", purpose="user_data", @@ -403,11 +395,19 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): def transform_file_content_request( self, - file_content_request, - optional_params: dict, - litellm_params: dict, - ) -> tuple[str, dict]: - raise NotImplementedError("GoogleAIStudioFilesHandler does not support file content retrieval") + file_content_request: FileContentRequest, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature + file_id: Final = file_content_request.get("file_id") + if file_id is None: + raise ValueError("file_id is required to download file content") + api_base: Final = litellm_params.get("api_base") + url: Final = join_api_base( + self.get_api_base(api_base if isinstance(api_base, str) else None), + f"download/v1beta/{gemini_file_name(file_id)}:download", + ) + return url, {"alt": "media"} def transform_file_content_response( self, @@ -415,4 +415,4 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): logging_obj: LiteLLMLoggingObj, litellm_params: dict, ) -> HttpxBinaryResponseContent: - raise NotImplementedError("GoogleAIStudioFilesHandler does not support file content retrieval") + return HttpxBinaryResponseContent(response=raw_response) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index addd642eca7..81d39f848fb 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4833,7 +4833,7 @@ OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: set[str] = { } FILE_CONTENT_STREAMING_PROVIDERS: Final[frozenset[str]] = frozenset( - {*OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS, LlmProviders.VERTEX_AI.value} + {*OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS, LlmProviders.VERTEX_AI.value, LlmProviders.GEMINI.value} ) LITELLM_EXECUTED_BATCH_PROVIDERS: Final[frozenset[str]] = frozenset({LlmProviders.HOSTED_VLLM.value}) diff --git a/litellm/utils.py b/litellm/utils.py index 156247897c3..7c73d1cc4d5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10866,6 +10866,10 @@ class ProviderConfigManager: from litellm.llms.mistral.batches.transformation import MistralBatchesConfig return MistralBatchesConfig() + elif LlmProviders.GEMINI == provider: + from litellm.llms.gemini.batches.transformation import GeminiBatchesConfig + + return GeminiBatchesConfig() return None @staticmethod diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index bef78554a36..2c8114f8736 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -14,27 +14,28 @@ maps (litellm.completion_cost, batch_cost_calculator), the tokenizer deterministic stand-ins so the arithmetic under test is the only variable. """ -import asyncio, json, time +import asyncio +import json import logging +import time from typing import Final, Literal +from unittest.mock import AsyncMock, patch import httpx import pytest import respx - import litellm import litellm.batches.batch_utils as bu -from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage -from litellm.batches.batch_utils import( +from litellm.batches.batch_utils import ( _aggregate_batch_cost_usage_models, - get_file_content_as_dictionary, _get_response_from_batch_job_output_file, calculate_batch_cost_and_usage, + get_file_content_as_dictionary, ) from litellm.cost_calculator import batch_cost_calculator +from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from unittest.mock import AsyncMock, patch # --------------------------------------------------------------------------- # # Builders for batch OUTPUT file rows. @@ -1188,7 +1189,9 @@ async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monk @pytest.mark.asyncio -async def test_handle_completed_vertex_batch_regional_location_costs_uplift_over_global(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_handle_completed_vertex_batch_regional_location_costs_uplift_over_global( + monkeypatch: pytest.MonkeyPatch, +) -> None: import litellm.files.main as files_main rows = [_vertex_openai_row("request-1", "gemini-3.6-flash", 10, 5)] @@ -2188,8 +2191,7 @@ GROUNDED_USAGE_METADATA = { PASSTHROUGH_OUTPUT_URI = ( - "gs://litellm-bucket/litellm-vertex-files/passthrough/publishers/google/models/gemini-2.5-flash/u/" - "predictions.jsonl" + "gs://litellm-bucket/litellm-vertex-files/passthrough/publishers/google/models/gemini-2.5-flash/u/predictions.jsonl" ) @@ -2555,6 +2557,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -2564,6 +2567,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): original_state = _copy_litellm_state() @@ -2580,6 +2584,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + def _copy_litellm_state(): state = {} for attr in _CALLBACK_ATTRS: @@ -2591,6 +2596,7 @@ def _copy_litellm_state(): state[attr] = getattr(litellm, attr) return state + _CALLBACK_ATTRS = ( "callbacks", "success_callback", @@ -2613,6 +2619,7 @@ _SCALAR_ATTRS = ( "cohere_key", ) + def _clear_logging_queue(loop=None) -> None: from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -2621,6 +2628,7 @@ def _clear_logging_queue(loop=None) -> None: return asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) + def _reset_litellm_callbacks() -> None: for attr in _CALLBACK_ATTRS: if hasattr(litellm, attr): @@ -2630,11 +2638,13 @@ def _reset_litellm_callbacks() -> None: if callable(reset): reset() + def _restore_litellm_state(state) -> None: for attr, value in state.items(): if hasattr(litellm, attr): setattr(litellm, attr, value) + def _make_batch_output_line(prompt_tokens: int = 10, completion_tokens: int = 5): """Return a single successful batch output line (OpenAI JSONL format).""" return { @@ -2663,11 +2673,13 @@ def _make_batch_output_line(prompt_tokens: int = 10, completion_tokens: int = 5) "error": None, } + CUSTOM_MODEL_INFO = { "input_cost_per_token_batches": 0.00125, "output_cost_per_token_batches": 0.005, } + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_batch_cost_calculator_explicit_zero_pricing_not_overridden_by_global( monkeypatch, @@ -2699,6 +2711,7 @@ def test_batch_cost_calculator_explicit_zero_pricing_not_overridden_by_global( assert prompt_cost == 0.0 assert completion_cost == 0.0 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_batch_cost_calculator_uses_custom_model_info(): """batch_cost_calculator should use model_info override when provided.""" @@ -2718,6 +2731,7 @@ def test_batch_cost_calculator_uses_custom_model_info(): f"Expected completion cost {expected_completion}, got {completion_cost}" ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_aggregate_batch_cost_uses_custom_model_info(): """_aggregate_batch_cost_usage_models should thread model_info to batch_cost_calculator.""" @@ -2732,6 +2746,7 @@ def test_aggregate_batch_cost_uses_custom_model_info(): expected = (10 * 0.00125) + (5 * 0.005) assert result.cost == pytest.approx(expected), f"Expected total cost {expected}, got {result.cost}" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("data_residency", ["eu", "us", "fedramp"]) def test_batch_cost_calculator_applies_data_residency_uplift(data_residency, monkeypatch): @@ -2779,6 +2794,7 @@ async def test_calculate_batch_cost_and_usage_uses_custom_model_info(): assert result.usage.prompt_tokens == 10 assert result.usage.completion_tokens == 5 + @pytest.fixture def sample_file_content(): return b""" @@ -2786,6 +2802,7 @@ def sample_file_content(): {"id": "batch_req_6769ca597e588190920666612634e2b4", "custom_id": "request-2", "response": {"status_code": 200, "request_id": "82e04f4c001fe2c127cbad199f5fd31b", "body": {"id": "chatcmpl-AhjSNgVB4Oa4Hq0NruTRsBaEbRWUP", "object": "chat.completion", "created": 1734986203, "model": "gpt-4o-mini-2024-07-18", "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello! What can I do for you today?", "refusal": null}, "logprobs": null, "finish_reason": "length"}], "usage": {"prompt_tokens": 22, "completion_tokens": 10, "total_tokens": 32, "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, "system_fingerprint": "fp_0aa8d3e20b"}}, "error": null} """ + @pytest.fixture def sample_file_content_dict(): return [ @@ -2877,6 +2894,7 @@ def sample_file_content_dict(): }, ] + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_get_file_content_as_dictionary(sample_file_content): result = get_file_content_as_dictionary(sample_file_content) @@ -2886,6 +2904,7 @@ def test_get_file_content_as_dictionary(sample_file_content): assert result[0]["response"]["status_code"] == 200 assert result[0]["response"]["body"]["usage"]["total_tokens"] == 30 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_get_batch_job_total_usage_from_file_content(sample_file_content_dict): with patch("litellm.completion_cost", return_value=0.0): @@ -2894,6 +2913,7 @@ def test_get_batch_job_total_usage_from_file_content(sample_file_content_dict): assert result.usage.prompt_tokens == 42 # 20 + 22 assert result.usage.completion_tokens == 20 # 10 + 10 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_batch_cost_calculator(sample_file_content_dict): @@ -2913,6 +2933,7 @@ async def test_batch_cost_calculator(sample_file_content_dict): assert result.prompt_cost == pytest.approx(0.6) assert result.completion_cost == pytest.approx(0.4) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_get_response_from_batch_job_output_file(sample_file_content_dict): result = _get_response_from_batch_job_output_file(sample_file_content_dict[0]) @@ -2920,6 +2941,7 @@ def test_get_response_from_batch_job_output_file(sample_file_content_dict): assert result["object"] == "chat.completion" assert result["usage"]["total_tokens"] == 30 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_batch_retrieve_cost_tracking_with_completed_batch_no_explicit_cost(): @@ -3016,6 +3038,7 @@ async def test_batch_retrieve_cost_tracking_with_completed_batch_no_explicit_cos assert mock_batch._hidden_params["batch_failed_requests"] == 0 assert mock_batch.usage == expected_usage + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_handle_completed_batch_computes_real_cost_from_output_file( @@ -3059,6 +3082,7 @@ async def test_handle_completed_batch_computes_real_cost_from_output_file( assert result.successful_requests == 2 assert result.failed_requests == 0 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_batch_retrieve_cost_tracking_with_explicit_cost_data(): @@ -3146,6 +3170,7 @@ async def test_batch_retrieve_cost_tracking_with_explicit_cost_data(): assert mock_batch._hidden_params["batch_models"] == explicit_models assert mock_batch.usage == explicit_usage + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_batch_retrieve_explicit_cost_split_sets_cost_breakdown(): @@ -3196,6 +3221,7 @@ async def test_batch_retrieve_explicit_cost_split_sets_cost_breakdown(): assert logging_obj.cost_breakdown["output_cost"] == 0.04 assert logging_obj.cost_breakdown["total_cost"] == 0.10 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_batch_retrieve_cost_tracking_with_unified_file_id_incomplete_batch(): @@ -3276,6 +3302,7 @@ async def test_batch_retrieve_cost_tracking_with_unified_file_id_incomplete_batc assert "batch_models" not in mock_batch._hidden_params assert not hasattr(mock_batch, "usage") or mock_batch.usage is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_batch_retrieve_cost_tracking_with_partial_explicit_data(): @@ -3445,7 +3472,10 @@ def test_batch_ocr_preserves_additive_page_quantities_without_changing_cost(): assert captured["pages_processed"] == 5 assert captured["doc_size_bytes"] == 300 assert result.cost == pytest.approx(5 * 0.01) - assert sorted(row["usage"]["pages_processed"] for row in result.usage.model_dump()["batch_usage_by_request"]) == [2, 3] + assert sorted(row["usage"]["pages_processed"] for row in result.usage.model_dump()["batch_usage_by_request"]) == [ + 2, + 3, + ] def test_native_vertex_capture_retains_unknown_models_without_changing_legacy_counts(): @@ -3456,3 +3486,76 @@ def test_native_vertex_capture_retains_unknown_models_without_changing_legacy_co assert captured["batch_usage_by_request"] == ({"model": "missing-usage-model", "usage": None},) assert captured["batch_unknown_usage_requests"] == 1 assert (result.cost, result.successful_requests, result.failed_requests) == (0, 0, 1) + + +def test_gemini_output_lines_bill_camelcase_usage_with_thinking_tokens_at_batch_rates(): + row = _success_row( + model="gemini-3.8-flash", + usage={"promptTokens": 9, "completionTokens": 1, "totalTokens": 75}, + ) + + result = bu._aggregate_batch_cost_usage_models( + entries=[row], + custom_llm_provider="gemini", + model_info=ModelInfo( + key="gemini/gemini-3.8-flash", + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=7.5e-7, + output_cost_per_token=3.75e-6, + litellm_provider="gemini", + mode="chat", + supported_openai_params=None, + input_cost_per_token_batches=3.75e-7, + output_cost_per_token_batches=1.875e-6, + ), + ) + + assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (9, 66, 75) + assert result.cost == pytest.approx((9 * 3.75e-7) + (66 * 1.875e-6)) + assert (result.successful_requests, result.failed_requests, result.models) == (1, 0, ["gemini-3.8-flash"]) + + +@pytest.mark.asyncio +async def test_gemini_native_output_rows_are_priced_from_usage_metadata(monkeypatch): + native_row = { + "key": "r0", + "response": { + "candidates": [{"content": {"parts": [{"text": "0"}], "role": "model"}, "finishReason": "STOP"}], + "usageMetadata": { + "promptTokenCount": 3, + "candidatesTokenCount": 1, + "totalTokenCount": 65, + "thoughtsTokenCount": 61, + }, + "modelVersion": "gemini-3.8-flash", + }, + } + failed_first_row = {"key": "bad", "status": {"code": 3, "message": "Invalid request"}} + jsonl = "\n".join(json.dumps(row) for row in (failed_first_row, native_row)).encode() + monkeypatch.setattr(bu, "_fetch_batch_managed_file_content", AsyncMock(return_value=jsonl)) + monkeypatch.setattr(bu, "count_error_file_failed_requests", AsyncMock(return_value=0)) + + result = await bu.handle_completed_batch( + batch=_batch("batch-la6"), + custom_llm_provider="gemini", + model_name="gemini-3.8-flash", + model_info=ModelInfo( + key="gemini/gemini-3.8-flash", + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=7.5e-7, + output_cost_per_token=3.75e-6, + litellm_provider="gemini", + mode="chat", + supported_openai_params=None, + input_cost_per_token_batches=3.75e-7, + output_cost_per_token_batches=1.875e-6, + ), + ) + + assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (3, 62, 65) + assert result.cost == pytest.approx((3 * 3.75e-7) + (62 * 1.875e-6)) + assert (result.successful_requests, result.failed_requests) == (1, 1) diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index 8cd74c7d83a..b12ea53bce3 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -1382,3 +1382,28 @@ async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: assert first.usage is not None assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2) assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued") + + +# =========================================================================== # +# gemini - a provider-config provider, like mistral, so it requires `model` +# =========================================================================== # + + +def test_create__gemini_routes_to_base_http_handler_with_gemini_config(seams): + result = bm.create_batch(**CREATE_KW, custom_llm_provider="gemini", model="gemini/gemini-3.8-flash") + + assert result is seams.base_http.create_batch.return_value + _assert_only(seams.base_http.create_batch, seams, "create_batch") + forwarded = seams.base_http.create_batch.call_args.kwargs + assert type(forwarded["provider_config"]).__name__ == "GeminiBatchesConfig" + assert forwarded["model"] == "gemini-3.8-flash" + + +def test_retrieve__gemini_routes_to_base_http_handler_with_gemini_config(seams): + result = bm.retrieve_batch(batch_id="la6", custom_llm_provider="gemini", model="gemini/gemini-3.8-flash") + + assert result is seams.base_http.retrieve_batch.return_value + _assert_only(seams.base_http.retrieve_batch, seams, "retrieve_batch") + forwarded = seams.base_http.retrieve_batch.call_args.kwargs + assert type(forwarded["provider_config"]).__name__ == "GeminiBatchesConfig" + assert forwarded["batch_id"] == "la6" diff --git a/tests/unit/llms/gemini/batches/__init__.py b/tests/unit/llms/gemini/batches/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/gemini/batches/test_gemini_batches_transformation.py b/tests/unit/llms/gemini/batches/test_gemini_batches_transformation.py new file mode 100644 index 00000000000..c8f15c9c590 --- /dev/null +++ b/tests/unit/llms/gemini/batches/test_gemini_batches_transformation.py @@ -0,0 +1,310 @@ +""" +Regression tests for ``GeminiBatchesConfig``, the BaseBatchesConfig implementation behind +``custom_llm_provider="gemini"`` on /v1/batches. + +Locks the ``models/{model}:batchGenerateContent`` create URL, the ``input_config.file_name`` +request shape, Gemini -> OpenAI state mapping, ``batchStats`` -> ``request_counts``, bare ids at +the LiteLLM boundary (``batches/`` and ``files/`` stripped) and ``x-goog-api-key`` auth. +Everything runs for real against canned httpx responses; only the API key env var is set. +""" + +import json + +import httpx +import pytest + +from litellm.llms.gemini.batches.transformation import ( + GeminiBatchesConfig, + is_gemini_batch_usage_shape, + is_native_gemini_batch_output_row, + usage_from_batch_output, +) +from litellm.llms.gemini.common_utils import GeminiError +from litellm.types.utils import LiteLLMBatch, LlmProviders + +STATUS_MAP = { + "BATCH_STATE_UNSPECIFIED": "validating", + "BATCH_STATE_PENDING": "validating", + "BATCH_STATE_RUNNING": "in_progress", + "BATCH_STATE_SUCCEEDED": "completed", + "BATCH_STATE_FAILED": "failed", + "BATCH_STATE_CANCELLED": "cancelled", + "BATCH_STATE_EXPIRED": "expired", +} + + +def _operation(**metadata_overrides): + metadata = { + "@type": "type.googleapis.com/google.ai.generativelanguage.v1main.GenerateContentBatch", + "model": "models/gemini-3.8-flash", + "displayName": "nightly-eval", + "inputConfig": {"fileName": "files/haf0y9b1kydd"}, + "output": {"responsesFile": "files/batch-la6uw1zzqrect7ugge7te7fueeslt1ms9xiw"}, + "createTime": "2026-11-10T21:18:34.714998843Z", + "endTime": "2026-11-10T21:21:01.499150334Z", + "updateTime": "2026-11-10T21:21:01.499150354Z", + "batchStats": {"requestCount": "3", "successfulRequestCount": "2", "failedRequestCount": "1"}, + "state": "BATCH_STATE_SUCCEEDED", + "name": "batches/la6uw1zzqrect7ugge7te7fueeslt1ms9xiw", + } + return { + "name": "batches/la6uw1zzqrect7ugge7te7fueeslt1ms9xiw", + "metadata": {**metadata, **metadata_overrides}, + "done": True, + } + + +def _response(payload: dict, status_code: int = 200) -> httpx.Response: + return httpx.Response( + status_code=status_code, + content=json.dumps(payload).encode(), + request=httpx.Request("GET", "https://generativelanguage.googleapis.com/v1beta/batches/x"), + ) + + +@pytest.fixture +def config() -> GeminiBatchesConfig: + return GeminiBatchesConfig() + + +@pytest.fixture +def api_key(monkeypatch) -> str: + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + monkeypatch.setenv("GEMINI_API_KEY", "AIza-test") + return "AIza-test" + + +def test_custom_llm_provider(config): + assert config.custom_llm_provider == LlmProviders.GEMINI + + +@pytest.mark.parametrize( + "input_file_id", + ["haf0y9b1kydd", "files/haf0y9b1kydd", "https://generativelanguage.googleapis.com/v1beta/files/haf0y9b1kydd"], +) +def test_create_request_names_the_uploaded_file_in_input_config(config, input_file_id): + body = config.transform_create_batch_request( + model="gemini-3.8-flash", + create_batch_data={ + "input_file_id": input_file_id, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + optional_params={}, + litellm_params={}, + ) + assert body == { + "batch": {"display_name": "litellm-haf0y9b1kydd", "input_config": {"file_name": "files/haf0y9b1kydd"}} + } + + +def test_create_request_uses_display_name_from_metadata(config): + body = config.transform_create_batch_request( + model="gemini-3.8-flash", + create_batch_data={ + "input_file_id": "files/abc", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "metadata": {"display_name": "nightly-eval"}, + }, + optional_params={}, + litellm_params={}, + ) + assert body["batch"]["display_name"] == "nightly-eval" + + +def test_create_request_requires_input_file(config): + with pytest.raises(ValueError, match="input_file_id"): + config.transform_create_batch_request( + model="gemini-3.8-flash", + create_batch_data={"endpoint": "/v1/chat/completions", "completion_window": "24h"}, + optional_params={}, + litellm_params={}, + ) + + +@pytest.mark.parametrize( + "api_base,model,expected", + [ + ( + None, + "gemini-3.8-flash", + "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:batchGenerateContent", + ), + ( + "https://proxy.example/", + "models/gemini-3.8-flash", + "https://proxy.example/v1beta/models/gemini-3.8-flash:batchGenerateContent", + ), + ], +) +def test_create_url_targets_the_model_batch_route(config, api_base, model, expected): + data = {"input_file_id": "files/abc", "endpoint": "/v1/chat/completions", "completion_window": "24h"} + assert config.get_complete_batch_url(api_base, None, model, {}, {}, data) == expected + + +def test_validate_environment_uses_goog_api_key_header(config, api_key): + headers = config.validate_environment({"x-trace": "1"}, "gemini-3.8-flash", [], {}, {}) + assert headers == {"x-trace": "1", "x-goog-api-key": api_key} + + +def test_validate_environment_explicit_key_wins(config, api_key): + headers = config.validate_environment({}, "gemini-3.8-flash", [], {}, {}, api_key="AIza-deployment") + assert headers["x-goog-api-key"] == "AIza-deployment" + + +def test_validate_environment_without_key_raises(config, monkeypatch): + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + with pytest.raises(ValueError, match="GEMINI_API_KEY"): + config.validate_environment({}, "gemini-3.8-flash", [], {}, {}) + + +def test_create_response_maps_operation_onto_openai_batch(config): + batch = config.transform_create_batch_response( + model="gemini-3.8-flash", raw_response=_response(_operation()), logging_obj=None, litellm_params={} + ) + assert isinstance(batch, LiteLLMBatch) + assert batch.model_dump(exclude_none=True) == { + "id": "la6uw1zzqrect7ugge7te7fueeslt1ms9xiw", + "object": "batch", + "endpoint": "/v1/chat/completions", + "input_file_id": "haf0y9b1kydd", + "output_file_id": "batch-la6uw1zzqrect7ugge7te7fueeslt1ms9xiw", + "completion_window": "24h", + "status": "completed", + "created_at": 1794345514, + "completed_at": 1794345661, + "request_counts": {"total": 3, "completed": 2, "failed": 1}, + "metadata": {"display_name": "nightly-eval"}, + } + + +def test_retrieve_request_is_presigned_get_with_bare_or_prefixed_id(config, api_key): + for batch_id in ("la6", "batches/la6"): + req = config.transform_retrieve_batch_request( + batch_id=batch_id, optional_params={}, litellm_params={"api_base": "https://proxy.example"} + ) + assert req == { + "method": "GET", + "url": "https://proxy.example/v1beta/batches/la6", + "headers": {"x-goog-api-key": api_key}, + }, batch_id + + +def test_retrieve_request_prefers_litellm_params_api_key(config, api_key): + req = config.transform_retrieve_batch_request( + batch_id="la6", optional_params={}, litellm_params={"api_key": "AIza-deployment"} + ) + assert req["headers"]["x-goog-api-key"] == "AIza-deployment" + + +@pytest.mark.parametrize("gemini_state,openai_status", sorted(STATUS_MAP.items())) +def test_retrieve_response_status_mapping(config, gemini_state, openai_status): + batch = config.transform_retrieve_batch_response( + model=None, raw_response=_response(_operation(state=gemini_state)), logging_obj=None, litellm_params={} + ) + assert batch.status == openai_status + + +@pytest.mark.parametrize( + "gemini_state,populated_field", + [ + ("BATCH_STATE_SUCCEEDED", "completed_at"), + ("BATCH_STATE_FAILED", "failed_at"), + ("BATCH_STATE_EXPIRED", "expired_at"), + ("BATCH_STATE_CANCELLED", "cancelled_at"), + ], +) +def test_end_time_lands_on_the_terminal_timestamp_for_its_state(config, gemini_state, populated_field): + batch = config.transform_retrieve_batch_response( + model=None, raw_response=_response(_operation(state=gemini_state)), logging_obj=None, litellm_params={} + ) + timestamps = {f: getattr(batch, f) for f in ("completed_at", "failed_at", "expired_at", "cancelled_at")} + assert timestamps == {f: (1794345661 if f == populated_field else None) for f in timestamps} + + +def test_running_batch_has_no_output_file_or_terminal_timestamp(config): + running = _operation(state="BATCH_STATE_RUNNING", batchStats={"requestCount": "3", "pendingRequestCount": "3"}) + del running["metadata"]["output"] + del running["metadata"]["endTime"] + batch = config.transform_retrieve_batch_response( + model=None, raw_response=_response(running), logging_obj=None, litellm_params={} + ) + assert batch.status == "in_progress" + assert batch.output_file_id is None + assert batch.completed_at is None + assert batch.request_counts.model_dump() == {"total": 3, "completed": 0, "failed": 0} + + +def test_failed_operation_error_is_reported_in_errors(config): + failed = {**_operation(state="BATCH_STATE_FAILED"), "error": {"code": 3, "message": "Input file is empty."}} + batch = config.transform_retrieve_batch_response( + model=None, raw_response=_response(failed), logging_obj=None, litellm_params={} + ) + assert batch.status == "failed" + assert [(e.code, e.message) for e in batch.errors.data] == [("3", "Input file is empty.")] + + +def test_error_class_is_gemini_error(config): + err = config.get_error_class("quota", 429, {"retry-after": "7"}) + assert isinstance(err, GeminiError) + assert (err.status_code, err.message) == (429, "quota") + + +# Gemini writes the usage of an OpenAI-shaped output row as camelCase, with thinking tokens +# counted in totalTokens but not in completionTokens (observed 2026-11-10 on gemini-3.8-flash). +@pytest.mark.parametrize( + "reported,expected", + [ + ( + {"promptTokens": 9, "completionTokens": 1, "totalTokens": 75}, + {"prompt_tokens": 9, "completion_tokens": 66, "total_tokens": 75, "reasoning_tokens": 65}, + ), + ( + {"promptTokens": 9, "completionTokens": 4, "totalTokens": 13}, + {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13, "reasoning_tokens": None}, + ), + ], +) +def test_batch_output_usage_folds_thinking_tokens_into_completion(reported, expected): + usage = usage_from_batch_output(reported) + details = usage.completion_tokens_details + assert { + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + "reasoning_tokens": details.reasoning_tokens if details else None, + } == expected + + +@pytest.mark.parametrize( + "usage_object,expected", + [({"promptTokens": 9}, True), ({"prompt_tokens": 9, "completion_tokens": 1}, False), ({}, False)], +) +def test_batch_usage_shape_detection(usage_object, expected): + assert is_gemini_batch_usage_shape(usage_object) is expected + + +def test_create_request_rejects_metadata_gemini_cannot_carry(config): + with pytest.raises(ValueError, match=r"\['run_id'\]"): + config.transform_create_batch_request( + model="gemini-3.8-flash", + create_batch_data={ + "input_file_id": "files/abc", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "metadata": {"display_name": "nightly", "run_id": "42"}, + }, + optional_params={}, + litellm_params={}, + ) + + +def test_native_output_row_detection(): + native = {"key": "r0", "response": {"candidates": [], "usageMetadata": {"promptTokenCount": 3}}} + openai_shaped = {"custom_id": "r0", "response": {"status_code": 200, "body": {"usage": {"promptTokens": 3}}}} + assert (is_native_gemini_batch_output_row(native), is_native_gemini_batch_output_row(openai_shaped)) == ( + True, + False, + ) diff --git a/tests/unit/llms/gemini/files/test_gemini_files_transformation.py b/tests/unit/llms/gemini/files/test_gemini_files_transformation.py index a2f95724688..0baf4e7b0fb 100644 --- a/tests/unit/llms/gemini/files/test_gemini_files_transformation.py +++ b/tests/unit/llms/gemini/files/test_gemini_files_transformation.py @@ -43,9 +43,7 @@ class TestGoogleAIStudioFilesTransformation: # CRITICAL: params should be empty dict, not contain Content-Type or any other params # These would be incorrectly interpreted as query parameters assert params == {}, f"Expected empty params dict, got: {params}" - assert ( - "Content-Type" not in params - ), "Content-Type should not be in query params" + assert "Content-Type" not in params, "Content-Type should not be in query params" def test_transform_retrieve_file_request_with_file_name_only(self): """ @@ -66,9 +64,7 @@ class TestGoogleAIStudioFilesTransformation: # CRITICAL: params should be empty dict assert params == {}, f"Expected empty params dict, got: {params}" - assert ( - "Content-Type" not in params - ), "Content-Type should not be in query params" + assert "Content-Type" not in params, "Content-Type should not be in query params" def test_transform_retrieve_file_request_with_raw_id_only(self): """ @@ -86,9 +82,7 @@ class TestGoogleAIStudioFilesTransformation: litellm_params=litellm_params, ) - assert ( - url == "https://generativelanguage.googleapis.com/v1beta/files/cctqueckiggb" - ) + assert url == "https://generativelanguage.googleapis.com/v1beta/files/cctqueckiggb" assert "key=" not in url assert params == {} @@ -103,8 +97,7 @@ class TestGoogleAIStudioFilesTransformation: ) assert ( - url - == "https://generativelanguage.googleapis.com/v1beta/files/..%2F..%2Fmodels%2Fgemini-pro%3Fx%3D1%23frag" + url == "https://generativelanguage.googleapis.com/v1beta/files/..%2F..%2Fmodels%2Fgemini-pro%3Fx%3D1%23frag" ) assert params == {} @@ -358,7 +351,49 @@ class TestGoogleAIStudioFilesTransformation: ) assert ( - url - == "https://generativelanguage.googleapis.com/v1beta/files/..%2F..%2Fmodels%2Fgemini-pro%3Fx%3D1%23frag" + url == "https://generativelanguage.googleapis.com/v1beta/files/..%2F..%2Fmodels%2Fgemini-pro%3Fx%3D1%23frag" ) assert params == {} + + +@pytest.mark.parametrize( + "file_id", + ["batch-la6", "files/batch-la6", "https://generativelanguage.googleapis.com/v1beta/files/batch-la6"], +) +def test_file_content_request_downloads_the_file_as_media(file_id): + handler = GoogleAIStudioFilesHandler() + url, params = handler.transform_file_content_request( + file_content_request={"file_id": file_id}, + optional_params={}, + litellm_params={"api_base": "https://proxy.example"}, + ) + assert (url, params) == ("https://proxy.example/download/v1beta/files/batch-la6:download", {"alt": "media"}) + + +def test_file_content_response_passes_the_jsonl_bytes_through(): + handler = GoogleAIStudioFilesHandler() + body = b'{"custom_id":"r0","response":{"status_code":200,"body":{"object":"chat.completion"}}}\n' + raw = httpx.Response(200, content=body, request=httpx.Request("GET", "https://generativelanguage.googleapis.com/x")) + assert ( + handler.transform_file_content_response(raw_response=raw, logging_obj=Mock(), litellm_params={}).content == body + ) + + +def test_retrieve_file_response_accepts_nanosecond_create_time(): + handler = GoogleAIStudioFilesHandler() + raw = httpx.Response( + 200, + json={ + "name": "files/batch-la6", + "displayName": "Results of batch la6", + "mimeType": "application/jsonl", + "sizeBytes": "1024", + "createTime": "2026-11-10T21:35:00.902340270Z", + "updateTime": "2026-11-10T21:35:00.902340270Z", + "uri": "https://generativelanguage.googleapis.com/v1beta/files/batch-la6", + "state": "ACTIVE", + }, + request=httpx.Request("GET", "https://generativelanguage.googleapis.com/v1beta/files/batch-la6"), + ) + file_object = handler.transform_retrieve_file_response(raw_response=raw, logging_obj=Mock(), litellm_params={}) + assert (file_object.created_at, file_object.bytes, file_object.status) == (1794346500, 1024, "processed") From 8f42db985d0c9aee79960cde1198b97f18f7dfef Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 11 Nov 2026 12:18:44 -0800 Subject: [PATCH 105/189] fix(router): map Gemini stream-opening errors (#51218) * fix(router): map errors when opening Gemini streams * fix(streaming): preserve stream-open exception causes * fix(streaming): limit open-error mapping to provider and transport errors --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../litellm_core_utils/streaming_handler.py | 17 ++- .../test_streaming_handler.py | 130 +++++++++++++++--- .../test_streaming_characterization_chat.py | 51 ++++++- 3 files changed, 176 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 956e202a65a..079b29d5509 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -46,6 +46,7 @@ from litellm.litellm_core_utils.model_response_utils import ( ) from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject, drop_token_ids from litellm.litellm_core_utils.thread_pool_executor import executor +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router import GenericLiteLLMParams @@ -2534,7 +2535,10 @@ class CustomStreamWrapper: def fetch_sync_stream(self): if self.completion_stream is None and self.make_call is not None: # Call make_call to get the completion stream - self.completion_stream = self.make_call(client=litellm.module_level_client) + try: + self.completion_stream = self.make_call(client=litellm.module_level_client) + except (BaseLLMException, httpx.HTTPError) as e: + self._raise_open_error(e) self._stream_iter = self.completion_stream.__iter__() return self.completion_stream @@ -2542,7 +2546,10 @@ class CustomStreamWrapper: async def fetch_stream(self): if self.completion_stream is None and self.make_call is not None: # Call make_call to get the completion stream - self.completion_stream = await self.make_call(client=litellm.module_level_aclient) + try: + self.completion_stream = await self.make_call(client=litellm.module_level_aclient) + except (BaseLLMException, httpx.HTTPError) as e: + self._raise_open_error(e) self._stream_iter = self.completion_stream.__aiter__() return self.completion_stream @@ -2864,6 +2871,12 @@ class CustomStreamWrapper: if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta): self._emitted_disqualifying_content = True + def _raise_open_error(self, error: BaseLLMException | httpx.HTTPError) -> NoReturn: + try: + raise self._map_stream_exception(error) + except Exception as mapped: + raise mapped from error + def _map_stream_exception(self, original_exception: Exception) -> Exception: return exception_type( model=self.model, diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index cb0ef9a9a33..beb1b58ec27 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -38,9 +38,10 @@ from litellm.litellm_core_utils.streaming_handler import ( _ProviderChunkParsed, ) from litellm.llms.anthropic.chat.handler import ModelResponseIterator as AnthropicModelResponseIterator -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import HTTPHandler, MaskedHTTPStatusError from litellm.llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler +from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( ModelResponseIterator as GeminiModelResponseIterator, ) @@ -1433,6 +1434,102 @@ def test_sync_streaming_bad_request_not_midstream(logging_obj: Logging): assert "invalid maxOutputTokens" in str(excinfo.value) +def _gemini_stream_error(status_code: int, sync_mode: bool) -> Exception: + if not sync_mode: + return VertexAIError(status_code=status_code, message="upstream said no", headers=None) + request: Final = httpx.Request("POST", "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash") + body: Final = b'{"error": {"message": "upstream said no"}}' + response: Final = httpx.Response(status_code, request=request, content=body) + return MaskedHTTPStatusError( + httpx.HTTPStatusError("upstream said no", request=request, response=response), + message=body, # pyright: ignore[reportArgumentType] # the sync stream handler passes the raw body bytes + text=body, # pyright: ignore[reportArgumentType] # the sync stream handler passes the raw body bytes + ) + + +def _gemini_stream_opened_by(error: BaseException, sync_mode: bool, logging_obj: Logging) -> CustomStreamWrapper: + def make_sync_call(client: object) -> NoReturn: + raise error + + async def make_call(client: object) -> NoReturn: + raise error + + return CustomStreamWrapper( + completion_stream=None, + model="gemini-2.5-flash", + logging_obj=logging_obj, + custom_llm_provider="gemini", + make_call=make_sync_call if sync_mode else make_call, + ) + + +async def _fetch(stream: CustomStreamWrapper, sync_mode: bool) -> object: + return stream.fetch_sync_stream() if sync_mode else await stream.fetch_stream() + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize( + ("status_code", "expected_error"), + [ + (400, litellm.BadRequestError), + (429, litellm.RateLimitError), + (503, litellm.ServiceUnavailableError), + ], +) +async def test_a_gemini_stream_that_fails_to_open_raises_the_mapped_error_from_fetch( + logging_obj: Logging, + sync_mode: bool, + status_code: int, + expected_error: type[litellm.BadRequestError | litellm.RateLimitError | litellm.ServiceUnavailableError], +) -> None: + provider_error: Final = _gemini_stream_error(status_code, sync_mode) + stream: Final = _gemini_stream_opened_by(provider_error, sync_mode, logging_obj) + with pytest.raises(expected_error, match="upstream said no") as excinfo: + await _fetch(stream, sync_mode) + assert (excinfo.value.status_code, excinfo.value.llm_provider) == (status_code, "gemini") + assert excinfo.value.__cause__ is provider_error + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_a_stream_open_error_that_is_already_mapped_is_raised_as_is( + logging_obj: Logging, sync_mode: bool +) -> None: + already_mapped: Final = litellm.BadRequestError( + message="already mapped", model="gemini-2.5-flash", llm_provider="gemini" + ) + stream: Final = _gemini_stream_opened_by(already_mapped, sync_mode, logging_obj) + with pytest.raises(litellm.BadRequestError) as excinfo: + await _fetch(stream, sync_mode) + assert excinfo.value is already_mapped + assert excinfo.value.__cause__ is None + + +async def test_a_cancelled_stream_open_stays_cancelled(logging_obj: Logging) -> None: + stream: Final = _gemini_stream_opened_by(asyncio.CancelledError(), False, logging_obj) + with pytest.raises(asyncio.CancelledError): + await stream.fetch_stream() + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_a_stream_that_opens_is_opened_once(logging_obj: Logging, sync_mode: bool) -> None: + def make_sync_call(client: object) -> ModelResponseListIterator: + return ModelResponseListIterator(model_responses=[]) + + async def make_call(client: object) -> ModelResponseListIterator: + return ModelResponseListIterator(model_responses=[]) + + stream: Final = CustomStreamWrapper( + completion_stream=None, + model="gemini-2.5-flash", + logging_obj=logging_obj, + custom_llm_provider="gemini", + make_call=make_sync_call if sync_mode else make_call, + ) + opened: Final = await _fetch(stream, sync_mode) + assert isinstance(opened, ModelResponseListIterator) + assert await _fetch(stream, sync_mode) is opened + + def _bedrock_error_event(exception_type: str): """A mocked botocore event-stream error event: status_code is botocore's hard-coded 400, with the real type in the :exception-type header.""" @@ -2113,37 +2210,32 @@ async def test_in_body_stream_error_500_wraps_for_midstream_fallback( assert "internal engine crash" in str(excinfo.value) -@pytest.mark.asyncio -async def test_async_streaming_read_timeout_triggers_midstream_fallback( - logging_obj: Logging, -): - """A mid-stream httpx.ReadTimeout must wrap into MidStreamFallbackError so - the Router's FallbackStreamWrapper can switch to a fallback model. +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_a_read_timeout_while_opening_the_stream_triggers_midstream_fallback( + logging_obj: Logging, sync_mode: bool +) -> None: + read_timeout: Final = httpx.ReadTimeout("Timeout on reading data from socket") - Previously __anext__ caught httpx.TimeoutException and re-raised it raw, - which bypassed _handle_stream_fallback_error and prevented stream_timeout - from triggering fallbacks the way connection-phase timeout does. - """ - import httpx + def make_sync_call(client: object) -> NoReturn: + raise read_timeout - from litellm.exceptions import MidStreamFallbackError + async def make_call(client: object) -> NoReturn: + raise read_timeout - async def _raise_read_timeout(**kwargs): - raise httpx.ReadTimeout("Timeout on reading data from socket") - - response = CustomStreamWrapper( + response: Final = CustomStreamWrapper( completion_stream=None, model="gpt-4", logging_obj=logging_obj, custom_llm_provider="openai", - make_call=_raise_read_timeout, + make_call=make_sync_call if sync_mode else make_call, ) with pytest.raises(MidStreamFallbackError) as excinfo: - await response.__anext__() + next(response) if sync_mode else await response.__anext__() assert excinfo.value.is_pre_first_chunk is True assert isinstance(excinfo.value.original_exception, litellm.Timeout) + assert excinfo.value.original_exception.__cause__ is read_timeout def _raw_read_timeout() -> NoReturn: diff --git a/tests/unit/test_router/core/test_streaming_characterization_chat.py b/tests/unit/test_router/core/test_streaming_characterization_chat.py index 590fafede8e..8f727c5f0ee 100644 --- a/tests/unit/test_router/core/test_streaming_characterization_chat.py +++ b/tests/unit/test_router/core/test_streaming_characterization_chat.py @@ -10,14 +10,17 @@ while one that raises later fails while the caller iterates. import asyncio from collections.abc import Callable, Mapping -from typing import Final +from typing import Final, NoReturn +import httpx import pytest import litellm from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError +from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.router_core import RouterCore from litellm.types.utils import Usage from tests.unit.test_router.core.scripted_host import ( @@ -35,6 +38,7 @@ from tests.unit.test_router.core.streaming_helpers import ( collect_texts, collect_texts_sync, drain_into, + request_logging, ) MakeCore = Callable[..., RouterCore] @@ -107,6 +111,51 @@ async def test_a_first_chunk_bad_request_is_raised_from_the_call_without_retry( assert host.deployments_called() == ["a"] +def _gemini_stream_that_fails_to_open(sync: bool) -> Callable[..., CustomStreamWrapper]: + request: Final = httpx.Request("POST", "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash") + body: Final = b'{"error": {"code": 400, "message": "gemini said no", "status": "INVALID_ARGUMENT"}}' + sync_error: Final = MaskedHTTPStatusError( + httpx.HTTPStatusError("gemini said no", request=request, response=httpx.Response(400, request=request)), + message=body, # pyright: ignore[reportArgumentType] # the sync stream handler passes the raw body bytes + text=body, # pyright: ignore[reportArgumentType] # the sync stream handler passes the raw body bytes + ) + + def make_sync_call(client: object) -> NoReturn: + raise sync_error + + async def make_call(client: object) -> NoReturn: + raise VertexAIError(status_code=400, message=body.decode(), headers=None) + + def answer(**_request: object) -> CustomStreamWrapper: # kwargs-ok: the host passes the request kwargs + return CustomStreamWrapper( + completion_stream=None, + model="gemini-2.5-flash", + custom_llm_provider="gemini", + logging_obj=request_logging(), + make_call=make_sync_call if sync else make_call, + ) + + return answer + + +async def test_a_gemini_stream_that_fails_to_open_raises_the_mapped_bad_request( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", _gemini_stream_that_fails_to_open(sync=False)) + core: Final = make_core(model_list=[deployment("a", "a")], num_retries=0) + with pytest.raises(litellm.BadRequestError, match="gemini said no"): + await open_stream(core) + + +def test_a_sync_gemini_stream_that_fails_to_open_raises_the_mapped_bad_request( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", _gemini_stream_that_fails_to_open(sync=True)) + core: Final = make_core(model_list=[deployment("a", "a")], num_retries=0) + with pytest.raises(litellm.BadRequestError, match="gemini said no"): + core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + async def test_the_first_chunk_already_read_is_still_delivered_to_the_caller( make_core: MakeCore, host: ScriptedHost ) -> None: From deb7697a8e515671774b3ccda6dd6d67123e41cc Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 11 Nov 2026 12:22:12 -0800 Subject: [PATCH 106/189] fix(auth): enforce model access on Google and image URL routes (#51226) * fix(auth): authorize URL-selected Google and image models * fix(auth): preserve server model selection and budget fallbacks * test(auth): isolate URL model authorization settings * fix(auth): type handler model selection boundaries * fix(auth): validate request model selection inputs * fix(auth): tolerate absent request path metadata * fix(auth): support requests without path metadata --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/proxy/auth/auth_utils.py | 46 +++++++- litellm/proxy/auth/user_api_key_auth.py | 3 + .../proxy/common_utils/http_parsing_utils.py | 6 ++ litellm/proxy/proxy_server.py | 4 + .../spend_tracking/budget_reservation.py | 24 ++++- tests/unit/proxy/auth/test_auth_utils.py | 101 ++++++++++++++++++ .../unit/proxy/auth/test_user_api_key_auth.py | 48 +++++++++ .../test_user_api_key_auth_request_flow.py | 40 +++++++ .../common_utils/test_http_parsing_utils.py | 20 ++++ .../spend_tracking/test_budget_reservation.py | 34 ++++++ 10 files changed, 320 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 7413490e44f..900e993cadb 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -37,7 +37,11 @@ from litellm.llms.azure.passthrough.transformation import azure_router_model_in_ from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.proxy._types import * from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata +from litellm.proxy.common_utils.http_parsing_utils import ( + extract_nested_form_metadata, + read_request_path_params, + resolve_inference_model, +) from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_ENDPOINT_MARKER, ) @@ -2672,6 +2676,7 @@ def get_model_from_request( llm_router: Router | None = None, request: Request | None = None, team_id: str | None = None, + path_params: Mapping[str, object] | None = None, ) -> str | list[str] | None: """Resolve the model(s) a request targets, for model-access and budget checks. @@ -2690,6 +2695,12 @@ def get_model_from_request( if request_dispatched_to_pass_through_endpoint(request): return None + handler_request_data: Final = _MODEL_REQUEST_DATA.validate_python(request_data) + handler_path_params: Final = read_request_path_params(request) if request is not None else path_params + handler_model: Final = _handler_selected_model(route, handler_request_data, handler_path_params) + if handler_model is not None: + return handler_model + candidates: Final = _extract_model_candidates_from_request( request_data=request_data, route=route, @@ -2719,6 +2730,39 @@ def get_model_from_request( return model +_HandlerModelKind: TypeAlias = Literal["completion", "image_generation", "image_edit"] +_MODEL_REQUEST_DATA: Final = TypeAdapter(Mapping[str, object]) +_HANDLER_MODEL_ROUTES: Final[tuple[tuple[re.Pattern[str], str, _HandlerModelKind], ...]] = ( + (re.compile(r"^(?:/v1beta)?/models/(.+):(?:generateContent|streamGenerateContent)$"), "model_name", "completion"), + (re.compile(r"^/openai/deployments/(.+)/images/generations$"), "model", "image_generation"), + (re.compile(r"^/openai/deployments/(.+)/images/edits$"), "model", "image_edit"), +) + + +def _handler_selected_model( + route: str, request_data: Mapping[str, object], path_params: Mapping[str, object] | None +) -> str | None: + selection: Final[tuple[re.Match[str], str, _HandlerModelKind] | None] = next( + ((match, name, kind) for pattern, name, kind in _HANDLER_MODEL_ROUTES if (match := pattern.match(route))), + None, + ) + if selection is None: + return None + from litellm.proxy.proxy_server import general_settings_view, user_model_view + + route_match, path_param_name, kind = selection + route_model: Final = route_match.group(1) + path_model: Final = path_params.get(path_param_name) if path_params is not None else None + model: Final = resolve_inference_model( + request_data.get("model"), + general_settings_view(), + user_model_view(), + path_model if isinstance(path_model, str) and path_model else route_model, + kind=kind, + ) + return model if isinstance(model, str) and model else route_model + + def _model_from_route_path(route: str) -> str | None: """ Google generateContent-style routes allow "/" inside the model id, e.g. diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index d7a8d4a0700..1b2e4e791e3 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -153,6 +153,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg populate_request_with_path_params, read_raw_json_body, read_request_body, + read_request_path_params, resolve_realtime_route_model, rewrite_request_model, safe_get_request_headers, @@ -4252,6 +4253,7 @@ async def _reserve_budget_after_common_checks( reserve_budget_for_request, ) + reservation_path_params: Final = read_request_path_params(request) user_api_key_auth_obj.budget_reservation = await reserve_budget_for_request( request_body=request_data, route=route, @@ -4270,6 +4272,7 @@ async def _reserve_budget_after_common_checks( or user_api_key_auth_obj.billing_agent_policy is not None ), raw_body=await read_raw_json_body(request=request), + path_params=reservation_path_params, ) if request is not None: reservation: Final = user_api_key_auth_obj.budget_reservation diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 80ba2f49745..2786b4271e4 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -37,6 +37,12 @@ _BINARY_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-protobu _ANNOTATION_QUALIFIERS: Final[frozenset[object]] = frozenset({Annotated, NotRequired, ReadOnly, Required}) _DECODED_JSON: Final = TypeAdapter(object) _JSON_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object]) +_REQUEST_PATH_PARAMS: Final = TypeAdapter(Mapping[str, object]) + + +def read_request_path_params(request: Request | None) -> Mapping[str, object] | None: + path_params: Final = _DECODED_JSON.validate_python(getattr(request, "path_params", None)) + return _REQUEST_PATH_PARAMS.validate_python(path_params) if isinstance(path_params, Mapping) else None class _ReadableUpload(Protocol): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 23d45e07dfa..6fbe562b5d1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3183,6 +3183,10 @@ def general_settings_view() -> Mapping[str, object]: return _GENERAL_SETTINGS_VIEW.validate_python(general_settings) +def user_model_view() -> str | None: + return user_model if isinstance(user_model, str) else None + + config_passthrough_endpoints: list[dict[str, object]] | None = None log_file: Final = "api_log.json" worker_config: Final = None diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 8f0584bd0d0..096578264e3 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -262,12 +262,13 @@ async def reserve_budget_for_request( apply_user_budget_to_team_keys: bool = False, fail_closed_budget_enforcement: bool = False, raw_body: bytes | None = None, + path_params: Mapping[str, object] | None = None, ) -> dict | None: if valid_token is None or not RouteChecks.is_llm_api_route(route=route): return None if RouteChecks.is_unbilled_route(route): return None - if get_model_from_request(request_body, route, llm_router=llm_router) is None: + if get_model_from_request(request_body, route, llm_router=llm_router, path_params=path_params) is None: return None counters: Final = await _get_budget_counters( @@ -291,6 +292,7 @@ async def reserve_budget_for_request( llm_router=llm_router, team_id=valid_token.team_id, raw_body=raw_body, + path_params=path_params, ) reservation_cost = estimate_request_max_cost( @@ -298,6 +300,7 @@ async def reserve_budget_for_request( route=route, llm_router=llm_router, input_token_counts=input_token_counts, + path_params=path_params, ) # estimate_request_max_cost still returns None when the model is unknown # to the cost map (no token-priced cost fields, e.g. image/audio routes). @@ -334,6 +337,7 @@ async def reserve_budget_for_request( route=route, llm_router=llm_router, input_token_counts=input_token_counts, + path_params=path_params, ) budget_reservation: Final = { "reserved_cost": reservation_cost, @@ -1319,6 +1323,7 @@ def estimate_request_max_cost( route: str, llm_router: Router | None, input_token_counts: Mapping[str, int] | None = None, + path_params: Mapping[str, object] | None = None, ) -> float | None: estimates = [ _estimate_request_max_cost_for_model( @@ -1328,7 +1333,9 @@ def estimate_request_max_cost( llm_router=llm_router, input_tokens=(input_token_counts or {}).get(model_name), ) - for model_name in _get_request_models(request_body=request_body, route=route, llm_router=llm_router) + for model_name in _get_request_models( + request_body=request_body, route=route, llm_router=llm_router, path_params=path_params + ) ] estimates = [estimate for estimate in estimates if estimate is not None] if not estimates: @@ -1341,6 +1348,7 @@ def estimate_request_input_cost( route: str, llm_router: Router | None, input_token_counts: Mapping[str, int] | None = None, + path_params: Mapping[str, object] | None = None, ) -> float | None: """Cost of the request's input tokens alone. @@ -1357,7 +1365,9 @@ def estimate_request_input_cost( llm_router=llm_router, input_tokens=(input_token_counts or {}).get(model_name), ) - for model_name in _get_request_models(request_body=request_body, route=route, llm_router=llm_router) + for model_name in _get_request_models( + request_body=request_body, route=route, llm_router=llm_router, path_params=path_params + ) ] estimates = [estimate for estimate in estimates if estimate is not None] if not estimates: @@ -1609,8 +1619,9 @@ def _get_request_models( request_body: dict, route: str, llm_router: Router | None, + path_params: Mapping[str, object] | None = None, ) -> Sequence[str]: - model: Final = get_model_from_request(request_body, route, llm_router=llm_router) + model: Final = get_model_from_request(request_body, route, llm_router=llm_router, path_params=path_params) if model is None: return () return (model,) if isinstance(model, str) else tuple(model) @@ -1622,11 +1633,14 @@ async def count_request_input_tokens( llm_router: Router | None, team_id: str | None, raw_body: bytes | None = None, + path_params: Mapping[str, object] | None = None, ) -> Mapping[str, int]: """Input-token count per candidate model, counted once per request. The counts are reused by both the max-cost and the input-cost estimate.""" - models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) + models: Final = _get_request_models( + request_body=request_body, route=route, llm_router=llm_router, path_params=path_params + ) if not models: return MappingProxyType({}) counted_models: Final = MappingProxyType( diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 73a2c534628..58a78aede3c 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -1379,6 +1379,107 @@ def test_get_model_from_request_supports_google_model_names_with_slashes(): ) +_URL_SELECTED_MODEL_ROUTES: Final = ( + ("/v1beta/models/premium-model:generateContent", "premium-model"), + ("/models/premium-model:generateContent", "premium-model"), + ("/v1beta/models/premium-model:streamGenerateContent", "premium-model"), + ("/models/provider/premium-model:7b:streamGenerateContent", "provider/premium-model:7b"), + ("/openai/deployments/premium-model/images/generations", "premium-model"), + ("/openai/deployments/provider/premium-model/images/edits", "provider/premium-model"), +) + + +@pytest.fixture +def no_inference_model_override(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + + +@pytest.mark.usefixtures("no_inference_model_override") +@pytest.mark.parametrize("route, url_model", _URL_SELECTED_MODEL_ROUTES) +@pytest.mark.parametrize( + "request_data", + [ + {"model": "cheap-model"}, + {"target_model_names": ["cheap-model"]}, + {"model": "cheap-model", "target_model_names": "cheap-model"}, + ], +) +def test_get_model_from_request_checks_the_url_model_on_routes_served_from_the_url( + route: str, url_model: str, request_data: dict[str, object] +) -> None: + assert get_model_from_request(request_data=request_data, route=route) == url_model + + +@pytest.mark.usefixtures("no_inference_model_override") +@pytest.mark.parametrize( + "route, path_params", + [ + ("/v1beta/models/premium-model:generateContent", {"model_name": "cheap-model"}), + ("/models/premium-model:streamGenerateContent", {"model_name": "cheap-model"}), + ("/openai/deployments/premium-model/images/generations", {"model": "cheap-model"}), + ("/openai/deployments/premium-model/images/edits", {"model": "cheap-model"}), + ], +) +def test_get_model_from_request_follows_the_handlers_path_param_over_route_and_body( + route: str, path_params: dict[str, str] +) -> None: + request: Final = Request(scope={"type": "http", "path": route, "path_params": path_params, "headers": []}) + body: Final = {"model": "body-model"} + + assert get_model_from_request(request_data=body, route=route, request=request) == "cheap-model" + assert get_model_from_request(request_data=body, route=route, path_params=path_params) == "cheap-model" + assert get_model_from_request(request_data=body, route=route, path_params={"other": "x"}) == "premium-model" + + +@pytest.mark.parametrize( + "route, settings, cli_model, expected", + [ + ("/v1beta/models/url-model:generateContent", {"completion_model": "configured"}, None, "configured"), + ("/models/url-model:streamGenerateContent", {}, "cli", "cli"), + ("/v1beta/models/url-model:generateContent", {"image_generation_model": "configured"}, None, "url-model"), + ("/openai/deployments/url-model/images/generations", {}, "cli", "cli"), + ("/openai/deployments/url-model/images/generations", {"completion_model": "configured"}, None, "url-model"), + ( + "/openai/deployments/url-model/images/generations", + {"image_generation_model": "configured"}, + None, + "url-model", + ), + ("/openai/deployments/url-model/images/edits", {"completion_model": "configured"}, "cli", "configured"), + ("/openai/deployments/url-model/images/edits", {}, "cli", "cli"), + ("/openai/deployments/url-model/images/edits", {"image_generation_model": "configured"}, None, "url-model"), + ], +) +def test_get_model_from_request_applies_the_handlers_server_model_precedence( + monkeypatch: pytest.MonkeyPatch, + route: str, + settings: dict[str, str], + cli_model: str | None, + expected: str, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", settings) + monkeypatch.setattr(proxy_server, "user_model", cli_model) + + assert get_model_from_request(request_data={"model": "body-model"}, route=route) == expected + + +@pytest.mark.usefixtures("no_inference_model_override") +@pytest.mark.parametrize("route", ["/v1/images/generations", "/images/edits", "/v1/chat/completions"]) +def test_get_model_from_request_keeps_the_body_model_where_the_url_names_none(route: str) -> None: + assert get_model_from_request(request_data={"model": "cheap-model"}, route=route) == "cheap-model" + + +@pytest.mark.parametrize("route, url_model", _URL_SELECTED_MODEL_ROUTES) +def test_get_model_from_request_skips_url_model_on_configured_pass_through(route: str, url_model: str) -> None: + request: Final = _request_dispatched_to(_pass_through_endpoint()) + assert get_model_from_request(request_data={"model": url_model}, route=route, request=request) is None + + def test_get_model_from_request_vertex_passthrough_still_works(): route = "/vertex_ai/v1/projects/p/locations/l/publishers/google/models/gemini-1.5-pro:generateContent" assert get_model_from_request(request_data={}, route=route) == "gemini-1.5-pro" diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index ce4a844550b..7746fc96673 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -1293,6 +1293,54 @@ def test_get_model_from_request(route, request_data, expected_model): assert get_model_from_request(request_data, route) == expected_model +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route_template", + [ + "/v1beta/models/{model}:generateContent", + "/models/{model}:streamGenerateContent", + "/openai/deployments/{model}/images/generations", + "/openai/deployments/{model}/images/edits", + ], +) +@pytest.mark.parametrize("body", [{"model": "cheap-model"}, {"target_model_names": ["cheap-model"]}]) +async def test_key_model_allowlist_applies_to_the_model_selected_by_the_url( + monkeypatch: pytest.MonkeyPatch, route_template: str, body: dict[str, object] +) -> None: + from litellm.proxy._types import ProxyErrorTypes + from litellm.proxy.auth.user_api_key_auth import enforce_key_and_fallback_model_access + + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", None) + monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {}) + monkeypatch.setattr(litellm.proxy.proxy_server, "user_model", None) + router: Final = litellm.Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": f"openai/{name}", "api_key": "sk-fake"}} + for name in ("cheap-model", "premium-model") + ] + ) + token: Final = UserAPIKeyAuth(models=["cheap-model"]) + + async def authorize(url_model: str) -> None: + route: Final = route_template.format(model=url_model) + request: Final = StarletteRequest( + {"type": "http", "method": "POST", "path": route, "headers": [], "query_string": b""} + ) + await enforce_key_and_fallback_model_access( + valid_token=token, + request_data=dict(body), + route=route, + request=request, + llm_model_list=list(router.model_list), + llm_router=router, + ) + + await authorize("cheap-model") + with pytest.raises(ProxyException) as exc_info: + await authorize("premium-model") + assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied + + @pytest.mark.asyncio async def test_jwt_non_admin_team_route_access(monkeypatch): """ diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 0d2433d9707..5824b5fccab 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -10952,6 +10952,42 @@ class TestCheckTeamMemberModelBudgetWithFallback: assert request_data["model"] == "gpt-4o-mini" + @pytest.mark.asyncio + @pytest.mark.parametrize("operation", ["generations", "edits"]) + async def test_image_deployment_key_fallback_is_the_model_later_checks_and_reservation_see( + self, monkeypatch: pytest.MonkeyPatch, operation: str + ) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth.auth_utils import get_model_from_request + + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + route: Final = f"/openai/deployments/gpt-4o/images/{operation}" + valid_token: Final = UserAPIKeyAuth( + token="test-key", + models=["gpt-4o", "gpt-4o-mini"], + budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + model_max_budget={ + "gpt-4o": {"max_budget": 5.0, "budget_duration": "1d"}, + "gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}, + }, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={"gpt-4o": 10.0}) + request_data: Final[dict[str, object]] = {"model": "gpt-4o"} + request: Final = self._request("gpt-4o") + assert get_model_from_request(request_data, route, request=request) == "gpt-4o" + + await _check_key_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + model_name="gpt-4o", + request_data=request_data, + request=request, + ) + + assert get_model_from_request(request_data, route, request=request) == "gpt-4o-mini" + assert get_model_from_request(request_data, route, path_params=request.path_params) == "gpt-4o-mini" + @pytest.mark.asyncio async def test_fallback_outside_member_allowed_models_is_rejected(self): valid_token: Final = UserAPIKeyAuth( @@ -13440,6 +13476,10 @@ async def test_managed_actor_cannot_access_provider_resource_routes(monkeypatch, ("/v1/chat/completions", {}, "forbidden-model"), ("/v1/images/generations", {"image_generation_model": "forbidden-model"}, None), ("/v1/images/edits", {"image_generation_model": "forbidden-model"}, None), + ("/v1beta/models/test-model:generateContent", {"completion_model": "forbidden-model"}, None), + ("/models/test-model:streamGenerateContent", {}, "forbidden-model"), + ("/openai/deployments/test-model/images/generations", {}, "forbidden-model"), + ("/openai/deployments/test-model/images/edits", {"completion_model": "forbidden-model"}, None), ], ) async def test_managed_agent_cannot_bypass_grants_with_server_default( diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index 8423dee9886..b8a686d3f67 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -31,6 +31,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( numeric_form_fields, populate_request_with_path_params, read_raw_json_body, + read_request_path_params, ) from fastapi import Request as Request_http_parsing from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -1786,3 +1787,22 @@ async def test_convert_upload_files_to_file_data_reads_every_upload_into_file_tu "tags": ["red", "blue"], "display_title": "My skill", } + + +@pytest.mark.parametrize( + "scope_path_params, expected", + [ + ({"model": "deployment/name"}, {"model": "deployment/name"}), + ({}, {}), + (None, None), + ], +) +def test_read_request_path_params_returns_mappings_and_drops_non_mapping_metadata( + scope_path_params: Mapping[str, object] | None, expected: Mapping[str, object] | None +) -> None: + request: Final = Request({"type": "http", "path_params": scope_path_params}) + assert read_request_path_params(request) == expected + + +def test_read_request_path_params_without_request_is_none() -> None: + assert read_request_path_params(None) is None diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 72927fa0197..087434c981f 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -223,6 +223,40 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None: assert math.isclose(after, before * 1000) +def _priced_group(model_name: str, cost_per_token: float) -> dict[str, object]: + return Deployment( + model_name=model_name, + litellm_params=LiteLLM_Params(model="dashscope/qwen3-max", api_key="sk-fake"), + model_info=ModelInfo( + id=model_name, + max_output_tokens=1000, + tiered_pricing=[ + {"input_cost_per_token": cost_per_token, "output_cost_per_token": cost_per_token, "range": [0, 128000]} + ], + ), + ).model_dump() + + +def test_google_route_reserves_on_the_model_its_path_param_selects(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + router: Final = Router(model_list=[_priced_group("cheap-group", 1e-06), _priced_group("premium-group", 1e-03)]) + route: Final = "/v1beta/models/premium-group:generateContent" + body: Final = {"model": "cheap-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10} + + url_model_cost: Final = estimate_request_max_cost(request_body=body, route=route, llm_router=router) + path_param_cost: Final = estimate_request_max_cost( + request_body=body, route=route, llm_router=router, path_params={"model_name": "cheap-group"} + ) + cheap_cost: Final = estimate_request_max_cost( + request_body=body, route="/v1beta/models/cheap-group:generateContent", llm_router=router + ) + + assert url_model_cost is not None and cheap_cost is not None + assert path_param_cost == cheap_cost + assert math.isclose(url_model_cost, cheap_cost * 1000) + + @pytest.mark.asyncio @pytest.mark.parametrize( "expiry_offset, expected_max_budget", From f889b2b587d522944e2c5bd80e10112a890539eb Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 11 Nov 2026 12:36:02 -0800 Subject: [PATCH 107/189] test(integration): fix the baseline accounting and series cap flakes (#51096) * test(integration): run baseline accounting store tests in a private schema The accounting workers share their database with two gateway proxies, and every spend-log tick on those proxies projects any dirty baseline comparison it finds. When one of them published the test's scope between its last append and its own project() call, the test saw "unchanged" instead of "published". The module fixture now copies the twelve tables the store touches into a fresh schema and points DATABASE_URL at it for the module, so the gateway proxies, which only read public, can no longer publish these rows. * test(integration): wait for H9's own prometheus increments before it returns H9 stopped as soon as both workers' sample files already counted `other`, so its own chat's increment could still be in flight and land inside the next cell's window, which then saw two requests instead of one. H9 now takes a before scrape, counts the chats it sends, and finishes with the same _overflowed wait every other TestCapped cell uses. --- .../test_prometheus_series_cap.py | 23 ++++++----- .../spend/test_baseline_accounting.py | 38 ++++++++++++++++++- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/tests/integration/observability/test_prometheus_series_cap.py b/tests/integration/observability/test_prometheus_series_cap.py index ecdb6dabfc5..68c96a257b9 100644 --- a/tests/integration/observability/test_prometheus_series_cap.py +++ b/tests/integration/observability/test_prometheus_series_cap.py @@ -7,6 +7,7 @@ from __future__ import annotations import time from collections.abc import Iterator, Sequence +from itertools import count from typing import Final import anthropic @@ -221,19 +222,21 @@ class TestCapped: """H9: fresh connections reach both workers, and each worker's own sample file names only the warm aliases while counting the fourth key on `other`, since the admitted sets live in the shared directory.""" extra: Final = capped.key("h9") + before: Final = scrape(capped.gateway) + sent: Final = count(1) - def send_on_a_fresh_connection() -> tuple[WorkerSamples, ...]: + def send_on_a_fresh_connection() -> tuple[int, tuple[WorkerSamples, ...]]: assert capped.chat(extra, Call.new()).status_code == 200 - return worker_samples(capped.prom_dir, REQUESTS) + return next(sent), worker_samples(capped.prom_dir, REQUESTS) - workers: Final = eventually( - send_on_a_fresh_connection, - lambda found: ( - sum(1 for worker in found if worker.overflow > 0) >= 2 - or any(extra.alias in worker.aliases for worker in found) - ), - seconds=90, - ) + def reached_both_workers(attempt: tuple[int, tuple[WorkerSamples, ...]]) -> bool: + _, found = attempt + return sum(1 for worker in found if worker.overflow > 0) >= 2 or any( + extra.alias in worker.aliases for worker in found + ) + + chats, workers = eventually(send_on_a_fresh_connection, reached_both_workers, seconds=90) + _overflowed(capped, extra, before, chats) assert all(extra.alias not in worker.aliases for worker in workers), workers assert sum(1 for worker in workers if worker.overflow > 0) >= 2, workers assert frozenset().union(*(worker.aliases for worker in workers)) == capped.warm_aliases, workers diff --git a/tests/integration/spend/test_baseline_accounting.py b/tests/integration/spend/test_baseline_accounting.py index 60c5c32511a..d384d509fd7 100644 --- a/tests/integration/spend/test_baseline_accounting.py +++ b/tests/integration/spend/test_baseline_accounting.py @@ -1,11 +1,13 @@ import asyncio import json +import os import uuid from collections.abc import AsyncIterator, Callable from contextlib import asynccontextmanager from dataclasses import replace from datetime import datetime, timedelta, timezone from typing import Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit import pytest import pytest_asyncio @@ -29,10 +31,42 @@ from tests.integration._support.prisma_clients import connected_prisma pytestmark = pytest.mark.asyncio(loop_scope="session") +_ISOLATED_TABLES: Final = ( + "LiteLLM_AutoRouterBaselineComparison", + "LiteLLM_AutoRouterBaselineObservation", + "LiteLLM_AutoRouterDailySpend", + "LiteLLM_AutoRouterSession", + "LiteLLM_AutoRouterUserSession", + "LiteLLM_DailyAgentSpend", + "LiteLLM_DailyEndUserSpend", + "LiteLLM_DailyOrganizationSpend", + "LiteLLM_DailyTagSpend", + "LiteLLM_DailyTeamSpend", + "LiteLLM_DailyUserSpend", + "LiteLLM_SpendLogs", +) + + +def _with_schema(database_url: str, schema: str) -> str: + parts: Final = urlsplit(database_url) + kept: Final = tuple((name, value) for name, value in parse_qsl(parts.query) if name != "schema") + return urlunsplit(parts._replace(query=urlencode((*kept, ("schema", schema))))) + + @pytest_asyncio.fixture(scope="module", loop_scope="session") async def db() -> AsyncIterator[Prisma]: - async with connected_prisma() as client: - yield client + schema: Final = f"baseline_accounting_{uuid.uuid4().hex}" + async with connected_prisma() as admin: + await admin.execute_raw(f'CREATE SCHEMA "{schema}"') + try: + for table in _ISOLATED_TABLES: + await admin.execute_raw(f'CREATE TABLE "{schema}"."{table}" (LIKE "{table}" INCLUDING ALL)') + with pytest.MonkeyPatch.context() as environment: + environment.setenv("DATABASE_URL", _with_schema(os.environ["DATABASE_URL"], schema)) + async with connected_prisma() as client: + yield client + finally: + await admin.execute_raw(f'DROP SCHEMA "{schema}" CASCADE') @asynccontextmanager From f52f26d9c636038c6e3609cc7d17cf625d434c3e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 11 Nov 2026 12:37:28 -0800 Subject: [PATCH 108/189] fix(proxy): let /team/delete transactions wait for a pooled connection (#51099) The team-model and team-key transactions in /team/delete used Prisma's 2s default max_wait, so a busy writer pool failed the delete with P2028 while every plain query in the same request waited up to the 60s pool timeout. Both now get the 30s team-delete budget for max_wait, and the key transaction gets it for timeout too. --- .../model_management_endpoints.py | 7 +- litellm/proxy/utils.py | 4 +- .../test_team_delete_large_membership.py | 92 +++++++++++++++++++ 3 files changed, 100 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 7846922bae2..0b4f266f838 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -295,7 +295,9 @@ class _TxModelTables(Protocol): @runtime_checkable class _TransactionFactory(Protocol): - def __call__(self, *, timeout: datetime.timedelta = ...) -> AbstractAsyncContextManager[_TxModelTables]: ... + def __call__( + self, *, timeout: datetime.timedelta = ..., max_wait: datetime.timedelta = ... + ) -> AbstractAsyncContextManager[_TxModelTables]: ... class _ModelTransactionClient(LiteLLMBaseModel): @@ -2449,7 +2451,8 @@ async def delete_team_models( """ deleted_model_ids: Final[list[str]] = [] transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db) - async with transaction_client.tx(timeout=datetime.timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx: + tx_budget: Final = datetime.timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS) + async with transaction_client.tx(max_wait=tx_budget, timeout=tx_budget) as tx: for team_id in team_ids: rows = await _get_team_deployments(team_id, prisma_client, table=tx.litellm_proxymodeltable) model_ids = [row.model_id for row in rows] diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bf00bdfb5fb..51f1c8a614d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -73,6 +73,7 @@ from litellm.constants import ( SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS, SPEND_LOG_WRITE_BATCH_MAX_BYTES, SPEND_LOG_WRITE_BATCH_MAX_ROWS, + TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS, VLLM_CHOICE_FIELDS, ) from litellm.cost_calculator import BaseTokenUsageProcessor @@ -5361,6 +5362,7 @@ _deprecated_key_cache: Final[LimitedSizeOrderedDict] = LimitedSizeOrderedDict(ma _DEPRECATED_KEY_CACHE_TTL_SECONDS: Final = 60 _PRISMA_DEFAULT_TX_TIMEOUT: Final = timedelta(seconds=5) _PRISMA_DEFAULT_TX_MAX_WAIT: Final = timedelta(seconds=2) +_TEAM_DELETE_TX_BUDGET: Final = timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS) _SPEND_LOG_READ_STATEMENT_TIMEOUT: Final = timedelta(seconds=SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS) _STATEMENT_TIMEOUT_TRANSACTION_GRACE: Final = timedelta(seconds=5) @@ -6983,7 +6985,7 @@ class PrismaClient: return {"deleted_teams": team_id_list} elif table_name == "key" and team_id_list is not None and isinstance(team_id_list, list): # admin only endpoint -> `/team/delete` - async with self.tx() as transaction: + async with self.tx(max_wait=_TEAM_DELETE_TX_BUDGET, timeout=_TEAM_DELETE_TX_BUDGET) as transaction: _ = await transaction.query_raw(_LOCK_KEYS_BY_TEAM_SQL, team_id_list) _ = await transaction.litellm_verificationtoken.delete_many(where={"team_id": {"in": team_id_list}}) except Exception as e: diff --git a/tests/integration/management/test_team_delete_large_membership.py b/tests/integration/management/test_team_delete_large_membership.py index 2a8508ff3c9..361cbf037a3 100644 --- a/tests/integration/management/test_team_delete_large_membership.py +++ b/tests/integration/management/test_team_delete_large_membership.py @@ -21,6 +21,7 @@ import httpx import psycopg import pytest import yaml +from psycopg.rows import TupleRow from pydantic import JsonValue from redis import Redis @@ -55,7 +56,11 @@ FROM generate_series(1, %s::int) AS n """ TAKE_TEAM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))" LOCK_MODEL_ROW_SQL: Final = 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s FOR UPDATE' +LOCK_TEAM_KEY_ROWS_SQL: Final = 'SELECT token FROM "LiteLLM_VerificationToken" WHERE team_id = %s FOR UPDATE' +LOCK_BUDGET_ROW_SQL: Final = 'SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_id = %s FOR UPDATE' LOCK_HELD_PAST_DEFAULT_TRANSACTION_SECONDS: Final = 7 +POOL_HELD_PAST_DELETE_RETRIES_SECONDS: Final = 12 +SATURATED_POOL_LIMIT: Final = 2 OLDEST_TRANSACTION_BLOCKED_BY_SQL: Final = """ SELECT coalesce(max(extract(epoch FROM clock_timestamp() - xact_start)), 0)::float AS age FROM pg_stat_activity @@ -146,6 +151,16 @@ def _waiters_on_lock_held_by(backend_pid: int) -> int: return waiting +def _sessions_blocked_by(backend_pid: int) -> int: + rows: Final = read_rows( + "SELECT count(*)::int AS blocked FROM pg_stat_activity WHERE %s::int = ANY(pg_blocking_pids(pid))", + (str(backend_pid),), + ) + blocked: Final = rows[0]["blocked"] + assert isinstance(blocked, int) + return blocked + + def _oldest_transaction_blocked_by(backend_pid: int) -> float: rows: Final = read_rows(OLDEST_TRANSACTION_BLOCKED_BY_SQL, (str(backend_pid),)) age: Final = rows[0]["age"] @@ -468,6 +483,83 @@ def test_delete_completes_after_waiting_on_the_team_lock_past_the_default_transa assert len(_tombstones(team)) == 1, _tombstones(team) +def _park_on_team_key_rows(holder: psycopg.Connection[TupleRow], team_id: str) -> None: + _ = holder.execute(LOCK_TEAM_KEY_ROWS_SQL, (team_id,)) + + +def _park_on_deleted_key_records(holder: psycopg.Connection[TupleRow], _team_id: str) -> None: + _ = holder.execute('LOCK TABLE "LiteLLM_DeletedVerificationToken" IN SHARE MODE') + + +def _requests_in_flight_including_this_one(proxy: Gateway) -> JsonValue: + return proxy.get("/health/backlog")["in_flight_requests"] + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize( + "park", + [ + pytest.param(_park_on_team_key_rows, id="next-transaction-is-the-team-model-delete"), + pytest.param(_park_on_deleted_key_records, id="next-transaction-is-the-team-key-delete"), + ], +) +def test_delete_waits_for_a_pooled_connection_when_every_one_is_busy( + gateway: Gateway, + tmp_path: Path, + park: Callable[[psycopg.Connection[TupleRow], str], None], +) -> None: + database_url: Final = os.environ["DATABASE_URL"] + with ( + gateway.scenario() as scenario, + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": database_url}, + config=_config_with_pool_limit(tmp_path, SATURATED_POOL_LIMIT), + remove_environment=("DATABASE_URL_READ_REPLICA",), + ) as owned, + ThreadPoolExecutor(max_workers=3) as pool, + ): + team: Final = string_value( + owned.gateway.post("/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}"})["team_id"] + ) + scenario.cleanups.callback(_remove_team_by_sql, team) + scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,)) + keys: Final = _seeded_team_keys(team, 3) + budget: Final = f"integration-{uuid.uuid4().hex}" + owned.gateway.post("/budget/new", {"budget_id": budget, "max_budget": 100}) + scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', (budget,)) + update_budget: Final = {"budget_id": budget, "max_budget": 1} + + with psycopg.connect(database_url) as budget_holder: + budget_holder.execute(LOCK_BUDGET_ROW_SQL, (budget,)) + first_connection_taker: Final = _post_in_background(pool, owned.gateway, "/budget/update", update_budget) + eventually(lambda: _sessions_blocked_by(budget_holder.info.backend_pid), lambda count: count == 1) + with psycopg.connect(database_url) as parking: + park(parking, team) + pending_delete: Final = _post_in_background(pool, owned.gateway, "/team/delete", {"team_ids": [team]}) + eventually(lambda: _sessions_blocked_by(parking.info.backend_pid), lambda count: count == 1) + second_connection_taker: Final = _post_in_background( + pool, owned.gateway, "/budget/update", update_budget + ) + eventually(lambda: _requests_in_flight_including_this_one(owned.gateway), lambda count: count == 4) + eventually( + pending_delete.done, + lambda done: done, + seconds=POOL_HELD_PAST_DELETE_RETRIES_SECONDS, + return_last_on_timeout=True, + ) + + response: Final = pending_delete.result(timeout=60) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + takers: Final = (first_connection_taker, second_connection_taker) + assert [taker.result(timeout=60).status_code for taker in takers] == [200, 200] + assert _team_rows(team) == [] + assert [_live_token(_hashed(key)) for key in keys] == [[], [], []] + assert len(_tombstones(team)) == 1, _tombstones(team) + + def test_deleting_two_teams_sharing_a_member_in_one_call_clears_both_from_the_member(gateway: Gateway) -> None: with gateway.scenario() as scenario: user: Final = scenario.user() From a1f3dcd01bc2ca586780312ae9a06826f2a82c46 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:59:19 -0800 Subject: [PATCH 109/189] fix(completion)!: answer 400 for a chat message without a role (#51124) * fix(completion): answer 400 for a chat message without a role A message with no role was filled with role assistant and sent upstream, so the caller's text reached the model as its own earlier reply and the request returned 200. Now a missing, null, or empty role answers 400 with OpenAI's wording, naming the message index. A role-less turn that carries tool_calls is still sent as assistant, since only an assistant writes tool calls. * fix(completion): treat a role-less message as the caller's error The 400 now raises MalformedRequestFieldError with OpenAI's param and code and the provider name, so it no longer counts toward the deployment's failure rate, cools it down, or fires an llm_exceptions alert * fix(completion): keep the assistant fill for a role-less function_call turn * refactor(completion): type the role-less call turn check and the request kwargs Read role, tool_calls and function_call off the cleaned message copy, and validate completion's kwargs into a typed mapping before naming the provider on the 400 * fix(completion): keep the alert and failure-count exclusions to the role error Raise MissingMessageRoleError, a MalformedRequestFieldError subclass, so a malformed tools 400 alerts and counts as on main, and fill assistant whenever a role-less turn has a tool_calls or function_call key * fix(completion): keep assistant-only role-less turns and skip SDK and PagerDuty alerts A role-less turn carrying any assistant-only key, null values included, is still filled as assistant, so a dumped assistant message or a reasoning turn keeps working. The Router's alerting_config alert and the PagerDuty failure count skip the role error, like the proxy alert already does. * fix(utils): drop the string code from the missing-role 400 so the Errors tab counts it as 400 The spend log stored code missing_required_parameter as the error code, which the Usage page Errors tab cannot parse as a status, so every role-less request landed under no status. The malformed tools 400 leaves code out the same way. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../pagerduty/pagerduty.py | 3 + litellm/__init__.py | 1 + litellm/exceptions.py | 4 ++ litellm/main.py | 4 +- litellm/proxy/utils.py | 3 + litellm/router_utils/cooldown_handlers.py | 8 ++- litellm/router_utils/handle_error.py | 4 ++ litellm/utils.py | 42 +++++++++---- .../test_pagerduty_alerting.py | 45 ++++++++++++++ ..._utils_model_creation_and_error_logging.py | 13 ++++ .../router_utils/test_cooldown_handlers.py | 44 ++++++++++++++ .../test_fallback_event_handlers.py | 19 ++++++ tests/unit/router_utils/test_handle_error.py | 33 +++++++++- tests/unit/test_utils.py | 60 ++++++++++++++++++- 14 files changed, 264 insertions(+), 19 deletions(-) diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py b/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py index f920aa7ac13..a0c896b7e05 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py @@ -17,6 +17,7 @@ from typing import List, Optional, Union from litellm._logging import verbose_logger from litellm.caching import DualCache +from litellm.exceptions import MissingMessageRoleError from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -86,6 +87,8 @@ class PagerDutyAlerting(SlackAlerting): Record a failure event. Only send an alert to PagerDuty if the configured *failure* threshold is exceeded in the specified window. """ + if isinstance(kwargs.get("exception"), MissingMessageRoleError): + return now = datetime.now(timezone.utc) standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( "standard_logging_object" diff --git a/litellm/__init__.py b/litellm/__init__.py index 8a73f3447b8..bfe0e2b5f3e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1457,6 +1457,7 @@ from .exceptions import ( BadRequestError, ImageFetchError, MalformedRequestFieldError as MalformedRequestFieldError, + MissingMessageRoleError as MissingMessageRoleError, VectorStoreSearchError, NotFoundError, PermissionDeniedError, diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 56a5f136b62..f8c6a50a589 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -369,6 +369,10 @@ class MalformedRequestFieldError(BadRequestError): pass +class MissingMessageRoleError(MalformedRequestFieldError): + pass + + class UnsupportedEndpointError(BadRequestError): pass diff --git a/litellm/main.py b/litellm/main.py index 2d6cd18162b..df8e8f29483 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5660,7 +5660,9 @@ def completion( if model is None: raise ValueError("model param not passed in.") # validate messages - messages = validate_and_fix_openai_messages(messages=messages) + messages = validate_and_fix_openai_messages( + messages=messages, model=model, request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs) + ) tools = validate_and_fix_openai_tools(tools=tools) # validate tool_choice requested_tool_choice: Final = cast( # cast-ok: widening an untyped dict to object is always sound diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 51f1c8a614d..eac2700551f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -151,6 +151,7 @@ from litellm.caching.dual_cache import LimitedSizeOrderedDict from litellm.caching.in_memory_cache import InMemoryCache from litellm.exceptions import ( GuardrailRaisedException, + MissingMessageRoleError, RejectedRequestError, SensitiveDataRouteException, ) @@ -671,6 +672,8 @@ class _BudgetAlertEmailer(Protocol): def _is_client_error_exception(exc: Exception) -> bool: + if isinstance(exc, MissingMessageRoleError): + return True if isinstance(exc, HTTPException): return exc.status_code < 500 if isinstance(exc, ProxyException): diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 190ef43a3e3..22bec9e1ead 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -24,7 +24,7 @@ from litellm.constants import ( INTERNAL_CALL_ORIGIN_METADATA_KEY, SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD, ) -from litellm.exceptions import MidStreamFallbackError +from litellm.exceptions import MidStreamFallbackError, MissingMessageRoleError from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials from litellm.router_deprecated import renamed_router_param @@ -83,8 +83,10 @@ def mark_deployment_selection_failure(exception: BaseException) -> None: def is_failure_outside_selected_deployment(exception: BaseException | None) -> bool: - return is_advisor_orchestration_failure(exception) or bool( - getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False) + return ( + is_advisor_orchestration_failure(exception) + or isinstance(exception, MissingMessageRoleError) + or bool(getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False)) ) diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index a1e6d2b2cad..319959b5cfd 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final from litellm._logging import redact_secrets, verbose_router_logger from litellm.constants import MAX_EXCEPTION_MESSAGE_LENGTH +from litellm.exceptions import MissingMessageRoleError from litellm.router_deprecated import renamed_router_param from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments_with_debug_info, @@ -46,6 +47,9 @@ async def send_llm_exception_alert( if slack_alerting_logger is None: return + if isinstance(original_exception, MissingMessageRoleError): + return + if "proxy_server_request" in request_kwargs: # Do not send any alert if it's a request from litellm proxy server request # the proxy is already instrumented to send LLM API call failures diff --git a/litellm/utils.py b/litellm/utils.py index 7c73d1cc4d5..7fa9d32ae64 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -534,6 +534,7 @@ from .exceptions import ( ContentPolicyViolationError, ContextWindowExceededError, MalformedRequestFieldError, + MissingMessageRoleError, ModelNotMappedError, NotFoundError, OpenAIError, @@ -9496,16 +9497,22 @@ def convert_list_message_to_dict( return [dict(cleanup_none_field_in_message(message=_as_message_value(message))) for message in messages] -def validate_and_fix_openai_messages(messages: list): +_ASSISTANT_ONLY_MESSAGE_KEYS: Final = frozenset( + {"tool_calls", "function_call", "reasoning_content", "reasoning_items", "thinking_blocks", "refusal", "audio"} +) + + +def validate_and_fix_openai_messages( + messages: list, model: str = "", request_kwargs: Mapping[str, object] | None = None +): """ Ensures all messages are valid OpenAI chat completion messages. - Handles missing role for assistant messages. + Only an assistant turn carries the keys in _ASSISTANT_ONLY_MESSAGE_KEYS, so one sent without a role but with + any of them, even set to null, gets "assistant" (issue #8661). """ new_messages: Final = [] for message in messages: - if not message.get("role"): - message["role"] = "assistant" if message.get("tool_calls"): message["tool_calls"] = jsonify_tools(tools=message["tool_calls"]) @@ -9516,8 +9523,10 @@ def validate_and_fix_openai_messages(messages: list): ), ) cleaned_message = cleanup_none_field_in_message(message=convert_msg_to_dict) + if not cleaned_message.get("role") and not _ASSISTANT_ONLY_MESSAGE_KEYS.isdisjoint(convert_msg_to_dict): + cleaned_message["role"] = "assistant" new_messages.append(cleaned_message) - return validate_chat_completion_user_messages(messages=new_messages) + return validate_chat_completion_user_messages(messages=new_messages, model=model, request_kwargs=request_kwargs) def validate_and_fix_openai_tools(tools: list | None) -> list[dict] | None: @@ -9570,21 +9579,36 @@ def cleanup_none_field_in_message(message: AllMessageValues): return {k: v for k, v in new_message.items() if v is not None} -def validate_chat_completion_user_messages(messages: list[AllMessageValues]): +def validate_chat_completion_user_messages( + messages: list[AllMessageValues], model: str = "", request_kwargs: Mapping[str, object] | None = None +): """ - Ensures all user messages are valid OpenAI chat completion messages. + Ensures every message has a role and all user messages are valid OpenAI chat completion messages. Args: messages: List of message dictionaries + model: Model named on the error raised for a message without a role + request_kwargs: The request's provider settings, read to name the provider on that error message_content_type: Type to validate content against Returns: List[dict]: The validated messages Raises: + MissingMessageRoleError: If a message has no role ValueError: If any message is invalid """ for idx, m in enumerate(messages): + if "role" not in m or not m["role"]: + raise MissingMessageRoleError( + message=f"Missing required parameter: 'messages[{idx}].role'.", + model=model, + llm_provider=_deployment_provider({**(request_kwargs or {}), "model": model}), + body={ + "type": "invalid_request_error", + "param": f"messages[{idx}].role", + }, + ) try: if m["role"] == "user": user_content = m.get("content") @@ -9597,10 +9621,6 @@ def validate_chat_completion_user_messages(messages: list[AllMessageValues]): if item.get("type") not in ValidUserMessageContentTypes: raise Exception(f"invalid content type={item.get('type')}") except Exception as e: - if isinstance(e, KeyError): - raise Exception( - f"Invalid message at index {idx}. Please ensure all messages are valid OpenAI chat completion messages." - ) if "invalid content type" in str(e): raise Exception( f"Invalid user message at index {idx}. Please ensure all user messages are valid OpenAI chat completion messages." diff --git a/tests/unit/enterprise/enterprise_callbacks/test_pagerduty_alerting.py b/tests/unit/enterprise/enterprise_callbacks/test_pagerduty_alerting.py index c26966eeb38..84b4dc0a93e 100644 --- a/tests/unit/enterprise/enterprise_callbacks/test_pagerduty_alerting.py +++ b/tests/unit/enterprise/enterprise_callbacks/test_pagerduty_alerting.py @@ -74,6 +74,51 @@ async def test_failure_alerts_follow_the_configured_threshold( assert recent_errors[0]["error_llm_provider"] == "openai" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("exception", "request_count"), + [ + ( + litellm.MissingMessageRoleError( + message="Missing required parameter: 'messages[0].role'.", model="gpt-5.6", llm_provider="openai" + ), + 0, + ), + ( + litellm.MalformedRequestFieldError( + message="tools: Input should be a valid list", model="gpt-5.6", llm_provider="openai" + ), + 1, + ), + ], +) +async def test_only_a_role_less_request_stays_out_of_the_failure_count( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + exception: Exception, + request_count: int, +) -> None: + route: Final = respx_mock.post(PAGERDUTY_URL).mock(return_value=httpx.Response(202)) + logger: Final = _pagerduty_logger(monkeypatch) + payload: Final = cast( + StandardLoggingPayload, + { + "error_information": { + "error_class": type(exception).__name__, + "error_code": "400", + "llm_provider": "openai", + }, + "metadata": {}, + }, + ) + + await logger.async_log_failure_event( + {"standard_logging_object": payload, "exception": exception}, None, None, None + ) + + assert route.call_count == request_count + + @pytest.mark.asyncio async def test_hanging_requests_alert_at_the_default_threshold( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 867259a9b3b..5c399f96e83 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -2007,6 +2007,19 @@ class TestPostCallFailureHookLLMExceptionAlerting: async def test_http_exception_does_not_alert(self): assert (await self._alerted(HTTPException(status_code=400, detail="blocked"))).called is False + @pytest.mark.asyncio + async def test_role_less_message_does_not_alert(self): + with pytest.raises(litellm.MalformedRequestFieldError) as exc_info: + litellm.completion(model="openai/gpt-5.6", messages=[{"content": "hi"}], mock_response="ok") + assert (await self._alerted(exc_info.value)).called is False + + @pytest.mark.asyncio + async def test_malformed_tools_field_still_alerts(self): + exc = litellm.MalformedRequestFieldError( + message="tools: Input should be a valid list", model="openai/gpt-5.6", llm_provider="openai" + ) + assert (await self._alerted(exc)).called is True + @pytest.mark.asyncio async def test_genuine_llm_api_error_still_alerts(self): assert (await self._alerted(Exception("upstream 503"))).called is True diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 3a61ca48f26..cc3165a036c 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -3,6 +3,7 @@ import copy import datetime import importlib import itertools +import json import random import time import uuid @@ -37,6 +38,7 @@ from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments, cast_exception_status_to_int, get_cooldown_deployments, + is_failure_outside_selected_deployment, is_response_lookup_not_found, mark_advisor_orchestration_failure, set_cooldown_deployments, @@ -1939,6 +1941,48 @@ async def test_should_cooldown_deployment(testing_litellm_router): ) +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +@pytest.mark.asyncio +async def test_role_less_requests_do_not_count_toward_the_deployments_failure_rate(): + """A caller's role-less message is not the deployment failing, so it must not push a healthy one into cooldown.""" + litellm_params: Final = {"model": "house-model", "custom_llm_provider": "openai", "mock_response": "ok"} + router: Final = Router( + model_list=[ + {"model_name": "house", "litellm_params": litellm_params, "model_info": {"id": "d1"}}, + {"model_name": "house", "litellm_params": litellm_params, "model_info": {"id": "d2"}}, + ] + ) + for _ in range(6): + with pytest.raises(litellm.MalformedRequestFieldError): + await router.acompletion(model="d1", messages=json.loads('[{"content": "hi"}]')) + await asyncio.sleep(1) + + assert _should_cooldown_deployment(router, "d1", 500, Exception("Test")) is False + + +@pytest.mark.parametrize( + ("exception", "outside_deployment"), + [ + ( + litellm.MissingMessageRoleError( + message="Missing required parameter: 'messages[0].role'.", model="m", llm_provider="openai" + ), + True, + ), + ( + litellm.MalformedRequestFieldError( + message="tools: Input should be a valid list", model="m", llm_provider="openai" + ), + False, + ), + ], +) +def test_only_a_role_less_request_is_a_failure_outside_the_deployment( + exception: Exception, outside_deployment: bool +) -> None: + assert is_failure_outside_selected_deployment(exception) is outside_deployment + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_should_cooldown_deployment_allowed_fails_set_on_router(): diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 9c034b77fe7..47d58a3b869 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -2503,6 +2503,25 @@ def test_is_bad_request_without_fallback_only_stops_on_a_real_400_when_the_flag_ assert is_bad_request_without_fallback(exception, fallback_on_unsupported_params=True) is False +@pytest.mark.asyncio +async def test_a_role_less_request_is_not_retried_on_the_fallback_group() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "house", + "litellm_params": {"model": "house-model", "custom_llm_provider": "openai", "mock_response": "ok"}, + }, + {"model_name": "backup", "litellm_params": {"model": "openai/gpt-5.6", "mock_response": "ok"}}, + ], + fallbacks=[{"house": ["backup"]}], + fallback_on_unsupported_params=False, + ) + + with pytest.raises(litellm.MalformedRequestFieldError, match="Missing required parameter") as exc_info: + await router.acompletion(model="house", messages=json.loads('[{"content": "hi"}]')) + assert "Fallback to backup" not in str(exc_info.value) + + @pytest.mark.asyncio async def test_run_async_fallback_skips_a_target_the_rate_limit_check_rejects() -> None: checked: Final = [] # mutable-ok: records the targets the check saw diff --git a/tests/unit/router_utils/test_handle_error.py b/tests/unit/router_utils/test_handle_error.py index 22bd6e74190..00350cc357c 100644 --- a/tests/unit/router_utils/test_handle_error.py +++ b/tests/unit/router_utils/test_handle_error.py @@ -1,6 +1,6 @@ import asyncio import importlib -from typing import List +from typing import Final, List from unittest.mock import AsyncMock, MagicMock import pytest @@ -40,6 +40,37 @@ async def test_send_llm_exception_alert_success(): slack_alerting_logger.send_alert.assert_called_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("exception", "alerts_sent"), + [ + ( + litellm.MissingMessageRoleError( + message="Missing required parameter: 'messages[0].role'.", model="gpt-5.6", llm_provider="openai" + ), + 0, + ), + ( + litellm.MalformedRequestFieldError( + message="tools: Input should be a valid list", model="gpt-5.6", llm_provider="openai" + ), + 1, + ), + ], +) +async def test_send_llm_exception_alert_skips_only_a_role_less_request(exception: Exception, alerts_sent: int): + slack_alerting_logger: Final = AsyncMock() + ctx: Final = MagicMock() + ctx.observers.return_value = RoutingObservers(slack_alerting_logger=slack_alerting_logger) + request_kwargs: Final = {"model": "gpt-5.6", "messages": [{"content": "Hello"}]} + + from litellm.router_utils.handle_error import send_llm_exception_alert + + await send_llm_exception_alert(ctx, request_kwargs, "Traceback", exception) + + assert slack_alerting_logger.send_alert.call_count == alerts_sent + + @pytest.mark.asyncio async def test_send_llm_exception_alert_no_logger(): """ diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6fece4d8c66..a1400f53ef8 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -57,6 +57,8 @@ from litellm.litellm_core_utils.thread_pool_executor import executor as logging_ from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, headers from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.proxy.db.request_error_tracking import status_code_from_metadata from litellm.proxy.utils import is_valid_api_key from litellm.responses.main import aresponses, responses from litellm.types.caching import CachingSupportedCallTypes @@ -119,6 +121,7 @@ from litellm.utils import ( trim_messages, validate_environment, validate_chat_completion_tool_choice, + validate_and_fix_openai_messages, ) from litellm.litellm_core_utils.model_cost_table import publish @@ -8939,6 +8942,57 @@ def test_validate_tool_choice_without_model_is_still_a_400(): assert exc_info.value.model == "" +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +@pytest.mark.parametrize( + ("messages", "index"), + [ + ([{"content": "Say hi in two words."}], 0), + ([{"role": "user", "content": "hi"}, {"role": None, "content": "hello"}], 1), + ([{"role": "user", "content": "hi"}, {"role": "", "content": "hello"}], 1), + ], +) +def test_completion_message_without_role_is_a_400(messages: list[dict[str, str | None]], index: int): + """A message with no role is the caller's mistake, rejected before any provider sees it, never sent as assistant.""" + with pytest.raises( + litellm.MissingMessageRoleError, match=re.escape(f"Missing required parameter: 'messages[{index}].role'.") + ) as exc_info: + litellm.completion(model=MODEL, messages=messages, mock_response="Hello there!") + assert exc_info.value.status_code == 400 + assert exc_info.value.model == MODEL + assert exc_info.value.llm_provider == "anthropic" + assert exc_info.value.param == f"messages[{index}].role" + error_information: Final = StandardLoggingPayloadSetup.get_error_information(exc_info.value) + assert status_code_from_metadata({"error_information": error_information}) == 400 + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +@pytest.mark.parametrize( + "call_turn", + [ + { + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + {"content": None, "function_call": {"name": "get_weather", "arguments": "{}"}}, + {"content": "ok", "tool_calls": []}, + {"content": "ok", "tool_calls": None}, + {"content": "ok", "function_call": None}, + {"content": "Sunny.", "reasoning_content": "The user wants the weather."}, + {"content": "Sunny.", "reasoning_items": []}, + {"content": "Sunny.", "thinking_blocks": [{"type": "thinking", "thinking": "Weather.", "signature": "sig"}]}, + {"content": None, "refusal": "I can't help with that."}, + {"content": None, "audio": {"id": "audio_1"}}, + ], +) +def test_completion_role_less_assistant_turn_is_sent_as_assistant(call_turn: dict[str, object]): + """Only an assistant turn carries these keys, so one replayed without a role still goes upstream as assistant.""" + messages: Final = [{"role": "user", "content": "Weather in Paris?"}, call_turn] + + assert validate_and_fix_openai_messages(messages)[1]["role"] == "assistant" + + @pytest.fixture() def _vcr_outcome_gate_local_testing(request, vcr): install_live_call_probe(request, vcr) @@ -10731,12 +10785,12 @@ def test_dict_to_response_format_helper(): def test_validate_user_messages_invalid_content_type(): from litellm.utils import validate_chat_completion_user_messages - messages = [{"content": [{"type": "invalid_type", "text": "Hello"}]}] + messages = [{"role": "user", "content": [{"type": "invalid_type", "text": "Hello"}]}] - with pytest.raises(Exception, match="Please ensure all messages are valid OpenAI chat completion") as e: + with pytest.raises(Exception, match="Please ensure all user messages are valid OpenAI chat completion") as e: validate_chat_completion_user_messages(messages) - assert "Invalid message" in str(e) + assert "Invalid user message at index 0" in str(e) print(e) From b63656a5de6a95487d2f4826e2cbe4cccca0d02e Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:39:19 -0800 Subject: [PATCH 110/189] fix(proxy): stream the guardrail block response when the block comes after an SSE keepalive ping (#51114) * fix(proxy): stream the guardrail block response when the block lands after an SSE keepalive ping * docs(proxy): keep the keepalive docstring's reasons together * fix(proxy): stream the /v1/messages block after a keepalive ping and always end a late block stream A passthrough guardrail block on a streamed /v1/messages request that landed after the first keepalive ping still ended as an api_error event. The route now hands its block stream to the keepalive relay, as chat and text completions do. If a route's block stream itself fails after a ping, the relay sends the usual late failure frame instead of raising, so the stream always ends. The passthrough tests now drive the real routes with a real guardrail. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../proxy/anthropic_endpoints/endpoints.py | 56 +++-- litellm/proxy/common_request_processing.py | 26 ++- litellm/proxy/proxy_server.py | 162 ++++++++------- litellm/utils.py | 1 + .../anthropic_endpoints/test_endpoints.py | 81 ++++++-- .../proxy/test_common_request_processing.py | 63 ++++++ ...t_modify_response_streaming_passthrough.py | 196 +++++++++++++++++- 7 files changed, 457 insertions(+), 128 deletions(-) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1688992f6fa..636ba987ee8 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -7,7 +7,7 @@ from types import MappingProxyType from typing import Final from fastapi import APIRouter, Depends, HTTPException, Request, Response, status -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, StreamingResponse from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -55,6 +55,7 @@ from litellm.proxy.litellm_pre_call_utils import ( forwarded_anthropic_oauth_token, should_auto_drop_params_for_agentic_cli, ) +from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.types.utils import TokenCountResponse @@ -126,6 +127,32 @@ def _strip_total_tokens_from_anthropic_response(response: object) -> None: usage.pop("total_tokens", None) +async def _record_passthrough_block( + e: ModifyResponseException, proxy_logging_obj: ProxyLogging, user_api_key_dict: UserAPIKeyAuth +) -> None: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=e.request_data, + ) + + +async def _stream_passthrough_block( + e: ModifyResponseException, + proxy_logging_obj: ProxyLogging, + user_api_key_dict: UserAPIKeyAuth, + headers: Mapping[str, str], +) -> StreamingResponse | JSONResponse: + await _record_passthrough_block(e, proxy_logging_obj, user_api_key_dict) + block_chunks: Final = AnthropicMessagesHandler().build_block_sse_chunks(e) + + async def _blocked_stream() -> AsyncGenerator[str, None]: + for chunk in block_chunks: + yield chunk.decode() + + return await create_response(generator=_blocked_stream(), media_type="text/event-stream", headers=headers) + + @router.post( "/v1/messages", tags=["Anthropic `/v1/messages`"], @@ -174,6 +201,9 @@ async def anthropic_response( user_max_tokens=user_max_tokens, user_api_base=user_api_base, version=version, + on_passthrough_block=lambda block: _stream_passthrough_block( + block, proxy_logging_obj, user_api_key_dict, _NO_HEADERS + ), ) # Optionally strip the non-Anthropic `usage.total_tokens` field # LiteLLM adds internally. Anthropic's official /v1/messages spec @@ -189,30 +219,12 @@ async def anthropic_response( _strip_total_tokens_from_anthropic_response(result) return result except ModifyResponseException as e: - # Guardrail flagged content in passthrough mode - return 200 with violation message - _data: Final = e.request_data - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=e, - request_data=_data, - ) - block_headers: Final = litellm_call_id_headers(base_llm_response_processor.litellm_call_id) or _NO_HEADERS - block_handler: Final = AnthropicMessagesHandler() - if data.get("stream") is True: - block_chunks: Final = block_handler.build_block_sse_chunks(e) - - async def _blocked_stream() -> AsyncGenerator[str, None]: - for chunk in block_chunks: - yield chunk.decode() - - return await create_response( - generator=_blocked_stream(), media_type="text/event-stream", headers=block_headers - ) - + return await _stream_passthrough_block(e, proxy_logging_obj, user_api_key_dict, block_headers) + await _record_passthrough_block(e, proxy_logging_obj, user_api_key_dict) fastapi_response.headers.update(block_headers) - return block_handler.build_block_response(e) + return AnthropicMessagesHandler().build_block_response(e) except AnthropicContextManagementError as e: if e.status_code >= 500: # Server-side polyfill failures hit the failure hook for spend/alert diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index c6981015417..1b004423181 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1662,6 +1662,18 @@ async def _relay_late_response(produced: Response, framing: LateSseFraming) -> A await _aclose_late_response(produced) +async def _late_block_or_failure( + exc: Exception, + on_passthrough_block: Callable[[ModifyResponseException], Awaitable[Response]] | None, +) -> Response | Exception: + if not isinstance(exc, ModifyResponseException) or on_passthrough_block is None: + return exc + try: + return await on_passthrough_block(exc) + except Exception as block_failure: # noqa: BLE001 # the client still needs a frame that ends the stream + return block_failure + + async def _sanitized_late_failure( exc: Exception, on_late_failure: "Callable[[Exception], Awaitable[HTTPException | None]] | None", @@ -1692,6 +1704,7 @@ async def open_sse_before_first_byte( media_type: str = "text/event-stream", on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None, late_framing: LateSseFraming = OPENAI_LATE_SSE_FRAMING, + on_passthrough_block: Callable[[ModifyResponseException], Awaitable[Response]] | None = None, ) -> _LateResponseT | StreamingResponse: """Write SSE keepalive comments while the upstream LLM call is still in flight. @@ -1708,6 +1721,8 @@ async def open_sse_before_first_byte( the first ping reaches the client as an SSE error frame under a 200 rather than as an HTTP error status, and LiteLLM's own `x-litellm-*` response headers are not yet known. Both are why this stays off until an operator sets an interval. + A passthrough guardrail block is not a failure: when the route passes + `on_passthrough_block`, the stream it builds for the block is relayed. """ interval: Final = coerce_keepalive_interval(ping_interval_seconds) if interval is None: @@ -1730,14 +1745,19 @@ async def open_sse_before_first_byte( try: produced: Final = produce_task.result() except Exception as exc: # noqa: BLE001 # the status line is already sent; surface it as a frame + late: Final = await _late_block_or_failure(exc, on_passthrough_block) + if isinstance(late, Response): + async for chunk in _relay_late_response(late, late_framing): + yield chunk + return verbose_proxy_logger.exception( - "request failed after its SSE keepalive had opened the response: %s", exc + "request failed after its SSE keepalive had opened the response: %s", late ) # The caller's own `except` never sees this, so its failure hook # would never fire and the failure would go unaudited. The hook # also gets to sanitize what reaches the client, by returning or # raising a replacement, so its answer decides the frame. - for frame in late_framing.failure(await _sanitized_late_failure(exc, on_late_failure)): + for frame in late_framing.failure(await _sanitized_late_failure(late, on_late_failure)): yield frame.encode() return async for chunk in _relay_late_response(produced, late_framing): @@ -3156,6 +3176,7 @@ class ProxyBaseLLMRequestProcessing: contents: list[object] | None = None, skip_pre_call_logic: bool = False, sse_framing: SseFraming = OPENAI_SSE_FRAMING, + on_passthrough_block: Callable[[ModifyResponseException], Awaitable[Response]] | None = None, ) -> object: """Run the request, sending SSE keepalives while the upstream is still silent. @@ -3207,6 +3228,7 @@ class ProxyBaseLLMRequestProcessing: ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router), on_late_failure=_audit_late_failure, late_framing=ANTHROPIC_LATE_SSE_FRAMING if answers_as_anthropic else sse_framing.late, + on_passthrough_block=on_passthrough_block, ) def total_timeout_for( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6fbe562b5d1..8f5db97e7a3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -210,8 +210,10 @@ from litellm.rust_bridge.catalog import Target from litellm.types.utils import ( PRICING_OVERRIDES_KEY, CallTypes, + Choices, CountContextManagement, LlmProviders, + Message, ModelResponse, ModelResponseStream, TextCompletionResponse, @@ -13263,6 +13265,70 @@ def _blocked_response_usage(original_response: object | None) -> "litellm.Usage" return litellm.Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) +def _passthrough_block_response(e: ModifyResponseException) -> ModelResponse: + return ModelResponse( + model=e.model, + choices=[Choices(message=Message(content=e.message), finish_reason="content_filter")], + usage=_blocked_response_usage(e.original_response), + ) + + +async def _record_passthrough_block(e: ModifyResponseException, user_api_key_dict: UserAPIKeyAuth) -> None: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=e.request_data, + ) + + +async def _stream_chat_passthrough_block( + e: ModifyResponseException, user_api_key_dict: UserAPIKeyAuth, request: Request +) -> StreamingResponse: + # Read before post_call_failure_hook, which pops it from the request data. + logging_obj: Final = e.request_data.get("litellm_logging_obj") + await _record_passthrough_block(e, user_api_key_dict) + streaming_response: Final = litellm.CustomStreamWrapper( + completion_stream=litellm.utils.ModelResponseIterator( + model_response=_passthrough_block_response(e), convert_to_delta=True + ), + model=e.model, + custom_llm_provider="cached_response", + logging_obj=logging_obj, + ) + return StreamingResponse( + select_data_generator( + response=streaming_response, + user_api_key_dict=user_api_key_dict, + request_data=e.request_data, + request=request, + ), + media_type="text/event-stream", + status_code=200, + ) + + +async def _stream_text_completion_passthrough_block( + e: ModifyResponseException, user_api_key_dict: UserAPIKeyAuth, request: Request +) -> StreamingResponse: + await _record_passthrough_block(e, user_api_key_dict) + streaming_response: Final = litellm.TextCompletionStreamWrapper( + completion_stream=litellm.utils.ModelResponseIterator( + model_response=_passthrough_block_response(e), convert_to_delta=True + ), + model=e.model, + ) + return StreamingResponse( + select_data_generator( + response=streaming_response, + user_api_key_dict=user_api_key_dict, + request_data=e.request_data, + request=request, + ), + media_type="text/event-stream", + status_code=200, + ) + + @router.post( "/v1/chat/completions", dependencies=[Depends(user_api_key_auth)], @@ -13352,50 +13418,17 @@ async def chat_completion( user_max_tokens=user_max_tokens, user_api_base=user_api_base, version=version, + on_passthrough_block=lambda block: _stream_chat_passthrough_block(block, user_api_key_dict, request), ) if isinstance(result, BaseModel): return model_dump_with_preserved_fields(result, exclude_unset=True) else: return result except ModifyResponseException as e: - # Guardrail flagged content in passthrough mode - return 200 with violation message - _data = e.request_data - # Capture logging_obj before post_call_failure_hook pops it from _data. - _logging_obj: Final = _data.get("litellm_logging_obj") - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=e, - request_data=_data, - ) - _chat_response = litellm.ModelResponse() - _chat_response.model = e.model - _chat_response.choices[0].message.content = e.message - _chat_response.choices[0].finish_reason = "content_filter" - # Report the blocked LLM response's real usage (set before the stream - # branch so both paths carry it); zero for pre-call blocks. - _chat_response.usage = _blocked_response_usage(e.original_response) - - if data.get("stream", None) is not None and data["stream"] is True: - _iterator = litellm.utils.ModelResponseIterator(model_response=_chat_response, convert_to_delta=True) - _streaming_response = litellm.CustomStreamWrapper( - completion_stream=_iterator, - model=e.model, - custom_llm_provider="cached_response", - logging_obj=_logging_obj, - ) - selected_data_generator = select_data_generator( - response=_streaming_response, - user_api_key_dict=user_api_key_dict, - request_data=_data, - request=request, - ) - - return StreamingResponse( - selected_data_generator, - media_type="text/event-stream", - status_code=200, # Return 200 for passthrough mode - ) - return _chat_response + if data.get("stream") is True: + return await _stream_chat_passthrough_block(e, user_api_key_dict, request) + await _record_passthrough_block(e, user_api_key_dict) + return _passthrough_block_response(e) except RejectedRequestError as e: _data = e.request_data await proxy_logging_obj.post_call_failure_hook( @@ -13509,49 +13542,20 @@ async def completion( user_max_tokens=user_max_tokens, user_api_base=user_api_base, version=version, + on_passthrough_block=lambda block: _stream_text_completion_passthrough_block( + block, user_api_key_dict, request + ), ) except ModifyResponseException as e: - # Guardrail flagged content in passthrough mode - return 200 with violation message - _data = e.request_data - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=e, - request_data=_data, - ) - - if _data.get("stream", None) is not None and _data["stream"] is True: - _text_response: Final = litellm.ModelResponse() - # Set text attribute dynamically for text completion format - setattr(_text_response.choices[0], "text", e.message) - _text_response.model = e.model - _usage = _blocked_response_usage(e.original_response) - # Set usage attribute dynamically (ModelResponse accepts usage in __init__ but it's not in type definition) - setattr(_text_response, "usage", _usage) - _iterator = litellm.utils.ModelResponseIterator(model_response=_text_response, convert_to_delta=True) - _streaming_response = litellm.TextCompletionStreamWrapper( - completion_stream=_iterator, - model=e.model, - ) - - selected_data_generator = select_data_generator( - response=_streaming_response, - user_api_key_dict=user_api_key_dict, - request_data=_data, - request=request, - ) - - return StreamingResponse( - selected_data_generator, - media_type="text/event-stream", - status_code=200, # Return 200 for passthrough mode - ) - else: - _response = litellm.TextCompletionResponse() - _response.choices[0].text = e.message - _response.model = e.model - _usage = _blocked_response_usage(e.original_response) - _response.usage = _usage - return _response + if e.request_data.get("stream") is True: + return await _stream_text_completion_passthrough_block(e, user_api_key_dict, request) + await _record_passthrough_block(e, user_api_key_dict) + text_completion_response: Final = litellm.TextCompletionResponse() + text_completion_response.choices[0].text = e.message + text_completion_response.choices[0].finish_reason = "content_filter" + text_completion_response.model = e.model + text_completion_response.usage = _blocked_response_usage(e.original_response) + return text_completion_response except RejectedRequestError as e: _data = e.request_data await proxy_logging_obj.post_call_failure_hook( diff --git a/litellm/utils.py b/litellm/utils.py index 7fa9d32ae64..fb974b24283 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9228,6 +9228,7 @@ class ModelResponseIterator: if convert_to_delta is True: _stream_response: Final = ModelResponseStream() _stream_response.choices[0].delta.content = model_response.choices[0].message.content + _stream_response.choices[0].finish_reason = model_response.choices[0].finish_reason self.model_response: ModelResponse | ModelResponseStream = _stream_response else: self.model_response = model_response diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 117d148c287..072de5c0eed 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -2,6 +2,7 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objects in streaming responses """ +import asyncio import datetime import json import logging @@ -85,10 +86,15 @@ _BLOCK_CALL_ID: Final = "messages-block" _BLOCK_TEXT: Final = "blocked by guardrail" -class _BlockEveryRequest(CustomGuardrail): +class _BlockEveryRequestOnceReleased(CustomGuardrail): + def __init__(self, released: asyncio.Event) -> None: + super().__init__(guardrail_name="deny-list", event_hook=GuardrailEventHooks.pre_call, default_on=True) + self.released: Final = released + async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict[str, object], call_type: str ) -> None: + await self.released.wait() self.raise_passthrough_exception(violation_message=_BLOCK_TEXT, request_data=data) @@ -117,21 +123,29 @@ def _messages_request(body: Mapping[str, object]) -> Request: async def _blocked_messages_reply( - body: Mapping[str, object], fastapi_response: Response, rails: tuple[CustomGuardrail, ...] + body: Mapping[str, object], + fastapi_response: Response, + rails: tuple[CustomGuardrail, ...], + keepalive_interval: float | None = None, ) -> tuple[object, str]: import litellm import litellm.proxy.anthropic_endpoints.endpoints as ep import litellm.proxy.proxy_server as proxy_server from litellm.proxy.utils import ProxyLogging - blocker: Final = _BlockEveryRequest( - guardrail_name="deny-list", event_hook=GuardrailEventHooks.pre_call, default_on=True - ) + released: Final = asyncio.Event() + if keepalive_interval is None: + released.set() with ( patch.object( proxy_server, "proxy_logging_obj", new=ProxyLogging(user_api_key_cache=DualCache()) ), # test-quality-ok: module global imported at call time; no injection seam - patch.object(litellm, "callbacks", new=[blocker, *rails]), # test-quality-ok: guardrails register through this global + patch.object( + litellm, "callbacks", new=[_BlockEveryRequestOnceReleased(released), *rails] + ), # test-quality-ok: guardrails register through this global + patch.object( + litellm, "sse_keepalive_ping_interval_seconds", new=keepalive_interval + ), # test-quality-ok: the operator's keepalive interval is read from this global ): response = await ep.anthropic_response( fastapi_response=fastapi_response, @@ -140,12 +154,34 @@ async def _blocked_messages_reply( ) if not isinstance(response, StreamingResponse): return response, "" - streamed: Final = b"".join( - [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator] - ).decode() + + async def chunks() -> AsyncGenerator[bytes, None]: + async for chunk in response.body_iterator: + released.set() + yield chunk if isinstance(chunk, bytes) else str(chunk).encode() + + streamed: Final = b"".join([chunk async for chunk in chunks()]).decode() return response, streamed +def _anthropic_events(streamed: str) -> tuple[tuple[str, Mapping[str, object]], ...]: + frames: Final = tuple(frame for frame in streamed.split("\n\n") if frame.strip() and frame != ": ping") + return tuple( + (frame.split("\n")[0].removeprefix("event: "), json.loads(frame.split("\n")[1].removeprefix("data: "))) + for frame in frames + ) + + +_BLOCK_EVENTS: Final = ( + "message_start", + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", +) + + class TestBlockedResponseUsage: """Blocked responses report the blocked LLM response's real usage.""" @@ -220,26 +256,27 @@ class TestBlockedResponseUsage: {"model": "claude-haiku-5-5", "stream": True}, Response(), rails ) - frames: Final = tuple(frame for frame in streamed.split("\n\n") if frame.strip()) - events: Final = tuple( - (frame.split("\n")[0].removeprefix("event: "), json.loads(frame.split("\n")[1].removeprefix("data: "))) - for frame in frames - ) + events: Final = _anthropic_events(streamed) assert isinstance(response, StreamingResponse) assert response.media_type == "text/event-stream" assert response.headers["x-litellm-call-id"] == _BLOCK_CALL_ID - assert [name for name, _ in events] == [ - "message_start", - "content_block_start", - "content_block_delta", - "content_block_stop", - "message_delta", - "message_stop", - ] + assert [name for name, _ in events] == list(_BLOCK_EVENTS) assert all(payload["type"] == name for name, payload in events) assert events[2][1]["delta"] == {"type": "text_delta", "text": _BLOCK_TEXT} assert events[4][1]["delta"]["stop_reason"] == "end_turn" + @pytest.mark.asyncio + async def test_a_streaming_block_after_a_keepalive_ping_answers_the_block_events(self) -> None: + _, streamed = await _blocked_messages_reply( + {"model": "claude-haiku-5-5", "stream": True}, Response(), (), keepalive_interval=0.05 + ) + + events: Final = _anthropic_events(streamed) + assert streamed.startswith(": ping\n\n") + assert [name for name, _ in events] == list(_BLOCK_EVENTS) + assert events[2][1]["delta"] == {"type": "text_delta", "text": _BLOCK_TEXT} + assert events[4][1]["delta"]["stop_reason"] == "end_turn" + @pytest.mark.asyncio async def test_json_block_answers_with_the_call_id(self) -> None: fastapi_response: Final = Response() diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index bfe381cb848..ccfac3d4e30 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -11608,6 +11608,69 @@ async def test_a_broken_hook_does_not_replace_the_real_error_with_its_own_bug(): assert "audit backend" not in collected[-2].decode() +async def _block_once_released(release: asyncio.Event) -> Response: + await release.wait() + raise ModifyResponseException( + message="blocked by policy", model="gpt-4o", request_data={"stream": True}, guardrail_name="g" + ) + + +async def _drain_releasing(response: StreamingResponse, release: asyncio.Event) -> tuple[bytes, ...]: + async def chunks() -> AsyncGenerator[bytes, None]: + async for chunk in response.body_iterator: + release.set() + yield chunk if isinstance(chunk, bytes) else str(chunk).encode() + + return tuple([chunk async for chunk in chunks()]) + + +@pytest.mark.asyncio +async def test_a_late_passthrough_block_streams_the_routes_block_response_instead_of_an_error_frame(): + block_frames: Final = ('data: {"choices": [{"delta": {"content": "blocked by policy"}}]}\n\n', "data: [DONE]\n\n") + release: Final = asyncio.Event() + stream_block: Final = AsyncMock(side_effect=lambda _block: _sse_response(block_frames)) + record_failure: Final = AsyncMock(return_value=None) + + response: Final = await open_sse_before_first_byte( + _block_once_released(release), + ping_interval_seconds=0.05, + on_late_failure=record_failure, + on_passthrough_block=stream_block, + ) + collected: Final = await _drain_releasing(response, release) + + assert collected[0] == TTFT_PING + assert [chunk for chunk in collected if chunk != TTFT_PING] == [frame.encode() for frame in block_frames] + assert [call.args[0].message for call in stream_block.await_args_list] == ["blocked by policy"] + record_failure.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_a_late_passthrough_block_on_a_route_without_a_block_stream_still_ends_the_stream(): + release: Final = asyncio.Event() + response: Final = await open_sse_before_first_byte(_block_once_released(release), ping_interval_seconds=0.05) + collected: Final = await _drain_releasing(response, release) + + assert collected[0] == TTFT_PING + assert "error" in json.loads(collected[-2].decode().removeprefix("data: ").strip()) + assert collected[-1] == b"data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_a_late_passthrough_block_whose_block_stream_fails_still_ends_the_stream(): + release: Final = asyncio.Event() + broken_block_stream: Final = AsyncMock(side_effect=RuntimeError("block stream unavailable")) + + response: Final = await open_sse_before_first_byte( + _block_once_released(release), ping_interval_seconds=0.05, on_passthrough_block=broken_block_stream + ) + collected: Final = await _drain_releasing(response, release) + + assert collected[0] == TTFT_PING + assert "error" in json.loads(collected[-2].decode().removeprefix("data: ").strip()) + assert collected[-1] == b"data: [DONE]\n\n" + + @pytest.mark.parametrize( "exc,expect_traceback", [ diff --git a/tests/unit/proxy/test_modify_response_streaming_passthrough.py b/tests/unit/proxy/test_modify_response_streaming_passthrough.py index f4d3b659ee1..5c46f1afac4 100644 --- a/tests/unit/proxy/test_modify_response_streaming_passthrough.py +++ b/tests/unit/proxy/test_modify_response_streaming_passthrough.py @@ -14,16 +14,28 @@ Reading it from the outer body passed ``logging_obj=None`` to ``AttributeError: 'NoneType' object has no attribute 'model_call_details'``. """ +import asyncio +import json +from collections.abc import AsyncGenerator, Mapping +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import Request, Response +from fastapi.responses import StreamingResponse +from starlette.types import Message +import litellm +import litellm.proxy.proxy_server as proxy_server +from litellm.caching.dual_cache import DualCache from litellm.exceptions import RejectedRequestError -from litellm.integrations.custom_guardrail import ModifyResponseException +from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.proxy_server import chat_completion - +from litellm.proxy.proxy_server import chat_completion, completion +from litellm.proxy.utils import ProxyLogging +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import ModelResponse, ModelResponseStream, TextCompletionResponse async def _run_streaming_block_and_get_wrapper(exception): """Drive chat_completion's streaming guardrail-passthrough handler for the @@ -108,3 +120,181 @@ async def test_streaming_rejected_request_uses_request_data_logging_obj(): mock_csw.assert_called_once() assert mock_csw.call_args.kwargs["logging_obj"] is sentinel_logging_obj + + +_BLOCK_TEXT: Final = "blocked by guardrail" +_PING: Final = ": ping" + + +class _BlockEveryRequestOnceReleased(CustomGuardrail): + def __init__(self, released: asyncio.Event) -> None: + super().__init__(guardrail_name="deny-list", event_hook=GuardrailEventHooks.pre_call, default_on=True) + self.released: Final = released + self.failures: tuple[Exception, ...] = () + + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict[str, object], call_type: str + ) -> None: + await self.released.wait() + self.raise_passthrough_exception(violation_message=_BLOCK_TEXT, request_data=data) + + async def async_post_call_failure_hook( + self, + request_data: dict[str, object], + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: str | None = None, + ) -> None: + self.failures = (*self.failures, original_exception) + + +class _RejectEveryRequest(CustomLogger): + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict[str, object], call_type: str + ) -> None: + raise RejectedRequestError( + message="rejected by the pre-call hook", model="gpt-4o", llm_provider="openai", request_data=data + ) + + +def _post(path: str, body: Mapping[str, object]) -> Request: + async def receive() -> Message: + return {"type": "http.request", "body": json.dumps(body).encode(), "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": path, + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + }, + receive, + ) + + +async def _frames_releasing_the_block(response: StreamingResponse, released: asyncio.Event) -> tuple[str, ...]: + async def chunks() -> AsyncGenerator[str, None]: + async for chunk in response.body_iterator: + released.set() + yield chunk if isinstance(chunk, str) else bytes(chunk).decode() + + streamed: Final = "".join([chunk async for chunk in chunks()]) + return tuple(frame for frame in streamed.split("\n\n") if frame) + + +async def _reply( + path: str, + body: Mapping[str, object], + rail: CustomLogger, + released: asyncio.Event, + keepalive_interval: float | None, +) -> tuple[object, tuple[str, ...]]: + endpoint: Final = chat_completion if path == "/v1/chat/completions" else completion + with ( + patch.object( + proxy_server, "proxy_logging_obj", new=ProxyLogging(user_api_key_cache=DualCache()) + ), # test-quality-ok: module global imported at call time; no injection seam + patch.object(litellm, "callbacks", new=[rail]), # test-quality-ok: guardrails register through this global + patch.object( + litellm, "sse_keepalive_ping_interval_seconds", new=keepalive_interval + ), # test-quality-ok: the operator's keepalive interval is read from this global + ): + response: Final = await endpoint( + request=_post(path, body), fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth() + ) + if not isinstance(response, StreamingResponse): + return response, () + return response, await _frames_releasing_the_block(response, released) + + +async def _blocked_reply( + path: str, body: Mapping[str, object], keepalive_interval: float | None +) -> tuple[object, tuple[str, ...], _BlockEveryRequestOnceReleased]: + released: Final = asyncio.Event() + if keepalive_interval is None: + released.set() + blocker: Final = _BlockEveryRequestOnceReleased(released) + response, frames = await _reply(path, body, blocker, released, keepalive_interval) + return response, frames, blocker + + +def _data_frames(frames: tuple[str, ...]) -> tuple[str, ...]: + return tuple(frame.removeprefix("data: ") for frame in frames if frame.startswith("data: {")) + + +KEEPALIVE_INTERVALS: Final = pytest.mark.parametrize( + "keepalive_interval", [0.05, None], ids=["blocked-after-a-keepalive-ping", "keepalive-off"] +) + + +@pytest.mark.asyncio +@KEEPALIVE_INTERVALS +async def test_a_blocked_chat_stream_gets_the_block_message_and_a_content_filter_finish( + keepalive_interval: float | None, +) -> None: + _, frames, blocker = await _blocked_reply( + "/v1/chat/completions", {"model": "gpt-4o", "messages": [], "stream": True}, keepalive_interval + ) + + chunks: Final = tuple(ModelResponseStream.model_validate_json(frame) for frame in _data_frames(frames)) + assert (frames[0] == _PING) is (keepalive_interval is not None) + assert frames[-1] == "data: [DONE]" + assert not any('"error"' in frame for frame in frames) + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == _BLOCK_TEXT + assert chunks[-1].choices[0].finish_reason == "content_filter" + assert [type(failure) for failure in blocker.failures] == [ModifyResponseException] + + +@pytest.mark.asyncio +@KEEPALIVE_INTERVALS +async def test_a_blocked_text_completion_stream_gets_the_block_message(keepalive_interval: float | None) -> None: + _, frames, blocker = await _blocked_reply( + "/v1/completions", {"model": "gpt-4o", "prompt": "hi", "stream": True}, keepalive_interval + ) + + chunks: Final = tuple(TextCompletionResponse.model_validate_json(frame) for frame in _data_frames(frames)) + assert (frames[0] == _PING) is (keepalive_interval is not None) + assert frames[-1] == "data: [DONE]" + assert not any('"error"' in frame for frame in frames) + assert "".join(chunk.choices[0].text or "" for chunk in chunks) == _BLOCK_TEXT + assert chunks[-1].choices[0].finish_reason == "content_filter" + assert [type(failure) for failure in blocker.failures] == [ModifyResponseException] + + +@pytest.mark.asyncio +async def test_a_blocked_chat_request_without_streaming_gets_the_block_message_and_a_content_filter_finish() -> None: + response, _, blocker = await _blocked_reply( + "/v1/chat/completions", {"model": "gpt-4o", "messages": [], "stream": False}, None + ) + + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == _BLOCK_TEXT + assert response.choices[0].finish_reason == "content_filter" + assert [type(failure) for failure in blocker.failures] == [ModifyResponseException] + + +@pytest.mark.asyncio +async def test_a_blocked_text_completion_without_streaming_gets_the_block_text_and_a_content_filter_finish() -> None: + response, _, blocker = await _blocked_reply( + "/v1/completions", {"model": "gpt-4o", "prompt": "hi", "stream": False}, None + ) + + assert isinstance(response, TextCompletionResponse) + assert response.choices[0].text == _BLOCK_TEXT + assert response.choices[0].finish_reason == "content_filter" + assert [type(failure) for failure in blocker.failures] == [ModifyResponseException] + + +@pytest.mark.asyncio +async def test_a_rejected_text_completion_stream_ends_with_a_stop_finish() -> None: + released: Final = asyncio.Event() + released.set() + + _, frames = await _reply( + "/v1/completions", {"model": "gpt-4o", "prompt": "hi", "stream": True}, _RejectEveryRequest(), released, None + ) + + chunks: Final = tuple(TextCompletionResponse.model_validate_json(frame) for frame in _data_frames(frames)) + assert frames[-1] == "data: [DONE]" + assert chunks[-1].choices[0].finish_reason == "stop" From 5e74515c29d62982694fa50aab5b416c2c252949 Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Wed, 11 Nov 2026 13:57:41 -0800 Subject: [PATCH 111/189] feat(otel): let a team or key OTel v2 destination add its own request metadata keys (#50813) * feat(otel): let a team or key OTel v2 destination add its own request metadata keys A team or key callback can now set baggage_metadata_keys in its callback_vars, a comma-separated list of requester_metadata. paths. The tenant fan-out adds them as litellm.metadata. to that destination's spans only, on top of the global LITELLM_OTEL_BAGGAGE_METADATA_KEYS; the operator's exporter and other destinations never see them. Values are bound when each span ends, an attribute the span already carries wins, and names of proxy identity fields are rejected when saved, along with lists over 20 keys. Resolves LIT-9545 * fix(otel): refuse request headers in a destination's baggage_metadata_keys The proxy copies the request headers into requester_metadata.headers with credential headers masked, but a header a gateway in front of it added (an IAP or OIDC assertion, an internal token) stays as sent. A team or key admin listing requester_metadata.headers. could export it to an endpoint the team controls. Entries under requester_metadata.headers are now rejected when saved, so exporting a header stays an operator decision in LITELLM_OTEL_BAGGAGE_METADATA_KEYS. Also drops an unused import. * fix(otel): hold destination metadata to the configured span limits A destination's extra litellm.metadata.* attributes were added to a span view built from a plain mapping, so they skipped the limits the SDK applies in set_attribute: a 1 MiB caller value survived an operator's OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT. Values are now cut to the length limit, and keys past the attribute-count limit are dropped and counted instead of evicting an attribute the span already carries. The per-destination views are built in one Final tuple ahead of the forwarding loop, since a Final cannot be assigned inside the loop. * fix(otel): apply destination metadata after the destination view, within the provider's limits The span limits were checked against the span as it ended, before the destination view strips prompt content and backend-only attributes, so a model-call span full of llm.input_messages.* could drop the team's keys on a no_content destination that had room. The values are now bound when the span ends and added as the last step before export, counted against what that destination actually sends, so their dropped count also survives the view. The limits come from the tracer provider the fan-out is attached to, read from the live span at start, instead of a separate env-built SpanLimits. * refactor(otel): consolidate span limits, destination validators and metadata key parsing Reads a span's limits in one guarded place that the emitter and the fan-out share, replaces the two parallel lists of per-destination callback-var validators with one mapping, splits and strips a baggage_metadata_keys list once, drops a filter the post-view merge already enforces, and folds the repeated fan-out test setup into one helper. No behavior change. * ci: retrigger checks stranded by the GitHub Actions outage --- litellm/integrations/otel/README.md | 8 +- litellm/integrations/otel/emitter.py | 9 +- litellm/integrations/otel/logger.py | 2 + litellm/integrations/otel/model/baggage.py | 58 +++- .../integrations/otel/model/destination.py | 11 +- litellm/integrations/otel/plumbing/context.py | 19 ++ .../integrations/otel/plumbing/providers.py | 93 ++++++- .../integrations/otel/presets/destinations.py | 2 + .../initialize_dynamic_callback_params.py | 23 +- litellm/proxy/_types.py | 18 +- .../callback_config_validation.py | 34 ++- litellm/proxy/litellm_pre_call_utils.py | 6 +- .../team_callback_endpoints.py | 5 +- litellm/types/utils.py | 7 + .../test_otel_destination_metadata_keys.py | 262 ++++++++++++++++++ .../otel/test_otel_v2_destinations.py | 127 +++++++++ .../test_callback_config_validation.py | 71 ++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 1 + 18 files changed, 700 insertions(+), 56 deletions(-) create mode 100644 tests/integration/observability/test_otel_destination_metadata_keys.py diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 06c2da117b5..70384e49f53 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -325,7 +325,13 @@ nothing here imports outside it: on `langfuse_otel`, `arize`, `weave_otel` and `newrelic` callbacks. - [`baggage.py`](./model/baggage.py) — the single definition of which request-identity values are promoted into Baggage (so child spans inherit them) and under which - attribute keys. + attribute keys. A team or key destination can list extra `requester_metadata.` + paths in `baggage_metadata_keys` in its own `callback_vars`. They are never written + into Baggage: the tenant fan-out adds them as `litellm.metadata.` to that + destination's view of each span, on top of the global list, so the operator's + exporter and other destinations never see them. A key the span already carries + wins, and names of proxy identity fields and `requester_metadata.headers.*` (the + request headers the proxy copies there) are rejected when saved. - [`utils.py`](./model/utils.py) — value coercion, JSON serialization, and extractor-table application, shared across the package. diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 87a432ee7ce..813ba6aaa73 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -6,8 +6,7 @@ from types import MappingProxyType from typing import Final from opentelemetry.context import Context -from opentelemetry.sdk.trace import ReadableSpan, SpanLimits -from opentelemetry.sdk.trace import Span as SdkSpan +from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.trace import Link, Span, Tracer from opentelemetry.trace.status import Status, StatusCode @@ -36,7 +35,7 @@ from litellm.integrations.otel.model.spans import ( ) from litellm.integrations.otel.plumbing.context import destination_backends from litellm.integrations.otel.plumbing.events import GenAIEventRecorder -from litellm.integrations.otel.plumbing.providers import to_otel_span_kind +from litellm.integrations.otel.plumbing.providers import span_limits, to_otel_span_kind # Roles emit() knows how to name and emit. PROXY_REQUEST and the management # routes are SERVER spans owned by the mounted FastAPI instrumentor, so they @@ -88,9 +87,7 @@ def error_attributes(error: SpanError) -> Mapping[str, AttrValue]: def span_attribute_limit(span: Span) -> int | None: """The attribute count limit ``span`` was built with, ``None`` when unbounded.""" - if not isinstance(span, SdkSpan): - return SpanLimits().max_span_attributes - return span._limits.max_span_attributes # pyright: ignore[reportPrivateUsage] # SDK has no public getter + return span_limits(span).max_span_attributes def _carried_keys( diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 41811f836c7..dfb03211927 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -66,6 +66,7 @@ from litellm.integrations.otel.plumbing.context import ( resolve_request_span_context, resolve_service_span_context, set_request_baggage, + set_request_metadata, set_request_root_span, ) from litellm.integrations.otel.plumbing.events import GenAIEventRecorder @@ -748,6 +749,7 @@ class OpenTelemetryV2(CustomLogger): """ try: identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict, request_metadata) + set_request_metadata(identity.metadata) bag: Final = promoted_baggage( identity, model, diff --git a/litellm/integrations/otel/model/baggage.py b/litellm/integrations/otel/model/baggage.py index 606fd79f218..5832a56b076 100644 --- a/litellm/integrations/otel/model/baggage.py +++ b/litellm/integrations/otel/model/baggage.py @@ -18,7 +18,11 @@ from collections.abc import Callable, Mapping from types import MappingProxyType from typing import Final -from litellm.integrations.otel.model.metadata import RequestIdentity, allowlisted_metadata +from litellm.integrations.otel.model.metadata import ( + REQUESTER_METADATA_PATH, + RequestIdentity, + allowlisted_metadata, +) from litellm.integrations.otel.model.semconv import GenAI, LiteLLM # Attribute key -> value extractor over (identity, request_model, @@ -65,6 +69,58 @@ DEFAULT_BAGGAGE_METADATA_KEYS: Final[tuple[str, ...]] = ( # explicitly allowlisted via ``config.baggage_team_metadata_keys``. DEFAULT_BAGGAGE_TEAM_METADATA_KEYS: Final[tuple[str, ...]] = () +# A team or key OTel v2 destination can add caller metadata keys on top of the +# global ``baggage_metadata_keys``, for its own destination only. Each key names +# a ``requester_metadata.`` path, so only what the caller sent can be read, +# and a key whose exported name is a proxy-owned identity field is refused. The +# proxy copies the request headers into ``requester_metadata.headers``, credentials +# masked but anything a gateway in front of it added left as sent, so exporting a +# header stays an operator decision made in the global list. +MAX_DESTINATION_METADATA_KEYS: Final = 20 +_RESERVED_METADATA_PREFIX: Final = "user_api_key_" +_REQUEST_HEADERS_KEY: Final = "headers" + + +def destination_metadata_keys_error(value: str) -> str | None: + """Why ``value`` is not a usable destination ``baggage_metadata_keys`` list, or ``None``.""" + keys: Final = _listed_keys(value) + named: Final = tuple((key, key.removeprefix(REQUESTER_METADATA_PATH)) for key in keys) + if any(name == key or not name for key, name in named): + return f"Invalid baggage_metadata_keys {value!r}: use a comma-separated list of {REQUESTER_METADATA_PATH} entries" + reserved: Final = tuple(key for key, name in named if _is_reserved_metadata_name(name)) + if reserved: + return ( + f"Invalid baggage_metadata_keys {value!r}: {', '.join(reserved)} would overwrite a proxy identity attribute" + ) + headers: Final = tuple(key for key, name in named if _is_request_header(name)) + if headers: + return ( + f"Invalid baggage_metadata_keys {value!r}: {', '.join(headers)} would export request headers, " + "which only the proxy-wide LITELLM_OTEL_BAGGAGE_METADATA_KEYS can do" + ) + if len(frozenset(keys)) > MAX_DESTINATION_METADATA_KEYS: + return f"Invalid baggage_metadata_keys {value!r}: at most {MAX_DESTINATION_METADATA_KEYS} keys" + return None + + +def parse_destination_metadata_keys(value: object) -> tuple[str, ...] | None: + """A destination's ``baggage_metadata_keys`` as a key tuple, ``None`` when unset or invalid.""" + if not isinstance(value, str) or destination_metadata_keys_error(value) is not None: + return None + return tuple(dict.fromkeys(_listed_keys(value))) + + +def _listed_keys(value: str) -> tuple[str, ...]: + return tuple(key.strip() for key in value.split(",")) + + +def _is_reserved_metadata_name(name: str) -> bool: + return name in DEFAULT_BAGGAGE_METADATA_KEYS or name.startswith(_RESERVED_METADATA_PREFIX) + + +def _is_request_header(name: str) -> bool: + return name.partition(".")[0].strip().lower() == _REQUEST_HEADERS_KEY + def promoted_baggage( identity: RequestIdentity, diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py index e760ba3600b..6af28f7c2a7 100644 --- a/litellm/integrations/otel/model/destination.py +++ b/litellm/integrations/otel/model/destination.py @@ -61,6 +61,13 @@ class OtelDestination(LiteLLMBaseModel): default=None, description="An explicit set replaces the global database-span exclusions; ``None`` follows the global set.", ) + metadata_keys: tuple[str, ...] = Field( + default=(), + description=( + "``requester_metadata.`` paths exported as ``litellm.metadata.`` on this destination's " + "spans only, on top of the global ``baggage_metadata_keys``." + ), + ) def captures_content(self, default: str) -> bool: setting: Final = self.capture_message_content @@ -79,8 +86,8 @@ class OtelDestination(LiteLLMBaseModel): def cache_key(self) -> tuple[str, tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], str | None]: """Identity for processor reuse, so one destination means one exporter. - ``span_scope``, the sampling rates, ``capture_message_content`` and - ``excluded_db_systems`` are left out on purpose: they decide which spans reach + ``span_scope``, the sampling rates, ``capture_message_content``, + ``excluded_db_systems`` and ``metadata_keys`` are left out on purpose: they decide which spans reach the processor and what they carry, not how the processor exports them, so two views of the same account share one exporter. """ diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py index b328b281dde..03d53965360 100644 --- a/litellm/integrations/otel/plumbing/context.py +++ b/litellm/integrations/otel/plumbing/context.py @@ -4,6 +4,7 @@ import os from collections.abc import Generator, Mapping from contextlib import contextmanager from contextvars import ContextVar, Token +from types import MappingProxyType from typing import TYPE_CHECKING, Final from opentelemetry import baggage @@ -478,6 +479,24 @@ def request_destinations() -> 'tuple["OtelDestination", ...]': return _request_destinations.get() +# The request's flattened metadata as identity seeding last read it, kept beside the +# destinations so the fan-out can export the extra ``metadata_keys`` a destination asked +# for onto that destination's spans alone, never into the Baggage every exporter reads. +_request_metadata: Final["ContextVar[Mapping[str, str]]"] = ContextVar( + "litellm_otel_request_metadata", default=MappingProxyType({}) +) + + +def set_request_metadata(metadata: Mapping[str, str]) -> None: + """Record the flattened metadata of the request on this context.""" + _request_metadata.set(metadata) + + +def request_metadata() -> Mapping[str, str]: + """The flattened metadata seeded for this request, empty before seeding or outside one.""" + return _request_metadata.get() + + #: ``litellm_settings: otel_tenant_destination_mode`` and its env equivalent. ADDITIVE_DESTINATION_MODE: Final = "additive" OTEL_TENANT_DESTINATION_MODE_ENV: Final = "LITELLM_OTEL_TENANT_DESTINATION_MODE" diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index b5a5b85fc11..0941c065e74 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -24,7 +24,7 @@ from opentelemetry.sdk._logs.export import ( ) from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import Event, ReadableSpan, SpanProcessor, TracerProvider +from opentelemetry.sdk.trace import Event, ReadableSpan, SpanLimits, SpanProcessor, TracerProvider from opentelemetry.sdk.trace import Span as SDKSpan from opentelemetry.sdk.trace.export import ( BatchSpanProcessor, @@ -49,6 +49,7 @@ from litellm.integrations.otel.mappers.langfuse import ( LANGFUSE_TRACE_NAME, LangfuseMapper, ) +from litellm.integrations.otel.model.baggage import promoted_metadata from litellm.integrations.otel.model.config import ( CaptureMessageContent, ExporterOwner, @@ -71,6 +72,7 @@ from litellm.integrations.otel.model.semconv import ( from litellm.integrations.otel.model.spans import LiteLLMSpanKind from litellm.integrations.otel.plumbing.context import ( request_destinations, + request_metadata, request_traces_excluded, suppressed_backends, ) @@ -469,9 +471,13 @@ def without_message_content(span: ReadableSpan) -> ReadableSpan: class _CountedSpanView(_SpanView): """A ``_SpanView`` of ``inner`` with other attributes that reports the dropped counts ``inner`` reports.""" - def __init__(self, inner: ReadableSpan, attributes: Attributes) -> None: + def __init__(self, inner: ReadableSpan, attributes: Attributes, dropped_attributes: int = 0) -> None: super().__init__(inner, inner.resource, attributes, inner.events, inner.status, parent=inner.parent) - self._dropped_counts: Final = (inner.dropped_attributes, inner.dropped_events, inner.dropped_links) + self._dropped_counts: Final = ( + inner.dropped_attributes + dropped_attributes, + inner.dropped_events, + inner.dropped_links, + ) @property def dropped_attributes(self) -> int: @@ -486,6 +492,60 @@ class _CountedSpanView(_SpanView): return self._dropped_counts[2] +def span_limits(span: object) -> SpanLimits: + """The limits ``span`` was built with; the env-configured defaults for a span the SDK did not build.""" + if not isinstance(span, SDKSpan): + return SpanLimits() + return span._limits # pyright: ignore[reportPrivateUsage] # the SDK has no public accessor for a span's limits + + +class _PendingMetadataSpan(_CountedSpanView): + """``inner`` as it ended, carrying the destination metadata added once the destination's view is built.""" + + def __init__(self, inner: ReadableSpan, metadata: tuple[tuple[str, str], ...], limits: SpanLimits) -> None: + super().__init__(inner, inner.attributes) + self.destination_metadata: Final = metadata + self.limits: Final = limits + + +def with_destination_metadata( + span: ReadableSpan, metadata_keys: tuple[str, ...], metadata: Mapping[str, str], limits: SpanLimits +) -> ReadableSpan: + """``span`` carrying the ``litellm.metadata.*`` values a destination's own ``metadata_keys`` select. + + Bound when the span ends, so a tree held for sampling keeps its own request's values, along + with the span limits it ended under. :func:`with_pending_metadata` adds them once the + destination's view is built. + """ + if not metadata_keys: + return span + promoted: Final = tuple(promoted_metadata(metadata, metadata_keys).items()) + if not promoted: + return span + return _PendingMetadataSpan(span, promoted, limits) + + +def with_pending_metadata(ended: ReadableSpan, exported: ReadableSpan) -> ReadableSpan: + """``exported``, the destination's view of ``ended``, plus the metadata ``ended`` carries. + + Counted against what the destination exports rather than the span as it ended, since the view + can drop prompt content and backend-specific attributes. An exported attribute always wins, so a + destination key never replaces a value the global allowlist or the proxy's identity wrote. + Values are cut to the length limit, and keys past the count limit are dropped and counted. + """ + if not isinstance(ended, _PendingMetadataSpan): + return exported + attributes: Final = exported.attributes or _NO_ATTRIBUTES + extra: Final = tuple((key, value) for key, value in ended.destination_metadata if key not in attributes) + max_attributes: Final = ended.limits.max_span_attributes + max_length: Final = ended.limits.max_span_attribute_length + room: Final = len(extra) if max_attributes is None else max(0, max_attributes - len(attributes)) + kept: Final = {key: value if max_length is None else value[:max_length] for key, value in extra[:room]} + return _CountedSpanView( + exported, MappingProxyType({**attributes, **kept}), dropped_attributes=len(extra) - len(kept) + ) + + def _is_litellm_span(span: ReadableSpan) -> bool: """Whether LiteLLM's own tracer wrote ``span``, rather than application or instrumentation code.""" scope: Final = span.instrumentation_scope @@ -761,9 +821,11 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary() self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) + self._span_limits = SpanLimits() def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: context: Final = span.context + self._span_limits = span_limits(span) with self._lock: request: Final = self._request_of(span, trace.get_current_span(parent_context)) if context is not None: @@ -782,8 +844,11 @@ class TenantFanOutSpanProcessor(SpanProcessor): if span.context is not None: self._roots.pop(span.context.span_id, None) failed: Final = span.status.status_code is StatusCode.ERROR - for destination in request_destinations(): - if ( + metadata: Final = request_metadata() + deliveries: Final = tuple( + (destination, with_destination_metadata(span, destination.metadata_keys, metadata, self._span_limits)) + for destination in request_destinations() + if not ( self._operator_already_writes(span, destination, suppressed) or not _in_scope(span, destination.span_scope) or _is_excluded_database_span( @@ -792,12 +857,13 @@ class TenantFanOutSpanProcessor(SpanProcessor): if destination.excluded_db_systems is None else destination.excluded_db_systems, ) - ): - continue + ) + ) + for destination, view in deliveries: if not _is_sampled(destination): - self._forward(span, destination) + self._forward(view, destination) continue - for held_span, held_destination in self._route(request, span, destination, failed): + for held_span, held_destination in self._route(request, view, destination, failed): self._forward(held_span, held_destination) for held_span, held_destination in self._settle(request, span, failed): self._forward(held_span, held_destination) @@ -834,12 +900,11 @@ class TenantFanOutSpanProcessor(SpanProcessor): if processor is None: return try: - processor.on_end( - _scoped( - _for_destination(span, destination, self._default_capture_for(destination), self._langfuse_mapped), - destination.span_scope, - ) + exported: Final = _scoped( + _for_destination(span, destination, self._default_capture_for(destination), self._langfuse_mapped), + destination.span_scope, ) + processor.on_end(with_pending_metadata(span, exported)) except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span verbose_logger.debug("OTel V2 fan-out: forwarding to %s failed: %s", destination.endpoint, exc) finally: diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index 984b385a647..0bd40824066 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -208,6 +208,7 @@ def destination_for( service_name: str | None = None, capture_message_content: str | None = None, excluded_db_systems: frozenset[str] | None = None, + metadata_keys: tuple[str, ...] | None = None, ) -> OtelDestination | None: """The destination ``params`` names for ``callback_name``, or ``None``. @@ -240,6 +241,7 @@ def destination_for( span_scope=_span_scope(callback_name, params), capture_message_content=capture_message_content, excluded_db_systems=excluded_db_systems, + metadata_keys=metadata_keys or (), success_sampling_rate=success_sampling_rate, error_sampling_rate=error_sampling_rate, ) diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index c3769686e45..cdb009fcf77 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -1,12 +1,16 @@ import re -from collections.abc import Generator, Iterator, Mapping +from collections.abc import Callable, Generator, Iterator, Mapping from contextlib import contextmanager from contextvars import ContextVar +from types import MappingProxyType from typing import Any, Final from litellm.types.utils import ( ARIZE_OTLP_PROTOCOLS, + BAGGAGE_METADATA_KEYS_VAR, CAPTURE_MESSAGE_CONTENT_VALUES, + CAPTURE_MESSAGE_CONTENT_VAR, + EXCLUDED_SERVICES_VAR, OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams, @@ -99,6 +103,14 @@ def validate_capture_message_content_value(value: str) -> None: ) +def validate_baggage_metadata_keys_value(value: str) -> None: + from litellm.integrations.otel.model.baggage import destination_metadata_keys_error + + error: Final = destination_metadata_keys_error(value) + if error is not None: + raise ValueError(error) + + def validate_excluded_services_value(value: str) -> None: from litellm.integrations.otel.model.config import invalid_excluded_service_names @@ -111,6 +123,15 @@ def validate_excluded_services_value(value: str) -> None: ) +OTEL_DESTINATION_VALIDATORS: Final[Mapping[str, Callable[[str], None]]] = MappingProxyType( + { + CAPTURE_MESSAGE_CONTENT_VAR: validate_capture_message_content_value, + EXCLUDED_SERVICES_VAR: validate_excluded_services_value, + BAGGAGE_METADATA_KEYS_VAR: validate_baggage_metadata_keys_value, + } +) + + # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict _supported_callback_params: Final[tuple[str, ...]] = ( "langfuse_public_key", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0d40faf55fb..80819542d68 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -31,9 +31,8 @@ from typing_extensions import ( from litellm._uuid import uuid from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + OTEL_DESTINATION_VALIDATORS, validate_arize_otlp_protocol_value, - validate_capture_message_content_value, - validate_excluded_services_value, validate_langfuse_environment_value, validate_langfuse_span_scope_value, validate_no_callback_env_reference, @@ -96,8 +95,7 @@ from litellm.types.router import AllowedModelRegion, RouterErrors, UpdateRouterC from litellm.types.router_weights import validate_router_settings_dict from litellm.types.secret_managers.main import KeyManagementSystem from litellm.types.utils import ( - CAPTURE_MESSAGE_CONTENT_VAR, - EXCLUDED_SERVICES_VAR, + OTEL_DESTINATION_VARS, AzureSpillover, CallTypes, CostBreakdown, @@ -2136,9 +2134,7 @@ class BlockModelRequest(LiteLLMPydanticObjectBase): model_id: str # required -CALLBACK_VAR_NAMES: Final = frozenset( - {*StandardCallbackDynamicParams.__annotations__, CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR} -) +CALLBACK_VAR_NAMES: Final = frozenset({*StandardCallbackDynamicParams.__annotations__, *OTEL_DESTINATION_VARS}) class AddTeamCallback(LiteLLMPydanticObjectBase): @@ -2153,7 +2149,7 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): for key, value in callback_vars.items(): if key not in CALLBACK_VAR_NAMES: raise ValueError(f"Invalid callback variable: {key}. Must be one of {set(CALLBACK_VAR_NAMES)}") - if key in (CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR) and not isinstance(value, str): + if key in OTEL_DESTINATION_VARS and not isinstance(value, str): raise ValueError(f"{key} must be a string, got {type(value).__name__}") callback_vars[key] = str(value) validate_no_callback_env_reference(key, callback_vars[key], source="key/team callback metadata") @@ -2163,10 +2159,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): validate_langfuse_span_scope_value(callback_vars[key]) if key == "arize_otlp_protocol": validate_arize_otlp_protocol_value(callback_vars[key]) - if key == CAPTURE_MESSAGE_CONTENT_VAR: - validate_capture_message_content_value(callback_vars[key]) - if key == EXCLUDED_SERVICES_VAR: - validate_excluded_services_value(callback_vars[key]) + if (destination_validator := OTEL_DESTINATION_VALIDATORS.get(key)) is not None: + destination_validator(callback_vars[key]) return values diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py index a486d6665fc..a1c2ff8eb52 100644 --- a/litellm/proxy/common_utils/callback_config_validation.py +++ b/litellm/proxy/common_utils/callback_config_validation.py @@ -14,7 +14,11 @@ from typing import Final, Literal, cast # noqa: TID251 # preserve arbitrary ke from pydantic import TypeAdapter, ValidationError from litellm.litellm_core_utils.credential_accessor import CredentialAccessor -from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR +from litellm.types.utils import ( + BAGGAGE_METADATA_KEYS_VAR, + EXCLUDED_SERVICES_VAR, + OTEL_DESTINATION_VARS, +) _NEWRELIC_CALLBACK: Final = "newrelic" _NEWRELIC_VAR_PREFIX: Final = "newrelic_" @@ -42,18 +46,12 @@ def callback_config_error( ) if langfuse_error is not None: return langfuse_error - from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( - validate_capture_message_content_value, - validate_excluded_services_value, - ) + from litellm.litellm_core_utils.initialize_dynamic_callback_params import OTEL_DESTINATION_VALIDATORS destination_setting_error: Final = next( ( error - for variable_name, validator in ( - (CAPTURE_MESSAGE_CONTENT_VAR, validate_capture_message_content_value), - (EXCLUDED_SERVICES_VAR, validate_excluded_services_value), - ) + for variable_name, validator in OTEL_DESTINATION_VALIDATORS.items() if ( error := _destination_setting_error( variable_name, callback_name, callback_vars, callback_type, validator @@ -269,11 +267,17 @@ def conflicting_callback_var_error( def _same_callback_var_value(variable_name: str, first: str, second: str) -> bool: - if variable_name != EXCLUDED_SERVICES_VAR: - return first == second - from litellm.integrations.otel.model.config import parse_excluded_services + if variable_name == EXCLUDED_SERVICES_VAR: + from litellm.integrations.otel.model.config import parse_excluded_services - return parse_excluded_services(first) == parse_excluded_services(second) + return parse_excluded_services(first) == parse_excluded_services(second) + if variable_name == BAGGAGE_METADATA_KEYS_VAR: + from litellm.integrations.otel.model.baggage import parse_destination_metadata_keys + + first_keys: Final = parse_destination_metadata_keys(first) + second_keys: Final = parse_destination_metadata_keys(second) + return first_keys is not None and second_keys is not None and frozenset(first_keys) == frozenset(second_keys) + return first == second def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str | None: @@ -331,7 +335,7 @@ def _conflicting_entry_setting_error( return next( ( error - for variable_name in (CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR) + for variable_name in OTEL_DESTINATION_VARS if (error := conflicting_callback_var_error(callback_name, variable_name, callback_vars, earlier)) is not None ), @@ -364,7 +368,7 @@ def _logging_entry_error(entry: object) -> str | None: destination_var_error: Final = next( ( f"{variable_name} must be a string, got {type(callback_vars[variable_name]).__name__}" - for variable_name in (CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR) + for variable_name in OTEL_DESTINATION_VARS if variable_name in callback_vars and not isinstance(callback_vars[variable_name], str) ), None, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index ff0883b6958..b1eb8f7d8ae 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -332,8 +332,10 @@ from litellm.types.router import BaggageMappedTagsStamp, holds_api_key_secret_re from litellm.types.services import ServiceTypes from litellm.types.utils import ( API_KEY_SECRET_REFERENCE_REJECTION, + BAGGAGE_METADATA_KEYS_VAR, CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR, + OTEL_DESTINATION_VARS, CustomPricingLiteLLMParams, LlmProviders, ProviderSpecificHeader, @@ -342,7 +344,7 @@ from litellm.types.utils import ( SupportedCacheControls, ) -_DESTINATION_ONLY_CALLBACK_VARS: Final[frozenset[str]] = frozenset({CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR}) +_DESTINATION_ONLY_CALLBACK_VARS: Final[frozenset[str]] = frozenset(OTEL_DESTINATION_VARS) _CALLBACK_CREDENTIAL_KEYS: Final = frozenset(StandardCallbackDynamicParams.__annotations__) | frozenset( {TRUSTED_CALLBACK_VARS_FIELD} @@ -1479,6 +1481,7 @@ def resolve_tenant_otel_destinations( def _tenant_destination( name: str, callbacks: Sequence[AddTeamCallback], service_name: str | None ) -> "OtelDestination | None": + from litellm.integrations.otel.model.baggage import parse_destination_metadata_keys from litellm.integrations.otel.model.config import parse_capture_message_content, parse_excluded_services from litellm.integrations.otel.presets.destinations import destination_for @@ -1495,6 +1498,7 @@ def _tenant_destination( service_name, parse_capture_message_content(callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)), parse_excluded_services(callback_vars.get(EXCLUDED_SERVICES_VAR)), + parse_destination_metadata_keys(callback_vars.get(BAGGAGE_METADATA_KEYS_VAR)), ) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 5b778392664..1d35d34cbcf 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -60,7 +60,7 @@ from litellm.proxy.management_endpoints.team_endpoints import ( # noqa: F401 # ) from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.repositories.team_repository import TeamRepository -from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR +from litellm.types.utils import OTEL_DESTINATION_VARS router: Final = APIRouter() @@ -304,6 +304,7 @@ async def add_team_callbacks( An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out - excluded_services: For OTel v2 callbacks, a comma-separated list of datastore names such as "redis,postgres", or "none" to exclude nothing. An explicit value replaces the global exclusions; omitted follows them + - baggage_metadata_keys: For OTel v2 callbacks, a comma-separated list of request metadata keys such as "requester_metadata.correlation_id", exported as litellm.metadata. on this destination's spans only, on top of the global LITELLM_OTEL_BAGGAGE_METADATA_KEYS. At most 20 keys; names of proxy identity fields (user_api_key_*, requester_ip_address) and request headers (requester_metadata.headers.*) are rejected - gcs_bucket_name: The name of the GCS bucket - gcs_path_service_account: The path to the GCS service account - langsmith_api_key: The API key for the Langsmith callback @@ -376,7 +377,7 @@ async def add_team_callbacks( destination_setting_error: Final = next( ( error - for variable_name in (CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR) + for variable_name in OTEL_DESTINATION_VARS if ( error := conflicting_callback_var_error( data.callback_name, variable_name, data.callback_vars, stored_callbacks diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 81d39f848fb..5990135a5cb 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4029,6 +4029,13 @@ ARIZE_OTLP_PROTOCOLS: Final[frozenset[str]] = frozenset(get_args(ArizeOtlpProtoc CAPTURE_MESSAGE_CONTENT_VAR: Final = "capture_message_content" EXCLUDED_SERVICES_VAR: Final = "excluded_services" +BAGGAGE_METADATA_KEYS_VAR: Final = "baggage_metadata_keys" +#: Team and key callback vars that tune that callback's own OTel v2 destination, never sent to the backend. +OTEL_DESTINATION_VARS: Final[tuple[str, ...]] = ( + CAPTURE_MESSAGE_CONTENT_VAR, + EXCLUDED_SERVICES_VAR, + BAGGAGE_METADATA_KEYS_VAR, +) class CaptureMessageContent(str): diff --git a/tests/integration/observability/test_otel_destination_metadata_keys.py b/tests/integration/observability/test_otel_destination_metadata_keys.py new file mode 100644 index 00000000000..33b18fa4c2e --- /dev/null +++ b/tests/integration/observability/test_otel_destination_metadata_keys.py @@ -0,0 +1,262 @@ +"""A team's or key's own OTel v2 destination can list extra request metadata keys in ``baggage_metadata_keys``, +exported as ``litellm.metadata.*`` on that destination's spans only, on top of the operator's global allowlist.""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal + +import httpx +import pytest +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment +from integration._support.otlp_sink import ( + ConnectSink, + Span, + SpanSinks, + audit_configuration_writer, + owned_connect_sink, + owned_sinks, + recorded_spans, + spans_for_trace, +) +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +OPERATION: Final = "gen_ai.operation.name" +SERVER_KIND: Final = 2 +METADATA_PREFIX: Final = "litellm.metadata." +TEAM_KEYS: Final = "requester_metadata.tenant_id, requester_metadata.order.id" + +Backend = Literal["newrelic", "arize"] + + +def _chat(_request: Request) -> Reply: + return Reply( + body=( + b'{"id":"chatcmpl-meta","object":"chat.completion","created":1,"model":"gpt-4o-mini",' + b'"choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],' + b'"usage":{"prompt_tokens":7,"completion_tokens":2,"total_tokens":9}}' + ) + ) + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + sinks: SpanSinks + newrelic: ConnectSink + upstream: Wire + + def sink(self, backend: Backend) -> str: + return self.newrelic.control_url if backend == "newrelic" else self.sinks.arize + + def destination(self, backend: Backend, extra: Mapping[str, str]) -> dict[str, JsonValue]: + tag: Final = uuid.uuid4().hex + credentials: Final[Mapping[str, str]] = ( + {"newrelic_api_key": f"nr-{tag}", "newrelic_region": "us"} + if backend == "newrelic" + else {"arize_space_key": f"space-{tag}", "arize_api_key": f"arize-{tag}"} + ) + return { + "callback_name": backend, + "callback_type": "success_and_failure", + "callback_vars": {**credentials, **extra}, + } + + def team_key(self, scenario: Scenario, backend: Backend, extra: Mapping[str, str]) -> str: + team: Final = scenario.team() + attached: Final = self.proxy.request("POST", f"/team/{team}/callback", self.destination(backend, extra)) + assert attached.status_code == 200, attached.text + return scenario.key(team_id=team) + + def send(self, key: str, model: str, metadata: Mapping[str, JsonValue]) -> str: + trace_id: Final = uuid.uuid4().hex + response: Final = self.proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "hi"}], "metadata": dict(metadata)}, + key=key, + headers={"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"}, + ) + assert response.status_code == 200, response.text + return trace_id + + def request_spans(self, url: str, since: int, trace_id: str) -> tuple[Span, ...]: + return _server_and_model_spans( + eventually( + lambda: spans_for_trace(recorded_spans(url, since)[1], trace_id), + lambda found: len(_server_and_model_spans(found)) >= 2, + seconds=40, + ) + ) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("destination-metadata-keys") + with ( + wire_server(_chat) as upstream, + owned_sinks(directory / "sinks") as sinks, + owned_connect_sink(directory / "newrelic") as newrelic, + gateway_from_environment() as gateway, + owned_proxy_process( + gateway, + directory, + { + "LITELLM_OTEL_V2": "1", + "OTEL_BSP_SCHEDULE_DELAY": "300", + "LITELLM_OTEL_BAGGAGE_METADATA_KEYS": "requester_metadata.correlation_id", + "ARIZE_HTTP_ENDPOINT": sinks.arize + "/v1/traces", + "HTTPS_PROXY": newrelic.proxy_url, + "NO_PROXY": "127.0.0.1,localhost", + "REQUESTS_CA_BUNDLE": newrelic.ca_pem, + }, + config=audit_configuration_writer(sinks)(directory, {}), + remove_environment=("OTEL_ENDPOINT", "OTEL_EXPORTER", "OTEL_EXPORTER_OTLP_ENDPOINT"), + workers=2, + ) as owned, + ): + yield Rig(owned.gateway, sinks, newrelic, upstream) + + +def _server_and_model_spans(spans: tuple[Span, ...]) -> tuple[Span, ...]: + return tuple(span for span in spans if span["kind"] == SERVER_KIND or OPERATION in span["attributes"]) + + +def _exported_metadata(spans: tuple[Span, ...]) -> list[dict[str, JsonValue]]: + return [ + {name: value for name, value in span["attributes"].items() if name.startswith(METADATA_PREFIX)} + for span in spans + ] + + +def _request_metadata(marker: str) -> dict[str, JsonValue]: + return { + "correlation_id": f"corr-{marker}", + "tenant_id": f"tenant-{marker}", + "order": {"id": f"order-{marker}"}, + "unlisted": f"unlisted-{marker}", + } + + +@pytest.mark.parametrize("level", ["team", "key"]) +@pytest.mark.parametrize("backend", ["newrelic", "arize"]) +def test_destination_keys_reach_that_destination_and_not_the_operator( + rig: Rig, backend: Backend, level: Literal["team", "key"] +) -> None: + with rig.proxy.scenario() as scenario: + model: Final = scenario.model(api_base=rig.upstream.url + "/v1") + keys: Final = {"baggage_metadata_keys": TEAM_KEYS} + key: Final = ( + rig.team_key(scenario, backend, keys) + if level == "team" + else scenario.key(metadata={"logging": [rig.destination(backend, keys)]}) + ) + tenant_since: Final = recorded_spans(rig.sink(backend))[0] + operator_since: Final = recorded_spans(rig.sinks.operator)[0] + marker: Final = uuid.uuid4().hex + trace_id: Final = rig.send(key, model, _request_metadata(marker)) + tenant: Final = rig.request_spans(rig.sink(backend), tenant_since, trace_id) + operator: Final = rig.request_spans(rig.sinks.operator, operator_since, trace_id) + + assert sorted(span["kind"] == SERVER_KIND for span in tenant) == [False, True], [span["name"] for span in tenant] + assert _exported_metadata(tenant) == [ + { + "litellm.metadata.correlation_id": f"corr-{marker}", + "litellm.metadata.tenant_id": f"tenant-{marker}", + "litellm.metadata.order.id": f"order-{marker}", + } + ] * len(tenant), [(span["name"], sorted(span["attributes"])) for span in tenant] + assert _exported_metadata(operator) == [{"litellm.metadata.correlation_id": f"corr-{marker}"}] * len(operator), [ + (span["name"], sorted(span["attributes"])) for span in operator + ] + + +@pytest.mark.parametrize("backend", ["newrelic", "arize"]) +def test_a_destination_without_keys_and_a_request_body_cannot_add_them(rig: Rig, backend: Backend) -> None: + with rig.proxy.scenario() as scenario: + model: Final = scenario.model(api_base=rig.upstream.url + "/v1") + listing: Final = rig.team_key(scenario, backend, {"baggage_metadata_keys": TEAM_KEYS}) + sibling: Final = rig.team_key(scenario, backend, {}) + since: Final = recorded_spans(rig.sink(backend))[0] + marker: Final = uuid.uuid4().hex + listed_trace: Final = rig.send(listing, model, _request_metadata(f"listed-{marker}")) + sibling_trace: Final = rig.send( + sibling, + model, + { + **_request_metadata(marker), + "baggage_metadata_keys": TEAM_KEYS, + "callback_vars": {"baggage_metadata_keys": TEAM_KEYS}, + }, + ) + listed: Final = rig.request_spans(rig.sink(backend), since, listed_trace) + unlisted: Final = rig.request_spans(rig.sink(backend), since, sibling_trace) + + assert {"litellm.metadata.tenant_id": f"tenant-listed-{marker}"}.items() <= _exported_metadata(listed)[0].items() + assert _exported_metadata(unlisted) == [{"litellm.metadata.correlation_id": f"corr-{marker}"}] * len(unlisted), [ + (span["name"], sorted(span["attributes"])) for span in unlisted + ] + + +def _attach(rig: Rig, team: str, keys: str, *, key: str | None = None) -> httpx.Response: + return rig.proxy.request( + "POST", + f"/team/{team}/callback", + rig.destination("newrelic", {"baggage_metadata_keys": keys}), + key=key, + ) + + +def test_a_team_admin_can_set_keys_and_a_plain_member_cannot(rig: Rig) -> None: + with rig.proxy.scenario() as scenario: + team: Final = scenario.team() + admin_key: Final = scenario.key(user_id=scenario.member(team, role="admin"), team_id=team) + member_key: Final = scenario.key(user_id=scenario.member(team), team_id=team) + + refused: Final = _attach(rig, team, "requester_metadata.tenant_id", key=member_key) + accepted: Final = _attach(rig, team, "requester_metadata.tenant_id", key=admin_key) + + assert refused.status_code == 403, refused.text + assert accepted.status_code == 200, accepted.text + + +@pytest.mark.parametrize( + ("keys", "reason"), + [ + ("requester_metadata.user_api_key_user_id", "proxy identity attribute"), + ("tenant_id", "requester_metadata."), + ("requester_metadata.headers.x-internal-token", "would export request headers"), + (",".join(f"requester_metadata.k{index}" for index in range(21)), "at most 20 keys"), + ], +) +def test_unusable_keys_are_refused_on_team_and_key_callbacks(rig: Rig, keys: str, reason: str) -> None: + with rig.proxy.scenario() as scenario: + team_response: Final = _attach(rig, scenario.team(), keys) + key_response: Final = rig.proxy.request( + "POST", + "/key/generate", + {"metadata": {"logging": [rig.destination("newrelic", {"baggage_metadata_keys": keys})]}}, + ) + + assert [(response.status_code, reason in response.text) for response in (team_response, key_response)] == [ + (422, True), + (400, True), + ], [team_response.text, key_response.text] + + +def test_unset_keys_leave_the_destination_on_the_global_allowlist(rig: Rig) -> None: + with rig.proxy.scenario() as scenario: + model: Final = scenario.model(api_base=rig.upstream.url + "/v1") + key: Final = rig.team_key(scenario, "newrelic", MappingProxyType({})) + since: Final = recorded_spans(rig.newrelic.control_url)[0] + marker: Final = uuid.uuid4().hex + spans: Final = rig.request_spans( + rig.newrelic.control_url, since, rig.send(key, model, _request_metadata(marker)) + ) + + assert _exported_metadata(spans) == [{"litellm.metadata.correlation_id": f"corr-{marker}"}] * len(spans) diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 6dd9ff35026..256b9e5e480 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -61,6 +61,7 @@ from litellm.integrations.otel.plumbing.context import ( destination_backends, request_destinations, set_request_destinations, + set_request_metadata, ) from litellm.integrations.otel.plumbing.providers import ( TenantFanOutSpanProcessor, @@ -589,6 +590,132 @@ class TestFanOut: "chat gpt-4", } + def test_destination_metadata_keys_reach_only_that_destination(self): + team, sibling, operator = InMemorySpanExporter(), InMemorySpanExporter(), InMemorySpanExporter() + exporters: Final = {"http://team.local": team, "http://sibling.local": sibling} + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator)) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda destination: SimpleSpanProcessor(exporters[destination.endpoint]) + ) + ) + + def run(): + set_request_destinations( + ( + OtelDestination( + endpoint="http://team.local", + metadata_keys=("requester_metadata.tenant_id", "requester_metadata.order.id"), + ), + OtelDestination(endpoint="http://sibling.local"), + ) + ) + set_request_metadata( + MappingProxyType( + { + "requester_metadata.tenant_id": "tenant-1", + "requester_metadata.order.id": "order-1", + "requester_metadata.unlisted": "nope", + } + ) + ) + emit(provider) + + in_fresh_context(run) + + team_attributes: Final = {key: value for key, value in (team.get_finished_spans()[0].attributes or {}).items()} + assert {key: value for key, value in team_attributes.items() if key.startswith("litellm.metadata.")} == { + "litellm.metadata.tenant_id": "tenant-1", + "litellm.metadata.order.id": "order-1", + } + assert [ + key + for span in (*sibling.get_finished_spans(), *operator.get_finished_spans()) + for key in span.attributes or {} + if key.startswith("litellm.metadata.") + ] == [] + + @staticmethod + def _team_export( + metadata: Mapping[str, str], + attributes: Mapping[str, str] = MappingProxyType({}), + limits: SpanLimits | None = None, + ) -> ReadableSpan: + team: Final = InMemorySpanExporter() + provider: Final = TracerProvider() if limits is None else TracerProvider(span_limits=limits) + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=lambda _: SimpleSpanProcessor(team))) + + def run(): + set_request_destinations((OtelDestination(endpoint="http://team.local", metadata_keys=tuple(metadata)),)) + set_request_metadata(MappingProxyType(dict(metadata))) + with get_tracer(provider, "litellm").start_as_current_span("chat gpt-4") as span: + span.set_attributes(dict(attributes)) + + in_fresh_context(run) + return team.get_finished_spans()[0] + + def test_destination_metadata_key_never_replaces_an_attribute_the_span_carries(self): + exported: Final = self._team_export( + {"requester_metadata.tenant_id": "from-caller"}, + {"litellm.metadata.tenant_id": "from-operator-allowlist"}, + ) + + assert (exported.attributes or {})["litellm.metadata.tenant_id"] == "from-operator-allowlist" + + def test_destination_metadata_obeys_the_span_value_length_limit(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", "8") + exported: Final = self._team_export({"requester_metadata.tenant_id": "x" * 1_048_576}) + + assert (exported.attributes or {})["litellm.metadata.tenant_id"] == "x" * 8 + + def test_destination_metadata_obeys_a_providers_own_span_limits(self): + exported: Final = self._team_export( + {"requester_metadata.a": "abcdefgh", "requester_metadata.b": "2", "requester_metadata.c": "3"}, + {"litellm.team.id": "team-a", "litellm.metadata.correlation_id": "corr"}, + SpanLimits(max_span_attributes=3, max_span_attribute_length=4), + ) + + assert dict(exported.attributes or {}) == { + "litellm.team.id": "team", + "litellm.metadata.correlation_id": "corr", + "litellm.metadata.a": "abcd", + } + assert exported.dropped_attributes == 2 + + def test_destination_metadata_counts_against_the_view_after_content_is_redacted(self): + prompt: Final = {f"llm.input_messages.{index}.message.content": "hello" for index in range(4)} + exported: Final = self._team_export( + {"requester_metadata.correlation_id": "corr-1"}, prompt, SpanLimits(max_span_attributes=4) + ) + + assert dict(exported.attributes or {}) == {"litellm.metadata.correlation_id": "corr-1"} + assert exported.dropped_attributes == 0 + + def test_sampled_destination_keeps_the_metadata_of_the_request_that_ended_its_spans(self): + team: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=lambda _: SimpleSpanProcessor(team))) + destination: Final = OtelDestination( + endpoint="http://team.local", + metadata_keys=("requester_metadata.tenant_id",), + success_sampling_rate=1.0, + error_sampling_rate=1.0, + ) + + def run(tenant: str): + set_request_destinations((destination,)) + set_request_metadata(MappingProxyType({"requester_metadata.tenant_id": tenant})) + emit(provider) + + in_fresh_context(run, "tenant-a") + in_fresh_context(run, "tenant-b") + + assert sorted(span.attributes["litellm.metadata.tenant_id"] for span in team.get_finished_spans()) == [ + "tenant-a", + "tenant-b", + ] + def test_team_exclusions_replace_empty_global_and_operator_keeps_all_database_spans(self): team, inherited, operator = InMemorySpanExporter(), InMemorySpanExporter(), InMemorySpanExporter() exporters: Final = {"http://team.local": team, "http://inherited.local": inherited} diff --git a/tests/unit/proxy/common_utils/test_callback_config_validation.py b/tests/unit/proxy/common_utils/test_callback_config_validation.py index ac1fe2d785e..19cac032c8a 100644 --- a/tests/unit/proxy/common_utils/test_callback_config_validation.py +++ b/tests/unit/proxy/common_utils/test_callback_config_validation.py @@ -345,7 +345,7 @@ def test_an_unsupported_capture_message_content_is_rejected_on_key_logging_metad @pytest.mark.usefixtures("otel_v2_on") -@pytest.mark.parametrize("callback_var", ["capture_message_content", "excluded_services"]) +@pytest.mark.parametrize("callback_var", ["capture_message_content", "excluded_services", "baggage_metadata_keys"]) def test_key_logging_metadata_rejects_non_string_destination_callback_vars(callback_var: str) -> None: error: Final = logging_metadata_config_error( { @@ -575,3 +575,72 @@ def test_a_non_string_credential_name_is_not_a_reference(name: object) -> None: credential_reference_rejection({"logging": [entry]}, None, is_proxy_admin=False, is_defined=lambda _: False) is None ) + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.parametrize("callback_name", ["langfuse_otel", "arize", "weave_otel", "newrelic"]) +def test_baggage_metadata_keys_is_accepted_on_every_otel_v2_destination(callback_name: str) -> None: + assert ( + callback_config_error( + callback_name, {"baggage_metadata_keys": "requester_metadata.tenant_id, requester_metadata.order.id"} + ) + is None + ) + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.parametrize("callback_name", ["langfuse", "datadog", "otel", "arize_phoenix", None]) +def test_baggage_metadata_keys_is_rejected_where_it_would_never_take_effect(callback_name: str | None) -> None: + error: Final = callback_config_error(callback_name, {"baggage_metadata_keys": "requester_metadata.tenant_id"}) + assert error is not None and "baggage_metadata_keys" in error and "newrelic" in error + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.parametrize( + ("value", "reason"), + [ + ("tenant_id", "requester_metadata."), + ("requester_metadata.", "requester_metadata."), + ("requester_metadata.tenant_id,", "requester_metadata."), + ("user_api_key_user_id", "requester_metadata."), + ("requester_metadata.user_api_key_user_id", "proxy identity attribute"), + ("requester_metadata.user_api_key_hash", "proxy identity attribute"), + ("requester_metadata.requester_ip_address", "proxy identity attribute"), + ("requester_metadata.headers.x-internal-token", "would export request headers"), + ("requester_metadata.headers", "would export request headers"), + ("requester_metadata.Headers.authorization", "would export request headers"), + (",".join(f"requester_metadata.k{index}" for index in range(21)), "at most 20 keys"), + ], +) +def test_unusable_baggage_metadata_keys_are_rejected(value: str, reason: str) -> None: + error: Final = callback_config_error("newrelic", {"newrelic_api_key": "nr-key", "baggage_metadata_keys": value}) + assert error is not None and reason in error, error + + +def test_baggage_metadata_keys_is_rejected_while_otel_v2_is_off(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + try: + error: Final = callback_config_error("newrelic", {"baggage_metadata_keys": "requester_metadata.tenant_id"}) + finally: + is_otel_v2_enabled.cache_clear() + assert error is not None and "baggage_metadata_keys" in error and "LITELLM_OTEL_V2" in error + + +@pytest.mark.parametrize( + ("stored", "rejected"), + [ + ("requester_metadata.b, requester_metadata.a", False), + ("requester_metadata.a", True), + ], +) +def test_entries_for_one_backend_share_one_baggage_metadata_keys_list(stored: str, rejected: bool) -> None: + error: Final = conflicting_callback_var_error( + "newrelic", + "baggage_metadata_keys", + {"baggage_metadata_keys": "requester_metadata.a,requester_metadata.b"}, + [("newrelic", {"baggage_metadata_keys": stored})], + ) + assert (error is not None) is rejected, error diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f5f75e65c83..6daedc3bfbd 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -18283,6 +18283,7 @@ export interface paths { * An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. * "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out * - excluded_services: For OTel v2 callbacks, a comma-separated list of datastore names such as "redis,postgres", or "none" to exclude nothing. An explicit value replaces the global exclusions; omitted follows them + * - baggage_metadata_keys: For OTel v2 callbacks, a comma-separated list of request metadata keys such as "requester_metadata.correlation_id", exported as litellm.metadata. on this destination's spans only, on top of the global LITELLM_OTEL_BAGGAGE_METADATA_KEYS. At most 20 keys; names of proxy identity fields (user_api_key_*, requester_ip_address) and request headers (requester_metadata.headers.*) are rejected * - gcs_bucket_name: The name of the GCS bucket * - gcs_path_service_account: The path to the GCS service account * - langsmith_api_key: The API key for the Langsmith callback From 89149b4544e056d52b46774bf5723b2ac57c385a Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:11:47 -0800 Subject: [PATCH 112/189] refactor(typing): clear 1141 basedpyright Any errors across 182 files (#50711) * refactor(typing): clear 340 basedpyright Any errors (wave 1) * refactor(typing): clear 497 basedpyright Any errors (wave 2) * refactor(typing): clear 74 basedpyright Any errors (wave 3) * refactor(typing): clear 143 basedpyright Any errors (wave 4) * refactor(typing): clear 106 basedpyright Any errors (wave 5) * fix(typing): keep router fence visibility and py310-safe Self import Bind the Protocol-narrowed routers in pass_through_endpoints and rag ingestion to router-named locals so router_surface_check still sees the aadapter_completion and aocr uses, and import Self from typing_extensions in prisma_client for check_py310_typing_imports. * perf(core_helpers): fast-path dicts and lists in the checked JSON helpers as_str_mapping now tries the concrete dict check before the Mapping ABC check, and iter_items returns lists before the runtime-checkable Protocol check. Both return the same object as before, and roughly halve the overhead CodSpeed flagged on the A2A text-extraction benchmarks. * refactor(realtime): keep the base TypeError for a non-mapping translation expires_after * fix(proxy): restore the require_attributes import the main merge dropped * test: cover the retyped realtime, vector store, agent, stream, video and provider response paths * fix(realtime): name the public request types in non-mapping TypeErrors keyword_arguments wraps expires_after and transcription_session so a non-mapping value raises the same TypeError text as before the typing pass. Tests narrow Greptile's pegasus case with isinstance and freeze its model. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../proxy/hooks/managed_vector_stores.py | 2 +- litellm/assistants/main.py | 51 ++- litellm/batches/batch_utils.py | 9 +- litellm/batches/main.py | 13 +- litellm/caching/caching_handler.py | 28 +- litellm/caching/qdrant_semantic_cache.py | 8 +- litellm/google_genai/adapters/handler.py | 9 +- litellm/google_genai/main.py | 13 +- litellm/harness/endpoint.py | 6 +- litellm/images/main.py | 11 +- .../clickhouse/clickhouse_spend_logger.py | 7 +- .../cloudzero/cz_resource_names.py | 40 ++- litellm/integrations/custom_logger.py | 2 +- .../integrations/dotprompt/prompt_manager.py | 16 +- .../focus/destinations/gcs_destination.py | 11 +- litellm/integrations/galileo.py | 77 ++++- .../gitlab/gitlab_prompt_manager.py | 23 +- litellm/integrations/langfuse/langfuse.py | 7 +- .../opik/opik_payload_builder/extractors.py | 19 +- .../handler.py | 12 +- .../chat_completion_agentic_loop.py | 15 +- litellm/litellm_core_utils/core_helpers.py | 36 ++- .../exception_mapping_utils.py | 34 +- .../json_validation_rule.py | 18 +- litellm/litellm_core_utils/litellm_logging.py | 152 ++++++--- .../litellm_core_utils/llm_cost_calc/utils.py | 2 +- .../convert_dict_to_response.py | 19 +- .../prompt_templates/factory.py | 34 +- .../huggingface_template_handler.py | 8 +- .../litellm_core_utils/realtime_streaming.py | 34 +- litellm/litellm_core_utils/redact_messages.py | 2 +- .../streaming_chunk_builder_utils.py | 20 +- .../litellm_core_utils/streaming_handler.py | 234 ++++++++------ litellm/llms/a2a/common_utils.py | 30 +- .../aiohttp_openai/chat/transformation.py | 15 +- litellm/llms/anthropic/chat/transformation.py | 80 +++-- litellm/llms/anthropic/common_utils.py | 6 +- .../messages/adapters/transformation.py | 22 +- .../anthropic/messages/native/mcp_handler.py | 7 +- .../messages/native/streaming_iterator.py | 9 +- litellm/llms/azure_ai/agents/handler.py | 64 ++-- .../azure_ai/vector_stores/transformation.py | 15 +- litellm/llms/base_llm/base_model_iterator.py | 13 +- litellm/llms/base_llm/chat/transformation.py | 4 +- .../llms/base_llm/videos/transformation.py | 2 +- ...mazon_twelvelabs_pegasus_transformation.py | 16 +- .../amazon_nova_canvas_transformation.py | 23 +- .../guardrail_translation/handler.py | 89 ++++-- .../llms/bedrock/realtime/transformation.py | 25 +- .../bedrock/vector_stores/transformation.py | 17 +- .../codestral/completion/transformation.py | 16 +- litellm/llms/cohere/common_utils.py | 8 +- .../llms/cohere/embed/v1_transformation.py | 22 +- litellm/llms/custom_httpx/llm_http_handler.py | 72 +++-- .../llms/deprecated_providers/aleph_alpha.py | 33 +- litellm/llms/edenai/videos/transformation.py | 2 +- .../llms/fireworks_ai/chat/transformation.py | 7 +- litellm/llms/gigachat/chat/transformation.py | 30 +- .../llms/gigachat/embedding/transformation.py | 42 ++- .../huggingface/embedding/transformation.py | 66 +++- litellm/llms/minimax/videos/transformation.py | 2 +- litellm/llms/nlp_cloud/chat/transformation.py | 14 +- .../audio_transcription/riva_grpc.py | 24 +- litellm/llms/ollama/chat/transformation.py | 23 +- litellm/llms/oobabooga/chat/oobabooga.py | 13 +- .../llms/openai/chat/gpt_transformation.py | 27 +- litellm/llms/openai/cost_calculation.py | 16 +- .../llms/openai/image_edit/transformation.py | 14 +- .../llms/openai/image_variations/handler.py | 18 +- .../guardrail_translation/handler.py | 47 ++- .../transcriptions/whisper_transformation.py | 17 +- .../openai/vector_stores/transformation.py | 21 +- litellm/llms/openai/videos/transformation.py | 136 +++++--- litellm/llms/openai_like/chat/handler.py | 32 +- litellm/llms/openai_like/embedding/handler.py | 11 +- litellm/llms/predibase/chat/transformation.py | 11 +- .../ragflow/vector_stores/transformation.py | 7 +- litellm/llms/replicate/chat/transformation.py | 7 +- .../sagemaker/completion/transformation.py | 32 +- litellm/llms/snowflake/chat/transformation.py | 8 +- .../audio_transcription/transformation.py | 27 +- .../stability/image_edit/transformations.py | 18 +- litellm/llms/vertex_ai/common_utils.py | 48 +-- .../llms/vertex_ai/files/transformation.py | 11 +- .../image_generation_handler.py | 16 +- .../llms/vertex_ai/realtime/transformation.py | 7 +- .../llama/transformation.py | 3 +- .../volcengine/embedding/transformation.py | 14 +- .../llms/volcengine/videos/transformation.py | 2 +- litellm/llms/watsonx/common_utils.py | 17 +- .../llms/watsonx/completion/transformation.py | 14 +- litellm/llms/xai/chat/transformation.py | 54 +++- litellm/llms/you_com/search/transformation.py | 27 +- litellm/main.py | 18 +- litellm/models/base.py | 39 ++- litellm/proxy/_lazy_openapi_snapshot.py | 4 +- .../proxy/agent_endpoints/a2a_endpoints.py | 11 +- .../model_deprecation_alerts.py | 8 +- litellm/proxy/auth/handle_jwt.py | 13 +- litellm/proxy/auth/litellm_license.py | 26 +- litellm/proxy/client/cli/commands/keys.py | 38 +-- litellm/proxy/client/cli/commands/models.py | 11 +- litellm/proxy/common_request_processing.py | 59 ++-- litellm/proxy/common_utils/callback_utils.py | 10 +- .../common_utils/encrypt_decrypt_utils.py | 15 +- .../proxy/common_utils/reset_budget_job.py | 6 +- .../proxy/db/budget_window_spend_writer.py | 17 +- litellm/proxy/db/db_spend_update_writer.py | 64 +++- litellm/proxy/db/exception_handler.py | 29 +- litellm/proxy/db/model_usage_rollup.py | 9 +- litellm/proxy/db/prisma_client.py | 164 ++++++++++ litellm/proxy/db/routing_prisma_wrapper.py | 10 + litellm/proxy/db/spend_log_tool_index.py | 12 +- .../agent_skills_archive.py | 21 +- .../proxy/fine_tuning_endpoints/endpoints.py | 48 ++- .../cato_networks/cato_networks.py | 33 +- .../guardrail_hooks/custom_code/primitives.py | 8 +- .../guardrail_hooks/enkryptai/enkryptai.py | 10 +- .../generic_guardrail_api.py | 21 +- .../mcp_jwt_signer/mcp_jwt_signer.py | 6 +- .../guardrail_hooks/microsoft_purview/base.py | 6 +- .../guardrail_hooks/pangea/pangea.py | 40 ++- .../panw_prisma_airs/panw_prisma_airs.py | 27 +- .../guardrail_hooks/tool_permission.py | 26 +- .../proxy/guardrails/guardrail_registry.py | 15 +- litellm/proxy/hooks/batch_rate_limiter.py | 9 +- litellm/proxy/litellm_pre_call_utils.py | 25 +- .../access_group_endpoints.py | 24 +- .../internal_user_endpoints.py | 4 +- .../key_management_endpoints.py | 13 +- .../management_v1/spend_logs.py | 24 +- ...model_access_group_management_endpoints.py | 29 +- .../model_management_endpoints.py | 39 ++- .../organization_endpoints.py | 17 +- .../team_callback_endpoints.py | 4 +- litellm/proxy/mcp_server/db.py | 26 +- .../mcp_server/discoverable_endpoints.py | 35 ++- .../proxy/mcp_server/mcp_server_manager.py | 40 +-- litellm/proxy/mcp_server/operations.py | 2 +- litellm/proxy/mcp_server/rest_endpoints.py | 6 +- litellm/proxy/mcp_server/tool_search.py | 15 +- .../admission_control_middleware.py | 10 +- .../billable_request_metrics_middleware.py | 32 +- .../per_request_root_path_middleware.py | 14 +- .../request_size_limit_middleware.py | 20 +- .../openai_files_endpoints/common_utils.py | 4 +- .../file_content_streaming_handler.py | 12 +- .../openai_files_endpoints/files_endpoints.py | 16 +- .../llm_passthrough_endpoints.py | 32 +- .../anthropic_passthrough_logging_handler.py | 8 +- .../pass_through_endpoints.py | 25 +- litellm/proxy/proxy_server.py | 75 +++-- litellm/proxy/rag_endpoints/endpoints.py | 9 +- litellm/proxy/rerank_endpoints/endpoints.py | 4 +- .../proxy/response_api_endpoints/endpoints.py | 2 +- .../proxy/response_polling/polling_handler.py | 15 +- .../search_tool_management.py | 14 +- .../search_endpoints/search_tool_registry.py | 10 +- .../spend_tracking/key_metadata_recovery.py | 15 +- .../spend_tracking/ptu_flat_cost_rollup.py | 13 +- .../proxy_setting_endpoints.py | 16 +- litellm/proxy/utils.py | 34 +- .../management_endpoints.py | 11 +- litellm/proxy/vector_store_endpoints/utils.py | 17 +- litellm/proxy/video_endpoints/utils.py | 13 +- litellm/rag/ingestion/base_ingestion.py | 29 +- litellm/realtime_api/main.py | 26 +- litellm/repositories/table_repositories.py | 14 +- litellm/rerank_api/main.py | 18 +- .../mcp/litellm_proxy_mcp_handler.py | 3 +- litellm/router.py | 44 ++- litellm/router_api.py | 291 +++++++++++++++++- litellm/router_rust/invoke/streaming.py | 15 +- litellm/router_rust/surface/legacy_members.py | 8 +- .../quality_router/quality_router.py | 2 +- litellm/router_utils/cooldown_callbacks.py | 5 +- litellm/router_utils/cooldown_handlers.py | 2 +- .../secret_managers/secret_manager_handler.py | 41 +-- .../text_completions/streaming_iterator.py | 28 +- litellm/utils.py | 78 +++-- .../vector_stores/vector_store_registry.py | 17 +- litellm/videos/main.py | 58 +++- .../test_streaming_handler.py | 131 ++++++++ .../unit/llms/azure_ai/agents/test_handler.py | 58 +++- ...t_azure_ai_vector_stores_transformation.py | 36 +++ .../test_twelvelabs_pegasus_transformation.py | 46 +++ ...est_bedrock_vector_store_transformation.py | 35 +++ .../llms/cohere/test_cohere_common_utils.py | 20 +- ...st_huggingface_embedding_transformation.py | 106 +++++++ .../test_nlp_cloud_chat_transformation.py | 52 ++++ .../ollama/test_ollama_chat_transformation.py | 59 ++++ tests/unit/realtime_api/test_main.py | 89 +++++- tests/unit/test_main.py | 56 +++- tests/unit/videos/test_main.py | 79 +++++ 194 files changed, 4116 insertions(+), 1332 deletions(-) create mode 100644 tests/unit/llms/huggingface/embedding/test_huggingface_embedding_transformation.py diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py b/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py index fcaf58099b9..6e16ed8f187 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py @@ -123,7 +123,7 @@ class _PROXY_LiteLLMManagedVectorStores( """ # Use the router to create the vector store response: Final = await llm_router.avector_store_create(model=model, **request_data) - return cast(VectorStoreCreateResponse, response) + return response # ============================================================================ # VECTOR STORE CRUD OPERATIONS diff --git a/litellm/assistants/main.py b/litellm/assistants/main.py index a5721c6f047..90a083bed2d 100644 --- a/litellm/assistants/main.py +++ b/litellm/assistants/main.py @@ -5,14 +5,16 @@ import contextvars import os from collections.abc import Coroutine, Iterable, Mapping, Sequence from functools import partial -from typing import Any, Final, Literal +from typing import Any, Final, Literal, Protocol import httpx from openai import AsyncOpenAI, OpenAI from openai.types.beta.assistant import Assistant from openai.types.beta.assistant_deleted import AssistantDeleted +from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.types.router import GenericLiteLLMParams from litellm.utils import ( exception_type, @@ -31,6 +33,21 @@ from .utils import get_optional_params_add_message ####### ENVIRONMENT VARIABLES ################### openai_assistants_api: Final = OpenAIAssistantsAPI() azure_assistants_api: Final = AzureAssistantsAPI() +_EXTRA_BODY: Final = TypeAdapter(object) + + +class _PopsKeys(Protocol): + def pop(self, key: str, default: None, /) -> object: ... + + +def _pop_azure_ad_token(extra_body: object) -> object: + return require_attributes(extra_body, _PopsKeys, "pop").pop("azure_ad_token", None) + + +def _popped_azure_ad_token(extra_body: object) -> str | None: + token: Final = _pop_azure_ad_token(extra_body) + return token if isinstance(token, str) else None + ### ASSISTANTS ### @@ -146,10 +163,10 @@ def get_assistants( or get_secret("AZURE_API_KEY") ) - extra_body: Final = optional_params.get("extra_body", {}) + extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {})) azure_ad_token: str | None = None if extra_body is not None: - azure_ad_token = extra_body.pop("azure_ad_token", None) + azure_ad_token = _popped_azure_ad_token(extra_body) else: azure_ad_token = get_secret("AZURE_AD_TOKEN") @@ -331,10 +348,10 @@ def create_assistants( or get_secret("AZURE_API_KEY") ) - extra_body: Final = optional_params.get("extra_body", {}) + extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {})) azure_ad_token: str | None = None if extra_body is not None: - azure_ad_token = extra_body.pop("azure_ad_token", None) + azure_ad_token = _popped_azure_ad_token(extra_body) else: azure_ad_token = get_secret("AZURE_AD_TOKEN") @@ -481,10 +498,10 @@ def delete_assistant( or get_secret("AZURE_API_KEY") ) - extra_body: Final = optional_params.get("extra_body", {}) + extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {})) azure_ad_token: str | None = None if extra_body is not None: - azure_ad_token = extra_body.pop("azure_ad_token", None) + azure_ad_token = _popped_azure_ad_token(extra_body) else: azure_ad_token = get_secret("AZURE_AD_TOKEN") @@ -657,10 +674,10 @@ def create_thread( api_version: str | None = optional_params.api_version or litellm.api_version or get_secret("AZURE_API_VERSION") - extra_body: Final = optional_params.get("extra_body", {}) + extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {})) azure_ad_token: str | None = None if extra_body is not None: - azure_ad_token = extra_body.pop("azure_ad_token", None) + azure_ad_token = _popped_azure_ad_token(extra_body) else: azure_ad_token = get_secret("AZURE_AD_TOKEN") @@ -804,10 +821,10 @@ def get_thread( or get_secret("AZURE_API_KEY") ) - extra_body: Final = optional_params.get("extra_body", {}) + extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {})) azure_ad_token: str | None = None if extra_body is not None: - azure_ad_token = extra_body.pop("azure_ad_token", None) + azure_ad_token = _popped_azure_ad_token(extra_body) else: azure_ad_token = get_secret("AZURE_AD_TOKEN") @@ -982,10 +999,10 @@ def add_message( or get_secret("AZURE_API_KEY") ) - extra_body: Final = optional_params.get("extra_body", {}) + extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {})) azure_ad_token: str | None = None if extra_body is not None: - azure_ad_token = extra_body.pop("azure_ad_token", None) + azure_ad_token = _popped_azure_ad_token(extra_body) else: azure_ad_token = get_secret("AZURE_AD_TOKEN") @@ -1134,10 +1151,10 @@ def get_messages( or get_secret("AZURE_API_KEY") ) - extra_body: Final = optional_params.get("extra_body", {}) + extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {})) azure_ad_token: str | None = None if extra_body is not None: - azure_ad_token = extra_body.pop("azure_ad_token", None) + azure_ad_token = _popped_azure_ad_token(extra_body) else: azure_ad_token = get_secret("AZURE_AD_TOKEN") @@ -1333,10 +1350,10 @@ def run_thread( or get_secret("AZURE_API_KEY") ) - extra_body: Final = optional_params.get("extra_body", {}) + extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {})) azure_ad_token = None if extra_body is not None: - azure_ad_token = extra_body.pop("azure_ad_token", None) + azure_ad_token = _pop_azure_ad_token(extra_body) else: azure_ad_token = get_secret("AZURE_AD_TOKEN") diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 62ccb07f695..1c830e45464 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -12,6 +12,7 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils import batch_utils as core_batch_utils from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo @@ -849,14 +850,14 @@ def _get_batch_job_usage_from_response_body( return usage -def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> Mapping[str, Any]: +def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, object]) -> Mapping[str, object]: """ Get the ``result`` object from a line of an Anthropic message batch results JSONL file. Anthropic batch results lines look like: ``{"custom_id": ..., "result": {"type": "succeeded", "message": {..., "usage": {...}}}}`` """ - return batch_results_line.get("result", None) or {} + return decoded_json_object(batch_results_line.get("result", None) or {}) def _get_response_from_batch_job_output_file( @@ -866,7 +867,9 @@ def _get_response_from_batch_job_output_file( Get the response from the batch job output file """ if custom_llm_provider == "anthropic": - return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("message", None) or {} + return decoded_json_object( + _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("message", None) or {} + ) if custom_llm_provider == "bedrock": return batch_job_output_file.get("modelOutput", None) or {} _response: Final = cast( # cast-ok: batch output response comes from provider JSON diff --git a/litellm/batches/main.py b/litellm/batches/main.py index d88f549b06b..212205a9d7b 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -15,13 +15,15 @@ import contextvars import os from collections.abc import Coroutine from functools import partial -from typing import Any, Final, Literal, cast +from typing import Any, Final, Literal, Protocol, cast import httpx from openai.types.batch import BatchRequestCounts +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler from litellm.llms.azure.batches.handler import AzureBatchesAPI @@ -60,10 +62,15 @@ azure_batches_instance: Final = AzureBatchesAPI() vertex_ai_batches_instance: Final = VertexAIBatchPrediction(gcs_bucket_name="") anthropic_batches_instance: Final = AnthropicBatchesHandler() xai_batches_instance: Final = XAIBatchesHandler() +_EXTRA_BODY: Final = TypeAdapter(object) base_llm_http_handler = BaseLLMHTTPHandler() ################################################# +class _PopsKeys(Protocol): + def pop(self, key: str, default: None, /) -> object: ... + + def _resolve_timeout( optional_params: GenericLiteLLMParams, kwargs: dict[str, Any], @@ -485,9 +492,9 @@ def _handle_retrieve_batch_providers_without_provider_config( or get_secret_str("AZURE_API_KEY") ) - extra_body: Final = optional_params.get("extra_body", {}) + extra_body: Final = _EXTRA_BODY.validate_python(optional_params.get("extra_body", {})) if extra_body is not None: - extra_body.pop("azure_ad_token", None) + require_attributes(extra_body, _PopsKeys, "pop").pop("azure_ad_token", None) else: get_secret_str("AZURE_AD_TOKEN") diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index ab5d2332c21..a3cf54916f1 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -19,7 +19,7 @@ import datetime import inspect import time from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping -from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar +from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeVar from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError from typing_extensions import TypedDict @@ -72,6 +72,8 @@ from litellm.litellm_core_utils.core_helpers import ( as_object_list, as_str_mapping, get_parent_otel_span_from_kwargs, + json_contains, + require_attributes, ) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper @@ -246,6 +248,15 @@ def _stream_replay_requested(kwargs: Mapping[str, object]) -> bool: return converted_stream_requested(kwargs) and not kwargs.get("_agentic_loop_depth") +class _CacheControls(Protocol): + def get(self, key: str, default: bool, /) -> object: ... + + +def _cache_reads_disabled(kwargs: Mapping[str, object]) -> bool: + cache_controls: Final = require_attributes(kwargs.get("cache", {}), _CacheControls, "get") + return cache_controls.get("no-cache", False) is True + + def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> bool: """ When the cache hit is replayed as a stream, do not run success callbacks at cache-hit time. @@ -403,7 +414,7 @@ class LLMCachingHandler: if ( (kwargs.get("caching", None) is None and litellm.cache is not None) or kwargs.get("caching", False) is True ) and ( - kwargs.get("cache", {}).get("no-cache", False) is not True + not _cache_reads_disabled(kwargs) ): # allow users to control returning cached responses from the completion function args = args or () final_embedding_cached_response: EmbeddingResponse | None = None @@ -521,7 +532,7 @@ class LLMCachingHandler: kwargs: dict[str, Any], args: tuple[object, ...] | None = None, ) -> CachingHandlerResponse: - cached_result: Any | None = None + cached_result: object | None = None # Check if caching should be performed BEFORE doing expensive kwargs copy if _is_response_cache_excluded(model=model, kwargs=kwargs): @@ -550,7 +561,7 @@ class LLMCachingHandler: verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss") return CachingHandlerResponse(cached_result=None) if cached_result is not None: - if "detail" in cached_result: + if json_contains(cached_result, "detail"): # implies an error occurred pass else: @@ -1460,6 +1471,7 @@ class LLMCachingHandler: Returns: None """ + request_kwargs: Final = dict[str, object](kwargs) litellm_params: Final = { "logger_fn": kwargs.get("logger_fn", None), "acompletion": is_async, @@ -1473,7 +1485,7 @@ class LLMCachingHandler: if litellm.cache is not None: litellm_params["preset_cache_key"] = ( - self.preset_cache_key or litellm.cache.get_preset_cache_key_from_kwargs(**kwargs) + self.preset_cache_key or litellm.cache.get_preset_cache_key_from_kwargs(**request_kwargs) ) else: litellm_params["preset_cache_key"] = None @@ -1483,11 +1495,11 @@ class LLMCachingHandler: user=kwargs.get("user", None), optional_params={}, litellm_params=litellm_params, - input=(kwargs.get("messages", "") if not is_embedding else kwargs.get("input", "")), - api_key=kwargs.get("api_key", None), + input=(request_kwargs.get("messages", "") if not is_embedding else request_kwargs.get("input", "")), + api_key=request_kwargs.get("api_key", None), original_response=str(cached_result), additional_args=None, - stream=kwargs.get("stream", False), + stream=request_kwargs.get("stream", False), custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 560a5ba06b8..501ac2037e1 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -41,6 +41,7 @@ from .base_cache import BaseCache _WAIT_FOR_INDEXING: Final = MappingProxyType({"wait": "true"}) _EMBEDDING_VECTOR: Final = TypeAdapter(object) +_CACHED_RESPONSE: Final = TypeAdapter(object) if TYPE_CHECKING: from litellm.router import Router @@ -175,14 +176,13 @@ class QdrantSemanticCache(BaseCache): else: raise Exception("Error while creating new collection") - def _get_cache_logic(self, cached_response: Any): + def _get_cache_logic(self, cached_response: Any) -> object: if cached_response is None: return cached_response try: - cached_response = json.loads(cached_response) # Convert string to dictionary + return _CACHED_RESPONSE.validate_python(json.loads(cached_response)) except Exception: - cached_response = ast.literal_eval(cached_response) - return cached_response + return _CACHED_RESPONSE.validate_python(ast.literal_eval(cached_response)) def _get_qdrant_cache_key_filter(self, key: str) -> dict: return { diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index 7caa0c62571..db090c9b007 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -2,6 +2,7 @@ from collections.abc import AsyncIterator, Coroutine, Mapping from typing import Final, cast from openai.types.chat import ChatCompletionStreamOptionsParam +from typing_extensions import TypedDict import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -15,6 +16,10 @@ from .transformation import GoogleGenAIAdapter GOOGLE_GENAI_ADAPTER: Final = GoogleGenAIAdapter() +class _TranslatedCompletionKwargs(TypedDict, total=False, extra_items=object): + pass + + def _mark_failure_logged_by_inner_completion(adapter_kwargs: Mapping[str, object]) -> None: outer_logging_obj: Final = adapter_kwargs.get("litellm_logging_obj") if isinstance(outer_logging_obj, LiteLLMLoggingObj): @@ -55,7 +60,7 @@ class GenerateContentToCompletionHandler: **(extra_kwargs or {}), ) - completion_kwargs: Final = dict(completion_request) + completion_kwargs: Final = dict[str, object](completion_request) # Forward extra_kwargs that should be passed to completion call if extra_kwargs is not None: @@ -76,7 +81,7 @@ class GenerateContentToCompletionHandler: if stream_options is not None: completion_kwargs["stream_options"] = stream_options - return GenerateContentCompletionKwargs(**completion_kwargs) + return GenerateContentCompletionKwargs(**_TranslatedCompletionKwargs(**completion_kwargs)) @staticmethod async def async_generate_content_handler( diff --git a/litellm/google_genai/main.py b/litellm/google_genai/main.py index 40c9c7bc36c..71000fec81a 100644 --- a/litellm/google_genai/main.py +++ b/litellm/google_genai/main.py @@ -40,6 +40,7 @@ else: # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() _GENERATE_CONTENT_RESULT: Final = TypeAdapter(object) +_GENERATE_CONTENT_PAYLOAD: Final = TypeAdapter(object) ################################################# @@ -368,8 +369,8 @@ def generate_content( response: Final = _GENERATE_CONTENT_RESULT.validate_python( base_llm_http_handler.generate_content_handler( model=setup_result.model, - contents=contents, - tools=tools, + contents=_GENERATE_CONTENT_PAYLOAD.validate_python(contents), + tools=_GENERATE_CONTENT_PAYLOAD.validate_python(tools), generate_content_provider_config=setup_result.generate_content_provider_config, generate_content_config_dict=setup_result.generate_content_config_dict, custom_llm_provider=setup_result.custom_llm_provider, @@ -465,10 +466,10 @@ async def agenerate_content_stream( return _GENERATE_CONTENT_RESULT.validate_python( await base_llm_http_handler.generate_content_handler( model=setup_result.model, - contents=contents, + contents=_GENERATE_CONTENT_PAYLOAD.validate_python(contents), generate_content_provider_config=setup_result.generate_content_provider_config, generate_content_config_dict=setup_result.generate_content_config_dict, - tools=tools, + tools=_GENERATE_CONTENT_PAYLOAD.validate_python(tools), custom_llm_provider=setup_result.custom_llm_provider, litellm_params=setup_result.litellm_params, logging_obj=setup_result.litellm_logging_obj, @@ -556,10 +557,10 @@ def generate_content_stream( return _GENERATE_CONTENT_RESULT.validate_python( base_llm_http_handler.generate_content_handler( model=setup_result.model, - contents=contents, + contents=_GENERATE_CONTENT_PAYLOAD.validate_python(contents), generate_content_provider_config=setup_result.generate_content_provider_config, generate_content_config_dict=setup_result.generate_content_config_dict, - tools=tools, + tools=_GENERATE_CONTENT_PAYLOAD.validate_python(tools), custom_llm_provider=setup_result.custom_llm_provider, litellm_params=setup_result.litellm_params, logging_obj=setup_result.litellm_logging_obj, diff --git a/litellm/harness/endpoint.py b/litellm/harness/endpoint.py index 249bb2cebb7..fa8105b30e8 100644 --- a/litellm/harness/endpoint.py +++ b/litellm/harness/endpoint.py @@ -37,7 +37,7 @@ from litellm.constants import ( from litellm.harness.context import GatewayTarget from litellm.harness.errors import HarnessError, HarnessInstallFailed from litellm.harness.types import Harness, Usage -from litellm.litellm_core_utils.core_helpers import has_attributes +from litellm.litellm_core_utils.core_helpers import decoded_json_object, has_attributes from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.llms.custom_http import httpxSpecialProvider @@ -722,7 +722,7 @@ class ModelEndpoint: return await litellm.anthropic.messages.acreate(**kwargs) if route == ROUTE_CHAT: if not kwargs.get("stream"): - return await litellm.acompletion(**kwargs) + return await litellm.acompletion(**decoded_json_object(kwargs)) stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field return await litellm.acompletion( **{ @@ -731,7 +731,7 @@ class ModelEndpoint: STREAM_USAGE_INJECTED_FIELD: "include_usage" not in stream_options, } ) - return await litellm.aresponses(**kwargs) + return await litellm.aresponses(**decoded_json_object(kwargs)) async def _call_sdk(self, route: str, body: Mapping[str, object]) -> Response: kwargs = self._sdk_kwargs(body) diff --git a/litellm/images/main.py b/litellm/images/main.py index 79bb07fc394..b2da69bb452 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -6,6 +6,7 @@ from functools import partial from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload from pydantic import TypeAdapter, ValidationError +from typing_extensions import TypedDict if TYPE_CHECKING: from litellm.images.utils import ImageEditRequestUtils @@ -698,7 +699,7 @@ async def aimage_variation(*args, **kwargs) -> ImageResponse: init_response = await loop.run_in_executor(None, func_with_context) if isinstance(init_response, dict) or isinstance(init_response, ImageResponse): ## CACHING SCENARIO if isinstance(init_response, dict): - init_response = ImageResponse(**init_response) + init_response = _image_response_from_fields(init_response) response = init_response elif asyncio.iscoroutine(init_response): response = await init_response @@ -717,6 +718,14 @@ async def aimage_variation(*args, **kwargs) -> ImageResponse: ) +class _ImageResponseFields(TypedDict, total=False, extra_items=object): + pass + + +def _image_response_from_fields(fields: Mapping[str, object]) -> ImageResponse: + return ImageResponse(**_ImageResponseFields(**fields)) + + @client def image_variation( image: FileTypes, diff --git a/litellm/integrations/clickhouse/clickhouse_spend_logger.py b/litellm/integrations/clickhouse/clickhouse_spend_logger.py index ef7442143ca..bc1a142d133 100644 --- a/litellm/integrations/clickhouse/clickhouse_spend_logger.py +++ b/litellm/integrations/clickhouse/clickhouse_spend_logger.py @@ -13,6 +13,7 @@ from types import MappingProxyType from typing import Any, Final, Protocol from pydantic import JsonValue, TypeAdapter, ValidationError +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -57,6 +58,10 @@ class _SupportsItems(Protocol): def items(self) -> Iterable[tuple[object, object]]: ... +class _SpendLogKwargs(TypedDict, total=False, extra_items=object): + standard_logging_object: ReadOnly[StandardLoggingPayload | None] + + def _request_metadata_fields(value: object) -> Iterator[tuple[str, JsonValue]]: if value is None: return @@ -243,7 +248,7 @@ class ClickHouseSpendLogger(ClickHouseBatchLogger): async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: self._log(kwargs) - def _log(self, kwargs: Mapping[str, Any]) -> None: + def _log(self, kwargs: _SpendLogKwargs) -> None: try: payload = kwargs.get("standard_logging_object") if payload is None or _is_trace_ingest(payload): diff --git a/litellm/integrations/cloudzero/cz_resource_names.py b/litellm/integrations/cloudzero/cz_resource_names.py index 9b685996a41..77079818908 100644 --- a/litellm/integrations/cloudzero/cz_resource_names.py +++ b/litellm/integrations/cloudzero/cz_resource_names.py @@ -17,10 +17,16 @@ """CloudZero Resource Names (CZRN) generation and validation for LiteLLM resources.""" import re +from collections.abc import Mapping from enum import Enum -from typing import Any, Final, cast +from typing import Final, Protocol, cast import litellm +from litellm.litellm_core_utils.core_helpers import require_attributes + + +class _SupportsLower(Protocol): + def lower(self) -> str: ... class CZEntityType(str, Enum): @@ -35,7 +41,7 @@ class CZRNGenerator: def __init__(self): """Initialize CZRN generator.""" - def create_from_litellm_data(self, row: dict[str, Any]) -> str: + def create_from_litellm_data(self, row: Mapping[str, object]) -> str: """Create a CZRN from LiteLLM daily spend data. CZRN format: czrn:::::: @@ -49,19 +55,21 @@ class CZRNGenerator: - cloud-local-id: model """ service_type: Final = "litellm" - provider: Final = self._normalize_provider(row.get("custom_llm_provider", "unknown")) + provider: Final = self._normalize_provider( + require_attributes(row.get("custom_llm_provider", "unknown"), _SupportsLower, "lower") + ) region: Final = "cross-region" # Use the actual entity_id (team_id or user_id) as the owner account team_id: Final = row.get("team_id", "unknown") - owner_account_id: Final = self._normalize_component(team_id) + owner_account_id: Final = self._normalize_owner(team_id) resource_type: Final = "llm-usage" # Create a unique identifier with just the model (entity info already in owner_account_id) model: Final = row.get("model", "unknown") - cloud_local_id: Final = model + cloud_local_id: Final = f"{model}" return self.create_from_components( service_type=service_type, @@ -112,7 +120,7 @@ class CZRNGenerator: return cast(tuple[str, str, str, str, str, str], match.groups()) - def _normalize_provider(self, provider: str) -> str: + def _normalize_provider(self, provider: _SupportsLower) -> str: """Normalize provider names to standard CZRN format.""" # Map common provider names to CZRN standards provider_map: Final = { @@ -141,16 +149,14 @@ class CZRNGenerator: return "unknown" # Convert to lowercase unless uppercase is allowed - if not allow_uppercase: - component = component.lower() + return self._clean_component(component if allow_uppercase else component.lower()) - # Replace invalid characters with hyphens - component = re.sub(r"[^a-zA-Z0-9-]", "-", component) + def _normalize_owner(self, owner: object) -> str: + if not owner: + return "unknown" + return self._clean_component(require_attributes(owner, _SupportsLower, "lower").lower()) - # Remove consecutive hyphens - component = re.sub(r"-+", "-", component) - - # Remove leading/trailing hyphens - component = component.strip("-") - - return component or "unknown" + def _clean_component(self, component: str) -> str: + hyphenated: Final = re.sub(r"[^a-zA-Z0-9-]", "-", component) + collapsed: Final = re.sub(r"-+", "-", hyphenated) + return collapsed.strip("-") or "unknown" diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 385522c2435..b582d6b8c4f 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -759,7 +759,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac response: object, plan: AgenticLoopPlan, kwargs: dict, - ) -> Any: + ) -> object: """ Post-process the response returned by the agentic-loop follow-up call. diff --git a/litellm/integrations/dotprompt/prompt_manager.py b/litellm/integrations/dotprompt/prompt_manager.py index 0e52ca0b62f..dfa3cb97f5f 100644 --- a/litellm/integrations/dotprompt/prompt_manager.py +++ b/litellm/integrations/dotprompt/prompt_manager.py @@ -5,14 +5,14 @@ Based on Google's GenAI Kit dotprompt implementation: https://google.github.io/d import re from collections.abc import Mapping from pathlib import Path -from typing import Any, Final +from typing import Any, Final, Protocol import yaml from jinja2 import DictLoader, select_autoescape from jinja2.sandbox import ImmutableSandboxedEnvironment from typing_extensions import NotRequired, ReadOnly, TypedDict -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, require_attributes class _PromptFileJson(TypedDict): @@ -27,6 +27,14 @@ class PromptJsonEntry(TypedDict): metadata: ReadOnly[dict[str, object]] +class _SupportsGet(Protocol): + def get(self, key: str, default: object = None, /) -> object: ... + + +def _metadata_section(metadata: Mapping[str, object], name: str) -> _SupportsGet: + return require_attributes(metadata.get(name, {}), _SupportsGet, "get") + + def strip_version_suffix(prompt_id: str) -> str | None: base, separator, version = prompt_id.rpartition(".v") if separator and base and version.isdigit(): @@ -51,8 +59,8 @@ class PromptTemplate: restricted_keys: Final = ["model", "input", "output"] self.model = self.metadata.get("model") self.input_schema = self.metadata.get("input", {}).get("schema", {}) - self.output_format = self.metadata.get("output", {}).get("format") - self.output_schema = self.metadata.get("output", {}).get("schema", {}) + self.output_format = _metadata_section(self.metadata, "output").get("format") + self.output_schema = _metadata_section(self.metadata, "output").get("schema", {}) self.optional_params = {} for key in self.metadata: if key not in restricted_keys: diff --git a/litellm/integrations/focus/destinations/gcs_destination.py b/litellm/integrations/focus/destinations/gcs_destination.py index 2f7052597ff..82f1a65c217 100644 --- a/litellm/integrations/focus/destinations/gcs_destination.py +++ b/litellm/integrations/focus/destinations/gcs_destination.py @@ -2,8 +2,9 @@ from __future__ import annotations +from collections.abc import Mapping from datetime import timezone -from typing import Any, Final +from typing import Final from litellm._logging import verbose_logger from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase @@ -21,14 +22,14 @@ class FocusGCSDestination(GCSBucketBase, FocusDestination): self, *, prefix: str, - config: dict[str, Any] | None = None, + config: Mapping[str, str] | None = None, ) -> None: - config = config or {} - bucket_name: Final = config.get("bucket_name") + resolved_config: Final[Mapping[str, str]] = config or {} + bucket_name: Final = resolved_config.get("bucket_name") if not bucket_name: raise ValueError("bucket_name must be provided for GCS destination") super().__init__(bucket_name=bucket_name) - service_account_json: Final = config.get("service_account_json") + service_account_json: Final = resolved_config.get("service_account_json") if service_account_json is not None: self.path_service_account_json = service_account_json self.prefix = prefix.rstrip("/") diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index 0f7dcdbcc81..f04604ddad8 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -6,7 +6,7 @@ import re import uuid from collections.abc import Callable, Mapping, Sequence from datetime import datetime, timezone, tzinfo -from typing import Any, Final, Protocol, cast +from typing import Final, Protocol, cast import httpx from pydantic import ConfigDict, Field, TypeAdapter @@ -109,9 +109,58 @@ class LLMResponse(LiteLLMBaseModel): user_metadata: dict[str, object] | None = None +class _GalileoRecord(TypedDict, total=False): + latency_ms: ReadOnly[int] + status_code: ReadOnly[int] + input_text: ReadOnly[str] + output_text: ReadOnly[str] + node_type: ReadOnly[str] + model: ReadOnly[str] + num_input_tokens: ReadOnly[int] + num_output_tokens: ReadOnly[int] + num_total_tokens: ReadOnly[int] + cost: ReadOnly[float | None] + output_logprobs: ReadOnly[Mapping[str, object] | None] + created_at: ReadOnly[str] + tags: ReadOnly[Sequence[str] | None] + user_metadata: ReadOnly[Mapping[str, object] | None] + messages: ReadOnly[Sequence[object]] + + +class _GalileoLoggingKwargs(TypedDict, total=False, extra_items=object): + standard_logging_object: ReadOnly[GalileoStandardLoggingFields | None] + + +def _record_messages(messages: object) -> object: + return messages.get("messages") if isinstance(messages, dict) else messages + + +def _galileo_record(response: LLMResponse, messages: object) -> _GalileoRecord: + record: Final[_GalileoRecord] = { + "latency_ms": response.latency_ms, + "status_code": response.status_code, + "input_text": response.input_text, + "output_text": response.output_text, + "node_type": response.node_type, + "model": response.model, + "num_input_tokens": response.num_input_tokens, + "num_output_tokens": response.num_output_tokens, + "num_total_tokens": response.num_total_tokens, + "cost": response.cost, + "output_logprobs": response.output_logprobs, + "created_at": response.created_at, + "tags": response.tags, + "user_metadata": response.user_metadata, + } + record_messages: Final = _record_messages(messages) + if isinstance(record_messages, list) and record_messages: + return {**record, "messages": record_messages} + return record + + class GalileoObserve(CustomLogger): def __init__(self) -> None: - self.in_memory_records: list[Mapping[str, object]] = [] + self.in_memory_records: list[_GalileoRecord] = [] self.batch_size = 1 self.api_key = os.getenv("GALILEO_API_KEY") self.project_id = os.getenv("GALILEO_PROJECT_ID") @@ -278,7 +327,7 @@ class GalileoObserve(CustomLogger): return created_at @staticmethod - def _token_metrics_from_record(record: Mapping[str, Any]) -> dict[str, object]: + def _token_metrics_from_record(record: _GalileoRecord) -> dict[str, object]: num_input_tokens: Final = int(record.get("num_input_tokens") or 0) num_output_tokens: Final = int(record.get("num_output_tokens") or 0) num_total_tokens = int(record.get("num_total_tokens") or 0) @@ -296,7 +345,7 @@ class GalileoObserve(CustomLogger): @staticmethod def _record_to_v2_span( - record: Mapping[str, Any], + record: _GalileoRecord, *, trace_id: str, span_id: str, @@ -322,12 +371,13 @@ class GalileoObserve(CustomLogger): **GalileoObserve._token_metrics_from_record(record), }, } - if record.get("tags"): - span["tags"] = record["tags"] + tags: Final = record.get("tags") + if tags: + span["tags"] = tags return span @staticmethod - def _record_to_v2_trace(record: Mapping[str, Any]) -> dict[str, object]: + def _record_to_v2_trace(record: _GalileoRecord) -> dict[str, object]: trace_id: Final = str(uuid.uuid4()) span_id: Final = str(uuid.uuid4()) created_at: Final = GalileoObserve._normalize_created_at(record.get("created_at", "")) @@ -347,7 +397,7 @@ class GalileoObserve(CustomLogger): "spans": [GalileoObserve._record_to_v2_span(record, trace_id=trace_id, span_id=span_id)], } - def _build_traces_payload(self, records: Sequence[Mapping[str, object]]) -> dict[str, object]: + def _build_traces_payload(self, records: Sequence[_GalileoRecord]) -> dict[str, object]: payload: Final[dict[str, object]] = { "traces": [self._record_to_v2_trace(record) for record in records], "logging_method": "api_direct", @@ -668,7 +718,7 @@ class GalileoObserve(CustomLogger): async def async_log_success_event( self, - kwargs: Mapping[str, object], + kwargs: _GalileoLoggingKwargs, response_obj: object, start_time: object, end_time: object, @@ -686,7 +736,7 @@ class GalileoObserve(CustomLogger): async def _async_log_success_event_impl( self, - kwargs: Mapping[str, Any], + kwargs: _GalileoLoggingKwargs, response_obj: object, start_time: object, end_time: object, @@ -752,12 +802,7 @@ class GalileoObserve(CustomLogger): created_at=GalileoObserve._format_created_at(start_ts), ) - request_dict: Final = request_record.model_dump() - if isinstance(messages, dict): - messages = messages.get("messages") - if isinstance(messages, list) and messages: - request_dict["messages"] = messages - self.in_memory_records.append(request_dict) + self.in_memory_records.append(_galileo_record(request_record, messages)) verbose_logger.debug("Galileo Logger: queued record, in_memory=%d", len(self.in_memory_records)) # Bound the buffer so persistent flush failures cannot grow it diff --git a/litellm/integrations/gitlab/gitlab_prompt_manager.py b/litellm/integrations/gitlab/gitlab_prompt_manager.py index adaacd615a8..c51c513b633 100644 --- a/litellm/integrations/gitlab/gitlab_prompt_manager.py +++ b/litellm/integrations/gitlab/gitlab_prompt_manager.py @@ -3,13 +3,14 @@ GitLab prompt manager with configurable prompts folder. """ from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, TypeVar +from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar from jinja2 import DictLoader, select_autoescape from jinja2.sandbox import ImmutableSandboxedEnvironment from typing_extensions import ReadOnly, TypedDict from litellm.integrations.custom_prompt_management import CustomPromptManagement +from litellm.litellm_core_utils.core_helpers import require_attributes if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -40,6 +41,18 @@ class GitLabCachedPrompt(TypedDict): optional_params: ReadOnly[Mapping[str, object]] +class _SupportsGet(Protocol): + def get(self, key: str, default: object = None, /) -> object: ... + + +def _input_schema(metadata: Mapping[str, object]) -> object: + return require_attributes(metadata.get("input", {}), _SupportsGet, "get").get("schema", {}) + + +def _optional_params(metadata: Mapping[str, object]) -> Mapping[str, object]: + return {k: v for k, v in metadata.items() if k not in ["model", "input", "content"]} + + def encode_prompt_id(raw_id: str) -> str: """Convert GitLab path IDs like 'invoice/extract' → 'gitlab::invoice::extract'""" if raw_id.startswith(GITLAB_PREFIX): @@ -68,8 +81,8 @@ class GitLabPromptTemplate: self.model = model or metadata.get("model") self.temperature = metadata.get("temperature") self.max_tokens = metadata.get("max_tokens") - self.input_schema = metadata.get("input", {}).get("schema", {}) - self.optional_params = {k: v for k, v in metadata.items() if k not in ["model", "input", "content"]} + self.input_schema = _input_schema(metadata) + self.optional_params = _optional_params(metadata) def __repr__(self): return f"GitLabPromptTemplate(id='{self.template_id}', model='{self.model}')" @@ -84,7 +97,7 @@ class GitLabTemplateManager: def __init__( self, - gitlab_config: dict[str, Any], + gitlab_config: Mapping[str, Any], prompt_id: str | None = None, ref: str | None = None, gitlab_client: GitLabClient | None = None, @@ -297,7 +310,7 @@ class GitLabPromptManager(CustomPromptManagement): def __init__( self, - gitlab_config: dict[str, object], + gitlab_config: Mapping[str, object], prompt_id: str | None = None, ref: str | None = None, # tag/branch/SHA override gitlab_client: GitLabClient | None = None, diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index bd68ffe8cd6..842eaaf847d 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -22,6 +22,7 @@ from litellm.integrations.langfuse.langfuse_mock_client import ( from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, filter_exceptions_from_params, + json_contains, reconstruct_model_name, safe_deep_copy, ) @@ -1311,11 +1312,11 @@ def _end_grounding_span(*, tracing: "LangfuseTracing", parent: "LangfuseObservat ).end() -def log_requester_metadata(clean_metadata: Mapping[str, Any]): +def log_requester_metadata(clean_metadata: Mapping[str, object]): returned_metadata: Final = {} - requester_metadata: Final = clean_metadata.get("requester_metadata") or {} + requester_metadata: Final[object] = clean_metadata.get("requester_metadata") or {} for k, v in clean_metadata.items(): - if k not in requester_metadata: + if not json_contains(requester_metadata, k): returned_metadata[k] = v returned_metadata.update({"requester_metadata": requester_metadata}) diff --git a/litellm/integrations/opik/opik_payload_builder/extractors.py b/litellm/integrations/opik/opik_payload_builder/extractors.py index 1ceb9aa763f..d4fefe54333 100644 --- a/litellm/integrations/opik/opik_payload_builder/extractors.py +++ b/litellm/integrations/opik/opik_payload_builder/extractors.py @@ -2,15 +2,28 @@ import json from collections.abc import Mapping -from typing import Any, Final +from typing import Any, Final, Protocol from pydantic import TypeAdapter from litellm import _logging +from litellm.litellm_core_utils.core_helpers import require_attributes _DECODED_JSON: Final = TypeAdapter(object) +class _SpanIdentifiers(Protocol): + @property + def trace_id(self) -> str | None: ... + + @property + def id(self) -> str | None: ... + + +def _trace_and_span_ids(span: _SpanIdentifiers) -> tuple[str | None, str | None]: + return span.trace_id, span.id + + def normalize_provider_name(provider: str | None) -> str | None: """ Normalize LiteLLM provider names to standardized string names. @@ -77,7 +90,7 @@ def extract_opik_metadata( def extract_span_identifiers( - current_span_data: Any, + current_span_data: object, ) -> tuple[str | None, str | None]: """ Extract trace_id and parent_span_id from current_span_data. @@ -95,7 +108,7 @@ def extract_span_identifiers( return (current_span_data.get("trace_id"), current_span_data.get("id")) try: - return current_span_data.trace_id, current_span_data.id + return _trace_and_span_ids(require_attributes(current_span_data, _SpanIdentifiers, "trace_id", "id")) except AttributeError: _logging.verbose_logger.warning("Unexpected current_span_data format: %s", type(current_span_data)) return None, None diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py index f95b5611831..628f35edd50 100644 --- a/litellm/interactions/litellm_responses_transformation/handler.py +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -2,8 +2,8 @@ Handler for transforming interactions API requests to litellm.responses requests. """ -from collections.abc import AsyncIterator, Callable, Coroutine, Iterator -from typing import Any, Final +from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping +from typing import Final import litellm from litellm.interactions.litellm_responses_transformation.streaming_iterator import ( @@ -72,6 +72,7 @@ class LiteLLMResponsesInteractionsHandler: model=model, input=input, optional_params=optional_params, + custom_llm_provider=custom_llm_provider, **kwargs, ) @@ -108,10 +109,11 @@ class LiteLLMResponsesInteractionsHandler: async def async_interactions_api_handler( self, - responses_request: dict[str, Any], + responses_request: Mapping[str, object], model: str, input: InteractionInput | None, optional_params: InteractionsAPIOptionalRequestParams, + custom_llm_provider: str | None = None, **kwargs, ) -> InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]: """Async handler for interactions API requests.""" @@ -128,7 +130,7 @@ class LiteLLMResponsesInteractionsHandler: litellm_custom_stream_wrapper=responses_response, request_input=input, optional_params=optional_params, - custom_llm_provider=responses_request.get("custom_llm_provider"), + custom_llm_provider=custom_llm_provider, litellm_metadata=kwargs.get("litellm_metadata", {}), ) @@ -139,5 +141,5 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, - custom_llm_provider=responses_request.get("custom_llm_provider"), + custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 53451b04515..93799c72209 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -99,6 +99,11 @@ def _post_hook_overridden(callback: CustomLogger) -> bool: _STREAM_OPTIONS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +_HOOK_RESPONSE: Final = TypeAdapter(object) + + +def _plan_response_override(plan: AgenticLoopPlan) -> object | None: + return plan.response_override def _stashed_stream_options(kwargs: Mapping[str, object]) -> Mapping[str, object] | None: @@ -396,8 +401,10 @@ async def _execute_chat_completion_agentic_plan( ) if _post_hook_overridden(callback): try: - response_followup = await callback.async_post_agentic_loop_response_hook( - response=response_followup, plan=plan, kwargs=kwargs + response_followup = _HOOK_RESPONSE.validate_python( + await callback.async_post_agentic_loop_response_hook( + response=response_followup, plan=plan, kwargs=kwargs + ) ) except Exception as e: _call_id = getattr(logging_obj, "litellm_call_id", "unknown") @@ -646,8 +653,8 @@ async def run_chat_completion_agentic_hooks( kwargs=hook_kwargs, ) - if plan.response_override is not None: - return plan.response_override + if (response_override := _plan_response_override(plan)) is not None: + return response_override if plan.terminate: verbose_logger.debug( "Agentic chat loop terminated by callback=%s reason=%s", type(callback).__name__, plan.stop_reason diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 8dec2941e02..71b20f92cf7 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -102,7 +102,7 @@ _LAX_BOOL: Final = TypeAdapter(bool) def as_str_mapping(value: object) -> Mapping[str, object] | None: - return value if isinstance(value, Mapping) else None + return value if isinstance(value, (dict, Mapping)) else None def as_object_list(value: object) -> Sequence[object] | None: @@ -180,7 +180,24 @@ def require_attributes(value: object, shape: type[_Shape], *names: str) -> _Shap raise AttributeError(f"'{type(value).__name__}' object has no attribute '{missing}'", name=missing, obj=value) +class KeywordArguments(Protocol): + def keys(self) -> Iterable[str]: ... + + def __getitem__(self, key: str, /) -> object: ... + + +def keyword_arguments(value: object, target: type[object]) -> KeywordArguments: + if has_attributes(value, KeywordArguments, "keys"): + return value + raise TypeError( + f"{target.__module__}.{target.__qualname__}() argument after ** must be a mapping, not {type(value).__name__}" + ) + + def iter_items(value: object) -> Iterable[object]: + listed: Final = as_object_list(value) + if listed is not None: + return listed if isinstance(value, _SupportsIter): return value if isinstance(value, _SupportsGetItem): @@ -746,17 +763,22 @@ def process_response_headers( return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})} -def preserve_upstream_non_openai_attributes( - model_response: "ModelResponseStream", original_chunk: "ModelResponseStream" -): +def preserve_upstream_non_openai_attributes(model_response: "ModelResponseStream", original_chunk: object): """ Preserve non-OpenAI attributes from the original chunk. """ # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings expected_keys: Final = frozenset(type(model_response).model_fields) | frozenset({"usage"}) + chunk_model: Final = original_chunk if isinstance(original_chunk, BaseModel) else None + if chunk_model is None: + raise AttributeError( + f"type object '{type(original_chunk).__name__}' has no attribute 'model_fields'", + name="model_fields", + obj=type(original_chunk), + ) candidate_keys: Final = tuple( key - for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {})) + for key in (*type(chunk_model).model_fields, *(chunk_model.__pydantic_extra__ or {})) if key not in expected_keys ) if not candidate_keys: @@ -766,7 +788,7 @@ def preserve_upstream_non_openai_attributes( key, cast( # cast-ok: Pydantic extras are scalar-checked before assignment object, - getattr(original_chunk, key), + getattr(chunk_model, key), ), ) for key in candidate_keys @@ -774,7 +796,7 @@ def preserve_upstream_non_openai_attributes( if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values): _assign_attributes(model_response, candidate_values) return - _assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items()) + _assign_attributes(model_response, chunk_model.model_dump(include=set(candidate_keys)).items()) def _assign_attributes(target: BaseModel, values: Iterable[tuple[str, object]]) -> None: diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 6d39f272f3e..16cd076db39 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -17,7 +17,7 @@ from litellm.litellm_core_utils.bug_report import ( build_bug_report, should_report_bug, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, json_contains, subscript from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string from litellm.llms.bedrock.common_utils import bedrock_model_reaches_converse_by_default from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error @@ -1859,26 +1859,32 @@ def _map_together_ai_exception( extra_information: str, ) -> None: try: - error_response = json.loads(error_str) + error_response = _DECODED_ERROR_BODY.validate_python(json.loads(error_str)) except Exception: error_response = {"error": error_str} - if "error" in error_response and "`inputs` tokens + `max_new_tokens` must be <=" in error_response["error"]: + if json_contains(error_response, "error") and json_contains( + subscript(error_response, "error"), "`inputs` tokens + `max_new_tokens` must be <=" + ): raise ContextWindowExceededError( - message=f"TogetherAIException - {error_response['error']}", + message=f"TogetherAIException - {subscript(error_response, 'error')}", model=model, llm_provider="together_ai", response=getattr(original_exception, "response", None), ) - elif "error" in error_response and "invalid private key" in error_response["error"]: + elif json_contains(error_response, "error") and json_contains( + subscript(error_response, "error"), "invalid private key" + ): raise AuthenticationError( - message=f"TogetherAIException - {error_response['error']}", + message=f"TogetherAIException - {subscript(error_response, 'error')}", llm_provider="together_ai", model=model, response=getattr(original_exception, "response", None), ) - elif "error" in error_response and "INVALID_ARGUMENT" in error_response["error"]: + elif json_contains(error_response, "error") and json_contains( + subscript(error_response, "error"), "INVALID_ARGUMENT" + ): raise BadRequestError( - message=f"TogetherAIException - {error_response['error']}", + message=f"TogetherAIException - {subscript(error_response, 'error')}", model=model, llm_provider="together_ai", response=getattr(original_exception, "response", None), @@ -1890,13 +1896,13 @@ def _map_together_ai_exception( llm_provider="together_ai", ) elif ( - "error" in error_response - and "API key doesn't match expected format." in error_response["error"] - or "error_type" in error_response - and error_response["error_type"] == "validation" + json_contains(error_response, "error") + and json_contains(subscript(error_response, "error"), "API key doesn't match expected format.") + or json_contains(error_response, "error_type") + and subscript(error_response, "error_type") == "validation" ): raise BadRequestError( - message=f"TogetherAIException - {error_response['error']}", + message=f"TogetherAIException - {subscript(error_response, 'error')}", model=model, llm_provider="together_ai", response=getattr(original_exception, "response", None), @@ -1910,7 +1916,7 @@ def _map_together_ai_exception( ) elif original_exception.status_code == 422: raise BadRequestError( - message=f"TogetherAIException - {error_response['error']}", + message=f"TogetherAIException - {subscript(error_response, 'error')}", model=model, llm_provider="together_ai", response=getattr(original_exception, "response", None), diff --git a/litellm/litellm_core_utils/json_validation_rule.py b/litellm/litellm_core_utils/json_validation_rule.py index b6b16aff7a7..030bee48530 100644 --- a/litellm/litellm_core_utils/json_validation_rule.py +++ b/litellm/litellm_core_utils/json_validation_rule.py @@ -1,9 +1,11 @@ import json -from typing import Any, Final +from collections.abc import Mapping +from typing import Final from pydantic import TypeAdapter from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict _DECODED_RESPONSE: Final = TypeAdapter(object) @@ -76,7 +78,7 @@ def normalize_json_schema_types( return schema -def normalize_tool_schema(tool: dict[str, Any]) -> dict[str, Any]: +def normalize_tool_schema(tool: Mapping[str, object]) -> Mapping[str, object]: """ Normalize a tool's parameter schema to use standard JSON Schema lowercase types. @@ -92,10 +94,14 @@ def normalize_tool_schema(tool: dict[str, Any]) -> dict[str, Any]: normalized_tool: Final = tool.copy() # Normalize function parameters if present - if "function" in tool and isinstance(tool["function"], dict): - normalized_tool["function"] = tool["function"].copy() - if "parameters" in tool["function"]: - normalized_tool["function"]["parameters"] = normalize_json_schema_types(tool["function"]["parameters"]) + if "function" not in tool: + return normalized_tool + function: Final = tool["function"] + if is_str_keyed_dict(function): + normalized_function: Final = function.copy() + normalized_tool["function"] = normalized_function + if "parameters" in function: + normalized_function["parameters"] = normalize_json_schema_types(function["parameters"]) return normalized_tool diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8f3101915be..5d21b2762cd 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -25,6 +25,7 @@ from httpx import Response, Timeout from openai import AsyncStream as OpenAIAsyncStream from openai import Stream as OpenAIStream from pydantic import BaseModel, JsonValue, TypeAdapter +from typing_extensions import TypedDict import litellm from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase @@ -91,8 +92,11 @@ from litellm.litellm_core_utils.classifier_logging import ( ) from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, + attribute_or_none, + decoded_json_object, finite_cost, get_provider_response_headers_from_hidden_params, + has_attributes, is_expected_client_error, native_messages_reported_cost, provider_reported_usage_cost, @@ -103,6 +107,7 @@ from litellm.litellm_core_utils.core_helpers import ( require_attributes, set_response_cost_in_hidden_params, single_log_line, + subscript, ) from litellm.litellm_core_utils.error_normalization import normalize_error from litellm.litellm_core_utils.get_litellm_params import get_litellm_params @@ -669,6 +674,10 @@ class UsageObservation(Enum): ESTIMATED = "estimated" +class _UsageFields(TypedDict, total=False, extra_items=object): + pass + + class _A2AUsageSlot(Protocol): usage: Mapping[str, object] | None @@ -685,6 +694,31 @@ class _ParamsLookup(Protocol): def get(self, key: str, default: object, /) -> object: ... +class _HiddenParamsSlot(Protocol): + get: Callable[[str], object] + __setitem__: Callable[[str, object], None] + + +class _HiddenParamsCopier(Protocol): + copy: Callable[[], object] + + +class _MetadataDefaults(Protocol): + setdefault: Callable[[str, Mapping[str, object]], object] + + +class _MetadataSlot(Protocol): + __setitem__: Callable[[str, object], None] + + +class _MetadataUpdate(Protocol): + update: Callable[[Mapping[str, object]], None] + + +class _ExceptionHeaders(Protocol): + headers: object + + def _callback_control_params(litellm_params: object) -> dict[str, object]: mapping: Final = as_str_mapping(litellm_params) if isinstance(mapping, dict): @@ -1682,8 +1716,10 @@ class Logging(LiteLLMLoggingBaseClass): ) # log raw request to provider (like LangFuse) -- if opted in. if self.log_raw_request_response is True or litellm.log_raw_request_response is True: - _litellm_params: Final = self.model_call_details.get("litellm_params", {}) - _metadata: Final = _litellm_params.get("metadata", {}) or {} + _litellm_params: Final = require_attributes( + self._model_call_details_view.get("litellm_params", dict[str, object]()), _ParamsLookup, "get" + ) + _metadata: Final = _litellm_params.get("metadata", dict[str, object]()) or dict[str, object]() try: self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict( raw_request_api_base=self._get_masked_api_base(str(additional_args.get("api_base") or "")), @@ -1697,6 +1733,8 @@ class Logging(LiteLLMLoggingBaseClass): error=None, ) if should_redact_message_logging(self.model_call_details): + if not has_attributes(_metadata, _MetadataSlot, "__setitem__"): + raise TypeError(f"'{type(_metadata).__name__}' object does not support item assignment") _metadata["raw_request"] = REDACTED_BY_LITELLM else: curl_command: Final = self._get_request_curl_command( @@ -1705,15 +1743,21 @@ class Logging(LiteLLMLoggingBaseClass): additional_args=additional_args, data=additional_args.get("complete_input_dict", {}), ) - _metadata["raw_request"] = redact_string(str(curl_command)) + redacted_curl_command: Final = redact_string(str(curl_command)) + if not has_attributes(_metadata, _MetadataSlot, "__setitem__"): + raise TypeError(f"'{type(_metadata).__name__}' object does not support item assignment") + _metadata["raw_request"] = redacted_curl_command except Exception as e: self.model_call_details["raw_request_typed_dict"] = RawRequestTypedDict( error=str(e), ) - _metadata["raw_request"] = redact_string( + unlogged_raw_request: Final = redact_string( f"Unable to Log \ raw request: {e}" ) + if not has_attributes(_metadata, _MetadataSlot, "__setitem__"): + raise TypeError(f"'{type(_metadata).__name__}' object does not support item assignment") + _metadata["raw_request"] = unlogged_raw_request if getattr(self, "logger_fn", None) and callable(self.logger_fn): try: self.logger_fn( @@ -2344,7 +2388,7 @@ class Logging(LiteLLMLoggingBaseClass): def _record_zero_cost_diagnostic( self, result: object, - response_cost: float | None, + response_cost: object, *, calculation_failed: bool = False, litellm_model_name: str | None = None, @@ -2375,7 +2419,7 @@ class Logging(LiteLLMLoggingBaseClass): def _zero_cost_finding( self, result: object, - response_cost: float | None, + response_cost: object, *, calculation_failed: bool, litellm_model_name: str | None, @@ -3079,19 +3123,21 @@ class Logging(LiteLLMLoggingBaseClass): elif self.relays_llm_passthrough and isinstance(result, Response): from litellm.utils import ProviderConfigManager - provider_config: Final = ProviderConfigManager.get_provider_passthrough_config( - provider=self.model_call_details.get("custom_llm_provider", ""), - model=self.model, - ) - if provider_config is not None: - logging_result = provider_config.logging_non_streaming_response( + passthrough_provider: Final = self._model_call_details_view.get("custom_llm_provider", "") + if isinstance(passthrough_provider, str): + provider_config: Final = ProviderConfigManager.get_provider_passthrough_config( + provider=passthrough_provider, model=self.model, - custom_llm_provider=self.model_call_details.get("custom_llm_provider", ""), - httpx_response=result, - request_data=self.model_call_details.get("request_data", {}), - logging_obj=self, - endpoint=self.model_call_details.get("endpoint", ""), ) + if provider_config is not None: + logging_result = provider_config.logging_non_streaming_response( + model=self.model, + custom_llm_provider=passthrough_provider, + httpx_response=result, + request_data=self.model_call_details.get("request_data", {}), + logging_obj=self, + endpoint=self.model_call_details.get("endpoint", ""), + ) return logging_result def _surface_response_headers_from_result(self, logging_result: object) -> None: @@ -3138,19 +3184,25 @@ class Logging(LiteLLMLoggingBaseClass): """ if logging_result is None: return - hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, None) + hidden_params: Final = attribute_or_none(logging_result, HIDDEN_PARAMS_ATTR) if not hidden_params: return if self.model_call_details.get("litellm_params") is None: return - metadata_hidden_params: Final = hidden_params.copy() + metadata_hidden_params: Final = require_attributes( + require_attributes(hidden_params, _HiddenParamsCopier, "copy").copy(), _HiddenParamsSlot, "get" + ) response_cost: Final[object] = self.model_call_details.get("response_cost") if metadata_hidden_params.get("response_cost") is None and response_cost is not None: metadata_hidden_params["response_cost"] = response_cost - litellm_params: Final = self.model_call_details["litellm_params"] - metadata: Final = litellm_params.get("metadata") or {} + litellm_params: Final = require_attributes( + self._model_call_details_view["litellm_params"], _HiddenParamsSlot, "get" + ) + metadata: Final = litellm_params.get("metadata") or dict[str, object]() litellm_params["metadata"] = metadata + if not has_attributes(metadata, _MetadataSlot, "__setitem__"): + raise TypeError(f"'{type(metadata).__name__}' object does not support item assignment") metadata["hidden_params"] = metadata_hidden_params def _process_hidden_params_and_response_cost( @@ -3166,7 +3218,9 @@ class Logging(LiteLLMLoggingBaseClass): hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, {}) if hidden_params: if self.model_call_details.get("litellm_params") is not None: - self.model_call_details["litellm_params"].setdefault("metadata", {}) + require_attributes( + self._model_call_details_view["litellm_params"], _MetadataDefaults, "setdefault" + ).setdefault("metadata", {}) if self.model_call_details["litellm_params"]["metadata"] is None: self.model_call_details["litellm_params"]["metadata"] = {} self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = getattr( @@ -3179,8 +3233,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["response_cost"] = 0.0 self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result)) elif "response_cost" in hidden_params: - self.model_call_details["response_cost"] = hidden_params["response_cost"] - self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"]) + hidden_response_cost: Final = subscript(hidden_params, "response_cost") + self.model_call_details["response_cost"] = hidden_response_cost + self._record_zero_cost_diagnostic(logging_result, hidden_response_cost) self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result)) elif (existing_cost := self.model_call_details.get("response_cost")) is not None and ( existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped @@ -4659,10 +4714,23 @@ class Logging(LiteLLMLoggingBaseClass): elif self.model_call_details.get("combined_usage_object") is None: self.model_call_details["response_cost"] = 0 - if hasattr(exception, "headers") and isinstance(exception.headers, dict): + raised_exception: Final[object] = exception + exception_headers: Final = ( + as_str_mapping(raised_exception.headers) + if has_attributes(raised_exception, _ExceptionHeaders, "headers") + else None + ) + if isinstance(exception_headers, dict): self.model_call_details.setdefault("litellm_params", {}) - metadata: Final = self.model_call_details["litellm_params"].get("metadata", {}) or {} - metadata.update(exception.headers) + metadata: Final = require_attributes( + require_attributes(self._model_call_details_view["litellm_params"], _ParamsLookup, "get").get( + "metadata", dict[str, object]() + ) + or dict[str, object](), + _MetadataUpdate, + "update", + ) + metadata.update(exception_headers) ## STANDARDIZED LOGGING PAYLOAD @@ -6412,7 +6480,7 @@ def build_custom_logger_compatible_class( return callback # Get global BitBucket config - gitlab_config: Final = getattr(litellm, "global_gitlab_config", None) + gitlab_config: Final = litellm.global_gitlab_config if hasattr(litellm, "global_gitlab_config") else None if gitlab_config is None: raise ValueError("Gitlab configuration not found. Please set litellm.global_gitlab_config first.") @@ -7198,11 +7266,15 @@ class StandardLoggingPayloadSetup: if isinstance(reported, Usage): return reported normalized: Final = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj) - return Usage(**normalized) if normalized else Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) + return ( + Usage(**_UsageFields(**decoded_json_object(normalized))) + if normalized + else Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) + ) @staticmethod def get_usage_as_dict( - response_obj: dict[str, Any] | None, + response_obj: Mapping[str, object] | None, combined_usage_object: Usage | None = None, ) -> dict[str, Any]: """ @@ -7226,19 +7298,23 @@ class StandardLoggingPayloadSetup: reported: Final = response_obj.get("usage_info") if isinstance(reported, BaseModel): return reported.model_dump(exclude_none=True) + reported_fields: Final = as_str_mapping(reported) return ( - {key: value for key, value in reported.items() if value is not None} - if isinstance(reported, dict) + {key: value for key, value in reported_fields.items() if value is not None} + if isinstance(reported_fields, dict) else dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty) ) if isinstance(_raw, ResponseAPIUsage): return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump() - if isinstance(_raw, dict): - if ResponseAPILoggingUtils.is_response_api_usage(_raw): - return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump() - if InteractionsUsageObjectTransformation.is_interactions_usage_object(_raw): - return InteractionsUsageObjectTransformation.transform_interactions_usage_object(_raw).model_dump() - return _raw + raw_fields: Final = as_str_mapping(_raw) + if isinstance(raw_fields, dict): + if ResponseAPILoggingUtils.is_response_api_usage(raw_fields): + return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(raw_fields).model_dump() + if InteractionsUsageObjectTransformation.is_interactions_usage_object(raw_fields): + return InteractionsUsageObjectTransformation.transform_interactions_usage_object( + raw_fields + ).model_dump() + return raw_fields if isinstance(_raw, Usage): return _raw.model_dump() return _empty diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 1f6a66f699a..4ff82f705d9 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1429,7 +1429,7 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | residency: Final = data_residency.lower() if residency not in _VALID_DATA_RESIDENCIES: return 1.0 - multiplier: Final = model_info.get(f"regional_processing_uplift_multiplier_{residency}") + multiplier: Final[object] = model_info.get(f"regional_processing_uplift_multiplier_{residency}") if multiplier is None: return 1.0 if not isinstance(multiplier, (str, Buffer, SupportsFloat, SupportsIndex)): diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 0fed2cb2fce..20d1205c005 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -18,6 +18,7 @@ from litellm.constants import ( STREAM_FINISH_CHOICE_FIELDS, VLLM_CHOICE_FIELDS, ) +from litellm.litellm_core_utils.core_helpers import iter_items, subscript from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( extract_reasoning_content, @@ -68,6 +69,14 @@ _MODEL_RESPONSE_FIELDS: Final[frozenset] = frozenset(ModelResponse.model_fields. _ANNOTATION_ENTRIES: Final = TypeAdapter(tuple[object, ...]) +class _ImageResponseFields(TypedDict, total=False, extra_items=object): + pass + + +def _image_response_from_fields(fields: Mapping[str, object]) -> ImageResponse: + return ImageResponse(**_ImageResponseFields(**fields)) + + def _is_provider_specific_annotation(annotation: object) -> bool: match annotation: case {"type": "url_citation"}: @@ -639,7 +648,7 @@ class LiteLLMResponseObjectHandler: model_response_dict: Final = model_response_object.model_dump() model_response_dict.update(response_object) - model_response_object = ImageResponse(**model_response_dict) + model_response_object = _image_response_from_fields(model_response_dict) return model_response_object @staticmethod @@ -683,12 +692,12 @@ class LiteLLMResponseObjectHandler: choices_list: Final[list[TextChoices]] = [] # Convert each choice to TextChoices - for choice in response["choices"]: + for choice in iter_items(subscript(response, "choices")): text_choices = TextChoices() - text_choices["text"] = choice["message"]["content"] - text_choices["index"] = choice["index"] + text_choices["text"] = subscript(subscript(choice, "message"), "content") + text_choices["index"] = subscript(choice, "index") text_choices["logprobs"] = transformed_logprobs - text_choices["finish_reason"] = choice["finish_reason"] + text_choices["finish_reason"] = subscript(choice, "finish_reason") choices_list.append(text_choices) text_completion_response["choices"] = choices_list diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index ae36484b294..704af2b9ebd 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -13,7 +13,7 @@ from types import MappingProxyType from typing import Annotated, Any, Final, Protocol, TypeAlias, TypedDict, TypeVar, cast, overload from jinja2.sandbox import ImmutableSandboxedEnvironment -from pydantic import BaseModel, BeforeValidator, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator +from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError, model_validator from pydantic_core import ErrorDetails, PydanticCustomError import litellm @@ -22,7 +22,12 @@ import litellm.types.llms from litellm import verbose_logger from litellm._uuid import uuid from litellm.constants import REDACTED_BY_LITELLM -from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, require_attributes +from litellm.litellm_core_utils.core_helpers import ( + as_object_list, + as_str_mapping, + attribute_or_none, + require_attributes, +) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( anthropic_system_messages, find_anthropic_server_tool_result, @@ -1455,22 +1460,25 @@ def convert_to_gemini_tool_call_invoke( raise Exception(f"Unable to convert openai tool calls={message} to gemini tool calls. Received error={e}") -def _json_children(node: JsonValue) -> Iterable[JsonValue]: - if isinstance(node, dict): - return node.values() - if isinstance(node, list): - return (child for child in node if not isinstance(child, list)) +def _json_children(node: object) -> Iterable[object]: + mapping: Final = as_str_mapping(node) + if mapping is not None: + return mapping.values() + items: Final = as_object_list(node) + if items is not None: + return (child for child in items if not isinstance(child, list)) return () -def _next_json_level(level: tuple[JsonValue, ...], _: None) -> tuple[JsonValue, ...]: +def _next_json_level(level: tuple[object, ...], _: None) -> tuple[object, ...]: return tuple(itertools.chain.from_iterable(_json_children(node) for node in level)) -def _is_part_reference(node: JsonValue) -> bool: - if not isinstance(node, dict): +def _is_part_reference(node: object) -> bool: + mapping: Final = as_str_mapping(node) + if mapping is None: return False - ref: Final[JsonValue] = node.get("$ref") + ref: Final = mapping.get("$ref") return isinstance(ref, str) and ref != "" @@ -1482,7 +1490,7 @@ def _may_spell_ref_key(raw_json: str) -> bool: return '"$ref"' in raw_json or _REF_CHAR_ESCAPE.search(raw_json) is not None -def _contains_part_reference(payload: JsonValue) -> bool: +def _contains_part_reference(payload: object) -> bool: """Gemini reads a {"$ref": ""} object in a function_response as a pointer to one of its parts, unless it sits under a list nested directly in a list @@ -1739,7 +1747,7 @@ def convert_to_gemini_tool_call_result( try: if content_str.strip().startswith("{") or content_str.strip().startswith("["): # Try to parse as JSON (for Computer Use structured responses) - parsed: Final[JsonValue] = json.loads(content_str) + parsed: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(content_str))) if isinstance(parsed, dict) and not (_may_spell_ref_key(content_str) and _contains_part_reference(parsed)): response_data = parsed # Use the parsed JSON directly else: diff --git a/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py b/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py index d2531e70d6b..39baaed25e6 100644 --- a/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py +++ b/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py @@ -1,6 +1,6 @@ import json from datetime import datetime -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -28,6 +28,10 @@ class _ChatTemplateFileResult(TypedDict): chat_template: NotRequired[ReadOnly[str]] +class _AddedToken(TypedDict, total=False, extra_items=object): + content: ReadOnly[str] + + def strftime_now(fmt: str) -> str: """ Custom function for templates that need current date/time formatting (e.g., gpt-oss) @@ -159,7 +163,7 @@ async def aget_chat_template_file(hf_model_name: str) -> _ChatTemplateFileResult _aget_chat_template_file = aget_chat_template_file -def extract_token_value(token_value: None | str | dict[str, Any]) -> str: +def extract_token_value(token_value: None | str | _AddedToken) -> str: """ Extract token string from various formats (string, dict, etc.) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 01aa7296b0f..6d40d963d76 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -26,7 +26,7 @@ from litellm.constants import ( REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, json_contains from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError from litellm.llms.base_llm.realtime.transformation import ( @@ -220,9 +220,13 @@ def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, objec } +def _decode_json(payload: str) -> object: + return _DECODED_JSON.validate_python(json.loads(payload)) + + def _decode_json_object(payload: str) -> Mapping[str, object]: """Decode a realtime frame into its top-level field mapping.""" - return json.loads(payload) + return decoded_json_object(_decode_json(payload)) _GUARDRAIL_TURN_DETECTION_GATE: Final[Mapping[str, object]] = MappingProxyType( @@ -511,7 +515,7 @@ class RealTimeStreaming: self._held_client_content: tuple[str, ...] = () self._held_client_content_bytes: int = 0 self._held_client_content_deadline: float | None = None - self._replay_scope: Final = client_websocket_scope(websocket) + self._replay_scope: Final = client_websocket_scope(self.websocket) self.backend_ws = backend_ws self.logging_obj = logging_obj self._logging_worker = logging_worker @@ -533,7 +537,7 @@ class RealTimeStreaming: self._translation_usage_finalized = False # Detect whether the client is explicitly opting into the beta protocol. - self._client_wants_beta = self.detect_beta_header(websocket) + self._client_wants_beta = self.detect_beta_header(self.websocket) self._backend_uses_beta_protocol = not translation_session and ( self._client_wants_beta if backend_uses_beta_protocol is None else backend_uses_beta_protocol ) @@ -674,7 +678,7 @@ class RealTimeStreaming: if not self._should_store_message(message_obj): return try: - event_type: Final = message_obj.get("type", "") + event_type: Final = decoded_json_object(message_obj).get("type", "") if event_type in self._SESSION_EVENT_TYPES: typed_obj: OpenAIRealtimeEvents = OpenAIRealtimeStreamSessionEvents(**message_obj) else: @@ -1020,7 +1024,7 @@ class RealTimeStreaming: sent = True continue try: - msg_obj = _decode_json_object(msg) + msg_obj = as_str_mapping(_decode_json(msg)) except (json.JSONDecodeError, TypeError): msg_obj = None if isinstance(msg_obj, dict) and self.provider_config.is_setup_message(msg_obj): @@ -1162,13 +1166,15 @@ class RealTimeStreaming: return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS) async def _hold_session_update(self, message: str) -> None: - client_event: Final = _decode_json_object(message) + client_event: Final = _decode_json(message) if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES: await self.websocket.send_text( - json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id"))) + json.dumps( + _HELD_SESSION_UPDATE_TOO_LARGE.error_event(decoded_json_object(client_event).get("event_id")) + ) ) return - session: Final = client_event.get("session") + session: Final = decoded_json_object(client_event).get("session") update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update) self._held_session_update = held @@ -1182,7 +1188,7 @@ class RealTimeStreaming: _session_update_frame(session), self.model, None ) return next( - (frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None + (frame for frame in preview if isinstance(frame, str) and json_contains(_decode_json(frame), "setup")), None ) def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool: @@ -1194,7 +1200,7 @@ class RealTimeStreaming: return ( applied_setup is not None and merged_setup is not None - and _decode_json_object(applied_setup) == _decode_json_object(merged_setup) + and _decode_json(applied_setup) == _decode_json(merged_setup) ) async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None: @@ -1522,8 +1528,8 @@ class RealTimeStreaming: ``return_new_content_delta_events`` modality lookup, ...). """ try: - message_obj: Final = _decode_json_object(transformed_message) - if "setup" in message_obj: + message_obj: Final = _decode_json(transformed_message) + if json_contains(message_obj, "setup"): self.session_configuration_request = transformed_message except (json.JSONDecodeError, TypeError): return @@ -1852,7 +1858,7 @@ class RealTimeStreaming: def _parse_backend_event(raw_response: str) -> dict[str, object] | None: """Parse a backend frame once. Returns None for non-JSON or non-object frames.""" try: - event: Final = _decode_json_object(raw_response) + event: Final = as_str_mapping(_decode_json(raw_response)) except (json.JSONDecodeError, TypeError): return None return event if isinstance(event, dict) else None diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index c13c9bc31fb..3760d7f3059 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -351,7 +351,7 @@ def _redacted_web_search_action(action: object) -> object: def _redacted_web_search_call(item: object) -> object: if isinstance(item, BaseModel): - action: Final = getattr(item, "action", None) + action: Final = attribute_or_none(item, "action") return item if action is None else item.model_copy(update={"action": _redacted_web_search_action(action)}) mapping: Final = _as_mapping(item) if mapping is None or mapping.get("action") is None: diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index e2b651f8885..ee70152636f 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDic from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import ReadOnly, Required +from typing_extensions import TypedDict as ExtraItemsTypedDict import litellm from litellm._logging import verbose_logger @@ -19,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import ( chunk_usage, finite_cost, provider_reported_usage_cost, + require_attributes, stamp_litellm_bill_on_usage, ) from litellm.litellm_core_utils.hidden_params import ( @@ -82,6 +84,18 @@ if TYPE_CHECKING: ) +class _UsageFields(ExtraItemsTypedDict, total=False, extra_items=object): + pass + + +def _usage_fields(fields: Mapping[str, object]) -> _UsageFields: + return _UsageFields(**fields) + + +class _SupportsModelDump(Protocol): + def model_dump(self) -> Mapping[str, object]: ... + + class _ThinkingBlockFragment(TypedDict, total=False): type: str | None data: str | None @@ -946,12 +960,12 @@ class ChunkProcessor: usage_chunk = chunk.hidden_params.get("usage", None) if isinstance(usage_chunk, dict): - return Usage(**usage_chunk) + return Usage(**_usage_fields(usage_chunk)) if isinstance(usage_chunk, UnreportedUsage): return None if usage_chunk is None or isinstance(usage_chunk, Usage): return usage_chunk - return Usage(**usage_chunk.model_dump()) + return Usage(**_usage_fields(require_attributes(usage_chunk, _SupportsModelDump, "model_dump").model_dump())) def _calculate_usage_per_chunk( self, @@ -1275,7 +1289,7 @@ class ChunkProcessor: if value is not None } - returned_usage = Usage(**returned_usage.model_dump(), **provider_pricing_fields) + returned_usage = Usage(**_usage_fields(returned_usage.model_dump()), **provider_pricing_fields) return returned_usage diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 079b29d5509..62ba7bb0e04 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -27,11 +27,17 @@ from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_F from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import ( UPSTREAM_USAGE_COST, + as_str_mapping, attribute_or_none, chunk_usage, decoded_json_object, + has_attributes, + json_contains, provider_reported_usage_cost, + require_attributes, stamp_litellm_bill_on_usage, + subscript, + without, ) from litellm.litellm_core_utils.hidden_params import ( HIDDEN_PARAMS_ATTR, @@ -195,6 +201,10 @@ def _response_choice_index(response_obj: Mapping[str, object]) -> int: return _upstream_choice_index(response_obj.get("original_chunk")) +def _delta_choice_index(completion_fields: Mapping[str, object], response_fields: Mapping[str, object]) -> object: + return completion_fields.get("index", response_fields.get("choice_index")) + + def _non_null_upstream_choice_fields(upstream_chunk: object, fields: Sequence[str]) -> Mapping[str, object]: first_choice: Final = _upstream_first_choice(upstream_chunk) upstream_extras: Final = first_choice.model_extra if first_choice is not None else None @@ -231,13 +241,21 @@ def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool: return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields) +class _KeywordFields(TypedDict, total=False, extra_items=object): + pass + + +def _keyword_fields(fields: Mapping[str, object]) -> _KeywordFields: + return _KeywordFields(**fields) + + def _delta_from_upstream(upstream_delta: "BaseModel | _UpstreamDeltaLike | None") -> Delta | None: if upstream_delta is None: return Delta() if not isinstance(upstream_delta, BaseModel): return None try: - return Delta(**dict(iter(upstream_delta))) + return Delta(**_keyword_fields(dict(iter(upstream_delta)))) except Exception: return None @@ -275,7 +293,7 @@ def print_verbose(print_statement: object): @dataclass(frozen=True, slots=True) class _ProviderChunkParsed: - response_obj: dict[str, object] + response_obj: Mapping[str, object] @dataclass(frozen=True, slots=True) @@ -286,19 +304,8 @@ class _ProviderChunkEarlyReturn: _ProviderChunkResult = _ProviderChunkParsed | _ProviderChunkEarlyReturn -class _PredibaseStreamData(TypedDict): - token: NotRequired[Mapping[str, str]] - details: Mapping[str, str] - generated_text: str | None - error: str | None - - -class _NlpCloudStreamData(TypedDict): - generated_text: str - - -class _AlephAlphaStreamData(TypedDict): - completions: Sequence[Mapping[str, str]] +class _Replaceable(Protocol): + def replace(self, old: str, new: str, /) -> object: ... class _DeltaDumpDict(TypedDict): @@ -365,6 +372,32 @@ class _VertexChunkLike(Protocol): candidates: Sequence[_VertexCandidateLike] +class _TokenTotalsLike(Protocol): + prompt_tokens: int + completion_tokens: int + total_tokens: int + + +class _ParamsLookupLike(Protocol): + def get(self, key: str, default: object, /) -> object: ... + + +class _IdentifiedChunkLike(Protocol): + id: str + + +class _Decodable(Protocol): + def decode(self, encoding: str, /) -> str: ... + + +class _VllmRequestOutputLike(Protocol): + outputs: object + + +class _VllmCompletionOutputLike(Protocol): + text: object + + class _ParsedChunkHiddenParams(LiteLLMBaseModel): provider_specific_fields: Mapping[str, object] | None = None upstream_provider: str | None = None @@ -783,28 +816,33 @@ class CustomStreamWrapper: self.holding_chunk = "" return hold, curr_chunk - def handle_predibase_chunk(self, chunk): + def handle_predibase_chunk(self, chunk: object): try: if not isinstance(chunk, str): - chunk = chunk.decode("utf-8") # DO NOT REMOVE this: This is required for HF inference API + Streaming + chunk = require_attributes(chunk, _Decodable, "decode").decode( + "utf-8" + ) # DO NOT REMOVE this: This is required for HF inference API + Streaming text = "" is_finished = False finish_reason = "" print_verbose(f"chunk: {chunk}") if chunk.startswith("data:"): - data_json: Final[_PredibaseStreamData] = json.loads(chunk[5:]) + data_json: Final = _DECODED_JSON.validate_python(json.loads(chunk[5:])) print_verbose(f"data json: {data_json}") - if "token" in data_json and "text" in data_json["token"]: - text = data_json["token"]["text"] - if data_json.get("details", False) and data_json["details"].get("finish_reason", False): + if json_contains(data_json, "token") and json_contains(subscript(data_json, "token"), "text"): + text = subscript(subscript(data_json, "token"), "text") + data_lookup: Final = require_attributes(data_json, _ParamsLookupLike, "get") + if data_lookup.get("details", False) and require_attributes( + subscript(data_json, "details"), _ParamsLookupLike, "get" + ).get("finish_reason", False): is_finished = True - finish_reason = data_json["details"]["finish_reason"] - elif data_json.get("generated_text", False): # if full generated text exists, then stream is complete + finish_reason = subscript(subscript(data_json, "details"), "finish_reason") + elif data_lookup.get("generated_text", False): # if full generated text exists, then stream is complete text = "" # don't return the final bos token is_finished = True finish_reason = "stop" - elif data_json.get("error", False): - raise Exception(data_json.get("error")) + elif data_lookup.get("error", False): + raise Exception(data_lookup.get("error", None)) return { "text": text, "is_finished": is_finished, @@ -828,11 +866,11 @@ class CustomStreamWrapper: if self.model and "dolphin" in self.model: chunk = self.process_chunk(chunk=chunk) else: - data_json: Final[_NlpCloudStreamData] = json.loads(chunk) - chunk = data_json["generated_text"] + data_json: Final = _DECODED_JSON.validate_python(json.loads(chunk)) + chunk = subscript(data_json, "generated_text") text = chunk - if "[DONE]" in text: - text = text.replace("[DONE]", "") + if json_contains(text, "[DONE]"): + text = require_attributes(text, _Replaceable, "replace").replace("[DONE]", "") is_finished = True finish_reason = "stop" return { @@ -845,9 +883,9 @@ class CustomStreamWrapper: def handle_aleph_alpha_chunk(self, chunk): chunk = chunk.decode("utf-8") - data_json: Final[_AlephAlphaStreamData] = json.loads(chunk) + data_json: Final = _DECODED_JSON.validate_python(json.loads(chunk)) try: - text: Final = data_json["completions"][0]["completion"] + text: Final = subscript(subscript(subscript(data_json, "completions"), 0), "completion") is_finished: Final = True finish_reason: Final = "stop" return { @@ -1014,20 +1052,17 @@ class CustomStreamWrapper: def model_response_creator( self, - chunk: dict | None = None, + chunk: dict[str, object] | None = None, hidden_params: Mapping[str, object] | None = None, delta: Delta | None = None, ) -> ModelResponseStream: _model: Final = self._served_openrouter_model() or self._cached_model_name _logging_obj_llm_provider: Final = self._cached_logging_llm_provider - if chunk is None: - args: dict[str, Any] = {"model": _model} - else: + if chunk is not None: chunk.pop("model", None) - args = {"model": _model} - if chunk: - args.update({k: v for k, v in chunk.items() if k != "stream"}) + chunk_fields: Final[Mapping[str, object]] = {k: v for k, v in chunk.items() if k != "stream"} if chunk else {} + args: Final = _keyword_fields({"model": _model, **chunk_fields}) model_response: Final = ( ModelResponseStream(**args) @@ -1108,7 +1143,8 @@ class CustomStreamWrapper: provider_specific_fields: Final = getattr(original_chunk, "provider_specific_fields", None) if provider_specific_fields is not None: model_response.provider_specific_fields = provider_specific_fields - _apply_provider_fields(model_response, provider_specific_fields) + if model_response.provider_specific_fields is not None: + _apply_provider_fields(model_response, model_response.provider_specific_fields) return model_response def is_chunk_non_empty( @@ -1169,10 +1205,8 @@ class CustomStreamWrapper: model_response.choices[0].delta["role"] = "assistant" self.sent_first_chunk = True elif self.sent_first_chunk is True and hasattr(model_response.choices[0].delta, "role"): - _initial_delta: Final = model_response.choices[0].delta.model_dump() - - _initial_delta.pop("role", None) - set_extra_attribute(model_response.choices[0], "delta", Delta(**_initial_delta)) + _initial_delta: Final = without(model_response.choices[0].delta.model_dump(), "role") + set_extra_attribute(model_response.choices[0], "delta", Delta(**_keyword_fields(_initial_delta))) return model_response def _prebuilt_plain_delta(self, chunk: object) -> Delta | None: @@ -1275,7 +1309,7 @@ class CustomStreamWrapper: self, completion_obj: dict[str, Any], model_response: ModelResponseStream, - response_obj: dict[str, Any], + response_obj: Mapping[str, Any], plain_delta_created: bool = False, ): from litellm.litellm_core_utils.core_helpers import ( @@ -1300,19 +1334,15 @@ class CustomStreamWrapper: if hold is False: ## check if openai/azure chunk - original_chunk: Final = response_obj.get("original_chunk", None) + original_chunk: Final[object] = response_obj.get("original_chunk", None) if original_chunk: - upstream_chunk: Final[_UpstreamChunkLike] = original_chunk + upstream_chunk: Final = require_attributes(original_chunk, _UpstreamChunkLike, "choices") if len(upstream_chunk.choices) > 0: - original_chunk_for_reuse: Final[object] = cast( # cast-ok: runtime type-check before access - object, - original_chunk, - ) upstream_choice: Final = ( - original_chunk_for_reuse.choices[0] + original_chunk.choices[0] if plain_delta_created - and isinstance(original_chunk_for_reuse, (ChatCompletionChunk, ModelResponseStream)) - and len(original_chunk_for_reuse.choices) == 1 + and isinstance(original_chunk, (ChatCompletionChunk, ModelResponseStream)) + and len(original_chunk.choices) == 1 else None ) upstream_delta: Final = cast( # cast-ok: [LIT006] runtime delta may be absent @@ -1343,13 +1373,13 @@ class CustomStreamWrapper: for choice in upstream_chunk.choices: try: if isinstance(choice, BaseModel): - choice_json = choice.model_dump() - choice_json.pop( - "finish_reason", None - ) # for mistral etc. which return a value in their last chunk (not-openai compatible). - choice_json.pop("logprobs", None) choices.append( - StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None)) + StreamingChoices( + **_keyword_fields( + without(choice.model_dump(), "finish_reason", "logprobs") + ), # for mistral etc. which return a value in their last chunk (not-openai compatible). + logprobs=getattr(choice, "logprobs", None), + ) ) except Exception: choices.append(StreamingChoices()) @@ -1377,10 +1407,10 @@ class CustomStreamWrapper: self.sent_first_chunk = True if response_obj.get("provider_specific_fields") is not None: completion_obj["provider_specific_fields"] = response_obj["provider_specific_fields"] - model_response.choices[0].delta = Delta(**completion_obj) - _index: Final[int | None] = completion_obj.get("index", response_obj.get("choice_index")) + model_response.choices[0].delta = Delta(**_keyword_fields(completion_obj)) + _index: Final = _delta_choice_index(completion_obj, response_obj) if _index is not None: - model_response.choices[0].index = _index + set_extra_attribute(model_response.choices[0], "index", _index) _apply_choice_fields(model_response.choices[0], chunk_choice_fields) if sibling_finish_reason is not None: @@ -1436,7 +1466,7 @@ class CustomStreamWrapper: # Preserve custom attributes from original chunk (applies to both # empty and non-empty delta final chunks). - _original_chunk: Final = response_obj.get("original_chunk", None) + _original_chunk: Final[object] = response_obj.get("original_chunk", None) if _original_chunk is not None: preserve_upstream_non_openai_attributes( model_response=model_response, @@ -1504,7 +1534,7 @@ class CustomStreamWrapper: def _dispatch_provider_chunk( self, - chunk: Any, + chunk: object, model_response: ModelResponseStream, completion_obj: dict[str, object], ) -> _ProviderChunkResult: @@ -1532,19 +1562,23 @@ class CustomStreamWrapper: chunk.choices[0].finish_reason = None return _ProviderChunkEarlyReturn(chunk) + chunk_fields: Final = as_str_mapping(chunk) + generic_chunk: Final = chunk_fields if isinstance(chunk_fields, dict) else None if ( - isinstance(chunk, dict) - and generic_chunk_has_all_required_fields(chunk=chunk) # check if chunk is a generic streaming chunk + # check if chunk is a generic streaming chunk + generic_chunk is not None and generic_chunk_has_all_required_fields(chunk=generic_chunk) ) or (self.custom_llm_provider and self.custom_llm_provider in litellm._custom_providers): if self.received_finish_reason is not None: - _chunk_has_content: Final = isinstance(chunk, dict) and ( - bool(chunk.get("text", "")) - or chunk.get("tool_use") is not None + _chunk_has_content: Final = generic_chunk is not None and ( + bool(generic_chunk.get("text", "")) + or generic_chunk.get("tool_use") is not None # Usage-only final chunks are valid and needed to surface # finish_reason/usage to downstream translators. - or chunk.get("usage") is not None + or generic_chunk.get("usage") is not None ) - if not _chunk_has_content and (not isinstance(chunk, dict) or "provider_specific_fields" not in chunk): + if not _chunk_has_content and ( + generic_chunk is None or "provider_specific_fields" not in generic_chunk + ): raise StopIteration anthropic_response_obj: Final[GChunk] = cast(GChunk, chunk) completion_obj["content"] = anthropic_response_obj["text"] @@ -1582,7 +1616,12 @@ class CustomStreamWrapper: if response_obj["is_finished"]: self.received_finish_reason = response_obj["finish_reason"] elif self.custom_llm_provider and self.custom_llm_provider == "vllm": - completion_obj["content"] = chunk[0].outputs[0].text + vllm_output: Final = require_attributes( + subscript(require_attributes(subscript(chunk, 0), _VllmRequestOutputLike, "outputs").outputs, 0), + _VllmCompletionOutputLike, + "text", + ) + completion_obj["content"] = vllm_output.text elif ( self.custom_llm_provider and self.custom_llm_provider == "aleph_alpha" ): # aleph alpha doesn't provide streaming @@ -1689,7 +1728,13 @@ class CustomStreamWrapper: if response_obj["is_finished"] and response_obj["sibling_finish_reason"] is None: self.received_finish_reason = response_obj["finish_reason"] if response_obj["usage"] is not None: - _text_completion_usage: Final[Usage] = response_obj["usage"] + _text_completion_usage: Final = require_attributes( + _DECODED_JSON.validate_python(response_obj["usage"]), + _TokenTotalsLike, + "prompt_tokens", + "completion_tokens", + "total_tokens", + ) setattr( model_response, "usage", @@ -1711,7 +1756,13 @@ class CustomStreamWrapper: if response_obj["is_finished"]: self.received_finish_reason = response_obj["finish_reason"] if "usage" in response_obj is not None: - _codestral_usage: Final[Usage] = response_obj["usage"] + _codestral_usage: Final = require_attributes( + _DECODED_JSON.validate_python(response_obj["usage"]), + _TokenTotalsLike, + "prompt_tokens", + "completion_tokens", + "total_tokens", + ) setattr( model_response, "usage", @@ -1813,14 +1864,14 @@ class CustomStreamWrapper: ) return _ProviderChunkParsed(response_obj) - def chunk_creator(self, chunk: Any): + def chunk_creator(self, chunk: object): if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields: self.logging_obj.record_native_bedrock_guardrail(chunk.provider_specific_fields) if self.custom_llm_provider != "cached_response" and isinstance( chunk, (ChatCompletionChunk, ModelResponseStream) ): self._ollama_response_ids.observe(chunk) - if hasattr(chunk, "id"): + if has_attributes(chunk, _IdentifiedChunkLike, "id"): self.response_id = chunk.id if self._ollama_response_ids.response_id is not None: self.response_id = self._ollama_response_ids.response_id @@ -1840,10 +1891,10 @@ class CustomStreamWrapper: ), delta=prebuilt_delta, ) - response_obj: dict[str, Any] = {} + response_obj: Mapping[str, object] = {} try: # return this for all models - completion_obj: Final[dict[str, Any]] = {"content": ""} + completion_obj: Final[dict[str, object]] = {"content": ""} dispatch_result: Final = self._dispatch_provider_chunk( chunk=chunk, model_response=model_response, @@ -1855,24 +1906,23 @@ class CustomStreamWrapper: model_response.model = self._served_openrouter_model() or self.model ## FUNCTION CALL PARSING - original_chunk: Final[_UpstreamChunkLike | None] = ( - response_obj.get("original_chunk") if response_obj is not None else None - ) + original_chunk: Final = response_obj.get("original_chunk") if response_obj is not None else None if ( original_chunk is not None ): # function / tool calling branch - only set for openai/azure compatible endpoints # enter this branch when no content has been passed in response - if hasattr(original_chunk, "id"): + if has_attributes(original_chunk, _IdentifiedChunkLike, "id"): model_response = self.set_model_id(original_chunk.id, model_response) if stored_attribute(original_chunk, "provider_specific_fields") is not None: model_response = self.copy_model_response_level_provider_specific_fields( original_chunk, model_response ) - if original_chunk.choices and len(original_chunk.choices) > 0: - delta: Final = original_chunk.choices[0].delta + upstream_chunk: Final = require_attributes(original_chunk, _UpstreamChunkLike, "choices") + if upstream_chunk.choices and len(upstream_chunk.choices) > 0: + delta: Final = upstream_chunk.choices[0].delta if delta is not None and (delta.function_call is not None or delta.tool_calls is not None): try: - model_response.system_fingerprint = original_chunk.system_fingerprint + model_response.system_fingerprint = upstream_chunk.system_fingerprint ## AZURE - check if arguments is not None if delta.function_call is not None: if delta.function_call.arguments is None: @@ -1922,7 +1972,7 @@ class CustomStreamWrapper: if "tool_calls" in completion_obj and _entry_len(completion_obj, "tool_calls") > 0: if self.is_function_call is True: # user passed in 'functions' param - completion_obj["function_call"] = completion_obj["tool_calls"][0]["function"] + completion_obj["function_call"] = subscript(subscript(completion_obj["tool_calls"], 0), "function") completion_obj["tool_calls"] = None self.tool_call = True @@ -1931,7 +1981,7 @@ class CustomStreamWrapper: if isinstance(chunk_usage, Usage): model_response.usage = chunk_usage elif isinstance(chunk_usage, BaseModel): - model_response.usage = Usage(**chunk_usage.model_dump()) + model_response.usage = Usage(**_keyword_fields(chunk_usage.model_dump())) ## RETURN ARG if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields: @@ -2126,7 +2176,11 @@ class CustomStreamWrapper: else: asyncio.run(self.logging_obj.async_success_handler(processed_chunk, None, None, cache_hit)) ## SYNC LOGGING — only for sync SDK entrypoints; async proxy paths export via async_success_handler - litellm_params: Final[Mapping[str, object]] = self.logging_obj.model_call_details.get("litellm_params", {}) + litellm_params: Final = require_attributes( + _DECODED_JSON.validate_python(self.logging_obj.model_call_details.get("litellm_params", {})), + _ParamsLookupLike, + "get", + ) if self.logging_obj.is_sync_litellm_request(litellm_params): self.logging_obj.success_handler(processed_chunk, None, None, cache_hit) @@ -3015,9 +3069,9 @@ class CustomStreamWrapper: _strip_sse_data_from_chunk = strip_sse_data_from_chunk -def _usage_from_payload(payload: Mapping[str, Any]) -> Usage: +def _usage_from_payload(payload: Mapping[str, object]) -> Usage: """Build ``Usage`` from a wire-shaped usage dict, forwarding every key.""" - return litellm.Usage(**payload) + return litellm.Usage(**_keyword_fields(payload)) def _cache_token_count(details: PromptTokensDetailsWrapper | None, keys: tuple[str, ...]) -> int: diff --git a/litellm/llms/a2a/common_utils.py b/litellm/llms/a2a/common_utils.py index f6ff6d3bc47..d9dd4b2b395 100644 --- a/litellm/llms/a2a/common_utils.py +++ b/litellm/llms/a2a/common_utils.py @@ -2,11 +2,12 @@ Common utilities for A2A (Agent-to-Agent) Protocol """ -from collections.abc import Awaitable, Callable, Mapping -from typing import Any, Final +from collections.abc import Awaitable, Callable, Mapping, Sequence +from typing import Final from pydantic import BaseModel +from litellm.litellm_core_utils.core_helpers import decoded_json_object, iter_items from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) @@ -67,7 +68,15 @@ def convert_messages_to_prompt(messages: list[AllMessageValues]) -> str: return "\n".join(conversation_parts) -def extract_text_from_a2a_message(message: dict[str, Any], depth: int = 0, max_depth: int = 10) -> str: +def _join_with_spaces(texts: Sequence[object]) -> str: + strings: Final = [text for text in texts if isinstance(text, str)] + if len(strings) == len(texts): + return " ".join(strings) + first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str)) + raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found") + + +def extract_text_from_a2a_message(message: object, depth: int = 0, max_depth: int = 10) -> str: """ Extract text content from A2A message parts. @@ -82,19 +91,20 @@ def extract_text_from_a2a_message(message: dict[str, Any], depth: int = 0, max_d if message is None or depth >= max_depth: return "" - parts: Final = message.get("parts", []) - text_parts: Final[list[str]] = [] + parts: Final = iter_items(decoded_json_object(message).get("parts", [])) + text_parts: Final[list[object]] = [] for part in parts: - if part.get("kind") == "text": - text_parts.append(part.get("text", "")) + part_fields = decoded_json_object(part) + if part_fields.get("kind") == "text": + text_parts.append(part_fields.get("text", "")) # Handle nested parts if they exist - elif "parts" in part: - nested_text = extract_text_from_a2a_message(part, depth + 1, max_depth) + elif "parts" in part_fields: + nested_text = extract_text_from_a2a_message(part_fields, depth + 1, max_depth) if nested_text: text_parts.append(nested_text) - return " ".join(text_parts) + return _join_with_spaces(text_parts) def extract_text_from_a2a_response(response_dict: Mapping[str, object], max_depth: int = 10) -> str: diff --git a/litellm/llms/aiohttp_openai/chat/transformation.py b/litellm/llms/aiohttp_openai/chat/transformation.py index 9253ca389ee..9ad5a585e66 100644 --- a/litellm/llms/aiohttp_openai/chat/transformation.py +++ b/litellm/llms/aiohttp_openai/chat/transformation.py @@ -11,7 +11,8 @@ from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final from aiohttp import ClientResponse -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, JsonValue, TypeAdapter +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig @@ -31,6 +32,16 @@ _JSON_OBJECTS: Final = TypeAdapter( ) +class _ChatCompletionBody(TypedDict, extra_items=object): + id: ReadOnly[str] + created: ReadOnly[int] + model: ReadOnly[str] + object: ReadOnly[str] + choices: ReadOnly[JsonValue] + system_fingerprint: ReadOnly[NotRequired[str | None]] + service_tier: ReadOnly[NotRequired[str | None]] + + class AiohttpOpenAIChatConfig(OpenAILikeChatConfig): def get_complete_url( self, @@ -73,7 +84,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: - _json_response: Final = await raw_response.json() + _json_response: Final[_ChatCompletionBody] = await raw_response.json() model_response.id = _json_response.get("id") model_response.choices = [ Choices.model_validate(choice) for choice in _JSON_OBJECTS.validate_python(_json_response.get("choices")) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index c777f1f6ac9..1d24cfab131 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -6,7 +6,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from functools import partial from itertools import chain from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, NoReturn, cast +from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, NoReturn, Protocol, cast, runtime_checkable import httpx from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch @@ -31,6 +31,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.asyncify import is_event_loop_running from litellm.litellm_core_utils.core_helpers import ( + as_object_list, as_str_mapping, coerce_token_limit, drops_unsupported_params, @@ -450,6 +451,46 @@ def _enum_conflicts_with_declared_type(schema: Mapping[str, object]) -> bool: return check is not None and not all(check(value) for value in enum_values) +def _as_json_object(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, dict) else None + + +def _filtered_output_subschema(value: object) -> object: + subschema: Final = _as_json_object(value) + return value if subschema is None else AnthropicConfig.filter_anthropic_output_schema(subschema) + + +def _filtered_output_subschemas(value: object) -> object: + subschemas: Final = _as_json_object(value) + if subschemas is None: + return value + return {name: _filtered_output_subschema(subschema) for name, subschema in subschemas.items()} + + +def _filtered_output_branches(branches: Sequence[object]) -> Sequence[object]: + return [_filtered_output_subschema(branch) for branch in branches] + + +@runtime_checkable +class _SupportsAdd(Protocol): + def __add__(self, other: object, /) -> object: ... + + +def _concatenated(left: object, right: str | Sequence[object]) -> object: + if isinstance(left, _SupportsAdd): + return left + right + raise TypeError(f"unsupported operand type(s) for +: '{type(left).__name__}' and '{type(right).__name__}'") + + +def _extended_any_of(result: Mapping[str, object], branches: Sequence[object]) -> object: + filtered: Final = _filtered_output_branches(branches) + return _concatenated(result["anyOf"], filtered) if "anyOf" in result else filtered + + +def _described_with_note(description: object, note: str) -> object: + return _concatenated(_concatenated(description, " "), note) if description else note + + # Single, internal-only key on ``litellm_params`` used to thread the per- # request reverse map (sanitized -> original) from request build to response # parsing. ``litellm_params`` is never serialized to a provider; ``optional_ @@ -927,7 +968,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return params @staticmethod - def filter_anthropic_output_schema(schema: dict[str, Any]) -> dict[str, Any]: + def filter_anthropic_output_schema(schema: Mapping[str, object]) -> Mapping[str, object]: """ Filter out unsupported fields from JSON schema for Anthropic's output_format API. @@ -1019,16 +1060,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): note_value = json.dumps(value) if isinstance(value, (dict, list)) else value constraint_descriptions.append(label.format(note_value)) - result: Final[dict[str, Any]] = {} + result: Final[dict[str, object]] = {} # Update description with removed constraint info if constraint_descriptions: - existing_desc: Final = schema.get("description", "") constraint_note: Final = "Note: " + ", ".join(constraint_descriptions) + "." - if existing_desc: - result["description"] = existing_desc + " " + constraint_note - else: - result["description"] = constraint_note + result["description"] = _described_with_note(schema.get("description"), constraint_note) drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema) @@ -1041,26 +1078,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Already handled above continue - if key in ("properties", "$defs", "definitions") and isinstance(value, dict): - nested_schemas: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict - result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in nested_schemas.items()} - elif key == "items" and isinstance(value, dict): - nested_schema: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict - result[key] = AnthropicConfig.filter_anthropic_output_schema(nested_schema) - elif key == "anyOf" and isinstance(value, list): - result["anyOf"] = result.get("anyOf", []) + [ - AnthropicConfig.filter_anthropic_output_schema(item) for item in value - ] - elif key == "allOf" and isinstance(value, list): - result[key] = [AnthropicConfig.filter_anthropic_output_schema(item) for item in value] + branches = as_object_list(value) + if key in ("properties", "$defs", "definitions"): + result[key] = _filtered_output_subschemas(value) + elif key == "items": + result[key] = _filtered_output_subschema(value) + elif key == "anyOf" and branches is not None: + result["anyOf"] = _extended_any_of(result, branches) + elif key == "allOf" and branches is not None: + result[key] = _filtered_output_branches(branches) elif key == "$ref" and isinstance(value, str): result[key] = schema_fragment_ref(value) - elif key == "oneOf" and isinstance(value, list): + elif key == "oneOf" and branches is not None: # Anthropic rejects oneOf ("Schema type 'oneOf' is not supported"); # the Anthropic SDK rewrites it to anyOf, so do the same. - result["anyOf"] = result.get("anyOf", []) + [ - AnthropicConfig.filter_anthropic_output_schema(item) for item in value - ] + result["anyOf"] = _extended_any_of(result, branches) else: result[key] = value diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 937afeae402..e32754adf03 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -2570,7 +2570,7 @@ def changed_server_tool_call_arguments( def write_anthropic_hosted_web_call_arguments(holder: Message | Delta, arguments_by_tool_id: Mapping[str, str]) -> None: - fields: Final = holder.provider_specific_fields + fields: Final[Mapping[str, object]] = holder.provider_specific_fields or {} if not arguments_by_tool_id or not fields or fields.get("web_search_calls") is None: return holder.provider_specific_fields = { # rebind-ok: the hosted copy of a rewritten call must not keep its old input @@ -2598,7 +2598,7 @@ def _jsonable_hosted_web_call(item: object, arguments_by_tool_id: Mapping[str, s call: Final = _with_hosted_web_call_arguments(item, arguments_by_tool_id) if not isinstance(call, ResponseFunctionWebSearch): return call - return _WEB_SEARCH_CALL_ADAPTER.dump_python(call, mode="json", exclude_none=True) + return _DECODED_JSON.validate_python(_WEB_SEARCH_CALL_ADAPTER.dump_python(call, mode="json", exclude_none=True)) def _with_hosted_web_call_arguments(item: object, arguments_by_tool_id: Mapping[str, str]) -> object: @@ -2613,7 +2613,7 @@ def _with_hosted_web_call_arguments(item: object, arguments_by_tool_id: Mapping[ def without_anthropic_server_tool_calls(holder: Message, tool_ids: frozenset[str]) -> None: - fields: Final = holder.provider_specific_fields or {} + fields: Final[Mapping[str, object]] = holder.provider_specific_fields or {} holder.provider_specific_fields = { # rebind-ok: a denied call must leave every copy of the message it was on **fields, "server_tool_calls": [ diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 0c564a194dc..e68360e69b9 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -4,12 +4,17 @@ import json from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast from pydantic import JsonValue, TypeAdapter import litellm -from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items, normalize_drop_params +from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, + attribute_items, + normalize_drop_params, + require_attributes, +) from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.messages.utils import ( @@ -254,6 +259,11 @@ if TYPE_CHECKING: ToolResultContent: TypeAlias = str | list[ToolMessageContentPart] +class _CarriesUsage(Protocol): + @property + def usage(self) -> Usage: ... + + def target_supports_mid_conversation_system(model: str | None, custom_llm_provider: str | None) -> bool: if not model: return False @@ -681,7 +691,7 @@ class LiteLLMAnthropicMessagesAdapter: ## ASSISTANT MESSAGE ## assistant_message_str: str | None = None - assistant_content_list: list[dict[str, Any]] = [] # For content blocks with cache_control + assistant_content_list: list[ChatCompletionTextObject] = [] # For content blocks with cache_control has_cache_control_in_text = False tool_calls: list[ChatCompletionAssistantToolCall] = [] thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] = [] @@ -694,7 +704,7 @@ class LiteLLMAnthropicMessagesAdapter: assistant_message_str = str(content) elif isinstance(content, dict): if content.get("type") == "text": - text_block: dict[str, object] = { + text_block: ChatCompletionTextObject = { "type": "text", "text": content.get("text", ""), } @@ -753,7 +763,7 @@ class LiteLLMAnthropicMessagesAdapter: ): # Use list format if any text block has cache_control, otherwise use string if has_cache_control_in_text and len(assistant_content_list) > 0: - assistant_content: Any = assistant_content_list + assistant_content: str | Sequence[ChatCompletionTextObject] | None = assistant_content_list elif len(assistant_content_list) > 0 and not has_cache_control_in_text: # Concatenate text blocks into string when no cache_control assistant_content = "".join(block.get("text", "") for block in assistant_content_list) @@ -1795,7 +1805,7 @@ class LiteLLMAnthropicMessagesAdapter: else translated_finish_reason ) # extract usage - usage: Final[Usage] = getattr(response, "usage") + usage: Final = require_attributes(response, _CarriesUsage, "usage").usage message_usage: Final = self.translate_openai_usage_to_anthropic_usage(usage) polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None anthropic_usage: Final[AnthropicUsage] = ( diff --git a/litellm/llms/anthropic/messages/native/mcp_handler.py b/litellm/llms/anthropic/messages/native/mcp_handler.py index 8fe18d79f24..cefd6c6a6d9 100644 --- a/litellm/llms/anthropic/messages/native/mcp_handler.py +++ b/litellm/llms/anthropic/messages/native/mcp_handler.py @@ -46,6 +46,10 @@ def _get_stop_reason(response: AnthropicMessagesResponse) -> str | None: return stop_reason if isinstance(stop_reason, str) else None +def _requests_stream(kwargs: Mapping[str, object]) -> bool: + return bool(kwargs.get("stream", False)) + + def _build_tool_result_message(tool_results: Sequence[Mapping[str, object]]) -> AnthropicMessagesUserMessageParam: """Turn executed tool results into the user message Anthropic expects.""" return AnthropicMessagesUserMessageParam( @@ -120,7 +124,8 @@ async def anthropic_messages_with_mcp( should_auto_execute: Final = LiteLLM_Proxy_MCP_Handler.should_auto_execute_tools( mcp_tools_with_litellm_proxy=mcp_references ) - stream: Final = bool(kwargs.pop("stream", False)) + stream: Final = _requests_stream(kwargs) + kwargs.pop("stream", None) base_call_args: Final[Mapping[str, object]] = { "max_tokens": max_tokens, diff --git a/litellm/llms/anthropic/messages/native/streaming_iterator.py b/litellm/llms/anthropic/messages/native/streaming_iterator.py index 52b3f1042f8..1195652dfd2 100644 --- a/litellm/llms/anthropic/messages/native/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/streaming_iterator.py @@ -28,6 +28,7 @@ from litellm.llms.anthropic.messages.utils import anthropic_sse_frame from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) +from litellm.responses.mcp.parked_stream_dispatch import DeferredStreamRelease from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.types.utils import GenericStreamingChunk, ModelResponseStream @@ -647,8 +648,12 @@ class BaseAnthropicMessagesStreamingIterator: disconnect (GeneratorExit / CancelledError) re-raises past it. Without this dispatch that window loses the spend row entirely. """ - deferred_cb: Final = getattr(self.litellm_logging_obj, "_on_deferred_stream_complete", None) - deferred_args: Final = getattr(self.litellm_logging_obj, "_deferred_stream_complete_args", None) + deferred_cb: Final[DeferredStreamRelease | None] = getattr( + self.litellm_logging_obj, "_on_deferred_stream_complete", None + ) + deferred_args: Final[tuple[object, ...] | None] = getattr( + self.litellm_logging_obj, "_deferred_stream_complete_args", None + ) if deferred_cb is None or deferred_args is None: return self.litellm_logging_obj._on_deferred_stream_complete = None diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index 5dae6cc9fc3..5ed1030aba9 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -29,6 +29,7 @@ import httpx from typing_extensions import ReadOnly from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import decoded_json_object, subscript from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.azure_ai.agents.transformation import ( @@ -81,19 +82,6 @@ class _AzureMessagesData(TypedDict, total=False): data: ReadOnly[list[_AzureMessage]] -class _CreatedObject(TypedDict): - id: ReadOnly[str] - - -class _RunError(TypedDict, total=False): - message: ReadOnly[str] - - -class _RunStatus(TypedDict, total=False): - status: ReadOnly[str] - last_error: ReadOnly[_RunError] - - class _SSEDelta(TypedDict, total=False): content: ReadOnly[list[_AzureContentItem]] @@ -114,6 +102,14 @@ class _AsyncAgentRequest(Protocol): ) -> Awaitable[httpx.Response]: ... +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class AzureAIAgentsHandler: """ Handler for Azure AI Agent Service. @@ -133,21 +129,21 @@ class AzureAIAgentsHandler: def _build_thread_url(self, api_base: str, api_version: str) -> str: return f"{join_api_base(api_base, 'threads')}?api-version={api_version}" - def _build_messages_url(self, api_base: str, thread_id: str, api_version: str) -> str: + def _build_messages_url(self, api_base: str, thread_id: object, api_version: str) -> str: encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id") return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/messages')}?api-version={api_version}" - def _build_runs_url(self, api_base: str, thread_id: str, api_version: str) -> str: + def _build_runs_url(self, api_base: str, thread_id: object, api_version: str) -> str: encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id") return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/runs')}?api-version={api_version}" - def _build_run_status_url(self, api_base: str, thread_id: str, run_id: str, api_version: str) -> str: + def _build_run_status_url(self, api_base: str, thread_id: object, run_id: object, api_version: str) -> str: encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id") encoded_run_id: Final = encode_url_path_segment(run_id, field_name="run_id") run_status_route: Final = f"threads/{encoded_thread_id}/runs/{encoded_run_id}" return f"{join_api_base(api_base, run_status_route)}?api-version={api_version}" - def _build_list_messages_url(self, api_base: str, thread_id: str, api_version: str) -> str: + def _build_list_messages_url(self, api_base: str, thread_id: object, api_version: str) -> str: encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id") return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/messages')}?api-version={api_version}" @@ -217,7 +213,7 @@ class AzureAIAgentsHandler: model: str, content: str, model_response: ModelResponse, - thread_id: str, + thread_id: object, messages: list[dict[str, object]], annotations: list[_TransformedAnnotation] | None = None, ) -> ModelResponse: @@ -354,10 +350,10 @@ class AzureAIAgentsHandler: api_base: str, api_version: str, agent_id: str, - thread_id: str | None, + thread_id: object, messages: list[dict[str, object]], optional_params: dict, - ) -> tuple[str, str, list[_TransformedAnnotation] | None]: + ) -> tuple[object, str, list[_TransformedAnnotation] | None]: """Execute the agent flow synchronously. Returns (thread_id, content, annotations).""" # Step 1: Create thread if not provided @@ -365,8 +361,7 @@ class AzureAIAgentsHandler: verbose_logger.debug("Creating thread at: %s", self._build_thread_url(api_base, api_version)) response = make_request("POST", self._build_thread_url(api_base, api_version), {}) self._check_response(response, [200, 201], "Failed to create thread") - thread_data: Final[_CreatedObject] = response.json() - thread_id = thread_data["id"] + thread_id = subscript(_json_body(response), "id") verbose_logger.debug("Created thread: %s", thread_id) # At this point thread_id is guaranteed to be a string @@ -386,8 +381,7 @@ class AzureAIAgentsHandler: response = make_request("POST", self._build_runs_url(api_base, thread_id, api_version), run_payload) self._check_response(response, [200, 201], "Failed to create run") - run_data: Final[_CreatedObject] = response.json() - run_id: Final = run_data["id"] + run_id: Final = subscript(_json_body(response), "id") verbose_logger.debug("Created run: %s", run_id) # Step 4: Poll for completion @@ -396,15 +390,15 @@ class AzureAIAgentsHandler: response = make_request("GET", status_url) self._check_response(response, [200], "Failed to get run status") - status_data: _RunStatus = response.json() + status_data = decoded_json_object(_json_body(response)) status = status_data.get("status") verbose_logger.debug("Run status: %s", status) if status == "completed": break elif status in ["failed", "cancelled", "expired"]: - error_data: _RunStatus = response.json() - error_msg = error_data.get("last_error", {}).get("message", "Unknown error") + error_data = decoded_json_object(_json_body(response)) + error_msg = decoded_json_object(error_data.get("last_error", {})).get("message", "Unknown error") raise AzureAIAgentsError(status_code=500, message=f"Run {status}: {error_msg}") time.sleep(self.config.POLL_INTERVAL_SECONDS) @@ -482,10 +476,10 @@ class AzureAIAgentsHandler: api_base: str, api_version: str, agent_id: str, - thread_id: str | None, + thread_id: object, messages: list[dict[str, object]], optional_params: dict, - ) -> tuple[str, str, list[_TransformedAnnotation] | None]: + ) -> tuple[object, str, list[_TransformedAnnotation] | None]: """Execute the agent flow asynchronously. Returns (thread_id, content, annotations).""" # Step 1: Create thread if not provided @@ -493,8 +487,7 @@ class AzureAIAgentsHandler: verbose_logger.debug("Creating thread at: %s", self._build_thread_url(api_base, api_version)) response = await make_request("POST", self._build_thread_url(api_base, api_version), {}) self._check_response(response, [200, 201], "Failed to create thread") - thread_data: Final[_CreatedObject] = response.json() - thread_id = thread_data["id"] + thread_id = subscript(_json_body(response), "id") verbose_logger.debug("Created thread: %s", thread_id) # At this point thread_id is guaranteed to be a string @@ -514,8 +507,7 @@ class AzureAIAgentsHandler: response = await make_request("POST", self._build_runs_url(api_base, thread_id, api_version), run_payload) self._check_response(response, [200, 201], "Failed to create run") - run_data: Final[_CreatedObject] = response.json() - run_id: Final = run_data["id"] + run_id: Final = subscript(_json_body(response), "id") verbose_logger.debug("Created run: %s", run_id) # Step 4: Poll for completion @@ -524,15 +516,15 @@ class AzureAIAgentsHandler: response = await make_request("GET", status_url) self._check_response(response, [200], "Failed to get run status") - status_data: _RunStatus = response.json() + status_data = decoded_json_object(_json_body(response)) status = status_data.get("status") verbose_logger.debug("Run status: %s", status) if status == "completed": break elif status in ["failed", "cancelled", "expired"]: - error_data: _RunStatus = response.json() - error_msg = error_data.get("last_error", {}).get("message", "Unknown error") + error_data = decoded_json_object(_json_body(response)) + error_msg = decoded_json_object(error_data.get("last_error", {})).get("message", "Unknown error") raise AzureAIAgentsError(status_code=500, message=f"Run {status}: {error_msg}") await asyncio.sleep(self.config.POLL_INTERVAL_SECONDS) diff --git a/litellm/llms/azure_ai/vector_stores/transformation.py b/litellm/llms/azure_ai/vector_stores/transformation.py index a4777be146f..59254ac6c0a 100644 --- a/litellm/llms/azure_ai/vector_stores/transformation.py +++ b/litellm/llms/azure_ai/vector_stores/transformation.py @@ -4,7 +4,9 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.base_llm.vector_store.transformation import ( @@ -30,6 +32,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + class AzureAIVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAzureLLM): """ @@ -230,13 +234,14 @@ class AzureAIVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAzureLLM # Build attributes with all available metadata # Exclude system fields and already-processed fields - attributes = {} - for key, value in result.items(): - if key not in ["id", "content", "contentVector", "@search.score"]: - attributes[key] = value + metadata = { + key: value + for key, value in decoded_json_object(_DECODED_JSON.validate_python(result)).items() + if key not in ["id", "content", "contentVector", "@search.score"] + } # Always include document_id in attributes - attributes["document_id"] = document_id + attributes = {**metadata, "document_id": document_id} result_obj = VectorStoreSearchResult( score=score, diff --git a/litellm/llms/base_llm/base_model_iterator.py b/litellm/llms/base_llm/base_model_iterator.py index 3afc685b3dd..a705cfe20ad 100644 --- a/litellm/llms/base_llm/base_model_iterator.py +++ b/litellm/llms/base_llm/base_model_iterator.py @@ -1,7 +1,10 @@ import json from abc import abstractmethod +from collections.abc import Mapping from typing import TYPE_CHECKING, Final, cast +from typing_extensions import TypedDict + import litellm from litellm.constants import STREAM_SSE_DONE_STRING @@ -27,10 +30,18 @@ def _indexed_thinking_block( return {**block, "index": index} +class _DeltaFields(TypedDict, total=False, extra_items=object): + pass + + +def _message_fields_without_thinking_blocks(message: Message) -> Mapping[str, object]: + return message.model_dump(exclude={"thinking_blocks"}) + + def _streaming_delta(message: Message) -> Delta: thinking_blocks: Final = message.thinking_blocks if hasattr(message, "thinking_blocks") else None return Delta( - **message.model_dump(exclude={"thinking_blocks"}), + **_DeltaFields(**_message_fields_without_thinking_blocks(message)), thinking_blocks=( None if thinking_blocks is None diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index b87a783e244..08f986de85a 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -73,8 +73,8 @@ _NO_ATTRIBUTION_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({}) def with_attribution_headers( attribution_headers: Mapping[str, str], - headers: dict[str, str] | None, # mutable-ok: returned as-is when there is nothing to add -) -> dict[str, str] | None: # mutable-ok: becomes the request's outbound headers + headers: dict[str, object] | None, # mutable-ok: returned as-is when there is nothing to add +) -> dict[str, object] | None: # mutable-ok: becomes the request's outbound headers """ `headers` plus any attribution header the caller didn't already set (names compared case-insensitively). Builds a new dict; `headers` is never mutated. diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index d7b643eb7f4..3e48c6f2ff2 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -235,7 +235,7 @@ class BaseVideoConfig(ABC): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, - ) -> dict[str, str]: + ) -> object: pass @abstractmethod diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py index dc14ef005b5..f9e834e9d37 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -7,10 +7,11 @@ https://docs.twelvelabs.io/docs/models/pegasus import json import time +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final import httpx -from pydantic import ConfigDict, TypeAdapter, with_config +from pydantic import BaseModel, ConfigDict, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -43,6 +44,7 @@ class _PegasusResponse(TypedDict): _PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse) +_OPTIONAL_PARAM: Final = TypeAdapter(object) class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): @@ -65,7 +67,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): def map_openai_params( self, - non_default_params: dict, + non_default_params: Mapping[str, object], optional_params: dict, model: str, drop_params: bool, @@ -79,7 +81,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): optional_params["responseFormat"] = self._normalize_response_format(value) return optional_params - def _normalize_response_format(self, value: Any) -> object: + def _normalize_response_format(self, value: object) -> object: """Normalize response_format to TwelveLabs format. TwelveLabs expects: @@ -110,7 +112,11 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): return value # Otherwise return the dict as is return value - return type_to_response_format_param(response_format=value) or value + if value is None: + return None + if isinstance(value, type) and issubclass(value, BaseModel): + return type_to_response_format_param(response_format=value) or value + raise TypeError(f"Unsupported response_format type - {value}") def transform_request( self, @@ -134,7 +140,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): # Handle responseFormat - transform to TwelveLabs format if "responseFormat" in optional_params: - response_format: Final = optional_params["responseFormat"] + response_format: Final = _OPTIONAL_PARAM.validate_python(optional_params["responseFormat"]) transformed_format: Final = self._normalize_response_format(response_format) if transformed_format: request_data["responseFormat"] = transformed_format diff --git a/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py b/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py index 6f972986e56..5685a7b3ef7 100644 --- a/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py @@ -1,8 +1,9 @@ import types from collections.abc import Mapping -from typing import Any, Final +from typing import Final from openai.types.image import Image +from typing_extensions import TypedDict from litellm.llms.bedrock.common_utils import get_cached_model_info from litellm.types.llms.bedrock import ( @@ -19,6 +20,10 @@ from litellm.types.llms.bedrock import ( from litellm.types.utils import ImageResponse +class _NovaCanvasTaskFields(TypedDict, total=False, extra_items=object): + pass + + class AmazonNovaCanvasConfig: """ Reference: https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/model-catalog/serverless/amazon.nova-canvas-v1:0 @@ -82,10 +87,12 @@ class AmazonNovaCanvasConfig: image_generation_config = {**image_generation_config, **optional_params} if task_type == "TEXT_IMAGE": - text_to_image_params: dict[str, Any] = image_generation_config.pop("textToImageParams", {}) + text_to_image_params: Mapping[str, object] = image_generation_config.pop("textToImageParams", {}) text_to_image_params = {"text": text, **text_to_image_params} try: - text_to_image_params_typed: Final = AmazonNovaCanvasTextToImageParams(**text_to_image_params) + text_to_image_params_typed: Final = AmazonNovaCanvasTextToImageParams( + **_NovaCanvasTaskFields(**text_to_image_params) + ) except Exception as e: raise ValueError( f"Error transforming text to image params: {e}. Got params: {text_to_image_params}, Expected params: {AmazonNovaCanvasTextToImageParams.__annotations__}" @@ -104,7 +111,7 @@ class AmazonNovaCanvasConfig: imageGenerationConfig=image_generation_config_typed, ) if task_type == "COLOR_GUIDED_GENERATION": - color_guided_generation_params: dict[str, Any] = image_generation_config.pop( + color_guided_generation_params: Mapping[str, object] = image_generation_config.pop( "colorGuidedGenerationParams", {} ) color_guided_generation_params = { @@ -113,7 +120,7 @@ class AmazonNovaCanvasConfig: } try: color_guided_generation_params_typed: Final = AmazonNovaCanvasColorGuidedGenerationParams( - **color_guided_generation_params + **_NovaCanvasTaskFields(**color_guided_generation_params) ) except Exception as e: raise ValueError( @@ -133,10 +140,12 @@ class AmazonNovaCanvasConfig: imageGenerationConfig=image_generation_config_typed, ) if task_type == "INPAINTING": - inpainting_params: dict[str, Any] = image_generation_config.pop("inpaintingParams", {}) + inpainting_params: Mapping[str, object] = image_generation_config.pop("inpaintingParams", {}) inpainting_params = {"text": text, **inpainting_params} try: - inpainting_params_typed: Final = AmazonNovaCanvasInpaintingParams(**inpainting_params) + inpainting_params_typed: Final = AmazonNovaCanvasInpaintingParams( + **_NovaCanvasTaskFields(**inpainting_params) + ) except Exception as e: raise ValueError( f"Error transforming inpainting params: {e}. Got params: {inpainting_params}, Expected params: {AmazonNovaCanvasInpaintingParams.__annotations__}" diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index a20eae0b59d..be23e450b19 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -8,13 +8,13 @@ from dataclasses import dataclass from functools import partial, reduce from itertools import accumulate, chain from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias, TypeVar +from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypeVar, runtime_checkable from pydantic import InstanceOf, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object +from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, subscript from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, @@ -171,10 +171,33 @@ def _reads_flat_texts_only(guardrail: "CustomGuardrail") -> bool: return is_compression_guardrail(guardrail) or not guardrail.supports_structured_messages() -_StringHolder = tuple[Any, str | int] +_StringHolder: TypeAlias = tuple[object, str | int] +_StringLeaf: TypeAlias = tuple[_StringHolder, str] _LeafIdentity: TypeAlias = tuple[int, str | int] +@runtime_checkable +class _SupportsSetItem(Protocol): + def __setitem__(self, key: str | int, value: object, /) -> None: ... + + +@runtime_checkable +class _SupportsDelItem(Protocol): + def __delitem__(self, key: str | int, /) -> None: ... + + +def _assignable(container: object) -> _SupportsSetItem: + if isinstance(container, _SupportsSetItem): + return container + raise TypeError(f"'{type(container).__name__}' object does not support item assignment") + + +def _deletable(container: object) -> _SupportsDelItem: + if isinstance(container, _SupportsDelItem): + return container + raise TypeError(f"'{type(container).__name__}' object doesn't support item deletion") + + @dataclass(frozen=True, slots=True) class _Assign: holder: _StringHolder @@ -194,7 +217,7 @@ _OBJECT_BY_REFERENCE: Final = TypeAdapter(InstanceOf[Mapping[str, object]]) _ARRAY_BY_REFERENCE: Final = TypeAdapter(InstanceOf[list[object]]) -def _collect_strings(node: object, holders: list[_StringHolder]) -> None: +def _collect_strings(node: object, leaves: list[_StringLeaf]) -> None: """ Record a (container, key) holder for every non-empty string value nested under an arbitrary JSON node, so prompt content a caller hides in fields @@ -209,28 +232,28 @@ def _collect_strings(node: object, holders: list[_StringHolder]) -> None: for key, value in current.items(): if isinstance(value, str): if value: - holders.append((current, key)) + leaves.append(((current, key), value)) else: stack.append(value) elif isinstance(current, list): for index, value in enumerate(current): if isinstance(value, str): if value: - holders.append((current, index)) + leaves.append(((current, index), value)) else: stack.append(value) -def _collect_block_text(block: dict, holders: list[_StringHolder]) -> None: +def _collect_block_text(block: dict, leaves: list[_StringLeaf]) -> None: text: Final = block.get("text") if isinstance(text, str) and text: - holders.append((block, "text")) + leaves.append(((block, "text"), text)) -def _request_field_holders(body: Mapping[str, object]) -> tuple[_StringHolder, ...]: - holders: Final = list[_StringHolder]() - _collect_strings(body.get("additionalModelRequestFields"), holders) - return tuple(holders) +def _request_field_leaves(body: Mapping[str, object]) -> tuple[_StringLeaf, ...]: + leaves: Final = list[_StringLeaf]() + _collect_strings(body.get("additionalModelRequestFields"), leaves) + return tuple(leaves) def _extract_converse_texts( @@ -267,15 +290,15 @@ def _extract_converse_texts( that model. Scanning it stays the fail-closed default. Its strings come last, in the same order as they follow the latest user message in the rows. """ - system_holders: Final = list[_StringHolder]() + system_leaves: Final = list[_StringLeaf]() if not skip_system and not scan_only_tool_results: for block in body.get("system") or []: if isinstance(block, dict): - _collect_block_text(block, system_holders) + _collect_block_text(block, system_leaves) - request_field_holders: Final = () if scan_only_tool_results else _request_field_holders(body) + request_field_leaves: Final = () if scan_only_tool_results else _request_field_leaves(body) - message_holders: Final = list[_StringHolder]() + message_leaves: Final = list[_StringLeaf]() for message in body.get("messages") or []: if not isinstance(message, dict): continue @@ -286,20 +309,19 @@ def _extract_converse_texts( continue if scan_only_tool_results and "toolResult" not in block: continue - _collect_block_text(block, message_holders) + _collect_block_text(block, message_leaves) tool_use = block.get("toolUse") if isinstance(tool_use, dict): - _collect_strings(tool_use.get("input"), message_holders) + _collect_strings(tool_use.get("input"), message_leaves) tool_result = block.get("toolResult") if isinstance(tool_result, dict): for inner in tool_result.get("content") or []: if isinstance(inner, dict): - _collect_block_text(inner, message_holders) - _collect_strings(inner.get("json"), message_holders) + _collect_block_text(inner, message_leaves) + _collect_strings(inner.get("json"), message_leaves) - holders: Final = [*system_holders, *message_holders, *request_field_holders] - texts: Final = [container[key] for container, key in holders] - return texts, holders + leaves: Final = (*system_leaves, *message_leaves, *request_field_leaves) + return [text for _, text in leaves], [holder for holder, _ in leaves] @dataclass(frozen=True, slots=True) @@ -464,9 +486,9 @@ def _apply(writes: Sequence[_Assign | _Remove]) -> None: for write in writes: match write: case _Assign(holder=(container, key), value=value): - container[key] = value + _assignable(container)[key] = value case _Remove(holder=(container, key)): - del container[key] + del _deletable(container)[key] def _block_text_holder(block: object) -> _ContentHolder | None: @@ -545,7 +567,7 @@ def _edits_drop_a_text_change( if writes is None: return True _apply(writes) - edited: Final = [container[key] for container, key in holders] + edited: Final = [subscript(container, key) for container, key in holders] return any( returned != before and after == before for before, returned, after in zip(texts, guardrailed_texts, edited) ) @@ -562,26 +584,25 @@ def _extract_converse_output_texts( ``citationsContent.content[].text`` -- while leaving structural values such as reasoning signatures and citation sources untouched. """ - holders: Final[list[_StringHolder]] = [] + leaves: Final[list[_StringLeaf]] = [] for block in content_blocks: if not isinstance(block, dict): continue - _collect_block_text(block, holders) + _collect_block_text(block, leaves) tool_use = block.get("toolUse") if isinstance(tool_use, dict): - _collect_strings(tool_use.get("input"), holders) + _collect_strings(tool_use.get("input"), leaves) reasoning = block.get("reasoningContent") if isinstance(reasoning, dict): reasoning_text = reasoning.get("reasoningText") if isinstance(reasoning_text, dict): - _collect_block_text(reasoning_text, holders) + _collect_block_text(reasoning_text, leaves) citations = block.get("citationsContent") if isinstance(citations, dict): for cited in citations.get("content") or []: if isinstance(cited, dict): - _collect_block_text(cited, holders) - texts: Final = [container[key] for container, key in holders] - return texts, holders + _collect_block_text(cited, leaves) + return [text for _, text in leaves], [holder for holder, _ in leaves] def _write_back_texts( @@ -598,7 +619,7 @@ def _write_back_texts( for idx, (container, key) in enumerate(holders): if idx >= len(guardrailed_texts): break - container[key] = guardrailed_texts[idx] + _assignable(container)[key] = guardrailed_texts[idx] _GroupKey: TypeAlias = str | tuple[str, int] diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 25498fef46f..c59baf2e07c 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -18,6 +18,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import decoded_json_object, subscript from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection from litellm.llms.bedrock.common_utils import BedrockError @@ -918,7 +919,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): def transform_content_start_event( self, - event: dict, + event: object, current_response_id: str | None, current_output_item_id: str | None, current_conversation_id: str | None, @@ -942,7 +943,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): Returns: Tuple of (events, response_id, output_item_id, conversation_id, delta_type) """ - content_start: Final = event["contentStart"] + content_start: Final = decoded_json_object(subscript(event, "contentStart")) role: Final = content_start.get("role") if role != "ASSISTANT": @@ -1228,7 +1229,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): def transform_content_end_event( self, - event: dict, + event: object, current_output_item_id: str | None, current_response_id: str | None, current_delta_type: str | None, @@ -1247,7 +1248,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): Returns: Tuple of (events, reset_delta_chunks) """ - content_end: Final = event["contentEnd"] + content_end: Final = subscript(event, "contentEnd") verbose_logger.debug("Handling contentEnd: %s", content_end) if self._assistant_final_text_active: @@ -1338,7 +1339,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): def transform_prompt_end_event( self, - event: dict, + event: object, current_response_id: str | None, current_conversation_id: str | None, ) -> tuple[ @@ -1400,7 +1401,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): def transform_tool_use_event( self, - event: dict, + event: object, current_response_id: str | None, current_conversation_id: str | None, ) -> tuple[list[OpenAIRealtimeEvents], str, str]: @@ -1416,7 +1417,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): Tuple of (events, response_id, conversation_id) """ verbose_logger.debug("Handling toolUse") - tool_use: Final = BedrockToolUse.model_validate(event["toolUse"]) + tool_use: Final = BedrockToolUse.model_validate(subscript(event, "toolUse")) verbose_logger.debug("Tool use event: %s (ID: %s)", tool_use.toolName, tool_use.toolUseId) response_id: Final = current_response_id or f"resp_{uuid.uuid4()}" @@ -1668,7 +1669,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): current_conversation_id, current_delta_type, ) = self.transform_content_start_event( - event, + _JSON_VALUE.validate_python(event), current_response_id, current_output_item_id, current_conversation_id, @@ -1710,7 +1711,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): elif "contentEnd" in event: events, current_delta_chunks = self.transform_content_end_event( - event, + _JSON_VALUE.validate_python(event), current_output_item_id, current_response_id, current_delta_type, @@ -1735,7 +1736,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): elif "toolUse" in event: events, current_response_id, current_conversation_id = self.transform_tool_use_event( - event, current_response_id, current_conversation_id + _JSON_VALUE.validate_python(event), current_response_id, current_conversation_id ) returned_messages.extend(events) @@ -1745,7 +1746,9 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): current_output_item_id, current_response_id, current_delta_type, - ) = self.transform_prompt_end_event(event, current_response_id, current_conversation_id) + ) = self.transform_prompt_end_event( + _JSON_VALUE.validate_python(event), current_response_id, current_conversation_id + ) returned_messages.extend(events) return { diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index b2b3a9b0faf..afcbe61cd75 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -4,9 +4,11 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, cast, runtime_checkable from urllib.parse import urlparse import httpx +from pydantic import TypeAdapter +from typing_extensions import TypedDict from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import decoded_json_object +from litellm.litellm_core_utils.core_helpers import decoded_json_object, keyword_arguments from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM @@ -61,6 +63,13 @@ def _vector_search_configuration(retrieval_config: object) -> Mapping[str, objec return decoded_json_object(decoded_json_object(retrieval_config).get("vectorSearchConfiguration", {})) +_DECODED_JSON: Final = TypeAdapter(object) + + +class _KBResponseFields(TypedDict, total=False, extra_items=object): + pass + + class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): """Vector store configuration for AWS Bedrock Knowledge Bases.""" @@ -361,7 +370,11 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj ) -> VectorStoreSearchResponse: try: - response_data: Final = BedrockKBResponse(**response.json()) + response_data: Final = BedrockKBResponse( + **_KBResponseFields( + **keyword_arguments(_DECODED_JSON.validate_python(response.json()), BedrockKBResponse) + ) + ) results: Final[list[VectorStoreSearchResult]] = [] for item in response_data.get("retrievalResults", []) or []: content: BedrockKBContent | None = item.get("content") diff --git a/litellm/llms/codestral/completion/transformation.py b/litellm/llms/codestral/completion/transformation.py index 636760b093d..c22bdf9d887 100644 --- a/litellm/llms/codestral/completion/transformation.py +++ b/litellm/llms/codestral/completion/transformation.py @@ -1,10 +1,20 @@ import json from typing import Final +from pydantic import TypeAdapter +from typing_extensions import TypedDict + import litellm +from litellm.litellm_core_utils.core_helpers import keyword_arguments from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig from litellm.types.llms.databricks import GenericStreamingChunk +_DECODED_JSON: Final = TypeAdapter(object) + + +class _StreamChunkFields(TypedDict, total=False, extra_items=object): + pass + class CodestralTextCompletionConfig(OpenAITextCompletionConfig): """ @@ -100,7 +110,11 @@ class CodestralTextCompletionConfig(OpenAITextCompletionConfig): "finish_reason": finish_reason, } - original_chunk: Final = litellm.ModelResponseStream(**chunk_data_dict) + original_chunk: Final = litellm.ModelResponseStream( + **_StreamChunkFields( + **keyword_arguments(_DECODED_JSON.validate_python(chunk_data_dict), litellm.ModelResponseStream) + ) + ) _choices: Final = chunk_data_dict.get("choices", []) or [] if len(_choices) == 0: return { diff --git a/litellm/llms/cohere/common_utils.py b/litellm/llms/cohere/common_utils.py index 4ba5f9bd57b..f5d084ea800 100644 --- a/litellm/llms/cohere/common_utils.py +++ b/litellm/llms/cohere/common_utils.py @@ -397,8 +397,12 @@ class CohereV2ModelResponseIterator: if index != -1: str_line = str_line[index:] - data_json: Final = json.loads(str_line) - return self.chunk_parser(chunk=data_json) + data_json: Final = _DECODED_JSON.validate_python(json.loads(str_line)) + if isinstance(data_json, dict): + return self.chunk_parser(chunk=data_json) + raise ValueError( + f"Failed to parse v2 chunk: {type(data_json).__name__!r} object has no attribute 'get', chunk: {data_json}" + ) # Async iterator def __aiter__(self): diff --git a/litellm/llms/cohere/embed/v1_transformation.py b/litellm/llms/cohere/embed/v1_transformation.py index 056b5bce742..ca096e369e1 100644 --- a/litellm/llms/cohere/embed/v1_transformation.py +++ b/litellm/llms/cohere/embed/v1_transformation.py @@ -2,12 +2,14 @@ Legacy /v1/embedding transformation logic for Bedrock Cohere. """ -from collections.abc import Sized +from collections.abc import Mapping, Sized from typing import Final, Protocol import httpx +from pydantic import TypeAdapter from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE +from litellm.litellm_core_utils.core_helpers import as_str_mapping, subscript from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.custom_httpx.http_handler import header_value from litellm.types.llms.bedrock import ( @@ -24,6 +26,17 @@ class _SupportsEncode(Protocol): def encode(self, text: str, /) -> Sized: ... +_DECODED_JSON: Final = TypeAdapter(object) + + +def _embed_response_body(response_json: object) -> Mapping[str, object]: + body: Final = as_str_mapping(response_json) + if body is None: + subscript(response_json, "embeddings") + raise TypeError(f"'{type(response_json).__name__}' object is not subscriptable") + return body + + def _bedrock_billed_input_tokens(response: httpx.Response) -> int | None: count: Final = header_value(response.headers, "x-amzn-bedrock-input-token-count") return int(count) if count is not None and count.isdigit() else None @@ -119,7 +132,7 @@ class CohereEmbeddingConfig: encoding: _SupportsEncode, input: list, ) -> EmbeddingResponse: - response_json: Final = response.json() + response_json: Final = _DECODED_JSON.validate_python(response.json()) ## LOGGING logging_obj.post_call( input=input, @@ -127,13 +140,14 @@ class CohereEmbeddingConfig: additional_args={"complete_input_dict": data}, original_response=response_json, ) + billed_input_tokens: Final = _bedrock_billed_input_tokens(response) if "images" in data else None return self._populate_embedding_response( - response_json=response_json, + response_json=dict(_embed_response_body(response_json)), model_response=model_response, model=model, encoding=encoding, input=input, - billed_input_tokens=_bedrock_billed_input_tokens(response) if "images" in data else None, + billed_input_tokens=billed_input_tokens, ) def transform_response( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index efb8b302664..f65c4205944 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -632,6 +632,7 @@ _OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object]) _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _JSON_OBJECTS: Final = TypeAdapter(list[dict[str, object]]) _DECODED_JSON: Final = TypeAdapter(object) +_AGENTIC_HOOK_RESULT: Final = TypeAdapter(object) def _decoded_body_headers(response: httpx.Response) -> httpx.Headers: @@ -906,7 +907,7 @@ class BaseLLMHTTPHandler: stream: bool | None = False, fake_stream: bool = False, api_key: str | None = None, - headers: dict[str, Any] | None = None, + headers: dict[str, object] | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, provider_config: BaseConfig | None = None, shared_session: Optional["ClientSession"] = None, @@ -1407,7 +1408,7 @@ class BaseLLMHTTPHandler: api_key: str | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, aembedding: bool | None = False, - headers: dict[str, Any] | None = None, + headers: dict[str, object] | None = None, ) -> EmbeddingResponse: provider_config: Final = ProviderConfigManager.get_provider_embedding_config( model=model, provider=litellm.LlmProviders(custom_llm_provider) @@ -1588,7 +1589,7 @@ class BaseLLMHTTPHandler: api_key: str | None = None, api_base: str | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, - litellm_params: dict[str, Any] | None = None, + litellm_params: dict[str, object] | None = None, ) -> RerankResponse: # get config from model, custom llm provider headers = provider_config.validate_environment( @@ -6146,7 +6147,7 @@ class BaseLLMHTTPHandler: fingerprint: str, stream: bool = False, callback: Optional["CustomLogger"] = None, - ) -> AnthropicMessagesResponse | AsyncIterator[object]: + ) -> object: from litellm.anthropic_interface import messages as anthropic_messages patch: Final = plan.request_patch or AgenticLoopRequestPatch() @@ -6187,7 +6188,7 @@ class BaseLLMHTTPHandler: kwargs_for_followup["max_agentic_loops"] = max_loops kwargs_for_followup["_agentic_loop_fingerprints"] = [*fingerprints, fingerprint] - response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate( + response: Final[AnthropicMessagesResponse | AsyncIterator[object]] = await anthropic_messages.acreate( **{ "max_tokens": max_tokens, "messages": patch.messages, @@ -6200,9 +6201,7 @@ class BaseLLMHTTPHandler: if callback is not None: try: - response = await callback.async_post_agentic_loop_response_hook( - response=response, plan=plan, kwargs=kwargs - ) + return await callback.async_post_agentic_loop_response_hook(response=response, plan=plan, kwargs=kwargs) except Exception as e: _call_id: Final = getattr(logging_obj, "litellm_call_id", "unknown") verbose_logger.exception( @@ -6227,7 +6226,7 @@ class BaseLLMHTTPHandler: fingerprints: Sequence[str], fingerprint: str, callback: Optional["CustomLogger"] = None, - ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: + ) -> object: patch: Final = plan.request_patch or AgenticLoopRequestPatch() if patch.messages is None: raise ValueError("Agentic loop plan missing patched responses input") @@ -6263,7 +6262,7 @@ class BaseLLMHTTPHandler: try: with internal_sub_call(logging_obj.litellm_call_id): - response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator = await litellm.aresponses( + response: Final[ResponsesAPIResponse | BaseResponsesAPIStreamingIterator] = await litellm.aresponses( model=patch.model or model, input=patch.messages, # pyright: ignore[reportArgumentType] # pre-existing mismatch surfaced by the Router import; patch messages are valid response input at runtime **optional_params, @@ -6272,7 +6271,7 @@ class BaseLLMHTTPHandler: if callback is not None: try: - response = await callback.async_post_agentic_loop_response_hook( + return await callback.async_post_agentic_loop_response_hook( response=response, plan=plan, kwargs=kwargs ) except Exception as e: @@ -6501,21 +6500,23 @@ class BaseLLMHTTPHandler: callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan ) if not build_plan_overridden: - agentic_result: object = await bill_first_round_on_failure( - callback.async_run_agentic_loop( - tools=tool_calls, - model=model, - messages=messages, - response=response, - anthropic_messages_provider_config=anthropic_messages_provider_config, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + agentic_result: object = _AGENTIC_HOOK_RESULT.validate_python( + await bill_first_round_on_failure( + callback.async_run_agentic_loop( + tools=tool_calls, + model=model, + messages=messages, + response=response, + anthropic_messages_provider_config=anthropic_messages_provider_config, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + logging_obj=logging_obj, + stream=stream, + kwargs=kwargs_with_provider, + ), logging_obj=logging_obj, + response=response, stream=stream, - kwargs=kwargs_with_provider, - ), - logging_obj=logging_obj, - response=response, - stream=stream, + ) ) return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface) @@ -6536,8 +6537,9 @@ class BaseLLMHTTPHandler: stream=stream, ) - if plan.response_override is not None: - return self._maybe_wrap_in_fake_stream(plan.response_override, logging_obj, api_surface) + response_override: object = plan.response_override + if response_override is not None: + return self._maybe_wrap_in_fake_stream(response_override, logging_obj, api_surface) if plan.terminate: verbose_logger.debug( "Agentic loop terminated by callback=%s reason=%s", @@ -6545,10 +6547,12 @@ class BaseLLMHTTPHandler: plan.stop_reason, ) return self._maybe_wrap_in_fake_stream( - await callback.async_post_agentic_loop_response_hook( - response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls), - plan=plan, - kwargs=kwargs_with_provider, + _AGENTIC_HOOK_RESULT.validate_python( + await callback.async_post_agentic_loop_response_hook( + response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls), + plan=plan, + kwargs=kwargs_with_provider, + ) ), logging_obj, api_surface, @@ -12396,10 +12400,10 @@ class BaseLLMHTTPHandler: def generate_content_handler( self, model: str, - contents: Any, + contents: object, generate_content_provider_config: BaseGoogleGenAIGenerateContentConfig, generate_content_config_dict: dict, - tools: Any, + tools: object, custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, @@ -12517,10 +12521,10 @@ class BaseLLMHTTPHandler: async def async_generate_content_handler( self, model: str, - contents: Any, + contents: object, generate_content_provider_config: BaseGoogleGenAIGenerateContentConfig, generate_content_config_dict: dict, - tools: Any, + tools: object, custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, diff --git a/litellm/llms/deprecated_providers/aleph_alpha.py b/litellm/llms/deprecated_providers/aleph_alpha.py index cb56717993e..fae2c49a978 100644 --- a/litellm/llms/deprecated_providers/aleph_alpha.py +++ b/litellm/llms/deprecated_providers/aleph_alpha.py @@ -1,14 +1,30 @@ import json import time import types -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sized from typing import Final import httpx +from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.core_helpers import iter_items, json_contains, subscript from litellm.utils import Choices, Message, ModelResponse, Usage +_DECODED_JSON: Final = TypeAdapter(object) +_STR: Final = TypeAdapter(str) + + +def _has_length(value: object) -> bool: + if isinstance(value, Sized): + return len(value) > 0 + raise TypeError(f"object of type '{type(value).__name__}' has no len()") + + +def _completion_message(item: object) -> Message: + completion: Final = subscript(item, "completion") + return Message(content=_STR.validate_python(completion) if _has_length(completion) else None) + class AlephAlphaError(Exception): def __init__(self, status_code, message): @@ -249,22 +265,19 @@ def completion( ) print_verbose(f"raw model_response: {response.text}") ## RESPONSE OBJECT - completion_response: Final = response.json() - if "error" in completion_response: + completion_response: Final = _DECODED_JSON.validate_python(response.json()) + if json_contains(completion_response, "error"): raise AlephAlphaError( - message=completion_response["error"], + message=subscript(completion_response, "error"), status_code=response.status_code, ) else: try: choices_list: Final = [] - for idx, item in enumerate(completion_response["completions"]): - if len(item["completion"]) > 0: - message_obj = Message(content=item["completion"]) - else: - message_obj = Message(content=None) + for idx, item in enumerate(iter_items(subscript(completion_response, "completions"))): + message_obj = _completion_message(item) choice_obj = Choices( - finish_reason=item["finish_reason"], + finish_reason=subscript(item, "finish_reason"), index=idx + 1, message=message_obj, ) diff --git a/litellm/llms/edenai/videos/transformation.py b/litellm/llms/edenai/videos/transformation.py index 0cb9c5afaec..97235a0ffb8 100644 --- a/litellm/llms/edenai/videos/transformation.py +++ b/litellm/llms/edenai/videos/transformation.py @@ -142,7 +142,7 @@ class EdenAIVideoConfig(OpenAIVideoConfig): raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str | None = None, - ) -> dict[str, str]: # mutable-ok: inherited contract + ) -> object: raw_response.raise_for_status() # the shared GET helpers return error bodies instead of raising return super().transform_video_list_response( raw_response=raw_response, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 55a75423042..ec8de5af5e2 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Final, Literal, cast import httpx from pydantic import ConfigDict, TypeAdapter +from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger @@ -65,6 +66,10 @@ _STR: Final = TypeAdapter(str) _DECODED_JSON: Final = TypeAdapter(object) +class _FunctionFields(TypedDict, total=False, extra_items=object): + pass + + def _map_reasoning_effort(value: object) -> object: effort: Final[object] = cast(Mapping[str, object], value).get("effort") if isinstance(value, Mapping) else value if effort is True: @@ -718,7 +723,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): """ if tool_calls is not None and message.content is not None and message.tool_calls is None: try: - function: Final = Function(**json.loads(message.content)) + function: Final = Function(**_FunctionFields(**_JSON_DICT.validate_python(json.loads(message.content)))) if function.name != RESPONSE_FORMAT_TOOL_NAME and function.name in [ tool["function"]["name"] for tool in tool_calls ]: diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 7bc783b960f..59058b03d67 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -17,6 +17,7 @@ import httpx from pydantic import TypeAdapter from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -452,36 +453,37 @@ class GigaChatConfig(BaseConfig): choices: Final[list[Choices]] = [] # mutable-ok: accumulator for building response choices for choice in response_json.get("choices", []): message_data = choice.get("message", {}) - finish_reason = choice.get("finish_reason", "stop") + finish_reason = _JSON_VALUE.validate_python(choice.get("finish_reason", "stop")) - if finish_reason == "function_call" and message_data.get("function_call"): - func_call = message_data["function_call"] + function_call = ( + decoded_json_object(_JSON_VALUE.validate_python(message_data)).get("function_call") + if finish_reason == "function_call" + else None + ) + if function_call: + func_call = decoded_json_object(function_call) args = func_call.get("arguments", {}) + args_mapping = as_str_mapping(args) if is_structured_output: - if isinstance(args, dict): - content = json.dumps(args, ensure_ascii=False) - else: - content = str(args) - message_data["content"] = content - message_data.pop("function_call", None) - message_data.pop("functions_state_id", None) + message_data["content"] = ( + json.dumps(args_mapping, ensure_ascii=False) if args_mapping is not None else str(args) + ) finish_reason = "stop" else: - if isinstance(args, dict): - args = json.dumps(args, ensure_ascii=False) + arguments = json.dumps(args_mapping, ensure_ascii=False) if args_mapping is not None else args message_data["tool_calls"] = [ { "id": f"call_{uuid.uuid4().hex[:24]}", "type": "function", "function": { "name": func_call.get("name", ""), - "arguments": args, + "arguments": arguments, }, } ] - message_data.pop("function_call", None) finish_reason = "tool_calls" + message_data.pop("function_call", None) message_data.pop("functions_state_id", None) diff --git a/litellm/llms/gigachat/embedding/transformation.py b/litellm/llms/gigachat/embedding/transformation.py index 30ca7147865..5a988edc144 100644 --- a/litellm/llms/gigachat/embedding/transformation.py +++ b/litellm/llms/gigachat/embedding/transformation.py @@ -9,12 +9,13 @@ from __future__ import annotations import types from collections.abc import Mapping -from typing import Final +from typing import Final, Protocol, runtime_checkable import httpx from pydantic import ConfigDict, TypeAdapter from litellm import LlmProviders +from litellm.litellm_core_utils.core_helpers import iter_items, json_contains, subscript from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -26,6 +27,28 @@ from litellm.types.utils import EmbeddingResponse from ..authenticator import get_access_token _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) + + +@runtime_checkable +class _SupportsDelItem(Protocol): + def __delitem__(self, key: str, /) -> None: ... + + +@runtime_checkable +class _SupportsSetItem(Protocol): + def __setitem__(self, key: str, value: object, /) -> None: ... + + +def _add_prompt_tokens(total_tokens: float, embedding: object) -> float: + if not (json_contains(embedding, "usage") and json_contains(subscript(embedding, "usage"), "prompt_tokens")): + return total_tokens + prompt_tokens: Final = subscript(subscript(embedding, "usage"), "prompt_tokens") + if isinstance(prompt_tokens, int | float): + return total_tokens + prompt_tokens + raise TypeError( + f"unsupported operand type(s) for +=: '{type(total_tokens).__name__}' and '{type(prompt_tokens).__name__}'" + ) class GigaChatEmbeddingError(BaseLLMException): @@ -144,7 +167,7 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig): "model": "Embeddings" } """ - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) # Log response logging_obj.post_call( @@ -155,16 +178,19 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig): ) # Calculate total tokens from individual embeddings - total_tokens = 0 - if "data" in response_json: - for emb in response_json["data"]: - if "usage" in emb and "prompt_tokens" in emb["usage"]: - total_tokens += emb["usage"]["prompt_tokens"] + total_tokens: float = 0 + if json_contains(response_json, "data"): + for emb in iter_items(subscript(response_json, "data")): + total_tokens = _add_prompt_tokens(total_tokens, emb) # Remove usage from individual embeddings (not part of OpenAI format) - if "usage" in emb: + if json_contains(emb, "usage"): + if not isinstance(emb, _SupportsDelItem): + raise TypeError(f"'{type(emb).__name__}' object does not support item deletion") del emb["usage"] # Set overall usage + if not isinstance(response_json, _SupportsSetItem): + raise TypeError(f"'{type(response_json).__name__}' object does not support item assignment") response_json["usage"] = { "prompt_tokens": total_tokens, "total_tokens": total_tokens, diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index bb836eb1b35..e3210482b2a 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -1,13 +1,15 @@ import json import os import time -from collections.abc import Mapping, Sequence +from collections.abc import Mapping, Sequence, Sized from copy import deepcopy from typing import TYPE_CHECKING, Any, Final, Protocol import httpx +from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.core_helpers import iter_items, subscript from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) @@ -44,6 +46,27 @@ def _header_value(headers: Mapping[str, str], key: str, default: str) -> str: return headers.get(key, default) +_DECODED_JSON: Final = TypeAdapter(object) + + +def _json_len(value: object) -> int: + if isinstance(value, Sized): + return len(value) + raise TypeError(f"object of type '{type(value).__name__}' has no len()") + + +def _generated_text(value: object) -> str: + if isinstance(value, str): + return value + raise AttributeError(f"'{type(value).__name__}' object has no attribute 'strip'", name="strip", obj=value) + + +def _add_logprob(total: float, logprob: object) -> float: + if isinstance(logprob, (int, float)): + return total + logprob + raise TypeError(f"unsupported operand type(s) for +=: '{type(total).__name__}' and '{type(logprob).__name__}'") + + tgi_models_cache = None conv_models_cache = None @@ -407,33 +430,41 @@ class HuggingFaceEmbeddingConfig(BaseConfig): headers=None, ) - if len(completion_response[0]["generated_text"]) > 0: - model_response.choices[0].message.content = output_parser(completion_response[0]["generated_text"]) + if _json_len(_DECODED_JSON.validate_python(completion_response[0]["generated_text"])) > 0: + model_response.choices[0].message.content = output_parser( + _generated_text(_DECODED_JSON.validate_python(completion_response[0]["generated_text"])) + ) ## GETTING LOGPROBS + FINISH REASON if "details" in completion_response[0] and "tokens" in completion_response[0]["details"]: model_response.choices[0].finish_reason = completion_response[0]["details"]["finish_reason"] sum_logprob = 0 - for token in completion_response[0]["details"]["tokens"]: - if token["logprob"] is not None: - sum_logprob += token["logprob"] + for token in iter_items(_DECODED_JSON.validate_python(completion_response[0]["details"]["tokens"])): + logprob = subscript(token, "logprob") + if logprob is not None: + sum_logprob = _add_logprob(sum_logprob, logprob) setattr(model_response.choices[0].message, "_logprob", sum_logprob) if "best_of" in optional_params and optional_params["best_of"] > 1: if "details" in completion_response[0] and "best_of_sequences" in completion_response[0]["details"]: choices_list: Final = [] - for idx, item in enumerate(completion_response[0]["details"]["best_of_sequences"]): + for idx, item in enumerate( + iter_items( + _DECODED_JSON.validate_python(completion_response[0]["details"]["best_of_sequences"]) + ) + ): sum_logprob = 0 - for token in item["tokens"]: - if token["logprob"] is not None: - sum_logprob += token["logprob"] - if len(item["generated_text"]) > 0: + for token in iter_items(subscript(item, "tokens")): + logprob = subscript(token, "logprob") + if logprob is not None: + sum_logprob = _add_logprob(sum_logprob, logprob) + if _json_len(subscript(item, "generated_text")) > 0: message_obj = Message( - content=output_parser(item["generated_text"]), + content=output_parser(_generated_text(subscript(item, "generated_text"))), logprobs=sum_logprob, ) else: message_obj = Message(content=None) choice_obj = Choices( - finish_reason=item["finish_reason"], + finish_reason=subscript(item, "finish_reason"), index=idx + 1, message=message_obj, ) @@ -442,8 +473,13 @@ class HuggingFaceEmbeddingConfig(BaseConfig): elif task == "text-classification": model_response.choices[0].message.content = json.dumps(completion_response) else: - if isinstance(completion_response, list) and len(completion_response[0]["generated_text"]) > 0: - model_response.choices[0].message.content = output_parser(completion_response[0]["generated_text"]) + if ( + isinstance(completion_response, list) + and _json_len(_DECODED_JSON.validate_python(completion_response[0]["generated_text"])) > 0 + ): + model_response.choices[0].message.content = output_parser( + _generated_text(_DECODED_JSON.validate_python(completion_response[0]["generated_text"])) + ) ## CALCULATING USAGE prompt_tokens = 0 try: diff --git a/litellm/llms/minimax/videos/transformation.py b/litellm/llms/minimax/videos/transformation.py index aff99542d9b..fd014a3d521 100644 --- a/litellm/llms/minimax/videos/transformation.py +++ b/litellm/llms/minimax/videos/transformation.py @@ -803,7 +803,7 @@ class MinimaxVideoConfig(BaseVideoConfig): params.update(extra_query) return f"{api_base}/v2/query/video_generation", params - def transform_video_list_response( # pyright: ignore[reportIncompatibleMethodOverride] # base declares dict[str, str] but the payload is a heterogeneous list body + def transform_video_list_response( self, raw_response: httpx.Response, logging_obj: "Logging", diff --git a/litellm/llms/nlp_cloud/chat/transformation.py b/litellm/llms/nlp_cloud/chat/transformation.py index 0cca299de93..8301bee9b90 100644 --- a/litellm/llms/nlp_cloud/chat/transformation.py +++ b/litellm/llms/nlp_cloud/chat/transformation.py @@ -1,9 +1,10 @@ import json import time -from collections.abc import Mapping +from collections.abc import Mapping, Sized from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, @@ -23,6 +24,15 @@ else: LoggingClass = Any +_DECODED_JSON: Final = TypeAdapter(object) + + +def _json_len(value: object) -> int: + if isinstance(value, Sized): + return len(value) + raise TypeError(f"object of type '{type(value).__name__}' has no len()") + + class NLPCloudConfig(BaseConfig): """ Reference: https://docs.nlpcloud.com/#generation @@ -199,7 +209,7 @@ class NLPCloudConfig(BaseConfig): ) else: try: - if len(completion_response["generated_text"]) > 0: + if _json_len(_DECODED_JSON.validate_python(completion_response["generated_text"])) > 0: model_response.choices[0].message.content = completion_response["generated_text"] except Exception: raise NLPCloudError( diff --git a/litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py b/litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py index 8367853673f..8f8111be3ce 100644 --- a/litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py +++ b/litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py @@ -6,11 +6,12 @@ request carries them exactly once over both plaintext and TLS. """ from collections.abc import Iterator, Mapping, Sequence -from typing import Any, Final, TypeAlias +from typing import Final, Protocol, SupportsIndex, SupportsInt, TypeAlias, runtime_checkable import grpc from google.protobuf.message import Message from pydantic import TypeAdapter +from typing_extensions import Buffer from litellm.llms.nvidia_riva.audio_transcription import riva_asr_schema as schema from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException @@ -38,16 +39,29 @@ def open_channel(api_base: str, use_ssl: bool) -> grpc.Channel: return grpc.insecure_channel(api_base) -def recognition_config(config: Mapping[str, Any]) -> Message: +@runtime_checkable +class _SupportsTrunc(Protocol): + def __trunc__(self) -> int: ... + + +def _native_int(value: object) -> int: + if isinstance(value, (str, Buffer, SupportsInt, SupportsIndex, _SupportsTrunc)): + return int(value) + raise TypeError( + f"int() argument must be a string, a bytes-like object or a real number, not '{type(value).__name__}'" + ) + + +def recognition_config(config: Mapping[str, object]) -> Message: endpointing: Final = _endpointing_config(config.get("endpointing_config")) return schema.RecognitionConfig( encoding=schema.LINEAR_PCM, - sample_rate_hertz=int(config["sample_rate_hertz"]), + sample_rate_hertz=_native_int(config["sample_rate_hertz"]), language_code=config["language_code"], - audio_channel_count=int(config["audio_channel_count"]), + audio_channel_count=_native_int(config["audio_channel_count"]), enable_automatic_punctuation=bool(config.get("enable_automatic_punctuation", True)), enable_word_time_offsets=bool(config.get("enable_word_time_offsets", False)), - max_alternatives=int(config.get("max_alternatives", 1)), + max_alternatives=_native_int(config.get("max_alternatives", 1)), model=config.get("model", "") or "", verbatim_transcripts=bool(config.get("verbatim_transcripts", False)), profanity_filter=bool(config.get("profanity_filter", False)), diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 1e3e6186dd1..22e09584b1d 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, TypeAdapter import litellm from litellm._uuid import uuid -from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.core_helpers import decoded_json_object, map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_images_from_message, @@ -47,6 +47,15 @@ else: LiteLLMLoggingObj = Any _REQUEST_PARAMS: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) + + +def _json_document(value: object) -> str | bytes | bytearray: + if isinstance(value, (str, bytes, bytearray)): + return value + raise TypeError(f"the JSON object must be str, bytes or bytearray, not {value.__class__.__name__}") + + _TOOL_CALL_ARGUMENTS: Final = TypeAdapter(dict[str, object]) @@ -372,7 +381,7 @@ class OllamaChatConfig(BaseConfig): # remap 'thinking' to 'reasoning_content' response_json_message["reasoning_content"] = response_json_message["thinking"] del response_json_message["thinking"] - elif response_json_message.get("content") is not None: + elif decoded_json_object(_DECODED_JSON.validate_python(response_json_message)).get("content") is not None: # parse reasoning content from content from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( parse_content_for_reasoning, @@ -387,14 +396,20 @@ class OllamaChatConfig(BaseConfig): and litellm_params.get("function_name") is not None and response_json_message is not None ): - function_call: Final = json.loads(response_json_message["content"]) + function_call: Final = decoded_json_object( + _DECODED_JSON.validate_python( + json.loads(_json_document(_DECODED_JSON.validate_python(response_json_message["content"]))) + ) + ) message: Final = litellm.Message( content=None, tool_calls=[ { "id": f"call_{uuid.uuid4()}", "function": { - "name": function_call.get("name", litellm_params.get("function_name")), + "name": function_call.get( + "name", _DECODED_JSON.validate_python(litellm_params.get("function_name")) + ), "arguments": json.dumps(function_call.get("arguments", function_call)), }, "type": "function", diff --git a/litellm/llms/oobabooga/chat/oobabooga.py b/litellm/llms/oobabooga/chat/oobabooga.py index b4dc57ff5ed..a473ef18630 100644 --- a/litellm/llms/oobabooga/chat/oobabooga.py +++ b/litellm/llms/oobabooga/chat/oobabooga.py @@ -1,7 +1,9 @@ import json -from collections.abc import Callable +from collections.abc import Callable, Sized from typing import TYPE_CHECKING, Final +from pydantic import TypeAdapter + import litellm from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import get_httpx_client @@ -15,6 +17,13 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj oobabooga_config: Final = OobaboogaConfig() +_DECODED_JSON: Final = TypeAdapter(object) + + +def _json_len(value: object) -> int: + if isinstance(value, Sized): + return len(value) + raise TypeError(f"object of type '{type(value).__name__}' has no len()") def completion( @@ -149,7 +158,7 @@ def embedding( } ] - num_tokens: Final = len(completion_response["data"][0]["embedding"]) + num_tokens: Final = _json_len(_DECODED_JSON.validate_python(completion_response["data"][0]["embedding"])) # Adding metadata to response setattr( model_response, diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index f7bd46536a1..c4c72ab6be6 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -8,15 +8,21 @@ import os from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from contextvars import ContextVar from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast, overload +from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, cast, overload from urllib.parse import urlparse import httpx from pydantic import TypeAdapter, ValidationError +from typing_extensions import TypedDict import litellm from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason +from litellm.litellm_core_utils.core_helpers import ( + drops_unsupported_params, + map_finish_reason, + require_attributes, + subscript, +) from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( extract_reasoning_content, handle_invalid_parallel_tool_calls, @@ -147,6 +153,14 @@ _CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _DECODED_JSON: Final = TypeAdapter(object) +class _ModelResponseStreamFields(TypedDict, total=False, extra_items=object): + pass + + +class _SupportsGetWithDefault(Protocol): + def get(self, key: str, default: str, /) -> object: ... + + def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -> str | None: value: Final = litellm_params.get(key) if litellm_params is not None else None return value if isinstance(value, str) else None @@ -808,7 +822,10 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): convert_tool_call_to_json_mode=json_mode, ): # to support response_format on claude models - json_mode_content_str: str | None = str(new_tool_calls[0]["function"].get("arguments", "")) or None + tool_call_function = require_attributes( + subscript(new_tool_calls[0], "function"), _SupportsGetWithDefault, "get" + ) + json_mode_content_str: str | None = str(tool_call_function.get("arguments", "")) or None if json_mode_content_str is not None: translated_message = Message(content=json_mode_content_str) finish_reason = "stop" @@ -1086,7 +1103,7 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator): choices: Final = self._map_reasoning_to_reasoning_content(raw_choices) fingerprint: Final = _OBJECT_MAPPING.validate_python(chunk).get("system_fingerprint") - kwargs: Final[dict[str, Any]] = { + kwargs: Final[dict[str, object]] = { "id": chunk.get("id"), "system_fingerprint": fingerprint, "object": "chat.completion.chunk", @@ -1099,6 +1116,6 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator): service_tier: Final = chunk.get("service_tier") if isinstance(service_tier, str) and service_tier: kwargs["service_tier"] = service_tier - return ModelResponseStream(**kwargs) + return ModelResponseStream(**_ModelResponseStreamFields(**kwargs)) except Exception as e: raise e diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index bc8aaab3185..a6dbc79254e 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -4,7 +4,9 @@ Helper util for handling openai-specific cost calculation """ from collections.abc import Mapping -from typing import Any, Final, Literal +from typing import Final, Literal, SupportsFloat, SupportsIndex + +from typing_extensions import Buffer from litellm._logging import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token @@ -100,8 +102,14 @@ def _video_resolution_to_cost_field_suffix(resolution: str) -> str | None: return safe +def _native_float(value: object) -> float: + if isinstance(value, (str, Buffer, SupportsFloat, SupportsIndex)): + return float(value) + raise TypeError(f"float() argument must be a string or a real number, not '{type(value).__name__}'") + + def video_output_cost_per_second( - model_info: Mapping[str, Any], + model_info: Mapping[str, object], video_resolution: str | None, ) -> float | None: """ @@ -118,10 +126,10 @@ def video_output_cost_per_second( tier_key: Final = f"output_cost_per_second_{suffix}" tier_rate: Final = model_info.get(tier_key) if tier_rate is not None: - return float(tier_rate) + return _native_float(tier_rate) out: Final = model_info.get("output_cost_per_second") if out is not None: - return float(out) + return _native_float(out) return None diff --git a/litellm/llms/openai/image_edit/transformation.py b/litellm/llms/openai/image_edit/transformation.py index 0bba58096ef..92fdffebe34 100644 --- a/litellm/llms/openai/image_edit/transformation.py +++ b/litellm/llms/openai/image_edit/transformation.py @@ -3,9 +3,12 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles +from pydantic import TypeAdapter +from typing_extensions import TypedDict import litellm from litellm.images.utils import ImageEditRequestUtils +from litellm.litellm_core_utils.core_helpers import keyword_arguments from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.secret_managers.main import get_secret_str @@ -28,6 +31,13 @@ else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + + +class _ImageResponseFields(TypedDict, total=False, extra_items=object): + pass + + class OpenAIImageEditConfig(BaseImageEditConfig): """ Base configuration for OpenAI image edit API. @@ -149,10 +159,10 @@ class OpenAIImageEditConfig(BaseImageEditConfig): ) -> ImageResponse: """No transform applied since outputs are in OpenAI spec already""" try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code) - return ImageResponse(**raw_response_json) + return ImageResponse(**_ImageResponseFields(**keyword_arguments(raw_response_json, ImageResponse))) def validate_environment( self, diff --git a/litellm/llms/openai/image_variations/handler.py b/litellm/llms/openai/image_variations/handler.py index 504dfb50015..0200d9acbb0 100644 --- a/litellm/llms/openai/image_variations/handler.py +++ b/litellm/llms/openai/image_variations/handler.py @@ -7,6 +7,8 @@ from typing import Final import httpx from openai import AsyncOpenAI, OpenAI +from pydantic import BaseModel +from typing_extensions import TypedDict import litellm from litellm.types.utils import FileTypes, ImageResponse, LlmProviders @@ -18,6 +20,14 @@ from ..common_utils import OpenAIError, upstream_error_response from ..workload_identity import build_async_openai_client, build_openai_client +class _ImageResponseFields(TypedDict, total=False, extra_items=object): + pass + + +def _dumped_fields(model: BaseModel) -> Mapping[str, object]: + return model.model_dump() + + class OpenAIImageVariationsHandler: def get_sync_client( self, @@ -94,7 +104,7 @@ class OpenAIImageVariationsHandler: raw_response: Final = await client.images.with_raw_response.create_variation(**data) response: Final = raw_response.parse() - response_json: Final = response.model_dump() + response_json: Final = _dumped_fields(response) ## LOGGING logging_obj.post_call( @@ -109,7 +119,7 @@ class OpenAIImageVariationsHandler: ## RESPONSE OBJECT return provider_config.transform_response_image_variation( model=model, - model_response=ImageResponse(**response_json), + model_response=ImageResponse(**_ImageResponseFields(**response_json)), raw_response=httpx.Response( status_code=200, request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object @@ -215,7 +225,7 @@ class OpenAIImageVariationsHandler: raw_response: Final = client.images.with_raw_response.create_variation(**json_data) response: Final = raw_response.parse() - response_json: Final = response.model_dump() + response_json: Final = _dumped_fields(response) ## LOGGING logging_obj.post_call( @@ -230,7 +240,7 @@ class OpenAIImageVariationsHandler: ## RESPONSE OBJECT return provider_config.transform_response_image_variation( model=model, - model_response=ImageResponse(**response_json), + model_response=ImageResponse(**_ImageResponseFields(**response_json)), raw_response=httpx.Response( status_code=200, request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 8e9f2a0a0bd..77c0223b375 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -36,7 +36,7 @@ from collections.abc import Iterable, Mapping, MutableSequence, Sequence from dataclasses import dataclass from itertools import accumulate, chain, groupby, repeat from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast, runtime_checkable from openai.types.responses.response_output_refusal import ResponseOutputRefusal from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError @@ -52,7 +52,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i OpenAiResponsesToChatCompletionStreamIterator, tool_call_dict_from_output_item, ) -from litellm.litellm_core_utils.core_helpers import has_attributes +from litellm.litellm_core_utils.core_helpers import as_object_list, has_attributes from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamingScanKey, @@ -694,6 +694,24 @@ def _is_textless_tool_row(message: Mapping[str, object], texts: tuple[str, ...] return _is_string_tool_row(message) and (not message.get("content") or texts == ()) +@runtime_checkable +class _SupportsSetItem(Protocol): + def __setitem__(self, key: str, value: object, /) -> None: ... + + +def _assignable(item: object) -> _SupportsSetItem: + if isinstance(item, _SupportsSetItem): + return item + raise TypeError(f"'{type(item).__name__}' object does not support item assignment") + + +def _summarized_encrypted_content(item: Mapping[str, object]) -> str | None: + if compaction_summary(item) is None: + return None + encrypted_content: Final = item["encrypted_content"] + return encrypted_content if isinstance(encrypted_content, str) else None + + def _rewritten_whole_text(item: object, field: str, guardrail_response: str) -> object | None: if isinstance(stream_item_field(item, field), str): return guardrail_response @@ -1134,7 +1152,7 @@ class OpenAIResponsesHandler(BaseTranslation): async def _apply_guardrail_responses_to_input( self, - messages: Any, # Can be List[Dict[str, Any]] or ResponseInputParam + messages: Sequence[Mapping[str, object]], responses: Sequence[str], task_mappings: Sequence[tuple[int, int | None]], ) -> None: @@ -1149,26 +1167,27 @@ class OpenAIResponsesHandler(BaseTranslation): for guardrail_response, mapping in zip(responses, task_mappings): msg_idx = cast(int, mapping[0]) content_idx_optional = cast(int | None, mapping[1]) + item = messages[msg_idx] - if compaction_summary(messages[msg_idx]) is not None: - messages[msg_idx]["encrypted_content"] = with_compaction_summary( # rebind-ok: in-place input rewrite - messages[msg_idx]["encrypted_content"], guardrail_response - ) + encrypted_content = _summarized_encrypted_content(item) + if encrypted_content is not None: + _assignable(item)["encrypted_content"] = with_compaction_summary(encrypted_content, guardrail_response) continue - field = _scanned_text_field(messages[msg_idx]) - content = messages[msg_idx].get(field, None) + field = _scanned_text_field(item) + content = item.get(field, None) if content is None: continue + content_parts = as_object_list(content) if content_idx_optional is None: - rewritten = _rewritten_whole_text(messages[msg_idx], field, guardrail_response) + rewritten = _rewritten_whole_text(item, field, guardrail_response) if rewritten is not None: - messages[msg_idx][field] = rewritten + _assignable(item)[field] = rewritten - elif isinstance(content, list): + elif content_parts is not None: # Replace specific text item in list content - self._write_content_part_text(content[content_idx_optional], guardrail_response) + self._write_content_part_text(content_parts[content_idx_optional], guardrail_response) async def process_output_response( self, @@ -1299,7 +1318,7 @@ class OpenAIResponsesHandler(BaseTranslation): request_data: dict | None = None, stream_transform_sink: StreamTransformSink | None = None, deliver_ended_stream_rewrites: bool = False, - ) -> list[Any]: + ) -> Sequence[object]: """ Process output streaming response by applying guardrails to text content. diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index 947d6175122..21fdf202a52 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -3,7 +3,10 @@ from collections.abc import Mapping from typing import Final from httpx import Headers, Response +from pydantic import TypeAdapter +from typing_extensions import TypedDict +from litellm.litellm_core_utils.core_helpers import json_contains, keyword_arguments from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, @@ -19,6 +22,12 @@ from litellm.types.utils import FileTypes, TranscriptionResponse from ..common_utils import OpenAIError +_DECODED_JSON: Final = TypeAdapter(object) + + +class _TranscriptionResponseFields(TypedDict, total=False, extra_items=object): + pass + def _header_value(headers: Mapping[str, str], key: str, default: str) -> str: return headers.get(key, default) @@ -131,14 +140,16 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): raw_response: Response, ) -> TranscriptionResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except json.JSONDecodeError: content_type: Final = _header_value(raw_response.headers, "content-type", "").lower() if "application/json" in content_type: raise return TranscriptionResponse(text=raw_response.text) - if any(key in raw_response_json for key in TranscriptionResponse.model_fields): - return TranscriptionResponse(**raw_response_json) + if any(json_contains(raw_response_json, key) for key in TranscriptionResponse.model_fields): + return TranscriptionResponse( + **_TranscriptionResponseFields(**keyword_arguments(raw_response_json, TranscriptionResponse)) + ) else: raise ValueError("Invalid response format. Received response does not match the expected format.") diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index d108802f7f9..7fee55ade1c 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -1,8 +1,11 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx +from pydantic import TypeAdapter +from typing_extensions import TypedDict import litellm +from litellm.litellm_core_utils.core_helpers import keyword_arguments from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.openai.workload_identity import resolve_openai_bearer_token @@ -27,6 +30,12 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + + +class _VectorStoreResponseFields(TypedDict, total=False, extra_items=object): + pass + class OpenAIVectorStoreConfig(BaseVectorStoreConfig): ASSISTANTS_HEADER_KEY = "OpenAI-Beta" @@ -118,8 +127,10 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj ) -> VectorStoreSearchResponse: try: - response_json: Final = response.json() - return VectorStoreSearchResponse(**response_json) + response_json: Final = _DECODED_JSON.validate_python(response.json()) + return VectorStoreSearchResponse( + **_VectorStoreResponseFields(**keyword_arguments(response_json, VectorStoreSearchResponse)) + ) except Exception as e: raise self.get_error_class( error_message=str(e), @@ -149,8 +160,10 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): def transform_create_vector_store_response(self, response: httpx.Response) -> VectorStoreCreateResponse: try: - response_json: Final = response.json() - return VectorStoreCreateResponse(**response_json) + response_json: Final = _DECODED_JSON.validate_python(response.json()) + return VectorStoreCreateResponse( + **_VectorStoreResponseFields(**keyword_arguments(response_json, VectorStoreCreateResponse)) + ) except Exception as e: raise self.get_error_class( error_message=str(e), diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index abd157a9a08..4d3efc29fa7 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -2,12 +2,21 @@ import mimetypes from collections.abc import Mapping from io import BufferedReader, BytesIO from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, Protocol, cast from urllib.parse import quote import httpx from httpx._types import FileContent, FileTypes, RequestFiles +from pydantic import TypeAdapter +from litellm.litellm_core_utils.core_helpers import ( + as_object_list, + decoded_json_object, + iter_items, + json_contains, + require_attributes, + subscript, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.videos.async_task_utils import unsupported_video_fields from litellm.llms.base_llm.videos.transformation import BaseVideoConfig @@ -39,6 +48,59 @@ else: BaseLLMException = Any +_REQUEST_VALUE: Final = TypeAdapter(object) + + +def _dict_fields(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, dict) else None + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + +def _model_label(model: object) -> str | None: + return format(model) if model else None + + +def _encoded_video_id(video_id: object, provider: str, model: object) -> object: + if not video_id: + return video_id + return encode_video_id_with_provider( + require_attributes(video_id, str, "startswith"), + provider, + _model_label(model), + ) + + +def _with_encoded_id(video: object, provider: str) -> object: + fields: Final = _dict_fields(video) + if fields is None or "id" not in fields: + return video + return {**fields, "id": _encoded_video_id(fields["id"], provider, fields.get("model"))} + + +def _edge_model(data: object, index: int) -> object: + if not data: + return None + edge: Final = _dict_fields(subscript(data, index)) + return None if edge is None else edge.get("model") + + +def _decoded_character(character: object) -> object: + fields: Final = _dict_fields(character) + if fields is None: + return character + character_id: Final = fields.get("id") + if not isinstance(character_id, str): + return character + return {**fields, "id": extract_original_character_id(character_id)} + + class OpenAIVideoConfig(BaseVideoConfig): """ Configuration class for OpenAI video generation. @@ -157,25 +219,11 @@ class OpenAIVideoConfig(BaseVideoConfig): `character_`, convert it back to the original provider id before forwarding upstream. """ - raw_characters: Final = request_dict.get("characters") - if not isinstance(raw_characters, list): + raw_characters: Final = as_object_list(_REQUEST_VALUE.validate_python(request_dict.get("characters"))) + if raw_characters is None: return request_dict - decoded_characters: Final[list[Any]] = [] - for character in raw_characters: - if not isinstance(character, dict): - decoded_characters.append(character) - continue - - character_id = character.get("id") - if isinstance(character_id, str): - decoded_character = dict(character) - decoded_character["id"] = extract_original_character_id(character_id) - decoded_characters.append(decoded_character) - else: - decoded_characters.append(character) - - request_dict["characters"] = decoded_characters + request_dict["characters"] = [_decoded_character(character) for character in raw_characters] return request_dict def transform_video_create_response( @@ -343,41 +391,27 @@ class OpenAIVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, - ) -> dict[str, str]: - response_data: Final = raw_response.json() + ) -> object: + response_data: Final = _json_body(raw_response) + if not custom_llm_provider or not json_contains(response_data, "data"): + return response_data - if custom_llm_provider and "data" in response_data: - for video_obj in response_data.get("data", []): - if isinstance(video_obj, dict) and "id" in video_obj: - video_obj["id"] = encode_video_id_with_provider( - video_obj["id"], - custom_llm_provider, - video_obj.get("model"), - ) + fields: Final = decoded_json_object(response_data) + data: Final = fields.get("data", []) + encoded_items: Final = [_with_encoded_id(item, custom_llm_provider) for item in iter_items(data)] + encoded_data: Final = encoded_items if as_object_list(data) is not None else data - # Encode pagination cursor IDs so they remain consistent - # with the wrapped data[].id format - data_list: Final = response_data.get("data", []) - if response_data.get("first_id"): - first_model = None - if data_list and isinstance(data_list[0], dict): - first_model = data_list[0].get("model") - response_data["first_id"] = encode_video_id_with_provider( - response_data["first_id"], - custom_llm_provider, - first_model, - ) - if response_data.get("last_id"): - last_model = None - if data_list and isinstance(data_list[-1], dict): - last_model = data_list[-1].get("model") - response_data["last_id"] = encode_video_id_with_provider( - response_data["last_id"], - custom_llm_provider, - last_model, - ) - - return response_data + # Encode pagination cursor IDs so they remain consistent + # with the wrapped data[].id format + first_id: Final = fields.get("first_id") + encoded_first: Final[Mapping[str, object]] = ( + {"first_id": _encoded_video_id(first_id, custom_llm_provider, _edge_model(data, 0))} if first_id else {} + ) + last_id: Final = fields.get("last_id") + encoded_last: Final[Mapping[str, object]] = ( + {"last_id": _encoded_video_id(last_id, custom_llm_provider, _edge_model(data, -1))} if last_id else {} + ) + return {**fields, "data": encoded_data, **encoded_first, **encoded_last} def transform_video_delete_request( self, diff --git a/litellm/llms/openai_like/chat/handler.py b/litellm/llms/openai_like/chat/handler.py index a450947afdf..5a5d038979b 100644 --- a/litellm/llms/openai_like/chat/handler.py +++ b/litellm/llms/openai_like/chat/handler.py @@ -5,14 +5,16 @@ For handling OpenAI-like chat completions, like IBM WatsonX, etc. """ import json -from collections.abc import Callable, Mapping, Sequence -from typing import Final, TypedDict +from collections.abc import Callable +from typing import Final import httpx -from typing_extensions import ReadOnly +from pydantic import TypeAdapter +from typing_extensions import TypedDict import litellm from litellm import LlmProviders +from litellm.litellm_core_utils.core_helpers import keyword_arguments from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -26,22 +28,16 @@ from litellm.utils import CustomStreamWrapper, ProviderConfigManager from ..common_utils import OpenAILikeBase, OpenAILikeError from .transformation import OpenAILikeChatConfig - -class _OpenAILikeChatCompletion(TypedDict, total=False): - """The chat-completion JSON body an OpenAI-like provider returns for a non-streamed call.""" - - id: ReadOnly[str] - choices: ReadOnly[Sequence[Mapping[str, object]]] - created: ReadOnly[int] - model: ReadOnly[str] - system_fingerprint: ReadOnly[str] - usage: ReadOnly[Mapping[str, object]] - object: ReadOnly[str] +_DECODED_JSON: Final = TypeAdapter(object) -def _fake_streamed_model_response(payload: _OpenAILikeChatCompletion) -> ModelResponse: +class _ModelResponseFields(TypedDict, total=False, extra_items=object): + pass + + +def _fake_streamed_model_response(payload: object) -> ModelResponse: """Build the single response a fake-streamed provider call replays as one chunk.""" - return ModelResponse(**payload) + return ModelResponse(**_ModelResponseFields(**keyword_arguments(payload, ModelResponse))) async def make_call( @@ -63,7 +59,7 @@ async def make_call( if streaming_decoder is not None: completion_stream = streaming_decoder.aiter_bytes(response.aiter_bytes(chunk_size=1024)) elif fake_stream: - model_response: Final = _fake_streamed_model_response(response.json()) + model_response: Final = _fake_streamed_model_response(_DECODED_JSON.validate_python(response.json())) completion_stream = MockResponseIterator(model_response=model_response) else: completion_stream = ModelResponseIterator( @@ -103,7 +99,7 @@ def make_sync_call( if streaming_decoder is not None: completion_stream = streaming_decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) elif fake_stream: - model_response: Final = _fake_streamed_model_response(response.json()) + model_response: Final = _fake_streamed_model_response(_DECODED_JSON.validate_python(response.json())) completion_stream = MockResponseIterator(model_response=model_response) else: completion_stream = ModelResponseIterator( diff --git a/litellm/llms/openai_like/embedding/handler.py b/litellm/llms/openai_like/embedding/handler.py index 540feeca6e7..b064505b445 100644 --- a/litellm/llms/openai_like/embedding/handler.py +++ b/litellm/llms/openai_like/embedding/handler.py @@ -10,9 +10,11 @@ from typing import Final import httpx from pydantic import TypeAdapter +from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import keyword_arguments from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -40,6 +42,11 @@ _TRANSPORT_HEADER_NAMES: Final = frozenset( ) _HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+") _HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*") +_DECODED_JSON: Final = TypeAdapter(object) + + +class _EmbeddingResponseFields(TypedDict, total=False, extra_items=object): + pass def _header_text(name: str, value: object) -> str | None: @@ -108,7 +115,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): response.raise_for_status() - response_json: Final = response.json() + response_json: Final = _DECODED_JSON.validate_python(response.json()) except httpx.HTTPStatusError as e: raise OpenAILikeError( status_code=e.response.status_code, @@ -126,7 +133,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): additional_args={"complete_input_dict": data}, original_response=response_json, ) - return EmbeddingResponse(**response_json) + return EmbeddingResponse(**_EmbeddingResponseFields(**keyword_arguments(response_json, EmbeddingResponse))) except Exception as e: ## LOGGING logging_obj.post_call( diff --git a/litellm/llms/predibase/chat/transformation.py b/litellm/llms/predibase/chat/transformation.py index b30713fa119..94e64dfb95b 100644 --- a/litellm/llms/predibase/chat/transformation.py +++ b/litellm/llms/predibase/chat/transformation.py @@ -5,10 +5,11 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Literal from httpx import Headers, Response +from pydantic import TypeAdapter import litellm from litellm.constants import DEFAULT_MAX_TOKENS -from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.core_helpers import json_contains, map_finish_reason, subscript from litellm.litellm_core_utils.prompt_templates.factory import ( prompt_factory, render_custom_prompt_template, @@ -28,6 +29,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + class PredibaseConfig(BaseConfig): """ @@ -163,13 +166,13 @@ class PredibaseConfig(BaseConfig): additional_args={"complete_input_dict": request_data}, ) try: - completion_response: Final = raw_response.json() + completion_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raise PredibaseError(message=raw_response.text, status_code=422) - if "error" in completion_response: + if json_contains(completion_response, "error"): raise PredibaseError( - message=str(completion_response["error"]), + message=str(subscript(completion_response, "error")), status_code=raw_response.status_code, ) elif not isinstance(completion_response, dict): diff --git a/litellm/llms/ragflow/vector_stores/transformation.py b/litellm/llms/ragflow/vector_stores/transformation.py index ef44b6e4c0f..421bd769778 100644 --- a/litellm/llms/ragflow/vector_stores/transformation.py +++ b/litellm/llms/ragflow/vector_stores/transformation.py @@ -41,6 +41,11 @@ class _RAGFlowCreateResponseRoot(RootModel[_RAGFlowCreateResponse]): model_config = ConfigDict(frozen=True, hide_input_in_errors=True) +class _RAGFlowStatus(TypedDict): + code: ReadOnly[NotRequired[int]] + message: ReadOnly[NotRequired[str]] + + _RAGFLOW_EMPTY_DATA: Final[_RAGFlowDataset] = {} @@ -197,7 +202,7 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): } """ try: - response_json: Final = response.json() + response_json: Final[_RAGFlowStatus] = response.json() # Check for RAGFlow error response if response_json.get("code") != 0: diff --git a/litellm/llms/replicate/chat/transformation.py b/litellm/llms/replicate/chat/transformation.py index 62dfac43bd7..4553a02e451 100644 --- a/litellm/llms/replicate/chat/transformation.py +++ b/litellm/llms/replicate/chat/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH @@ -33,6 +34,10 @@ _TEXTS: Final = TypeAdapter(Iterable[str], config=ConfigDict(strict=True, hide_i _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) +class _ReplicatePrediction(TypedDict): + urls: ReadOnly[NotRequired[Mapping[str, str]]] + + class ReplicateConfig(BaseConfig): """ Reference: https://replicate.com/meta/llama-2-70b-chat/api @@ -292,7 +297,7 @@ class ReplicateConfig(BaseConfig): "urls":{"cancel":"https://api.replicate.com/v1/predictions/gqsmqmp1pdrj00cknr08dgmvb4/cancel","get":"https://api.replicate.com/v1/predictions/gqsmqmp1pdrj00cknr08dgmvb4","stream":"https://stream-b.svc.rno2.c.replicate.net/v1/streams/eot4gbydowuin4snhncydwxt57dfwgsc3w3snycx5nid7oef7jga"} } """ - response_json: Final = response.json() + response_json: Final[_ReplicatePrediction] = response.json() prediction_url: Final = response_json.get("urls", {}).get("get") if prediction_url is None: raise ReplicateError( diff --git a/litellm/llms/sagemaker/completion/transformation.py b/litellm/llms/sagemaker/completion/transformation.py index 5f6bef4246e..88f953091b7 100644 --- a/litellm/llms/sagemaker/completion/transformation.py +++ b/litellm/llms/sagemaker/completion/transformation.py @@ -11,9 +11,11 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from httpx._models import Headers, Response +from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import as_object_list, json_contains, subscript from litellm.litellm_core_utils.prompt_templates.factory import ( prompt_factory, render_custom_prompt_template, @@ -33,6 +35,14 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + + +def _concatenated(prefix: str, value: object) -> str: + if isinstance(value, str): + return prefix + value + raise TypeError(f'can only concatenate str (not "{type(value).__name__}") to str') + class SagemakerConfig(BaseConfig): """ @@ -199,7 +209,7 @@ class SagemakerConfig(BaseConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: - completion_response: Final = raw_response.json() + completion_response: Final = _DECODED_JSON.validate_python(raw_response.json()) ## LOGGING logging_obj.post_call( input=messages, @@ -212,15 +222,19 @@ class SagemakerConfig(BaseConfig): ## RESPONSE OBJECT try: - if isinstance(completion_response, list): - completion_response_choices = completion_response[0] - else: - completion_response_choices = completion_response + completion_response_list: Final = as_object_list(completion_response) + completion_response_choices: Final = ( + completion_response if completion_response_list is None else completion_response_list[0] + ) completion_output = "" - if "generation" in completion_response_choices: - completion_output += completion_response_choices["generation"] - elif "generated_text" in completion_response_choices: - completion_output += completion_response_choices["generated_text"] + if json_contains(completion_response_choices, "generation"): + completion_output = _concatenated( + completion_output, subscript(completion_response_choices, "generation") + ) + elif json_contains(completion_response_choices, "generated_text"): + completion_output = _concatenated( + completion_output, subscript(completion_response_choices, "generated_text") + ) # check if the prompt template is part of output, if so - filter it out if completion_output.startswith(prompt) and "" in prompt: diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index 928dcc3bc52..161cf908a6f 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -17,7 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict import httpx from typing_extensions import ReadOnly -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_process_openai_file_message, convert_to_anthropic_tool_result, @@ -354,13 +354,15 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): if isinstance(block, Mapping) and block.get("type") == "text" ) elif role == "assistant": - tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None) + tool_calls: object = ( + msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None) + ) thinking_blocks = _signed_thinking_blocks(msg) if tool_calls: content_blocks: list[dict[str, object]] = list(thinking_blocks) # mutable-ok: JSON wire blocks if content: content_blocks.append({"type": "text", "text": content}) - for tc in tool_calls: + for tc in iter_items(tool_calls): func = tc.get("function", {}) if isinstance(tc, dict) else getattr(tc, "function", {}) tc_id = tc.get("id", "") if isinstance(tc, dict) else getattr(tc, "id", "") func_name = func.get("name", "") if isinstance(func, dict) else getattr(func, "name", "") diff --git a/litellm/llms/soniox/audio_transcription/transformation.py b/litellm/llms/soniox/audio_transcription/transformation.py index 30d872f248f..cefb195f4a5 100644 --- a/litellm/llms/soniox/audio_transcription/transformation.py +++ b/litellm/llms/soniox/audio_transcription/transformation.py @@ -13,11 +13,13 @@ import re from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final, NoReturn +from typing import Any, Final, NoReturn, SupportsFloat, SupportsIndex from httpx import Headers, Response from pydantic import TypeAdapter, ValidationError +from typing_extensions import Buffer +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, @@ -172,6 +174,12 @@ def _nested_form_param(param: str, fields: tuple[_BracketedField, ...]) -> Mappi } +def _as_float(value: object) -> float: + if isinstance(value, (str, Buffer, SupportsFloat, SupportsIndex)): + return float(value) + raise TypeError(f"float() argument must be a string or a real number, not {type(value).__name__!r}") + + def _is_soniox_context(value: object) -> bool: try: _STRUCTURED_CONTEXT.validate_python(value) @@ -345,7 +353,7 @@ class SonioxAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_format: str | None = None, ) -> TranscriptionResponse: """Shared response-building logic (also used by the handler).""" - transcription_meta: dict[str, Any] = {} + transcription_meta: object = {} transcript: dict[str, Any] if isinstance(payload, dict) and "transcript" in payload: @@ -385,9 +393,10 @@ class SonioxAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response["task"] = "transcribe" # Best-effort metadata fields matching OpenAI's verbose_json shape. - if transcription_meta.get("audio_duration_ms") is not None: + audio_duration_ms: Final = decoded_json_object(transcription_meta).get("audio_duration_ms") + if audio_duration_ms is not None: try: - response["duration"] = float(transcription_meta["audio_duration_ms"]) / 1000.0 + response["duration"] = _as_float(audio_duration_ms) / 1000.0 except (TypeError, ValueError): pass @@ -403,10 +412,12 @@ class SonioxAudioTranscriptionConfig(BaseAudioTranscriptionConfig): words: Final[list[dict[str, object]]] = [] for token in tokens: word_entry: dict[str, object] = {"word": token.get("text", "")} - if token.get("start_ms") is not None: - word_entry["start"] = float(token["start_ms"]) / 1000.0 - if token.get("end_ms") is not None: - word_entry["end"] = float(token["end_ms"]) / 1000.0 + start_ms: object = token.get("start_ms") + if start_ms is not None: + word_entry["start"] = _as_float(start_ms) / 1000.0 + end_ms: object = token.get("end_ms") + if end_ms is not None: + word_entry["end"] = _as_float(end_ms) / 1000.0 words.append(word_entry) if words: response["words"] = words diff --git a/litellm/llms/stability/image_edit/transformations.py b/litellm/llms/stability/image_edit/transformations.py index 974996c5e48..65277973d39 100644 --- a/litellm/llms/stability/image_edit/transformations.py +++ b/litellm/llms/stability/image_edit/transformations.py @@ -6,13 +6,13 @@ Handles transformation between OpenAI-compatible format and Stability AI API for API Reference: https://platform.stability.ai/docs/api-reference """ -from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles -from typing_extensions import ReadOnly, TypedDict +from pydantic import TypeAdapter +from litellm.litellm_core_utils.core_helpers import decoded_json_object, json_contains, subscript from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams @@ -31,12 +31,7 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any - -class _StabilityImageEditResponse(TypedDict, total=False): - image: ReadOnly[str] - finish_reason: ReadOnly[str] - seed: ReadOnly[int] - errors: ReadOnly[Sequence[str]] +_DECODED_JSON: Final = TypeAdapter(object) class StabilityImageEditConfig(BaseImageEditConfig): @@ -268,7 +263,7 @@ class StabilityImageEditConfig(BaseImageEditConfig): OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp} """ try: - response_data: Final[_StabilityImageEditResponse] = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error parsing Stability AI response: {e}", @@ -277,14 +272,15 @@ class StabilityImageEditConfig(BaseImageEditConfig): ) # Check for errors in response - if "errors" in response_data: + if json_contains(response_json, "errors"): raise self.get_error_class( - error_message=f"Stability AI error: {response_data['errors']}", + error_message=f"Stability AI error: {subscript(response_json, 'errors')}", status_code=raw_response.status_code, headers=raw_response.headers, ) # Check finish_reason + response_data: Final = decoded_json_object(response_json) finish_reason: Final = response_data.get("finish_reason", "") if finish_reason == "CONTENT_FILTERED": raise self.get_error_class( diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index d0de71e0305..aa275f254b4 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -7,7 +7,7 @@ from enum import Enum from functools import lru_cache from itertools import accumulate, chain, repeat, takewhile from types import MappingProxyType -from typing import Any, Final, Literal, TypeVar, cast, get_type_hints +from typing import Final, Literal, TypeVar, cast, get_type_hints import httpx from pydantic import JsonValue, TypeAdapter, ValidationError @@ -1384,6 +1384,13 @@ class VertexAIModelInfo(BaseLLMModelInfo): raise NotImplementedError("Vertex AI models are not supported yet") +_COUNT_TOKENS_REQUEST_FIELDS: Final = frozenset({"model", "contents", "system_instruction", "tools"}) + + +class _ForwardedCountTokensParams(TypedDict, total=False, extra_items=object): + pass + + class VertexAITokenCounter(BaseTokenCounter): """Token counter implementation for Google AI Studio provider.""" @@ -1446,7 +1453,7 @@ class VertexAITokenCounter(BaseTokenCounter): model_to_use: str, messages: list[dict[str, object]] | None, contents: list[dict[str, object]] | None, - deployment: dict[str, Any] | None = None, + deployment: dict[str, object] | None = None, request_model: str = "", tools: list[dict[str, object]] | None = None, system: object | None = None, @@ -1458,6 +1465,7 @@ class VertexAITokenCounter(BaseTokenCounter): ) -> TokenCountResponse | None: import copy + from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.llms.anthropic.common_utils import context_management_with_edits from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_publisher from litellm.llms.vertex_ai.vertex_ai_partner_models.main import ( @@ -1467,19 +1475,19 @@ class VertexAITokenCounter(BaseTokenCounter): VertexAIPartnerModels, ) - deployment = deployment or {} - count_tokens_params_request: Final = copy.deepcopy(deployment.get("litellm_params", {})) + resolved_deployment: Final = deployment or {} + raw_litellm_params: Final = copy.deepcopy(resolved_deployment.get("litellm_params", {})) if not VertexAIPartnerModels.is_vertex_partner_model(model_to_use) and not uses_vertex_gemini_api( - model_to_use, count_tokens_params_request + model_to_use, decoded_json_object(raw_litellm_params) if raw_litellm_params else None ): return None without_count_route: Final = ( VertexAIPartnerModels.is_vertex_partner_model(model_to_use) and count_tokens_publisher(model_to_use) is None ) - model_info: Final[Mapping[str, object]] = deployment.get("model_info") or {} - if without_count_route and messages and model_info.get("custom_tokenizer") is not None: + model_info: Final[object] = resolved_deployment.get("model_info") or {} + if without_count_route and messages and decoded_json_object(model_info).get("custom_tokenizer") is not None: return None if without_count_route: return TokenCountResponse( @@ -1497,7 +1505,7 @@ class VertexAITokenCounter(BaseTokenCounter): translated_count: Final = await self._count_translated_claude_request( model_to_use=model_to_use, messages=messages, - deployment=deployment, + deployment=resolved_deployment, request_model=request_model, tools=tools, system=system, @@ -1513,7 +1521,7 @@ class VertexAITokenCounter(BaseTokenCounter): partner_models_handler: Final = VertexAIPartnerModels() # Extract vertex-specific params from litellm_params - partner_litellm_params: Final[dict[str, object]] = count_tokens_params_request + partner_litellm_params: Final = dict(decoded_json_object(raw_litellm_params)) vertex_project = partner_litellm_params.get("vertex_project") or partner_litellm_params.get( "vertex_ai_project" ) @@ -1585,11 +1593,12 @@ class VertexAITokenCounter(BaseTokenCounter): ) from litellm.llms.vertex_ai.count_tokens.handler import VertexAITokenCounter + gemini_litellm_params: Final = decoded_json_object(raw_litellm_params) request_view: Final = await inference_request_view( messages or [], system, messages_format, - count_context_edits(context_management, tools, count_tokens_params_request, model_to_use), + count_context_edits(context_management, tools, gemini_litellm_params, model_to_use), ) if isinstance(request_view, InferenceRequestViewError): return count_context_error_response(request_view, request_model, model_to_use) @@ -1600,7 +1609,7 @@ class VertexAITokenCounter(BaseTokenCounter): return count_context_error_response(count_context, request_model, model_to_use) blocks_error: Final = unconvertible_messages_error( request_view.messages, messages_format - ) or malformed_tool_result_text_count_error(messages, messages_format, deployment) + ) or malformed_tool_result_text_count_error(messages, messages_format, resolved_deployment) if blocks_error is not None: return count_context_error_response(blocks_error, request_model, model_to_use) @@ -1608,22 +1617,21 @@ class VertexAITokenCounter(BaseTokenCounter): contents if contents is not None else await gemini_count_contents( - request_view.messages, messages_format, model_to_use, "vertex_ai", count_tokens_params_request + request_view.messages, messages_format, model_to_use, "vertex_ai", gemini_litellm_params ) ) if isinstance(resolved_contents, VertexGeminiCountContextError): return count_context_error_response(resolved_contents, request_model, model_to_use) - count_tokens_params: Final = { - "model": model_to_use, - "contents": resolved_contents, - "system_instruction": count_context.system_instruction, - "tools": count_context.tools, - } - count_tokens_params_request.update(count_tokens_params) try: result = await VertexAITokenCounter().acount_tokens( - **count_tokens_params_request, + **_ForwardedCountTokensParams( + **{k: v for k, v in gemini_litellm_params.items() if k not in _COUNT_TOKENS_REQUEST_FIELDS} + ), + model=model_to_use, + contents=resolved_contents, + system_instruction=count_context.system_instruction, + tools=count_context.tools, ) except litellm.APIError as error: return provider_error_response(error, request_model, model_to_use) diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 1f03de4625c..7a7afcb83f1 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -10,7 +10,7 @@ from contextlib import aclosing from dataclasses import dataclass from functools import partial from types import MappingProxyType -from typing import IO, Any, Final, TypedDict +from typing import IO, Final, TypedDict from urllib.parse import quote, unquote import httpx @@ -827,7 +827,7 @@ def _openai_batch_jsonl_entry_to_vertex_anthropic_row( def _openai_batch_jsonl_entry_to_vertex_rows( - openai_entry: dict[str, Any], + openai_entry: Mapping[str, object], map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], batch_model: str | None = None, deployment_params: Mapping[str, object] = MappingProxyType({}), @@ -842,7 +842,7 @@ def _openai_batch_jsonl_entry_to_vertex_rows( Example Vertex jsonl {"request":{"contents": [{"role": "user", "parts": [{"text": "What is the relation between the following video and image samples?"}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/video/animals.mp4", "mimeType": "video/mp4"}}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/image/cricket.jpeg", "mimeType": "image/jpeg"}}]}]}} """ - entry_body: Final = openai_entry.get("body") + entry_body: Final = as_str_mapping(openai_entry.get("body")) model: Final = batch_model or (str(entry_body.get("model") or "") if isinstance(entry_body, dict) else "") is_anthropic_model: Final = is_vertex_anthropic_batch_model(model) if _is_embeddings_batch_entry(openai_entry): @@ -855,11 +855,10 @@ def _openai_batch_jsonl_entry_to_vertex_rows( if not isinstance(entry_body, dict): raise TypeError("`body` on batch requests must be a JSON object, but was missing or not an object") - openai_request_body: Final = openai_entry["body"] chat_request_body: Final = ( - responses_batch_body_to_chat_body(openai_request_body, custom_llm_provider="vertex_ai") + responses_batch_body_to_chat_body(entry_body, custom_llm_provider="vertex_ai") if _is_responses_batch_entry(openai_entry) - else openai_request_body + else entry_body ) chat_messages: Final = map_developer_role_to_system_role(chat_request_body.get("messages", [])) if is_anthropic_model: diff --git a/litellm/llms/vertex_ai/image_generation/image_generation_handler.py b/litellm/llms/vertex_ai/image_generation/image_generation_handler.py index ec915e10fcd..56843d2e1a1 100644 --- a/litellm/llms/vertex_ai/image_generation/image_generation_handler.py +++ b/litellm/llms/vertex_ai/image_generation/image_generation_handler.py @@ -7,6 +7,7 @@ from openai.types.image import Image from pydantic import ConfigDict, TypeAdapter import litellm +from litellm.litellm_core_utils.core_helpers import json_contains, subscript from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -20,23 +21,24 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj _PREDICTIONS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class VertexImageGeneration(VertexLLM): def process_image_generation_response( self, - json_response: Mapping[str, object], + json_response: object, model_response: ImageResponse, model: str | None = None, ) -> ImageResponse: - if "predictions" not in json_response: + if not json_contains(json_response, "predictions"): raise litellm.InternalServerError( message=f"image generation response does not contain 'predictions', got {json_response}", llm_provider="vertex_ai", model=model, ) - predictions: Final = _PREDICTIONS.validate_python(json_response["predictions"]) + predictions: Final = _PREDICTIONS.validate_python(subscript(json_response, "predictions")) response_data: Final[list[Image]] = [] for prediction in predictions: @@ -168,7 +170,7 @@ class VertexImageGeneration(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - json_response: Final[Mapping[str, object]] = response.json() + json_response: Final = _DECODED_JSON.validate_python(response.json()) return self.process_image_generation_response(json_response, model_response, model) async def aimage_generation( @@ -272,11 +274,11 @@ class VertexImageGeneration(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - json_response: Final[Mapping[str, object]] = response.json() + json_response: Final = _DECODED_JSON.validate_python(response.json()) return self.process_image_generation_response(json_response, model_response, model) - def is_image_generation_response(self, json_response: dict[str, Any]) -> bool: + def is_image_generation_response(self, json_response: Mapping[str, object]) -> bool: if "predictions" in json_response: - if "bytesBase64Encoded" in json_response["predictions"][0]: + if json_contains(subscript(json_response["predictions"], 0), "bytesBase64Encoded"): return True return False diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 7635cd32a06..5781481dde8 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -20,7 +20,10 @@ import json from collections.abc import Awaitable, Callable from typing import Final +from pydantic import TypeAdapter + from litellm import verbose_logger +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import ( @@ -30,6 +33,8 @@ from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import ( from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +_DECODED_JSON: Final = TypeAdapter(object) + class VertexAIRealtimeConfig(GeminiRealtimeConfig): """ @@ -217,7 +222,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): ``setup`` closes the connection with a 1007 policy error. """ json_message: Final = json.loads(message) - msg_type: Final = json_message.get("type") + msg_type: Final = decoded_json_object(_DECODED_JSON.validate_python(json_message)).get("type") if msg_type == "session.update": if session_configuration_request is None: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama/transformation.py index d11be440172..e26b26af278 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama/transformation.py @@ -131,7 +131,8 @@ class VertexAILlama3Config(OpenAIGPTConfig): ## RESPONSE OBJECT try: - completion_response: Final = OpenAIChatCompletionResponse(**raw_response.json()) + payload: Final[OpenAIChatCompletionResponse] = raw_response.json() + completion_response: Final = OpenAIChatCompletionResponse(**payload) except Exception as e: response_headers: Final = getattr(raw_response, "headers", None) raise VertexAIError( diff --git a/litellm/llms/volcengine/embedding/transformation.py b/litellm/llms/volcengine/embedding/transformation.py index cc1e7d59caa..283b013917b 100644 --- a/litellm/llms/volcengine/embedding/transformation.py +++ b/litellm/llms/volcengine/embedding/transformation.py @@ -8,7 +8,10 @@ from collections.abc import Mapping from typing import Final import httpx +from pydantic import TypeAdapter +from typing_extensions import TypedDict +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -18,6 +21,12 @@ from litellm.types.utils import EmbeddingResponse from ..common_utils import get_volcengine_base_url, get_volcengine_headers +_DECODED_JSON: Final = TypeAdapter(object) + + +class _VolcengineEmbeddingFields(TypedDict, total=False, extra_items=object): + pass + class VolcEngineEmbeddingConfig(BaseEmbeddingConfig): """ @@ -159,9 +168,10 @@ class VolcEngineEmbeddingConfig(BaseEmbeddingConfig): ) -> EmbeddingResponse: """Transform Volcengine response to EmbeddingResponse""" try: - response_json: Final = raw_response.json() + decoded_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise ValueError(f"Failed to parse Volcengine response as JSON: {e}") + response_json: Final = decoded_json_object(decoded_response) # Volcengine response format matches OpenAI format closely # Just need to ensure all required fields are present @@ -177,7 +187,7 @@ class VolcEngineEmbeddingConfig(BaseEmbeddingConfig): transformed_response["id"] = response_json["id"] # Create EmbeddingResponse from transformed data - return EmbeddingResponse(**transformed_response) + return EmbeddingResponse(**_VolcengineEmbeddingFields(**transformed_response)) def validate_environment( self, diff --git a/litellm/llms/volcengine/videos/transformation.py b/litellm/llms/volcengine/videos/transformation.py index 5af1080b9a0..ca4d3546636 100644 --- a/litellm/llms/volcengine/videos/transformation.py +++ b/litellm/llms/volcengine/videos/transformation.py @@ -874,7 +874,7 @@ class VolcEngineVideoConfig(BaseVideoConfig): **(extra_query or {}), } - def transform_video_list_response( # pyright: ignore[reportIncompatibleMethodOverride] # base declares dict[str, str] but the payload is a heterogeneous list body + def transform_video_list_response( self, raw_response: httpx.Response, logging_obj: "Logging", diff --git a/litellm/llms/watsonx/common_utils.py b/litellm/llms/watsonx/common_utils.py index 6d3686e9f85..5d2aa418173 100644 --- a/litellm/llms/watsonx/common_utils.py +++ b/litellm/llms/watsonx/common_utils.py @@ -1,10 +1,12 @@ from typing import Final, cast import httpx +from pydantic import TypeAdapter import litellm from litellm import verbose_logger from litellm.caching import InMemoryCache +from litellm.litellm_core_utils.core_helpers import subscript from litellm.litellm_core_utils.prompt_templates import factory as ptf from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -23,6 +25,13 @@ class WatsonXAIError(BaseLLMException): iam_token_cache: Final = InMemoryCache() +_IAM_TOKEN_RESPONSE: Final = TypeAdapter(object) + + +def _ttl_with_buffer(expires_in: object) -> float: + if isinstance(expires_in, (int, float)): + return expires_in - 10 + raise TypeError(f"unsupported operand type(s) for -: '{type(expires_in).__name__}' and 'int'") def get_watsonx_iam_url(): @@ -30,7 +39,7 @@ def get_watsonx_iam_url(): def generate_iam_token(api_key=None, **params) -> str: - result: str | None = iam_token_cache.get_cache(api_key) + result: object = iam_token_cache.get_cache(api_key) if result is None: headers: Final = {} @@ -58,13 +67,13 @@ def generate_iam_token(api_key=None, **params) -> str: ) response: Final = litellm.module_level_client.post(url=iam_token_url, data=data, headers=headers) response.raise_for_status() - json_data: Final = response.json() + json_data: Final = _IAM_TOKEN_RESPONSE.validate_python(response.json()) - result = json_data["access_token"] + result = subscript(json_data, "access_token") iam_token_cache.set_cache( key=api_key, value=result, - ttl=json_data["expires_in"] - 10, # leave some buffer + ttl=_ttl_with_buffer(subscript(json_data, "expires_in")), ) return cast(str, result) diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py index 9ea062ea2f8..fd1ecaf29dd 100644 --- a/litellm/llms/watsonx/completion/transformation.py +++ b/litellm/llms/watsonx/completion/transformation.py @@ -4,7 +4,9 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock @@ -30,6 +32,9 @@ else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + + class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): """ Reference: https://cloud.ibm.com/apidocs/watsonx-ai#text-generation @@ -305,12 +310,15 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): completion_tokens: Final = json_resp["results"][0]["generated_token_count"] model_response.choices[0].message.content = generated_text model_response.choices[0].finish_reason = map_finish_reason(json_resp["results"][0]["stop_reason"]) - if json_resp.get("created_at"): + created_at: Final = decoded_json_object(_DECODED_JSON.validate_python(json_resp)).get("created_at") + if created_at: + if not isinstance(created_at, str): + raise TypeError("fromisoformat: argument must be str") try: - created_datetime = datetime.fromisoformat(json_resp["created_at"]) + created_datetime = datetime.fromisoformat(created_at) except ValueError: # datetime.fromisoformat cannot handle 'Z' in Python 3.10 - created_datetime = datetime.fromisoformat(f"{json_resp['created_at'].rstrip('Z')}+00:00") + created_datetime = datetime.fromisoformat(f"{created_at.rstrip('Z')}+00:00") model_response.created = int(created_datetime.timestamp()) else: model_response.created = int(time.time()) diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index f53b71a5a9a..da8d673ce9f 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -1,15 +1,17 @@ import copy from collections.abc import AsyncIterator, Iterator, Mapping from types import MappingProxyType -from typing import Any, Final +from typing import Final, Protocol, SupportsIndex, SupportsInt, runtime_checkable import httpx from pydantic import ConfigDict, TypeAdapter +from typing_extensions import Buffer import litellm from litellm._logging import verbose_logger from litellm.constants import XAI_API_BASE from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.prompt_templates.common_utils import ( filter_value_from_dict, strip_name_from_messages, @@ -36,6 +38,28 @@ _RESPONSE_BODY: Final = TypeAdapter(dict[object, object], config=ConfigDict(hide _FILTERED_TOOL: Final = TypeAdapter(object) +@runtime_checkable +class _SupportsTrunc(Protocol): + def __trunc__(self) -> int: ... + + +def _token_count(value: object) -> int: + if isinstance(value, (str, Buffer, SupportsInt, SupportsIndex, _SupportsTrunc)): + return int(value) + raise TypeError( + f"int() argument must be a string, a bytes-like object or a real number, not '{type(value).__name__}'" + ) + + +def _reasoning_tokens(details: object) -> int: + if not details: + return 0 + details_fields: Final = as_str_mapping(details) + if details_fields is not None and isinstance(details, dict): + return _token_count(details_fields.get("reasoning_tokens") or 0) + return _token_count(attribute_or_none(details, "reasoning_tokens") or 0) + + def _usage_restated_from_xai_ticks(usage: Usage | None) -> Usage | None: reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None)) if usage is None or reported_cost is None: @@ -324,7 +348,7 @@ class XAIChatConfig(OpenAIGPTConfig): @staticmethod def fold_reasoning_tokens_into_completion( - target: ModelResponse | Usage | dict[str, Any] | None, + target: ModelResponse | Usage | dict[str, object] | None, ) -> None: """Reconcile xAI Usage to the OpenAI invariant. @@ -341,24 +365,20 @@ class XAIChatConfig(OpenAIGPTConfig): return if isinstance(target, ModelResponse): - usage: Usage | dict[str, Any] | None = getattr(target, "usage", None) + usage: Usage | dict[str, object] | None = getattr(target, "usage", None) else: usage = target if usage is None: return if isinstance(usage, dict): - details = usage.get("completion_tokens_details") or {} - if isinstance(details, dict): - reasoning_tokens = int(details.get("reasoning_tokens") or 0) - else: - reasoning_tokens = int(getattr(details, "reasoning_tokens", 0) or 0) + reasoning_tokens = _reasoning_tokens(usage.get("completion_tokens_details")) if reasoning_tokens <= 0: return - prompt_tokens = int(usage.get("prompt_tokens") or 0) - completion_tokens = int(usage.get("completion_tokens") or 0) - total_tokens = int(usage.get("total_tokens") or 0) + prompt_tokens = _token_count(usage.get("prompt_tokens") or 0) + completion_tokens = _token_count(usage.get("completion_tokens") or 0) + total_tokens = _token_count(usage.get("total_tokens") or 0) if total_tokens == prompt_tokens + completion_tokens: return @@ -370,8 +390,8 @@ class XAIChatConfig(OpenAIGPTConfig): usage["completion_tokens"] = completion_tokens + reasoning_tokens return - details = getattr(usage, "completion_tokens_details", None) - reasoning_tokens = int(getattr(details, "reasoning_tokens", 0) or 0) if details else 0 + details = attribute_or_none(usage, "completion_tokens_details") + reasoning_tokens = _token_count(attribute_or_none(details, "reasoning_tokens") or 0) if details else 0 if reasoning_tokens <= 0: return @@ -407,15 +427,15 @@ class XAIChatConfig(OpenAIGPTConfig): @staticmethod def _normalize_openai_compatible_usage_totals( - usage: Usage | dict[str, Any] | None, + usage: Usage | dict[str, object] | None, ) -> None: if usage is None: return if isinstance(usage, dict): - prompt_tokens = int(usage.get("prompt_tokens") or 0) - completion_tokens = int(usage.get("completion_tokens") or 0) + prompt_tokens = _token_count(usage.get("prompt_tokens") or 0) + completion_tokens = _token_count(usage.get("completion_tokens") or 0) expected_total = prompt_tokens + completion_tokens - if int(usage.get("total_tokens") or 0) < expected_total: + if _token_count(usage.get("total_tokens") or 0) < expected_total: usage["total_tokens"] = expected_total return prompt_tokens = int(usage.prompt_tokens or 0) diff --git a/litellm/llms/you_com/search/transformation.py b/litellm/llms/you_com/search/transformation.py index 9228deda26b..b56b547a949 100644 --- a/litellm/llms/you_com/search/transformation.py +++ b/litellm/llms/you_com/search/transformation.py @@ -8,7 +8,9 @@ OpenAPI spec: https://you.com/specs/openapi_search_v1.yaml from typing import Final, TypedDict import httpx +from pydantic import TypeAdapter +from litellm.litellm_core_utils.core_helpers import decoded_json_object, iter_items, subscript from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.search.transformation import ( @@ -18,6 +20,8 @@ from litellm.llms.base_llm.search.transformation import ( ) from litellm.secret_managers.main import get_secret_str +_DECODED_JSON: Final = TypeAdapter(object) + class _YouComSearchRequestRequired(TypedDict): """Required fields for You.com Search API request.""" @@ -169,23 +173,26 @@ class YouComSearchConfig(BaseSearchConfig): - snippets[0] → SearchResult.snippet (falls back to `description`) - page_age → SearchResult.date """ - response_json: Final = raw_response.json() - raw_results: Final = response_json.get("results") or {} + response_json: Final = decoded_json_object(_DECODED_JSON.validate_python(raw_response.json())) + raw_results: Final = decoded_json_object(response_json.get("results") or {}) web_results: Final = raw_results.get("web") or [] news_results: Final = raw_results.get("news") or [] results: Final[list[SearchResult]] = [] - for item in list(web_results) + list(news_results): + for entry in list(iter_items(web_results)) + list(iter_items(news_results)): + item = decoded_json_object(entry) snippets = item.get("snippets") or [] - snippet = snippets[0] if snippets else item.get("description", "") + snippet = subscript(snippets, 0) if snippets else item.get("description", "") results.append( - SearchResult( - title=item.get("title", ""), - url=item.get("url", ""), - snippet=snippet, - date=item.get("page_age"), - last_updated=None, + SearchResult.model_validate( + { + "title": item.get("title", ""), + "url": item.get("url", ""), + "snippet": snippet, + "date": item.get("page_age"), + "last_updated": None, + } ) ) diff --git a/litellm/main.py b/litellm/main.py index df8e8f29483..c86a888aa96 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3657,8 +3657,10 @@ def _complete_openrouter(ctx: CompletionDispatchContext) -> _CompletionDispatchR or get_secret_str("OR_API_KEY") ) - openrouter_site_url: Final = get_secret("OR_SITE_URL") or "https://litellm.ai" - openrouter_app_name: Final = get_secret("OR_APP_NAME") or "liteLLM" + openrouter_site_url: Final = _OPAQUE_VALUE_ADAPTER.validate_python( + get_secret("OR_SITE_URL") or "https://litellm.ai" + ) + openrouter_app_name: Final = _OPAQUE_VALUE_ADAPTER.validate_python(get_secret("OR_APP_NAME") or "liteLLM") openrouter_headers: Final = { "HTTP-Referer": openrouter_site_url, @@ -3762,8 +3764,10 @@ def _complete_vercel_ai_gateway( api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") - vercel_site_url: Final = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" - vercel_app_name: Final = get_secret("VERCEL_APP_NAME") or "liteLLM" + vercel_site_url: Final = _OPAQUE_VALUE_ADAPTER.validate_python( + get_secret("VERCEL_SITE_URL") or "https://litellm.ai" + ) + vercel_app_name: Final = _OPAQUE_VALUE_ADAPTER.validate_python(get_secret("VERCEL_APP_NAME") or "liteLLM") vercel_headers: Final = { "http-referer": vercel_site_url, @@ -7349,8 +7353,10 @@ def embedding( or get_secret_str("OR_API_KEY") ) - openrouter_site_url: Final = get_secret("OR_SITE_URL") or "https://litellm.ai" - openrouter_app_name: Final = get_secret("OR_APP_NAME") or "liteLLM" + openrouter_site_url: Final = _OPAQUE_VALUE_ADAPTER.validate_python( + get_secret("OR_SITE_URL") or "https://litellm.ai" + ) + openrouter_app_name: Final = _OPAQUE_VALUE_ADAPTER.validate_python(get_secret("OR_APP_NAME") or "liteLLM") openrouter_headers: Final = { "HTTP-Referer": openrouter_site_url, diff --git a/litellm/models/base.py b/litellm/models/base.py index 3105fc9b91d..728fea1cff7 100644 --- a/litellm/models/base.py +++ b/litellm/models/base.py @@ -2,14 +2,34 @@ Base model class for domain models. """ +from collections.abc import Iterable, Mapping from datetime import datetime -from typing import Any +from typing import Final, Protocol from pydantic import ConfigDict +from typing_extensions import TypedDict +from litellm.litellm_core_utils.core_helpers import as_str_mapping, has_attributes from litellm.types.llms.base import LiteLLMBaseModel +class _RecordFields(TypedDict, total=False, extra_items=object): + pass + + +class _ModelDumpRecord(Protocol): + def model_dump(self) -> Mapping[str, object]: ... + + +class _DictRecord(Protocol): + def dict(self) -> Mapping[str, object]: ... + + +def _as_dict(record: object) -> Mapping[str, object] | None: + fields: Final = as_str_mapping(record) + return fields if isinstance(record, dict) else None + + class DomainModel(LiteLLMBaseModel): """Base class for all domain models.""" @@ -23,17 +43,18 @@ class DomainModel(LiteLLMBaseModel): updated_at: datetime | None = None @classmethod - def from_db_record(cls, record: Any) -> "DomainModel": + def from_db_record(cls, record: Mapping[str, object] | Iterable[tuple[str, object]] | None) -> "DomainModel": """Create a domain model from a database record.""" if record is None: raise ValueError("Cannot create domain model from None record") - if isinstance(record, dict): - return cls(**record) - if hasattr(record, "model_dump") and callable(record.model_dump): - return cls(**record.model_dump()) - if hasattr(record, "dict") and callable(record.dict): - return cls(**record.dict()) - return cls(**dict(record)) + fields: Final = _as_dict(record) + if fields is not None: + return cls(**_RecordFields(**fields)) + if has_attributes(record, _ModelDumpRecord, "model_dump") and callable(record.model_dump): + return cls(**_RecordFields(**record.model_dump())) + if has_attributes(record, _DictRecord, "dict") and callable(record.dict): + return cls(**_RecordFields(**record.dict())) + return cls(**_RecordFields(**dict(record))) def to_db_dict(self, exclude_unset: bool = False) -> dict[str, object]: """Convert domain model to a dictionary for database operations.""" diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py index c5a63b0f990..e428017badd 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.py +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -13,7 +13,7 @@ the JSON, then run `npm run gen:api` in ui/litellm-dashboard and commit schema.d import json import re import sys -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from types import MappingProxyType @@ -43,7 +43,7 @@ def _stabilize_multi_method_route_ids(routes) -> None: """FastAPI derives route IDs from a set of methods; make snapshots stable.""" for route in routes: - methods = sorted(getattr(route, "methods", None) or []) + methods: Sequence[str] = sorted(getattr(route, "methods", None) or []) if len(methods) <= 1 or not getattr(route, "path_format", None): continue diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index f0a4d47f1e4..bff270b8983 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -23,7 +23,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, keyword_arguments from litellm.litellm_core_utils.url_utils import SSRFError, validate_url from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header from litellm.llms.azure.common_utils import caused_by_entra_rejection @@ -145,19 +145,20 @@ def _to_jsonrpc_object(chunk: object) -> object: return chunk -def _build_message_send_params(params: dict[str, Any]) -> "MessageSendParams": +def _build_message_send_params(params: object) -> "MessageSendParams": """Build MessageSendParams from wire (0.3) or A2A 1.0 JSON-RPC params.""" from a2a.compat.v0_3.types import MessageSendParams + fields: Final = dict(**keyword_arguments(params, MessageSendParams)) try: - return MessageSendParams(**params) + return MessageSendParams.model_validate(fields) except ValidationError: from a2a.compat.v0_3.conversions import pb2_v10, to_compat_send_message_request from google.protobuf.json_format import ParseDict, ParseError pb: Final = pb2_v10.SendMessageRequest() try: - ParseDict(params, pb, ignore_unknown_fields=True) + ParseDict(fields, pb, ignore_unknown_fields=True) except ParseError as e: raise ValueError(f"Invalid message/send params: {e}") from e return to_compat_send_message_request(pb, "").params @@ -1027,7 +1028,7 @@ async def invoke_agent_a2a( from a2a.compat.v0_3.types import SendMessageRequest try: - message_send_params: Final = _build_message_send_params(params) + message_send_params: Final = _build_message_send_params(_DECODED_JSON.validate_python(params)) except (ValidationError, ValueError) as e: return _jsonrpc_error(request_id, -32602, f"Invalid params: {e}") diff --git a/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py index a69dd6ee025..0e9fe830f3f 100644 --- a/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py +++ b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py @@ -68,7 +68,13 @@ async def update_deprecation_alert_state( ) -> DeprecationAlertState: _require_admin(user, write=True) state: Final = DeprecationAlertState( - **data.model_dump(), updated_at=datetime.now(timezone.utc), updated_by=user.user_id + model=data.model, + deprecation_date=data.deprecation_date, + status=data.status, + last_seen_date=data.last_seen_date, + last_seen_requests=data.last_seen_requests, + updated_at=datetime.now(timezone.utc), + updated_by=user.user_id, ) await repository.set_param(_state_key(data.model, data.deprecation_date), state.model_dump(mode="json")) return state diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 4c4e506ce38..6b39511d89b 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -30,6 +30,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import json_contains, subscript from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value from litellm.llms.custom_httpx.httpx_handler import HTTPHandler from litellm.proxy._types import ( @@ -164,6 +165,10 @@ class _UserInfoResponse(Protocol): def json(self) -> dict[str, object]: ... +class _JWKSResponse(Protocol): + def json(self) -> object: ... + + @dataclass(frozen=True, slots=True) class JWTIdentity: user_id: str | None @@ -218,6 +223,10 @@ def _userinfo_document(response: _UserInfoResponse) -> dict[str, object]: return response.json() +def _jwks_document(response: _JWKSResponse) -> object: + return response.json() + + def jwks_unavailable_exception(error: JWKSUnreachableError) -> ProxyException: return ProxyException( message=( @@ -960,12 +969,12 @@ class JWTHandler: ) try: - response_json: Final = response.json() + response_json: Final = _jwks_document(response) except Exception as e: verbose_proxy_logger.error("Error parsing response: %s. Original Response: %s", e, response.text) raise Exception(f"Error parsing response: {e}. Check server logs for original response.") - keys: Final = response_json["keys"] if "keys" in response_json else response_json + keys: Final = subscript(response_json, "keys") if json_contains(response_json, "keys") else response_json return cast(JWKKeyValue, keys) # cast-ok: JWTKeyItem declares only `kid`, validating would drop key material async def _get_public_key_from_jwks_url(self, jwks_url: str, kid: str | None) -> JWTKeyItem: diff --git a/litellm/proxy/auth/litellm_license.py b/litellm/proxy/auth/litellm_license.py index e1d6b2e090c..e828afe999c 100644 --- a/litellm/proxy/auth/litellm_license.py +++ b/litellm/proxy/auth/litellm_license.py @@ -3,14 +3,17 @@ import base64 import json import os +from collections.abc import Mapping from datetime import datetime from typing import TYPE_CHECKING, Final import httpx from pydantic import TypeAdapter +from typing_extensions import TypedDict from litellm._logging import verbose_proxy_logger from litellm.constants import NON_LLM_CONNECTION_TIMEOUT +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.custom_httpx.http_handler import HTTPHandler if TYPE_CHECKING: @@ -23,6 +26,23 @@ AUTO_ROUTER_LICENSE_REMEDY: Final = "A LiteLLM license with the 'auto_router' fe _LICENSE_VERDICT: Final = TypeAdapter(object) +class _LicenseFields(TypedDict, total=False, extra_items=object): + pass + + +def _license_fields(license_data: object) -> Mapping[str, object]: + fields: Final = as_str_mapping(license_data) + if fields is None: + raise TypeError(f"'{type(license_data).__name__}' object is not subscriptable") + return fields + + +def _expiration_text(expiration_date: object) -> str: + if isinstance(expiration_date, str): + return expiration_date + raise TypeError(f"strptime() argument 1 must be str, not {type(expiration_date).__name__}") + + class LicenseCheck: """ - Check if license in env @@ -201,15 +221,15 @@ class LicenseCheck: ) # Decode and parse the data - license_data: Final = json.loads(message.decode()) + license_data: Final = _license_fields(_LICENSE_VERDICT.validate_python(json.loads(message.decode()))) # Check expiration date - expiration_date: Final = datetime.strptime(license_data["expiration_date"], "%Y-%m-%d") + expiration_date: Final = datetime.strptime(_expiration_text(license_data["expiration_date"]), "%Y-%m-%d") if expiration_date < datetime.now(): self.airgapped_license_data = None return False, "License has expired" - self.airgapped_license_data = EnterpriseLicenseData(**license_data) + self.airgapped_license_data = EnterpriseLicenseData(**_LicenseFields(**license_data)) return True diff --git a/litellm/proxy/client/cli/commands/keys.py b/litellm/proxy/client/cli/commands/keys.py index 558c9139f5b..1b6db2b0d5c 100644 --- a/litellm/proxy/client/cli/commands/keys.py +++ b/litellm/proxy/client/cli/commands/keys.py @@ -2,7 +2,7 @@ import builtins import json from collections.abc import Mapping, Sequence from datetime import datetime -from typing import Any, Final, Literal, Protocol +from typing import Final, Literal, Protocol import click import requests @@ -33,6 +33,22 @@ class _JsonBodyView(TypedDict): body: ReadOnly[object] +class _KeyImportFields(TypedDict, total=False, extra_items=object): + pass + + +_KEY_IMPORT_FIELD_NAMES: Final = ( + "models", + "aliases", + "spend", + "key_alias", + "team_id", + "user_id", + "budget_id", + "config", +) + + def _cli_context(view: _CliContextView) -> _CliContext: return view.obj @@ -302,25 +318,9 @@ def _display_dry_run_table(source_keys: Sequence[Mapping[str, object]]) -> None: rich.print(table) -def _prepare_key_import_data(key: Mapping[str, object]) -> dict[str, Any]: +def _prepare_key_import_data(key: Mapping[str, object]) -> _KeyImportFields: """Prepare key data for import by extracting relevant fields.""" - import_data: Final = {} - - # Copy relevant fields if they exist - for field in [ - "models", - "aliases", - "spend", - "key_alias", - "team_id", - "user_id", - "budget_id", - "config", - ]: - if key.get(field): - import_data[field] = key[field] - - return import_data + return _KeyImportFields(**{field: key[field] for field in _KEY_IMPORT_FIELD_NAMES if key.get(field)}) def _import_keys_to_destination( diff --git a/litellm/proxy/client/cli/commands/models.py b/litellm/proxy/client/cli/commands/models.py index 538b42a8c83..ae3751cff69 100644 --- a/litellm/proxy/client/cli/commands/models.py +++ b/litellm/proxy/client/cli/commands/models.py @@ -10,12 +10,17 @@ from typing import TYPE_CHECKING, Any, Final, Literal import click import rich import yaml +from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm.litellm_core_utils.core_helpers import json_contains, subscript + # local imports from ... import Client from ._cli_context import cli_context_values +_DECODED_YAML: Final = TypeAdapter(object) + if TYPE_CHECKING: from rich.console import JustifyMethod @@ -385,10 +390,10 @@ def _print_summary_table(provider_counts): def get_model_list_from_yaml_file(yaml_file: str) -> list[dict[str, object]]: """Load and validate the model list from a YAML file.""" with open(yaml_file, "r") as f: - data: Final = yaml.safe_load(f) - if not data or "model_list" not in data: + data: Final = _DECODED_YAML.validate_python(yaml.safe_load(f)) + if not data or not json_contains(data, "model_list"): raise click.ClickException("YAML file must contain a 'model_list' key with a list of models.") - model_list: Final = data["model_list"] + model_list: Final = subscript(data, "model_list") if not isinstance(model_list, list): raise click.ClickException("'model_list' must be a list of model definitions.") return model_list diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 1b004423181..e585cd7b1ff 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -73,6 +73,7 @@ from litellm.litellm_core_utils.core_helpers import ( is_expected_client_error, proxy_request_headers, redact_nested_match_and_regex_keys, + require_attributes, stream_cost_injection_requested, ) from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer @@ -175,7 +176,7 @@ from litellm.types.guardrails import GuardrailEventHooks from litellm.types.router import RouterRateLimitError from litellm.types.router_weights import RouterWeights, validate_router_weights -_LateResponseT = TypeVar("_LateResponseT", bound=Response) +_LateResponseT = TypeVar("_LateResponseT") _LlmCallT = TypeVar("_LlmCallT") _ResponseT: Final = TypeVar("_ResponseT") _ChunkT: Final = TypeVar("_ChunkT") @@ -1365,7 +1366,17 @@ class LateSseFraming: """How a response the keepalive already opened ends once its upstream fails late.""" failure: Callable[[Exception], tuple[str, ...]] - body: Callable[[Response], tuple[bytes, ...]] + body: Callable[[object], tuple[bytes, ...]] + + +class _RenderedBody(Protocol): + @property + def body(self) -> bytes | memoryview: ... + + +class _CarriesStatusCode(Protocol): + @property + def status_code(self) -> int: ... def _openai_late_failure_frames(exc: Exception) -> tuple[str, ...]: @@ -1373,8 +1384,11 @@ def _openai_late_failure_frames(exc: Exception) -> tuple[str, ...]: return _sse_error_frames(error_obj) -def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]: - return b"data: " + (bytes(produced.body) or b"{}") + b"\n\n", b"data: [DONE]\n\n" +def _openai_late_body_frames(produced: object) -> tuple[bytes, ...]: + return ( + b"data: " + (bytes(require_attributes(produced, _RenderedBody, "body").body) or b"{}") + b"\n\n", + b"data: [DONE]\n\n", + ) def _anthropic_late_failure_frames(exc: Exception) -> tuple[str, ...]: @@ -1393,10 +1407,14 @@ def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]: return _responses_error_frames(sse_error_payload(exc)[1]) -def _responses_late_body_frames(produced: Response) -> tuple[bytes, ...]: - body: Final = bytes(produced.body) +def _responses_late_body_frames(produced: object) -> tuple[bytes, ...]: + body: Final = bytes(require_attributes(produced, _RenderedBody, "body").body) error_obj: Final = sse_error_object(body or b"{}") or {"message": body.decode()} - with_status: Final = {**error_obj, "code": error_obj.get("code") or str(produced.status_code)} + with_status: Final = { + **error_obj, + "code": error_obj.get("code") + or str(require_attributes(produced, _CarriesStatusCode, "status_code").status_code), + } return tuple(frame.encode() for frame in _responses_error_frames(with_status)) @@ -1620,7 +1638,7 @@ def ttft_keepalive_interval(request_data: Mapping[str, object], llm_router: Rout return resolve_ttft_keepalive_interval(deployments, litellm.sse_keepalive_ping_interval_seconds) -async def _aclose_late_response(produced: Response) -> None: +async def _aclose_late_response(produced: object) -> None: """Run the cleanup Starlette would have run, for a response it never called. Closing an already-closed async generator is a no-op, so this is safe to call @@ -1643,7 +1661,7 @@ async def _aclose_late_response(produced: Response) -> None: verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc) -async def _relay_late_response(produced: Response, framing: LateSseFraming) -> AsyncGenerator[bytes, None]: +async def _relay_late_response(produced: object, framing: LateSseFraming) -> AsyncGenerator[bytes, None]: """Replay a Response that was built after a keepalive had already opened the wire.""" if not isinstance(produced, StreamingResponse): # The status line is already on the wire, so a non-streaming body, an error @@ -3303,7 +3321,7 @@ class ProxyBaseLLMRequestProcessing: contents: list[object] | None = None, # Add contents parameter skip_pre_call_logic: bool = False, first_chunk_framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING, - ) -> Any: + ) -> object: """ Common request processing logic for both chat completions and responses API endpoints """ @@ -3571,20 +3589,21 @@ class ProxyBaseLLMRequestProcessing: ) # Non-streaming response - fall through to normal response handling elif select_data_generator: - selected_data_generator = select_data_generator( + produced_generator: Final[AsyncGenerator[str, None]] = select_data_generator( response=response, user_api_key_dict=user_api_key_dict, request_data=self.data, request=request, ) - if route_type in ("aresponses", "aget_responses"): - selected_data_generator = ( - ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership( - original_stream_response=response, - wrapped_generator=selected_data_generator, - user_api_key_dict=user_api_key_dict, - ) + selected_data_generator = ( + ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership( + original_stream_response=response, + wrapped_generator=produced_generator, + user_api_key_dict=user_api_key_dict, ) + if route_type in ("aresponses", "aget_responses") + else produced_generator + ) if selected_data_generator is not None: return await create_response( generator=selected_data_generator, @@ -3826,9 +3845,9 @@ class ProxyBaseLLMRequestProcessing: @staticmethod async def _wrap_responses_stream_for_container_ownership( original_stream_response: object, - wrapped_generator: Any, + wrapped_generator: AsyncIterator[_ChunkT], user_api_key_dict: UserAPIKeyAuth, - ): + ) -> AsyncGenerator[_ChunkT, None]: """Forward SSE chunks and record container ownership before the terminal chunk goes out. Streaming ``/v1/responses`` short-circuits out of diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 0aa8d2291ef..82120023b21 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -93,6 +93,10 @@ class _CallbackNotDispatchable: _CallbackLoadError: TypeAlias = _CallbackResolvedToClass | _CallbackNotDispatchable +class _PresidioOptions(TypedDict, total=False, extra_items=object): + pass + + def _classify_loaded_callback(entry: str, loaded: object) -> CustomLogger | Callable[..., object] | _CallbackLoadError: """ Decide whether what a ``litellm_settings.callbacks`` dotted path resolved to can be dispatched. @@ -193,15 +197,15 @@ def initialize_callbacks_on_proxy( if presidio_logging_only is not None: presidio_logging_only = bool(presidio_logging_only) # validate boolean given - _presidio_params = {} + _presidio_params: Mapping[str, object] = {} if "presidio" in callback_specific_params and isinstance(callback_specific_params["presidio"], dict): _presidio_params = callback_specific_params["presidio"] - params: dict[str, Any] = { + params: Mapping[str, object] = { "logging_only": presidio_logging_only, **_presidio_params, } - pii_masking_object = OPTIONAL_PresidioPIIMasking(**params) + pii_masking_object = OPTIONAL_PresidioPIIMasking(**_PresidioOptions(**params)) imported_list.append(pii_masking_object) elif isinstance(callback, str) and callback == "llamaguard_moderations": try: diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index 644a35ef4ef..5b7ce5c8f3e 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -3,7 +3,7 @@ import ctypes import os import re from collections.abc import Mapping -from typing import Final, Literal, cast +from typing import Final, Literal, TypeVar, cast, overload from pydantic import TypeAdapter, ValidationError @@ -201,7 +201,18 @@ def decrypt_bearer_token(token: str, prefix: str) -> str | None: return None -def encrypt_value_helper(value: str, new_encryption_key: str | None = None): +_Passthrough: Final = TypeVar("_Passthrough") + + +@overload +def encrypt_value_helper(value: str, new_encryption_key: str | None = None) -> str: ... + + +@overload +def encrypt_value_helper(value: _Passthrough, new_encryption_key: str | None = None) -> _Passthrough: ... + + +def encrypt_value_helper(value: object, new_encryption_key: str | None = None) -> object: signing_key: Final = new_encryption_key or get_salt_key() try: diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 9e3071f020e..ae91cbb1f2a 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -1200,7 +1200,7 @@ class ResetBudgetJob: if updated_keys: await self._write_key_reset_updates(updated_keys=updated_keys) for k in updated_keys: - token = getattr(k.row, "token", None) + token = k.row.token if token: await self._invalidate_spend_counter(key_counter(str(token))) @@ -1317,7 +1317,7 @@ class ResetBudgetJob: if updated_users: await self._write_user_reset_updates(updated_users=updated_users) for u in updated_users: - user_id = getattr(u.row, "user_id", None) + user_id = u.row.user_id if user_id: await self._invalidate_spend_counter(user_counter(str(user_id))) if user_id == LITELLM_PROXY_BUDGET_NAME: @@ -1438,7 +1438,7 @@ class ResetBudgetJob: if updated_teams: await self._write_team_reset_updates(updated_teams=updated_teams) for t in updated_teams: - team_id = getattr(t.row, "team_id", None) + team_id = t.row.team_id if team_id: await self._invalidate_spend_counter(team_counter(str(team_id))) diff --git a/litellm/proxy/db/budget_window_spend_writer.py b/litellm/proxy/db/budget_window_spend_writer.py index d2b69f24397..ccabd9dcde6 100644 --- a/litellm/proxy/db/budget_window_spend_writer.py +++ b/litellm/proxy/db/budget_window_spend_writer.py @@ -18,6 +18,7 @@ from collections.abc import Mapping, Sequence from contextlib import nullcontext from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from types import TracebackType from typing import TYPE_CHECKING, Final, Protocol from pydantic import TypeAdapter @@ -149,7 +150,13 @@ class _UpsertBatch(Protocol): class _UpsertBatchManager(Protocol): async def __aenter__(self) -> _UpsertBatch: ... - async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> bool | None: ... class _UpsertTransaction(Protocol): @@ -161,7 +168,13 @@ class _UpsertTransaction(Protocol): class _UpsertTransactionManager(Protocol): async def __aenter__(self) -> _UpsertTransaction: ... - async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> bool | None: ... def _seed_totals_query(prisma_client: "PrismaClient") -> _SeedTotalsQuery: diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 12267120965..5a9a52447b7 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -18,7 +18,7 @@ from contextlib import asynccontextmanager from contextvars import ContextVar from datetime import datetime, timedelta, timezone from functools import reduce -from types import MappingProxyType +from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, overload from urllib.parse import quote, unquote @@ -249,21 +249,43 @@ class SpendUpsertTable(Protocol): class _SpendBatch(Protocol): - litellm_usertable: BatchTable - litellm_endusertable: SpendUpsertTable - litellm_dailymodelusage: SpendUpsertTable + @property + def litellm_usertable(self) -> BatchTable: ... + + @property + def litellm_endusertable(self) -> SpendUpsertTable: ... + + @property + def litellm_dailymodelusage(self) -> SpendUpsertTable: ... def execute_raw(self, query: LiteralString, *args: object) -> None: ... - litellm_verificationtoken: BatchTable - litellm_teamtable: BatchTable - litellm_teammembership: BatchTable - litellm_organizationtable: BatchTable - litellm_organizationmembership: BatchTable - litellm_projecttable: BatchTable - litellm_tagtable: BatchTable - litellm_agentstable: BatchTable - litellm_modelaccessgroupbudgettable: BatchTable + @property + def litellm_verificationtoken(self) -> BatchTable: ... + + @property + def litellm_teamtable(self) -> BatchTable: ... + + @property + def litellm_teammembership(self) -> BatchTable: ... + + @property + def litellm_organizationtable(self) -> BatchTable: ... + + @property + def litellm_organizationmembership(self) -> BatchTable: ... + + @property + def litellm_projecttable(self) -> BatchTable: ... + + @property + def litellm_tagtable(self) -> BatchTable: ... + + @property + def litellm_agentstable(self) -> BatchTable: ... + + @property + def litellm_modelaccessgroupbudgettable(self) -> BatchTable: ... _EntitySpendTable: TypeAlias = Literal[ @@ -288,7 +310,13 @@ def _entity_spend_table(batcher: _SpendBatch, table_accessor: _EntitySpendTable) class SpendBatchManager(Protocol): async def __aenter__(self) -> _SpendBatch: ... - async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> bool | None: ... class _SpendTransaction(Protocol): @@ -300,7 +328,13 @@ class _SpendTransaction(Protocol): class _SpendTransactionManager(Protocol): async def __aenter__(self) -> _SpendTransaction: ... - async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> bool | None: ... _DailySpendTransactionT = TypeVar("_DailySpendTransactionT", bound=BaseDailySpendTransaction) diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index c2d58512fea..33d973cccfb 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -1,11 +1,12 @@ import re -from collections.abc import Awaitable, Callable, Iterator +from collections.abc import Awaitable, Callable, Iterator, Mapping from http import HTTPStatus from typing import Final, Protocol, TypeVar from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.proxy._types import ( DB_CONNECTION_ERROR_TYPES, DB_RETRY_SAFE_ERROR_TYPES, @@ -93,6 +94,28 @@ def _exception_types(*candidates: object) -> tuple[type[BaseException], ...]: return tuple(c for c in candidates if isinstance(c, type) and issubclass(c, BaseException)) +class _CodedError(Protocol): + @property + def code(self) -> object: ... + + +def _error_code(error: _CodedError) -> object: + return error.code + + +class _FrameGlobals(Protocol): + @property + def f_globals(self) -> Mapping[str, object]: ... + + +class _PrefixMatchable(Protocol): + def startswith(self, prefix: str, /) -> bool: ... + + +def _frame_module_name(frame: _FrameGlobals) -> _PrefixMatchable: + return require_attributes(frame.f_globals.get("__name__", ""), _PrefixMatchable, "startswith") + + def _prisma_error_code(e: Exception) -> str | None: import prisma.engine.errors @@ -317,7 +340,7 @@ class PrismaDBExceptionHandler: if not isinstance(e, _exception_types(prisma.errors.PrismaError)): return False - if isinstance(e, prisma.errors.DataError) and e.code == _PRISMA_POOL_TIMEOUT_CODE: + if isinstance(e, prisma.errors.DataError) and _error_code(e) == _PRISMA_POOL_TIMEOUT_CODE: return True if PrismaDBExceptionHandler.postgres_sqlstate(e) == "53300": return True @@ -433,7 +456,7 @@ class PrismaDBExceptionHandler: return False tb = e.__traceback__ if hasattr(e, "__traceback__") else None while tb is not None: - if tb.tb_frame.f_globals.get("__name__", "").startswith("prisma.engine"): + if _frame_module_name(tb.tb_frame).startswith("prisma.engine"): return True tb = tb.tb_next return False diff --git a/litellm/proxy/db/model_usage_rollup.py b/litellm/proxy/db/model_usage_rollup.py index f8cdabe9296..1a5f15fe294 100644 --- a/litellm/proxy/db/model_usage_rollup.py +++ b/litellm/proxy/db/model_usage_rollup.py @@ -6,6 +6,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime from itertools import groupby +from types import TracebackType from typing import TYPE_CHECKING, Final, Protocol from pydantic import TypeAdapter, ValidationError @@ -42,7 +43,13 @@ class ModelUsageBatch(Protocol): class ModelUsageBatchManager(Protocol): async def __aenter__(self) -> ModelUsageBatch: ... - async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> bool | None: ... @dataclass(frozen=True, slots=True) diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index a7f18769461..e0be2e1e5ef 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -35,12 +35,170 @@ from litellm.proxy.db.token_auth import ( from litellm.secret_managers.main import str_to_bool if TYPE_CHECKING: + from collections.abc import Sequence + from types import TracebackType + from prisma import Prisma from prisma import models as prisma_models from prisma.types import HttpConfig + from typing_extensions import Self from litellm.repositories.prisma_protocols import TableActions + _DEFAULT_TX_MAX_WAIT: Final = timedelta(seconds=2) + _DEFAULT_TX_TIMEOUT: Final = timedelta(seconds=5) + + class PrismaBatchActions(Protocol): + def create(self, data: Mapping[str, object], include: Mapping[str, object] | None = None) -> None: ... + + def create_many(self, data: Sequence[Mapping[str, object]], *, skip_duplicates: bool | None = None) -> None: ... + + def delete(self, where: Mapping[str, object], include: Mapping[str, object] | None = None) -> None: ... + + def update( + self, + data: Mapping[str, object], + where: Mapping[str, object], + include: Mapping[str, object] | None = None, + ) -> None: ... + + def upsert( + self, + where: Mapping[str, object], + data: Mapping[str, object], + include: Mapping[str, object] | None = None, + ) -> None: ... + + def update_many(self, data: Mapping[str, object], where: Mapping[str, object]) -> None: ... + + def delete_many(self, where: Mapping[str, object] | None = None) -> None: ... + + class PrismaBatchSession(Protocol): + @property + def litellm_agentstable(self) -> PrismaBatchActions: ... + + @property + def litellm_budgettable(self) -> PrismaBatchActions: ... + + @property + def litellm_dailymodelusage(self) -> PrismaBatchActions: ... + + @property + def litellm_dailyteamspend(self) -> PrismaBatchActions: ... + + @property + def litellm_dailytoolspend(self) -> PrismaBatchActions: ... + + @property + def litellm_endusertable(self) -> PrismaBatchActions: ... + + @property + def litellm_modelaccessgroupbudgettable(self) -> PrismaBatchActions: ... + + @property + def litellm_organizationmembership(self) -> PrismaBatchActions: ... + + @property + def litellm_organizationtable(self) -> PrismaBatchActions: ... + + @property + def litellm_projecttable(self) -> PrismaBatchActions: ... + + @property + def litellm_proxymodeltable(self) -> PrismaBatchActions: ... + + @property + def litellm_tagtable(self) -> PrismaBatchActions: ... + + @property + def litellm_teammembership(self) -> PrismaBatchActions: ... + + @property + def litellm_teamtable(self) -> PrismaBatchActions: ... + + @property + def litellm_usertable(self) -> PrismaBatchActions: ... + + @property + def litellm_verificationtoken(self) -> PrismaBatchActions: ... + + def execute_raw(self, query: str, *args: object) -> None: ... + + async def commit(self) -> None: ... + + async def __aenter__(self) -> Self: ... + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + exc_tb: TracebackType | None, + /, + ) -> None: ... + + class PrismaTransaction(Protocol): + @property + def litellm_accessgrouptable(self) -> TableActions[prisma_models.LiteLLM_AccessGroupTable]: ... + + @property + def litellm_agentstable(self) -> TableActions[prisma_models.LiteLLM_AgentsTable]: ... + + @property + def litellm_budgettable(self) -> TableActions[prisma_models.LiteLLM_BudgetTable]: ... + + @property + def litellm_invitationlink(self) -> TableActions[prisma_models.LiteLLM_InvitationLink]: ... + + @property + def litellm_mcpservertable(self) -> TableActions[prisma_models.LiteLLM_MCPServerTable]: ... + + @property + def litellm_mcpuserenvvars(self) -> TableActions[prisma_models.LiteLLM_MCPUserEnvVars]: ... + + @property + def litellm_objectpermissiontable(self) -> TableActions[prisma_models.LiteLLM_ObjectPermissionTable]: ... + + @property + def litellm_organizationtable(self) -> TableActions[prisma_models.LiteLLM_OrganizationTable]: ... + + @property + def litellm_proxymodeltable(self) -> TableActions[prisma_models.LiteLLM_ProxyModelTable]: ... + + @property + def litellm_teamtable(self) -> TableActions[prisma_models.LiteLLM_TeamTable]: ... + + @property + def litellm_usertable(self) -> TableActions[prisma_models.LiteLLM_UserTable]: ... + + @property + def litellm_verificationtoken(self) -> TableActions[prisma_models.LiteLLM_VerificationToken]: ... + + @property + def litellm_workflowevent(self) -> TableActions[prisma_models.LiteLLM_WorkflowEvent]: ... + + @property + def litellm_workflowrun(self) -> TableActions[prisma_models.LiteLLM_WorkflowRun]: ... + + def batch_(self) -> PrismaBatchSession: ... + + async def execute_raw(self, query: str, *args: object) -> int: ... + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... + + async def query_first(self, query: str, *args: object) -> Mapping[str, object] | None: ... + + class PrismaTransactionManager(Protocol): + async def __aenter__(self) -> PrismaTransaction: ... + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + exc_tb: TracebackType | None, + /, + ) -> None: ... + + _ENVIRONMENT_VARIABLES_REVISION_MIGRATION: Final = "20261015120000_add_environment_variables_revision_trigger" __all__ = ( @@ -925,6 +1083,12 @@ class PrismaWrapper: async def query_first(self, query: str, *args: object) -> Mapping[str, object] | None: ... + def tx( + self, *, max_wait: int | timedelta = _DEFAULT_TX_MAX_WAIT, timeout: int | timedelta = _DEFAULT_TX_TIMEOUT + ) -> PrismaTransactionManager: ... + + def batch_(self) -> PrismaBatchSession: ... + @property def litellm_deprecatedverificationtoken( self, diff --git a/litellm/proxy/db/routing_prisma_wrapper.py b/litellm/proxy/db/routing_prisma_wrapper.py index c71e2206f45..e6a436a9b1e 100644 --- a/litellm/proxy/db/routing_prisma_wrapper.py +++ b/litellm/proxy/db/routing_prisma_wrapper.py @@ -19,8 +19,12 @@ if TYPE_CHECKING: from prisma import models as prisma_models from prisma.types import HttpConfig + from litellm.proxy.db.prisma_client import PrismaBatchSession, PrismaTransactionManager from litellm.repositories.prisma_protocols import TableActions + _DEFAULT_TX_MAX_WAIT: Final = timedelta(seconds=2) + _DEFAULT_TX_TIMEOUT: Final = timedelta(seconds=5) + # Per-model action methods that read from the database. These are routed to # the read replica when one is configured. _MODEL_READ_METHODS: Final = frozenset( @@ -311,6 +315,12 @@ class RoutingPrismaWrapper: async def query_first(self, query: str, *args: object) -> Mapping[str, object] | None: ... + def tx( + self, *, max_wait: int | timedelta = _DEFAULT_TX_MAX_WAIT, timeout: int | timedelta = _DEFAULT_TX_TIMEOUT + ) -> PrismaTransactionManager: ... + + def batch_(self) -> PrismaBatchSession: ... + @property def litellm_deprecatedverificationtoken( self, diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index 92181d546eb..e2cebc691a5 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -19,6 +19,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from itertools import chain, groupby +from types import TracebackType from typing import TYPE_CHECKING, Final, Literal, Protocol from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError @@ -175,7 +176,8 @@ class _ToolRollupTable(Protocol): class _ToolRollupBatch(Protocol): - litellm_dailytoolspend: _ToolRollupTable + @property + def litellm_dailytoolspend(self) -> _ToolRollupTable: ... def execute_raw(self, query: LiteralString, *args: object) -> None: ... @@ -183,7 +185,13 @@ class _ToolRollupBatch(Protocol): class _ToolRollupBatchManager(Protocol): async def __aenter__(self) -> _ToolRollupBatch: ... - async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> bool | None: ... def _tool_rollup_batch(prisma_client: PrismaClient) -> _ToolRollupBatchManager: diff --git a/litellm/proxy/discovery_endpoints/agent_skills_archive.py b/litellm/proxy/discovery_endpoints/agent_skills_archive.py index e45ca7e10f2..1665cb4caaf 100644 --- a/litellm/proxy/discovery_endpoints/agent_skills_archive.py +++ b/litellm/proxy/discovery_endpoints/agent_skills_archive.py @@ -17,7 +17,7 @@ import zlib from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import ClassVar, Final +from typing import ClassVar, Final, Protocol import yaml from pydantic import ConfigDict, TypeAdapter, ValidationError @@ -64,6 +64,15 @@ _YAML_PAIRS: Final = TypeAdapter( ) +class _YamlValueNode(Protocol): + @property + def value(self) -> object: ... + + +def _yaml_node_value(node: _YamlValueNode) -> object: + return node.value + + class _AgentSkillFrontmatterLoader(yaml.BaseLoader): yaml_implicit_resolvers: ClassVar[Mapping[str | None, Sequence[tuple[str, re.Pattern[str]]]]] = { # pyright: ignore[reportIncompatibleVariableOverride] # PyYAML resolver tables are class state None: list(_AGENT_SKILL_YAML_SCALARS.items()) @@ -259,7 +268,7 @@ def agent_skill_frontmatter(manifest: bytes) -> Mapping[str, str]: return MappingProxyType({}) if not isinstance(node, yaml.MappingNode) or not _valid_frontmatter_node(node): return MappingProxyType({}) - pairs: Final = _YAML_PAIRS.validate_python(node.value) + pairs: Final = _YAML_PAIRS.validate_python(_yaml_node_value(node)) fields: Final = { key_text: value_text for key, value in pairs @@ -273,7 +282,7 @@ def agent_skill_frontmatter(manifest: bytes) -> Mapping[str, str]: def _frontmatter_string(node: yaml.Node) -> str | None: if not isinstance(node, yaml.ScalarNode) or node.tag != "tag:yaml.org,2002:str": return None - value: Final = _DECODED_YAML.validate_python(node.value) + value: Final = _yaml_node_value(node) return value if isinstance(value, str) else None @@ -282,18 +291,18 @@ def _valid_frontmatter_node(node: yaml.Node, depth: int = 0) -> bool: return False if isinstance(node, yaml.ScalarNode): pattern: Final = _AGENT_SKILL_YAML_SCALARS.get(node.tag) - scalar: Final = _DECODED_YAML.validate_python(node.value) + scalar: Final = _yaml_node_value(node) return isinstance(scalar, str) and ( node.tag == "tag:yaml.org,2002:str" or (pattern is not None and pattern.fullmatch(scalar) is not None) ) if isinstance(node, yaml.SequenceNode): - values: Final = _YAML_NODES.validate_python(node.value) + values: Final = _YAML_NODES.validate_python(_yaml_node_value(node)) return node.tag == "tag:yaml.org,2002:seq" and all( _valid_frontmatter_node(value, depth + 1) for value in values ) if not isinstance(node, yaml.MappingNode) or node.tag != "tag:yaml.org,2002:map": return False - pairs: Final = _YAML_PAIRS.validate_python(node.value) + pairs: Final = _YAML_PAIRS.validate_python(_yaml_node_value(node)) keys: Final = tuple(_frontmatter_string(key) for key, _ in pairs) return ( None not in keys diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index 030f8d7a46a..850a8508428 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -6,12 +6,14 @@ ########################################################################## import asyncio -from typing import Final, cast +from typing import Final from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -25,6 +27,7 @@ from litellm.proxy.utils import handle_exception_on_proxy from litellm.types.utils import LiteLLMFineTuningJob router: Final = APIRouter() +_DECODED_JSON: Final = TypeAdapter(object) from litellm.types.llms.openai import LiteLLMFineTuningJobCreate @@ -50,7 +53,7 @@ def set_fine_tuning_config(config): # Function to search for specific custom_llm_provider and return its configuration def get_fine_tuning_provider_config( - custom_llm_provider: str, + custom_llm_provider: object, ): global fine_tuning_config if fine_tuning_config is None: @@ -61,6 +64,13 @@ def get_fine_tuning_provider_config( return None +async def _decoded_request_body(request: Request) -> object: + try: + return _DECODED_JSON.validate_python(await request.json()) + except Exception: + return {} + + @router.post( "/v1/fine_tuning/jobs", dependencies=[Depends(user_api_key_auth)], @@ -163,7 +173,7 @@ async def create_fine_tuning_job( detail={"error": "LLM Router not initialized. Ensure models added to proxy."}, ) - response = cast(LiteLLMFineTuningJob, await llm_router.acreate_fine_tuning_job(**data)) + response = await llm_router.acreate_fine_tuning_job(**data) response.training_file = training_file response.validation_file = fine_tuning_request.validation_file set_hidden_param(response, "unified_file_id", unified_file_id) @@ -285,12 +295,8 @@ async def retrieve_fine_tuning_job( route_type=CallTypes.aretrieve_fine_tuning_job.value, ) - try: - request_body = await request.json() - except Exception: - request_body = {} - - custom_llm_provider = request_body.get("custom_llm_provider", None) or custom_llm_provider + request_body: Final = decoded_json_object(await _decoded_request_body(request)) + requested_provider: Final = request_body.get("custom_llm_provider", None) or custom_llm_provider ## CHECK IF MANAGED FILE ID unified_finetuning_job_id: str | Literal[False] = False @@ -305,16 +311,13 @@ async def retrieve_fine_tuning_job( status_code=500, detail={"error": "LLM Router not initialized. Ensure models added to proxy."}, ) - response = cast( - LiteLLMFineTuningJob, - await llm_router.aretrieve_fine_tuning_job( - **data, - ), + response = await llm_router.aretrieve_fine_tuning_job( + **data, ) set_hidden_param(response, "unified_finetuning_job_id", unified_finetuning_job_id) - elif custom_llm_provider: + elif requested_provider: # get configs for custom_llm_provider - llm_provider_config: Final = get_fine_tuning_provider_config(custom_llm_provider=custom_llm_provider) + llm_provider_config: Final = get_fine_tuning_provider_config(custom_llm_provider=requested_provider) if llm_provider_config is not None: data.update(llm_provider_config) @@ -560,11 +563,7 @@ async def cancel_fine_tuning_job( route_type=CallTypes.acancel_fine_tuning_job.value, ) - try: - request_body = await request.json() - except Exception: - request_body = {} - + request_body: Final = decoded_json_object(await _decoded_request_body(request)) custom_llm_provider: Final = request_body.get("custom_llm_provider", None) ## CHECK IF MANAGED FILE ID @@ -580,11 +579,8 @@ async def cancel_fine_tuning_job( status_code=500, detail={"error": "LLM Router not initialized. Ensure models added to proxy."}, ) - response = cast( - LiteLLMFineTuningJob, - await llm_router.acancel_fine_tuning_job( - **data, - ), + response = await llm_router.acancel_fine_tuning_job( + **data, ) set_hidden_param(response, "unified_finetuning_job_id", unified_finetuning_job_id) else: diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index c773e88c769..03f83616fea 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -11,7 +11,7 @@ import os import ssl from collections.abc import AsyncGenerator, AsyncIterable, Iterator, Mapping, Sequence from ssl import SSLContext -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias from fastapi import HTTPException from pydantic import BaseModel @@ -23,7 +23,7 @@ from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm._version import version as litellm_version from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.litellm_core_utils.core_helpers import attribute_items, iter_items +from litellm.litellm_core_utils.core_helpers import attribute_items, iter_items, require_attributes, subscript from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field from litellm.llms.anthropic.common_utils import ( anthropic_server_tool_calls, @@ -60,6 +60,19 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_ToolCallKind: TypeAlias = Literal["server_tool_call", "tool_call"] +_OutputTarget: TypeAlias = tuple[Literal["content"], None] | tuple[_ToolCallKind, int] + + +class _ToolCallFunction(Protocol): + arguments: str + + +class _FunctionToolCall(Protocol): + @property + def function(self) -> _ToolCallFunction: ... + + class CatoNetworksGuardrailMissingSecrets(Exception): pass @@ -522,14 +535,16 @@ class CatoNetworksGuardrail(CustomGuardrail): ) @staticmethod - def _output_fragments(message: Message) -> Sequence[tuple[tuple[str, int | None], str]]: + def _output_fragments(message: Message) -> Sequence[tuple[_OutputTarget, str]]: """Assistant text the proxy returns to the caller: ``content`` plus every tool call's ``function.arguments`` string, Claude's server tool calls first, each tagged with where a redaction must be written back. ``content`` is only included when present so a tool-call-only choice keeps its ``None`` content (the text-vs-tool-call signal downstream consumers rely on) while its arguments are still inspected.""" - content: Final = () if message.content is None else ((("content", None), message.content),) + content: Final[tuple[tuple[_OutputTarget, str], ...]] = ( + () if message.content is None else ((("content", None), message.content),) + ) return ( *content, *CatoNetworksGuardrail._tool_call_fragments("server_tool_call", anthropic_server_tool_calls(message)), @@ -537,7 +552,9 @@ class CatoNetworksGuardrail(CustomGuardrail): ) @staticmethod - def _tool_call_fragments(kind: str, tool_calls: Sequence[object]) -> tuple[tuple[tuple[str, int | None], str], ...]: + def _tool_call_fragments( + kind: _ToolCallKind, tool_calls: Sequence[object] + ) -> tuple[tuple[tuple[_ToolCallKind, int], str], ...]: return tuple( ((kind, idx), arguments) for idx, arguments in enumerate( @@ -547,14 +564,16 @@ class CatoNetworksGuardrail(CustomGuardrail): ) @staticmethod - def _apply_output_fragment(message: Any, target: tuple[str, int | None], redacted: str) -> None: + def _apply_output_fragment(message: Message, target: _OutputTarget, redacted: str) -> None: match target: case ("content", _): message.content = redacted case ("server_tool_call", int() as idx): write_anthropic_server_tool_call_arguments(message, idx, redacted) case (_, idx): - message.tool_calls[idx].function.arguments = redacted + require_attributes( + subscript(message.tool_calls, idx), _FunctionToolCall, "function" + ).function.arguments = redacted @staticmethod def _responses_output_field(item: object, key: str) -> str | Sequence[object] | None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py index 55f7abd40a8..51b849751c8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py @@ -13,7 +13,7 @@ from typing import Final, Literal from urllib.parse import urlparse import httpx -from pydantic import JsonValue +from pydantic import JsonValue, TypeAdapter from typing_extensions import ReadOnly, TypedDict import litellm @@ -22,6 +22,8 @@ from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, vali from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.types.llms.custom_http import httpxSpecialProvider +_DECODED_JSON: Final = TypeAdapter(object) + # ============================================================================= # Result Types - Used by Starlark code to return guardrail decisions # ============================================================================= @@ -200,7 +202,7 @@ class JsonSchemaNode(TypedDict, total=False): properties: ReadOnly[Mapping[str, "JsonSchemaNode"]] -def json_parse(text: str) -> JsonValue: +def json_parse(text: str) -> object: """ Parse a JSON string into a Python object. @@ -211,7 +213,7 @@ def json_parse(text: str) -> JsonValue: Parsed Python object, or None if parsing fails """ try: - return json.loads(text) + return _DECODED_JSON.validate_python(json.loads(text)) except (json.JSONDecodeError, TypeError) as e: verbose_proxy_logger.debug("Starlark json_parse error: %s", e) return None diff --git a/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py b/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py index 1b9bfc725db..8c698934796 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py @@ -8,7 +8,7 @@ import os from collections.abc import AsyncGenerator, AsyncIterable from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Literal, Optional +from typing import TYPE_CHECKING, Final, Literal, Optional import httpx @@ -186,7 +186,6 @@ class EnkryptAIGuardrails(CustomGuardrail): details: Final = response.get("details", {}) detected_attacks: Final[list[str]] = [] - attack_details: Final[dict[str, Any]] = {} for key, value in summary.items(): # Check if attack is detected @@ -195,13 +194,14 @@ class EnkryptAIGuardrails(CustomGuardrail): if key == "toxicity": if isinstance(value, list) and len(value) > 0: detected_attacks.append(key) - attack_details[key] = details.get(key, {}) else: if value == 1: detected_attacks.append(key) - attack_details[key] = details.get(key, {}) - return {"attacks_detected": detected_attacks, "attack_details": attack_details} + return { + "attacks_detected": detected_attacks, + "attack_details": {key: details.get(key, {}) for key in detected_attacks}, + } def _determine_guardrail_status(self, response_json: EnkryptAIResponse) -> GuardrailStatus: """ diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 7e052796545..85136c46a67 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -21,7 +21,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, @@ -46,6 +46,7 @@ _CHAT_ROW: Final[TypeAdapter[AllMessageValues]] = TypeAdapter(AllMessageValues) _SENT_ROW: Final = TypeAdapter(Mapping[str, object]) _SENT_ROWS: Final = TypeAdapter(Sequence[Mapping[str, object]]) _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_DECODED_JSON: Final = TypeAdapter(object) # Headers whose values are forwarded as-is (case-insensitive). Glob patterns supported (e.g. x-stainless-*, x-litellm*). _HEADER_VALUE_ALLOWLIST: Final = frozenset( @@ -242,6 +243,16 @@ def _structured_rows_to_write_back( ) +def _guardrail_response_from_json(response_json: object) -> GenericGuardrailAPIResponse: + if isinstance(response_json, dict): + return GenericGuardrailAPIResponse.from_dict(response_json) + raise AttributeError( + f"{type(response_json).__name__!r} object has no attribute 'get'", + name="get", + obj=response_json, + ) + + class GenericGuardrailAPI(CustomGuardrail): """ Generic Guardrail API integration for LiteLLM. @@ -396,7 +407,7 @@ class GenericGuardrailAPI(CustomGuardrail): input_type: Literal["request", "response"], logging_obj: Optional["LiteLLMLoggingObj"], error: Exception, - http_status_code: int | None = None, + http_status_code: object = None, ) -> GenericGuardrailAPIInputs: status_suffix: Final = f" http_status_code={http_status_code}" if http_status_code else "" error_type: Final = "HTTPStatusError" if isinstance(error, httpx.HTTPStatusError) else type(error).__name__ @@ -482,7 +493,7 @@ class GenericGuardrailAPI(CustomGuardrail): ) -> GenericGuardrailAPIInputs: unreachable_fail_open: Final = is_unreachable and self.unreachable_fallback == "fail_open" if unreachable_fail_open or not self.fail_on_error: - http_status_code: Final = getattr(getattr(error, "response", None), "status_code", None) + http_status_code: Final = attribute_or_none(attribute_or_none(error, "response"), "status_code") return self._fail_open_passthrough( inputs=inputs, input_type=input_type, @@ -608,11 +619,11 @@ class GenericGuardrailAPI(CustomGuardrail): ) response.raise_for_status() - response_json: Final = response.json() + response_json: Final = _DECODED_JSON.validate_python(response.json()) verbose_proxy_logger.debug("Generic Guardrail API response: %s", response_json) - guardrail_response: Final = GenericGuardrailAPIResponse.from_dict(response_json) + guardrail_response: Final = _guardrail_response_from_json(response_json) # Handle the response if guardrail_response.action == "BLOCKED": diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py index 9463023854e..85bf39c481b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py @@ -135,7 +135,7 @@ _mcp_jwt_signer_instance: Optional["MCPJWTSigner"] = None _MCP_JWT_CALL_TYPES: Final = frozenset({"call_mcp_tool", "list_mcp_tools"}) # Simple in-memory JWKS cache: keyed by JWKS URI → (keys_list, fetched_at). -_jwks_cache: Final[dict[str, tuple[Sequence[Mapping[str, object]], float]]] = {} +_jwks_cache: Final[dict[str, tuple[object, float]]] = {} _JWKS_CACHE_TTL: Final = 3600 # 1 hour @@ -181,7 +181,7 @@ def _compute_kid(public_key: RSAPublicKey) -> str: return hashlib.sha256(der_bytes).hexdigest()[:16] -async def _fetch_jwks(jwks_uri: str, timeout: float | httpx.Timeout | None = None) -> Sequence[Mapping[str, object]]: +async def _fetch_jwks(jwks_uri: str, timeout: float | httpx.Timeout | None = None) -> object: """ Fetch and cache a JWKS from the given URI. @@ -202,7 +202,7 @@ async def _fetch_jwks(jwks_uri: str, timeout: float | httpx.Timeout | None = Non client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) resp: Final = await client.get(jwks_uri, headers={"Accept": "application/json"}, timeout=timeout) resp.raise_for_status() - jwks_body: Final[Mapping[str, Sequence[Mapping[str, object]]]] = resp.json() + jwks_body: Final = decoded_json_object(_DECODED_JSON.validate_python(resp.json())) fetched_keys: Final = jwks_body.get("keys", []) _jwks_cache[jwks_uri] = (fetched_keys, now) return fetched_keys diff --git a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py index 1790f335548..16a820e1c0b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py +++ b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py @@ -9,7 +9,7 @@ import httpx from typing_extensions import NotRequired, TypedDict from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.core_helpers import decoded_json_object, iter_items, json_contains +from litellm.litellm_core_utils.core_helpers import attribute_or_none, decoded_json_object, iter_items, json_contains from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) @@ -445,10 +445,10 @@ class PurviewGuardrailBase: tool_calls: Final = tool_calls_with_server_tool_calls(message) if tool_calls: for tc in tool_calls: - fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) + fn = tc.get("function") if isinstance(tc, dict) else attribute_or_none(tc, "function") if fn is None: continue - arguments = fn.get("arguments") if isinstance(fn, dict) else getattr(fn, "arguments", None) + arguments = fn.get("arguments") if isinstance(fn, dict) else attribute_or_none(fn, "arguments") if isinstance(arguments, str) and arguments.strip(): args.append(arguments) diff --git a/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py b/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py index 152515c50ed..1f7f93bb9cb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py @@ -1,6 +1,6 @@ # litellm/proxy/guardrails/guardrail_hooks/pangea.py import os -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sized from typing import TYPE_CHECKING, Final from fastapi import HTTPException @@ -12,7 +12,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) -from litellm.litellm_core_utils.core_helpers import decoded_json_object +from litellm.litellm_core_utils.core_helpers import decoded_json_object, iter_items, subscript from litellm.llms.anthropic.common_utils import ( message_dict_with_server_tool_calls, message_dict_with_server_tool_calls_split_out, @@ -40,6 +40,12 @@ if TYPE_CHECKING: _DECODED_JSON: Final = TypeAdapter(object) +def _json_len(value: object) -> int: + if isinstance(value, Sized): + return len(value) + raise TypeError(f"object of type '{type(value).__name__}' has no len()") + + def _choice_with_message(choice: object, rewrite: Callable[[object], object]) -> object: mapping: Final = validated_object_mapping(choice) if mapping is None or "message" not in mapping: @@ -59,9 +65,9 @@ class _TextCompletionRequest: return [{"role": "user", "content": self.body["prompt"]}] # This mutates the original dict, but we'll still return it anyways - def update_original_body(self, prompt_messages: list[dict]) -> dict[str, object]: - assert len(prompt_messages) == 1 - self.body["prompt"] = prompt_messages[0]["content"] + def update_original_body(self, prompt_messages: object) -> dict[str, object]: + assert _json_len(prompt_messages) == 1 + self.body["prompt"] = subscript(subscript(prompt_messages, 0), "content") return self.body @@ -117,7 +123,7 @@ class PangeaHandler(CustomGuardrail): self.api_base, ) - async def _call_pangea_ai_guard(self, api: str, payload: dict, hook_name: str) -> dict: + async def _call_pangea_ai_guard(self, api: str, payload: dict, hook_name: str) -> Mapping[str, object]: """ Makes the API call to the Pangea AI Guard endpoint. The function itself will raise an error in the case that a response @@ -153,8 +159,7 @@ class PangeaHandler(CustomGuardrail): ) response.raise_for_status() - result: Final = response.json() - decoded_result: Final = decoded_json_object(_DECODED_JSON.validate_python(result)) + decoded_result: Final = decoded_json_object(_DECODED_JSON.validate_python(response.json())) verdict: Final = decoded_json_object(decoded_result.get("result", {})) if verdict.get("blocked"): @@ -172,7 +177,7 @@ class PangeaHandler(CustomGuardrail): "Pangea Guardrail (%s): Request passed. Response: %s", hook_name, verdict.get("detectors") ) - return result + return decoded_result async def _async_pre_call_hook( self, @@ -200,14 +205,15 @@ class PangeaHandler(CustomGuardrail): ai_guard_response = await self._call_pangea_ai_guard("v1beta/guard", ai_guard_payload, "async_pre_call_hook") add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) - if not ai_guard_response.get("result", {}).get("transformed"): + pangea_result: Final = decoded_json_object(ai_guard_response.get("result", {})) + if not pangea_result.get("transformed"): return - output: Final = ai_guard_response.get("result", {}).get("output", {}) + output: Final = pangea_result.get("output", {}) if call_type == "text_completion" or call_type == "atext_completion": - data = transformer.update_original_body(output["messages"]) + data = transformer.update_original_body(subscript(output, "messages")) else: - data["messages"] = output["messages"] + data["messages"] = subscript(output, "messages") return data @log_guardrail_information @@ -290,12 +296,14 @@ class PangeaHandler(CustomGuardrail): ai_guard_response = await self._call_pangea_ai_guard("v1beta/guard", ai_guard_payload, "async_pre_call_hook") add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) - if not ai_guard_response.get("result", {}).get("transformed"): + pangea_result: Final = decoded_json_object(ai_guard_response.get("result", {})) + if not pangea_result.get("transformed"): return - output: Final = ai_guard_response.get("result", {}).get("output", {}) + output: Final = pangea_result.get("output", {}) response.choices = [ # pyright: ignore[reportAttributeAccessIssue] # Pangea returns untyped choice dicts - _choice_with_message(choice, message_dict_with_server_tool_calls_split_out) for choice in output["choices"] + _choice_with_message(choice, message_dict_with_server_tool_calls_split_out) + for choice in iter_items(subscript(output, "choices")) ] return response diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 73ccd82d37f..f4b153d86ba 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -28,7 +28,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) -from litellm.litellm_core_utils.core_helpers import decoded_json_object +from litellm.litellm_core_utils.core_helpers import decoded_json_object, json_contains from litellm.llms.anthropic.common_utils import ( anthropic_server_tool_call_params, tool_calls_with_server_tool_calls, @@ -145,6 +145,7 @@ class _ResponsesInputItem(LiteLLMBaseModel): _ResponsesInput: TypeAlias = str | tuple[_ResponsesInputItem, ...] | None _RESPONSES_INPUT: Final[TypeAdapter[_ResponsesInput]] = TypeAdapter(_ResponsesInput) _DECODED_ARGUMENTS: Final = TypeAdapter(object) +_DECODED_JSON: Final = TypeAdapter(object) _NO_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) @@ -394,7 +395,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): metadata: Mapping[str, object] | None = None, call_id: object = None, tool_event: _AirsToolEvent | None = None, - ) -> dict[str, object]: + ) -> Mapping[str, object]: """Call PANW Prisma AIRS API to scan content or a tool_event.""" if tool_event is None and not content.strip(): @@ -506,30 +507,32 @@ class PanwPrismaAirsHandler(CustomGuardrail): ) response.raise_for_status() - result: Final[dict[str, object]] = response.json() + result: Final = _DECODED_JSON.validate_python(response.json()) # Validate response format - if "action" not in result: + if not json_contains(result, "action"): verbose_proxy_logger.error("PANW Prisma AIRS: Invalid API response format: %s", result) return {"action": "block", "category": "api_error"} + body: Final = decoded_json_object(result) + # Check for profile-related errors from PANW API - if result.get("action") == "block" and "error" in result: - error_msg: Final = str(result.get("error", "")).lower() + if body.get("action") == "block" and "error" in body: + error_msg: Final = str(body.get("error", "")).lower() if "profile" in error_msg and ( "not found" in error_msg or "required" in error_msg or "invalid" in error_msg ): verbose_proxy_logger.error( "PANW Prisma AIRS: Profile configuration error. Ensure your API key has a linked profile in Strata Cloud Manager, or provide 'profile_name' or 'profile_id' in config/metadata. PANW API response: %s", - result, + body, ) verbose_proxy_logger.debug( "PANW Prisma AIRS: Scan result - Action: %s, Category: %s", - result.get("action"), - result.get("category", "unknown"), + body.get("action"), + body.get("category", "unknown"), ) - return result + return body except httpx.HTTPStatusError as e: status: Final = e.response.status_code @@ -839,7 +842,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): def _handle_api_error_with_logging( self, - scan_result: dict[str, object], + scan_result: Mapping[str, object], data: dict[str, object], start_time: datetime, event_type: GuardrailEventHooks, @@ -1363,7 +1366,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): assembled_model_response: ModelResponse, request_data: dict, start_time: datetime, - ) -> tuple[bool, ModelResponse, dict[str, object]]: + ) -> tuple[bool, ModelResponse, Mapping[str, object]]: """ Scan assembled streaming response and apply masking if needed. Returns (content_was_modified, response, scan_result). diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index f2697c0aa11..09f92f9c8ee 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -5,11 +5,11 @@ from dataclasses import dataclass from functools import partial, reduce from itertools import chain, groupby from types import MappingProxyType -from typing import Any, Final, Literal, TypeAlias, TypedDict, TypeVar +from typing import Final, Literal, TypeAlias, TypeVar from fastapi import HTTPException from pydantic import TypeAdapter -from typing_extensions import ReadOnly, Required +from typing_extensions import TypedDict from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache @@ -22,7 +22,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, keyword_arguments from litellm.litellm_core_utils.tool_fields import declared_function_name, qualified_member_name from litellm.llms.anthropic.common_utils import ( anthropic_server_tool_call_params, @@ -472,19 +472,13 @@ def _rewritten_tool_config(tool_config: object, removed_names: frozenset[str], f } -class _ToolPermissionRuleFields(TypedDict, total=False): - """The config-file shape a :class:`ToolPermissionRule` is built from.""" - - id: ReadOnly[Required[str]] - tool_name: ReadOnly[str | None] - tool_type: ReadOnly[str | None] - decision: ReadOnly[Required[Literal["allow", "deny"]]] - allowed_param_patterns: ReadOnly[dict[str, str] | None] +class _ToolPermissionRuleFields(TypedDict, total=False, extra_items=object): + pass -def _rule_from_fields(fields: _ToolPermissionRuleFields) -> ToolPermissionRule: +def _rule_from_fields(fields: object) -> ToolPermissionRule: """Validate one config-file rule entry into a :class:`ToolPermissionRule`.""" - return ToolPermissionRule(**fields) + return ToolPermissionRule.model_validate(_ToolPermissionRuleFields(**keyword_arguments(fields, ToolPermissionRule))) def _is_tool_use_block(block: object) -> bool: @@ -529,7 +523,7 @@ class ToolPermissionGuardrail(CustomGuardrail): self.default_action, ) - def _load_rules(self, rules: list[Any] | None) -> None: + def _load_rules(self, rules: Sequence[object] | None) -> None: """Parse ``rules`` and (re)build the compiled target/pattern lookups. ``self.rules`` plus ``_compiled_rule_targets`` / ``_compiled_rule_patterns`` @@ -948,10 +942,10 @@ class ToolPermissionGuardrail(CustomGuardrail): return is_allowed, None, message @staticmethod - def _get_mapping_value(item: object, key: str) -> Any: + def _get_mapping_value(item: object, key: str) -> object: if isinstance(item, dict): return item.get(key) - return getattr(item, key, None) + return attribute_or_none(item, key) @staticmethod def _legacy_function_call_id(choice_index: int) -> str: diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 2f2294db522..cafd79626ef 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -10,6 +10,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, TypeAlias, TypeVar, cast from pydantic import BaseModel, TypeAdapter, ValidationError +from typing_extensions import TypedDict import litellm from litellm import Router @@ -17,6 +18,7 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH, GUARDRAIL_ROTATION_ATTEMPTS from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.core_helpers import as_str_mapping, keyword_arguments, subscript from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, @@ -102,6 +104,10 @@ def _as_json_array(value: object) -> list[object] | None: return _JSON_ARRAY.validate_python(value) if isinstance(value, list) else None +class _CategoryThresholdFields(TypedDict, total=False, extra_items=object): + pass + + def _extra_param_items(litellm_params: LitellmParams) -> Iterable[tuple[str, object]]: return ( litellm_params.model_dump(exclude_none=True).items() @@ -775,7 +781,11 @@ class InMemoryGuardrailHandler: litellm_params = litellm_params_data if "category_thresholds" in litellm_params_data and litellm_params_data["category_thresholds"]: - lakera_category_thresholds: Final = LakeraCategoryThresholds(**litellm_params_data["category_thresholds"]) + lakera_category_thresholds: Final = LakeraCategoryThresholds( + **_CategoryThresholdFields( + **keyword_arguments(subscript(litellm_params_data, "category_thresholds"), LakeraCategoryThresholds) + ) + ) litellm_params.category_thresholds = lakera_category_thresholds if litellm_params.api_key and litellm_params.api_key.startswith("os.environ/"): @@ -871,7 +881,8 @@ class InMemoryGuardrailHandler: params: Final = guardrail.get("litellm_params") if not isinstance(params, (dict, LitellmParams)): return - litellm_params: Final = LitellmParams(**params) if isinstance(params, dict) else params + params_mapping: Final = as_str_mapping(params) + litellm_params: Final = LitellmParams(**params_mapping) if params_mapping is not None else params guardrail_name: Final = guardrail.get("guardrail_name", "Unknown") for custom_guardrail_callback in self._tracked_callbacks(guardrail_id): for error in ( diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index b4d343b3932..625e1e2df7e 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -26,6 +26,7 @@ from urllib.parse import parse_qs, urlsplit from fastapi import HTTPException from pydantic import Field, TypeAdapter, ValidationError +from typing_extensions import TypedDict import litellm from litellm._internal_context import with_service_target @@ -105,6 +106,10 @@ _DECODED_JSON: Final = TypeAdapter(object) _WINDOW_START_ADAPTER: Final[TypeAdapter[int | float | str | None]] = TypeAdapter(int | float | str | None) +class _FileContentFetchKwargs(TypedDict, total=False, extra_items=object): + pass + + def _batch_create_routing_model(data: Mapping[str, object]) -> str | None: body_model: Final = data.get("model") if isinstance(body_model, str) and body_model: @@ -537,7 +542,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): custom_llm_provider: str, data: dict, user_api_key_dict: UserAPIKeyAuth | None = None, - ) -> tuple[str, dict[str, Any]]: + ) -> tuple[str, dict[str, object]]: """ Map proxy-facing file IDs to provider file IDs and credentials. @@ -954,7 +959,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): file_content = await litellm.afile_content( file_id=provider_file_id, user_api_key_dict=user_api_key_dict, - **fetch_kwargs, + **_FileContentFetchKwargs(**fetch_kwargs), ) file_content_bytes: Final = getattr(file_content, "content", None) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index b1eb8f7d8ae..8abc91a72ed 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -54,7 +54,10 @@ from litellm.integrations.custom_guardrail import merged_guardrail_entries, with from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, attribute_or_none, + decoded_json_object, is_codex_user_agent, + iter_items, + keyword_arguments, normalize_drop_params, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor @@ -1251,6 +1254,10 @@ def convert_key_logging_metadata_to_callback( return team_callback_settings_obj +class _CallbackModelFields(TypedDict, total=False, extra_items=object): + pass + + def get_validated_callback_metadata(item: object, *, source: str) -> AddTeamCallback | None: if not isinstance(item, Mapping): raise TypeError(f"{source} metadata.logging must be a list of objects, got an item of {type(item).__name__}") @@ -1280,9 +1287,9 @@ class KeyAndTeamLoggingSettings: return None @staticmethod - def get_team_dynamic_logging_settings(user_api_key_dict: UserAPIKeyAuth): + def get_team_dynamic_logging_settings(user_api_key_dict: UserAPIKeyAuth) -> object: if user_api_key_dict.team_metadata is not None and "logging" in user_api_key_dict.team_metadata: - return decrypt_callback_vars(user_api_key_dict.team_metadata).get("logging") + return decoded_json_object(decrypt_callback_vars(user_api_key_dict.team_metadata)).get("logging") return None @@ -1291,14 +1298,14 @@ def get_dynamic_logging_metadata( ) -> TeamCallbackMetadata | None: callback_settings_obj: TeamCallbackMetadata | None = None key_dynamic_logging_settings: Final = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(user_api_key_dict) - team_dynamic_logging_settings: Final[dict | None] = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings( + team_dynamic_logging_settings: Final = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings( user_api_key_dict ) ######################################################################################### # Key-based callbacks ######################################################################################### if key_dynamic_logging_settings is not None: - for item in key_dynamic_logging_settings: + for item in iter_items(key_dynamic_logging_settings): callback = get_validated_callback_metadata(item=item, source="key-level") if callback is None: continue @@ -1310,7 +1317,7 @@ def get_dynamic_logging_metadata( # Team-based callbacks ######################################################################################### elif team_dynamic_logging_settings is not None: - for item in team_dynamic_logging_settings: + for item in iter_items(team_dynamic_logging_settings): callback = get_validated_callback_metadata(item=item, source="team-level") if callback is None: continue @@ -1331,9 +1338,11 @@ def get_dynamic_logging_metadata( } } """ - team_metadata: Final = decrypt_callback_vars(user_api_key_dict.team_metadata) + team_metadata: Final = decoded_json_object(decrypt_callback_vars(user_api_key_dict.team_metadata)) callback_settings: Final = team_metadata.get("callback_settings", None) or {} - callback_settings_obj = TeamCallbackMetadata(**callback_settings) + callback_settings_obj = TeamCallbackMetadata( + **_CallbackModelFields(**keyword_arguments(callback_settings, TeamCallbackMetadata)) + ) verbose_proxy_logger.debug("Team callback settings activated: %s", callback_settings_obj) ######################################################################################### # Enter here when configured on the config.yaml file. @@ -1465,7 +1474,7 @@ def resolve_tenant_otel_destinations( disabled: Final = _dynamically_disabled_backends(user_api_key_dict, request_headers) callbacks: Final = tuple( callback - for item in entries + for item in iter_items(entries) if (callback := get_validated_callback_metadata(item=item, source="otel-destination")) is not None if callback.callback_name.lower() not in disabled ) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 4cbc2155af0..10bdd1d2353 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -1,7 +1,7 @@ import asyncio from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass -from types import MappingProxyType +from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Final, Protocol from fastapi import APIRouter, Depends, HTTPException, status @@ -98,11 +98,11 @@ class _KeyRecord(Protocol): class _AccessGroupTable(Protocol): async def find_unique(self, where: Mapping[str, object]) -> _AccessGroupRecord | None: ... - async def find_many(self, order: Mapping[str, object]) -> Sequence[_AccessGroupRecord]: ... + async def find_many(self, *, order: Mapping[str, object]) -> Sequence[_AccessGroupRecord]: ... async def create(self, data: Mapping[str, object]) -> _AccessGroupRecord: ... - async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> _AccessGroupRecord: ... + async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> _AccessGroupRecord | None: ... async def delete(self, where: Mapping[str, object]) -> object: ... @@ -118,9 +118,9 @@ class _TeamTable(Protocol): class _KeyTable(Protocol): async def find_unique(self, where: Mapping[str, object]) -> _KeyRecord | None: ... - async def find_many(self, where: Mapping[str, object]) -> Sequence[_KeyRecord]: ... + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_KeyRecord]: ... - async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... + async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... class _AgentRecord(Protocol): @@ -132,9 +132,9 @@ class _AgentRecord(Protocol): class _AgentTable(Protocol): - async def find_many(self, where: Mapping[str, object]) -> Sequence[_AgentRecord]: ... + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_AgentRecord]: ... - async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... + async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... class _HasSomeFilter(TypedDict): @@ -170,7 +170,13 @@ class _AccessGroupTx(Protocol): class _AccessGroupTransaction(Protocol): async def __aenter__(self) -> _AccessGroupTx: ... - async def __aexit__(self, *exc_info: object) -> None: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> None: ... def _access_group_transaction(prisma_client: PrismaClient) -> _AccessGroupTransaction: @@ -731,6 +737,8 @@ async def update_access_group( from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + if record is None: + raise AttributeError("'NoneType' object has no attribute 'dict'", name="dict", obj=None) await _cache_access_group_record(record) await _patch_team_caches_add_access_group(teams_to_add, access_group_id, user_api_key_cache, proxy_logging_obj) await _patch_team_caches_remove_access_group( diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 21ac82754b4..3307f3050d9 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -1122,13 +1122,13 @@ async def _target_in_caller_admin_org( caller_user_id: str, target_user_id: str, ) -> bool: - admin_memberships: Final = await prisma_client.db.litellm_organizationmembership.find_many( + admin_memberships: Final = await _organization_membership_table(prisma_client).find_many( where={"user_id": caller_user_id, "user_role": LitellmUserRoles.ORG_ADMIN.value} ) if not admin_memberships: return False admin_org_ids: Final = {membership.organization_id for membership in admin_memberships} - target_memberships: Final = await prisma_client.db.litellm_organizationmembership.find_many( + target_memberships: Final = await _organization_membership_table(prisma_client).find_many( where={"user_id": target_user_id} ) return any(membership.organization_id in admin_org_ids for membership in target_memberships) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index b59606a8dcf..90779f732c7 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -20,7 +20,7 @@ import secrets import traceback from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from datetime import datetime, timedelta, timezone -from types import MappingProxyType +from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, TypeVar, cast import fastapi @@ -245,13 +245,20 @@ class _UserRowLike(Protocol): class _TxTables(Protocol): - litellm_proxymodeltable: TableActions[object] + @property + def litellm_proxymodeltable(self) -> TableActions[object]: ... class _ModelReencryptTransaction(Protocol): async def __aenter__(self) -> _TxTables: ... - async def __aexit__(self, *exc_info: object) -> None: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> None: ... class _ModelParamsUpdate(TypedDict): diff --git a/litellm/proxy/management_endpoints/management_v1/spend_logs.py b/litellm/proxy/management_endpoints/management_v1/spend_logs.py index da84c2d0013..378d3b34d12 100644 --- a/litellm/proxy/management_endpoints/management_v1/spend_logs.py +++ b/litellm/proxy/management_endpoints/management_v1/spend_logs.py @@ -1,12 +1,11 @@ """`/management/v1/spend_logs` facets.""" -from collections.abc import Sequence from datetime import datetime, timezone from functools import partial -from typing import Annotated, Final, Literal, Protocol +from typing import Annotated, Final, Literal from fastapi import APIRouter, Depends, Query, Request -from typing_extensions import ReadOnly, TypedDict +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth @@ -21,7 +20,7 @@ from litellm.proxy.list_api.common import ( reject_unknown_query_params, ) from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX -from litellm.proxy.utils import PrismaClient +from litellm.proxy.utils import query_raw_rows from litellm.types.proxy.management_endpoints.management_v1 import ( FacetListResponse, PageMeta, @@ -35,18 +34,7 @@ router: Final = APIRouter(prefix=MANAGEMENT_V1_PREFIX) # query, so both reads of the same table stop at the same depth. SPEND_LOGS_FACET_SCAN_CAP: Final = 10000 - -class _FacetRow(TypedDict, total=False): - end_user: ReadOnly[str | None] - user: ReadOnly[str | None] - - -class _FacetQuery(Protocol): - async def __call__(self, query: str, *args: object) -> Sequence[_FacetRow]: ... - - -def _facet_query(prisma_client: PrismaClient) -> _FacetQuery: - return prisma_client.db.query_raw +_FACET_VALUES: Final = TypeAdapter(list[str]) def _as_utc(value: datetime) -> datetime: @@ -141,8 +129,8 @@ async def _list_spend_log_facet( f" ORDER BY {column_sql} ASC" f" LIMIT ${scan_idx + 1} OFFSET ${scan_idx + 2}" ) - rows: Final = await _facet_query(prisma_client)(facet_sql, *params) - values: Final[list[str]] = [value for row in rows if (value := row.get(column))] + rows: Final = await query_raw_rows(prisma_client.db, facet_sql, *params) + values: Final = _FACET_VALUES.validate_python([value for row in rows if (value := row.get(column))]) has_more: Final = len(values) > page_size return FacetListResponse( diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index 3a2de12d108..abc2fe4651a 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -20,13 +20,14 @@ import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from datetime import datetime, timedelta from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Any, Final, Protocol +from typing import TYPE_CHECKING, Annotated, Final, Protocol from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import json_contains, require_attributes from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.config_sync_pubsub import publish_model_rows_written @@ -104,6 +105,14 @@ def _http_error(status_code: int, message: str) -> HTTPException: return HTTPException(status_code=status_code, detail=detail) +class _AppendableAccessGroups(Protocol): + def append(self, value: str, /) -> None: ... + + +class _RemovableAccessGroups(Protocol): + def remove(self, value: str, /) -> None: ... + + class _DeploymentRow(Protocol): @property def model_id(self) -> str: ... @@ -376,7 +385,7 @@ async def _missing_models_after_read_through( return tuple(still_missing) -def add_access_group_to_deployment(model_info: dict[str, Any], access_group: str) -> tuple[dict[str, Any], bool]: +def add_access_group_to_deployment(model_info: dict[str, object], access_group: str) -> tuple[dict[str, object], bool]: """ Add an access group to a deployment's model_info. @@ -385,16 +394,16 @@ def add_access_group_to_deployment(model_info: dict[str, Any], access_group: str access_group: The access group name to add Returns: - Tuple[Dict[str, Any], bool]: (updated_model_info, was_modified) + tuple[dict[str, object], bool]: (updated_model_info, was_modified) """ access_groups: Final = model_info.get("access_groups", []) # Check if access group already exists - if access_group in access_groups: + if json_contains(access_groups, access_group): return model_info, False # Add the access group - access_groups.append(access_group) + require_attributes(access_groups, _AppendableAccessGroups, "append").append(access_group) model_info["access_groups"] = access_groups return model_info, True @@ -729,7 +738,9 @@ async def _find_deployment_or_400(model_id: str, prisma_client: PrismaClient) -> return deployment.model_info -def remove_access_group_from_deployment(model_info: dict[str, Any], access_group: str) -> tuple[dict[str, Any], bool]: +def remove_access_group_from_deployment( + model_info: dict[str, object], access_group: str +) -> tuple[dict[str, object], bool]: """ Remove an access group from a deployment's model_info. @@ -738,16 +749,16 @@ def remove_access_group_from_deployment(model_info: dict[str, Any], access_group access_group: The access group name to remove Returns: - Tuple[Dict[str, Any], bool]: (updated_model_info, was_modified) + tuple[dict[str, object], bool]: (updated_model_info, was_modified) """ access_groups: Final = model_info.get("access_groups", []) # Check if access group exists - if access_group not in access_groups: + if not json_contains(access_groups, access_group): return model_info, False # Remove the access group - access_groups.remove(access_group) + require_attributes(access_groups, _RemovableAccessGroups, "remove").remove(access_group) model_info["access_groups"] = access_groups return model_info, True diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 0b4f266f838..6d4dc01a8c1 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -553,6 +553,21 @@ def _effective_complexity_router_config( ).effective +def _encrypted_litellm_params( + dumped: Mapping[str, object], + incoming_params: GenericLiteLLMParams | None, + existing_params: GenericLiteLLMParams | None, +) -> Mapping[str, object]: + return { + name: ( + _effective_complexity_router_config(incoming_params, existing_params) + if name == "complexity_router_config" + else encrypt_value_helper(value) + ) + for name, value in dumped.items() + } + + def _dump_without_none(model: BaseModel) -> Mapping[str, object]: return model.model_dump(exclude_none=True) @@ -1511,14 +1526,11 @@ def update_db_model( # update litellm params if updated_patch.litellm_params: # Encrypt any sensitive values - encrypted_params: Final = { - k: ( - _effective_complexity_router_config(updated_patch.litellm_params, db_model.litellm_params) - if k == "complexity_router_config" - else encrypt_value_helper(v) - ) - for k, v in updated_patch.litellm_params.model_dump(exclude_none=True, exclude_unset=True).items() - } + encrypted_params: Final = _encrypted_litellm_params( + updated_patch.litellm_params.model_dump(exclude_none=True, exclude_unset=True), + updated_patch.litellm_params, + db_model.litellm_params, + ) merged_litellm_params.update(encrypted_params) @@ -3471,14 +3483,9 @@ async def update_model( ### ENCRYPT PARAMS ### encrypted_params: Final = MappingProxyType( - { - k: ( - _effective_complexity_router_config(model_params.litellm_params, deployment.litellm_params) - if k == "complexity_router_config" - else encrypt_value_helper(value=v) - ) - for k, v in _new_litellm_params_dict.items() - } + _encrypted_litellm_params( + _new_litellm_params_dict, model_params.litellm_params, deployment.litellm_params + ) ) ### MERGE WITH EXISTING DATA ### diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index a185c0dda35..853f29ecb62 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -206,7 +206,19 @@ class _ObjectPermissionTxClient(Protocol): class _BudgetTxClient(Protocol): - async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> "PrismaBudgetTable | None": ... + async def update( + self, *, where: Mapping[str, object], data: Mapping[str, object] + ) -> "PrismaBudgetTable | None": ... + + +class _OrganizationTxClient(Protocol): + async def update( + self, + *, + where: Mapping[str, object], + data: Mapping[str, object], + include: Mapping[str, object] | None = None, + ) -> "PrismaOrganizationTable | None": ... class _TransactionTables(Protocol): @@ -217,7 +229,7 @@ class _TransactionTables(Protocol): def litellm_budgettable(self) -> "_BudgetTxClient": ... @property - def litellm_organizationtable(self) -> "_OrganizationTableClient": ... + def litellm_organizationtable(self) -> "_OrganizationTxClient": ... class _TransactionManager(Protocol): @@ -228,6 +240,7 @@ class _TransactionManager(Protocol): exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: "TracebackType | None", + /, ) -> bool | None: ... diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 1d35d34cbcf..24675406567 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -164,7 +164,9 @@ def _resolve_team_callbacks(team_metadata: object) -> TeamCallbackMetadata: for entry in logging_entries if isinstance(logging_entries, list) else (): if not isinstance(entry, dict): continue - callback = get_validated_callback_metadata(item=entry, source="team-level read") + callback = get_validated_callback_metadata( + item=_DECODED_JSON.validate_python(entry), source="team-level read" + ) if callback is None: continue resolved = convert_key_logging_metadata_to_callback(data=callback, team_callback_settings_obj=resolved) diff --git a/litellm/proxy/mcp_server/db.py b/litellm/proxy/mcp_server/db.py index 27f8b90a586..f29acc3b380 100644 --- a/litellm/proxy/mcp_server/db.py +++ b/litellm/proxy/mcp_server/db.py @@ -5,6 +5,7 @@ import json from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from types import TracebackType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, cast from fastapi import HTTPException @@ -68,8 +69,11 @@ if TYPE_CHECKING: class _UserEnvVarsTransactionClient(Protocol): - litellm_mcpuserenvvars: "TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]" - litellm_mcpservertable: "TableActions[prisma_db_models.LiteLLM_MCPServerTable]" + @property + def litellm_mcpuserenvvars(self) -> "TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]": ... + + @property + def litellm_mcpservertable(self) -> "TableActions[prisma_db_models.LiteLLM_MCPServerTable]": ... async def execute_raw(self, query: str, *args: object) -> int: ... @@ -77,7 +81,13 @@ class _UserEnvVarsTransactionClient(Protocol): class _UserEnvVarsTransaction(Protocol): async def __aenter__(self) -> _UserEnvVarsTransactionClient: ... - async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> bool | None: ... @dataclass(frozen=True, slots=True) @@ -139,6 +149,10 @@ def _blank_to_none(value: object) -> str | None: return value.strip() or None +def _submitted_or(submitted: Mapping[str, object], field: str, fallback: str | None) -> object: + return submitted.get(field, fallback) + + # Token-exchange settings with dedicated columns that also exist on # ``MCPCredentials`` as a legacy shape (rows and REST callers that predate the # columns). Every write lifts blob values into the columns and strips them from @@ -1379,8 +1393,10 @@ async def update_mcp_server( "credentials": safe_dumps( _bind_submitted_oauth_client( safe_submitted, - _blank_to_none(cleared_auth_fields.get("issuer", prepared_data.get("issuer", old_issuer))), - _blank_to_none(prepared_data.get("url", existing.url if existing else None)), + _blank_to_none( + cleared_auth_fields.get("issuer", _submitted_or(prepared_data, "issuer", old_issuer)) + ), + _blank_to_none(_submitted_or(prepared_data, "url", existing.url if existing else None)), ) ) } diff --git a/litellm/proxy/mcp_server/discoverable_endpoints.py b/litellm/proxy/mcp_server/discoverable_endpoints.py index ecc40b6e98f..f36609aa8b3 100644 --- a/litellm/proxy/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/mcp_server/discoverable_endpoints.py @@ -6,7 +6,17 @@ import time from collections.abc import AsyncIterator, Callable, Mapping from contextlib import asynccontextmanager from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Final, Literal, Optional +from typing import ( + TYPE_CHECKING, + Final, + Literal, + Optional, + Protocol, + SupportsIndex, + SupportsInt, + overload, + runtime_checkable, +) from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse import httpx @@ -15,6 +25,7 @@ from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Resp from mcp.client.auth.utils import get_client_metadata_scopes, union_scopes from mcp.shared.auth import OAuthMetadata from pydantic import ConfigDict, Field, SecretStr, TypeAdapter, ValidationError +from typing_extensions import Buffer from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache @@ -673,10 +684,24 @@ def _validate_token_response( ) +@runtime_checkable +class _SupportsTrunc(Protocol): + def __trunc__(self) -> int: ... + + +class _OAuthTokenResponseFields(Protocol): + @overload + def get(self, key: Literal["access_token", "refresh_token"], /) -> str | None: ... + @overload + def get(self, key: Literal["expires_in"], /) -> object: ... + @overload + def get(self, key: Literal["scope"], default: str | None, /) -> object: ... + + async def _store_per_user_token_server_side( server: MCPServer, user_id: str, - token_response: dict[str, Any], + token_response: _OAuthTokenResponseFields, identity_binding_proof: str | None = None, cimd_client_id: str | None = None, requested_scope: str | None = None, @@ -699,7 +724,11 @@ async def _store_per_user_token_server_side( raw_expires: Final = token_response.get("expires_in") try: - expires_in: int | None = int(raw_expires) if raw_expires is not None else None + expires_in: int | None = ( + int(raw_expires) + if isinstance(raw_expires, (str, Buffer, SupportsInt, SupportsIndex, _SupportsTrunc)) + else None + ) except (TypeError, ValueError): expires_in = None diff --git a/litellm/proxy/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py index cc02fc43ef3..3d55f63c73d 100644 --- a/litellm/proxy/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -29,7 +29,7 @@ from dataclasses import dataclass, replace from functools import lru_cache from itertools import chain, groupby from types import EllipsisType, MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypedDict, cast +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypedDict, cast from urllib.parse import ParseResult, urlparse import anyio @@ -73,6 +73,7 @@ from litellm.constants import ( MCP_UPSTREAM_SESSION_SHUTDOWN_TIMEOUT_SECONDS, ) from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException +from litellm.litellm_core_utils.core_helpers import as_object_list, decoded_json_object from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.mcp_client.client import ( @@ -469,14 +470,6 @@ class MCPServerConfig(TypedDict, total=False): rpm: ReadOnly[int | None] -class _ProtectedResourceMetadataPayload(TypedDict, total=False): - """The RFC 9728 protected-resource metadata document fields this gateway reads.""" - - authorization_servers: Sequence[object] - scopes_supported: Sequence[str] - scopes: Sequence[str] - - class _AuthorizationServerMetadataPayload(TypedDict, total=False): """The RFC 8414 / OpenID Discovery authorization-server metadata fields this gateway reads.""" @@ -1910,6 +1903,10 @@ def _listing_task_was_cancelled(error: asyncio.CancelledError) -> bool: return error.args != (SESSION_CLEANUP_TIMEOUT_MESSAGE,) +class _InternalNetworkSettings(Protocol): + def get(self, key: Literal["mcp_internal_ip_ranges"], /) -> list[str] | None: ... + + class MCPServerManager: _STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$") @@ -5369,7 +5366,7 @@ class MCPServerManager: try: response: Final = await self._fetch_oauth_discovery_url(resource_metadata_url, server_url) response.raise_for_status() - data: Final[_ProtectedResourceMetadataPayload] = response.json() + decoded: Final = _UNTYPED_VALUE.validate_python(response.json()) except SSRFError as exc: verbose_logger.warning( "MCP OAuth discovery: refusing to fetch resource metadata from %s " @@ -5387,11 +5384,13 @@ class MCPServerManager: ) return [], None - raw_servers: Final = data.get("authorization_servers") - if isinstance(raw_servers, list): - authorization_servers = [entry for entry in raw_servers if isinstance(entry, str) and entry.strip() != ""] - else: - authorization_servers = [] + data: Final = decoded_json_object(decoded) + raw_servers: Final = as_object_list(data.get("authorization_servers")) + authorization_servers: Final = ( + [entry for entry in raw_servers if isinstance(entry, str) and entry.strip() != ""] + if raw_servers is not None + else [] + ) scopes: Final = self._extract_scopes(data.get("scopes_supported") or data.get("scopes")) @@ -5631,13 +5630,14 @@ class MCPServerManager: "aws_session_name": credentials_dict.get("aws_session_name"), } - def _extract_scopes(self, scopes_value: str | Sequence[object] | None) -> list[str] | None: + def _extract_scopes(self, scopes_value: object) -> list[str] | None: if isinstance(scopes_value, str): scopes = [s.strip() for s in scopes_value.split() if s.strip()] return scopes or None - if isinstance(scopes_value, list): - scopes = [s for s in scopes_value if isinstance(s, str) and s.strip()] - return scopes or None + scope_entries: Final = as_object_list(scopes_value) + if scope_entries is not None: + listed_scopes: Final = [s for s in scope_entries if isinstance(s, str) and s.strip()] + return listed_scopes or None return None async def _fetch_tools_with_timeout( @@ -7354,7 +7354,7 @@ class MCPServerManager: servers.append(server) return servers - def _get_general_settings(self) -> dict[str, Any]: + def _get_general_settings(self) -> _InternalNetworkSettings: """Get general_settings, importing lazily to avoid circular imports.""" try: from litellm.proxy.proxy_server import ( diff --git a/litellm/proxy/mcp_server/operations.py b/litellm/proxy/mcp_server/operations.py index 8422d516d8c..afa3a373f3c 100644 --- a/litellm/proxy/mcp_server/operations.py +++ b/litellm/proxy/mcp_server/operations.py @@ -2080,7 +2080,7 @@ async def _list_tools_before_first_call( async def execute_mcp_tool( name: str, - arguments: dict[str, object], + arguments: Mapping[str, object], allowed_mcp_servers: list[MCPServer], start_time: datetime, user_api_key_auth: UserAPIKeyAuth | None = None, diff --git a/litellm/proxy/mcp_server/rest_endpoints.py b/litellm/proxy/mcp_server/rest_endpoints.py index 815c1d6270c..165fc23d754 100644 --- a/litellm/proxy/mcp_server/rest_endpoints.py +++ b/litellm/proxy/mcp_server/rest_endpoints.py @@ -15,7 +15,7 @@ import httpx2 from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from pydantic import TypeAdapter, ValidationError from starlette.datastructures import Headers -from typing_extensions import ReadOnly +from typing_extensions import Never, ReadOnly from litellm._logging import verbose_logger from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_TOOL_LISTING_TIMEOUT @@ -118,6 +118,8 @@ _MCP_GUARDRAIL_REJECTIONS: Final = ( _CLIENT_FORWARDED_TOKEN_AUTH_TYPES: Final = frozenset((MCPAuth.true_passthrough, MCPAuth.oauth_delegate)) +_NO_TOOL_ARGUMENTS: Final[Mapping[str, Never]] = MappingProxyType({}) + def _connection_error_message(exc: BaseException, url: str | None, timeout_seconds: float) -> str: reference: Final = uuid4().hex @@ -356,7 +358,7 @@ if MCP_AVAILABLE: status_code=403, detail={"error": "forbidden", "message": f"{tool_name} requires mcp_tool_search_enabled on the key"}, ) - tool_arguments: Final = data.get("arguments") or {} + tool_arguments: Final = data.get("arguments") or _NO_TOOL_ARGUMENTS if tool_name == AGENT_SEARCH_TOOL_NAME: return await handle_agent_search( query=str(tool_arguments.get("query", "")), diff --git a/litellm/proxy/mcp_server/tool_search.py b/litellm/proxy/mcp_server/tool_search.py index 96bcf8fbacf..e65fcf61f49 100644 --- a/litellm/proxy/mcp_server/tool_search.py +++ b/litellm/proxy/mcp_server/tool_search.py @@ -10,13 +10,13 @@ from dataclasses import dataclass from datetime import datetime from itertools import chain, islice from types import EllipsisType, MappingProxyType -from typing import TYPE_CHECKING, Any, Final, TypedDict +from typing import TYPE_CHECKING, Final, Protocol, SupportsIndex, SupportsInt, TypedDict, runtime_checkable import anyio from anyio import to_process from anyio.lowlevel import RunVar from pydantic import JsonValue, SecretStr, ValidationError -from typing_extensions import ReadOnly, Required, assert_never +from typing_extensions import Buffer, ReadOnly, Required, assert_never import litellm from litellm.llms.litellm_proxy.skills.skill_search import DEFAULT_SKILL_SEARCH_TOP_K @@ -131,7 +131,14 @@ async def _tool_argument_validation_error( return "Tool argument validation worker failed" -def coerce_top_k(value: Any, default: int = 5) -> int: +@runtime_checkable +class _SupportsTrunc(Protocol): + def __trunc__(self) -> int: ... + + +def coerce_top_k(value: object, default: int = 5) -> int: + if not isinstance(value, (str, Buffer, SupportsInt, SupportsIndex, _SupportsTrunc)): + return default try: return int(value) except (TypeError, ValueError): @@ -699,7 +706,7 @@ async def handle_mcp_proxy_tool( async def handle_mcp_tool_call( tool_name: str, - arguments: dict[str, Any], + arguments: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth, client_ip: str | None = None, mcp_servers: list[str] | None = None, diff --git a/litellm/proxy/middleware/admission_control_middleware.py b/litellm/proxy/middleware/admission_control_middleware.py index 83503d0c3d4..1b1af7fff95 100644 --- a/litellm/proxy/middleware/admission_control_middleware.py +++ b/litellm/proxy/middleware/admission_control_middleware.py @@ -3,7 +3,7 @@ import os from collections.abc import Callable, Mapping from dataclasses import dataclass from functools import lru_cache -from typing import Annotated, Final, Protocol, TypeAlias, runtime_checkable +from typing import Annotated, Final, Literal, Protocol, TypeAlias, runtime_checkable from pydantic import Field, TypeAdapter, ValidationError from starlette.responses import JSONResponse @@ -133,6 +133,12 @@ class AdmissionControlState: return self._metrics +class _RoutedScope(Protocol): + def __getitem__(self, key: Literal["path"], /) -> str: ... + + def get(self, key: Literal["root_path"], default: str, /) -> str: ... + + class _AdmissionLease: def __init__(self, state: AdmissionControlState) -> None: self.state: Final = state @@ -207,7 +213,7 @@ class AdmissionControlMiddleware: state.record_release() -def _get_route_path(scope: Scope) -> str: +def _get_route_path(scope: _RoutedScope) -> str: """Strip the ASGI root_path (SERVER_ROOT_PATH) the same way Starlette does before route matching.""" path: Final[str] = scope["path"] root_path: Final[str] = scope.get("root_path", "") diff --git a/litellm/proxy/middleware/billable_request_metrics_middleware.py b/litellm/proxy/middleware/billable_request_metrics_middleware.py index 95e5c9d1293..14b99b0e25d 100644 --- a/litellm/proxy/middleware/billable_request_metrics_middleware.py +++ b/litellm/proxy/middleware/billable_request_metrics_middleware.py @@ -19,7 +19,7 @@ import re import threading from collections.abc import Callable, Sequence from enum import Enum -from typing import Final, Protocol, runtime_checkable +from typing import Final, Literal, Protocol, runtime_checkable from starlette.types import ASGIApp, Message, Receive, Scope, Send @@ -196,6 +196,26 @@ def _extract_model_id(headers: Sequence[tuple[bytes, bytes]]) -> str | None: ) +class _RequestLineFields(Protocol): + def get(self, key: Literal["path", "method"], default: str, /) -> str: ... + + +class _ResponseStartFields(Protocol): + def __getitem__(self, key: Literal["status"], /) -> int: ... + + def get( + self, key: Literal["headers"], default: Sequence[tuple[bytes, bytes]], / + ) -> Sequence[tuple[bytes, bytes]]: ... + + +def _request_line_fields(scope: _RequestLineFields) -> _RequestLineFields: + return scope + + +def _response_start_fields(message: _ResponseStartFields) -> _ResponseStartFields: + return message + + class BillableRequestMetricsMiddleware: """ Pure ASGI middleware that classifies each request once and fans the result @@ -263,7 +283,10 @@ class BillableRequestMetricsMiddleware: await self.app(scope, receive, send) return - classification: Final = classify_billable_request(scope.get("path", ""), scope.get("method", "POST")) + request_line: Final = _request_line_fields(scope) + classification: Final = classify_billable_request( + request_line.get("path", ""), request_line.get("method", "POST") + ) if classification is None: await self.app(scope, receive, send) return @@ -275,8 +298,9 @@ class BillableRequestMetricsMiddleware: async def send_wrapper(message: Message) -> None: nonlocal status_code, model_id if message["type"] == "http.response.start": - status_code = message["status"] - model_id = _extract_model_id(message.get("headers", [])) + response_start: Final = _response_start_fields(message) + status_code = response_start["status"] + model_id = _extract_model_id(response_start.get("headers", [])) await send(message) await self.app(scope, receive, send_wrapper) diff --git a/litellm/proxy/middleware/per_request_root_path_middleware.py b/litellm/proxy/middleware/per_request_root_path_middleware.py index 3afa268b182..fdc374a58d7 100644 --- a/litellm/proxy/middleware/per_request_root_path_middleware.py +++ b/litellm/proxy/middleware/per_request_root_path_middleware.py @@ -14,7 +14,7 @@ Opt-in: with ``SERVER_ROOT_PATHS`` unset the middleware is not added at all. import os from collections.abc import Sequence from contextvars import ContextVar -from typing import Final +from typing import Final, Literal, Protocol from starlette.types import ASGIApp, Receive, Scope, Send @@ -89,6 +89,14 @@ def get_server_root_paths() -> tuple[str, ...]: return normalize_root_paths(configured.split(",")) +class _RootPathFields(Protocol): + def get(self, key: Literal["path", "root_path"], default: str, /) -> str: ... + + +def _root_path_fields(scope: _RootPathFields) -> _RootPathFields: + return scope + + class PerRequestRootPathMiddleware: """Sets ``scope["root_path"]`` to the configured prefix matching the request path on a whole-segment boundary. ``scope["path"]`` is left @@ -104,7 +112,7 @@ class PerRequestRootPathMiddleware: async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] in ("http", "websocket"): - path: Final[str] = scope.get("path", "") + path: Final[str] = _root_path_fields(scope).get("path", "") for prefix in self.root_paths: if path == prefix or path.startswith(prefix + "/"): scope["root_path"] = prefix # rebind-ok: ASGI middleware contract; Router and base_url read it @@ -113,7 +121,7 @@ class PerRequestRootPathMiddleware: # existing value when nothing matched — i.e. FastAPI's scalar # SERVER_ROOT_PATH) so code that emits URLs off the request path # picks the same prefix the router will resolve the request under. - token: Final = _request_root_path_var.set(str(scope.get("root_path", ""))) + token: Final = _request_root_path_var.set(str(_root_path_fields(scope).get("root_path", ""))) try: await self.app(scope, receive, send) finally: diff --git a/litellm/proxy/middleware/request_size_limit_middleware.py b/litellm/proxy/middleware/request_size_limit_middleware.py index 329a3c38d91..06649d72153 100644 --- a/litellm/proxy/middleware/request_size_limit_middleware.py +++ b/litellm/proxy/middleware/request_size_limit_middleware.py @@ -1,7 +1,7 @@ import json -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping from types import MappingProxyType -from typing import Final +from typing import Final, Literal, Protocol from fastapi import HTTPException from starlette._utils import get_route_path @@ -14,6 +14,18 @@ MaxRequestSizeGetter = Callable[[], int | float | None] RequestSizeLimitEnabledGetter = Callable[[], bool] +class _RequestHeadersFields(Protocol): + def get(self, key: Literal["headers"], /) -> Iterable[tuple[bytes, bytes]] | None: ... + + +class _RequestBodyFields(Protocol): + def get(self, key: Literal["body"], default: bytes, /) -> bytes: ... + + +def _request_body_fields(message: _RequestBodyFields) -> _RequestBodyFields: + return message + + class RequestEntityTooLarge(HTTPException): def __init__(self, max_request_size_mb: float) -> None: super().__init__(status_code=413, detail=f"Request size is too large. Max size is {max_request_size_mb} MB") @@ -100,7 +112,7 @@ class RequestSizeLimitMiddleware: if message["type"] != "http.request": return message - received_body_bytes += len(message.get("body", b"")) + received_body_bytes += len(_request_body_fields(message).get("body", b"")) if received_body_bytes > max_request_size_bytes: raise RequestEntityTooLarge(max_request_size_mb) return message @@ -128,7 +140,7 @@ def _mb_to_bytes(max_request_size_mb: float | None) -> int | None: return int(max_request_size_mb * 1024 * 1024) -def _get_content_length(scope: Scope) -> int | None: +def _get_content_length(scope: _RequestHeadersFields) -> int | None: headers: Final = dict(scope.get("headers") or []) raw_content_length: Final = headers.get(b"content-length") if raw_content_length is None: diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index e5e68a930c9..a81fced9021 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -25,7 +25,7 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import MAX_FILE_LIST_LIMIT from litellm.litellm_core_utils.batch_utils import batch_cost_is_final from litellm.litellm_core_utils.cloud_storage_security import REQUEST_S3_BUCKET_PARAMS -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.llms.base_llm.managed_resources.utils import is_base64_encoded_unified_id from litellm.proxy._types import ProxyException from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler @@ -1567,7 +1567,7 @@ async def resolve_output_file_ids_to_unified(response, prisma_client) -> None: if not prisma_client: return for attr in ("output_file_id", "error_file_id"): - raw_id = getattr(response, attr, None) + raw_id = attribute_or_none(response, attr) if not raw_id or is_base64_encoded_unified_file_id(raw_id): continue try: diff --git a/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py b/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py index 3b809688d6c..fb67cc93e37 100644 --- a/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py +++ b/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py @@ -1,5 +1,5 @@ from collections.abc import AsyncIterator -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast from fastapi.responses import StreamingResponse @@ -12,6 +12,14 @@ if TYPE_CHECKING: from litellm.proxy.utils import ProxyLogging +class _CallIdFields(Protocol): + def get(self, key: Literal["litellm_call_id"], default: str, /) -> str: ... + + +def _call_id_fields(data: _CallIdFields) -> _CallIdFields: + return data + + class FileContentStreamingHandler: @staticmethod def resolve_streaming_request_params( @@ -77,7 +85,7 @@ class FileContentStreamingHandler: async for chunk in stream_iterator: yield chunk await proxy_logging_obj.update_request_status( - litellm_call_id=data.get("litellm_call_id", ""), status="success" + litellm_call_id=_call_id_fields(data).get("litellm_call_id", ""), status="success" ) except Exception as e: await proxy_logging_obj.post_call_failure_hook( diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 8b8d9ec81be..df51740b553 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -11,7 +11,7 @@ import itertools import traceback from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass -from typing import Any, BinaryIO, Final, NoReturn, TypedDict, cast, get_args +from typing import Any, BinaryIO, Final, NoReturn, Protocol, TypedDict, cast, get_args import httpx from fastapi import ( @@ -35,7 +35,7 @@ from litellm.constants import ANTHROPIC_DEFAULT_FILE_LIST_LIMIT from litellm.litellm_core_utils.cloud_storage_security import ( is_managed_cloud_storage_uri, ) -from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket +from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket, has_attributes from litellm.llms.base_llm.files.litellm_db_storage_backend import LITELLM_DB_STORAGE_BACKEND_NAME from litellm.llms.base_llm.files.transformation import BaseFileEndpoints, FileListModelScope from litellm.llms.base_llm.managed_resources.isolation import build_list_page @@ -185,6 +185,11 @@ _LISTED_FILES_ADAPTER: Final = TypeAdapter(list[OpenAIFileObject]) _REQUEST_BODY_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +class _CarriesPrismaClient(Protocol): + @property + def prisma_client(self) -> PrismaClient: ... + + class UploadedFileInfo(TypedDict): filename: ReadOnly[str | None] content_type: ReadOnly[str | None] @@ -1067,8 +1072,11 @@ async def get_file_content( ) # Check if file is stored in a storage backend (check DB) - if hasattr(managed_files_obj, "prisma_client") and getattr(managed_files_obj, "prisma_client", None): - prisma_client: Final[PrismaClient] = getattr(managed_files_obj, "prisma_client") + if ( + has_attributes(managed_files_obj, _CarriesPrismaClient, "prisma_client") + and managed_files_obj.prisma_client + ): + prisma_client: Final = managed_files_obj.prisma_client db_file: Final = await ManagedFileRepository(prisma_client).table.find_first( where={"unified_file_id": file_id} ) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index a2e2e36d458..f1a9e282bc9 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -2510,21 +2510,23 @@ async def _relay_router_model( user_api_key_dict: UserAPIKeyAuth, ) -> Response: try: - result: Final = await llm_router.allm_passthrough_route( - model=model, - method=request.method, - endpoint=endpoint, - request_query_params=request.query_params, - request_headers=safe_get_request_headers(request), - stream=is_streaming_request, - content=None, - data=None, - files=None, - json=(request_body if is_json_content_type(request.headers.get("content-type", "")) else None), - params=None, - headers=None, - cookies=None, - litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict, request), + result: Final = _DECODED_JSON.validate_python( + await llm_router.allm_passthrough_route( + model=model, + method=request.method, + endpoint=endpoint, + request_query_params=request.query_params, + request_headers=safe_get_request_headers(request), + stream=is_streaming_request, + content=None, + data=None, + files=None, + json=(request_body if is_json_content_type(request.headers.get("content-type", "")) else None), + params=None, + headers=None, + cookies=None, + litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict, request), + ) ) except httpx.HTTPStatusError as upstream_error: return await _relay_upstream_response(upstream_error.response) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index c1fc6479421..dafbf6f74b6 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -11,6 +11,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ANTHROPIC_BATCHES_ROUTE from litellm.litellm_core_utils.core_helpers import ( + json_contains, native_messages_reported_cost, override_stamped_response_cost, response_cost_in_hidden_params, @@ -58,6 +59,7 @@ else: EndpointType = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) def _event_payload(event: str) -> Mapping[str, object] | None: @@ -609,10 +611,10 @@ class AnthropicPassthroughLoggingHandler: from litellm.types.utils import Choices, SpecialEnums try: - _json_response: Final = httpx_response.json() + _json_response: Final = _DECODED_JSON.validate_python(httpx_response.json()) # Only handle successful batch job creation (POST requests with 201 status) - if httpx_response.status_code == 200 and "id" in _json_response: + if httpx_response.status_code == 200 and json_contains(_json_response, "id"): # Transform Anthropic response to LiteLLM batch format anthropic_batches_config: Final = AnthropicBatchesConfig() litellm_batch_response: Final = anthropic_batches_config.transform_retrieve_batch_response( @@ -626,7 +628,7 @@ class AnthropicPassthroughLoggingHandler: litellm_batch_response.status = "validating" # Extract batch ID from the response - batch_id: Final = _json_response.get("id", "") + batch_id: Final = litellm_batch_response.id # Get model from request body (batch response doesn't include model) request_body = request_body or {} diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 42a7c458ef2..00e2aeb9e6e 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -6,7 +6,16 @@ import posixpath import re import traceback from base64 import b64encode -from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Mapping, MutableMapping, Sequence +from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Callable, + Coroutine, + Iterable, + Mapping, + MutableMapping, + Sequence, +) from dataclasses import dataclass from datetime import datetime from functools import reduce @@ -278,6 +287,11 @@ async def set_env_variables_in_header(custom_headers: dict | None) -> dict | Non return headers +class _AdapterCompletionRouter(Protocol): + @property + def aadapter_completion(self) -> Callable[..., Coroutine[object, object, object]]: ... + + async def chat_completion_pass_through_endpoint( fastapi_response: Response, request: Request, @@ -368,7 +382,8 @@ async def chat_completion_pass_through_endpoint( and data["model"] is not None and llm_router.registry().resolve(data["model"], kind="recognized") ): - llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) + adapter_router = require_attributes(llm_router, _AdapterCompletionRouter, "aadapter_completion") + llm_response = asyncio.create_task(adapter_router.aadapter_completion(**data)) elif ( llm_router is not None and data["model"] not in router_model_names @@ -377,11 +392,13 @@ async def chat_completion_pass_through_endpoint( or len(llm_router.registry().patterns().patterns) > 0 ) ): # check for wildcard routes or default deployment before checking deployment_names - llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) + adapter_router = require_attributes(llm_router, _AdapterCompletionRouter, "aadapter_completion") + llm_response = asyncio.create_task(adapter_router.aadapter_completion(**data)) elif llm_router is not None and data["model"] in llm_router.registry().names( kind="deployment_names" ): # model in router deployments, calling a specific deployment on the router (lowest priority) - llm_response = asyncio.create_task(llm_router.aadapter_completion(**data, specific_deployment=True)) + adapter_router = require_attributes(llm_router, _AdapterCompletionRouter, "aadapter_completion") + llm_response = asyncio.create_task(adapter_router.aadapter_completion(**data, specific_deployment=True)) elif user_model is not None: # `litellm --model ` llm_response = asyncio.create_task(litellm.aadapter_completion(**data)) else: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8f5db97e7a3..b01dcfd2f03 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -394,6 +394,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, get_parent_otel_span_from_kwargs, iter_items, + json_contains, normalize_drop_params, require_attributes, ) @@ -2532,6 +2533,7 @@ class _InvitationAcceptTxManager(Protocol): exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, + /, ) -> bool | None: ... @@ -13806,7 +13808,7 @@ async def moderations( llm_router=llm_router, user_model=user_model, ) - response: Final = await llm_call + response: Final = _UNTYPED_VALUE.validate_python(await llm_call) ### ALERTING ### asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success")) @@ -13856,14 +13858,20 @@ async def moderations( ) +class _SpeechBytes(Protocol): + def aiter_bytes(self, chunk_size: int | None = None) -> Awaitable[AsyncIterator[bytes]]: ... + + async def _audio_speech_chunk_generator( - _response: HttpxBinaryResponseContent, + _response: object, ) -> AsyncGenerator[bytes, None]: # chunk_size has a big impact on latency, it can't be too small or too large # too small: latency is high # too large: latency is low, but memory usage is high # 8192 is a good compromise - _generator: Final = await _response.aiter_bytes(chunk_size=AUDIO_SPEECH_CHUNK_SIZE) + _generator: Final = await require_attributes(_response, _SpeechBytes, "aiter_bytes").aiter_bytes( + chunk_size=AUDIO_SPEECH_CHUNK_SIZE + ) async for chunk in _generator: yield chunk @@ -13924,7 +13932,7 @@ async def audio_speech( llm_router=llm_router, user_model=user_model, ) - response: Final = await llm_call + response: Final = _UNTYPED_VALUE.validate_python(await llm_call) ### ALERTING ### asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success")) @@ -14088,7 +14096,7 @@ async def audio_transcriptions( llm_router=llm_router, user_model=user_model, ) - response: Final = await llm_call + response: Final = _UNTYPED_VALUE.validate_python(await llm_call) except Exception as e: raise e finally: @@ -14665,7 +14673,7 @@ async def get_assistants( # for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch if llm_router is None: raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) - response: Final = await llm_router.aget_assistants(**data) + response: Final = _UNTYPED_VALUE.validate_python(await llm_router.aget_assistants(**data)) ### ALERTING ### asyncio.create_task( @@ -14741,7 +14749,7 @@ async def create_assistant( # for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch if llm_router is None: raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) - response: Final = await llm_router.acreate_assistants(**data) + response: Final = _UNTYPED_VALUE.validate_python(await llm_router.acreate_assistants(**data)) ### ALERTING ### asyncio.create_task( @@ -14816,7 +14824,9 @@ async def delete_assistant( # for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch if llm_router is None: raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) - response: Final = await llm_router.adelete_assistant(assistant_id=assistant_id, **data) + response: Final = _UNTYPED_VALUE.validate_python( + await llm_router.adelete_assistant(assistant_id=assistant_id, **data) + ) ### ALERTING ### asyncio.create_task( @@ -14891,7 +14901,7 @@ async def create_threads( # for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch if llm_router is None: raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) - response: Final = await llm_router.acreate_thread(**data) + response: Final = _UNTYPED_VALUE.validate_python(await llm_router.acreate_thread(**data)) ### ALERTING ### asyncio.create_task( @@ -14964,7 +14974,7 @@ async def get_thread( # for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch if llm_router is None: raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) - response: Final = await llm_router.aget_thread(thread_id=thread_id, **data) + response: Final = _UNTYPED_VALUE.validate_python(await llm_router.aget_thread(thread_id=thread_id, **data)) ### ALERTING ### asyncio.create_task( @@ -15041,7 +15051,7 @@ async def add_messages( # for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch if llm_router is None: raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) - response: Final = await llm_router.a_add_message(thread_id=thread_id, **data) + response: Final = _UNTYPED_VALUE.validate_python(await llm_router.a_add_message(thread_id=thread_id, **data)) ### ALERTING ### asyncio.create_task( @@ -15114,7 +15124,7 @@ async def get_messages( # for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch if llm_router is None: raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) - response: Final = await llm_router.aget_messages(thread_id=thread_id, **data) + response: Final = _UNTYPED_VALUE.validate_python(await llm_router.aget_messages(thread_id=thread_id, **data)) ### ALERTING ### asyncio.create_task( @@ -15194,7 +15204,9 @@ async def run_thread( if "stream" in data and data["stream"] is True: # use generate_responses to stream responses async def produce_run_stream() -> StreamingResponse | JSONResponse: - run_stream: Final = await router.arun_thread(thread_id=thread_id, **data) + run_stream: Final = _UNTYPED_VALUE.validate_python( + await router.arun_thread(thread_id=thread_id, **data) + ) return await create_response( generator=async_assistants_data_generator( user_api_key_dict=user_api_key_dict, @@ -15222,7 +15234,7 @@ async def run_thread( on_late_failure=audit_late_failure, ) - response: Final = await router.arun_thread(thread_id=thread_id, **data) + response: Final = _UNTYPED_VALUE.validate_python(await router.arun_thread(thread_id=thread_id, **data)) ### ALERTING ### asyncio.create_task( @@ -16175,11 +16187,11 @@ async def _count_request_tokens( tools: Final = request.tools system: Final = request.system - deployment: dict[str, Any] | None = None if routed is None else dict(routed) + deployment: dict[str, object] | None = None if routed is None else dict(routed) litellm_model_name = None model_info: Final = None if routed is None else routed.get("model_info", {}) if deployment is not None: - litellm_model_name = deployment.get("litellm_params", {}).get("model") + litellm_model_name = _with_get(deployment.get("litellm_params", _EMPTY_MAPPING)).get("model") deployment = { # rebind-ok: merge the named credential into a copy **deployment, "litellm_params": _with_chat_timeout( @@ -16187,8 +16199,8 @@ async def _count_request_tokens( ), } # remove the custom_llm_provider_prefix in the litellm_model_name - if "/" in litellm_model_name: - litellm_model_name = litellm_model_name.split("/", 1)[1] + if json_contains(litellm_model_name, "/"): + litellm_model_name = _split_once(litellm_model_name, "/")[1] model_to_use: str = ( litellm_model_name if isinstance(litellm_model_name, str) and litellm_model_name else request.model @@ -17071,6 +17083,14 @@ class _SupportsLower(Protocol): def lower(self) -> object: ... +class _SupportsSplit(Protocol): + def split(self, sep: str, maxsplit: int, /) -> Sequence[object]: ... + + +def _split_once(value: object, sep: str) -> Sequence[object]: + return require_attributes(value, _SupportsSplit, "split").split(sep, 1) + + def _lowered(value: object) -> object: return require_attributes(value, _SupportsLower, "lower").lower() @@ -17239,18 +17259,19 @@ async def _apply_search_filter_to_models( def _is_byok_outside_caller_teams(model_info_dict: object) -> bool: return _byok_row_outside_caller_teams(model_info_dict, allowed_team_ids) - def _model_matches_search(m: dict[str, Any]) -> bool: + def _model_matches_search(m: Mapping[str, object]) -> bool: # Team BYOK models persist an internal `model_name` # (e.g. `model_name_{team_id}_{uuid}`) and expose the user-facing # name via `model_info.team_public_model_name`. Match both so the # name shown in the UI is searchable. - if search_lower in (m.get("model_name") or "").lower(): + if json_contains(_lowered(m.get("model_name") or ""), search_lower): return True - team_public_model_name: Final = (m.get("model_info") or {}).get("team_public_model_name") or "" - if search_lower in team_public_model_name.lower(): + team_public_model_name: Final = ( + _with_get(m.get("model_info") or _EMPTY_MAPPING).get("team_public_model_name") or "" + ) + if json_contains(_lowered(team_public_model_name), search_lower): return True - litellm_params: Final[Mapping[str, object]] = m.get("litellm_params") or MappingProxyType({}) - litellm_model: Final = litellm_params.get("model") + litellm_model: Final = _with_get(m.get("litellm_params") or _EMPTY_MAPPING).get("model") return isinstance(litellm_model, str) and search_lower in litellm_model.lower() # Filter models in router by search term, dropping BYOK rows that @@ -19481,7 +19502,7 @@ async def async_queue_request( return StreamingResponse( async_data_generator( user_api_key_dict=user_api_key_dict, - response=await router.schedule_acompletion(**data), + response=_UNTYPED_VALUE.validate_python(await router.schedule_acompletion(**data)), request_data=data, request=request, ), @@ -19500,7 +19521,7 @@ async def async_queue_request( on_late_failure=audit_late_failure, ) - response: Final = await router.schedule_acompletion(**data) + response: Final = _UNTYPED_VALUE.validate_python(await router.schedule_acompletion(**data)) fastapi_response.headers.update({"x-litellm-priority": str(queued_priority)}) return response except Exception as e: @@ -20147,7 +20168,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): ) if user_obj and hasattr(user_obj, "__dict__"): - user_obj.__dict__.pop("password", None) + vars(user_obj).pop("password", None) # The password just changed via an invitation/reset link; any UI session # minted under the old password may be in hostile hands. Revoke them all — diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index 47050f01644..38041ed1862 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -19,6 +19,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import ORJSONResponse, StreamingResponse from pydantic import TypeAdapter from starlette.datastructures import UploadFile +from typing_extensions import TypedDict import litellm from litellm._logging import verbose_proxy_logger @@ -91,6 +92,10 @@ _EMPTY_TEAM_PROVIDER_CREDENTIALS: Final[Mapping[str, object]] = MappingProxyType _TEAM_CREDENTIAL_PROVIDER: Final = "openai" +class _ForwardedRequestFields(TypedDict, total=False, extra_items=object): + pass + + def _as_string_keyed_mapping(value: object) -> Mapping[str, object] | None: if isinstance(value, Mapping): return value @@ -904,7 +909,7 @@ async def rag_ingest( vertex_target_error: Final = _vertex_target_error(base) if vertex_target_checked else None if vertex_target_error is not None: raise HTTPException(status_code=400, detail={"error": vertex_target_error}) - request_data: Final[dict[str, Any]] = await add_litellm_data_to_request( + request_data: Final[dict[str, object]] = await add_litellm_data_to_request( data={}, request=request, general_settings=general_settings, @@ -955,7 +960,7 @@ async def rag_ingest( file_url=file_url, file_id=ingest_file_id, router=llm_router, - **request_data, + **_ForwardedRequestFields(**request_data), ) if ingest_response.get("status") == "failed": raise HTTPException( diff --git a/litellm/proxy/rerank_endpoints/endpoints.py b/litellm/proxy/rerank_endpoints/endpoints.py index 4905751a85d..b80220b1aac 100644 --- a/litellm/proxy/rerank_endpoints/endpoints.py +++ b/litellm/proxy/rerank_endpoints/endpoints.py @@ -6,6 +6,7 @@ from typing import Final import orjson from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import ORJSONResponse +from pydantic import TypeAdapter from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -23,6 +24,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( from litellm.types.utils import CallTypes router: Final = APIRouter() +_DECODED_JSON: Final = TypeAdapter(object) @router.post( @@ -85,7 +87,7 @@ async def rerank( llm_router=llm_router, user_model=user_model, ) - response: Final = await llm_call + response: Final = _DECODED_JSON.validate_python(await llm_call) ### ALERTING ### asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success")) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 11a0b947abe..369d167f623 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -2686,7 +2686,7 @@ async def _responses_websocket_session( llm_router=llm_router, user_model=user_model, ) - outcome: Final = await llm_call + outcome: Final = _DECODED_JSON.validate_python(await llm_call) except asyncio.CancelledError: if refused_turns.first is not None: _ = bill_socket_failure(refused_turns.first) diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 0a13e3a9e99..11be7a736ed 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -21,6 +21,13 @@ if TYPE_CHECKING: _RESPONSE_POLLING_TARGET: Final = "response_polling" _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _JSON_LIST: Final = TypeAdapter(list[object]) +_CACHED_VALUE: Final = TypeAdapter(object) + + +def _json_document(value: object) -> str | bytes | bytearray: + if isinstance(value, (str, bytes, bytearray)): + return value + raise TypeError(f"the JSON object must be str, bytes or bytearray, not {value.__class__.__name__}") class ResponsePollingHandler: @@ -156,13 +163,13 @@ class ResponsePollingHandler: cache_key: Final = self.get_cache_key(polling_id) # Get current state - cached_state: Final = await self.redis_cache.async_get_cache(cache_key) + cached_state: Final = _CACHED_VALUE.validate_python(await self.redis_cache.async_get_cache(cache_key)) if not cached_state: verbose_proxy_logger.warning("No cached state found for polling_id: %s", polling_id) return # Parse existing ResponsesAPIResponse from cache - state: Final = _JSON_OBJECT.validate_python(json.loads(cached_state)) + state: Final = _JSON_OBJECT.validate_python(json.loads(_json_document(cached_state))) # Update status (using OpenAI native status values) if status: @@ -239,10 +246,10 @@ class ResponsePollingHandler: return None cache_key: Final = self.get_cache_key(polling_id) - cached_state: Final = await self.redis_cache.async_get_cache(cache_key) + cached_state: Final = _CACHED_VALUE.validate_python(await self.redis_cache.async_get_cache(cache_key)) if cached_state: - return _JSON_OBJECT.validate_python(json.loads(cached_state)) + return _JSON_OBJECT.validate_python(json.loads(_json_document(cached_state))) return None diff --git a/litellm/proxy/search_endpoints/search_tool_management.py b/litellm/proxy/search_endpoints/search_tool_management.py index a2c4066a14f..dee8c8d5c60 100644 --- a/litellm/proxy/search_endpoints/search_tool_management.py +++ b/litellm/proxy/search_endpoints/search_tool_management.py @@ -7,6 +7,7 @@ from datetime import datetime from typing import Any, Final, TypeAlias from fastapi import APIRouter, Depends, HTTPException +from typing_extensions import ReadOnly, Required, TypedDict from litellm._logging import verbose_proxy_logger from litellm.constants import UI_SESSION_TOKEN_TEAM_ID @@ -68,7 +69,18 @@ async def _refresh_router_search_tools() -> None: verbose_proxy_logger.exception("Search tool router refresh failed after a management write: %s", e) -def _with_loaded_tools_where_undecryptable(db_search_tools: Sequence[dict[str, object]]) -> list[dict[str, Any]]: +class _ListedSearchTool(TypedDict, total=False): + search_tool_id: ReadOnly[str | None] + search_tool_name: ReadOnly[Required[str]] + litellm_params: ReadOnly[Required[Mapping[str, object]]] + search_tool_info: ReadOnly[dict[object, object] | None] + created_at: ReadOnly[str | None] + updated_at: ReadOnly[str | None] + + +def _with_loaded_tools_where_undecryptable( + db_search_tools: Sequence[_ListedSearchTool], +) -> Sequence[_ListedSearchTool]: from litellm.proxy.proxy_server import llm_router kept_search_tools: Final = keep_loaded_search_tools_that_do_not_decrypt( diff --git a/litellm/proxy/search_endpoints/search_tool_registry.py b/litellm/proxy/search_endpoints/search_tool_registry.py index c8cd1100e08..39510076d22 100644 --- a/litellm/proxy/search_endpoints/search_tool_registry.py +++ b/litellm/proxy/search_endpoints/search_tool_registry.py @@ -5,7 +5,7 @@ Search Tool Registry for managing search tool configurations. import os from collections.abc import Iterator, Mapping, Sequence from datetime import datetime, timezone -from typing import Final, Protocol +from typing import Final, Protocol, TypeVar from pydantic import TypeAdapter, ValidationError @@ -172,9 +172,13 @@ def _did_not_decrypt(search_tool: Mapping[str, object]) -> bool: ) +_DbSearchToolT: Final = TypeVar("_DbSearchToolT", bound=Mapping[str, object]) +_LoadedSearchToolT: Final = TypeVar("_LoadedSearchToolT", bound=Mapping[str, object]) + + def keep_loaded_search_tools_that_do_not_decrypt( - db_search_tools: Sequence[Mapping[str, object]], loaded_search_tools: Sequence[Mapping[str, object]] -) -> Sequence[Mapping[str, object]]: + db_search_tools: Sequence[_DbSearchToolT], loaded_search_tools: Sequence[_LoadedSearchToolT] +) -> Sequence[_DbSearchToolT | _LoadedSearchToolT]: """Replace each DB search tool whose params do not decrypt with the current key by its loaded version.""" loaded_by_id: Final = {tool.get("search_tool_id"): tool for tool in loaded_search_tools} kept: Final = tuple( diff --git a/litellm/proxy/spend_tracking/key_metadata_recovery.py b/litellm/proxy/spend_tracking/key_metadata_recovery.py index b3b5e41328d..f48f853ba2a 100644 --- a/litellm/proxy/spend_tracking/key_metadata_recovery.py +++ b/litellm/proxy/spend_tracking/key_metadata_recovery.py @@ -1,10 +1,11 @@ import asyncio from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet +from contextlib import AbstractAsyncContextManager from dataclasses import dataclass from datetime import datetime, timedelta from types import MappingProxyType -from typing import Final, Literal, TypeVar +from typing import Final, Literal, Protocol, TypeVar from pydantic import ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict @@ -168,6 +169,16 @@ async def _db_or_empty( return None +class _SpendLogTransaction(Protocol): + async def execute_raw(self, query: str, /) -> int: ... + + async def query_raw(self, query: str, /, *args: object) -> object: ... + + +def _spend_log_transaction(prisma_client: PrismaClient) -> AbstractAsyncContextManager[_SpendLogTransaction]: + return prisma_client.db.tx(timeout=_SPEND_LOG_TRANSACTION_TIMEOUT) + + async def _rows_within_the_statement_timeout( prisma_client: PrismaClient, sql: str, @@ -177,7 +188,7 @@ async def _rows_within_the_statement_timeout( ) -> Sequence[Mapping[str, object]]: async with ( db_span("recover_key_metadata", table), - prisma_client.db.tx(timeout=_SPEND_LOG_TRANSACTION_TIMEOUT) as transaction, + _spend_log_transaction(prisma_client) as transaction, ): await transaction.execute_raw(_SPEND_LOG_STATEMENT_TIMEOUT_SQL) for setting in planner_settings: diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py index e4a98619bfd..86e8ef31cf4 100644 --- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py +++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py @@ -18,7 +18,7 @@ from collections.abc import Awaitable, Callable, Iterator, Mapping from dataclasses import dataclass from datetime import date, datetime, time, timedelta, timezone from itertools import chain, groupby -from types import MappingProxyType +from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Final, Protocol from pydantic import TypeAdapter @@ -71,13 +71,20 @@ class _SentinelBatchTable(Protocol): class _SentinelBatch(Protocol): - litellm_dailyteamspend: _SentinelBatchTable + @property + def litellm_dailyteamspend(self) -> _SentinelBatchTable: ... class _SentinelBatchManager(Protocol): async def __aenter__(self) -> _SentinelBatch: ... - async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + /, + ) -> bool | None: ... class _SupportsGet(Protocol): diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 5b961f1320b..2de589564ac 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -3,7 +3,7 @@ import asyncio import json import os from collections import Counter -from collections.abc import Mapping, MutableMapping, Sequence +from collections.abc import ItemsView, Mapping, MutableMapping, Sequence from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -1306,6 +1306,16 @@ async def _load_environment_variables_param( return param_value +def _submitted_sso_values(submitted: Mapping[str, object]) -> ItemsView[str, object]: + return submitted.items() + + +def _environment_value(value: object) -> str: + if isinstance(value, str): + return value + raise TypeError(f"str expected, not {type(value).__name__}") + + def _decrypted_sso_fields(environment_variables: Mapping[str, object]) -> Mapping[str, str | None]: return MappingProxyType( { @@ -1398,11 +1408,11 @@ async def update_sso_settings( field_name: submitted[field_name] if field_name in submitted else fallback_values.get(field_name) for field_name in SSOConfig.model_fields } - for field_name, value in submitted.items(): + for field_name, value in _submitted_sso_values(submitted): if field_name in SSO_FIELD_ENV_VARS: env_var_name = SSO_FIELD_ENV_VARS[field_name] if value: - os.environ[env_var_name] = value + os.environ[env_var_name] = _environment_value(value) else: # Clear environment variable if value is null/empty os.environ.pop(env_var_name, None) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index eac2700551f..874fcbc8d09 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -169,6 +169,7 @@ from litellm.litellm_core_utils.core_helpers import ( attribute_or_none, coerce_token_limit, get_or_create_metadata_bucket, + has_attributes, independent_snapshot, is_expected_client_error, require_attributes, @@ -2373,10 +2374,12 @@ class ProxyLogging: modified_args = self._extract_modified_arguments_from_content(new_content, request_obj) if modified_args is not None: # Return the masked/redacted arguments for the MCP call to use - return MCPPreCallResponseObject( - should_proceed=True, - error_message=None, - modified_arguments=modified_args, + return MCPPreCallResponseObject.model_validate( + { + "should_proceed": True, + "error_message": None, + "modified_arguments": modified_args, + } ) else: # Could not parse modified arguments, allow original call but warn @@ -2395,7 +2398,7 @@ class ProxyLogging: return None - def _extract_modified_arguments_from_content(self, masked_content: str, request_obj) -> dict | None: + def _extract_modified_arguments_from_content(self, masked_content: str, request_obj) -> object: """ Extract modified/masked arguments from the guardrail response content. """ @@ -2416,7 +2419,7 @@ class ProxyLogging: # Try to parse as JSON first try: - modified_args = json.loads(args_text) + modified_args = _DECODED_JSON.validate_python(json.loads(args_text)) verbose_proxy_logger.debug("Successfully parsed JSON args: %s", modified_args) return modified_args except json.JSONDecodeError as e: @@ -5643,8 +5646,13 @@ _LOCK_TEAMS_BY_ID_SQL: Final = ( ) -def _user_unique_conflict_message(metadata: object, user_id: object) -> str: - match metadata: +class _UniqueConflictMetadata(Protocol): + @property + def meta(self) -> object: ... + + +def _user_unique_conflict_message(error: _UniqueConflictMetadata, user_id: object) -> str: + match error.meta: case {"target": ["user_id"]}: return f"User with id {user_id} already exists" case {"target": ["sso_user_id"]}: @@ -5658,7 +5666,7 @@ def _raise_user_unique_conflict(error: "UniqueViolationError", user_id: object, raise error raise HTTPException( status_code=409, - detail={"error": _user_unique_conflict_message(error.meta, user_id)}, + detail={"error": _user_unique_conflict_message(error, user_id)}, ) from error @@ -8338,6 +8346,10 @@ class PrismaClient: ### HELPER FUNCTIONS ### +class _JsonDumpableRow(Protocol): + def model_dump_json(self) -> object: ... + + async def cache_user_row(user_id: str, cache: DualCache, db: PrismaClient) -> None: """ Check if a user_id exists in cache, @@ -8349,8 +8361,8 @@ async def cache_user_row(user_id: str, cache: DualCache, db: PrismaClient) -> No user_row: Final = await db.get_data(user_id=user_id) if user_row is not None: print_verbose(f"User Row: {user_row}, type = {type(user_row)}") - if hasattr(user_row, "model_dump_json") and callable(getattr(user_row, "model_dump_json")): - cache_value: Final[str] = user_row.model_dump_json() + if has_attributes(user_row, _JsonDumpableRow, "model_dump_json") and callable(user_row.model_dump_json): + cache_value: Final = user_row.model_dump_json() cache.set_cache(key=cache_key, value=cache_value, ttl=600) # store for 10 minutes diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 4b9547300db..62af5663ea9 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -10,6 +10,7 @@ All /vector_store management endpoints import copy import json +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from fastapi import APIRouter, Depends, HTTPException @@ -54,8 +55,16 @@ def _vector_store_table(prisma_client: "PrismaClient") -> "TableActions[_VectorS return ManagedVectorStoresRepository(prisma_client).table +class _VectorStoreRowFields(TypedDict, total=False, extra_items=object): + pass + + +def _dumped_row(row: "_VectorStoreRow") -> Mapping[str, object]: + return row.model_dump() + + def _row_to_vector_store(row: "_VectorStoreRow") -> LiteLLM_ManagedVectorStore: - return LiteLLM_ManagedVectorStore(**row.model_dump()) + return LiteLLM_ManagedVectorStore(**_VectorStoreRowFields(**_dumped_row(row))) class _ConfigOwnedDetail(TypedDict): diff --git a/litellm/proxy/vector_store_endpoints/utils.py b/litellm/proxy/vector_store_endpoints/utils.py index 9004d1951b8..4044e50f8c1 100644 --- a/litellm/proxy/vector_store_endpoints/utils.py +++ b/litellm/proxy/vector_store_endpoints/utils.py @@ -2,10 +2,11 @@ import json import re from collections.abc import Iterable, Mapping from types import MappingProxyType -from typing import Final, Literal +from typing import Final, Literal, Protocol from fastapi import HTTPException, Request from pydantic import ConfigDict, TypeAdapter +from typing_extensions import TypedDict import litellm from litellm._logging import verbose_proxy_logger @@ -27,6 +28,18 @@ from litellm.utils import ProviderConfigManager _JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) +class _ManagedVectorStoreFields(TypedDict, total=False, extra_items=object): + pass + + +class _DumpableRow(Protocol): + def model_dump(self) -> Mapping[str, object]: ... + + +def _managed_vector_store_from_row(row: _DumpableRow) -> LiteLLM_ManagedVectorStore: + return LiteLLM_ManagedVectorStore(**_ManagedVectorStoreFields(**row.model_dump())) + + def normalize_litellm_params( vector_store: LiteLLM_ManagedVectorStore, ) -> LiteLLM_ManagedVectorStore: @@ -316,7 +329,7 @@ async def get_litellm_managed_vector_store( ) if not rows: return None - return normalize_litellm_params(LiteLLM_ManagedVectorStore(**rows[0].model_dump())) + return normalize_litellm_params(_managed_vector_store_from_row(rows[0])) except Exception as e: verbose_proxy_logger.warning( "Failed to resolve vector store id=%s from shared cache: %s", diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 24fd443b68a..8cca272b0b3 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -1,8 +1,9 @@ -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, Protocol import orjson from pydantic import BaseModel, ConfigDict, TypeAdapter +from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_utils import video_access_group from litellm.proxy.openai_files_endpoints.common_utils import authorize_model_for_key @@ -63,7 +64,11 @@ def get_custom_provider_from_data(data: dict[str, Any]) -> str | None: return None -def encode_character_id_in_response(response: Any, custom_llm_provider: str, model_id: str | None) -> object: +class _CarriesId(Protocol): + id: object + + +def encode_character_id_in_response(response: object, custom_llm_provider: str, model_id: str | None) -> object: if isinstance(response, dict) and response.get("id"): response["id"] = encode_character_id_with_provider( character_id=response["id"], @@ -72,7 +77,9 @@ def encode_character_id_in_response(response: Any, custom_llm_provider: str, mod ) return response - character_id: Final = getattr(response, "id", None) + if not has_attributes(response, _CarriesId, "id"): + return response + character_id: Final = response.id if isinstance(character_id, str) and character_id: response.id = encode_character_id_with_provider( character_id=character_id, diff --git a/litellm/rag/ingestion/base_ingestion.py b/litellm/rag/ingestion/base_ingestion.py index 8b2cf6e0a4e..0d75fe4e531 100644 --- a/litellm/rag/ingestion/base_ingestion.py +++ b/litellm/rag/ingestion/base_ingestion.py @@ -14,12 +14,13 @@ from __future__ import annotations import base64 from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, Protocol, cast import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid4 from litellm.constants import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE +from litellm.litellm_core_utils.core_helpers import has_attributes, require_attributes from litellm.litellm_core_utils.url_utils import async_safe_get from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -31,6 +32,8 @@ from litellm.rag.text_splitters import RecursiveCharacterTextSplitter from litellm.types.rag import RAGIngestOptions, RAGIngestResponse if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Iterable + from litellm import Router @@ -38,6 +41,21 @@ class RAGIngestBadRequestError(ValueError): status_code: Final = 400 +class _OcrRouter(Protocol): + @property + def aocr(self) -> Callable[..., Awaitable[object]]: ... + + +class _OcrPages(Protocol): + @property + def pages(self) -> Iterable[object]: ... + + +class _OcrPage(Protocol): + @property + def markdown(self) -> str: ... + + class BaseRAGIngestion(ABC): """ Base class for RAG ingestion. @@ -168,7 +186,8 @@ class BaseRAGIngestion(ABC): # Use router if available if self.router is not None: - ocr_response = await self.router.aocr( + ocr_router: Final = require_attributes(self.router, _OcrRouter, "aocr") + ocr_response = await ocr_router.aocr( model=ocr_model, document={"type": doc_type, url_key: data_url}, ) @@ -179,8 +198,10 @@ class BaseRAGIngestion(ABC): ) # Extract text from pages - if hasattr(ocr_response, "pages") and ocr_response.pages: - return "\n\n".join(page.markdown for page in ocr_response.pages if hasattr(page, "markdown")) + if has_attributes(ocr_response, _OcrPages, "pages") and ocr_response.pages: + return "\n\n".join( + page.markdown for page in ocr_response.pages if has_attributes(page, _OcrPage, "markdown") + ) return None diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 85d87554023..9bd8148b286 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -21,7 +21,7 @@ from litellm.constants import ( request_timeout, ) from litellm.exceptions import BadRequestError -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, keyword_arguments from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig @@ -394,7 +394,7 @@ async def acreate_realtime_client_secret( async def acreate_realtime_translation_client_secret( model: str | None = None, session: Mapping[str, Any] | None = None, - expires_after: Mapping[str, Any] | None = None, + expires_after: Mapping[str, object] | None = None, timeout: float | None = None, **kwargs: object, # kwargs-ok: public API preserves provider-specific request options ) -> httpx.Response: @@ -409,7 +409,11 @@ async def acreate_realtime_translation_client_secret( req: Final = RealtimeClientSecretRequest( model=requested_model_name, session=session_config, - expires_after=RealtimeExpiresAfter(**expires_after) if expires_after else None, + expires_after=RealtimeExpiresAfter( + **_ExpiresAfterFields(**keyword_arguments(expires_after, RealtimeExpiresAfter)) + ) + if expires_after + else None, ) litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") if not isinstance(litellm_logging_obj, LiteLLMLogging): @@ -488,10 +492,18 @@ def _deployment_base_model(kwargs: Mapping[str, object], custom_llm_provider: st return None if base_model is None else base_model.removeprefix(f"{custom_llm_provider}/") +class _ExpiresAfterFields(TypedDict, total=False, extra_items=object): + pass + + +class _TranscriptionSessionFields(TypedDict, total=False, extra_items=object): + pass + + @wrapper_client async def acreate_realtime_transcription_session( model: str | None = None, - transcription_session: dict[str, Any] | None = None, + transcription_session: Mapping[str, object] | None = None, timeout: float | None = None, **kwargs, ): @@ -506,7 +518,9 @@ async def acreate_realtime_transcription_session( """ req: Final = RealtimeTranscriptionSessionRequest( model=model, - **(transcription_session or {}), + **_TranscriptionSessionFields( + **keyword_arguments(transcription_session or {}, RealtimeTranscriptionSessionRequest) + ), ) model_name = req.resolved_model() or "gpt-realtime-whisper" litellm_logging_obj: Final[LiteLLMLogging] = kwargs.get("litellm_logging_obj") @@ -626,7 +640,7 @@ async def arealtime_translation_calls( openai_ephemeral_key: str, sdp_body: bytes, model: str | None = None, - session: Mapping[str, Any] | None = None, + session: Mapping[str, object] | None = None, timeout: float | None = None, **kwargs: object, # kwargs-ok: public API preserves provider-specific request options ) -> httpx.Response: diff --git a/litellm/repositories/table_repositories.py b/litellm/repositories/table_repositories.py index fbef73a6114..3c3b87c9b8c 100644 --- a/litellm/repositories/table_repositories.py +++ b/litellm/repositories/table_repositories.py @@ -34,6 +34,16 @@ class _PrismaWriterClientView(Protocol): def writer_db(self) -> object: ... +class _LedgerWriterClientView(Protocol): + @property + def writer_db(self) -> LedgerDatabase: ... + + +class _RawQueryWriterClientView(Protocol): + @property + def writer_db(self) -> LedgerTransaction: ... + + class PrismaTableRepository(Generic[RowT_co]): """Base for repositories that expose a single Prisma table.""" @@ -143,7 +153,7 @@ class SpendLogsRepository(PrismaTableRepository["prisma_models.LiteLLM_SpendLogs ) if not ledger: return legacy_count - database: Final[LedgerDatabase] = self.prisma_client.writer_db + database: Final = require_attributes(self._connected_client(), _LedgerWriterClientView, "writer_db").writer_db async with database.tx(timeout=timedelta(seconds=60)) as transaction: accepted: Final = await self._insert_ledger(transaction, ledger, write_spend_logs=write_spend_logs) return legacy_count + accepted @@ -211,7 +221,7 @@ class SpendLogsRepository(PrismaTableRepository["prisma_models.LiteLLM_SpendLogs return frozenset(str(row["request_id"]) for row in rows) async def has_delivery(self, delivery_id: str) -> bool: - database: Final[LedgerTransaction] = self.prisma_client.writer_db + database: Final = require_attributes(self._connected_client(), _RawQueryWriterClientView, "writer_db").writer_db rows: Final = await database.query_raw( 'SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE delivery_id = $1', delivery_id ) diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index e344ce46be7..8c2a0a36212 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -188,14 +188,16 @@ def rerank( model_response: Final = RerankResponse() - rerank_litellm_params: Final = { - "litellm_call_id": litellm_call_id, - "proxy_server_request": proxy_server_request, - "model_info": model_info, - "preset_cache_key": None, - "stream_response": {}, - **optional_params.model_dump(exclude_unset=True), - } + rerank_litellm_params: Final = dict[str, object]( + { + "litellm_call_id": litellm_call_id, + "proxy_server_request": proxy_server_request, + "model_info": model_info, + "preset_cache_key": None, + "stream_response": {}, + **optional_params.model_dump(exclude_unset=True), + } + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index c3166b26a74..b2c98282da8 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -21,6 +21,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import ( parse_allowed_tools_tool_choice, tool_calls_the_choice_excludes, ) +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager from litellm.proxy.mcp_server.utils import ( @@ -1178,7 +1179,7 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def should_raise_mcp_guardrail_block(error: Exception) -> bool: - return getattr(error, "mcp_block_behavior", None) == "raise" + return attribute_or_none(error, "mcp_block_behavior") == "raise" @staticmethod def create_follow_up_messages_for_chat( diff --git a/litellm/router.py b/litellm/router.py index 98b016f590f..2bf4db967fd 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -234,11 +234,29 @@ def _class_descriptor(cls: type, name: str) -> bool: if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Awaitable from typing import ParamSpec, overload + import httpx + from openai.pagination import AsyncCursorPage + from openai.types.beta.assistant import Assistant + from openai.types.beta.assistant_deleted import AssistantDeleted + from openai.types.beta.threads.message import Message as OpenAIMessage + from openai.types.file_deleted import FileDeleted + + from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper - from litellm.types.llms.openai import AllMessageValues, OpenAIFileObject - from litellm.types.utils import EmbeddingResponse, LiteLLMBatch, ModelResponse + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse + from litellm.types.llms.openai import ( + AllMessageValues, + HttpxBinaryResponseContent, + OpenAIFileObject, + ResponsesAPIResponse, + Thread, + ) + from litellm.types.utils import EmbeddingResponse, LiteLLMBatch, LiteLLMFineTuningJob, ModelResponse + from litellm.types.vector_stores import VectorStoreCreateResponse _P: Final = ParamSpec("_P") @@ -281,6 +299,28 @@ if TYPE_CHECKING: async def acreate_file(self, model: str, **kwargs: object) -> OpenAIFileObject: ... # pyright: ignore[reportIncompatibleMethodOverride] # see acompletion async def acreate_batch(self, model: str, **kwargs: object) -> LiteLLMBatch: ... # pyright: ignore[reportIncompatibleMethodOverride] # see acompletion async def aretrieve_batch(self, model: str | None = None, **kwargs: object) -> LiteLLMBatch: ... # pyright: ignore[reportIncompatibleMethodOverride] # see acompletion + + aget_assistants: Callable[..., Awaitable[AsyncCursorPage[Assistant]]] + acreate_assistants: Callable[..., Awaitable[Assistant]] + adelete_assistant: Callable[..., Awaitable[AssistantDeleted]] + acreate_thread: Callable[..., Awaitable[Thread]] + aget_thread: Callable[..., Awaitable[Thread]] + a_add_message: Callable[..., Awaitable[OpenAIMessage]] + aget_messages: Callable[..., Awaitable[AsyncCursorPage[OpenAIMessage]]] + arun_thread: Callable[..., Awaitable[object]] + adecisions: Callable[..., Awaitable[OpenAIDecisionResponse | DecisionsResponse]] + acreate_fine_tuning_job: Callable[..., Awaitable[LiteLLMFineTuningJob]] + aretrieve_fine_tuning_job: Callable[..., Awaitable[LiteLLMFineTuningJob]] + acancel_fine_tuning_job: Callable[..., Awaitable[LiteLLMFineTuningJob]] + alist_fine_tuning_jobs: Callable[..., Awaitable[object]] + alist_batches: Callable[..., Awaitable[Mapping[str, object]]] + acancel_batch: Callable[..., Awaitable[LiteLLMBatch]] + aget_responses: Callable[..., Awaitable[ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]] + afile_content: Callable[..., Awaitable[HttpxBinaryResponseContent | FileContentStreamingResult]] + afile_delete: Callable[..., Awaitable[FileDeleted]] + avector_store_create: Callable[..., Awaitable[VectorStoreCreateResponse]] + allm_passthrough_route: Callable[..., Awaitable[httpx.Response | AsyncGenerator[bytes, bytes]]] + def __getattr__(self, name: str) -> Any: ... # the endpoints served by name else: diff --git a/litellm/router_api.py b/litellm/router_api.py index a75397ea43d..1b0f970a537 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -60,6 +60,8 @@ if TYPE_CHECKING: from litellm.messages.dispatch import MessagesResult from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.router import Router + from litellm.router_rust.surface.model_list import HostedModelList + from litellm.router_rust.surface.registry import UntypedEntry from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter from litellm.router_strategy.auto_router.auto_router import AutoRouter from litellm.router_strategy.budget_limiter import RouterBudgetLimiting @@ -102,14 +104,25 @@ if TYPE_CHECKING: # Re-exported for litellm.types, which may not import router_utils. AttemptedFallbackTargets: TypeAlias = _AttemptedFallbackTargets + ModelGroupAliases: TypeAlias = dict[str, str | RouterModelGroupAliasItem] + RoutingGroups: TypeAlias = dict[str, RoutingGroup] + AccessGroups: TypeAlias = dict[str, list[str]] + AutoRouters: TypeAlias = dict[str, list[TaggedPreRoutingStrategy[AutoRouter]]] + ComplexityRouters: TypeAlias = dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] + QualityRouters: TypeAlias = dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] + AdaptiveRouters: TypeAlias = dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] + SwitchyardRouters: TypeAlias = dict[str, list[TaggedPreRoutingStrategy[SwitchyardRouter]]] + RoutingPlugins: TypeAlias = list[RoutingPlugin] + ModelNameSet: TypeAlias = set[str] + # What registry().plugins(kind) returns, one member per kind. RegistryPlugins: TypeAlias = ( - dict[str, list[TaggedPreRoutingStrategy[AutoRouter]]] - | dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] - | dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] - | dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] - | dict[str, list[TaggedPreRoutingStrategy[SwitchyardRouter]]] - | list[RoutingPlugin] + AutoRouters + | ComplexityRouters + | QualityRouters + | AdaptiveRouters + | SwitchyardRouters + | RoutingPlugins | RouterBudgetLimiting | DualCache | None @@ -351,6 +364,220 @@ def _json_default_stable_id(obj: object) -> str: return stable_id(obj) +class _RegistryHost(Protocol): + """The host registry members ``RouterRegistry`` reads, typed as the view hands them on.""" + + @property + def _registry_version(self) -> int: ... + + @property + def _core_model_list(self) -> "HostedModelList": ... + + @property + def _core_model_group_alias(self) -> "ModelGroupAliases": ... + + @property + def _core_routing_groups(self) -> "RoutingGroups": ... + + @property + def _core_model_names(self) -> AbstractSet[str]: ... + + @property + def _core_deployment_names(self) -> Sequence[str]: ... + + @property + def _core_team_public_model_names(self) -> frozenset[str]: ... + + @property + def _core_model_name_to_deployment_indices(self) -> Mapping[str, Sequence[int]]: ... + + @property + def _core_team_model_to_deployment_indices(self) -> Mapping[tuple[str, str], Sequence[int]]: ... + + @property + def _core_pattern_router(self) -> "PatternMatchRouter": ... + + @property + def _core_team_pattern_routers(self) -> "Mapping[str, PatternMatchRouter]": ... + + @property + def _core_cached_deployment_model_info(self) -> "_lru_cache_wrapper[ModelMapInfo | None]": ... + + @property + def _core_auto_routers(self) -> "AutoRouters": ... + + @property + def _core_complexity_routers(self) -> "ComplexityRouters": ... + + @property + def _core_quality_routers(self) -> "QualityRouters": ... + + @property + def _core_adaptive_routers(self) -> "AdaptiveRouters": ... + + @property + def _core_switchyard_routers(self) -> "SwitchyardRouters": ... + + @property + def _core_routing_plugins(self) -> "RoutingPlugins": ... + + @property + def _core_prompt_caching_pin_cache(self) -> "DualCache": ... + + def _core_get_deployment(self, model_id: str) -> "Deployment | None": ... + + def _core_has_model_id(self, candidate_id: str) -> bool: ... + + def _core_get_model_info(self, id: str) -> "UntypedEntry | None": ... + + def _core_get_deployment_model_info(self, model_id: str, model_name: str) -> "ModelMapInfo | None": ... + + def _core_get_router_model_info( + self, + deployment: "Mapping[str, object] | Deployment | None", + received_model_name: str, + id: str | None = None, + ) -> "ModelMapInfo": ... + + def _core_get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: ... + + def _core_get_model_list( + self, model_name: str | None = None, team_id: str | None = None + ) -> Sequence[Mapping[str, object]] | None: ... + + def _core_config_deployments(self) -> Iterator[Mapping[str, object]]: ... + + def _core_get_model_list_from_model_alias( + self, model_name: str | None = None + ) -> Sequence[Mapping[str, object]]: ... + + def _core_get_model_list_from_routing_groups( + self, model_name: str | None = None + ) -> Sequence[Mapping[str, object]]: ... + + def _core_get_all_deployments( + self, model_name: str, model_alias: str | None = None, team_id: str | None = None + ) -> Sequence[Mapping[str, object]]: ... + + def _core_deployments_named(self, model_name: str) -> tuple[Mapping[str, object], ...] | None: ... + + def _core_get_model_list_of_routed_group( + self, model_group: str, team_id: str | None = None + ) -> Sequence[Mapping[str, object]]: ... + + def _core_wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: ... + + def _core_get_routable_deployments( + self, model: str, request_kwargs: Mapping[str, object], call_type: str | None = None + ) -> tuple[Mapping[str, object], ...]: ... + + def _core_get_deployment_by_model_group_name(self, model_group_name: str) -> "Deployment | None": ... + + def _core_deployments_for_request( + self, model: str, request_kwargs: Mapping[str, object] + ) -> Sequence[Mapping[str, object]]: ... + + def _core_should_include_deployment( + self, model_name: str, model: Mapping[str, object], team_id: str | None = None + ) -> bool: ... + + def _core_get_model_names(self, team_id: str | None = None) -> Sequence[str]: ... + + def _core_get_model_ids( + self, model_name: str | None = None, exclude_team_models: bool = False + ) -> Sequence[str]: ... + + def team_public_model_names_of(self, team_id: object) -> frozenset[str]: ... + + def _core_resolved_litellm_models( + self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False + ) -> tuple[str, ...]: ... + + def _core_wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: ... + + def _core_get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: ... + + def _core_resolve_model_name_from_model_id( + self, model_id: str | None, custom_llm_provider: str | None = None + ) -> str | None: ... + + def _core_map_team_model( + self, team_model_name: str | None, team_id: str, custom_llm_provider: str | None = None + ) -> str | None: ... + + def _core_routing_group_has_alternatives(self, model_group: str | None) -> bool: ... + + def _core_routable_model_group(self, model_name: str) -> str: ... + + def _core_get_model_from_alias(self, model: str) -> str | None: ... + + def _core_resolve_served_model_name(self, model_name: str, served_model: str) -> str: ... + + def _core_is_recognized_model(self, model: str) -> bool: ... + + def _core_get_model_group(self, id: str) -> Sequence[Mapping[str, object]] | None: ... + + def _core_is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: ... + + def _core_team_model_has_alternatives(self, deployment_id: str) -> bool: ... + + def _core_get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: ... + + def _core_get_routable_upstream_provider(self, model_name: str, team_id: str | None = None) -> str | None: ... + + def _core_get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: ... + + def _core_get_model_group_info( + self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None + ) -> "ModelGroupInfo | None": ... + + def _core_cached_model_group_info(self, model_group: str) -> "ModelGroupInfo | None": ... + + def _core_get_configured_display_name(self, model_name: str) -> str | None: ... + + def _core_get_configured_mode(self, model_name: str) -> str | None: ... + + def _core_get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: ... + + def _core_get_pinned_reasoning_efforts(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: ... + + def _core_get_configured_max_input_tokens( + self, model_name: str, team_id: str | None = None + ) -> tuple[int | None, ...]: ... + + def _core_get_routable_deployment_model_info( + self, model_name: str, team_id: str | None = None + ) -> "tuple[ModelMapInfo | None, ...]": ... + + def _core_get_configured_token_limits(self, model_name: str) -> tuple[int | None, int | None]: ... + + def _core_get_model_listing_info(self, model_name: str) -> "DeploymentModelListingInfo | None": ... + + def get_routing_group(self, model_name: str) -> "RoutingGroup | None": ... + + def _get_client( + self, deployment: Mapping[str, object], kwargs: Mapping[str, object], client_type: str | None = None + ) -> object: ... + + def _core_get_model_access_groups( + self, model_name: str | None = None, model_access_group: str | None = None, team_id: str | None = None + ) -> "AccessGroups": ... + + def _core_get_deployment_credentials(self, model_id: str) -> Mapping[str, object] | None: ... + + def _core_get_deployment_credentials_with_provider( + self, model_id: str, team_id: str | None = None + ) -> Mapping[str, object] | None: ... + + def _core_get_credential_deployment(self, model_id: str, team_id: str | None = None) -> "Deployment | None": ... + + def _core_get_router_deployment_budget_limiter(self) -> "RouterBudgetLimiting | None": ... + + def _core_decryptable_reasoning( + self, model_group: str, request_input: Sequence[object], targets: "Sequence[Deployment]" + ) -> tuple[object, ...]: ... + + class RouterRegistry: """ Read-only queries over the configured deployments (``registry()``). @@ -361,7 +588,7 @@ class RouterRegistry: __slots__ = ("_router",) - def __init__(self, router: "Router") -> None: + def __init__(self, router: _RegistryHost) -> None: self._router = router @property @@ -407,12 +634,8 @@ class RouterRegistry: id: str | None = None, ) -> "ModelMapInfo": """Model info used for routing decisions (was ``get_router_model_info``).""" - r: Final = self._router - # The core's overloads split (deployment, id) by which one is None; this forwards either form unchanged. - return r._core_get_router_model_info( # pyright: ignore[reportCallIssue] # see above - deployment=deployment, # pyright: ignore[reportArgumentType] # see above - received_model_name=received_model_name, - id=id, # pyright: ignore[reportArgumentType] # see above + return self._router._core_get_router_model_info( + deployment=deployment, received_model_name=received_model_name, id=id ) def discovered_model_info(self, id: str | None) -> Mapping[str, int]: @@ -1316,12 +1539,52 @@ class RouterSettings: return self._owner("_core_deployment_call_timeout")._core_deployment_call_timeout(litellm_params) +class _HealthHost(Protocol): + """The router members ``RouterHealth`` reads, typed as the view hands them on.""" + + @property + def _core_cache(self) -> "DualCache": ... + + @property + def _core_cooldown_cache(self) -> "CooldownCache": ... + + @property + def _core_health_state_cache(self) -> "DeploymentHealthCache": ... + + @property + def _core_deployment_latency_map(self) -> "UntypedDict": ... + + @property + def _core_ctx(self) -> "RoutingContext": ... + + @property + def _core_cooldown_time(self) -> float | None: ... + + def _core_get_fully_blocked_model_names(self) -> "ModelNameSet": ... + + async def _core_async_get_fully_unhealthy_model_names(self) -> "ModelNameSet": ... + + async def _core_get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: ... + + async def _core_get_remaining_model_group_usage(self, model_group: str) -> Mapping[str, int]: ... + + def _core_team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None": ... + + def _core_split_request_priority( + self, model: str, kwargs: Mapping[str, object] + ) -> tuple[int | None, Mapping[str, object]]: ... + + def priority_header_update( + self, model: str, kwargs: Mapping[str, object], header: str | None + ) -> Mapping[str, int | str]: ... + + class RouterHealth: """Cooldowns, blocked deployments, health checks and usage (``health()``).""" __slots__ = ("_router",) - def __init__(self, router: "Router") -> None: + def __init__(self, router: _HealthHost) -> None: self._router = router @property diff --git a/litellm/router_rust/invoke/streaming.py b/litellm/router_rust/invoke/streaming.py index 9286e9aa06c..b1de96fd91b 100644 --- a/litellm/router_rust/invoke/streaming.py +++ b/litellm/router_rust/invoke/streaming.py @@ -42,6 +42,7 @@ from collections.abc import ( Mapping, Sequence, ) +from collections.abc import Set as AbstractSet from typing import ( TYPE_CHECKING, Final, @@ -629,8 +630,8 @@ def continuation_routing(core: "RustCore", request: Mapping[str, object]) -> Map registry: Final = core.registry() excluded: Final = [ deployment.model_info.id - for record in registry.all() - if (deployment := registry.deployment(str(_record_id(record)))) is not None + for record_id in _record_ids(registry.all()) + if (deployment := registry.deployment(str(record_id))) is not None and not deployment_can_continue(deployment.model_dump(exclude_none=True)) ] return {"body": {CONTINUATION_EXCLUDED_IDS: excluded}} @@ -641,13 +642,21 @@ def _record_id(record: Mapping[str, object]) -> object: return model_info.get("id") if isinstance(model_info, Mapping) else None # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # a plain record +def _record_ids(records: Iterable[Mapping[str, object]]) -> Iterator[object]: + return (_record_id(record) for record in records) + + +def _record_model_names(records: Iterable[Mapping[str, object]]) -> AbstractSet[object]: + return {record.get("model_name") for record in records} + + def _names_deployment_that_cannot_continue(core: "RustCore", target: object) -> bool: from litellm.router_utils.fallback_event_handlers import fallback_target_name from litellm.router_utils.pre_call_checks.continuation_prefill_check import deployment_can_continue name: Final = fallback_target_name(target) registry: Final = core.registry() - model_names: Final = {record.get("model_name") for record in registry.all()} + model_names: Final = _record_model_names(registry.all()) if name is None or name in model_names or not registry.has_deployment(name): return False deployment: Final = registry.deployment(name) diff --git a/litellm/router_rust/surface/legacy_members.py b/litellm/router_rust/surface/legacy_members.py index 046501f38f3..640fdf739c0 100644 --- a/litellm/router_rust/surface/legacy_members.py +++ b/litellm/router_rust/surface/legacy_members.py @@ -289,8 +289,8 @@ class RustLegacyMembers(RustRegistryMembers, RustOptionalChecks): validate_routing_strategy(routing_strategy) # pyright: ignore[reportArgumentType] # the strategy as given # `_reset_custom_routing_strategy`. self._custom_strategy = None - _ = self.__dict__.pop("get_available_deployment", None) - _ = self.__dict__.pop("_core_async_get_available_deployment", None) + self.__dict__.pop("get_available_deployment", None) + self.__dict__.pop("_core_async_get_available_deployment", None) self._sel.init_default(routing_strategy, routing_strategy_args, configured=False) def set_optional_pre_call_checks(self, optional_pre_call_checks: Iterable[object] | None) -> None: @@ -301,7 +301,7 @@ class RustLegacyMembers(RustRegistryMembers, RustOptionalChecks): """``initialize_assistants_endpoint`` binds the endpoints; this core answers them from ``__getattr__``, so what is left to do is to drop an assigned replacement.""" for name in _ASSISTANTS_ENDPOINTS: - _ = self.__dict__.pop(name, None) + self.__dict__.pop(name, None) def initialize_router_endpoints(self) -> None: """``initialize_router_endpoints``, as ``initialize_assistants_endpoint`` above.""" @@ -310,7 +310,7 @@ class RustLegacyMembers(RustRegistryMembers, RustOptionalChecks): ) for name in (*_ROUTER_ENDPOINTS, *_generated_endpoints): - _ = self.__dict__.pop(name, None) + self.__dict__.pop(name, None) # -- pre-routing strategies -------------------------------------------------------------------- diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index c3b038b5f26..094c61e0550 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -103,7 +103,7 @@ class QualityRouter(CustomLogger): self._tier_to_models_cache = self._build_tier_index() return self._tier_to_models_cache - def _get_routing_preferences(self, deployment: object) -> dict[str, Any] | None: + def _get_routing_preferences(self, deployment: object) -> dict[str, object] | None: """ Extract litellm_routing_preferences from a deployment, handling both dict-shaped and Pydantic-object-shaped deployments. diff --git a/litellm/router_utils/cooldown_callbacks.py b/litellm/router_utils/cooldown_callbacks.py index 9e90b72ba1b..09bbeb92836 100644 --- a/litellm/router_utils/cooldown_callbacks.py +++ b/litellm/router_utils/cooldown_callbacks.py @@ -42,8 +42,7 @@ async def router_cooldown_event_callback( temp_litellm_params = dict(temp_litellm_params) _model_name: Final = _deployment.model_name or "" _api_base: Final = litellm.get_api_base(model=_model_name, optional_params=temp_litellm_params) or "" - model_info: Final = _deployment["model_info"] - model_id: Final = model_info.id + model_id: Final = _deployment.model_info.id litellm_model_name: Final = temp_litellm_params.get("model") or "" llm_provider = "" @@ -68,7 +67,7 @@ async def router_cooldown_event_callback( prometheusLogger.increment_deployment_cooled_down( litellm_model_name=_model_name, - model_id=model_id, + model_id=str(model_id), api_base=_api_base, api_provider=llm_provider, exception_status=str(exception_status), diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 22bec9e1ead..9e3e0189fd0 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -149,7 +149,7 @@ _EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = ( ) -def _first_present(*sources: Mapping[str, Any] | None, key: str) -> int | float | None: +def _first_present(*sources: Mapping[str, object] | None, key: str) -> int | float | None: """Return *key* from the first source mapping where it's set, so callers can support a setting living in more than one deployment config location. Sources are checked in order from most to least specific to that setting. Only a number diff --git a/litellm/secret_managers/secret_manager_handler.py b/litellm/secret_managers/secret_manager_handler.py index f1f38c384cc..afed7532d74 100644 --- a/litellm/secret_managers/secret_manager_handler.py +++ b/litellm/secret_managers/secret_manager_handler.py @@ -7,22 +7,13 @@ Handles retrieving secrets from different secret management systems. import base64 import os from collections.abc import Mapping -from typing import Any, Final, Generic, Protocol, TypeVar - -from typing_extensions import ReadOnly, TypedDict +from typing import Final, Protocol import litellm from litellm._logging import print_verbose +from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem -_ClientT = TypeVar("_ClientT") - - -class _SecretManagerClientView(TypedDict, Generic[_ClientT]): - """Typed read of the untyped secret manager handle configured for this key manager.""" - - client: ReadOnly[_ClientT] - class _AzureKeyVaultSecret(Protocol): @property @@ -74,7 +65,7 @@ def _is_base64(s: str) -> bool: def get_secret_from_manager( - client: Any, + client: object, key_manager: str, secret_name: str, key_management_settings: KeyManagementSettings | None = None, @@ -96,16 +87,14 @@ def get_secret_from_manager( Exception: For other errors during secret retrieval """ secret = None - raw_view: Final[_SecretManagerClientView[object]] = {"client": client} - client_object: Final = raw_view["client"] + client_object: Final = client if ( key_manager == KeyManagementSystem.AZURE_KEY_VAULT.value or type(client_object).__module__ + "." + type(client_object).__name__ == "azure.keyvault.secrets._client.SecretClient" ): # support Azure Secret Client - from azure.keyvault.secrets import SecretClient - azure_view: Final[_SecretManagerClientView[_AzureKeyVaultClient]] = {"client": client} - azure_client: Final = azure_view["client"] + azure_client: Final = require_attributes(client, _AzureKeyVaultClient, "get_secret") secret = azure_client.get_secret(secret_name).value elif ( @@ -122,8 +111,7 @@ def get_secret_from_manager( raise ValueError( "Google KMS requires the encrypted secret to be encoded in base64" ) # fix for this vulnerability https://huntr.com/bounties/ae623c2f-b64b-4245-9ed4-f13a0a5824ce - google_kms_view: Final[_SecretManagerClientView[_GoogleKmsClient]] = {"client": client} - google_kms_client: Final = google_kms_view["client"] + google_kms_client: Final = require_attributes(client, _GoogleKmsClient, "decrypt") google_kms_response: Final = google_kms_client.decrypt( request={ "name": litellm._google_kms_resource_name, @@ -143,8 +131,7 @@ def get_secret_from_manager( ciphertext_blob: Final = base64.b64decode(encrypted_value) # Perform the decryption - aws_kms_view: Final[_SecretManagerClientView[_AwsKmsClient]] = {"client": client} - aws_kms_client: Final = aws_kms_view["client"] + aws_kms_client: Final = require_attributes(client, _AwsKmsClient, "decrypt") aws_kms_response: Final = aws_kms_client.decrypt(CiphertextBlob=ciphertext_blob) # Extract and decode the plaintext @@ -171,8 +158,9 @@ def get_secret_from_manager( elif key_manager == KeyManagementSystem.GOOGLE_SECRET_MANAGER.value: try: - google_secret_manager_view: Final[_SecretManagerClientView[_GoogleSecretManagerClient]] = {"client": client} - google_secret_manager_client: Final = google_secret_manager_view["client"] + google_secret_manager_client: Final = require_attributes( + client, _GoogleSecretManagerClient, "get_secret_from_google_secret_manager" + ) secret = google_secret_manager_client.get_secret_from_google_secret_manager(secret_name) print_verbose(f"secret from google secret manager: [set={secret is not None}]") if secret is None: @@ -183,8 +171,7 @@ def get_secret_from_manager( elif key_manager == KeyManagementSystem.HASHICORP_VAULT.value: try: - hashicorp_view: Final[_SecretManagerClientView[_SyncSecretReader]] = {"client": client} - hashicorp_client: Final = hashicorp_view["client"] + hashicorp_client: Final = require_attributes(client, _SyncSecretReader, "sync_read_secret") secret = hashicorp_client.sync_read_secret(secret_name=secret_name) if secret is None: raise ValueError(f"No secret found in Hashicorp Secret Manager for {secret_name}") @@ -194,8 +181,7 @@ def get_secret_from_manager( elif key_manager == KeyManagementSystem.CYBERARK.value: try: - cyberark_view: Final[_SecretManagerClientView[_SyncSecretReader]] = {"client": client} - cyberark_client: Final = cyberark_view["client"] + cyberark_client: Final = require_attributes(client, _SyncSecretReader, "sync_read_secret") secret = cyberark_client.sync_read_secret(secret_name=secret_name) if secret is None: raise ValueError(f"No secret found in CyberArk Secret Manager for {secret_name}") @@ -224,8 +210,7 @@ def get_secret_from_manager( secret = os.getenv(secret_name) else: # assume the default is infisicial client - infisical_view: Final[_SecretManagerClientView[_InfisicalClient]] = {"client": client} - infisical_client: Final = infisical_view["client"] + infisical_client: Final = require_attributes(client, _InfisicalClient, "get_secret") secret = infisical_client.get_secret(secret_name).secret_value return secret diff --git a/litellm/text_completions/streaming_iterator.py b/litellm/text_completions/streaming_iterator.py index daf65b77a85..517a499ec8c 100644 --- a/litellm/text_completions/streaming_iterator.py +++ b/litellm/text_completions/streaming_iterator.py @@ -1,11 +1,15 @@ -from typing import Final +from typing import Final, Protocol from litellm.constants import VLLM_CHOICE_FIELDS -from litellm.litellm_core_utils.core_helpers import decoded_json_object +from litellm.litellm_core_utils.core_helpers import decoded_json_object, json_contains, require_attributes, subscript from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse +class _SupportsGet(Protocol): + def get(self, key: str, /) -> object: ... + + class TextCompletionStreamWrapper: def __init__( self, @@ -43,16 +47,22 @@ class TextCompletionStreamWrapper: raise Exception if chunk["choices"]: - chat_choice: Final = chunk["choices"][0] + chat_choice: Final = subscript(subscript(chunk, "choices"), 0) choice_fields: Final = decoded_json_object( - {field: chat_choice[field] for field in VLLM_CHOICE_FIELDS if field in chat_choice} + { + field: subscript(chat_choice, field) + for field in VLLM_CHOICE_FIELDS + if json_contains(chat_choice, field) + } ) text_choices: Final = TextChoices(index=0, **choice_fields) - delta: Final = chat_choice["delta"] - text_choices["text"] = delta["content"] - text_choices["reasoning_content"] = delta.get("reasoning_content") - text_choices["index"] = chat_choice["index"] - text_choices["finish_reason"] = chat_choice["finish_reason"] + delta: Final = subscript(chat_choice, "delta") + text_choices["text"] = subscript(delta, "content") + text_choices["reasoning_content"] = require_attributes(delta, _SupportsGet, "get").get( + "reasoning_content" + ) + text_choices["index"] = subscript(chat_choice, "index") + text_choices["finish_reason"] = subscript(chat_choice, "finish_reason") response["choices"] = [text_choices] else: response["choices"] = [] diff --git a/litellm/utils.py b/litellm/utils.py index fb974b24283..7012741148c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -95,7 +95,9 @@ from litellm.litellm_core_utils.core_helpers import ( bind_budget_reservation_to_callbacks, has_attributes, iter_items, + json_contains, normalize_drop_params, + require_attributes, unbind_budget_reservation_from_callbacks, ) from litellm.litellm_core_utils.fallback_generalizations import ( @@ -581,6 +583,7 @@ additional_details: Final[dict[str, str] | None] = {} local_cache: Final[dict[str, str] | None] = {} last_fetched_at: Final = None last_fetched_at_keys: Final = None +_OPAQUE_VALUE: Final = TypeAdapter(object) ######## Model Response ######################### # All liteLLM Model responses will be in this format, Follows the OpenAI Format @@ -765,14 +768,20 @@ def _is_same_registered_custom_logger(existing: object, callback_class: CustomLo ) -def get_request_guardrails(kwargs: dict[str, Any]) -> list[str]: +class _SupportsGet(Protocol): + def get(self, key: str, /) -> object: ... + + +def _metadata_field(metadata: object, key: str) -> object: + return require_attributes(metadata, _SupportsGet, "get").get(key) if metadata else None + + +def get_request_guardrails(kwargs: Mapping[str, object]) -> object: """ Get the request guardrails from the kwargs """ - metadata: Final = kwargs.get("metadata") or {} - requester_metadata: Final = metadata.get("requester_metadata") or {} - applied_guardrails: Final = requester_metadata.get("guardrails") or [] - return applied_guardrails + requester_metadata: Final = _metadata_field(kwargs.get("metadata"), "requester_metadata") + return _metadata_field(requester_metadata, "guardrails") or [] def get_applied_guardrails(kwargs: dict[str, object]) -> list[str]: @@ -787,7 +796,7 @@ def get_applied_guardrails(kwargs: dict[str, object]) -> list[str]: for callback in litellm.callbacks: if callback is not None and isinstance(callback, CustomGuardrail): if callback.guardrail_name is not None: - if callback.default_on is True or callback.guardrail_name in request_guardrails: + if callback.default_on is True or json_contains(request_guardrails, callback.guardrail_name): applied_guardrails.append(callback.guardrail_name) return applied_guardrails @@ -1431,7 +1440,7 @@ def function_setup( level="info", ) if "logger_fn" in kwargs: - user_logger_fn = kwargs["logger_fn"] + user_logger_fn = _OPAQUE_VALUE.validate_python(kwargs["logger_fn"]) # INIT LOGGER - for user-specified integrations model: Final = args[0] if len(args) > 0 else kwargs.get("model", None) call_type: Final = original_function @@ -1515,9 +1524,9 @@ def function_setup( elif call_type == CallTypes.embedding.value or call_type == CallTypes.aembedding.value: messages = args[1] if len(args) > 1 else kwargs.get("input", None) elif call_type == CallTypes.image_generation.value or call_type == CallTypes.aimage_generation.value: - messages = args[0] if len(args) > 0 else kwargs["prompt"] + messages = _OPAQUE_VALUE.validate_python(args[0] if len(args) > 0 else kwargs["prompt"]) elif call_type == CallTypes.moderation.value or call_type == CallTypes.amoderation.value: - messages = args[1] if len(args) > 1 else kwargs["input"] + messages = _OPAQUE_VALUE.validate_python(args[1] if len(args) > 1 else kwargs["input"]) elif call_type == CallTypes.atext_completion.value or call_type == CallTypes.text_completion.value: messages = args[0] if len(args) > 0 else kwargs.get("prompt") elif call_type == CallTypes.rerank.value or call_type == CallTypes.arerank.value: @@ -1544,10 +1553,10 @@ def function_setup( kwargs["metadata"] = {"file_checksum": file_checksum} messages = file_checksum elif call_type == CallTypes.aspeech.value or call_type == CallTypes.speech.value: - messages = kwargs.get("input", "speech") + messages = _OPAQUE_VALUE.validate_python(kwargs.get("input", "speech")) elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value: # Handle both 'input' (standard Responses API) and 'messages' (Cursor chat format) - messages = ( + messages = _OPAQUE_VALUE.validate_python( args[0] if len(args) > 0 else kwargs.get("input") or kwargs.get("messages", "default-message-value") ) elif ( @@ -1727,9 +1736,9 @@ def _schedule_async_success_logging( async def _client_async_logging_helper( logging_obj: LiteLLMLoggingObject, - result, - start_time, - end_time, + result: object, + start_time: datetime.datetime, + end_time: datetime.datetime, is_completion_with_fallbacks: bool, ): if ( @@ -1748,7 +1757,7 @@ async def _client_async_logging_helper( ) -def _get_wrapper_num_retries(kwargs: dict[str, Any], exception: Exception) -> tuple[int | None, dict[str, Any]]: +def _get_wrapper_num_retries(kwargs: dict[str, object], exception: Exception) -> tuple[object, dict[str, object]]: """ Get the number of retries from the kwargs and the retry policy. Used for the wrapper functions. @@ -1773,6 +1782,17 @@ def _get_wrapper_num_retries(kwargs: dict[str, Any], exception: Exception) -> tu return num_retries, kwargs +@runtime_checkable +class _ModelFallbackLookup(Protocol): + def __getitem__(self, model: str | None, /) -> object: ... + + +def _context_window_fallback_model(fallbacks: object, model: str | None) -> object: + if isinstance(fallbacks, _ModelFallbackLookup): + return fallbacks[model] + raise TypeError(f"'{type(fallbacks).__name__}' object is not subscriptable") + + def _get_wrapper_timeout(kwargs: dict[str, object]) -> object: from litellm.litellm_core_utils.litellm_logging import Logging @@ -2661,7 +2681,7 @@ def client(original_function): retry_call_type: Final = _get_call_type_from_public_name(original_function.__name__) num_retries, kwargs = _get_wrapper_num_retries(kwargs=kwargs, exception=e) if retry_call_type == CallTypes.acompletion.value: - context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {}) + context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict") is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs) @@ -2689,13 +2709,13 @@ def client(original_function): elif ( isinstance(e, litellm.exceptions.ContextWindowExceededError) and context_window_fallback_dict - and model in context_window_fallback_dict + and json_contains(context_window_fallback_dict, model) and not is_acompletion_litellm_router_call ): if len(args) > 0: - args[0] = context_window_fallback_dict[model] + args[0] = _context_window_fallback_model(context_window_fallback_dict, model) else: - kwargs["model"] = context_window_fallback_dict[model] + kwargs["model"] = _context_window_fallback_model(context_window_fallback_dict, model) result = await original_function(*args, **kwargs) return result elif retry_call_type == CallTypes.aresponses.value: @@ -4469,7 +4489,7 @@ def get_optional_params_transcription( passed_params.pop("custom_llm_provider") passed_params.pop("drop_params") normalized_drop_params: Final = normalize_drop_params(drop_params) - special_params: Final[Mapping[str, object]] = passed_params.pop("kwargs") + special_params: Final = as_str_mapping(_OPAQUE_VALUE.validate_python(passed_params.pop("kwargs"))) or {} from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( transcription_stream_flag, transcription_stream_unset, @@ -4494,9 +4514,10 @@ def get_optional_params_transcription( message="`stream` must be a boolean. To drop it from the call, set `litellm.drop_params = True`.", ) + passed_entries: Final[Mapping[str, object]] = MappingProxyType(passed_params) non_default_params: Final = { k: v - for k, v in passed_params.items() + for k, v in passed_entries.items() if k in default_params and v != default_params[k] and not (k == "stream" and (stream_unset or malformed_stream)) } optional_params = {} @@ -4735,7 +4756,7 @@ def get_optional_params_embeddings( # retrieve all parameters passed to the function passed_params: Final = locals() passed_params.pop("custom_llm_provider", None) - special_params: Final = passed_params.pop("kwargs") + special_params: Final = as_str_mapping(_OPAQUE_VALUE.validate_python(passed_params.pop("kwargs"))) or {} passed_params.pop("drop_params", None) drop_params = normalize_drop_params(drop_params) @@ -5196,7 +5217,7 @@ class PreProcessNonDefaultParams: @staticmethod def base_pre_process_non_default_params( passed_params: dict, - special_params: dict, + special_params: Mapping[str, object], custom_llm_provider: str, additional_drop_params: list[str] | None, default_param_values: dict, @@ -5241,7 +5262,7 @@ class PreProcessNonDefaultParams: @staticmethod def embedding_pre_process_non_default_params( passed_params: dict, - special_params: dict, + special_params: Mapping[str, object], custom_llm_provider: str, additional_drop_params: list[str] | None, model: str, @@ -5262,7 +5283,7 @@ class PreProcessNonDefaultParams: def pre_process_non_default_params( passed_params: dict, - special_params: dict, + special_params: Mapping[str, object], custom_llm_provider: str, additional_drop_params: list[str] | None, model: str, @@ -7585,7 +7606,8 @@ def get_model_info_helper( token_count_tokenizer=_model_info.get("token_count_tokenizer", None), vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None), ) - for cost_key, cost_value in _model_info.items(): + cost_entries: Final[Mapping[str, object]] = MappingProxyType(_model_info) + for cost_key, cost_value in cost_entries.items(): if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None: returned_model_info[cost_key] = cost_value return returned_model_info @@ -7805,7 +7827,7 @@ def function_to_dict(input_function) -> dict: # may represent a set of acceptable values # translating as enum for function calling try: - param_enum = str(list(literal_eval(param_type))) + param_enum = str(list(iter_items(_OPAQUE_VALUE.validate_python(literal_eval(param_type))))) param_type = "string" except Exception: pass @@ -10774,7 +10796,7 @@ class ProviderConfigManager: @staticmethod def get_provider_passthrough_config( model: str, - provider: LlmProviders, + provider: LlmProviders | str, ) -> BasePassthroughConfig | None: if LlmProviders.BEDROCK == provider: from litellm.llms.bedrock.passthrough.transformation import ( diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py index b83c9ef48b1..2adaf17ac44 100644 --- a/litellm/vector_stores/vector_store_registry.py +++ b/litellm/vector_stores/vector_store_registry.py @@ -1,6 +1,6 @@ # litellm/proxy/vector_stores/vector_store_registry.py import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from datetime import datetime, timezone from itertools import chain from typing import ( @@ -12,6 +12,7 @@ from typing import ( ) from pydantic import ConfigDict, TypeAdapter, ValidationError +from typing_extensions import TypedDict from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import remove_items_at_indices @@ -39,6 +40,14 @@ _TOOL: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True)) _STORE_ID_ITEMS: Final = TypeAdapter(list[object] | tuple[object, ...], config=ConfigDict(strict=True)) +class _ManagedVectorStoreRowFields(TypedDict, total=False, extra_items=object): + pass + + +def _row_fields(row: Iterable[tuple[str, object]]) -> Mapping[str, object]: + return dict(row) + + def _named_store_ids(store_ids: object) -> tuple[str, ...]: if isinstance(store_ids, str): return (store_ids,) @@ -538,8 +547,10 @@ class VectorStoreRegistry: order={"created_at": "desc"}, ) for vector_store in _vector_stores_from_db: - _dict_vector_store = dict(vector_store) - _litellm_managed_vector_store = LiteLLM_ManagedVectorStore(**_dict_vector_store) + _dict_vector_store = _row_fields(vector_store) + _litellm_managed_vector_store = LiteLLM_ManagedVectorStore( + **_ManagedVectorStoreRowFields(**_dict_vector_store) + ) vector_stores_from_db.append(_litellm_managed_vector_store) return vector_stores_from_db diff --git a/litellm/videos/main.py b/litellm/videos/main.py index 66eb10e30db..f2681b03cb2 100644 --- a/litellm/videos/main.py +++ b/litellm/videos/main.py @@ -6,10 +6,13 @@ from functools import partial from typing import Final, Literal, overload from httpx._types import FileContent +from pydantic import TypeAdapter +from typing_extensions import TypedDict import litellm from litellm.constants import DEFAULT_VIDEO_ENDPOINT_MODEL from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT +from litellm.litellm_core_utils.core_helpers import iter_items, keyword_arguments from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.videos.transformation import BaseVideoConfig @@ -26,6 +29,16 @@ from litellm.types.videos.utils import decode_video_id_with_provider from litellm.utils import ProviderConfigManager, client from litellm.videos.utils import VideoGenerationRequestUtils +_DECODED_JSON: Final = TypeAdapter(object) + + +class _VideoObjectFields(TypedDict, total=False, extra_items=object): + pass + + +class _CharacterObjectFields(TypedDict, total=False, extra_items=object): + pass + def _unsupported_video_call( call_name: str, custom_llm_provider: str, model: str | None = None @@ -204,9 +217,11 @@ def video_generation( mock_response = kwargs.get("mock_response", None) if mock_response is not None: if isinstance(mock_response, str): - mock_response = json.loads(mock_response) + mock_response = _DECODED_JSON.validate_python(json.loads(mock_response)) - response: Final = VideoObject(**mock_response) + response: Final = VideoObject.model_validate( + _VideoObjectFields(**keyword_arguments(mock_response, VideoObject)) + ) return response # get llm provider logic @@ -593,9 +608,11 @@ def video_remix( mock_response = kwargs.get("mock_response", None) if mock_response is not None: if isinstance(mock_response, str): - mock_response = json.loads(mock_response) + mock_response = _DECODED_JSON.validate_python(json.loads(mock_response)) - response: Final = VideoObject(**mock_response) + response: Final = VideoObject.model_validate( + _VideoObjectFields(**keyword_arguments(mock_response, VideoObject)) + ) return response # Try to decode provider from video_id if not explicitly provided @@ -808,8 +825,11 @@ def video_list( mock_response = kwargs.get("mock_response", None) if mock_response is not None: if isinstance(mock_response, str): - mock_response = json.loads(mock_response) - return [VideoObject(**item) for item in mock_response] + mock_response = _DECODED_JSON.validate_python(json.loads(mock_response)) + return [ + VideoObject.model_validate(_VideoObjectFields(**keyword_arguments(item, VideoObject))) + for item in iter_items(mock_response) + ] # Ensure custom_llm_provider is not None - default to openai if not provided if custom_llm_provider is None: @@ -1026,9 +1046,11 @@ def video_status( mock_response = kwargs.get("mock_response", None) if mock_response is not None: if isinstance(mock_response, str): - mock_response = json.loads(mock_response) + mock_response = _DECODED_JSON.validate_python(json.loads(mock_response)) - response: Final = VideoObject(**mock_response) + response: Final = VideoObject.model_validate( + _VideoObjectFields(**keyword_arguments(mock_response, VideoObject)) + ) return response # Try to decode provider from video_id if not explicitly provided @@ -1173,8 +1195,10 @@ def video_create_character( mock_response = kwargs.get("mock_response", None) if mock_response is not None: if isinstance(mock_response, str): - mock_response = json.loads(mock_response) - return CharacterObject(**mock_response) + mock_response = _DECODED_JSON.validate_python(json.loads(mock_response)) + return CharacterObject.model_validate( + _CharacterObjectFields(**keyword_arguments(mock_response, CharacterObject)) + ) if custom_llm_provider is None: custom_llm_provider = "openai" @@ -1298,8 +1322,10 @@ def video_get_character( mock_response = kwargs.get("mock_response", None) if mock_response is not None: if isinstance(mock_response, str): - mock_response = json.loads(mock_response) - return CharacterObject(**mock_response) + mock_response = _DECODED_JSON.validate_python(json.loads(mock_response)) + return CharacterObject.model_validate( + _CharacterObjectFields(**keyword_arguments(mock_response, CharacterObject)) + ) if custom_llm_provider is None: custom_llm_provider = "openai" @@ -1430,8 +1456,8 @@ def video_edit( mock_response = kwargs.get("mock_response", None) if mock_response is not None: if isinstance(mock_response, str): - mock_response = json.loads(mock_response) - return VideoObject(**mock_response) + mock_response = _DECODED_JSON.validate_python(json.loads(mock_response)) + return VideoObject.model_validate(_VideoObjectFields(**keyword_arguments(mock_response, VideoObject))) if custom_llm_provider is None: decoded: Final = decode_video_id_with_provider(video_id) @@ -1564,8 +1590,8 @@ def video_extension( mock_response = kwargs.get("mock_response", None) if mock_response is not None: if isinstance(mock_response, str): - mock_response = json.loads(mock_response) - return VideoObject(**mock_response) + mock_response = _DECODED_JSON.validate_python(json.loads(mock_response)) + return VideoObject.model_validate(_VideoObjectFields(**keyword_arguments(mock_response, VideoObject))) if custom_llm_provider is None: decoded: Final = decode_video_id_with_provider(video_id) diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index beb1b58ec27..793c40bcf83 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -9935,3 +9935,134 @@ def test_chunk_creator_usage_only_chunk_with_dict_usage_keeps_details( assert usage.prompt_tokens_details.cached_tokens == 80 assert usage.completion_tokens_details is not None assert usage.completion_tokens_details.reasoning_tokens == 5 + + +def _stream_of(chunks: Sequence[object], sync_mode: bool) -> Iterator[object] | AsyncIterator[object]: + return iter(chunks) if sync_mode else _as_async(iter(chunks)) + + +def _streamed_text_and_finish_reasons(received: Sequence[ModelResponseStream]) -> tuple[str, list[str]]: + return ( + "".join(chunk.choices[0].delta.content or "" for chunk in received if chunk.choices), + [chunk.choices[0].finish_reason for chunk in received if chunk.choices and chunk.choices[0].finish_reason], + ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_predibase_byte_stream_yields_token_text_then_the_reported_finish_reason( + sync_mode: bool, logging_obj: Logging +) -> None: + lines: Final = ( + b'data: {"token": {"id": 1, "text": "Hel"}}', + b'data: {"token": {"id": 2, "text": "lo"}, "details": {"finish_reason": "length"}}', + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=_stream_of(lines, sync_mode), + model="llama-3-1-8b-instruct", + custom_llm_provider="predibase", + logging_obj=logging_obj, + ) + + received: Final = await _collect(wrapper, sync_mode) + + assert _streamed_text_and_finish_reasons(received) == ("Hello", ["length"]) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_predibase_error_event_fails_the_stream_with_the_upstream_message( + sync_mode: bool, logging_obj: Logging +) -> None: + lines: Final = ( + b'data: {"token": {"id": 1, "text": "Hel"}}', + b'data: {"error": "Request failed during generation: CUDA out of memory"}', + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=_stream_of(lines, sync_mode), + model="llama-3-1-8b-instruct", + custom_llm_provider="predibase", + logging_obj=logging_obj, + ) + + with pytest.raises(MidStreamFallbackError) as raised: + await _collect(wrapper, sync_mode) + + assert "CUDA out of memory" in str(raised.value) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_nlp_cloud_stream_drops_the_done_marker_and_finishes_with_stop( + sync_mode: bool, logging_obj: Logging +) -> None: + lines: Final = ('{"generated_text": "Hello"}', '{"generated_text": " world[DONE]"}') + wrapper: Final = CustomStreamWrapper( + completion_stream=_stream_of(lines, sync_mode), + model="finetuned-llama-3-70b", + custom_llm_provider="nlp_cloud", + logging_obj=logging_obj, + ) + + received: Final = await _collect(wrapper, sync_mode) + + assert _streamed_text_and_finish_reasons(received) == ("Hello world", ["stop"]) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_aleph_alpha_stream_yields_the_completion_text_and_stops(sync_mode: bool, logging_obj: Logging) -> None: + lines: Final = (b'{"completions": [{"completion": " Paris", "finish_reason": "maximum_tokens"}]}',) + wrapper: Final = CustomStreamWrapper( + completion_stream=_stream_of(lines, sync_mode), + model="luminous-base", + custom_llm_provider="aleph_alpha", + logging_obj=logging_obj, + ) + + received: Final = await _collect(wrapper, sync_mode) + + assert _streamed_text_and_finish_reasons(received) == (" Paris", ["stop"]) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_generic_tool_use_streams_as_a_function_call_when_the_caller_passed_functions( + sync_mode: bool, logging_obj: Logging +) -> None: + logging_obj.update_environment_variables( + model="bedrock/invoke-endpoint", + user=None, + optional_params={"functions": [{"name": "get_weather", "parameters": {"type": "object", "properties": {}}}]}, + litellm_params={"litellm_call_id": "functions-stream"}, + ) + tool_use: Final = { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + "index": 0, + } + chunks: Final = [ + { + "text": "", + "tool_use": tool_use, + "is_finished": True, + "finish_reason": "tool_calls", + "usage": None, + "index": 0, + } + ] + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=chunks), + model="bedrock/invoke-endpoint", + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + + received: Final = await _collect(wrapper, sync_mode) + + deltas: Final = [chunk.choices[0].delta for chunk in received if chunk.choices] + assert [ + (delta.function_call.name, delta.function_call.arguments) for delta in deltas if delta.function_call is not None + ] == [("get_weather", '{"city": "Paris"}')] + assert [delta.tool_calls for delta in deltas if delta.tool_calls] == [] diff --git a/tests/unit/llms/azure_ai/agents/test_handler.py b/tests/unit/llms/azure_ai/agents/test_handler.py index 14b6094747f..04bc8e2bf97 100644 --- a/tests/unit/llms/azure_ai/agents/test_handler.py +++ b/tests/unit/llms/azure_ai/agents/test_handler.py @@ -1,8 +1,10 @@ import datetime import json +from collections.abc import Mapping from typing import Final import httpx +import openai import pytest import respx @@ -22,7 +24,12 @@ def _httpx_only_transport(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") -def _assistants_api(thread_id: str, answer: str, project_path: str = "/api/projects/litellm-test") -> respx.MockRouter: +def _assistants_api( + thread_id: str, + answer: str, + project_path: str = "/api/projects/litellm-test", + run: Mapping[str, object] | None = None, +) -> respx.MockRouter: thread_path: Final = f"{project_path}/threads/{thread_id}" router: Final = respx.mock(assert_all_called=False) router.post(path=f"{project_path}/threads").respond( @@ -42,7 +49,10 @@ def _assistants_api(thread_id: str, answer: str, project_path: str = "/api/proje }, ) router.get(path=f"{thread_path}/runs/run_1").respond( - 200, json={"id": "run_1", "object": "thread.run", "thread_id": thread_id, "status": "completed"} + 200, + json={"id": "run_1", "object": "thread.run", "thread_id": thread_id, "status": "completed"} + if run is None + else run, ) router.get(path=f"{thread_path}/messages").respond( 200, @@ -172,6 +182,50 @@ async def test_handler_acompletion_with_thread_id_reuses_that_thread() -> None: assert response._hidden_params["thread_id"] == "thread_existing" +async def _complete(sync_mode: bool, request: Mapping[str, object]) -> object: + if sync_mode: + return litellm.completion(**request) + return await litellm.acompletion(**request) + + +@pytest.mark.parametrize("status", ["failed", "cancelled", "expired"]) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_a_run_that_ends_without_completing_raises_its_last_error_and_reads_no_messages( + sync_mode: bool, status: str +) -> None: + router: Final = _assistants_api( + "thread_new", + "never read", + run={ + "id": "run_1", + "object": "thread.run", + "thread_id": "thread_new", + "status": status, + "last_error": {"code": "server_error", "message": "Sorry, something went wrong."}, + }, + ) + request: Final = { + "model": f"azure_ai/agents/{AGENT_ID}", + "messages": [{"role": "user", "content": "hi"}], + "api_base": API_BASE, + "api_key": "azure-ad-token", + "stream": False, + } + + with router: + with pytest.raises(openai.APIError) as raised: + await _complete(sync_mode, request) + recorded: Final = _recorded(router) + + assert raised.value.status_code == 500 + assert f"Run {status}: Sorry, something went wrong." in raised.value.message + assert [(method, url.split("?")[0]) for method, url, _, _ in recorded][-1] == ( + "GET", + "/api/projects/litellm-test/threads/thread_new/runs/run_1", + ) + + def _sse(event: str, data: dict[str, object] | str) -> str: return f"event: {event}\ndata: {data if isinstance(data, str) else json.dumps(data, ensure_ascii=False)}\n\n" diff --git a/tests/unit/llms/azure_ai/vector_stores/test_azure_ai_vector_stores_transformation.py b/tests/unit/llms/azure_ai/vector_stores/test_azure_ai_vector_stores_transformation.py index 7a0df7d96f0..5f86c5adc58 100644 --- a/tests/unit/llms/azure_ai/vector_stores/test_azure_ai_vector_stores_transformation.py +++ b/tests/unit/llms/azure_ai/vector_stores/test_azure_ai_vector_stores_transformation.py @@ -1,4 +1,5 @@ import json +from typing import Final from unittest.mock import MagicMock import httpx @@ -180,3 +181,38 @@ def test_transform_requires_embedding_model(): with pytest.raises(ValueError, match="litellm_embedding_model is required"): AzureAIVectorStoreConfig().transform_search_vector_store_request(**transform_kwargs) + + +class _SearchLogging: + def __init__(self, query: str) -> None: + self.model_call_details = {"input": query} + + +def test_search_response_keeps_document_fields_as_attributes_without_search_internals() -> None: + body: Final = { + "value": [ + { + "@search.score": 0.8, + "id": "doc-1", + "content": "Run litellm --config config.yaml", + "contentVector": [0.5, -0.25], + "title": "Proxy setup", + "tags": ["proxy", "setup"], + } + ] + } + + response: Final = AzureAIVectorStoreConfig().transform_search_vector_store_response( + httpx.Response(200, json=body), _SearchLogging("how do I start the proxy") + ) + + assert response["search_query"] == "how do I start the proxy" + assert response["data"] == [ + { + "score": 0.8, + "content": [{"text": "Run litellm --config config.yaml", "type": "text"}], + "file_id": "doc-1", + "filename": "Document doc-1", + "attributes": {"title": "Proxy setup", "tags": ["proxy", "setup"], "document_id": "doc-1"}, + } + ] diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_twelvelabs_pegasus_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_twelvelabs_pegasus_transformation.py index 5b3cb2e0c92..fa4780053c2 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_twelvelabs_pegasus_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_twelvelabs_pegasus_transformation.py @@ -1,8 +1,10 @@ +from collections.abc import Mapping from typing import Final from unittest.mock import Mock import httpx import pytest +from pydantic import BaseModel, ConfigDict from litellm.llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import ( AmazonTwelveLabsPegasusConfig, @@ -112,3 +114,47 @@ def test_transform_response_accepts_null_finish_reason(finish_reason: str | None assert result.choices[0].message.content == "A cat jumps." assert result.choices[0].finish_reason == expected + + +class _VideoSummary(BaseModel): + model_config = ConfigDict(frozen=True) + + title: str + duration_seconds: int + + +def _request_with_response_format(response_format: object) -> Mapping[str, object]: + config: Final = AmazonTwelveLabsPegasusConfig() + optional_params: Final = config.map_openai_params( + non_default_params={"response_format": response_format}, + optional_params={}, + model="twelvelabs.pegasus-1-2-v1:0", + drop_params=False, + ) + return config.transform_request( + model="twelvelabs.pegasus-1-2-v1:0", + messages=_make_messages(), + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + +def test_pydantic_response_format_is_sent_as_the_json_schema_of_its_fields() -> None: + response_format: Final = _request_with_response_format(_VideoSummary)["responseFormat"] + assert isinstance(response_format, Mapping) + json_schema: Final = response_format["jsonSchema"] + assert isinstance(json_schema, Mapping) + + assert set(json_schema["properties"]) == {"title", "duration_seconds"} + + +def test_null_response_format_sends_no_response_format() -> None: + request: Final = _request_with_response_format(None) + + assert "responseFormat" not in request + + +def test_response_format_that_is_neither_a_schema_nor_a_model_raises_type_error() -> None: + with pytest.raises(TypeError): + _request_with_response_format("json") diff --git a/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py b/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py index 87c57023c17..e69fb34038f 100644 --- a/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py +++ b/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py @@ -6,6 +6,7 @@ import httpx import pytest import respx +from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.vector_stores.transformation import BedrockVectorStoreConfig @@ -375,3 +376,37 @@ async def test_bedrock_search_with_credentials_managed_registry( assert search_response["data"][0]["score"] == 0.9 assert search_response["data"][0]["content"][0]["text"] == "Mock search result" + + +class _SearchLogging: + def __init__(self, query: str) -> None: + self.model_call_details: Final = {"query": query} + + +def test_search_response_keeps_each_text_result_with_its_score_and_metadata() -> None: + metadata: Final = {"x-amz-bedrock-kb-source-uri": "s3://docs/setup.md", "x-amz-bedrock-kb-chunk-id": "chunk-1"} + body: Final = { + "retrievalResults": [ + {"content": {"text": "Run litellm --config config.yaml"}, "metadata": metadata, "score": 0.42}, + {"content": {"type": "IMAGE"}, "score": 0.9}, + ] + } + + response: Final = BedrockVectorStoreConfig().transform_search_vector_store_response( + httpx.Response(200, json=body), _SearchLogging("how do I start the proxy") + ) + + assert response["search_query"] == "how do I start the proxy" + assert [(result["score"], result["content"][0]["text"], result["attributes"]) for result in response["data"]] == [ + (0.42, "Run litellm --config config.yaml", metadata) + ] + + +@pytest.mark.parametrize("body", [[], [{"retrievalResults": []}], "retrievalResults"]) +def test_search_response_that_is_not_a_json_object_raises_a_bedrock_error(body: object) -> None: + with pytest.raises(BedrockError) as raised: + BedrockVectorStoreConfig().transform_search_vector_store_response( + httpx.Response(200, json=body), _SearchLogging("query") + ) + + assert raised.value.status_code == 200 diff --git a/tests/unit/llms/cohere/test_cohere_common_utils.py b/tests/unit/llms/cohere/test_cohere_common_utils.py index 1d17824a181..0fde387b5ea 100644 --- a/tests/unit/llms/cohere/test_cohere_common_utils.py +++ b/tests/unit/llms/cohere/test_cohere_common_utils.py @@ -2,7 +2,7 @@ from typing import Final import pytest -from litellm.llms.cohere.common_utils import ModelResponseIterator +from litellm.llms.cohere.common_utils import CohereV2ModelResponseIterator, ModelResponseIterator def test_stream_parses_object_chunks_and_rejects_a_chunk_that_is_not_an_object() -> None: @@ -16,3 +16,21 @@ def test_stream_parses_object_chunks_and_rejects_a_chunk_that_is_not_an_object() assert (first["text"], first["index"], first["is_finished"]) == ("Hi", 1, False) with pytest.raises(AttributeError, match=r"^'list' object has no attribute 'get'$"): next(stream) + + +def test_v2_stream_parses_object_chunks_and_rejects_a_chunk_that_is_not_an_object() -> None: + stream: Final = CohereV2ModelResponseIterator( + streaming_response=iter( + [ + '{"type": "content-delta", "index": 0, "delta": {"message": {"content": {"text": "Hi"}}}}', + "[1, 2]", + ] + ), + sync_stream=True, + ) + + first: Final = next(stream) + + assert (first["text"], first["index"], first["is_finished"]) == ("Hi", 0, False) + with pytest.raises(RuntimeError): + next(stream) diff --git a/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_transformation.py b/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_transformation.py new file mode 100644 index 00000000000..37016d69b53 --- /dev/null +++ b/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_transformation.py @@ -0,0 +1,106 @@ +from datetime import datetime, timezone +from typing import Final + +import httpx +import pytest + +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig +from litellm.types.utils import ModelResponse + + +def _transform( + body: object, *, task: str | None = None, optional_params: dict[str, object] | None = None +) -> ModelResponse: + return HuggingFaceEmbeddingConfig().transform_response( + model="tgi-endpoint", + raw_response=httpx.Response(200, json=body), + model_response=ModelResponse(), + logging_obj=Logging( + model="tgi-endpoint", + messages=[{"role": "user", "content": "Capital of France?"}], + stream=False, + call_type="completion", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="huggingface-transform", + function_id="huggingface-transform", + ), + request_data={"inputs": "Capital of France?"}, + messages=[{"role": "user", "content": "Capital of France?"}], + optional_params={} if optional_params is None else optional_params, + litellm_params={} if task is None else {"task": task}, + encoding=None, + ) + + +def test_tgi_response_strips_chat_tokens_and_sums_the_token_logprobs() -> None: + response: Final = _transform( + [ + { + "generated_text": "Paris", + "details": { + "finish_reason": "length", + "tokens": [{"logprob": -0.5}, {"logprob": None}, {"logprob": -0.25}], + }, + } + ] + ) + + assert response.choices[0].message.content == "Paris" + assert response.choices[0].finish_reason == "length" + assert response.choices[0].message._logprob == -0.75 + + +def test_tgi_response_with_best_of_adds_one_choice_per_extra_sequence() -> None: + response: Final = _transform( + [ + { + "generated_text": "Paris", + "details": { + "finish_reason": "length", + "tokens": [{"logprob": -0.5}], + "best_of_sequences": [ + { + "generated_text": "Lyon", + "finish_reason": "length", + "tokens": [{"logprob": -1.0}, {"logprob": None}, {"logprob": -0.5}], + }, + {"generated_text": "", "finish_reason": "stop", "tokens": []}, + ], + }, + } + ], + optional_params={"best_of": 2}, + ) + + assert [(choice.index, choice.message.content, choice.finish_reason) for choice in response.choices] == [ + (0, "Paris", "length"), + (1, "Lyon", "length"), + (2, None, "stop"), + ] + assert response.choices[1].message.logprobs == -1.5 + + +def test_text_generation_task_reads_the_first_generated_text() -> None: + response: Final = _transform([{"generated_text": "<|assistant|>Paris"}], task="text-generation") + + assert response.choices[0].message.content == "Paris" + + +@pytest.mark.parametrize( + ("body", "error"), + [ + pytest.param([{"generated_text": 42}], TypeError, id="generated-text-without-a-length"), + pytest.param([{"generated_text": ["Paris"]}], AttributeError, id="generated-text-that-is-not-a-string"), + pytest.param( + [{"generated_text": "Paris", "details": {"finish_reason": "length", "tokens": [{"logprob": "-0.5"}]}}], + TypeError, + id="logprob-that-is-not-a-number", + ), + ], +) +def test_tgi_response_with_malformed_fields_raises_the_python_error_for_that_field( + body: object, error: type[Exception] +) -> None: + with pytest.raises(error): + _transform(body) diff --git a/tests/unit/llms/nlp_cloud/chat/test_nlp_cloud_chat_transformation.py b/tests/unit/llms/nlp_cloud/chat/test_nlp_cloud_chat_transformation.py index 35f8d933cee..3cf2518921a 100644 --- a/tests/unit/llms/nlp_cloud/chat/test_nlp_cloud_chat_transformation.py +++ b/tests/unit/llms/nlp_cloud/chat/test_nlp_cloud_chat_transformation.py @@ -1,6 +1,13 @@ +from datetime import datetime, timezone +from typing import Final + +import httpx import pytest +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.nlp_cloud.chat.transformation import NLPCloudConfig +from litellm.llms.nlp_cloud.common_utils import NLPCloudError +from litellm.types.utils import ModelResponse @pytest.mark.parametrize( @@ -45,3 +52,48 @@ def test_init_called_again_with_none_keeps_the_value_stored_earlier() -> None: ScopedNLPCloudConfig(top_k=None, top_p=0.5) assert ScopedNLPCloudConfig.get_config() == {"top_k": 40, "top_p": 0.5} + + +def _transform(body: object) -> ModelResponse: + return NLPCloudConfig().transform_response( + model="finetuned-llama-3-70b", + raw_response=httpx.Response(200, json=body), + model_response=ModelResponse(), + logging_obj=Logging( + model="finetuned-llama-3-70b", + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="completion", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="nlp-cloud-transform", + function_id="nlp-cloud-transform", + ), + request_data={"text": "Hi"}, + messages=[{"role": "user", "content": "Hi"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + +def test_transform_response_returns_the_generated_text_and_reported_token_counts() -> None: + response: Final = _transform({"generated_text": "Hello there", "nb_input_tokens": 3, "nb_generated_tokens": 2}) + + assert response.choices[0].message.content == "Hello there" + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (3, 2, 5) + + +@pytest.mark.parametrize( + "body", + [ + pytest.param({"nb_input_tokens": 3, "nb_generated_tokens": 0}, id="missing-generated-text"), + pytest.param( + {"generated_text": 42, "nb_input_tokens": 3, "nb_generated_tokens": 1}, id="generated-text-without-a-length" + ), + ], +) +def test_transform_response_without_usable_generated_text_raises_an_nlp_cloud_error(body: dict[str, object]) -> None: + with pytest.raises(NLPCloudError) as raised: + _transform(body) + + assert raised.value.status_code == 200 diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index 36df47deaab..a51acd6dd44 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -5,8 +5,10 @@ import logging import os import sys import traceback +from datetime import datetime, timezone from typing import Final, cast +import httpx import pytest import respx from pydantic import BaseModel @@ -28,6 +30,7 @@ from unittest.mock import ANY, MagicMock, patch import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.utils import ( ChatCompletionDeltaToolCall, Choices, @@ -1444,3 +1447,59 @@ def test_reasoning_effort_with_a_matching_think_value_does_not_warn( assert result["think"] is think assert _warnings_naming(caplog, "reasoning_effort") == () + + +def _transform_json_format_function_call(content: object) -> ModelResponse: + return OllamaChatConfig().transform_response( + model="llama3.2", + raw_response=httpx.Response( + 200, + json={ + "model": "llama3.2", + "message": {"role": "assistant", "content": content}, + "done": True, + "prompt_eval_count": 12, + "eval_count": 8, + }, + ), + model_response=ModelResponse(), + logging_obj=Logging( + model="llama3.2", + messages=[{"role": "user", "content": "Weather in Paris?"}], + stream=False, + call_type="completion", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="ollama-json-function-call", + function_id="ollama-json-function-call", + ), + request_data={"format": "json"}, + messages=[{"role": "user", "content": "Weather in Paris?"}], + optional_params={}, + litellm_params={"function_name": "lookup_weather"}, + encoding=None, + ) + + +@pytest.mark.parametrize( + ("content", "name", "arguments"), + [ + pytest.param( + '{"name": "get_weather", "arguments": {"city": "Paris"}}', "get_weather", {"city": "Paris"}, id="named-call" + ), + pytest.param('{"city": "Paris"}', "lookup_weather", {"city": "Paris"}, id="bare-arguments"), + ], +) +def test_json_format_response_for_a_function_call_becomes_a_tool_call( + content: str, name: str, arguments: dict[str, object] +) -> None: + response: Final = _transform_json_format_function_call(content) + + choice: Final = response.choices[0] + tool_call: Final = choice.message.tool_calls[0] + assert (choice.finish_reason, choice.message.content) == ("tool_calls", None) + assert (tool_call.function.name, json.loads(tool_call.function.arguments)) == (name, arguments) + + +def test_json_format_function_call_response_without_text_content_raises_type_error() -> None: + with pytest.raises(TypeError): + _transform_json_format_function_call(None) diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index ad3036560a3..4e5b6d665e2 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -5,7 +5,7 @@ import time from collections.abc import Awaitable, Callable, Mapping from datetime import datetime, timezone from pathlib import Path -from types import TracebackType +from types import MappingProxyType, TracebackType from typing import ClassVar, Final, Literal, Protocol from unittest.mock import AsyncMock, MagicMock, patch @@ -1135,6 +1135,93 @@ async def test_realtime_session_requests_forward_an_explicit_null_upstream( assert [key for key, value in session_fields.items() if value is None] == [] +def _create_translation_client_secret(expires_after: object, gateway: _GatewayThatRecordsTheBody) -> Awaitable[object]: + return realtime_main.acreate_realtime_translation_client_secret.__wrapped__( + model="gpt-realtime-translate", + expires_after=expires_after, + custom_llm_provider="openai", + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=_realtime_http_logging("translation_client_secrets"), + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "expires_after", + [ + pytest.param({"anchor": "created_at", "seconds": 600}, id="dict"), + pytest.param(MappingProxyType({"anchor": "created_at", "seconds": 600}), id="read-only-mapping"), + ], +) +async def test_translation_client_secret_forwards_a_mapping_expires_after(expires_after: Mapping[str, object]) -> None: + gateway: Final = _GatewayThatRecordsTheBody() + + await _create_translation_client_secret(expires_after, gateway) + + assert gateway.body["expires_after"] == {"anchor": "created_at", "seconds": 600} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "expires_after", + [ + pytest.param([("anchor", "created_at"), ("seconds", 600)], id="list-of-pairs"), + pytest.param("seconds=600", id="string"), + ], +) +async def test_translation_client_secret_rejects_an_expires_after_that_is_not_a_mapping(expires_after: object) -> None: + gateway: Final = _GatewayThatRecordsTheBody() + + with pytest.raises(TypeError, match=r"realtime\.RealtimeExpiresAfter\(\) argument after \*\* must be a mapping"): + await _create_translation_client_secret(expires_after, gateway) + + assert gateway.body is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "transcription_session", + [ + pytest.param([("input_audio_format", "pcm16")], id="list-of-pairs"), + pytest.param("input_audio_format=pcm16", id="string"), + ], +) +async def test_transcription_session_rejects_a_session_that_is_not_a_mapping(transcription_session: object) -> None: + gateway: Final = _GatewayThatRecordsTheBody() + + with pytest.raises( + TypeError, match=r"realtime\.RealtimeTranscriptionSessionRequest\(\) argument after \*\* must be a mapping" + ): + await realtime_main.acreate_realtime_transcription_session.__wrapped__( + model="openai/gpt-transcribe", + transcription_session=transcription_session, + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + assert gateway.body is None + + +@pytest.mark.asyncio +async def test_transcription_session_accepts_a_read_only_mapping() -> None: + gateway: Final = _GatewayThatRecordsTheBody() + + await realtime_main.acreate_realtime_transcription_session.__wrapped__( + model="openai/gpt-transcribe", + transcription_session=MappingProxyType({"input_audio_format": "pcm16"}), + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + assert gateway.body["input_audio_format"] == "pcm16" + + @pytest.mark.asyncio async def test_transcription_client_secret_sends_the_deployment_model_as_the_session_model() -> None: gateway: Final = _GatewayThatRecordsTheSession() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 577d4b4b11d..97df9cdbfbf 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -20,7 +20,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx -from openai import APITimeoutError, AsyncOpenAI, OpenAI +from openai import APIError, APITimeoutError, AsyncOpenAI, OpenAI from openai._legacy_response import LegacyAPIResponse from openai.types.chat.chat_completion import ChatCompletion from pydantic import BaseModel, ConfigDict, TypeAdapter @@ -12016,3 +12016,57 @@ def test_supplied_openai_client_base_url_decides_data_residency(monkeypatch: pyt ) assert captured["logging_obj"].litellm_params.get("data_residency") is None + + +_ALEPH_ALPHA_COMPLETE_URL: Final = "https://api.aleph-alpha.com/complete" + + +def _aleph_alpha_completion() -> object: + return litellm.completion( + model="luminous-base", + custom_llm_provider="aleph_alpha", + messages=[{"role": "user", "content": "Capital of France?"}], + api_key="aleph-test-key", + ) + + +def test_aleph_alpha_completion_returns_each_upstream_completion_as_a_choice(respx_mock: respx.MockRouter) -> None: + respx_mock.post(_ALEPH_ALPHA_COMPLETE_URL).respond( + 200, + json={ + "model_version": "2022-04", + "completions": [ + {"completion": " Paris", "finish_reason": "maximum_tokens"}, + {"completion": " Lyon", "finish_reason": "end_of_text"}, + ], + }, + ) + + response: Final = _aleph_alpha_completion() + + assert [choice.message.content for choice in response.choices] == [" Paris", " Lyon"] + + +def test_aleph_alpha_error_in_a_successful_body_fails_the_call_with_the_upstream_message( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(_ALEPH_ALPHA_COMPLETE_URL).respond( + 200, json={"error": "Prompt rejected by the content filter", "code": "PROMPT_REJECTED"} + ) + + with pytest.raises(APIError) as raised: + _aleph_alpha_completion() + + assert "Prompt rejected by the content filter" in raised.value.message + + +@pytest.mark.parametrize("completion", [42, None]) +def test_aleph_alpha_completion_that_is_not_text_fails_the_call( + respx_mock: respx.MockRouter, completion: object +) -> None: + respx_mock.post(_ALEPH_ALPHA_COMPLETE_URL).respond( + 200, json={"completions": [{"completion": completion, "finish_reason": "end_of_text"}]} + ) + + with pytest.raises(APIError): + _aleph_alpha_completion() diff --git a/tests/unit/videos/test_main.py b/tests/unit/videos/test_main.py index d43141bdea3..26a6c0687d0 100644 --- a/tests/unit/videos/test_main.py +++ b/tests/unit/videos/test_main.py @@ -30,6 +30,7 @@ helper runs for real against genuinely-encoded ids, so the provider assertions reflect production. """ +from collections.abc import Callable, Mapping from contextlib import ExitStack from dataclasses import dataclass from typing import Any, Dict, Final @@ -38,6 +39,7 @@ from unittest.mock import MagicMock, patch from litellm.types.videos.main import VideoCreateOptionalRequestParams import pytest +from pydantic import BaseModel import litellm @@ -486,3 +488,80 @@ def test_video_generation__custom_provider_without_a_video_config_is_a_400(monke with pytest.raises(litellm.BadRequestError, match="video generation is not supported for image-only"): videos_main.video_generation(model="image-only/seedance-1", prompt="a cat") handler.video_generation_handler.assert_not_called() + + +_VIDEO_JSON: Final = '{"id": "video_1", "object": "video", "status": "completed", "model": "sora-2"}' +_CHARACTER_JSON: Final = '{"id": "char_1", "object": "character", "created_at": 1767225600, "name": "hero"}' + + +def _fields(value: object) -> object: + if isinstance(value, list): + return [_fields(item) for item in value] + return value.model_dump() if isinstance(value, BaseModel) else value + + +@pytest.mark.parametrize( + ("operation", "arguments", "mock_response", "expected"), + [ + pytest.param( + videos_main.video_generation, + {"prompt": "a cat surfing", "model": "sora-2"}, + _VIDEO_JSON, + VideoObject(id="video_1", object="video", status="completed", model="sora-2"), + id="generation", + ), + pytest.param( + videos_main.video_remix, + {"video_id": "video_1", "prompt": "make it night"}, + _VIDEO_JSON, + VideoObject(id="video_1", object="video", status="completed", model="sora-2"), + id="remix", + ), + pytest.param( + videos_main.video_status, + {"video_id": "video_1"}, + _VIDEO_JSON, + VideoObject(id="video_1", object="video", status="completed", model="sora-2"), + id="status", + ), + pytest.param( + videos_main.video_edit, + {"video_id": "video_1", "prompt": "add rain"}, + _VIDEO_JSON, + VideoObject(id="video_1", object="video", status="completed", model="sora-2"), + id="edit", + ), + pytest.param( + videos_main.video_extension, + {"video_id": "video_1", "prompt": "keep going", "seconds": "4"}, + _VIDEO_JSON, + VideoObject(id="video_1", object="video", status="completed", model="sora-2"), + id="extension", + ), + pytest.param( + videos_main.video_list, + {}, + f"[{_VIDEO_JSON}]", + [VideoObject(id="video_1", object="video", status="completed", model="sora-2")], + id="list", + ), + pytest.param( + videos_main.video_create_character, + {"name": "hero", "video": b"\x00\x00\x00\x18ftypmp42"}, + _CHARACTER_JSON, + CharacterObject(id="char_1", object="character", created_at=1767225600, name="hero"), + id="create-character", + ), + pytest.param( + videos_main.video_get_character, + {"character_id": "char_1"}, + _CHARACTER_JSON, + CharacterObject(id="char_1", object="character", created_at=1767225600, name="hero"), + id="get-character", + ), + ], +) +def test_a_json_string_mock_response_is_parsed_into_the_typed_result( + operation: Callable[..., object], arguments: Mapping[str, object], mock_response: str, expected: object +) -> None: + assert _fields(operation(**arguments, mock_response=mock_response)) == _fields(expected) From a45e3b66054219babb4553d40ae2a5738b5f3505 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 11 Nov 2026 14:24:26 -0800 Subject: [PATCH 113/189] fix(proxy): reset a deleted customer's admission spend counter so a re-created one is not refused (#51224) * fix(proxy): reset a deleted customer's admission spend counter so a re-created one is not refused /customer/delete only dropped the committed twin, so spend:end_user: kept the old spend and a re-created customer with the same id stayed over budget. Zero the counter and its DB floor marker on every worker after the delete commits, the same way a key spend reset does. * fix(proxy): drop a deleted customer's spend counter before zeroing it A Redis that rejects writes but still serves reads, such as one at its memory limit, kept the old counter when the zeroing SET failed, so the re-created customer stayed refused. Delete the counter first, which Redis still allows in that state, then write zero and the floor. * test(proxy): type the customer delete spend counter test parameters --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../customer_endpoints.py | 15 +- .../test_customer_endpoints.py | 131 ++++++++++++++++-- 2 files changed, 128 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index bd202b85f73..5ba13e598c3 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -815,7 +815,12 @@ async def delete_end_user( See below for all params ``` """ - from litellm.proxy.proxy_server import prisma_client, spend_counter_store + from litellm.proxy.proxy_server import ( + invalidate_spend_counter, + prisma_client, + set_spend_counter_with_floor_and_broadcast, + spend_counter_store, + ) try: if prisma_client is None: @@ -845,9 +850,11 @@ async def delete_end_user( verbose_proxy_logger.debug("received response from updating prisma client. response=%s", response) await _evict_end_user_cache_keys(_end_user_cache_keys(data.user_ids)) - await drop_committed_spend( - spend_counter_store, tuple(end_user_counter(user_id) for user_id in data.user_ids) - ) + counter_keys: Final = tuple(end_user_counter(user_id) for user_id in data.user_ids) + for counter_key in counter_keys: + await invalidate_spend_counter(counter_key=counter_key) + await set_spend_counter_with_floor_and_broadcast(counter_key=counter_key, value=0.0) + await drop_committed_spend(spend_counter_store, counter_keys) return DeleteCustomersResponse( deleted_customers=response, diff --git a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py index 83b0c1c8c6b..f96bbd5f609 100644 --- a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py @@ -10,11 +10,15 @@ from fastapi.routing import APIRoute from fastapi.testclient import TestClient from pydantic import BaseModel +import litellm +from litellm.caching.caching import DualCache from litellm.proxy._types import ( + LiteLLM_BudgetTable, LiteLLM_EndUserTable, LitellmUserRoles, ProxyException, ) +from litellm.proxy.auth.auth_checks import check_end_user_budget from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.management_endpoints.customer_endpoints import _should_update_field, router from litellm.types.proxy.management_endpoints.common_daily_activity import ( @@ -26,6 +30,7 @@ from litellm.types.proxy.management_endpoints.customer_endpoints import ( DeleteCustomersResponse, UnblockUsersResponse, ) +from tests.unit.redis.stores.spend_counters_double import FakeSpendCounters app = FastAPI() @@ -250,20 +255,47 @@ def test_delete_customer_not_found(mock_prisma_client, mock_user_api_key_auth): assert response_json["error"]["code"] == "404" -def test_delete_customer_drops_its_committed_spend_so_a_recreated_customer_starts_from_its_own_row( - monkeypatch, mock_prisma_client, mock_user_api_key_auth -): - from types import SimpleNamespace - - from tests.unit.redis.stores.spend_counters_double import FakeSpendCounters - - spend: Final = FakeSpendCounters( - redis=None, - memory={"spend:committed:end_user:cust-gone": 7.0, "spend:committed:end_user:cust-stays": 3.0}, - ) +@pytest.fixture(autouse=True) +def spend_counters(monkeypatch: pytest.MonkeyPatch) -> FakeSpendCounters: + """A fresh spend counter store and floor cache per test, so no counter a delete writes outlives its test.""" + spend: Final = FakeSpendCounters(redis={}) monkeypatch.setattr("litellm.proxy.proxy_server.spend_counter_store", spend.store()) + monkeypatch.setattr("litellm.proxy.proxy_server.spend_counter_cache", DualCache()) + return spend + + +def _over_budget_customers(spend: FakeSpendCounters) -> None: + from litellm.proxy.proxy_server import spend_counter_cache + + assert spend.redis is not None + for user_id, spent in (("cust-gone", 7.0), ("cust-stays", 3.0)): + spend.redis[f"spend:end_user:{user_id}"] = spent + spend.redis[f"spend:committed:end_user:{user_id}"] = spent + spend_counter_cache.in_memory_cache.set_cache(key=f"spend_db_floor:spend:end_user:{user_id}", value=spent) + + +async def _check_budget_of_new_customer(user_id: str) -> None: + await check_end_user_budget( + end_user_obj=LiteLLM_EndUserTable( + user_id=user_id, blocked=False, spend=0.0, litellm_budget_table=LiteLLM_BudgetTable(max_budget=1.0) + ), + route="/chat/completions", + ) + + +@pytest.mark.parametrize("fail_closed", [False, True]) +@pytest.mark.asyncio +async def test_a_deleted_customer_recreated_under_the_same_id_is_admitted_while_others_stay_over_budget( + monkeypatch: pytest.MonkeyPatch, + mock_prisma_client: MagicMock, + mock_user_api_key_auth: None, + spend_counters: FakeSpendCounters, + fail_closed: bool, +) -> None: + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"fail_closed_budget_enforcement": fail_closed}) + _over_budget_customers(spend_counters) mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock( - return_value=[SimpleNamespace(user_id="cust-gone")] + return_value=[LiteLLM_EndUserTable(user_id="cust-gone", blocked=False)] ) mock_prisma_client.db.litellm_endusertable.delete_many = AsyncMock(return_value=1) @@ -271,8 +303,79 @@ def test_delete_customer_drops_its_committed_spend_so_a_recreated_customer_start "/customer/delete", json={"user_ids": ["cust-gone"]}, headers={"Authorization": "Bearer test-key"} ) - assert response.status_code == 200 - assert spend.memory == {"spend:committed:end_user:cust-stays": 3.0} + assert response.status_code == 200, response.text + await _check_budget_of_new_customer("cust-gone") + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_budget_of_new_customer("cust-stays") + assert exc_info.value.current_cost == 3.0 + assert spend_counters.redis is not None + assert "spend:committed:end_user:cust-gone" not in spend_counters.redis + assert spend_counters.redis["spend:committed:end_user:cust-stays"] == 3.0 + + +@pytest.mark.parametrize( + ("existing", "delete_error"), + [ + pytest.param([], None, id="customer-missing"), + pytest.param( + [LiteLLM_EndUserTable(user_id="cust-gone", blocked=False)], RuntimeError("db down"), id="db-fails" + ), + ], +) +@pytest.mark.asyncio +async def test_a_failed_customer_delete_keeps_its_spend_enforced( + mock_prisma_client: MagicMock, + mock_user_api_key_auth: None, + spend_counters: FakeSpendCounters, + existing: list[LiteLLM_EndUserTable], + delete_error: Exception | None, +) -> None: + _over_budget_customers(spend_counters) + mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock(return_value=existing) + mock_prisma_client.db.litellm_endusertable.delete_many = AsyncMock(side_effect=delete_error, return_value=1) + + response: Final = client.post( + "/customer/delete", json={"user_ids": ["cust-gone"]}, headers={"Authorization": "Bearer test-key"} + ) + + assert response.status_code >= 400 + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_budget_of_new_customer("cust-gone") + assert exc_info.value.current_cost == 7.0 + + +@pytest.mark.parametrize( + ("failing", "redis_down"), + [ + pytest.param(frozenset({"set"}), False, id="redis-rejects-writes"), + pytest.param(frozenset(), True, id="redis-down"), + ], +) +@pytest.mark.parametrize("fail_closed", [False, True]) +@pytest.mark.asyncio +async def test_a_recreated_customer_is_admitted_when_redis_rejects_the_counter_reset( + monkeypatch: pytest.MonkeyPatch, + mock_prisma_client: MagicMock, + mock_user_api_key_auth: None, + failing: frozenset[str], + redis_down: bool, + fail_closed: bool, +) -> None: + spend: Final = FakeSpendCounters(redis={}, failing=failing, fail=redis_down) + monkeypatch.setattr("litellm.proxy.proxy_server.spend_counter_store", spend.store()) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"fail_closed_budget_enforcement": fail_closed}) + _over_budget_customers(spend) + mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock( + return_value=[LiteLLM_EndUserTable(user_id="cust-gone", blocked=False)] + ) + mock_prisma_client.db.litellm_endusertable.delete_many = AsyncMock(return_value=1) + + response: Final = client.post( + "/customer/delete", json={"user_ids": ["cust-gone"]}, headers={"Authorization": "Bearer test-key"} + ) + + assert response.status_code == 200, response.text + await _check_budget_of_new_customer("cust-gone") def test_error_schema_consistency(mock_prisma_client, mock_user_api_key_auth): From 36c7f10c8434894fea8d8ee715363c7b32cf1543 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:26:47 -0800 Subject: [PATCH 114/189] fix(proxy): answer x-litellm-call-id on every token count response (#51037) * fix(proxy): answer x-litellm-call-id on every token count response The token count log line names a litellm_call_id, but count responses never carried it, so a client that sent no id of its own could not find its count's log line. Every count route (Anthropic Messages, Responses input_tokens, Gemini countTokens, Bedrock count-tokens, and /utils/token_counter) now answers the id on success and on errors, and a crash answers the JSON 500 the Anthropic and Gemini routes already used. * fix(proxy): keep the gateway's crash answers on count routes and stamp Responses opt-in bodies A crash on /utils/token_counter or /v1/responses/input_tokens goes back to the gateway's generic 500, or 503 when the database is down, now with the x-litellm-call-id header. Responses count rejections carry error.litellm_call_id when include_call_id_in_error_body is on * test(proxy): count through count_request_tokens in the token counter loop tests The token_counter route now also takes the response it sets x-litellm-call-id on, so these tests call the shared count function they exercise * fix(proxy): keep a count crash's own exception and log pre-count Responses failures The call id now reaches the crash handler through request.state instead of a wrapper exception, so logs and Sentry see the error the count raised. A Responses count that fails before counting writes its count line for any error, so its header names a line * fix(proxy): keep the token counter's injected args out of its API docs Tests the opt-in error body id on an Anthropic count the proxy rejects before counting * fix(proxy): keep a count route's own HTTP error when adding the call id The wrapper raised a new HTTPException with the same status, detail and headers, so a subclass lost its type and error tracking saw a new exception. It now adds the header to the original and raises it again --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../proxy/anthropic_endpoints/endpoints.py | 28 ++- litellm/proxy/common_utils/token_count_log.py | 31 ++- litellm/proxy/google_endpoints/endpoints.py | 20 +- .../llm_passthrough_endpoints.py | 16 +- litellm/proxy/proxy_server.py | 23 +- .../proxy/response_api_endpoints/endpoints.py | 81 +++++-- .../anthropic_endpoints/test_endpoints.py | 87 ++++++- .../common_utils/test_token_count_log.py | 52 ++++- .../proxy/google_endpoints/test_endpoints.py | 2 + .../test_google_api_endpoints.py | 22 +- .../test_llm_pass_through_endpoints.py | 45 +++- .../response_api_endpoints/test_endpoints.py | 134 ++++++++++- tests/unit/proxy/test_custom_tokenizer_bug.py | 10 +- ...test_proxy_server_endpoints_and_startup.py | 10 +- tests/unit/proxy/test_proxy_token_counter.py | 215 ++++++++++-------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 +- 16 files changed, 627 insertions(+), 157 deletions(-) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 636ba987ee8..ed3abbdc5b6 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -42,13 +42,14 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg read_request_body, ) from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, error_status_code, litellm_call_id_headers, openai_error_param, openai_error_type, with_litellm_call_id, ) -from litellm.proxy.common_utils.token_count_log import rejected_token_count +from litellm.proxy.common_utils.token_count_log import answered_with_litellm_call_id, rejected_token_count from litellm.proxy.common_utils.token_count_settings import count_source_fields from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost from litellm.proxy.litellm_pre_call_utils import ( @@ -336,6 +337,7 @@ def _provider_context_management(response: TokenCountResponse) -> Mapping[str, o ) async def count_tokens( request: Request, + fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # Used for auth ): """ @@ -358,9 +360,17 @@ async def count_tokens( Returns: {"input_tokens": } """ + litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)) + return await answered_with_litellm_call_id( + _count_tokens(request, user_api_key_dict, litellm_call_id), litellm_call_id, request, fastapi_response + ) + + +async def _count_tokens( + request: Request, user_api_key_dict: UserAPIKeyAuth, litellm_call_id: str +) -> JSONResponse | Mapping[str, object]: from litellm.proxy.proxy_server import count_request_tokens, llm_router, proxy_config - litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id")) try: request_data: Final = await read_request_body(request=request) data: Final[dict] = {**request_data} @@ -389,7 +399,9 @@ async def count_tokens( ) if isinstance(token_request, ProxyException): return rejected_token_count( - _anthropic_error_json_response(token_request, request), litellm_call_id, request_model + _anthropic_error_json_response(with_litellm_call_id(token_request, litellm_call_id), request), + litellm_call_id, + request_model, ) token_response: Final = await count_request_tokens( @@ -401,13 +413,17 @@ async def count_tokens( **_count_tokens_body_options(data, auto_drop_params), ) except HTTPException as e: - return _anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request) + return _anthropic_error_json_response( + with_litellm_call_id(proxy_exception_from_http_exception(e, {}), litellm_call_id), request + ) except ProxyException as e: - return _anthropic_error_json_response(e, request) + return _anthropic_error_json_response(with_litellm_call_id(e, litellm_call_id), request) except Exception as e: answered: Final = with_answered_status(e) log_llm_api_exception(answered, litellm_call_id) - return _anthropic_error_json_response(proxy_exception_from_route_error(answered), request) + return _anthropic_error_json_response( + with_litellm_call_id(proxy_exception_from_route_error(answered), litellm_call_id), request + ) else: return { "input_tokens": token_response.total_tokens, diff --git a/litellm/proxy/common_utils/token_count_log.py b/litellm/proxy/common_utils/token_count_log.py index 04c07a29117..83520ef1ade 100644 --- a/litellm/proxy/common_utils/token_count_log.py +++ b/litellm/proxy/common_utils/token_count_log.py @@ -6,10 +6,16 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Final, TypeVar -from fastapi import status +from fastapi import HTTPException, Request, Response, status from litellm._logging import verbose_proxy_logger -from litellm.proxy.common_utils.openai_error_payload import error_status_code +from litellm.proxy._types import ProxyException +from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, + error_status_code, + headers_with_litellm_call_id, + with_litellm_call_id, +) from litellm.types.token_count import TokenCountSource from litellm.types.utils import TokenCountResponse @@ -28,6 +34,7 @@ def _logfmt_value(value: str | None) -> str: _Counted: Final = TypeVar("_Counted") _Rejection: Final = TypeVar("_Rejection") _ProviderCount: Final = TypeVar("_ProviderCount", bound=Mapping[str, object]) +_Answer: Final = TypeVar("_Answer") @dataclass(frozen=True, slots=True) @@ -131,3 +138,23 @@ async def logged_provider_token_count( ) return await _logged(count, _CountRequest(litellm_call_id, request_model, model_id), provider_fields) + + +async def answered_with_litellm_call_id( + answer: Awaitable[_Answer], litellm_call_id: str, request: Request, fastapi_response: Response +) -> _Answer: + try: + answered: Final = await answer + except ProxyException as e: + raise with_litellm_call_id(e, litellm_call_id) + except HTTPException as e: + e.headers = headers_with_litellm_call_id(e.headers, litellm_call_id) + raise + except Exception: + request.state.litellm_call_id = litellm_call_id # rebind-ok: the crash handler reads the id from request.state + raise + if isinstance(answered, Response): + answered.headers.setdefault(LITELLM_CALL_ID_HEADER, litellm_call_id) + return answered + fastapi_response.headers[LITELLM_CALL_ID_HEADER] = litellm_call_id # rebind-ok: FastAPI's only way to add a header + return answered diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index c018f071b3c..31b88e8110c 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -22,7 +22,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg read_request_body, ) from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER -from litellm.proxy.common_utils.token_count_log import rejected_token_count +from litellm.proxy.common_utils.token_count_log import answered_with_litellm_call_id, rejected_token_count from litellm.proxy.common_utils.token_count_settings import count_source_fields from litellm.types.llms.vertex_ai import TokenCountDetailsResponse @@ -327,7 +327,7 @@ async def google_stream_generate_content( dependencies=[Depends(user_api_key_auth)], response_model=TokenCountDetailsResponse, ) -async def google_count_tokens(request: Request, model_name: str): +async def google_count_tokens(request: Request, model_name: str, fastapi_response: Response): """ ```json return { @@ -342,10 +342,18 @@ async def google_count_tokens(request: Request, model_name: str): } ``` """ - from litellm.proxy.common_utils.http_parsing_utils import read_request_body - from litellm.proxy.proxy_server import token_counter as internal_token_counter - litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)) + return await answered_with_litellm_call_id( + _google_count_tokens(request, model_name, litellm_call_id), litellm_call_id, request, fastapi_response + ) + + +async def _google_count_tokens( + request: Request, model_name: str, litellm_call_id: str +) -> JSONResponse | TokenCountDetailsResponse: + from litellm.proxy.common_utils.http_parsing_utils import read_request_body + from litellm.proxy.proxy_server import count_request_tokens + data: Final = await read_request_body(request=request) try: token_request: Final = _google_count_tokens_request(model_name, data) @@ -365,7 +373,7 @@ async def google_count_tokens(request: Request, model_name: str): ) try: - token_response: Final = await internal_token_counter( + token_response: Final = await count_request_tokens( request=token_request, call_endpoint=True, litellm_call_id=litellm_call_id ) except ProxyException as error: diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index f1a9e282bc9..321c921b5bd 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -133,6 +133,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) from litellm.proxy.common_utils.token_count_log import ( + answered_with_litellm_call_id, deployment_model_id, logged_provider_token_count, rejected_token_count, @@ -1526,7 +1527,7 @@ async def handle_bedrock_count_tokens( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, request_body: dict[str, object], -) -> dict[str, object]: +) -> Mapping[str, object]: """ Handle AWS Bedrock CountTokens API requests. @@ -1534,11 +1535,22 @@ async def handle_bedrock_count_tokens( - /v1/messages/count_tokens - /v1/messages/count-tokens """ + litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)) + return await answered_with_litellm_call_id( + _bedrock_count_tokens(user_api_key_dict, request_body, litellm_call_id), + litellm_call_id, + request, + fastapi_response, + ) + + +async def _bedrock_count_tokens( + user_api_key_dict: UserAPIKeyAuth, request_body: dict[str, object], litellm_call_id: str +) -> Mapping[str, object]: from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler from litellm.proxy.proxy_server import llm_router - litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)) try: # Initialize the handler handler: Final = BedrockCountTokensHandler() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b01dcfd2f03..fc4d4e65660 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -626,6 +626,7 @@ from litellm.proxy.common_utils.timezone_utils import ( get_budget_reset_time, ) from litellm.proxy.common_utils.token_count_log import ( + answered_with_litellm_call_id, deployment_model_id, logged_token_count, rejected_token_count, @@ -2595,6 +2596,9 @@ async def otel_unhandled_exception_handler(request: Request, exc: Exception): raise exc response: Final = await _unhandled_exception_response(request, exc) response.headers.update(trace_response_headers(getattr(request.state, "parent_otel_span", None))) + litellm_call_id: Final = attribute_of(request.state, "litellm_call_id") + if isinstance(litellm_call_id, str): + response.headers.setdefault(LITELLM_CALL_ID_HEADER, litellm_call_id) return response @@ -16090,19 +16094,30 @@ def _system_message(system: object) -> ChatCompletionSystemMessage | None: ) async def token_counter( request: TokenCountRequest, + http_request: Request, + fastapi_response: Response, call_endpoint: bool = False, litellm_call_id: Annotated[str | None, Header(alias=LITELLM_CALL_ID_HEADER)] = None, ): """ + Returns: + TokenCountResponse + Args: request: TokenCountRequest call_endpoint: bool - When set to "True" it will call the token counting endpoint - e.g Anthropic or Google AI Studio Token Counting APIs. litellm_call_id: the x-litellm-call-id header, logged with the count so it joins the inference call's logs - - Returns: - TokenCountResponse + \f + http_request: carries the call id to the error handler when the count fails + fastapi_response: answers the x-litellm-call-id header the count's log line names """ - return await count_request_tokens(request=request, call_endpoint=call_endpoint, litellm_call_id=litellm_call_id) + call_id: Final = resolve_litellm_call_id(litellm_call_id) + return await answered_with_litellm_call_id( + count_request_tokens(request=request, call_endpoint=call_endpoint, litellm_call_id=call_id), + call_id, + http_request, + fastapi_response, + ) async def count_request_tokens( diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 369d167f623..b4eec96ecd9 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -11,6 +11,7 @@ from itertools import chain from types import MappingProxyType from typing import ( TYPE_CHECKING, + Annotated, Any, Final, Literal, @@ -80,8 +81,9 @@ from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, error_status_code, openai_error_type_for_status, + with_litellm_call_id, ) -from litellm.proxy.common_utils.token_count_log import rejected_token_count +from litellm.proxy.common_utils.token_count_log import answered_with_litellm_call_id, rejected_token_count from litellm.proxy.common_utils.token_count_settings import count_source_fields from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity, decrypted_response_id @@ -1575,6 +1577,12 @@ def _proxy_managed_files_hook() -> _ManagedFileAccess | None: _managed_files_hook_dep: Final = Depends(_proxy_managed_files_hook) +def _proxy_general_settings() -> Mapping[str, object]: + from litellm.proxy.proxy_server import general_settings_view + + return general_settings_view() + + async def _with_managed_file_id_mapping( data: Mapping[str, object], managed_files: _ManagedFileAccess | None, user_api_key_dict: UserAPIKeyAuth ) -> Mapping[str, object]: @@ -1607,7 +1615,9 @@ def _with_authorized_previous_response( return {**data, "previous_response_id": authorized_id, ADDRESSED_RESPONSE_ID_FIELD: previous_response_id} -def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse: +def _responses_invalid_request_response( + message: str, param: str | None, code: str | None, body_call_id: str | None +) -> JSONResponse: body: Final[_ResponsesApiErrorBody] = { "error": { "message": message, @@ -1616,7 +1626,7 @@ def _responses_invalid_request_response(message: str, param: str | None, code: s "code": code, } } - return JSONResponse(status_code=400, content=body) + return JSONResponse(status_code=400, content={"error": with_call_id({**body["error"]}, body_call_id)}) class _ProviderErrorMessage(TypedDict): @@ -1681,16 +1691,17 @@ def _provider_count_error_body(raw_message: str, status_code: int) -> _Responses } -def _provider_count_error_response(exc: ProxyException, request: Request) -> JSONResponse: +def _provider_count_error_response( + exc: ProxyException, request: Request, general_settings: Mapping[str, object] +) -> JSONResponse: from litellm.proxy.proxy_server import ( _close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; a route that answers an error itself must stamp the OTel server span like the global ProxyException handler does - general_settings_view, ) status_code: Final = error_status_code(exc, 500) _close_dangling_otel_server_span(request, status_code, exc=exc) body: Final = _provider_count_error_body(exc.message, status_code) - call_id: Final = error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER)) + call_id: Final = error_body_call_id(general_settings, exc.headers.get(LITELLM_CALL_ID_HEADER)) return JSONResponse( status_code=status_code, content={"error": with_call_id({**body["error"]}, call_id)}, @@ -1698,11 +1709,12 @@ def _provider_count_error_response(exc: ProxyException, request: Request) -> JSO ) -def _missing_responses_param_response(param: str) -> JSONResponse: +def _missing_responses_param_response(param: str, body_call_id: str | None) -> JSONResponse: return _responses_invalid_request_response( message=f"Missing required parameter: '{param}'.", param=param, code="missing_required_parameter", + body_call_id=body_call_id, ) @@ -1744,6 +1756,8 @@ def _responses_input_as_token_count_messages( ) async def responses_input_tokens( request: Request, + fastapi_response: Response, + general_settings: Annotated[Mapping[str, object], Depends(_proxy_general_settings)], token_counter: _TokenCounter = _token_counter_dep, llm_router: Router | None = _llm_router_dep, user_api_key_dict: UserAPIKeyAuth = _user_api_key_auth_dep, @@ -1768,19 +1782,50 @@ async def responses_input_tokens( Returns: `{"object": "response.input_tokens", "input_tokens": }` """ litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)) + return await answered_with_litellm_call_id( + _responses_input_tokens( + request, + token_counter, + llm_router, + user_api_key_dict, + response_id_security, + managed_files, + litellm_call_id, + general_settings, + ), + litellm_call_id, + request, + fastapi_response, + ) + + +async def _responses_input_tokens( + request: Request, + token_counter: _TokenCounter, + llm_router: Router | None, + user_api_key_dict: UserAPIKeyAuth, + response_id_security: ResponsesIDSecurity | None, + managed_files: _ManagedFileAccess | None, + litellm_call_id: str, + general_settings: Mapping[str, object], +) -> JSONResponse | _ResponsesInputTokensResult: + body_call_id: Final = error_body_call_id(general_settings, litellm_call_id) data: Final = await read_request_body(request=request) model_name: Final = data.get("model") input_value: Final = data.get("input") if not isinstance(model_name, str) or not model_name: - return rejected_token_count(_missing_responses_param_response("model"), litellm_call_id, None) + return rejected_token_count(_missing_responses_param_response("model", body_call_id), litellm_call_id, None) if input_value is None: - return rejected_token_count(_missing_responses_param_response("input"), litellm_call_id, model_name) + return rejected_token_count( + _missing_responses_param_response("input", body_call_id), litellm_call_id, model_name + ) if isinstance(input_value, (str, list)) and not input_value: return rejected_token_count( _responses_invalid_request_response( message="""One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""", param=None, code="missing_required_parameter", + body_call_id=body_call_id, ), litellm_call_id, model_name, @@ -1792,7 +1837,9 @@ async def responses_input_tokens( token_fields: Final = resolve_counted_tool_fields(counted_fields, llm_router, user_api_key_dict.team_id) if isinstance(token_fields, ProxyException): return rejected_token_count( - _responses_invalid_request_response(message=token_fields.message, param=token_fields.param, code=None), + _responses_invalid_request_response( + message=token_fields.message, param=token_fields.param, code=None, body_call_id=body_call_id + ), litellm_call_id, model_name, ) @@ -1807,7 +1854,10 @@ async def responses_input_tokens( except ValidationError as e: return rejected_token_count( _responses_invalid_request_response( - message=first_validation_error(e), param=str(e.errors(include_url=False)[0]["loc"][0]), code=None + message=first_validation_error(e), + param=str(e.errors(include_url=False)[0]["loc"][0]), + code=None, + body_call_id=body_call_id, ), litellm_call_id, model_name, @@ -1815,7 +1865,7 @@ async def responses_input_tokens( except Exception as e: return rejected_token_count( _responses_invalid_request_response( - message=f"Invalid request for token counting: {e}", param=None, code=None + message=f"Invalid request for token counting: {e}", param=None, code=None, body_call_id=body_call_id ), litellm_call_id, model_name, @@ -1835,8 +1885,9 @@ async def responses_input_tokens( managed_files, user_api_key_dict, ) - except ProxyException as e: - raise rejected_token_count(e, litellm_call_id, model_name) + except Exception as e: + rejected_token_count(e, litellm_call_id, model_name) + raise try: token_response: Final = await token_counter( request=token_request, @@ -1847,7 +1898,7 @@ async def responses_input_tokens( except ProxyException as e: if e.type != _PROVIDER_COUNT_ERROR_TYPE: raise - return _provider_count_error_response(e, request) + return _provider_count_error_response(with_litellm_call_id(e, litellm_call_id), request, general_settings) result: Final[_ResponsesInputTokensResult] = { "object": "response.input_tokens", "input_tokens": token_response.total_tokens, diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 072de5c0eed..f8706efb4bd 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -557,7 +557,9 @@ class TestProxyExceptionAnthropicEnvelope: proxy_server, "count_request_tokens", new=AsyncMock(side_effect=unavailable) ), ): - response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens( + request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth() + ) body: Final = json.loads(response.body) assert response.status_code == expected_status @@ -840,7 +842,9 @@ class TestErrorLogCarriesCallId: ), # test-quality-ok: module global imported at call time; the test targets the endpoint's except block caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"), ): - response = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response = await ep.count_tokens( + request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth() + ) assert response.status_code == 500 record = self._error_record(caplog) @@ -883,7 +887,9 @@ class TestCountTokensCompactedHistory: patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam patch.object(proxy_server, "llm_router", None), # test-quality-ok: module global; with no router the count runs on the local tokenizer, the path that raised ): - return await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + return await ep.count_tokens( + request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth() + ) counted = await count(compacted) @@ -934,7 +940,9 @@ async def test_count_tokens_answers_400_for_malformed_tools_without_counting(mod patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts it is never reached patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route ): - response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens( + request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth() + ) assert response.status_code == 400 assert json.loads(response.body)["error"]["type"] == "invalid_request_error" @@ -957,7 +965,9 @@ async def test_count_tokens_counts_malformed_tool_entries_for_a_model_the_router patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route ): - response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens( + request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth() + ) assert response == {"input_tokens": 7} assert counter.await_args.kwargs["request"].tools == _MALFORMED_TOOL_ENTRY @@ -990,7 +1000,9 @@ async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set( patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives ): - response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens( + request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth() + ) assert response == {"input_tokens": 7} counted: Final = counter.await_args.kwargs["request"] @@ -1017,7 +1029,9 @@ async def test_count_tokens_keeps_claude_code_malformed_tools_when_operator_sets patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route patch.object(proxy_server, "proxy_config", new=operator_config), # test-quality-ok: module global imported at call time; no injection seam on this route ): - response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens( + request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth() + ) assert response.status_code == 400 assert json.loads(response.body)["error"]["type"] == "invalid_request_error" @@ -1055,7 +1069,9 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives patch.object(proxy_server, "llm_router", new=router), # test-quality-ok: module global imported at call time; no injection seam on this route ): - response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + response: Final = await ep.count_tokens( + request=request, fastapi_response=Response(), user_api_key_dict=UserAPIKeyAuth() + ) assert response == {"input_tokens": 7} assert counter.await_args.kwargs["request"].tools is None @@ -1246,7 +1262,7 @@ def test_count_tokens_rejected_before_counting_logs_the_400_it_answers( ) -> None: response: Final = _count_on_local_model(monkeypatch, body) - assert response.status_code == 400, response.text + assert (response.status_code, response.headers.get("x-litellm-call-id")) == (400, "count-log-local"), response.text assert token_count_log_lines() == ( { "litellm_call_id": "count-log-local", @@ -1303,6 +1319,59 @@ def test_count_tokens_on_bedrock_answers_400_for_a_message_without_content_witho assert len(respx_mock.calls) == 0 +def _claude_count_reply(status_code: int) -> Callable[[httpx.Request], httpx.Response]: + reply: Final = ( + {"input_tokens": 77} + if status_code == 200 + else {"type": "error", "error": {"type": "invalid_request_error", "message": "invalid base64 data"}} + ) + return lambda request: httpx.Response(status_code, json=reply, request=request) + + +@pytest.mark.parametrize("client_call_id", [None, "count-echo-claude"]) +@pytest.mark.parametrize("provider_status", [200, 400]) +def test_count_tokens_answers_the_call_id_its_count_line_names( + monkeypatch: pytest.MonkeyPatch, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], + provider_status: int, + client_call_id: str | None, +) -> None: + response: Final = _count_on_claude( + monkeypatch, + _claude_count_reply(provider_status), + {} if client_call_id is None else {"x-litellm-call-id": client_call_id}, + ) + + (line,) = token_count_log_lines() + assert (response.status_code, response.headers.get("x-litellm-call-id")) == ( + provider_status, + line["litellm_call_id"], + ), response.text + assert client_call_id in (None, line["litellm_call_id"]) + + +@pytest.mark.parametrize( + ("provider_status", "extra_body"), + [(400, MappingProxyType({})), (200, MappingProxyType({"tools": 5}))], + ids=["provider_rejection", "malformed_tools_rejection"], +) +def test_count_tokens_error_body_names_the_call_id_its_header_carries_when_opted_in( + monkeypatch: pytest.MonkeyPatch, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], + provider_status: int, + extra_body: Mapping[str, object], +) -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {"include_call_id_in_error_body": True}) + + response: Final = _count_on_claude(monkeypatch, _claude_count_reply(provider_status), {}, extra_body) + + (line,) = token_count_log_lines() + assert response.json()["error"]["litellm_call_id"] == response.headers.get("x-litellm-call-id") + assert response.headers.get("x-litellm-call-id") == line["litellm_call_id"] + + class TestEventLoggingBatchEndpoint: """Test the stubbed event logging batch endpoint""" diff --git a/tests/unit/proxy/common_utils/test_token_count_log.py b/tests/unit/proxy/common_utils/test_token_count_log.py index 94b976d88cf..b34bb55eb82 100644 --- a/tests/unit/proxy/common_utils/test_token_count_log.py +++ b/tests/unit/proxy/common_utils/test_token_count_log.py @@ -1,9 +1,12 @@ import asyncio from collections.abc import Callable +from typing import Final import pytest +from fastapi import HTTPException, Request, Response -from litellm.proxy.common_utils.token_count_log import logged_token_count +from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER +from litellm.proxy.common_utils.token_count_log import answered_with_litellm_call_id, logged_token_count from litellm.types.utils import TokenCountResponse @@ -15,6 +18,10 @@ async def _count_ending_in(error: BaseException) -> TokenCountResponse: raise error +def _count_request() -> Request: + return Request({"type": "http", "method": "POST", "path": "/utils/token_counter", "headers": []}) + + @pytest.mark.asyncio async def test_a_count_ending_in_a_base_exception_logs_a_500( token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], @@ -43,3 +50,46 @@ async def test_a_cancelled_count_writes_no_line( await logged_token_count(_count_ending_in(asyncio.CancelledError()), "count-log-cancelled", "count-log-gpt", None) assert token_count_log_lines() == () + + +@pytest.mark.asyncio +async def test_an_http_error_answers_its_status_and_detail_with_the_call_id_beside_its_own_headers() -> None: + rejection: Final = HTTPException(status_code=429, detail={"error": "slow down"}, headers={"retry-after": "3"}) + + with pytest.raises(HTTPException) as answered: + await answered_with_litellm_call_id(_count_ending_in(rejection), "count-echo-http", _count_request(), Response()) + + assert (answered.value.status_code, answered.value.detail, dict(answered.value.headers or {})) == ( + 429, + {"error": "slow down"}, + {"retry-after": "3", LITELLM_CALL_ID_HEADER: "count-echo-http"}, + ) + + +class _CountTooLarge(HTTPException): + pass + + +@pytest.mark.asyncio +async def test_an_http_error_reaches_its_handler_as_the_same_error_with_the_call_id() -> None: + too_large: Final = _CountTooLarge(status_code=413, detail="count body too large") + + with pytest.raises(_CountTooLarge) as answered: + await answered_with_litellm_call_id(_count_ending_in(too_large), "count-echo-same", _count_request(), Response()) + + assert answered.value is too_large + assert dict(answered.value.headers or {}) == {LITELLM_CALL_ID_HEADER: "count-echo-same"} + + +@pytest.mark.asyncio +async def test_a_cancelled_answer_stays_cancelled_and_names_no_call_id() -> None: + cancelled: Final = asyncio.CancelledError() + fastapi_response: Final = Response() + + with pytest.raises(asyncio.CancelledError) as raised: + await answered_with_litellm_call_id( + _count_ending_in(cancelled), "count-echo-cancelled", _count_request(), fastapi_response + ) + + assert raised.value is cancelled + assert LITELLM_CALL_ID_HEADER not in fastapi_response.headers diff --git a/tests/unit/proxy/google_endpoints/test_endpoints.py b/tests/unit/proxy/google_endpoints/test_endpoints.py index 9dd2e267e64..8e1a39dd176 100644 --- a/tests/unit/proxy/google_endpoints/test_endpoints.py +++ b/tests/unit/proxy/google_endpoints/test_endpoints.py @@ -8,6 +8,7 @@ import pytest from dotenv import load_dotenv from pydantic import JsonValue, ValidationError from starlette.requests import Request +from starlette.responses import Response from litellm.proxy.google_endpoints.endpoints import _google_count_tokens_request, google_count_tokens from litellm.types.llms.vertex_ai import TokenCountDetailsResponse @@ -102,6 +103,7 @@ async def test_proxy_gemini_to_openai_like_model_token_counting(): } ), model_name="volcengine/foo", + fastapi_response=Response(), ) assert response.get("totalTokens") > 0 diff --git a/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py b/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py index 4867ca9ab63..0509e6c68a3 100644 --- a/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py +++ b/tests/unit/proxy/google_endpoints/test_google_api_endpoints.py @@ -172,7 +172,7 @@ def test_google_count_tokens_unchanged(): ) with patch( - "litellm.proxy.proxy_server.token_counter", + "litellm.proxy.proxy_server.count_request_tokens", new_callable=AsyncMock, return_value=fake_response, ): @@ -194,7 +194,7 @@ def test_google_count_tokens_unexpected_failure_uses_the_google_error_envelope() pytest.skip(f"Skipping test due to missing dependency: {e}") with patch( - "litellm.proxy.proxy_server.token_counter", + "litellm.proxy.proxy_server.count_request_tokens", new_callable=AsyncMock, side_effect=RuntimeError("tokenizer unavailable"), ): @@ -227,7 +227,7 @@ def test_google_count_tokens_uses_normalized_total_when_provider_shape_differs() ) with patch( - "litellm.proxy.proxy_server.token_counter", + "litellm.proxy.proxy_server.count_request_tokens", new_callable=AsyncMock, return_value=fake_response, ): @@ -284,6 +284,7 @@ def test_google_count_tokens_logs_one_line_with_the_callers_call_id_model_and_co ) assert (response.status_code, response.json()["totalTokens"]) == (200, 77), response.text + assert response.headers.get("x-litellm-call-id") == "count-log-gemini" assert token_count_log_lines() == ( { "litellm_call_id": "count-log-gemini", @@ -318,3 +319,18 @@ def test_google_count_tokens_rejected_body_logs_the_400_it_answers( "status_code": 400, }, ) + + +@pytest.mark.parametrize("client_call_id", [None, "count-echo-gemini"]) +def test_google_count_tokens_rejected_body_answers_the_call_id_its_count_line_names( + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], client_call_id: str | None +) -> None: + response: Final = _build_test_client().post( + "/v1beta/models/count-log-claude:countTokens", + json={"contents": "not a list"}, + headers={} if client_call_id is None else {"x-litellm-call-id": client_call_id}, + ) + + (line,) = token_count_log_lines() + assert (response.status_code, response.headers.get("x-litellm-call-id")) == (400, line["litellm_call_id"]) + assert client_call_id in (None, line["litellm_call_id"]) diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 8e88fd7449a..7f118fb3186 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8250,6 +8250,7 @@ async def _bedrock_native_count( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, aws_reply: httpx.Response, + fastapi_response: Response, request_body: Mapping[str, object] = _BEDROCK_COUNT_BODY, ) -> dict[str, object]: from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( @@ -8288,7 +8289,7 @@ async def _bedrock_native_count( return await handle_bedrock_count_tokens( endpoint=f"model/{_BEDROCK_COUNT_ALIAS}/count-tokens", request=request, - fastapi_response=Response(), + fastapi_response=fastapi_response, user_api_key_dict=MagicMock(), request_body=dict(request_body), ) @@ -8301,9 +8302,13 @@ async def test_bedrock_native_count_tokens_logs_the_count_against_the_router_mod respx_mock: respx.MockRouter, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], ) -> None: - result: Final = await _bedrock_native_count(monkeypatch, respx_mock, httpx.Response(200, json={"inputTokens": 12})) + fastapi_response: Final = Response() + result: Final = await _bedrock_native_count( + monkeypatch, respx_mock, httpx.Response(200, json={"inputTokens": 12}), fastapi_response + ) assert result == {"input_tokens": 12} + assert fastapi_response.headers.get("x-litellm-call-id") == "bedrock-native-count" assert len(respx_mock.calls) == 1 assert respx_mock.calls.last.request.url.path == "/model/anthropic.claude-haiku-4-5-20251001-v1:0/count-tokens" assert token_count_log_lines() == ( @@ -8328,9 +8333,12 @@ async def test_bedrock_native_count_tokens_logs_the_provider_status_when_the_cou ) -> None: rejected: Final = httpx.Response(400, json={"message": "The provided model doesn't support counting tokens."}) with pytest.raises(HTTPException) as exc_info: - await _bedrock_native_count(monkeypatch, respx_mock, rejected) + await _bedrock_native_count(monkeypatch, respx_mock, rejected, Response()) - assert exc_info.value.status_code == 400 + assert (exc_info.value.status_code, (exc_info.value.headers or {}).get("x-litellm-call-id")) == ( + 400, + "bedrock-native-count", + ) assert token_count_log_lines() == ( { "litellm_call_id": "bedrock-native-count", @@ -8363,7 +8371,7 @@ async def test_bedrock_native_count_tokens_answers_400_for_a_malformed_body_with ) -> None: with pytest.raises(HTTPException) as exc_info: await _bedrock_native_count( - monkeypatch, respx_mock, httpx.Response(200, json={"inputTokens": 12}), request_body + monkeypatch, respx_mock, httpx.Response(200, json={"inputTokens": 12}), Response(), request_body ) assert (exc_info.value.status_code, exc_info.value.detail) == (400, {"error": problem}) @@ -8458,6 +8466,33 @@ async def test_bedrock_native_count_tokens_without_a_model_logs_the_400_it_answe ) +@pytest.mark.asyncio +async def test_bedrock_native_count_tokens_without_a_model_answers_the_call_id_its_count_line_names( + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], +) -> None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + handle_bedrock_count_tokens, + ) + + request: Final = Request( + {"type": "http", "method": "POST", "path": f"/bedrock/model/{_BEDROCK_COUNT_ALIAS}/count-tokens", "headers": []} + ) + with pytest.raises(HTTPException) as exc_info: + await handle_bedrock_count_tokens( + endpoint=f"model/{_BEDROCK_COUNT_ALIAS}/count-tokens", + request=request, + fastapi_response=Response(), + user_api_key_dict=MagicMock(), + request_body={"messages": [{"role": "user", "content": "hi"}]}, + ) + + (line,) = token_count_log_lines() + assert (exc_info.value.status_code, (exc_info.value.headers or {}).get("x-litellm-call-id")) == ( + 400, + line["litellm_call_id"], + ) + + @pytest.mark.asyncio async def test_bedrock_native_count_tokens_with_a_region_only_model_answers_the_validation_error_and_logs_it( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index e4e4b27e1c2..0a28bb507d2 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -5416,10 +5416,13 @@ class TestResponsesInputTokens: response_id_security: object | None = None, managed_files: object | None = None, headers: Mapping[str, str] = EMPTY_MAPPING, + general_settings: Mapping[str, object] | None = None, + raise_server_exceptions: bool = False, ) -> tuple[Response, AsyncMock]: from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.response_api_endpoints.endpoints import ( + _proxy_general_settings, _proxy_llm_router, _proxy_managed_files_hook, _proxy_response_id_security, @@ -5446,8 +5449,10 @@ class TestResponsesInputTokens: app.dependency_overrides[_proxy_response_id_security] = lambda: response_id_security if managed_files is not None: app.dependency_overrides[_proxy_managed_files_hook] = lambda: managed_files + if general_settings is not None: + app.dependency_overrides[_proxy_general_settings] = lambda: general_settings try: - client = TestClient(app) + client = TestClient(app, raise_server_exceptions=raise_server_exceptions) response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876", **headers}) return response, token_counter_mock finally: @@ -5456,6 +5461,7 @@ class TestResponsesInputTokens: app.dependency_overrides.pop(_proxy_llm_router, None) app.dependency_overrides.pop(_proxy_response_id_security, None) app.dependency_overrides.pop(_proxy_managed_files_hook, None) + app.dependency_overrides.pop(_proxy_general_settings, None) def test_string_input_returns_openai_input_tokens_shape(self): response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"}) @@ -5738,6 +5744,7 @@ class TestResponsesInputTokens: ) assert (response.status_code, response.json().get("input_tokens")) == (200, 7), response.text + assert response.headers.get("x-litellm-call-id") == "count-log-responses" assert token_count_log_lines() == ( { "litellm_call_id": "count-log-responses", @@ -5771,6 +5778,81 @@ class TestResponsesInputTokens: }, ) + def test_count_rejected_before_counting_answers_the_call_id_its_count_line_names( + self, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] + ) -> None: + response, _ = self._post_input_tokens({"model": "count-log-gpt"}) + + (line,) = token_count_log_lines() + assert (response.status_code, response.headers.get("x-litellm-call-id")) == (400, line["litellm_call_id"]) + + @pytest.mark.parametrize( + ("raised", "status_code"), + [ + ( + ProxyException( + message="Token counting is disabled", type="token_counting_disabled", param="model", code=503 + ), + 503, + ), + ( + ProxyException( + message="LiteLLM does not support counting tokens for this model.", + type="token_counting_error", + param="model", + code=400, + ), + 400, + ), + ], + ids=["proxy-error", "provider-count-error"], + ) + def test_a_failed_count_answers_the_callers_call_id(self, raised: Exception, status_code: int) -> None: + response, _ = self._post_input_tokens( + {"model": "gpt-4o", "input": "hi"}, + counter=AsyncMock(side_effect=raised), + headers={"x-litellm-call-id": "count-echo-failed"}, + ) + + assert (response.status_code, response.headers.get("x-litellm-call-id")) == ( + status_code, + "count-echo-failed", + ), response.text + + def test_a_crashed_count_answers_the_gateways_internal_error_with_the_callers_call_id(self) -> None: + response, _ = self._post_input_tokens( + {"model": "gpt-4o", "input": "hi"}, + counter=AsyncMock(side_effect=RuntimeError("tokenizer crashed reading /etc/litellm/config.yaml")), + headers={"x-litellm-call-id": "count-echo-crashed"}, + ) + + assert (response.status_code, response.headers.get("x-litellm-call-id")) == (500, "count-echo-crashed") + assert response.json() == {"error": {"message": "Internal server error", "type": "internal_server_error"}} + + def test_a_crashed_count_reaches_error_tracking_as_the_error_it_raised(self) -> None: + crash: Final = RuntimeError("tokenizer crashed") + + with pytest.raises(RuntimeError) as raised: + self._post_input_tokens( + {"model": "gpt-4o", "input": "hi"}, + counter=AsyncMock(side_effect=crash), + raise_server_exceptions=True, + ) + + assert raised.value is crash + + def test_a_count_during_a_database_outage_answers_503_with_the_callers_call_id(self) -> None: + from prisma.errors import ClientNotConnectedError + + response, _ = self._post_input_tokens( + {"model": "gpt-4o", "input": "hi"}, + counter=AsyncMock(side_effect=ClientNotConnectedError()), + headers={"x-litellm-call-id": "count-echo-db-down"}, + ) + + assert (response.status_code, response.headers.get("x-litellm-call-id")) == (503, "count-echo-db-down") + assert response.json()["error"]["type"] == "no_db_connection" + def test_a_managed_file_the_key_cannot_read_answers_403_without_a_count(self) -> None: from litellm.integrations.custom_logger import CustomLogger @@ -5792,6 +5874,28 @@ class TestResponsesInputTokens: assert response.json()["error"]["message"] == denied counter.assert_not_awaited() + def test_a_managed_file_lookup_crash_answers_the_call_id_its_count_line_names( + self, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] + ) -> None: + from litellm.integrations.custom_logger import CustomLogger + + class ManagedFiles(CustomLogger): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + raise RuntimeError("managed file table unreadable") + + response, counter = self._post_input_tokens( + { + "model": "count-log-gpt", + "input": [{"role": "user", "content": [{"type": "input_file", "file_id": "litellm-managed-file"}]}], + }, + managed_files=ManagedFiles(), + ) + + (line,) = token_count_log_lines() + assert (response.status_code, response.headers.get("x-litellm-call-id")) == (500, line["litellm_call_id"]) + assert line["status_code"] == 500 + counter.assert_not_awaited() + def test_malformed_tools_follow_the_counted_deployment_even_when_fallbacks_are_configured(self) -> None: router: Final = litellm.Router( model_list=[ @@ -6057,6 +6161,34 @@ class TestResponsesInputTokens: } + def test_provider_count_error_body_names_the_call_id_its_header_carries_when_opted_in(self) -> None: + failing_counter: Final = AsyncMock( + side_effect=ProxyException( + message="LiteLLM does not support counting tokens for this model.", + type="token_counting_error", + param="model", + code="400", + ) + ) + with patch("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True}): + response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, counter=failing_counter) + + assert response.json()["error"]["litellm_call_id"] == response.headers.get("x-litellm-call-id") + + @pytest.mark.parametrize( + "body", + [{"input": "hi"}, {"model": "gpt-4o"}, {"model": "gpt-4o", "input": ""}], + ) + def test_count_rejection_body_names_the_call_id_its_header_carries_when_opted_in( + self, body: Mapping[str, object] + ) -> None: + response, counter = self._post_input_tokens(dict(body), general_settings={"include_call_id_in_error_body": True}) + + counter.assert_not_awaited() + assert response.status_code == 400, response.text + assert response.json()["error"]["litellm_call_id"] == response.headers.get("x-litellm-call-id") + + class TestGuardrailBlockedResponsesShape: """A pre_call block raises ModifyResponseException before any provider call. diff --git a/tests/unit/proxy/test_custom_tokenizer_bug.py b/tests/unit/proxy/test_custom_tokenizer_bug.py index 29625ea1a32..9b2c0498264 100644 --- a/tests/unit/proxy/test_custom_tokenizer_bug.py +++ b/tests/unit/proxy/test_custom_tokenizer_bug.py @@ -19,7 +19,7 @@ import litellm.proxy.proxy_server import litellm.utils from litellm import Router from litellm.proxy._types import ProxyException, TokenCountRequest -from litellm.proxy.proxy_server import token_counter +from litellm.proxy.proxy_server import count_request_tokens def _fake_hf_tokenizer(num_tokens: int) -> MagicMock: @@ -60,7 +60,7 @@ async def test_custom_tokenizer_from_model_info_is_used(monkeypatch): with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: mock_tokenizer_cls.from_pretrained.return_value = _fake_hf_tokenizer(7) - response = await token_counter( + response = await count_request_tokens( request=TokenCountRequest( model="my-embedding-model", messages=[{"role": "user", "content": "Bonjour le monde"}], @@ -92,7 +92,7 @@ async def test_model_without_custom_tokenizer_uses_default(monkeypatch): monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: - response = await token_counter( + response = await count_request_tokens( request=TokenCountRequest( model="gpt-4", messages=[{"role": "user", "content": "hello"}], @@ -133,7 +133,7 @@ async def test_custom_tokenizer_counts_a_vertex_partner_model_without_a_count_ro with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: mock_tokenizer_cls.from_pretrained.return_value = _fake_hf_tokenizer(9) - response: Final = await token_counter( + response: Final = await count_request_tokens( request=TokenCountRequest(model="vertex-glm", messages=[{"role": "user", "content": "Return exactly OK."}]), call_endpoint=True, ) @@ -151,7 +151,7 @@ async def test_contents_only_count_of_a_vertex_partner_model_without_a_count_rou monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", _vertex_glm_router_with_its_own_tokenizer()) with pytest.raises(ProxyException) as raised: - await token_counter( + await count_request_tokens( request=TokenCountRequest( model="vertex-glm", contents=[{"role": "user", "parts": [{"text": "Return exactly OK."}]}] ), diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 5b423421458..4f96af7f02c 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -15778,7 +15778,7 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun warm_tokenizer("claude-fable-5") response, took, lags = await timed_with_loop_lags( - lambda: proxy_server_module.token_counter(TokenCountRequest(model="claude-fable-5", prompt=text * 100)) + lambda: proxy_server_module.count_request_tokens(TokenCountRequest(model="claude-fable-5", prompt=text * 100)) ) assert response.total_tokens > 0 @@ -15819,7 +15819,7 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp litellm.utils._select_custom_tokenizer_helper.cache_clear() try: - response: Final = await proxy_server_module.token_counter( + response: Final = await proxy_server_module.count_request_tokens( TokenCountRequest(model="self-hosted", prompt="count me off the loop") ) assert loader_threads.get_nowait() is not loop_thread @@ -15860,7 +15860,7 @@ async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revisi litellm.utils._select_custom_tokenizer_helper.cache_clear() try: responses: Final = [ - await proxy_server_module.token_counter(TokenCountRequest(model="self-hosted", prompt="count me once")) + await proxy_server_module.count_request_tokens(TokenCountRequest(model="self-hosted", prompt="count me once")) for _ in range(3) ] assert from_pretrained.call_args_list == [mock.call("my-org/tokenizer", revision="main", token=None)] @@ -15868,8 +15868,8 @@ async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revisi assert len({response.total_tokens for response in responses}) == 1 assert responses[0].total_tokens > 0 - await proxy_server_module.token_counter(TokenCountRequest(model="self-hosted-pinned", prompt="count me once")) - await proxy_server_module.token_counter(TokenCountRequest(model="self-hosted-private", prompt="count me once")) + await proxy_server_module.count_request_tokens(TokenCountRequest(model="self-hosted-pinned", prompt="count me once")) + await proxy_server_module.count_request_tokens(TokenCountRequest(model="self-hosted-private", prompt="count me once")) assert from_pretrained.call_args_list == [ mock.call("my-org/tokenizer", revision="main", token=None), mock.call("my-org/tokenizer", revision="v2", token=None), diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 34868606012..bbd3ce8c78d 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -38,7 +38,7 @@ load_dotenv() # this file is to test litellm/proxy -from fastapi import Request +from fastapi import Request, Response from fastapi.responses import JSONResponse from starlette.types import Message @@ -83,7 +83,6 @@ from litellm.proxy.proxy_server import ( _provider_token_count, _try_provider_token_count, count_request_tokens, - token_counter, ) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted @@ -139,7 +138,7 @@ async def test_vLLM_token_counting(): setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - response = await token_counter( + response = await count_request_tokens( request=TokenCountRequest( model="special-alias", messages=[{"role": "user", "content": "hello"}], @@ -172,7 +171,7 @@ async def test_token_counting_model_not_in_model_list(): setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - response = await token_counter( + response = await count_request_tokens( request=TokenCountRequest( model="special-alias", messages=[{"role": "user", "content": "hello"}], @@ -209,8 +208,8 @@ async def test_asking_the_provider_endpoint_leaves_an_openrouter_deployments_cou model="or-gpt", messages=[{"role": "user", "content": "Count these tokens, ünïcödé 你好 🙂"}] ) - via_provider_endpoint: Final = await token_counter(request=request, call_endpoint=True) - counted_locally: Final = await token_counter(request=request, call_endpoint=False) + via_provider_endpoint: Final = await count_request_tokens(request=request, call_endpoint=True) + counted_locally: Final = await count_request_tokens(request=request, call_endpoint=False) assert (via_provider_endpoint.total_tokens, via_provider_endpoint.model_used) == ( counted_locally.total_tokens, @@ -238,7 +237,7 @@ async def test_gpt_token_counting(): setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - response = await token_counter( + response = await count_request_tokens( request=TokenCountRequest( model="gpt-4", messages=[{"role": "user", "content": "hello"}], @@ -281,7 +280,7 @@ async def test_token_counter_leaves_the_routers_deployment_without_the_credentia ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - response = await token_counter( + response = await count_request_tokens( request=TokenCountRequest(model="devstral", messages=[{"role": "user", "content": "hello"}]) ) @@ -312,10 +311,10 @@ async def test_token_counter_reads_an_oidc_api_key_only_when_it_calls_the_provid request = TokenCountRequest(model="claude", messages=[{"role": "user", "content": "hello"}]) if not call_endpoint: - assert (await token_counter(request=request, call_endpoint=call_endpoint)).total_tokens > 0 + assert (await count_request_tokens(request=request, call_endpoint=call_endpoint)).total_tokens > 0 return with pytest.raises(ProxyException) as excinfo: - await token_counter(request=request, call_endpoint=call_endpoint) + await count_request_tokens(request=request, call_endpoint=call_endpoint) assert excinfo.value.code == "401" assert str(tmp_path) not in excinfo.value.message @@ -343,7 +342,7 @@ async def test_token_counter_answers_503_when_the_metadata_server_behind_an_oidc 500, text="metadata server unavailable" ) with pytest.raises(ProxyException) as excinfo: - await token_counter( + await count_request_tokens( request=TokenCountRequest(model="claude", messages=[{"role": "user", "content": "hello"}]), call_endpoint=True, ) @@ -463,7 +462,7 @@ async def test_anthropic_messages_count_tokens_endpoint(): try: # Call the endpoint - response = await count_tokens(mock_request, mock_user_api_key_dict) + response = await count_tokens(mock_request, Response(), mock_user_api_key_dict) # Verify response format matches Anthropic spec assert isinstance(response, dict) @@ -545,7 +544,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): try: # Call the endpoint - response = await count_tokens(mock_request, mock_user_api_key_dict) + response = await count_tokens(mock_request, Response(), mock_user_api_key_dict) # Verify response format matches Anthropic spec assert isinstance(response, dict) @@ -586,7 +585,7 @@ async def test_internal_token_counter_anthropic_provider_detection(): # Test with is_direct_request=False (simulating call from Anthropic endpoint) # The provider counter is engaged: its API error surfaces instead of a local count with pytest.raises(ProxyException) as exc_info: - await token_counter( + await count_request_tokens( request=TokenCountRequest( model="claude-test", messages=[{"role": "user", "content": "hello"}], @@ -610,7 +609,7 @@ async def test_internal_token_counter_anthropic_provider_detection(): setattr(litellm.proxy.proxy_server, "llm_router", llm_router) with pytest.raises(ProxyException) as missing_key: - await token_counter( + await count_request_tokens( request=TokenCountRequest( model="gpt-test", messages=[{"role": "user", "content": "hello"}], @@ -653,7 +652,7 @@ async def test_anthropic_endpoint_error_handling(): anthropic_endpoints.read_request_body = mock_read_request_body try: - response = await count_tokens(mock_request, mock_user_api_key_dict) + response = await count_tokens(mock_request, Response(), mock_user_api_key_dict) assert response.status_code == 400 assert json.loads(response.body) == { @@ -947,7 +946,7 @@ async def test_vertex_ai_anthropic_token_counting(): mock_handle_count_tokens.return_value = mock_token_response # Test with messages format and call_endpoint=True - response = await token_counter( + response = await count_request_tokens( request=TokenCountRequest( model="vertex_ai/claude-3-5-sonnet-20241022", messages=[ @@ -1161,7 +1160,7 @@ async def test_token_counter_httpx_status_error_raises_proxy_exception(): litellm.proxy.proxy_server.llm_router = mock_router with pytest.raises(ProxyException) as exc_info: - await token_counter( + await count_request_tokens( request=TokenCountRequest( model="claude-4-6-sonnet", messages=[{"role": "user", "content": "hello"}], @@ -1227,7 +1226,7 @@ async def test_proxy_token_counter_error_raises_exception_when_disabled(): litellm.proxy.proxy_server._get_provider_token_counter = mock_get_provider_token_counter with pytest.raises(ProxyException) as exc_info: - await token_counter( + await count_request_tokens( request=TokenCountRequest( model="claude-bedrock", messages=[{"role": "user", "content": "hello"}], @@ -1292,7 +1291,7 @@ async def test_proxy_token_counter_error_raises_when_enabled(): with patch("litellm.token_counter") as mock_local_counter: with pytest.raises(ProxyException) as exc_info: - await token_counter( + await count_request_tokens( request=TokenCountRequest( model="claude-bedrock", messages=[{"role": "user", "content": "hello"}], @@ -1356,7 +1355,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format(): proxy_server.count_request_tokens = mock_token_counter_error try: - response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict) + response: Final[object] = await anthropic_count_tokens(mock_request, Response(), mock_user_api_key_dict) assert isinstance(response, JSONResponse) assert response.status_code == 400 @@ -1415,7 +1414,7 @@ async def test_anthropic_endpoint_403_permission_error_format(): proxy_server.count_request_tokens = mock_token_counter_error try: - response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict) + response: Final[object] = await anthropic_count_tokens(mock_request, Response(), mock_user_api_key_dict) assert isinstance(response, JSONResponse) assert response.status_code == 403 @@ -1474,7 +1473,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): proxy_server.count_request_tokens = mock_token_counter_error try: - response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict) + response: Final[object] = await anthropic_count_tokens(mock_request, Response(), mock_user_api_key_dict) assert isinstance(response, JSONResponse) assert response.status_code == 429 @@ -1550,7 +1549,7 @@ async def test_anthropic_count_tokens_drops_a_malformed_tool_result_text_when_dr route: Final = respx.post("https://api.openai.com/v1/responses/input_tokens").mock( return_value=httpx.Response(200, json={"input_tokens": 9}) ) - response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + response: Final = await anthropic_count_tokens(request, Response(), UserAPIKeyAuth()) assert response["input_tokens"] == 9 sent: Final = json.loads(route.calls.last.request.content) @@ -1595,7 +1594,7 @@ async def _count_on_a_model_without_a_count_api( monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) with respx.mock: - return await anthropic_count_tokens(request, UserAPIKeyAuth()) + return await anthropic_count_tokens(request, Response(), UserAPIKeyAuth()) @pytest.mark.asyncio @@ -1676,7 +1675,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock( return_value=httpx.Response(200, json={"input_tokens": 7}) ) - response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + response: Final = await anthropic_count_tokens(request, Response(), UserAPIKeyAuth()) assert response["input_tokens"] == 7 sent: Final = json.loads(route.calls.last.request.content) @@ -2191,8 +2190,8 @@ async def test_framed_estimate_that_cannot_read_the_body_answers_the_provider_er ) with pytest.raises(ProxyException) as raised: - await token_counter(request=TokenCountRequest(model="kimi", **unreadable_body), call_endpoint=True) - readable: Final = await token_counter( + await count_request_tokens(request=TokenCountRequest(model="kimi", **unreadable_body), call_endpoint=True) + readable: Final = await count_request_tokens( request=TokenCountRequest(model="kimi", messages=[{"role": "user", "content": "What is new?"}]), call_endpoint=True, ) @@ -2223,7 +2222,7 @@ async def test_framed_estimate_counts_a_json_schema_shorthand_like_its_explicit_ ) async def count(schema: object) -> TokenCountResponse: - return await token_counter( + return await count_request_tokens( request=TokenCountRequest( model="kimi", messages=[{"role": "user", "content": "What is new?"}], @@ -2277,8 +2276,8 @@ async def test_framed_estimate_counts_google_contents_like_the_same_conversation _framed_kimi(monkeypatch) google_request: Final = _google_count_tokens_request("kimi", {"contents": contents}) - from_google: Final = await token_counter(request=google_request, call_endpoint=True) - from_messages: Final = await token_counter( + from_google: Final = await count_request_tokens(request=google_request, call_endpoint=True) + from_messages: Final = await count_request_tokens( request=TokenCountRequest(model="kimi", messages=google_request.messages), call_endpoint=True ) @@ -2309,7 +2308,7 @@ async def test_framed_deployment_whose_custom_tokenizer_cannot_load_answers_the_ ) with pytest.raises(ProxyException) as raised: - await token_counter( + await count_request_tokens( request=TokenCountRequest(model="kimi", messages=[{"role": "user", "content": "What is new?"}]), call_endpoint=True, ) @@ -2361,7 +2360,7 @@ def _framed_kimi_with_custom_tokenizer(monkeypatch: pytest.MonkeyPatch, identifi async def _count_status(model: str = "kimi") -> str: try: - response: Final = await token_counter( + response: Final = await count_request_tokens( request=TokenCountRequest(model=model, messages=[{"role": "user", "content": "What is new?"}]), call_endpoint=True, ) @@ -2625,8 +2624,8 @@ async def test_local_count_loads_the_tokenizer_again_after_a_failed_load(monkeyp request: Final = TokenCountRequest(model="kimi", messages=[{"role": "user", "content": "What is new?"}]) with pytest.raises(OSError, match="hub unreachable"): - await token_counter(request=request) - retried: Final = await token_counter(request=request) + await count_request_tokens(request=request) + retried: Final = await count_request_tokens(request=request) assert retried.count_source == "local_estimate" and retried.total_tokens > 0 @@ -2656,15 +2655,15 @@ async def test_framed_count_loads_the_tokenizer_again_only_after_the_window_of_a monkeypatch.setattr(litellm.proxy.proxy_server, "TOKENIZER_LOAD_MAX_RUNNING", 1) request: Final = TokenCountRequest(model="kimi", messages=[{"role": "user", "content": "What is new?"}]) - first: Final = await token_counter(request=request, call_endpoint=True) - second: Final = await token_counter(request=request, call_endpoint=True) + first: Final = await count_request_tokens(request=request, call_endpoint=True) + second: Final = await count_request_tokens(request=request, call_endpoint=True) assert (first.tokenizer_type, second.tokenizer_type) == tokenizer_types async def _fork_guard_traceback_frames() -> int: with pytest.raises(ForkedAfterNativeRuntimeStarted) as raised: - await token_counter( + await count_request_tokens( request=TokenCountRequest(model="kimi", messages=[{"role": "user", "content": "What is new?"}]), call_endpoint=True, ) @@ -2703,7 +2702,7 @@ async def test_swallowed_estimate_error_is_logged_by_type_without_the_request_co verbose_proxy_logger.addHandler(caplog.handler) try: with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name), pytest.raises(ProxyException): - await token_counter( + await count_request_tokens( request=TokenCountRequest( model="kimi", messages=[ @@ -2748,9 +2747,9 @@ async def test_local_estimate_adds_the_configured_framing_once_per_request(monke ) messages: Final = [{"role": "system", "content": "be brief"}, {"role": "user", "content": "hello there"}] - plain: Final = await token_counter(request=TokenCountRequest(model="plain", messages=messages)) - framed: Final = await token_counter(request=TokenCountRequest(model="framed", messages=messages)) - framed_prompt: Final = await token_counter(request=TokenCountRequest(model="framed", prompt="hello there")) + plain: Final = await count_request_tokens(request=TokenCountRequest(model="plain", messages=messages)) + framed: Final = await count_request_tokens(request=TokenCountRequest(model="framed", messages=messages)) + framed_prompt: Final = await count_request_tokens(request=TokenCountRequest(model="framed", prompt="hello there")) assert plain.total_tokens == litellm.token_counter(model="gpt-5.4-mini", messages=messages) assert framed.total_tokens == plain.total_tokens + 100 @@ -2782,9 +2781,9 @@ async def test_framing_comes_from_the_row_the_deployment_provider_or_base_model_ messages: Final = [{"role": "user", "content": "hello there"}] framing: Final = cost_map_entry(kimi)["token_count_framing"] - prefixed: Final = await token_counter(request=TokenCountRequest(model="prefixed", messages=messages)) - split: Final = await token_counter(request=TokenCountRequest(model="split", messages=messages)) - aliased: Final = await token_counter(request=TokenCountRequest(model="aliased", messages=messages)) + prefixed: Final = await count_request_tokens(request=TokenCountRequest(model="prefixed", messages=messages)) + split: Final = await count_request_tokens(request=TokenCountRequest(model="split", messages=messages)) + aliased: Final = await count_request_tokens(request=TokenCountRequest(model="aliased", messages=messages)) assert prefixed.total_tokens == litellm.token_counter(model="system.ai.kimi-k3", messages=messages) + framing assert split.total_tokens == prefixed.total_tokens @@ -2964,7 +2963,7 @@ async def test_local_estimate_matches_the_billed_prompt_with_the_shipped_framing ), ) - counted: Final = await token_counter( + counted: Final = await count_request_tokens( request=TokenCountRequest( model="billed", messages=[{"role": "user", "content": "What is the weather in Paris?"}], @@ -3363,7 +3362,9 @@ async def test_bedrock_kimi_k3_deployment_with_a_custom_tokenizer_keeps_counting monkeypatch.setattr(litellm.utils, "select_tokenizer", lambda model, custom_tokenizer=None: custom) messages: Final = [{"role": "user", "content": "What is new?"}] - counted: Final = await token_counter(request=TokenCountRequest(model="kimi", messages=messages), call_endpoint=True) + counted: Final = await count_request_tokens( + request=TokenCountRequest(model="kimi", messages=messages), call_endpoint=True + ) assert counted.total_tokens == ( litellm.token_counter(model=counted.model_used, messages=messages, custom_tokenizer=custom) @@ -3398,7 +3399,7 @@ async def test_bedrock_kimi_k3_without_its_tokenizer_adds_the_template_offsets_t ): _framed_kimi(monkeypatch) - counted: Final = await token_counter( + counted: Final = await count_request_tokens( request=TokenCountRequest(model="kimi", messages=messages, tools=tools), call_endpoint=True ) estimate: Final = litellm.token_counter( @@ -3427,7 +3428,7 @@ async def test_slow_cost_map_lookup_does_not_stall_other_requests_on_the_worker( monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", None) monkeypatch.setattr(litellm, "get_model_info", slow_get_model_info) counting: Final = asyncio.create_task( - token_counter( + count_request_tokens( request=TokenCountRequest(model="slow-lookup-model", messages=[{"role": "user", "content": "hi"}]) ) ) @@ -3462,7 +3463,7 @@ async def test_uncountable_model_keeps_the_provider_error_when_local_counting_is monkeypatch.setattr(litellm, "disable_token_counter", True) with pytest.raises(ProxyException) as raised: - await token_counter( + await count_request_tokens( request=TokenCountRequest(model="kimi", messages=[{"role": "user", "content": "hi"}]), call_endpoint=True ) @@ -3512,7 +3513,7 @@ async def test_configured_token_count_model_overrides_the_shipped_one_and_an_emp litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock") ) - counted: Final = await token_counter( + counted: Final = await count_request_tokens( request=TokenCountRequest(model="sonnet", messages=[{"role": "user", "content": "hi"}]), call_endpoint=True ) @@ -3550,7 +3551,7 @@ async def test_shipped_token_count_model_keeps_the_prompt_in_the_deployment_geog litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock") ) - await token_counter( + await count_request_tokens( request=TokenCountRequest(model="sonnet", messages=[{"role": "user", "content": "hi"}]), call_endpoint=True ) @@ -3608,7 +3609,7 @@ async def test_region_lookup_for_a_shipped_count_model_leaves_the_event_loop_fre monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) async def count() -> TokenCountResponse: - return await token_counter( + return await count_request_tokens( request=TokenCountRequest(model="sonnet", messages=[{"role": "user", "content": "hi"}]), call_endpoint=True, ) @@ -3636,7 +3637,7 @@ async def test_framing_still_applies_while_the_deployment_is_cooling_down(monkey ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) messages: Final = [{"role": "user", "content": "hello there"}] - warm: Final = await token_counter(request=TokenCountRequest(model="framed", messages=messages)) + warm: Final = await count_request_tokens(request=TokenCountRequest(model="framed", messages=messages)) router.cooldown_cache.add_deployment_to_cooldown( model_id="framed-deployment", @@ -3644,7 +3645,7 @@ async def test_framing_still_applies_while_the_deployment_is_cooling_down(monkey exception_status=429, cooldown_time=60, ) - cooling: Final = await token_counter(request=TokenCountRequest(model="framed", messages=messages)) + cooling: Final = await count_request_tokens(request=TokenCountRequest(model="framed", messages=messages)) assert warm.total_tokens == litellm.token_counter(model="gpt-5.4-mini", messages=messages) + 100 assert cooling.total_tokens - 100 == litellm.token_counter(model="framed", messages=messages) @@ -3738,7 +3739,7 @@ async def _count_through_the_messages_endpoint( return_value=AsyncHTTPHandler(transport=httpx.MockTransport(peer)), ), ): - response: Final = await anthropic_count_tokens(mock_request, MagicMock()) + response: Final = await anthropic_count_tokens(mock_request, Response(), MagicMock()) return response, peer.request_bodies @@ -4098,7 +4099,7 @@ async def _messages_endpoint_count(router: Router, request_body: Mapping[str, Js patch.object(proxy_server, "llm_router", router), patch.object(anthropic_endpoints, "read_request_body", AsyncMock(return_value=dict(request_body))), ): - return await anthropic_count_tokens(mock_request, MagicMock()) + return await anthropic_count_tokens(mock_request, Response(), MagicMock()) class _MessagesCountReply(BaseModel): @@ -4651,10 +4652,10 @@ async def test_call_endpoint_counts_a_prompt_like_the_same_text_sent_as_a_user_m client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_gemini_word_counting_peer)) with patch.object(gemini_count_tokens_handler, "get_async_httpx_client", return_value=client): - from_prompt: Final = await token_counter( + from_prompt: Final = await count_request_tokens( request=TokenCountRequest(model="gemini-counted", prompt=text), call_endpoint=True ) - from_message: Final = await token_counter( + from_message: Final = await count_request_tokens( request=TokenCountRequest(model="gemini-counted", messages=[{"role": "user", "content": text}]), call_endpoint=True, ) @@ -4700,7 +4701,7 @@ async def test_call_endpoint_counts_a_prompt_on_a_vertex_ai_deployment(monkeypat with patch.object( VertexAIPartnerModelsTokenCounter, "handle_count_tokens_request", side_effect=_vertex_partner_word_count ): - from_prompt: Final = await token_counter( + from_prompt: Final = await count_request_tokens( request=TokenCountRequest(model="vertex-counted", prompt=text), call_endpoint=True ) @@ -4729,7 +4730,7 @@ async def test_prompt_next_to_a_conversation_answers_400_without_drop_params( patch.object(gemini_count_tokens_handler, "get_async_httpx_client", return_value=client), pytest.raises(ProxyException) as rejected, ): - await token_counter( + await count_request_tokens( request=TokenCountRequest(model="gemini-counted", prompt="hi", **conversation), call_endpoint=call_endpoint ) @@ -4764,10 +4765,10 @@ async def test_drop_params_counts_a_conversation_without_the_prompt( client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_gemini_word_counting_peer)) with patch.object(gemini_count_tokens_handler, "get_async_httpx_client", return_value=client): - with_prompt: Final = await token_counter( + with_prompt: Final = await count_request_tokens( request=TokenCountRequest(model="gemini-counted", prompt="hi", **conversation), call_endpoint=call_endpoint ) - conversation_only: Final = await token_counter( + conversation_only: Final = await count_request_tokens( request=TokenCountRequest(model="gemini-counted", **conversation), call_endpoint=call_endpoint ) @@ -4784,7 +4785,7 @@ async def test_call_endpoint_prompt_next_to_an_empty_conversation_counts_the_pro client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_gemini_word_counting_peer)) with patch.object(gemini_count_tokens_handler, "get_async_httpx_client", return_value=client): - counted: Final = await token_counter( + counted: Final = await count_request_tokens( request=TokenCountRequest(model="gemini-counted", prompt=_FOX, **empty_conversation), call_endpoint=True ) @@ -4797,8 +4798,10 @@ async def test_local_count_of_a_prompt_next_to_empty_messages_counts_the_prompt( ) -> None: monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", _gemini_router()) - with_empty: Final = await token_counter(request=TokenCountRequest(model="gemini-counted", prompt=_FOX, messages=[])) - alone: Final = await token_counter(request=TokenCountRequest(model="gemini-counted", prompt=_FOX)) + with_empty: Final = await count_request_tokens( + request=TokenCountRequest(model="gemini-counted", prompt=_FOX, messages=[]) + ) + alone: Final = await count_request_tokens(request=TokenCountRequest(model="gemini-counted", prompt=_FOX)) assert with_empty.error is False assert with_empty.total_tokens == alone.total_tokens > 0 @@ -4826,13 +4829,13 @@ async def test_local_count_of_a_prompt_with_system_or_tools_counts_them_like_the ) -> None: monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", _gemini_router()) - from_prompt: Final = await token_counter( + from_prompt: Final = await count_request_tokens( request=TokenCountRequest(model="gemini-counted", prompt=_FOX, **empty_conversation, **chat_fields) ) - from_message: Final = await token_counter( + from_message: Final = await count_request_tokens( request=TokenCountRequest(model="gemini-counted", messages=[{"role": "user", "content": _FOX}], **chat_fields) ) - prompt_alone: Final = await token_counter(request=TokenCountRequest(model="gemini-counted", prompt=_FOX)) + prompt_alone: Final = await count_request_tokens(request=TokenCountRequest(model="gemini-counted", prompt=_FOX)) assert from_prompt.error is False assert from_prompt.total_tokens == from_message.total_tokens > prompt_alone.total_tokens @@ -4903,8 +4906,8 @@ async def test_call_endpoint_sends_a_prompt_to_the_provider_as_one_user_message( ) with _count_apis_answering(count_api): - from_prompt: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) - from_message: Final = await token_counter(request=_FOX_AS_MESSAGE, call_endpoint=True) + from_prompt: Final = await count_request_tokens(request=_FOX_AS_PROMPT, call_endpoint=True) + from_message: Final = await count_request_tokens(request=_FOX_AS_MESSAGE, call_endpoint=True) assert (from_prompt.total_tokens, from_prompt.error) == (41, False) assert from_prompt.count_source != "local_estimate" @@ -4925,9 +4928,9 @@ async def test_call_endpoint_answers_a_provider_error_on_a_prompt_like_on_the_sa with _count_apis_answering(count_api): with pytest.raises(ProxyException) as from_prompt: - await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + await count_request_tokens(request=_FOX_AS_PROMPT, call_endpoint=True) with pytest.raises(ProxyException) as from_message: - await token_counter(request=_FOX_AS_MESSAGE, call_endpoint=True) + await count_request_tokens(request=_FOX_AS_MESSAGE, call_endpoint=True) assert from_prompt.value.code == str(status) assert f"scripted {status}" in from_prompt.value.message @@ -4944,11 +4947,11 @@ async def test_framed_stand_in_counts_a_prompt_like_the_same_text_sent_as_a_user ) -> None: _framed_kimi(monkeypatch) - from_prompt: Final = await token_counter(request=TokenCountRequest(model="kimi", prompt=_FOX), call_endpoint=True) - from_message: Final = await token_counter( + from_prompt: Final = await count_request_tokens(request=TokenCountRequest(model="kimi", prompt=_FOX), call_endpoint=True) + from_message: Final = await count_request_tokens( request=TokenCountRequest(model="kimi", messages=[{"role": "user", "content": _FOX}]), call_endpoint=True ) - local_prompt: Final = await token_counter(request=TokenCountRequest(model="kimi", prompt=_FOX)) + local_prompt: Final = await count_request_tokens(request=TokenCountRequest(model="kimi", prompt=_FOX)) assert (from_prompt.count_source, from_prompt.error) == ("local_estimate", False) assert from_prompt.total_tokens == from_message.total_tokens > local_prompt.total_tokens @@ -4970,9 +4973,9 @@ async def test_call_endpoint_counts_a_prompt_on_a_provider_without_a_count_api_a return httpx.Response(200, json={"input_tokens": 41}, request=request) with _count_apis_answering(count_api): - from_prompt: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) - from_message: Final = await token_counter(request=_FOX_AS_MESSAGE, call_endpoint=True) - local_prompt: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=False) + from_prompt: Final = await count_request_tokens(request=_FOX_AS_PROMPT, call_endpoint=True) + from_message: Final = await count_request_tokens(request=_FOX_AS_MESSAGE, call_endpoint=True) + local_prompt: Final = await count_request_tokens(request=_FOX_AS_PROMPT, call_endpoint=False) assert sent == [] assert (from_prompt.count_source, from_prompt.error) == ("local_estimate", False) @@ -5008,8 +5011,8 @@ async def test_call_endpoint_counts_a_prompt_on_a_non_chat_deployment_as_raw_tex return httpx.Response(200, json={"input_tokens": 41}, request=request) with _count_apis_answering(count_api): - from_endpoint: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) - local: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=False) + from_endpoint: Final = await count_request_tokens(request=_FOX_AS_PROMPT, call_endpoint=True) + local: Final = await count_request_tokens(request=_FOX_AS_PROMPT, call_endpoint=False) assert sent == [] assert (from_endpoint.total_tokens, from_endpoint.error) == (local.total_tokens, False) @@ -5044,7 +5047,7 @@ async def test_call_endpoint_sends_a_prompt_on_a_chat_mode_deployment_to_the_pro return httpx.Response(200, json={"object": "response.input_tokens", "input_tokens": 41}, request=request) with _count_apis_answering(count_api): - from_prompt: Final = await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + from_prompt: Final = await count_request_tokens(request=_FOX_AS_PROMPT, call_endpoint=True) assert (from_prompt.total_tokens, from_prompt.error) == (41, False) assert len(sent) == 1 @@ -5061,8 +5064,8 @@ async def test_call_endpoint_counts_a_prompt_for_a_model_without_a_deployment_as ) request: Final = TokenCountRequest(model="deepseek/deepseek-v4-flash", prompt=_FOX) - from_endpoint: Final = await token_counter(request=request, call_endpoint=True) - local: Final = await token_counter(request=request, call_endpoint=False) + from_endpoint: Final = await count_request_tokens(request=request, call_endpoint=True) + local: Final = await count_request_tokens(request=request, call_endpoint=False) assert (from_endpoint.total_tokens, from_endpoint.error) == (local.total_tokens, False) @@ -5102,7 +5105,7 @@ async def test_call_endpoint_answers_a_timeout_when_a_stalled_count_api_outlasts with pytest.raises(ProxyException) as error: async with asyncio.timeout(10): - await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + await count_request_tokens(request=_FOX_AS_PROMPT, call_endpoint=True) assert error.value.code == "408" @@ -5144,7 +5147,7 @@ async def test_call_endpoint_names_the_status_of_a_count_api_error_with_an_empty with _count_apis_answering(lambda request: httpx.Response(404, request=request)): with pytest.raises(ProxyException) as error: - await token_counter(request=_FOX_AS_PROMPT, call_endpoint=True) + await count_request_tokens(request=_FOX_AS_PROMPT, call_endpoint=True) assert error.value.code == "404" assert "404" in error.value.message @@ -5494,8 +5497,8 @@ async def test_token_counter_counts_a_custom_azure_deployment_name_with_its_base ) messages: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}] - named: Final = await token_counter(request=TokenCountRequest(model="named", messages=messages)) - served: Final = await token_counter(request=TokenCountRequest(model="served", messages=messages)) + named: Final = await count_request_tokens(request=TokenCountRequest(model="named", messages=messages)) + served: Final = await count_request_tokens(request=TokenCountRequest(model="served", messages=messages)) assert named.total_tokens == served.total_tokens @@ -5527,7 +5530,9 @@ def _utils_token_counter( ), ) monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-test")) - return TestClient(app).post("/utils/token_counter", json=dict(body), headers=dict(headers)) + return TestClient(app, raise_server_exceptions=False).post( + "/utils/token_counter", json=dict(body), headers=dict(headers) + ) def test_utils_token_counter_logs_one_line_with_the_callers_call_id_model_and_count( @@ -5609,6 +5614,38 @@ def test_utils_token_counter_failed_count_logs_its_status_in_place_of_a_count( ) +@pytest.mark.parametrize( + ("body", "headers", "status_code"), + [ + (_COUNTED_HELLO, {}, 200), + (_COUNTED_HELLO, {"x-litellm-call-id": "count-echo-utils"}, 200), + ({"model": "count-log-gpt"}, {}, 400), + ({**_COUNTED_HELLO, "prompt": "and a prompt"}, {"x-litellm-call-id": "count-echo-utils"}, 400), + ( + {"model": "count-log-gpt", "messages": [{"role": "user", "content": "hi", "tool_calls": "x"}]}, + {"x-litellm-call-id": "count-echo-utils"}, + 500, + ), + ], + ids=["counted", "counted-with-client-id", "no-messages", "failed-count-with-client-id", "crash-with-client-id"], +) +def test_utils_token_counter_answers_the_call_id_its_count_line_names( + monkeypatch: pytest.MonkeyPatch, + token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], + body: Mapping[str, JsonValue], + headers: Mapping[str, str], + status_code: int, +) -> None: + response: Final = _utils_token_counter(monkeypatch, body, headers) + + (line,) = token_count_log_lines() + assert (response.status_code, response.headers.get("x-litellm-call-id")) == ( + status_code, + line["litellm_call_id"], + ), response.text + assert headers.get("x-litellm-call-id") in (None, line["litellm_call_id"]) + + def test_utils_token_counter_json_line_keeps_its_own_fields_when_the_call_id_is_a_dict( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6daedc3bfbd..11c06527413 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -19885,13 +19885,13 @@ export interface paths { put?: never; /** * Token Counter - * @description Args: + * @description Returns: + * TokenCountResponse + * + * Args: * request: TokenCountRequest * call_endpoint: bool - When set to "True" it will call the token counting endpoint - e.g Anthropic or Google AI Studio Token Counting APIs. * litellm_call_id: the x-litellm-call-id header, logged with the count so it joins the inference call's logs - * - * Returns: - * TokenCountResponse */ post: operations["token_counter_utils_token_counter_post"]; delete?: never; From d88a73c870426d35c4f24a5a824369adfc23f8fb Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 11 Nov 2026 14:39:53 -0800 Subject: [PATCH 115/189] feat(rate-limit): name the limit that rejected a request in a rate_limited_by header on the 429 (#51228) * feat(rate-limit): name the limit that rejected a request in a rate_limited_by header on the 429 * test(rate-limit): expect rate_limited_by in the exact-headers assertion --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/parallel_request_limiter_v3.py | 1 + .../management/test_customer_model_limits.py | 18 ++++++++++++++++++ .../hooks/test_parallel_request_limiter_v3.py | 4 ++++ 3 files changed, 23 insertions(+) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 14b9c48b0d6..24c35d78c2e 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -3347,6 +3347,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): headers={ "retry-after": str(self.window_size), "rate_limit_type": str(status["rate_limit_type"]), + "rate_limited_by": descriptor_key, "reset_at": reset_time_formatted, }, rate_limit_type=map_v3_rate_limit_type(status["rate_limit_type"]), diff --git a/tests/integration/management/test_customer_model_limits.py b/tests/integration/management/test_customer_model_limits.py index f1c207661d2..72de2c40d09 100644 --- a/tests/integration/management/test_customer_model_limits.py +++ b/tests/integration/management/test_customer_model_limits.py @@ -53,3 +53,21 @@ def test_customer_per_model_rpm_limit_is_shared_across_keys_and_isolated_per_cus _assert_customer_model_limit_error(_chat(gateway, key_two, capped, acme), acme, capped) assert _chat(gateway, key_one, capped, globex).status_code == 200 assert _chat(gateway, key_one, open_model, acme).status_code == 200 + + +@pytest.mark.parametrize( + ("limit_field", "limited_by"), + (("model_rpm_limit", "model_per_end_user"), ("rpm_limit", "end_user")), + ids=("customer-model", "customer-wide"), +) +def test_customer_rate_limit_429_names_the_limit_in_a_header(gateway: Gateway, limit_field: str, limited_by: str) -> None: + with gateway.scenario() as scenario: + capped: Final = scenario.model() + customer: Final = _customer(scenario, **{limit_field: {capped: 1} if limit_field.startswith("model_") else 1}) + key: Final = scenario.key() + + assert _chat(gateway, key, capped, customer).status_code == 200 + rejected: Final = _chat(gateway, key, capped, customer) + assert rejected.status_code == 429, rejected.text + assert rejected.headers.get("rate_limit_type") == "requests", dict(rejected.headers) + assert rejected.headers.get("rate_limited_by") == limited_by, dict(rejected.headers) diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index e3a63d097ec..c6e25dbeec3 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -4688,6 +4688,7 @@ async def test_mcp_per_key_rejection_does_not_consume_shared_server_rpm_v3() -> assert key_rejected.value.headers is not None assert key_rejected.value.headers["retry-after"] == str(handler.window_size) assert key_rejected.value.headers["rate_limit_type"] == "requests" + assert key_rejected.value.headers["rate_limited_by"] == "mcp_per_key" assert key_rejected.value.headers["reset_at"] await handler.enforce_mcp_server_rate_limits(second_key, server) @@ -8606,6 +8607,7 @@ def test_rate_limit_error_reports_reset_time_in_utc_on_a_non_utc_proxy(process_t assert exc_info.value.headers == { "retry-after": str(handler.window_size), "rate_limit_type": "requests", + "rate_limited_by": "api_key", "reset_at": expected_reset, } assert exc_info.value.detail == ( @@ -9938,6 +9940,8 @@ async def test_end_user_model_rpm_limit_is_shared_across_keys_and_isolated_per_c await _rpm_request(handler, cache, acme_via_key_two, "target") assert exc.value.status_code == 429 assert "model_per_end_user: acme:target" in str(exc.value.detail), exc.value.detail + assert exc.value.headers is not None + assert exc.value.headers["rate_limited_by"] == "model_per_end_user" await _rpm_request(handler, cache, globex, "target") await _rpm_request(handler, cache, acme_via_key_one, "other-model") From 935d8e6fcc3ca7b02dbd28ade5a4aa002998e576 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 11 Nov 2026 14:44:57 -0800 Subject: [PATCH 116/189] chore(ui): remove the deprecated Learning Resources sidebar link (#51251) Learning Resources is on the finalized deprecation list in https://github.com/BerriAI/litellm/discussions/32090 for low usage, with no replacement. Drop its Developer Tools entry, the now unused BookOpen icon import, its page description mapping and both en.json strings, and take it out of the top-level label list in leftnav.test.tsx. --- ui/litellm-dashboard/src/components/leftnav.test.tsx | 1 - ui/litellm-dashboard/src/components/leftnav.tsx | 8 -------- ui/litellm-dashboard/src/components/page_metadata.ts | 1 - ui/litellm-dashboard/src/i18n/locales/en.json | 2 -- 4 files changed, 12 deletions(-) diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index 1ffe8f6eaa9..f3e02d13e56 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -284,7 +284,6 @@ describe("Sidebar (leftnav)", () => { "Budgets", "API Reference", "AI Hub", - "Learning Resources", "Experimental", "Settings", ]; diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 6040da32f6c..652a3596980 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -25,7 +25,6 @@ import { Bell, Blocks, Bot, - BookOpen, Building2, Boxes, ChevronRight, @@ -434,13 +433,6 @@ const menuGroups: MenuGroup[] = [ label: "shell.nav.items.aiHub", icon: , }, - { - key: "learning-resources", - page: "learning-resources", - label: "shell.nav.items.learningResources", - icon: , - external_url: "https://models.litellm.ai/cookbook", - }, { key: "caching", page: "caching", diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 8459b9a555b..b2d86dbe7b8 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -36,7 +36,6 @@ export const pageDescriptions: Record = { budgets: "shell.pageDescriptions.budgets", api_ref: "shell.pageDescriptions.apiRef", "model-hub-table": "shell.pageDescriptions.modelHubTable", - "learning-resources": "shell.pageDescriptions.learningResources", caching: "shell.pageDescriptions.caching", "transform-request": "shell.pageDescriptions.transformRequest", "cost-tracking": "shell.pageDescriptions.costTracking", diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index 25bf2817e27..ebf8b5dc134 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -8037,7 +8037,6 @@ "budgets": "Budgets", "apiReference": "API Reference", "aiHub": "AI Hub", - "learningResources": "Learning Resources", "responseCache": "Response Cache", "experimental": "Experimental", "prompts": "Prompts", @@ -8085,7 +8084,6 @@ "budgets": "Set and monitor spending budgets", "apiRef": "Browse API documentation and endpoints", "modelHubTable": "Explore available AI models and providers", - "learningResources": "Access tutorials and documentation", "caching": "Configure response caching and coordination Redis settings", "transformRequest": "Set up request transformation rules", "costTracking": "Track and analyze API costs", From 3671d6b380746aee0f1d50f3f725ab0773de2cd3 Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Wed, 11 Nov 2026 14:49:30 -0800 Subject: [PATCH 117/189] feat(helm): add scheme and tlsConfig to the gateway ServiceMonitor (#50759) * feat(helm): add scheme and tlsConfig to the gateway ServiceMonitor The componentized chart rendered scheme: http as a literal, so Prometheus could not scrape the gateway metrics port through a sidecar under STRICT mTLS. gateway.serviceMonitor.scheme defaults to http and tlsConfig is passed through as the Prometheus Operator TLSConfig, so the default render is unchanged * chore(helm): drop the comments on the new serviceMonitor values --- helm/litellm/templates/_workload.tpl | 6 ++++- .../tests/hpa_workload_metrics_tests.yaml | 26 +++++++++++++++++++ helm/litellm/values.yaml | 2 ++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/helm/litellm/templates/_workload.tpl b/helm/litellm/templates/_workload.tpl index 8f52cd52f2a..2b6b1434be9 100644 --- a/helm/litellm/templates/_workload.tpl +++ b/helm/litellm/templates/_workload.tpl @@ -507,7 +507,11 @@ spec: path: /metrics/ interval: {{ .Values.gateway.serviceMonitor.interval }} scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }} - scheme: http + scheme: {{ .Values.gateway.serviceMonitor.scheme }} + {{- with .Values.gateway.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.gateway.serviceMonitor.relabelings }} relabelings: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/tests/hpa_workload_metrics_tests.yaml b/helm/litellm/tests/hpa_workload_metrics_tests.yaml index a6c17942a11..9b790070819 100644 --- a/helm/litellm/tests/hpa_workload_metrics_tests.yaml +++ b/helm/litellm/tests/hpa_workload_metrics_tests.yaml @@ -201,6 +201,32 @@ tests: scrapeTimeout: 10s scheme: http + - it: scrapes over https with the given tlsConfig, for a mesh with STRICT mTLS + template: gateway/servicemonitor.yaml + set: + gateway.serviceMonitor.enabled: true + gateway.metricsServer.enabled: true + gateway.serviceMonitor.scheme: https + gateway.serviceMonitor.tlsConfig: + caFile: /etc/prom-certs/root-cert.pem + certFile: /etc/prom-certs/cert-chain.pem + keyFile: /etc/prom-certs/key.pem + insecureSkipVerify: true + asserts: + - equal: + path: spec.endpoints + value: + - port: metrics + path: /metrics/ + interval: 15s + scrapeTimeout: 10s + scheme: https + tlsConfig: + caFile: /etc/prom-certs/root-cert.pem + certFile: /etc/prom-certs/cert-chain.pem + keyFile: /etc/prom-certs/key.pem + insecureSkipVerify: true + - it: carries litellm-helm annotations, relabelings and namespaceSelector onto the ServiceMonitor template: gateway/servicemonitor.yaml release: diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 9bfc3ca32e9..b4795cd9ca6 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -407,6 +407,8 @@ gateway: # Empty scrapes the release namespace. namespaceSelector: matchNames: [] + scheme: http + tlsConfig: {} # Opt-in `collector` sidecar (same image, `python -m litellm.proxy.collector`) # that runs the post-response spend pipeline (cost calculation, spend logs, # spend counters, budget reservation reconciliation) so the uvicorn workers From 534a60d28dbe4b7191a79a43f2c1a2db640c992d Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:50:23 -0800 Subject: [PATCH 118/189] fix(proxy): name the deployment in stream error alerts on chat, completions, responses and messages (#51034) * fix(proxy): name the deployment in stream error alerts on every provider and route The llm_exceptions alert for a cut stream on Anthropic chat, /v1/completions, native /v1/responses and native /v1/messages had no API Base, model_group or deployment line. The shared exception mapper now attaches its debug info at its single exit, mid-stream fallback errors inherit it from the error they wrap, and the native relays add it to the same error object they re-raise. * fix(proxy): keep the prompt out of attached alert debug info and name the deployment on more stream exits * fix(proxy): name the deployment on stream errors that arrive already mapped exception_type returned errors that were already litellm types before it attached the debug info, so a finish_reason=abort chunk or the max streaming duration timeout still sent an alert with no deployment. It now builds the info first and attaches the prompt-free copy on that early return too, and native Responses streams attach it on their generic error exit Also adds tests for the dropped `Messages: None` line and the mapping-crash exit * fix(proxy): read the router's deployment from litellm_metadata on Messages and Responses errors /v1/messages and /v1/responses keep the router's model_group, deployment and api_base in litellm_metadata, so their errors before the stream named only the model in alerts. The debug info now reads litellm_metadata first, falls back to metadata, and takes API Base from it when the call passes none get_first_chars_messages returns an empty string again for input that is not a mapping, as it did before this branch * fix(proxy): merge both metadata buckets when error alerts name the deployment A pre_call guardrail adds litellm_metadata to a chat request while the router keeps model_group and deployment in metadata, and generic routes such as moderations keep the key alias and team in metadata and the deployment in litellm_metadata. Reading only one bucket dropped lines the merge base showed. The debug info now merges both, and litellm_metadata wins when both carry the same key, so a caller's own metadata cannot replace the router's values * fix(proxy): keep a reused error's alert details on the request that raised it An exception instance raised for more than one request, such as an SDK mock_response exception or a handler that raises one shared error, kept the first request's Key Name, model_group and deployment in every later alert The attached details now record the call id that set them. A later call with a different id replaces them, and a layer that re-maps the error without the request keeps them * fix(proxy): drop alert details from an error object raised by more than one request One exception object raised by two concurrent requests (a shared mock_response error, or a handler that raises one error object for every request) let a request's alert show the other request's Key Name and deployment, since the later attach replaced the earlier one. An error object tagged by one call id and attached again under another now drops its details for good, so no alert names the wrong request. One request re-mapping its own error, with or without a call id, keeps the lines it already has --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/exceptions.py | 5 +- .../exception_mapping_utils.py | 165 +++++++---- .../messages/native/streaming_iterator.py | 26 +- litellm/main.py | 15 +- .../streaming_handler.py | 2 + litellm/responses/streaming_iterator.py | 54 ++-- litellm/utils.py | 11 +- .../test_stream_error_alert_api_base.py | 100 ++++++- .../test_exception_mapping_utils.py | 260 ++++++++++++++++++ .../native/test_streaming_iterator.py | 61 ++++ .../unit/responses/test_streaming_iterator.py | 131 +++++++++ tests/unit/test_utils.py | 5 + .../test_streaming_iterator.py | 29 ++ 13 files changed, 762 insertions(+), 102 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index f8c6a50a589..61810f650af 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1289,7 +1289,10 @@ class MidStreamFallbackError(ServiceUnavailableError): self.model = model self.llm_provider = llm_provider self.original_exception = original_exception - self.litellm_debug_info = litellm_debug_info + inherited_debug_info: Final[object] = getattr(original_exception, "litellm_debug_info", None) + self.litellm_debug_info = litellm_debug_info or ( + inherited_debug_info if isinstance(inherited_debug_info, str) else None + ) self.max_retries = max_retries self.num_retries = num_retries self.generated_content = generated_content diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 16cd076db39..338902aa919 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -3,6 +3,7 @@ import json import re import traceback from collections.abc import Mapping +from dataclasses import dataclass from types import MappingProxyType from typing import Final, Protocol, cast @@ -2514,15 +2515,113 @@ _HOST_WITH_OPTIONAL_PORT: Final = re.compile( ) +def _router_metadata(params: Mapping[str, object]) -> Mapping[str, object]: + metadata: Final = as_str_mapping(params.get("metadata")) or _NO_LITELLM_PARAMS + litellm_metadata: Final = as_str_mapping(params.get("litellm_metadata")) or _NO_LITELLM_PARAMS + return MappingProxyType({**metadata, **litellm_metadata}) + + +def _deployment_api_base(metadata: Mapping[str, object]) -> str | None: + api_base: Final = metadata.get("api_base") + return api_base if isinstance(api_base, str) and api_base else None + + def with_deployment_api_base(litellm_params: object) -> Mapping[str, object]: params: Final = as_str_mapping(litellm_params) or _NO_LITELLM_PARAMS - metadata: Final = as_str_mapping(params.get("metadata")) - deployment_api_base: Final = None if metadata is None else metadata.get("api_base") - if not isinstance(deployment_api_base, str) or not deployment_api_base: + deployment_api_base: Final = _deployment_api_base(_router_metadata(params)) + if deployment_api_base is None: return params return MappingProxyType({**params, "api_base": deployment_api_base}) +_DEBUG_INFO_ATTRIBUTE: Final = "litellm_debug_info" +_DEBUG_INFO_CALL_ID_ATTRIBUTE: Final = "_litellm_debug_info_call_id" +_RAISED_BY_SEVERAL_CALLS: Final = object() + + +@dataclass(frozen=True, slots=True) +class _RequestDebugInfo: + with_prompt: str + without_prompt: str + call_id: str + + +_NO_REQUEST_DEBUG_INFO: Final = _RequestDebugInfo(with_prompt="", without_prompt="", call_id="") + + +def _request_debug_info(model: object, extra_kwargs: object, completion_kwargs: object) -> _RequestDebugInfo: + params: Final = as_str_mapping(extra_kwargs) or _NO_LITELLM_PARAMS + try: + metadata: Final = _router_metadata(params) + api_base: Final = litellm.get_api_base(model=model, optional_params=params) or _deployment_api_base(metadata) + messages: Final = litellm.get_first_chars_messages( + kwargs=as_str_mapping(completion_kwargs) or _NO_LITELLM_PARAMS + ) + shown_api_base: Final = _api_base_without_credentials(api_base) if api_base else "" + head: Final = f"\nModel: {model}" + (f"\nAPI Base: `{shown_api_base}`" if shown_api_base else "") + prompt: Final = ( + f"\nMessages: `{messages}`" if messages and litellm.redact_messages_in_exceptions is False else "" + ) + labelled_values: Final = ( + ("model_group", metadata.get("model_group")), + ("deployment", metadata.get("deployment")), + ("vertex_project", params.get("vertex_project")), + ("vertex_location", params.get("vertex_location")), + ) + tail: Final = "".join(f"\n{label}: `{value}`\n" for label, value in labelled_values if value is not None) + call_id: Final = params.get("litellm_call_id") + return _RequestDebugInfo( + with_prompt=add_key_name_and_team_to_alert(request_info=head + prompt + tail, metadata=metadata), + without_prompt=add_key_name_and_team_to_alert(request_info=head + tail, metadata=metadata), + call_id=call_id if isinstance(call_id, str) else "", + ) + except Exception: + return _NO_REQUEST_DEBUG_INFO + + +def _attach_debug_info(exception: Exception, request_debug_info: _RequestDebugInfo) -> None: + attached_call_id: Final = getattr(exception, _DEBUG_INFO_CALL_ID_ATTRIBUTE, None) + if attached_call_id is _RAISED_BY_SEVERAL_CALLS: + return + if ( + isinstance(attached_call_id, str) + and attached_call_id + and request_debug_info.call_id + and attached_call_id != request_debug_info.call_id + ): + setattr(exception, _DEBUG_INFO_ATTRIBUTE, None) + setattr(exception, _DEBUG_INFO_CALL_ID_ATTRIBUTE, _RAISED_BY_SEVERAL_CALLS) + return + if request_debug_info.without_prompt and getattr(exception, _DEBUG_INFO_ATTRIBUTE, None) is None: + setattr(exception, _DEBUG_INFO_ATTRIBUTE, request_debug_info.without_prompt) + setattr(exception, _DEBUG_INFO_CALL_ID_ATTRIBUTE, request_debug_info.call_id) + + +def mapped_stream_exception( + original_exception: Exception, model: str, custom_llm_provider: str, litellm_params: object +) -> Exception: + try: + return exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=original_exception, + completion_kwargs={}, + extra_kwargs=with_deployment_api_base(litellm_params), + ) + except Exception as mapped_exception: + return mapped_exception + + +def with_deployment_debug_info( + error: Exception, model: str, custom_llm_provider: str, litellm_params: object +) -> Exception: + if not (model or custom_llm_provider): + return error + request_debug_info: Final = _request_debug_info(model, with_deployment_api_base(litellm_params), _NO_LITELLM_PARAMS) + _attach_debug_info(error, request_debug_info) + return error + + def _api_base_without_credentials(api_base: str) -> str: scheme, separator, _ = api_base.partition("://") scheme_prefix: Final = scheme + separator if separator and _URL_SCHEME.fullmatch(scheme) else "" @@ -2546,11 +2645,17 @@ def exception_type( model, original_exception, custom_llm_provider, - completion_kwargs={}, - extra_kwargs={}, + completion_kwargs: object = _NO_LITELLM_PARAMS, + extra_kwargs: object = _NO_LITELLM_PARAMS, ): """Maps an LLM Provider Exception to OpenAI Exception Format""" - if any(isinstance(original_exception, exc_type) for exc_type in litellm.LITELLM_EXCEPTION_TYPES): + request_debug_info: Final = ( + _request_debug_info(model, extra_kwargs, completion_kwargs) + if model or custom_llm_provider + else _NO_REQUEST_DEBUG_INFO + ) + if isinstance(original_exception, tuple(litellm.LITELLM_EXCEPTION_TYPES)): + _attach_debug_info(original_exception, request_debug_info) return original_exception if _is_guardrail_block(original_exception): return original_exception @@ -2572,11 +2677,11 @@ def exception_type( print() # noqa: T201 litellm_response_headers: Final = _get_response_headers(original_exception=original_exception) + extra_information: Final = request_debug_info.with_prompt try: error_str = ( redact_secret_string(str(original_exception)) if _ENABLE_SECRET_REDACTION else str(original_exception) ) - extra_information = "" if model or custom_llm_provider: if hasattr(original_exception, "message"): error_str = ( @@ -2589,47 +2694,8 @@ def exception_type( else: exception_type = "" - ################################################################################ - # Common Extra information needed for all providers - # We pass num retries, api_base, vertex_deployment etc to the exception here - ################################################################################ - try: - _api_base: Final = litellm.get_api_base(model=model, optional_params=extra_kwargs) - messages: Final = litellm.get_first_chars_messages(kwargs=completion_kwargs) - _vertex_project: Final = extra_kwargs.get("vertex_project") - _vertex_location: Final = extra_kwargs.get("vertex_location") - _metadata: Final = extra_kwargs.get("metadata", {}) or {} - _model_group: Final = _metadata.get("model_group") - _deployment: Final = _metadata.get("deployment") - extra_information = f"\nModel: {model}" - - if isinstance(custom_llm_provider, str) and len(custom_llm_provider) > 0: - exception_provider = custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception" - - shown_api_base: Final = _api_base_without_credentials(_api_base) if _api_base else "" - if shown_api_base: - extra_information += f"\nAPI Base: `{shown_api_base}`" - if messages and len(messages) > 0 and litellm.redact_messages_in_exceptions is False: - extra_information += f"\nMessages: `{messages}`" - - if _model_group is not None: - extra_information += f"\nmodel_group: `{_model_group}`\n" - if _deployment is not None: - extra_information += f"\ndeployment: `{_deployment}`\n" - if _vertex_project is not None: - extra_information += f"\nvertex_project: `{_vertex_project}`\n" - if _vertex_location is not None: - extra_information += f"\nvertex_location: `{_vertex_location}`\n" - - # on litellm proxy add key name + team to exceptions - extra_information = add_key_name_and_team_to_alert(request_info=extra_information, metadata=_metadata) - except Exception: - # DO NOT LET this Block raising the original exception - pass - - ################################################################################ - # End of Common Extra information Needed for all providers - ################################################################################ + if isinstance(custom_llm_provider, str) and len(custom_llm_provider) > 0: + exception_provider = custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception" ################################################################################ #################### Start of Provider Exception mapping #################### @@ -2923,17 +2989,20 @@ def exception_type( # don't let an error with mapping interrupt the user from receiving an error from the llm api calls if exception_mapping_worked: setattr(e, "litellm_response_headers", litellm_response_headers) + _attach_debug_info(e, request_debug_info) raise e else: for error_type in litellm.LITELLM_EXCEPTION_TYPES: if isinstance(e, error_type): setattr(e, "litellm_response_headers", litellm_response_headers) + _attach_debug_info(e, request_debug_info) raise e # it's already mapped raised_exc: Final = APIConnectionError( message=f"{original_exception}\n{redact_string(traceback.format_exc())}", llm_provider="", model="", ) + _attach_debug_info(raised_exc, request_debug_info) setattr(raised_exc, "litellm_response_headers", litellm_response_headers) carry_agentic_followup_mark(source=original_exception, mapped=raised_exc) raise raised_exc diff --git a/litellm/llms/anthropic/messages/native/streaming_iterator.py b/litellm/llms/anthropic/messages/native/streaming_iterator.py index 1195652dfd2..5a7633698ec 100644 --- a/litellm/llms/anthropic/messages/native/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/streaming_iterator.py @@ -15,6 +15,7 @@ from litellm.constants import ( ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE, ) from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_debug_info from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP @@ -563,6 +564,7 @@ class BaseAnthropicMessagesStreamingIterator: start_time=self.start_time, passthrough_success_handler_obj=GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ, url_route="/v1/messages", + on_upstream_error=self._with_deployment_debug_info, ) def _convert_chunk_to_sse_format(self, chunk: dict | Any) -> bytes: @@ -610,8 +612,9 @@ class BaseAnthropicMessagesStreamingIterator: An upstream failure (Bedrock read / decode / chunk-conversion error) that happens while the client is still connected is forwarded through the queue and re-raised here, so the original provider exception (and - its status) reaches the proxy's failure handling unchanged rather than - being masked by a generic incomplete-stream event. + its status) reaches the proxy's failure handling with the deployment's + debug info attached, rather than being masked by a generic + incomplete-stream event. This method provides the common logic for both Anthropic and Bedrock implementations. """ @@ -802,18 +805,29 @@ class BaseAnthropicMessagesStreamingIterator: """ from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler + described: Final = self._with_deployment_debug_info(exc) await PassThroughStreamingHandler.schedule_stream_failure_logging( litellm_logging_obj=self.litellm_logging_obj, endpoint_type=EndpointType.ANTHROPIC, request_body=self.request_body, raw_bytes=collected_chunks, - exception=exc, + exception=described, ) - if not client_detached.is_set() and await self._enqueue_for_client(queue, client_detached, exc): + if not client_detached.is_set() and await self._enqueue_for_client(queue, client_detached, described): await client_detached.wait() - if not _exception_left_unconsumed(queue, exc): + if not _exception_left_unconsumed(queue, described): return - await self._fire_detached_failure_hook(exc) + await self._fire_detached_failure_hook(described) + + def _with_deployment_debug_info(self, exc: Exception) -> Exception: + details: Final = self.litellm_logging_obj.model_call_details + provider: Final = details.get("custom_llm_provider") + return with_deployment_debug_info( + exc, + self.litellm_logging_obj.model, + provider if isinstance(provider, str) and provider else "anthropic", + details.get("litellm_params"), + ) async def _fire_detached_failure_hook(self, exc: Exception) -> None: from litellm._logging import verbose_proxy_logger diff --git a/litellm/main.py b/litellm/main.py index c86a888aa96..239a85cef4c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -90,6 +90,7 @@ from litellm.litellm_core_utils.chat_completion_agentic_loop import ( ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind from litellm.litellm_core_utils.dd_tracing import tracer +from litellm.litellm_core_utils.exception_mapping_utils import mapped_stream_exception from litellm.litellm_core_utils.get_litellm_params import ( AGENTIC_LOOP_CONTROL_KWARGS_KEYS, AGENTIC_LOOP_TOOL_CHOICE_KEY, @@ -1033,7 +1034,7 @@ def _text_completion_response_from_cached_dict(cached_response_dict: Mapping[str return TextCompletionResponse(stream=bool(cached_response_dict.get("stream", False)), **fields) -async def _async_streaming(response, model, custom_llm_provider, args): +async def _async_streaming(response, model, custom_llm_provider, litellm_params: Mapping[str, object]): try: print_verbose(f"received response in _async_streaming: {response}") if asyncio.iscoroutine(response): @@ -1042,11 +1043,11 @@ async def _async_streaming(response, model, custom_llm_provider, args): print_verbose(f"line in async streaming: {line}") yield line except Exception as e: - custom_llm_provider = custom_llm_provider or "openai" - raise exception_type( - model=model, - custom_llm_provider=custom_llm_provider, - original_exception=e, + raise mapped_stream_exception( + e, + model if isinstance(model, str) else "", + custom_llm_provider if isinstance(custom_llm_provider, str) and custom_llm_provider else "openai", + litellm_params, ) @@ -8090,7 +8091,7 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp response=response, model=model, custom_llm_provider=custom_llm_provider, - args=args, + litellm_params=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), ), model=model, custom_llm_provider=custom_llm_provider, diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 345e17c435d..722e58211ae 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -173,6 +173,7 @@ class PassThroughStreamingHandler: passthrough_success_handler_obj: PassThroughEndpointLogging, url_route: str, route_streaming_logging: RouteStreamingLogging | None = None, + on_upstream_error: Callable[[Exception], object] = lambda _: None, ): resolved_route_streaming_logging: Final[RouteStreamingLogging] = ( route_streaming_logging or PassThroughStreamingHandler.route_streaming_logging_to_handler @@ -271,6 +272,7 @@ class PassThroughStreamingHandler: litellm_logging_obj._deferred_stream_complete_args = (_build_logging_coroutine(),) except Exception as e: verbose_proxy_logger.error("Error in chunk_processor: %s", e) + on_upstream_error(e) if response.status_code < 400: logging_scheduled = True await PassThroughStreamingHandler.schedule_stream_failure_logging( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index c2191375cb4..347537277df 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -47,7 +47,7 @@ from litellm.litellm_core_utils.core_helpers import ( set_response_cost_in_hidden_params, stamp_litellm_bill_on_usage, ) -from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base +from litellm.litellm_core_utils.exception_mapping_utils import mapped_stream_exception, with_deployment_debug_info from litellm.litellm_core_utils.internal_call_metadata import ( is_background_response, requests_background, @@ -344,27 +344,21 @@ def map_stream_error_to_exception( return _mapped_exception(provider_exception, model, custom_llm_provider, logging_obj) +def _logged_litellm_params(logging_obj: LiteLLMLoggingObj | None) -> object: + return ( + None + if logging_obj is None + else _typed_gets_litellm_params(logging_obj.model_call_details.get)("litellm_params", {}) + ) + + def _mapped_exception( original_exception: Exception, model: str, custom_llm_provider: str, logging_obj: LiteLLMLoggingObj | None, ) -> Exception: - litellm_params: Final = ( - None - if logging_obj is None - else _typed_gets_litellm_params(logging_obj.model_call_details.get)("litellm_params", {}) - ) - try: - return litellm.exception_type( - model=model, - custom_llm_provider=custom_llm_provider, - original_exception=original_exception, - completion_kwargs={}, - extra_kwargs=with_deployment_api_base(litellm_params), - ) - except Exception as mapped_exception: - return mapped_exception + return mapped_stream_exception(original_exception, model, custom_llm_provider, _logged_litellm_params(logging_obj)) def _event_field(event: object, name: str) -> object: @@ -1219,12 +1213,17 @@ class BaseResponsesAPIStreamingIterator: model=self.model or "", ) + def _with_deployment_debug_info(self, error: Exception) -> Exception: + return with_deployment_debug_info( + error, self.model or "", self.custom_llm_provider or "", _logged_litellm_params(self.logging_obj) + ) + def _raise_if_ended_without_terminal_event(self) -> None: if self.completed_response is not None: return if self._replays_stored_stream and not self._yielded_first_chunk: return - error: Final = self._stream_ended_early_error() + error: Final = self._with_deployment_debug_info(self._stream_ended_early_error()) self._handle_failure(error) if self._output_started: raise error @@ -1236,10 +1235,11 @@ class BaseResponsesAPIStreamingIterator: raise _mapped_exception(error, self.model or "", self.custom_llm_provider or "", self.logging_obj) from error def _raise_for_transport_error(self, error: httpx.ReadError | httpx.RemoteProtocolError) -> NoReturn: - self._handle_failure(error) + described: Final = self._with_deployment_debug_info(error) + self._handle_failure(described) if self._output_started: - raise error - raise self._fallback_error(error) from error + raise described + raise self._fallback_error(described) from described async def call_post_streaming_hooks_for_testing( @@ -1341,14 +1341,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): self.finished = True raise StopAsyncIteration from e self._raise_for_transport_timeout(e) - except httpx.HTTPError as e: - # Handle HTTP errors - self.finished = True - self._handle_failure(e) - raise e except Exception as e: self.finished = True - self._handle_failure(e) + self._handle_failure(self._with_deployment_debug_info(e)) raise e def _handle_logging_completed_response(self): @@ -1434,14 +1429,9 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): self.finished = True raise StopIteration from e self._raise_for_transport_timeout(e) - except httpx.HTTPError as e: - # Handle HTTP errors - self.finished = True - self._handle_failure(e) - raise e except Exception as e: self.finished = True - self._handle_failure(e) + self._handle_failure(self._with_deployment_debug_info(e)) raise e def _handle_logging_completed_response(self): diff --git a/litellm/utils.py b/litellm/utils.py index 7012741148c..2820638cd28 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6583,13 +6583,10 @@ def get_model_region(litellm_params: LiteLLM_Params, mode: str | None) -> str | return None -def get_first_chars_messages(kwargs: dict) -> str: - try: - _messages = kwargs.get("messages") - _messages = str(_messages)[:100] - return _messages - except Exception: - return "" +def get_first_chars_messages(kwargs: object) -> str: + request: Final = as_str_mapping(kwargs) + messages: Final = None if request is None else request.get("messages") + return "" if messages is None else str(messages)[:100] def count_characters(text: str) -> int: diff --git a/tests/integration/observability/test_stream_error_alert_api_base.py b/tests/integration/observability/test_stream_error_alert_api_base.py index d7f325da8c7..b91cf7e3025 100644 --- a/tests/integration/observability/test_stream_error_alert_api_base.py +++ b/tests/integration/observability/test_stream_error_alert_api_base.py @@ -13,7 +13,15 @@ import openai import pytest import yaml from anthropic.types import RawContentBlockDeltaEvent, TextDelta -from integration._support.anthropic_sse import DROP_PAUSE, dropping_reply, stream_reply +from integration._support.anthropic_sse import ( + CONTENT_BLOCK_START, + DROP_PAUSE, + dropping_reply, + error_body, + message_start, + stream_reply, + text_delta, +) from integration._support.client import ( JSON_OBJECT, Gateway, @@ -30,6 +38,7 @@ from integration._support.responses_ws_peer import PROVIDER_MODEL, Script, event from integration._support.tls import write_self_signed_cert from integration._support.vertex import service_account_json from integration._support.wire import Reply, Request, Wire, wire_server +from openai.types import Completion from openai.types.chat import ChatCompletionChunk from openai.types.responses import ResponseFailedEvent, ResponseTextDeltaEvent from pydantic import JsonValue @@ -39,6 +48,7 @@ pytestmark: Final = [pytest.mark.timeout(2 * graceful_stop_seconds() + 120), pyt AZURE_KEY: Final = "synthetic-azure-key" API_VERSION: Final = "2025-04-01-preview" AZURE_CHAT_BACKEND: Final = "azure/gpt-4o-mini" +ANTHROPIC_BACKEND: Final = "anthropic/claude-opus-5-5" PARTIAL_TEXT: Final = "partial answer" ALERT_SECONDS: Final = 40 NO_RETRY_NO_CACHE: Final[Mapping[str, JsonValue]] = MappingProxyType({"num_retries": 0, "cache": {"no-cache": True}}) @@ -234,6 +244,10 @@ def responses_text(item: object) -> str: return item.delta if isinstance(item, ResponseTextDeltaEvent) else "" +def completion_text(item: object) -> str: + return "".join(choice.text for choice in item.choices) if isinstance(item, Completion) else "" + + def failure_of(item: object) -> str: if isinstance(item, Exception): return str(item) @@ -296,6 +310,19 @@ def responses_outcome(gateway: Gateway, key: str, model: str) -> Outcome: return outcome(items, responses_text) +def completions_outcome(gateway: Gateway, key: str, model: str) -> Outcome: + with openai_client(gateway, key) as client: + items: Final = tuple( + items_or_error( + lambda: client.completions.create( + model=model, prompt="stream an answer", stream=True, extra_body=dict(NO_RETRY_NO_CACHE) + ), + openai.APIError, + ) + ) + return outcome(items, completion_text) + + ENTRYPOINTS: Final[Mapping[str, Callable[[Gateway, str, str], Outcome]]] = MappingProxyType( {"chat": chat_outcome, "messages": messages_outcome, "responses-bridge": responses_outcome} ) @@ -480,3 +507,74 @@ def test_fallback_mid_stream_failure_alert_names_the_fallback_deployment(alertin alert: Final = alert_naming(alerting.alerts, marker) assert_names_deployment(alert, backup_wire.url, backup, AZURE_CHAT_BACKEND, caller) assert primary_wire.url not in alert, alert + + +PING_NEVER_SENT: Final = b"event: ping\ndata: {}\n\n" + + +def anthropic_cut_after_text(marker: str) -> Reply: + return dropping_reply( + ( + message_start(f"msg_{marker}", ANTHROPIC_BACKEND), + CONTENT_BLOCK_START, + text_delta(PARTIAL_TEXT), + PING_NEVER_SENT, + ), + abort_after=3, + ) + + +def responses_cut_after_text(marker: str) -> Reply: + identity: Final = f"resp_{marker}" + return dropping_reply( + (frame(created(identity)), frame(delta(identity, PARTIAL_TEXT)), b": never sent\n\n"), abort_after=2 + ) + + +@dataclass(frozen=True, slots=True) +class CutPath: + backend: str + reply: Callable[[str], Reply] + call: Callable[[Gateway, str, str], Outcome] + + +CUT_PATHS: Final[Mapping[str, CutPath]] = MappingProxyType( + { + "anthropic-chat": CutPath(ANTHROPIC_BACKEND, anthropic_cut_after_text, chat_outcome), + "completions": CutPath(AZURE_CHAT_BACKEND, cut_after_text, completions_outcome), + "native-responses": CutPath(AZURE_CHAT_BACKEND, responses_cut_after_text, responses_outcome), + "native-messages": CutPath(ANTHROPIC_BACKEND, anthropic_cut_after_text, messages_outcome), + } +) + + +def scripted_model(scenario: Scenario, backend: str, api_base: str) -> str: + if backend == AZURE_CHAT_BACKEND: + return azure_chat(scenario, api_base) + return scenario.model(model=backend, api_base=api_base, api_key="synthetic-anthropic-key", num_retries=0) + + +@pytest.mark.parametrize("path", list(CUT_PATHS)) +def test_stream_cut_alert_names_the_deployment(alerting: AlertingProxy, path: str) -> None: + marker: Final = uuid.uuid4().hex + cut: Final = CUT_PATHS[path] + with wire_server(chat_upstream(cut.reply, marker)) as wire, alerting.gateway.scenario() as scenario: + model: Final = scripted_model(scenario, cut.backend, wire.url) + caller: Final = keyed_caller(scenario) + seen: Final = cut.call(alerting.gateway, caller.key, model) + assert seen.text == PARTIAL_TEXT and seen.failure, seen + assert_names_deployment(alert_naming(alerting.alerts, model), wire.url, model, cut.backend, caller) + + +def test_anthropic_rejection_before_stream_alert_names_the_deployment(alerting: AlertingProxy) -> None: + marker: Final = uuid.uuid4().hex + + def rejected(_: str) -> Reply: + return Reply(status=500, body=error_body(500, provider_message(marker))) + + with wire_server(chat_upstream(rejected, marker)) as wire, alerting.gateway.scenario() as scenario: + model: Final = scripted_model(scenario, ANTHROPIC_BACKEND, wire.url) + caller: Final = keyed_caller(scenario) + seen: Final = chat_outcome(alerting.gateway, caller.key, model) + assert provider_message(marker) in seen.failure and not seen.text, seen + assert_names_deployment(alert_naming(alerting.alerts, marker), wire.url, model, ANTHROPIC_BACKEND, caller) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 32a99a37870..bc7827f4f07 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.exception_mapping_utils import ( exception_type, extract_and_raise_litellm_exception, ) +from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -157,6 +158,265 @@ def test_api_base_line_keeps_an_at_sign_in_the_path(api_base: str) -> None: assert f"API Base: `{api_base}`" in _debug_info_for_api_base(api_base) +@pytest.mark.parametrize( + ("provider_error", "mapped_class"), + [ + ( + httpx.ReadError("peer closed connection", request=httpx.Request("POST", "https://relay.example/v1/messages")), + litellm.APIConnectionError, + ), + (RuntimeError("stream decoder failed"), litellm.APIConnectionError), + (AnthropicError(status_code=500, message="[Errno 104] Connection reset by peer"), litellm.InternalServerError), + (AnthropicError(status_code=429, message="rate_limit_error"), litellm.RateLimitError), + ], + ids=["unmapped_with_request", "unmapped_without_request", "anthropic_500", "anthropic_429"], +) +def test_mapped_provider_error_carries_the_deployment_debug_info( + provider_error: Exception, mapped_class: type[Exception] +) -> None: + with pytest.raises(mapped_class) as caught: + exception_type( + model="claude-opus-5-5", + original_exception=provider_error, + custom_llm_provider="anthropic", + completion_kwargs={"messages": [{"role": "user", "content": "CONFIDENTIAL patient record"}]}, + extra_kwargs={ + "api_base": "https://relay.example", + "metadata": {"model_group": "claude-alias", "deployment": "anthropic/claude-opus-5-5"}, + }, + ) + debug_info: Final = caught.value.litellm_debug_info + assert isinstance(debug_info, str) + assert "API Base: `https://relay.example`" in debug_info + assert "model_group: `claude-alias`" in debug_info + assert "deployment: `anthropic/claude-opus-5-5`" in debug_info + assert "CONFIDENTIAL" not in debug_info + + +_ROUTER_METADATA: Final = { + "api_base": "https://relay.example/v1", + "model_group": "claude-alias", + "deployment": "anthropic/claude-haiku-5-5", +} + + +@pytest.mark.parametrize( + ("provider_error", "custom_llm_provider", "extra_kwargs"), + [ + ( + AnthropicError(status_code=529, message="Overloaded"), + "anthropic", + {"litellm_metadata": _ROUTER_METADATA}, + ), + ( + OpenAIError(status_code=500, message="upstream failed"), + "hosted_vllm", + { + "api_base": "https://relay.example/v1", + "metadata": {"user_id": "set by the caller"}, + "litellm_metadata": _ROUTER_METADATA, + }, + ), + ], + ids=["native_messages_without_api_base", "bridged_responses_with_request_metadata"], +) +def test_router_metadata_under_litellm_metadata_names_the_deployment( + provider_error: Exception, custom_llm_provider: str, extra_kwargs: dict[str, object] +) -> None: + with pytest.raises(litellm.InternalServerError) as caught: + exception_type( + model="claude-haiku-5-5", + original_exception=provider_error, + custom_llm_provider=custom_llm_provider, + extra_kwargs=extra_kwargs, + ) + debug_info: Final = caught.value.litellm_debug_info + assert isinstance(debug_info, str) + assert "API Base: `https://relay.example/v1`" in debug_info + assert "model_group: `claude-alias`" in debug_info + assert "deployment: `anthropic/claude-haiku-5-5`" in debug_info + + +_KEY_METADATA: Final = {"user_api_key_alias": "alias-key", "user_api_key_team_alias": "team-a"} + + +@pytest.mark.parametrize( + "extra_kwargs", + [ + {"metadata": _ROUTER_METADATA, "litellm_metadata": _KEY_METADATA}, + {"metadata": _KEY_METADATA, "litellm_metadata": _ROUTER_METADATA}, + ], + ids=["chat_with_guardrail_litellm_metadata", "generic_route_with_key_fields_in_metadata"], +) +def test_debug_info_reads_lines_from_both_metadata_buckets(extra_kwargs: dict[str, object]) -> None: + with pytest.raises(litellm.InternalServerError) as caught: + exception_type( + model="claude-haiku-5-5", + original_exception=AnthropicError(status_code=529, message="Overloaded"), + custom_llm_provider="anthropic", + extra_kwargs=extra_kwargs, + ) + debug_info: Final = caught.value.litellm_debug_info + assert isinstance(debug_info, str) + assert "Key Name: `alias-key`" in debug_info + assert "Team: `team-a`" in debug_info + assert "API Base: `https://relay.example/v1`" in debug_info + assert "model_group: `claude-alias`" in debug_info + assert "deployment: `anthropic/claude-haiku-5-5`" in debug_info + + +def test_litellm_metadata_wins_over_caller_metadata_for_the_same_key() -> None: + with pytest.raises(litellm.InternalServerError) as caught: + exception_type( + model="claude-haiku-5-5", + original_exception=AnthropicError(status_code=529, message="Overloaded"), + custom_llm_provider="anthropic", + extra_kwargs={ + "metadata": {"deployment": "set by the caller", "api_base": "https://caller.example"}, + "litellm_metadata": _ROUTER_METADATA, + }, + ) + debug_info: Final = caught.value.litellm_debug_info + assert isinstance(debug_info, str) + assert "deployment: `anthropic/claude-haiku-5-5`" in debug_info + assert "API Base: `https://relay.example/v1`" in debug_info + assert "caller" not in debug_info + + +def test_mapper_that_passes_its_debug_info_keeps_the_prompt_line() -> None: + request: Final = httpx.Request("POST", "https://gateway.example/v1/chat/completions") + original: Final = OpenAIError( + status_code=500, message="upstream failed", request=request, response=httpx.Response(500, request=request) + ) + with pytest.raises(litellm.InternalServerError) as caught: + exception_type( + model="gpt-5.4-mini", + original_exception=original, + custom_llm_provider="openai", + completion_kwargs={"messages": [{"role": "user", "content": "hello there"}]}, + extra_kwargs={"metadata": {"deployment": "openai/gpt-5.4-mini"}}, + ) + debug_info: Final = caught.value.litellm_debug_info + assert isinstance(debug_info, str) + assert "Messages: `[{'role': 'user', 'content': 'hello there'}]`" in debug_info + assert "deployment: `openai/gpt-5.4-mini`" in debug_info + + +def test_debug_info_has_no_messages_line_when_the_request_has_no_messages() -> None: + assert "Messages" not in _debug_info_for_api_base("https://gateway.example/v1") + + +@pytest.mark.parametrize( + "already_mapped", + [ + litellm.InternalServerError( + message="Upstream generation aborted (finish_reason=abort)", model="gpt-5.4-mini", llm_provider="openai" + ), + litellm.Timeout( + message="Stream exceeded max streaming duration of 10s", model="gpt-5.4-mini", llm_provider="openai" + ), + ], + ids=["finish_reason_abort", "max_streaming_duration"], +) +def test_already_mapped_error_gains_the_deployment_debug_info(already_mapped: Exception) -> None: + returned: Final = exception_type( + model="gpt-5.4-mini", + original_exception=already_mapped, + custom_llm_provider="openai", + completion_kwargs={"messages": [{"role": "user", "content": "CONFIDENTIAL patient record"}]}, + extra_kwargs={ + "api_base": "https://relay.example/v1", + "metadata": {"model_group": "gpt-alias", "deployment": "openai/gpt-5.4-mini"}, + }, + ) + assert returned is already_mapped + debug_info: Final = returned.litellm_debug_info + assert isinstance(debug_info, str) + assert "API Base: `https://relay.example/v1`" in debug_info + assert "deployment: `openai/gpt-5.4-mini`" in debug_info + assert "CONFIDENTIAL" not in debug_info + + +def test_error_instance_raised_by_several_requests_names_none_of_them() -> None: + shared: Final = litellm.RateLimitError(message="mock rate limit", llm_provider="openai", model="gpt-5.4-mini") + + def debug_info_for(alias: str) -> object: + with pytest.raises(litellm.RateLimitError) as caught: + completion( + model="openai/gpt-5.4-mini", + messages=[{"role": "user", "content": "hi"}], + mock_response=shared, + metadata={"user_api_key_alias": alias, "model_group": f"group-{alias}"}, + ) + assert caught.value is shared + return caught.value.litellm_debug_info + + first: Final = debug_info_for("alias-a") + assert isinstance(first, str) and "Key Name: `alias-a`" in first + assert debug_info_for("alias-b") is None + assert debug_info_for("alias-c") is None + + +_RELAY_DEPLOYMENT_KWARGS: Final = { + "api_base": "https://relay.example/v1", + "metadata": {"deployment": "openai/gpt-5.4-mini"}, +} + + +@pytest.mark.parametrize( + ("first_kwargs", "second_kwargs"), + [ + ({"litellm_call_id": "call-1", **_RELAY_DEPLOYMENT_KWARGS}, {}), + (_RELAY_DEPLOYMENT_KWARGS, {"litellm_call_id": "call-1"}), + ], + ids=["second-mapping-has-no-request", "first-mapping-had-no-call-id"], +) +def test_mapping_again_for_the_same_request_keeps_the_deployment_already_named( + first_kwargs: dict[str, object], second_kwargs: dict[str, object] +) -> None: + already_mapped: Final = litellm.InternalServerError( + message="Upstream generation aborted (finish_reason=abort)", model="gpt-5.4-mini", llm_provider="openai" + ) + exception_type( + model="gpt-5.4-mini", + original_exception=already_mapped, + custom_llm_provider="openai", + extra_kwargs=first_kwargs, + ) + exception_type( + model="gpt-5.4-mini", + original_exception=already_mapped, + custom_llm_provider="openai", + completion_kwargs={}, + extra_kwargs=second_kwargs, + ) + debug_info: Final = already_mapped.litellm_debug_info + assert isinstance(debug_info, str) + assert "API Base: `https://relay.example/v1`" in debug_info + assert "deployment: `openai/gpt-5.4-mini`" in debug_info + + +class _ErrorWhoseMessageCannotBeRead(Exception): + @property + def message(self) -> str: + raise RuntimeError("message unavailable") + + +def test_error_that_breaks_the_mapping_still_names_the_deployment() -> None: + with pytest.raises(litellm.APIConnectionError) as caught: + exception_type( + model="claude-opus-5-5", + original_exception=_ErrorWhoseMessageCannotBeRead("upstream failed"), + custom_llm_provider="anthropic", + completion_kwargs={}, + extra_kwargs={"api_base": "https://relay.example", "metadata": {"deployment": "anthropic/claude-opus-5-5"}}, + ) + debug_info: Final = caught.value.litellm_debug_info + assert isinstance(debug_info, str) + assert "API Base: `https://relay.example`" in debug_info + assert "deployment: `anthropic/claude-opus-5-5`" in debug_info + + # Test cases for is_error_str_context_window_exceeded # Tuple format: (error_message, expected_result) context_window_test_cases = [ diff --git a/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py index dd50f60250a..39f7a4a05cb 100644 --- a/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py @@ -645,6 +645,67 @@ async def test_async_sse_wrapper_reraises_upstream_error_to_connected_client(): assert detached_hook.exceptions == [], "the relay re-raised the error, so the proxy failure hook already ran" +_DEPLOYMENT_PARAMS: Final = { + "metadata": { + "api_base": "https://relay.example/anthropic", + "model_group": "claude-alias", + "deployment": "anthropic/claude-opus-5-5", + } +} + + +def _assert_names_the_deployment(error: Exception) -> None: + debug_info: Final = getattr(error, "litellm_debug_info", None) + assert isinstance(debug_info, str) + assert "API Base: `https://relay.example/anthropic`" in debug_info + assert "model_group: `claude-alias`" in debug_info + assert "deployment: `anthropic/claude-opus-5-5`" in debug_info + + +@pytest.mark.asyncio +async def test_relayed_upstream_error_reaches_the_client_naming_the_deployment(): + async def _cut_stream(): + yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}} + raise _ProviderStreamError("Response payload is not completed", status_code=529) + + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("test_upstream_error_names_the_deployment"), + request_body={}, + ) + iterator.litellm_logging_obj.model_call_details["litellm_params"] = _DEPLOYMENT_PARAMS + + with pytest.raises(_ProviderStreamError) as excinfo: + async for _ in iterator.async_sse_wrapper(_cut_stream()): + pass + + assert excinfo.value.status_code == 529 + _assert_names_the_deployment(excinfo.value) + + +@pytest.mark.asyncio +async def test_native_anthropic_stream_cut_reaches_the_client_naming_the_deployment() -> None: + logging_obj: Final = _make_logging_obj("native_anthropic_stream_cut") + logging_obj.model_call_details["litellm_params"] = _DEPLOYMENT_PARAMS + cut: Final = httpx.ReadError("Response payload is not completed") + + async def _cut_upstream() -> AsyncIterator[bytes]: + yield b'event: message_start\ndata: {"type": "message_start"}\n\n' + raise cut + + stream: Final = AnthropicMessagesConfig().get_async_streaming_response_iterator( + model=logging_obj.model, + httpx_response=httpx.Response(200, content=_cut_upstream()), + request_body={"model": logging_obj.model}, + litellm_logging_obj=logging_obj, + ) + + with pytest.raises(httpx.ReadError) as excinfo: + _ = [chunk async for chunk in stream] + + assert excinfo.value is cut + _assert_names_the_deployment(cut) + + @pytest.mark.asyncio async def test_async_sse_wrapper_logs_failure_on_upstream_error_after_disconnect(): """ diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 17a5112ce6f..ec14f67ce17 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -8,6 +8,7 @@ import functools import gc import json import threading +import time import weakref from collections.abc import Callable, Mapping from datetime import datetime @@ -367,6 +368,136 @@ def test_sync_transport_read_timeout_raises_litellm_timeout(prefix): assert "Request timed out. Error_str: ReadTimeout" in str(exc_info.value) +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix", _READ_TIMEOUT_PREFIXES) +@pytest.mark.parametrize("error_class", [httpx.ReadError, httpx.RemoteProtocolError]) +async def test_transport_cut_error_names_the_deployment(prefix, error_class): + logging_obj: Final = _logging_obj_stub() + logging_obj.model_call_details = { + "litellm_params": { + "metadata": { + "api_base": "https://relay.example/openai/v1", + "model_group": "responses-alias", + "deployment": "openai/gpt-4o-mini", + } + } + } + trailing_error: Final = error_class("peer closed connection without sending complete message body") + iterator: Final = _make_iterator(sse_events=prefix, logging_obj=logging_obj, trailing_error=trailing_error) + + with pytest.raises((error_class, MidStreamFallbackError)) as exc_info: + async for _ in iterator: + pass + + debug_info: Final = exc_info.value.litellm_debug_info + assert isinstance(debug_info, str) + assert "API Base: `https://relay.example/openai/v1`" in debug_info + assert "model_group: `responses-alias`" in debug_info + assert "deployment: `openai/gpt-4o-mini`" in debug_info + + +class _FailureDebugInfoRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.debug_info: object = None + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.debug_info = getattr(kwargs["exception"], "litellm_debug_info", None) + + +@pytest.mark.parametrize("prefix", _READ_TIMEOUT_PREFIXES) +def test_sync_transport_cut_failure_callbacks_see_the_deployment(prefix): + recorder: Final = _FailureDebugInfoRecorder() + logging_obj: Final = _real_logging_obj( + call_type="responses", + litellm_params={ + "metadata": { + "api_base": "https://relay.example/openai/v1", + "model_group": "responses-alias", + "deployment": "openai/gpt-4o-mini", + } + }, + callbacks=(recorder,), + ) + iterator: Final = _make_sync_iterator( + sse_events=prefix, + logging_obj=logging_obj, + trailing_error=httpx.ReadError("peer closed connection without sending complete message body"), + ) + + with pytest.raises((httpx.ReadError, MidStreamFallbackError)): + for _ in iterator: + pass + + assert isinstance(recorder.debug_info, str) + assert "API Base: `https://relay.example/openai/v1`" in recorder.debug_info + assert "model_group: `responses-alias`" in recorder.debug_info + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix", _READ_TIMEOUT_PREFIXES) +async def test_stream_ending_without_terminal_event_names_the_deployment(prefix): + logging_obj: Final = _failure_tracking_logging_obj() + logging_obj.model_call_details = { + "litellm_params": { + "metadata": { + "api_base": "https://relay.example/openai/v1", + "model_group": "responses-alias", + "deployment": "openai/gpt-4o-mini", + } + } + } + iterator: Final = _make_iterator(sse_events=prefix, logging_obj=logging_obj) + + with pytest.raises((litellm.APIConnectionError, MidStreamFallbackError)) as exc_info: + async for _ in iterator: + pass + + debug_info: Final = exc_info.value.litellm_debug_info + assert isinstance(debug_info, str) + assert "API Base: `https://relay.example/openai/v1`" in debug_info + assert "deployment: `openai/gpt-4o-mini`" in debug_info + + +_DEPLOYMENT_LITELLM_PARAMS: Final = { + "metadata": { + "api_base": "https://relay.example/openai/v1", + "model_group": "responses-alias", + "deployment": "openai/gpt-4o-mini", + } +} + + +async def _drain_async(iterator: ResponsesAPIStreamingIterator) -> None: + async for _ in iterator: + pass + + +async def _drain_sync(iterator: SyncResponsesAPIStreamingIterator) -> None: + for _ in iterator: + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("make_iterator", "drain"), [(_make_iterator, _drain_async), (_make_sync_iterator, _drain_sync)] +) +async def test_max_streaming_duration_timeout_names_the_deployment(monkeypatch, make_iterator, drain): + monkeypatch.setattr(streaming_module, "LITELLM_MAX_STREAMING_DURATION_SECONDS", 10.0) + logging_obj: Final = _failure_tracking_logging_obj() + logging_obj.model_call_details = {"litellm_params": _DEPLOYMENT_LITELLM_PARAMS} + iterator: Final = make_iterator(sse_events=_PARTIAL_OUTPUT_EVENTS, logging_obj=logging_obj) + iterator._stream_created_time = time.time() - 20 + + with pytest.raises(litellm.Timeout, match="max streaming duration") as exc_info: + await drain(iterator) + + debug_info: Final = exc_info.value.litellm_debug_info + assert isinstance(debug_info, str) + assert "API Base: `https://relay.example/openai/v1`" in debug_info + assert "deployment: `openai/gpt-4o-mini`" in debug_info + + @pytest.mark.asyncio @pytest.mark.parametrize("trailer", [[], [_DONE_MARKER]], ids=["eof", "done-marker"]) async def test_stream_ending_after_partial_output_without_terminal_event_raises(trailer): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a1400f53ef8..b5a5c609dae 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -11850,6 +11850,11 @@ async def test_success_enqueue_is_tracked_without_starting_callbacks_synchronous assert logging_obj.model_call_details["has_logged_async_success"] is True +@pytest.mark.parametrize("kwargs", [None, "not a mapping"], ids=["none", "string"]) +def test_get_first_chars_messages_returns_empty_text_for_input_that_is_not_a_mapping(kwargs: object) -> None: + assert litellm.get_first_chars_messages(kwargs=kwargs) == "" + + def test_get_response_string_joins_the_text_parts_of_a_completed_responses_event() -> None: from litellm.types.llms.openai import ResponseCompletedEvent from litellm.utils import get_response_string diff --git a/tests/unit/text_completions/test_streaming_iterator.py b/tests/unit/text_completions/test_streaming_iterator.py index 571451164d9..097546f7b31 100644 --- a/tests/unit/text_completions/test_streaming_iterator.py +++ b/tests/unit/text_completions/test_streaming_iterator.py @@ -2,12 +2,14 @@ from collections.abc import AsyncIterator, Iterator, Sequence from itertools import chain from typing import Final +import httpx import pytest from openai.types.completion import Completion from litellm.exceptions import APIConnectionError from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.main import _async_streaming from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper from litellm.types.utils import Choices, Delta, ModelResponseStream, StreamingChoices, TextCompletionResponse, Usage @@ -288,6 +290,33 @@ def test_sync_text_completion_stream_maps_upstream_failure_after_content() -> No assert (error.value.model, error.value.llm_provider) == ("test-model", "openai") +_DEPLOYMENT_PARAMS: Final = { + "api_base": "https://relay.example/v1", + "metadata": {"model_group": "instruct-alias", "deployment": "text-completion-openai/gpt-instruct"}, +} + + +@pytest.mark.asyncio +async def test_async_text_completion_upstream_cut_names_the_deployment() -> None: + async def cut_stream() -> AsyncIterator[ModelResponseStream]: + raise httpx.ReadError("Response payload is not completed") + yield + + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=_async_streaming(cut_stream(), "test-model", None, _DEPLOYMENT_PARAMS), + model="test-model", + ) + + with pytest.raises(APIConnectionError, match="OpenAIException") as error: + await anext(wrapper) + + debug_info: Final = error.value.litellm_debug_info + assert isinstance(debug_info, str) + assert "API Base: `https://relay.example/v1`" in debug_info + assert "model_group: `instruct-alias`" in debug_info + assert "deployment: `text-completion-openai/gpt-instruct`" in debug_info + + def _stream_with_trailing_usage_chunk() -> tuple[ModelResponseStream, ...]: return ( ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="Hi"))]), From 63d2f4fef7aedb8de7cefaef748fb88817ee2f0b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 11 Nov 2026 14:55:32 -0800 Subject: [PATCH 119/189] refactor(ui): remove the deprecated API Reference page (#51246) The API Reference page was already behind a deprecation banner. This deletes the /ui/api-reference route and its components, the Developer Tools sidebar entry and page description, the legacy ?page=api_ref redirects, and the i18n strings only that page used. Old ?page=api_ref and ?page=api-reference bookmarks now fall through to the dashboard like any other unknown page id, since there is no replacement page. Tests that listed the page are updated: the sidebar and legacy-route vitest files, the uiHref examples, the e2e sidebar fixtures and Internal Viewer nav check, and the UI path detection unit test. --- tests/e2e/ui/fixtures/menuMappings.ts | 1 - tests/e2e/ui/fixtures/migratedPages.ts | 5 - tests/e2e/ui/fixtures/pages.ts | 1 - .../internal-viewer/internalViewer.spec.ts | 1 - tests/unit/proxy/test_ui_path_detection.py | 2 +- .../_components/APIReferenceView.test.tsx | 92 ------------ .../_components/APIReferenceView.tsx | 142 ------------------ .../api-reference/_components/DocLink.tsx | 33 ---- .../app/(dashboard)/api-reference/page.tsx | 22 --- .../app/(dashboard)/legacyPageRoutes.test.ts | 2 - .../src/app/(dashboard)/legacyPageRoutes.ts | 2 - .../src/components/leftnav.test.tsx | 2 - .../src/components/leftnav.tsx | 8 - .../src/components/networking.test.ts | 2 +- .../src/components/page_metadata.ts | 1 - ui/litellm-dashboard/src/i18n/locales/en.json | 16 -- ui/litellm-dashboard/src/utils/uiHref.test.ts | 10 +- ui/litellm-dashboard/src/utils/uiHref.ts | 4 +- 18 files changed, 9 insertions(+), 337 deletions(-) delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.test.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx diff --git a/tests/e2e/ui/fixtures/menuMappings.ts b/tests/e2e/ui/fixtures/menuMappings.ts index 4cf90c00cd0..56b4356bde7 100644 --- a/tests/e2e/ui/fixtures/menuMappings.ts +++ b/tests/e2e/ui/fixtures/menuMappings.ts @@ -14,7 +14,6 @@ export const menuLabelToPage: Record = { "Internal Users": Page.Users, "Internal User": Page.Users, // Legacy label support Organizations: Page.Organizations, - "API Reference": Page.ApiRef, "AI Hub": Page.ModelHubTable, "Model Hub": Page.ModelHubTable, Logs: Page.Logs, diff --git a/tests/e2e/ui/fixtures/migratedPages.ts b/tests/e2e/ui/fixtures/migratedPages.ts index e49168ebb4a..84e7ed760d6 100644 --- a/tests/e2e/ui/fixtures/migratedPages.ts +++ b/tests/e2e/ui/fixtures/migratedPages.ts @@ -13,11 +13,6 @@ export const MIGRATED_E2E_PAGES: Readonly> = { linkName: "Models + Endpoints", content: { role: "heading", name: "Model Management" }, }, - api_ref: { - segment: "api-reference", - linkName: "API Reference", - content: { role: "heading", name: "OpenAI Compatible Proxy: API Reference" }, - }, "llm-playground": { segment: "playground", linkName: "Playground", content: { role: "tab", name: "Chat" } }, projects: { segment: "projects", diff --git a/tests/e2e/ui/fixtures/pages.ts b/tests/e2e/ui/fixtures/pages.ts index a850dcb4755..909cc93f6f8 100644 --- a/tests/e2e/ui/fixtures/pages.ts +++ b/tests/e2e/ui/fixtures/pages.ts @@ -10,7 +10,6 @@ export enum Page { Teams = "teams", Organizations = "organizations", AdminPanel = "admin-panel", - ApiRef = "api_ref", LoggingAndAlerts = "logging-and-alerts", Budgets = "budgets", Guardrails = "guardrails", diff --git a/tests/e2e/ui/tests/internal-viewer/internalViewer.spec.ts b/tests/e2e/ui/tests/internal-viewer/internalViewer.spec.ts index dd40976341d..4c2a80b5f2d 100644 --- a/tests/e2e/ui/tests/internal-viewer/internalViewer.spec.ts +++ b/tests/e2e/ui/tests/internal-viewer/internalViewer.spec.ts @@ -32,7 +32,6 @@ test.describe("Internal Viewer", () => { "Usage", "Logs", "Teams", - "API Reference", "AI Hub", ]; for (const label of expectedVisible) { diff --git a/tests/unit/proxy/test_ui_path_detection.py b/tests/unit/proxy/test_ui_path_detection.py index 5b5d43f647e..6ecdd037304 100644 --- a/tests/unit/proxy/test_ui_path_detection.py +++ b/tests/unit/proxy/test_ui_path_detection.py @@ -121,7 +121,7 @@ class TestUIDetectionLogic: # Verify routes exist found_routes = 0 - expected_routes = ["login", "guardrails", "logs", "api-reference"] + expected_routes = ["login", "guardrails", "logs"] for route in expected_routes: route_index = os.path.join(self.temp_dir, route, "index.html") if os.path.exists(route_index): diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.test.tsx deleted file mode 100644 index dae19032e20..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.test.tsx +++ /dev/null @@ -1,92 +0,0 @@ -import { render, screen } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { describe, expect, it, vi } from "vitest"; -import APIReferenceView from "./APIReferenceView"; - -vi.mock("@/components/CodeBlock", () => ({ - __esModule: true, - default: ({ code }: { code: string }) => 
{code}
, -})); - -describe("APIReferenceView", () => { - const codeBlockTestId = "api-reference-code-block"; - - it("uses the API doc base url when provided", () => { - const apiDocUrl = "https://docs.litellm.test"; - render(); - - const codeBlocks = screen.getAllByTestId(codeBlockTestId); - expect(codeBlocks[0]).toHaveTextContent(new RegExp(apiDocUrl)); - }); - - it("falls back to the proxy base url when the docs url is missing", () => { - const proxyUrl = "https://proxy.litellm.test"; - render(); - - const codeBlocks = screen.getAllByTestId(codeBlockTestId); - expect(codeBlocks[0]).toHaveTextContent(new RegExp(proxyUrl)); - }); - - it("prefers the docs url when both urls are provided", () => { - const apiDocUrl = "https://docs-preferred.litellm.test"; - const proxyUrl = "https://proxy-backup.litellm.test"; - - render( - , - ); - - const codeBlocks = screen.getAllByTestId(codeBlockTestId); - const renderedCode = codeBlocks[0].textContent ?? ""; - expect(renderedCode).toContain(apiDocUrl); - expect(renderedCode).not.toContain(proxyUrl); - }); - - it("renders the page title, blurb and docs link", () => { - render(); - - expect(screen.getByText("OpenAI Compatible Proxy: API Reference")).toBeInTheDocument(); - expect(screen.getByText(/LiteLLM is OpenAI Compatible/)).toBeInTheDocument(); - - const docsLink = screen.getByRole("link", { name: /API Reference Docs/ }); - expect(docsLink).toHaveAttribute("href", "https://docs.litellm.ai/docs/proxy/user_keys"); - expect(docsLink).toHaveAttribute("target", "_blank"); - }); - - it("exposes the three SDK tabs with the first selected by default", () => { - render(); - - expect(screen.getAllByRole("tab").map((tab) => tab.textContent)).toEqual([ - "OpenAI Python SDK", - "LlamaIndex", - "Langchain Py", - ]); - expect(screen.getAllByRole("tab").map((tab) => tab.getAttribute("aria-selected"))).toEqual([ - "true", - "false", - "false", - ]); - }); - - it.each([ - ["OpenAI Python SDK", "import openai"], - ["LlamaIndex", "from llama_index.llms import AzureOpenAI"], - ["Langchain Py", "from langchain.chat_models import ChatOpenAI"], - ])("selecting %s shows its snippet wired to the base url", async (tabName, marker) => { - const proxyUrl = "https://proxy.litellm.test"; - const user = userEvent.setup(); - render(); - - await user.click(screen.getByRole("tab", { name: tabName })); - - expect(screen.getByRole("tab", { name: tabName })).toHaveAttribute("aria-selected", "true"); - - const selectedPanel = screen.getByRole("tabpanel"); - expect(selectedPanel).toHaveTextContent(new RegExp(marker)); - expect(selectedPanel).toHaveTextContent(new RegExp(proxyUrl)); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx deleted file mode 100644 index f55dc2e4c82..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx +++ /dev/null @@ -1,142 +0,0 @@ -"use client"; -import React from "react"; -import { useTranslation } from "react-i18next"; -import CodeBlock from "@/components/CodeBlock"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import DocLink from "./DocLink"; -import { productNames } from "@/i18n/untranslated/productNames"; - -interface ApiRefProps { - proxySettings: { - PROXY_BASE_URL?: string; - LITELLM_UI_API_DOC_BASE_URL?: string | null; - }; -} - -const APIReferenceView: React.FC = ({ proxySettings }) => { - const { t } = useTranslation(); - let base_url = ""; - const customDocBaseUrl = proxySettings?.LITELLM_UI_API_DOC_BASE_URL; - if (customDocBaseUrl && customDocBaseUrl.trim()) { - base_url = customDocBaseUrl; - } else if (proxySettings?.PROXY_BASE_URL) { - base_url = proxySettings.PROXY_BASE_URL; - } - - return ( -
-
- {/* Header row with Docs link on the right */} -
-

{t("apiReference.view.title")}

- -
- -

{t("apiReference.view.description")}

- - - - - {productNames.openAIPythonSdk} - - - {productNames.llamaIndex} - - - {productNames.langchainPython} - - - - - - - - - - - - - - -
-
- ); -}; - -export default APIReferenceView; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx deleted file mode 100644 index cee174d7fdb..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx +++ /dev/null @@ -1,33 +0,0 @@ -import React from "react"; -import { ExternalLink } from "lucide-react"; -import { useTranslation } from "react-i18next"; - -import { cn } from "@/lib/cva.config"; - -export type DocLinkProps = { - href?: string; - className?: string; -}; - -const DocLink = ({ href, className }: DocLinkProps) => { - const { t } = useTranslation(); - return ( - - {t("apiReference.docLink.label")} - - {t("apiReference.docLink.newTab")} - - ); -}; - -export default DocLink; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx deleted file mode 100644 index 4d8498a1be4..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx +++ /dev/null @@ -1,22 +0,0 @@ -"use client"; - -import { useTranslation } from "react-i18next"; -import APIReferenceView from "./_components/APIReferenceView"; -import { DeprecationBanner } from "@/components/DeprecationBanner"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import useProxySettings from "@/app/(dashboard)/hooks/proxySettings/useProxySettings"; - -const APIReferencePage = () => { - const { t } = useTranslation(); - const { accessToken } = useAuthorized(); - const proxySettings = useProxySettings(accessToken); - - return ( - <> - - - - ); -}; - -export default APIReferencePage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.test.ts index 315d988db54..70b2f9b0ab7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.test.ts @@ -14,8 +14,6 @@ describe("legacyPageRedirectHref", () => { }); it("keeps the older aliases for renamed pages", () => { - expect(redirect("page=api_ref")).toBe("/ui/api-reference"); - expect(redirect("page=api-reference")).toBe("/ui/api-reference"); expect(redirect("page=claude-code-plugins")).toBe("/ui/skills"); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts index 90b49e56c5a..98cdd1e79bb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts @@ -5,8 +5,6 @@ const LEGACY_PAGE_ROUTES: ReadonlyMap = new Map( home: "home", "api-keys": "api-keys", models: "models-and-endpoints", - api_ref: "api-reference", - "api-reference": "api-reference", "llm-playground": "playground", projects: "projects", chat: "chat", diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index f3e02d13e56..e57a17c7136 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -282,7 +282,6 @@ describe("Sidebar (leftnav)", () => { "Organizations", "Access Groups", "Budgets", - "API Reference", "AI Hub", "Experimental", "Settings", @@ -701,7 +700,6 @@ describe("Sidebar (leftnav)", () => { expectHref("Models + Endpoints", "/ui/models-and-endpoints"); expectHref("Usage", "/ui/usage"); expectHref("Alerts", "/ui/alerts"); - expectHref("API Reference", "/ui/api-reference"); }); it("never links a leaf to the legacy ?page= switch", () => { diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 652a3596980..4a968fa39ec 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -28,7 +28,6 @@ import { Building2, Boxes, ChevronRight, - Code2, Database, ExternalLink, FileText, @@ -420,13 +419,6 @@ const menuGroups: MenuGroup[] = [ { groupLabel: "DEVELOPER TOOLS", items: [ - { - key: "api_ref", - page: "api_ref", - route: "api-reference", - label: "shell.nav.items.apiReference", - icon: , - }, { key: "model-hub-table", page: "model-hub-table", diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index 47a1aef87c0..bc614f1a53a 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -388,7 +388,7 @@ describe("UI config and public endpoints", () => { await Networking.getUiConfig(); expect(Networking.serverRootPath).toBe("/litellm"); - expect(uiHref("api-reference")).toBe("/litellm/ui/api-reference"); + expect(uiHref("logs")).toBe("/litellm/ui/logs"); }); it("sends model group connection probes to the configured gateway", async () => { diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index b2d86dbe7b8..d818f33c4d9 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -34,7 +34,6 @@ export const pageDescriptions: Record = { projects: "shell.pageDescriptions.projects", "access-groups": "shell.pageDescriptions.accessGroups", budgets: "shell.pageDescriptions.budgets", - api_ref: "shell.pageDescriptions.apiRef", "model-hub-table": "shell.pageDescriptions.modelHubTable", caching: "shell.pageDescriptions.caching", "transform-request": "shell.pageDescriptions.transformRequest", diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index ebf8b5dc134..854a49fd546 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -8035,7 +8035,6 @@ "organizations": "Organizations", "accessGroups": "Access Groups", "budgets": "Budgets", - "apiReference": "API Reference", "aiHub": "AI Hub", "responseCache": "Response Cache", "experimental": "Experimental", @@ -8082,7 +8081,6 @@ "projects": "Manage projects within teams", "accessGroups": "Manage access groups for role-based permissions", "budgets": "Set and monitor spending budgets", - "apiRef": "Browse API documentation and endpoints", "modelHubTable": "Explore available AI models and providers", "caching": "Configure response caching and coordination Redis settings", "transformRequest": "Set up request transformation rules", @@ -8554,20 +8552,6 @@ "empty": "No posts yet" } }, - "apiReference": { - "page": { - "featureName": "The API Reference tab" - }, - "view": { - "title": "OpenAI Compatible Proxy: API Reference", - "description": "LiteLLM is OpenAI Compatible. This means your API Key works with the OpenAI SDK. Just replace the base_url to point to your litellm proxy. Example Below" - }, - "docLink": { - "title": "Open documentation in a new tab", - "label": "API Reference Docs", - "newTab": "(opens in a new tab)" - } - }, "uiTheme": { "settings": { "toasts": { diff --git a/ui/litellm-dashboard/src/utils/uiHref.test.ts b/ui/litellm-dashboard/src/utils/uiHref.test.ts index dae2594929c..53cfd4f7971 100644 --- a/ui/litellm-dashboard/src/utils/uiHref.test.ts +++ b/ui/litellm-dashboard/src/utils/uiHref.test.ts @@ -9,16 +9,16 @@ afterEach(() => { describe("uiHref", () => { it("builds a /ui-rooted path when serverRootPath is /", () => { - expect(uiHref("api-reference")).toBe("/ui/api-reference"); + expect(uiHref("logs")).toBe("/ui/logs"); }); it("prefixes a non-root serverRootPath without duplicating slashes", () => { setServerRootPath("/team-x/"); - expect(uiHref("api-reference")).toBe("/team-x/ui/api-reference"); + expect(uiHref("logs")).toBe("/team-x/ui/logs"); }); it("tolerates a leading slash in the route segment", () => { - expect(uiHref("/api-reference")).toBe("/ui/api-reference"); + expect(uiHref("/logs")).toBe("/ui/logs"); }); it("stays root-relative under next dev, which serves the app at /", () => { @@ -29,8 +29,8 @@ describe("uiHref", () => { describe("routeSegmentForPathname", () => { it("strips the /ui base and any trailing slash", () => { - expect(routeSegmentForPathname("/ui/api-reference")).toBe("api-reference"); - expect(routeSegmentForPathname("/ui/api-reference/")).toBe("api-reference"); + expect(routeSegmentForPathname("/ui/logs")).toBe("logs"); + expect(routeSegmentForPathname("/ui/logs/")).toBe("logs"); }); it("returns an empty segment for the dashboard root", () => { diff --git a/ui/litellm-dashboard/src/utils/uiHref.ts b/ui/litellm-dashboard/src/utils/uiHref.ts index 83e65b4d5a3..dcb75ca43b2 100644 --- a/ui/litellm-dashboard/src/utils/uiHref.ts +++ b/ui/litellm-dashboard/src/utils/uiHref.ts @@ -10,12 +10,12 @@ function uiBase(): string { return `${root}/ui`; } -/** Absolute (same-origin) href for a dashboard route segment, e.g. "api-reference" -> "/ui/api-reference". */ +/** Absolute (same-origin) href for a dashboard route segment, e.g. "logs" -> "/ui/logs". */ export function uiHref(routeSegment: string): string { return `${uiBase()}/${routeSegment.replace(/^\/+/, "")}`; } -/** First route segment under the UI base, e.g. "/ui/api-reference/" -> "api-reference" and "/ui/" -> "". */ +/** First route segment under the UI base, e.g. "/ui/logs/" -> "logs" and "/ui/" -> "". */ export function routeSegmentForPathname(pathname: string): string { const base = uiBase(); const relative = pathname.startsWith(base) ? pathname.slice(base.length) : pathname; From 587902becf7d48d843befca9d7ea359a216e39d7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 11 Nov 2026 14:56:36 -0800 Subject: [PATCH 120/189] test(e2e): remove the secret manager suite (#51241) The secret manager lanes ran the proxy against a Vault dev container and a Conjur container that we start ourselves, so nothing in them needed a real third-party service. They have not run a single test since late October, because the co-located proxy cannot boot in the lane pod, and they were soft_fail outside the promotion gate, so they only added noise. The AWS test in the folder was skipped on every lane, since no aws lane ever existed. The four coverage registry rows go with the tests, because the registry is the e2e denominator and this behavior belongs in the integration tier if it gets covered again. The Vault and CyberArk clients keep their unit tests. --- tests/e2e/AGENTS.md | 1 - tests/e2e/CONTRIBUTING.md | 25 --- tests/e2e/conftest.py | 7 - tests/e2e/coverage_registry/other.yaml | 4 - tests/e2e/e2e_config.py | 1 - .../secret_manager_cyberark_ci_config.yml | 8 - ...cret_manager_hashicorp_vault_ci_config.yml | 8 - tests/e2e/pytest.ini | 1 - tests/e2e/secret_manager/backend.sh | 76 --------- tests/e2e/secret_manager/conftest.py | 57 ------- tests/e2e/secret_manager/secret_backends.py | 25 --- tests/e2e/secret_manager/secret_store.py | 29 ---- .../secret_manager/secret_store_cyberark.py | 133 --------------- .../secret_store_hashicorp_vault.py | 117 ------------- .../test_aws_secret_manager_e2e.py | 34 ---- .../secret_manager/test_secret_manager_e2e.py | 155 ------------------ 16 files changed, 681 deletions(-) delete mode 100644 tests/e2e/gateway/secret_manager_cyberark_ci_config.yml delete mode 100644 tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml delete mode 100755 tests/e2e/secret_manager/backend.sh delete mode 100644 tests/e2e/secret_manager/conftest.py delete mode 100644 tests/e2e/secret_manager/secret_backends.py delete mode 100644 tests/e2e/secret_manager/secret_store.py delete mode 100644 tests/e2e/secret_manager/secret_store_cyberark.py delete mode 100644 tests/e2e/secret_manager/secret_store_hashicorp_vault.py delete mode 100644 tests/e2e/secret_manager/test_aws_secret_manager_e2e.py delete mode 100644 tests/e2e/secret_manager/test_secret_manager_e2e.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index abdc1ecafbb..008a8f80cf0 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -46,7 +46,6 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `router/` - routing and reliability behavior (fallbacks, cooldowns) - `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, run by hand since its weekly workflow never had working Bedrock credentials), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). The memory tests (`test_reliability_memory_e2e.py`, run by the perf lane on its own fresh stack: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the check for a DB-backed boot that idles near the pod limit the way v1.100.x did, and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a loose ceiling over the measured batches, the check for the v1.100.0 retry-breadcrumb leak). New perf tests go here too, marked `perf`, behind the same `E2E_PERF_TESTS` opt-in. The perf cases (`test_perf_cases.py`, one `PerformanceTestCase` per module in `perf/cases/`, no `_e2e` suffix) are the request-performance runs. A case names what the gateway was deployed with (model groups and guardrails, written out in the gateway's config: the `proxy_config` block of project-releaser's perf helm values in the lane, `gateway/perf_ci_config.yml` locally, with each deployment's mock knobs in an `x-mock-knobs` extra header so the model ids are real ones like `gpt-6-luna` and `anthropic.claude-sonnet-5`) and lists what the test creates and sends: teams, Keycloak JWT users, virtual keys, `RequestShape`s (a coding-agent session from `perf/setup/corpus/` whose input-token size is drawn per request from the shape's range, with or without tools, and the output cap), a `Traffic` plan (seed, rps, request count, endpoint, shape and stream shares, or a `Ramp` that steps the rate up through the run, built with `ramped()`, whose plan carries each request's send second and whose run reports `sustained_rps`, the last step the pod held, scored in `perf/setup/ramp.py`, with every step's latency, completed rate, non-200s, CPU per request, Redis latency and Postgres connections cut from the samplers' time-stamped `Series` (`perf/setup/series.py`) to the step's window and emitted as `perf_ramp_step__` JUnit properties for the dashboard) and `Limit`s (one metric at one percentile under one ceiling, or over one floor for `sustained_rps`). The infra side lives in `perf/setup/`: `provision.py` checks the deployed config matches the case and builds the callers, `plan.py` draws the request list from the seed, `run.py` replays it from a separate Locust process (`locustfile.py`) that builds each body right before sending and records status, header, first-byte, first-token and total time per request, and `observe.py` joins all of that with what the mocks received and the workers' RSS to score every `Limit`. Limits are about what the caller sees and what the gateway costs: every non-blocked request is a 200 (`unexpected_status_count`), the seconds the gateway added per request at p50 and p99 (`gateway_added_seconds`, to the first token on streams), the client's time to the first token of a stream at p50 and p99 (`time_to_first_token_seconds`, mock delay included), worker memory growth (`memory_growth_megabytes`), and the Postgres statements and Redis commands a request waits on before it is forwarded (`postgres_calls_per_request`, `redis_calls_per_request`: auth and routing reads, the rate-limiter script), with the work the gateway does after the response or on a timer (spend writes, daily aggregates, config reloads) reported separately as `postgres_background_calls_per_request` and `redis_background_calls_per_request`; the same overhead as the gateway itself reports it, read from the `x-litellm-overhead-duration-ms` header per request (`litellm_overhead_seconds`, gated next to `gateway_added_seconds`, the outside reading of the same thing, so the two can be charted together) and the gateway's own end-to-end time from `x-litellm-response-duration-ms` (`litellm_response_seconds`), the run mean of the gateway's `litellm_redis_latency` histogram from a `/metrics` scrape before and after the run (`telemetry.py`, `prometheus_redis_latency_mean_seconds`), and the database's own view of the run from the same `pg_stat_statements` rows and a once-a-second `pg_stat_activity` sample: execution seconds per wall second (`postgres_active_sessions`, the average-active-sessions number RDS reports as DB load), the slowest statement's mean (`postgres_slowest_statement_seconds`, named in the detail) and the peak client connection count (`postgres_peak_connections`), each under a ceiling set from the first local runs and due a retune once the lane has its ten-run baseline; `datastore.py` reads `pg_stat_statements` before and after the run and holds a Redis `MONITOR` connection during it, sorts each statement by verb and table and each command by key, and the run output lists the top statements and commands per bucket so a red run names what grew. Whether a body, a guardrail call or a spend row is right is printed as a diagnostic, never a limit, that is the functional suites' job. The providers and the guardrail are out-of-process servers in `perf/setup/mocks/` (`server.py openai|anthropic|bedrock|guardrail`, each answering with a reply generated for the request in that provider's wire format, sized from the prompt and output cap, a tool call when tools were offered, waiting the case's `mock_latency` first, request logs at `/mock/requests` and `/mock/calls`), reached through `E2E_PERF_MOCK_*_URL` by both the gateway config and the test. Adding a perf test is adding a module with a `CASE` to `perf/cases/`; a new deployment or guardrail is a config change; a new metric is a name in `Metric` plus a branch in `observe.observe` - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at, see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite -- `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py `, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 4f0c7624e31..94fb0b3372f 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -92,31 +92,6 @@ The suites run against a live proxy, so bring one up first by running the litell Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and start the proxy with `LITELLM_OTEL_V2=true`, the `otel` callback, and `OTEL_EXPORTER_OTLP_ENDPOINT` pointing at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. -### Secret manager lanes - -`key_management_system` is global to the proxy, so the `secret_manager/` tests run once per backend, each against its own proxy. The backends are `hashicorp_vault` and `cyberark` (CyberArk Conjur). `E2E_SECRET_MANAGER` opts in and names the backend (a key of `secret_backends.BACKENDS`). The proxy boots from `gateway/secret_manager__ci_config.yml`, and the tests reach the same manager through that backend's `SecretStore`. The managers are enterprise features, so the proxy needs a license. `secret_manager/backend.sh` runs any backend in Docker and writes its env, so every lane runs the same way locally: - -```bash -bash tests/e2e/secret_manager/backend.sh up cyberark -(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/proxy.env; set +a; env -u OPENAI_API_KEY LITELLM_LICENSE=... \ - LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" DATABASE_URL=... uv run litellm --config tests/e2e/gateway/secret_manager_cyberark_ci_config.yml --port 4000) -(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/tests.env; set +a; OPENAI_API_KEY=... \ - uv run --group e2e-dev pytest tests/e2e/secret_manager/ -v) -bash tests/e2e/secret_manager/backend.sh down cyberark -``` - -`E2E_SECRET_MANAGER_PORT` moves the manager off its usual port (8200 for Vault, 8080 for Conjur), and `E2E_SECRET_MANAGER_DIR` moves the env files. Keep that directory private, because both files hold a working admin credential. Keep `OPENAI_API_KEY` out of the proxy's environment. The tests copy the runner's key into the manager under a fresh name per test, so a passing call proves the key came through the manager rather than the `os.environ` fallback `get_secret` takes when the manager errors - -A backend declares what it supports in its `SecretBackend.capabilities`, and a test that needs something not every backend does carries `@pytest.mark.requires_capability(...)`, so it is deselected, not failed or skipped, on the lanes that lack it. CyberArk has no `deletes_stored_keys`, because the proxy's delete answers `not_supported` and Conjur keeps the key, so the delete test runs only on the Vault lane - -To add a backend, leave the tests and markers alone and add: - -1. `secret_manager/secret_store_.py`: a `SecretStore` (`write`, `read` returning None when absent, idempotent `destroy`) over the manager's own API through `e2e_http`'s external helpers, read from `E2E__*` env vars, and a `SecretBackend` whose `system` is the litellm `KeyManagementSystem` value and whose `capabilities` lists what it supports -2. its entry in `secret_backends.BACKENDS` -3. `gateway/secret_manager__ci_config.yml`, a copy of an existing lane's with only `key_management_system` changed -4. an `up_` function in `secret_manager/backend.sh` that starts the manager and writes `proxy.env` and `tests.env` -5. a CI step that runs `backend.sh up ` (or the same containers as sidecars), boots the proxy with `proxy.env` and a license, and runs pytest with `tests.env` - ### Record and replay Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 8fb96bef2f9..526651e88a1 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -38,7 +38,6 @@ from e2e_config import ( PROXY_WIDE_CONFIG_OPT_IN_ENV, SAGEMAKER_NOVA2_LITE_OPT_IN_ENV, SAGEMAKER_NOVA_OPT_IN_ENV, - SECRET_MANAGER_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV, unique_marker, ) @@ -74,7 +73,6 @@ OPT_IN_MARKERS: Final = MappingProxyType( "owned_gateway": OWNED_GATEWAY_OPT_IN_ENV, "otel_v2": OTEL_V2_OPT_IN_ENV, "otel_tls": OTEL_TLS_OPT_IN_ENV, - "secret_manager": SECRET_MANAGER_OPT_IN_ENV, "sagemaker_nova": SAGEMAKER_NOVA_OPT_IN_ENV, "sagemaker_nova2_lite": SAGEMAKER_NOVA2_LITE_OPT_IN_ENV, "proxy_wide_config": PROXY_WIDE_CONFIG_OPT_IN_ENV, @@ -181,11 +179,6 @@ def pytest_configure(config: pytest.Config) -> None: "markers", "otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set", ) - config.addinivalue_line( - "markers", - "secret_manager: needs a proxy booted from gateway/secret_manager__ci_config.yml against that live " - "secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py)", - ) config.addinivalue_line( "markers", "sagemaker_nova: needs a deployed SageMaker Nova endpoint the proxy's AWS credentials can invoke; " diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index ff0f2263515..fc345786861 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -45,10 +45,6 @@ - {id: other.config.runtime_update.applies_at_runtime, module: other, tier: P0, area: config, assertions: [applies_at_runtime], source: "proxy_server.py:14014-14060", rationale: "/config/update persists to DB + invalidates cache"} - {id: other.config.passthrough.headers_forwarded, module: other, tier: P0, area: config, assertions: [headers_forwarded], source: "passthrough/utils.py forward_headers_from_request", rationale: "Custom pass-through static headers and x-pass-* client headers reach the upstream"} - {id: other.config.general_settings.alert_webhook_side_effect, module: other, tier: P1, area: config, assertions: [alert_webhook_side_effect], source: "proxy_server.py:14215", rationale: "alert_to_webhook_url auto-enables slack alerting"} -- {id: other.config.secret_resolution.kms_integration, module: other, tier: P1, area: config, assertions: [kms_integration], source: "secret_managers/main.py get_secret / secret_manager/test_secret_manager_e2e.py", rationale: "A deployment whose api_key is os.environ/ gets its key from the configured secret manager when that name exists only in the manager"} -- {id: other.config.secret_resolution.manager_value_used, module: other, tier: P1, area: config, assertions: [manager_value_used], source: "secret_managers/main.py get_secret / secret_manager/test_secret_manager_e2e.py", rationale: "The value the manager holds is what reaches the provider: a bogus key in the manager is rejected by the provider with 401, so a passing resolution test cannot be an env fallback"} -- {id: other.config.secret_manager.virtual_key_stored, module: other, tier: P1, area: config, assertions: [virtual_key_stored], source: "key_management_event_hooks.py _store_virtual_key_in_secret_manager", rationale: "With store_virtual_keys, /key/generate writes the new key under prefix_for_stored_virtual_keys + key_alias in the manager"} -- {id: other.config.secret_manager.virtual_key_deleted, module: other, tier: P1, area: config, assertions: [virtual_key_deleted], source: "key_management_event_hooks.py _delete_virtual_keys_from_secret_manager", rationale: "/key/delete removes the stored key from the manager, so a revoked key does not linger there"} - {id: other.config.overrides.audit_logged, module: other, tier: P1, area: config, assertions: [audit_logged], source: "config_override_endpoints.py:67-100", rationale: "Config override mutations audit-logged, values redacted"} - {id: other.key_mgmt.regenerate.grace_period_honored, module: other, tier: P1, area: auth, assertions: [grace_period_honored], source: "key_management_endpoints.py:4503-4560", rationale: "Old key valid during grace_period then revoked"} - {id: other.key_mgmt.spend_reset.resets_to_value, module: other, tier: P1, area: auth, assertions: [resets_to_value], source: "key_management_endpoints.py:4841", rationale: "reset_spend resets accumulated spend"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 6316d22d3fa..bdd77d6fd6d 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -207,7 +207,6 @@ PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE" OWNED_GATEWAY_OPT_IN_ENV: Final = "E2E_OWNED_GATEWAY" OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2" OTEL_TLS_OPT_IN_ENV: Final = "E2E_OTEL_EXPORTER_ENDPOINT" -SECRET_MANAGER_OPT_IN_ENV: Final = "E2E_SECRET_MANAGER" SAGEMAKER_NOVA_OPT_IN_ENV: Final = "SAGEMAKER_NOVA_ENDPOINT" SAGEMAKER_NOVA2_LITE_OPT_IN_ENV: Final = "SAGEMAKER_NOVA2_LITE_ENDPOINT" PROXY_WIDE_CONFIG_OPT_IN_ENV: Final = "E2E_PROXY_WIDE_CONFIG_STACK" diff --git a/tests/e2e/gateway/secret_manager_cyberark_ci_config.yml b/tests/e2e/gateway/secret_manager_cyberark_ci_config.yml deleted file mode 100644 index 32602783b02..00000000000 --- a/tests/e2e/gateway/secret_manager_cyberark_ci_config.yml +++ /dev/null @@ -1,8 +0,0 @@ -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - store_model_in_db: true - key_management_system: cyberark - key_management_settings: - access_mode: read_and_write - store_virtual_keys: true - prefix_for_stored_virtual_keys: litellm-e2e/virtual-keys/ diff --git a/tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml b/tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml deleted file mode 100644 index 8e6b7e74f8e..00000000000 --- a/tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml +++ /dev/null @@ -1,8 +0,0 @@ -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - store_model_in_db: true - key_management_system: hashicorp_vault - key_management_settings: - access_mode: read_and_write - store_virtual_keys: true - prefix_for_stored_virtual_keys: litellm-e2e/virtual-keys/ diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 891c5fc282c..d0707d02d6c 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -17,7 +17,6 @@ markers = owned_gateway: boots its own proxy from source against the stack's Postgres, so it needs DATABASE_URL on the pytest host; deselected unless E2E_OWNED_GATEWAY is set otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set - secret_manager: needs a proxy booted from gateway/secret_manager__ci_config.yml against that live secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py) sagemaker_nova: needs a deployed SageMaker Nova endpoint the proxy's AWS credentials can invoke; deselected unless SAGEMAKER_NOVA_ENDPOINT names it sagemaker_nova2_lite: needs a deployed SageMaker Nova 2 Lite endpoint the proxy's AWS credentials can invoke; deselected unless SAGEMAKER_NOVA2_LITE_ENDPOINT names it proxy_wide_config: turns on a proxy-wide setting through /config/update, so it runs on a stack no other suite is driving; deselected unless E2E_PROXY_WIDE_CONFIG_STACK is set diff --git a/tests/e2e/secret_manager/backend.sh b/tests/e2e/secret_manager/backend.sh deleted file mode 100755 index 70272c237c5..00000000000 --- a/tests/e2e/secret_manager/backend.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -umask 077 - -usage() { - local systems - systems=$(declare -F | sed -n 's/^declare -f up_//p' | paste -sd '|' -) - echo "usage: $0 up|down $systems" >&2 - exit 2 -} - -action=${1:-} -system=${2:-} -dir=${E2E_SECRET_MANAGER_DIR:-$HOME/.cache/litellm-e2e-secret-manager}/$system -name=litellm-e2e-$system - -wait_for() { - local url=$1 - for _ in $(seq 1 90); do - if curl -sf -o /dev/null "$url"; then - return 0 - fi - sleep 2 - done - echo "$system did not answer at $url" >&2 - return 1 -} - -down() { - docker rm -f "$name" "$name-db" >/dev/null 2>&1 || true - docker network rm "$name" >/dev/null 2>&1 || true - rm -rf "$dir" -} - -up_hashicorp_vault() { - local port=${E2E_SECRET_MANAGER_PORT:-8200} - local token - token=e2e-$(openssl rand -hex 16) - docker run -d --name "$name" -p "127.0.0.1:$port:8200" --cap-add IPC_LOCK \ - -e VAULT_DEV_ROOT_TOKEN_ID="$token" hashicorp/vault:1.20 >/dev/null - wait_for "http://127.0.0.1:$port/v1/sys/health" - printf 'HCP_VAULT_ADDR=http://127.0.0.1:%s\nHCP_VAULT_TOKEN=%s\n' "$port" "$token" >"$dir/proxy.env" - printf 'E2E_VAULT_ADDR=http://127.0.0.1:%s\nE2E_VAULT_TOKEN=%s\n' "$port" "$token" >"$dir/tests.env" -} - -up_cyberark() { - local port=${E2E_SECRET_MANAGER_PORT:-8080} - local data_key api_key - docker network create "$name" >/dev/null - docker run -d --name "$name-db" --network "$name" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:15 >/dev/null - data_key=$(docker run --rm cyberark/conjur:1.24 data-key generate) - docker run -d --name "$name" --network "$name" -p "127.0.0.1:$port:80" \ - -e DATABASE_URL="postgres://postgres@$name-db/postgres" -e CONJUR_DATA_KEY="$data_key" \ - -e CONJUR_AUTHENTICATORS=authn cyberark/conjur:1.24 server >/dev/null - wait_for "http://127.0.0.1:$port/" - docker exec "$name" conjurctl account create --name default >/dev/null - api_key=$(docker exec "$name" conjurctl role retrieve-key default:user:admin | tr -d '\r\n') - printf 'CYBERARK_API_BASE=http://127.0.0.1:%s\nCYBERARK_ACCOUNT=default\nCYBERARK_USERNAME=admin\nCYBERARK_API_KEY=%s\n' \ - "$port" "$api_key" >"$dir/proxy.env" - printf 'E2E_CYBERARK_API_BASE=http://127.0.0.1:%s\nE2E_CYBERARK_ACCOUNT=default\nE2E_CYBERARK_USERNAME=admin\nE2E_CYBERARK_API_KEY=%s\n' \ - "$port" "$api_key" >"$dir/tests.env" -} - -[[ $# -eq 2 && -n $system ]] && declare -F "up_$system" >/dev/null || usage - -case $action in - up) - down - mkdir -p "$dir" - "up_$system" - echo "E2E_SECRET_MANAGER=$system" >>"$dir/tests.env" - echo "$system is up; env in $dir/proxy.env (proxy) and $dir/tests.env (pytest)" - ;; - down) down ;; - *) usage ;; -esac diff --git a/tests/e2e/secret_manager/conftest.py b/tests/e2e/secret_manager/conftest.py deleted file mode 100644 index 46ef598d711..00000000000 --- a/tests/e2e/secret_manager/conftest.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -import os -from dataclasses import dataclass -from typing import Final - -import pytest - -from e2e_config import SECRET_MANAGER_OPT_IN_ENV -from proxy_client import ProxyClient -from secret_backends import BACKENDS, selected_backend -from secret_store import SecretBackend, SecretStore - -REQUIRES_CAPABILITY: Final = "requires_capability" - - -def pytest_configure(config: pytest.Config) -> None: - config.addinivalue_line( - "markers", - f"{REQUIRES_CAPABILITY}(capability): secret_manager test deselected when the backend " - f"{SECRET_MANAGER_OPT_IN_ENV} names lacks the capability (secret_store.Capability)", - ) - - -def _lacks_capability(item: pytest.Item, backend: SecretBackend) -> bool: - marker: Final = item.get_closest_marker(REQUIRES_CAPABILITY) - return marker is not None and marker.args[0] not in backend.capabilities - - -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - backend: Final = BACKENDS.get(os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip()) - if backend is None: - return - deselected: Final = [item for item in items if _lacks_capability(item, backend)] - if deselected: - config.hook.pytest_deselected(items=deselected) - items[:] = [item for item in items if not _lacks_capability(item, backend)] - - -@dataclass(frozen=True, slots=True) -class SecretManagerClient: - proxy: ProxyClient - - -@pytest.fixture(scope="session") -def client(proxy: ProxyClient) -> SecretManagerClient: - return SecretManagerClient(proxy) - - -@pytest.fixture(scope="session") -def backend() -> SecretBackend: - return selected_backend() - - -@pytest.fixture(scope="session") -def store(backend: SecretBackend) -> SecretStore: - return backend.from_env() diff --git a/tests/e2e/secret_manager/secret_backends.py b/tests/e2e/secret_manager/secret_backends.py deleted file mode 100644 index 578b56970a3..00000000000 --- a/tests/e2e/secret_manager/secret_backends.py +++ /dev/null @@ -1,25 +0,0 @@ -from __future__ import annotations - -import os -from types import MappingProxyType -from typing import Final - -import pytest - -from e2e_config import SECRET_MANAGER_OPT_IN_ENV -from secret_store import SecretBackend -from secret_store_cyberark import CYBERARK -from secret_store_hashicorp_vault import HASHICORP_VAULT - -BACKENDS: Final = MappingProxyType({backend.system: backend for backend in (HASHICORP_VAULT, CYBERARK)}) - - -def selected_backend() -> SecretBackend: - system: Final = os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip() - backend: Final = BACKENDS.get(system) - if backend is None: - pytest.fail( - f"{SECRET_MANAGER_OPT_IN_ENV}={system!r} names no secret manager backend; " - f"set it to one of {sorted(BACKENDS)}" - ) - return backend diff --git a/tests/e2e/secret_manager/secret_store.py b/tests/e2e/secret_manager/secret_store.py deleted file mode 100644 index cff5005b73a..00000000000 --- a/tests/e2e/secret_manager/secret_store.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from typing import Final, Literal, Protocol - -SECRET_MANAGER_CONFIG_DIR: Final = "gateway" - - -class SecretStore(Protocol): - def write(self, name: str, value: str) -> None: ... - - def read(self, name: str) -> str | None: ... - - def destroy(self, name: str) -> None: ... - - -Capability = Literal["deletes_stored_keys"] - - -@dataclass(frozen=True, slots=True) -class SecretBackend: - system: str - from_env: Callable[[], SecretStore] - capabilities: frozenset[Capability] - - @property - def proxy_config(self) -> str: - return f"{SECRET_MANAGER_CONFIG_DIR}/secret_manager_{self.system}_ci_config.yml" diff --git a/tests/e2e/secret_manager/secret_store_cyberark.py b/tests/e2e/secret_manager/secret_store_cyberark.py deleted file mode 100644 index 22c7c458d86..00000000000 --- a/tests/e2e/secret_manager/secret_store_cyberark.py +++ /dev/null @@ -1,133 +0,0 @@ -from __future__ import annotations - -import base64 -import os -import time -from dataclasses import dataclass, field -from typing import Final, Literal -from urllib.parse import quote - -import pytest -import yaml -from e2e_http import ExternalWrite, Headers, send_text_external -from e2e_metadata import step -from pydantic import Field - -from secret_store import SecretBackend - -CYBERARK_API_BASE_ENV: Final = "E2E_CYBERARK_API_BASE" -CYBERARK_ACCOUNT_ENV: Final = "E2E_CYBERARK_ACCOUNT" -CYBERARK_USERNAME_ENV: Final = "E2E_CYBERARK_USERNAME" -CYBERARK_API_KEY_ENV: Final = "E2E_CYBERARK_API_KEY" - -# The same defaults CyberArkSecretManager falls back to for CYBERARK_*. -DEFAULT_API_BASE: Final = "http://127.0.0.1:8080" -DEFAULT_ACCOUNT: Final = "default" -DEFAULT_USERNAME: Final = "admin" - -SYSTEM: Final = "cyberark" - -_POLICY_LOAD_ATTEMPTS: Final = 5 -_POLICY_LOAD_RETRY_DELAY_SECONDS: Final = 0.2 - -_START_HINT: Final = ( - f"Start one with `bash tests/e2e/secret_manager/backend.sh up {SYSTEM}`, which writes the env for " - f"the proxy (booted from gateway/secret_manager_{SYSTEM}_ci_config.yml) and for the tests" -) - - -class ConjurHeaders(Headers): - authorization: str = Field(repr=False) - content_type: str | None = Field(default=None, serialization_alias="Content-Type") - - -def _policy_scalar(name: str) -> str: - # Quoted the way CyberArkSecretManager._ensure_variable_exists quotes it. - return yaml.safe_dump(name, default_style='"').strip() - - -@dataclass(frozen=True, slots=True) -class Conjur: - base_url: str - account: str - username: str - api_key: str = field(repr=False) - - def _fail_unless_reached(self, result: ExternalWrite, action: str) -> None: - if result.status_code == -1: - pytest.fail(f"No live Conjur at {self.base_url}: {result.body}. {_START_HINT}") - if result.status_code == 401: - pytest.fail(f"Conjur rejected {self.username}'s credentials while trying to {action}. {_START_HINT}") - - def _headers(self, content_type: str | None = None) -> ConjurHeaders: - # Tokens last about eight minutes, so each call authenticates afresh rather than - # letting a long session outlive a cached one. - auth: Final = send_text_external( - "POST", - f"{self.base_url}/authn/{self.account}/{quote(self.username, safe='')}/authenticate", - headers=Headers(), - content=self.api_key, - ) - self._fail_unless_reached(auth, "authenticate") - if not auth.ok: - pytest.fail(f"Conjur refused to authenticate {self.username}: HTTP {auth.status_code} {auth.body[:300]}") - token: Final = base64.b64encode(auth.body.encode()).decode() - return ConjurHeaders(authorization=f'Token token="{token}"', content_type=content_type) - - def _secret_url(self, name: str) -> str: - return f"{self.base_url}/secrets/{self.account}/variable/{quote(name, safe='')}" - - def _load_root_policy(self, method: Literal["POST", "PATCH"], policy: str, attempt: int = 0) -> ExternalWrite: - result: Final = send_text_external( - method, - f"{self.base_url}/policies/{self.account}/policy/root", - headers=self._headers(content_type="application/x-yaml"), - content=policy, - ) - if result.status_code != 409 or attempt + 1 == _POLICY_LOAD_ATTEMPTS: - return result - time.sleep(_POLICY_LOAD_RETRY_DELAY_SECONDS * (1 << attempt)) - return self._load_root_policy(method, policy, attempt + 1) - - def _update_root_policy(self, method: Literal["POST", "PATCH"], policy: str, action: str) -> None: - result: Final = self._load_root_policy(method, policy) - self._fail_unless_reached(result, action) - if not result.ok: - pytest.fail(f"Conjur refused to {action}: HTTP {result.status_code} {result.body[:300]}") - - @step("Write the secret {name} to CyberArk Conjur") - def write(self, name: str, value: str) -> None: - self._update_root_policy("POST", f"- !variable {_policy_scalar(name)}\n", f"declare {name}") - result: Final = send_text_external("POST", self._secret_url(name), headers=self._headers(), content=value) - self._fail_unless_reached(result, f"write {name}") - if not result.ok: - pytest.fail(f"Conjur refused to write {name}: HTTP {result.status_code} {result.body[:300]}") - - @step("Read the secret {name} from CyberArk Conjur") - def read(self, name: str) -> str | None: - result: Final = send_text_external("GET", self._secret_url(name), headers=self._headers()) - self._fail_unless_reached(result, f"read {name}") - if result.status_code == 404: - return None - if not result.ok: - pytest.fail(f"Conjur refused to read {name}: HTTP {result.status_code} {result.body[:300]}") - return result.body - - @step("Delete the secret {name} from CyberArk Conjur") - def destroy(self, name: str) -> None: - self._update_root_policy("PATCH", f"- !delete\n record: !variable {_policy_scalar(name)}\n", f"destroy {name}") - - -def conjur_from_env() -> Conjur: - api_key: Final = os.environ.get(CYBERARK_API_KEY_ENV, "").strip() - if not api_key: - pytest.fail(f"The {SYSTEM} lane needs {CYBERARK_API_KEY_ENV} to reach its Conjur. {_START_HINT}") - return Conjur( - base_url=os.environ.get(CYBERARK_API_BASE_ENV, "").strip().rstrip("/") or DEFAULT_API_BASE, - account=os.environ.get(CYBERARK_ACCOUNT_ENV, "").strip() or DEFAULT_ACCOUNT, - username=os.environ.get(CYBERARK_USERNAME_ENV, "").strip() or DEFAULT_USERNAME, - api_key=api_key, - ) - - -CYBERARK: Final = SecretBackend(system=SYSTEM, from_env=conjur_from_env, capabilities=frozenset()) diff --git a/tests/e2e/secret_manager/secret_store_hashicorp_vault.py b/tests/e2e/secret_manager/secret_store_hashicorp_vault.py deleted file mode 100644 index 5954719ccbb..00000000000 --- a/tests/e2e/secret_manager/secret_store_hashicorp_vault.py +++ /dev/null @@ -1,117 +0,0 @@ -from __future__ import annotations - -import os -from dataclasses import dataclass, field -from typing import Final - -import pytest -from e2e_http import ( - Headers, - NetworkError, - Success, - UnknownApiError, - delete_external, - get_external, - post_json_external, -) -from e2e_metadata import step -from pydantic import BaseModel, Field - -from secret_store import SecretBackend - -VAULT_ADDR_ENV: Final = "E2E_VAULT_ADDR" -VAULT_TOKEN_ENV: Final = "E2E_VAULT_TOKEN" -VAULT_MOUNT_ENV: Final = "E2E_VAULT_MOUNT_NAME" - -DEFAULT_VAULT_ADDR: Final = "http://127.0.0.1:8200" -DEFAULT_MOUNT: Final = "secret" - -SYSTEM: Final = "hashicorp_vault" - -_START_HINT: Final = ( - f"Start one with `bash tests/e2e/secret_manager/backend.sh up {SYSTEM}`, which writes the env for " - f"the proxy (booted from gateway/secret_manager_{SYSTEM}_ci_config.yml) and for the tests" -) - - -class VaultHeaders(Headers): - x_vault_token: str = Field(serialization_alias="X-Vault-Token", repr=False) - - -class KvData(BaseModel): - key: str = Field(repr=False) - - -class KvWriteBody(BaseModel): - data: KvData - - -class KvReadData(BaseModel): - data: KvData - - -class KvReadResponse(BaseModel): - data: KvReadData - - -@dataclass(frozen=True, slots=True) -class Vault: - base_url: str - token: str = field(repr=False) - mount: str = DEFAULT_MOUNT - - def _headers(self) -> VaultHeaders: - return VaultHeaders(x_vault_token=self.token) - - def _data_url(self, name: str) -> str: - return f"{self.base_url}/v1/{self.mount}/data/{name}" - - def _metadata_url(self, name: str) -> str: - return f"{self.base_url}/v1/{self.mount}/metadata/{name}" - - @step("Write the secret {name} to HashiCorp Vault") - def write(self, name: str, value: str) -> None: - write: Final = post_json_external( - self._data_url(name), headers=self._headers(), json=KvWriteBody(data=KvData(key=value)) - ) - if write.status_code == -1: - pytest.fail(f"No live Vault at {self.base_url}: {write.body}. {_START_HINT}") - if not write.ok: - pytest.fail(f"Vault refused to write {name}: HTTP {write.status_code} {write.body[:300]}") - - @step("Read the secret {name} from HashiCorp Vault") - def read(self, name: str) -> str | None: - result: Final = get_external(self._data_url(name), headers=self._headers(), response_type=KvReadResponse) - match result: - case Success(data=body): - return body.data.data.key - case UnknownApiError(status_code=404): - return None - case NetworkError(message=message): - return pytest.fail(f"No live Vault at {self.base_url}: {message}. {_START_HINT}") - case _: - return pytest.fail(f"Vault refused to read {name}: {result}") - - @step("Delete the secret {name} from HashiCorp Vault") - def destroy(self, name: str) -> None: - write: Final = delete_external(self._metadata_url(name), headers=self._headers()) - if not write.ok and write.status_code != 404: - pytest.fail(f"Vault refused to destroy {name}: HTTP {write.status_code} {write.body[:300]}") - - -def vault_from_env() -> Vault: - token: Final = os.environ.get(VAULT_TOKEN_ENV, "").strip() - if not token: - pytest.fail(f"The hashicorp_vault lane needs {VAULT_TOKEN_ENV} to reach its Vault. {_START_HINT}") - return Vault( - base_url=os.environ.get(VAULT_ADDR_ENV, DEFAULT_VAULT_ADDR).rstrip("/"), - token=token, - mount=os.environ.get(VAULT_MOUNT_ENV, "").strip() or DEFAULT_MOUNT, - ) - - -HASHICORP_VAULT: Final = SecretBackend( - system=SYSTEM, - from_env=vault_from_env, - capabilities=frozenset({"deletes_stored_keys"}), -) diff --git a/tests/e2e/secret_manager/test_aws_secret_manager_e2e.py b/tests/e2e/secret_manager/test_aws_secret_manager_e2e.py deleted file mode 100644 index 5bd67abd2f6..00000000000 --- a/tests/e2e/secret_manager/test_aws_secret_manager_e2e.py +++ /dev/null @@ -1,34 +0,0 @@ -import hashlib -import os -from typing import Final - -import pytest -from e2e_config import SECRET_MANAGER_OPT_IN_ENV -from e2e_metadata import Domain, Subject, meta -from pydantic import TypeAdapter - -from litellm.secret_managers.aws_secret_manager_v2 import AWSSecretsManagerV2 -from litellm.secret_managers.main import get_secret - -pytestmark = [ - pytest.mark.e2e, - pytest.mark.secret_manager, - pytest.mark.skipif( - os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip() != "aws", - reason=f"runs only on the AWS Secrets Manager lane ({SECRET_MANAGER_OPT_IN_ENV}=aws)", - ), -] - -_AWS_FIXTURE_MASTER_KEY_SHA256: Final = "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b" -_STRING_MAP: Final = TypeAdapter(dict[str, str]) -_SECRET_STRING: Final = TypeAdapter(str) - - -class TestAwsSecretManager: - @meta(Subject(domain=Domain.DEPLOY_OPS)) - def test_aws_secret_manager(self) -> None: - AWSSecretsManagerV2.load_aws_secret_manager(use_aws_secret_manager=True) - secret_value: Final = _STRING_MAP.validate_json( - _SECRET_STRING.validate_python(get_secret("litellm_master_key")) - ) - assert hashlib.sha256(secret_value["litellm_master_key"].encode()).hexdigest() == _AWS_FIXTURE_MASTER_KEY_SHA256 diff --git a/tests/e2e/secret_manager/test_secret_manager_e2e.py b/tests/e2e/secret_manager/test_secret_manager_e2e.py deleted file mode 100644 index 8eaabcaae6f..00000000000 --- a/tests/e2e/secret_manager/test_secret_manager_e2e.py +++ /dev/null @@ -1,155 +0,0 @@ -from __future__ import annotations - -import os -import time -from collections.abc import Callable -from typing import Final - -import pytest - -from e2e_config import unique_marker -from e2e_http import Result, Success, UnauthorizedError, unwrap -from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody, LiteLLMParamsBody -from proxy_client import ProxyClient -from secret_store import SecretStore -from e2e_metadata import Domain, Mode, Provider, Subject, meta - -pytestmark = [pytest.mark.e2e, pytest.mark.secret_manager] - -BACKEND_MODEL: Final = "openai/gpt-4o-mini" -VIRTUAL_KEY_PREFIX: Final = "litellm-e2e/virtual-keys/" -PROVIDER_KEY_ENV: Final = "OPENAI_API_KEY" - - -# The proxy's env never holds OPENAI_API_KEY and each test seeds it under a fresh name, so a passing -# call proves the key came from the manager and not get_secret's os.environ fallback. -def _provider_key() -> str: - key: Final = os.environ.get(PROVIDER_KEY_ENV, "").strip() - if not key: - pytest.fail(f"The secret manager suite seeds the manager with the runner's {PROVIDER_KEY_ENV}, which is unset") - return key - - -def _seed(store: SecretStore, resources: ResourceManager, value: str) -> str: - name: Final = f"litellm-e2e-openai-{unique_marker()}" - store.write(name, value) - resources.defer(lambda: store.destroy(name)) - return name - - -def _deploy(proxy: ProxyClient, resources: ResourceManager, secret_name: str) -> str: - model_name: Final = f"secret-manager-backed-{unique_marker()}" - model_id: Final = proxy.create_model( - model_name, - LiteLLMParamsBody(model=BACKEND_MODEL, api_key=f"os.environ/{secret_name}"), - provider_live=True, - ) - resources.defer(lambda: proxy.delete_model(model_id)) - return model_name - - -def _chat(proxy: ProxyClient, key: str, model: str) -> Result[ChatResponse]: - return proxy.chat( - key, - ChatBody( - model=model, - messages=[ChatMessage(role="user", content=f"reply with one word {unique_marker()}")], - max_tokens=16, - ), - ) - - -def _eventually(proxy: ProxyClient, read: Callable[[], str | None], expected: str | None, context: str) -> None: - deadline: Final = time.monotonic() + proxy.poll_timeout - last: str | None = read() - while last != expected and time.monotonic() < deadline: - time.sleep(proxy.poll_interval) - last = read() - if last != expected: - pytest.fail( - f"{context}: the secret manager still holds {'a value' if last is not None else 'nothing'} after the deadline" - ) - - -class TestSecretManager: - @pytest.mark.covers("other.config.secret_resolution.kms_integration") - @meta( - Subject( - domain=Domain.DEPLOY_OPS, - providers=(Provider.OPENAI,), - models=(BACKEND_MODEL,), - mode=Mode.NONSTREAM, - ) - ) - def test_deployment_key_resolves_from_the_manager( - self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore, scoped_key: str - ) -> None: - model: Final = _deploy(proxy, resources, _seed(store, resources, _provider_key())) - - response: Final = unwrap(_chat(proxy, scoped_key, model)) - - assert response.choices, f"the manager-backed deployment answered with no choices: {response}" - - @pytest.mark.covers("other.config.secret_resolution.manager_value_used") - @meta( - Subject( - domain=Domain.DEPLOY_OPS, - providers=(Provider.OPENAI,), - models=(BACKEND_MODEL,), - mode=Mode.NONSTREAM, - ) - ) - def test_deployment_uses_the_value_the_manager_holds( - self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore, scoped_key: str - ) -> None: - bogus: Final = f"sk-litellm-e2e-not-a-key-{unique_marker()}" - model: Final = _deploy(proxy, resources, _seed(store, resources, bogus)) - - result: Final = _chat(proxy, scoped_key, model) - - match result: - case UnauthorizedError(body=body): - assert "AuthenticationError" in body, f"the 401 did not come from the provider: {body[:300]}" - case Success(): - pytest.fail("a deployment whose managed secret is not a real key still reached the provider") - case _: - pytest.fail(f"expected the provider to reject the manager-held key with 401, got {result}") - - @pytest.mark.covers("other.config.secret_manager.virtual_key_stored") - @meta( - Subject( - domain=Domain.DEPLOY_OPS, - ) - ) - def test_generated_key_is_written_to_the_manager( - self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore - ) -> None: - alias: Final = f"litellm-e2e-vk-{unique_marker()}" - secret_name: Final = f"{VIRTUAL_KEY_PREFIX}{alias}" - resources.defer(lambda: store.destroy(secret_name)) - key: Final = proxy.generate_key(KeyGenerateBody(key_alias=alias)) - resources.defer(lambda: proxy.delete_key(key)) - - _eventually(proxy, lambda: store.read(secret_name), key, f"the generated key {alias}") - - @pytest.mark.requires_capability("deletes_stored_keys") - @pytest.mark.covers("other.config.secret_manager.virtual_key_deleted") - @meta( - Subject( - domain=Domain.DEPLOY_OPS, - ) - ) - def test_deleted_key_is_removed_from_the_manager( - self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore - ) -> None: - alias: Final = f"litellm-e2e-vk-{unique_marker()}" - secret_name: Final = f"{VIRTUAL_KEY_PREFIX}{alias}" - resources.defer(lambda: store.destroy(secret_name)) - key: Final = proxy.generate_key(KeyGenerateBody(key_alias=alias)) - resources.defer(lambda: proxy.delete_key(key)) - _eventually(proxy, lambda: store.read(secret_name), key, f"the generated key {alias}") - - proxy.delete_key(key) - - _eventually(proxy, lambda: store.read(secret_name), None, f"the deleted key {alias}") From d5a1602968e36b4517f836ce8404eee4adc9243e Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 11 Nov 2026 15:10:32 -0800 Subject: [PATCH 121/189] fix(bedrock): forward the compact-2026-09-04 compaction param on Invoke /v1/messages (#51223) * fix(bedrock): forward the compact-2026-09-04 `compaction` param on Invoke /v1/messages The compact-2026-09-04 beta is a top-level `compaction` request parameter ({"type": "summarize"}), not a context_management edit. The Bedrock Invoke allowlist is built from BedrockInvokeAnthropicMessagesRequest, which had no `compaction` field, so the field was dropped and the request reached Bedrock as an ordinary turn. The bedrock beta map also dropped compact-2026-09-04, so adopting requests lost the beta. - add `compaction` to BedrockInvokeAnthropicMessagesRequest - add compact-2026-09-04 to the bedrock beta map (Invoke only, not Converse) - add the beta automatically when a `compaction` param is present Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-compaction.html Co-Authored-By: Claude Opus 5.5 * fix(bedrock): send `compaction` only with its beta, and add the beta for an adopted block Follow-ups to forwarding the compact-2026-09-04 `compaction` parameter on Bedrock Invoke: - `compaction` stays in the body only when it is an object and compact-2026-09-04 survives the route's beta filter. bedrock_mantle has no entry for that beta, so a mantle request keeps the earlier strip instead of reaching Bedrock with the field and no beta (400 "Extra inputs are not permitted"). A null or non-object value is dropped the same way. - A request that adopts the signed compaction block now gets the beta even without a client anthropic-beta header, as the Anthropic route does (carries_signed_compaction_block). - The compaction tests pin the bundled beta config (local_beta_headers_config) so they do not read the remote copy an earlier test left cached. Co-Authored-By: Claude Opus 5.5 * fix(bedrock): type the compaction helper and field as read-only views The type discipline gate rejects a bare dict or set annotation and a TypedDict field without ReadOnly; the helper only reads the request and grows the caller's beta set in place. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): keep the compaction helper within the basedpyright ceiling Read the request's messages through a typed adapter and delete the dropped compaction key, so the new call adds no unknown member or argument diagnostics. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(bedrock): drop comments that restate the compaction checks and type the new test params Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Homen Shum <317650355+homenshum-cheiron@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/anthropic_beta_headers_config.json | 1 + .../anthropic_claude_transformation.py | 24 ++++ litellm/types/llms/bedrock.py | 7 ++ .../test_anthropic_claude_transformation.py | 106 ++++++++++++++++++ tests/unit/llms/bedrock/test_mantle.py | 23 ++++ .../test_anthropic_beta_headers_filtering.py | 4 +- 6 files changed, 164 insertions(+), 1 deletion(-) diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 1b659770181..a1359d98c4a 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -201,6 +201,7 @@ "bash_20250124": null, "code-execution-2025-08-25": null, "compact-2026-01-12": "compact-2026-01-12", + "compact-2026-09-04": "compact-2026-09-04", "computer-use-2025-01-24": "computer-use-2025-01-24", "computer-use-2025-11-24": "computer-use-2025-11-24", "context-1m-2025-08-07": "context-1m-2025-08-07", diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index cf48fcf5c2f..814da685b8c 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -23,6 +23,7 @@ from litellm.llms.anthropic.common_utils import ( OBJECT_MAPPING_ADAPTER, AnthropicModelInfo, carried_thinking_keys, + carries_signed_compaction_block, validated_object_mapping, ) from litellm.llms.anthropic.messages.native.transformation import ( @@ -82,6 +83,7 @@ else: _MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +_MESSAGE_LIST: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) class AmazonAnthropicClaudeMessagesConfig( @@ -571,6 +573,19 @@ class AmazonAnthropicClaudeMessagesConfig( "edits": retained_edits, } + @staticmethod + def _add_compaction_beta_for_bedrock_invoke( + anthropic_messages_request: Mapping[str, object], + beta_set: set[str], # mutable-ok: the caller's beta set is grown in place, like the sibling helpers + ) -> None: + """``compaction`` and an adopted signed block both need the compact-2026-09-04 beta on Bedrock. + + Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-compaction.html + """ + messages: Final = _MESSAGE_LIST.validate_python(anthropic_messages_request.get("messages") or ()) + if isinstance(anthropic_messages_request.get("compaction"), dict) or carries_signed_compaction_block(messages): + beta_set.add("compact-2026-09-04") + def strips_beta_gated_fields_from_final_headers(self) -> bool: return False @@ -639,6 +654,7 @@ class AmazonAnthropicClaudeMessagesConfig( anthropic_messages_request=anthropic_messages_request, beta_set=beta_set, ) + self._add_compaction_beta_for_bedrock_invoke(anthropic_messages_request, beta_set) # pyright: ignore[reportUnknownArgumentType] # built from the base signature's bare dict self._get_tool_search_beta_header_for_bedrock( model=model, @@ -858,6 +874,14 @@ class AmazonAnthropicClaudeMessagesConfig( if filtered_betas: anthropic_messages_request["anthropic_beta"] = filtered_betas + if "compaction" in anthropic_messages_request and not ( + isinstance(anthropic_messages_request["compaction"], dict) and "compact-2026-09-04" in filtered_betas + ): + verbose_logger.debug( + "Bedrock Invoke: dropping `compaction`; compact-2026-09-04 is not served on this route" + ) + del anthropic_messages_request["compaction"] + final_request: Final = OBJECT_MAPPING_ADAPTER.validate_python(anthropic_messages_request) # pyright: ignore[reportUnknownArgumentType] # built from the base signature's bare dict gated: Final = strip_beta_gated_native_request( {}, diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index ff4f2c4fd1e..e42a1a8f057 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -1257,6 +1257,13 @@ class BedrockInvokeAnthropicMessagesRequest(TypedDict, total=False): # other edit types (e.g. `clear_thinking_20251015`) never reach Bedrock. context_management: dict + # `compaction` ({"type": "summarize", "instructions"?: str}) is the + # `compact-2026-09-04` beta: one summarization call that returns a single + # signed compaction block. Bedrock rejects it together with + # `context_management`. Ref: + # https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-compaction.html + compaction: ReadOnly[Mapping[str, object]] + class BedrockBatchRecordKind(Enum): """ diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index c4614dd636f..991a0c36f03 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -1547,6 +1547,112 @@ def test_bedrock_messages_preserves_compact_context_management_and_adds_beta(): assert result["max_tokens"] == 4096 +@pytest.mark.usefixtures("local_beta_headers_config") +def test_bedrock_messages_keeps_compaction_param_and_adds_beta(): + """ + The ``compact-2026-09-04`` beta uses a top-level ``compaction`` parameter, not a + ``context_management`` edit. Bedrock InvokeModel accepts it with that beta and + returns one signed compaction block. Before this fix the top-level allowlist + dropped ``compaction``, so the request reached Bedrock as an ordinary turn. + + Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-compaction.html + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": [{"type": "text", "text": "Is CHG-TR-0042 Type IB in TR-TITCK?"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Type IB."}]}, + {"role": "user", "content": [{"type": "text", "text": "Now KR-MFDS."}]}, + ] + optional_params = {"max_tokens": 1024, "compaction": {"type": "summarize", "instructions": "Keep rule ids."}} + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-5", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["compaction"] == {"type": "summarize", "instructions": "Keep rule ids."} + assert "compact-2026-09-04" in result["anthropic_beta"] + assert "context_management" not in result + assert set(result).issubset(cfg.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS) + + +@pytest.mark.usefixtures("local_beta_headers_config") +def test_bedrock_messages_keeps_compact_2026_09_04_beta_for_adopting_request(): + """ + A request that adopts the signed block sends it first in ``messages`` and must + carry the ``compact-2026-09-04`` beta again. The bedrock beta map has to keep it. + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + block = {"type": "compaction", "content": "TR-TITCK: Type IB.", "signature": "sig"} + messages = [{"role": "user", "content": [block, {"type": "text", "text": "Now KR-MFDS."}]}] + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-5", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 512}, + litellm_params=GenericLiteLLMParams(), + headers={"anthropic-beta": "compact-2026-09-04"}, + ) + + assert "compact-2026-09-04" in result["anthropic_beta"] + assert result["messages"][0]["content"][0] == block + + +@pytest.mark.usefixtures("local_beta_headers_config") +def test_bedrock_messages_adds_compact_2026_09_04_beta_for_signed_block_without_client_header(): + """ + A client that adopts the signed block but sends no anthropic-beta header still + needs ``compact-2026-09-04``; the Anthropic route adds it for the block, and so + does Bedrock Invoke. + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + block = {"type": "compaction", "content": "TR-TITCK: Type IB.", "signature": "sig"} + messages = [{"role": "user", "content": [block, {"type": "text", "text": "Now KR-MFDS."}]}] + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-5", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 512}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["anthropic_beta"] == ["compact-2026-09-04"] + assert result["messages"][0]["content"][0] == block + + +@pytest.mark.usefixtures("local_beta_headers_config") +@pytest.mark.parametrize("compaction", [None, "summarize", []]) +def test_bedrock_messages_drops_compaction_that_is_not_an_object(compaction: object): + """ + Only an object ``compaction`` gets the beta. Any other value is dropped, as it + was before the field was allowed, instead of reaching Bedrock without its beta. + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-5", + messages=[{"role": "user", "content": [{"type": "text", "text": "Hi"}]}], + anthropic_messages_optional_request_params={"max_tokens": 512, "compaction": compaction}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert "compaction" not in result + assert "compact-2026-09-04" not in result.get("anthropic_beta", []) + + def test_bedrock_messages_filters_unsupported_context_management_edits(): """ Mixed edit lists must drop the LiteLLM-internal ``clear_thinking_20251015`` diff --git a/tests/unit/llms/bedrock/test_mantle.py b/tests/unit/llms/bedrock/test_mantle.py index 63af0105f5b..908e0bbbced 100644 --- a/tests/unit/llms/bedrock/test_mantle.py +++ b/tests/unit/llms/bedrock/test_mantle.py @@ -796,3 +796,26 @@ async def test_mantle_anthropic_messages_streaming_sends_stream_and_passes_throu assert "event: message_start" in text assert '"text": "pong"' in text assert "event: message_stop" in text + + +@pytest.mark.usefixtures("local_beta_headers_config") +def test_mantle_messages_drops_compaction_without_its_beta(): + """ + The bedrock_mantle beta map has no compact-2026-09-04, so the beta is filtered + out on this route. ``compaction`` must go with it, or the body carries a field + that Bedrock refuses without the beta. + """ + from litellm.types.router import GenericLiteLLMParams + + config = AmazonMantleMessagesConfig() + headers: dict[str, str] = {} + request = config.transform_anthropic_messages_request( + model="mantle/anthropic.claude-mythos-preview", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={"max_tokens": 100, "compaction": {"type": "summarize"}}, + litellm_params=GenericLiteLLMParams(), + headers=headers, + ) + + assert "compaction" not in request + assert "compact-2026-09-04" not in headers.get("anthropic-beta", "") diff --git a/tests/unit/test_anthropic_beta_headers_filtering.py b/tests/unit/test_anthropic_beta_headers_filtering.py index a9bc741f800..d0fb6fe0f6e 100644 --- a/tests/unit/test_anthropic_beta_headers_filtering.py +++ b/tests/unit/test_anthropic_beta_headers_filtering.py @@ -99,7 +99,9 @@ class TestAnthropicBetaHeadersFiltering: filtered = filter_and_transform_beta_headers( beta_headers=all_headers, provider=provider ) - assert ("compact-2026-09-04" in filtered) is (provider in ("anthropic", "azure_ai", "vertex_ai")) + # Bedrock InvokeModel serves the compact-2026-09-04 `compaction` parameter; Converse does not. + # https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-compaction.html + assert ("compact-2026-09-04" in filtered) is (provider in ("anthropic", "azure_ai", "vertex_ai", "bedrock")) assert ("mid-conversation-tool-changes-2026-07-01" in filtered) is (provider != "bedrock_converse") for header in unsupported_headers: From 2c7ed0713a59c6076602c2359cd7d35e0bf4e002 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 11 Nov 2026 15:26:23 -0800 Subject: [PATCH 122/189] fix(fallback): preserve concurrent writes and serve saved configuration (#51245) * fix(fallback): serialize fallback writes and wait for replicas on read-back POST and DELETE /fallback now read, edit and write router_settings in one writer transaction under the same per-param advisory lock the config section writer takes, so concurrent writes on any replica no longer drop each other's rules. The cache is invalidated and the router updated only after commit. The Terraform provider now polls the read after a litellm_fallback create or update until the proxy serves the written list, within new create/update timeouts, instead of clearing the ID on a stale 404 or saving a stale list. Refresh 404 still removes the resource, and error responses are not retried. Fixes #47888 * fix(fallback): bound provider requests by the apply context and inject the fallback repository The provider's fallback write and read-back requests now carry the apply context through a context-aware MakeRequest variant, so a cancelled apply sends nothing and the create/update timeout also ends a hung request. The POST and DELETE /fallback routes take a RouterFallbackRepositoryDep that returns a RouterFallbackStore. Router settings parsing moved into RouterFallbackRepository, which edits the rules through the locked ConfigRepository operation. Unit tests now check the outcome of the race and rollback through a fake lock instead of pinning the lock SQL. They also cover a racing config section write, a 404 delete inside the lock, route resolution of the repository and the no-database path. * refactor(fallback): keep atomic edits in the config repository * fix(terraform): release hung test handlers and audit MakeRequestWithContext calls The hung-request cancellation test now releases its handler through a test owned channel before closing the server, since the server does not notice the client's cancel while the POST body is unread. The endpoint audit now reads the method and path of MakeRequestWithContext calls from their shifted argument positions, and skips only MakeRequest's own forwarding call inside the helper files, so POST /fallback counts as covered again and dynamic paths elsewhere still fail closed. * test(fallback): keep stored-rule regressions with their helpers * fix(fallback): read saved rules without Terraform changes * fix(fallback): preserve valid rules when stored entries are malformed --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/models/fallback.py | 25 + litellm/proxy/deps/repositories.py | 9 + .../fallback_management_endpoints.py | 210 +++--- litellm/proxy/proxy_server.py | 5 +- litellm/repositories/config_repository.py | 74 ++- .../test_fallback_management_public_names.py | 64 ++ .../test_fallback_management_endpoints.py | 605 +++++++++++++++--- 7 files changed, 782 insertions(+), 210 deletions(-) create mode 100644 litellm/models/fallback.py diff --git a/litellm/models/fallback.py b/litellm/models/fallback.py new file mode 100644 index 00000000000..b8532a7f3cb --- /dev/null +++ b/litellm/models/fallback.py @@ -0,0 +1,25 @@ +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Annotated, Final, Protocol, TypeAlias + +from pydantic import Field, JsonValue, TypeAdapter + +FallbackRule = dict[str, list[str]] +StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_mode="left_to_right")] +STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback]) +SavedFallbackRule: TypeAlias = JsonValue + + +@dataclass(frozen=True, slots=True) +class FallbackRulesEdit: + rules: Sequence[StoredFallback] + replaced: bool + + +FallbackRulesEditor: TypeAlias = Callable[[Sequence[StoredFallback]], FallbackRulesEdit] + + +class RouterFallbackStore(Protocol): + async def saved_rules(self, fallback_key: str) -> Sequence[SavedFallbackRule] | None: ... + + async def edit_rules(self, fallback_key: str, edit: FallbackRulesEditor) -> FallbackRulesEdit: ... diff --git a/litellm/proxy/deps/repositories.py b/litellm/proxy/deps/repositories.py index 323cf57dfec..503314640c2 100644 --- a/litellm/proxy/deps/repositories.py +++ b/litellm/proxy/deps/repositories.py @@ -3,11 +3,13 @@ from typing import Annotated, TypeAlias from fastapi import Depends from litellm.models.deployment_warning import DeploymentWarningIgnoreStore +from litellm.models.fallback import RouterFallbackStore from litellm.models.gateway_activity import GatewayActivityStore from litellm.models.plugin_upload import PluginUploadStore from litellm.models.skill_submission import SkillSubmissionStore from litellm.models.user_banner import UserBannerStore from litellm.proxy.deps.database import DatabaseDep +from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository from litellm.repositories.gateway_activity_repository import GatewayActivityRepository from litellm.repositories.plugin_upload_repository import PluginUploadRepository @@ -54,3 +56,10 @@ async def get_skill_submission_repository(database: DatabaseDep) -> SkillSubmiss SkillSubmissionRepositoryDep: TypeAlias = Annotated[ SkillSubmissionStore | None, Depends(get_skill_submission_repository) ] + + +async def get_router_fallback_repository(database: DatabaseDep) -> RouterFallbackStore | None: + return None if database is None else ConfigRepository(database.prisma, use_writer=True) + + +RouterFallbackRepositoryDep: TypeAlias = Annotated[RouterFallbackStore | None, Depends(get_router_fallback_repository)] diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 477cf44073f..f736ac1306a 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -10,11 +10,11 @@ DELETE /fallback/{model} - Delete fallbacks for a specific model # pyright: reportMissingImports=false -import json from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Annotated, Final, Literal +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal -from pydantic import Field, JsonValue, TypeAdapter, ValidationError +from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -22,7 +22,7 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_utils import fallback_target_model_name from litellm.proxy.auth.model_checks import get_all_fallbacks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.utils import PrismaClient, invalidate_config_param +from litellm.proxy.utils import invalidate_config_param if TYPE_CHECKING: from fastapi import APIRouter, Depends, HTTPException, status @@ -36,8 +36,16 @@ else: # fastapi is only required for proxy, not for SDK usage pass -from litellm.repositories.config_repository import ConfigRepository -from litellm.router_api import ChangeSet +from litellm.models.fallback import ( + STORED_FALLBACKS, + FallbackRulesEdit, + FallbackRulesEditor, + RouterFallbackStore, + StoredFallback, +) +from litellm.proxy.deps.repositories import RouterFallbackRepositoryDep +from litellm.repositories.config_repository import ROUTER_SETTINGS_PARAM +from litellm.router_api import ChangeSet, fallback_chain from litellm.types.management_endpoints.router_settings_endpoints import ( FallbackCreateRequest, FallbackDeleteResponse, @@ -48,12 +56,10 @@ from litellm.types.router import ContentPolicyFallbackRule, ContentPolicyFallbac router: Final = APIRouter() -ROUTER_SETTINGS_PARAM: Final = "router_settings" -FallbackRule = dict[str, list[str]] -StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_mode="left_to_right")] -_STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback]) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) -_STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue]) +_FALLBACK_KEY_BY_TYPE: Final[Mapping[str, str]] = MappingProxyType( + {"general": "fallbacks", "context_window": "context_window_fallbacks", "content_policy": "content_policy_fallbacks"} +) _OPTIONAL_JSON_RULES: Final[TypeAdapter[Sequence[JsonValue] | None]] = TypeAdapter(Sequence[JsonValue] | None) _CONTENT_POLICY_FALLBACK_RULE: Final[TypeAdapter[ContentPolicyFallbackRule]] = TypeAdapter(ContentPolicyFallbackRule) _FALLBACK_TARGETS: Final[TypeAdapter[FallbackTargets]] = TypeAdapter(FallbackTargets) @@ -104,18 +110,20 @@ def _checked_content_policy_rules( return tuple((candidate, _checked_content_policy_rule(candidate)) for candidate in candidates) +def _layered_checked_rules( + checked_rules: Sequence[tuple[object, ContentPolicyFallbackRule | ValidationError]], +) -> Sequence[Mapping[str, object]]: + return _layered_over_config_rules( + tuple(result for _, result in checked_rules if not isinstance(result, ValidationError)), + _config_file_rules( + litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list + ), + ) + + def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None: checked_rules: Final = _checked_content_policy_rules(rules) - runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError)) - layered_rules: Final = ( - _layered_over_config_rules( - runtime_rules, - _config_file_rules( - litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list - ), - ) - or None - ) + layered_rules: Final = _layered_checked_rules(checked_rules) or None for rule, result in checked_rules: if log_skipped_rules and isinstance(result, ValidationError): _log_ignored_content_policy_rules(rule, result) @@ -135,54 +143,52 @@ async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None: await proxy_config.get_config() -async def _stored_router_settings(prisma_client: PrismaClient) -> Mapping[str, JsonValue]: - try: - row: Final = await ConfigRepository(prisma_client, use_writer=True).get_param(ROUTER_SETTINGS_PARAM) - except json.JSONDecodeError: - return {} - if row is None or not isinstance(row.param_value, Mapping): - return {} - return _STORED_ROUTER_SETTINGS.validate_python(row.param_value) +def _with_rule(rules: Sequence[StoredFallback], model: str, fallback_models: Sequence[str]) -> FallbackRulesEdit: + covering: Final = next((index for index, rule in enumerate(rules) if _rule_covers(rule, model)), None) + rule: Final[StoredFallback] = {model: list(fallback_models)} + if covering is None: + return FallbackRulesEdit(rules=[*rules, rule], replaced=False) + return FallbackRulesEdit(rules=[*rules[:covering], rule, *rules[covering + 1 :]], replaced=True) -def _editable_rules( - proxy_config: "ProxyConfig", stored_settings: Mapping[str, JsonValue], fallback_key: str -) -> list[StoredFallback]: - file_settings: Final = proxy_config.router_settings - if not file_settings.owned_by_config(fallback_key): - return _STORED_FALLBACKS.validate_python(stored_settings.get(fallback_key) or []) - file_rules: Final = file_settings.get(fallback_key) - return _STORED_FALLBACKS.validate_python(file_rules if isinstance(file_rules, list) else []) +def _without_rule(rules: Sequence[StoredFallback], model: str, fallback_type: str) -> FallbackRulesEdit: + kept: Final = [rule for rule in rules if not _rule_covers(rule, model)] + if len(kept) == len(rules): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"}, + ) + return FallbackRulesEdit(rules=kept, replaced=True) + + +def _edit_config_file_rules( + proxy_config: "ProxyConfig", + llm_router: "Router", + fallback_key: str, + edit: FallbackRulesEditor, +) -> FallbackRulesEdit: + file_rules: Final = proxy_config.router_settings.get(fallback_key) + edited: Final = edit(STORED_FALLBACKS.validate_python(file_rules if isinstance(file_rules, list) else [])) + proxy_config.reject_config_owned_writes( + section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(edited.rules)} + ) + llm_router.apply(ChangeSet(assign={fallback_key: edited.rules})) + return edited async def _write_rules( - prisma_client: PrismaClient, + fallbacks: RouterFallbackStore, proxy_config: "ProxyConfig", llm_router: "Router", - stored_settings: Mapping[str, JsonValue], fallback_key: str, - rules: Sequence[StoredFallback], -) -> None: - proxy_config.reject_config_owned_writes( - section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)} - ) + edit: FallbackRulesEditor, +) -> FallbackRulesEdit: if proxy_config.router_settings.owned_by_config(fallback_key): - llm_router.apply(ChangeSet(assign={fallback_key: rules})) - return - await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules}) - _apply_fallback_rules(llm_router, fallback_key, rules) - - -async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None: - router_settings_json: Final = json.dumps(router_settings) - await ConfigRepository(prisma_client).table.upsert( - where={"param_name": ROUTER_SETTINGS_PARAM}, - data={ - "create": {"param_name": ROUTER_SETTINGS_PARAM, "param_value": router_settings_json}, - "update": {"param_value": router_settings_json}, - }, - ) + return _edit_config_file_rules(proxy_config, llm_router, fallback_key, edit) + edited: Final = await fallbacks.edit_rules(fallback_key, edit) await invalidate_config_param(ROUTER_SETTINGS_PARAM) + _apply_fallback_rules(llm_router, fallback_key, edited.rules) + return edited @router.post( @@ -195,6 +201,8 @@ async def _persist_router_settings(prisma_client: PrismaClient, router_settings: async def create_fallback( data: FallbackCreateRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + *, + fallbacks: RouterFallbackRepositoryDep, ): """ Create or update fallbacks for a specific model. @@ -218,7 +226,6 @@ async def create_fallback( """ from litellm.proxy.proxy_server import ( llm_router, - prisma_client, proxy_config, store_model_in_db, ) @@ -262,7 +269,7 @@ async def create_fallback( ) # Check if we need to store in DB - if store_model_in_db is not True or prisma_client is None: + if store_model_in_db is not True or fallbacks is None: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail={ @@ -271,7 +278,6 @@ async def create_fallback( ) await _reload_config_file_ownership(proxy_config) - router_settings: Final = await _stored_router_settings(prisma_client) # Get the appropriate fallback list based on type fallback_key = "fallbacks" @@ -280,23 +286,13 @@ async def create_fallback( elif data.fallback_type == "content_policy": fallback_key = "content_policy_fallbacks" - # Get existing fallbacks - existing_fallbacks: Final = _editable_rules(proxy_config, router_settings, fallback_key) - - # Update or add the fallback configuration - fallback_updated = False - for i, rule in enumerate(existing_fallbacks): - if _rule_covers(rule, data.model): - # Update existing fallback - existing_fallbacks[i] = {data.model: data.fallback_models} - fallback_updated = True - break - - if not fallback_updated: - # Add new fallback - existing_fallbacks.append({data.model: data.fallback_models}) - - await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, existing_fallbacks) + edited: Final = await _write_rules( + fallbacks, + proxy_config, + llm_router, + fallback_key, + lambda rules: _with_rule(rules, data.model, data.fallback_models), + ) verbose_proxy_logger.info( "Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type @@ -306,7 +302,7 @@ async def create_fallback( model=data.model, fallback_models=data.fallback_models, fallback_type=data.fallback_type, - message=f"Fallback configuration {'updated' if fallback_updated else 'created'} successfully", + message=f"Fallback configuration {'updated' if edited.replaced else 'created'} successfully", ) except HTTPException: @@ -319,6 +315,26 @@ async def create_fallback( ) +async def _saved_rules_in_effect( + proxy_config: "ProxyConfig", fallbacks: RouterFallbackStore, fallback_key: str +) -> Sequence[object]: + if proxy_config.router_settings.owned_by_config(fallback_key): + file_rules: Final = proxy_config.router_settings.get(fallback_key) + return file_rules if isinstance(file_rules, list) else () + saved: Final = await fallbacks.saved_rules(fallback_key) + if fallback_key == "content_policy_fallbacks": + return _layered_checked_rules(_checked_content_policy_rules(saved)) + return saved or () + + +def _fallback_chain_for(rules: Sequence[object], model: str) -> object: + chain, _ = fallback_chain( + fallbacks=rules, # pyright: ignore[reportArgumentType] # the chain also resolves the string and rich-target rules the router accepts + model_group=model, + ) + return chain or [] + + @router.get( "/fallback/{model}", tags=["Fallback Management"], @@ -329,6 +345,8 @@ async def get_fallback( model: str, fallback_type: Literal["general", "context_window", "content_policy"] = "general", user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + *, + fallbacks: RouterFallbackRepositoryDep, ): """ Get fallback configuration for a specific model. @@ -342,7 +360,7 @@ async def get_fallback( GET /fallback/gpt-3.5-turbo?fallback_type=general ``` """ - from litellm.proxy.proxy_server import llm_router + from litellm.proxy.proxy_server import llm_router, proxy_config, store_model_in_db try: if llm_router is None: @@ -351,9 +369,12 @@ async def get_fallback( detail={"error": "Router not initialized"}, ) - # Get fallbacks using the existing utility function fallback_targets: Final = _FALLBACK_TARGETS.validate_python( get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type) + if store_model_in_db is not True or fallbacks is None + else _fallback_chain_for( + await _saved_rules_in_effect(proxy_config, fallbacks, _FALLBACK_KEY_BY_TYPE[fallback_type]), model + ) ) fallback_models: Final = [fallback_target_model_name(target) or model for target in fallback_targets] @@ -389,6 +410,8 @@ async def delete_fallback( model: str, fallback_type: Literal["general", "context_window", "content_policy"] = "general", user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + *, + fallbacks: RouterFallbackRepositoryDep, ): """ Delete fallback configuration for a specific model. @@ -404,7 +427,6 @@ async def delete_fallback( """ from litellm.proxy.proxy_server import ( llm_router, - prisma_client, proxy_config, store_model_in_db, ) @@ -416,7 +438,7 @@ async def delete_fallback( detail={"error": "Router not initialized"}, ) - if store_model_in_db is not True or prisma_client is None: + if store_model_in_db is not True or fallbacks is None: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail={ @@ -425,7 +447,6 @@ async def delete_fallback( ) await _reload_config_file_ownership(proxy_config) - router_settings: Final = await _stored_router_settings(prisma_client) # Get the appropriate fallback list based on type fallback_key = "fallbacks" @@ -434,18 +455,13 @@ async def delete_fallback( elif fallback_type == "content_policy": fallback_key = "content_policy_fallbacks" - # Get existing fallbacks - existing_fallbacks: Final = _editable_rules(proxy_config, router_settings, fallback_key) - - updated_fallbacks: Final = [rule for rule in existing_fallbacks if not _rule_covers(rule, model)] - - if len(updated_fallbacks) == len(existing_fallbacks): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"}, - ) - - await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, updated_fallbacks) + await _write_rules( + fallbacks, + proxy_config, + llm_router, + fallback_key, + lambda rules: _without_rule(rules, model, fallback_type), + ) verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index fc4d4e65660..d8320667a3a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5748,7 +5748,6 @@ def should_load_db_object(object_type: str | SupportedDBObjectType) -> bool: _CONFIG_PERSISTED_SECTIONS: Final = ("general_settings", "router_settings", "litellm_settings") _CONFIG_UNMANAGED_EXCLUSIONS: Final = frozenset(("environment_variables", "model_list")) _CONFIG_SECTION_VALUES: Final = TypeAdapter(Mapping[str, JsonValue]) -_CONFIG_SECTION_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock(hashtext($1))" class _ConfigParamWhere(TypedDict): @@ -6301,7 +6300,7 @@ class ProxyConfig: prisma_client: PrismaClient, ) -> Mapping[str, JsonValue] | None: async with prisma_client.tx() as tx: - await tx.query_raw(_CONFIG_SECTION_LOCK_SQL, section_name) + await tx.query_raw(CONFIG_PARAM_LOCK_SQL, section_name) config_table: Final = cast("TableActions[_ConfigParamRow]", tx.litellm_config) config_where: Final[_ConfigParamWhere] = {"param_name": section_name} existing_row: Final[_ConfigParamRow | None] = await config_table.find_first(where=config_where) @@ -15287,7 +15286,7 @@ from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCoun from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient -from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.config_repository import CONFIG_PARAM_LOCK_SQL, ConfigRepository from litellm.repositories.model_repository import ModelRepository from litellm.repositories.table_repositories import ( AccessGroupRepository, diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index 36c6ab7f711..e9973e1f674 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -4,9 +4,14 @@ from __future__ import annotations import json from collections.abc import Mapping, Sequence +from contextlib import AbstractAsyncContextManager from datetime import datetime from typing import TYPE_CHECKING, Final, Protocol, cast +from pydantic import JsonValue, TypeAdapter, ValidationError + +from litellm.models.fallback import STORED_FALLBACKS, FallbackRulesEdit, FallbackRulesEditor, SavedFallbackRule + if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -39,10 +44,42 @@ class _ConfigTable(Protocol): async def delete_many(self, *, where: Mapping[str, object]) -> int: ... -class _ConfigDatabase(Protocol): +class _ConfigTransaction(Protocol): + @property + def litellm_config(self) -> _ConfigTable: ... + async def query_raw(self, query: str, *args: object) -> object: ... +class _ConfigWriter(_ConfigTransaction, Protocol): + def tx(self) -> AbstractAsyncContextManager[_ConfigTransaction]: ... + + +CONFIG_PARAM_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock(hashtext($1))" +ROUTER_SETTINGS_PARAM: Final = "router_settings" +_JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_JSON_RULES: Final = TypeAdapter(list[JsonValue]) +_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue]) + + +def _stored_router_settings(param_value: object) -> Mapping[str, JsonValue]: + try: + decoded: Final[object] = _JSON_VALUE.validate_json(param_value) if isinstance(param_value, str) else param_value + except ValidationError: + return {} + if not isinstance(decoded, Mapping): + return {} + return _ROUTER_SETTINGS.validate_python(decoded) + + +def _edited_router_settings( + param_value: object, fallback_key: str, edit: FallbackRulesEditor +) -> tuple[Mapping[str, JsonValue], FallbackRulesEdit]: + router_settings: Final = _stored_router_settings(param_value) + edited: Final = edit(STORED_FALLBACKS.validate_python(router_settings.get(fallback_key) or [])) + return {**router_settings, fallback_key: _JSON_RULES.validate_python(edited.rules)}, edited + + class ConfigParam: """Simple wrapper for config parameter from DB.""" @@ -73,6 +110,12 @@ class ConfigRepository: def table(self) -> _ConfigTable: return self._config_table + @property + def _writer(self) -> _ConfigWriter: + return cast( # cast-ok: Prisma delegates database methods and transactions dynamically + _ConfigWriter, self.prisma_client.writer_db + ) + async def get_param(self, param_name: str) -> ConfigParam | None: """Get a config parameter from the database.""" record: Final = await self._config_table.find_unique(where={"param_name": param_name}) @@ -96,9 +139,7 @@ class ConfigRepository: return ConfigParam(param_name=param_name, param_value=param_value) async def set_param_if_revision(self, param_name: str, param_value: object, revision: int) -> bool: - delegate: Final = self.prisma_client.writer_db - database: Final = cast(_ConfigDatabase, delegate) # cast-ok: Prisma delegates database methods dynamically - rows: Final = await database.query_raw( + rows: Final = await self._writer.query_raw( """INSERT INTO "LiteLLM_Config" (param_name, param_value, last_run_at) SELECT $1, $2::jsonb, NOW() WHERE $3::int = 0 ON CONFLICT (param_name) DO UPDATE @@ -115,6 +156,31 @@ class ConfigRepository: ) return bool(rows) + async def saved_rules(self, fallback_key: str) -> Sequence[SavedFallbackRule] | None: + row: Final = await self._config_table.find_unique(where={"param_name": ROUTER_SETTINGS_PARAM}) + if row is None: + return None + rules: Final = _stored_router_settings(row.param_value).get(fallback_key) + if rules is None: + return None + return rules if isinstance(rules, list) else () + + async def edit_rules(self, fallback_key: str, edit: FallbackRulesEditor) -> FallbackRulesEdit: + param_name: Final = ROUTER_SETTINGS_PARAM + async with self._writer.tx() as transaction: + await transaction.query_raw(CONFIG_PARAM_LOCK_SQL, param_name) + row: Final = await transaction.litellm_config.find_unique(where={"param_name": param_name}) + value, result = _edited_router_settings(None if row is None else row.param_value, fallback_key, edit) + value_json: Final = json.dumps(dict(value)) + await transaction.litellm_config.upsert( + where={"param_name": param_name}, + data={ + "create": {"param_name": param_name, "param_value": value_json}, + "update": {"param_value": value_json}, + }, + ) + return result + async def delete_param(self, param_name: str) -> bool: """Delete a config parameter from the database.""" try: diff --git a/tests/integration/management/test_fallback_management_public_names.py b/tests/integration/management/test_fallback_management_public_names.py index 6f97165fb3c..56f8309d6f4 100644 --- a/tests/integration/management/test_fallback_management_public_names.py +++ b/tests/integration/management/test_fallback_management_public_names.py @@ -238,6 +238,70 @@ def test_consecutive_creates_within_the_cache_window_keep_every_rule(gateway: Ga assert frozenset(primaries) <= covered, (primaries, covered) +def _create_over_a_fresh_connection(instance: Gateway, model: str, target: str) -> httpx.Response: + with httpx.Client(base_url=instance.client.base_url, timeout=30, trust_env=False) as client: + return Gateway(client, instance.key, instance.upstream_url).request( + "POST", "/fallback", _fallback_body(model, [target]) + ) + + +def _delete_over_a_fresh_connection(instance: Gateway, model: str) -> httpx.Response: + with httpx.Client(base_url=instance.client.base_url, timeout=30, trust_env=False) as client: + return Gateway(client, instance.key, instance.upstream_url).request( + "DELETE", f"/fallback/{model}", params={"fallback_type": "general"} + ) + + +def test_concurrent_writes_through_both_instances_keep_every_rule(gateway: Gateway, peer: Gateway) -> None: + with gateway.scenario() as scenario: + target: Final = scenario.model(model=f"openai/n7t-{uuid.uuid4().hex}") + primaries: Final = tuple(scenario.model(model=f"openai/n7{tag}-{uuid.uuid4().hex}") for tag in "abcdefgh") + for instance in (gateway, peer): + for model in (target, *primaries): + _wait_until_served(instance, model) + for primary in primaries: + scenario.cleanups.callback(_forget_fallback, gateway, primary, "general") + instances: Final = tuple(gateway if index % 2 == 0 else peer for index in range(len(primaries))) + with ThreadPoolExecutor(max_workers=len(primaries)) as pool: + created: Final = tuple( + pool.map(partial(_create_over_a_fresh_connection, target=target), instances, primaries) + ) + assert all(response.status_code == 200 for response in created), [response.text for response in created] + assert frozenset(primaries) <= _covered_models(_stored_fallbacks()) + + dropped: Final = primaries[::2] + with ThreadPoolExecutor(max_workers=len(dropped)) as pool: + removed: Final = tuple(pool.map(_delete_over_a_fresh_connection, instances[1::2], dropped)) + assert all(response.status_code == 200 for response in removed), [response.text for response in removed] + covered: Final = _covered_models(_stored_fallbacks()) & frozenset(primaries) + assert covered == frozenset(primaries[1::2]), covered + + +def _read_fallback(instance: Gateway, model: str) -> httpx.Response: + return instance.request("GET", f"/fallback/{model}", params={"fallback_type": "general"}) + + +def test_the_other_instance_reads_each_write_immediately(gateway: Gateway, peer: Gateway) -> None: + with gateway.scenario() as scenario: + primary: Final = scenario.model(model=f"openai/n8p-{uuid.uuid4().hex}") + first: Final = scenario.model(model=f"openai/n8a-{uuid.uuid4().hex}") + second: Final = scenario.model(model=f"openai/n8b-{uuid.uuid4().hex}") + for model in (primary, first, second): + _wait_until_served(gateway, model) + scenario.cleanups.callback(_forget_fallback, gateway, primary, "general") + + assert gateway.request("POST", "/fallback", _fallback_body(primary, [first])).status_code == 200 + created: Final = _read_fallback(peer, primary) + assert gateway.request("POST", "/fallback", _fallback_body(primary, [second, first])).status_code == 200 + updated: Final = _read_fallback(peer, primary) + assert gateway.request("DELETE", f"/fallback/{primary}", params={"fallback_type": "general"}).status_code == 200 + deleted: Final = _read_fallback(peer, primary) + + assert created.status_code == 200 and _fallback_models(created) == [first], created.text + assert updated.status_code == 200 and _fallback_models(updated) == [second, first], updated.text + assert deleted.status_code == 404, deleted.text + + def test_unknown_model_404_lists_team_public_and_internal_names(gateway: Gateway) -> None: with gateway.scenario() as scenario: team: Final = scenario.team() diff --git a/tests/unit/proxy/test_fallback_management_endpoints.py b/tests/unit/proxy/test_fallback_management_endpoints.py index e3d0d072e0a..06adc0ed340 100644 --- a/tests/unit/proxy/test_fallback_management_endpoints.py +++ b/tests/unit/proxy/test_fallback_management_endpoints.py @@ -8,34 +8,44 @@ Tests: 4. Validation tests (invalid models, duplicate fallbacks, etc.) """ +import asyncio import copy import json -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping, Sequence +from contextlib import asynccontextmanager from types import MappingProxyType, SimpleNamespace -from typing import Final, Literal, get_args +from typing import Final, Literal, TypeAlias, get_args from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException +from fastapi.testclient import TestClient from pydantic import JsonValue, TypeAdapter from redis.exceptions import NoPermissionError import litellm from litellm import Router +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.config_resolvers.settings_rules import Section +from litellm.proxy.deps.database import get_database from litellm.proxy.management_endpoints.fallback_management_endpoints import ( FallbackCreateRequest, FallbackDeleteResponse, + FallbackGetResponse, FallbackResponse, create_fallback, delete_fallback, get_fallback, ) -from litellm.proxy.proxy_server import ProxyConfig +from litellm.proxy.proxy_server import ProxyConfig, app from litellm.proxy.utils import evict_config_param, get_config_param, litellm_config_cache +from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.database import Database from litellm.router_api import ChangeSet from litellm.router_api import RouterSettings as RouterSettingsSnapshot from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict +from tests._support.native_database import native_database @pytest.fixture(autouse=True) @@ -79,9 +89,7 @@ class TestFallbackCreateRequest: def test_duplicate_fallback_models(self): """Test that duplicate fallback models raise validation error""" - with pytest.raises( - ValueError, match="fallback_models must not contain duplicates" - ): + with pytest.raises(ValueError, match="fallback_models must not contain duplicates"): FallbackCreateRequest( model="gpt-3.5-turbo", fallback_models=["gpt-4", "gpt-4"], @@ -180,13 +188,12 @@ class TestCreateFallbackForTeamScopedModels: return _team_router() @pytest.fixture - def prisma_client(self) -> MagicMock: - client: Final = MagicMock() - client.writer_db = client.db - client.db.litellm_config.find_unique = AsyncMock(return_value=None) - client.db.litellm_config.upsert = AsyncMock() - client.get_generic_data = AsyncMock(return_value=None) - return client + def stored(self) -> "_StoredRouterSettings": + return _StoredRouterSettings(None) + + @pytest.fixture + def prisma_client(self, stored: "_StoredRouterSettings") -> MagicMock: + return stored.prisma_client @pytest.fixture def proxy_config(self) -> ProxyConfig: @@ -197,14 +204,13 @@ class TestCreateFallbackForTeamScopedModels: ) -> FallbackResponse: with ( patch("litellm.proxy.proxy_server.llm_router", router), - patch("litellm.proxy.proxy_server.prisma_client", prisma_client), patch("litellm.proxy.proxy_server.proxy_config", proxy_config), patch("litellm.proxy.proxy_server.store_model_in_db", True), ): - return await create_fallback(request, MagicMock()) + return await create_fallback(request, MagicMock(), fallbacks=ConfigRepository(prisma_client)) async def test_public_names_create_a_rule_keyed_on_the_public_name( - self, router: Router, prisma_client: MagicMock, proxy_config: MagicMock + self, router: Router, stored: "_StoredRouterSettings", prisma_client: MagicMock, proxy_config: MagicMock ) -> None: request: Final = FallbackCreateRequest(model="team-primary", fallback_models=["team-fallback"]) @@ -213,10 +219,7 @@ class TestCreateFallbackForTeamScopedModels: assert response.model == "team-primary" assert response.fallback_models == ["team-fallback"] assert router.settings().fallbacks == [{"team-primary": ["team-fallback"]}] - persisted: Final = json.loads( - prisma_client.db.litellm_config.upsert.call_args.kwargs["data"]["create"]["param_value"] - ) - assert persisted["fallbacks"] == [{"team-primary": ["team-fallback"]}] + assert stored.stored_fallbacks() == [{"team-primary": ["team-fallback"]}] async def test_internal_names_keep_working( self, router: Router, prisma_client: MagicMock, proxy_config: MagicMock @@ -279,8 +282,45 @@ class _ConfigFileProxyConfig(ProxyConfig): return {"router_settings": copy.deepcopy(self.config_file)} +NATIVE: Final = native_database() + + +class _ConfigTransaction: + def __init__(self, stored: "_StoredRouterSettings") -> None: + self._stored: Final = stored + self.staged: SimpleNamespace | None = None + self.holds_lock = False + + @property + def litellm_config(self) -> "_ConfigTransaction": + return self + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, int]]: + if "router_settings" not in args or self.holds_lock: + return () + if self._stored.lock.locked(): + self._stored.lock_contended.set() + await self._stored.lock.acquire() + self.holds_lock = True + return ({"locked": 1},) + + async def find_unique(self, where: Mapping[str, str]) -> SimpleNamespace | None: + visible: Final = self.staged or copy.deepcopy(self._stored.row) + await self._stored.after_read() + return visible + + async def find_first(self, where: Mapping[str, str]) -> SimpleNamespace | None: + return await self.find_unique(where) + + async def upsert(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]]) -> None: + if self._stored.fail_writes: + raise RuntimeError("the database refused the write") + self.staged = _config_row(json.loads(data["update"]["param_value"])) + + class _StoredRouterSettings: - """The LiteLLM_Config router_settings row, with the proxy objects that read it through the config cache""" + """The LiteLLM_Config router_settings row behind a writer with PostgreSQL's transaction semantics: a + transaction's write lands only when it commits, and the advisory lock waits for the transaction holding it""" def __init__( self, @@ -288,23 +328,50 @@ class _StoredRouterSettings: config_file: RouterSettings | None = None, *, lagging_replica: bool = False, + hold_first_read: bool = False, + fail_writes: bool = False, ) -> None: self.row: SimpleNamespace | None = None if router_settings is None else _config_row(router_settings) + self.writes = 0 + self.fail_writes: Final = fail_writes + self.lock: Final = asyncio.Lock() + self.lock_contended: Final = asyncio.Event() + self._holding_next_read = hold_first_read + self.read_held: Final = asyncio.Event() + self.held_read_released: Final = asyncio.Event() self.prisma_client: Final = MagicMock() self.prisma_client.get_generic_data = AsyncMock( side_effect=lambda value, **_: self.row if value == "router_settings" else None ) - self.prisma_client.writer_db.litellm_config.find_unique = AsyncMock( - side_effect=lambda **_: copy.deepcopy(self.row) - ) + self.prisma_client.writer_db.tx = self._transaction + self.prisma_client.tx = self._transaction self.prisma_client.db.litellm_config.find_unique = AsyncMock( side_effect=lambda **_: None if lagging_replica else copy.deepcopy(self.row) ) - self.prisma_client.db.litellm_config.upsert = AsyncMock(side_effect=self._upsert) self.proxy_config: Final = _ConfigFileProxyConfig(config_file) + self.prisma_client.writer_db.litellm_config.find_unique = AsyncMock( + side_effect=lambda **_: copy.deepcopy(self.row) + ) + self.fallbacks: Final = ConfigRepository(self.prisma_client, use_writer=True) - async def _upsert(self, where: dict[str, str], data: dict[str, dict[str, str]]) -> None: - self.row = _config_row(json.loads(data["update"]["param_value"])) + @asynccontextmanager + async def _transaction(self) -> AsyncIterator[_ConfigTransaction]: + transaction: Final = _ConfigTransaction(self) + try: + yield transaction + if transaction.staged is not None: + self.row = transaction.staged + self.writes += 1 + finally: + if transaction.holds_lock: + self.lock.release() + + async def after_read(self) -> None: + if not self._holding_next_read: + return + self._holding_next_read = False + self.read_held.set() + await self.held_read_released.wait() def written_by_another_instance(self, router_settings: RouterSettings) -> None: self.row = _config_row(router_settings) @@ -318,6 +385,7 @@ class _StoredRouterSettings: return row.param_value["fallbacks"] +FallbackWrite: TypeAlias = Callable[[], Awaitable[object]] TEAM_RULE: Final = {"team-primary": ["team-fallback"]} GATEWAY_RULE: Final = {"gpt-5.4-mini": ["team-fallback"]} MALFORMED_GATEWAY_RULE: Final = {"gpt-5.4-mini": "team-fallback"} @@ -347,20 +415,260 @@ class TestFallbackWritesSeeTheLatestStoredRules: async def _create(self, request: FallbackCreateRequest, stored: _StoredRouterSettings) -> FallbackResponse: with ( patch("litellm.proxy.proxy_server.llm_router", _team_router()), - patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client), patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), patch("litellm.proxy.proxy_server.store_model_in_db", True), ): - return await create_fallback(request, MagicMock()) + return await create_fallback(request, MagicMock(), fallbacks=stored.fallbacks) async def _delete(self, model: str, stored: _StoredRouterSettings) -> FallbackDeleteResponse: with ( patch("litellm.proxy.proxy_server.llm_router", _team_router()), - patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client), patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), patch("litellm.proxy.proxy_server.store_model_in_db", True), ): - return await delete_fallback(model, "general", MagicMock()) + return await delete_fallback(model, "general", MagicMock(), fallbacks=stored.fallbacks) + + async def _get( + self, + model: str, + stored: _StoredRouterSettings, + fallback_type: Literal["general", "context_window", "content_policy"] = "general", + router: Router | None = None, + ) -> FallbackGetResponse: + with ( + patch("litellm.proxy.proxy_server.llm_router", router or _team_router()), + patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + return await get_fallback(model, fallback_type, MagicMock(), fallbacks=stored.fallbacks) + + async def _read_status(self, model: str, stored: _StoredRouterSettings, router: Router | None = None) -> int: + try: + await self._get(model, stored, router=router) + except HTTPException as refused: + return refused.status_code + return 200 + + async def test_a_replica_that_has_not_reloaded_reads_each_write_as_saved(self) -> None: + stored: Final = _StoredRouterSettings(None, lagging_replica=True) + stale_router: Final = _team_router() + stale_router.apply(ChangeSet(assign={"fallbacks": [{"team-primary": ["gpt-5.4-mini"]}]})) + + await self._create(FallbackCreateRequest(model="team-primary", fallback_models=["team-fallback"]), stored) + created: Final = await self._get("team-primary", stored, router=stale_router) + await self._create( + FallbackCreateRequest(model="team-primary", fallback_models=["gpt-5.4-mini", "team-fallback"]), stored + ) + updated: Final = await self._get("team-primary", stored, router=stale_router) + await self._delete("team-primary", stored) + + assert created.fallback_models == ["team-fallback"] + assert updated.fallback_models == ["gpt-5.4-mini", "team-fallback"] + assert await self._read_status("team-primary", stored, router=stale_router) == 404 + assert stale_router.settings().fallbacks == [{"team-primary": ["gpt-5.4-mini"]}] + + @pytest.mark.parametrize( + ("fallback_type", "fallback_key"), + [ + ("general", "fallbacks"), + ("context_window", "context_window_fallbacks"), + ("content_policy", "content_policy_fallbacks"), + ], + ) + async def test_a_read_names_every_saved_target_of_each_fallback_type( + self, fallback_type: Literal["general", "context_window", "content_policy"], fallback_key: str + ) -> None: + stored: Final = _StoredRouterSettings( + {fallback_key: [{"team-primary": [{"model": "gpt-5.4-mini", "api_key": "sk-test"}, "team-fallback"]}]} + ) + + response: Final = await self._get("team-primary", stored, fallback_type) + + assert response.fallback_models == ["gpt-5.4-mini", "team-fallback"] + assert response.fallback_type == fallback_type + + @pytest.mark.parametrize( + ("fallback_type", "fallback_key"), + [("general", "fallbacks"), ("content_policy", "content_policy_fallbacks")], + ) + async def test_a_read_skips_a_malformed_saved_rule_and_serves_the_valid_ones( + self, fallback_type: Literal["general", "content_policy"], fallback_key: str + ) -> None: + stored: Final = _StoredRouterSettings(None) + stored.row = SimpleNamespace(param_name="router_settings", param_value={fallback_key: [42, TEAM_RULE, None]}) + + response: Final = await self._get("team-primary", stored, fallback_type) + + assert response.fallback_models == ["team-fallback"] + + async def test_a_read_resolves_the_saved_wildcard_rule(self) -> None: + stored: Final = _StoredRouterSettings({"fallbacks": [GATEWAY_RULE, {"*": ["gpt-5.4-mini"]}]}) + + response: Final = await self._get("team-primary", stored) + + assert response.fallback_models == ["gpt-5.4-mini"] + + @pytest.mark.parametrize( + "router_settings", + [None, {"routing_strategy": "latency-based-routing"}, {"fallbacks": None}, {"fallbacks": "team-primary"}], + ) + async def test_a_read_does_not_answer_with_rules_only_the_routers_memory_holds( + self, router_settings: RouterSettings | None + ) -> None: + stored: Final = _StoredRouterSettings(router_settings) + stale_router: Final = _team_router() + stale_router.apply(ChangeSet(assign={"fallbacks": [TEAM_RULE]})) + + assert await self._read_status("team-primary", stored, router=stale_router) == 404 + + async def test_a_content_policy_read_layers_the_saved_rules_over_the_litellm_settings_rules( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "content_policy_fallbacks", [TEAM_RULE, {"gpt-5.4-mini": ["team-primary"]}]) + stored: Final = _StoredRouterSettings({"content_policy_fallbacks": [GATEWAY_RULE]}) + emptied: Final = _StoredRouterSettings({"content_policy_fallbacks": []}) + + saved_rule: Final = await self._get("gpt-5.4-mini", stored, "content_policy") + file_rule: Final = await self._get("team-primary", stored, "content_policy") + after_delete: Final = await self._get("gpt-5.4-mini", emptied, "content_policy") + + assert saved_rule.fallback_models == ["team-fallback"] + assert file_rule.fallback_models == ["team-fallback"] + assert after_delete.fallback_models == ["team-primary"] + + async def test_a_read_outside_database_mode_answers_from_the_router(self) -> None: + stored: Final = _StoredRouterSettings({"fallbacks": [TEAM_RULE]}) + router: Final = _team_router() + router.apply(ChangeSet(assign={"fallbacks": [GATEWAY_RULE]})) + + with ( + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.store_model_in_db", False), + ): + response: Final = await get_fallback("gpt-5.4-mini", "general", MagicMock(), fallbacks=stored.fallbacks) + + assert response.fallback_models == ["team-fallback"] + + async def test_a_read_of_a_router_settings_row_that_fails_answers_500(self) -> None: + stored: Final = _StoredRouterSettings({"fallbacks": [TEAM_RULE]}) + stored.prisma_client.writer_db.litellm_config.find_unique = AsyncMock(side_effect=RuntimeError("db down")) + + with pytest.raises(HTTPException) as failed: + await self._get("team-primary", stored) + + assert failed.value.status_code == 500 + + async def _while_a_write_holds_the_rules( + self, stored: _StoredRouterSettings, holder: FallbackWrite, contender: FallbackWrite + ) -> None: + with ( + patch("litellm.proxy.proxy_server.llm_router", _team_router()), + patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + holding: Final = asyncio.create_task(holder()) + await stored.read_held.wait() + contending: Final = asyncio.create_task(contender()) + blocked: Final = asyncio.create_task(stored.lock_contended.wait()) + await asyncio.wait((contending, blocked), return_when=asyncio.FIRST_COMPLETED) + blocked.cancel() + stored.held_read_released.set() + await asyncio.gather(holding, contending) + + async def test_concurrent_first_creates_keep_both_rules(self) -> None: + stored: Final = _StoredRouterSettings(None, hold_first_read=True) + + await self._while_a_write_holds_the_rules( + stored, + lambda: create_fallback( + FallbackCreateRequest(model="team-primary", fallback_models=["team-fallback"]), + MagicMock(), + fallbacks=stored.fallbacks, + ), + lambda: create_fallback( + FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), + MagicMock(), + fallbacks=stored.fallbacks, + ), + ) + + assert stored.stored_fallbacks() == [TEAM_RULE, GATEWAY_RULE] + + async def test_a_delete_racing_a_create_keeps_the_created_rule(self) -> None: + stored: Final = _StoredRouterSettings( + {"fallbacks": [TEAM_RULE], "routing_strategy": "latency-based-routing"}, hold_first_read=True + ) + + await self._while_a_write_holds_the_rules( + stored, + lambda: create_fallback( + FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), + MagicMock(), + fallbacks=stored.fallbacks, + ), + lambda: delete_fallback("team-primary", "general", MagicMock(), fallbacks=stored.fallbacks), + ) + + assert stored.row is not None + assert stored.row.param_value == {"fallbacks": [GATEWAY_RULE], "routing_strategy": "latency-based-routing"} + + async def test_a_config_section_write_racing_a_fallback_write_keeps_both_changes(self) -> None: + stored: Final = _StoredRouterSettings({"fallbacks": [TEAM_RULE]}, hold_first_read=True) + + await self._while_a_write_holds_the_rules( + stored, + lambda: create_fallback( + FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), + MagicMock(), + fallbacks=stored.fallbacks, + ), + lambda: stored.proxy_config._upsert_changed_config_section( + section_name="router_settings", + changed_keys={"routing_strategy": "latency-based-routing"}, + removed_keys=frozenset(), + prisma_client=stored.prisma_client, + ), + ) + + assert stored.row is not None + assert stored.row.param_value == { + "fallbacks": [TEAM_RULE, GATEWAY_RULE], + "routing_strategy": "latency-based-routing", + } + + async def test_a_failed_write_changes_neither_the_stored_rules_nor_the_router(self) -> None: + stored: Final = _StoredRouterSettings({"fallbacks": [TEAM_RULE]}, fail_writes=True) + router: Final = _team_router() + router.apply(ChangeSet(assign={"fallbacks": [TEAM_RULE]})) + + with ( + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + pytest.raises(HTTPException) as failed, + ): + await create_fallback( + FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), + MagicMock(), + fallbacks=stored.fallbacks, + ) + + assert failed.value.status_code == 500 + assert stored.stored_fallbacks() == [TEAM_RULE] + assert router.settings().fallbacks == [TEAM_RULE] + + async def test_a_delete_of_a_missing_rule_answers_404_and_leaves_the_row_and_lock_untouched(self) -> None: + settings: Final = {"fallbacks": [TEAM_RULE], "routing_strategy": "latency-based-routing"} + stored: Final = _StoredRouterSettings(copy.deepcopy(settings)) + + with pytest.raises(HTTPException) as missing: + await self._delete("gpt-5.4-mini", stored) + await self._create(FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), stored) + + assert missing.value.status_code == 404 + assert stored.writes == 1 + assert stored.row is not None + assert stored.row.param_value == {**settings, "fallbacks": [TEAM_RULE, GATEWAY_RULE]} async def test_a_create_reads_the_rules_from_the_writer_while_the_replica_lags(self) -> None: stored: Final = _StoredRouterSettings({"fallbacks": [TEAM_RULE]}, lagging_replica=True) @@ -397,7 +705,6 @@ class TestFallbackWritesSeeTheLatestStoredRules: stored: Final = _StoredRouterSettings(None) with ( patch("litellm.proxy.proxy_server.llm_router", router), - patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client), patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), patch("litellm.proxy.proxy_server.store_model_in_db", True), ): @@ -406,10 +713,15 @@ class TestFallbackWritesSeeTheLatestStoredRules: model="gpt-5.4-mini", fallback_models=["team-fallback"], fallback_type="content_policy" ), MagicMock(), + fallbacks=stored.fallbacks, + ) + after_create: Final = await get_fallback( + "team-primary", "content_policy", MagicMock(), fallbacks=stored.fallbacks + ) + await delete_fallback("gpt-5.4-mini", "content_policy", MagicMock(), fallbacks=stored.fallbacks) + after_delete: Final = await get_fallback( + "team-primary", "content_policy", MagicMock(), fallbacks=stored.fallbacks ) - after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock()) - await delete_fallback("gpt-5.4-mini", "content_policy", MagicMock()) - after_delete: Final = await get_fallback("team-primary", "content_policy", MagicMock()) assert after_create.fallback_models == ["team-fallback"] assert after_delete.fallback_models == ["team-fallback"] @@ -419,7 +731,6 @@ class TestFallbackWritesSeeTheLatestStoredRules: stored: Final = _StoredRouterSettings({"content_policy_fallbacks": [MALFORMED_GATEWAY_RULE]}) with ( patch("litellm.proxy.proxy_server.llm_router", router), - patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client), patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), patch("litellm.proxy.proxy_server.store_model_in_db", True), ): @@ -428,8 +739,11 @@ class TestFallbackWritesSeeTheLatestStoredRules: model="team-primary", fallback_models=["team-fallback"], fallback_type="content_policy" ), MagicMock(), + fallbacks=stored.fallbacks, + ) + after_create: Final = await get_fallback( + "team-primary", "content_policy", MagicMock(), fallbacks=stored.fallbacks ) - after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock()) assert after_create.fallback_models == ["team-fallback"] @@ -445,7 +759,7 @@ class TestFallbackWritesSeeTheLatestStoredRules: **{router_setting: [{"team-primary": [{"model": "gpt-5.4-mini", "api_key": "sk-test"}, "team-fallback"]}]} ) with patch("litellm.proxy.proxy_server.llm_router", router): - response: Final = await get_fallback("team-primary", fallback_type, MagicMock()) + response: Final = await get_fallback("team-primary", fallback_type, MagicMock(), fallbacks=None) assert response.fallback_models == ["gpt-5.4-mini", "team-fallback"] @@ -459,7 +773,7 @@ class TestFallbackWritesSeeTheLatestStoredRules: router: Final = _team_router() router.update_settings(**{router_setting: [{"team-primary": [{"temperature": 0}, "team-fallback"]}]}) with patch("litellm.proxy.proxy_server.llm_router", router): - response: Final = await get_fallback("team-primary", fallback_type, MagicMock()) + response: Final = await get_fallback("team-primary", fallback_type, MagicMock(), fallbacks=None) assert response.fallback_models == ["team-primary", "team-fallback"] @@ -467,7 +781,7 @@ class TestFallbackWritesSeeTheLatestStoredRules: router: Final = _team_router() router.update_settings(fallbacks=[{"team-primary": "team-fallback"}]) with patch("litellm.proxy.proxy_server.llm_router", router), pytest.raises(HTTPException) as raised: - await get_fallback("team-primary", "general", MagicMock()) + await get_fallback("team-primary", "general", MagicMock(), fallbacks=None) assert raised.value.status_code == 500 @@ -530,6 +844,79 @@ class TestFallbackWritesSeeTheLatestStoredRules: assert stored.row.param_value == {"fallbacks": [GATEWAY_RULE]} +class TestFallbackRoutesResolveTheirRepository: + @pytest.fixture + def stored(self) -> Iterator[_StoredRouterSettings]: + stored: Final = _StoredRouterSettings(None, lagging_replica=True) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + with ( + patch("litellm.proxy.proxy_server.llm_router", _team_router()), + patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + yield stored + app.dependency_overrides.pop(user_api_key_auth, None) + app.dependency_overrides.pop(get_database, None) + + def test_a_create_stores_the_rule_through_the_request_database(self, stored: _StoredRouterSettings) -> None: + app.dependency_overrides[get_database] = lambda: Database(prisma=stored.prisma_client, native=NATIVE) + + response: Final = TestClient(app).post( + "/fallback", json={"model": "team-primary", "fallback_models": ["team-fallback"]} + ) + + assert response.status_code == 200, response.text + assert stored.stored_fallbacks() == [TEAM_RULE] + + @pytest.mark.parametrize( + ("method", "path", "body"), + [ + ("POST", "/fallback", {"model": "team-primary", "fallback_models": ["team-fallback"]}), + ("DELETE", "/fallback/team-primary", None), + ], + ) + def test_writes_without_a_database_are_refused( + self, stored: _StoredRouterSettings, method: str, path: str, body: Mapping[str, JsonValue] | None + ) -> None: + app.dependency_overrides[get_database] = lambda: None + + response: Final = TestClient(app).request(method, path, json=body) + + assert response.status_code == 400, response.text + assert "Database storage not enabled" in response.text + assert stored.row is None + + def test_a_read_serves_the_saved_rule_the_routers_memory_lacks(self, stored: _StoredRouterSettings) -> None: + stored.written_by_another_instance({"fallbacks": [TEAM_RULE]}) + app.dependency_overrides[get_database] = lambda: Database(prisma=stored.prisma_client, native=NATIVE) + + response: Final = TestClient(app).get("/fallback/team-primary", params={"fallback_type": "general"}) + + assert response.status_code == 200, response.text + assert response.json() == { + "model": "team-primary", + "fallback_models": ["team-fallback"], + "fallback_type": "general", + } + + def test_a_read_without_a_database_answers_from_the_router(self, stored: _StoredRouterSettings) -> None: + stored.written_by_another_instance({"fallbacks": [TEAM_RULE]}) + app.dependency_overrides[get_database] = lambda: None + + response: Final = TestClient(app).get("/fallback/team-primary", params={"fallback_type": "general"}) + + assert response.status_code == 404, response.text + + def test_an_unknown_model_is_refused_before_the_missing_database(self, stored: _StoredRouterSettings) -> None: + app.dependency_overrides[get_database] = lambda: None + + response: Final = TestClient(app).post( + "/fallback", json={"model": "team-missing", "fallback_models": ["team-fallback"]} + ) + + assert response.status_code == 404, response.text + + CONFIG_RULE: Final = {"gpt-5.4-mini": ["team-fallback"]} CONFIG_WINDOW_RULE: Final = {"team-primary": ["gpt-5.4-mini"]} @@ -557,20 +944,18 @@ class TestFallbackWritesHonourTheConfigFile: ) -> FallbackResponse: with ( patch("litellm.proxy.proxy_server.llm_router", router), - patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client), patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), patch("litellm.proxy.proxy_server.store_model_in_db", True), ): - return await create_fallback(request, MagicMock()) + return await create_fallback(request, MagicMock(), fallbacks=stored.fallbacks) async def _delete(self, model: str, router: Router, stored: _StoredRouterSettings) -> FallbackDeleteResponse: with ( patch("litellm.proxy.proxy_server.llm_router", router), - patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client), patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), patch("litellm.proxy.proxy_server.store_model_in_db", True), ): - return await delete_fallback(model, "general", MagicMock()) + return await delete_fallback(model, "general", MagicMock(), fallbacks=stored.fallbacks) async def test_a_create_that_adds_a_rule_to_a_config_owned_list_is_refused( self, router: Router, stored: _StoredRouterSettings @@ -584,7 +969,7 @@ class TestFallbackWritesHonourTheConfigFile: assert refused.value.status_code == 400 assert "key 'fallbacks' is set in the config file" in str(refused.value.detail) assert stored.stored_fallbacks() == [CONFIG_RULE] - stored.prisma_client.db.litellm_config.upsert.assert_not_awaited() + assert stored.writes == 0 assert router.settings().fallbacks == [CONFIG_RULE] async def test_a_create_after_the_config_file_takes_over_the_key_is_refused(self, router: Router) -> None: @@ -610,7 +995,7 @@ class TestFallbackWritesHonourTheConfigFile: assert refused.value.status_code == 400 assert "key 'fallbacks' is set in the config file" in str(refused.value.detail) assert stored.stored_fallbacks() == [CONFIG_RULE] - stored.prisma_client.db.litellm_config.upsert.assert_not_awaited() + assert stored.writes == 0 assert router.settings().fallbacks == [CONFIG_RULE] async def test_a_delete_of_a_rule_only_the_config_file_holds_is_refused(self, router: Router) -> None: @@ -629,10 +1014,14 @@ class TestFallbackWritesHonourTheConfigFile: self, router: Router, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.delenv("LITELLM_TEST_UNSET_FALLBACKS", raising=False) - stored: Final = _StoredRouterSettings(None, config_file={"fallbacks": "os.environ/LITELLM_TEST_UNSET_FALLBACKS"}) + stored: Final = _StoredRouterSettings( + None, config_file={"fallbacks": "os.environ/LITELLM_TEST_UNSET_FALLBACKS"} + ) with pytest.raises(HTTPException) as refused: - await self._create(FallbackCreateRequest(model="team-primary", fallback_models=["team-fallback"]), router, stored) + await self._create( + FallbackCreateRequest(model="team-primary", fallback_models=["team-fallback"]), router, stored + ) with pytest.raises(HTTPException) as not_found: await self._delete("team-primary", router, stored) @@ -696,13 +1085,30 @@ class TestFallbackWritesHonourTheConfigFile: {"fallbacks": [TEAM_RULE]}, config_file={"routing_strategy": "latency-based-routing"} ) - await self._create(FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), router, stored) + await self._create( + FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), router, stored + ) assert stored.row is not None assert stored.row.param_value == {"fallbacks": [TEAM_RULE, GATEWAY_RULE]} await self._delete("team-primary", router, stored) assert stored.row.param_value == {"fallbacks": [GATEWAY_RULE]} + async def test_a_read_of_a_config_owned_list_answers_from_the_file_not_the_shadowed_row(self) -> None: + stored: Final = _StoredRouterSettings({"fallbacks": [TEAM_RULE]}, config_file={"fallbacks": [CONFIG_RULE]}) + + with ( + patch("litellm.proxy.proxy_server.llm_router", _team_router()), + patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + from_file: Final = await get_fallback("gpt-5.4-mini", "general", MagicMock(), fallbacks=stored.fallbacks) + with pytest.raises(HTTPException) as shadowed: + await get_fallback("team-primary", "general", MagicMock(), fallbacks=stored.fallbacks) + + assert from_file.fallback_models == ["team-fallback"] + assert shadowed.value.status_code == 404 + async def test_a_sibling_list_the_config_file_does_not_set_stays_writable( self, router: Router, stored: _StoredRouterSettings ) -> None: @@ -737,15 +1143,12 @@ class TestCreateFallback: return router @pytest.fixture - def mock_prisma_client(self): - """Create a mock prisma client""" - client = MagicMock() - client.writer_db = client.db - client.db.litellm_config.find_unique = AsyncMock(return_value=None) - client.db.litellm_config.upsert = AsyncMock() - client.get_generic_data = AsyncMock(return_value=None) - client.jsonify_object = lambda x: x - return client + def stored(self) -> _StoredRouterSettings: + return _StoredRouterSettings(None) + + @pytest.fixture + def mock_prisma_client(self, stored: _StoredRouterSettings) -> MagicMock: + return stored.prisma_client @pytest.fixture def mock_proxy_config(self): @@ -758,7 +1161,7 @@ class TestCreateFallback: return MagicMock() async def test_create_fallback_success( - self, mock_router, mock_prisma_client, mock_proxy_config, mock_user_api_key_dict + self, mock_router, stored, mock_prisma_client, mock_proxy_config, mock_user_api_key_dict ): """Test successful fallback creation""" request = FallbackCreateRequest( @@ -785,18 +1188,16 @@ class TestCreateFallback: True, ), ): - response = await create_fallback(request, mock_user_api_key_dict) + response = await create_fallback( + request, mock_user_api_key_dict, fallbacks=ConfigRepository(mock_prisma_client) + ) assert response.model == "gpt-3.5-turbo" assert response.fallback_models == ["gpt-4", "claude-3-haiku"] assert response.fallback_type == "general" - assert ( - "created" in response.message.lower() - or "updated" in response.message.lower() - ) + assert "created" in response.message.lower() or "updated" in response.message.lower() - # Verify database was updated - mock_prisma_client.db.litellm_config.upsert.assert_called_once() + assert stored.stored_fallbacks() == [{"gpt-3.5-turbo": ["gpt-4", "claude-3-haiku"]}] async def test_create_fallback_router_not_initialized( self, mock_prisma_client, mock_proxy_config, mock_user_api_key_dict @@ -814,7 +1215,7 @@ class TestCreateFallback: ), pytest.raises(HTTPException) as exc_info, ): - await create_fallback(request, mock_user_api_key_dict) + await create_fallback(request, mock_user_api_key_dict, fallbacks=ConfigRepository(mock_prisma_client)) assert exc_info.value.status_code == 500 assert "Router not initialized" in str(exc_info.value.detail) @@ -843,7 +1244,7 @@ class TestCreateFallback: ), pytest.raises(HTTPException) as exc_info, ): - await create_fallback(request, mock_user_api_key_dict) + await create_fallback(request, mock_user_api_key_dict, fallbacks=ConfigRepository(mock_prisma_client)) assert exc_info.value.status_code == 404 assert "not found in router" in str(exc_info.value.detail) @@ -872,7 +1273,7 @@ class TestCreateFallback: ), pytest.raises(HTTPException) as exc_info, ): - await create_fallback(request, mock_user_api_key_dict) + await create_fallback(request, mock_user_api_key_dict, fallbacks=ConfigRepository(mock_prisma_client)) assert exc_info.value.status_code == 400 assert "Invalid fallback models" in str(exc_info.value.detail) @@ -901,14 +1302,12 @@ class TestCreateFallback: ), pytest.raises(HTTPException) as exc_info, ): - await create_fallback(request, mock_user_api_key_dict) + await create_fallback(request, mock_user_api_key_dict, fallbacks=ConfigRepository(mock_prisma_client)) assert exc_info.value.status_code == 400 assert "cannot be its own fallback" in str(exc_info.value.detail) - async def test_create_fallback_db_not_enabled( - self, mock_router, mock_user_api_key_dict - ): + async def test_create_fallback_db_not_enabled(self, mock_router, mock_user_api_key_dict): """Test error when database storage is not enabled""" request = FallbackCreateRequest( model="gpt-3.5-turbo", @@ -926,7 +1325,7 @@ class TestCreateFallback: ), pytest.raises(HTTPException) as exc_info, ): - await create_fallback(request, mock_user_api_key_dict) + await create_fallback(request, mock_user_api_key_dict, fallbacks=None) assert exc_info.value.status_code == 400 assert "Database storage not enabled" in str(exc_info.value.detail) @@ -959,7 +1358,9 @@ class TestCreateFallback: True, ), ): - response = await create_fallback(request, mock_user_api_key_dict) + response = await create_fallback( + request, mock_user_api_key_dict, fallbacks=ConfigRepository(mock_prisma_client) + ) assert response.fallback_type == "context_window" # Verify the correct setting was updated @@ -988,25 +1389,19 @@ class TestGetFallback: """Create a mock user API key dict""" return MagicMock() - async def test_get_fallback_success( - self, mock_router_with_fallbacks, mock_user_api_key_dict - ): + async def test_get_fallback_success(self, mock_router_with_fallbacks, mock_user_api_key_dict): """Test successful fallback retrieval""" with patch( "litellm.proxy.proxy_server.llm_router", mock_router_with_fallbacks, ): - response = await get_fallback( - "gpt-3.5-turbo", "general", mock_user_api_key_dict - ) + response = await get_fallback("gpt-3.5-turbo", "general", mock_user_api_key_dict, fallbacks=None) assert response.model == "gpt-3.5-turbo" assert response.fallback_models == ["gpt-4", "claude-3-haiku"] assert response.fallback_type == "general" - async def test_get_fallback_not_found( - self, mock_router_with_fallbacks, mock_user_api_key_dict - ): + async def test_get_fallback_not_found(self, mock_router_with_fallbacks, mock_user_api_key_dict): """Test error when fallback is not found""" with ( patch( @@ -1015,7 +1410,7 @@ class TestGetFallback: ), pytest.raises(HTTPException) as exc_info, ): - await get_fallback("gpt-4", "general", mock_user_api_key_dict) + await get_fallback("gpt-4", "general", mock_user_api_key_dict, fallbacks=None) assert exc_info.value.status_code == 404 assert "No general fallbacks configured" in str(exc_info.value.detail) @@ -1029,7 +1424,7 @@ class TestGetFallback: ), pytest.raises(HTTPException) as exc_info, ): - await get_fallback("gpt-3.5-turbo", "general", mock_user_api_key_dict) + await get_fallback("gpt-3.5-turbo", "general", mock_user_api_key_dict, fallbacks=None) assert exc_info.value.status_code == 500 assert "Router not initialized" in str(exc_info.value.detail) @@ -1051,17 +1446,12 @@ class TestDeleteFallback: return router @pytest.fixture - def mock_prisma_client(self): - """Create a mock prisma client""" - client = MagicMock() - client.writer_db = client.db - client.db.litellm_config.find_unique = AsyncMock( - return_value=_config_row({"fallbacks": [{"gpt-3.5-turbo": ["gpt-4", "claude-3-haiku"]}]}) - ) - client.db.litellm_config.upsert = AsyncMock() - client.get_generic_data = AsyncMock(return_value=None) - client.jsonify_object = lambda x: x - return client + def stored(self) -> _StoredRouterSettings: + return _StoredRouterSettings({"fallbacks": [{"gpt-3.5-turbo": ["gpt-4", "claude-3-haiku"]}]}) + + @pytest.fixture + def mock_prisma_client(self, stored: _StoredRouterSettings) -> MagicMock: + return stored.prisma_client @pytest.fixture def mock_proxy_config(self): @@ -1076,6 +1466,7 @@ class TestDeleteFallback: async def test_delete_fallback_success( self, mock_router_with_fallbacks, + stored, mock_prisma_client, mock_proxy_config, mock_user_api_key_dict, @@ -1100,15 +1491,17 @@ class TestDeleteFallback: ), ): response = await delete_fallback( - "gpt-3.5-turbo", "general", mock_user_api_key_dict + "gpt-3.5-turbo", + "general", + mock_user_api_key_dict, + fallbacks=ConfigRepository(mock_prisma_client), ) assert response.model == "gpt-3.5-turbo" assert response.fallback_type == "general" assert "deleted" in response.message.lower() - # Verify database was updated - mock_prisma_client.db.litellm_config.upsert.assert_called_once() + assert stored.stored_fallbacks() == [] async def test_delete_fallback_not_found( self, @@ -1137,7 +1530,9 @@ class TestDeleteFallback: ), pytest.raises(HTTPException) as exc_info, ): - await delete_fallback("gpt-4", "general", mock_user_api_key_dict) + await delete_fallback( + "gpt-4", "general", mock_user_api_key_dict, fallbacks=ConfigRepository(mock_prisma_client) + ) assert exc_info.value.status_code == 404 assert "No general fallbacks configured" in str(exc_info.value.detail) @@ -1151,14 +1546,12 @@ class TestDeleteFallback: ), pytest.raises(HTTPException) as exc_info, ): - await delete_fallback("gpt-3.5-turbo", "general", mock_user_api_key_dict) + await delete_fallback("gpt-3.5-turbo", "general", mock_user_api_key_dict, fallbacks=None) assert exc_info.value.status_code == 500 assert "Router not initialized" in str(exc_info.value.detail) - async def test_delete_fallback_db_not_enabled( - self, mock_router_with_fallbacks, mock_user_api_key_dict - ): + async def test_delete_fallback_db_not_enabled(self, mock_router_with_fallbacks, mock_user_api_key_dict): """Test error when database storage is not enabled""" with ( patch( @@ -1171,7 +1564,7 @@ class TestDeleteFallback: ), pytest.raises(HTTPException) as exc_info, ): - await delete_fallback("gpt-3.5-turbo", "general", mock_user_api_key_dict) + await delete_fallback("gpt-3.5-turbo", "general", mock_user_api_key_dict, fallbacks=None) assert exc_info.value.status_code == 400 assert "Database storage not enabled" in str(exc_info.value.detail) From 89a6b5fbb26b632ef0faf633b0d3cf317d73deaa Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 11 Nov 2026 15:38:26 -0800 Subject: [PATCH 123/189] chore: remove dead files from the repo root (#50936) --- GEMINI.md | 1 - examples/bigmodel.md | 63 - examples/lar1_ollama_config.yaml | 45 - mcp_servers.json | 23 - package-lock.json | 5325 ------------------------------ package.json | 21 - qa_sticky_session.sh | 59 - router_plugins.json | 28 - taplo.toml | 23 - 9 files changed, 5588 deletions(-) delete mode 100644 GEMINI.md delete mode 100644 examples/bigmodel.md delete mode 100644 examples/lar1_ollama_config.yaml delete mode 100644 mcp_servers.json delete mode 100644 package-lock.json delete mode 100644 package.json delete mode 100755 qa_sticky_session.sh delete mode 100644 router_plugins.json delete mode 100644 taplo.toml diff --git a/GEMINI.md b/GEMINI.md deleted file mode 100644 index 5fc00e0b5ae..00000000000 --- a/GEMINI.md +++ /dev/null @@ -1 +0,0 @@ -Read @AGENTS.md for coding guidelines diff --git a/examples/bigmodel.md b/examples/bigmodel.md deleted file mode 100644 index 88a69796ebe..00000000000 --- a/examples/bigmodel.md +++ /dev/null @@ -1,63 +0,0 @@ -# BigModel (Zhipu AI) - -Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL - -```yaml -model_list: - - model_name: glm - litellm_params: - model: bigmodel/glm-5.3 - api_key: os.environ/BIGMODEL_API_KEY - api_base: https://open.bigmodel.cn -``` - -One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API - -| Proxy endpoint | BigModel endpoint | -| --- | --- | -| `/v1/chat/completions` | `/api/paas/v4/chat/completions` | -| `/v1/responses` | `/api/v1/responses` | -| `/v1/messages` | `/api/anthropic/v1/messages` | - -`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix - -A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above - -The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY` - -```python -import litellm - -chat = litellm.completion( - model="bigmodel/glm-5.3", - messages=[{"role": "user", "content": "Hello"}], - thinking={"type": "enabled"}, - reasoning_effort="low", -) - -response = litellm.responses( - model="bigmodel/glm-5.3", - input="Hello", - reasoning={"effort": "low"}, - store=True, -) - -async def messages(): - return await litellm.anthropic.messages.acreate( - model="bigmodel/glm-5.3", - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - thinking={"type": "adaptive"}, - output_config={"effort": "low"}, - ) -``` - -Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route - -GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters - -Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search - -The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices - -API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27 diff --git a/examples/lar1_ollama_config.yaml b/examples/lar1_ollama_config.yaml deleted file mode 100644 index 998cbf169b3..00000000000 --- a/examples/lar1_ollama_config.yaml +++ /dev/null @@ -1,45 +0,0 @@ -model_list: - - model_name: agent-router - litellm_params: - model: ollama/qwen3.5:9b - api_base: http://127.0.0.1:11434 - model_info: - id: cloud-smart - type: cloud-smart - - - model_name: agent-router - litellm_params: - model: ollama/phi4-mini:latest - api_base: http://127.0.0.1:11434 - model_info: - id: cloud-fast - type: cloud-fast - - - model_name: agent-router - litellm_params: - model: ollama/llama3.2:3b - api_base: http://127.0.0.1:11434 - model_info: - id: local - type: local - - - model_name: agent-router - litellm_params: - model: ollama/lfm2.5-thinking:latest - api_base: http://127.0.0.1:11434 - model_info: - id: deep - type: deep - -router_settings: - routing_strategy: lar1 - routing_strategy_args: - confidence_threshold_low: 0.3 - confidence_threshold_medium: 0.5 - confidence_threshold_high: 0.7 - -general_settings: - master_key: sk-lar1-demo - -litellm_settings: - set_verbose: true diff --git a/mcp_servers.json b/mcp_servers.json deleted file mode 100644 index 3dd21c0d65f..00000000000 --- a/mcp_servers.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "zapier": { - "sse_url": "https://mcp.zapier.com/api/mcp" - }, - "deepwiki": { - "http_url": "https://mcp.deepwiki.com/mcp", - "sse_url": "https://mcp.deepwiki.com/sse", - "description": "The DeepWiki MCP server provides programmatic access to DeepWiki's repository documentation (Devin Wiki) and search capabilities (Devin Search)." - }, - "jira": { - "sse_url": "https://mcp.atlassian.com/v1/sse", - "description": "The Jira MCP server provides programmatic access to Jira's issue tracking capabilities." - }, - "linear": { - "sse_url": "https://mcp.linear.app/sse", - "description": "The Linear MCP server provides programmatic access to Linear's issue tracking capabilities." - }, - "scrapegraph": { - "http_url": "https://smithery.ai/api/mcp/scrapegraph-mcp", - "sse_url": "https://smithery.ai/api/mcp/scrapegraph-mcp/sse", - "description": "The ScrapeGraph MCP server provides programmatic access to ScrapeGraph AI's web scraping capabilities, including smart scraping, web crawling, search scraping, and agentic scraping workflows." - } -} diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index c1c90370ddf..00000000000 --- a/package-lock.json +++ /dev/null @@ -1,5325 +0,0 @@ -{ - "name": "litellm-dependency-refresh", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "prism-react-renderer": "2.4.1", - "prisma": "5.17.0", - "react-copy-to-clipboard": "5.1.1" - }, - "devDependencies": { - "@testing-library/jest-dom": "6.8.0", - "@testing-library/react": "14.3.1", - "@types/react-copy-to-clipboard": "5.0.7", - "jest": "29.7.0" - } - }, - "node_modules/@adobe/css-tools": { - "version": "4.4.4", - "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.4.4.tgz", - "integrity": "sha512-Elp+iwUx5rN5+Y8xLt5/GRoG20WGoDCQ/1Fb+1LiGtvwbDavuSk0jhD/eZdckHAuzcDzccnkv+rEjyWfRx18gg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/code-frame": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", - "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.27.1", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/compat-data": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.5.tgz", - "integrity": "sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.5.tgz", - "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.5", - "@babel/helper-compilation-targets": "^7.27.2", - "@babel/helper-module-transforms": "^7.28.3", - "@babel/helpers": "^7.28.4", - "@babel/parser": "^7.28.5", - "@babel/template": "^7.27.2", - "@babel/traverse": "^7.28.5", - "@babel/types": "^7.28.5", - "@jridgewell/remapping": "^2.3.5", - "convert-source-map": "^2.0.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.3", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/generator": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.5.tgz", - "integrity": "sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.28.5", - "@babel/types": "^7.28.5", - "@jridgewell/gen-mapping": "^0.3.12", - "@jridgewell/trace-mapping": "^0.3.28", - "jsesc": "^3.0.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.27.2", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", - "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^7.27.2", - "@babel/helper-validator-option": "^7.27.1", - "browserslist": "^4.24.0", - "lru-cache": "^5.1.1", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-globals": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", - "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", - "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/traverse": "^7.27.1", - "@babel/types": "^7.27.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz", - "integrity": "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-module-imports": "^7.27.1", - "@babel/helper-validator-identifier": "^7.27.1", - "@babel/traverse": "^7.28.3" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/@babel/helper-plugin-utils": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", - "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", - "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.28.4", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.4.tgz", - "integrity": "sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.27.2", - "@babel/types": "^7.28.4" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.5.tgz", - "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.28.5" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/plugin-syntax-async-generators": { - "version": "7.8.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", - "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-bigint": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", - "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-properties": { - "version": "7.12.13", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", - "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.12.13" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-static-block": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", - "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-attributes": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.27.1.tgz", - "integrity": "sha512-oFT0FrKHgF53f4vOsZGi2Hh3I35PfSmVs4IBFLFj4dnafP+hIWDLg3VyKmUHfLoLHlyxY4C7DGtmHuJgn+IGww==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.27.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-meta": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", - "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-json-strings": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", - "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-jsx": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.27.1.tgz", - "integrity": "sha512-y8YTNIeKoyhGd9O0Jiyzyyqk8gdjnumGTQPsz0xOZOQ2RmkVJeZ1vmmfIvFEKqucBG6axJGBZDE/7iI5suUI/w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.27.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-logical-assignment-operators": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", - "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", - "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-numeric-separator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", - "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-object-rest-spread": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", - "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-catch-binding": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", - "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-chaining": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", - "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-private-property-in-object": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", - "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-top-level-await": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", - "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-typescript": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz", - "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.27.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/runtime": { - "version": "7.28.4", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.4.tgz", - "integrity": "sha512-Q/N6JNWvIvPnLDvjlE1OUBLPQHH6l3CltCEsHIujp45zQUSSh8K+gHnaEX45yAT1nyngnINhvWtzN+Nb9D8RAQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/template": { - "version": "7.27.2", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", - "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.27.1", - "@babel/parser": "^7.27.2", - "@babel/types": "^7.27.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.5.tgz", - "integrity": "sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.5", - "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.28.5", - "@babel/template": "^7.27.2", - "@babel/types": "^7.28.5", - "debug": "^4.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/types": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.5.tgz", - "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@bcoe/v8-coverage": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", - "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", - "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/console": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", - "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/core": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", - "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/reporters": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-changed-files": "^29.7.0", - "jest-config": "^29.7.0", - "jest-haste-map": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-resolve-dependencies": "^29.7.0", - "jest-runner": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "jest-watcher": "^29.7.0", - "micromatch": "^4.0.4", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/core/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@jest/core/node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/core/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jest/environment": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", - "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/fake-timers": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-mock": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/expect": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", - "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "expect": "^29.7.0", - "jest-snapshot": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/expect-utils": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", - "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-get-type": "^29.6.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/fake-timers": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", - "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@sinonjs/fake-timers": "^10.0.2", - "@types/node": "*", - "jest-message-util": "^29.7.0", - "jest-mock": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/globals": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", - "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/expect": "^29.7.0", - "@jest/types": "^29.6.3", - "jest-mock": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/reporters": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", - "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@jridgewell/trace-mapping": "^0.3.18", - "@types/node": "*", - "chalk": "^4.0.0", - "collect-v8-coverage": "^1.0.0", - "exit": "^0.1.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-instrument": "^6.0.0", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", - "istanbul-reports": "^3.1.3", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "jest-worker": "^29.7.0", - "slash": "^3.0.0", - "string-length": "^4.0.1", - "strip-ansi": "^6.0.0", - "v8-to-istanbul": "^9.0.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/schemas": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", - "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.27.8" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/source-map": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", - "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.18", - "callsites": "^3.0.0", - "graceful-fs": "^4.2.9" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/test-result": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", - "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/istanbul-lib-coverage": "^2.0.0", - "collect-v8-coverage": "^1.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/test-sequencer": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", - "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/test-result": "^29.7.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/transform": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", - "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/types": "^29.6.3", - "@jridgewell/trace-mapping": "^0.3.18", - "babel-plugin-istanbul": "^6.1.1", - "chalk": "^4.0.0", - "convert-source-map": "^2.0.0", - "fast-json-stable-stringify": "^2.1.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-util": "^29.7.0", - "micromatch": "^4.0.4", - "pirates": "^4.0.4", - "slash": "^3.0.0", - "write-file-atomic": "^4.0.2" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/types": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", - "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/remapping": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", - "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@prisma/debug": { - "version": "5.17.0", - "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.17.0.tgz", - "integrity": "sha512-l7+AteR3P8FXiYyo496zkuoiJ5r9jLQEdUuxIxNCN1ud8rdbH3GTxm+f+dCyaSv9l9WY+29L9czaVRXz9mULfg==", - "license": "Apache-2.0" - }, - "node_modules/@prisma/engines": { - "version": "5.17.0", - "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.17.0.tgz", - "integrity": "sha512-+r+Nf+JP210Jur+/X8SIPLtz+uW9YA4QO5IXA+KcSOBe/shT47bCcRMTYCbOESw3FFYFTwe7vU6KTWHKPiwvtg==", - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@prisma/debug": "5.17.0", - "@prisma/engines-version": "5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053", - "@prisma/fetch-engine": "5.17.0", - "@prisma/get-platform": "5.17.0" - } - }, - "node_modules/@prisma/engines-version": { - "version": "5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053", - "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053.tgz", - "integrity": "sha512-tUuxZZysZDcrk5oaNOdrBnnkoTtmNQPkzINFDjz7eG6vcs9AVDmA/F6K5Plsb2aQc/l5M2EnFqn3htng9FA4hg==", - "license": "Apache-2.0" - }, - "node_modules/@prisma/fetch-engine": { - "version": "5.17.0", - "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.17.0.tgz", - "integrity": "sha512-ESxiOaHuC488ilLPnrv/tM2KrPhQB5TRris/IeIV4ZvUuKeaicCl4Xj/JCQeG9IlxqOgf1cCg5h5vAzlewN91Q==", - "license": "Apache-2.0", - "dependencies": { - "@prisma/debug": "5.17.0", - "@prisma/engines-version": "5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053", - "@prisma/get-platform": "5.17.0" - } - }, - "node_modules/@prisma/get-platform": { - "version": "5.17.0", - "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.17.0.tgz", - "integrity": "sha512-UlDgbRozCP1rfJ5Tlkf3Cnftb6srGrEQ4Nm3og+1Se2gWmCZ0hmPIi+tQikGDUVLlvOWx3Gyi9LzgRP+HTXV9w==", - "license": "Apache-2.0", - "dependencies": { - "@prisma/debug": "5.17.0" - } - }, - "node_modules/@sinclair/typebox": { - "version": "0.27.8", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", - "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "type-detect": "4.0.8" - } - }, - "node_modules/@sinonjs/fake-timers": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", - "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@sinonjs/commons": "^3.0.0" - } - }, - "node_modules/@testing-library/dom": { - "version": "9.3.4", - "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-9.3.4.tgz", - "integrity": "sha512-FlS4ZWlp97iiNWig0Muq8p+3rVDjRiYE+YKGbAqXOu9nwJFFOdL00kFpz42M+4huzYi86vAK1sOOfyOG45muIQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.10.4", - "@babel/runtime": "^7.12.5", - "@types/aria-query": "^5.0.1", - "aria-query": "5.1.3", - "chalk": "^4.1.0", - "dom-accessibility-api": "^0.5.9", - "lz-string": "^1.5.0", - "pretty-format": "^27.0.2" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@testing-library/dom/node_modules/aria-query": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.1.3.tgz", - "integrity": "sha512-R5iJ5lkuHybztUfuOAznmboyjWq8O6sqNqtK7CLOqdydi54VNbORp49mb14KbWgG1QD3JFO9hJdZ+y4KutfdOQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "deep-equal": "^2.0.5" - } - }, - "node_modules/@testing-library/dom/node_modules/dom-accessibility-api": { - "version": "0.5.16", - "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", - "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@testing-library/jest-dom": { - "version": "6.8.0", - "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.8.0.tgz", - "integrity": "sha512-WgXcWzVM6idy5JaftTVC8Vs83NKRmGJz4Hqs4oyOuO2J4r/y79vvKZsb+CaGyCSEbUPI6OsewfPd0G1A0/TUZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@adobe/css-tools": "^4.4.0", - "aria-query": "^5.0.0", - "css.escape": "^1.5.1", - "dom-accessibility-api": "^0.6.3", - "picocolors": "^1.1.1", - "redent": "^3.0.0" - }, - "engines": { - "node": ">=14", - "npm": ">=6", - "yarn": ">=1" - } - }, - "node_modules/@testing-library/react": { - "version": "14.3.1", - "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-14.3.1.tgz", - "integrity": "sha512-H99XjUhWQw0lTgyMN05W3xQG1Nh4lq574D8keFf1dDoNTJgp66VbJozRaczoF+wsiaPJNt/TcnfpLGufGxSrZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.12.5", - "@testing-library/dom": "^9.0.0", - "@types/react-dom": "^18.0.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "react": "^18.0.0", - "react-dom": "^18.0.0" - } - }, - "node_modules/@types/aria-query": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", - "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/babel__core": { - "version": "7.20.5", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", - "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.20.7", - "@babel/types": "^7.20.7", - "@types/babel__generator": "*", - "@types/babel__template": "*", - "@types/babel__traverse": "*" - } - }, - "node_modules/@types/babel__generator": { - "version": "7.27.0", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", - "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__template": { - "version": "7.4.4", - "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", - "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.1.0", - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__traverse": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", - "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.28.2" - } - }, - "node_modules/@types/graceful-fs": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", - "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/istanbul-lib-coverage": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", - "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/istanbul-lib-report": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", - "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/istanbul-lib-coverage": "*" - } - }, - "node_modules/@types/istanbul-reports": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", - "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/istanbul-lib-report": "*" - } - }, - "node_modules/@types/node": { - "version": "24.10.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", - "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/prismjs": { - "version": "1.26.5", - "resolved": "https://registry.npmjs.org/@types/prismjs/-/prismjs-1.26.5.tgz", - "integrity": "sha512-AUZTa7hQ2KY5L7AmtSiqxlhWxb4ina0yd8hNbl4TWuqnv/pFP0nDMb3YrfSBf4hJVGLh2YEIBfKaBW/9UEl6IQ==", - "license": "MIT" - }, - "node_modules/@types/prop-types": { - "version": "15.7.15", - "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", - "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/react": { - "version": "18.3.27", - "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.27.tgz", - "integrity": "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/prop-types": "*", - "csstype": "^3.2.2" - } - }, - "node_modules/@types/react-copy-to-clipboard": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/@types/react-copy-to-clipboard/-/react-copy-to-clipboard-5.0.7.tgz", - "integrity": "sha512-Gft19D+as4M+9Whq1oglhmK49vqPhcLzk8WfvfLvaYMIPYanyfLy0+CwFucMJfdKoSFyySPmkkWn8/E6voQXjQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/react": "*" - } - }, - "node_modules/@types/react-dom": { - "version": "18.3.7", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", - "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "@types/react": "^18.0.0" - } - }, - "node_modules/@types/stack-utils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", - "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/yargs": { - "version": "17.0.35", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", - "integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/yargs-parser": "*" - } - }, - "node_modules/@types/yargs-parser": { - "version": "21.0.3", - "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", - "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/ansi-escapes": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", - "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "type-fest": "^0.21.3" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/anymatch": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, - "license": "ISC", - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/aria-query": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.2.tgz", - "integrity": "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/array-buffer-byte-length": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz", - "integrity": "sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.3", - "is-array-buffer": "^3.0.5" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/available-typed-arrays": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", - "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "possible-typed-array-names": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/babel-jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", - "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/transform": "^29.7.0", - "@types/babel__core": "^7.1.14", - "babel-plugin-istanbul": "^6.1.1", - "babel-preset-jest": "^29.6.3", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.8.0" - } - }, - "node_modules/babel-plugin-istanbul": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", - "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/helper-plugin-utils": "^7.0.0", - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-instrument": "^5.0.4", - "test-exclude": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", - "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.12.3", - "@babel/parser": "^7.14.7", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/babel-plugin-jest-hoist": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", - "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.3.3", - "@babel/types": "^7.3.3", - "@types/babel__core": "^7.1.14", - "@types/babel__traverse": "^7.0.6" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/babel-preset-current-node-syntax": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", - "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/plugin-syntax-async-generators": "^7.8.4", - "@babel/plugin-syntax-bigint": "^7.8.3", - "@babel/plugin-syntax-class-properties": "^7.12.13", - "@babel/plugin-syntax-class-static-block": "^7.14.5", - "@babel/plugin-syntax-import-attributes": "^7.24.7", - "@babel/plugin-syntax-import-meta": "^7.10.4", - "@babel/plugin-syntax-json-strings": "^7.8.3", - "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", - "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", - "@babel/plugin-syntax-numeric-separator": "^7.10.4", - "@babel/plugin-syntax-object-rest-spread": "^7.8.3", - "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", - "@babel/plugin-syntax-optional-chaining": "^7.8.3", - "@babel/plugin-syntax-private-property-in-object": "^7.14.5", - "@babel/plugin-syntax-top-level-await": "^7.14.5" - }, - "peerDependencies": { - "@babel/core": "^7.0.0 || ^8.0.0-0" - } - }, - "node_modules/babel-preset-jest": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", - "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", - "dev": true, - "license": "MIT", - "dependencies": { - "babel-plugin-jest-hoist": "^29.6.3", - "babel-preset-current-node-syntax": "^1.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/baseline-browser-mapping": { - "version": "2.8.30", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.30.tgz", - "integrity": "sha512-aTUKW4ptQhS64+v2d6IkPzymEzzhw+G0bA1g3uBRV3+ntkH+svttKseW5IOR4Ed6NUVKqnY7qT3dKvzQ7io4AA==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "baseline-browser-mapping": "dist/cli.js" - } - }, - "node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/browserslist": { - "version": "4.28.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", - "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "baseline-browser-mapping": "^2.8.25", - "caniuse-lite": "^1.0.30001754", - "electron-to-chromium": "^1.5.249", - "node-releases": "^2.0.27", - "update-browserslist-db": "^1.1.4" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/bser": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", - "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "node-int64": "^0.4.0" - } - }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/call-bind": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", - "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.0", - "es-define-property": "^1.0.0", - "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/call-bound": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "get-intrinsic": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001756", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001756.tgz", - "integrity": "sha512-4HnCNKbMLkLdhJz3TToeVWHSnfJvPaq6vu/eRP0Ahub/07n484XHhBF5AJoSGHdVrS8tKFauUQz8Bp9P7LVx7A==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "CC-BY-4.0" - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/char-regex": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", - "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/ci-info": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", - "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/cjs-module-lexer": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", - "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/clsx": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", - "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">= 1.0.0", - "node": ">= 0.12.0" - } - }, - "node_modules/collect-v8-coverage": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.3.tgz", - "integrity": "sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw==", - "dev": true, - "license": "MIT" - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "license": "MIT" - }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/copy-to-clipboard": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.3.tgz", - "integrity": "sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==", - "license": "MIT", - "dependencies": { - "toggle-selection": "^1.0.6" - } - }, - "node_modules/create-jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", - "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-config": "^29.7.0", - "jest-util": "^29.7.0", - "prompts": "^2.0.1" - }, - "bin": { - "create-jest": "bin/create-jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/css.escape": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz", - "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==", - "dev": true, - "license": "MIT" - }, - "node_modules/csstype": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", - "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/dedent": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.0.tgz", - "integrity": "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "babel-plugin-macros": "^3.1.0" - }, - "peerDependenciesMeta": { - "babel-plugin-macros": { - "optional": true - } - } - }, - "node_modules/deep-equal": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.3.tgz", - "integrity": "sha512-ZIwpnevOurS8bpT4192sqAowWM76JDKSHYzMLty3BZGSswgq6pBaH3DhCSW5xVAZICZyKdOBPjwww5wfgT/6PA==", - "dev": true, - "license": "MIT", - "dependencies": { - "array-buffer-byte-length": "^1.0.0", - "call-bind": "^1.0.5", - "es-get-iterator": "^1.1.3", - "get-intrinsic": "^1.2.2", - "is-arguments": "^1.1.1", - "is-array-buffer": "^3.0.2", - "is-date-object": "^1.0.5", - "is-regex": "^1.1.4", - "is-shared-array-buffer": "^1.0.2", - "isarray": "^2.0.5", - "object-is": "^1.1.5", - "object-keys": "^1.1.1", - "object.assign": "^4.1.4", - "regexp.prototype.flags": "^1.5.1", - "side-channel": "^1.0.4", - "which-boxed-primitive": "^1.0.2", - "which-collection": "^1.0.1", - "which-typed-array": "^1.1.13" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/deepmerge": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", - "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/define-properties": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", - "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "dev": true, - "license": "MIT", - "dependencies": { - "define-data-property": "^1.0.1", - "has-property-descriptors": "^1.0.0", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/detect-newline": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", - "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/diff-sequences": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", - "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/dom-accessibility-api": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.6.3.tgz", - "integrity": "sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==", - "dev": true, - "license": "MIT" - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/electron-to-chromium": { - "version": "1.5.259", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.259.tgz", - "integrity": "sha512-I+oLXgpEJzD6Cwuwt1gYjxsDmu/S/Kd41mmLA3O+/uH2pFRO/DvOjUyGozL8j3KeLV6WyZ7ssPwELMsXCcsJAQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/emittery": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", - "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sindresorhus/emittery?sponsor=1" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/error-ex": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", - "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-arrayish": "^0.2.1" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-get-iterator": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz", - "integrity": "sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.1.3", - "has-symbols": "^1.0.3", - "is-arguments": "^1.1.1", - "is-map": "^2.0.2", - "is-set": "^2.0.2", - "is-string": "^1.0.7", - "isarray": "^2.0.5", - "stop-iteration-iterator": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-string-regexp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", - "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/execa": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.0", - "human-signals": "^2.1.0", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.1", - "onetime": "^5.1.2", - "signal-exit": "^3.0.3", - "strip-final-newline": "^2.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "node_modules/exit": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", - "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", - "dev": true, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/expect": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", - "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/expect-utils": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/fb-watchman": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", - "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "bser": "2.1.1" - } - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "license": "MIT", - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/for-each": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", - "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-callable": "^1.2.7" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/functions-have-names": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", - "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-package-type": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", - "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dev": true, - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/glob": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.0.tgz", - "integrity": "sha512-tvZgpqk6fz4BaNZ66ZsRaZnbHvP/jG3uKJvAZOwEVUL4RTA5nJeeLYfyN9/VA8NX/V3IBG+hkeuGpKjvELkVhA==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "minimatch": "^10.1.1", - "minipass": "^7.1.2", - "path-scurry": "^2.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/has-bigints": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.1.0.tgz", - "integrity": "sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/html-escaper": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", - "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", - "dev": true, - "license": "MIT" - }, - "node_modules/human-signals": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=10.17.0" - } - }, - "node_modules/import-local": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", - "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", - "dev": true, - "license": "MIT", - "dependencies": { - "pkg-dir": "^4.2.0", - "resolve-cwd": "^3.0.0" - }, - "bin": { - "import-local-fixture": "fixtures/cli.js" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/indent-string": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/internal-slot": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.1.0.tgz", - "integrity": "sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "hasown": "^2.0.2", - "side-channel": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/is-arguments": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", - "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-array-buffer": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", - "integrity": "sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.8", - "call-bound": "^1.0.3", - "get-intrinsic": "^1.2.6" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-arrayish": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", - "dev": true, - "license": "MIT" - }, - "node_modules/is-bigint": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.1.0.tgz", - "integrity": "sha512-n4ZT37wG78iz03xPRKJrHTdZbe3IicyucEtdRsV5yglwc3GyUfbAfpSeD0FJ41NbUNSt5wbhqfp1fS+BgnvDFQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-bigints": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-boolean-object": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.2.2.tgz", - "integrity": "sha512-wa56o2/ElJMYqjCjGkXri7it5FbebW5usLw/nPmCMs5DeZ7eziSYZhSmPRn0txqeW4LnAmQQU7FgqLpsEFKM4A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.3", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-core-module": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", - "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", - "dev": true, - "license": "MIT", - "dependencies": { - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-date-object": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.1.0.tgz", - "integrity": "sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-generator-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", - "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/is-map": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.3.tgz", - "integrity": "sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-number-object": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.1.1.tgz", - "integrity": "sha512-lZhclumE1G6VYD8VHe35wFaIif+CTy5SJIi5+3y4psDgWu4wPDoBhF8NxUOinEc7pHgiTsT6MaBb92rKhhD+Xw==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.3", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-regex": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", - "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "gopd": "^1.2.0", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-set": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.3.tgz", - "integrity": "sha512-iPAjerrse27/ygGLxw+EBR9agv9Y6uLeYVJMu+QNCoouJ1/1ri0mGrcWpfCqFZuzzx3WjtwxG098X+n4OuRkPg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-shared-array-buffer": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.4.tgz", - "integrity": "sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-string": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.1.1.tgz", - "integrity": "sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.3", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-symbol": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.1.1.tgz", - "integrity": "sha512-9gGx6GTtCQM73BgmHQXfDmLtfjjTUDSyoxTCbp5WtoixAhfgsDirWIcVQ/IHpvI5Vgd5i/J5F7B9cN/WlVbC/w==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "has-symbols": "^1.1.0", - "safe-regex-test": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-weakmap": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", - "integrity": "sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-weakset": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.4.tgz", - "integrity": "sha512-mfcwb6IzQyOKTs84CQMrOwW4gQcaTOAWJ0zzJCl2WSPDrWk/OzDaImWFH3djXhb24g4eudZfLRozAvPGw4d9hQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.3", - "get-intrinsic": "^1.2.6" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/isarray": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", - "dev": true, - "license": "MIT" - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, - "license": "ISC" - }, - "node_modules/istanbul-lib-coverage": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", - "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", - "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.23.9", - "@babel/parser": "^7.23.9", - "@istanbuljs/schema": "^0.1.3", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^7.5.4" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-instrument/node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-report": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", - "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^4.0.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-source-maps": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", - "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-reports": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", - "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", - "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/core": "^29.7.0", - "@jest/types": "^29.6.3", - "import-local": "^3.0.2", - "jest-cli": "^29.7.0" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-changed-files": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", - "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", - "dev": true, - "license": "MIT", - "dependencies": { - "execa": "^5.0.0", - "jest-util": "^29.7.0", - "p-limit": "^3.1.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-circus": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", - "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/expect": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "co": "^4.6.0", - "dedent": "^1.0.0", - "is-generator-fn": "^2.0.0", - "jest-each": "^29.7.0", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "p-limit": "^3.1.0", - "pretty-format": "^29.7.0", - "pure-rand": "^6.0.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-circus/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-circus/node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-circus/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-cli": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", - "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/core": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "create-jest": "^29.7.0", - "exit": "^0.1.2", - "import-local": "^3.0.2", - "jest-config": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "yargs": "^17.3.1" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-config": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", - "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/test-sequencer": "^29.7.0", - "@jest/types": "^29.6.3", - "babel-jest": "^29.7.0", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "deepmerge": "^4.2.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "jest-circus": "^29.7.0", - "jest-environment-node": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-runner": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "micromatch": "^4.0.4", - "parse-json": "^5.2.0", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@types/node": "*", - "ts-node": ">=9.0.0" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "ts-node": { - "optional": true - } - } - }, - "node_modules/jest-config/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-config/node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-config/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-diff": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", - "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "^29.6.3", - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-diff/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-diff/node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-diff/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-docblock": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", - "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", - "dev": true, - "license": "MIT", - "dependencies": { - "detect-newline": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-each": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", - "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "jest-get-type": "^29.6.3", - "jest-util": "^29.7.0", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-each/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-each/node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-each/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-environment-node": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", - "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/fake-timers": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-mock": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-get-type": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", - "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-haste-map": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", - "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/graceful-fs": "^4.1.3", - "@types/node": "*", - "anymatch": "^3.0.3", - "fb-watchman": "^2.0.0", - "graceful-fs": "^4.2.9", - "jest-regex-util": "^29.6.3", - "jest-util": "^29.7.0", - "jest-worker": "^29.7.0", - "micromatch": "^4.0.4", - "walker": "^1.0.8" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "optionalDependencies": { - "fsevents": "^2.3.2" - } - }, - "node_modules/jest-leak-detector": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", - "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-leak-detector/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-leak-detector/node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-leak-detector/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-matcher-utils": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", - "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "jest-diff": "^29.7.0", - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-matcher-utils/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-matcher-utils/node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-matcher-utils/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-message-util": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", - "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "^29.6.3", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.4", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-message-util/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-message-util/node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-message-util/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-mock": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", - "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-pnp-resolver": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", - "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - }, - "peerDependencies": { - "jest-resolve": "*" - }, - "peerDependenciesMeta": { - "jest-resolve": { - "optional": true - } - } - }, - "node_modules/jest-regex-util": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", - "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-resolve": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", - "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-pnp-resolver": "^1.2.2", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "resolve": "^1.20.0", - "resolve.exports": "^2.0.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-resolve-dependencies": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", - "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-regex-util": "^29.6.3", - "jest-snapshot": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-runner": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", - "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/environment": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "emittery": "^0.13.1", - "graceful-fs": "^4.2.9", - "jest-docblock": "^29.7.0", - "jest-environment-node": "^29.7.0", - "jest-haste-map": "^29.7.0", - "jest-leak-detector": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-resolve": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-util": "^29.7.0", - "jest-watcher": "^29.7.0", - "jest-worker": "^29.7.0", - "p-limit": "^3.1.0", - "source-map-support": "0.5.13" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-runtime": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", - "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/fake-timers": "^29.7.0", - "@jest/globals": "^29.7.0", - "@jest/source-map": "^29.6.3", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "cjs-module-lexer": "^1.0.0", - "collect-v8-coverage": "^1.0.0", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-mock": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "slash": "^3.0.0", - "strip-bom": "^4.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-snapshot": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", - "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@babel/generator": "^7.7.2", - "@babel/plugin-syntax-jsx": "^7.7.2", - "@babel/plugin-syntax-typescript": "^7.7.2", - "@babel/types": "^7.3.3", - "@jest/expect-utils": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "babel-preset-current-node-syntax": "^1.0.0", - "chalk": "^4.0.0", - "expect": "^29.7.0", - "graceful-fs": "^4.2.9", - "jest-diff": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "natural-compare": "^1.4.0", - "pretty-format": "^29.7.0", - "semver": "^7.5.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-snapshot/node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-snapshot/node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/jest-util": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", - "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "graceful-fs": "^4.2.9", - "picomatch": "^2.2.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-validate": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", - "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "camelcase": "^6.2.0", - "chalk": "^4.0.0", - "jest-get-type": "^29.6.3", - "leven": "^3.1.0", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-validate/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/jest-validate/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-validate/node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-validate/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/jest-watcher": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", - "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "emittery": "^0.13.1", - "jest-util": "^29.7.0", - "string-length": "^4.0.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-worker": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", - "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "jest-util": "^29.7.0", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-worker/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "license": "MIT" - }, - "node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/jsesc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", - "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", - "dev": true, - "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true, - "license": "MIT" - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/kleur": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", - "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, - "license": "MIT" - }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/loose-envify": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", - "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", - "license": "MIT", - "dependencies": { - "js-tokens": "^3.0.0 || ^4.0.0" - }, - "bin": { - "loose-envify": "cli.js" - } - }, - "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/lz-string": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", - "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", - "dev": true, - "license": "MIT", - "bin": { - "lz-string": "bin/bin.js" - } - }, - "node_modules/make-dir": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", - "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^7.5.3" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/make-dir/node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/makeerror": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", - "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "tmpl": "1.0.5" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", - "dev": true, - "license": "MIT" - }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/min-indent": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", - "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-int64": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", - "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-releases": { - "version": "2.0.27", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", - "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/object-inspect": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", - "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/object-is": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.6.tgz", - "integrity": "sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.7", - "define-properties": "^1.2.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/object.assign": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.7.tgz", - "integrity": "sha512-nK28WOo+QIjBkDduTINE4JkF/UJJKyf2EJxvJKfblDpyg0Q+pkOHNTL0Qwy6NP6FhE/EnzV73BxxqcJaXY9anw==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.8", - "call-bound": "^1.0.3", - "define-properties": "^1.2.1", - "es-object-atoms": "^1.0.0", - "has-symbols": "^1.1.0", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-locate/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/parse-json": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", - "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.0.0", - "error-ex": "^1.3.1", - "json-parse-even-better-errors": "^2.3.0", - "lines-and-columns": "^1.1.6" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, - "license": "MIT" - }, - "node_modules/path-scurry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", - "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/path-scurry/node_modules/lru-cache": { - "version": "11.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", - "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", - "dev": true, - "license": "ISC", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", - "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pirates": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", - "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/pkg-dir": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", - "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "find-up": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/possible-typed-array-names": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", - "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/pretty-format": { - "version": "27.5.1", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", - "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1", - "ansi-styles": "^5.0.0", - "react-is": "^17.0.1" - }, - "engines": { - "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" - } - }, - "node_modules/pretty-format/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/prism-react-renderer": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/prism-react-renderer/-/prism-react-renderer-2.4.1.tgz", - "integrity": "sha512-ey8Ls/+Di31eqzUxC46h8MksNuGx/n0AAC8uKpwFau4RPDYLuE3EXTp8N8G2vX2N7UC/+IXeNUnlWBGGcAG+Ig==", - "license": "MIT", - "dependencies": { - "@types/prismjs": "^1.26.0", - "clsx": "^2.0.0" - }, - "peerDependencies": { - "react": ">=16.0.0" - } - }, - "node_modules/prisma": { - "version": "5.17.0", - "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.17.0.tgz", - "integrity": "sha512-m4UWkN5lBE6yevqeOxEvmepnL5cNPEjzMw2IqDB59AcEV6w7D8vGljDLd1gPFH+W6gUxw9x7/RmN5dCS/WTPxA==", - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@prisma/engines": "5.17.0" - }, - "bin": { - "prisma": "build/index.js" - }, - "engines": { - "node": ">=16.13" - } - }, - "node_modules/prompts": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", - "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "kleur": "^3.0.3", - "sisteransi": "^1.0.5" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/prop-types": { - "version": "15.8.1", - "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", - "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", - "license": "MIT", - "dependencies": { - "loose-envify": "^1.4.0", - "object-assign": "^4.1.1", - "react-is": "^16.13.1" - } - }, - "node_modules/prop-types/node_modules/react-is": { - "version": "16.13.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", - "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", - "license": "MIT" - }, - "node_modules/pure-rand": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", - "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/dubzzz" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fast-check" - } - ], - "license": "MIT" - }, - "node_modules/react": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", - "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", - "license": "MIT", - "peer": true, - "dependencies": { - "loose-envify": "^1.1.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/react-copy-to-clipboard": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/react-copy-to-clipboard/-/react-copy-to-clipboard-5.1.1.tgz", - "integrity": "sha512-s+HrzLyJBxrpGTYXF15dTgMjAJpEPZT/Yp6NytAtZMRngejxt6Pt5WrfFxLAcsqUDU6sY1Jz6tyHwIicE1U2Xg==", - "license": "MIT", - "dependencies": { - "copy-to-clipboard": "^3.3.3", - "prop-types": "^15.8.1" - }, - "peerDependencies": { - "react": ">=15.3.0" - } - }, - "node_modules/react-dom": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", - "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "loose-envify": "^1.1.0", - "scheduler": "^0.23.2" - }, - "peerDependencies": { - "react": "^18.3.1" - } - }, - "node_modules/react-is": { - "version": "17.0.2", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", - "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", - "dev": true, - "license": "MIT" - }, - "node_modules/redent": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", - "integrity": "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==", - "dev": true, - "license": "MIT", - "dependencies": { - "indent-string": "^4.0.0", - "strip-indent": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/regexp.prototype.flags": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", - "integrity": "sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.8", - "define-properties": "^1.2.1", - "es-errors": "^1.3.0", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "set-function-name": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/resolve": { - "version": "1.22.11", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", - "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-core-module": "^2.16.1", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/resolve-cwd": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", - "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve.exports": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", - "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/safe-regex-test": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", - "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "is-regex": "^1.2.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/scheduler": { - "version": "0.23.2", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", - "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "loose-envify": "^1.1.0" - } - }, - "node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/set-function-length": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", - "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "dev": true, - "license": "MIT", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/set-function-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.2.tgz", - "integrity": "sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "functions-have-names": "^1.2.3", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", - "side-channel-map": "^1.0.1", - "side-channel-weakmap": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-map": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", - "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-weakmap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", - "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3", - "side-channel-map": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/sisteransi": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", - "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", - "dev": true, - "license": "MIT" - }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-support": { - "version": "0.5.13", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", - "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/stack-utils": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", - "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "escape-string-regexp": "^2.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/stop-iteration-iterator": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.1.0.tgz", - "integrity": "sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "internal-slot": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/string-length": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", - "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "char-regex": "^1.0.2", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-bom": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", - "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-final-newline": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/strip-indent": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", - "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "min-indent": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/test-exclude": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", - "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/tmpl": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", - "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/toggle-selection": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/toggle-selection/-/toggle-selection-1.0.6.tgz", - "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==", - "license": "MIT" - }, - "node_modules/type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "dev": true, - "license": "MIT" - }, - "node_modules/update-browserslist-db": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz", - "integrity": "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "escalade": "^3.2.0", - "picocolors": "^1.1.1" - }, - "bin": { - "update-browserslist-db": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/v8-to-istanbul": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", - "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", - "dev": true, - "license": "ISC", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.12", - "@types/istanbul-lib-coverage": "^2.0.1", - "convert-source-map": "^2.0.0" - }, - "engines": { - "node": ">=10.12.0" - } - }, - "node_modules/walker": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", - "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "makeerror": "1.0.12" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/which-boxed-primitive": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.1.1.tgz", - "integrity": "sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-bigint": "^1.1.0", - "is-boolean-object": "^1.2.1", - "is-number-object": "^1.1.1", - "is-string": "^1.1.1", - "is-symbol": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/which-collection": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.2.tgz", - "integrity": "sha512-K4jVyjnBdgvc86Y6BkaLZEN933SwYOuBFkdmBu9ZfkcAbdVbpITnDmjvZ/aQjRXQrv5EPkTnD1s39GiiqbngCw==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-map": "^2.0.3", - "is-set": "^2.0.3", - "is-weakmap": "^2.0.2", - "is-weakset": "^2.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/which-typed-array": { - "version": "1.1.19", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz", - "integrity": "sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==", - "dev": true, - "license": "MIT", - "dependencies": { - "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.8", - "call-bound": "^1.0.4", - "for-each": "^0.3.5", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/write-file-atomic": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", - "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", - "dev": true, - "license": "ISC", - "dependencies": { - "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.7" - }, - "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true, - "license": "ISC" - }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - } - } -} diff --git a/package.json b/package.json deleted file mode 100644 index be91e4170ab..00000000000 --- a/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "dependencies": { - "prism-react-renderer": "2.4.1", - "prisma": "5.17.0", - "react-copy-to-clipboard": "5.1.1" - }, - "devDependencies": { - "@testing-library/jest-dom": "6.8.0", - "@testing-library/react": "14.3.1", - "@types/react-copy-to-clipboard": "5.0.7", - "jest": "29.7.0" - }, - "overrides": { - "glob": "13.0.0", - "minimatch": "10.2.5", - "picomatch": "2.3.2", - "@isaacs/brace-expansion": "5.0.0", - "@babel/traverse": "7.28.5", - "braces": "3.0.3" - } -} diff --git a/qa_sticky_session.sh b/qa_sticky_session.sh deleted file mode 100755 index 2c5a1a66c4c..00000000000 --- a/qa_sticky_session.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# QA: code interpreter sandbox stickiness via metadata.session_id -# bash qa_sticky_session.sh -# LITELLM_BASE_URL=http://localhost:4000 LITELLM_KEY="$LITELLM_MASTER_KEY" bash qa_sticky_session.sh - -set -euo pipefail - -BASE="${LITELLM_BASE_URL:-http://localhost:4000}" -KEY="${LITELLM_KEY:-${LITELLM_MASTER_KEY:?set LITELLM_KEY or LITELLM_MASTER_KEY}}" -MODEL="${LITELLM_MODEL:-gpt-4o-mini}" -# proxy running at http://localhost:4000 -SESSION_A="qa-session-$(date +%s)-A" -SESSION_B="qa-session-$(date +%s)-B" - -content() { - echo "$1" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('choices',[{}])[0].get('message',{}).get('content',''))" -} - -call() { - local session="${1:-}" code="$2" meta="" - [[ -n "$session" ]] && meta=", \"metadata\": {\"session_id\": \"$session\"}" - curl -s -X POST "$BASE/chat/completions" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $KEY" \ - -d "{\"model\":\"$MODEL\"$meta,\"tools\":[{\"type\":\"code_interpreter\"}],\"messages\":[{\"role\":\"user\",\"content\":\"Run this Python code and tell me the result: $code\"}]}" -} - -assert_match() { - local label="$1" body="$2" pattern="$3" - if echo "$body" | grep -qiE "$pattern"; then - echo "PASS $label" - else - echo "FAIL $label (expected /$pattern/)" - echo " $(content "$body")" - exit 1 - fi -} - -echo "=== Sticky Session Sandbox QA ===" -echo "base: $BASE session A: $SESSION_A session B: $SESSION_B" -echo - -R=$(call "$SESSION_A" "x = 42; print(x)") -assert_match "same session_id reuses sandbox (set x=42)" "$R" "42" - -R=$(call "$SESSION_A" "print(x)") -assert_match "same session_id keeps state (x still 42)" "$R" "42" - -R=$(call "$SESSION_B" "print(x)") -assert_match "different session_id is isolated" "$R" "not defined|NameError|undefined|error" - -R=$(call "" "y = 99; print(y)") -assert_match "no session_id runs code" "$R" "99" - -R=$(call "" "print(y)") -assert_match "no session_id gets fresh sandbox each request" "$R" "not defined|NameError|undefined|error" - -echo -echo "All checks passed." diff --git a/router_plugins.json b/router_plugins.json deleted file mode 100644 index ffcddf89fd1..00000000000 --- a/router_plugins.json +++ /dev/null @@ -1,28 +0,0 @@ -[ - { - "name": "TEMPLATE: copy this block for a new plugin, then delete this entry", - "description": "One line on what the plugin does and the routing signal it publishes.", - "author": "Plugin author's name.", - "repo": "https://github.com// (public source repository).", - "commit": "Full 40-char git SHA to pin when the plugin is not yet on PyPI; omit once 'pypi' is set.", - "version": "Plugin release version, e.g. 1.0.0.", - "pypi": "PyPI spec pinned to a version, e.g. my-plugin==1.0.0, or null if unpublished.", - "litellm_version": "Minimum compatible litellm version, e.g. >=1.94.0.", - "entrypoint": "Dotted import path to the plugin instance, e.g. my_plugin.plugin.instance.", - "license": "SPDX license id, e.g. MIT.", - "tags": ["searchable", "keywords"] - }, - { - "name": "language-detector", - "description": "Detects the user's language and publishes a routing signal.", - "author": "Jean Nuñez", - "repo": "https://github.com/jeann2013/language-detector", - "commit": "9e712819269173fc25a16f59ca3e9890f7864ac1", - "version": "1.0.0", - "pypi": null, - "litellm_version": ">=1.94.0", - "entrypoint": "litellm_plugin_language_detector.plugin.language_detector_plugin", - "license": "MIT", - "tags": ["language", "classification", "routing"] - } -] diff --git a/taplo.toml b/taplo.toml deleted file mode 100644 index e5065a69616..00000000000 --- a/taplo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[formatting] - -# Keep your table/key order as written (project, project.scripts, dependency-groups, tool.uv, ...) -reorder_keys = false - -# Force arrays to stay multiline once expanded -array_auto_expand = true -array_auto_collapse = false - -# Keep nice spacing inside arrays and inline tables -compact_arrays = false -compact_inline_tables = true - -# Don’t align `=` vertically (matches your example) -align_entries = true - -# Reasonable defaults -align_comments = true -trailing_newline = true -reorder_arrays = true -reorder_inline_tables = false -allowed_blank_lines = 2 -crlf = false From 2c44f26288c57e0b646e01985b4265a0e2ce856d Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 11 Nov 2026 15:38:27 -0800 Subject: [PATCH 124/189] chore: remove cookbook and preserve live assets (#50944) --- .dockerignore | 2 +- .github/workflows/test-unit.yml | 1 - cookbook/Benchmarking_LLMs_by_use_case.ipynb | 753 ------------ ...pic)_with_Streaming_liteLLM_Examples.ipynb | 406 ------- cookbook/Evaluating_LLMs.ipynb | 579 ---------- .../LiteLLM_Azure_and_OpenAI_example.ipynb | 422 ------- cookbook/LiteLLM_Bedrock.ipynb | 310 ----- cookbook/LiteLLM_CometAPI.ipynb | 474 -------- cookbook/LiteLLM_Comparing_LLMs.ipynb | 441 -------- cookbook/LiteLLM_Completion_Cost.ipynb | 241 ---- cookbook/LiteLLM_HuggingFace.ipynb | 252 ----- cookbook/LiteLLM_NovitaAI_Cookbook.ipynb | 97 -- cookbook/LiteLLM_OpenRouter.ipynb | 179 --- cookbook/LiteLLM_Petals.ipynb | 568 ---------- cookbook/LiteLLM_PromptLayer.ipynb | 130 --- cookbook/LiteLLM_User_Based_Rate_Limits.ipynb | 224 ---- cookbook/LiteLLM_batch_completion.ipynb | 163 --- ...teLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb | 568 ---------- cookbook/Parallel_function_calling.ipynb | 478 -------- cookbook/Proxy_Batch_Users.ipynb | 208 ---- cookbook/TogetherAI_liteLLM.ipynb | 1006 ----------------- ..._Nemo_Guardrails_with_LiteLLM_Server.ipynb | 157 --- cookbook/VLLM_Model_Testing.ipynb | 404 ------- .../chatgpt_subscription_passthrough.md | 114 -- .../claude_code_quickstart/guide.md | 294 ----- cookbook/ai_coding_tool_guides/index.json | 134 --- cookbook/anthropic_agent_sdk/README.md | 150 --- .../anthropic_agent_sdk/agent_with_mcp.py | 144 --- cookbook/anthropic_agent_sdk/common.py | 164 --- .../anthropic_agent_sdk/config.example.yaml | 25 - cookbook/anthropic_agent_sdk/main.py | 99 -- cookbook/anthropic_agent_sdk/requirements.txt | 2 - cookbook/benchmark/benchmark.py | 90 -- .../auto_evals.py | 30 - cookbook/benchmark/readme.md | 181 --- cookbook/codellama-server/README.MD | 153 --- .../codellama-server/imgs/code-output.png | Bin 238123 -> 0 bytes .../imgs/promptlayer_logging.png | Bin 300105 -> 0 bytes cookbook/codellama-server/main.py | 102 -- cookbook/community-resources/get_hf_models.py | 89 -- cookbook/community-resources/max_tokens.json | 93 -- cookbook/gollem_go_agent_framework/README.md | 119 -- .../gollem_go_agent_framework/basic/main.go | 41 - cookbook/gollem_go_agent_framework/go.mod | 5 - cookbook/gollem_go_agent_framework/go.sum | 2 - .../proxy_config.yaml | 16 - .../streaming/main.go | 56 - .../gollem_go_agent_framework/tools/main.go | 64 -- cookbook/google_adk_litellm_tutorial.ipynb | 412 ------- cookbook/gpt_realtime_translate.py | 245 ---- cookbook/liteLLM_A121_Jurrasic_example.ipynb | 251 ---- cookbook/liteLLM_Baseten.ipynb | 146 --- cookbook/liteLLM_Getting_Started.ipynb | 411 ------- cookbook/liteLLM_IBM_Watsonx.ipynb | 300 ----- cookbook/liteLLM_Langchain_Demo.ipynb | 195 ---- cookbook/liteLLM_Ollama.ipynb | 289 ----- cookbook/liteLLM_Replicate_Demo.ipynb | 238 ---- cookbook/liteLLM_Streaming_Demo.ipynb | 226 ---- cookbook/liteLLM_VertextAI_Example.ipynb | 199 ---- cookbook/liteLLM_clarifai_Demo.ipynb | 187 --- cookbook/liteLLM_function_calling.ipynb | 331 ------ .../litellm_Test_Multiple_Providers.ipynb | 571 ---------- cookbook/litellm_model_fallback.ipynb | 51 - .../batch_api/bedrock/README.md | 41 - .../batch_api/bedrock/bedrock.py | 27 - .../bedrock/bedrock_batch_completions.jsonl | 128 --- .../braintrust_prompt_wrapper_README.md | 279 ----- .../braintrust_prompt_wrapper_server.py | 274 ----- .../litellm_proxy_server/cli_token_usage.py | 63 -- .../mcp/mcp_with_litellm_proxy.py | 39 - cookbook/litellm_proxy_server/readme.md | 177 --- .../custom_secret_manager_config.yaml | 20 - .../secret_manager/my_secret_manager.py | 80 -- cookbook/litellm_router/error_log.txt | 1004 ---------------- cookbook/litellm_router/load_test_proxy.py | 148 --- cookbook/litellm_router/load_test_queuing.py | 164 --- cookbook/litellm_router/load_test_router.py | 143 --- cookbook/litellm_router/request_log.txt | 48 - cookbook/litellm_router/response_log.txt | 0 .../test_questions/question1.txt | 43 - .../test_questions/question2.txt | 65 -- .../test_questions/question3.txt | 50 - .../memory_usage/router_endpoint.py | 65 -- .../memory_usage/router_memory_usage copy.py | 91 -- .../memory_usage/router_memory_usage.py | 91 -- .../memory_usage/send_request.py | 28 - .../test_loadtest_openai_client.py | 73 -- .../test_loadtest_router.py | 87 -- .../test_loadtest_router_withs3_cache.py | 93 -- cookbook/litellm_test_multiple_llm_demo.ipynb | 55 - cookbook/livekit_agent_sdk/README.md | 116 -- .../livekit_agent_sdk/config.example.yaml | 20 - cookbook/livekit_agent_sdk/main.py | 121 -- cookbook/livekit_agent_sdk/requirements.txt | 2 - .../logging_observability/LiteLLM_Arize.ipynb | 172 --- .../LiteLLM_Langfuse.ipynb | 197 ---- .../LiteLLM_Lunary.ipynb | 348 ------ .../LiteLLM_Proxy_Langfuse.ipynb | 252 ----- .../litellm_proxy_langfuse.png | Bin 315558 -> 0 bytes .../RELEASE_NOTES_GENERATION_INSTRUCTIONS.md | 543 --------- cookbook/misc/add_new_models.py | 71 -- cookbook/misc/config.yaml | 72 -- cookbook/misc/dev_release.txt | 11 - cookbook/misc/migrate_proxy_config.py | 97 -- cookbook/misc/openai_timeouts.py | 33 - cookbook/misc/sagmaker_streaming.py | 55 - cookbook/misc/test_responses_api.py | 51 - cookbook/misc/update_json_caching.py | 54 - ...flow_langchain_tracing_litellm_proxy.ipynb | 315 ------ .../mock_bedrock_guardrail_server.py | 540 --------- .../mock_prompt_management_server/README.md | 292 ----- .../mock_prompt_management_server.py | 390 ------- cookbook/nova_sonic_realtime.py | 292 ----- cookbook/result.html | 22 - cookbook/veo_video_generation.py | 321 ------ .../grafana_dashboard.json | 0 .../dashboard_all_metrics/readme.md | 0 .../grafana_dashboard.json | 0 .../dashboard_genai_otel/readme.md | 0 .../dashboard_v2/grafana_dashboard.json | 0 .../grafana_dashboards}/readme.md | 0 .../skill_scanner_adapter}/README.md | 4 +- .../skill_scanner_adapter}/server.py | 0 pyproject.toml | 3 + pyrightconfig.json | 2 +- ruff.toml | 2 + scripts/check_type_discipline.py | 8 +- .../router_surface_check.py | 1 - .../security/test_skill_scanner.py | 2 +- tests/unit/cookbook/__init__.py | 0 .../cookbook/test_gpt_realtime_translate.py | 141 --- ...test_prometheus_metric_name_consistency.py | 2 +- .../test_skill_scanning.py | 4 +- tests/unit/test_check_type_discipline.py | 13 + 134 files changed, 32 insertions(+), 22195 deletions(-) delete mode 100644 cookbook/Benchmarking_LLMs_by_use_case.ipynb delete mode 100644 cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb delete mode 100644 cookbook/Evaluating_LLMs.ipynb delete mode 100644 cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb delete mode 100644 cookbook/LiteLLM_Bedrock.ipynb delete mode 100644 cookbook/LiteLLM_CometAPI.ipynb delete mode 100644 cookbook/LiteLLM_Comparing_LLMs.ipynb delete mode 100644 cookbook/LiteLLM_Completion_Cost.ipynb delete mode 100644 cookbook/LiteLLM_HuggingFace.ipynb delete mode 100644 cookbook/LiteLLM_NovitaAI_Cookbook.ipynb delete mode 100644 cookbook/LiteLLM_OpenRouter.ipynb delete mode 100644 cookbook/LiteLLM_Petals.ipynb delete mode 100644 cookbook/LiteLLM_PromptLayer.ipynb delete mode 100644 cookbook/LiteLLM_User_Based_Rate_Limits.ipynb delete mode 100644 cookbook/LiteLLM_batch_completion.ipynb delete mode 100644 cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb delete mode 100644 cookbook/Parallel_function_calling.ipynb delete mode 100644 cookbook/Proxy_Batch_Users.ipynb delete mode 100644 cookbook/TogetherAI_liteLLM.ipynb delete mode 100644 cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb delete mode 100644 cookbook/VLLM_Model_Testing.ipynb delete mode 100644 cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md delete mode 100644 cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md delete mode 100644 cookbook/ai_coding_tool_guides/index.json delete mode 100644 cookbook/anthropic_agent_sdk/README.md delete mode 100644 cookbook/anthropic_agent_sdk/agent_with_mcp.py delete mode 100644 cookbook/anthropic_agent_sdk/common.py delete mode 100644 cookbook/anthropic_agent_sdk/config.example.yaml delete mode 100644 cookbook/anthropic_agent_sdk/main.py delete mode 100644 cookbook/anthropic_agent_sdk/requirements.txt delete mode 100644 cookbook/benchmark/benchmark.py delete mode 100644 cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py delete mode 100644 cookbook/benchmark/readme.md delete mode 100644 cookbook/codellama-server/README.MD delete mode 100644 cookbook/codellama-server/imgs/code-output.png delete mode 100644 cookbook/codellama-server/imgs/promptlayer_logging.png delete mode 100644 cookbook/codellama-server/main.py delete mode 100644 cookbook/community-resources/get_hf_models.py delete mode 100644 cookbook/community-resources/max_tokens.json delete mode 100644 cookbook/gollem_go_agent_framework/README.md delete mode 100644 cookbook/gollem_go_agent_framework/basic/main.go delete mode 100644 cookbook/gollem_go_agent_framework/go.mod delete mode 100644 cookbook/gollem_go_agent_framework/go.sum delete mode 100644 cookbook/gollem_go_agent_framework/proxy_config.yaml delete mode 100644 cookbook/gollem_go_agent_framework/streaming/main.go delete mode 100644 cookbook/gollem_go_agent_framework/tools/main.go delete mode 100644 cookbook/google_adk_litellm_tutorial.ipynb delete mode 100644 cookbook/gpt_realtime_translate.py delete mode 100644 cookbook/liteLLM_A121_Jurrasic_example.ipynb delete mode 100644 cookbook/liteLLM_Baseten.ipynb delete mode 100644 cookbook/liteLLM_Getting_Started.ipynb delete mode 100644 cookbook/liteLLM_IBM_Watsonx.ipynb delete mode 100644 cookbook/liteLLM_Langchain_Demo.ipynb delete mode 100644 cookbook/liteLLM_Ollama.ipynb delete mode 100644 cookbook/liteLLM_Replicate_Demo.ipynb delete mode 100644 cookbook/liteLLM_Streaming_Demo.ipynb delete mode 100644 cookbook/liteLLM_VertextAI_Example.ipynb delete mode 100644 cookbook/liteLLM_clarifai_Demo.ipynb delete mode 100644 cookbook/liteLLM_function_calling.ipynb delete mode 100644 cookbook/litellm_Test_Multiple_Providers.ipynb delete mode 100644 cookbook/litellm_model_fallback.ipynb delete mode 100644 cookbook/litellm_proxy_server/batch_api/bedrock/README.md delete mode 100644 cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py delete mode 100644 cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl delete mode 100644 cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md delete mode 100644 cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py delete mode 100644 cookbook/litellm_proxy_server/cli_token_usage.py delete mode 100644 cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py delete mode 100644 cookbook/litellm_proxy_server/readme.md delete mode 100644 cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml delete mode 100644 cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py delete mode 100644 cookbook/litellm_router/error_log.txt delete mode 100644 cookbook/litellm_router/load_test_proxy.py delete mode 100644 cookbook/litellm_router/load_test_queuing.py delete mode 100644 cookbook/litellm_router/load_test_router.py delete mode 100644 cookbook/litellm_router/request_log.txt delete mode 100644 cookbook/litellm_router/response_log.txt delete mode 100644 cookbook/litellm_router/test_questions/question1.txt delete mode 100644 cookbook/litellm_router/test_questions/question2.txt delete mode 100644 cookbook/litellm_router/test_questions/question3.txt delete mode 100644 cookbook/litellm_router_load_test/memory_usage/router_endpoint.py delete mode 100644 cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py delete mode 100644 cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py delete mode 100644 cookbook/litellm_router_load_test/memory_usage/send_request.py delete mode 100644 cookbook/litellm_router_load_test/test_loadtest_openai_client.py delete mode 100644 cookbook/litellm_router_load_test/test_loadtest_router.py delete mode 100644 cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py delete mode 100644 cookbook/litellm_test_multiple_llm_demo.ipynb delete mode 100644 cookbook/livekit_agent_sdk/README.md delete mode 100644 cookbook/livekit_agent_sdk/config.example.yaml delete mode 100644 cookbook/livekit_agent_sdk/main.py delete mode 100644 cookbook/livekit_agent_sdk/requirements.txt delete mode 100644 cookbook/logging_observability/LiteLLM_Arize.ipynb delete mode 100644 cookbook/logging_observability/LiteLLM_Langfuse.ipynb delete mode 100644 cookbook/logging_observability/LiteLLM_Lunary.ipynb delete mode 100644 cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb delete mode 100644 cookbook/logging_observability/litellm_proxy_langfuse.png delete mode 100644 cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md delete mode 100644 cookbook/misc/add_new_models.py delete mode 100644 cookbook/misc/config.yaml delete mode 100644 cookbook/misc/dev_release.txt delete mode 100644 cookbook/misc/migrate_proxy_config.py delete mode 100644 cookbook/misc/openai_timeouts.py delete mode 100644 cookbook/misc/sagmaker_streaming.py delete mode 100644 cookbook/misc/test_responses_api.py delete mode 100644 cookbook/misc/update_json_caching.py delete mode 100644 cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb delete mode 100644 cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py delete mode 100644 cookbook/mock_prompt_management_server/README.md delete mode 100644 cookbook/mock_prompt_management_server/mock_prompt_management_server.py delete mode 100644 cookbook/nova_sonic_realtime.py delete mode 100644 cookbook/result.html delete mode 100644 cookbook/veo_video_generation.py rename {cookbook/litellm_proxy_server/grafana_dashboard => litellm/integrations/prometheus_helpers/grafana_dashboards}/dashboard_all_metrics/grafana_dashboard.json (100%) rename {cookbook/litellm_proxy_server/grafana_dashboard => litellm/integrations/prometheus_helpers/grafana_dashboards}/dashboard_all_metrics/readme.md (100%) rename {cookbook/litellm_proxy_server/grafana_dashboard => litellm/integrations/prometheus_helpers/grafana_dashboards}/dashboard_genai_otel/grafana_dashboard.json (100%) rename {cookbook/litellm_proxy_server/grafana_dashboard => litellm/integrations/prometheus_helpers/grafana_dashboards}/dashboard_genai_otel/readme.md (100%) rename {cookbook/litellm_proxy_server/grafana_dashboard => litellm/integrations/prometheus_helpers/grafana_dashboards}/dashboard_v2/grafana_dashboard.json (100%) rename {cookbook/litellm_proxy_server/grafana_dashboard => litellm/integrations/prometheus_helpers/grafana_dashboards}/readme.md (100%) rename {cookbook/skill_scanner => litellm/proxy/discovery_endpoints/skill_scanner_adapter}/README.md (94%) rename {cookbook/skill_scanner => litellm/proxy/discovery_endpoints/skill_scanner_adapter}/server.py (100%) delete mode 100644 tests/unit/cookbook/__init__.py delete mode 100644 tests/unit/cookbook/test_gpt_realtime_translate.py diff --git a/.dockerignore b/.dockerignore index 8459d0cd482..f0fdcf9d479 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,5 @@ docs -cookbook +litellm/proxy/discovery_endpoints/skill_scanner_adapter .circleci .github tests diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index d0726e0daa0..6f980b458fc 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -460,7 +460,6 @@ jobs: tests/unit/ocr tests/unit/passthrough tests/unit/rag - tests/unit/cookbook tests/unit/realtime_api tests/unit/rerank_api tests/unit/search diff --git a/cookbook/Benchmarking_LLMs_by_use_case.ipynb b/cookbook/Benchmarking_LLMs_by_use_case.ipynb deleted file mode 100644 index 6ea6211bfb6..00000000000 --- a/cookbook/Benchmarking_LLMs_by_use_case.ipynb +++ /dev/null @@ -1,753 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "4Cq-_Y-TKf0r" - }, - "source": [ - "# LiteLLM - Benchmark Llama2, Claude1.2 and GPT3.5 for a use case\n", - "In this notebook for a given use case we run the same question and view:\n", - "* LLM Response\n", - "* Response Time\n", - "* Response Cost\n", - "\n", - "## Sample output for a question\n", - "![Screenshot 2023-09-07 at 4.45.37 PM.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAACXAAAALmCAYAAADhDS65AAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEAIEHoNvQkiNYCUEFoA6UWwEZIAocQYCCp2ZFHBtYsFbOiqiGIHxI7YWRR7X1BRUdbFgl15kwK67ivfm++bO//958x/zpw7c+8dAGgnuGJxHqoBQL6oUBIfFsQYnZrGID0FREADegAFRlxegZgVGxsFYBls/17e3QCIrL3qJNP6Z/9/LZp8QQEPACQW4gx+AS8f4gMA4NU8saQQAKKMt5xcKJZhWIG2BAYI8XwZzlLgahnOUOA9cpvEeDbErQCoqHG5kiwA1C9DnlHEy4Ia6n0Qu4j4QhEANAbE/vn5E/kQp0NsB23EEMv0mRk/6GT9TTNjSJPLzRrCirnIi0qwsECcx536f6bjf5f8POmgDxtY1bIl4fGyOcO83cqdGCnDahD3ijKiYyDWgviDkC+3hxilZEvDkxT2qDGvgA1zBnQhduFzgyMhNoY4VJQXHaXkMzKFoRyI4QpBpwgLOYkQ60M8X1AQkqC02SiZGK/0hTZkStgsJX+OK5H7lfl6IM1NYin1X2cLOEp9TL04OzEFYgrEVkXC5GiI1SF2LshNiFTajCzOZkcP2kik8bL4rSCOF4jCghT6WFGmJDReaV+eXzA4X2xjtpATrcT7CrMTwxX5wVp5XHn8cC7YZYGIlTSoIygYHTU4F74gOEQxd+yZQJSUoNT5IC4MileMxSnivFilPW4hyAuT8RYQuxcUJSjH4smFcEEq9PFMcWFsoiJOvDiHGxGriAdfAqIAGwQDBpDCmgEmghwgbO9t7IV3ip5QwAUSkAUEwEnJDI5IkfeI4DUBFIM/IRKAgqFxQfJeASiC/NchVnF1Apny3iL5iFzwBOJ8EAny4L1UPko05C0ZPIaM8B/eubDyYLx5sMr6/z0/yH5nWJCJUjLSQY8M2qAlMYQYTAwnhhLtcUPcH/fFo+A1EFZXnIl7D87juz3hCaGD8JBwndBJuD1BWCL5KcpRoBPqhypzkfFjLnAbqOmBB+F+UB0q47q4IXDC3aEfFh4APXtAlq2MW5YVxk/af5vBD09DaUd2IaNkPXIg2e7nkeoO6h5DKrJc/5gfRawZQ/lmD/X87J/9Q/b5sI382RKbj+3HzmInsfPYEawRMLDjWBPWhh2V4aHV9Vi+uga9xcvjyYU6wn/4G3yyskwWuNS59Lh8UfQVCqbI3tGAPVE8VSLMyi5ksOAXQcDgiHjOwxiuLq5uAMi+L4rX15s4+XcD0W37zs39AwC/4wMDA4e/cxHHAdjrBbf/oe+cHRN+OlQBOHeIJ5UUKThcdiHAtwQN7jQDYAosgR2cjyvwBL4gEISACBADEkEqGA+jz4brXAImg+lgDigDFWAJWAnWgg1gM9gOdoF9oBEcASfBGXARXAbXwV24errBC9AH3oHPCIKQECpCRwwQM8QacURcESbij4QgUUg8koqkI1mICJEi05G5SAWyDFmLbEJqkb3IIeQkch7pQG4jXUgP8hr5hGKoGqqNmqA26HCUibLQSDQRHYdmoZPQYrQUXYSuRmvQnWgDehK9iF5HO9EXaD8GMFVMFzPHnDAmxsZisDQsE5NgM7FyrBKrweqxZvicr2KdWC/2ESfidJyBO8EVHI4n4Tx8Ej4TX4ivxbfjDXgrfhXvwvvwbwQqwZjgSPAhcAijCVmEyYQyQiVhK+Eg4TTcS92Ed0QiUZdoS/SCezGVmEOcRlxIXEfcTTxB7CA+IvaTSCQDkiPJjxRD4pIKSWWkNaSdpOOkK6Ru0gcVVRUzFVeVUJU0FZFKiUqlyg6VYypXVJ6qfCZrkK3JPuQYMp88lbyYvIXcTL5E7iZ/pmhSbCl+lERKDmUOZTWlnnKaco/yRlVV1ULVWzVOVag6W3W16h7Vc6pdqh/VtNQc1NhqY9WkaovUtqmdULut9oZKpdpQA6lp1ELqImot9RT1AfWDOl3dWZ2jzlefpV6l3qB+Rf0ljUyzprFo42nFtEraftolWq8GWcNGg63B1ZipUaVxSOOmRr8mXXOEZoxmvuZCzR2a5zWfaZG0bLRCtPhapVqbtU5pPaJjdEs6m86jz6VvoZ+md2sTtW21Odo52hXau7Tbtft0tHTcdZJ1puhU6RzV6dTFdG10Obp5uot19+ne0P2kZ6LH0hPoLdCr17ui917fSD9QX6Bfrr9b/7r+JwOGQYhBrsFSg0aD+4a4oYNhnOFkw/WGpw17jbSNfI14RuVG+4zuGKPGDsbxxtOMNxu3GfebmJqEmYhN1picMuk11TUNNM0xXWF6zLTHjG7mbyY0W2F23Ow5Q4fBYuQxVjNaGX3mxubh5lLzTebt5p8tbC2SLEosdlvct6RYMi0zLVdYtlj2WZlZjbKablVndceabM20zrZeZX3W+r2NrU2KzTybRptntvq2HNti2zrbe3ZUuwC7SXY1dtfsifZM+1z7dfaXHVAHD4dshyqHS46oo6ej0HGdY8cwwjDvYaJhNcNuOqk5sZyKnOqcupx1naOcS5wbnV8OtxqeNnzp8LPDv7l4uOS5bHG5O0JrRMSIkhHNI167OrjyXKtcr7lR3ULdZrk1ub1yd3QXuK93v+VB9xjlMc+jxeOrp5enxLPes8fLyivdq9rrJlObGctcyDznTfAO8p7lfcT7o4+nT6HPPp+/fJ18c313+D4baTtSMHLLyEd+Fn5cv01+nf4M/3T/jf6dAeYB3ICagIeBloH8wK2BT1n2rBzWTtbLIJcgSdDBoPdsH/YM9olgLDgsuDy4PUQrJClkbciDUIvQrNC60L4wj7BpYSfCCeGR4UvDb3JMODxOLacvwitiRkRrpFpkQuTayIdRDlGSqOZR6KiIUctH3Yu2jhZFN8aAGE7M8pj7sbaxk2IPxxHjYuOq4p7Ej4ifHn82gZ4wIWFHwrvEoMTFiXeT7JKkSS3JtOSxybXJ71OCU5aldI4ePnrG6IuphqnC1KY0Ulpy2ta0/jEhY1aO6R7rMbZs7I1xtuOmjDs/3nB83vijE2gTuBP2pxPSU9J3pH/hxnBruP0ZnIzqjD4em7eK94IfyF/B7xH4CZYJnmb6ZS7LfJbll7U8qyc7ILsyu1fIFq4VvsoJz9mQ8z43Jndb7kBeSt7ufJX89PxDIi1Rrqh1ounEKRM7xI7iMnHnJJ9JKyf1SSIlWwuQgnEFTYXa8Ee+TWon/UXaVeRfVFX0YXLy5P1TNKeIprRNdZi6YOrT4tDi36bh03jTWqabT58zvWsGa8ammcjMjJktsyxnlc7qnh02e/scypzcOb+XuJQsK3k7N2Vuc6lJ6ezSR7+E/VJXpl4mKbs5z3fehvn4fOH89gVuC9Ys+FbOL79Q4VJRWfFlIW/hhV9H/Lr614FFmYvaF3suXr+EuES05MbSgKXbl2kuK172aPmo5Q0rGCvKV7xdOWHl+Ur3yg2rKKukqzpXR61uWmO1ZsmaL2uz116vCqraXW1cvaD6/Tr+uivrA9fXbzDZULHh00bhxlubwjY11NjUVG4mbi7a/GRL8pazvzF/q91quLVi69dtom2d2+O3t9Z61dbuMN6xuA6tk9b17By78/Ku4F1N9U71m3br7q7YA/ZI9zzfm773xr7IfS37mfvrD1gfqD5IP1jegDRMbehrzG7sbEpt6jgUcail2bf54GHnw9uOmB+pOqpzdPExyrHSYwPHi4/3nxCf6D2ZdfJRy4SWu6dGn7rWGtfafjry9LkzoWdOnWWdPX7O79yR8z7nD11gXmi86Hmxoc2j7eDvHr8fbPdsb7jkdanpsvfl5o6RHceuBFw5eTX46plrnGsXr0df77iRdOPWzbE3O2/xbz27nXf71Z2iO5/vzr5HuFd+X+N+5QPjBzV/2P+xu9Oz82hXcFfbw4SHdx/xHr14XPD4S3fpE+qTyqdmT2ufuT470hPac/n5mOfdL8QvPveW/an5Z/VLu5cH/gr8q61vdF/3K8mrgdcL3xi82fbW/W1Lf2z/g3f57z6/L/9g8GH7R+bHs59SPj39PPkL6cvqr/Zfm79Ffrs3kD8wIOZKuPJfAQxWNDMTgNfbAKCmAkCH5zPKGMX5T14QxZlVjsB/woozorx4AlAP/9/jeuHfzU0A9myBxy+oTxsLQCwVgERvgLq5DdXBs5r8XCkrRHgO2Jj4NSM/A/ybojhz/hD3zy2QqbqDn9t/AendfFfdLXJmAAAAimVYSWZNTQAqAAAACAAEARoABQAAAAEAAAA+ARsABQAAAAEAAABGASgAAwAAAAEAAgAAh2kABAAAAAEAAABOAAAAAAAAAJAAAAABAAAAkAAAAAEAA5KGAAcAAAASAAAAeKACAAQAAAABAAAJcKADAAQAAAABAAAC5gAAAABBU0NJSQAAAFNjcmVlbnNob3Q0Yv8EAAAACXBIWXMAABYlAAAWJQFJUiTwAAAB12lUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNi4wLjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczpleGlmPSJodHRwOi8vbnMuYWRvYmUuY29tL2V4aWYvMS4wLyI+CiAgICAgICAgIDxleGlmOlBpeGVsWURpbWVuc2lvbj43NDI8L2V4aWY6UGl4ZWxZRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+MjQxNjwvZXhpZjpQaXhlbFhEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlVzZXJDb21tZW50PlNjcmVlbnNob3Q8L2V4aWY6VXNlckNvbW1lbnQ+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgqAaPP7AAAAHGlET1QAAAACAAAAAAAAAXMAAAAoAAABcwAAAXMAAp5dSS8XJAAAQABJREFUeAHsnQe8HUX5v4cgRTD0EkqoMSAdIlIkGIr0KlIEBARpBpSmIkVAqvRepUmTQKSIIII06SUC0iMl0iEUQaSJ/PPMjzn/uZs95+4pN7lJnvfzufdsmZ2dfXZmd3bnu+872fLLL/95mAitb9++E+FReUgSkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQwPgk8P7778fd//vf/w6TTz55mGyyyUKfPn3iH9NpvmoZJ1PAVRWV6SQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBSZ2AAq6KNUAPXBVBmUwCEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISqExAAVdFVAq4KoIymQQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCRQmYACroqoFHBVBGUyCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSKAyAQVcFVEp4KoIymQSkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQQGUCCrgqolLAVRGUySQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBygQUcFVGZUIJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAI9Q+Df//53mHzyycNkk00W+vTpE/+YTvNV9zrZ8ssv/3nVxKaTgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggRAUcFkLJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJjCcCCrjGE3h3KwEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAGXdUACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkMB4IqCAazyBd7cSkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQUcFkHxgmBGWecMcw///zhkUceCZ999tk42ac7mTQITDnllGGxxRYLI0eODO+///4kcdDzzDNPmGaaacJTTz01SRyvBykBCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKYmAko4JqYz24vObbZZ589HHHEEbE0d955Zzj//PN7Sckm7GLMNddcYciQIWHyyScPH3/8cRg+fHj473//O2EfVAulP+OMMwIirtGjR4ef//znLeQwbjYZPHhwFDHme6PM119/fb6o2+kNNtggbLjhhjHd8ccfHx5//PFutxkfCWaaaaaw5pprhimmmKK2+88//zzceuut4aWXXqotK5toZ9uy/FwmAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAIS6M0EFHC1eHbmnHPOsNBCC4XnnnsujBo1qsVcJozNFlxwwfD888+H//3vfy0VeMkllww//vGP47bwOvzww1vKx426Ejj00EMD9TDZSSedFB599NE0O85+260f7Rb0N7/5TZhsssnCJ598Enbdddd2s+uR7aeaaqpw+umnl+a9ww47lC6vt3DHHXcMyy+/fFx9zTXXhGuvvbZe0i7Lx/V52mOPPcLiiy/epQzMPPzww+GUU04Za3m+oJ1t83xamf7Sl74UVl555TDffPOFOeaYI8wwwwyBG+ULL7wQhg0bFj788MNWsnUbCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQQF0CCrjqohl7BSKRoUOHBgRJffr0qSXAqwzCpGOPPTaKSGorJoKJ733ve2H11VePodqOOeaYlo6IcG8HHXRQ3PaJJ54Ixx13XEv5uFFXAt/85jfD2muvHUUmrBkfAq5O1I+uR9X83FlnnRUQ3fznP/8Ju+++e/MZjKMtdttttyi4w2MaHsOmm266uOdmBVybbbZZ9GzFxhdffHH0aNXdIYyP87TEEkuEjTfeOHz5y1+OAruZZ545/hJG9eSTT25Y5Ha2bZhxNyu5rtOOCE9ZZni4Yz3XMU0CEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCTQKQIKuCqSnHHGGcOBBx4Ypp9++toWDOYjHEmGZ5ajjjqq2/BgKf2E8IvnLARrb731VvjZz37WcpEXXXTR6MlmxIgRerBpmWL5hueee25cMT4EXJ2qH+VHVm3pV7/61dC/f//o2entt9+uttF4TrXIIouEvffeO5aiWQEX3rxWWGGF8M4770SPawhIu7PecJ5+/etfh1lmmSVUEXAVj6edbYt5NZondGMSqiIIxKPdp59+GhCUpWs/nt7gyXJNAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQk0AkCCrgqUjzyyCPDbLPNFlMTAoywbQi2pphiirD55puHVVZZJa5j0J/B/Sqiioq7Hq/JkvADscg+++wzXsvizssJ9AYBl/Wj/NzUW9qOgKteno2W94Z23I4Iq51tG3EprsNb2Kmnnhqv3zvttFOXsLHML7fccnETvL7df//9xc2dl4AEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCbREQAFXBWyEqtt+++1jSgbtGbwv2gYbbBA23HDDuPhPf/pTuOKKK0Lfvn3DnnvuWQvHdfvtt4cbbrghpiF82h577FFbh+Drt7/9bXjyySeLWcf5hRdeOArF8GCDyAD74IMPwj//+c9wzjnnhPfeey8uK/7Da8wWW2wRFltssYDnHkKE4TmG9AjQ5phjjvD73/8+3HjjjcVN43yrwo+vfOUrYa+99qodH5n973//CxdccEF45plnSveVLyTE3DbbbBPwrgQrhHKUF09gN910U7jrrrvy5PG48GhEmDbsscceC3fccUfAu9Hss89eC7F38803h2uvvbbLtvVm4Ixg429/+1v417/+VS9ZS8s5D3g0m2GGGWrbw+fyyy+PHopqC8dMbL311gEPZoTwxKgrV199dbjvvvvifBJwEZZumWWWCUsttVSYdtppAx7iXnrppXDKKafULT/loN4iKIId2xHiDy9DL7/8cjjjjDNCI69WrdaPWPA2/u26665h3nnn7ZJD3r66rCjMzD///LFN4LWLeoZ9/PHHYfTo0bFtUd8OO+ywHvWk16yAi/RbbrllF49/lPnwww+vFLa1nfOE90Ha0VxzzRVo19RD9v3mm2+Giy66KDz77LMFwuWz7YiwutsWT4i0pxSW8oUXXgicZ+r0Z599Fr2zXXnlldHrGddQjPpNvohuc1t33XXjteaWW27JF0cPXMcff3xc9pe//CVceumlXdY7IwEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEmiVgAKuCuQITYdwAYHNbrvtFsULZZudfvrpUSSFcGbnnXeOYpqhQ4fWkr766qvhgAMOiPO5KCwluOeee6JnrzSffhGBIcCqZwh6EBMUBQeIGk488cSa4Kve9v/4xz8CHsaSrbfeelHUg7inO6sXDm3ppZeOrIrbX3/99WH48OHFxV3mCQ+33XbbdRGrdEkwZgaWRx99dE24loc+I23ygJZET/n2d955Zzj//PPzRWNNb7bZZmGNNdaoiaYefPDBKGYaK2GLCxDOIRIqGmIxPADllupVvuypp56qhXpLAq5iSM+UnuW77757qdCH+onoq55R5xHivf/++7UknagftcxanMADXvHcImY85JBDGuY4zzzzhF/+8pdjbVvcCLFPElsW13VivlkBF9eTb3zjG2PtmusJbaHMOnGeEKbyV2Sd76+eqDVPw3R3Iqxi+ny+u205rwcddFC+yVjTXBOKx4H47IgjjhgrbdmCVVddNWy11VZx1bBhw+qKXsu2dZkEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUigEQEFXI3ofLEuiUVGjhwZjjrqqLpb4KVmxRVXjOsRXeHlas011wx4dMGz0WuvvRb233//uB5x1frrrx+mnnrqMHjw4Cj8KhMJ/eAHPwgrrbRS3AYPVHiPQuSDNy0EBSuvvHIUJSBOQBCE55lkP/rRj8KgQYPi7HXXXRfwPoXXrsUXXzx6FEOUhj333HPRk0+cGfMPz1ff+ta30mzD3+K2eWKOHWEVRlkRhHUn4CoKbPBIhietN954IwwcODCyTOV+/fXXw3777VfbJZxgmjzssAKBxkMPPRQQlA0YMKDGCo9ERc87tYzGTJx99tnRE1VaBl9EUJyDTtnaa68dvvvd78bsEB89+uij4bbbbguEI8wNsQ8CviQoS1618BiFJQEX03hGQsiHxzLq4jTTTMPi6LHsvPPOi9P5vyQOpF48/fTTYcSIEbGuIjDkXGB4MzvhhBNqm3WqftQybGGC84lXOgxhE56Xqgi48KCEVzq8MuENDvERHscIgfqd73ynds6vuuqqQJvpKWtWwEWd//a3vx2vF1wz0jWhkYCr3fOUizAR8uHhjLZIu+G6stFGG9U8mFXh1Z0IqxHrKtsOGTIkIFrDYxhGO8HbH9cyzjFGO37ggQeixzmYMv/DH/4wrmv0b9NNN43XcgRgbPPTn/50rHbaaHvXSUACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABBoRUMDViM6YdbmnpO48NyFYwnMTRphFxCHYvvvuG0MB5gKuuOKLf8ccc0wUOhUFXIQs+9WvfhVTIaBAbMMJyw3vScnLF+EFCSOWDLHZrLPOGkVbCJZyQ4iAmIwwY4h2TjvttHx1LVTaLrvsEoUyhBAserhBBNNIBJVnSCg+wtV1J+BKAhu2RXBUDJVIuSkH4e+wa665pktIxFzwVtxXHuYSL1cI4eoZ5w+RXW54oup0KMXjjjsuhlHMhXBzzz13FIu9+OKLNW9chHNMnrkQUyGqSpYEXJwL6gget5IlIVpR7JbW84v4qSwEJ3UCsRDHzLHnhviFc9Gp+pHn3ew09XiBBRaoJOBKYsyiKI19IuxCBInoDW4pRGWz5amSvlkBV54nAqVjjz02Lmok4CJBO+cptVnyKWsv8MILHu0EUdNOO+0UvRSSvsyqiLDKtmNZ1W0Jrfn1r389XpcQXGI77rhjWH755eN08qyW80foivCxzJZccsnoDTCFZuQ4y7wdlm3rMglIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkEBVAgq4uiGFdx/Cl2FFsVBx01xMhfcevNJgrQq48M6ElybsxhtvjOKnOFP4R0i4mWeeeSxvMnvvvXf0NENyxEoPP/xwePfdd6PnGDxaffrppzF83uOPP15XwIDwCxEDXqH22Wefwp6rzyYxSFFUleeQi5RyQVOehmlEKYSGRED08ssvx5B4KU0ScKUwlmk5v3379o3bMd2dx6Ci9yI8gSXRDNt3yrbeeuvo/SkvLyK8hRZaKO4CgQyesVL4NkRzCGVySwKusmNC4INID3FXErTk2zLNejyXUYdghJEeoRACLuoJQq0y61T9KMu76rJmBFynnHJKFGghxLn11lsD4UMRqCF+xKMZnppox0l8WbUMzabLBUR47mvGmhFwpXybPU9zzjlnOPTQQ+PmeLJCZFpmW265ZVhuueXiKsSITzzxRFmyuKyqCKssg6rbJgFXLmbFw+F2Y0KyYlzDuJYhPkMsih144IHhlVdeidP5v9lmm61LaFk8vOG1bdSoUXkypyUgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAJtE1DA1Q1CxC2IYDDCh1144YV1tyDE3eabbx7XI6q5++6743SrAq4kTKm7w5IV7OvNN9+Mawg5iBgIoVOZffLJJ+GRRx4JF1988VievVL6ZoUfabvibxUB1worrFALZzZs2LAoWivmk+aToAPPOXjQSZYEXHiUwhtV0ZJnre7EeGxHKEbCEOIZrUzgUcy7lflcJHLkkUdGQdGZZ54ZQyCSH+EfTz/99PDzn/88hpAsE7YlAddJJ50UwzDm5UgCsXoCrpRvvk1xOheXFdd1qn4U821mPrWTKiEUCfuHWK2eUW/wtDd8+PB6STqyvLcLuAhFuPHGGzd1rH/4wx/C1VdfXXeb1Ga55px88sl105WtqLptmYCLcJNcF/BiiDcuDG+AXJMwPPohUisaYTVpP4j9EHm9+uqrxSTOS0ACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABDpCQAFXNxgJD4boB3vqqafqeqJhPV5e8PaC5UKqVgVchxxySCCcHpaHxYsLSv7hSQhBDoKDZAhFEGLMO++80btQWp7/NvKu1SmBThUBV+4pJxfA5WVN04lN0SNVdwKuJI6qIuBK++rp3+QV6p577gm33HJLDG3JceENKgmvEr9LLrkkpsnL1KqAa9111w3f+c53YlYffvhh9CyEpzGEUP369QurrbZaFLFNTAIuDnb11VePHs0Qz9UTN+KBK7X7nHWnpnu7gItQsISExRA+8dfIEIPipYu6U8+qirDKtq+6bScFXIMGDYriUNrGbrvtVlYsl0lAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQ6QkABVwWMSWCDqAZvT/XEVCkdAqof/vCHtZyTgCsP61VbOWYCbzTTTjtt9PSUvMKwntCNhHDE2C/eppqxeeaZJwq3/vrXv8bNEIMRfnCqqaYKc8wxR8DLDmELMQRRZeILwu4ttdRS0UPXT37yk5i2lX9JgNQohOJMM81UE8jhBen888+vuyu8UnEciNb22muvWrpOCrgWW2yxgPcePAbde++9XYRxtR12YCKdZ46FcJZ4/cLbG4I2BEYnnHBC9CZGvaIeIJbJrVUBV/JGRjjN/fbbb6zjS967Ggm4OlU/8uNpdroZD1xf//rXY11GjDnFFFNEgSRhIml/eNtDtATzRsfcbPnK0o9rAVez52nppZeuiZbwOkh9bNeqirDK9lN1204KuCgHHF588cUYXrOsXC6TgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJdIKAAq4KFDfccMOwwQYbxJS33nprDDlY3AyvPt/73vfiYkLu5UKsoUOHhmWWWaZUFILIihBeWHG7FMKLdYiwLrjgAiZLbaGFFopeckaOHFlbT0g+vAzVC22Wh3wkjCLHVjSEaIQ2rCdoQezCPl5//fXipl3mqwi42CCJivD4s88++0SBVpeMxszgUQzxGfa3v/0tnHrqqXGaf50ScO2xxx5h8cUXr+VLiLV0nmoLOzSx4IILRgEV2eFxa5pppgmHHXZY2H777cOcc84ZCOs33XTTRREJHtaK1qqAK203YsSIcNppp3XJFqEf9Yey1Dv3bNCp+tFl503ONCPg+s1vfhNzP+KIIwLhKItGyFHaErbTTjsFRJs9YeNawNXseULYmdoVIVkRodaz2WefPfTv3z9ev+qlYXlVEVZZHlW37ZSAC3HoDjvsEBZeeOEo+Pvzn/8cbrvttrKiuUwCEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCTQNgEFXBUQEsrupJNOqnmrQkyFV5oUqjAXeCF2wdtN7iUpX/+Pf/wjDB8+PDzzzDOBEF0IK6accspYiqIYCXHUiSeeGL1mkQCvVBdddFEU1LAOEQgeusgnedLaZZddwqeffhrzO+6448IMM8wQpwnPh2AnlZljOvTQQwPiC6yeBy6EUgimsLvuuitcccUVUdSy3HLLBbwZDRw4MPTp0ydcd9114aqrrorp+EfoSfaRjOPgOBFCXH311WlxPJZcJIMHJMK3YR999FEUwj322GNxnmPeZpttap6p2A5BEyEgkyG6wIMVFXvPPffsEvqNciIko2yNPIGRF0If9pcb+SGm6glLAjfyToKpjTbaKKy//vq13cHu8ssvr80zwTGdc845cRmCG+pQbvD61re+FQgDhwe1nHXaJ2I56hXnl3O01lprRU9UeKjCKA8iRH6L1mr9KObTzDzHnMrGdj/96U+j9yxEdgizknFcqS2kZUm0xrrLLrusSzjKvn37hmOPPTbWD7ajLXXS8jaBMIjwpBhe1ZLRPvNrR1rOeUn1kTadjvNXv/pVeO2111Kyul76WjlP1He80GHPP/98FPmltoY3v+WXXz4su+yyMcwmaWB79913M1kzhFDJKOsss8wSaM940EtGvcrrZVreyrZJLPv2228HxHjwxIsewk7OOaI8lsGT+o9RrlGjRqXdxt/kFS8tZBtEnVxXNAlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkECnCSjgqkgU4cHhhx8exR1swoA+IQ0RGSRhBUIEPBk9+uijXXJFXIUALBc0sX3aLk/M8r333rvmeYqwbngYSmlZj8AjF3Sk7YuecnIBF2nY9oMPPoiiH8qU8nz44YcD4R/LDA9M5MP+GtnZZ58d7rvvvpgkiagapc/XFcVjCHIQuCRD3IGgJmfNsSBcSvuknIjEcsZsf8kll0SRDqIwxGG5kS8ee8rEI5xHQuslY38I8xBC9YQRBnLRRReNWSMmQVQy44wzRkFR2h9itdGjR6fZKABacskla/NMvPvuu7H+5GK1PEEutEveyvL1+TSCF/JJVgxXyfJW6kfKr5VfhIOIcKpa0ftcEnCl7TlGvJ5xnBxLsr/85S/h0ksvTbNt/xYFQY0yJJQpbSLZ0UcfHWaeeeY02/CXa1IuCEuJWzlPiOSOP/74Llxoh7AqtjOufb/4xS8CwikMD26E/szrTypL2S/X1uQRrZVtCU+L+C7fH20Wr4WcY65JyRBn4u3wzDPPTIsCAtfknY2FRQ98LDvmmGMCoTc1CUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQQKcJKOBqgihh7LbbbrtQFM0gEHjxxRej2KGehxa2QcSRe5VhOzxBLbHEEoFQihjL8ByTPN2wDCEP4qF5552X2S6GSIF933TTTWN5vzn44INjaDPyRKyVBFspA7Z98skno7iszLtSSsf+DzjggJo3r7QcIRmexPAohugkGYwGDx6cZhv+UgY8gRU94OA1Z8stt+zCK2WEpyUEZ7mYCbEJHqjwcpSMvPFYBZtNNtkkrLPOOmlV/EXsgleqopcmVhISc9NNN435we+WW26JHpu6ZNDBGQRrCNewYcOGhRtvvDFOI6CZfvrpo+cfyppbmSgIAQ35cK4RqOQ82LYoaMILVFl9xtvbtddeGxAPJVFMyjsvA9PN1o/i9s3ML7300tEbWLEu18ujeLwpRCf1vciGPFjOuS56OquXf9Xl2267bVh55ZUrJUfIhKApGd62kqe8tKzebz0BF+lbOU8ItRD64W2rjDl1Ao+Ef/rTn7p4DuM6d/LJJ5cyLpaddpqHtGxl21dffTXuL9XVtA88JT799NORZyp/EpsiqkUsht1xxx3Rq2LajtClXPPS9frZZ5+teT1LafyVgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJdIqAAq4mSW6++eZhjTXWiB6yXnnllYDnlwceeCB6ZkH8gces999/v4u4KN8FQoz55psvIDjIRU95mnrTiCnInz9C+SHcIh8EEGWGOAFx2P333x+FKQjAECbgWerll1+OoikEH1UNccRXv/rVGL6OsvdUOMG8PBwDYRrxQISwhVBuCKrGlXG+3njjjbqMO1kORGOE8ctDUXK+Vl111Sgwaba+VC0bwkQEZLB+/PHHw+uvv1510y7pxkf96FKACjNzzTVXDOP3yCOPRHHRgAEDwqyzzhrPL3wRB47L+lWhyB1P0up5oi7SFrnecO3hr0z82PECj8cMqS+I1HrK8954PDR3LQEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACvYiAAq4mTgYh+AjFV88QfiCOICQbHrM0CUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACjQgo4GpEp7Buzz33DIsttlj0ZoX3Lbz1zDHHHPEPr1YYIi5Civ32t78tbO2sBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBrgQUcHXl0XAO71qLLLJIeOyxx8ZKN+WUU4Z+/fo1HRZxrIxcIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlMMgQUcE0yp9oDlYAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAR6GwEFXL3tjFgeCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSGCSIaCAa5I51R6oBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJNDbCCjg6m1nxPJIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAApMMAQVck8yp9kAlIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggd5GQAFXbzsjlkcCEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISmGQIKOCaZE61ByoBCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJ9DYCCrh62xmxPBKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpDAJENAAdckc6o9UAlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUigtxFQwNXbzojlkYAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQmGQIKuCaZU+2BSkACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAK9jYACrt52RiyPBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJDDJEFDANcmcag9UAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEuhtBBRw9bYzYnkkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggUmGgAKuSeZUe6ASkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQQG8joICrt50RyyMBCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJTDIEFHBNMqfaA5WABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEehsBBVy97YxYHglIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhgkiGggGuSOdUeqAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCTQ2wgo4OptZ8TySEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAKTDAEFXJPMqfZAJSABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIIHeRkABV287I5ZHAhKQgAQkMJ4JfOUrXwkLLLBAGDlyZPjwww/Hc2ncvQQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAT+j4DjWNaEiZWAAq6J9cx6XBKQwCRBYM455wwrrrhimGaaaWrH+/nnn4c///nP4fXXX68t68TE0ksvHRZddNHQp0+fWnbs48Ybb6zNOzHhE/jGN74Rdt5553gg//rXv8Jee+014R+URyABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJDDBE3Aca4I/hR5AAwIKuBrAcZUEJCCB3k7gzDPPDFNMMcVYxbzrrrvCeeedN9byVhd87WtfC/vss0/p5jvssEPp8t60cJ555gkDBgwIo0aNCs8991xA5DYx2qyzzhpWXnnlKN678847WzrErbfeOqyyyipx288++yzstNNOLeXjRhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhLoJAHHsTpJc/znNd1004XFF188FuTFF18M//znP8OXv/zlsNxyy4URI0aE9957b/wX8osSDBw4MAwaNCg89NBD4ZlnnumRcing6hGsZioBCUyMBOgQfPOb3wxf+tKX4uFxwzjssMPCO++80+Vw11tvvbDOOuvUhFUff/xxOProo+MNp0vCDsycffbZYfLJJx8rp/vuuy+wrlO2zDLLhKFDh5Zm11sFXDPOOGPYe++9Q79+/cJkk01WKzvirbfffjuceOKJ4ZVXXqktnxgmTj311Nip4ViOPfbY8OSTTzZ9WN///vfDkCFD4nb//e9/a964ms7IDSQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSGCiJcDgOuMmU089dekx/u9//wsffPBB/LD6kksuCaNHjy5N58JJm4D1aOI//zvuuGNYdtllu0T4aXTUjOPhjOHII48sTeY4VimWCWoh47Y4kGD8OY27pwNgbJL7x5RTThnHc3/605+mVeP1F6HZCSecEMtAHaVe94TDEAVc4/U0u3MJSGBCInDWWWeNdRN59tlnwxFHHFE7DBTBp5xyShfBECs77REr7XD11VcPgwcPjp0ewikm67SAi3x33XXXwD5mnnnmMNVUU6Vdhd4o4EJot9122zXsDHJTveKKKyaqEJDnnntu7bxcf/31Yfjw4bX5qhMLLbRQDJtI5wkBWOqMVN3edBKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJDDxE/jlL38Z5p133koHyvv44447rqWPjivtwEQTLAHr0QR76ioX/KSTTgpf+cpXKqcnYSMHA45jNYWy1yWeffbZw7777hsQRHVnn376adhll126SzZO1uehO9nhgQce2COOQhRwjZPT6U4kIIGJgcB+++0XFlxwwS6HwkMHNw46EtgGG2wQNtxwwy5pmDn99NOjO8WxVnRwAcKxaaaZJubYEwKuVNQtt9wyrLbaamm21wm4ZplllnDUUUeNJaKrFbgwgRe1559/vrC0d8726dMnqs7rlS4XcN1www3hyiuvrJfU5RKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBlglssskmMRpJ1QyIVvKjH/2oavK20/GRMn94cplUjIgtn3322QR1uNaj3ne6Ol2PTjvttLqe+uodPeOvP/zhD+utdvkETKBM0Md1i2v1FFNM0eXIxrWAq1HdV8DV5dQ4IwEJSKB3EMCrEx6vcrvsssvCzTffHBchHJp11llrq+lg7L777uHDDz+sLWNivvnmC1tssUWYbbbZouqchwgUtW+99Va49tprw6OPPtolfT7DFy0rrbRSzANPWK+++mr405/+FL0mdSfgQgDEfhdddNEwwwwzRE9an3zySQwDeffdd4c//vGP+a5Kp3u7gOuggw4K88wzT63sHB/hEp9++ukw99xzB1xt5kp/wikm95vf/e53w1JLLVXztMY2559/fswreTtL3sfefPPN+MVQbUfZBEK/zTbbLNaFtC/OLyEbL7jggrquovv27Rvgu9hii8XObHq4xM30v/71r9C/f/8Y63nPPffM9vb/J9sRcPGQtPTSS9eOnVwRJhL+s7v40hzjtttuG7lPP/30MQ/qPGzvv//+seoV9ZSO9xxzzFF7gCb843e+853w1a9+NUw77bSBThmxrjl3//nPf/7/QTolAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACvYLA1772tbDPPvvUykKoxAceeCC+yyZSxvLLL19bx8RPfvKTOBaSL8QLC++Xef/NNOG0eCfMeAlRJh577LE8eW2a8Q4+ql955ZXjO2UG3hmTQSj22muvRe9gvGPH00sK3zhgwID47j55fiHKChFUeC/P2A558D7+5ZdfDr/5zW/iO+7aDrMJ8mUMYPHFFw8zzjhjbayF9/gjR44MF198cWBsIretttoqLLLIInEfLGcsCE80cGK8BgHDu+++Gx0CjBo1Kt+0Ns3YB+Hj5pprrhhejBVpO8pEBJVHHnkknHzyybVt8olWxy7yPHpiuhP1qNXxr1bqEWMijHFw/uDOeAjMESgSyYZxJOoh9W7YsGHh8ccfL8XWSj361re+FVZdddVapJwnnngi/P3vfw+bbrppPP/U4ffffz9cffXV4fbbby/db7v1qNk2e+aZZ9aEOYceemhsG1w3GE/CYIWzBexXv/pVZFom4GpmHIu6/r3vfS+OB5LXbbfdFsdW+/XrF4VCMHvooYdiO2ZsjnZE2Eai0hTbLuWC+5AhQyJjxrhIT3t/5plnwoUXXhjHtEinNSZAPV1rrbVqieB4xhlnhL/97W9xGW3q4IMPrl3fyjyxtdJu0g7bqfsKuBJFfyUgAQn0IgLc7BHy5IaA6oADDojer/CCVbQ8xCAdQR5QEOg0MjxC4U44F35xQ9p+++3Diiuu2GjTuK7MAxdxhOlQJgFSWSZ0NogpjTipnvVmAReCuGJM7F/84hfhjTfeqB0Ox4/aH57J6JC98MILAdV3bjyo/fjHP46Lzj777NqDFQvKOo90jBFX8bBRz9jupptuCpdffnmXJJSHDn4S4XVZWZjB6xsCJ2y99daLD6nsu5GxXx528cxVNEJ/IqAqM9jUe0AmPR7nKAN1u54hAIMtjLGyL2ropJUdQ34O4sb+k4AEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggV5BALEKgoxkvEdnfCJZLtxgWTHkFO+K11577S7v69O26fcf//hH+PWvfz2WJ6299947CqJSunq/iMpuueWWuLoYro/35lg+XhAXjPnHuiuuuCLceOONaVH8XWKJJcLOO+/c0KMQooNLL720JqApciKjeu/E2W8uOks7Z/zjiCOOKC1rSsMv4jccC+TWzthFnk9PTRf5NFuP2hn/aqUebbzxxnFcJOfBOUd8WGaIU4pjMK3UI/I+66yzuuynUR3O634qVzv1iDxaabN5mRmnpMw4D0BwiH300Udh6NChcfqcc86pjTfl46vNjmMxVtfdWGzcYeFfUQAJL+oI0X/qGeeecj/44IP1krj8CwL5+WUR7SKJtxKkFVZYoeZ9reiBq9V2Q96t1v299tor3mvK7hOpzPxyTUeI+M9//jNf3PS0IRSbRuYGEpDApExguxIPXHQ06Fiss846Y3XYYLXjjjvWHiya6TDgfQiVcbLddtstekhK841+iwIuPE+RV35zodzcTIodSsQ2dEbquRXuzQIuFPDbbLNNDQ0er3ggLFrxPCBq+v3vfx87vrkQKRcPIa7CM1Qy+NHRzA1PXgsvvHC+KHqx4uEoZ08CFPl33HFHLe3WW28dVlllldo8YjoeTPl6Z/755++y/a677lr7AoDOAJ6sqhhfHe2///5jJaVsPBDlx54SNRJwUV7KXcX4YoFODqJERIh5x7u77fFa9te//rW7ZK6XgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBcUigkfCGaAsIkZLxjph328nWWGONsPnmm6fZ+Mt7d6z4Pr0oqih6bCJvPOGw3UILLdRl3COPooKHpEGDBsV9VPlHeXgHnz5O5l08grVi+erlhfCMciE+wQFA1e3wDkR0itwQb+GdJhljSK+//nr0XJYvLxNwtTN2kfbXk7/t1KN2xr9arUdFTzxV2FxzzTXR6xppW61HbMuYTfIgx3wjQwiCU4nc2qlHrbbZ3EFCGhuqJ+DKxV4pLeWn7TQzjsW15utf/3p+6JWmc49PjJ/Cu+h4AVFRMdQf14r99tuvi0OJSjuchBIhguOamKysfqZ1XGe5LjBOmhx3tNNuyLfVul8UnaUylv3eeuut0QNj2bqqyxRwVSVlOglIQAJjCPBVBR0zDIFNcu951VVXxdCK3HzoHNMZTx3x5BKY8HSIsHLDlTBekegQIABDEJO2I92VV14ZPSbxwPGzn/0s3zTgpQu3qwiGcPubW1HAdfzxx9fKSjq8P3EToZNB5xaRGb/J+BoFZX6Z9WYB1w9+8IPoAjWV+957742q9zSffvEYxRcSyXAxi8czzh8PMklJnwu4cImLAIkQllhRwLXccsuFnXbaKWUZXePyQPbSSy/FjhxukBE7pfOLQA7uyfLOKsvw6oZ3NwwRF57AcO2Zdx5ZVzwWlpUZ5cUlM39lxvknfCS/yy67bC1JPQHXlFNOGV05p+NhAwRz5513XnRPDQ881uUCQVxHE2YU48uHPfbYo8aDZbCifGybP0Tz1QIuVDUJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhLoPQSKwhtCofHhOOKGosAhF2EhiOCj6fR+mffevE9P0SB4f4wAY+qpp64dLO+WeceM8XE1XlqSXXfddYFxGox30nyMT/hG8ickWwpJyEfMKapE2pbf+++/PyD0Iowi4zi5QObtt9+O4wakO+aYY8JMM83EZDQ+WOYDZARXjNMgOEnjRiSAB6IxDFZEDMnFIIxBEOqObdZdd90aD8afGI/IjZCOiRdjO/DhvT/GOBHzjGPg/eWQQw6pbdru2EUtox6caLUeUaR2xr9arUfsd9FFF40RWdI5YRkf0RMBBkEhEXUY20vGueIcce7aqUecY8YdF1hggZR1bHM333xzHFPaYosturSbXATFBq3Wo3baLPWRsSccSOBgAcvHxHIPXAcddFAcCysT9zQzjoVjBdoe417JGHMljzy0K2EmKdf666+fktUcEBQFn3iK+u1vfxvTc43AG9lKK61U2y5FbKotcKILAVgxjpuMkJWHH354mh3rl3FI2lKydtoNebRa9xmfz9tyKk/xl/sY9ZrQvO2YAq526LmtBCQwyRFARLPkkkvG40b0M3DgwPgwkIu5EE8h8kqdtp///OdRzINYhXjoyYoiK5YXxTgIWuisFLct3tToyNNZTJbnTWcuDw1IzG06GLnhWQpxWrJGnYzeLOAiZnYevhDPWnTIipa732Qd8exxnYzleeQCLtYRW37IkCFMjiXgKn7BwkMbD3a5Edu5f//+tUV0TDiXWOqUppV05uk0Uja+7kHERNxybtx5hyWlpzPK1wfJcOuMODAZD81VrPhVVD0B1+DBg+NDcMozf4hNyxCcwTW1haL4LP+SAkEbArj0wJm71kasyFdOmgQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCfQeAkXhTb2S8bE1g+fp/e/qq68ePwBO6e+8884ookrz/DLmseaaa9YW5R+eI5zgo/jc+Liej4wZV0FowXt1BFu8Z89trrnmiqKutCwXlbCsuJ4yI/QphnDjnTZjM7y3T4Z4DGHaVFNNlRbFd+S858eK4wC5KI2PuonGgTEGgNgnt/x9Ossp1xtvvBFFO3j5wmEAojm8cuXW7thFnldPTbdaj9od/2qnHsEiPyecD0JXIupLlq9nGWN1iBDzcIqt1KNiuW+77bZw0UUXxd0iUNxggw1SEUIao0wLimWqWo/abbOMF+E1jv1h9QRcrKMd0H5TWpblVnUcK48iQ144VUDQlTvbQFCGUCj3DIVwC/El54l2jzHGRtsulom82B4rOm6IC/1XI4DTB+pRsrLQomld8bcT199W6z5lYZwT4V8emQlhYrq2k6bqOCxpG5kCrkZ0XCcBCUigQCDv6D766KPxplwMmcfXHHS0Uzg6LuB88VB0a5o8cxV20UUBnL7OoOOQvEKRHm9EeSzlYmclF3DlHZTivurNp/2Wre/NAi6+JCA8X7K77747nHvuuWm29suDHR3cZLk74lYFXHwdlH85k/Ju9IvA6vrrr49JiuEfy7bj5k8owdQRL6bJj7WeeK24TXG+WJfqCbhyMSN5DBs2LCAaK1qx7tIZTg/MeWepKNLKv5rJBXbF/J2XgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCB8UOgivAGQQvCllz4sPfee4dFFlmkqULj1YQQWBiesgirlT4eLsuI/T399NNxPIUB8WRFgVZZyCuEGnz4niwJr3JvRoimctFHSluMFJIirbA+5cM0nph22WUXJqMNHTo0LLPMMnGasYA84gcLix/6x4SFf4jYeFfPOEKydscuUj49+dtqPWp3/KudegSPfIwD71v7779/F0xF5wuMoSAwbLceFQVcuUir6HEtjVGmgrVaj9pts2n/6beRgCulqfdbdRwrrx9JFIkwNHm34xqBIKdv375dQpYiykKMdfrpp9crQt3l5I3TD21sAquuumrYaqutaivya3ptYZ0Jxn7bbTet1v1UpGLo1AMPPDCKhtP6Tv0q4OoUSfORgAQmCQJ0vpJb0hEjRsQvGnLPVUn4lMdzxqXjU089FZX1fA2QjO3wSFS0fNuk1sblau4qmIcdOuLJ+KqDjmKyXMBVvCGmNPV+6bDw4EHnqcx6s4BrrbXWCni5SlZ0FZyW5w9CLMvdK+cCLhjDOhkubwmFiKWOXVrX7EMQ5/7ggw+uhUkkH76K4Bjyr3NS/vlv/jVFvnx8CrjwOJY/FKZypTjVaT7/oih/uEEQmXuKO/bYY2PoSLZTwJXo+SsBCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIoPcQKApvCEf28MMPBz4AzsVVd9xxR7jwwgtrBc/fw9cWNpjgffwf/vCHcM0119RSES1l66237hLSsLYym8Aj1X777VdbUhRw5Z69UqKigIuP+4n+wRhBsvzD8LSM36KXmfx9fi7gYpCeD/2TMWaEQAArE3ARhQPBFx6EkgOBtG3xN/8wuxNjF8X8Oz3faj3qxPhXq/UIBvkYR1lkGz5oJxxoMurvW2+91XY9Kgq4cmELIkDGwJIVBVyt1qNOtNlUJn57u4CLMuae0pjvzmjTnHPGdrWxCRSvvWWhYsfe6v+WMDbb7vW31bqfyqSAK5HwVwISkEAvIkBHh/jIGDHREVvhDSvFcsfN7/nnn9+l08YNHjeQxDYnBnoywhjyMJMb60mXLN28+Kpk9tlnT4vjgwpx0ZMVO6m5gIsvCBDNJEPUhKisniFCa+TmsTcLuIgTf9xxx9UeDHmo46uAXO3ODZpzxm8yvpJBtIblHeriFzC5qKgo4CJU4LzzzpuyjF/e8CVFmbFt7kaXNJQdd9CIoAizuPTSSwceWvr16xfDdOb5FB/s0rpcwFV8IE5puvut+uVC0YvZY489Fr3M5fnjNpaHQwSGGMfNA2YSLuYPNwq4cnJOS0ACEpCABCQgAQlIQAISkIAEJCABCUhAAhLo/QSKwps///nP4fLLLw/FMG68G+YdOiEOsY022iisv/76tQMkUgVRJcqMbT/66KMuHrx458z7dPJ79913AxEuEAfMMccctZBneV6EI8QDD9adiKC4nv3jpYd3+ESOSPbBBx9EoVqaT7+HHXZYLEea5z0440lYOwIuwsotu+yyMarHSiutFAYOHBj3wxhQLpZjP/n7+nbHLsivp63VetTu+Fc79Qgm+RgH9QThFGNsyfL1LGMsCkFhu/WoHQFXq/Wo3TabmKTf3i7gYgzvzDPPrI3/Ml6IkLPe+CnjXukak47R37EJEEo3v14x7vrkk0+OnXDMEsSPMGX8thPX31brfipcUcCVHLik9Z361QNXp0iajwQkMEkQyIVUKTwfgpf11lsvPiQgyuLmnd/UuRndc889YeONN47pEihu5nTWnnvuubgIgRbuFlM8ZRYSJhGxUf7lBct5MOCLES7ipD/88MPjzYt1WC7gYj736sU8IjPEZsnwDDZ48OCwxhprhOmmm24sgVhKx29vFnBRvjxOPPMwwtPVO++8E1nxsDLbbLOxKlrRre0222wTH/bSerxCIS5aYYUVusQ2Tg9tKd3mm28e+aV5vqIgnCb7T8YD1brrrhtw0Uqng848+WDp6wXm8WaVn5+FFloodgxTp4a6k3t+S/nnHR9Eazwo8osYjPNLPnQyKdebb76ZNuvyW1XAVXygIpNLLrkk8LUShngLMSJxzZMlQWKazx9e8gdK1udiOR7CaRuaBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJ9B4CxffEScBFCXOBBvOjR48OhHrD5ptvvi7vfBlX4T1+EniRhjETxizwvILQhpCJhNzC8o/aEUfxrjkZ4dD4qD2PasK4AJEesKJAi2WM91x22WUxNCNesRALJMs9K+XvtFnPx/tEUEnv+Yvet1jOu/wk+mhHwJU8aTE+wPEgBEpWFMzxcTn7wtodu0j76MnfVusRZWpn/KudesS+i/WB+ov3NsZ/dtxxx+i1jXQY5+1HP/pRrAvF7ZqtR80IuBgPGjVq1P8VYsz/VutRu202FQDvcYx10Z5nnnnmuBiBJl77aC9VvFdVHcdqJ4QiAi7Ycb1IxpjtOeecU3NSwHVp0KBBceyXOtxMSMCU56T2W4yQRFthPDBd26kbq6++ehT4pjC2qQ63225arfvpHC2xxBJdvCY+8cQT0UsbTl4QdzEOS31544034lh12q7ZXwVczRIzvQQkMEkSoEOBy9k8BCIgCI2Ye7OaZZZZAiHjEK8ko8OBN6SLL764NA9ELXTeZ5xxxi6qYzpzqLnfe++9+LDADSx3i0snhi9LeJDIvUmxX/b5/PPPR2EX89/+9rfDFltswWTN2B4RD4KtXDRGAkRliMIwhEvbbbddzYtSXFjnX70vTuok75HFiLMQLhWZIFxKntLSjuHEg07+UDhkyJDw/e9/PyWJv5wLOmJFY/mJJ54Y1eHsj+lpppmmSzJutHQ+6YgmAVZKkMclp7OaC8s4B4i4EJjhwpeviZIVXT6n5cTj7i78Imlvvvnm+DDKNF8kIY6qsl3xITr3VkZeGHWAji31uXgOkjc6OjBwLzLlPHDu+Pokf7gmX9oB+9MkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhg/BNg3AFxUPG9d4q2wLtnxkvy9byT5703HzEz/rHwwgt3ORAG8/kYm/fpxffHV111Vbjuuuti+uKH2IyzIB574YUXYhSVTTfdtLY9YyF44OJ9PlYm4IorSv5R3v33378mlsoFISk5Yw+UmbGa4nt2Im5wrBjvvXNhGMs4XqKI8IF3HuGDdZSXcRqiqmD5B9yUCzEJwh/eya+11lpdxEJEf8HhANbu2EXMpAf/tVuP2hn/aqcegaQoKGmEKf8Avp16hMAQIUlu1HHKgrBptdVWG3bZV5IAAEAASURBVKvNDR8+vObhrtV6xP7aabOMryLqLLaR/DiY5lgOHiPmTIJLlrUyjrXIIovEcdH8+vOf//wncsrHmpLTA65VyRgLQzREOyMyU54Hy2jvtCvGV/N1tGeuNVp9AozTIqQqjtdybhhLLRvzRqSJ45J22g0laqfus/1MM83URRPAsnqWRGf11jdaroCrER3XSUACEviCALHN6UAXjRs1rnOTFcPKpeVJ2IQCm68eig8eKV36JV9U+jzoJENxzNcbVa1YNtTrCIG6s+K+99prr+gxqrvt0vr8S5a0bFz/cr546Mk7TsUy0AnEExlf1uTGNnlYzHxd2fQDDzwQPa6xDm9TPMx1d35Ji1gp9ypVFHCRpsw4P0XRWUr3gx/8IOA6uZGx/b777hu/diLdBhtsEN1ZN9omX4cb6yuvvDIuooOKFznEWt1Z/uVV8euQfFs6UHmbytfRgeYBXJOABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIIHxSyD3JpWXBPHDTjvtFBcVI3qwkEF6vLDwHh1BR1HUlOeVphFZILrg42GsKLxJ6cp+CenI++lkVQVcjCHwYT5iqNyKHmTydfk0Xo8YxMcQoPDxcpldeOGFYdttty1b1eVj7Fx8UJr4i4UIIRD5UP5k7YxdpDx66rfdekS5Wh3/aqcesd+qAi6i9HD+cmulHjXaJ1FO+vfvX9qeci9y7dSjdtosnvS23377HEHd6RtvvDEMGzastr6VcSzEbAMGDKjlkSbId7PNNkuz8bds2TXXXBOuvfba6AWQ9I3GG1Nmf/nLX8Kll16aZv2tQ4DQiLvvvnulsdRiFKVW2w1Faafup0NhTBRnLo2sXYcUCrga0XWdBCQggS8I4PkKRXDuFQghDHF3ecBIxkWbMIu55yHSpS9KSIe6HIEKYe2KN/yUJwKi999/P2Vb+0WYhIvVopcnPGmNHDkyrLjiirW0Za46l1lmmbDdGG9aye1kLfGYCfaNov3cc8+tfdHB+uWWWy66ei2WNd82TeMRDOFUbzDU/AikijdSHlq44eNR7e233y4tKiEO6TzkCnC+duFBb6uttqptwzLcI+dCO84vD6eI5cqY8fBEZ73YiUNURYeykbEtHUm+2qlnlA8Ver5vzi0dBsRPF110Ufw6IG0PH77iqSI64+sBHjLzLx/Ih84zoSHL8iCUJCFFU6hQ0iNk5AuKvJ2wHLfO5E/YyuJXGHxBRd3iWDQJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhIYvwQIb1gUNvD+9vHHH4/v31PpGNNgbIJ31qy/9957u4hZNt544+hBquz9Mu/giYSCUIb348nwOIWXrUaGkIyxmeSJKqUtCrgQahCJZIYZZohjAozN8AE2YyX1xhB4/48YpRi1hX18/PHH4Xe/+12MzJL2yS9jEnjsyQ0xG5E6ePeNyCo3RGuEguS9PlYlAgdjRbyPT9vEDb/41+rYRZ5HT0x3qh61Mv7VTj2CRS7gGjFiRKxDjH/AmnPL+cD7FeKqMmulHu2xxx5h8cUX75Id7YT6uuCCC8bwc/lKxsTwXnf99dfHxe3WIzJppc0SCYg2kI+75eVM04xD4fAgeZ5jeSvjWIynEpkoHyvD2QbRjlJ4UfKmnR1wwAGxraW08MShAGNWGAJMhEP8lhnXiauvvjrcddddZatdVkKA+oADkfnnn7/LOUpJETEhiENEV7RW2g15dKLuU24EowMHDuxSLOoM9YCwioz/cv9p1RRwtUrO7SQgAQl0gAAh87jI82CCAKsojKm3C75IQaFMx+uRRx7p8uBSb5t8OSEeEQvNPffcUdhDJ5KQi+3cUPL8e8M0x8eDT+pwjR49Ogq3/v73v8cbKMIiFP88YOF2Of8ahfKz3XzzzRfdFiPUy8MsVj2+2WefPXJG+Adjzi/lKDO8WPGQ+PDDDwfKhntX3ESzLfMvvvhirbNYtn1xGXWLjg8dzKr1qphHs/N9+/aN9Rk3ojwgItqamOpUszxMLwEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAtUIIG5ivIT34oQo40PsXMSR58KYCh8y41mIAXO269evXxRI8TEz78TrvZ8uCrgIkVZvP/k+y6YZa8HLD2MtlJdxnuQlrCx9O8sQ57AvRCKIYDhmRGewgAPv5OsJzor7bWbsorhtb59vZvyrnXoEh1zAVfQa1QynCbUeNdNmm+HRW9Pi7INQp7RFhF+M+3HtQHCjtUaAsVh4MiaK0weuozgoeeONN7rNsNl208lrKHUB0S3XUsaQuWd1yhRwdYqk+UhAAhKQQK8hgCjruOOOK/UIVVbIyy67LLoiLlvnMglIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCZ8AogFllpqqbDbbrvVDgavLAigEAzgRUWTQBUCxYg8d999d7jhhhsCkWpyb3FV8jKNBCQggURAAVci4a8EJCABCUw0BNZZZ52wySabVD4e4h4T1lCTgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhg4iNAxIr999+/FrWjeIRE6SA0Vk950Cruz/kJkwAiwFNPPTVGT6l3BGeccUZ48MEH6612uQQkIIG6BBRw1UXjCglIQAISmFAJEMLvsMMOix64cF15//33R9fFCy+8cOCvf//+Ydppp41ulO+8887AlxGaBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEyeBjTbaKKy//voND46QeIwnaBKoR2C22WYLRx55ZL3VcTniLURcmgQkIIFmCSjgapaY6SUgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIYIIhwIffhxxySKnnpM8//zy89dZb4cADDzSM4gRzRsdfQXEggJALb1xFI3ziaaedFnAuoElAAhJoloACrmaJmV4CEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkECHCCjg6hBIs5GABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJNAsAQVczRIzvQQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggQ4RUMDVIZBmIwEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIoFkCCriaJWZ6CUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACHSKggKtDIM1GAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpBAswQUcDVLzPQSkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQ6REABV4dAmo0EJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIIFmCSjgapaY6SUgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCXSIgAKuDoE0GwlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAs0SUMDVLDHTS0ACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAIS6BABBVwdAmk2EpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEmiWggKtZYqaXgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCTQIQIKuDoE0mwkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQk0S0ABV7PETC8BCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSKBDBBRwdQPya1/7WjcpXC0BCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSKA5Ak8++WTcQAFXN9z69u3bTQpXS0ACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISaI7A+++/HzdQwNUNNwVc3QBytQQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCTQNAEFXBWRKeCqCMpkEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkEBlAgq4KqJSwFURlMkkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggcoEFHBVRKWAqyIok0lAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEAClQko4KqISgFXRVAmk4AEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQqE1DAVRGVAq6KoEwmAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCVQmoICrIioFXBVBmUwCEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISqExAAVdFVAq4KoIymQQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCRQmYACroqoFHBVBGUyCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSKAyAQVcFVEp4KoIymQSkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQQGUCCrgqolLAVRGUySQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBygQUcFVEpYCrIiiTSUACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAKVCSjgqohKAVdFUCaTgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCoTUMBVEZUCroqgTCYBCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJVCaggKsiKgVcFUGZTAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKoTEABV0VUCrgqgjKZBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJFCZgAKuiqgUcFUEZTIJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIoDIBBVwVUSngqgjKZBKQgAQkIAEJ1AissMIKYfbZZw/PPvts+Pvf/15b3qmJeeedNyy99NIxu4ceeii8+OKL3Wa98MILh2WWWSamu/TSS7tN324C+lCrrrpqmGyyycL9998fXnnllZgl5eAPu/baa8P//ve/OO0/CUhg/BP40pe+FNZZZ53Qp0+fMGLEiPDPf/6z5UJNNdVUYe21147bP/jgg+Gll14Ka665Zvjyl78cnnvuufDoo4+GZZddNsw111wxTX6dKO6U6x3XPeypp56Kf8U0zH/1q18Niy66aFw1atSo8Le//S18+9vfDrPOOmt48803w0033VS2Wa9Y9s1vfjOW85133gm33357j5Rp+umnD0OGDInX5ZEjR4bHH3+80n46fd3+xje+Eeacc87w3nvvhVtuuaVSGcZnoqWWWioMHDgwlvnjjz+Odfmee+4Jo0eP7lKs1VdfPXzlK18Jqe59/etfD3PPPXf417/+FW699dYuaXvTzITSRrpjtuSSS9ba/7jo53RXnmbXV7lmkucCCywQllhiiZj9ddddF/773//G6XFxDWFH9fp39coVC9eD/7g3cO779+8fJp988vDqq6+Ghx9+OHCNyy3dR3gZ+pe//CXeU1j2+eefhz/+8Y81jvk2E9r0hN4GqvDmuWLdddcN9Fd66jmrSjlM838EJoU6N7Ge63buOeOyHda75/SGvmSjft9HH33U8fdBPf2OiWfAKaecMjTzLDLzzDOHwYMHx2Zy7733htdee62tJjO++hJtFfqLjcdlu+hEec1jwiMw9dRTh4033jg+y9922221d6yNjmRcPR80KoPrJCABCUigOQIKuCryUsBVEZTJJCABCUhAAhKoEfjRj34UB3sZ6P35z39eW96piS233DLw0hJDkMDAU3f2/e9/PwwaNCgm22OPPbpL3vZ6Bry32267mA9CrTRIn9iw4qc//Wn49NNPYxr/SWB8EGDwIgmIXn755UCbrWKtblcl7/GZZsYZZwwHHXRQLMLNN98cEAe0anPMMUft+sdg+R/+8Idw/PHHR3EYApcTTjgh7LbbbmHAgAFxF4888kg4//zzS3d31FFHBV5YYohmDjvssNJ0++yzTxTMsBLR2MUXXxyOPPLIKBr797//HQ444IDS7Tq5sNW6kcr54Ycfhl/84hedLFItLwRFW2+9dZx/+umnwxlnnFFb12ii3nW70bE2WverX/0qTDfddLG99cQ9stGxNLNuhhlmCEOHDo3CuuJ2iD4QcQ0bNqy26rjjjosCEq4lxxxzTNh///3jtp988kn42c9+Vks3PiYanY9U98ZVG2n3+Oebb754HXn77bfDu+++W8sur6fjop9T23GHJqpcM9kVfSv6WBh17IMPPojT6TwWryH9+vUL00wzTfjPf/7T9sAqO6rXv6tXrli4HviHWIt9Lr744qW5c5/hGscgOkYbRDj62Wefhb333jtsuummgUEt7JBDDokD1nFmAvg3sbaBKuinmGKKeH0l7TPPPBNOP/30KpuZpocI9NR1t14d76HDmCSzbeee06gddvrc1bvn9Ia+ZKN+3wsvvNDx90GpvfXUO6YTTzwxtgX6C/vtt1+ldoF4a5NNNolphw8fHv7617/WtmtUF+qt475e1sepZdqLJxq1i15c7Fi0Rs8Jvb3sk1L5+FiNZ1PsiiuuCHfddVe3h1/v+aDbDU0gAQlIQALjjYACroroFXBVBGUyCUhAAhKQgARqBHghlQYPzz333I574VLAVUPthATaIrDWWmsF/jDEQ4iIqlir21XJe3ym6aSAC09ESWiFeAsR17HHHhu9ZqRB129961vxK1KOGSHGwQcfPNbh52ViJcIZBt/LvPcdffTR8ctx0p155pnRU1d6aTmuxCmt1o1UzqL4gmPplHVawNXoWBut6w2DblWYHnHEEVH4QlrqHR7DGJxBDJPs+uuvD3/+85/j7K9//evAAMjzzz8fTjrppJAEhT15TlM5uvttdD5S3RtXbaS7sjZaj6eHAw88MCbBw96FF15YS54GNlmQ+mC1lRPARJVrJodRb3AzncdifUNMSL3FE1wS6LaDo95ger1ytbOvRtvm55v2yYAz4ize4eEFA0tiYaZ/8pOfhPnnn78mHF1vvfUC3lOwCemDgom5DcST0c2/CXmAvJtDmyBX5+2wU9fdRnV8goTUSwvdzj2nXjvsiXNX757TG/qSjfp9Z511Vq0v0qn3QT39jqmTAq5GdaHRunHdl+hk86zXLjq5j57Kq9FzQk/t03ybJ6CAq3lmbiEBCUhgQiSggKviWVPAVRGUySQgAQlIQAIS6ELg8MMPD9NOO20MQYbXmU7ahCzgItRG8pZAiKMyEUYnWZmXBBoRyF9WnnfeeTGsX6P0aV2r26Xte+tvLpZq1wMXA+h42cLwUnT33XeHNNBB+ER4I4RBJIMxAL/nnnvG6fwf4TzWWGONfFG44IILYoisfCHhAfGiguV5JVHDuBKntFo3CGNH6N233nor3HDDDfmhdWy6VQFXvet2o2NttK43DLp1B3WDDTaIYYBJR0g2xIHUK4ywa9tuu22czutauu8T5hMB4Y9//OMY7g7h1y9/+cuYfnz9a3Q+xnUbaYdBo0G/nhAStFPWZretcs0kz3qDm/WuIROjgAshFoIsDA93tD0EahjXUQRZhNjDTjvttBhOceeddw5f+9rXotALzyKE+aadY50SnsTMevjfxNwGqqCjnWyxxRbR2yHXWrxtauOPQE9cdxvV8fF3pBPfntu559Rrhz1x7nqzgKu7fl9aT0j6Tr0P6ok8U+1uRcDFPZf+B0ZIt5deeilON6oLjdbV6+PETHv5v3rtopcXOxav0XPChFD+SaWMrQi46j0fTCrMPE4JSEACEyIBBVwVz5oCroqgTCYBCUhAAhKQQBcC66+/flhttdXigC/hsFIIly6JWpyZkAVcLR6ym0mgRwi0+rKy1e165CA6mGknBVwUCwEXL7OTd7M06HDfffeFyy67LJY893KUBtrzQyLEHmFecnv88cfDOeecky8KQ4YMCRtttFFc9uabb8YBfWbGtTilN9eNVgVcXUBnM42OtdG6CUHAhRgQUSACrTKPb+kYwEEInRdffDEKCNkmeYbacccdw6KLLhpFeYceemhGbtxPNjof47qNtHP0jQb9ekJI0E5ZW9m2yjWz2cHNiVHAtc0224RlllkmIi7zbPKDH/wgIDzF7rjjjvD73/8+pG2SJzJCkdOfzkWYcYNe/m9ibwO9HL/FKxDoietuozpe2L2zbRLo9D2nJ85dbxZwpb5ivX5fT7wP6ok8UzVqRcCVti3+NqoLjdY128cp7tf51gg0ek5oLUe36gkCrQi4eqIc5ikBCUhAAj1LQAFXRb4KuCqCMpkEJCABCUhAAl0ITD311FE4gHjhlltuCddee22X9f37949fKxLu5Xe/+12YYYYZwiqrrBLDu+CVBgECX5an0Ez5xp0QcA0ePDgOfvHV5Keffhpef/31WE72Wc/mGxMacsUVVwzzzDNP9JzDNo899lhAjFEUqNV72YrnkoUWWiiGsLnqqqvG2hWD3wjf2BdM8FxCme69994wevToWnry4Q/j63s8+pTZd7/73TDddNNFjwtwzq2Z48m3y6fx4EA+2I033hhefvnlOJ3/oz+56aabxkXPPvtsuP3222urZ5111rDuuuuGueaaK5aTc8G5v/POO8NDDz1US5cmvvOd70Qub7zxRrjuuuvS4trvEkssERBp/Pe//w14OOO3O9twww0DL1IJ+8WXs5y75ZdfPsw555zxK1o8J/3973+P2fDSaKWVVgrzzjtv9J7GV7YMjv7jH/8o3U2fPn1iiCK241g5vpEjR8Zz9txzz5Vuw0LaB1xmm2228OUvfzl62OABBibwK3puS8fwwgsvxHpM/cbTW79+/WJd48tnjg2RBUbYkM022ywKgygXhpcd2BMS7bXXXovLiv+a2a7Zc1vcV9k8dYkX93gfwcMfHD744IPIFK9NTJcZPDintCnOCeEKqau0wXyb7gRcyy67bM2DXr06mO+fr83ZXxJmESKRMnAurr766ph0p512Cossskicpt5feeWVeRbxi3XyePvtt+N1h2srobLwoJJbPoj417/+NQwfPjyuzsUpDLQwaM/APm2OfKgbhHd85ZVX8uxq01XrYjN1o5Z5NsHXsbRDykGbyg1mXEMQsnHeud7Cn2sf19+q1qqAq3jdbnSsnNshY8R0lLVe20rip48//jgg0CtaM9fmdC9FhME5x+sO9Z1rIe2DsKjw5F7COgYoBg4cGGaaaaZYp7ge/fGPfxzrmoIoa/LJJw9cd1LIvryc22+/fdwHyy6//PJwzz33hP333z8eM/cr7jff//73w6BBg+K1Be9z3RnXKzzOYfQZKC+eZuDIec77EZ2sl+22kWav82uvvXasH1yDuG8Syg4xDuEnYVjPYME1ij4ExrWLe+rDDz8cRowYEfJrAB6VEM9R5xdccMEo0mF/cKwXKrfZ6+tyyy0X98G1iesZ3p24znJuqDvcH2mjDOxWtSrXzHqDm8VrCO1wgQUWiNds+qK0B/ps3N+4z+XWTJur178rKxd1g+UYfQnu0TDivMCb/h3X6+Sxgz4m+XP94PpMeu5t3LNyw8MW13CMUKXFvg71CcEW9uSTTwZCWXEN/eY3vxn7koT3pX4gsqQfjkizGSN/QgBTHwmZRNhKysi9BIFxmXXiOt7TbYByc27SPZJ5+k30sWljXGdzS31xrpNXXHFF7Jevueaa8dxedNFF8fym9FX79yl9vV/qFP03jLZffG6p2t+pl39a3my/Mm2XfjvdB+R+sPHGGwf607Qd2jPPRfS/eR7Izw19RJ4psT/96U+l/RvuT9Rdzm2ZFzPulYsttli8V3I9w4spbYlrXd7e2r3uJl7pt7s6ntLx28m6yv2Ea2in+xJ5ecumYct+aXM8r3Kt5trHMwl9k3rPIwMGDIjPSfQbuXfSBkmL91yum1Wt1XtOWTvsqXNX755T1pfsqT5hPZ7d9ft64n1QT+SZjq8VARfvC1ZeeeWYBdcb7oWN6gJ98EZ9OfoMnHMMvvlzKsuauZf0VB+EcpRZWbvI03Xq/pTyrPoskNKX/TZ6piu+E+H4CD1d9b1Od/39cd1e2703tnLdbZZZOkdzzz134DmDfjx58N7s/vvvD1NOOWUYOnRoTEa/66677kqb1P0tPh+QkDxpaxjPsDxP0X/jGGeZZZbwzjvvxHtJo/dLcWP/SUACEpBAjxCgb4/xDMbzAs8IXLv5YzrNV935ZGNewnR9kq+6ZS9Px4OxJgEJSEACEpCABFohsNdee0WxEy+eioOSvMDZZJNNYra87CJEGB2xovGwfvLJJ3dZ3K6AiwFrXryUGYObyTNOvp5BAwaryozwNZQxDcCRpt7L1vxFPwNwCHqSsQ0DCnROi8YLdYQd6SUFg3BJFMXgKS9xi8bLhwMOOCAuLoawavZ4inmn+TQgyDwDo7/5zW/SqtpvHoorfaHLSgYq2Z6Od5mNGjUqnHHGGV3EcYTx4sVNvXBwed2ACWy6szR4zwMCggq4FY0688QTTwQGtIrGuTn77LPjwE6+jsErQhzxYrDMchb5evaRBp7y5WmaMjL4mh5oWJ6OAVEL4hZEhkVjsOvUU0+NL8B42fyzn/2smCTOMxDGAEiZVd2ulXNbtr98GW2W9lOvvnAeGKTOBzNpS1x7EKuUGYNw1Nm0TSMBF+2fdoOxr0suuaR00C/fz7HHHhtFKIhXGIzCGyGiUV4IJ3EqorCtttoqbkaaXOiy8MILh1122SWuo+0zUMUyDFFNXgeSdy/WJY9ITKe6wYAYx8sAWdEYwGfQIgn80vpm6mLVupHyLv6mciIEgFOy7urSM888E8P1Ub+7MwY7t95665js6aefjteX7rZhffG6Tduu134QS6WBnGLeqW2VDbqltM1em/N7KQJPhBXFNkJ9PeWUU8J2Y15Ul51/hD3UVdIlI/whg9uIMbhHFy15XWD5QQcdFMO3ISThhXsSbXOPp3xcy1M40WI++Xx+32SfvEhPx8L1l+ss1ul6mepeK22klet8Ov8MDnBvRhCcrFEouySqS2nTL+Ljk046qUs95bwh0CkzhKu5kJo0rVxfU7vg+oH4tF4fiUH4m266qawoYy2rcs2kHpcNbqbzmK4hZd4L2SF9trz9Ntvm8nqKqJD6jpWVizaEBzCM80T7YFnREIUgzEMEVzTu+VzfqSvJ6Dtxn2egGA9cRUvetlj+hz/8IQqrEIUziIVIlr4U9Y6wvUUexbyK87l4rLiOefrCnMfcOnUd78k2QN+SOj3fFx8l5OVnGrEQx5V/sJHOOddO7ulcs5LxLJEExs3079P29X7zOsX97/TTT49Jm+3v1Ms/LU/tqWq/Mm3Hb3fnu6x/n29fnEbci8iwrO2QFq9y3GOS0JHzwMA5xsccDPoWLXleKoZ2o89Ou6C/VWY8t9EHTyFL03WQtM1ed8vy766Os01P1FXyTM/lnexLlB1jWoYQh/sh+65nqT+RryeEc/qIKV/ONG0RYXRZv6WYlvlW7zll7bCnzl29e07qS+QfA/RUn7CMHcuq9Pt64n1QT+TJ8bQi4MqZ8yEFouxGdYF7d9m7ntSXS/cVylMUcDV7L8nraSf7IJStzPL99eT9iX038yxQVta0rOrza0/09/O6My6uu+3cG1u57rbCjPPCx5Lcj9IzYDpX/PJuGFEXVlXAlfoz6fmAbfO6yjs5+uB8RFw03mPTB+cZUZOABCQggXFHIL3rVsDVDXMFXN0AcrUEJCABCUhAAnUJpK/7ScALsfyL2PyFRcqAl+J4taL/wdeFydKgd5rPRToMCjI42J0lTyB5Ol6+v/XWW1Gwkw9qF18GIK5AZIHxYpjBDF4AIMTAO1JazgB58qpU72Vr/qI/F3Dx5TEhb5IxCIH4CDFRXjZeSPOH2I2Xzkn0hjee4ouF733ve/HLNfLMObVyPKlcxV/OEwP5WL1BwIO/8DpEmqOOOip+IY13GDy4JGNgjEFFXvTANR1XepmZ0vWkgCvtI51jvvYvG8DhOKk7iILSC9j85Tn54CWIkGHpOBjwQ5xD3SbP9EKqeHy8kNp9993jesrBIBeD/LxcpC6k/RUHSNOLqXQM/MKUssIzbZfOEUIlhEEMVFFWjGOinLS35HEsrsj+Vdmu1XOb7WasSb6uh2ca5KENIjrhHDCwx3qMawjtKtkPf/jD6EGBec4RA3W0E754TaIuxAcIhmBTT8CFoI4XxRjn5be//W0lrzKIsShbElsxAMm+EWMieMB4gUi9pk4gsGIwJFk+CE9751iTACkXgeUvIYveVIp1g/LjaY1zjZcXtsWY33fffdOu48vRZupilbpRy7xkIpUzf7lKe2GACjaUm+s1IjPqLV6FUvtC3MZ1uzvrlICL/ddrP7d94YGrUdsqG3Sj7K1cm8vupdRx7iF48OM6lhscubdQLyljsmuuuSbceuutabbh75Ah/z9cZ15vdtttt/jlcrpnM4DOYEE+iNMo4/y+maejzHiaufDCC3ukXqa6l/ZZtY20ep1P5z/tL/0iQmRQsp4hqqM/kETGXNM4z1yvYZ73L1IeXNe5h3BvTdd61nFfTmKHVq+vZfvjBRv5UkYG55Nx/ab9dmdVrpn1BjfTeUzXEAZ/8HRBO+AaAl+uffA488wzY1FaaXN5PW1GwJWOnes89zA8UpUNFvGyknsZ19TUV8DLEALsKoZIjHtN2jYJLNN9LPU76Itwb+J6UfToWG8/eIvD+wTGvYZyce/EG1heXsRMyctZJ6/jPdUGOJ5f/vKXXfoF9EnpO+XXUa53pOOYsbwuct1IzJm++OKLo9fUZvv3MeMG//L7fX5tbba/02AXcVVqT3m6Rv3KlK4n+oAIptL9CiEjfTnODX3m1P+jbTPIirUzSJ1fn6nj9N25psw3RtiX+ktcy1JY4LLrYJXrbuJV/O2ujpO+J+oq16Ik4Epl6um+RC4GZZCcZ2jOJ8886XxTlvTsyHQK/cp0Oj9cT2mnXPuSJc+3ab7eb6v3nLJ22FPnrt49J9XV/Bl0XPcJq/T7euJ9UE/kSR3plICrUV3gutWoL5ffV3IBVyv3kryepjbQk32QfH89eX9q9X1JYpD/Vnl+7an+/rhur63eG1u57rbKbIUVVgibb7557RRx/+W5gvOUP1eQoPjOtrZRYSL1Z9LzAavzupqS03fDkyPPzjzHpD5d6jendP5KQAISkEDPE1DAVZExL1s0CUhAAhKQgAQk0CqB9GIUD0a8TE1WfGGBNxQ84TBohfESmTQYQo3kwYD5Tgi4hg0bFsPZkB+We4niRShCBh7iGeDCixUP8LzwwstF7qUmF1jkX3HXe9mav+jPBVxHHHFEbSCv+DIC9+G44qcMSYBDmfO8krCL5clSnhwHAhUGnlo9npRn2W8+kAAfXnIkoy+ZBjlyr1m5t6CicIAXJpQ3iY7wLpEERT0t4OLcs4800FwU/lGO3NsFg+AMwmJ5GKP83BCu6vzzz09IogiHtElYwUslhIsY3lcYJMLwqsDLz2T5l+p5PWB9ejHFNPWU/aUwRgyCMIDLiyosHwghRAx/2HnnnVc3FGdMkP1rtF2r5zbLfqxJvIbgPQRL4dnyRHhTYcAHY0AjeV7jusFxM8hDm07XF9LlL8iToIqXg7DC8EKGKIKQpoRtxGhLcEr1MS5s8A+xFueNQW62pW6zD64htMdkaRCG+VzowDRtKAnTaBMIubgW5CI+PBHwZSyWL2c+rxvUG8LFpJA0lA0BZhoA5dwxCIq1Whcb1Y2YcZ1/qZz5y9U8L8KPcd1ORr2GDyyqChA6JeBK9SgvX7H9NFqXznc+6Nbqtbl4L0VAlsJzwoh9JaEb7YDzzz0VI8zqDjvsEKe5XpxzzjlxutG/3PsjdTr3escxIIxE+MBgLC/vaZe0x3RNbZR3ft8kHQP1eN2iTifriXqZ6h77aKaNtHqdT+ef/cEQ4Rz3cPbdncE4hbQsenHMy0O+CN4QviVLHiuYzwU2rV5fi/tDKJ5EO+yD+zgCYixdT+NMg39Vrpn5tTsf3EznMb+GsKt0H0BUka7vLG+1zeX1tFkBF22PAeJ0Dcn7EJSJ0KPc4zCu/Vyfab95/ymurPOP9oZ4K/Wf8usBAg2EVqk9ki9CWHjlbaxO1nFx6oNxLaHvRztPhleRJEDM+8P5tbAT1/GeaAN4eyDkOAYfvKekY6MPQV1Oou+8v53XRbblWYd2x/lKlvrizFft36dty37zQcd8gDzV86r9nbK882WpPbGsmX5lp/uA9JuSN+FcpEW5qMO06fTRD173uK+2Okid39+4HlPHU1+NDwgQkqUPCZIwv3gdrHrdpfz1rFEd76m6Oq77EghYuBdiCAOpN9y3kuXP5Xmby+9jRZFWfm3O20bKs+y31XtOvXbYE+cuP678npP6EnlfclyfR463Sr+vJ94H9USenRJwUdca1YVG6/L7St7HaeVektdTytTTfZB8f3kb7PT9qdVnARjUs7yvUnymy6/zzbzXSW2UfZb198d1e2313tjKdbdVZnk9z/uwMCR8NB4+kxX7VGl58Tf1Z/Lng7yukp5nBCIqpOdVPOLuvPPOMSv6QPkHdsX8nZeABCQggc4TUMBVkakCroqgTCYBCUhAAhKQQCkBhEdjQk3HlxY8+DKwgOUvLOiYpQHJlAkvyBkowoqDbu0KuOoNVOchoZLHsKFDh9bCLfI1/YMPPpiKWPtNLxp4MYNIhBep9V625i8zkoArD81WFAilneDtJYVOu+CCC+Kg7IABAwJfvmK8/GZgIRmDeClEUC7oaPV4Ur5lv/ng76OPPhoFLild/gKekE2Ebho0aFAMFUmavGxpG36pM9QdLA+jmQYP6w1m5nWDl2ZJyBMzqvMvvdRhdfGFXS5wYDCHwaF8gCEPIYlHD0Lx5V7Jyuo2+8kHiRhg55xivCDj3FHnk/AtrvjiXwoJRRkI8ZIsP4ayesoLqBSaCXEXLx+xRi8rU95lv/W2a+fclu0nLdtoo42iO32Om3aaD46ShvVDxngFwvBSgmAKSyFyOHe8BE+D5qxDeJdezBGi5YEHHojiqjTAj+CAtpyEY+wbgQuh3DpteViCFG4McRViOyz/8pNrJS/+uZYyUI/lQkNCx9xwww1xOf/yupGuHbWVYyZ23XXXGDaAZXn9b7Uu1qsb+T7LplM585eruUA2H8BL2xP6DA87DLRX8U7TmwVcrV6b83tp8V4Jp1wgUsYQQReD4LlHkcS3+Mv1EM+JiOawS8aEEaXddMry+yYvyxFN5G2W/fREvUx1j/yrtpF2rvP5gE5xcIIyNLJGg355/yK/r6T88vtOLv5q9fqa76+s75J/RZ/vL5Wn1d96g5vpPObXEPaRBg6L7aPVNpfX03wwvaxc+QAR9xCEKEkYRNnya3cuemIdxn0L72nFe/7/re36n3sa6dkn9v/YOw9oS4qijzcGkh5ABYEl7BIkiSDg6hJEgqCL5CxBEBQRFcSEYNgPRfw+xbAoQRBRAQExICw5LCsiSbLCquCCREEJHkVR0W9/DXWt19s909Mz9777dqvOeW/mTujp+Xd1d3X1f6qxV7EHuhQmzCGHsWQ2dSUUWTZK26Rdt+P9qAP0tfS54Eybia5ogfyGrY9ALIXYg+gyD5dA5nypfc+9KdE6pSfIm9o7qfTluNQnfufalf2wAbUdHqsjLBvMOAThQyD6stJJah3pi4lciarsE5/9Ty+3Kh+f6HawSbsraca2VTreL10dpC3BOzOGlSXC+YhCiKuCB9G0ZMJcEz/1WF1/kCL30a5DXmccwHJ2XYiu50KoSdXDfpRdqs8RWyJF4Ar7PLDo0iZsgm0//EH9SHNYCVylfYnW037bIOiDfl4/+6fSsUCVzqbGr/209wfd7pb2jU3b3VLMtA0Rs6koP52Xrghc1A3sOiELiJ5oQjqkQRNDwBAwBAyBwSEgbTK+f/wP+CDxW/LHvvzOzdF8syeZ/vupSO5dY+A6I3CNgUKyLBoChoAhYAgYAkOMgP7C9vzzz3dXXHGFz612WAixJ3wNIeuE0VU0SUcvDRjer3/rSTImoHAKhCLLPXEcp+/VV1/tSTTYQ1VLG7373e92hPJHJFpUytmqHf1C4CICChMUiBDH/A/1T4cuv+mmm9xpp53mz8rXpzgemOyWL8X1hJmekJaIPk3fR2Vljl3IduQDA1o7kblQO4qZuMT41l/PacKITpjILThNEO2AFp3oB4ELDDUpimdT9kKkijmSIPfIxNHZZ5/trr32WqdJIiyTx8REKEQ8AjOEsPDglBImIvhCnEkq0gbnMK8y0aZJPTo97ZTUzi59PFUWOh3ZT93Xpmwl7dwtGBJNBAIPZSARRzSBSzve0E0mtKnXEoEqfJaOwAX5TyJucJ3U7fCeLn7rpUCEsKid9RAfaScRTRqUCUacjpLXkLgouhHqjORbL4mldUPO622OLqZ0Q6cT25d8avKFJoBwD5HLfvrTn3oCa0jsiaUZHtN1k6iPJ5xwQnhJ9Hes3ebCqnetOhebdCttm3Vfev3117szzzxzxDsccsghboUVVvDHaD+JjqWFiG5EA4RUwsRkSiA3UJ+kntXpSiqdquO632xC+Gmrl6J7TeqI1qWm7byUP89jEoptrlRNDms9JTLaXXfdNSJZlmCVZVIhG8sygiMuev5HTvuqnwdxR0cnJRnaJCG61D0vlofUsdhkOtdKOeo2hOMpAldpndN62oTAFSNJshwhbTACkZj0tEBgkGXB6iaPhNzL/RCsILMMQtBJbJRNZpOoJzwfQVQTuLpux7uuAziBIbIiLMUmRK0QOyH36HZC66Imx8u9bex7SSPcpibIm9o7Ybrhb6lPTezKftiAmsxOHlkCFUIPhB9x7Id5L52kFhJcaozEeIcJZoSPgSDz6Xawbbsr75HS8X7q6qBsCXnH2Jb3Y2lvovlttdVWfgzGdZrApT9IoZzo57CPIYv0S3Q9LyVwtSm7VJ8jtoQeew9DOcbKoR/+oH6kOawErtK+RPcX/bZBKHf9PE3g6rp/iulYzlggdp8cS43b+mnvD7q+lvaNTdvdUsy0/zLld9EfjuaORcWe0eMDrav4heSjOdEHtozR6I9knz7HxBAwBAwBQ2AwCMg4zwhcNXgbgasGIDttCBgChoAhYAgYArUIyDI6mqiiHRYs90QUilDkK+OuCVyQdGITpvorb3EWSzQB8ha7h+MQakSEaJFytmpHvxC4BB/SSD2Dc/IcvRzl29/+dscSi4hE7mFfiF2QHHiOSOn7yP2prZ5kZCKMr+RxpElUMD2ZR9QwvrxGmEwW0lmYthALdNjyfhK4tANc8qKJZLEJ6BiBa8cdd3Qbb7yxJJEsUylP3l8m1bmJrxYpVyZCmUSX63oJzt7RE4gcF8dU7B04r8lA2tmVclZyT5Wk7mtTtlXP4xwkE6IfrLXWWn5pNiZDYqIJXBMnTvRLroYYolPoKBOAN9xwQy8ZTeDqHXx+JxW5L7yu5Df5o96wFX3QxBtdT3RUAvL+ve99b0SkMYm8J/mo0w1NINC6wf0lupjSDclPaiv51M5VyEJHHHGEjzgW3gexE5ICpAcmJXJEO5SHjcBV2jbrvlSTSQSPgw8+2K244or+Z6zvk3a2jsCl2zXqDUu9dS2639T9WficrvVSdC/VfsbqiMaD/KX6bml7pF5zrUy6EhkwrK+cr5LUxD73aPtCJpl1WpqQHOp/Sfuqn6fbKHmmJl6Ez5NrSraxyXTSkXLUbQjHUwSu0jqn9VTXuVi+9ASRkHPJk8hmm23ml/Dmd7i0N8e0bVVF4NLLF2p7izS6FtoTljknMpgsJxc+Q+eh63a86zqgseM96uoy18hSfbrMIStC6NHSxr7X6eh9rVN6grypvaPTjO1LfUq1izG7sl824F577eU/YAjzSV1nTASxWyKvck3JJHUqylr4zPC3bgebtrthWvI7peP91NVB2RLyjrIlMiRLldOvC0FczslWxuT8pt2h36QeaKHeshz9zTff7PUhNbbU9+Tu63ouZZyqh/0ou1SfI7aErqOjVY45WEp72JU/iGd2neawErjkPXnnVB/FObE5xVek9bSfNgjPRvTz+tk/8aySsQD3pSQ1fu2nvT/o+lrSN4JX03a3FDMii/NBACJtrf+h/q2xxhrugAMO8EdCv4W6bMSu2DN6fKB1FfshFslbR+olb7KSxIjE7YchYAgYAoZAXxAwAlcmrEbgygTKLjMEDAFDwBAwBAyBJAIQjCCkIAygcbBqhwUEBE2ikIQGTeAikg+OeARnMU4BvsZuIrKkRsrZqh39QuDSXyXmPEs7GTThRCIHjB8/vhdJSkcxwalX+j51+dIRwmQJER3ZR09yEolr8cUXn4OEFD5DnC366/86Atc73/lOt/baa/ukcGw3WUIxFtVLE7j4ypuv67XECFw6Ipu+NrWvHe+bbrqpn8wVB6zcg7MWIiMT4kxw8FtHCxOsYu9AGrGJNo6nnJWcq5LUfW3Ktup5OPMgNEEyCAX8EMhuiCZw8ftVr3qVw5FI9JkQV86DGZFYSEfXJ87xpSX3yH3HHXecnyzkXNeio6dAhiBPOBf1ck3yTE1IIOqHTOzHnPN1uhEjp/CcUl1M6YbkPbWVfGrnKtcKsQTylZRxmIaO7hie07+HlcCFfpW2zXV9qSZwiZ5oTHIJXDqdfjmxU/2mzm8/9FJ0L9V+xupIm3Y+Numq37FqPzU5zD3avoiVdYrAVdq+1j1vmAlcbepcSk/rJvljJGBN4IIUGS4jlkvg0umkbOoqvco9l9J7bBIiI0kkSE3gIu0u2/Gu64C2X3NxkInFWJnrNNrY9zodva8nHfUEOdc0sXd0mrH9unYxZlf2ywYkfxtttJHbcsstHVF3YqKxyJmkFqKGLMsIMZF+DtHLVseepY/VtYOpdlenEe6ndLyfujooW0LeFfIn9QgSRih8ZEF/zLKwiCZw8RtbfbfddvP6HiN9MW6cOnWq/1CD69tKrJ6n6mE/yi7V58RsiUGXYxNsu/YH8eyu05R2Ifx4sOo9NeYSwZ3rU7pQdy6mb6V9idbTftoggo9+nm6TOd9l/1Q6FpB8xrap8WvK7omlwTHt14nVUX2f1p2Y7abHXjG7PncMJ88s6Rvl3ibtbilmRKSXdj/2vuSFMQvjUKQrAlesbpC+EbhAwcQQMAQMgdFBwAhcmbgbgSsTKLvMEDAEDAFDwBAwBCoRkEgHt912m4NwUOewILF+EbggPzz44INz5JfJgZ133tkf/9GPfuS/4CUqDpF+iJTBcmV1wvJUkD5Szlbt6BcCl/6qEschjusqgZSEY1FElk9jAo2oACwXKUsyMgEDYU6k9H3k/tSWyVCcSDjSxXGl88XX0rLkWSqyUJi26IwmsNQRuPTX/xBgciLzVE1SlRC4iEyBfiNEBYiRE/W7MknB18hELCPPQhZ64IEH/Jfk1Bl5D3AcN27c0BK42pStxiTcJ5Ib+CDoA8uI8pU99Zj6tu222zomsJGQwOUPzv7HBDLRu9ZZZx0fkYiyFREnsyZwUQ8h1TA5IBHVIBcx4dSPEPp66SGiSsn76CVTJb8a52uuucYRwQCRCIByHdsq/eZ8jJzSRhdTDnCeVSWSz5DApe9ZcsklHVFG+PqWZXakrnANBDgZ5Ot79P6wErjIY2nbXNeXduX8F6d6GNVR49t2P9VvSrr90kvRvSYErtJ2nnepm9CR941tqyYEtX0Rm/hIEQlK29e65w0zgQtsS+tcSk9jk656MjM2QaSJV20IXHqZcMoTAlXXoskj2JtEVWNZYqKTUncQsdtCApfOS9t2vOs6oD96wKYIl6HVeWcf20CWYY6Vub6+rX2v05J9rVNiu8g52ebYO3JtalvXLsYIXNo2iUXlk2eJnmj7Xs7VbbHd6MsZ51B24CHCkuUsaVs3Sa3bJiFwQSQ68sgjfVLoAWPFHKlrB1PtblXaKR3vp64OypaQ99YT/NgVjHVoTyDPMY7ExmNMi4QELkmDLaQQlrXkIywhkHK8CQGH66skVs9T9bAfZZfqc2K2xKDLsQq32Dmp+134gyT9LtMcVgJXaV+i9bSfNoiUhX5ev/qnNmMByWdsmxq/9tPeH3R9LekbY1jVtbulmOmPFqZMmeKIth2KHssbgStEx34bAoaAITD3ICC+XfwMzDPh+2Vujj/25XfuG883adKk/+RePJauMwLXWCoty6shYAgYAoaAITC8CEhkJImmVOew4E36ReDSX0dqxN7znve41Vdf3R/iy12cyDKxyWQVX3vFiBtMIghhikhTGJgpZ6t29AuBSzuxifBEpKdQiFjFl+fIjTfeOCIK0BZbbOGIBIXwfBxQfNmM0wPnh5bS99FppPb1u5144onuwAMP9JfK5Ijct/vuu7vZtrP/ecYZZ/j3kXOy1ZMUEJkghyFC4BKSmFwvW/2F6mgRuPQkpyxxJ/mTLYONXXbZxZOKwIcJp0022cRtv/32/pLUclOyNCb6OIwRuNqUrWATbnHUorcIEzJEeAjroSbuCYGLpQaFeHXhhRd6kpxOmwkfJr0lXZbq0wSuK664whHZCTn66KMdy+sgqTL1J1v8W2GFFXyUMZKgnZSoArGoX3riVF8rUUF0NuomYWMErja6mHKA6zzF9iWfmsA1efJkXyZMps6YMWPEbeADSVSiN8SWHxtxw+wf2umbqmPhPfzWbZu02xyveteqc7FJt9K2ua4v7YrABcGZCVWIGdOnT+f1O5dUvykP6pdeiu41IXCVtvO8S6z85R3rtqnJYe7TeppL4CptX3Oep0kSTepbHQaxyXTukXLUbQjHdcRCbROV1rmUnsbypScz+zl5utpqq3k7FNsAEks/RPezsUkzTVTRBK6u2/Gu6wB9CQQAhA8kqJ8xYZk3yGfoF2RpJFbm+t629r1OS/a1TskEeYm9I+mltlKfUu2itkNEH/phA2Krrbrqqv5Dmphuy/iS95ClzhkvbbXVVv7Vzj33XHfVVVf5ffmno/boMQqkfexzveSt3MMWsjy2O3LxxRf7v5J21ydQ8S+l4/3U1UHZEvLaejxDnSOCnxYi7Gy33Xb+kBC4ICbysQPCRxyUtxYit2AXyhLrfPTCB1htJVbPY/WQ5/Sj7FJ9TsyWGHQ5NsVW6msX/iB5dpdpDiuBq7Qv0XraTxtEykI/r1/9U5uxgOQztk2N2/pp7w+6vpb0jSXtLm3WHnvs4WG8TuJTAABAAElEQVRO+U1ivjD9MYJ8TBuWFelSJojYHuE14W+xZ/T4QOtqrG6QhkXgCpG034aAIWAIDA4BI3BlYm0Erkyg7DJDwBAwBAwBQ8AQqESASQ++HkT4uh2CEV9nIbGQ4RzvF4ELQ5BJPE0CYaIRkgZOX74ExunLRJiesLrsssvcBRdcQNZ6gjOdfOIEID2+HGObcrZqR78QAbQjSpOVeg+ZvaO/SAsJHTwbYpNMOvAuiEwu+B/P/yt9H51Gap+v8XB0IBCsZLmz73znO46lHEWIgLTPPvv4n0SWgmgVinaYyLKUXCNRvdgPl/HaYIMNeo59zo8WgUuTgLQukScRvfQiSyYReYOJCiYskOuvv36OKBAa334QuIiMxxfROaKdnPq+NmWbeu4yyyzjqCtITF8gIBAdCAcjIgQultLEqY9AkItN/MkEkkzW6bK7/PLL3bRp0/z9IfZEbrn//vv9uS7/yVfkkqZMcMhv2WpihBxLRRsQp2VqEjZG4GqjiyndkHymtpJP7VwlGgYELdpU6jt6r0VP8MWij+lr2e8ngUvXA55VhUNs0q20bR60859365ek+k15Xr/0UnSvSR3RbUWTdp53iZW/vGPdNjU5zH3avsglcJW2rznP0+3UMBK4SutcSk/rJvljE0RdReCq05suzhMNBxInwiT3vffe6/flH8QWiQapCVxdt+Nd1wHyL7YA+7EovXrJHv1usTInDZG29r2ko7d60lEmyEvsHZ1mbL+uXYwRuPphAzJeIF0kpneQ6xm/Ib/97W8dYyQIjfIRyR133OFOOeUUf55/jPNog7EdEU3gwpaU5ZtOPvlkR53VEvtIpKTd1WnG9qt0vF+6OmhbQuxd7LrwAynGs3xUscQSS3h4hMBF2XEf5x999NFeuWsMdbse+6hBX5u7H6vnsXpIev0ou1SfE7MlBl2OuRjKdV36g/qR5rASuEr7Eq2ng7BB9PP61T+1GQuIzsS2qXFbP+39QdfXkr6xpN3FzysfTDQZI2myXOxDVPyu+DzZIkbgimmyHTMEDAFDYO5AwAhcmeVoBK5MoOwyQ8AQMAQMAUPAEKhFQBzjLOkHoWK0CFxkFIc9hA62Sy21lPvABz7gZEk1HXmH5RhYmgtnMU5mHAU4khGc/NyHsxYhchYRtJCUs1U7+oXAxfV6UoAlJIhMxfNwUEBCWXPNNbnM/fWvf/VLuPkf6h/RmIhaJcK9TLaFXx6Xvo+kW7fVEwtcy1IzkM9CkQk9jjPhQsQuyCo4iYhCJZGTwkhb8pUt9xGV56STTvL38TX91ltv7cuJc8hoEbh4to7mxiQDhB9IQgiEGQhcolM44PnyXE/AUW7cwzI9TE7h4MOxyD0I5fuhD32oR2gpmWgjHe04xNH6rW99q5dPzqek6r7Ssk09i3cGC3l38nj77bf7usGSiG9/+9s9IVTuZ6KOCTuch+gj96GH1E10DYHsxVeoEtWOiWic9tpJqwlc3KPrbsmyP6RRJ3qCnGurlvFBv/VY7c477/T1IXxGnW7ECFxtdLFKN8K86d+ST03g2n///XvRDWlfmVAV4i2TQLS/MglL/8JSpFWiCVyQddD5KoFQy3KWuux1u131rlXnYpNupW2zfk6MDN1VBC4c8YsssojvU4SMXYVdyblUvylp9UsvRfeaELjIU0k7z32x8ud4jmiyAvYAeYe8Sb3QeppL4CptX8lr3fOGhcAl5Uu/CTmHfhW8SutcSk/rJvn7OXlKW8nSssjXvva1OchV/kTLf5pIgx3PsuLoIDYwE6r0xyJ64q3rdrzrOkCe9QcA2J0QwYWkzdJs73rXu3p2ho72GCtzwUC2be17SUe2sQnyEntH0kttpd6k2sUYgYu0urYBdb9NH0+ULFlWiXEbZQDJHuEjGz62YewkUdWo9/SNLEe98sore12FuCqiCVzUoQMOOMCfYkzyzW9+04/taCf33HNPTwLnpB6L1bWDOjJdLpG1Ssf7pauDsiUEd60nfMjC+BrMiUYL1kSeFiHa1ne/+13/U9u+RMllzC5CNGzGiYwlGUfxMVYXEqvnsXrIs/pRdqk+J2ZL9KscdXkRFRv7uFS68gfp53eVphC40MW6j5rQMT7A0pjrKO9VulB1LqZvvGtJX6L1tJ82iJSFfp4QuLrun9qMBSSfsa0ux9An0i97Xz+zn2M4ed/SvrGk3W2LGXnGP3PCCSf4D0Pp7/Ep4qsRMQKXIGFbQ8AQMATmPgSMwJVZpnpSIPMWu8wQMAQMAUPAEDAEDIEoAiw/ss022/hzONll2b+Yw4KL+hWBS2cOx74QQzgOMQMnJY47kR122MExUSHCPRBCcFKJQM6B6MVXZkjK2aod/ZoIwBdxODokLzyDtHB6iXCMCYmY01Q7s7j+vvvu85Mccq/elryPvr9qX4c+5zqcn0SmCYXJFrCQ9+U8mON0l2O8L0Q2CG0ienJFjukt98j9o0ngYuKa52sdoTwhDkn+yLeefMCpRhQBiVzGefRMIkvxW0c2Ay+WzGRpudKJNpxhOIS1pELW62uq7istW51+uK9JKJwLdQXSz0ILLeRvQwcgM0H2CfURZzvnqVdSDvzma86HH364ksBFmYIV5YSwjBzR4bqUbbfd1hGRRYSlHy+99FL5OWL7jne8w6277rq9Y6nlSOt0I0bgaqOLVbrRy2xkR/KpCVyQs5igol1AKCvKkPzpeiFRNyLJjjikJ4JHnEj8kIhvqXa76l2rzsUm3chCSds8KOe/LENHGejlWxPQFR1O9ZuSWL/0UnQvRVSI1RHyVNLOc1+q/DmXI5JfuVb0X+tpLoGLNErb17rnDQuBS+eT99VtTEmdS+lpbNJVT2b2c/IUQvfyyy/P6/kIREJW9gc6+gfphckz6TtJlr5Y+kTaBm0bY/Ogq7TZXbbjPLfrOkCaIYGad0Pk/dj/3e9+54lr7COxMn/uzH//t7Xv/5vSc3tap2SCnDNN7Z0w3fC3YJxqF1MErn7YgELOII/oGbYw5aJtbIisjMOk3Ii+JBGcwncjDdFjTeDiOh0BmN9ci8j1/IbkwTgL0e1Lk3bX31zxT/CXS6Sd53c/dHVQtoS8j/YJcCzEORwzQd5DD8LxLuXNtYydpIxILycqK9flSKyep+oh6XVddqk+J2ZL9KscNXkIX4QQXHPwC6/RZd/GH6TT7SpNIXDptFP76Cz2sMZcE7i4r0oXUudi+kZaJX2J1tN+2iDkD9HP61f/1GYs8Fwu4/+rxm39sve17sT8odo+j/UvkCkZD2tScfzt/nu0pG8saXdLMQv1nJxrf5jeNwLXf8vV9gwBQ8AQmNsQMAJXZokagSsTKLvMEDAEDAFDwBAwBGoRwOGCowEHK45o+WI6XGJPEkoRuHbffXc3adIkf9lFF13kLrnkErklud1rr738l9M4elkWjWhNelKGG3HiT506tef814mtv/76juVhhEigzxElB7KIjv5CJIL99tvPX6YJMSzpgWMC0QQufrNEy3vf+17/9S6/tUAsw7Ezc+ZMfXjEvkywc1CiEI24QP1o+j7q1spdvTQCF+Lwfeyxx6L38L5MfAjxRl/EhN/xxx8fjSJB2e+2224jHPXcy1d6OCeFGJhL4II0xZewsahK2pkXLsPCM/VSiKeffvoIshlp8n7LLrssl44Q3o8oAfxpSWEievvLX/7SL0UqenjVVVe5c8891xO/Uu9A+qmJNs7pLyT5Tf0g+lSdVN2Xeg/SrCrb1DOZmGGCGv3SgvMcTIjKddRRR/Wi6EHGIvoWdZw6J22Nvpd9JgEhPhHRC2G5PgicSBiBi2ObbLKJjxDHPs8mEpFEVuNYWyEa4Mc//vFeMkxGysC1d/D5HdoRWSKIvNCe4NQMpUq/uTZFTkmVYZ0ukmaVbnA+JpJPTa7gOiIqEPlFE7b0/bSJRMmIvbu+jn29vFN4LvZbCFxV7XbVu6bOSSSD8F3JQ9O2mSXL6JuQWF+a6/xHz9C3lEByhPgoE1ap69ocT/WbOs1+6KXoXqwP4NmpOsK5kna+qvxJs0622GILN3ny5J49MmvWLG+7aD2NTfSkIsGUtq91z+sXgUtHgqK9lDZYyjGsV7T/RFAScjR9kI7M0rTOpfQ0li/aLexeJEZo10sohjYE9+jls2NlyjWIjsIaLrP93BXd/Ke9IXqu2CCSKphjB7PEorRHnJNILV2246TbdR0gTYSInkR0DYV2DzuBNlaiQHJNrMzDe/lNu9XGvtdpap3SE+RN7R2dZmxf6lOqXayyK1PtNM8psQGJ8kZfho0WE5a1JMqqHm/QX1HPdSQn7oXsw6Tvjjvu6Em4IYGLa9BxJtZDYZKcsRhjDZG6djDV7sr9qW1Kx+X6rnV1ULaE5J9tSDqUc5QjRJo99tjDvfrVr/aHsT1ZahFhDA9hRxO2/InZ/6ifRE6VJdDleJttrJ6n6iHP6brsUn1OzJboVzkyxpKIt/hniGRZKl35g/Tzu0qzhMClMQ8JXFW6kDoX0zd516Z9idbTftogkj/9vH72T6k+JmeMKnmNbVPjNq7th72vdaefYzj9rqV9Y0m7W4IZeV1xxRW9zaRJ2hzHb0NUdokEnUvgEntGjw+0rsbqBs/ThG76H/TLxBAwBAwBQ2AwCIgfnI+JsPOw+/FB8Me+/M7NzXyzJ5Ke+ywn944xcp0RuMZIQVk2DQFDwBAwBAwBQ6AxAjgH+KoVJyQDdxzzVYLDY9VVV/UELBwKTO7feOON7vHHH6+6rdE5jFCWjyDSFMv7kDbRhPjqv2vpx/uQpixbR975OrhKML4nTJjgMeVLeaKLscRILMqYTgecuI+lbTDoIXFQHsMokHIoT4hc5BWiGcuBpARMiKyEHhBRgGshJIlgn6+00kr+K1dIZTJ5LedLtkyOEV2DQRJRBfQEZVV6VfeVlm3V85iIX3311T2RhMlUiKBMrCLUSUhNEALvueeeEfrABN7EiRP95DL5YllLdOyWW27p3V/13Hn1XBtdrNKNpngyYIcAQD1iAol6RLvNJCpLeY22VL1r1blUvvvRNqeeNRaPD4teauyatvP63pJ9yFH0f5Ah6FO66AdK29eS/I/GPfQd6A5tvyzBJvmwOidI1G8htmOjYNM89NBDvh+F5CMCiQt7gok22mixJ7pux/tRB3gHbCxII4wRIAbTx1x77bW+rsk7lmwHZd8Pi71DXcNOxy5rYt9XYUuZQMRmCSXKBjsAe68q4hw2A+XJPZC1GGPkTMKir4z5sLfRb4iy3D9IqdPxfunqIN+RvhP7Dr2F8MG4hn5NBP2hDBnjUQYi0g6hE5BzWYqe9ujWW2+tHc9LGv3czgtl10/85qa0q3Sh6lwKg0H1JanntzneZf/UZixQ9Q5147ZB2/tVeW1zrqRvLG13SzCjD2bcTx9Mn43fRyJftnlvu9cQMAQMAUNgbCBgBK7McjICVyZQdpkhYAgYAoaAIWAIGAKGgP8qmmglSO5XcQabIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAobAvImAEbgyy90IXJlA2WWGgCFgCBgChoAhYAjMowjw9SiRFgiTPmXKFB8dKVyiaB6Fxl7bEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUOgAgEjcFWAo08ZgUujYfuGgCFgCBgChoAhYAgYAiECe++9t19Oh5D+IpdffrmbNm2a/LStIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAITAHAkbgmgOS+AEjcMVxsaOGgCFgCBgChoAhYAgYAs8hAIFrvfXW68HxwAMPuGOOOab323YMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMgRgCRuCKoRI5ZgSuCCh2yBAwBAwBQ8AQMAQMAUOgh8Byyy3nJk6c6Fg28e6773YzZ87snbMdQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQyCFgBG4UsgEx43AFQBiPw0BQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDoDUCRuDKhNAIXJlA2WWGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIZCNgBG4MqEyAlcmUHaZIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAhkI2AErkyojMCVCZRdZggYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgC2QgYgSsTKiNwZQJllxkChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGQDYCRuDKhMoIXJlA2WWGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIZCNgBG4MqEyAlcmUHaZIWAIGAKGgCFgCAw1AltssYVbYokl3GOPPeYuu+yyoc6rZW50EXjta1/r1ltvPfeKV7zCPfzww+60004b3Qy1eDrv8MY3vtGncN1117lHHnmkRWrOvf71r3fjxo1zf/7zn92VV17ZKq1hvHn8+PFuww039Fk788wz3X/+8x8333zzube97W3uRS96kbvnnnvcHXfcMYxZH6o8DQNmg9bVQT9vtAp8scUWc5tssknR42lPr7/++qJ7w5uop/TpTzzxhJsxY0Z42n4PEQLYXyuttJJbdNFFfflfddVVQ5S7sZUV/HObbbaZ75duuOEG99BDD/kX6Lo+TJ482S2wwALud7/7nbv99tvHFkiW23kWAdoY7DVk+vTp3obvCozVVlvN8Yecd9557t///nfrpBdccEG3zTbbuKWXXtottNBC7qyzznL33Xdf63QHmUA/cKnKf8xOr7q+7bkuy2gYbOO2eFTdT9mss846/pKbbrrJ3X///VWX157regxb+8CCC7rUj4LH2y2zEdh6663dIoss4mbOnOluvvnmMYkJY8iVV17ZPf744+7iiy8ek+8QZjpml9JfrLvuuv7S733ve+Et9tsQMAQMAUPAEOghYASuHhTVO0bgqsbHzhoChoAhYAgYAobA8CDAZNMyyyzjM/Tggw+6Z555ppe5z3/+8945/pe//MV98pOf7B23nTgCTILgOIXAMmvWrPhFAzg6YcIE94IXvMA7tJ588sm+P3Hbbbf1k6PyIAYNn/rUp+TnmNtC3tppp518vn/4wx+6q6++utU7fOYzn/FOUurWYYcd1iqtYbx59913d5MmTfJZ+/CHP+yeffZZ9+IXv9h98Ytf9Md+85vfuOOPP34Ysz4qeUq1E8OA2aB1ddDPG5UCn/1QJhn22GOPosdD/Pz0pz9ddG94k/Tpf/vb39zhhx/eO73UUku5hRde2D399NNzEFarzvUSGIKdVL0agqw1zsJHPvIRt+yyy/bugwB7yimn9H4Pemes6EAKFwjm++67rz8NiUSI1Kn6UKVLVee++tWv+mdA4Dr22GNT2bHjQ4jAoO3mYYLgNa95jdt///19ls4++2x37bXXdpa9gw46yK2yyio+vY9+9KPun//8Z6u0559/fnfUUUc5tiKnn366+8UvfiE/x8S2a1zqXjpmp9fdU3q+tIxSbesw2MalWOTch22IjYjwsdwFF1zQu62k7+16DNvLTEc7VfqR0oGOHm3JKAS+8pWveFL7r3/9a3fCCSeoM+ndYesn8U0uvvjifuxyxBFHpDM+hs7E7NK9997bfyTJa3zwgx8cQ29jWTUEDAFDwBAYNAJG4MpE3AhcmUDZZYaAIWAIGAKGgCEw6gi89a1vdfwhp556qrvtttt6eRInghG4epBU7nzoQx9yyy+/vCdwHXrooZXX9uskBDIhT91yyy3uO9/5Tr8e1Uv3s5/9rBP79+9//7v/evi4447rnR9rO107v+d2kkpsYmhun3Bpo9OpdmIYMBu0rg76eW3Krc29w07ggmyJ/j311FNuypQpI1616tyIC0f5R6pejXK2Gj+eqIXHHHNM776//vWvnlAxbdq03rFB74wVHUjh0pTAVaVLVeeMwJUqgeE+Php28zAhMpYIXET6ZTIb4WMZxqdnnHGGjyIzTJjW5WVuJnCVllGqbR0G27iuPNucryJwlfS9XY9h27xb7N4q/UjpQCwdO9YOgaYErmHsJ43A1U4H7G5DwBAwBAyBuQ8BI3BllqlMYGVebpcZAoaAIWAIGAKGgCEwaghoAte3vvWtEcu+GIGrWbEMg+NxNBxs4gTsMlJMM+S7vXrJJZd0LF+FsGzVAw880OoBcztJJUbgYskTjr/whS/0E2tjLTpCqwKvuTnVTgwDZoPW1UE/r6Zo+nYaUg5LlYSy4oorur322ssf/uMf/xiNVEfEEnHEhPc3/U27Rvv2pz/9yV100UW926smCavO9RIYgp1UvRqCrDXKwhprrOEOOOAAfw+Eeoj1oy1jRQdSOKUIXKn6UKVLVeeMwJUqgeE+Php28zAh0k8C19prr+1IH2Hpp7ZLKNJfvu51r/Pp8YEKH6qMRekalzoMYnZ63T2l50vLKNW2DoNtXIpFzn1dE7i6HsPmvEOTa6r0I6UDTdK3a/MQEN9NbgSuYewn50YCV8wutQhceTptVxkChoAhYAi4nt+Qj1zwxWNHszoLf+zL71ys5pu91MZ/ci8eS9cZgWsslZbl1RAwBAwBQ8AQmLcRMAJXd+U/DI7H0XCwyaTl73//e/flL3+5O0DnkpTmdpLKICeG5gaVGIZ2IoXjoHV10M9LvfdoHYfAdfDBB/vH/+EPf3CQpkdDqgg6VedGI6+pZw5zvUrlOXZ8nXXWcfvss48/dfHFFzv+RlvGig6kcEoRuFLXV+lS1TmxhWwJxRSyw3l8NOzmYUKinwSurt/zne98p4P8hBD9FzKyST0Cg7TTS8uoqm2tf8Oxe0XXBK5hR6JKP+ZVHRiNMjMC12igXvZMI3CV4WZ3GQKGgCEwLyIgH34agaum9I3AVQOQnTYEDAFDwBAwBAyBUUfgpS99qdt1113d0ksv7ZZYYgmfn4cfftg99thj7sILL3SPPPKIn0xeaKGF/BIVRx55pGMZKBznyyyzjHv66acdhJ0rrrjCPfTQQ9H3geX/5je/2b3qVa/yzyCSyG9/+1tHNB4muEqEZ2+11VY+DwsvvLD7xz/+4R34P/nJT5JpbrbZZv4L8Fe+8pWOSChEiiLv55xzjmPJPy3ked999/WHrr32WnfPPfe4t7zlLW7llVd2iy++uHviiSfcvffe66OXsLQRstpqq7kNNtjArbLKKm7BBRf0x26//XbH+/LF+bPPPuuP8W/RRRd1m2++uZswYYJbbLHFfF5mzpzprrvuOkcEFpFll13Wbbnllv4nX6yfeeaZ7plnnpHTfssyBEy8/Otf//LvM27cOPeyl73Mrbrqqv48+SP/t956q7v55ptH3Jv6wVIV22+/vc/fy1/+cn8ZEyR33XWXu+CCC0bcBvmPd5j9UYY/DpY8C2x//vOfj7g29YOlHrifdMD+ySefdA8++KD78Y9/7ATf8F709W1ve5vXAaLagDN6+7Of/czddNNN4eW937m6A44bb7yxv4/Jc/KkZbnllvPPR5+oH+ggAyWePWPGjDkiDDQlqWgdBEf0DYyIOsA4A325+uqre5HB0D0mqKnL1EvRzzDf8g65OijXyxZd4znoJrp45513uhtuuMHrqejAhz/8Ya/vvAPtC4JOkGctvMc222zjVlhhBfeSl7zEY0Z50z4QGShV9jqNnP03velNHpf777/f6yT5px1jqdO//e1v7r777nPnnntu70ulMM2m+VxqqaV8+0Q65513nm9vmDhDZ3k3cEm1EywHVIUZbcuOO+7o6yZlSLuC3v30pz9111xzzYiscx116tFHH3WxpdfWWmstr0+0HbRRbJEqXeULLr4Mpg+g3vH1FvpGv0HbQJ+hpQqL66+/3uMTPo+2esLsthG55JJLfFvgf6h/lMkuu+zij9C+UedypEnbRnpveMMb3Ktf/Wr3+OOPex1ZffXVfT2k/oMFkfnoy9pEAKkjcEnfQn5o337zm9+w25N3vOMdXsdoA0877bTecXbQOeoYcvnll/t2mfKDrECfjd5ssskmjjxQtylP+ppf/vKXviwp29Q5bAQtlBntEPWKfhkyGulQzmEfm6MXOu26fcEoVa90/0t0K9p22krabt4RvT3//POjulb1bEhW6AHtPu3bnnvu6XWXfvzYY4/t3Uqdz7GD0M+ddtrJ2xnYGwg4zpo1y+vZ3Xff3ThNuWHy5Mn+nelbqVdbb721W3fddd0CCyzgPvGJT8hlc2yr9CPUgSZ21hwPihxo2t5EkugdShG4wvpQpUuUM/1clZ7VEbjow8WGJnP0S9iK2Ge0/7kibRP6e9ZZZ3m9w5bC1qbfp1+m7tEuUN95T/padB59p83Cvo1JaTvZj7yQvyaYCS5N2mz650HazbwTJE3+6L8Zh7DPOIO+jXY8tJe4JybUYWwc8k+5YdNQ/ozLfvWrX8VuiR4LCVyMc0gXW4/2C/sanYn1tXXtIO/GmASbEbs+lFx7lH53ww039Gnxvgh5Il1sHCYpUiJ1gPNE1aVekNaaa67p7U/a2RtvvDFZJ7gvN59ci7TFpbSvamKnP5fT+v+5eSkto6p2F/s0ZRvzPNo2zv/whz/0thDtIPYttgwRLLFzGPMz9mfMSvvN2JY2Apsc2zUVFa6JXVOPYvqKGIGrSd8bplw1hgWHHXbYwfcVtDe8OzYL9gX1qEk/JM/tQj9o82jPqvpXeV6Tukj7tW9Dn448J7Wdf/75vb2HzUz/HYswjf+G9jM2BmNsxhgGO5TxLzYy15EO/XZKmvSFkgZlTFsHruwzbqHdvOOOO/wHd7xDTgSu3H6yqf0g+azaUoYbbbSR97UxrqF/om5jk33kIx/xNjP2xxFHHDFHMiV1GPsbHxP9BjYy/TR2E+Mo/CsxyfUv5Yx/QruU54UELto56gvR9ujP6MOuvPLKStuhBIvYu+pjXdkgOk3bNwQMAUPAEGiHAP0WYgSuGhwxjEwMAUPAEDAEDAFDwBAYZgRw8H3sYx+LZhEnHo4KWUIRxwgT/LEloJgcZdKKiSgtOFkOOeQQB1EsJjiQWP6iiWy66aZu22239RPNsfsgQTGZJcJE8gc+8AHvJJNjekvezz77bD/ZJcdxPhFtAiGPTDyQTigQTD73uc/5CWDyxORlTD71qU/1yCFMIuKEYWIyFJymP/jBD3pEDJyspI/zCGHi/vjjj+/dhgNzypQpfnKFgxC8IH/E0mYCeOrUqb17Uzs4w9/3vvf1SGjhdQwCTjzxxB5x6P/+7/96+dPXMmjgvauEfDJxLCSx8Fr07Zvf/OYczihIAjg+cTrGBELOCSecMAdpoInu4BhjIh1hUgCylMh2223nSCslTCYdddRRvTLnupCkkrpXjmsdpOxwAnMsFIhH6CfEklDIB/rDxIWWJjqo79tvv/38pIg+xj56i2MTRzQiBC79DqHuMsl80EEHJcuQNL/xjW/MUfb+AQ3//c///I8nMtGGUWdpl0KhHaAtwhGvpSSfmigA+Y9JWdFViJA4hGNCfcGJL21PiBlEmgMPPNAxcRATJlgpb5EvfOEL/lrqLEtMhKInjNBPJrOQlK5CHuNc6vnci/MY0ppIFRaQ/0466aQ5nkfdZrIBgQBEGxCKbm9z+5GmbRvPREeZ9EA/IE8xoR0TJgAvu+yy2KnaY3UELiZsZIlFJhuPO+64Xprjx493hx56aO/3l770pRH9MG0YbRkC1mAufToT/Ycffrg77LDDenW3l9DsHSFGS70Oz2nbgcnAFDakA5lJL0Oboxf6eXX7Wh/Ca3X/C8Fq4sSJ4SX+N20OhJbvf//70fOxg0LWYSKJCWjpq2lnhBTVxA7CVqLviAmTbaeccoo/1SRNSUvqNRNf9AnojsgHP/hB2Z1jW6UfogMldtYcDwoOlLQ3QRIjfmqdo42irULC+lClS7QBTCLGRPRMdCKMwEW7SXvCBFpMmEA/5phj5rBbYtdyTNom9iFrQUoOhT6PZdHf+973Ru3C6dOnOz580NKmnSSdLvNSgpng0qTNpt0cpN0MThAK0EnanUsvvdTbCRxHsOWrJvGfu8q5j370o/4jBvkdbmlz0akc0QQuPsCAiBsTCKBf+9rXRuip6HyqHZQyIT3yzGSzCBjkjon48EUIyXK/bE8//fQoiULO6/ejLYW4JXaZXMOWess4Swjtcq5JPuWeNriU9lVN7XTJa9W2SV5Ky6iq3a2yjfVYDTIHZIKwXKlj6Cx1LubDQKepJ1ynpaldo+9tuq/tcWxJbMqcvjf1HI2LHsNCHmGMFhtPktZTTz3liMqU+vgn9ryu9AP7FCJYTKR/5VzTuqjHork+nVge9DEIpPhfEIiyJ598sj7t9z/96U97/wY2sNhKnKjzYTD2w8+iSYUlfSHPoj/HD4c/KRQ+vEEfqC85BK6cfrLEfgjzFf7GFgS/mK8IPwf9CbZzjMBVUoch3kI6jgltBB9AhFFxm/iX0F/aIiT0Eci4OLRLuVYTuBgL4p+ICbYDNkQoJViEaYS/u7RBwrTttyFgCBgChkA5AkbgysTOCFyZQNllhoAhYAgYAoaAITBqCOCAghCA44MvABGcd5AIIHDh5BYngmQS5wVEAa5hYlecgPz++Mc/Lpf59Fjagq/mEM7jLMJGgsAgDtZcYhFp4FDBsSICMYEJCt6DL9BEcKTJl+dC3uAcTh5IZjh8IMSIvcY7QXYgf4h29vkDs/9xDSQVnGBE4QrzzyS7fDEnjjK+hkNwhvL+RF1g2QARHKSQJkhPO5Vx6IhzCAcNhCoRmYTnt3bu8uXs17/+db8UF2mRJsK78hzKkjKtEnRAlxn4ghfOM74sFOIGaTLxj3MR/dH446hkghgsv/3tb1c9zr3rXe/ykyhcRJpMGuGAwwEojjom4XgW6SJ8Vc0EhQiTnkSSYUIbHRB9C/Wqqe6knN+QLSAEUv7oBGQx3hcyJJjLJGA4cSaT57wn5VYnMR1kQokvdPlyN0YoZKCGjlMeop+iF/K8Eh3k3v33399H6JF0IABAUiACmbyznKsjcEFyQM9En3gnJlAoc5zJQoLgXXAOthXdBkha1An0mzZM8k95MknAcaQ0n9o5K89jS/roOPWJNjDWTqDvMQIXbTSYSblSH0iLNCBiyHHtuO2awKUdtZQ9E51ghO6TP5H//d//9W0lv6uwICobpLmwbkBMJdojEk58+IOz/+ky1c+T8+G2pG0jDT3xLGmiH+gP9Z2yFKF8SpZyqiNwoZ9MLlLGQrqSZxJlTSIFcoxIEz/60Y/ktCcR0TbSVrMsDSJ9uqQFyQuSGnWZZ3AtfTztGtvUOSaYED15h45Tn0mb9phoPwjHmUCVqJs5euFvzPyX0//qyQ/yQ79BedG30fZIHSISF9FrckQm6MNr6dfR61Dv6uwg9AlCHriJTUC7Tr8IMYZ8NU1T8ib1TH7LVuuGHNPbKv0QHdD1MdfO0s+I7Ze0N7F05JjWuSoCV5UuEblnk9kR61LtN+UrOhESuGQyl/zQzqN/1G2JCstx7uc6sXc4lpJY24Tegb/UZX0v5Yy+075K38t5yL3S54W6xfE6G5A0+pEX0i3BLJaXujabJWwHaTfzbkzgopMI7ZG0P+xDRoKMUiVE0COqH4I+YeuhN7Rn2gaEHBZGyoulqwlOch57FT0Nx21EypG6z7Wi83KfbKUd1GVCvUZHkab2KNEwIfpgA4udKOMsPsSR/kWer7ex98Om5v1IT9pb7oGIduqpp/Zub5pPubEUl9K+qsROl7ymtk3zUlpGVe0uY+iUbazHavIO9JfYaLptlXPUL/QS/dF2K0RWCK0iJXaN3FuyjRG4cvre1LM0LprABUFc3ptxHOMI+iHsUKlT4ccgqWdwvEv9oN5R16r615K6GBtPowdVPp2qd+YcbWwJgYu2FFuM9p480CfTx1ImK620Us+HQURlIjOKlPSF2LWMaaVvoQ7hM9H2paSfQ+Cq6ydL7QfJQ2rL2Er766i/YKd9idwbErhK6jBEeOoiAl74shjT0JbgNxThQxpIVEhT/5K2RSU9tryTjIvDcRrndV3jN4LPFh0K/Yjojo7KXYLFc09I/+/aBkk/yc4YAoaAIWAINEXACFyZiImBkXm5XWYIGAKGgCFgCBgChsCoIcCSAvwhfK2vo9CIE4FzTA58+ctf7k3OM9nIRLs4/Yj+guMP0Q770BkOSYOQ50Jg4BnihPc3J/4dffTRPeIK0cE0IUk7EoS0wlIqb3/7231qGLHkjwkyEe0MkYkGzoXOPhwkRBDBSYIQ7eg973mP32cCgHcRYZKcL9dxxOjIKJzX+Q+dK+SV8PQ42kLSgs4nEypE9iBCDV/TIVzPJBxbBKcWTjskNzoN1x5wwAG9r1/DCRocvDpaFkuNnHHGGdzmRSYqcpyAcg9EFbDGSQb5TyZ1OK8nt4hKRuQLhDLESYiEDnccWJC9yCtCtBKIa4jGPkd3Us5vIpVMeD6CBl/p86WsCPUBxyyTo2EZyuR5KYELghMYC0Z6wpznE3WO6HMI4xDqJWQ2Ji519CWNQ64OMuHL88ThzMSWtBG8K4QtTaCsI3BBdGRpAiSMmMcxvrSFFISAG3WzjWisqJd8RayjAaHXkFwQHeGoNJ+hc5ZJEoiX+pmpdkK3PToCF0RJIvwg6IKQvPgN4RHMEa1fXRK4mNykLBAmAKiHYCmioydoAmoOFrG6oScriBwIIVME/cahj4T6LdeE29K2TfdjvC+REfRkOO2N6H7YroR5SP2uI3Bxn9ZRlghhogLRz+c3/Sj9qQgTn9RbrTPSpwuBS66V9pj+Tiam6s7R19C+8Az6QsqKySgRlnckIgbCRCH2A5KjF/7Chv9S9Uq3YfQ35JP8iBCViwkOBKIL/SckxTqRfo/rSJc+kT5X6obWnyZ2kJ4QCtvp0jSlnpFX8sdkNXVV7AaOV0lKP0rtrKpnlbY3VWlqnasicEkaKV3ifNU50QlN4CJq18477+yTpj+jDxL9os2nHgtpXbefkpfYVusB5cmHC0RvQEKyCmRMoqWK/attSpYngxyIdNVOdpGXUsxCXHLb7EHbzdrGBXtsDwjN9Gk5Iv077Q79pegT9zIGQUcR3e76A4l/oc7Qjn33u9/ttWVExnn3u9/t23qS0MRp0XmO17WDmsBVYo/yDD6EgciBiL3pf1T8C98v/MiD90P/EeoTRBcZ85XmswSX0r5K30f+c+30Csg82VNs/6b9ZkkZkZdU25qyjfVYjfsh2RKZGIEUQ78nH/bwDtgg2EOI1gkdRanUrvGJFv6LEbgkqVTfK+djW42LELgYI8t4MCRpgRF2H3qE8KER44kq0TrXpX6kdIC8lNRFrTuk0cSnw/UxKSVwaX/bz3/+8xERX9FX+VBHk5FK+0I9bsQux+7FTkfWW289H9kX2x3J9d1U9ZOl9oPPQOIfvi6iMiNggu0k7TK+RPxs4oPUmJXWYdE9nqdJWvzWNqQen2udDMeBMd+kTod0Yz6C2DhN223cR7RgdEhEj8Opu/jV6AtKsZB0U9uubZDUc+y4IWAIGAKGQHMEjMCViZkRuDKBsssMAUPAEDAEDAFDYNQR0A6lKgIXEZX4OkwLS7OwhBsi9+LUkwgqGI9CJtL3aacMaZJ2legJ1tDxyH04H3FyQt4REoP+ak9PNujnaEIQk+RMgmhnH84PJkfECJZ79X16+SFx/nCfJnCxvJQQriAVyVJIkh5b7WzTWONgYzJBiEtM9OgvNfkSHsKVSJWDTa4Jt7wzzhieBUmISZZQICiBIyIYyzUyUZHrBOQ+IRcwgQz2Qk7iHF/EC0mOpY4gjOFwxIGFhBGu/MHZ/yZNmuSJcPyWCdQS3Yk5v0mT8oVchANYSCQcF5GoaGH5y+R5iJvcF25DHcTprifotCMvNkEnhA+dj1Id1F/168kRyTPRwKgP4giWCTX9DtrZuf322/tlqMgb7UQ4Ycn5TWZHOUGIKgchs41oAteMGTPcj3/84xHJ6UkeJiGEDFWaT+2cxVnO5LzWbR6eaidimEGSo24i5A8CD3qkBUctzmwEXQFTca6mSE56wgj9FKJcTFdXXnll3z6RPoRKIQvyG+HLZCGy6kmJHCxiz9PkOciC9C0i2kkdK0+5TrZt2jZNBoi12+uvv77bbbfd/KOakGUlb2xzCFz6nTWhR5Y0oS5R/7T+EjmL/CN6icfYxADXVE0Sps4RHVKW8UgtYSWTG+QRPUV3c/SCPDWVVL3SbVgqIg39jSxFqwk+VXmQfo9rIHnrCDBt7KAUgatNmlLPyGusHed4laR0oNTOqnpWaXtTlabWOV2+qfqQ0iWeUXVOdELsD67HbsJ+og7QH2E/aKEPpZ4gTOZhc9aJbptuvvlmT7bR94h9xTHeUX8kodscidrYVTvZRV7IcylmGpcmbfag7WZN4CLKBwS7JsL1jHdYZjg2fpK+AcJ1allW/TxNZknZDHppKT6qwBZAROfZD9tBjukyEQJXqT1KeiXkIP1+2GXYyKEdpe1qiWbZJp8luJT2Vfq+WPuestPBMyU6zab9ZkkZkY9U2xqzjblej9Vi5HNt/8fIsRC68B9ACpExXaldQ35KRdvjsoSipJXqe+V8bKtxEQKXrgOxcSNke2xvhKXLhSgTS59j/dKPlA6U1kWtO/TBTXw6qXcvJXDpjxpi+oifCNud8T7jX6SkL9T1nXEBH0ax1aLb81zfTaqfbGM/6DyF+9q+xKYh6rkWHZFNE7hK6zA+CexsBNyJ0qaFdPEZ4JegXpX4l7QtmvIRxOxS3T9pwqnOn84//c+9997ro/iXjNN0urH9rm2Q2DPsmCFgCBgChkAZAjJ3xZiO8SJ+Quxd/tiX37mpzzd7kuW/n+7m3jUGrjMC1xgoJMuiIWAIGAKGgCFgCHgEcghcOL00IUmg01+XycQySyHstdde/hLt5Jd72PLFnExYsNQBTtYq2XXXXd0GG2zgLwlDy8t9TEoRDQUnLpEIZAIr5tSVe97//vc7JgsRidiknX04b3DihCIORo6zT9QO2Y9F4NKOTnGq+BvUPx26neVTTjvttN5ZouzwHIxtLeF1nEs52PR94b6e0KsiIlBmlF2oDzJRkesE5PmaBMdEyi9+8Qt39dVXz+Ewk7xqHRCyoJyTLSQ30kWk3PV9uboTc37LM8ItzjzKB+c3uk8ZhfjI5Dnv2XQJRT2xIM9m6RzqHgLBjQlpLZBpJNy/EAxLdVAcgrwT6YZOYJ6rI5PVEbh0PmUfnWLpHxzXTCBIFLWuCVy8C9FIQoFkSr1BcHSnotLk5FM7Z1N1SdqPUE902yOkN103Y+RV8syX9ThoaYeIpkK6XRK4eEYoOABYgpL2bquttuotcZEicKWwiNUNSGu0NdSlsM7oCTkhq4V50781fqk8cH2sbdMTz0yI6+hS3EPUHCFbhFELOZ8jOn9hBC25X7fp8hwdeQ1CHeRVRPKpJwSFnMz52MQAx6smCVPnZHIFvUOnY0LUFpZVQqSPzakjsbTqjqXqlSa0Ur9DQiXpbjKbNAppE4mRUPyJ4J/0e9LX6NNt7KAUgatNmlLPaBvAiW0TSelAqZ3V5Nlcm9PeVKWpdW6QBC7yLZHnWIpHiFphXmVpq7BPCK+T37ptCqOBco20/5rUKfcScZI6gdx1113uG9/4xggiaZt2sou8tMFM4yJtobw321SbrdvYqvfXaem2u+qeWN+iCVxESyJCX1vhHegXaMsmPB8ptoTAFbMpyZvGSOwTjle1g5zXZSIErlJ7lPRKyEGavAIJgHYrFL1kvbxfm3yW4FLaV5Xa6SEG+ndpXkijpIy4L9WHx2xjrtdjNSGjclzkkEMOcSussIL/SZvIkplaWJ6aaOAQZqRNLLVrdLpN97W91i8Cl/4Ii/wxFsJex36Uib4m+e6XfqR0oLQuat1p6tNJ4VFK4NIffZA2YwrIonzIGLNLS/tC3d6l7Fkd6TTXd6P7AN3ntekLUxhzPMe+FFtHE7hK67D+kINxDfYRHwrRH8SkxL+kbVGNoU4/Nk7TBC5sCojfoUyePNm95S1v8YeFuFmKRZh2zu82NkhO+naNIWAIGAKGQB4CYtcZgasGLyNw1QBkpw0BQ8AQMAQMAUNgaBDIIXCFk+iSeU0kEQLXjjvu6DbeeGO5JDlRKGQklnUhMkeVaKIVy4zg9KgSTeTRy6KF97CUm3xxKqHPtbNPlmMM79Nf9wlhhWtSjke91FXVxKlgEsvzTjvt5J3VkhcczhAYwvRSDja5L7bddNNN3XbbbedPEaEIh1VMNDFIf50oExW5TkDS5stFnNbyzvI8vkjk62CcyrK0D+e0DqAvshyQ3CdbcciTDvnV9+XoDunoSQFxgkn6fJ3J0pxMkEHoCfPPdeEkrEyep+qRpC1brYM6koec32yzzRxReZAwjD7HdDkJgatUB2XivirvLD9GeSJSH/Q7yGSYv2D2PyZM+NJ4rbXW8pHlcFLHpEsCV1gm+nl66Qf9zJJ8auesjnykn5dqJ2KYMRkrxJJwGTadZrgvTu1UNA09YYR+VkXgkrSJnLj55pv7L5SFZCfnZJsicKWwSNUNrcMQIGgTIEtKNJHciek2bZueeI61OXpCrEnbJ1ix1RMfKQIX1wkJQCYaqT9EJICgAQFRiCESOUSIbjhwdBTM2MQA6Us9j5GRUuckygv3h/0QxxDdPkrfklNHnru72f9UvRLsqmwNTWrRS05W5UD6vXtnf+Uu+3J9GzsoReBqk6bUs3B5X8lv3TamA23srLrncb5pe1OVpta5QRK49HJ25C+nnuQsX6XbJibSaRe0iM7rSU05r3VdCFxdtZNd5KUNZhqXJm32oO1mTeCCBEzktaZC38G4gPKE9ByT3H5ST/inoimSvkyoazK5tH2xdpB7dJkIgavUHiW9EnKQfj/GFSwdGgr2HuMHRHBrk88SXKTeNu2rpH1uaqeHGOjfpXkhjZIy4r5UHx6zjblej9V0u8455OCDD/Y2Fvt8hBa2vzJeFLuK60rtGu4tFW2P94vARd74uA4ieCgsqY3fAfsxN+pxv/QjpQOldVHrTlOfToiT/C4lcDFuIooy/U0o2N5EVIRAK9HPSvtCfARC5InVC3m2tOe545dUP9nGfpC8hFv9rJgfRK6n/4SYrW2d0jpMX8pHHuiMFtoNxmeQ4agj4n8q8S9pWzQ1Lo6N0zSBK9aWkV/dz8lYvBQL/f6p/S5tkNQz7LghYAgYAoZAcwSMwJWJmRG4MoGyywwBQ8AQMAQMAUNg1BHIIXClSAAxApeOuJHzclUOZ7lfvvTkd07UFaJq4VhBUl8fck47f6+44gp3/vnnj1hCMRWmvCmBS0eb4rl1EnMyEoGMr/1EUl+SaqdX6us+SUO2+itCiZIi5/RWT8S0JXCRLl+8MynN8m96ol+eid7x9SA6QrkTaQhHWiwanNwjji+JPNFUd0hH64UmcOGkhDgV5pU84TyE0IGDNsyjTJ7n6DrP1w7nmA5qAtdZZ501x5J2mvwiBK5SHRQHr57gII9aNLGgjsBFhAi+imeyLBTwQSDGIZpM5Q8U/BMiSxX2EPLe8IY3+NRPOukkH8GvNJ/aOZtynKcmKHS5C+lN181p06Y5iKY5Ukfg0hNsdQQuJoepR7K0hH4+REnqKcueIuI0Zj8Hi1Td0BEJZZldHfExhS3P1aLxa9q26fZO6pFOe5AELv1lOBMMtIFEQBPSl9RvflP2TEzSToVRGqV9ZMKOCTERmQDOJXCRNm1DE/nJT37ipk+fnqUXTdKVa1P1Stqw2LvJvbruNSVwxSaX2thBKQJXmzRT9Uzev24b0482dlbV80rbm6o0U21Rqj6kdIlnVJ0T0obohG7HqvKnz8VIUPo8+3Vtk0yux/rtGIGrn+1k07y0wawOl1SbPWi7WRO4cso7LP9UW4DtSWQdJrQRISKF94e/9cRv1YcO0pbqdEOdD9PWZSIELumvwmtTv/WYSNsuYm+m7pPj+v2qbAd5PwjttJlt8lmCizy/aV8l98Xqu2AQs9PlXGwraTbNC2mVlBH3pdpW3T+Lbcz1eqwGKU9/9MN5TeCK2XAhgauNXcPzSmVQBC7yt9FGG7ktt9zSEYEpJhrf2Hk51i/9SOlAaV3UuhMbT/M+KZ+OvGu4zSFwic0VkublwyCIdDLWDdPHF4VPqrQvpK6vt956PtkzzzzTEZ0uJkLuaUvgamM/xPLFMSL30s8hRNVnbB4TIfYJgattHaZsd9ttN++fin2ohG9p6tSp/qOiEv9SyhbV7xazS3MIXEQxp79FGIvzYS31tInIOK3unq5tkLrn2XlDwBAwBAyBfASMwJWJlRG4MoGyywwBQ8AQMAQMAUNg1BHomsClI0XxpVroUA1fmMl/llGsEhyvRDxCNHHIH4j8wxaD+IPEolnJLRLFhN/f/e53PdmrjbMv5XgUBxPPwakP6aFKmDzAGSXC+0BECZ1JMUdLyUSUJuJJlBR5tt4S0QCyFaK/AJSJilwnoE6TfRyaRGNi4pov+ojsISLOZL0URiyyglwvDkkiGvBlZlPdIR09KSAELqL/oFM4B5EHHnjA6wtRkeRrWYgV48aNG0oCV6kOysR96IT2IDz/Ty9tIRNquh5JGXK5LBXFPmUEwQSSJaQJliyAIAdBDemSwFW1zJueYOR9yUtpPnOcs6l2IoYZEQKJFIhIlED/o+ZfHYFLfzmMXosOy6SDJrxpRy3LfKDzLHk6a9YsT66ESCRLg3ZF4KKeMblGmyd5kS+tmaROLYUXwtKmbdN6EZv8S5EBwjxU/c6NwKX1iolKJhjA5qqrrnLnnnuun+hgwoPJBZbf3Xffff1jjz32WAeZRCQ2McA5qeexydrUOSKjET2PtoHn1AnLF1EP9btUTabXpReeT9UrqQtVUU30kpS33367Y5neOpF+T8g6+vo2dlCKwNUmzVi91vmt24/pQBs7q+p5pe1NVZopnUvVh5Qu8Yyqc6FOjB8/vkc4p19hMrVKsA0h59dJXdvUlDTVz3ayaV7aYFaHS6rNHrTdTPuMTiJNCVx6Up++ELub/pjldRlPIWIHa6KVP5H4pwlOkGwZW4Si7RM9rgp1PrxPl4kQuErtUdIuIQfp9/vZz37mfvCDH4TZ9FHM6CsQlqdCb9vkswSX0r5K2uemdvocIKgDpXkhiZIy4r5U26p1T48n9FitCwIXeSi1a7i3VAZJ4JI8Ms6GRETdoM0FYxHqB/WkSvqlHykdKK2LWncGSeCSfo8xE+1eTJZcckkfvXqNNdbwH2SIf4FriZwLEVc+WGtiP7CsPSQ9JBXlSROdcn03qX6yjf3gMxn5xwd7fLiHpKI7co7xK7aoELg41lUd5iNDiHAQo4QUTfryrBL/UsoWJV2RmF2qCVy09+hDKJAzd955Z3/4Rz/6kY8W1hUW+ln9sEF0+rZvCBgChoAh0A4BI3Bl4ocBYWIIGAKGgCFgCBgChsBYQKBrApce2KeWqsBxtMsuu3jiDktj1TkKdYScFMFICBcy2S9fhzKhIU6gsDy0Q5D7mexo4+xLOR71hOQ3vvENx7I1oeCsEofbjTfe6Ilnco0mTjHBLpFwmLxhYpYv7kVSDjY5H9vipOILVIQQ/t/85jfnuEwTe4PLFwAAQABJREFUKkKHpExU5DoBl1122d4ymxdeeOEcBD4cZjirEHGU7b777m7SpEn+2BlnnOHAKBQ98QfBCgJIie7oSQEhcOml7FLvKQ5bykWcruSx6eR5nQ6WROAq1UGJYMV7pJb6EWIN11QRuPQSeJQr9RJChxZNLOqSwMUzpkyZ4qg/oYgDWEhebfKZ45xNtRO63GWSSn9RqydN9Tvo6EzSjsnkirSH+nr29dfsdQQurddhe0NaevmMrghcpKsnfk888UR34IEHcth/+YxTOkfatG36+aNN4IIoxaQ87TBtG20oIo583Wbi2F9mmWU8mYv6qCU2MSDpoH9NCFzS59Le8ZywLpMuk4P8IZC16I9z6oi/oeG/VL1iIox+kXxClqDtCUWToy699FJHv1Qn0u/FCFxt7KAUgatNmk37oPDdhSAQ6kepnRWmr3+Xtjc6jXA/pXOp+pDSJdKtOhfqBCRL6i0iUX38j+AfS9MymUtkPGzcOqlrmwTDWESeWASufraTTfPSBrM6XLokcLXBrA2BS9tIsnS91hdNrCwhcIntodNkH/sbOxy55pprfHQP9kOd55gWXSZC4Cq1R0m3hBykCVwpMoDuQ+X92uSzBJfSvqrUTtflFO6X5qW0jLgv1bbGbGOu12O1rghcpXYN+SmVQRC40O9VV13VE+5jBEZdryCDYnNXSb/0I6UDpXVR605XBC4iDlPnEB0d0B+Y/Y/IWvR7iPaXTJ482RHhCRt9xowZ/rz8o99jLC3+ne9///s+claJ/aDbu9S4cbXVVuuNqVI+DcmbbFP+pTZ9oaQd24p9GUYMlmspW8a6jIvEV8S5kjrMBx5EEkP4qIw6oIUyp3wYiyF8RMS4QaJ35/omU7aoflbMLtUELvFL6XvY174AooTxkVUJFkIED9OX3/2wQSRt2xoChoAhYAi0R8AIXJkYGoErEyi7zBAwBAwBQ8AQMARGHQFN4Dr11FN9dBXJlDgRUiQo/dWdTCTo0PI4rnByMHGqhYgyRJZBrrvuOscycFWiQ6kTrUscZ3IPDhSIOgiEMCb3ZcKSY8cff7xjUkILjigIJDh+NMmhjbNPHI88R0/4a/KPEIt0XtjXy94dd9xxPQKXjsDD19VHHHGE+8AHPuC/mOU+IudAwBBJOdjkfGyLs5HIZmDBJDzRdCg7LRtvvLFf7pBjocNSJipynYBrr722n4QhrdTX8DLpJ1FT9KR2+M6kg+hlECQ6WYnu6EkBcZRtt912nqjCc1iOIIykoR2Yw0jgKtVB7TSPLcmpSXNgU0XgglQiXyLHyhDiFHVbllfsmsAVy7/WD1mOrk0+c5yzqXZCtz0yicpyYtQF6iZ6BZmT9kqEcSdtHed19IUYqU7uCZdjrSNwSTQPnh8SdXgubRKkAKRLApeuU7yzLDdStcSTvKNs27RteuJZt+eSdooMIOdztrkRuEhLE475LcvEsk+dgWRDeYjcd999cyyfIX16OCGSIuiQVuqcduJfdtll/kt/eTZbJqToV9Br+hX6OLY5dUSnk7ufqld6YjJmbzAhwyTHwgsv7B+l60PVs6XfixG42thBuq8Tu4p8tElT7CFt61S9W3gupQOSLtc3sbPC9PXv0vZGpxHup3QuVR9SukS6VediOiG2DPeCYxg5QUd/yyXc1LVN8sxcAlc/28mmeQEnuacpZnW4pNrsQdvNbQhc2OdEvUTQNwhJWvg4ZsMNN/SHcvVJT/jTRh999NH+gxadrhA2OMakOREVkZjO+xPP/9NlIgSuUnuUJHV7Lvamfl5sX78f/Sbtvf7whXsg+NPGIhAH6D/b5LMEF/1uTfqqUjvdv2ziX2leSE7fm1tG3JdqW2O2MdfrsVpXBK5Su4b8lMogCFz77LOPj3JNHmPtBvYP9R5JkX78yef/6TJuoqvcru8N9SOlA6V1UetOVwQu3kHIRUyS0jZqOeCAAxxRtRBN4DryyCM9QYs2lvcOfWP6YxghBJX0hURXo42TcSNjB3wpWvSHgbm+m1Q/2cZ+0HkK93kH/AIIUXGJjqtFR9HXBK6SOky62J5g9uijj/bqgn6eTpePQSbMXhmAthfJ9U2mbFH9nJhdqglc6Bz9FXokgm1D/WU8o32vOs9NxmmSbmzbDxsk9hw7ZggYAoaAIVCGgBG4MnEzAlcmUHaZIWAIGAKGgCFgCIw6AtoBCmkAJ4k4esSJ0ITAxQvpr8BwhECokjQhfUHgEscSE4ChIz0Gip48wLl4yimn+DSXX355h4MCogNy9tlnu2uvvXbEJDEOeyJfCYkLJxTEMiEE6GWc2jj79EQFy1pBTpLlEnXEG5Y7IYoUzjsmuXFmrrnmmj7/esKNfArJjJNCsMM5B27ci+il1bQTlrQoQxxb2tHjbwr+aecuRj8TjSxxh7C8IXkUcgBONSaGRGSiItcJqEkp4EPZUKYIRAQikUk0MianJH1xfnId1/OFMGWLw2r77bfvRfUKJ6mb6o6uE0Lg0qQziDLoNEsdgTfXQ4QUfChXnNDinJVJ7jBfvEdM6nSwJAIXzynRQU0QIo0rr7zSTZs2zesTzkvqnhCuOC/OeP0OQkYCH3ATnMQhix6jY5AwpR6TFnX8jjvuYNfjTP65F52BDJIjEML4claEusIf7RFlit7LF7Wy5ECbfOY4Z1PtRAwz8q2jyFE3qQ8Q4GgHIBYJeYrIftQlRE+QQBg46aSTfF2B7Lr11lv3yoBrNWElpqu63jFRA6mEerfCCiu4Pffc0xE9UIQvl1mOFsnBIvY8SYutnrzgd5Oy53qktG3T5TQMBC7Kjf5ThAl0JtJFdDlxTPRZzrOVPj0kcMlx2izafto26TNS51hShLaV+sJ96AUEPoQ6B9GYPgzRulmnF9p+oL2hf84RXV66/9WTWaRz0UUXuUsuucQnyaQHbZhENIuRsVLPln4pdY9+jyZ2UIrART5K06yrZ6l3lOMpHdBl2cTOknRjW63HTdqbWFpyTOdT23vyXmF9SOkS6VWdi+mEJsxiA0BMvv/++33WiLD4rne9q9fvEXVD6pDkPbbVeYi1TdJuantS0olF4OJcv9rJkryUYlaHS4rANWi7uQ2BS5cTpHOWr6WcaWv50ABbSiSMmCfHw60mOHGO9Piwhoi8tJ/77befX96cc9gT9BEiMZ2Xc2x1mQiBi+Ml9ij3adtG7E2OV0n4fthRRBqGpMU48L3vfa+bMNueRcK2oDSfJbiU9lWldrp/4cS/0ryQXEkZcZ/WFd2Hp2xjPVbrisBVateQf913EQGaD7ZypIrAJX1UzDZLpa1xkTEsyyXutdde/hbIJhCQJCIxZU2bxEcTSGrZPX/y+X/90o+UDvDYkrqodadLApfYVOSLiH3YFRBAGZ8TLVVEE7j233//XkRabOKTTz65Z2cTgRObWQhLjF0pp9K+kDZb+gJ8QJCTGDcy3qV+0iaK5PpuqvpJ3S819SFJPsKtjorIeAQfmPgEIPTR34k/QRO4SuswY2GZyz3//PPdFVdc0csSeIEb+OmPpZr6l1K2aO9Bs3ekzuu+SBO4uJaPVYmkx3appZbyukOdRMg3+UdKsfA3J/7psu7KBkk8yg4bAoaAIWAIFCBAP4wwh4efnb6S/os/9uV3btLzzQ7DPDIcQ+6dQ36ddPpDnk3LniFgCBgChoAhYAgYAt45j1NMi0z8ihOhKYGLSRIcITjORHBiQfQQZwvHQweJXBvb4lw/+OCDveEp53HoYIiKhBFHdFQrruF6nKBCfOJYGAmojbNPO01JG+ErPSZDCFfP5Ku8P/kAE01Y4RhONnH66ig64SSxdupxH5NkTLojUm7+x+x/OV/TMolBmen8MCEbGvixZTFloiLXCUi+QmcUDjHeg+drjCAoPPzww/5VcC7j3JXzHBQClxwjDchxkOREmuqOLkdxfqMzfN0opD/ShkyiyUtMzsp58oVDl2USxNE72gSuEh3kPXX0MX6DMXVJ6hG/BX+ZUNP1SAhc3EsdJuqQSFh+OCwXWmghf5p077zzTu/k1ssacg/PyZGQwCX36DxzLIwqV5rPHOes1i/JD+0EdUAmRjVmYMnEEM5zkbDtQ7doL9gifP3NV+Ap0e9fR+Biea9tttmmlxT3IlLmYdsuXyLnYFFXN8J24rbbbvNO/F5mMnZK2zY9kRQjSaTIABlZ6l3SJAKXjsBEAtpJz+8QK76SFn3gPCJ9g54Y4Lh+V37r81Xn9Bfw3Idu0C7qvh+yJJMc6AlSpxeHHHKIJwdyLe1nzpJyXJuqV/S/2267rZ9Y4zqEfOo2jGPkj4hh2AQ5Iv1e2DfLvaV2UBWBqzTNunomeU5tq3SgxM5KPYfjpe1NVZopnUvVhypdqjqX0gkdsYB80och0oeyn9IjzoWiyyPWNpWQpvrVTpbkhfctwawOl6o2W3RBsO6n3QxZAp1EZIwgz63bEiGUOif9L9ejT6JLYRtMu8a7sYRnSkKCk1xHWuFzGKNI9C2uS+m8pKHLRBO4Su3REnJQ7vvxvhBvdJS80nyW4lLaV5XY6VJGqW1pXkrKiDyk2taUbayv74rART5K7Bru0wQj6okQdTlXJVUELl1/SEPbZqk0NS4yhuVaPR5C17EPaTe0vQYRBntN+qjUMzjeD/3QeZdnSxtZUhf1WLRLApeOdCj51FtpO2l/JfI041hsMfFbcQ26TRloX0LY95T0hfR1PCv06cjkreSPPDfx3aT6yVL7QWMW2+djy3HjxvVOkW9E+iV5D03g4nxJHdYf6pEGdYDy493keRyX6GjsN/UvpWxR0hIRjHVdD8d3cq28v/zmw0v8Bbr+lmAh6cW2/bBBYs+xY4aAIWAIGAJlCBiBKxM3I3BlAmWXGQKGgCFgCBgChsBQIKCjOpAhIuwQpQbCCqQBHAKQA0KJLaEo13AfjkeJbCHH2eKwIpQ3f02ELxQhVshXZnIvDgy+yjvttNN6k8RyTjtG5ZhsY2QAnGg48JHYeY7rpfqEsMJxHGY4m/jiTQRHqBjRLJXDl96aiCHXgTEO6JkzZ/pDRKDaaqut/D6OGNLBQaVFT5xqIhrLLk6ePLnnJJw1a5abOnWqvjW6T774AlSWZ9EXgTFfRDOhHopMVDRxAuKwPPDAA3tf/IZp4liGiBWGzAdD9EpIPvo+9IplnO4NlpThmia6w/IzOGcR7fxOPZvyoc4QrYClCsQ5e9VVV3nM5Kts7YzziSf+1emgjsB1+umnjyCrkaTWi3Byt4kO6uyxtAPOeu3E5DzkOiLNCV5SH/Q7aDISjlCik1EeWtAv8CMqFxHepI6TPhPAbQlc6AZkMBy04TvgLCdyFcQTkdJ88sUzXz4jQoSVNGWbaifQD2l7NGbcxyQEdZOIg6EwmQpuOioe18z+WMrttttuc7wv9YOJDCIhIprAldLVlPP4scce8xO4tLMsR4lQH9CDHCxSz/MJzf6HnlCnRJgg45lNpaRto31iwggJ6xHHqsgAnM8R6qOQEWUJz6r7pE/mGshGQtrlN3klz0gq8orcH7ZFkGOJBITeI9QX+jKk6hzn119/fV//pd3jmAgT4UQZgNQnUqcXmsA1ffp0x3K4OZKqV9L/VuWTukNECsheuSJL6ISTbfr+EjuoisBF2iVp1tUznefYfp0ONLWzYs/Qx0raG31/uJ/SuVR9qNKlqnNVOqEjKer80fdh57A0LKTCHKlrm4Q0he5jO2rREbiwm4l0KdKPdrI0L+SpKWZ1uFS12YO0m3UEC5aykgjFUg51W2zUnXbaqWdryvW067S32PBik3GuLhqQJnwTeYuItmClhbaRCKohwbVK57lfl4kmcHGuxB4tIQdpAhcfoWAHSl9HPhD6PMZgt95663MH1P+SfLbBpbSvamqnq1dM7pbkpaSMyECqbU3ZxnqsFltaW3+IEbPhqBeMVWLtZNV7x+wa8q+XfAvtM86nZPfdd/f2Oud1hFB+1/W9XBOKxkWPYYnSByaLLrpoeIv/jR3EWKiJjV2FU8quqtKPlA6IHde0LuqxaFOfThQkdVC343KY/pwo8PSzlJ0mcHENbRH3ka+Y4AMiOqAej3Jd076Qexg3M97WPimOk8dvf/vbbtddd/Vj7Sa+m6p+ssR+ID9VwnidMTW4hYLvAZ1effXVvX/siCOOGHFJlW6m6jCRjvmIIPQTkDD2mUQh1w9q4l9K2aI6vZhdSvQ8ouiJv4l8Cmlb7iUaF74+rgmlBIswDf27axtEp237hoAhYAgYAu0QEJvJInDV4GgErhqA7LQhYAgYAoaAIWAIDB0COEH4qgqDj0hWuRNJdS9CaG8mByByYURCHmCZrTZCXllyEJuLL1yJnlM18brIIot4B89KK63knWlcj8MqJES1yZO+FwcjS0jhvJToUXIepxBLj4EJTjW+iodYQvSFLgUnKMvzMCkB5k0miCgzHGKQRZisgUxyzz33RJ1CbfPM8msTJ070E04QAFhqighkt9xyi3cyxtLnOr56hLCAk5TrKU+JXBa7R4411R25T7Y8e9111/VliP6gy7qM0Un0DMINk6NNcJdn9HtbqoM4Z3E+ot/oLRNd4WRebt5xbKNjfB3M5DUkCJzKCNhRtpD00Dv9DBybOLZjhNLYs+WLcyGk4DRn8o66AWEGIhPblJTmM5WePl7VTujr9D5lAFGKCVrIOyE++lr2KWvqCu9L+8vkgMYzvL7qN+0CTnTqLG0C+g2uIpQZEwakD2m0C0E/IADwHugcX5S3kUG2bW3yOVr3Uidp42hLZXkdyUvVOcpp1VVX9fWW+osO3HjjjZVRXyTd2FYmIGOTsrHr9bGqesWkPXWBP9oX9JSlbKqi0+i0S/f7YQf1I82696vSga7trNFob8L3r9KlqnNhOvIb+4D2m8h79GP0PUz26nZUrh3N7TC1k4PEbKzYzegGJHdsUcZWkLixmWXZc85jIzCm42MI7JymYzpsZSZ9uR9brwmpg+fnSqk9mps+12kClywPx6Q7S53RX9EPQO6owmgQ+dTvVNpXdWmnS35K8yL3N92WtK1Nn5Fzfdd2Tc4zU9dU9b2pe1LH6X8gihPVlX5IxhKMw0qkH/pRpQODrotVmMh4lXaY/hxfRJU/irQg3tAm4QdizMTYjDKoG4+W9oXjx4/vfbyE/4txYIzkU/We+lxdP9kP+4H+SHQWvx9+gxwfXkkdlr6VeoJuP/HEE76PpR+sKtu2/iWNcc4++SOiF7pD/1WVN9IrwaIqH4JTv2yQqmfbOUPAEDAEDIE0AkbgSmMz4gyGlYkhYAgYAoaAIWAIGAKGgCFgCBgChkA3COCEPvTQQ32kLr5QzpGQwJVzj10zPAjwlTGRHpFzzjnHXXPNNcOTOctJ3xCQZVPCiC19e6AlbAgYAoaAITBXIhAjcM2VL2ovZQgYAoaAIWAIGAKGgCFgCBgC8ywCRuDKLHojcGUCZZcZAoaAIWAIGAKGgCFgCBgChoAhUIMAUQVYPo8vr1kmkwhQOWIErhyUhusavu4m6gdlPmXKFP/VsERQG66cWm76gcD+++/voxOwHAhLdpkYAoaAIWAIGAKlCBiBqxQ5u88QMAQMAUPAEDAEDAFDwBAwBMYKAkbgyiwpI3BlAmWXGQKGgCFgCBgChoAhYAgYAoaAIVCDAKSeQw45xF188cV+qYCay3unjcDVg2LM7Oy9995+aSjIeiKXX365mzZtmvy07VyMwI477uiXR2b5RBNDwBAwBAwBQ6ANAkbgaoOe3WsIGAKGgCFgCBgChoAhYAgYAmMBASNwZZaSEbgygbLLDAFDwBAwBAwBQ8AQMAQMAUPAEOgTAkbg6hOwfUwWAtd6663Xe8IDDzzgjjnmmN5v2zEEDAFDwBAwBAwBQyAHASNw5aBk1xgChoAhYAgYAoaAIWAIGAKGwFhGwAhcmaVnBK5MoOwyQ8AQMAQMAUPAEDAEDAFDwBAwBPqEwMorr+yWWGIJ9/TTTzeK3NWn7FiyGQgst9xybuLEiY5lE++++243c+bMjLvsEkPAEDAEDAFDwBAwBEYisMACC/ionhxl+e0//elPIy+wX4aAIWAIGAKGgCFgCBgChoAhYAiMcQSMwJVZgEbgygTKLjMEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMgWwEjMCVCZURuDKBsssMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQyAbASNwZUJlBK5MoOwyQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBDIRsAIXJlQGYErEyi7zBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEshEwAlcmVEbgygTKLjMEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMgWwEjMCVCZURuDKBsssMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQyAbASNwZUJlBK5MoMboZVtssYVbYokl3GOPPeYuu+yyMfoWlu3RROAVr3iFe+Mb3+j+85//uCuvvNJtttlmbr755nM33HCDe+ihh3zWVlxxRbfWWmv5/WnTprl//etfo5lle/ZciMBYacs23HBD3+Y+8cQTbsaMGXNhSdgrpRCgXdxtt93cC17wAnfnnXe6W2+9NXXpmDiOfRhr78eCjm+99dZukUUWcTNnznQ333zzmMC7y0yOHz/eUU7ImWee6fvvLtPvKq0XvvCFbp111nGvf/3rHfr2ve99z91///1u22239fXonnvucXfccUdXj+ssHbGLSPC6665zjzzySGdpd50QfedKK63kFl10UXf99de7q666qutH1KZH+Y4bN879+c9/9nYkN9Bevu1tb3MvetGL3LCWc+2LzQUXoBeUAzJ9+nT38MMPd/ZWq622muMPOe+889y///3v1mkvuOCCbptttnFLL720W2ihhdxZZ53l7rvvvtbpzo0JWB3LL1UbY+RjNexXDkNZLrnkkm7zzTf3UOE7GQYbYSzY7sOoW/Q5O+ywg7dZsJ/E9zWMeZ2X8jTM/RtjUGxbbO6ubKp+pKn1RdLXx6r2//KXv7jLL7+86hI7N0QITJ482S2wwALud7/7nbv99tuHKGeWlXkVgS5tkrnNBzuv6oS9tyFgCMybCBiBK7Pc53UCF4bsMsss49F68MEH3TPPPJOJ3PBcVvUOn//8572Dm0HWJz/5yeHJtOUkG4EJEyb4iczHH3/cPfnkk9n3dXXh3nvv7dZbbz33j3/8w0+u7rvvvj5pJmNwSiIce+1rX+v3P/GJT7i//vWvft/+GQJdITDotqyqXa16J8nn3/72N3f44YdXXWrn5jIEXvziF7svfvGL/q1+9atfuZNPPnko3rC0D6FNj7X3Y0HHv/KVr/jJll//+tfuhBNOyCqHUpyyEh/wRbvvvrubNGmSf+qHP/xh9+yzzw44B/WPw9mGLjE5JnL66ae7X/ziF+6rX/2qP/Tb3/7WHXfccXJ64NulllrKLbzwwu7pp58eMQELqX2nnXby+fnhD3/orr766oHnLeeBH/nIR9yyyy7buxQy3CmnnNL7Paidz3zmM55QyRjrsMMO84/V7eVvfvMbd/zxxw8qO/YchcBrXvMat//++/sjZ599trv22mvV2Xa7Bx10kFtllVV8Ih/96EfdP//5z1YJzj///O6oo45ybEWkzZDftv0vAlbH/osFe1V2vdg1g/KXVOVlZK5H/pJ8zutjjCr8BKNBleXIEnrul7YRIKbz0dtoi+Ayr+tO03J41ate5d73vvf528455xx3zTXXNE3Crm+BACRzPprgQ9JZs2b1Uhrm/k3qGvmdOnVqL89tdvqRps6PjLv0sap97DnsOpOxgYCULwSuY489dmxk2nI5VyMgbVoXNonuD5r6YFN9zFwNvr2cIWAIGAJDhIARuDILY14ncL31rW91/CGnnnqqu+222zKRG57Lqt5BDKPRdGIND1JjLyc4LD71qU/5jN9yyy3uO9/5zsBf4uijj/YTmHfddZf/kiw2oW8EroEXyzz3wEG3ZVXtahX4ks8uBqNVz7Fzw4dAG+dBv96mTR8yLxG42uDUr7Jrk+5YIHARJUMi//CuEL9PO+00HzVNHM2jTeCCkEm9fuqpp9yUKVN6RaInZ4eVwMXX/8ccc0wvz+ALOYcoqYMWI3ANGvH8540lAhcfk/BRCcJkLmPbM844w7cZ+W8871ypbRIjSTrv70n5fMR2H5S/xMYY7ephFX6DLsvYm2gbwQhcMYTGzjEjcI1uWX3oQx9yyy+/vO/zDz300F5mhrl/23HHHd3GG2/s88xHC3wE21b6kabOk4y79LGqfSNwVaEzfOekfI3ANXxlM6/mSGy1Lnzmuj9oSuBK9THzarnYexsChoAhMGgEjMCVibgRuP5L4PrWt741JkPKaidW+A5iGA3KIZmpdnZZJgKjPamsn3/SSSf5r96NwJVZeHZZpwgMui2ralerXgxSAktn/OlPf3IXXXRR1aV2bi5DoI3zoF9Q6Da8KQk4ReAaCzreNAJXG5z6VXZt0h0LBK53vvOdbu211/avCTkd/RQRR/OwErho46kHCEvqPPDAA5L1odmuscYa7oADDvD54eMUPlIZLYkRuIjAhp6yjCZLnRJ5zWTwCPSTwEX9Jn0EEkPbJRT32msv97rXvc6nF7YZ/qD9G4GA1bERcIwgcI22v8TGGCPLpumvKvwGPV6M5X0YCVxjwXaPYTnax4zANbolkJpcH+b+7SUveYmPFkoeL774Yv/XFsV+pKnzJOOuv//97+7EE0/Up6L7ELhYvcRkbCAg5WsErrFRXvNCLru0Sdr4YFN9zLxQBvaOhoAhYAgMAwJG4MosBSNwGYErU1XsslFAYLQnlXfeeWe30UYb+eWXWIYpNaFvEbhGQTnmsUcO2iFfNTkwj0Fvr5uJQBvnQeYjGl/Wpg9JtfeNMzEKNxiBa/iXUNQErnB5NXE0j3bUmFQErlFQ6caPXGedddw+++zj7+tqAqlxJp6/IUbgKk3L7usWgX4SuLrNqXO6zfjsZz/rifJdP8PSm3sRqLLrbYwxtsp9mMoyhtwwErhi+bRj9QgYgaseo35eMVYn1xnXLLPMMu7Pf/6z+/SnP90JRP1IUzIm4y6i9X7iE5+Qw7adSxCQ8jUC11xSoPYaIxBo44Mdq33MCADshyFgCBgCYxgBI3BlFt68SuB66Utf6nbddVe39NJLuyWWWMKj9fDDD7vHHnvMXXjhhe6RRx7pIch5lnlhELbIIos4vjjhup/97Gfupptu6l0X7iy11FI+fPIKK6zgCTC//OUv/fIl//rXv9ymm27qwypfccUVji9dtLzgBS9wb37zmx0OA57N84hCwNfpGN0iOe+gHZJHHnmke/3rX+8jHvAuTz/9tPv973/vyMNDDz0kyY7Y5uZFbpo8ebLHlC9yLrnkErf11lu7dddd1y2wwAJzDAZxvq255pru5S9/uf8CnyhhLNN37rnnOjAKpWk5bLfddo7J61mzZvkICUxGT5o0yY0bN85HS/j5z3/u7rjjDv8YsIaoNH78eP+FONEUfvrTn7q77757RDZ22mknv5wO+fz1r3/t02OSjLw98cQT/tj555/vy1bfSNjrxRZbzD366KPR5WvWWmst/1U5781X6myJTPCyl73Mrbrqqj4pBtT33HOPu/XWW93NN9+sk/f38qU75Yrcf//9Ppoc17HMiBbyyx+N5DnnnOP33/KWtzjaAlm+SK5HZ1gXXAZ7qQn9OgIXOvCmN73Jvw8GNqFyn3zySa97hLnVgs6RHkIZ3XvvvR5nvronj0RquPrqq3sRLzbYYANPLKMuo9NcT+Qj0o9Jk7zE7rdjeQisttpqjrJBaCuZiNfyjne8w7HEE+0beqeF+rTNNtv4Q5dffrlvp9q2Zcstt5xvx1/5yle6hRZayIezpw7Qhs+YMaMXGSKnXdV5Dff5moh2hzaVNkQLbcAuu+zi20i+pKTtp02gbb/++uv1pbY/JAigozvssIPvj2l/iCDyxz/+0fcNLEWm29fQeXD22We79ddf30cgoZ97/PHHffvEfWG/L69L1ByWXaA9Q09p07BH6FdiX7pC0CCCDXpM27jnnnu6CRMm+DyiW7l9iDxfb1Ptfajj0qdwL7p8++2362R6+xCCsaF4p7POOqt3nB3a99w+TG6kPDbccEO3yiqr+L6BfpsoTvTrX/7ylx1fPtNPn3DCCXJLdFvX18r75fSZ9Jebb765LwPqO457yuW6667zZaIzoPs6lrajf6cvXnnlld3iiy/ubQrpz+j/YwL5gnJadtll3TPPPOPuvPNOd8MNN7gtt9zS95vcA/n62Wefjd3e+FgX/SdtMeWGbYN+IuSZukXdwBYUR3MqAhc6Tv/CsioLL7yw+8Mf/uCwsWlHU3WL5+Tet8kmm7gVV1zR1130iLyRPnWRMQJ2JPUUgRwl9obYnZTblVde6ZjEpYwYD1A+2NxE7MJGi0npuEGnRTuErYoOoUsI+GALUz+1XdvUxpfnNL0vRuAiDcZhCHYt9QR5wxve4F796lf79pLxwOqrr+51Gb2hraOe8x46Ypu/8fl/Cy64YG8MxT71CluS+xhbcYx7Y+2pTid3H7y33357r1u08wgROBknXHDBBXMk06Q9meNmdaCLukhyIYGLsQz2Om0KZcR4F7ywk0Kp6n+OPfZYP76gnqP7P/7xj8Pb/fgip72MtRnkiXSlzZgj8ciB3PaZd6cNRaj7Z555pn+WTpLlHMGO8Rr1GnuP/pvxHGOcn/zkJ47xPzaAjOOwCy+77LIRY3mdJvu5bRTX0l5stdVW7LrzzjvP29T0Z9jQtIVgE6tj/obn/zV5HuVA/09/Qh9OH4c/g/ekHUZXqMeXXnqpfsSI/aZj/6ZtzYiHPf8jx663McZzYKE/lCX98Pe///0RcErbxUH8C7TbWihb+kbaP/Rfy2abbebrC2Mw7GpsI+oNfoiwz67Sa55LPa7y37UtS51vvd9kPBASuOhvpC0QmwV7jTFuTOjrqGvYxdjN2CHYzvgq6Ve0n5L7qzCjLaB9CG137uuiTmML0WfT/5E/nse7oS/oA+WcGheQh5h0pS/y7rFnhMdo97E/sP1od/B9YZvOP//87n3ve5+/HH295pprRtza1D8pN+Ozo/1mi06wzJ7UHe3vpX2lnUWwN2M+W5YWxhYBZ+wjRMqWcSpLfaO/6CV9FP0akVnpt6iLnKP+Mp6S8Sr2N7rGtTFp0neA577P+/ZyxzviRyJP2G0I74ffCF8p7xXr34bFhkT/5SMKljPHBtVSks9+pCl5knFXGwJXlz4MbMgcaWIj4FukviLY6OiiFtrdPfbYw7e32Hbo/1NPPdW7pIndrXW+K382/kXacYSxJHWB8TTzKfgVGevdeOONc7wX10v5ik+fY1pK/DD6/ib7tEPMyWBDghNjaPpIxgkpf0fTdlbG413OA8k7NtE5uYdt6TuU+hRK86nzLPv9wDNmk8jz2DaxLUp8sHV9TFe+M/1Otm8IGAKGgCEwJwJG4JoTk+gRJsDmRcGh8LGPfSz66jg8xaHCBBET7jhOYnLffff5ycHQ+YRhCmkndh+Tizh9kTCEP4bdIYcc4nA2xgSHNctGIDnvIE4snD4MRHAChYJxglEfTig1yYukKZM0TADgEIAQJfLBD37Q7/Juhx56qCc5yDm9ZTBy1FFHjRgwlZSDvDuNAU5+JrNCwTmBkwmjNBQcAywbyABPRKJ6MAjHCYlTJxSeN3Xq1BGTtV/4whf8tamlLBksQq5DwJCJ/i996Ut+sipMn4EI6SM8/6CDDnI4UWIC0QCngdZPHChM+PJ+ONmZMBZhkgJnF0I5UQ7IGWec4QeEqQl9SZNr+WpLD77kazHOxQQsyaOINsB5VxxqHAuFiT0mRZjcC4Xy/tznPud1UJ9rmhd9r+03QwBHAMvdIOEkPO0CbYAIuq7bHyafaUMR6iB1VOpzSVtG/RbnpzxTb9EXdJ26m9Ou6nvDfcknE3iHH35473RdGwbBjZD1KUdpLyHbGRgCTIhAfom1P2QCpx59gpA3dNuFI4r7cQaGgp6hb+idFshXEydO1Id6+7TXOBrDCTVxyOGIxxEBWRqhDcbhHXu+7kN6D4jspNr7UMdxHmInIfRd9GGh0P9+8pOf9IexDeRr5JI+jESYmMBWYtIhFCaQwB77K4fAVdfXSv9W12eCF5MoMcy59wc/+MGIiR+tL9h29GdM4oRCWdKf0fZp2W+//fxEjD7GPs9ico8JTqQrAldX/SeEDSHo+gyqf6effrqffBK9DvsOLoVQCcEkJkyA4fQPJ0qa3nfYYYf18NPPIX3GDvRP9FMIE2OQyhGpG5AnsbsgmIVCG//1r399DhJH6bghTF/bbuE5xh+nnHKKP1xi43NjyX0yNqDNA1tE67+OtIZNy4QdYxPI36myZmIFMowW8H7/+98ftc0Zr9Fu4NDWYyl9f9N90mNiVyYXw/ux+enXtT7mtidhWvp3V3WRNDWBi4n2mM5yHX3a1772tRHjCamnsf6HsYCUJfeTZ8Z3Ik3ay5w2Q9JNbZs8j36FNlf6U62fpA+BZMqUKV6X+C1jJ9p/iF0IE/8yrvMHnv9H+zx9+nRPqNDH2W/atuk+mol9xnPid/h/9t4D3LKiSv+ubhqRIEERBBUaUAFhAMUAioqoKIiAGBDDYCQY8HHMOA4yjhhGnI8RUAQFCaIzioyoCIiALSISBgVB/wqIoJIkDdBkvvMreI/vra69zz51zrl9u7vW89y799mhwtpVq9Za9dYq9Ga2TSWKIJTWgWvD5udyj/wAudGfUspNEJbY/iWyJi0Lv7vo9ZLdS7qNQWQ7+QZTex6bhu2DISaK4ZkTwHnag+uB6DTvec97suMp7yLnWexAfxG1tWvaFgCbHMl/N8q3zKXLtWHtAe8rgBrRz3OEj+mwww6bcovxhDEz52PSgwDEAWWJ2niGLMCWFl/cPvVyDtun+bbY8vTTlOhH6EtM0GMrIS+70Ljbi+o+KG8WcaLTSX76897mUgDXINu+yU+MT2LHHXfM5kfeLPrQQhfkOgtkIYBL3lfixd4/+ScZw+mHkH9bFvkAPknrx3jEuI5ekvMRM+7jo+M5p2HHDtf3uto78AcgX44+/vGPR10kN75J75gJOqT8v7l+XlrOSaQJj6XPYXOWROAatw+jSxmG1REAjLAoH6JNo+fhKxdJP+c3+jvtTG1/WL3b2/y4/Nmur2PPAdxK+zRlR2YdeuihUxbE6/um+lmpH4Z8hiX8I3xXLXZJ32eu6ogjjugv6NH9Ejmr8W6c80CUZ9g2N446lPgUSsup8qbHSfBTabpOQr4luoX3t64+2EFjDG2nUuVA5UDlQOXA5DkgeYvuha6AboNPgT/O9btrSWb1UOJTLZeub87w5+SkmeHFHHvxWPW/1157RZAKKxYgHAxMuOAAQilmhRITZCIUfFY9oRDhwJLTMp0MxUAmuowIZRRAEw7f1BnjAC7KgeNM6VIWJiL5Rqy4kIKu/LrUQYqRyoIRwgpZ0mZyD2UH4vdHPvIRPRZXcQxTFr2oSRr91pEJK8KTQv4MxjV1RHGb2wMhqTysQCN/qPQ75OqOAoxjHn6mhKOJNoBRoQlYn2zieTlI9C78JE2+GRPU+kY+Qc2zMraHAXDts88+0Zki4BllAShA26SNQkyCywiCl7RPyq4VrjzDt+U56ge9ueekwdEHUX6VmXMmTxVVDgMXQ5frTADzDd1BiOMQByLkabrDlwhsRDyAKB+RHygHqw1pv8obIBlRLSBXwOOF3j/6EHzGEYhSnxICn4khT5O8mCQVlZRF79bj8BygHeL04xunhhngVkUvIWVWgJ5wwgn9TGhDyFmXG7n+3EWW4ehn8oBy0JZxpiKPmdChb6mvC0jYRa72C5o5UTm9zshw5J7KgHwDsMaE1nrrrdeX+azqxTlcaWZwAJAV3whCpuMYp73QpjSxSxvEAQjlZJfGON4TqIhnU2CKT/zSTpHltBNkJTJespJIXETNFMkhp986MoHGeIFDvm0M0fO5Y5O8T9s44x99XbrLvvvuuwDYaLfddosr28kH0AXgC6hkDIMfOFXFE3iMfkbEsnQCoguAa9BY6+Mb30b5cq4xkygJbO8lgvd8A3jvZWKCjD8o115IE/AVegrvKi/pfUr/bW97W38hANdonzjdGfsl0/TsOABc4xw/idSA04zxXP2I8Z26M5GLc1ntOu0nPkHA87yHrEUn5/tDXGdCinREw77HRB4gIvjJN2Asoq8zdgDI8YmxHIBL+XKkbaL3UEZ9G377IpJSu8Hz0TmTv0yqen9AR2JClYk/5Mew9obSLn1Per/r1N7+HVyiSS3lyRHdmT5Fn3CwFHYCchJCVvNbfYZ2QB/kedl58cHev3EAuFJeUEbGdfJDbsveo84AH2hDUBd5Eh9s+DfOvkgWPiGkLCkzY1BqfwJEoP2L1E/1W0cBOPxbOoBrWHmZkxkASCDJDOWdOw6bH2n4tln81mICzh3g6baGj+M8B9HvkB2M/5J3XGcxFu1QNKyM4j0fo5UOR9o+0ZGY7M9NcPNMSX4u90gDwvbiW9BW8HOIBKbRb8kAfksvarP90/41yCeifHLHLnq99Bq9Dw+XRBvD27CPbfDFwe58Q3QLEXoxuhQE8BbAOvSJT3wijvWc01aQkcgXFmfRZiB4jZ8EfxDU1q4B5DAOIO8l11P/Xem3jJk3/BvWHsj1FeqNfkfZ3Q9F1BGPcOhAAfQ6dBlkB3arbBKK+ZnPfKYfiauNZ8gC5I344vZprpwlfZryoIcyziEHNA5zfRgA17jbi+pOOZqIyGi77rpr/zb8Qd9Abri+wQMO4Cr1T3oUJdJEd8AHQX4CSHL98MMPD0SpHweAi/QgxiPq5n7Ch+481A8Zu2lr3s6IpgfoWFQydri+p3To9232DgsBFaFFi3U07uOPRQblxjfXO5TXwtIhJU8pK/1aeiDlKi3nJNKkPNLnSgBcKhPp8F3H4cNAZ2ijUh3Bgcg+30DEYhaAQOmYVKJ359r8qP7snL5OmvAbm9r9DSysAMQv0vdNAVwlfhilOezx7W9/ewSd8R7jIX41ZBKLYXw+g2+keYtSOavxTmXkm+IzGGUeqLTNjasO1KWLT6G0nOJV7jgJfipN10nI2+0FfnfRLXL9TbZGkw920BiD3VGpcqByoHKgcmDyHKgAro48lvOk4+OL3WOEi+YPcjAVv5mUlXMoNV5xHqFcajKGFe0Aa6ADDjigDzIBlOJbCTDBhyNZ5Hm6IZcq3Th+P/CBD/SjTaDwyIhtq4MUI/JDEWZVFsYyhFOCLfLkUKa+OCuh0rK4woWijMHPZKWUcI/UwTWMBilHTHbgINOkBxO0dOTS7+B1x0jAOaiJHjc0qS/fTlEJ+O0OJPiOcQQ5gIs0AZroHm0FfsrJACBFW6iVALjIDwcffIDSCSdWC7IlFYTDBccqRjeEEkv7lDHkk8Y+ecSzTJDi2MO54US0FNq5gC3ccwdhFwCX6o0CzbdW+UiLVf4C9fmKwVQBZxUFRicORci/Db9ZochKRQh5xjcASEB9FPGFeyVl4b1K5RwQEIsUHNThzhPuIcvoryL1M769nHPen4eRZUT+m/twhDpWozFRLEIGIrOQOaTpE+ptclXv544qpxujnhZh1D2KEk5STTrjxIBPlRY+B5B9kh+MiwJpUTLkC6vJNVnJZC7jQSq7aNeMudyDkHkAK5jYQJ7hzIVIB7nGdWQlERaRiSKicuEwh3D+MiZIlsohxz3ePa4XLZGxgvEXahtD4gMt/5rkfa6Nu87g442Sl15Euej/jPulYxjge0UzYDIQfmkcJvoJkf80edQFwEUZ2/jUZcxU/UjLJ3j4zTYZbEtEmVzOpO2FSS6iR0lPIbrknnvuSRKxfugikLcX+ImDVtvTIMuYVPVJoHEAuCYxfro+nJZR7doBXHwj+iR85Hvz3fn+IhZOAISCXKcofY90GH/4TukEpE96+iS3+gbvUka+jbaJRtYjN0gP8slXbz/D2A0xoYZ/PkmYtknvr8PYG6XvyTZAFnaNwEW1aN8APgXw55rrD0RL1oKGvffeu79VHdtN0D4kB9PoGqk+TbrD0h577BG3zOO9FNiEbYj+Ix2cLU2QzVAXeRIfbPg37r6YTgjBm6OPPrrPO7bEecc73tGXqd5u1U8pam788fbiAC5v72nbbJKX5NEmM7jfRKX5ua0omw87lugjEPIcmcQR8uf5nUaJI1Il70OuI5bKKB+jSZNJDoBmivjmY4yDJEvzc7lHfoyvREuQfebRc11/L7H9ve0MI6MoVxO5Lu7+F5532c337OovWdxsDAdieZtxm1389ejJbGVGlAxIMoK+DHgfkj9HPh+ueX8R6JPrg9o1z0ziW5JujkrsgbSveEQl8vDyu5+FCXjGS4iJWmwPjWNc84gRrmt34ZnauMuetJwlfZpvig2jb0tbYeGUdJ1Uf6IeOZpke8nlp2s+PpzZ25aMCO8ib9dc8/Gq1D/p+bkOQ/oOFBE4eFwALq8b+ijtTAtvGL+RechtyPUCdFjAZFDp2OFjEel0tXd4Fl8hbYp+4NHbPU2XVT52LGwdEh0Q/yfkfmF+l5ZzEmlSHtfnZIdyPUf4zdVP3CYdtw8jl7euOf+G0RGQsdhimsM5+eSTo51Bf9ZCIF9oRn4lere3T9IYhz/b+yVppou70NexTSDaPvMq+pb6vg7gKvXDxAwK/smmpp0QOEC6I0m5bQQAHCA4VCpnNd6RBvo731C8cN2D+13ngUrb3DjqMIxPobSc8KKJJsFPpek6idsLw+gWaX/r6oOlvk1jTBMv6vXKgcqByoHKgfFyoAK4OvKzArjyAC4mAVHuIHesOFvZu5tJOUjKsG8b5hNI/p4rcXIgYvwAPIFovALt+Huu0LCi7Kijjoq33Qmk9PSeFCN+8zzvOfmEh94dpSyapCEPdxQoT1/ByEQlfHPykNyA5nBGl3wH0vS6q27Kyw0gHLVMKLmDzJ3srDZncgYSsITzXPk9XQelyPBLQUWkA+W2UOQ6jhK1hXTCCQcpABTKjeMMZ4gTkapwEkHwUQ4EN5BY7frZz37WX4vngPpIn4lSXxnqDsIuAC7SxkC+5JJL+u3VM9NqXpyUtA3IFXDqxuSIwArcd6Mr18cEGuJdd/SUlIX8KpVzwB3N7vTUd+cb0cYw5LWSm6gnGJ6QT355f+4qy0gDo4xVy/QPRfXjukiRFNL20iZX9W7uqHK6MerAAne4633kHvWmnXvUON2vx+nngMvynJwBKMLKYIgJTJxCqexCdqfAWDnuvb15NCWPRui1Bsij7WJd9sohx7O5MaltDPH0c+dN8j7Xxn3lqstz0vXti1yfKhnDGNfQoSQ3AF0iP5x23333AHgFGjeAKzdmssWbJvRTJ6DK5aAzya+0vTBGy3jRe64vahtqby85Pct5RDopOEppD3OcxPjZBsZQu3YAF1vVERUH0laLaR28f+Ecxmlb+h5py9mcTkD6pGcTgCtXRu/HgLuYdCi1G9K6p7+bAFylOn7pe5RLtsGwAK5cf/JoGdKLvS+RB3qjTwxQBue93uN6CZEfej1yyMG4nhb6OTIO8np30cE9nfR83H3Rx7omG8VlqkfXUT+ljLnxxycRBOAqlZfk0SYzuJ+jUfLj+2KbaDEX8sijprp9SN5un7BYSt/fy4W9RqQESACYUhnlYzSTO4Abvd17v/AJ7tL8XO7lfBUAiOkXkMvMYW3/Cy+8sNgnEjNv+Nem10uv4VWN0Z5Mzl/C/cXRxpCN5otKFDlZdht19+jJ6Lvom/hUtBgGm0t+RvqCFhDyrsh1HOx37KBB7Zp3J/EtVab06DKyqz3gfcV9MEobPwuyHPK+gi6Nvggxia0FavFC7x+RywTm9wVBXXimNu72qZdzmD6tb0t7APjjYHrK6uO010/1yB2VJvfG3V5y+XHN/b05/Z5n8M+i/0DyZfh7btfEhx7+l/MT+4KcdHEQrwGoQu/Edya9YRwArtw38LEo55vgu1Ie7Fv5T0rHDh+LaDNd7R140jS57mn6+OZ6x0zQISUbnY/Uq6ScvAdNIk3X5x7Kpfm/9xW3Scftw2gqwSj2CGn6nArtEV+BFobl+mWJ3u3tkzzG4c/2sQidjzETOeHkeqiP0fq+mrPiHeRsyVyC5zfMueZR0BMou+ur6MVatMYOHyx8KZWzlEnjHefjmAcqbXPjqkNXn0JpOeFTG42bn+SlNF0nkR4wrG6R9reuPljK0TTGcK9S5UDlQOVA5cDkOaA5EHyR2ED44LBB+ONcv7uWpG6h2JVTi9hzTQ4gX3GVKn2qIg5dHE+QDGMH/sjI1/M67rTTTuGFL3xh/Km0fQLHneN6h6M7ewh/jeENNdWBe1KMUIIczMI9yFd6qbyjlEWTNOSHMsTRSYo7UUS4nxLOXxRdiBVf1E0rOsWr9J3cd+CZtrrjUJQzwo1Qpa3tA/nN9hznnHNOvKXyu4NS7+ioZ9zxOW4AF4IMxwpEOGABtVQGHeU09+/vk0eaQNTzOtI+aae8hyNWBpY7CB1E4GnKAau00iMOXkIlb7311mHuw5GRfMLfFfDU2UFabMlIu4Uw8CiHE45NHJyQJrz9vp8PKgvOt/XXX99fied8W/pLpXYOOHhEESr49gJr4ZiGx5AmshzM6G2prT/zfk6WcT1HrDylHIBwkHcoBN5HeKdNrubS1DWV041Rd2TzHI5unCoAatW39H49zgwO+OQ7JWLrNCZLaLNSMNOSuuzyaAL+HJPYbK8FMQYyFtLO2TI0lbf+HvJy5513jpeY3CRCCiSHnHSQeNH+eR900AL1Q19hLEmJ8Q5neJO8z7Vx0sDBiZ5CPRRli+sOYNTEaOkY5s5L5wP5iDyCwbgBXLkx053XfI8/9qL/pPSsZz0rArW5zjbFxxxzzBTAX9Nkv5xKvKf2oskk+Mx4lwLYeNajgowDwEWaKQ0aP9Pn099tYAy1awdwyanXpD+SPpGC2G4NUmTc0vdIoxTA5aBk0hH5uCKdu9RuUJpNxyYAV6mOX/oe5ZNtoAlJrrm8bJp8k17A8yKPPiC9wqPGNPV55K6iHrosVLrDHD2/trRcJsoGc305J0+GKYeeHaUvukzN6dTk4eOIfyv106bxxycoBeAqlZeUo01mcD9Ho+RHeuiKyF70RCfJcb/mE2dEdTjllFP8djxn4Zf0XkUZKJVRPkbn2mFTHyvNz8EebPnGAp+UZO+6DSy7uEl2p7Y/CxqIpAkN6xNJy+O/Xf6mPgXpNaktoPeXJBvD9Q6A0ERBUORDFqcxQYgOqYVq9A18EhxdPui7N8kHeMuWVYCWII3Zg9o1z07XtySvEnvA+wpbF7OdaUpsPU7Udu8r6TP8Rl9eY401YhSi7bffvg+KawJw5WQB6aiNu33q5RymTwvc01Z2AQHbvj/lEk2yvSiP9Oj2idpf+gyLdfALQtLbSv3E/t7ZZ5+d9SWhXxBFF76xZeg4AFznnntuOP7446dU7b3vfW9YZ5114jXkNtuwOal9AqrEToVKxw4fi4axd8hT8iiVzZ6myx3XO2aCDul9DP7h24RKyhlf7P2bRJrS5+BzCgpSvjoSHY7Fa9AkfRjKLz2OYo8oLXTytddeWz/jEdsNGxvQ7SAapHd7+yYdwGsAAEAASURBVByXP9v19auuuiouLE/L6dt/e7/Q9xWAq9QPk+Y3zG8HbNPGzj///DBv3rwsuJt0XV6mOpvyna55oNI2N446DONTKC2n+Nl0lP6QymGeL51XU5quk5TqFt7fhvHBUv6mMYZ7lSoHKgcqByoHJs8Bza9VANcAXmtl3IDHFtvbTQ4gdyjJeZVjgoxbVkEwkeYrk5j4UShqf9cn8qSMamWjnkM5ypEcyDjTKBfUVAfuSTHySROuixwMI6fEKGXRJE0O4NQUEUplyR1LvwNptdXdlX1NAHn+gwBcTYohacg4cQVXDu2m1e0OWoGHpA/5pIk75NKtDAa1F9LSNl8+ecQKuJyRqqhE6Sokd6p2BXDhiGJbDQAKOOlz1ATgkpHp72yzzTZxCwGusRUdDkynNgDXsGVxsIXn0WRI+TP1/CEOaAJTDkBF2YOHrI4R+FCrxLQiNF0J3NafySkny/QNmGxgG4+5PcAgABPJUd3n6P2V321ylftNpHK6MQqKnK0R6c8p4aAlQh0Tp3Lqpc/U3wuHA0wg4ghJiW8LsIQ2ixNT5M6DnOziOXeYC1ijPuLjutLUEfkpB7pvTSSHHKAhnesdjk1jCNsyS4fw5zk/88yHthBpkve5Ns579DG2QIE8ep7qB1hRAIrSMQz5z2QG5GNQvGD/NBHUBOawR+NpE5+4OWjM1MQmzzaNxdyT3BEoyduLtkrhOSfXKdVeBCpq0ut4ny03WekP6b34Y4R/w46fg7JqA2OoLYtXpKXJQM6b+Cwe84yih5a+RxridToB6ZMnuQhcTd/GowFJ5/ZvPIzdQPnaqAnAVarjl75HGWUbOF+8/fskg09q5ewvn0xX/3a+snU7EXxT8skK16fT57r81iIHnlU7y73n+qgiigySJ7l00mvj7Is+IZRb4a28JVPdLlA/bRp//FsKwFUqLylHm8xQOdPjKPkpLd8akGvos0RTSOWQA7gOO+ywcNlllymJ/tG3CRMAo1RG+RjtY64ya+pjpfm53GP7JHSFlBTRQcCOEtt/FFmTlsd/t+n10mtcRvm7S5KN4QsGvvnNb8aFC2oz6L2AS1jcJFsY0DTgaUgyxP0sPo47Tzl3f4u2kxvUrnlvUt+StHM0rD3gfaVJV5VvSH3F8yVCzIte9KIIlsOGzJHkB/e68Ext3O1TL2fXPu1jcG4RpMqqRYSp/qT7fpx0e/G8/Bz9GJAuhI2FbE/JtyWT3lbqn/T3vv71r8ct79P80t/jAHDl2uA+++wT0CUgwCzpeCYft/w3PCc5wHn6PNegnB7uY9Ew9g7pNU2ue5ozWYdE90S3px878Nv1o666LvyAJpGm9Dn/3g/l1v5fNv4kfBhNOY9DR0COIaMA0Yp80bau6Tis3u3tM+cTKvFnu77eBAymPvRdyP3r+r4qS6kfBh2plPBLMOfhMoK0mEcjuiWLJKmXyOVlro/oOckqzcdxXeNdTqfz8abrPFBpm5tUHaij274am0rLSXptNG5+kpfSlE4yim4xqL+RX84Hy/WmMYZ7lSoHKgcqByoHJs+BCuDqyOMK4MpvoYhjdtVVV43GqVZN51gqxUNOLHcU54xh0vCQ2gJwedSAXD7pNVdGuzixmoBDOYfkKGXJTdKo7Bg+OAugdM92PZMeS78D6ejb5OruijsOdhztTu5QdGNOExgyfvwdnSusNY4NtR056XJl4T2flOgC4HIQoPIddJRjyiePdM3fxcGAIYRxlYZUdwehO4Oa0mxqS/CGiDZEUoDcwHQFnChshx9+uBcvuMErx7I/4BNmHoGrpCwVwOWcLTv3bYuI5kafYCWxVm4L8Mhv+onanju5yLmtP3M/J8u4zkQrWzmmzgLaIE5zjEXavPdX3muTq9xvIpVTxqiew6ECeA1AECCyHJ100knh9NNPz92q1xYSB7baaquw7bbbBqI65cidxoNkF+/nnAcaV9omOTztHICraUxqAiZNAsC1yiqrhP322y+ySZEhWd2qcdBBE6VjGI4pRelkNTmrynOkSQYBPHLP+LUmPvFM0/im9yXD9HvQUZMX/k1zYx3pOLhHQCy1lzYnuzvw9N6gcrXdLxk/29Ljnus9aRnlaNbEL/Kbeg9DgHgAGJS8BwgIKgVwNel6OWdrqd0wiBdNAK6mb9mUnuyN0vdIN2cbePt3OeqTWq7DqXzu4FX/9vZOpBOfANB74wRw+Yrqpqgd5Ot1yQG4cjq4ytt0bPoObXp1U1pc9wmhtgllyR3X19VPm8Yfr78AXKXykrK2yQzu52iU/JQekaD55qKmCCIO4Gr6tr5NOBFYiMJVKqOabDKVM9fHSmUpMtHBHk39LAVwldj+TW1c9UqPklHp9fR3m14v3b1Jdi9JNobLWCIA/fCHP+xv24duRZTrXXfdNbKX39jljDfIIGxwfGK+rXZTtFQS8DaF/YMdNKhd894kviXpttEw9oDXq6mvyDfkAC4WuiE7WHiUEpPTtE9twdoE4HL/jKehNu72aZdypn3aJ/6bdFfylexts21Uvkm3F+WTHrVojOs5fYPrHjlck+Sl/km+LQtyINLgew6iLgAujcW+xeegb+sArlzdBYqQrTHK2OFjUVObydk78KZpct3TnOk65B577BEAAvpCJtePcvx3OSxd19vKuNNUG9L39rzazqUbtvVz/1bD+DCa8h2HjkB7RkYBMhfltgLnXlN+bXq31znX5kv82a6vN8l5yqtv4gvO9X2lr5f6YWgfoxARwrDZ8EPxDVJCJhKpDr2uVM6Spsa7nE5XMg/U1AbS8uu39NJJ1YF8cj6F0nKq3E3HcfOTfJSmdJJRdItB/Y38cj5YrjeNMdyrVDlQOVA5UDkweQ5UAFdHHlcAVx7A5QN8G+Jfk4REMSKakRufRFzBKZOSb58gAJev7mV1Y27ywdNBGWUbRWjcTqxRypKbpFG5fU9uN950P3cs/Q6kJaVwXIo7acog0uQ011JSvgL1cV9OulxZuO+rMzy8dtOksk+Kw8s0LDppOuH0Y8IBGjQZ7avD0wmIJqdqLk03DDFwcT4QKpmVLvABUv/xCaFBCniJwVtaFgAJGHkpOT/Te/X3VA54m8GJjdMfwBQT66z2laFJf2FbMdoSlDpR1K+a+lBucoWtEulPchBcc801gYmEX/3qV/1oV4DK1lxzzYkDuGKlHv7H6nVWoeHMA8ym8nEbAKgUGH+nni9cDiAHAN/hPEP+IqdE2gZpkOzieR/TBFrR+NC2etUnEH79618HdAcodcjFi/avaQyhD9L2cgS4Fged9113FKovyuHiaaAHAcxF5hPJkQlteAbxHkBNqHQMY/sYAHVQLuII1+lPmhDPOb15JqUmPvEcMgleQOmYyDUH4PA9GB/aCIcqumGX9uI6pdqLQEW5SKfK17cN03u6N+yxdPwclE8bGEPtWgAu0lJYferN+DCI2A6GLbtK3yN98TqdmPCJsVwErqZxKuds9W88jN0wqP5NAK5SHb/0PcqZsw28/Y86+eZ8bYoi4tuKOJh0EB9z913faIvAhf3IJAWkRT2D5EkuP12bRF/0CaGm6GX+rbxPqp9qQkjl1NEnKAXgKpWXpNkmM5RnehwlP9LCT8IkP2OmEwBRAT113QFc8IbIZCm57BCwo1RGNY3RytO/m/ex0vxyZVdeOqZgjxLbfxRZo3LkjuP2lyzONoZkNmMZtjtRueTXYBEKUVcgfFa0QxY5eHQ++g32F+QyI16wf4rKzCW2BsdGG9SueXbc35I0u1IXe6BLX5Hu7wAu2cSUBbAH9ir8Z+EjE8Lo7ejW0MICcDmwRAsSYoGSf/KZpfpT8lj8Oen2ksuTa77ojwUolDUl35JKAC635YbxEwPUmduLBg4J1B1/tPwbBODy7zFJABdFLB07fCzKgVlI23Vht1uaJtc9TR/fXO/oCoyatA4J6JW2BgkoX1LOmMDD/8adpvS5YQFckmOT8GF4ff18HDpCLqoifg0AN4x1olK929tnrs2X+LNdX2/aXhoQMN8E8giJ+r7S10v9MOLLqEcWtm6yySYR/A3Q3/3t6s+lcpayyVeVs8fJC/Ae1HUhf2mbm1QdKLvLLY1NpeUkvTYaNz/JS2nKn+hj2bC6xaD+Rn7+LbqMMbxTqXKgcqByoHJg8hzQ/CdjNj435nNYfMsf5/rdtSSzelGT8nvadU1hhj5XAVx5AJeDrI477rhw3nnnLfAFXfEFGMBKpde85jWBsOfQkUceGR0v6Yse1UcALjcOmkLi0mhJH4UXAx3FHRq3E2uUssjhp1UHad3lzGky8uAddYSI/sQqQyKWQcN8B56XUjguxZ00Vf6miVNW8WAQ8K3ckSnjtokvWqVIHl0AXAg1wE+Qr66JF+wf4fcBi6AYM8kEvXnAZLScSwAEmbRwanKq5tJ0UJqMCk/LnXWTBnCVlsXLW8/LOMCgS1ulTyAncThBTIwDPiSaDpNeEL8f//jH9ycJ4sWH/7X1Zx7xCVW1NyYcdt5555hCE5BDod8BnChSEC+0ydWYYMM/lVPGKI9tt912ATAg9TvrrLOmvElfBvSi1da5bUGnvFB/TAsHaJdsEYOsB6CVkk8kK/x6qfPAozYCEMoBv90hc+qpp8aICJQpdcil5WwDJqXPpr+b5H2ujevdl7zkJXFLHH4D+qIf4UxMJ3BKxzB3XjZNCm6wwQZhr732ikVq6vcqr45tfMqNb3qPo0+4NW2bRURXAc/QJyl7l/aSm9BgXFYEhqZtkAWko3zupOL3sDSp8dP7UFpGtWv/xtqOB1nN84CzUqJ98AfR/tD/St8jjekAcJXaDZSvjZoAXKU6ful7lDFnG3j7l7OeZ0smtdZZZ53omOX9JnDWDjvsEPWEtme414UcDMYWyEccccQCr6HzYBci6zzqwiB5skBCdmESfdFlqn8Hy3ZK5GiiRqFjQeqnmhDydzj3bykAV6m8JL02mcH9HI2SH+k5CI9xTLoacoh2DeBZ5ACuJoCxg2uRTbJ/AAMNK9uaxmiVp6mPlcrELqCUFMBFWWQ7d7X9sWvZZgca1icSX2r416bXS6/J+QxIbkmzMXxLbBatrbbaalMmg/FVYMfj60C/Qd6xFTzjrkjfvYmnPOcAS/WHQe2a98b9LUmziUrsgS59Rb4hB3C5TZrKF8rn2/cuLAAX5RCQJ9XtuQd5lN+mZx568u//J9le/p7L1DOX2SeccEIEJE59IkRZhP4Dyb9Q6if2ftUE/pZ8ls8Q24HFK1AOoO4LLycN4FLZhh2rfCzKgVmoW87e4fp0ALimQ4eUHsziEvq+60ddgWbww2mcaUqfGxbANUkfhtfVz0exR0jHdXjaMn0N8Ah08cUXByLrikr17kFtflQAFwsE9M1UVo7uV23T10v9MJ7XMOf4f5///OfHV4jqqUAESsPLrTGxVM6SZptOVwLgKm1zk6oDdcwBuErLSXptNG5+kpfSdJ95qW4xqL+RXwVwwYVKlQOVA5UDM48DFcDV8ZtUANffAVwOuPKJj7/97W/9VYTOVjc0tRLXV2k5gEfvsdLgrW99q37GKBpE0/Cth3D0ExUGg8LJt/Vjj3C2j4PcieV14J4UoybnWc4hOUpZZEjK6UAZnHzika3xMOKdUiAToVR33333+Mgw34EX2upeoriTppxLnGsFFeciX80zb968QGQGqG0yNd2aowuAizTl5ONcYBjORR6xRRME3Bs0eaSoWKyqxZnl1ORUzaXJClFFeMHATFei+6Sll2+QAl5i8JaWxetez8s54A56UtEqbs4BpNJ+cf6Lrrrqqn4EHV1r6888k5NlO+20U3R2c5+t1tJIdakDpwnAlcpV0msildON0f333z9O+gE4AHiQynZ3yDc5dJvyq9cnwwHGHfQAKCe/AOsecMAB8b5AJoNkFw/nnAc+Ke1je0y89w8QJE5zhfn3MULOu6YJ9DZgktJvOjbJ+1wbVxrwAMc0/ZnJWjlE0+14eb5kDGPshhekTz9CtpCPk0+4TweAy4GiAvN7eTj3Vf6HHHLISAAuByTkwCq+uIC8U3AU14ahSY2f3u7TMqpdq29RXnein3baaTECm9cDZzTgAdogshaecyx9j7SnA8BVajd43XPnbsdo0pHnSnX80vfIM2cbuLx04FDJpJanhX6BDMCWEgG6YZIJfQPK9Rs92+VIBBraGnKINkYf8fxIg4kKtgmBfCVxTl+OD3X4N4m+6AAu6sK4hk7upAk6riHfmYCE1E+bxh//lgJwlcpL8muTGdzP0Sj5OSAZMDcR8t7znvfECJLkldqlDgZgcgrdz/U9xkPGT41f0jlLZVTTGC0+eL/wPlaaXxdQSg7ANaztTzvUdszD+kRU99xx3P6SxdnGSPUI+OkALe+L4jVyHvCdSHKf34ceemigDTqhnxLthP7gfqNB7Zo0xv0tvVzpeYk90KWv5ABc8sEgN1KgOnxCBmkLvoUJ4BKAD14dfPDBcYxzvvmY0RXANcn24mXzc5/szpUTvZLvxBGSLuX6VToOKP2cn3ijjTaKiz54JrdQMgfG8kUpKbgE+xC+AQCGJg3gKh07fCwqBXBRPwc6eZo+vrne4c/zPuRRXmQjelqT0iEls+jbyD3kClsqQ13LGR+2f+NMU/rcsAAuHwvG7cOwqk45HcUewQ5gvgE9HjrllFMCCzEACiJjoa985SuB7YOhUr3b21SuzZf4s11fp53iD/FFBJQX3Qn+QMxb4FOF9H1dXy/xw8TECv5tuummUYfn1aboYSqPwP6lcpY85KvKzYGVzAOVtrlJ1YE65gBcpeUkvTYaNz/JS2m6z7xUtxjU38gv54PlukDCnOdkMdcrVQ5UDlQOVA5MjgMVwNWRt+MCcOFMmDt3bjTgU0WyY1EWymPuYMH4IyKWJgM16U7BmET68pe/HIEHGMtEddEqAnc68aw7IAh5/q1vfStuqUMkKZR13hcpAhe/99xzz7DhhhvGW6x2BIGusgBOAMAlpy95iM9tdZBilFNeySgHeuB6aVlU95QnpAmxZdgee+wRzzE8WLVO6Frq9YY3vCFuUcVNNx5Lv0Nb3UsUd8rlAC4czIT7B4CHMf6KV7wifl+e4zeTIID4IDduicKDYUj9cdQQFUAGI8/65LyDBOAJdWJVCnk78At+40C7+uqrSSI6BN7+9rfHyCf89qg+bZNHbhjyLd0RSzpNTtVcmu7wZNsstjyiDjhscXoDZhS502yQAl5i8JaWReWrx9E44JEvSEmrD5Wq93Gu5VbBtvVn3snJMncWMPmGTGUrUfoVchPHl/oefRYDjiPUJlfjAw3/VE43Rj3iAvIO8Cp9GCJCHhOCcr4y0ZWuSmvIql6eIAccVMH3QPYjpyDGD2QeAEBIkTYGyS6ezTkPHJTEMyeffHJ0KHKOoxmHuSLXueON+zmHHNdFbWOInmk6Nsn7XBv3NJiUZvJPRJ/CAUofdCodwwDBa/xgPGTCiwkUdCvGWsYxkZzz+t10bONTbnxL03HwOdvdEDGUejPpQ5k23njj+IrrNl3ai08ACeSE3s74LNnFpOr3v//9KFPQw2kvAqqQqd7j3HU7n4zlXhNNavx0vcjLSDnUrh3AxdacTApSb3jLRBoTmBARyZCj6BeQb4lQ+h7pqK2TH2Auxg9kt48Po26hSD7SnTkfxm7g+SZyp7EmHfWst4Nh7I3S91Q/tw28/Y86+Ua9XC7Qz4466qioayDLiWCBnBU5gIu+T/+lXbH9qba50bNNR+8XOB1oH7fddlt8HPlE+1YfZZJFoKgu8qRLnsPo1U3pcd31fn7DOxYIMaEFz+Ar25tA2C/UU6R+mo5Luu8TqQJwca9EXvJem8zgfhOV5OfgEtIVkB+e0J41of/jH/84yl+ecQAXvwHusU0ithR+EiYG1A59UUGpjGoao8kbaupjpfm53NP2jw/l9Pf/OQBXie1fKmv+XpIFz7z8qc9Hsn4Yf8nibmMQQdB1CSZX2ZIJShckus0jznv7xOdBhFL4DtG/WKyoSXTfptvf8+tKl+O4v6WnnZ6X2ANevqa+kgNwuT0MGIKxG94RIQg/GdHORGw3iR8K6sIztXH/Vl3KmevTHi2F8hGpmHEVPybR32WzUDb38fC7ibwO424vTXlyXfoJ5yw2/NKXvhQBhchq9AGBIbjvupR/q2H8xA5u4z2i/eDvZdEq+jtRiyH8x+ecc04ca7BzIHRR2tMFF1wQnvSkJ0V/GpHLRZMGcJWOHT4W5cAslD9n73Dd9QgikAH+QFfzNBemDkkZ5R9u0yH5rshQ9EIW+eL/GRXANc40pc+5nUrdBhH9RAureHacPoy2vEt1BOYh0EkgB1F6pCT8FYDsOLquP4ze7e0z1+ZL/Nmpvo7twVwKIC3G0r333jvM7fkBIJfz/Nb3dX291A9TYjd5W6WfoA8g/yD0DOw0RSpHDqu8pXJW411Op6PNYhdA7i+IF3r/PGiC5DD3StvcJOpAeXIArlHKybtNNAl+Kk1vq6W6xaD+Rr1yPliuN40x3KtUOVA5UDlQOTB5DlQAV0cejwLgQnljwgQDxEFJGN0YkABKOJ/J5AqcyinwAJOzDOhywHOf+lBXXcOQZqKOCTsRjm74omd0XUfe0T0HcDFRC3gHBUTEylOUWj3P9ZNOOimcfvrpeiQ6g6WE6qLqIMUop7zybA70wPXSssgJ4pM0pOfkBjrX4QekOvIbpV0rRkq/Q1vd/bsPo7jLQI8FfvhfWn4upw5Hd177uzr3NuEALu6rHnrWJzR9VRD31d80scA1N9T43TZ5hOHHtmFNxrs717yOuTRxKOH40nclb8qnslFnOWC4R1unrpoI49q4DN7SsqQANspUaXgO+Gog3kZ+IcdE6aQX7RoZ4qR+MIwso60BpNQEAenR5nxCgnx0n/ZJu2abQ5cRKofkqn7njiqnG6OAs5CNGidp+ziFKJ+Xxft2Lu16bXo5AJhOW9XxzWgrfDMfowEQ4Yyn7YziPNhxxx37AGBqSX4ARSQvuYaMZCIFsJJIDq5Uzus+R7VJXevazprkvdLzNq60OfqkJr9zEfW4DpWMYegn9CdNcpAO/JduBu807nQFcJGG6sU5JD7lxreHnvj7f1bI49hTvpSB7+Vl5BqTMETpgrq0F9eXHOTkkT9IK20vzgN/z51XyDltrUwaTTSp8bMNjKF2rW+gsr3yla+MDkv9pp6uR3CdiTD6JPwXlb7nTj3SUpv3Sc9xALhK7QbVL3dsA3CV6vil7+VsA2//45h8o6+hP2s8T3ni+qcDuBifmXyCeIb+0oXIh/y8j/M+MkBjPemkW8B1kSdN+U+iL6YTQsrbZQjXqBvyS9G3uKZ+2jT+eP9xAFeJvCS/NpnB/SYqyc8jJ6f180kv+MSkLODOVJdVeVJeYucgo5xKZFTTGK10m/oY90vyc7nXBErJgT3Iz8cyfsMTyMdMt/1LZU1MtOFfm16v8b/aGH9nnusLqWxExiEP9P1y9jop4QdwMA96Ld/edds0gtGgdk264/6WpNlGw9oDXfpKDsAF+IkFgaK0n6DXuF9QAIQuPFMblx5DHl3K2dSnAaQKLKDy5o5dAVy8O6n2kiuXrqXjA9fdV+DnDuAq9U/Cs3322WeKnkC/cL0htZs+9rGP9SOvqdw6+vgyaQAXeZaMHT4WNckKHyPcbvE2qjrDD/woApQvTB2SMkkXSuWkyqujZCp9kEW3owK4SHdcaaoOTT5g1SF3nKQPI5cf10p0BAce02+QwQIlM5ZhCzC2QLI/S/XuSfizu+rr1A0ANosuRPq+qT5b4ocptZtSHZk+TFmxo6RLpN+lVM5qvMvpdK4/DDMPVNLm4P8k6kC6TQCu0nKSZhNNgp9K03US8i/RLbqMMZKV5NFljEEWVppeDrD9tvww+I4E6siVgnb+2te+Ns5dco5+jI8CHYNFqsiSEkLnIAI3cgJbhXQInAKgnh0AcsTiCoCfAPGZO0CfQ76hozO3RJlSIpDH1ltvnV5e4DcRjFM+sKiDgDLkCyYDHZL2ysLWU089Nf5eIKHeBXyDW265ZVhzzTVj/dBZmMu4/PLL4wIFzpuohC9NaY3zOmMH3+u5z31uDIgALwiiwhwA7SDlXZe8S9OczvdYGOg7uzXVi/aA33MQCQdAe1eE+Nw76CHofOxuQP+gjdN+WJigYDK59wZd03dizCZdeEk/4o9z/R6Uju7P6kVPKpMASmGGHksBXLzH1jUIjCYC/IDRj/CayeRoespJJANW1EJsQ4cDetlll42//R/1QqCySiAlVhaywmO11VbrK6QISBo1EZto9FAadhx+kp87u5Q2+TFo5AaOpjoAXiBNVoXjjE6pCcDFcyVl0QqDVBFL833Vq14VnUbpdQYenMLpIFfyHdrq7op7Go6cMvnKi2OPPbYP0BOAiwlYJm8Uxl71QOCdccYZcaDWNR2JwLbrrrv224Ou036oL3lCKYCLAWm77bbrO3aIznDQQQfp9bDbbrvFSF79Cw+fUBbaGls9MpCJfCUR28woyhv3Fbo4nXDSu274OpilKU0GUr61O6VIi/ZBFCK2WGQrRRHGJpMgHKFf/epXcdW77nP0FUv+bfSMO/88DG5JWTTRrrTrsZwD6o+kwLjAdxbhPN1rr73izyZHr94fVpY1yQ7kMbKeKBOMZWqjZ555ZmCFJ9QkV+PNhn8qZyoDUXboJw7Y8iR++9vfxlV0OIorzQwOMI7jZEdBzRERVVhBqEiLfNs22UUaTc4D7mFQIQ/VFrkmIi/Gn9SZoDFJjkY978dBY4g/6+dN8r6pjfu7ONVxpkCsLmecbaJhxzDSwXlIxDxWgzsx7hF5B4OecX4YAFcTn5rGN8+Xc2QNxk9OL0ZuodvQz0Vd2kvThAZpsPUquiQGjRPOaFaoa2x1J5W3P3QVtv/uQpMYP9vAGHI0575fWz/BWY1ugTGZUsl7OF2JZipQEHo4UUPgh/jrAC71jaZxqsnZSllL7Ya0nvrdBuDimRIdv/S9nG3g7d8n39AF0Akg1+Hihd4/nHToEFDaPkiTdkUUEMlRZALtgQUzcgr61lOlExHkDw9ZsKPtwrkmIl90CYCSTl3lib/j5+Pui77AhMhbOAThsRPjDhFMHTzM/UHjj39LB3Dx7rDyknfaZAb322iY/IgCsP3228fk0BUBW6VOTbc1BEDxySmiULz0pS/tt0OVjWeJ6C29Qdc5DiujmsZopdnUx3R/2Pxc7mFb5hyCTWAP8hzW9i+VUapf7tik1w+S3U3+ksXZxthqq63Cq1/96sjGNPoeFx3k2Kbnvf71rw9sVZejnJ0/qF0rnXF/S6WbOw5rD3TpKzkAF3m7HPGyIDPQjeAnW/FByCfGtS48Uxt3+7RLOdv6NDIO0JmAzIx7pE9ELvRwxhLsF4GkvT5N55NqL035cR0QPfq77BY9y8Idxj78BJADuPjd1P+51+YnZvEz9iV2ihP8w1465phjpixCgL/onh6BjfeYsKRMbNcMrx3ANejbkr+ia+Z0LexZxhAmVVLA8bBjh49FuT5PXZrsHepF3d3eozy0M9ncC1OHpOyyWfDhoBs0EYAt/PwQ/t+5D0dLyvGfejfpup7+uNJUHdD3AMgNS21tYhQfRls5htURJHNJM7fdY7p1sLZSpC/NBH+2A7jw1bNgTvap+ITcwd9w0UUX6VI8tunrw/phSu0mJmWxC7Ctc4S8JTAC8xdOJXJW413OHi+dB6JMw7Y51WPcdSDdNp9CaTlV3vQ4CX4qTddJlO+wukWXMcZ9YO4baxpjBChQmepx8hxwHwkLJ3L+PEqBTs48ZqqzqYSuC+lal2O6i0T6Tg7wiQ+I9kQbbCJfOKhnBJjR76YjQEeiP4rYQYb+kfp/dR8MBnO6yFOndIGI3+Mc/fOLX/xiDP6R3ivhS5rGJH7j60NfoT3kiHlu5tbT+ZPcs7pWmuZ0v+c6vsqeO3qk+Nx9rqG/gVMQ5XRS7rXZZ7QfsDHMSZWQ5G0FcA3gXimAC0MK5OYgApjCZMpMJzo9KyxoOKx6csALnREDiwkFADuAOpg06ArugE8IDW2/hDMMpxjEpIa2QnQegT7GqQ6Qi0aMkUeo9DZqq0Pbe4PulZRlUJrcR4kn2tN6660XAWYAkxhsm2jU79CU7jDXZfzwPTB0MQBwSuKIAZyHsdQ2QDDQ0pYwtvmuTOamEyK58qBY8g5GGXk76Irn6cc483DE4DxgkAclzvMLm+DR05/+9NiWQcWjwGBMiVB66HsoGcgL73t6ZlzHmVSWcdWppjOYA8gO2iCrFZiEQ5ZqxR1v03+QQxgBOE69f41TriLzcL4g23F+IAMAstHuXTEfXKP6xHRyALkKGIJIcshXvhkrDUoV1Lay44hD1vMHcJxxEWNx1GiAg8aQtjJNx73SMQxnKw5Mxla2q2JMZUKrlEblE+VAztDHmWzgu1166aVZY7i0jP4eDjom7nAMkhc6yCCdQgZf0wS8p+/nM2n8ZDIL/RG9HLlNnc8777yB/aT0PQGC0Pulyztvxn1eYjeUlqFUxy99r7Scw75H/+N7CyyO7i2HhG/xonRxJCPfc4td9EzTEV7QRlh5SVtkMpExYhRZ1JQX1yfdF9F7cOaglyNTcmCjtvJ1vTfd8nLS+TnwYr/99ouygokqouOge+I7YJxqo1IZ1ZZm273pzm9Y25+yj1vWjFOvp3zVxoAL7bTiiitGGYmtBeiEfkB/SIGR7akseHfc33LBHKZemS57gDaPvQhgh/EE29R9Oug+2JGMN9gJC5vQnVm0io4kPw7AJ/pGGk2qS1kn1V7a8kY2obvTRhm7ARFQ9kE0in+S9ssW69hA+BHpF21+RL45/j7sUXym9KFJ6RmD6j3dYwd2DpEtAAO5D2dQOUvvD6tDluazOL03KR/GIB6NW0fI5TdpvTuXZ3rNAVw/+MEP4qJ+5iCYi8AWZiwAICkZnL7f9rvED1NqNzGuPfOZz4yLX5Cf119/fRw7mCNgEjhHo8jZXHqjXitpcwujDiXlHJU343x/3LpFW9mme4xpK8uSeE/+SdW9DcAl8B/P4rthfhQAFd8QXQ5qW9wcH0j+sU05cglCDvE+OiD+Iwedsngb3zyETgKgXHniN0E3Y64Te8G34Aaciq9SxDa9yELywi5qIoDcmgcgTcD3Iq6TH+MTuisyBkojLfrWpNzHdkCPYvygDPhHIOwMFiy4XlnCl5jYNPxzEBx8ZK4ZPAffDNkBoVMTzS0FtDUVrzTN6X6PRTJEhIfcPkzrha6QLib1Z2ijtGG1He7JX+rPpdFwaUP4WMGrKNgBugcYiTZMh6fp5xXA5dxoOUdZG5ZQzhEk6ui8z3ZYrGRgxSrCV4RAbVuJoucWlyPC4o1vfGOsDiu8CduXkgtrIkg0Karpe/X3wudACuBa+CWqJagcqByoHKgcqByoHKgc6MYBLcBII+J0e7s+NW4OVLthPBzF4YQDjUgxRx555AKJvu51rwtEwoUOOeSQKQBcrbonIucRRxyxwLv1QuVAFw7kAFxd3qvPVA5UDlQOLEocIBoii++YdCIKkk/2UA8mu4ioBOUi3cQb9V/lwAziwCg65AyqRi3KYsyBHIBrYVW32k0Li/OLd75Vt1i8v29aOyIbsgAOEIlAUHqmCcC1dW/bQaKFQ4ByCIiixSAASViIp7TQT7sEXQEARaRY8A3MzaeRhNilQbslAU5hgQLk0QsBtHzqU5+aAqD1+5Rx3333je/xTzsfsdCXaI9dyCMfpzspsPABnAHgeojdMLSN7rvf/e7wpCc9KV4X+Df+6P0D6ASvBXhyH1kpX5T2JI8sgFB0WL4Z5fZFcm28aipXaZrT/R7lVxRdAGolUVPFA+eTruUAXIrGzDNpVC8H+QEMpG0PSxXA1ZFjJQAuQvaxpZsoXVmlrTJ0H0c6qwGWBELwKdwx4DV4ocZI/T3sHCutDjzwwCWBLYtNHSuAa7H5lLUilQOVA5UDlQOVA0sUB972trfFqA7ufFiiGDADK1vthvF8FJxmOJogtqjz7Up9O0kmnAEviohix7s47XDYEe2kUuVACQcqgKuEa/WdyoHKgUWNA759pW9JTD2YyAAMo+1k0u1fFrW61vIuGRwo1SGXDO7UWs4EDswUAFe1m2ZCa1g8y1B1i8XzuzbVCtAUkUVz1ATgcsBJzm/jWwXmti7M5eXRjH7605+GE044Ycpj+IgAgwEMc8AMkZ0ATkGAiXzeXwnwnvRhgrcoQqLmlU877bQAqGoQkbewA+Qj8JK/51HMzj333HD88cfH2wLfpCAyvcu2jPxBp556avjhD38Yz0v5El+e8D8HDXldlS2gQKKhQ+yeoHPdzx1L05zu9yg79aGOABRpYyXk3xcwpLZkTgFcRCfeY489YhYAFdmWMiUBEgHTffjDH26NCpa+y2/1nbqFYo47dq0EwOX7B5NUuhUFyGm2tBGdf/754dhjj9XPxf7IxACr0iAaMI2csLYMTkID00EI/TgdW7Es9gyfxgpqoCVMJ+EBK1UOVA5UDlQOVA5UDlQOLAoc2GWXXeJWLWyfWGnmcKDaDaN/i1e84hUBp50I+4qoINheWo2ITXb44YfHbU31HAA67Nof/ehHS8xiI9W9HsfLgQrgGi8/a2qVA5UDM5MDrOQnwpZ2Y2BSaP78+XHrdyb3RWeeeWY48cQT9bMeKwdmLAdKdcgZW6FasMWOAzMFwFXtpsWuac2YClXdYsZ8imkpyI477hi3YFNmRKWXzyYH4ELnJPoVR3w873//+/Vq/8hiPgDZUBNgqf/wwyeAoVh8gJ+IBQh33XVX+kj2N1GumOdv23XMg9tQrhtuuCGwRTNANAi/1G9+85ts+n7Rt09ki++vfe1rfjue4/MCFAfdcsstMbIW/Pzc5z4XrzVtK7nTTjuFF77whfEZwFuAuKBSvsSXJ/zPgXyAh3Jbawu4RlE+8pGPDPyupWlO93vUh2/Kt73gggvCMcccw6WhyCMls+iVcZ3+B6UALg9CdPrpp8fd9+KD9g+bUFuNfvvb3w4/+9nP7O7g0wrgGsyj+EQJgIsQbY997GP7ORx88MFTwtW9+MUvDjvssEP/fpOg6D+wmJ0gxBlM2HM6R4C3WI2GUK20aHGgArgWre9VS1s5UDlQOVA5UDlQOVA5MJM5UO2G8XwdtrDffPPN+5PKnipOOZxdF198sV+u55UDY+NABXCNjZU1ocqByoEZzoEtt9wyvPrVr+4vTk2Lm4tikD5Tf1cOzCQOVB1yJn2NWpaUAzMFwJWWq/6uHBgnB6puMU5uLlppfeADH+gDunIALgcxte1mpWhARLsi6tUgAhQ2e/bsPuiJiEMbbbRRWHPNNcONN94YLr/88rgdeJoOaRO1iB3JvvGNb6S3Y5qAvAjigh+K5zl6ZHgAXJtsskkMAENa1157bVxoeM4550xJz99p2prco/rfc8894UMf+lBMA75Rhuuvv36BADIsugDfoSj28IKdGqBSvsSXJ/xP8/JpZH3PVtsMco3n+U5tVJrmdL8HgJE8oZ/85Cfx26611loxGhzBg6688sq4zSG8yRHtjChaAMAAK7L1JttsNgG4HP/De+SR0jbbbBMAZELz5s0L3/nOd9JHWn9XAFcre/5+swTAlYY6FJJUqW6xxRbhda97nX5GNGTJPpj9BBbRkyc+8YkBXgjsBpDtkksuyaJDF9EqLnHFZuBkYGRgRTBWqhyoHKgcqByoHKgcqByoHKgcGJUD1W4YlYMhRpjbeuut4yKaZZddNjprLr300rjQCKdZpcqBSXEAe5/V49B5550XV+ROKq+abuVA5UDlwMLmABNCL3jBCwK6y8orrxwnhy677LI4+cTkUaXKgUWNA8yNVB1yUftqS0Z5mXR9+tOfHivLdu+5SdQlgxO1los7B6pusbh/4Xz9BgG4nvWsZwWiAUFErQL8lCPf2jCNJpQ+76AnQGGAWlZfffX0sRjNi0hH6Lhd6R3veEcEgvH8bbfd1o+6BdAFwEsbEezlsMMO62MH0LEBtUF/bNgNykG+beC1ZzzjGREwhu4O8Ic6p+lOki8xsxH/aTcsttwDgJSjN7/5zWGzzTaLt44++uhw4YUX5h7rXytNc7rfW2ONNeI2hf2CZ06wwb75zW9m60yfAKyFX/Sggw6K7UnXSCrtM47/Se8pa/gMv6GmCHHxZsO/CuBqYEx6uQTApXBtSgtEHnvBitg+kW0URV33HNXz9Vg5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQObD4cGAQgMt3+jr33HPD8ccfn638vvvuG1ZbbbV4j20ABQ7JPcyuWWyvlxJbNAKCIkq+CMALYJZB4FmiWu25555h7bXX1qvhP//zP8MVV1wRf3Nvww037N8jXQBe5OXbkAPCofzs4AUdeOCB/WheBNEhOpiTA9dI833ve5/f7p87IEcXHWDGtUnwRXmNegTDQiQoiG+h8zRdogRvtdVW8fL3v//9GJUqfUa/S9OkHSr/YcpS+t6Pf/zjGCTIAybxrdkulDZLPbS9PXXzdsdvtspky0zII2W1AbiE/2lrUwDCBO4i0A3AsGFIfRRAHgBe6kBUPP441++uac7qRVJaLJftlgC4tNermIeApMGIHPnJtVQY6Ll6rByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcWPw5MAjAtcMOOwRAXBBbx33ve9/LMuWjH/1oP4rWfvvtt8C2gf7SBhtsEPbaa6/+JUAwbId4wQUXxGsrrrhioFwcoRtuuCEAnmqil7zkJeFlL3tZBKHwDKCXk08+OZx66qn9Vxxglm6HSCTvPfbYox8Vi128DjnkkPguwBsAOBDgMt5lW8BHP/rR4XnPe94U8FdbBC74s+qqq/bLGBPs/SPwDoAfgEjj5ovyGMeRCGnUAfrrX/8amnZ722WXXcLzn//8+NwPfvCDcNppp8Xz3L/SNIk2VVKW0veogwPTrrvuuvD5z38+aLvElVZaKW6HqF3g2CJRAEWu0fYAQ910000BwJ+oDcBF+nPmzIkAsfe///16ZcqRSG7arrQpQtyUF5IfFcCVMKTpZwmAC4GlPVJJN90H00Mbcp9G9elPf5rTSpUDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA0sYBwYBuAApvepVr4pc+fnPfx7+67/+K8uh/fffPwBkgRQVKPtg7+JTnvKU8M53vrN/+2tf+1rcAq5/oXcCQOpf/uVf4qWmKERPfvKTw+677x5WWGGF/qvsUvbVr361H3lLN9i6kAhX3L/00kt1uX9cZ511wnvf+974G2DOBz/4wf49B6f1Lz58QtkgADpE7frwhz/88J38ged6gYrCzjvvHNiiF7rkkkvCEUccMTa+5HMe7SpbPhIVCmoD1PE9nva0p8XnjjvuuHDeeefF89y/0jR/9atfFZWl9D3qQHskuhsR2375y18uUB3qQpQ1bY0JXgfAFv0CICLgPu5zTdQG4ALLs+yyy0YwYlNUt0022SS89a1vjcmpDSntLscK4OrCpd4zJQAuwvg95jGP6efAnp+g7EQelo1rf/jDH8LBBx+s2/VYOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDmwBHFgEICLbQfZfhACAHPkkUdmudMFcKIXAVwBZoFSsJSe4XjAAQf0I1ylAWzYzg4glIgoXmeccUZg275S8vyIoEQkJdHLX/7y8NznPjeCagBhkR/Rs4455piwzz77RADXMLugge3453/+5/je/PnzY0SpcfBF5Z3E8T/+4z8G1vNd73pXAFgHpVsJ5spUmuZ0v5cre3qNqHJEUYO+853vREDf61//+vgbcN+f/vSneK5/AMIE+CLqG/S73/0ubjsJeBHQGNQEiCTSGRHPoDZwZXwg868CuDJMyV0qAXCBuvP9XAldSAhD0Zve9Kaw+eab62f43//93/D1r3+9/7ueVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDoyPA7NiUg/2HFu9VZi987ges/fvwd6Fhxdnji+zmlLlwCLKgdmzHgzP3vj+sN3mc8IyD9zLkuVw9wOzw7V3zAkXXn5fuPj3D4Y758/u9ZmHetQiWs1a7MqByoHKgcqByoHKgcqByoHKgcqByoEZy4FBAC6ARgSTgdoiL33hC18Is2fP7hSFirQEwLn99tsjkIlrKb373e8ObG8IeQCbt73tbeEf/uEf+o9fdtll4aijjop59y8WnHjQnBTA5cmxMxqRvCDfBlBRkNjOccstt4z3tT1i/JH8A5QmbIjyG4UvSfJj//nv//7vMQJV21aRvlXlxz72sT6fmgpTmuZ0v9dUfr/uoMLvfve7EZz42te+1h8ZeA7Ii77EtolEjYMOOuigcOWVVy7wrm/reOKJJ4YzzzxzgWfaLlQAVxt37J46qV0aePqa17wmIj71IMi8L33pS/oZ6Bzac5OLIP7mzZvXv19PKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqB0bnAICURy51f1jzUXeFDVa9I6z5mHvC8svPCvfd82C44eZZ4XfXLx8uv6Xn6Lt3Trj/wdmjZ1hTqBxYhDmw0doPhNc/59bw88uXDZdeu3RYatac8PiV7w8bP/6OsOHqc8L82Q+G0y6dE86+aOlwZ2/hcwVyLcIfuxa9cqByoHKgcqByoHKgcqByoHKgcmBGcmAQgItCH3jggWGppZaK28DxPAAep8022yy8+c1vjpd++9vfhi9/+ct+O3vOdnxEHyKSFWCVHDnGgWd4lihYYCMgysF2ib/5zW9yr/evUT7ANdDpp58eTjvttP49PxEoSGUi0hYRpQCmARLLvbfbbruFZz/72TEZQGQXXXRRALTznOc8J1777//+73D22Wd7Nv3zT33qUz2f0fJTtskr5Us/0QmesD3kGmusEXPIRdfie372s5+NUbruvPPOAJhrEJWmOZ3vsW3iv/7rv8Z6sQteU/v+p3/6p7DWWmvFKtOW2Dpx1113bWQBuCD6FXTLLbfEI9t7sk3pTjvtFNhlD6L90I5SokzkAcFreD4MVQBXR26VALhWXnnl8IlPfKKfA3utIkivueaasOmmm4a3vOUtU+4hWBE8lSoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgdG5wDxgR71iHvD0x97U3jZZjeFjZ+xSnjE2k8NYYV1Q5izcu/u3SHMv7a3XPWycM1FV4TTz18m/OSK1cJ18x8ZHqjRhUb/ADWFRZIDuz337nDd/y0VfvLrOQuUf81Hzw7bbTonPOcpd4W/3DM/fPcXS4ff/GHpcPe9NRrXAsyqFyoHKgcqByoHKgcqByoHKgcqByoHKgcKOdAFwOVbw11wwQVx20DP7oMf/GB4/OMfHy+xxSJbLQ6ivffeO6y//vrxsZNPPjmccsopU17xyFbaYpAHvLyHHnpo+H//7/9NeS/3w6OINW1zuM0224Qdd9wxvu6RxhQRKwdIAtjzmc98JoJwwF7AB0BlALoAdkF/+ctfAqCslJ7ylKeEd77znfGyp13KlzT9SfwGlKaIUgCOHJ9Cfq94xSvCi170oph11x3hStOc7vcE7gOH89GPfnTK9ppUmGBKgKgA/bVFKIvMefifbzeZbpMIsA+AH0R6bKlItDoR0bkEfGQrz/3220+3Oh8rgKsjq0oAXCTNR1lllVWm5MKesQgOJ1CBhBisVDlQObBoc+BVr3pVWGaZZSLim0FwYRJA0Y022igW4Rvf+MbCLErNewAHtttuu9hurrjiivDrX/86Po3i+rznPS+e/+IXvwjXXtubWJwQPetZzwprrrlmQEH2rX5z5ZpQEfrJzpkzJ4awXWeddcKqq64a9yonNOlZZ50V7rnnnv5z9WTx4QAKrbaUxiCkDbzkJS+JFTz33HPDX//6135lt91227DccsvF8L65VTX9B+1E7ftvf/vbIhXpdCbK8B122CGuHGG11oUXXmhcHv50JtWPPemf8IQnBNpI6pAYvmZL9hs4hFiBhCMHo/jzn/98nyGsqKOvM77Rr4855pj+vcXhZHGsX04PGKSfIL/XW2+9sNJKKwVk+LAhwtO2QDj+jTfeOF4+77zzwp///Of0kdbfvU0Rw1or3B7+cbM/hee9pLfy7ak9h99KjDHr9f5W6v1hl7M6lZVw1/WwXOeFcO3/hD+fcWE4/MzVwwXXPSbcff9w0bi8zOecc0647rpeuh2Ilao4lW6++eao93R4pfER/Bc4OOmHv/zlL6NTsvHheqNyIMOBVz/z7nDeH5cJV92QufnwpXUft3R43bMfDBvPvT38/JoQjv3xI8Ktt7GtYvM7fge7GTkDnX/++XGx40tf+tKw7LLLBreL/J16XjmQ4wB6hyYjsGcnaTuT/6CxMFfGRfkaY8nLX/7yaKddfvnl4eKLL16Uq7PYlb3aMsN/0g022CA8/elPjy+O0186qb7SZmPlaj+p+uXyWpSuPe5xjwtbbLFFLPLPfvazcOONNy4SxceuQgZDZ5xxxhQf2aAK5Oy5Qe/U+8NxYEnTCYbjTn16HBxwQBSAHEUC8rTXXXfdsM8++/Qv/ehHP4q+GHAIb3rTmwJgJAigFSAWQC4iAaDuu+++CL7SdXwKimrE8/hLTz311AhWWXvttSO4CXsO8ihWAtIwj4NPaBCxlR0AGCJDKT38PocffnisK9G10PO333776N8gPY8uBU8IpgOR3/HHHx/PV1hhhQigET7jpz/9aTjhhBPiPaIqffrTn44Rxrjwhz/8IRx33HHRF0N+W2+9dZS7ir6EffG9730vvlvKl/jyhP9RdvivchP5jDoz54evjnkViO95wAEHxC03VaT9998/+vH4jS+XQERQaZrT/R7tn34A3XXXXTEKF7gbaMMNNwxvfetb+7gcbwvxgYZ/bQAuXgEohh0KMadAG8KPwfahRLzTd6CNM7c5LFUAV0eOlQK4mIBmH1h9qFx2CDKEEx+4UuXAks4BJv2Y1F1ttdWi45bBm/2K2UP22GOPjfvSduUR4BeFwuz6zsEHHzxl31+MO8IoEnqSEJOUBTAJDuYcQEsKDyE7DzvssK7ZTuQ5EOJSzlKE8EQy7CU6d+7cOKjfdNNNWWVyUvku6ukKwMsAjwIK0X4BBEK+xS6KrFZMoMzefXcvasSIpHCepEV4U1GuXLo3iSPtlT3Spax7HhgRhCdlEnJRIhwtGPMoxrm9sBeluqis426DrJ5hkhliVQwALRmd6QofydjUqFTZckcMMiYCWQXxz//8z7lHJn6tRDYuDBk+iBHif7oteNt7TXWfSfWTsc8YT/jvmUjj7neTqCMTBXvuuWffoUIe0j+8n3MdI/TjH/84p4sFLa71y+kBTfoJH9Idi/xmgpdw+aMQegHOD+gHP/hBNiR+U/qAt9Zf+dbw3hdcFdbbftMQ1npTCEv10iKq1oP3P/wa5z3n5SxAXL3z2QC6bgrh1h+GB875r/Cf310+/Piqx4V7HliqKZsFrnuZ3ZG5wIPJBY1XvnqVR7BHGBtZ9ZmCEpru+RYJOBodoJ9kW38uIRxoGo+bqv+Sje8LV/5tTvjD33H02UfnzJkdXrrZI8JuW8wPv7317vCNM5cOf/wz23dkH59yERtbtgdbZZx00knhC1/4QrQnr7rqqoDe0YWGrVuXNOszM5MDTTLPxybAGJO2GT0/t9VnJtdGLxWTf0wGQURRIJqCqPY/cWLqcTr5sijYMuJOUx/W/ek6MpmtRWSyV8a9lFNDAABAAElEQVSRd1tfKU2/zcZqSnNS9WvKj+sz5du2ldFl93SMFW1lGeYethD2BfStb30rsECkK+Xsua7v1ue6ccDb1ZKgE3TjSn1qnBxwP0sTgIv8AHpqQXQuf+YmAEWxBZyT5IS2JfR7u+yySwAo7kQ6AJZFLNAE3wA9+tGPjpGIdK/LEXuQOaknP/nJERTmaTM3DBDIKfUzsYAODIaIejBvwFyu0gL09slPfnLKDmjed/VuWjeug9vgXadh+eLvTvo8x8c0z3nz5sX5Rr+u7SK5xk5yV199df92aZrT+R4+MwIq+bwi3xNSO+AcMBv9iLY1iAYBuIjChW/7kY98ZGNSLOjE11dCFcDVkWulAC6SZ6L9ve99bx/N6VkyUYQRnEPN+nP1vHJgSeDA7rvvHp72tKc1VhWw4xe/+MUpg0fjw70bPnnS9pzfA8gC+AhiNRZGrwt4f5boL0wkOWlye0kEcAFS0WRs1xCczrsl+VyKchcA18te9rLAH9Q15O0g3s4EABftB3CN+hvKNgoVihAKN4TSxaSOK5CD6raw72tvbcr+vve9b2EXZyz5j7sNsoqGEL4QbYBvzgoGKHV+SMYuSgCuUtk4kwBO8WP0/on/XQFcbXWfSfVbFCY9xt3v9E3HeXS9iz4K2IQQ0hDODtlTrIRCjh9yyCHjzH6hprW41m8Y/YToiR5xDTuXyYXvf//7I30bb1fDALhwJ679qNvDh5//h7DeK57d+/HaEJZ6VAj39yJtzeqBsR5cpvc3p/cUzsCe4wYA16xepM8He8D4Wb3rs3sOmFvOD+HnR4XPfnulcOaf1wj3d9xO0cs8DgCXVrDmwq433asArpGa3WL3ctt43FTZp63zYA8CuVS46MrBjk3SeOoTHwz77nhfuH2pWeErp8wOv758ds9x3pT6Q9dZFf1v//Zv8QfgLUBcyBHkSQoSaUqppG5NadXrM58DTTLPJ2CmY1Le80vtlZnPxeFL2ARKqf0vz8vp5suiYMuIU019WPen6zgpgFNTXxmlXq5XpjZWU7qTql9TflyfKd+2rYwuu6djrGgryzD3KoBrGG5N/7PerpYEnWD6OVxzZD6BiFcQY34bloBgAFtttVV/fkXcY/wgCn5u60T5fZp87exsQYCLXJAaj3hFXi94wQvCK1/5SmXb6SgAFw8j78gLOzEl5lb+53/+J0YWS+9tueWWMRACdmRKBEAgYEIuCALzv294wxuydSM/osADnGWeKqVh+JK+O+nf8NEjQCk/6gR4S5HIdJ0jdrn4zgL7dPFgSZqkO53vEdGeeiv4Bfk70f6//vWvdwJv8d4gABfPEP3tQx/6UFx0yW8nIoCBZ8i1H3+u6bwCuJo4k1zXhENyeaifoE9prEQWQmiAFNUHGCqh+nDlwGLIgTe+8Y3hGc94RqwZA8n1118fwzTS90DqCtQxTMSG17/+9eGZz3zmQG4pbR4k8gYTTjhcHEzCNSJv0Y/pw3onXdGuye0K4PrfOBgOZH59IHJAirIDuAi/qVUTZ555Zj9sqU/if+1rX+tvuTgKK2cCgAtlnVX4UBrG1J1PhDjXRM8odZ6udxd3ANc42iBGFsYZBL+IlqU9xI8++ugpW/VJxjYZlbnviuGITEf3IqLXdFOpA38mAZzEM/G/ArjEkek7TkL2j7v02joePQ7jlW3jRWo7AHMF6tK9xeG4uNZvGP3kqU99athjjz3i58QpAsh8HOSTVsMAuB619L3hfU/7bdhq59VD2KA3xiy9bA/ANT880NsO8cG7Z/fwWcv2cFw9ENf9PajXUqzKuz88MB9w171h9iN6vyNmpQf4uvaccPtPvh8+cNKG4crbH9UDtAwmL/MwAC70PvQ/Vnj6eNU2IdZ0rwK4Bn+nJemJEl1kjVVmhSc+dnY47/f3xyB1Xfi14dx7w97bzA4rr/JA+OpPZoWfX7xUuPe+v6/OTtPApkZ+QkTa/fnPf97fOoNt5dEzB1FJ3QalWe/PXA40yTyfPJ2OSfkmW33mcm60ktFXX/e618XJLbZSJyI9VPtfnq/TzZcK4Mp/h7ar7mMaZwSupr7SVpZB99psrKZ3J1W/pvy43iSf296Z7ntrrbVWf/soQAA33NCyT/R0F64lP+YTsS+gGoGrhVEL6daSphMsJDbXbIfgABGrNtpoo4hFIMoQkY1/8YtfxIXxQyQz5VHAW+ycwlZ0nLPlIL5hFm5OgoiqxXZ4q666agQSsYU3C0HboiYRAAA/yBOf+MS4FSDzy9iUzOm2Ee+tt956YZ3ebmrM+xLcg7xYUMSccBtNN1/aypK7x7jHnDuL5YmUhr096jcrTXM63yMq6Prrrx/WXHPNGCCCOVf+ciC+HN9KrrFVJ7wGPMb8JfZSCoIbNl3hh9hVhraGnkn/5o9z/e6a7qzePtJdfJpd05sxz40DwDVjKlMLUjkwwziA8GGlLQKHST/Q4BdeeGG/lGxBxmQfz0Fy7vYfGOGEqC9Ef4GIEPDjH/84nhN2E0UBYr9fjyjgEyKgZ9///vfH5/inCbwK4KoArn6j6HCSmyBtem0Sk/gzAcClvsMqEhyQTijShOJFRqR9zp+biecVwDX4q7gzSs5T9Ykvf/nLgUkCkdrJMAAuvbuwjqUO/Argmr4vtihMekxC9o+bwxpLAG598IMfnJK8+jSOEyIpLm60uNZP9XKAedO3I4ou0XShH/3oR/Gv6dlhrjsYqiuAqwfRCs9d/frw8Z2vDmHzbcODK/YA4sveGx68Z6lw9yW9qEB/fUSYvdodYZlNHxXmLL9c7+n7w91/uDncd8kDYdYyc8IjNr0/zFm9BzqZf18vKFfP/rh8Xjj9uzeE/+/iDcLdPQDYIPIyDwPgakq3bUKs6Z7bK+mCk6Z86vXFlwMlusicXtPfaK3Z4Y/X9XYUvbNbFC44+MQ1HwzvfenssNbqd4Yvn7pU+OmvHhHuW3DBcp/Z0u0UWVhbN6SrufsvJCcldUuSqD8XIQ40ybzpBnAtQiybaFFr/8uzd7r5sijYMuJUUx/W/ek6LgyAU2nd2myspjQXRv1myrdt4smifN19ZhXAtSh/yVr2yoHKgcqByoHKge4cqACujryqAK6OjKqPVQ4UcMAnfJomiHbYYYfw4he/OKY+rlX9oGHZQxpQyO9///v+Vj78ZnKRI6huokgAFnBiW1RQ2ZCczZzLAQ2A69vf/nbYfvvtw9y5cyPK+eabb45IX0JUloZNJA8mcTfeeOMYDQxQGwhc8jvxxBOnlDOd/Ad5DwoYRDlAOaLR4BjPhU4lH+QeADfqCUobXoA6h1dEBXAEOqsxQRmDbIa4BzL+oosumgLGizfrvwU4kJsgBSGuPcaZCKUNvva1r41RqggHCoGcZ8XYD3/4wymIblDY9Bei1/Esk+l8N5Df9LGU5BAChU4kLFGuXM9+9rPjKg7Q+t/85jdj+8ZhTl6Ar375y1/GdkWeOC6JJkEbIqoSqHO212RLJSdQ8R/5yEfipXQfcz2nMvIb0OQofUhpTvK4wQYbhOc85zlxdYr2wWbVB3y54IILQg/wHrP/2c9+Fld0eFn+8R//MW4dw7MAWp34ntpuEMCpryIZ9rsr3a7vEUa3axtU2l2OrKjZZ599olzSNpOSpYBnAdGKdH0YABdtkLZIX2FrHghQICGSkfN8F60iVz4ct9122/CEJzwhRoRMtwAjPO5rXvOa2B+Rj2wJx8oe0kGuikaRjdMpw1Vejsj+5z73ubHtcg7/6bf0TY2NXSJwDar7qPWjrIxpm266aT80Miuk+J6AwBnnupJPegAY32mnnaJMI+omxhLjJWMs43gTlZQF2YksoD3RD5Gh5PXd7363P8aOq9+h86CTcFxuueUC21IT5YeVx7lxgXpus802/ejBhCEnehYyB0AKbV60+eabR35xZNsQeE8/YJwgH+onmcd76Aakw8ovpxIe8n7pe563n1OHnXfeOY5vtAEIXqFrpVtno5N1rZ/nwfm4x1P6HN8WfY2Q4Lk+oLGJ/HPjD9edcnpAqp+g8xGmn5WRWvxw3XXXhSuvvDLKRFZmirqON3qeo4OhugK4ll3qvvAvm18cNn/JquHe5TYJD965TJj1qLvDPX+aH2Zf8cyw1L2PDvOvPivM2v7OsOL2a4X5V9wcbj/8xrDc3ZuFpR/7uHDvaheFpTb7vzDrrkeFnjUQ5ix3fXjgovPDHt97arjmjuW9eNlzL/MwAC6NV3/5y19iNNKtt946rjpl0kZ2ySWXXBL1KfoX42fuHnphFwDXMH1n3O01yzi7OMn8hunjFGmSZSH9ku/AqmDGJVZAI19ZZYxdyJiNLsK4LRo0Huu53HH1lXurO3s3/npLdwAX6aw7d+nwrhfPCms+5sbwHyc9Mpx3GWNDPhIXugWygW11sVc0Jp/Zi0BMHduoa92G/eZtefq9rt8O3V32G2NJagvx7YgeTj9Hx0XWMe4iEyB4gV2AfoYfAN0TOcuWHmlaPA+YljSxOVgIgc47t+eTYCUw24c4da2Dv9NFf/Hnu+rN/k563iYPkXkpgAt9iii/+CcYG+HXpZde2l+wl6bPb3iE/cbqcL2DzEWncb2HZ9OxUFvZbLfddtE+IP9TTjkl4Mdia5ZlllkmRnvn3S7UtSxs3UKbgCQX0vRf/vKXxzGatqU2oWe66of0UexACD2OdtW1//HOMO1sSZa58AoaxhfH85Kb6GQltsywMhL/zgtf+EKyjoB99JaUABCRrmztQX04fT/3G7mGXMT+W3HFFaPMRB/CL4bcHCbKQQpwQobQV4mkg7xFZvzkJz+ZspiMMuG7wqaCAMgjq+kLyHhkBX6DtK/Eh+1f1/7dZmORdxuV1m9hfVvq0pUvPIveQ1vAL4hfknGGNkn5kd/4fpBTp556Ko/3yWX3GWecEb9z/2bvhO+IzEQ20sZoC6SF3YT8FHHv1a9+dfyJHyj1F+k5lQm7jMXo7kenvbHVGL50+sr8+fOjPwCf1W9+8xslEY8pgAu/BO/ir0I+U0Z0v7POOmvKe/zI2XP+0DDy2d9rO++aZukYhu0Jz9Cp8E2hCzO3xPeDN1xja+6cPUy5h7VJB43t3q7w30snEI+GzQ9/FD5Mtt5CDiGfaAPIH/z2+FLQDfF/5ajr2O7vEjyBAAf0Q/oTvh/6EFGb0B8rVQ5UDlQOVA5MPwcqgKsjzzGeKlUOVA5MhgMowi996Utj4gAScobPJptsEt761rfGZ9KIWCWlQnn+5Cc/GZ2fKL9snYhDC8IRvueee8ZzHG+sIkrJ93MmnOahhx4aHxG4gIk+FF4cDClhlOGgdlBC+kzuN5O4gBtQ4nOEYcnWcrfeemu87ZPjGH8AuHLERHFq5AHG4X2cyDnCCDrssMP6zowDDzwwW1cm7w466KBcEvWacSBnULsT+jvf+U4ExAEmzBF9RtHjMFgBGGrP6vR5jHomdp0EjuoC4PJ2BVgLgzslnGhsebL33ntn2wWOEkADIoxQJhYgwIE4151witEPaY9pGf25mXS+4447RvBDrkyAO+XscfAoz7KnvUBM/KZvuVGOo4K2AX3lK1/p86rku5PGMO/hlOjSBkl3GKIMyGCPrqbJPJzR7vyQjB0GwPXpT386AggBu7I1LoSTjK0IIOTj4YcfHs/9H85vwBsAULzeTOwA3mqSj4wJRA4DRDGKbPS+NmkZrnrjCEV+0OdSwjFOX6XeXQBcg+o+Sv0A4PH+3J5zKUc4mAD/pZNtuWe5RjtjzOZbI2Nyej/fE6crDiynkrKgG9DmBQ7y9DinfR9xxBFxjB1Hv8NxjExqarPUCce3CKf3e97znv62trquI32Vlb+MAZCD2vWMjvCTScuUMEI//vGPx8slPOTF0vfSsvhv+sC73vWuIOCt3+McOUL/lg5HdMhB9UvT0G/vA+MYT2lTyFPouOOOixP7yktH367YFyDofnrsop+gVzRtbeyg7GHGGy+Hg6G6ALjQXNdb4f/C57e9JDxijbXDnVduER657GbhgSvPDXdf/oewwopPCkvdc1+4569/DvNfeGdY4UNrhFtP/1u4/z/vCasss2aY85hHhztuvSbc/5hZYZmnbRfum3V3mLXCqWG5ZX8bDjt5pfDdPz6xB+nK68cqt5d5GACXxitshY9+9KMRVK/tpZU2RwEwm+4xZrUBuEr6zrjbq9cndz6p/Ibt45RtUmUZ5Tsgh5lMxCbNkfeVQeNx7n1dm91r6sssPSvMv6c7KFrvPmnd1cK7X3xjr//9LRz0vZXCb//Yi42XSYbxGr0DeYquQdtnAh1QTjrxqrR17FK3km+u9JuOw347JpmZEIawo4ky5pNhb37zm2Of5T7jDOMjoBxkCYQs5XduHAeEjS9Cvgyel+xmoRbAXY1TTBwzVkDD1oF3htFfeB4aRm9+6I38fx+//Anp6W47M+FIvXPEZC9+jJTYcr2pP5EHwDeN/bzr+WGrz5s3LyYpu5oJZCY/se1EijSs303HYcrC5D2Lm9Q2sDORDSIAAiwOgmh7jNf4qqBh9EMmyuUXk/+rS/8raWdLsszluwzri+OdUWyZEhmJ/xY/LtS0ZansdoAG2PaD+nBMrOUfujn9izbVRAAdBgGb9K4DnPDJwPccAco6/vjj+7dcvwKoAS/U//BjoV+nfaX/cu9kmP7dZmMNkiel9VsY3xb+DMMXnncZzHdg8R1+/pTSReL+Xtp2B41XbEH2pS99KfoYyAv9hSOyFd0l53uQnOQZ+gDjCUS0akA2TZTOeziAiz4F0DhHzGF88YtfnFIW6QQpL0rkcy5PvzZsmqVjmOQLfGK7s5xcwN/AfITrW5S1xCYdNLZ7u3KdoDQ/2YSAA2lXue+Nf+rggw9eYDHeMGO7vh1yDZmRm8Oi7aJbnH322Xq8HisHKgcqByoHpokDFcDVkdG5iZyOr9bHKgcqBwZwgJWkijR09NFHx+gT6Su77LJL/xlWvaRRadLnB/1mVSqrM6DUaGO1FIYbxKpJACUpseoVxyvEyiyUa0hGRPzR+4dzHUc0Bj0T31KGAbjsu+++eqzTUQYDDytdJniYwMahBuGMA5gGueMrXuj9A9yFMxGDhTqIcPgIJIGDlzRkAGEwYAQy0Uwd5AAGMKYtkoiewwokoi9ATNiSHs7mHCBP+dbjQxzIGdSpAcjq37322isCs/Tt+J4Yc/AYXnOdbyfHBfdof4xhAP/kWEqBdWpbKTgqV65cu2K1L+0Bw1l56NtiVNIuWc2jNsU9gDRMUHQhJtPlUEsBT13eXxjPsOKMyRr4LjAMsgJiAgDnObzSJK3K6LKOaz/96U8DUftEmqCHr2zPCJV+92HfA/Q0qA2qnMMcAYsccMABU4BSOD2Ra0RJ9AkpydiFBeCiL9Ff+HY4MmjbAOwATBLdUH0P5waT9qPIxlxfm5QM53sh45kwVB9mnMHZRfQ85LtTFwDXoLqX1o9yCFzHOeVk1TfjKzJI/Q35x3NykvJsE2nSQ/f5tqxiZawGIKFxj+vIWGSeqKQsb3/72/vREpC7OGHJiwkUgbqoF45g+uko/c6jnFJm5C6OTvozE+QiQIxa6ev8QLbTxiknDlbZRPDic5/7XBxjiC6BUxndQk5sdAfkFG3W8+J7oIcwNh111FEx+xIe8mLpezHTzL9UJsIr6s6EEQ52jWHwgm9D/fg2g+qXySpeyvWBUcZTX1zAal2N4cqfviEwdFcZqjTc4Z/qJ0SAAXjssgInA20aYBqryFPedtVPKDtRaQQW7wKG6knnsN3j/xLe+eIeQOu6x4YHf/eisPKqq4UHLulF5rvx2jB7Tm9bxN7WiPfceUe4c4elwgofWyPcfPot4d7P3Boee/cKYc6jlg4PsE3i8quEpf5h03Df8muG2+4+Kay4+WXh/ItC2P+CzcL9DZGExOtxAbgAbT/lKU/p61e0OWQTfYhj0z1Ahj7BmG6hWNJ3xt1exaum4yTyS9thlz5O+SZRFtId13egHthd2GEOPmW8Qk8ZNB5TllKa3dOHVlx+6Z6udn+4/a4F90nc4Mkrhw9tf3W4+d454cBvrxD+cv2CzwgIiw6C7ACMwniYAlFyZRxUt9JvnsvLr5V8OwHTSMdtdiIXvvvd747J+9jqk7XKG9mNzgPo3HWzNEK5ZLfe05ExBj0WKqnDMPoL4/2werPKmTu2yUNkno9Nep/xmog86Om+CC5dwMZComc+85nxNb4BOgw2GnoSYxvEdSbFGQ8hz88na2VXx4fsn9tudnmB05KyuP2I/si3BayH/orPSroxC6hYSAUNqx/mAFyD+h/5lLSzJVnmlvji4LPr7vymvXaxZUplZAnIZ1Afptxt5IAX2jd9EX6x2MUXL37mM5/pFInLAU7KF7sF+cx46jLW9U/Xr/QeR3hOdDr8y00ArmH7d5uNxdjZRqX1Wxjfdli+UG+XweIDdis+N8Ye/I8iX/Dq7/lcgC8a5z38IIy32LeMBfLzuC/V/ZQOnFe+Po47IMt3GEFmE62YMRN7E7tS/hhA7IDZIU9L6TPGUUbq6/5eQMyMiyLpBG7Pca9EPivNpmNJmiVjmPyCKofGbr4T/Vc8BEhNmUSpzOtqkw4a271duU5Qmp8AXCo3R9ok7YT2qLklfvti02HHdtIlouFb3vIWTiNhU6AvpnIQoCR/lSoHKgcqByoHpo8DFcDVkdearOj4eH2scqByYIwcQGkE7CSDiQk/DONSYjUmE26QO0+V3jve8Y5+tCo36HRfRxlBbhC4EUHarDRTiGQcrThuNRGcM/CUdnoE5EbEFwgFHQMEQwNiQhEwiCYW5fx2xxfGDFGXnG+AP7SKww1DXyGcRuUgP4wDnCQQRowmsjEYyRvKRXmKN+q/LAfUltygbjIA2a6JP4goV4SkF/k3Tx35gO8Aw8iBi0EoQJGMURwArAoT5crledCumPRXxKzUqcAEI44ltVV3IrX1LeXPkch7OFMg2v7+++/f71Px4gz/p34GrzyyloBYFB/5xkQ35JM7/HaAKL8lYzw6oH+TYb576XttbZAylhAAPdqJoo0xccckJIA9J9W/K/iAd+X8YIJz1AhcXne2fyMikwjHMZOlOIv4ngLplspG/z7TIcMZFxWtgO/AakX4DLF9wxvf+Ma+I6wLgIv32upeWr+tttqqH8GO8YdVrRpnmVyiDwkE1dXJ5JMeOH4BWaotonvQjwEvQQ6MKS2LAIo4bNk+ljxFHoXDJ6+97aWyX+/mjoAjAUlCaZRTdx7jOGYFJ9vW7LbbbvF5DEUm/iTDuehy3CeCuaexxNs/1yGNJ2nbKeVh6XsPlSb/f4899ghPfepT483U8Y2TFLmttgVoiShXoqb66X7umPaBUcdTB2gxUQwIAtkhYrEEDnIIUDhR3gaR6tVFP3GHsU90kYfXdZhxine9nXrkSe7lqLcOPrxpvSvCq55zQ7jjuhXDXSevER533zKht+l4L3xML5xQD3z14G13h/+746Yw/x/nhMe+53HhpnNvCzd//Prw+BtXDsut3pvwWboH4JrV4939c8Jds5YPt/5DDzCx1U3hit/dG97/82eF+x7opdNC4wJwKQvJDCYWFT1y0D2fYHQAV2nf8W84HfrfJPIr7eOTKMu4vsNpp53Wn9yjTbge6TK/bTxWW2o7zuqNhb1OER58YCoA66lzVwwvfvpq4cLfXR9+9pvbskm8eItHhvdt95fw/YufGI486a5w191T05DOxzhE28L2ZwKTcdjHn2zivYttdSv95k15cb302wEGoP9q0o1FYnw/xlmBhPiNjwBK7SqfNOY+4xX1g+CbR1aS7OYeugbjFfa5xoTSOkgWddVfXHfpqjdT5jZSGVJ56LYz76d+DC+LT6TTfrAP0N/Re9F/pQOSDtGriGIFKZIQ556fT9ZKF+IZ+A1gCn0UO3YQlZaFdF2XJUoMNpODC7zOPD+sfpgDcJFOW/8rbWdLsswt9cX59x/GlimVkSUgH9oL1NSHH7qb/4/8pG9BgBiQm5JnXPPo513tP7dpSAO7Hjkl8jTxk2GzkafrVzyLPxj9lD4GtfWVEllDmpIrORuL+zkqrd90f9tSuecymPpjY2LbyK72yPXuO/P33CdJmwJsAznYld/oJehXGr+/+tWvxgW0vnsHgEnScHIZzDiM/QixCAr/PWMpvn35MriHj14LNX3MSXUCxnQWwKsfoBMwlyHgkgMZpRO4PVcqnyljE42SpsuwLmOY/IKUhf6JnS7fFd8Rv7F83yyIZWEs5OPLMDap+iBp5MZ2b1euE5TmJx8m+VEvIvtpsR1+R/RJZA3k33rYsZ33/Z3Uhsc3xBaxtKsULMa7lSoHKgcqByoHJsuBCuDqyN8K4OrIqPpY5cCYOcCK+1133bVvKLG6BGNnFAJkhJEIofRjHDh5mGq2IyBEfI60xZcrsTIiUOgxQHAsOvnKBlZ2oih3IRyyWlXGxDKGl5OHm5ax6YYCwC2Ab04OCnPA1c477xwd09QBowfQgxP3t95663iJyV6c7VCb8y4+UP81ciBnUDcZgO589kl8VpjxvSAGd4HpPFP/5t4mZIwOC+C68MILo9PA81Af4JqDxPi97rrrxigAnKeh6LmWkkfDw2gFDIZjZFGiJgCXOwTdSPYQ6xjJOHWYhIeItEG/hgQALf3upe+Rd1Mb5N6kSe2L9gAgsQvJ+TEOAJdP4uScw8hivhNOOOQjVCobp1OGA/DB4ag2B1CXtufkkStTEI4/5+dtdS+tHw4qwH1N46wiulEOB1h7udJzdxi6I1fPkR/tCP54BIXSsqgdoz/gbJSjmfwAe2sbZ7YBkaO3pN8RTYIVzVDOqQw4jUkUnNGS/4AQZfe4MzAm8vA/d3BTfjmdNZbkJhc0zqVtp5SHpe95PfwcByj6Jd+Y76EIo/4M7YB8IfFL95vqp/u5o/eBcY2nbKcyd+7cmB1OfdIVOQCfscYnqPVMelS93OHfpJ80AbhGGW8oj+su6AFEb2uj2T3g1VuedHnY4RnX9lY5rBSuPemBsMx5S4fHPGKZsOwqvYmZ5R4Z7vvbHeGGR1wflvqn5cNq2z4m3PHn+eGqT10bHnv20mHVtdYMs5aZE8Itd4S7bpsfblv9rjDnlQ+G5de+J1xx6b3hn85+di8CF2CWZprJAK7SvjOJ9trMwakTLOPoH6P08UnUfRzfwbcoFS+33HLLaDvz2+27tvFY7zYdkYtPWH2FsMm6K4Q/XdsD298wP9x6+11h3ccvHd72mi3DZk95Qjjyv+aF/z7j6mwSs+b0tj1+xQPhhZveEz7/3ceFc399Q3/CMfvCkBeb6jbKN28rQum3I02XZ+gxjIkCz6fjtE/Wovcy3jL2OPkEvUftlezm2ZzvoLQOw+ovJXqz1y933gT+8LEpp/+xiE4Raxz85RPsxx57bDj//PMXyFaTm3wzQBx8B8/PJ2ulC5HImWeeGU488cQF0mu6UFoW0gNgQBuhv0LUg4jQEO0HkAD6GVSiH9Kf4D2kLRQ5b+p/3CttZ0uyzC31xZXYMqPIyOkG+RCpUItwWeCSbmnPQhv5BlKwKG0xRy4/AUawkCIl/GvosRBylYU8DuCibwHscVuuqa+M0r8lV3I2Vlpm/S6t33R/21K+uAzO+T8BSGnuwGW+vye7n8Vq8AtKwa7i5xZbbBGBLPx2u0hgLB8f9I58e/hVaJ88AzEWYX+zoOaoxE/Pfb0HWJG5AMh1AvdrxZsP/3N/DVvp0lcg6QRe7lL5/HBW2cMoaQ4zhpG59BHOc3qO80uLYkexSdUHyS83tnu7kk4wSn7yYZJfTjfBXwSAEALcBRitZGz3CN45u4L0faEn7dUDA3C/UuVA5UDlQOXA5DhQAVwdeauJjI6P18cqByoHRuQAyjuRdxTpieRwxGHoaKIQA2r99ddfICeMWgAROVpnnXUCAC1I0SbS5zCsFGkDYwzQWI5kVOUAXG1AMzkdtXUaE4JE11KEMc/rnHPOiQ4yGSc+cezPYZxidEI4H+CVO74OO+ywcNlll/krcTtEHJBQGmliyoO9Hzg8CecMMIFVgVp5VAFcKafKfucM6pwBSOpNk/g4aImQA7mxHi88/M8d14RFxtEHyRhtmpB2Q9/bVQ7gKAeGA49iJr1/hB/HsQzRHmmXTeSryknr85///MBJ26a0Fub1JgCXO7vV/4hAILAWTklkHKSJdt9GSqCJ0u9e+h7laWqD3Js0SRYuLACXT4xSVwAQTJjhxHDHrfPBv7VPpvozuXPva5OW4e7gyk2UUz5f+ZyCcHLl51pb3UvqxzgJeBpqA0EL9IyT1CPfxRcz/zTpkZNbelxbOvGbSIV879KyOAAKucsk27x581q3+yjpdw6C1baeqo+OAGsJw49jm2iK6mPu6NazOrLVExMpkFYgc66xJDe5oHHO207p9yx9jzI2kQOM2/opeihjadq2cvVrykvXvQ+Mazz1hQI+dnt0rtz3UZnSo+rlaTXpJ00ArlHGG8rj+gB9LwUwpGUGwPWGuZeHl2/257DSE1YN8+98ZPjzWbeE+fPuCivduExYedZyYf7se8P8HUN4wjtXC8us2PuevXf+fMqN4bZ/vzms+X8rh1nLLxVuXeq2cO8G94cVX7xiePS6s8Pdt/wt/P7SWeGj5z97kY3ANUrfmUR7Tb+d/x53fqP08XGXZVzfQfqh840ogdoqRvol99vGY38/dw4cZIN1etss77VpmLXiuuHaa64P11z9x7D+hk8Kc5++Y7juqsvDkUccF37yyytzr8dry/WiIbz/1deEdXrjzSeOWyFc/Zeb+xOZjS91vNFUt1G+eVPWo3w7pYlesvbaa+tnPKJ/ABTAhhe5bqZoFLqnIxFsmfyGHFQj2Z0by0epw7D6S4nerLo1HeVLSevmYxPb9zIxnxL2JOOhj4MCrjf5WUjDI7RL7/H8NFnLs9KF0BOwAzVhz71BVFoWpbvtttuG7bffXj/7RyKxo9uISvTDJlBKU/8bpZ0t6TJX30nHLr64ElsG/x7bYELD6r7TDfIRL/xIG2OreyIW0e41b1MC4EK/zy0Q2G677QJ1hdTPHcCV41tTXxmlf0uuuOxyXuTOHcA1TP2m+9uW8sVl8FlnnRXYGjcl+Sadb/6eAFwuE32RrKdHVCfGQMjHHwdNecTbjTbaKI4dPO/jM79zhCzFH7h1b7G0FuM0AbhY5EVeKbk89jylE8ieG0U+p3nq9zjS7DqGkad8Fj4fo7Lo+P+zdx7wdlVV/t/pBUghCYSEEiD0CApDVyABFKQqoNhGLAM4jiKK+gdULDMyMxYEBwRBZUQkoKL0Guk1gpAgIBISWgIECGQgneR/vxvWY72dve89Z99zb97LWyufvHPuKbv8dl/rd9aWZ6T8m1mTShtMje26Xklf0Ux8QuBK6ae0bkg+CNb1uKjuR390RD2BpBoKjhXQRSP33Xefu+CCC8JH7LchYAgYAoZAixAwAldBYGUhUPBxe8wQMAQyEWDSj7cI3NTL14NMkDEo43IYxZoI3hFQOoSSmuDynH6Hr0Oee672dX4gWmEU+5JDHpfFgP76Ra6lvlzgXfmKi3yhwGVrOyFSSdhy5MsOtjcUT12xr0nl2fCo8yFkD/0M/RqLZUQbVPmNYhNPMmxdx0KVcomJEbhiqJS/Fi6oCSG2AOS6Xqhp5QLbIrE9kkhKWSztii1JpN7JYrQsgStWr8S4LYtkSQ/HMgQuvQVMjLyiw+3K5ykCF2kWrCClgqV40qMPw4OatHv5ol4Us/oLw9xyz32PdKfqIPdaLdLHrioCF+RVtkZEORYKijy+okShxha6IlqRFlPyynPhsZ19uFZOa8VjmCbBPxwzwufkd7285+RPbylAHI36OZ4pQviItS3e1aK9SDA3oA7SvkXKpIWvI1GCSX8sYRAmWyVA4MT4qCWn3WmiVWi802HLuVZMs33pmWeeKbc6HQ844ABP5uai3qJLxpJY/y/jnK47ueUJ4SwX+04ZUT8mTpzoDjnkEH8FAwCGgJhokr+eR8byF3tfX2vUBmSMiOFZbzwVA7WeD+stLcJtpXSawnPJlyj8uZ+an6QIXM2MN8Sn58l4GGsktU3e3MFjn3IffMeTbuS4NV2/ISPcgoV93IuPznev3vuaW3TvYrd8YG+34VfXdetNqvXlS2sh1rZMXPDc6+6x/5ztlt690K2xdX83aOcBbui2Q9yQdQa43ktfdfPnzHPTHh7gTn1oh5oHrjc9nKTSopXholRPPauvi7JePvKQeynCAvdT97SBUfr13DbH/LBV9VXyGB6rjq+ZNl51WqoqB+bx4RaD2lOg7m/rjcch9rHfa60x0H3qQzu4Aw+pefQZuIFbsWC+6zW4trVw7+HuhX/c4n5+bm17mgfiHz1JeBO26uVOPvApd9vM3dzZlz5TW9e/6YlC7uceU3lrpsxTaWmm7CRMygiiuWztw/WLL77Y8eGWFk3gShGSCIM+H9GGXum7McTJuYTdTB7Kzl9y5s2SztQx1efpsUn6vDCMmDFfPsrj2SJzOZkj6PjEWEsYMheqZ1TmuZjkpkWHpdfQXIecz/ZuWsrOD3k3RUpJtb9m6llP73NzdHE5axmIpLlz33aTfKT+4sVw77339l6x5KNSuSfHHAIXetlY+9f9sISr51fiFV3i5phqK820b+lXYmsCHbc+1wSuMvlrd9nm4qL7YDwdojsPRTxCadz0e0Lg0n1ibG4l4coaS+uiIBGib0DE0xPnuh9jTc3aWgskc7Z5ZE3HB9kx0eO6rosxj0zyvuhrtFdPmQfIeq6Z/lniCY9VhVlkDCNuyefLL7/sx90wPfwW0jltmzbQzJpU2mBqbNf1SuYEzcQna8JQRy/51J6zZK2p63ER3Q9habxjfaDEJ3qrejoiedaOhoAhYAgYAtUhYASuglgagasgUPaYIdAEAni9YpHD9kciLIBwBxsjWmkyljzPURus9HVtANJfzOhnONdeblKTXu2OWS+MZBER7qWu45AtHGURodOln+OcRei0adM6voybOXOmO/3008PHor/1gjFm8EoRuPjqBy9lWqksEYjXA74CRIzAJcg0dwwX1IQWWwByPWXE118G81wj0QtBWYzqa7wfS1ejeiUGZyEl6XTUMzjr5ziXcDRBMnymO/yuR+DSbq/Z2gqlAgogUfyIwoHfGBxQGLFw1l895ZZ77ntgnqqD7SgP6WO10qxRvKL80HVp+PDh7pRTTvGvprZMkHYRKmlEqc4XddIXhmm44oor3JQpU/zllGEjfCf83aitVdmHawXvRRdd5Lc4DdPDb1GwaqNw7Dm5Vi/vOfnTX/9JHI2OMaJp+I4YPeqRpD/ykY+4nXfe2b/K2IeXE/kSMQwv9VunBa8ZKPWYA4hCTL9HfYVkLeNuTrsjPvpd5Bvf+MZKWyLr+DjHqxaKPyTliY17enyinlPfEWkzWknub9T+yHii605uefJFczPYS5r0UX+xKt419H051/W2SgJXbJ4m42DZ8VT377LNgZ4z8yHBvHnzJEt1j1JuovDnYV3+oqDmeorApdPDc40knIuIISw1vw/Dg1o1Ycg892/bT3ejNurv1hhe86g1eIR7o0bAXTBvoXtx+itueY20NWbPkW7Q6EG1vaVqJJLadworlr7h5tz5svu/Jxe4kVuu6dYat5brV3un1+JX3dLXXnEvPfO6u+KB9dwfnt7ELXfdk8CV2+aog7ruV1lfw/KT31XH10wbrzotrSyHVhG4GKc22bA2d/ryzm70ZtvVvvZZt9ZoauvBXn3dkldmuwfvuMVdd/OD7rZpL9eM4FKKnY+9+/ZyJ32ij9tm5Dz3jckj3Iynl3R+IPNXaq7RTJmnktJM2UmYYMkcX+s8Yh+NaWNtipBEmDI31kbMWN8t8Tebh7Lzl7LzZkln6liEwCVG+TCMkMBFWYBfGbnsssvcTTfdlBwLZS4UjmWN4mgmLTps3V9xHX1SuI1j2fkh4chYzLn27JJqf83UM52HKsabVqal6j43VxeXs5bho9nddtuNIu3kUddfUH90ecjctwjJR/ohPk4Rr8UEm2rDKsqVTtG/Um9lO0P9AHoB1k1sQ48I0Uo/EzvX698UwYkdCMg/IuFqAlesb461lWbbt/QrsTVWLG9cy81fO8u2GVz0eiTV5xclcLFOZhcQ0dGnMBW9Urgm0d6/0evhqVvIXrGxILU2In7WaOgZkBSBK2Wn4B2ZE+h3pS3Keq6ZPpE4YlJVmLq/IZ7YGMZ1yafkiWuhhLaXFO7he/Jbl520QX1NnuOo66Osj5uJT+qa1mHq+GIErpyxXXTOOux656mdbOq9Y/cMAUPAEDAE8hEwAldB7IzAVRAoe8wQyEQAbwosdOQrKgzmuGXFk1VKMMDjLSIUFvAxwhfbC6BwRGJfSkk42hNJyu3suHHjnCiT9ARWFhG438bwFhO86qCwla/ryTOkjZiweEPpibENefbZZ73CI/ZseE0vfCSt+pmU8V8WnzyLMRuyCIZc4sYD2sEHH+wmTZrkgzICl0Y0/zxcUBNSbAHI9ZQRn6+3eAfBY1PovcXfUH9YCLKNIpJajMbS1ahe5RqcVdL8qbSlZ555puPL8vCZ7vC7HoFLK/9QOn34wx/2faAoKWTBj4KI/vCoo47yWdZGntxyz32PBKTqoE9ci/9IvWgHgUvqMgo4yA8xYezCGwFbfNGPo4QUQWHERDtl2JDnUsdGba3KPpxtJ3BZj6TGR61g1SScVPq5Xi/vOfnjS3EU7AhjEmSzepKaD4TviNEjVMbq53R6+coW4lUVaWGMx9sl5Be+xNXzGm0gy2l3jP3MVxAxuPgfiT+6TtX7ulK8BRLMr3/9az9H4FzGkphxQcYTXXdyyzP3PdKYkn322ccdeOCB/rZ414g9y1fZlD2ijT2x/MXe19d0nYrN06QPKkvg0vgwR/3Zz37WQQDW2yfrtKTOJV9aOZ6an6QIXM2MN5Iu+n7WBvKFu1xPHdfqu8R9dev73cbjl7u11xnk+q0xpLYfeM2wN6BfzQpd89HVe4XrNbBvjbgFc6vGNvF9d+24eJlbsaRGz1pe68sXvVHzzvW6W75wnnv9lUVu3jOL3alTt3UzXquF1UC6qgcuXTfK9qGtqq8pKKuOr5k2XnVaWlkOVZMJdPmMHrWWO/W4YW7sZhs4N3inWrtZy7k+tbV4rzXcG4sW1NrI391dU25wV935nJv5Qq0tRZhcB+yzifvk9je7ax/e3P3hljfcq/+HC7zmJDXXaKbMUylqpuwkTLa85yMALegFMCAzDxHRBK7bb7/d/f73v5dbHUf9UZnWP8T6bnmpijwQVtH5i8TLsci8WT8fO0+RP/TYlDLmhwQuwodYgrdxxhjWWI1k9uzZXiei4xNjLe/KXChl5K0Xfm5aJEy9jbJcox2Sb+qHSNn5Ie/FSClcT7W/ZupZT+5zc3VxOWsZ6nDu3LcRyUePRVUQuEQvQp1jbc4Hs2xBzweutDXtBUmIVjxbTzTBiX6FeVEo2oPtpZfWPE3W9Gxah1OUwEW4zbRv6Vdia6wwzfI7N3/tLttcXHQfnOrzixK4+IB544039tDV88AlH7OFH31pT0sQZl944QV39NFH+/D0x5dc0EQn+mfWxtRltr1GR4tIPJqEpecEkIghE4ei+2m9lg/nBM30z2Gc8ruKMIuOYcQpekGwlp0LJC1yFBKU6HiaWZNKG0yN7bo+ypygmfgk7WUIXDlju/bART1Bf1ZP+FiAfsjEEDAEDAFDoD0IGIGrIM5G4CoIlD1mCGQiIKQmXmcRftZZZ/mFeWZw0dfkCxgWSRjjUxNTvnr7yle+4sPQ3rV0oBApWLgjl1xyif8ai3NZRAg5i2taMMyilMEQngpbPy/nEq7e9k7uccSV+BFHHOEvXXvttY7/jRRf2lArBtU111zTp4+AmJSjSNbbVnJdu+U1AheINC/hgpoQYwtArqeM+FoRkNpmg3pHPUHhjiINYwCSWozG0tWoXuUanH1C1B/c+WOUQJFRj8ipXumSp/UIXBgLUM5QLhDV8EKIiAJxhx128F9Ocg2FIlvGivKBa0huuee+R5ypOsi9Vov0hc0SuPhCF0U3okm4/kLtD561qMuIJnBB8IU8THmEW6xBxv3Wt77V8WWwjA0pw4YPvM6fRm2tyj5cKwS1sk8nb8stt3THHnusvyRjhr4fO6+X95z8gTFtBtGeJsK42VoDIyFjMWScRiJGD55LbbkoXycyJtKuc9NCO5ftbq+++uoOIq2kUbd7raTPaXfaa1iKlCSGIlFGShtLKQtJp1b08T7KZUTGEp1uf6P2R8YTXXdyMcx9T9ISO0Lwh+iPsBXqeeedt9Jj9NXMJYlf9ws8GMvfSgEEFxq1gWbGU1H6MmZgcArniEFSkj8lX80QuJoZbyRhzNtRXJx99tlyqe6xT42g9f51n3QHbfWMW3v9AW7IsIFuzot93d0PLnXrrLeGGz1mkBu8Rj83oH9v16tPjaxV424tXfJGbavFN9wrLy92Tzz+f26TMb3dhK1qnn0XLnYvzlnkHn28vzv9se1qvK4a8auBdFUCVzNtp5X1NQZn1fE108arTksry0EbzXV/W288juEfXhs0sJ879gOD3cTNp9X2NR3v+qz5Dtd30Djn+o6otaEakcvVvHEtW+Ceue9299OL/+H++o/5YRD+9xZbrO++dcQTbsH8pe6nVw9z0x5vHYGrmTKPJr52sZmyI0ydJnQSjL2UGcJ6Bw+QInpuFtsKkef0nEF/eBbruyXc3DzkzF9y5s2SztSxagKXzIMoD3RAoe6DdFAW/EcgbDBHSq3VZS4k8yr/UsE/uWkheOoRnlsx4CN4nBdvRSHZIGd+qIkB+gODVN+SW89Ie0/tc5vRxeWsZXR/VHbuy4c/fACExLavw2sx9QypgsAlc2LaKW0s9CSrt8zNIXAJ4cInWP3RHtPZCQFddS6Bq5n2Lf1KbI2lktvpVBO4yuSv3WWbi4vug5slcB155JFul1128fhdeOGFburUqZ2w5IcmKIUfmWq9DGRZ+twtttjCh8GaDI/6IlqfLlvfyT2OOqwUgUv3wfpd8kBekHpzgmb6Zx2fPm82zDJjGPGKziL0li9p4qN59Dfa9tLMmlTaYGps1/VR2lsz8claPqWTiXngyhnbNTn2nHPOcY888ohA2HHEO5187EnbQF9oYggYAoaAIdAeBIzAVRBnJlAmhoAh0BoE9BaCeM7iK5mqZcyYMd7DF+GGi61YXJpQxhaOfOElAqmERSbHkAwmiwienTx5srv77rvlNX/UE+rrr7/eYbwtIlohc+655zq2/NIihmWuobh76aWXGiq+9MJQFPwQRMTTDGEQlhaUSqQFAhBiBC6NTv55TMkeWwASgzbi67qpt4TDsIxHO+qnlgMOOMDtu+++/hJ1kzqKpBajsXRVrVD1CViN/wiBiyzGPKxoIgTPaIIW7QwjBUoHkSeffNIrK+R3brnnvke8qTooaWrlUfrYZglcpFHCYjKMtywtfDGJVy1EEzXwhogxBOMORp6wjWnlsRBmUoYNHV/svFFbq7IP1+Ri8kS9hDCsRXsekjFD34+d18t7Tv6IQxT4nAvZkXMRTcLWSk+5Hzsyrsm2G7EtNTV5Tc8hctKivyxNedSQcDVpO6fdsdUgSjkk5nkpZlyR8YB3INOjINZCmULupl8KFZjybsy4IONJWHckr8RRpjxz39N50eeQNpl/ki/aN0Q+2r4WiHd8YY2ExM9U/vT74XmjNiB5LOuBi3i0IYJyIn+0bdoxv4uK5KsZAlcz403RdMaeG9V/kfu3zR90Yzda5tYZs4Z7bVEv97vrX6sZURa7EcP6u959eru+vXu5XjUy9Yrav+VvLHfLamW/4PVlbmlt+nToPkPcFuP6uFdeWuwWzFniTntga/fI/LXhejWUrkrgIuFSrzgv0+ZaWV9JSyhVx9dMG686LeS1VeXQCgLXwJrnug/sPswdtPblrt/wxW75yFGu79BRbsDwjV3ffrU50ZJernevQa730DE1omNfd/avH3EPP/a2tx9dtmuPWNP98At93Iil/3A/v2oNd9UDK3vT1s8XOU/NNZop83rx5pYd83rW7KQLue666zxhmLWCzPV//vOfu4cfftjf1wQu1gfoH0LSAtuB08cizGtZJyCxvtvfeOtPTh5y5i8582adztg5/RZkIghKsh06z+m1c8qYH/PApY3pN9xQ8yB31VWdosUozfyAOJkfnHDCCf6o4xNjLS/KXCicI3UKNPEjNy0Epz3IPP300w6yCTotdFaIXvvnzA/LEriIM6ee8V5P7XOb0cXlrGWa6SP12igkn/KRGu0A3SFSBYFLPBIxlw2JlvSfJ510Use28TkELvQB9CeawMl4ShsiP1q/lkvgaqZ9S78SW2N5kCN/NIGrTP7aXba5uOg+ONXnF/XApb0Ix/TgwKt39JCtdDXsers+6hFjR0ie5XnWmLIDB2P1rFmzdDD+oxs+0Ea0LkPPCQifuikfUUkAkgZ+M97hMRKJzQly+2cfYOJPM2GWGcOIXnR5nMe2lNTeTm+77TbHON3MmlTaYGps1/VR5gTNxJdD4MoZ2/faay936KGHAmPSVsa8Rz42PvPMM43A5dGyP4aAIWAItAcBI3AVxNkIXAWBsscMgQwE9JaFGE7xqlFPMJppQlW9Z+We3u6nCHFKu0Dmiw62LOBLAzx6oFAaMuTN7VNCTyV6EYFyAa8HeENigYXxFa8gCOQDCDZaQSBpjR0hEogLZhS4eIbgywiUFR/72Mc6tmHQRr5Giq+Y8Z/wcGEtSuRf/vKXbtq0aX7xyRZPENBECUg6+UpYvCPxhQuLSIR0sOBAwVA0j/7FHvontqCOLQCBR1/HsE4ZCdFCfyGIK2nKUu6xjQgELsqWuskCVIwAqcVoLF2N6pUs2nVdlGIdNWqUO/nkk/1P6i9f+MSEuokCjbSigJZ0xJ7t6tc0XnydCmFDe/9j2wLKRgQlC8oWETF8yG9x3S+/OeaWe+579ergbrvt5j70oQ/55IV50WnOPZc+tgoCl9R70sIXinxRj5KFLWL5Wk5EE7i0UZ46DKFW+jjGhy984QsdymoU6ZBmcvtGXXdi5L+q+/BPf/rTfis/8k3fjaIcBSZK60996lMdnge4H5JwuBaTennPyR9x6DqGAg0iMf0Esvnmm7vPfvazHeOUeEHzN+v80UYPHqM+3Hjjjb78GPs++clPehy4By4SX05aGEPpJ+nfqMf0g/IVI8Zdvm6ULxxR6Er/V6/dka6UaEUu8TBuMy5suOGG3qOmjOkXX3yxu+uuuzp9Vc58g/QJiQsjOXMXMT6H24ZIm4oZFyQfYd3JwZC85r6XwonrlDMKfIRFMkZiFO8I9YB2IPMjDOlaaZ7Kn3858adRG2hmPF177bW9R0AdtSYf6uv1ziVfzRC4CD93vOHdww8/3O26666cOr6KZ1vvItKr1wr3rqEvuU9s/rAbPmaAGz5ioHtqrnO33/9/btGCmtebmvctyFi1KVGtXN/8v2zZCrdsRW/3zq3Wcu/cor9bsmCJe33uQnfFI2PcVbM3ckuW9ykStdNjBbgzJ6snbJ0CaUOU9aEnX7nO/I16yQcvMvak7qUMjLltp5X1NYZNK+LLbeOtSEuryiFF4Ko3Hsfw19fWHraGO+6wUW6tV//iXuo7xi3ovabbcPjTbuNhT7jeg4a5pfOXuxX9hrsBYzdzbu2t3ayn+rlL/jjL3XZn7Qv9WjvUMmhwf3fa19d2Y5ZMd7+6ur/741+H6ttZ5/Xyllvm9RKSW3b64wBNqtaEW/QOkKQ5amMt6WFcQgcASYtx+HOf+5wbN26cT2rYZ8T6bv/gW39y8pAzf9F9YdF5s05n7DzV5+l5UsqYHyNwMV4yV5I1Mh5RIIAgkPuZ2zP/NhC49wAAQABJREFUQfT6VccnxlqekblQysjLMynJTQtj5Ic//GEfLGMDHoHx2qF1SNwUD0Kcl50fpghc9dpfTj0jbT21z6UO5urictcyuX2k9rbD3IQ2x1Zx48ePd3hSh4wmEhK4Um1Yno8dtU4EMiLtlHUK296hD8VDjAjzRLZ4bySa4MSzpBO9L0c+Mqbty9b2U6ZMcVdccYUPMjW/kvhSbSW3fROu9CuxNZbEGx5z89fuss3FRffBqT6/KIEL7HQdY82M11/qGLoQCC7iQTvVt6PrP+iggzoVQ4wUrNscnrnYuhfdKeMMbYc1p4gmKodzAt7hg1y851FP0eVssskm/lW8xbNeEInNCXL7ZwkzdswNM2cME70g6WDco81ju6A/ohzQ6SH8xk7BDihI7ppU2mCq/HV91HOC3PiknyzjgYv8lR3beUc7BGBNynob3OgL0H9MmDCBx3w9FX2+v2B/DAFDwBAwBFqOgBG4CkJsBK6CQNljhkAGAscff7x3R1z01ZA0VeQ9JpmQRxAMYrg1biTac07sWYz6kExYRIvoRYRcix1RDMj2dbH7sWv6ix/uM6FGxJjIbxZm8uVtI8VXzPhPeF/84hc7Fn78lkWrxIOCeNCgQdzyacDgBIkBkUWG/1H7k1NW8m5POsYW1KkFIItzFlhahNQTup3mGeoppAApP66hfEIJJZJajMbS1aheNWNwlvToL+lTX8DJs139qMtR0kp/hMIF0V9l8VsrB/kdKt5i27vllnvue/XqoFZctcKjovSxVRC42FJMvm4E61DoU2k3msDFl8S0FxR5CM9gZEO5QTsTCfu+nL6xUVurug+nPpA3IfSQF93/Cx5cD0k4XEtJKu+5+SMe/eUqv0knQjmIaMKJXEsdQ6OHPKfzzLXwC3Ou5aQlbNfUIeICe+mr+Y2hUeYr9dod6UgJxl3GdamzPIeSU/8OPfvpryzledKj8Y31zTKWxIwLMp7E6k4OhqQr9z3ejQkGcTyPhm2AMtF4xbYprpe/WFxca9QGmh1Pw3od+zo5lTa5LvnS7UmPa1pBrb9eD7cEyR1vSIcmAIThSjpTx/6933C7jHjBHbHJ427Ndfu5IcMHuudf7eWm/WORm/fK4hq3BMMMfXmtXdSIWwMH9XNbbjLIbTa2r1taI3ktenmRu/Hxdd0Vz27k5i9904tJKi59XadZX0+d33PPPe6iiy7qmEeHZAxdVwhD30/dq2dgzGk7Op4YqbjZ+hpi04r4ctt4K9JCfltRDrQ1DJZI2N+mxmP/cJ0/fWtj6zojBrllSxa5xctqbaVXf7fDpr3dv+z4V+cWP+deG7C+67N0QG3bn4FuwMbruf5rb+xef2M9d/Pdr7mrb5jlZs5602hGFGM3WNf94It93Bov3efOv36w++P9zRO4CDeVt9wyJ8x6UrbsMMhiXEXC8Z0xBlIw4zwic8jQWOtv1v7wvswVuMZvtvfFYCsS67vlnhzL5oH3ys5fcufNksbYUbdH7kt/qMemlDE/RuAiDP2xH7/BlLUGRAwRyO8YRlkXIDo+PRbKXChl5JXwUseyaWE9wHgvc7SQXK+Nx2AFQZB5c9n5YYqUQj5S7Y97OfVMl3GV400r0lJ1n5uriwvnfGCPhP1FuJZppo/UOtY3Y3v7r443JHDp8uUNacNvv73ymdYxcJfwEekLQ52XJsn6ByN/wv5MHtFp5xofc0DukfVmvfkVz9drK2XbN+Eh0q/E1lhvPrHy39z8EVI7y5b4cnDRfXCqzy9D4GJLUeqm1CnSpXUh/KZuQG6B5BIK5c4YI+/zbMyrOeRG1tryHOEQj/ThvKfHH+o2fSzvsb4IJayvhMUHZ+J9i+dTc4KcPjGMP/xdNszcMUz0gjp+sEA0tuGYmLsmlTaYGtt1fdRzgtz4ZFwtS+AqO7aDF173mCsIbuBIvdM6Ea5Rr/gwycQQMAQMAUOgfQgYgasg1kwoTAwBQ6A1CMiiqmjootQs+jzPMdFkQYTREmJWEWHyintucRWr36HzZBGEAVOLLCIw7vE1mJDG5BniP//88/2XIXKtzPGwww7zysLwHcggLFr11orHHnusn4jzbEzxlTL+o8QBI7zJaGHCztc9eHzSSmaMyxhtELbnw6OaGDpnzpzpv/TU4dj5yghIvdF1G2IJBBNELwD5rRWx/L7yyiu9txjO+QoWxUOs3kIS4Csw/muRr81C5VksXY3qlRjwaCMoubVoD1yh8lA/p0lNMZKAfrarn7Ngx2MNXxaKgItMwLjGF2GUG0J/CPFJRLuw11/gyX055pQ77+a+l6qDWrmq+wZJZ7NHqZPUZXAtIvWUH/oLSAmLvg5PRNRXFHkoL2RrWZ7BoMZ7mrAl73J89NFHvYcEFG8iOX1jo7bWij4cQxv9v66v5AFMGLvwroaBMTQKSz5jx1Tec/MncegtieUaR9LK15eQVRhziwhkbPod6ixelfjKUBRY8v4tt9zi2BYzJmXTwnyE/FO/YoJiEOUw+dCSanf6mdg54zkGITEOyzNgRV98wQUXdBgk5d5HP/rRTp7o5DpHvKCyhW8ootiMGRdEcZyqO2UxlLhz35P3wyN9Il/cy9YW+j544UmRuhBKo/yFz/O7URtodjzVHm7L9Jk6rdLnFpmf1CNwEWbueKO9A5YlcBHvgBqJ613D5rqDN5jhRq2zwq05bIBb1ru/mz1vhZs7742aR7o3atvA9XbDhvR1Y9bu7YYOWOYWzV/slsxf5q56Yn335xfGliJvESdfK7PdWFERApfMB8L5GH0F3gWZoyO6PFP3NGFEiP46PWXbTqvrq04b562KL6eNtyot5LPqcqhHJkiNx6SjsdTc1KkNRIcNG+KO3GmZ23HI3e6aV95dMxQ+7w7Y8iW36ah+rteQ/m7gmPWcW2sj9/KC4W763xe4f8xc4hYv6efes33Nw93wB9ySZ592/3vzEHf5Q8MaR13giXp5yynzAlGWKjshDhGu3spO4tloo40cH7WJsJUicwUx1qJboE+RPkCeoy9AB/DAAw/IJX+M9d2dHnjrR9n6lzN/yZk3x9Iq11J9nl47pwjLUg6xeQoeQFh7ix5D4uMIOY4P1iCFiOj49Fo9ta6W94ocy6RFfwypt9uSeCAV8PGXGGJlvOF+mfkh6x6IggieWdlmW6Re++OZsvWsJ/e5ubq4ZtYyuX0kdYr1uPZ+RXmzdma+xq4GjEkhgSvVhnm3nqQISXjWYR7O2oUtxBBILOhy64lsscaz6NPwjE4fp4W047mOZ0Qaza/qtRXCKNO+Jc56ayx5Jjzm5o9w2l22xFkWF90Hp/p8sTXoPl8TbULi1wYbbOB1qfLBMukSYbyl35sVbHko9zlS5wgDCT+Q8hff+kPa0e2H4w3zf8Ya1qGiC+YV+l52AZFdOfC8hVcw2pcW7AJ48UvZKfR6Tt4r2z/Le/WOZcLMHcNkngOhiH4ztL2wbr/pppu8l/0wrTlr0kZju66Pek5A3DnxyZowtg0nYe65556e+Mh5uD4uM7bzPkK9xbMraQ2FNNBW0HOaGAKGgCFgCLQXAbEfQuhlnoqtgvkD/zmX30VT1WuXXXZ5k+5c9I1u8pwRuLpJQVkyDYEWIMCiaPvtt/eGVjpNXOfzv4jwLgtR+hDIT3gwaFborLfYYgu36aab+q/BIEmhWKhaUKxstdVWfvGOEZnFHosgBEUgxBIWtjNmzOi0QCTPbGPFApfFrWzhV3X6enp4eKniKyzqJMqBkKiA23e2TYDIxSBPWRTddqinY9uK/LMgZhsOFOuQRFolueWe816qDqLww0CSInm0Ku+pcOsRuHhH+jPaCm7sIZiIh7RUmPTDGKRoY5CeaGMQ7yDREkZM2t035vbhpB1DIoZCFgJsW4yyRiuuY/mrd61VeWdsRVHPdgEQ5sAe8h3jTzPCQoj8Ey5jH2M+xoh6kpMWDB477rijV9ISJ9usoYT861//2jHehnGm2l34XOw370JOI61sA0nZ1qvrKIqZBzDfIP88T/tACd4KycGQdOS+Vy8P9Inkna0mUYJjrGS+00w7qBdfK+5pQlXMa1gr4iwSZs54UyTces/0qW3jNnrg626fdZ9224140a01pJcbsGZf17tfX/fGipoCpHa/94rlbvGCpW7ZguXukRfWdNfMHueeeH1I4W0T68Vf1T3qJH0F/QSkbi317unn9Hkr2o4Ovyufd6U23s5yqGo8Zn6w/rpD3B47berue3Se+8eMmW7EWkvdQe9c4HYau8StNajm3W6Nfq7PsNqHkIOHu159B7sVy2tegRfUtv988Tm3qOYJ7ye3reumzuxsfGymzjTKWyvKvJVlpz1wXXXVVf4DHIxybPPN3BUdAHPtcA1YFsOcPJSdv+TMmxvlI6fPaxQmaxj0LOg5wJjxf+rUqe7ll19u9Grl99uZlrLzw1hmG7W/nHoWi6eKa+1MSyNcUvlpZh1HmDlrmdw+krUw6zE+hkEvyVqhyHw5pw2TRvpG+iDm5nyIotd9tF3SQ9uljywrrP3wsMW6nv613jqpbNj6+Xa2bx1v2fy1s2xJZztwqUfgIg20nXE1D9bUJUhBzLmp0xwbif6A45xzzqlrN+DDKuwM6KDwlsX6H6KMCCQu9L182IWOKTbW03djb+AZiNyyTaCEUfTYij6xFWHq/AiBCz03BE7wZH7EPAk9B3g0ar/0J+3Umbc7vrJjO3N7HBGACR90Mvdhx5Uq7Fi67OzcEDAEDAFDoDgCRuAqiBUTDxNDwBAwBAwBQ8AQMAQMgfoIHHzwwW7SpEnegxBGj1Ut4sUm5X58VafP4jcEDAFDoBUI6K0wQ8+PrYivq4eJ/6B+vZe7kf0XugnDXnZbrPmSGzpwsevfZ0Vt+8Re7rXFfdzjr6/tps0b6WYvHOwWL+9T8zfEWyaGgCEQQwBDT58+vWuG+uUdpOO+tfa08brL3G7jF7vt1l3qRvRf5nrXrvXuV/Ne0mupW1HjV69Y0sfdO3dNd+6tQ9yipdbGYthyLUbgSj1r1w0BQ8AQMAQMAUOgMQJ77bWX917Fk6EHrsZvp59gTvSDH/zAe2nXHr/Sb9idZhAICVzNhGXvGgKGgCFgCBgCXRUBI3AVLBkjcBUEyh4zBAwBQ8AQMAQMgR6LwM477+y3ycAb0sknn+y/BlxVYKBEGzNmjIPEwHkrtnRcVXmzeA0BQ8AQiCGApwc8n+K54fOf/7x/JNy6JvZeT7vWq0bN6t3BG6l5l8XBbG2cwNEspC1+mhgChkAeArWm5KmPI4a+4baq7SS0WY3IteEay91avVe4BW/0cg/P7eOuntbfvfJ6RyPMi2g1f8sIXKt5AVv2DAFDwBAwBNqKAFvtsTU7nhYRCFdskduMyNrrkEMOcRMnTvRBXXvttY7/Jq1DwAhcrcPWQjYEDAFDwBDoOggYgatgWRiBqyBQ9pghYAgYAoaAIWAI9FgE2HZuzz33dJdcconfemBVAvGjH/3I7w8uabjzzjt9uuS3HQ0BQ8AQWN0Q+Pa3v+2GDh3qSauSN/pCtpIwMQQMAUPAEOg+CBiBq/uUlaXUEDAEDAFDoGsjcNBBB7m99967I5Eral9tHH/88R2/c08gEiF8MIgsXbrUff3rX49ueegfsD+VIGAErkpgtEAMAUPAEDAEujgCRuAqWEBG4CoIlD1mCBgChoAhYAgYAoZAF0BAE7jmz5/vfvjDHzqOJoaAIWAIrK4IQOAaNmxYR/ZuvfVWd+mll3b8thNDwBAwBAyB7oGAEbi6RzlZKg0BQ8AQMAS6PgLaQxapveaaa9x1113XdMKFSERAy5cvd+eff76bNm1a0+FaAPURENxnzZrlfvKTn9R/2O4aAoaAIWAIGALdFAEjcBUsOCNwFQTKHjMEDAFDwBAwBAwBQ6ALILDrrru6/v37uxkzZnjX+HxlaWIIGAKGwOqMwLve9S43fvx4N2/ePPfwww+72bNnr87ZtbwZAoaAIbDaIsBWT9tvv73P32OPPeZeeuml1TavljFDwBAwBAwBQ6CVCIwePdpNmDDBzZ071z3++OPu9ddfryS6ffbZx388M2fOHDd9+nT7YLASVBsHwpqX7Sufe+45N3PmzMYv2BOGgCFgCBgChkA3RMAIXAULzQhcBYGyxwwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDoDACRuAqCJURuAoCZY8ZAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChkBhBIzAVRAqI3AVBMoeMwQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAyBwggYgasgVEbgKgiUPWYIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoURMAJXQaiMwFUQKHvMEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQKI2AEroJQGYGrIFD2mCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIFEbACFwFoTICV0Gg7DFDwBAwBAwBQ8AQ6FYIrLvuum7vvff2af7zn//snnvuuW6V/u22285ts802Ps2//e1vu0TaDzzwQDdkyBD36KOPuvvvvz+ZJp7r27evu+eee9ycOXOSz5W50YowU/GvscYabq+99nLbbrutY65MXpYuXermzp3rnnzySffYY4+5Rx55xC1fvjwVRLe/vv/++7sBAwa4J554wk2bNs3nBywmTZrkevXq5e699143e/bspvI5fvx4N2HCBB/GXXfd5Z5//vlC4e2+++5u1KhRbt68ee6WW24p9E69h2J5rfd8V7q35ZZbOv4jl19++WpdJ8lj1XWQMHNk4MCB7qCDDnLrrbeeGzRokJs8ebLvG3LC4p3uXAdz89zV3ys63nX1fEj66LcPOOAAP57NmDHDTZ8+3d/aY4893Prrr+9eeukld91118njduwiCOy0005uzJgxbv78+Y65rEjZPmN1LedUvRac2nUcMWKEe8973uOju/vuu1dac+y7775u0003dUOHDvVz45tvvtm9853vdOPGjXMor6dMmdKupFo8PRyBnjZvLFLcsh5hXXfFFVe4FStWFHmt8DOpuWu99czYsWPdxIkTHfoM+rkf/vCHheNbnR/caKONHLghF110UcOyqnqM6Ir6mTLlfdhhh/n1PXqMv/71r2VetWcNgVWGQNk57ypLqEVsCBgChoAhUBgBI3AVhIqFhIkhYAgYAoaAIWAIGALdFYHRo0e7wYMHuwULFnQymGBIQUmFQICCcNKd5F//9V/d5ptv7pP8pS99qUsk/bTTTvNK5L///e/uZz/7WTJNp556qic1zJw5051++unJ58rcaEWYsfj3228/9773vc/nM3ZfrmFM/fGPf+xeeeUVubRaHX/yk5/4/EDgOuOMM/w5xsajjjrKn0MW0sZkf7Hkn8985jPuHe94h3/rd7/7nbvjjjsKhSB1YeHChe7EE0/seAfDKEZUDC/UPS317sXyqt/tyue6n/jqV7/qiYZdOb3Npq3qOpiTnv79+7t///d/dxxFfvOb37i//OUv8rP0sTvXwdKZTbwAkaF3797u5Zdf7hL9atHxLpGdLne5X79+7gc/+IFPFyTks846y59/+9vfdsOGDXOvv/66O/nkk7tcunt6gr773e964vzixYvd17/+9Q44yvYZq6Kc29GmU/W6A6g2neg1xx/+8Ad32223dcR8wgkneJKkXIA8+Ytf/MKdcsopbvjw4W7JkiXua1/7mtzuFsd2lG23AGIVJzKnHHravLFIEX3iE59wO+ywg3+0FXPp1Nw1tZ6BZHfMMcd0Wot2FV1AETxb+cyRRx7pdtllFx/FV77yFffGG2/Uja7eGNFs++mOZSJzWwhc55xzTl3s7KYh0FUQKDvn7SrptnQYAoaAIWAIpBEwAlcam053jMDVCQ77YQgYAoaAIWAIGALdDAEMkijnXn31VW8MkeRrY4oRuASV5o6i9GtE4PrgBz/o8PYAkQaDI8apZqUVYYZpgpyEkl0LX2O/9tpr3msJnnb4klcEpTFGuIcfflgurTbHmKIsZYDIzXTVBK4vf/nLbsMNN/T17vjjj++UrHr3Ynnt9HIX/tHTDHFV18GcosXIh7EPoY+jf7jwwgu9Z8Kc8HinO9fB3Dzr9yBefvOb3/SX8Ajwv//7v/r2KjkvOt6tksRlRJoyYq4KYk9G8nvsK92VwNWuNp2q1+2uMHrNoQlceHDVnnMgSuJx9Morr+y2BK52lW27y7C7xZdbDj1t3likXA8//HD37ne/2z/aClJOau6aInDp9dGyZcv8R2rf+ta3imRltX+mKgJXFe2nFXWl1QUoc1sjcLUaaQu/SgR6+jq5SiwtLEPAEDAEugoCRuAqWBJG4CoIlD1mCBgChoAhYAgYAl0SASNwta9YROnXiMDFFoR4qIHsdO211/r/zaayFWHqNLHdJluiieDR7dxzz+3kyYn87Lbbbt6zG55iRDDOPfPMM/JztTjGFGUpA0RuhrWBoowHLrYiYksRtvy65pprOqKvR9Kqdy+W145Au/gJW3mIFzOIqqvztp4URdV1MKd4P/7xj7t/+qd/8q9CNKpiC5LuXAdzMAzfyTWkheFU+bvoeFdlnK0Mi/ELw2efPn082VA8xhmBq5WoNx+2EbjqY5iq1/Xfqv4ucxLmJgjbI8qccOutt3ZHH320v/7ggw+6X/3qV/6cP2xJzfaYeB28+uqrO6539ZOu2F93dcxakb7ccuhp88Yi2LM9F96XIeWHH4AUeb/RM6m5a2o9I975SA/e+ZYuXdooih5zvyyBKzVG5LYfTYA0AlePqXaW0VWMQE9fJ69i+C16Q8AQMARagoARuArCagSugkDZY4aAIWAIGAKGgCHQJREwAlf7iqWMQZstKMaOHevYarCqr4ZbESboodz9r//6r44t0Z588km/bWBqWwaUvijUBwwY4MHHUKc9LLSvRFoXU0xRljJA5KYil8CViq8eSavevVheU3HY9VWLQNV1MCc3n/rUpxwGUOR73/ueJxLmhKPf6el1MNeQpjGs+rzMeFd13O0Mzwhc7US7fFxG4CqPWVd6413vepf75Cc/6ZNU1QcNqzp/XbG/XtWYrIr4rRyqQx2v0Xh6Zt3HtnxVS9m5q/T7ELdY+5q8jUBZAtfbb3Y+y20/RuDqjKP9MgTagUBPXye3A2OLwxAwBAyBdiNgBK6CiBuBqyBQ9pghYAgYAoaAIWAIdCkE9tprL7fJJpt47zMQcPA889BDD7nnnnvOf8mutzPBM82zzz7rdt11V7fFFlu4wYMHu+eff95vfXfjjTcm8zVu3DjvcYlt2eQd4rjnnnvcokWLOt7jK3qeRa677jofl/+h/jDnOuKII/yVGTNmuFtuuUXdXfk0VBBus8023uvLpptu6r8QJj+kgy/6Y0J8eJTaeOONHd6rwIetW/7xj394r0Wcx4T3dt99d7f55ps7ziEn4WFm+vTp7sc//rEnOzXywEW42mgV81C18847O/KE54E//elPbquttnK77LKL22CDDbyHEOLFQ4j2btOKMEkr5XfwwQdz6pX3J510klu8eLH/nfqz7bbbuk9/+tMdtyGAjR492uebi6R92rRpHff1CVt1DBkyxG+JMXnyZH3L1yO8fDWqc7xEfO9///v9+5dffrnf5hHF9qhRo3zd4JrI9ttv7/bcc083fPhwv+XowoUL3SuvvOKmTJni/va3v8ljHceYoqysAaIjsMRJLoGLL9ZRvM+ePdvdeuutbsstt/TtlDo7cOBAHxvYY/i47777fL2K3aNfwFgTy6tOMt6WIOxASESefvppX7b333+/b4v6WanXeHCjbOkX6Is222wzj/e9997ry4a0kQfyQhtle076Luo72ysVFdoEfRr19Y9//GOn14YNG+b7nPXWW8/3AfRZL7zwgq+b9B1lBc89pBcsqL/0u+Rzzpw57qqrrvLpLxomXkloQ9TDyy67zGMg/TNhULY33HCDe+KJJzoF2agO0n8ccMABbp111vGYsn0rigHqAX2ueCgj/bRDBEzYziomEydO9GnDC8Idd9zhaEfgTTtCKC+w533i+tjHPuZxof6JhyMd7nvf+163/vrrrxRnozqow6CNMx6xdeMll1yib/n+hzqBUD8feOCBTvf3228/7/EF73XgLlKmbKse78gPeIIrwtjEGEnayYMIW6Udeuihvk2tvfba/jL5YBsY6l+ONDPelekXJG30BYxztE28ONIHM5bTdlNjMuXJmEw7Zjs22jHjI14LeV+k0XhAHf3Qhz7kHwfbRx991J9rAhdk60MOOcT3V2BM2yF9jNHz5s2TqFY60s8VHbdWejlyoUx9jLwevTR+/HjfN9DvQr4mb/S5zAFnzZrV6R0w4LmZM2d6b0r0O5Qb3pLA/s477/RzIl6ib2frrY022sj3L9xnXHr88cc7hSk/ivZR8rwY8uln2JJapEyfwTu55Uzdo23SB2rvUZIOcAIvBIISfXfRNi1hxI5F2wrtKFavdZiQM5hvUqcZrxj/2PqatkWZPvXUUx3zNcqHcY65AWM4bVXGAfrduXPn+rZz/fXX6yh8OMSDgAPt+bDDDnMjR4501D2EdQd1irGB+sGcjHbNhw4xD1xF274PvPan7Lxf5itl5uHNlK3E14r5UdnxoZVpoTzKjA+SlnaVA/UqNm+Ufo/+8M9//rOfu+LhlbGF/od2gnc55sBlhXUJczPm0cy/mAPTlm6//XY/PwvDazYt9Av77LOP75+Jm/hY/9L2wnklcTO3++d//mefT93PVjWPTs1dw/UMW3QzpnCkTtPv0l/RZvSajjRXPe4SZj0ps44E/6OOOsoHx5qGOR0ezugL6ROZT1DP8KKcmvtQ98CNOTP1jz6bNRTzaMZjBLJd6mMr/0DtT2yMaKYfW1X6maJjouQ7dZSPE5g7//73v/d6BOoSuiLKhfZx6aWXJnEt27YkHbR9dBYcGUtZM8laJNYmeY/1BvWANR3zX8ZK+iHmv1oPx7MydtNm2MaY58GMtSZrP/RlzM0Ig3ush9ANMC+g76V/YC0h60TC1FJ1e6tynpu7ptb5K4N1bnz9+/dvyTq57NyHMZD/rAOoS5zTPxHOBRdc0LE+0vjYuSFgCBgChkD1CNAPI+g1GRfRLzPP4D/n8rtozL1qE8QVRR/uTs8xQJkYAoaAIWAIGAKGgCHQ3RBAwYrhIxQUQnhHQmmD8QTBUAnJIyYosM4555yVbn3gAx/wxpWVbtQuEMcZZ5zRsUUKxCyMXAgEr/POO8+f6z8QhFCOIBj92XqrnmgFIQQbjE8xwfAbksFQ/vI+E96YoNwiz2LAlWdQfh133HFesSXX5IjRCyU+YRYhcPHef//3f3uvVjGMJX8oXlHgY8iKCco0iBwirQjzO9/5jhs6dKiPgriKkgFOPvlkT5biRcoAIoiQ9FAGYngNBcX1N77xDX859E5Wps4RgDYIYDBE+SRljqL75z//uY9HPJf5H5E/GJxDD2Ix47COD0MCBp5mJJfAdeqpp3pyDuSfE0880ZPvpG2F6aFuYVyPyTe/+U2vvIvlledRNFJPx9UU2zF58cUXPW5aiSz1mucxNOy0004rvYoh5pe//KX73Oc+5xeq4QM33XRTJ3JNeF//1vFRzhipEMgU1EWpD/odzh977DF39tlnJxXV4fMQ46jPYJIS6kNoYEo9+4lPfMIbqLifwol+Cix0mPXqIAY/DO0pwQDE1q4oClAKUOc5Eg/1SJejhPGjH/3IlxHPYGgSwqTcl+NvfvMbb6Bi2xuEPpstWEOBIIOxQMYpuZ+qg3JfH/H6JWt4+iBt/CIfbOWFQBagrWiBhEuedf9UtmyrHu8EY51OziE6nH766f4yY9PnP//5DoJm+CxKH+ozfVlRyR3vcvoFFFKUFWUfk2XLlvl5gx6TabuQhGWL1PA96uT555/fQeLWbSM2HkC+wWMpQvs/66yz/LkQe6iTtBGpW/7mW38waEEWvPvuu/Vlf1523FopgOBC2foYvB79iQckDDQxAUeI92AmImMMfQWYMG6HgrGPcVbIS/o+YTL+MvfRUqaPkveqJnCVLWe2E4OchsS2i4J8f8wxx/j7YMj/Im3avxD5U7atQHKI1WuCxkhM+iFwhMI4DBYQNF599VUnfbdeO5AXyAL0maFgcGYdIKLfw3jMPJ/xJiZ8EPGLX/zCxwl5NRwPyrZ94siZ98v8ocw8vJmylfhIb2rcz5kf5YwPrUpLzvggaVkV5aDnjdLvsZZhPsSHJKEwFvzP//xPlAQVPiu/G81H8Xr8s5/9rNMcrJm00N5Zy6655pqShE7H2BocIgflwHyKsRpplO4y82g9Puv1k+RT1jOkm486YqL736rH3Vh8+lrZdaTul8EbwiD9cSjg/R//8R+eoKbvMfeBeBMKYyvEa9H/FCFw6bTI3Keqfqwd+pmyY2KIWfhbCFyQpxj/CD8U6uOZZ5650pw6p20RNmsz9GCpNSlzS/1BG3XlC1/4Qkc5h+mjr7z44ov9OCL39BjMRzsQDsP4qD8//elP3VE1ciFE0lD4YIG1Ic9pqbq9VT3PzV1Tk8ccrHPjY74jc62q1sk5cx/Knz6ZcoaMj/5M5KKLLvKkWfltR0PAEDAEDIHWIWAEroLYxhR0BV+1xwwBQ8AQMAQMAUPAEFhlCEDOQuHKV3koaFAq8zUvXw9iyNWKHEkkhjgUfyh18RogEpKg8KKy4447+tss7lFmo8zCMI6nHESUQBhxIP9AAkJCQ4y/WPsjRlJ+/+d//mdDbzWi0Jf3OWJkIn8o0PhSUoSwxRMHniUw8AvRgrSjkMJwDAFLtv2D6IFCVoT7EFpE2YVyDIIK+Q2VXEUJXKLgISzi0l81xvKHEZ58YCxFuSWitylrRZhakfv//t//62REkDTEjniD2WuvvfwtDO8YbIUUwkU8eWGM0vKRj3zE8bU9osliZesc72uDAL9FqJt4WIEkeOCBB/qv0LlHOeD1gTrKF7Ao0qS8UWBpDxAxQomOTxsgJN6yx6oIXHg6kK/Y+aoWgbyC3FzzFkAZ0d7DeyixMVLF8sq7QrbhHOzwLIKiW74E5jrv8xyYIrF6DVmG9iZ9lX/wrT+0CZTo9CHSZrkFyY/20Eh0fGKIY32H0Z+ypS4QPh4T6Pfw4CfGaDxK8eVpEdHGGwwu9Hv0JXgv0UayIn0b8Uk71nHTVujDdT/FfeoxBiAkVQfxxoiyX/KMQZC+kvTRn4iBQpMVIQSh+EVCoijXIM5QRxHeg8CFAQKDh/SjUs8wJBBf1YppH3nwR2MHUeC2227reEL3ZdRZvRUQGH3xi1/0z0Js5Mt7pGzZVj3ekSbGGCHJME4zDkBywGsUYx1jgNRb2gX1mTGCfkzaDe9BYNPjjM9g5E8z411Ov/DZz37WTZgwwaeEdOJBgPoO8UBIXZQX6Ze+RG/VSTumH6F94I1C2hzvkB7apG4bOssyHuBxMEZ00XMT3uN54iF9GEmlrnOdciAdIjnjlrybOpatj6lw5Dok2o9+9KP+J3hBRmdeRH8MliIYKyFlIWJUl3vknXcYQ/TcUe5TZszPKEvpayhn7cklp48i/KoJXJLmouWcQ+Bq1KYlDbFj2bYSM85LuIKd/IY0T/9AHyZzH+6lCFzyHuM3fT1jK++K0D+JF1+95qBfnjp1qieP6Tk0SmraFeQlvJ8yXsQIXGXbPm00Z96v5w+Sp0bz8GbKNhZfs/Oj3PGhFWkBw5zxIZaWdpWDzBtJe9jvcY11IP0b61/p2/jNh1JFJPRWTHjMpVnHEqaM65qwTbi5aQnrA3N0+nzaLn23tPswPsY6+mvmcaznq55H6/FZr58kn0LgYs3G3BN8wEbGHvotPD0jrRh3fcCJPznrSN0vS7DkBR0M4yjzvVRZ6LUh79JvM8dhzJY6KGHmEriq7sdaqZ8pOyYKNqmjELjkvsyLKA/WX4IxYxV6DJHctgV5XrYRJiz6NtZUjH3ywQnX+egFUg+i56WMv8z5mVMxZ6NtItQnPuqjfSN6DPYXan/IA+sJvW6Xe7zP+MP4KXNq7uGdmI+HRFrR3qqe5+p1oaSbvDdaU/NsDta58VVN4Mqd+wiBi/xTD6Qv4pwPsiAAmhgChoAhYAi0HgEjcBXEWCY/BR+3xwwBQ8AQMAQMAUPAEOhSCGCQRFGoDTAkMFTkhF/34Tqd/4g26qPghTzBYh6vGHgA0dtFsMUCX/UhGGHxaIJopT3voBwWYb6FcQVBcSUemOR+7KgV+igUIDHo7bC+/OUvd3wdrck3EFnYpgIJ88w1lO6QGhCMW2KMPfbYYzu8lJFf8kD+EbZy+PjHP96h4ChK4MKYCS4I7vhxXS8S5g8ykyYQaU8yGMgwlCFVh4miEtIDgoIcXIsKuKDEQjDufv/73+9E4BFvFDo8nuFrR8qUPGJcyK1z2iBAHCi5IZFRn0XEY5k29Ms9vq6X/Oq6zP0YqUnHpw0QEl7Zo1bSQySCUFREQoOHvCNtAmwxOmupdy+WV7x2yRZ7tBHqCAYEhP6GshPihS7nsF6jkMZLC6IJQfzGUIRBhjqAaIUoZAuMvI1ExyeGON23sdWX3mYPJTV9Ef1bqJxPxQW5hr4CwfjG1/JgLKK9C2os5H7sqPPK/ZBApb08iWGL51J1EM8E497ylIZnIb6yF9FfOmvDo/Ycg5KbfGnRBK8LL7zQG+S5r43r2nhUtWJap0WfayKWeBPgvm7P8jz1VsYvthnDowQiRLvcsq16vKMPhECMhN4xjj76aMd2HQhEWQjaIvTf2rMVpAnKqpHkjne5/YLMU+iHIQljkBLRhgRIdZDrqEtgTDulrUEu0lvynXDCCR3kIzE26bZB2OF4oA2qut6Exhu8CkmdwYBM3ylEp1mzZnWMDbnjluQ7dsytj7Gw5Jr0/fzWJC1+a8w0JjLG8AxGQ8ZRiLBI2HeJNyV/s/ZH40k5yTwqp48iTCEhhYSw2LglaYgddbqof0XLOYfARfz12nQsfXKtbFtJ1Wu84opHVMZY8i9jLX0lhF/eRfT6IVw7MN/Fq660WT4e4RmEjyOEFKnf08RabbwO5zkxAldO28+d9+v5A/1M0Xl4btmG8VUxP8odH1qRltzxIUxLO8tB5o3UZ93v0W/htVEIFcwdqa/SZmQOwXv1hHkVxA9Exip5HhIPc2khjOCVjv4UyU2LxpIt0/S2r5BT6JPlYw7iEBI+aWBuhSEJklHV82g91uj1k+RTz3PJv/T74Ty9FeMu8dWTnHWk7pcJmz6WMUfGUT3/pq5RLggEWfpqmftQfmxJjkDWZ86tST96Du4fivzRadHjfFX9WKv1M2XHxAgEnS5pAhflgR5L1rd8oEKbFOK+Xpvlti3Re5AIrc/htyYHMsfFux8fufGxG0J7pA+RsZtreg7G+lzWp3oM5jk+4GL7b4T+i+eEMMpcnHwzhiN6fa49Q7WivbVinqsxIT+63PidWlPnYp0bX9Xr5Ny5j153gQ8fb9CO0dGaGAKGgCFgCLQPASNwFcTaCFwFgbLHDAFDwBAwBAwBQ6BLIiCKLW2AIaFakYMhU4hEkgmUU/I1q35XG+35Cusvf/mLvNJxFGUUBg+MsRjWtBIBZSNbpIloggNb7eHxq5FoRRnErfPPP7/TK9o4pQ3eeIXCiEDa+Io4VEZor1EoylCYQShCQYbCFKUWJC+OWvQWREUJXLyPUR4FGEpCIbFxXecvNIByf9ddd3Uf/vCHOV3JoF9lmHhAQQGMhIpyf7HOH4hw8hW6EPPGjx/v/u3f/s2/BdlFb6Gjn9ekwdw6pw0CKMBRuoqRUZJNHccwwdaeYR3iGfHYE6Y1ZhzW8WkDhMRV9tiVCVwYpyD+0I4wJtBHaKHN0A8gun/R9fr+++93v/71r/VrTivOtfGIhzQx55577nG48W8kOj4xxGmSaYxQxVYQeC9EYU8f0Eio0xBeEAgm4TZqkDvE+BISxlJha+UvxjLwDgXcMSYgQkRK1UFIGrQvykn3MxKmbLlLeWpynximdF8u70jboC8kfzyDrGoCF2mQtOk+64Mf/KDbY489fDrpyxFIs5BnEek3NYktt2yrHu9ShjQMb5QR+aFvo46HQjuV+hOSXMJn+d3MeJfbL0i7B3sIZ7qfpo7LFnRsQwoJTXtciHnKY0s7qcdi9NJtIzYepIyYup3FiKPgS19FGWiSc+64FSsTuZZbH+X92FFvkUz50d9oIR8Y98AR4g0iRnXOmcuJAZnf2tBHedK3SN/AfW0og2wo22Lm9lFiyA/rdmyMJv6U5JZzuwlcZdtKql4zDqDrpGww1AopUfDRc0y9BtBrB5TKQiyV9yAR0CchqfeaIXDltP2ceT/p1/OHMvPwVH9NmPVEx1fF/KiZ8aHqtJDv3PFBp6Xd5SDzRtKv+73Y+pdxCuINArEGglQ90R+56DWPfmeXXXZxRx55pL+ktyXNSYv2Dhpru0Si186xtbWkrep5tB6f9fpJ8lmUwNWKcVfynDrmrCN1v0wfjA5GjHQSjyb3yfaQel2oCTjyjp6/cW1VE7hidUjXsWb1M+Sx7JjIO/VEwqNcmBeE69vtttvOr3MIQz5Oy21beLPHgxUS+1AGQhV6PPQUMseRsZt3YnM2ruu6w5ya9aweu/XYzPOIngPF1sbME0iP1le1or21Yp6bu6bOxTo3vqoJXLlzH03gwoOb6IPfrCn21xAwBAwBQ6BdCMjcEFsKcwH0TYzF/OdcfhdNT6/awuLtz4yLvtUNnjMCVzcoJEuiIWAIGAKGgCFgCCQRKELgwpMNhslQ2O6OL3G1EVyUGdpQGb73L//yL26bbbbxl+WLYYw6KACYZIoSSt7TSqOcrdHOOecc98gjj0hw/siXxJDHkNArib+o/kBWY6spSBsY3pkcI0Lg0gbJmFGFZ/UXg2UIXFqhBrby9a02WAg5g3hEtKetMH9VhqlxRAGIIrCoaNz4ihtFPEI9AHOUo5Cq5MtRbRCATCUe1XLrnDYIaCVxo/RjfIO4tldta8Fxb3ktMgKX89sC8oU4C0YUuYgor2OYQs7D8E85C5lC1+vQExRhCGEIUpAQByVstkyR+kd7p903Eh2fGOK0YZr3MVxD5KG+aeJIo7Dr3QcjtpzBk8n73//+ji0tcghcbE143XXXrRQdRj2Me4h4JtJ1XhvBVnq5doGyoZ7jMZFtNumbdVnxjiam6vDo3+nnEf3FPr+7AoFLexWSbV/pa/BMQF+EsQXijfRL5J06zTHMD3nSUqRsqx7vUoQATWqs18fpPlfaos6TPtf9dpnxrpl+QRubmB9ADGfry5BMJOmkH6A/QOiTMGyHgmcfyhjDOG1ct40YVtqgquuAzE9ifZLESV9FW0IgLJGH3HFLwix6LFIf64WlSQfM6+hbIdKDQUrEqB72FzyvParGDD94QGWehbC16l133eXPY3+K9FFVE7jKlnO7CVxl20qqXosxVs/vwzIQIqw29ur5ZeqDCxnHddj6vWYIXDltP8wXvxvN+3lGzx/KzMNT/TVh1hMdXxXzo2bGh6rT0sz4oNPS7nKQeSPlJv1eqo/QXqlCb3KxctdeP0MirDyPdy7aPKLbYU5amOfhLRrR20T7C2/90R9vsbUa419Mqp5H6/FZzzUln0UJXO0ad2OY6GuN1pG6X059pKHnsZwzPgvhmrGXDyeoi6Fob5armsDVav0MeS87JoZ4hb+FwMVWpkJGDp8R3ZrUy9y2pfuA2McIxEs/ztqF9o/Hakmf7g/C9PGhHEQoRPRwegyOfQR13HHHuY033ti/Q77JvxbRCWpdULvaW7PzXE2oKrOmzsU6N76qCVy6/OS8yNxHE7iKkJElbDsaAoaAIWAIVIuAEbgK4mkEroJA2WOGgCFgCBgChoAh0CURECVTqOjRihytLNWZiBlhxKDDcygQY4IRXARvWhh5EJSNstUQxiO2pcNAJ16YQpKMhBE7aoU+Bh0USlq0ETEkVEFKw8vOtttu67euQDEUEyFw7b///u5973uffySFFTdF0RPGFwtbrhG3fF153333uQsuuMDf0vkTAoK8w1F7VQnjqzpMyVfKaKHTpc/f+973evIK11A6sn0hgut/3NIj2o29kAxCTzK5dU4bBHQ8PmL1B+Uo2/5ACIB4EZOwbsa8e+j46tWTWPixa/pL6yLGIAkjZfAQY0DM6F7vXpjXcCu6Iv2AkBp0vY61W6kD2vAr+aqKwAVB86STTvKe7yRsOdJP4o0NTz9CppR7jY58Vb733nt7cpCQQMN3cghcMQMI4eqtHSTcenUQ0hJtb1yNlIgCV/fTks6wbkBCo+wQITtxrssx3HatKxC49qqRL/nyGJk8ebL3iib9CGQ9jCFbbLGFN35h4NKEtJhnjZyyrXK8SxECJk6c6A455BCfTz3W+gvqj05L6ot9eTx3vGumX8ALwUc/+tGV6iSespgn4NVOb5kq/QRpFsOmpD911G0jNh5og2qMwJXyWEJ8mnwshDKpb9wv0kfWKz/C0JJTH/X7+px+FU+Z5F8LaabNQ+KjzQjRmmdkjAnJ+NzThIOQXM79egSunD6qagJX2XJuN4GrbFuJ1Ws9f4yR7CgnREjYev2g1w5sv4QXmFDE05Iex/V7zRC4cto+6Ss77+cdPc6VmYen+mvCrCc6virmR82MD1WnpZnxQael3eUQI3DF+j3Kdc899/RrS86LzNk1ySKWL8JBhDiht9Kr1wfzTiwt4oWU+0ijcYk+n3TFpOp5tB6f9fpJ8ilEGUmL9Pu6j+FeK8ddiTt2LLuO1P2yeAkNw9XejYSIJTqdVB0kDDw6MU4g8p7/kfij06LnPq3qx6rUz5ClsmNiAoaOy6LziHn7k4c0kY45QG7b0n0AW9TxcUE90fMrtrVj/RUTPc+SbRn1GKzbmLz/xS9+0ZPF+E2ewv5B+iFN4Gple6tynqsJVUXX1KwThDxbFuuc+C655BK/PTxb8SJ6q0p/4a0/eOvjI07tLZpboa5G3smZ+2gCF/HhSd3EEDAEDAFDoP0IGIGrIOZG4CoIlD1mCBgChoAhYAgYAl0SAVH2aQMMCdWKnNi2QDwTErgw+KPYKiOXXXaZu+mmm/wrO+20kzfS8kPc6h944IFun3328fdjCiV/I/JHK/RlawH9WEpBiJcMvjJEoREKClEEcgMiBC6tiGHbNr5cjIkoskJCVexZfe3oo492W2+9dactsBrlTxvgYvFVGabUA9IcMyrpvOhzrZiEEEP5IvoLQ/HgpLe80t5RmqlzKYOATqP2Fqevo7icN2+eV5Jx3Qhcb3vg0u1YY1bvXOpNo3otxlmtIJZwqyJwEZ4oNPlqWtq7xCPHK664wk2ZMkV+Jo+Q/sgf5INQMLjh8lq2OhSiVfhc+Fv3OYJd+AweA8ETkS+3U3UeQy5b1YakLeo5BjD6Ewxy/EZxr0WM+NyD6AHBUhT5MSNSMwQuMcoVVUzrdOpz3T9CHr366qs7trGkn4ZILFvQ8nvSpEkOj03kUXs1aKZsdTtpdrxLGdL01/vylb3GQc51u2tE4NJ1r8x4p/Mr8TY66rq92WabeSMYXh/Deko4tCO+9qfOiYEtrCf14ku1DXknZcTEAwntV28FK+/IUZOSmTfMmDGjqbmShBsem6mPYVj6N2My7YEyiJFPIW+ffvrpnkzHe2JUl62RdVjawBjzlKgNi9oDV24fJX1G2BeljFk6rfo8p5wxvDcicNGv4MkQ0dsSpdq0TlPqvExbidVrTaZJGQmJWzyb6PVDkbVDKwlcOW0/Z95P/nW/GVtn6HFGz8Nzy7ZRfGXnR82MD1WnpZnxoVFaWlkOMQJXrN+jvsRIU1xPCR6nR44cGZ136Xekv9Uf0ci1MmlJrXd0XPo87FP1Pc6rnEenxmfJZxECVzPrxTBvZX6ncGU+mVpH6n451QfHCFzS/8XWSJJmTSZa1QSuWL9ZpX5G8lxmTJR3UkfBmC1Q8T4UE9lyXdZMqToQe5dr0raYA7O2RYp4oMerFroVJOUhl3t6nGYdy3pWX4vp/TSBK1Zusu6Tuteq9taKea5e1+h1B1iJhGtqdFG5WOfEB+lX68dS/YLMecP1T2zOmzv30QSuFF6Cmx0NAUPAEDAEWoeAEbgKYmsEroJA2WOGgCFgCBgChoAh0CURqJLARQZl2xUUB3iaaCS4YMftP4KyBwUQBkJRXsmXZJoc0ChM7jdS6KcUhEJGIAwMsni9Qgn27LPP+nRCcsCYjwiBiy3Q8CaFxDx3cF0rsrQhh3uNBDIBpAFEvsBslL+UwULiqjJMMTIQdj2FocTNkfSheAcXRLyS+B+1P7rc8fCDsoutuxDew/OHSG6dSxkEJFxtUKL+UW5s3YXXEIwiiJDyjMD1NoFLk+1oN5A86gkkJtkKrVG9LmugrBcv93R82hCn38MbE19wQ6LE45TUWZ5BSS8LZ/2OPteKe8hNKP2pRzNnzvT9nPZilUPgQik7a9YsHaU/jynjY3UeL4cQXyRfbClHOyad4mUMYtaYMWOihkRtDMLjCqRLCKKI9hroL9T+NEPgkvIPvfDFFNMSX+ooSm7aMuWBVy4xgELaIy4E70Lgxja4c+fO7fjimnvNlC14VzXepQgBkJ8hQSP1PDjhRQNiFBL7st7feOtP7njXTL+g48cojHdMiC94tYAQJCLeIYRUrD2SyDOpY6xt6Ge1QVXi4b4Qe6Tu6HfkXPcz4m0wd9ySMGPHZupjLLzYNQyhO+ywg99Smq/8RbSnEzGqx8gDOQSuZvooaecyp5T0lu0zcsu5EYFLewOsisAleSzSVmL1Ws8fU95fiEMM2V2JwJXT9nPm/eRft+uYQVvjqOf9qf6aMOtJo/hkfBQDug4rRnBvZnyoOi3NjA+N0tLKctDzxnr9HmVRlsCltyyr54FL1iGaRJyTFjwNM29EmPdor5b+YvCH/p1tFItIs/Po1Pgs+SxC4CKdrRh36+U/dx2p++UUUSNG4BKdTkje0GnU3pu7E4Ert5/WeS8yJurnY+cy7tXzTvn973/fDR482Em9zG1bjCvjal6RkUYfV/CM1m3V8wqFl3n6I+TXv/61X/PF1oz+gbf+lCVw8Vor2lsr5rmaUFV0Tc0HCKydkbJY58RHf1yEwCXzgCLr5Nw2ZQQuX+z2xxAwBAyBVY6A6KGZk2NDQ7/Ibiv851x+F01or1122WVF0Ye703NG4OpOpWVpNQQMAUPAEDAEDIEQAVH2aQMMzzRS5PCMGEq04U6UARBeUAwKOYvnRSDiCBkHr0tMOEW0Iv7ss892xx57rL/FNkkogoqKDidmWNFKLjGs6O0ayRPEpDD92muUELjICwpRJKXE2XLLLTvyIvEVzQvPiRESwhu4N8pfymCh46wqzFBBfeKJJ3baTknHKedsh8V7SEh+4tq+++7rt1LinDqy3377+e0Lw3rK/dw6lzIIECaiyzq23YmuQ2EeYsbhRvG9GWvxv1oJH0tfKqSUwaPeNon17oV5ZfGIQQl5+eWXfd2NpYXtBDHqoOCGXII0qteimCxqoIzFq6/p+MQQxxZxKEkhn8n2rvIOeYNcKN602NIA0lU9kTTTJ9Lm+OJei97GKIfAlSKN6vpBG6GOxuqgJhCk+iadh9ADl24HGDQwIrL9IEJd02RLrqUIXHgxgiSBxEgDmlRVRDHtA6rzRwkbe8YAAEAASURBVHtFgnS2zjrrOG2QQTFP3iBt4QUDBYj2FEjQGpecstX1r5nxLkUIgGyDkQ9h68/zzjvPn+s/5EuIZCGu+jk5zx3vcvsFyMZ77LGHjx5PaaHBGDIRxhBE5iLi/YBrse09aL/UNfIO+ZE+LNY2eF9EG1RjBC6eE3KWvCNH8VTEfIK+FMkdtyTM2LHZ+hiGibETLz0IpE7Iy1pos+CLkhCB6InhWMaYqghczfRRMs+pisBFPsuUsyZwsTUvdVSLeEPlWrMErpy2kqrXYniNzblIK4RP2T5NP1Nk7dBKD1xl2z59Im0RKTPv53ndf8fWGal5eKq/Jsx60ig+af9F50fNjA9VpyV3fACvRmlpZTnIvJF01Ov3uF+WwHXkkUe6mh2FV92FF17opk6d6s/1H018g3zPWI7kpEWv5SAL4IUnFMbMI444wnvXYl1+++23h4/431XPo1Pjs+RTiDKSGOn3ZU4g11sx7krYsWPuOlL3y2UIXMxrxKNvbO5DGrku5OvuQuDK1c/kjImxctTXhMAV1jl5BpI69Yy2Ih995LYtvU5JfYQhdVrmOJK+2PxL0oiehvU3wvusDxuN3TkELklbrk5Q0quPMs51hTV1LtaawFVmDV/lOjm3TVEWRx11lF8zcW4euEDBxBAwBAyBVYOAEbgK4o5S18QQMAQMAUPAEDAEDIHuikDVBC6trLzhhhu8RyqNDUp6jDcoJzFm4llKk6S0QQFllGxfJp6ndFj1zhsp9DXpQEgLY8eOdSjjETzPyJd1Eg/KDpSjGDURIXBpZR0KpRiBSXs4kfgk3CJHCEz8J3yIZWy5gzt3pIzhSMdVZZiiLCd8FOZspxQSNyRu7cWMa4Kj3OdI/YCohgJ00aJF3mMX17WBk99Ibp1LGQTeDPVNIy3ekZDYF5kYMXbffXd/f3UicJGhsE4JgSt2LyRw8YwoWDmnj4EMpQW3/RgOEI1do3Yr4RY1UOo4Y+c6PjHEfec73/EELfol0kib06IJVymFun5evCMQTkhqpX5j1JctMnIIXBBaSLNOJwZLDFuEz3UhXcXq/CGHHOLIE8L2r6HHNN0n67B0HsVozn1wo5/X3iD0sykCF8+IMhxlBGFq0USHkGgUq4P63di5NnzKfU3Q0umU+/RzkBJFmi1bjW0z412KEMD4yXhLPaBcIJ6AnRbIUXhRQ2LEOf0s582Md9J+Cadov7Dddtt50h/vYCz+/e9/z2knkXAZKxhrtXEk5gVO1yUZU2JtQ0eiDaopAlfM0KrJ29rAnjtu6TSF583WxzA8iFmESf2R7YzDZ3Q+xJAjRvWYATHHA1czfZTMTcS4Kekv22dow3iZctZk9QsuuMB7JZQ00O/T/4MvInWR81Sb5l5KctpKql4LgZW4YnM06fO535UIXGXb/vTp07Pm/eRbzx/CORP3W0kcisUn/WDR+VEz40OjvJdNC3jJO5wXHR94tlFaWlkOMm8kHfX6Pe6XJXDp7VVja1LC1F6YLrvsMnfTTTdxOSst2rsL8wQIuXpeSbh6i9u7777bTZ48mcsrSdXz6NT4LJiHZBrp90MClx6vcnQUK2W0wQXmXDnrSN0vx8YbotVlL0Qs7Z2ILd7QnWgJ573ynn4mPNdp0XOfnDGKsBu11yr1MzljYpj/8LesU7hO/acdaNGkq+uvv95v0Z7btrbZZhvv6ZfwWesxD9Gy8847O+JD5ENHqftcO+ussxxlpoVyQ4/EvEPPi1pB4GpFe6t6ngs2et5QZk2di3VufKRV6l+z6+RcnSdpMAIXKJgYAoaAIbDqETACV8EyMAJXQaDsMUPAEDAEDAFDwBDokgiI8hMlLQpztjLD0NtIkUNmYh64+KoTww6KIcLEM5B4qOHLsS984QveMMX7uB8/55xzOO0kWonPjTLbIElAOQpC0ozXATGm/fKXv3TTpk3zZAS2bEJJ1r9/f4nC/eIXv3AYf5BPf/rTflsnzlEYo2BC4Y4BFCKAeBzjfkjgEmVMvXwSL7iQtttuu81/OdksgavKMNleTrZNI4+UPV9wz5gxw+cXA8qECRM8DuPe2g6A51J1gHvacwW/CRNlOB4+tOTWuZRBQMKGJIcBBYGMxjaPGMZQfmJQpk6IaAMm12LG4VR8u+22W4eXE/GwJuHWO2oPSxADMLDXE7aJe/jhhzsMO6HBQ7cZtsKDLEGdROrdi+VV5wkFMQbgp59+2odFvf3sZz/b0Za0FysdTxUGSh9hnT86PjHEaVypn+eee24HyZQvlunDIHMiKNNDj0BhdGLI4jqKfvpEtlvbeOON3cc+9jHv3UneKboFqVb+8i7EG7wlQC6CFAB2ECUQTcqK1UFt4KBt0QcyDrD9B+MARE/pE2mDkPk4asGb2kEHHaQvuZhxjAc0MSo0Hmll+B133OG972H4YNta8dhHGFUQuAgHbxVCyOU3fSxeuBDaN/26SNheuF5F2VYx3lFWbNmC0EcxrjMOMZbrfgwlD+M85DqEPFIeUr7yJb6/WedP7niX0y/ocYr+iDkDni4Ryo7ti2ULY/GmxXgDHuIZCmIMhjTwIA2Qb8kz9RhvFOASaxsagpQRkz5APF7wPPX2xhtv9P0C+IK/pIN5gfSDueOWTlN4XkV9DMPURJ4rrrjCTZkypeMR5jXUH/Knt2ySeWVVBK5m+ijpU7ShkgzExq2OjEVOcstZkzBIw09/+lPfv3KdbYxouyKawFWvTcvz4TGnraTqtfZsx3gFcRJCwFZbbeXo7/FsIqLnP0XWDq30wFW27VNHc+f9ev4Qm6+kiEM5ZQvWjeKTsaQogYswc8eHVqQlZ3wogksry0HmjaSjXr/H/bIELt7RfTrjHp46aY/0uYceemiHd8qwf8tNyzHHHOPbOHGzpqBtQIxG2HITApeMnfStoUdZ/2DtT9Xz6NT4LPkM52fS74cErmbGXY2NJvtLnmNH3b7KrCN1v1yGwIV9irzLnI50XnnllX7uw9obQo2e84Zz8FgedFo0gatV/ViMwEV+cvrpnDExhoG+JjobrjGHvPTSS72+g/kl6yXGR4T5KiRIriO6/pRpW5osTR+A7ok2ueGGG/ryJI/IxRdf7O66665Oc1n6CubMQuJCf0Ga5ONIPJxTR5BGY3eOB65m2ptPVOSP7hO70pq6DNa5a3jgkL6N82bWybltiniNwAUKJoaAIWAIrHoEjMBVsAyMwFUQKHvMEDAEDAFDwBAwBLokAloJTwJFCdpIkcOzMQIX1zFKoagWQcGFIgsloAjKJ5RSoTcQ7oeKjQcffND96le/klcLHXW+YoaVmIKQgLWCit+iKBdlKPgMGjSIW15xByEGggcGApQqokjjvn4XDCSMkMAlhkSeR5makuOOO86TPkgDRuBmCVzEU2WYfCmKMVcrh1N54TpGbhTLKdFGW5558skn/ZeHsedz6lzKICDh83UiHuKk3LhOGeFdCAnrNXUZYwIkGinTJ554whO/eD4Vnya/QJzBuFlEtIGkyPNC5EkZPHSbl/DEo0q9e7G88r7+8pzfYIcIfpxrfPjdqN3mGCgJNyU6PjHEQc6iLQvpgnKGnEC6dd1ObZcaxqXLl3uEh0i9ot4QrvyOfWXtX1B/wj5Sbul+hmss6ulnRWJ1kHxBdhGFPs/SX+u8YhyU+5QjSn+9vSR9u3jM433SEfNEyL16BC7t1Y5nQ5H8VUXgkv6PeML+l/IX70PcjxnRqijbsCxzxjvSJ+2ac0TqJ+UGCSccm6hvUsd5PrVlEvdCaWa8y+kXQoxoj9QF8iTtht/UQSHg6W14ST/3EXmec02YibUNnhFJGTFDYo88L3VVfkP2xvCmJWfc0u+H51XUxzDMcBymndD+qFcaS+2NUOpiVQSuZvooMXaFBIfUuBXmX37nljNthS009bgnYXIET7mn6yP3BEfOEWnTb/6K/y3bVlL1mtCZO2P0byRdicBFWsu2/dx5v54/xNYZKeIQacwp20bx5cyPcseHVqQFXHLGh0ZpaWU5yLxRl2ms3+N+DoELL53kT/e19BmM3XKNsYYtFvlIQ0TqV9m0gBXzBb1eD+eoxBGSeSVeOVY9j06Nz5JP0V1I/NLvhwQu7ueOu3q+yPxXtn6XOGPH3HWkkO0JMzb35HrMAxfXtcdKflM/IBHJOKPnJs0QuAhb8OccKTJGNWqvVetnyo6Jb+Yk/VcTuNJPOU961luM5rYtxmDGKD1fpzz171A/gu5Ck6x5nnKXOkC6Q69+ep3PB0GsCbTocTI23skHMSGBOLe96bj1eSvmuWEdkfh0W+FauKbmWg7WzcRX5TpZlyl5CceWlM7TCFygZWIIGAKGwKpHwAhcBcvACFwFgbLHDAFDwBAwBAwBQ6BLIoByGG84YpwXLwpsDYeSAEltX5gicPHOrrvu6t/XCiauI2ynBukp5bkGLzcY/0Uwfs2dO1d+Fjoee+yxjq2LkJiiKaUgBAc8zJAGLShxHnroIYdXLryUiHcbjMUYTRAU1rzLF4daePf888/3XpZ4L0XggjSBIiglELZQfCKzZs3qMKzF8lfPYKHDrzpM8scXvrJlhI5LziHvYWwQ72VyPXbEW4wYErTHs9izZeuc9rDDF7S33nrrSsHSDg477LBOilIeQqlFHSaf0k64jgITb1ii4NXK7FR8Whmp6xPh1RNNhKn3nNwTAhdkHb6cDg0e1Bm+zNX1F/IPC8N692J5lTj1VhJyjSNtAu929C3ydTLXG7VbMVDGlKh4noJwhsTIEv5G8EfHpw1xeJbhy3lNYtKvPvroo+68887r8FCm78XOU8pa+jWIBGyzBQESQYFaj8jJMzq8a665xr3vfe9bqY6inMdbg+47U3WQLS3pW4ScShwIaYFkSd9Hnyz9+c033+zw0qaFNBMOEhoT9HO63saMR9pjgbxHfeHLcsqYMSskcNWrgxJG7Pjud7/bHX744f4W4xL9jRa8M0l7SPU/uiz0u0XLtorxjnghLey///4dZTRz5ky/lS33aO94jov1y2BLWWpCHu80ktzxjnDL9gsYnWirlH1MIOcwptCnaMHL0cc//vFORivuU6/xEAdhRiTVNuQ+fQH9O6K9UJxyyikOL3H03WwHi6dJMazLu/WMvWXHLQkzdWy2PsbCPfDAA71XiTBfPEv/LV4+5F0ZY2LbqDJHYE6HxPppvUXXb37zmw5SQm4fJd4awvGubJ/RTDnT7vAoqkmU5B/iE9sqMmdCQgJXvTbtX4j8KdtWUvVagmZsYY4iaae/AEs8cn3oQx/ycwO9DXKRtUPMA5d+7w9/+IP3NEsatAczvFficUJE+ueQnCfvFW37ufN+PX8oOw/PKdtG8eXOj3LGh1alhbIrOz40Sku99VCz5aDnjfX6PfKVQ+DivVTfxz3W7GyTxppQSzNpoT4wH9QEEAmb+Bg7+d9IqpxHp8ZnyWfYv9cjcJHunHFXE7jYqpItK4tIzjqSj3lkvpEi9acIXKSJbdEPPvjgleYizFMgFcm6NTYHD/NUb4xotv3E+s2q9TNlx8Qw/+FvmTtAcMKTMusSLcyJ0PmE81GeyW1brBMg24juSeJjPGYexTyCdZEWvX2zvs55rE7pMTim99Nkn1i5CYErtj7PaW9hmvXvque5Orwya2pJU1msm42vqnVy7txHx8+29eKlUfCwoyFgCBgChkB7EDACV0GcjcBVECh7zBAwBAwBQ8AQMAS6NAJsi4JxHvIJRqUqBIPPFlts4YlUkHAgFEydOtV7KKoXPu9hiMBYiDcjFLHtFgzFYEJaUMJBxEFRhpAXyGGQHdgikHxp2WijjRxeK0g/W5tB9sBg3JOEurTJJpt4L2HrrLOO39YLHPgfKhmrxCW3ztVLAwrT7bff3hsz2OKQbYRkCzLewzjLV9YYEflSWhOS6oWr75FuCJExpap+rh3nGIvYEgyjLMp+LfXu6ef0OeslyEnUB0iKbCMCGQdDUFcWlP4YoNgeFLIKngwwqlDG5KGsjB492oc3cuRITwBB6a4xoE8hHvoTyDf1RCt/IRbQZ9Nn4aUAjwOQROl7yghtlnqOQYIw2M5Rlz/luOmmm/r+j7SHCltNzGLLDrafzBXpYzEggjX54avurirNlG2V4x2Gaki51CsMumEZkU7GNbZeoZ5BRGIMa2Z8yh3vcvoF2s6OO+7o+1zqK9vQMGehT5bxOVZHyC/tmHzivZK2QV/UCiFdjP/0d8wbaAeNxryqx61m6mMKExkHyRdGH7bOYjx84IEH2tY2m+2jUnnLuZ5TzhhhqYfMDWl3EF0bSaM2nXo/t62kwiMdzOVobzLHYUsrcKhH2E2F167rZdp+M/P+nPzklm1OXEXeacX4UCTe2DM540MsnCLXulo5xNJMOxtX88TDPBHCCO2QeRHHVgn1gf6KeRjzX+YUzAvLSNXz6DJxN3o2Z9wVkkuM4FIvPhk/wbJV68gwfshCEN9Yt6FLYawO9RXhOzm/291+cvvpqsdEwYr8Q1Ciz+JjFzxLN5LctsUWiHwkQFwyl623NhoyZIif87N2Yx7K3Jd+gzVeuyWnvdVLY5Xz3CrW1GWwriK+KtfJuW2qXvnYPUPAEDAEDIHWI2AEroIYM3EyMQQMAUPAEDAEDAFDwBCoDgG8Peyzzz4+wPCL++pisZAMga6DAF9KT5o0yXuRgeRoYgjUQyCm/K33fKvvQUjAexVf6se2rGl1/N05fBvvunPpWdoNgdUTAQi5ENMx+uJZIyR5YvDDAwxy9913u8mTJ6+eQFiuDAFDwBB4CwG8FI8ZM8Zpz2sGjiFgCHRvBNq9pm53fN27dCz1hoAhYAgYAikEjMCVQia4bgSuABD7aQgYAoaAIWAIGAKGQAYCfMGIByO+GMWjDF/q4UUEZamJIbA6I7Dzzjv7LWPwCMMWgLQDE0OgHgJdRflLv42Xp0MOOcRv2UKaw23A6uWjp96z8a6nlrzl2xDoHggcc8wx3nMHqb3zzjvdJZdc0pFwvICwpS6EXeTUU0/N8krZEaCdGAKGgCHQxRH4zGc+473oPvXUUw7vgyaGgCGweiDQ7jV1u+NbPUrJcmEIGAKGgCEQImAErhCRxG8jcCWAscuGgCFgCBgChoAhYAiUQABlBtt34clF5MYbb3RXXnml/LSjIbBaIsB2W3vuuac3kLJFn4kh0AiBrqL8Pe2003xSpd/GW8vXv/71ji22GuWjp9638a6nlrzl2xDoHgiMHz/ee9iSvh3PigsXLvRbh/OhhcjNN9/s/vSnP8lPOxoChoAhsFoi8MEPftBvXcf2iSaGgCGw+iDQ7jV1u+NbfUrKcmIIGAKGgCGgETACl0ajzrkRuOqAY7cMAUPAEDAEDAFDwBAoiIBWZvDKM88847dtKfi6PWYIGAKGQI9BQPeXeCx89dVXV0neIXCJgX/58uXu/PPPd9OmTVslaelOkeryI9023nWn0rO0GgI9A4Fdd93VHX744a5Pnz7RDN96663u0ksvjd6zi4aAIWAIGAKGgCFgCHR1BPSarB1r6nbH19Xxt/QZAoaAIWAI5CFgBK6CuBmBqyBQ9pghYAgYAoaAIWAIGAJ1ENhggw3cjjvu6LdNfPzxx92jjz5a52m7ZQgYAoZAz0Vg1KhRDg8pyNSpUx3bb64K2WeffdywYcPcnDlz3PTp0938+fNXRTK6XZw23nW7IrMEGwI9EgHIW3gIpc+ir3/hhRfcI4884h5++GE/X++RoFimDQFDwBAwBAwBQ2C1QKDda+p2x7daFJJlwhAwBAwBQ2AlBIzAtRIk8QtG4IrjYlcNAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQyAfASNwFcTOCFwFgbLHDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUOgMAJG4CoIlRG4CgJljxkChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGQGEEjMBVECojcBUEyh4zBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDIHCCBiBqyBURuAqCJQ9ZggYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChREwAldBqIzAVRAoe8wQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAojYASuglAZgasgUPaYIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAJtQmDs2LFu4sSJbt1113W9evVyP/zhD9sUc8+LZv/993cDBgxwTzzxhJs2bVrPA0DleN9993WjRo1yc+fOdTfccIO6Y6etRGCnnXZy48ePdy+//LK79tprWxmVhW0IVI4A49Tee+/tw/3zn//snnvuucrjWBUBvvOd73Q77LCDGzFihJszZ4674IILVkUyLM6KEWBOdcABB7i+ffu6GTNmuOnTp1ccgwVnCDg/d6+ynmHHnTRpkg/33nvvdbNnz16tYB44cKD7wAc+4PN38803d+Rv99139/PSefPmuVtuuWW1yrNlxhAwBAyBnoiAEbgKlroRuAoCZY8ZAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIdAGBLbcckt3zDHHeCOGRPelL31JTu1YMQI/+clPfIgQuM4444yKQ+8c3OjRo93gwYPdggULuiTJ4dRTT3WDBg1yr732mvvGN77ROfH2q2UIgPXIkSN9vTjppJNaFo8F3BmBoUOHenLOihUr3MyZMzvftF+FEXjPe97jDjvsMP/8b3/7Wwe5YFVLs2V78MEHe7KE5ANj0ze/+U35acdujEC/fv3cD37wA5+Dxx57zJ111lndODeW9K6KQNX1DELpUUcd5bN7+eWXO8iyq5Nsttlm7vOf/7zP0u9+9zt3xx13+HOZly5cuNCdeOKJq1OWLS+GgCFgCPRIBIzAVbDYjcBVECh7zBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAyBNiDwmc98xr3jHe/wMS1btsyTOr71rW+1IeaeGUU7CVwYjTHqvfrqq+6UU07pcoCLocwIXO0tGiNwtRdvie3LX/6y23DDDR0EruOPP14u27EkAl2RwNVs2X7ve99zYjdZtGiRe/rpp92ZZ55ZEhl7vCsiUDWxpivm0dK06hGoup4ZgcsIXKu+VlsKDAFDwBBoHgEjcBXEUBYiBR+3xwwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMARaiADEnuHDh3tSwde+9jW3dOnSFsZmQRuB6+06YASut7Fo55kRuNqJ9ttxNUvyeTuknn22OhK4TjvtNO8Fc/78+c4I1KtX/WYLxSOPPNL16dPHPfroo+4vf/nL6pVBy02XQKDqetZTCVxs7c02vS+99JK75pprukTZWiIMAUPAEDAE8hEwAldB7IzAVRAoe8wQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMgTYg8N3vftcNGTLEE7e++tWvtiHGnh2FEbjeLn8jcL2NRTvPjMDVTrTfjssIXG9j0czZ6kjgknHhqaeecj/+8Y+bgcfeNQQMAUOgaQR6KoGraeAsAEPAEDAEDIEuhYARuAoWhxG4CgJljxkChoAhYAgYAoaAIWAIGAKGgCFgCBgChkCXQmDixIlu44039mm6+OKL3euvv96RPr7WPuCAA/zv2bNnu2uvvbbjHicYQrbffnu3fPly99vf/tYtWbKk4z7X99xzT+8Fiy1QFi5c6F555RU3ZcoU97e//a3jua222srtuuuu/vf06dPd1KlTO+7pEzw9DB482LEt3SWXXKJvdTrfYYcd3GabbeY4Ei/bet1zzz1+C8XLL7+807OTJk3y2yyus846rm/fvg4vIRiaf/e73zm2e9IyevRo9/73v99fIhyeJ02jRo3y4Ydh63dHjBjh+Podufnmmz2pbPfdd3cTJkxwa6yxhnv++ed9vu+66y79WqfzoUOHur333tuNGzfODRs2zKcVrxd33323e/HFFzs9q39svfXWbo899nDrrbeeGzRokMfhueeec1dccYV79tln9aOumXSKof6JJ55wZ5xxRqdw+UG6d9ttN7/VGuVInh966CGPXYj1Si+/dWGvvfZym2yyiS8zvDJQ7wiD/Fx99dWdXitTtp1eTPwYP368bwtgNGDAAIfSlHhvvPFGN2vWrE5vaQLXd77zHbfTTju57bbbzo0dO9bjTx2jHdCmYrLBBhv4uKiXlBntivjuu+8+d8stt/h86/f2339/X76U53XXXecOPPBA3y5J58knn6wfraQcdIB4X6Fukz8Ik5TLggUL3Jw5c9xVV13lMdLPkzeef+ONN9zkyZN9XZY+iHoxd+5c783l+uuv1691nPfu3du9+93vdltuuaVve/PmzXMPPvigx+aEE05wI0eO9PGfdNJJHe8UOdlvv/18e1x77bW9Rxn6mUceecT96U9/cmzBGgp9y6GHHurx5B0Ezxa8Q75D2fn/s3cmcP9OZf6/ZaZSjUojQiFiRIlJREKElCVrWUpJlK1lmmKmUqF9poVI1pCUrSIhS5YQIVtlCZGmlKlm2mf+/X/vk88z1+849/297/P9Ps/v+fG5Xq/n+d7bWe7POec6y/W5r7Pmms3KK6/cPPDAAylO9N5aa63VgAcY3nvvvcmDzXXXXTdX0NpwMZLnP//5U/WP62xjd8MNNzTXXntt0o/xWR0/9rGPbbbeeuv0fugeyos6eMkllzSXX355eowyoE2vsMIKDc8jxIu3Q/oDwvSRvjpqtdVWa/hD8PZDWiXZdtttU12kHlLHoiwzQA+No/OVJrp6u+22S+0TXY+u+/nPf57yT78UJSdw0Z7pG1dcccXU96Ezb7nllqRzYrh4PETvdb3fbbfd1tDWasuW9sS7U8cR3vv6669P/eu3v/3tqSwPbUeqA9RF+mnON9lkk7RN4wknnNDceuutza677priJx10M3mgDWC/oc+89NJLU3vjIeov4xj6R+oLz+OdhrHKUOnbRy+11FLNxhtvnKKnDzv55JObP/7xj3Mlx/iFLaDRPfQXtDvGU+ST9su7kW/6FrYvZXx19913J90CNm0yRBf0wXr77bdPSVG2YJvLkPTG1XV9dFaeP+r4RhttlMaMjOXQXdR99AtjmZIw9nvlK1+ZwlCnKEPGYLfffntz1llnterUUly6NqTdjtuHKs3S75BxjsIPbcMK17e8KKOuejZ0DFJD4Npyyy3T+PjOO+9MY3jiQK8sscQSSZfQHpm/IMw/GKMsvfTSqW7Qt9N+qR8lqamDxIMeoc0wJiYO6ut3vvOd5tGPfnSz1157paTQkeqvGXcxfmXcSX5y6Ttny8P53AgYASNgBOYNAhrvMWelL2QOTn/AH8c675u7BeZ0bH/p+/D89JwJXPNTaTmvRsAIGAEjYASMgBEwAkbACBgBI2AEjIAQ2G233ZKhkPMzzjgjEUR0b5tttmkwKiMY49/+9rfrVvrdf//905YckKS4hyELweMVZJU2waDxsY99LN3G+LDvvvumYxag8KKTC4YKyBnIr3/964btEdtkv/32myKk5c+85S1vSZcgiuyzzz7JGJo/wznvCpkNY4gkGn0gsmE0ZmEMwah+5JFH6tGH/GKIBWcEIw/ELYWND2OA+cxnPvMQwghp77LLLmlxLj7PMdifeuqpU0aaeH+nnXZq1lhjjXhp6phwEMYiGW6cfHYRuDB2YnwuCeQkCF/UiVHyzne+s1hmxME2mUhN2Y5K97Wvfe0UcSR/FhwhTUVyowhcEAIwwENsyoU6BmYY46NgKITQ1CYY+g866KBEptEz8jYHmQkSIoZDieo855MqB8WNAZa0MRi2yYUXXthEcmMkqYAZRAYWmnMpEQFJj3IWYSqGAReM7094whMGEbh4/q1vfWsybMb4dEyc4I3ekWBAx0BKfkqCHjviiCPmqtNvfvObExmGcr/sssta2wPkr/PPP38q2tpwREC5EH6ZOaSlkkA6QA/nBEp08p577tlarpDsDj744GaLLbZoIB6U5N3vfvdcdbT0DNeG6ChIr5ChEIhw1L1cIPCpD8m37Rta/8fR+eQLchD5Lel67kM2op6o34xtAzIMBLmSQBL87Gc/O9etGr3X9X6QETH2l6RP2X74wx9ORNc8PMYmwiM17QhyFvlG70LypB+WQISClPjRj340XYJswdgBgkkuEDMhxkGkzAVdQv2m/vSVIX00BCDih2CLUA/o9yUQwRjjSC/yXpD9DjzwwESKo1+BaA/hKBf0y/HHH/8QcmONLhiCdf4ONemNo+v66qyIF/gxXqQPKAlkWrCMAumRsW6pTvEc/QTbhvYlANa026gnhvah8V3y46HjHMLXtGHCDSkvsFabzutZzRgk6j3GJoxRRonGdOgv9AP9TC4Q/5gLMIbLBX3FHAHdHaWmDhIeghjzs1LfwtgJfJFI4NI7QPZk7hZlyJwthvOxETACRsAIzDsETODqib0JXD2B8mNGwAgYASNgBIyAETACRsAIGAEjYASMwKxCAK8wu+++e8pTbhwRQUsZ/vjHPz4X4YQtkTAy4lHkkEMOSY/h/QePBgjGRL46h2ADoevJT37ylMEB46u8Jn3oQx+aIkN84AMfSJ5sUgQP/oO8hEcKBHJDycPNg48m70MQkTCMkDcMJ+QPIzlGZUSGUI4haECiwSiDsVdrPIT7yEc+kjwJ8Vw0+nAu4Tm8T+SGPt3nNxKjdB1iD1/C46UkEnzwJnTsscfqseQ553Wve93UOYZByAsYkGI4DHmRRBQxI4+khVGecoAAI8MPnrjwBoWMk882AlckaKgsMCDh3Q3vUgjXP/3pT7d6vEgPzfmHwQqPMHimIv+UKWQSiEsQIZCask0BW/7h4WTHHXdMd6nPeJaiPpEH6ovksMMOS547OJehTPd4P/IJUQYPKjL+cv6ud71LjyWjG8RC3o0weFTh3fDyQHnzdS0SCZCci8DFcRTwYXs7ZJLloDSi0Q9CAYZDSAnkNxrFad94K0Oi8Vnx0AbxLkTbg7wgwZsJHs4k6IbYPmkH4ATRRPWZZyE49PXAFbFT+VI/IT2pnGg3pI3gRYljkSsga6E/MCTTtkRmQ5+gP0XOETkhRfLgP8LSninbSAaLOrA2HEm85z3vmSK78W7oAOqQPA7yDHWQ59DRCOVG+sKT63gAgnQCMVDXIZRgsJZXD+4jlCMCiSEnhqUb4d9QHQXmEM6EPWVMWUd59atfnbyScC32FTX1fxydTz2lbqktU4eoJ+C73HLLTb0D3lEwsiOltkE9ou0QLhKqcrJ1jd7rej/KnDpJmjVlCwGQ/h49j1CP0GXoz+OOO666HYlURJy0fdVHjk888cREXBLZg2cQ+lp0Nn0thJlcMIChg+L4hHHLoYcemj9aPMf74NA+Gi898pJDpJA7IH8gkagc8xHLOD045x/6Az1Cv6L+ASwgyXFdUqMLhmCdjx1r0qvVdUN0FmQ4JNfj6CrqJu029ieQAD/5yU8KxkTmJT1EHlvBHcKMCHkiuE4F6jiIZdp3LFrSE3370Las1Ixzcgz79oVDy6uLwFUzBol6byiBS/jRxtAp6Maol3UffQeZTx49uY4up21Lcvz61kG8Mu6www6KZsq7MforjiN4oA+Bq2bONpW4D4yAETACRmCeIWACV0/otXjQ83E/ZgSMgBEwAkbACBgBI2AEjIARMAJGwAgYgVmBAAZQiFkYxXPiA9dlFCSzbLtx+umnp3yz1cree++djiFZQLZAID1BYoAwgBEvbsnIVj8ilGAghgCGQJDBgIREg3a6MOefCDEYTSBFjCIGEE7EjPyd2HIEIz/CwheeMGJ8kVQQvbxEow9hMd5hdO3jOSonRuVGQbYQe+Mb30i0ySiNxx8M/gjEOBmdozGGe7wL2zhShhiM5IUKEgyGQa5TDhggwVuCVy4IDQgEE4y9lNM4+SwRuDBs4Q2HfGBEJx/R49RrXvOatNUf+Yj1gfMuwUCPUS/3xlZbtl1pUV+pt0gkaXEe60Q0YKu+8gzlQj0XgQkDG9sqythL/cPgi+Ata5kHvSXhkYU4JYSjTtO2YllzX3WdY9rIRRddlMh8PIdMRzlAHiRdBE9OvAdpS6J3pkguzI3PP/zhD5ujjjoqESgIG73+sYWcyBjR+xJtGt2kNoJXFLxoCdO8zStP+W+ME6zQV9IF4Ew7FCFLXodop7RXBC9JIg5yjq5ky0p5CGM72JNOOolbyRMW5EMEnCAXicDKtUiWjfo0khqGhMNDB1sJIugx8JIupu2QnvIZywfijTw/RfyJB28r8sIYjdFqI+SPcugjtToq4hHzrTSlL8mL+ora+h/bN/EP0flsIcgfwvZa0dNhJDDEupq3DbbHjVtAxjgjibNW7/V5v5qyTS/94D/1C7Tzww8/fOpWbTuKpCIig0QIeVpkpUj24D51mDxAcEHoFyFyScAXnBHsO+hmxkLEJ09uerbtV3WO+337aJ6NYw3aE7oDnYSnOASdRB6kx2Peqd+06Tj+ILy8coEL/RVSqwuGYB37v9r0YtseoutqdFZMKyfN05/g8VXERfpziKkQbVUncpIWdQavaSIgQ9ShTLuktt3meqJvH9qVF7Vznuk7zqltw0PLK7bpWM9qxyBR79UQuChX5jkaf8R2DH542T366KM5TBLbLfWKsTBSUwcJF/XNxRdfnLZN5TrCVpN4fpREfaRxae6Bq2bOpvj9awSMgBEwAvMOARO4emJvAldPoPyYETACRsAIGAEjYASMgBEwAkbACBgBIzDrEIgeH/AIBIkhGusxpkHCwYiFEQCJnk4weuJdA8HLFUSGm266KXnaSBfDP5HCIH1AkEAwsGtbJYzkECkkePMRMQkSDN58+ohILdFATji8y2gdJ3oGinFCRuHreASjKMSHaPTBAAM5QEbhGLZ0HIlRhCXO3LgXjUAiyrHtoIy5uVFI6URjGJ5N8AYWt8WMns4Uht899thjavsoGbFq80l8MtTHre/wMIKnEQTvKNdcc006jv9kjKKOUfdyXOKzOm4jcNWWreIt/VK3ZZQt1RfeEUIGnlJOO+20FIUMZZyoTGLcb3rTm9L2XVw75phjpra7wohKfYeYxrvkonYKVpEoo7rO87lBj2vTUQ4QOKl7CFt4igSRLsz5F7c9jQSWaHxm4VntXuEgTGFQRCJBL5Yt3p3wThYlesHJ23x8Lh6jf+RNhW08qbtR4pZ7X/nKV9LWh+QNXUjbx/tHLhDtpKOoy/K4EY21pbYcvWrErbtqw5EH8kJdwYAMllEghdL2EOnciD3ETzxc5e2RNgrBAYHMANFFxv+8XqaHWv7V6qhIHI59CMnEviISnGrr/zg6P5JTS0Qz6haEPvoWeXqKbUNlEuGDoCgvkm1to6SjiKO2T6sp25hn9QuRwAUho7YdRVIRHpOEh9KMZA/qI3VUxEWeif1siTQsElTfulzbR5MX9Ag6SGMNSFfROxvkUEiikkgE+da3vpW2vNY9ftFl9AWQiWi/IlvW6ALiG4J1JNbUplej62p0FuNT+nWk1AdxPZJ7GVPRj8fxUanurL766skjIeEhJYvgw3lJYp82pN1GPVHKf8Qk6olSHnRt6Dintg3HvPXtY+iDROSO9ax2DBL1usa+wqHtN47p4piN52O9gGxJn4/+kLCNLvUJUZtmTFlTB/FEjA5DSvqP67Es+xC4auZspGMxAkbACBiBeYuACVw98dfCX8/H/ZgRMAJGwAgYASNgBIyAETACRsAIGAEjYARmDQLRsweeQiBcbL311s2LX/zi5KHpjjvuSEScaBTUFjl9yBIQtCCErb/++s0yD3oYyo3v0ejAsQhh0eAKUeSyyy7rhZtILXn+IH9gOO0ybOFZDOMQwpf0EC6i0ScSLPpkJhp4IJ6Qh1zilkoyUkWSA4bwu+66Kw+WPJdpi7/vfve7zQknnJAIYnjiwIgE+a1ENKMsttpqqxTftdde23z+85+fyxA1JJ9EIkN9JHDJQBm38stfgO072cYTEdb5M/l5G4Grtmzz+ON5JLrxHt///vcbjOeUUZvI2NdGAohb1kQDWyk+jPK0HYzDz3/+81PdzeNVXec6hAt+o0xXOcQ0OIY4wFZeeCzbbLPNpoiSbQSuEgmBeOQRIrbdPmVbCkd8baI42+onxmZt3XrzzTcnDyz77rtviq5LB2AQhWwTyymSEyCxRk90RIg3LBFXo2evmnCUg7wbss2TiFo5DiKwKZ9sA6b3yz3MKCxeaNBVYPad73wnvWMNyScSZYboKPIR8ZWXLa5H0lQkTtbW/3F0fiTkkTfKG2IuZJCSPuaZSMwA2y984QtcnkvYQhLPQEPbRm2f1la2a6211hQJNWaQfKHTJOoXIoEr1rOh7SiSithqGO9JUSKBCwINZR+F7Z3Rv8iFF17YQOCIgoccbY2LR8RRUttHK150OxgzJomivjxeiwSuOEaKz0CIZbyF0K4gjNfoAsIPwVpjllrdQ3o1ui7Wpb46C52+8847k2QaTzKuzCWSJdmmEuwjOZfnGaPSt0FelhE1j6frXP3P0LFo1BND+tCuvAwd50Tch7ThGK5veaHvSgSu0vv0GYNEvT6UwKW+MqaNXVh6pkSqevnLXz5F7DvllFOaK664Io3laupg7OPaxstsa0yaSBxfalyae+CK76LjPnM2PetfI2AEjIARmDcIaOzBxzSQ0xlL0g/yx7HO++ZugTmD+7lXEPqGnOXPmcA1ywvI2TMCRsAIGAEjYASMgBEwAkbACBgBI2AEWhGIBggIKp/97GeTNyQ8reB1C8MzX5EjkA/YlohfFoZK3mQw0rAVGiQiSBAlyQlckdQiD1SEk4cmCAMYV/ntIyK1REM3ni7wRILELYby+KLBRduZRaPP2WefnbZAy8O1nUcCV5thHiMVxnlE2EBOWGyxxdK1nJSTLj74j3JA9E4iOOBJDY85JaFsIFEg2iqtNp/EIUN9JHDJ2xr32/KvvPPMGWeckchRHHdJicA1Ttl2pQVOGMIhBkThfWgbkN+or9p6j2dkKIsemGLYSCKIBjaewTMD3u2WmUN0xIgc8VEcuRFRdR3vD+Q1l+kqB9LBs8SGG26Y8s3icUnaCFxnnnlmc/GcLYBykQcXtd3ooS/WrzzcEFJpyQNVHl9+vsEGGzRbbrllutxVVyMJRJ5VIjlBXg5j/JEcEMkuNeHiVrWk0aft4TUE0pFInfm2YjGv+XEbySd/Lp7X6ijiiN4foy5WnLl3tNr6P47Opy3gwUxkmvjuEDbwUAmBKHrpicSMNmJBTlIcR+/1eb+2ssX73JJLLhlfKx1HkjcX1C/EOj1OO4qkIto7nsqiRAJXSVe85CUvadjeFRFZPYaPbbcPgau2j45pxm1juY7HMDyH5e0WEhHbP+b6P8YVt7XDsxt9AmUoyePU9djPoAvou4ZgLQJXre4hvRpdt34govfVWfo4Qe8+CpM4joJ0A5E6F0gxjL8YC+CNc5SM026jnujbh47Kz9BxTm0bZnveoX1MbNOqZ/F9ho5Bot5r07Mxfo67xnSxLCP5WnHE+YQIXLV1EK96kD4Rxu/Ru6DSi1uyx/Gl3qFE4KqZsyk9/xoBI2AEjMC8QcAErp64m8DVEyg/ZgSMgBEwAkbACBgBI2AEjIARMAJGwAjMSgS0xRIGAQwDMnpjkPr6178+tS2YDFSvf/3r03t87nOfa/BOI4kelXSNX4xkeCzA0wwiklI6mfMPQgV5wJCo7aPw5qMtyErGWIUt/YrUIhIIz8Ttt+R1qhQ2GsguuOCC5mtf+9pcHrj6Gn0UdyRGdYWVR4YHHnggbYWkMlE8o34xHGK0VTxdnh2iUaxE4BqST/IlQ73KiXIkH0OEbeouuuiikUFKBK5xynZUgk9+8pObHXbYIXkeKpGUIC188pOfbNhWCZGhjC9iMcTn0kbgwigKuSAa0wlL26EeQ/Ihfc5LWyiWCGPTVQ4QM9ET2l4yviNeX3h3yAZIG4ELD0MQGnPJCVx4aEOvILfccktz5JFH5kHSucgUsc0XH5xzMXoCufPOO1P5tT2r69tvv32z9tprp9M27xfcjESEEoGrRAzpQ+DqG+4FL3hBI698yvuoX8oSQ7Pe76yzzmogr/aRNpJPV9haHUWctMf3vve9KXp5GFt66aWn2kT0CDNO/a8x9Md3hpTLVokQPiBjloS+hT4Gif1OW9vICVzj6L0+79dWtuMQuMZpR5FUVCIwxH6NcQnjkyiRwPXFL37xIVu/DiVw1fbRMU+0OTCRtG0VLQJXSc8rbCQ3oifx4FijCxgHDsFaxJpa3UN6UW/21XWxLvXVWW1jVGGY/+Z4v+hFL2o23njjZuGFF84fTefConjzwYvjtNs+eiLvQ7vyontDxjkR9yF9IV59h/YxsU1HbGvHIH30njDRb9eYLhK49AGMwvFbInDV1kHpAOIttRGuQ/DS9ql9CFxteRk1ZyMtixEwAkbACMw7BEzg6om9CVw9gfJjRsAIGAEjYASMgBEwAkbACBgBI2AEjMCsRCAa6yCj7LfffimfeIW69957E7kKQwVeh9gmBCNIvvVYNN6x+I/XjWuuuabhq3QIHYiIYTmBi3siYHCMd43NN9+8WWONNThNhthIFEsXO/6VCFzR05i8VZWiwOi+3nrrpVtsLQjZq8boo7gjgYstIEtb9mCMwjiPaBuWiAcEKYgxXQLxC/KKjPzRc0QeLhp5brjhhuaYY46ZawvFIfkk7pzAxTW2bsKNP15APvWpT3GpU+67775eHtZKBK5xyrYzU9lNto9jC6YVVlhhiozII5E01GXs49kSgQtDO9vwiLxFm6Pe4VVEXnrwrrXEEksMInCR3nSUQzT64fGIfNLWIUNh8I7ky3EJXGzbJyIc24iqrvFuUbRVXiyLeD8eQzxjGzJEBMZ4v3Qcy63LAxcetvBeiEC0QxfWkBMIXxMukpl4t5NPPpmoWgW9Amkkbr0kz4OtgcKNNpJPeOQhh7U6ShHJ2xrYQvJlq130LEL7o5+S1Nb/cXS+0tYvnhTpy/COQttQO+c+295hhOlDzBBuquPj6L0+79dWtpA9GA/korqk62qr0QPXOO0ojlNmA4Grto8WPpQfpIycGFwiM4u8kY+7FBe/UV/QT0IiFNl3iC4griFYi1hTq3vyvJfIKSWSa43Oih7P+CChRCImPxLGrmyjmAv1H3Imeof3hmgkGbXd9zjtto+eqCFwKe/8jhrn1LZhwkFoQvr2MW0ErtoxSB+9lzIY/nWN6WoIXLV1MBJMITHzkUYu1EltzziKwDXunC1P2+dGwAgYASMwcwiYwNUTawZdFiNgBIyAETACRsAIGAEjYASMgBEwAkbACMyvCLC1yZ577pmyD3lkqaWWauJ2SLvttlsyVHENbwl4H4BwI9IRAffee+/k5YrjaDjgHIlGqxKBKxqm8MS05pprJs9c+ZZYf42t+3+JwEUIeX5p847EM9Eoe9BBByVvYTVGH+JCIoGrjYACKQgSAnL55Zcn/KKBim0t+bo/F8gteINArr766rSND4QAtu6C3ICRG2N/LtGAdN555yUva7X5JG4Z6uWBi2tgBzGJfOARoLT9JWnyh+D1S0S/dKHlX4nAxaO1ZduSTDJ+yzMKZCqIiFHwMAWRBJIaAsEKslqXsY/novFT7SRuBRXJDjwv0RZx4CmjPPdU13NPIQo3HeUQ80L6eNeLErdYGpfARbwq29L2P9zHyIsughgjcgvXu0RxthEd2ZpJW8d+4xvfaO64445mr732SlGyBd5RRx31kOhJH9IrhIyotyKxoi85gchrwpE2RFlE3vzSSfaPrS8hFoEphDRIiaSHtBFc99hjj2allVZKz0g3tpF80kMt/2p1lKKLxA30xqabbpq26y15Hayt/+Po/Je97GXJUxikmW9961vKdvqlfNAb8l6nrfxi/9fXAxcRqh5PR59WU7bxZdUvRJ0GOaS2HQ0hFc2EB67aPloYRbIndVd1Ah2f61URuAjbRt4QiVUkr1pdQBpDsBaBa5z0anRdjc7C+6K8krVtaY0eR/dDgMOzJoR2xmgrrrhi6uNLJPzXve51zaqrrgp0aaxwxBFHpOO2f7Xtto+eGELg4h2HjnMgrNW04Zryol0w5kNUzziuHYPU6PWuMV0NgSsSp4bUQeYI1EPk9NNPT1t2ppPwj7pN/IjGlxzrHeIYatw5G/FajIARMAJGYN4gYAJXT9xN4OoJlB8zAkbACBgBI2AEjIARMAJGwAgYASNgBGYtAvKOpQzKExTnq622WvPa175Wt9IvWytC/pHgCQXvIgiGW8hKUTCIQYpASgQujH+QHzCeQUbRtlN49zn22GNjVCOPRWrJyRy6TgSf+cxnkkEoRgbxCW8/yoO2cKwx+ijeSIyCAAehICe8YJDFqwmCYe/uu+9uIqkHUh3Y5BK/yD/ssMMS8SIaEq+88sqGraKiQDgiD2xbiWD0xctTbT6JQ4b6SOCKxqHzzz+/Ofvss3l0SihvDI2QbzA48y4lktdUgAcP2ghctWWbx69zcKJNUBe0VZvu6Te+ozzCyFDWRqgoEbi23HLLBtITctVVVz3Ea1IkPQwlcMU8TqocpCtK5DzwOuCAA5pFF100vc8kCFwi4RAh3uLwGhcles3L23x8Lh5HQkS+FSzPxe3RaCOUJ/WV96OeohsgaUVhi6itt946XdKWppzUkBPGCSfjNnHQXiASRYke+KSL8QJION6PcoVcgh6WsP5NG+M+REUIi4hIPhyXyGlcz6VWRymeSNiDgId3HgSiHX9Rauv/ODof726QcagnkFfBM0okOMqbWx9iRu6Bizhr9V6f96sp2/ie6hcigYt+vbYdDSEVzQSBq7aPBqNIQqQ9oTP32Wef5M2J+/TJ6B1J1Fdxm1Ddj1vN4oGOfgip0QWEG4J1iVhDHH11D8/W6MganRW3YEV/o8fy9hm3vdMYivEv42CkNL5lPHXIIYek+20E2HTzwX+17baPnhhC4KoZ5+Bpr6YN15RXmweu2jFIH70Xy4njrjFdDYGrtg5G4leJrMyYmj6CX2QUgWvcOVtKxP+MgBEwAkZgniBgAldP2E3g6gmUHzMCRsAIGAEjYASMgBEwAkbACBgBI2AEZi0CkQxEJi+88MLkFYnjSK7iHIHoFD0mRQMXBkS2zcNbF6QoCCrPfe5z/xpwzv+S8YGbbN247LLLTj3HAYYitvgaIjKO5WSOaLyBTIVnK4yPCPnEmCfiGJ5dwACJ4eL1dHPEv0iM4lEW3PDeA0mLtN70pjc1yyyzTIolfh3PhUgiYYu6k046KRkbKQ9IEKusskoKB84QiBAMShBeIFog55xzTnPuueemY4gOEBrwsIZEwtU4+ZShPsa3yCKLpO3BRAjBmASZB8F7FcZqMEfwLkZZ9BEZ0zC6YiCmbkCUiGU0pGy70pRHE5752te+1lxwwQVTj4MXZYABNBJalL8hBC48dxAXQlxs+8Z7YRTGYIuHIZUn7w2xQkZn1fU2D1zTUQ4iqJBfDNyULZjTdnfaaacGz3ASvJexFSlSa3yOHuooawidN954Y4oTEgX6RfjkbT49VPjHdnZvfOMb0x3yTpukHhIP78BWREhsW1HH0Y6pf7/5zW/Sc+g3ylD5oA1CjkJqyAnjhFt77bWnvKpQLw499NDmnnvuSXnBC8ob3vCG5LGKC/IAxfGrX/3q5PmQY96Pdg2RBJ0COUukvNhe47s7NFy2AABAAElEQVSdeeaZyVvNqO1ea3UU+ZLghQ5PMBLaAwZp2k+U2vof9clQnS+PleQDrCAIUm8RvJ6h+/BOiEDMYYu2Pm2jROCK+Ryi92K4tverKdv0Ug/+U78QCVzcqm1HQ0hFM0Hg4l1q+uhIFCcO9BlEddoF+lwEjLjNXCRwEYZ7/EFgpP8AU/oiJHrnqdUFQ7COBK7a9GJdKxFBS1so8q41Oit6EoScTX8Ljoi2+UOPo1MoDwj3cWs62itEe8axCOUGXhCtEcjqkKW7JLa/Ie22j54YQuAijzXjnNo2PLS82ghctWOQiHub3svLrWtMR9mjA5DYLyqOSAY85ZRTmiuuuCLdqqmDBNR4j2M+kjn88MMT0Zp8MIfThyDcH0XgimU4dM4W9UOcK5KuxQgYASNgBKYfARO4emJsAldPoPyYETACRsAIGAEjYASMgBEwAkbACBgBIzBrEYiegchkTpyKBsQSOWXJJZdMBgQRGIgDw5SMkRjDMOxjkEHwfoBhhG2+JDmJCHIE200NFRk5SmSOnKiGYZ28KZ+klXu/qDH6KM/5O+k6aUasOMfLVvSWw9aWGHr0HM+AG54MJFzDGwFeuiRbbLFF85KXvESn6f14z/iOxEMZ867IOPmUoT4SuIgzekbinLzGOsA1DKdsqUZ++kg09PJ8JL3VlG1XmpFYxXPUZ/IJ8U5lwnV50eG4y9jH/djOZGCjXPDeIfIgz4ET2xtJolc68oHxke3ZVNfbCFyEn3Q5sP3e5ptvrqylcuVEmIARedc5Bm/0xzjGZ8iVSyyxRGua1C3SK7X5qUDZAdtAyejOLeJAlG/OqduQLRHKB2N3bH+UBc+LPMFz+bZIsc4OISfUhiMP0bsG5+QTiTogb6/oZgzj8s7H8+iN+G7UM3Qyv0gs03Rhzj95o9N56bdGR8V48rZJGUGqKElN/R9H50POol0KN+oRxDKwj206euqJOA7ZQpH3rdF7fd4v5km49ilbPat+ISdw1bajSBoo5SOSPWaKwFXTR9N+IBYieRuMBCjqDR60IPOiPyE+58Iz0lfci57/9GyNLhiCdSRwkWZNerW6rkZnQQZDj2ssSp7zPotrOWk7lgG4owNp0zEe+h/GM9K3xNMmNe02tsk2PTGUwJXrUvI+apxT24aHlhfPQ5RGYj2rHYP00Xt5eXWN6WoJXLV1MNc35DWOFeOxxpc8o3eI4+Vx5myQ3yHBI4xDGQNbjIARMAJGYOYQMIGrJ9YmcPUEyo8ZASNgBIyAETACRsAIGAEjYASMgBEwArMWgWiIwAiAcSnKjjvu2LCFBxK96sRn2CJxm222mTJc6x5GA7yQsMUiWylKICxF4hHXMdbIIBa9RylMn1+RWtrIHPFd8vhKWzbiXef1r399ejR6t8jDls4jMQpiB8aySNQhDMZ9jHHXX3/9Q6JguzO8dEVShR6C4Ea4H/zgB7o09fvCF74wYS0SwdSNOQd4BoLsgHchyTj5JC6MyJGQoHi78gFZjXoBwaevQLjBg5AwBDtt50YcQ8t2VLqveMUrGoyF0UiuMJBb8D5w1lln6VIiYlFWbeTDEoGLwJQzRuyFFlpoKi4OMKYS/0033dTsv//+U23r4osvbvB4JE8U0TA3VwQPnky6HHbZZZcGz1i53H///Yn0RDmwrRfCO7CVHPpB7f/4449v2AoslzbjM/jTBqmnuVx22WXJm9tKK600iMBFPOgrjOK50DZoWxBBolC2eFDSdrHxHkZ9ygSDZpQ999yzwfCKDCFw1YZT2tHbia7xSz7ZhpIyoA5HQffyfs94xjPi5XR83333pS0s5VmMixiiaX8ipHANAoMMC5y3SVedLOmoPJ7YVxx99NFTXtny5zjvSqukh8bR+aRHPcXDSSRscV2CzsbrG30t0qdtlDxwKb6heq/P+41TtuSrjcDFvZp2FD3GsMWnvCYRHwLW2mq41I9DaoY4iJx44okNXi2jREJNqZ3GZ+PxkD564403bjbbbLMUHL1IW2GcEiXmQ2RykYfo72655ZY0jsj7JPpfPFmqTsU4h+qCIVhHYo3SHJreOLpuqM4ij9Q/+lt5I1W++QVjPGjlXrTwnLbvvvum7VHj8zpGZ4E/fWBfGdpu++iJtj60K09DxznEVdOGCTekvGKbzutZzRikj94jj1Eg1/OupTFdnDfhFZR+KEr0wJXrnJo6SNzPfOYz05xA8ySlB6EQb3KME5FI4NI75OPE2jkbY3B5AI6eApUX/xoBI2AEjMD0IqB5Fh9VQiZnTMiaD38c67xvLhZYa6215t7wvW/IWf6cCVyzvICcPSNgBIyAETACRsAIGAEjYASMgBEwAkZgxhDAoLH66qsnwxgGf0gaGD4kEB/48htjA+SInEAg8hUkA4gBeAKYDll44YUbCB/LLbdcSgPPFXgKyY2p46YdiVHaWodttCDDYYC588470/ZJOQ4xXRbh2J6OL94hSuC1DCMu3ju6BJITW6bxBzGItNjmJXo9U/hJ5FNx5b94LFpxxRUTiYV3xih99dVXF/ORh207p+xYpIQAqK2M9Oyky1Z1GsMZmLKlEnUbwl0kwSn92l/eh7ZDWVMPIUr+9Kc/nYqONUjqKxhiLMwJDFMPthxMuhwWX3zxRFRhy0SMq+QJ47cE0hLeiChv6t4kBCP6aqutlrYKYltAiEjjtlkWvqmfYIuuIq8//vGPO7PLu1MHITrxfrz/HXfc0cvzSmfEE75JnYFIR92F2MFWSWzjFMuplCTGZcKhr/EAxLvxnm0CiQUPQZAYYp1te17Xh+gohan9nXT9H5UP6hV6Fb1NO8DIApb0e5TDpGXSek/5qy1bhe/6nV/aUdc7cK+2jx4Vr+5HAhfjIogtkMHp26lLfepUrS5QHob+znR6Q3UW70P9o31C5KJ93jVnWzr63S5Bl6oPQqdKP0Kgq5HpardD81I7zqltwzXllb8TaaNjZ3IMkudh3POaOkjfQr1lzAIRlHGQPIUOzY/KnTZQM2cbmp6fNwJGwAgYgfERMIGrJ4YMRi1GwAgYASNgBIyAETACRsAIGAEjYASMgBEwAuMhwBZp8qaEh5IjjjhivAhnQegSMWoWZOshWZhf8vmQjPuCETACRsAIGIGHMQI5geth/Kp+NSNgBIyAETACRsAIGIEOBEzg6gAn3jKBK6LhYyNgBIyAETACRsAIGAEjYASMgBEwAkbACPRHAE8SeK/AkwHkLW1NxjYweDaY32V+IUbNL/mc3+uD828EjIARMAJGYAgCJnANQcvPGgEjYASMgBEwAkbg4YuACVw9y9YErp5A+TEjYASMgBEwAkbACBgBI2AEjIARMAJGwAhkCIg4xJaJELkQtsX72Mc+lj05f57q/ci9tlCcjW8yv+RzNmLnPBkBI2AEjIARmC4ETOCaLmQdrxEwAkbACBgBI2AE5i8ETODqWV4mcPUEyo8ZASNgBIyAETACRsAIGAEjYASMgBEwAkYgQyASh7j1+9//vvnoRz/aPPDAA9mT8+dpfD8TuObPMnSujYARMAJGwAjMKwRM4JpXyDtdI2AEjIARMAJGwAjMLgRM4OpZHiZw9QTKjxkBI2AEjIARMAJGwAgYASNgBIyAETACRiBD4PGPf3yz4YYbNgsuuGBz9913NzfeeGPz5z//OXtq/j19zGMe06y++urpBW699dbml7/85ax8mfkln7MSPGfKCBgBI2AEjMA0IbD88ss3iy66aPO73/2u+d73vjdNqThaI2AEjIARMAJGwAgYgdmOgAlcPUvIBK6eQPkxI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBHojYAJXT6hM4OoJlB8zAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGoDcCJnD1hMoErp5A+TEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEeiNgAldPqEzg6gmUHzMCRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkagNwImcPWEygSunkD5MSNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYAR6I2ACV0+oTODqCZQfMwJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRqA3AiZw9YTKBK6eQPkxI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBHojYAJXT6hM4OoJlB8zAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGoDcCJnD1hMoErp5A+TEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEeiNgAldPqEzg6gmUHzMCRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkagNwImcPWEygSunkD5MSNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYAR6I2ACV0+oTODqCZQfMwJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRqA3AiZw9YTKBK6eQPkxI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBHojYAJXT6hM4OoJlB8zAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGoDcCJnD1hsoPGgEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEpgeB//7v/24WXHDBZoEFFmge9ahHpT+Odd431QXWWmutv/R92M8ZASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMQNOYwOVaYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARmEcImMA1j4B3skbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYARO4XAeMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMALzCAETuOYR8E7WCBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgApfrgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGYB4hYALXPALeyRoBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBEzgch0wAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAjMIwRM4JpHwDtZI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyACVyuA0bACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYgXmEgAlc8wh4J2sEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETCBy3XACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASMwjxAwgWseAe9kjYARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACJnC5DhgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBOYRAiZwzSPgnawRMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAELtcBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIzAPELABK55BLyTNQJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAImMDlOmAEjIARMAJGYD5H4OUvf3mz+OKLNz/72c+as846a9a+zQYbbNA89alPbR544IHm/PPPn7X5dMb+isCaa67ZLLPMMunkq1/9avP73//e0DzMEfjHf/zH5nnPe17zl7/8pTnmmGNm5ds+5jGPabbaaquUtyuuuKL58Y9/3GyxxRbNQgst1Nx2223Ntddem+6tsMIKzWqrrZaOTzvttOZ//ud/mgUWWKDZeuutm7/5m79pbr311ua6666btndceOGFm0033TSlefnllzf33ntvSmudddZpllpqqeZXv/pVc+65505b+ptttlnzd3/3d82dd97ZfOc732le+MIXNs94xjN6p/v3f//3zYYbbpjyd+mllzb33XfftOXVEY+HQG3fSl38h3/4h+b+++9v0PHjCu2S9hnb4bhxOvz4CNTWj/FT7hcDuvtVr3pV86hHPSrpROnKfqH9lBEwAkLgmc98ZrPGGmuk06uuuqq56667dKvq1+OAKthSoJLeXXnllZu11lor3T/66KN7R942nuwdwYQenB/mCLwqOK+yyirprb/85S83/+///b9qBGZy3lCdyRkMWKrXM5j8jCZVmq+1tcVHEi4zWgiPkMRG6SzWeldcccXmSU96UnPZZZc15513XvP85z+/WW655Zrf/OY3zTnnnDNfIsW6xMYbb9wsscQSab3kfe9733z5Hs60ETACRsAIGIHpRMAErulE13EbASNgBIyAEZgBBD796U83j3vc49IE/q1vfeu0p4iBdumll07p3H333c0f//jHXmkqn7/73e+affbZp1cYPzTvEPjwhz/cYLxBDjjggEQQVG5YMFpwwQWT4f8///M/ddm/8wkCbeX3T//0T81KK62U3mK33Xabkbdpy0tb4ksuuWTz/ve/P91mwfLUU09tPve5zyXj/49+9KPm4IMPTvfe9KY3pcVNTvbbb7+GSc/f/u3fNkcccUS6//3vf7/52Mc+lo75x+LhE57whPTcJMhKLKySB+RLX/rSFFnr3/7t35onPvGJzR/+8Idmr732Sven499nP/vZRFS75557mgMPPLA55JBDmsUWW6z505/+NJWvrnRf8pKXNDvttFN65KSTTmouvPDCrsfnm3tPfvKTm0UXXTQZ9G6//fb5Jt9dGa3tWz/4wQ8mUvWk+mQZpCFH0n9YZgcCtfVjpnIPifAd73hHSu7EE09sLrroopFJ145DR0bsB4zAfIzA61//+gbSAcIHPWecccbU29SMcR6u44ApUKbxoKR33/jGNzZ8HIMMGWO3jSenK/tt4/L5YY4AJjGfe+65Z/PnP/85QVUz/uuaN0wX/rM53lK9ns35HSdvpflaW1t8JOEyDqYOW0agTWfx9Hvf+970AZZC8vHZoYce2nz0ox9tFllkkd7zeoWfLb+Q1li3hiQrGdIvKox/RyNQ0/eNjtVPGAEjYASMwEwhYALXTCHtdIyAETACRsAITBMCWjTiC6yZIHBtueWWyeMNr/OZz3ym+e53v9vrzZTPSRmLeyXqh6oRaCNwQeqScf7qq6+eIsRUJ+SAM4pAV/nFBcSZWETryksbKHiV+sQnPpFuQ96CxCWyUiRlDSVwQezCUAMhERzGlbZF/pJBYNy0SuHRzZAcIClB1NECcF/9+3A13P7rv/5rs+yyyyYvc294wxtK0M1312r7VhO45ruirspwbf2oSqwiUA2Bq3YcWpE9BzEC8w0CXQSumjHOw3UcMBMFWtK78wOBq2tcPj/MESjbmM9I4KoZ/5nANXdrKdXruZ94+JyV5mttc7tHEi4PnxKePW/SprPwGM4ahwQD7iWXXNLgWXx+J3DtvffeU57S8ZL+29/+tnnb296mV/XvBBGo6fsmmLyjMgJGwAgYgTERMIFrTAAd3AgYASNgBIzAvEZAi0bzgsB12GGHTW1ZNgqHV7ziFWmrx1/84hfNmWeeOepx35/HCPQhcLE1W1xYmsdZdvI9EIjGmbz84gLiTBO48ry0vQpfah511FHp9uc///nmW9/6VoMeeuxjH5t0EcdIicBF2F133TV5j7v55psbtmCU1Bg3Fbb027bIXzIIlMKPe+2Tn/xk8ih20003Nf/+7//evPOd72zYVvLXv/51rwXSpz3taQ1bNiBs1cBWlQ8HeTguYtb2rSZwPRxq9Oh3qK0fo2OezBPjEriGjEMnk2PHYgRmJwKTJnA9XMcBM1F6Jb07vxG48nH5/DBHoGzZ6lFbqLMdvLZQrBn/dc0bZqIezbY0SvV6tuVxUvkpzdfa5naPJFwmha/j+T8E2nTWc5/73ORFnCevueaa5vDDD58KtOmmmzZLLbVU88tf/nIub5tTD8zyAxHQ/vKXv6Q1G3lKnOXZni+zV9P3zZcv6kwbASNgBB6mCJjA9TAtWL+WETACRsAIPHIQmF8IXI+cEnl4vKkJXA+PcszfYn4ncPE+ELgwqsgDoMhKl112WXPsscemVy4RuHIs4vnDjcDF9pC4zJeXvH333bdZddVVGwi0kLkeqeJFzP8reRO4/g8LH807BEzgmnfYO+WHFwKTJnA9vNCZ929jAtewMuiarwyL6a9Pe/xXg9ojN8wQAtcjFyW/+XQi8IIXvKDZY489UhJf/epXm6985SvTmdyMxq32BXELT4mW6UPAfd/0YeuYjYARMAIzgYAJXDOBstMwAkbACBgBIxAQWGWVVZr11lsvXbnooouaW265Jdxt0kQdl9lMaI888si57i222GLNtttum659/etfb+68884mErj++Z//uVlnnXUavhB8+tOfntxR8wzbjN17771zxaWTpZdeutlmm22Sd6yFFlqo+dOf/tTgzevKK69szj///KkvV9m67DWveU2z5JJLNuQDIc6f//zn6cuv++67T1EWf/k68SlPeUrzk5/8pPnmN7851zMQDXbZZZcU9xOe8ITm97//ffMf//EfKQ+QMiwzj0CJwPXa1762YUH92c9+dsoQA8lbb701fRV41VVX9crkwgsv3Gy33XbNcsst11Cn5Db9Bz/4QfLMRpxRXvrSlzZLLLFE8gB08cUXN2ussUaz9tprp23Q2A6O+v3FL34x1dlJhItxvPCFL0xfctOWIAzdddddactQvkrni8GSDH0/Fud4J9rcCSec0HC++eabN8Tzuc99rsGDEjKJNjKq/PKv6yH8gAGem/iC/Z577mloj23bpg5591F5KWGra+DyqEc9Km0fQL3RV5x4ijrllFPSYyUCF2HQMwhfsuKFa+ONN26e9axnpa/1KWPe8/rrr2/QZ2eccUZ6Vv+os+uvv36zzDLLNI9//OPTM9/73vcSJuisKG1faWvB8g9/+EOz1157pSDEh7cr6tRJJ52UtnJk66LVV1895Qe80Zl4z6JvYPuwlVZaKelTvrwFg9NPP31KVxPpIYcckvQ0Wy0cf/zxze67796stdZaSf++5z3viVktHvNV74YbbpjusWjM1pJRyN+LX/ziVC/B9YEHHkj1g7aYt+EYLh4TjnJC8KSGLtliiy2aFVdcsXnqU5+a4rzjjjuKekHx8HUy+aRfetzjHpf6PMqO7SWi17CVV145lR240c8h1157bepnjz766OZ///d/FWXn75prrtlstNFGCXu21qHcKYNvfOMbDXWhj0x6DNDVt4IxZcX702+TV96b/p16AM5t22oOqe+8NzgilCP9B97uaCe0KQiSJZ05CosUYfiHIYM2QP7RjWyDST4Zh3zoQx8KTzZJd/FV+xD9Td149atfneJE51IvaHcXXHBBw3itJH3rYB6WcIzXaGukQb/y7W9/O53jPeS//uu/kvc7wo3TVkr1g2uM/UYJ+TrxxBPneoy8bLbZZg3EK+oUY1V0EGVC2bfJM57xjGbddddN+hZde9ttt6X3ffSjH9284x3vSMFIqw1nHhgyDh2az1F9MXpt0nq6Dav8+iabbJL6g8UXXzzVf8qFsQ945X0P46VXvvKVKYovf/nL6XnaImVF/821PtJ3bpDHRTlvtdVWqd3R/v/4xz8m4jDpxvoxJJ/kfeutt05xPulJT0px0uYvvPDCpjT23H777ZtFF100bSF87rnnJh1Im6KfoO8FO8YLd999d8r+E5/4xGbnnXdOx8w96D9KQjksv/zySZfhAZTx8LyqE6X8lQhcQ8Y4eZwzMQ4QhqTNnJM56AYbbNA873nPS15Ef/rTnyY9wVimTWp0cM2YmrEJ9ZC6xRbVjNsZbzAfZ6wSpaR3cwIXfTNjCtoC701cjCUYm0ZpG0/GZ2rmKzE8x6PG5fPLHAFdzlgPTE8++eQ0/ll/zti9ZvxHP5LPGyJuQ+pEDNd2XKt3S/FFPYi+Y65JH0y7Rvcx7r7uuutSULABo2c+85lJv3GfeccPf/jDuaIu1Ws9UNOm+s4lJqEnho4JSvO1trZYwoX0JjHH6TtOVDmM+h2qL4eMu9vSHrI+0BZHfn0m6w51+2Uve1lqH4ssskjzq1/9KunpSy+9NM0/8rzpHF1Eu2LswZyR+RZzU9Z74rw611k8u+OOO6ZxGzoGoS9k3MxYm3bJXJR46Yfy9QqeZ62AdRz6K+oiY9gbb7wxrZmx9tZHRo2LtUZFXH3njPR5zB/os3hP5obgCDb5+LRvnKQ/ZEw5VBdEXdpnTEl+ovQdF8cwQ949hovHQ9Y+huqFmI6PjYARMAJGYHoRoA9fcMEFk02KPow/1tP447ivLDDHIFC2YvWNwc8ZASNgBIyAEXiEIMCEFaMjIvKBXp2Fs3/5l3/RafOBD3wgkUZ0YaeddkqGCM7xOnPDDTdMEbiY+GJIwxCRC0ZIDPoQUKIwIcUg0SYYOvbff/+0OMCC3/ve977ioxg7WMDuEhHNcmMxZDYWRxl8lASCG1uAaQuE0jO+NnkESgQutkvEeJ7L7bff3uDNZZSwQPz2t7+9taxZxPnEJz4xRVoiPpFzqDcYtDHk5cJCFHmDmCCpDUd4DMoYKlg8KQmGw/e///0PMZzWvJ+IRrz7WWedlchbShNvUhhcJ9VGRpVfNM5ARmHhrySQdDC0RRn67qPyEuPOjxUWAgqE0IMOOqhhqx8WMMEQEa4c77fffmnhkoVCiCTI97///QYvVeg0dFsuGJ3f/OY3T12G1MFiaUkwElH/I2GobZG/ZBBgERrdjrAwy+Jmrg+pHx/5yEfSV6olHQ+5jnfhOeS9731vw4IhxkAWRNV3/OhHP2oOPvjg9EzXv5gnSGUYyhEmjvQlGFNKQltE18dF3dJzXIvlgacwFhohYeXChJV+CB0QhX6U/rQk4IDBFyM70tXXvfWtb019XCmeeO3AAw9MBIJ4LR5T/m19ZHxuOscA++yzz1RSkJHID8TpXKjf/GFUyftknh1a3wmTE7hEIuQeXvPilqVcQ2L7k0e9v94p/1cakBqpL2ydilBHaOdIrf6GqEpdIHxJfvaznzUHHHDAXLeG1MEYkK/NMabmQr2FCCCSiMaK47SV0tgLXVKqF3l+0G0yQHKP/vdd73pXqjf5s5znW27pGfQJxqhcr3EfUg/YI6MIXH3HoTX5VJ9BGZT6YurFpPV0eumOf+hD8MZIVxL0LTru8ssvn7od+x7ItxDChTtzBuYOo6RLXxI2zg1iXMwnIOgrvXiPY4x0xx13XLrcN5+jxj/0afSrkcimOg8Zi+vLLrtsSjP+Y05BO8AgysIn4wp+KX/0aIxP4TT24BnGBy960YtmvE4oL6XfEoEr6tgYJh/jxHs6nolxAMS6vffeOyUJmQTiVqn+UE6M2XLDc40OHlWnSvPONp1NxqkPtDX+JKqDsX+NBC7m/xixS8K4X95kuR/bype+9KUGA7Kktr9T+Pir+h2vcaw53vwyR4j5pNwgtLatdYwa/8W+V/MG4TO0Tihc22+t3m2LT3UQggc6G8J+LtRD+gXSzoV6nc/JFWes14Qb2qaGziXG1RM1Y4LSfK2tLZZwiXWnZo7TVr8ol9I4MS+/0nmNvuwz7i6lpWtD1wcUru13pusOZY7uJt1cKAvGrRfP+cAwCn0YH2lRb0tCOLZD1Adxuc5ibkPbKwn95KGHHjq1PpaP0/nQgbVkiFslYa2YOXUkkJWe49qocTF9FTJkzkjazHFKsttuu01dHhIngWLb7Br71ugCte++Y8qpl5hzMGRcrHBD313h8t+uPiX2fTV6IU/L50bACBgBIzB9CJjANX3YOmYjYASMgBEwAkUEWABgkZTJfb4AhoFLXk8IzNePfD0qkUEUwwMeVRBNKvUMiwIYGjE+YOxiAQnhXIvknON1hq20yAdhMILgnQNvFSzyaaGCrzAhq/DF2dve9rbkAQEvWQhfoBEvBC59xZluFP4pn/GdMR6zQKY83H///YlkxnUMemKTszDC12qWmUOgROCivuD9QIvAGIDwvEPZt3ksUI75Wp3FKBnHWQSB8AARBOMk9xEWlli0lIiIpXN+SZNBLF/biVBGHWYxApIXUhuOsNG4DfkRT3O0B3m/4BnqPe2BhTOk9v20OEYcvANtQcd4msJgMKk2Mqr84gJiysScfyzwgTcLXmr33ONZLf7VvPuovCj90u9hhx2WiBtafILIxVfrkWgUce0icGGQZ3GZsgV7dCv6k3cGdyQubFFGWsAjjAhHXKfNYGRE4kJiNLiVDALRSJoCz/n329/+NuEb65zukRa6mraErpTEdPDGyFe7ItfiEQVDPobJj3/84wrS+hvzFHHFsI2BFaH948mEvC6zzDJThBDaDH2N2kZbItG4oWd4N0h53EPPqD3IgKjn5FGMc8JAYGN7SEhrEFMU7tRTT00eKCFN4SmFBW3pDL5mRiC00Z67BC+VeB1CMCBDOuH9SY++UelB/Ch9CR3jnq4xQOxbSQ/is+qH6gz1GwyUX57Lw9XUd+KRkQdsaAt4T3zVq17FrTS+yImDlAOESvICptomJAVo+ac08ttxa9Aa/Y3BA7yEC2V71113pXoI+UPXo2F9aB1UnqNu4hr405/RF2rMw3XKjLJAxmkrpbEX+rfNA5f6YtKNdQP9D0bKI22Gtkodi3Uqb6t46sOzi4Q40a+0U3nD071RBK4+49DafMZyAXuVOcf0xXiTEoFL+R1XTyuett84jmFshL5Fr6J3YtuGsEtZILHvifHyHnieFIk53ovHQ+cGCounhtiGIQ2glykzCNaST33qU8lbYZ984oVSnioJD3me8Rh9A+M/1cW8zqnOK02FBbvYp9LORdTGC5wINZEMrjgigUBk3dhP6rnprhNKp/RbInD1GeOU4uJafL/pGgdEXJUP+gPVnUhYp/5i9JbU6GDazdAxNV5/ZVhmfENbYxzIuDjqUdorhBhEdTDq0Ejg0jswj6ZeE1d8V+a8IgXEthLHecRR098p7fx31Lh8fpkjxHwynwS/2vFf7HsjgaumTuR4x/NavRvjyI9VB3WdPoB6yzuVPsJAH1Ifuac1IIhfUQcrzliva9rU0LnEOHqidkxQmq+1tcUSLrHuxDLoM8eJ4xHCgveocaLSaPut0ZfE1Wfc3ZZmzfpAW1y6PpN1By++GiOQPmNX5hu5vmZrw0jgjeVHu2POTtuLYzf6EtaRKNdcZzE/goTFGoP6BcZUjC8g7LOzgsaHjGtIT6J6y7n6K+oPHyVSJ5E4Z0oXWv7l75GPi/GAOnTOyHya9gyGjOGkl5ifykP40DjJfmyb8XXi2LdWF6h9x3jpt7vGlDw7dFxMmJp3J1xJ+qx91OqFUnq+ZgSMgBEwAtODAGMFxub0w/Sd/HGs876p2gNXX6T8nBEwAkbACBiBOQiIiAUYLEQwsUbkSSadzPmHgZl96yV4saCTlvGA63FSyeIbcbANA4JxjC+W5aUC7xGQExCO5WEIg37cypFwLABAEMgXBnDJzTZXCESK6PUoXWz5p3zGRb8YF6785S2FKKJBNYZpid6XJ4xAicBFEizscg9p87aRbmb/WDxn6xEkemHQY9FLAEYEFpcQLVBxzCIMZELqvyS2pbi4XhsuGqxYcCM9BswIC1+0Ly18xwW72veLi2OkgfEHA6uIaJNuI13lFxcQwZp8YDCToIvkySISVWrfvSsvSrP0i8EMHcUWAuQTYy5GYsimIuJEXLsIXIqfd6V8IaWBg4Q8sjUbehejIp62IHZIMFizQIdgXKd+IHEhMRrctLAajSKxzhE2bgWJHiSMDNXkASKM2kA0auA1DeM4Qr75w9hN/WXREkIv7UptKz3Y8i/mKRpuhROLwhh1IBVIIuYxjO7nv7lxA+zRLRB5EbbXg6SHRIIPW1nQvikT8kGZgL0kerhiQZg41IZVh6k3IscoXNcv3qEwRJAPvAgqPsLQJtRPxzrQFV/UW5MaA8R+cv05W3Zo2x+uo8eEK8QH8iuiTgxXW995Vxl5ROCKBC3KAeM1uEvwaMdXxgjbljJGGCVKg+coi2OOOSb1Q4o31tsh+hsjCh69kDi+4hx9I6OC2m1tHYz4kmfqAeR5hDb67ne/e6p/iXW0tq0Qb2nsxfWSxDIhfXSPxobUe22fjNcA2oSEOgWJSORI6pcIkugkyE9I1G2csy03njskowhcei72i/k4tDafUX+RTt4Xx7rF/fgutXqaeNoE706ve93r0m0Md4zZ1b9xMRpdqOuQdpHY93DOdouQ59VncK1LaucGsZzxyBtJ/ZEAS78N6aRPPvEWRrtAYj/KOf0+XopV5/BIoQ9JVOd5Dj1BXdUWt5QV/QdtCqHNMWeK/U3J214cH8mj4EzXiZThjn8lApceV9+dj3F0v/Qb3y/26YprEuOAOIYhDzkZj2195F0RnYT3DvqyWh0cdUffeafGDeSPuiOSFuexHkeCvOpg7F9zAhdzXvIgid4ymM8zxuKdYxqxHcTyGdLfKb3SL31U2xwvtgHyRT2YjXOEmE8IXBqnqhzJe9/xX+x74xxTcYFh3zpRwlvXavWuwpd+VQe5x9iFPlrj/7wuyquP4uGdmFchzHPQo4jijPW6pk0N1SG1eoI8144JSvO1trZYwiXWHfLRd45TO04kjTap1ZfEN2rc3ZYm12vXB7rinMm6o3IlP5FUyzljtF133TXNReNaKe2GMQ5zVHQNbSluRSoP2cQhfd6msyIBKB8fq43GtOPcjz6Euq9xI2u6jKn0IaU+giMfbTJqXBzrKjqi7xoJ6al9RV3C9do4Y9skntLYt1YXxHrQd0xJHoaOi2vfnbS6RP1V3veNoxe60vM9I2AEjIARmCwCrD2bwDVZTB2bETACRsAIGIGRCMRF2rggcOSRR6aOmQmWjNMssiEYzph4Iqeffnpz9tlnp+M4qeTL5LiYygNxshoNXUzm8LbF15YQZnLB4ItnpHyyFxfqYnx5+Pxc+YwT9Uh+4Ms1CDFRMO7yRTxfnLEYYpk5BCZN4Nphhx3S1/zUJzwdiKCkN+L+xhtvnE4pay12aYGKG7lHOq5FwynGJLWX2nAYASEHKZ8suEbha0jqMsLiEIZ/pPb94uIYX+WKKJAinfNv0m2ExaE+xpnc0wH52WCDDZqdd945ZS2S92rfvSsvKZEx/kVcxyFwRa8cMtrm2dICHXUGgxuLpnEhUQu0hNOCpYggXItGuJJxNdblSBokLIJ3GAheGGZKuvyvTw37H/MUDbciEfOO4CrDGLGzEPiWt7wlJcTWjaUt82IuonED7GhLEBWiRAO+PGDg3QtjDhKJhDEc+XjOc56TLrGFJPlB2hYx082Of/R1EAUgGkUPIAqivpsvcjEwj5LpGAPEvjV634LwJ5KQ8hW/Ko/haus78crIIwIX1/iCnK2hETDia21JJO3m20XrmfxXaXAdYqU83um5Gv2NMUNlSh8CoY76HYX8QVJCqFt4lKqpg7HultpyXEynTcjIXNtWyG9p7MX1XPDYR/kz9kSi3iJffAyA0EZFrEwXHvwXDUfqP9gOVn1yqX8jKPESP5IbqNLFwr+2cWhtPkki9hmlvEadOBN6OrZhkYxyKKJ+VD8X+x4MTdS5qKfzOPLzmrlBJM2WyE/0T7QxdKj6vlH5jHVHnoDzvK677rrJgMp1dAE6AVGd57jUb1N/IWwhcetWEXVjX54emvNP28uhIxiX8cxM1wnlpe13pghckxwHRGIG9XXfffd9iP6NZBPNA6IuHTIOqBlTRx1VaosY35mL0PcxXkJUB2P/Gt8jEu5jefJBFZ6OEQi+bJUW20rUyzX9XUyrdNw1Lo8kA+n4GMdsmSPEfE4XgaumTkSs8uMavZvHkZ+rDnI9X6+J7Q4CCF6G0GkSPgCgT0eYt9x8883pWHHGel3TpobqkJjfIXpinDFBab7W1hZLuNSO26JuGzJOTAXU8i/GOURfEt2ocXdLkuly7fpAV5wzVXfiBw05wVH5ix9/aA02egjDi2K+gwBzIuZGiAjtbTprKIErjhtLc6S4NV/sS/Q++e+ocfE4c0a1r6hLSL82ztg2S2PfcXSB2jf56zumrBkX1747+eqStrWPcfRCV3q+ZwSMgBEwApNFwASuyeLp2IyAETACRsAI9EKA7WZkaLjpppvStjTLzNmCioVhBA9FGCYQGTYxXuOyH5GhhmNNKqOxj+uS+OV7JIvpfvxlARpvE2uttVb609djMiLybJvhLMZTOlY+40Q931oHI83555+fSGhDDE6l9HxtPAQmTeAq5QYPMLiTZws7vtKUJ4U2AhcLG7ivz4W2RJtCWGxiMTqSXvqGY8EHMgyCq3stsKUL4Z8WyNranB7t835xcSwaEhXHpNtIX+MM74huigKBAX2EsJjP4lub9Hn3rry0xdv3esRV+jIupscv6YlTX/TmhnmVNR6E8HhSEgyOq666arolLyBxITEukmrBUkZsAkUjcNyiTWlBCFp++eXTKV/u4lUrigzLTOx410lIzFMkcEXCACQXSFoXXHDBlNfHIWnH8sADivq/GIcWHbkG/pSDvFfR/ijnUl8BGRTDARLJhopvVNtNAUf8Q+fQX26yySZTJKW+BK7pGAPEvlUGjrw+x1cSWSGGq63vxCsjTyRwRaJYJFig62lzjDFi+jF/pWOlAfFchHY9B0mkRn+zXbOIjyXyCfHj7QcyOfWP7UsgxbH9x9A6qLpLnOqrOI4iYluso7VthXhLY6+YHscYNuhH1QdfffXVqXz0XDREXHTRRYlopXv6RedTpxD6afrdaNiVbtTz+o0eGsYlcNXmk7zEPqPUF0edOBN6uk8b1la55F/4xr4nL0eeq5FRc4PoSa1ksCRN2hnbF9J2b7jhhrlIKaV8xjhz8oHeAe9c9ElI1HWq85HUrzD84kWYuQwS50WQPdZYY410PRJ/6d/p55HoZWmm60TKQMe/mSJwTXIcEIkZEI3zrXZ5XeYIGLgR4S9dOlQH14ypI+GPPuDGG29MH5TIO2HKWPZPdTD2b5HAxccaEEVz0XbXXNfYK7ZpjSdr+7s8vfy8a1weSQazeY4Q8zldBK6aOpFjPep8lN4dFV51MI4lFIZtDylDhDkFc4sor3zlK5tXvOIV6dLxxx/fXHLJJelYccZ6XdOmhuqQWj0xzpigNF8rtUWAKeFSO26TbiPeIeNEnm8TxTlUXxJf17i7Lb2u633WB7rCz1TdieQW8INMm0vcSpUPVPhQRVjzbIlExXWIWeyOwLonf206ayiBS+PGtjULPlhhjReBRMyHiF0yalw8zpxR7SvqEvJSG2dsm6Ux5Ti6QO17yJgyjmH7jotr372rDLnXtvahulqjF0al6ftGwAgYASMwOQRM4Joclo7JCBgBI2AEjMAgBDBIMHmX0V1fRTE5xMMDk0VEXxuLkJJ7P9CkEmM6X1DmstlmmzWQuJDcOIbRji1a+BqMvMjzQowjX/ibJIELN6BMHlkwzgVjDIsLeE7R1k/5Mz6fPgSmg8CFcZh6vvrqq6dtcTBAlKRE4MrrYQwHaYVtVhCFVXsZEg7iF4scEsKWJLYT2py8tdS8X1wc40vOfDFt0m2kr3FGpKf4/nHBX0Y03a959668KN7a34ir3iUupvclcMmzEvnoUx9OPvnkpLPjQqIMbsShBcs2Ald8lucRyCUYvxHItHk+ZpLAxQIoZOLYBsgX5Ee2D4R8DMGlj8Ty0FfIebj4NSrGT/pH9Z1sC8ECe0kg19C3IHFLvLZFzFIc+bVnPetZzU477ZSIO9qCIn+mL4GLcHqPSY0BtAgeyWGRNJXnFV35lKc8ZS4CVW19J24ZeSKBi+uqn3HhOxIfMAxiIOwjSgNDispX4eJWllzL24mei3UX/c0WfiL75VsDKkz+q7IbWgflMYX2ArmpJCKEkn+R52vbCvFrjKj6kadJH4M3EfQ7km/dzbUdd9yx2XDDDTlMMgpb4QJJAZIjIj2cTsK/uEVaPkYNj8112DYOrc0nkcc+o9QXxzo73Xo6EpPYro3xTEmikV3bFsa+J3rrLYVvuzZ0bhCJZLR3SLOjZFQ+Y5zoeW0DlMcr/RLblOp87GdjuJe+9KXNq171qnQp1jm8DuN9GIntIHoypiy0hd5M1omUqRH/ZorANclxQCRmfPvb357qR+KrMr6knBH1sbU6uGZMzXgCYi16OAp6kHqC4R4ie6yjqoNR70YCV2ksR9wRD23xGNuKdE9tf6f5SnyPeNw1Lo8kg5I+ny1zhJjP6SJw1dSJiHPpeKjeLcURr6kOlvRg7GP0IWEMG/uWUQSumjY1VIfEdjFET4wzJijN10ptEdyEdWzvteO22nFiLL/8uFZfEk/XuDtPp3Resz5QikfXZqruHHTQQc3Tnva0lGzbmJebmlNorCasuacPjzjukjadNYTA1eYhvivdUfdGjYvHmTOqfcU2Q35q44xtszT2HUcXqH2XdCl5Lo0p4xi277i49t3JQ5e0rX2ormrOVoqjbT2l9KyvGQEjYASMwPQgYALX9ODqWI2AETACRsAIjEQgbvHEBJnJFcYDGQ30hRnnfBnJ5I9FgiuvvHLKywSJaFKZE7uUgTYCF15Dtttuu6mFBz3PIgVbFrKNHItynMuIyDNthjOFb/tVPvOJuhZ2WJDhq7ySnHrqqc0555xTuuVr04TApAlceJjDkxDlnQsGBeoZJEJEJCyORcRqIyjyDCTEF73oRRwmbwx4d6gJx9fOGMCGiIwYte8XF8cUV57+JNtIX+OMtquLeWkzztS+e1deYro1xyVc42J6HwIX+pavWYeIjGtxIVHXiEcLlnERMBqBWSjHOBElErhK5SIDtohAMWztccyTvEAoLjwRsRCKRzYtXOsev/RF73rXu6aIjfFePI7l0badUInApS+Mo8eVGC/HMe577rmnOfDAA9MjbYuY6WbHP5Fq8kfQW7/85S+nSMgyLufPlc4nPQZQ3xq9xaALGUuURMYJhRunvhO/jDw5gStipy1GGNPgfRFp85CYbmb/2tLgsXXWWadKf2+99daJxEUcp512WgMRZpTU1kEtzreN10hX9SKOvWJ9HtJWiK9t7MU9JG5zSV3AkJQb+WMZ/jVU93/pN/XDPF3SXVyn/5D3vUim4V6btI1Da/NJOqU+I6YfdeJ062m2s8T4g0QPgjE/HMc8MUZmrNzW9+Rh285r5gb6ip84qb/5FtmltEbl84Mf/GDyfBfbQSke1e8SgautnZWMbYpbHhBIlzqBh0f1sfk4NOI/3XVC+ev6nSkCF3mY1DggEjOi17P8PaVz6W9pGzqvGQfUjKkXWWSRtF0n783cOBcIytRZiOyI6qX6V671IXA9+9nPnvIu2UXgqu3vGCd2Sde4PJIMSvp8tswRYj6ni8AFhkPrRBfuNXq3Kz7uqQ6W9GAkcOFR7hOf+MRc0Q0hcBGwpk0N0SG1emKcMUFpvtbWbwnr2N5rx22148S5CjA7GUdfdo27s2Qeclq7PvCQiLILM1F3tA6bJd16qg+RhHU+XmgNOOdGm84aQuCK3oRvv/321B91pdnnXte4eNw5o9pXbDPjxNnWNvWe4+gCte+SLiX+0phy6Lh4nHfXO7b9tq19qK7WjKPa0vJ1I2AEjIARmDwCJnBNHlPHaASMgBEwAkagFwJxosniyK677poWhc8777zmlFNOSdt1YIhlUZjFHCbRSO6Oe9SkskTggqiCkYLJIoKHEohh11577ZS3q9I2PjzbZjjjXpcon3Ginj/Pl264I8cjA2Q25Y/n2K6AibNlZhCYNIFLRjFyj5cp6htfrUOuwNU7ZMJNN900vVyJwNXmDp4AceGLekt9luF4SDgWoLVtIvk65phjUn7a/mEwZOs3pPb9uhbHSumO20amwzhT++5deSm9+5BrJVzjYnofAhfpsSUbnuLwzoZxbpSwFQl1Lur3hxuBSxjQXtgijy2n8E6FQUiSe2jT9fgby2MIKUVb/3V9MYrXH7z/IPRrfGWKtC1ippst/6KhFKM+24eiv/BaIKKCDPxDCFyxjkxiDKC+le3+VFfbtqTiVdVuFY5rtfWdsG1GHjx8Sq9i4Pj4xz8+RUjXVnuE7yNtaRA2pjNEf7NNEMZKRF6M0knHv9o6KM8KIjiVkigRV2rbCvF3jb122WWXZv3110/ZQG/RPthGMhc8z0FWQfAKO8rLHu0Cg0D0wEU/zbVcIO/rI4FxCVy1+SRPpT4j5nUmyTqRCCGvDjEvOpbnXs6ZJzCminol9j0K0/VbOzeIJEDIeBoXdaU1Kp9x6+AuD1wydjOuxHMaojo/xNimvEYvDczF2E4bcj2Sf0Azk3VC+ev6nUkCl/Ix7jggEjPatmfF4yXkX0TbvdXqYOVbvzVjasgDa665ZrPyyisnL5aKK/alqoPxWiRwaa6isPqNdarLo2ttf6d02n67xuVxrjWEwKWxBmkOmf915aUt/1yP+ZxOAlfMQ586EZ+Px7V6N8ZROlYdLOnBSRO4YvpD21QfHVKrJ8YZE4hgEsdrbf2WsI7tvXbcVjtOjGWQH4+jL7vG3Xk6+Xlt28/jaTufzrqjj1xIGzIOZO4u+cUvfpE8GgvrSCrvCse9Np01hMCFBz+86SLR83S6UPlv1Lh4nDmj2ldsM2SzNs62tqlXH0cXqH2XdCnxlwhcNePi2nfXO7b9tq19qK7WrKe0peXrRsAIGAEjMHkETOCaPKaO0QgYASNgBIxALwQgBWB4gKTERFveKLSoy+Iwi72I7pcWA0ZNKksEro033nhqy6A2Q7vcKudfv0+SwLXVVlulxe+77747GQUjcHzhDImIBQnk85//fMPXyJaZQWCSBC4Wp/V1L97d8NCAsThKdDVeInDxbJsBWAuEkawlAteQcNQ52iSir/zTSfbvZS97WfI+dT5r6gAAQABJREFUxKITBpZx3m/U4tik20iXQSQuIPY1zozz7l15ySAffFrCNS6m9yVwqW6hB9HHeb0lYxgW2BYUwWAOeaFtIVELltEgEA12LJTPVg9c9FEsUiJsT5CTMWKfRTvna9cuieUxhMAFiZltAikT0qAd5hIXas8666zmjDPOSI+0LWLm4eN51E0lgkkkWwwhcE16DBAXwfVVbbwW3wnsMYgz/ojP1NZ34u4y8micwhjmi1/8YrPzzjun7Hz1q19tvvKVr8SsdR53pVGrv6PHkzayjLxikTmIJRBFauqgvsim7rJ9I8TQXEQGjGOv2rZC3MI+ljPX8VqJ90rJpz71qbRttc7jL8R69Qlt2xdRlyCEkVe80Fx44YVzeZwRGSHGy3EknZTaV/48523j0Np8Emepz+C6ZKb1tNpwm8GIfEUDI/US/dPW9+g9un5r5wbRC2pbOUu3yCvFqHzyUcu6666bsgsWV1xxxUOyHkkszCO0/aHqfBt2JWObIo/6HLIQcaAjEPoPvCJLZrpOKN2239iWYr/H80cccURqm12eFvJ44/vJE+ekxwGRmNFGOI5jC5G8ascBQ8fUkFVf85rXpL4SgiTE7SjMUZmzaEt49Ah6XXUw6t1I4BKeMS6OY18DmRdPKqW2Utvf5enl513j8vlljhDzOR0ELkgjNXUix1rntXpX4dt+VQdLenCSBK6hbYrxwdC5RK2eGGdMUJqvldoi+Avr2N5rx22148S2esD1Wn1J2K5xN/fbZJz1gbY4a/qf2roTPTYxfsl1P3nkg5nNN988ZZe+gjlEXD8rbcdNn8EaFWNm9XltOmsIgYtMaNzYRsjhgw3G6QhzL+ZgXTJqXKxx3dA1EtJU+4pthuu1cba1TeJExtEFat8lXUrcpTFlzbi49t3JQ5e0rX2Moxe60vM9I2AEjIARmCwCJnBNFs/W2BicLbvsslP3WQwqLdhOPeADI2AEjMA0ILDRRhs1eKW4//77R07YpiF5R1lAIBpfuB0JWiwQsuBOHyLRRF/n/I6aVJYIXNtvv32Du37ksssua4499th0rH98Scq2Rkg0InIeDWd8ufPd736XyyNF+YwTdTxxPOlJT0qkCBa2SSsKeSSvCEbf888/P9728TQiEBegDjjggCmvHHFx/+qrr051dFQ2WHBjyyyELxTZEi4Ki3x8NUidR9oIXKWthOJ2Ydp+lDgigWtIOBEXiUNkSo4l0bOPyBrjvN+oxbFJt5Gu8osLiDLW6735jYZNET/HefeuvMR0a45LuMbF9DYC169+9aup7WtIN5J3coMo9zGiUWeIG3LXHnvskX7bFhK1YDk/ErjwtgXpBJERNZ2Ef2o/bYvH4dG5tjkcQuCKZXvppZc2xx13XIw2GVLZ+uJxj3tcus52jox7EC1iclyq41zPBT2w1FJLpcsYVu644465HolejKQT5nqg42SSY4DYt2oRmKQpE7yQRYmee2K42vpO3F1GnkjGoO6zXS79/V577fWQ7fpiPvPjrjR4VvWP4776G+8ujGUYa5XyhN6j3XKf+Tv1r7YO4kkIYxJSakMRpzj2irprSFshndLYi3UJvg7X+HKUIYetouhTEbwQ8P75eC1uuXTJJZc0xx9//FwGkxJxBP0J9ur7awhccRxam0/eK5ZpaTvjSGahHk430VZ9BXljHECfG4X+E+MLZRj7k7a+J4ZtO66dG8RxWMmrXiQLYrxH743KZzRcor/R47nEMUv0NqY6P8TYFuOORi28IFM/o4cvPTvTdULptv32IXDlY5y2uLge30+Eo0mPA6JxHawpZ+pQlDieP/jgg5PRO7bXIeOAoWPqCy64YOpjK7yxyZtkzF/sN6U7VAdj/xoJXNTNt7/97XN9FLDQQgs1EGkhg0U929ZWavq7mO/Scde4PLa30vhptswRYj5LBC7eu5T/Eh6x79W8IZLvh9SJUvxcq9W7bfHpuupgSQ9OksA1tE3RvofOJWr1xDhjAvXBffpXYR3be6w7Q8ZtteNElXvpt1ZfEteocXcpPa6Nsz7QFmdN/1NbdyKxss2jVdwOnn4KAtfuu+/erLXWWukVcq+dXKSPYKcBRGPvNp0Vx0H5+Fj9ouYlxKdrHJc+yojbQsa5Mc+XJNYb9W3xudj3DVkjIQ61r9hmuF4bZ1s/SZzIOLpA7bukS4m7ROCqGRfXvjt56JK2tY9YvkPGUV1p+Z4RMAJGwAhMHgETuCaPaTHGVVZZJW39pJsMnG644Qad+tcIGIEHEXjMYx6TCEac8iUtX+kOFSY6L37xi9P2a5ACWAzEEwTbtpxwwglFTw1dabDozARulEA+YFDfV/j6hq/3MfjydeVMiCZ1KH8mYZZ5j8A222zTQLCSaGsInVOv5IGKa6Uv20dNKksErrgAwsT/Ax/4QNryBIM3RL8ttthiyrCHkY7FCBnr4oI+i5ksIGOsHyXKZ5yox0WqG2+8MS02yMMNLviZyLIgjLC4kXt8GZWm79cj0Ebgoo5Qlgi6hEUBdKzKrZQiBkbcgstYLEIBxlvqIl+pYUSXsH3Addddl06lt3SPLa74o84RlsV5ffke20dtuPXWWy99WU169EHgQH+E4IWBOktfhcgr3DjvFxdPSotjk24jXeUXFxBLxo2ScWacd+/KSwJ4jH8lXONiugwxSkL6CT134IEHJn1InY5Gcu4xjpAnQHQzZET6coSvcyHOIG0LiVqwjAaBqFNnghiQMjjiX8yTDLeR6ALZGK964Ihg4OYrZLajQyA5QXbqklgeQ4wbGJ5IW/pEC+CkhQGUfgPDAXLrrbemNpxO5vyLdZztsfASxLt0CTqGbSIRSKIY99F9lDuGN3lf436JpML1NpnkGCD2rdFjCfUYkot0KmNa8i38Yrja+s77dRl5olFYWLQZRHS/9NuVBs/X6G/CxS+lGctDEoBsTF2DwLzYYovx2FQbr62DT3ziExMZR9hT/772ta81iy++eMOX8ZSbBH2jrQVr2wpxSbepnMk7/aP6XK7Thtrk4osvTm2EbaxZ00AwWkPC0diPcebWW2+d6hT5pg2KhCGdRzj0AsZe+lbygfELo4okN1Dpev4b9VM+Dq3NZ6nPiOnGNGdCT8c+hLksOk8kLnQP/RRESCSSl2K4eD2+S9vxOHODOGaMZQJZkA9CNG6C2AfBr08+RQwgv8RJ/woWjPl22GGHNF/hXuxPOVedH2JsI5wEL6vbbrutTtNvyTg5XXUivneJiDpXxsJJF4FLmORjnBD8IYfx/aZrHBCN62SAMiOvfLBEnYHktNxyy6W8SYdxUquDa8bUkRB96qmnNuecc07KD//IP2QU6qS8y3FdeMc8RwIXz0BmBFd+l1hiiTSe5L0Q0iAtpK2t1PZ3KdKWf13j8jh+ms1zhJjPSOCK1/uO/2Lfiw7SFmU1daIF8jSXFaFp6JpMW5xcVx0s6UHqGTYJhPUXeclOF+b8i4Rs6WzuKc5Yr4e2KdYMRJrvO5eo1RPkuXZMoLFL7F/a2mIJl1h3hsxxaseJvGub1OpL4hs17m5Lc5z1gbY4a+ah49SdSHjCCyhY0IeyfkWbfd7znpeyGtfWIxGXm3i5YpzPXAydzUc/YEM8eOiifUbdFHXWUAIXxDCt8TNWgsTF2gTpobPZshyJ+U0XWv6NGhePM2dU+4q6hGzUxtnWNuOr1eoCte+SLiX+EoGL60PHxbXvTlpdEutX7PvG0Qtd6fne7EGAseV73vOelCHm5tThcWVonMwlWB+kvqE70X2sETAXZG5VErwbsq6wzDLLpHUCjbFZZ/vyl7/c6jEc/YuHVNbsOWZMdc899yT+xTe+8Y0pe1aeZk246bIT53mrPUfvv/zlL2822GCDhD24/+xnP2tuvvnmBixq6kJtnDMdLseMMTtlzLoIa29twvox69l8sM86P5ix1sW6F3afn/zkJ3MFZd5NHe0rfERF31cj9Nu0H7CkPfDHsc77xrnAHIb53C4z+oZ8hDwXO25e2QSuR0jB+zUHI1Dr2UcJlRZcdY9fFDDt8bbbbouXO49ZONTWCV0PQmKI2yp0Pcs9fQlBnmSgGRVm3PsiNPSdtI2bnsOPRiB+DcTTccGW8/gVF+csFuTExlGTyhKBi86fSb2MP8TNIpq8IHDOgpXuswjAgOeb3/xmGgBq0Y/nkEic+euVh/5XPuNEHZIlgxgGIAjtgfcjHzEvbdsaPTQVX5kUAnHsEj1wEb/KUmnFRW1dy3/50u9Zz3rW1GUZ4Rh0ItQLDAcI9QCiO3VUeivdCP94RmG5DEkXz12S2nCEjx53OKdtkBbtRkI/ApFDUvt+oxbHpqONtJVfXODpa5zh/WvfnbBteeHeOFLCNS6m53U2vjvpRj0VvRVxj7pHnSA+CZM7FgbxmoC0LSRqwTIaBKKRdCaIAcpz12/Mkwy3PJ8bIFkYYVEaQ6vaI/hAzMgnuXl6sTyGGDeIZ7vttms23XTTqShJE50S+w3K4t3vfveU9y0eju+lwCXipO7xy2SexS+9H9dif5nXB9JljAcBaJRMcgwQ6yzp5nqMfCJ6D845zsPV1HfiHWXkyXUyWwXiHXGIjEqDuPL37qO/qYu0TfVBxEO91tiEc9osxhaNwWrrINsNacsV4s1F5cKv5gfjtBXpWJVzNPrnaZfOtQUMC18sgEW9R12nzalOET4nOKy88spJN8ZnYvuJx30JXNH4rDxrHFqbz1Kfobj5jbpjpvR09O5AHqiT1Is4Fsm9U7X1PYQfJePMDSDZMBaIbSZvQ9GLcJ98rrTSSonAk9cd8qlr4EF5xC0WVeeHGtuED3VcW8xyjTT22WefKcKinpuuOhGNxmwLqY8IlG7bbxeBq2uM0xZffL/pGgfkxnXlBcxVxlzjHCwg/kpqdHDNmDoSG0kbnYXuY44c8xg9RasOSu8SLh8/cQ3J3xVvb5AeGdMgXW2lpr9LkXb8U971iMbLsQ7N5jlCzGckQ8T6rHcbNf6Lfa9wIGxNnVCa+e84ejePK56rHEt6MPah4xK4atpU3hZGzSXG0RO1Y4LSfK2tLQrr2N5j3Rk6x6kZJ8ayLx3X6Evi6TPuLqXHtXHWB9rinMm6w0cLbGsrPY+uRvfrAwjyWOqbMJpDPpDwDKJ4OI4fH7XprKEELuKl72AnBUmeNud8YMV4bJSMGhcTvnbOqPYV24zyUxNnW9tUnPzW6gK175IuJd42AtfQcTFx1bw74bqkq++r1Qtd6fne7EGAnQHQIwh6ZhIfww+JE6+1bHffJvHjVz2z5JJLpnW3uKane/ot7a7Bx004JYjrFHqeX3mAjtc4rg03XXbiPH8158zF0fO8W0lYt6Z/xi7cV2rjnOlw+fvwoS0e/5EzzjijlTS4/PLLpw9p4jpGHhdk7DPPPHPqcvyYY+pixwFjXfrVGjGBqwa1HmGYhLEgDyMfQgoDhSgmcEU0fGwE/g+BSOCSh5b/u9t9FL0N8CSGM7wZ4cUHBrcmTBheWITVolh3rE3zwQ9+MIVnsoPCbROMe32MdQpvApeQ8K8mhSCBsfm+++6bAiV6cGzz7AHJ5fGPf3xxew8iKhG4uL7MHLY4A89osOQ6beO0005rrr/++uRhQYOY8847r+GrHYTFjOc85znpmH88j1ekLlE+84k6i3IssLYN0vlKgLAsmFtmDgFtIUOKeBmK+o2FKRYXVTfwIoiu7BIIHtRvdHIUdCt1DZ3PF8AsKiOQP9CrMvqju1lkZtFc+lzxQPBjAB3rSG04xRk9sugav+SX7cjY3hTjpKT2/eIkmK+Y5dFE8fI76TbSVn4Y6jG2I0OMM7XvTjpteeHeOFLCFR0DaQSJhhjOWezcd999p7yEUN/0RTz38ezJV7Oq81yTYFBER8VFkThZFLGA5+VVMerB9ed43iFupERqof2tsMIK6X6pXAjDu7UtLKaAA//FPEXDLXMc6klcHI5RQ3JhsT/fsi8+o+NYHtdcc00ymOuefuNCNIv2cezWVSZsZYgXpXxhgjkZBs+nPOUpSiKRS8CuS/h6bccdd3xI+VOOlD1bLO68885TUZBGNDRP3SgcTGoMEOsUyaAnWbBAf+TC9n0s6jDGyMPxbBe2pfpOmKOOOiqlmbct7iHRIFW7gDEqjb+mNLdHLV3jt01/c48FP9oaHoNyYT5BH0W9itKFU1sdJDxfydOW1d9xDZ3Dtnz0kbQv8ioC1zhtJR97xe3sSHeURAM340XapDzcxbCUKV/Snn322fFyOoa8zXgzX1RFX+DJiz+kL4GLZ7vGoTX5LPUZpCOJOnEm9TT1ZO2111Y25vplC3M8mURp63viM13H48wNqLssCMd6TVrUZTwAHnnkkVMk57755CtYjA75XIV4aTN4q8q3tVWdL217SLg2Yxv3JIw/SRuJxDPd53e66kRcEM7nhTH9/HjXXXdt1l133XQ5Goe5MGqMk8fFeXy/6RoHRGIG+o96oY+HlCfK+ZhjjmkYJ+RSo4NrxtR4y+RDwXz+QX6YC/A1O3Nhiepg7F/R53hA0TwbAz+6PQoGJuZTcawzqq0Mna/E9ErHbePy+WWOEPMZCVw147/Y9+Zjm6F1ooS1ro2jdxVH/qs6WNKDkcCFbsbzdZTogYtxlwiyijPWa8INbVND5xLj6omaMUFpvtbWFku4xLpTM8cZOk6M5dd2XKMv+467S2mOsz5Qio9rM113usYgtC36JkgIuUCaQOeT3yisVTFOxjOXpE1ndRG49KFlvl5BnDvttFP66EDx65c5MfmFUNhHRo2LFUdXvWqbM3YRuIh3aJxtbVN51G+NLlD7LulS4u0aUw4ZFyuPQ99d4dp+R/V9Xel1zWXb0vP12YEAa1dxbWoSBK4hcWrMCxrMA1mvZ74GsZEPdCTMeaRD8XqEDUJjY9YJGBfT9lgLjV6749iEuNROOUbPkhbrDqxZKL58HDdOuOmyE5OncSWSy8CedSzWOiHTiZ9Cf4BXNvqQPlIb50yHi+/Cmh5re1p/aiNw4fmUMZ/W+fEUy8dTzEcZn+s6WLJmddddd6VkqANtJLmYD80diRd7Q42YwFWDWo8wbBOkAi49bgJXCRVfMwJNIjyydRsylMBF57PiiiumsLlippPCmK/OCuVM591HyAeKG28wudehPuHbnjGBqw0ZX59JBOirID8ykGZAwdcM0WsKW6YxWGbQA8kmkktw88yAmMEgxo1IZhn6DixusDjHYhVfcjJAYaBJuxvi2W5oun6+HgH0KZMvBv2Uf6wbXbESBtIAi3qQPNDFDIYR6hn3iBsPV3iWiEQsCDVMwCBx4RmRusEiVKmO1IaLeaf+r7rqqqkN8MUl6bB9XheZd+j7xfS6jifdRmrLryuPte8+HXnpymfXPeofWLPgGAlZhOGLWwhu/FF/qZ8YHCO5sSvuh9M9iPGQCvhKjn4El9xMdOlD1J5n4n0pB3SBXKVDJmURaFSZMBlnkYiFydjndeUZgxeL6Szm0z/xriwoScCCPpHFJvTSOH2i4pzELwsLbAHJohdlBOkD49somXR9j4YI2o2+6h+Vj9r7NfqbtFjcR+8//elPT3WDbThp621SWweJD11DfaIOagEN4iFbKtaS3NryOcnrbJ8AiZ92pDHgVVdd1ZkE78oYjzEl5AT6/z4eALoiHTUOrclnV3rz6h4Li+DNPJd6gcdR9FyfdlyT53HmBqRHuTCmpw3eNWeRk/wyrq8V8sOXsfS9bGeKHuPjjr4k2Zp0o+eJaFyoiWs2heka4wzN5yTGAZGYofUbDJ7rrLNOGnPRp0N+6OpPa3RwzZhaYwB0GGmyBQxbs5C/2voNuZX+mT6Avrk2ntr+rq3Mp2NcPlvmCDXjvzacJlknxtW7bXmcqes1baqvDpmEngCH+XFMMOlxYo2+HLcO1bb9rnRnsu5gfGUMwhiW8Q1zTNYo++zqgQGZ8Rt9GEQExkPxg8OudxznHvWGMRP9Fds2QWYg/emSSc8Zyed0xKn3n2ldMHRcPB3v3tX3zQu9oLLw7+QQgLzJOJ81H5GWFHstgasmTsZGfJiN7mRdkI9tov2VLRUhiiPoJX3IFT9IYO2FXUDi+D/ej2SYjTfeuNlhhx1SfKyn8K6aI7PeB+EVnYjEjyxrwxHPdNmJiXscQdfwvgjYYxOh35GwY4k+ZD333HPTLju61/ZbG+dMhyP/2tqSd8w/CNI8M3/PTTbZpNl+++3TZdb9hB8XqMP7779/spdyfumllzbHHXcch73kwAMPTGuKlAVrfLX9MPND6jD5Yb7AH8c675WZOQ95C8UMKX2lkF2eOjWBawoKH8wnCDDA3WijjZLSogOFyMGexRi0UXZc48tDGfDp5BkwsLh8yy23pK9BWZxi0ZXFLq6xd7GMfBA22KsY4xfPIBjHMKyhZKNHojbI+HKMxab8izA9H7178YU48fYRtechYbriZSK1/hxvG0xkyS+CEQMsMWahlGGro4i5ri/eYpwMeDD4sBdv/NqTr2TA/corr0zYEw/EHHCEyS5CA8qfL33opPgil44VAwyLkLj/p4xKArmCgRETWHX6msD2xbMUr68ZASNgBNoQkN5iMhY9IrU9r+u14RTev0bACBgBIzA5BOJ2cGw3yrjzkSZ4cOFLZ4T5Re45iAV75jPMBRi7s2BkMQJGYGYRYA6Ol1Xm1NFAMLO5eGSkViJmPDLe3G9pBIxAXwQeSXrC48S+taLfc4+kutMPET9lBIzAwxWB6ME3f8daAldNnNh211tvvZSFCy64oPnCF74wV3biPAvbJN6+EXnm47htrUg7D/DM7rvvnmzRkZQEWQx7cxT61W233TZduvrqq9Mcj5PacISdtJ2YOCch0fPZZZdd1hx77LFzRQu5DzsJArkXD1mjpDbOmQ7He+Dhig8xS9JG4Io7PpR2RIlEND4ie//731+K/iHXYr3Dzn7++ec/5Jm+F0zg6ovUwOdgj8JglsD4REFJTOASEv6dHxBgz2y2jYl1WPnG5TaTIiR6zFJnxpexfEWOQSIXDDe4HMRAwdYzMKFL0mdrNuI//PDDU3Bcc6pDivFBVoJshrQp7vg8x1FR45Kzr7vhPJ54HvMRr/9/9s4E1rKi+P/nHzVoABlEBUQUEWUZR5BFdhj2TQhhkXVkUQkEQYRERhYBEZRREgIJCZDAzIAwA8g67MuAyCqyCIMIqMgmiKMYQhQTzX8+/eN7rdfTfe7Z7n1v5lUl792zdZ/uOt3V3VXfruKYSQrKYpWf9/HemDTRiHfny5sBOzgBiglxrEmRBTQAhmC3ZkyA8WbOnBmQxfbeKgt22TPpE+jM3uOY78nkbpC7oeN3+rlzwDmw+HPAyi0HcC3+39tr6BxwDiw+HGDOyOYONgugHIHsTsvFp6bVakJ4s4MXhDmD2IRBeDTtLGUnHesi1h7QjTfeWFx33XXh2P85B5wDg+eA5JVdq8ehCAdfivH1Bjeuj6/v7bV1DjThwHiSEz5PbNJC8mnGU9vJc8HvOAecA+OBA6xf8AYvwpmDbLFNAVxN8sR5BB7gcdhx1FFH9Rx9qFy5XzbP4DQCL4U4p0iRwhtzDw9d2JOJgoa9OpcOj2CKpiSHHzzfJB3vHYSdmHy7INmKyev73/9+MtKAnJ/wTAqwxHVLTfMcdjrKTKh6vDiL8Oo8YcKEcJrDAdA2aCPoLOFHTBavgPd4PGn1I7yEnnnmmaFdVk1TlqcDuMq40+E9K2DI1gFcHTLXsxooBwitdthhh/XewYA4f/78EHZGEwHdTAG4dI+BG0MFxgkEmcBgiiMOChiPUHjiQnBCoIERoAC4AIr1I1zQA37iPXHYI8KhYBRR3oAsq7gutOFeAFIRW50JEQBNvILhOhlPZHWIAWWXXXYJExrKCyn8FwMByuO2AK64PHjJIvavgBC6z3ch7BG7ixmUBDzl+tSpU3shiOAb4Cy+HwRYC+QxZSXEjNIRNogJmoxReo//OgecA86BphyQ3HIPXE056OmcA84B58DocAD5HW9kYtcac8jxSCglzzvvvKCcpP6sq/B6y4YL1kBaH+ERl40tzMednAPOgeFwgA1okPoh3rHZOODr2sHx343rg+Ot5+wcWFw4MJ7khM8Tu22146ntdMs5z8054BxY1DkAgIfIPVBTAFfMgyp5AozCdoiOgw18hJ/lD9shoREJ6UcouphOOumkoBPBVivnFPYZ8gTkhadydCR44MIOjK0TKvOOdP755we7JWs60hFetkk63jMIOzH5dkFypsIa9vDDD09mia0XvkHYoAEYlVHTPIedLlUHQnUSwQrKAbiIOLbMMssEOzch7WP67ne/W6y++urh8u23317Mnj07fmShc+GAaKf0gRijsFCCPhccwNWHQV3d1odTfg7gEif8d6xzACMD4CcoFnZHHHFEsf766/eqkANwAerBJSZGCggwEG4tBV664oorijvvvDPcs2EObX7hZs1/AKWYIOA5itjzApwRrgQkbBXae++9ix133LH0USYlxHdOCfqyhExOKBcCHdeSIuvSsqkHLvKC3xdffHHxyCOP9AxAAkJwnwEdZPyLL77IaZhg4QlAiH3LJ74fEy5o3rx5wbVpOFnwj8kT4DiFVCTkoxTgesZ/nQPOAedAUw5IbjmAqykHPZ1zwDngHBgdDkh+6+3M95n3j2dCSURoRK2vYl44eCvmiJ87B4bDASmaeRsKfrx7P/bYY8N5+Th9ixvXx+mH92o7B2pwYLzJCZ8n1mgcfR4db22nDzv8tnPAOTCOOFAFbFWXHf3yxMkDHp4gAFXYYfGCFBOesAiH+PTTT8e3suc2PJ6cgWyyySbF17/+9ZAmZz/lpg3PyPNN05HXIO3E5N+GBHzD6QYRnlJkbfkXXnhh8fDDD6ce611rmuew0/UKbA6qALjM4+EQz3XrrLNOcHICnkCRq2JsQ5xO5wceeGCx1VZbhdNUGEs9V+fXAVx1uNXiWQdwtWCeJx01DgCAErAoF+5ErgYppAVcWQUoIKHnn39+RD3sQgrvU4CZoC4BXKlYzRrkRxSm5OSYY44pJk2a1HsCsBWewZiELLnkkr3rhDLkWQAGVWnQAK4U360hjcH0gQceGFFcJluA9th5LGQ6u8BQYHMth+K2k7S6IIsRBfAT54BzwDkQcYDdDssvv3zwFEh42KrUNF3V/P0554BzwDngHCjnADsU2eHIZge81r7yyivlCcbRXXjDGoMNEHjCfeaZZwJYhLWKk3PAOTB8Duy8884FG6nYlAVwy/vi4L8BXrzx+A4hA/Ea7uQccA44BywHxquc8HmibQXNjsdr22nGLU/lHHAOLE4c6Ae2alLXfnkCQD799NMXyhoHE9hTsS+KOGdTG165yohNb4CRVl111d5jsneydttzzz3D9TKwDJ6mVlhhhfAceW222WaN0gGMGqSduFfBBgcAjbCDQ4rClMrGAoyIeHXzzTenHgvXmubJt2hSlqbpcnVoAuAiqhQALku01eOOO66vboD2jbc3vMVhPycko5zZ2PzqHjuAqy7HGj7vAK6GjPNko8qBKVOmFJMnTw5luOyyy4q5c+cuVB7iIe+www7hegrABbAJdG+KBPJSDGKeyQG4QMBuu+22qWyKyy+/vHj77bcXukeYRMI14iHKEgAsPEZVURDaQZ5QiTNmzOhlBTgAz1QKH/jss8+GEIWAmRgQFW6wl2DBAW5CUU5CgwRwUUcGl5gE4GIAycWUthMywkbgclWuRfHmBUo+RXx/QsAwsAn4l3rOrzkHnAPOAeeAc8A54BxwDjgHnAPOAeeAc8A54BxwDjgHnAPOAeeAc8A54BxwDjgHnAPdcMDa9oYVQnHixInFscce26uAIgPJyxOh6k455ZQQso6H3njjjeKEE07oPR8fEP5ut91269l1sTded911xZw5c8KjgLcAcUG33XZbceWVV4bj+B/2YXkCgxdbb711o3SEwmtiJ47LM4hz6kc9ITZNwucU7b///sU222wTbsWRtuLnm+bJxvsmZWmaTu0hLn8TANchhxxSbLTRRqHN4chERFsmhOfjjz+uSwv9Wt7SFmmTXZADuLrgYoU8HMBVgUn+yJjjAHFa11hjjVCu0047rXjppZcWKqN1O5kCcM2fP78gXmyK5L3LAn5yAC47KMd55cqm5xC4m2++ebHPPvsEgBHXn3jiieBpSs/kfldZ4C6RAYud8XgOiGm11VYLiHGuyztVDnHOMzZe7iABXDb8Ie8VCcBV5k4TYBe7vaCzzjorhHkEqAfZcJfhgvnH5EDhFwnF+Nprr5m7fugccA44B5wDzgHngHPAOeAccA44B5wDzgHngHPAOeAccA44B5wDzgHngHPAOeAccA50zYHRAHCttdZaI5xJWDux6vfRj3402Bo5t/Zg3ecXWzS2SYWv4xogFvJ77rnnOA0EEOuAAw4Ix/fee28xc+bM9+6M/LG4DEIoNk1Hrk3sxCNLM5gzIkURPQkqA8ZZmy+OVR588MFsgZrmCRCrSVmapsvVoQmAyzIDxy1f+9rXet7bymzpOHGhzu9///t7+ACbV5tjB3C14V6NtFZQkAzgSgoMUiNLf9Q5MHAOWIQyHpUYWGMCGHXwwQeHy3ZglnctQifi2jJFgIMYuO2APQgAl97NuygLgC7r9Uv3m/6ee+65vXCKuEdcdtllky5Dyb8rAJfiN8cezhRjmAkN/I1JAK6yUJKgjXEnCk2bNi2ENdhyyy3DObGsc2hjUOxrrrlmeM4BXIEN/s854BxwDjgHnAPOAeeAc8A54BxwDjgHnAPOAeeAc8A54BxwDjgHnAPOAeeAc8A5MFAOjAaAa+mlly7OOeecUC85uUhV8rzzzisIjQhNnTp1RBjFgw46qNhiiy16yf7zn/8ET0aE+4vpC1/4QgivyHXAP4SvS5HeJ/tz03SpvFPXYjvxP//5z9RjnV+TLb7M5mttt9iNLSAuVaCmeQ47XarsbQFc5Akwi3CQSy21VHgFkbgAVMWEpzgwDVBZ9Ko4XZVzB3BV4VIHzziAqwMmehZD54D1wEUMWEBPMdmBNQXgev3114sTTzwxThbONYDacH45ABcDOwCsFL366qvFLrvsUghkBEgrF0MZoSsEN2CrLgZR3vexj30sFI08AVWttNJKqaIWf/vb33qCvo0HLnhNuMJ4QlQVwGV5HheU0Iug5iFCKOJaUzGlZ82aVdxxxx1xknBOnGu8j0E5wF+46f+cA84B54BzwDngHHAOOAecA84B54BzwDngHHAOOAecA84B54BzwDngHHAOOAecA86BTjgwGgAuCi7gTpm3IiI14d0IOvPMMwuiCEHYVL/0pS+FY/49/fTTAZT17rvv9q7ZA2yxchpS5nXqoosuCkCcf/3rX8WRRx4ZbLhN0tl3lx3HduIubM9l79M9Qvx94AMfKP773/8W3/zmN3V5xK8NAZkDI9kETfMcdjpbZh33A3B9/vOfD57eeP7GG28s7rnnHiUd8WvbZS4yFQ5PPvShD4V0MShxRGYNThzA1YBpTZI4gKsJ1zzNaHPgwAMPLLbaaqtQDFDMoJljsmHzUgAuBllAQDEByAKRjDcsO8jmAFxx+vgcl4YCcF166aVZoZsK2xjnpfP111+/513slltuKW666SbdGvGrQakMFDUiwXsnOQAXHryQGdDvfve74AXrvSThZ4klluihypsCuMiI75KaBIlHGvDxqgVCG8qFnuQ7XnDBBSFGcFymkND/OQecA84B54BzwDngHHAOOAecA84B54BzwDngHHAOOAecA84B54BzwDngHHAOOAecA51zYLQAXNiPsVviOeuwww5L1gvQ1vLLLx/u8QzPTp48uZgyZUq4hj0SQMyTTz6ZTG8vYoskbB1pCA/IryVsu0cccUS4BCAMxx5Qk3SDthOHgrX4d9pppxWf/OQnQw6AyIiKZYmQiHwfbLjvvPNOcfTRR9vbyeOmeQ47XarwVQBcxx9/fEg6b968gmhXKTrmmGOKSZMmhVuESXz00UdHPLbiiisWRDGD/vrXvxbKc8RDLU4cwNWCeXWSOoCrDrf82bHCgY033jh4UqI8FmSl8q277roBuazzFICLewyKuA+0hIcm8ofuvvvu4mc/+1k4tgCuHGgsPBj9I+Qfof+gV155pQBYFpONxVxloLJI7pz7yR122KH46le/Gl6V4lFcBnsuABfXiMFsqQyxDkL6i1/8Yng8BktV9cBFYiZCgOgsWTeif/rTn4of/OAHYeLFt2WAZyIE8Iv3Wtp2222L/fbbL1xKgc7ss37sHHAOOAecA84B54BzwDngHHAOOAecA84B54BzwDngHHAOOAecA84B54BzwDngHHAOdMOB0QJw2ag+119/fXHDDTeMqJAFuxDpiYhPkHUQcvbZZxfPPPPMiHS5k+985zsFtkzo4YcfLi688MIRj5566qnFyiuvHK5ZO3OTdIO2E48oeIMTHJvg4AQiAhSRtSzttddexU477RQu/epXvypwSNKPmuY57HSpevQDcAH8A7MAYefG0xbOWSzhgAbQH89CgAGJvGVp7733Lnbcccdwac6cOcW1115rb7c+dgBXaxZWy8ABXNX45E+NPQ6APl1mmWVCwV544YVixowZQZhtvvnmQTgRC1aUA3AB+kEgPvbYYwXxhhFsAJ8gzgmxCPgJ2nrrrYsDDjggHP/2t78tyLOKq8n3ve99AZ0NmhgCRAQIigGLMm633XYhDCDPQbfeemtx1VVXhWP+CTAVe9Hi/YQqhF566aUAePr73/8e8tx5552L3XffPQCbuF8ldjDPiWzc4dmzZwcgmwYKy3dcOF555ZXFcsstF3i+6aabKovGIRSVAXnjWYw6Acg7/PDDQ924D3gLEBfE9Q022CAc4waVCRCgNoh0gLoAeEHf+973ir/85S/h2P+Vc4D2w84EUPH0D4hQoYSthO67777itddeC8eD+EdbAp3/1ltvhZjiekeqXLo3qF8mQ8Q5/9znPhfc2VKmP/7xjyFkZzw5GlQZPN/uObDeeusV66yzTpD1F198ceUXTJw4sbcIQ1bHu2gqZzRKDxLWd4UVVghjGxP4sUp42fz4xz8exspceNyxWnYv1+LHgdyY1GVNAZwzn3rxxReDgqfLvMdzXrvttlvPZfg111yzENBfvEE2LrXUUuH0rrvuCjvUdM/+Mh9AuQc99NBDYQ6+xx57BMXJc889Vzz++OP28eRx0/EnmZlfdA44B4bCgUGtg5A9hOqYMGFC8ctf/rK4/fbbh1KfsfySQfF6WHXucg6LHqPuGDOIehJ6Y9999w36mNtuuy1sTOQ9ZXX91Kc+VWy//fbFJz7xiaCPYce7KHWPd+y6667h2QcffDCMr3p+WL+joWsYVt14z5577hlkDZ4eMCY6FaG9jYU+1vRbMCdF/wsRnWGQOrqmZfR044sDY2XcGiTX8Xaz0UYbBf0wG/UJg5Yj9OqMLZDGNq1Prb6dkFkK0fbzn/+8Z6wvG2dz72xy/cMf/nCw6/D97r///t44Pww9SL/yfvnLXy4+85nPFNh8kHOrrrpqsANht2N9L3tVv3z8/tjgQFUAV84emqpFlTxp49g0aeO0HQBc6MTR6dOmsIXSX6HLLrusmDt3bjhWdCMiBdE3+hGh7MgTGxIh60S8j3UeoQQJI4gzDwiwGB6nKBPUNN0g7cShYC3+Yf/GQ5TARjjuuPzyy4P99itf+UrBHxTb4rkGaI51MsRaAhs41DTPYacLhY3+9QNw8bht08g+olKhK4YABtKGpL9k7nfyySeHe/bf6aefHtZhXBuETdwBXJbbAzx2ANcAmetZD5QDTG6JTczAmyKEvu7lAFxKp0FSz3MdYBLKKRFCEWFpKRdf1j7DsQV/6Z4tn66l3BnKc1Xs4nONNdYIkwtbZiYIDESWMCTh3rMOpcqr+MO4DZ28wH1ojlSvNh64bN7KT9fi+jC5OueccwoB5HgOXsEXywsmWXVAGnrfeP1Vu8MYCQAQsu0Cz3R4qIP4Bp/+9KfDMcC6VPjLcLPGPwEFFQdcSVPl0r1B/DKhBumuSbx9B4vEmTNnVprA23R+PFwOfPaznw0hVN98880ACNXbLVA19jSoZ1K/Nh0A0tjrXyrNsK+V9cnzzjuvYKcGCwB29gyayspS9m6V0+58Knt+UbmXa4+LSvnHazlzY1KX/EChwVwGhYQ1cnb5jvGYl1VkpTzvwpOll146zCXFn3vvvTeM7zq3v+QhxRPrAjZ1aIcgxwo1Tppcf7fjSJ3xx5ZjWMe5Ogzr/f6ewXEAUAXrWxRvo2XwbTpHGBxX8jnn1kH5FP3v2B3dPB2vc/vnsGg/kWuDg+D1MDmVm8Muu+yywdiLzoYNiFUII09ujKmSvqtn0D1pt7w1auXqyoYX1hlWV6XxLncPT+7onKCUZ4Su6lKWT07X0HQsbJqurIxt7skYivcIDGJtKdeH2+Y7zPRjpY/1q3OO11Ze0n4feOCBfln5/ZocaCK7a75isXp8UelTTZlOpBNt/CePfnq1lVZaKWxC51nAR1dffXUAfGGv+MMf/lCcccYZ3AreUwCGQbK/cJwbZ3MygTRNyIZzs/awYehB+pVXIc/k2MDapZib4CDBadHhgAWmoBfBcUOKNCeL7aGpZ6vmuf/++/ccEyif2O5ooyexqUQ2MT3f7xdHErKLWaBOKh3vJgLRb37zmxG3m6QbpJ14ROEanqy55poFXtDs2iDOykbC0j10bgIqWYce3G+a57DTqS76td8Xr1ipzfVsaCekZxm/yI/+QT9iLIoJcDFjjWRnfL/tuQO42nKwYvpp06aF3d56HNdr7MZxcg4sChxgICXeK95EJNAQSoBI8BqERy2Ido7nK0hKC57Bg5ViG4ebC/4xeIKIZsIak40tyz12Rdx8883xY8nzDTfcsDj00EN7Rh/7EO9kJ8b06dOD4LX3NGFJCVt2Z+CCEhR5TORJHZrs4mX3IxNkPEGIUMBpMCB2MzsgLPG+X/ziF4GfTBpiAJf4Hhu3lMdPfvKT4iMf+Ujx6quvBi9ZeMbRN9Uzd955ZwFoLibACMTxVTxle59yzZo1qyCtU3UOqN1VAXA1DS9aVprcIjFVrrJ82tzDBe2PfvSjXjukD9IHllxyyR6gi/YFol0e4dq8z9N2zwG72Ird8DY1oNt0YxXAVdYnpQDqp2jq6muUlaXsHSrn4gTgKmuPZbzwe6PPgdyY1GXJHMDVJTf/lxdKXealUMp1PdfxgIP3BdH8+fPDJhGd69f2YcZ/dr0B5koZ1+2zXY0/KsewfsvqMKwy+HsGxwHtJkZpztxmNKjpHGE0ymqN5HYjS9OyIDsUGoE8UICylka/MF4o1wa75vWw+Zmbw5500knBgwTjxze+8Y1KxRorhvC6AC42QMmTCGvod955pzj22GNDnXP3xiqAq+lY2DRdpYbR8CHp5LoCcOX6cMPijUqysdLH+lU+x2srL9GVOYCrHyfr328iu+u/ZfFJsaj0qaYcx24qGwxRWdAFY8/Ikd0oBHgLEJc2BFn7COGv6gC4cjIhV45+18cygAvPMauttloAxQCOwZukPA+OVZ1sP36P5/tEO8LjFcQatB+AK2UPjflXNU/SbbLJJsXBBx8cNnrH+eAN+ZJLLuldxkv+fvvt1zuvcmABXDxPNCc86cU2TupFWMVf//rXyWybpBuUnThZwAYXKR+yTlGolAVrI7zgp2y+VubiZSreeNYkT9477HSqK794HAPEBeUAXNwDqIs+U2MO1ywRHYgNq6k+tMoqq/S8cr3++ushyphN28WxA7i64KLn4RwYRxwAvINiSELrwAMPDAMkLLCIfCkttNMBFC8uYQGBMfF+9NFHgwI1xzoUMbh8x/hNHuygrEp4VsBzGO4weR8GItwfokBB6DUlwi6QJ+hcBjJAN+Rbp2ypdyPs2W1E2EGAVZZYlOGyEc9Lf/7zn4t58+a1qoPNm2MQwiwgqBeLmqeeeqqvpxsGtkmTJhWUG29mpAO4BxrZqR4HUkAp3LNj5IQABsptqTW+WG939d448umcsTxVrpEpuzvTLh9yZCKJe1fRYYcdVgDKhOgfLCidxh4HrOL8kUceGWEos0As7QivUgNCXzHRh/Dq11bOVnln3WfK+qSMSqMB4KojH1jQME4iy6+77rq6LBiTz5e1xzFZYC9UjwO5Man3QAcHDuDqgImJLKxhK+XpliR2xybnKJBQBqNUs7TTTjsVe+21V7iksR9FnBSAzIXZkAGV9fem40/IeIj/yuowxGL4qwbEga4NP02KWTZfaZLfINPk1kFN32nBKugfGAPGG+XaYNe8HjZfc3PYJiCA3Bgz7DrlAFy5umpzHuMphhrrsTh3jzFHYaasrmGYdU3pGpqOhU3TDbK+0oU6gOt/XB4rfex/JUof5eSlnec6gCvNu7ZXm8jutu9clNMvKn2qKY8lR//xj3/0gMllecEP0kBEcMDTszxE43iAYygH4MqNszmZEDJr8G8sA7hwKID9CZsfoebwgIYnNKiOLrcBWzzJYsoBAER4YcKGyDF2VBzasIF4EISNc+211y7WXXfdoGMCeHPffff1wibm3tk03aDsxLly1r1OSFTC0GKTx958zz33tOZ90zyHna4urxhDADxiH8fuzZoK/AL27jfeeKNudp0+7wCuTtnpmTkHFi8O4IpcuxaZ/N56660LVRDPOQCaUBqxQ55fSJNtAbgWSugXnAPOgSKlvMyxZRDGl5yxvE65cuWtel2yAnfMgEAtAcY8//zzww6KKu58bVo/Hh4HyhTni4oBvQm3yvrkogLgalLvsZ6mrD2O9bKP9/LlxqQu+eIAri65+b+88CqrUOKsBbR++N8TRW8XtL2WMoIRuh1lGISS6dJLL7VJRhyX9fdFZfwpq8OIyvrJIsmBrg0/TZhQNl9pkt+ilAZv1ni1hm644YYQMm5RKn8XZR0LbbCLelTNY1EGAeQAXLm6a94Ue2Xn+bJ7ufyGdT2la2g6FjZNN8i6Sr/hAK5BcnkweefkpQO4BsNvm+uiLLttPfy4Gw5onACA8cMf/rBSppK96JDxtqOQYNbbTw7AlXtBTibknu93fSwDuBQF5q233goh2HDCQISb3Nq+X139vnPAOeAccA6054ADuNrz0HNwDiy2HLDGGHbHA67Am4iIiRwTOghUKjFyRZo4O4BLHPFf58DCHNCi1IZQxMvdNttsEx7G0EDfI4TnSiut1AtFSqxwvFLgAtS6NWXXAC6OUf4SthRl7rPPPls89NBDYadDXAIpdv/1r38VRx55ZO92qlybbbZZ2MnAxGHGjBkFAE8UWeymAHyFG3l2NvBOFKl4EQO5TuhNyohnJkKWWALVTmhE6PHHH+8Zf+0zKiPX8Mi1KHh6w6PelClTwjdjpwMuv3GlyndAeZAivO3ts88+wWW1wqrixYQY7XznmAbxPew7Nt544wIvWCuvvHIA0OFtECUI31FAXZ4/6KCDwvdea621QnLaB+0ZLweE0YoN6OyGIW+8JOJV6+WXXw48id0ZY3Cjbf373/8e4d6XHWCE3XzhhReK2267LbRBPHXRP2jHKHjYTc6YlCLaHO6Zab+8/4knngjtknbLDjOuAVbmm+UI9+z9+qQFcAFGYKxEWQM/2dFGOXHrTl9OEV4XcVmOdyzGYvjA+EsbuuOOO3oeyaqUJZW/rrHTkPbGbpw4BC4ACsKdwe8lllgivJ++TEjj3//+98qi76/aKu159uzZYVff5ptvHsLbILPwMki9aFspqtM3+rXHVP7xNfiN+27aCH0ZmcPOTzwEzp07N348nOPhA7lNO0Tm8Y3hFSGa5EVRCakzikMIcDy8hB/0C1w2syONdyndlltuWWywwQYhb/LlebylyRMq+QwiT/IVtakf8mC33XYLgBwA/4wXqgPywpLkvcYk+iRu/KEbb7yxxxObBp4hbyHeRf8oozIAF55fd91115CcsZe5LPLOfq869VE5qvKPHa98b4i2huHPEkpVwoEhr3ADb4kxX16r6KPImLby0uZf5VhKcp4988wzR8gJvqO8aCJ3CeENpYybkp/cP+OMM4InXtq4vjPjC164+vX3puMP74XqyB6enzx5cjFx4sQgn1NefpCl2sV8/fXXB/nfrw7kmyPJ1i7nZXpX3bozZiOnGKcA3HFOX6J/XnTRRUGuKW/kKl7W2OFIWHeMBHxP5pDMbZsQ74P/yGDKzq5eZChlsbLS5o18YXcu4yz9CjlPv7nssssWmgPgdZh5LeMB4R4ot+QTIb+Z41GHOXPm9F6x/fbbh3kwcxR2dGrOwdgQz+sYbyg/7yE/nnnyySfD/MjORwhFIe+kGlN7L3zvgDAByANkGH0NT7ZV1hBxPvac/JgPMIeZMGFCCK3Ct7r77rvDXM8+yzHefXgnczzWMsxn+EYf/OAHi29/+9vx4yPO43WQvl9deUY72H///QMvBAjFm/Xzzz8f5hzsphV1XT+1F8YPwkBSFtZMtDfaAXNe5ly0OdoeADvmvMzH8BzO2u2aa67pzfVUTn7Zvc7cjbz4FrQt5gbM40hD2xH1a4M5Xit9Xb7U/UZ6T9PfeA6L/KUfwUvmchBeNxgzWduWrSFTY4wtF99viy22CPNCnmUuQ/ueNWtWI+/ozDeY+7FW5hvSLllLs3lJG5qQRZp3xnWlX7PeZ+5I+6KtIUMl+3L3rrrqqt46nfqxNkfuWaJs9GH6O/Lo3XffDZ56ScscSERZAWtTfvgsz5i6zy/lZk2DjLRhS2NdQ9OxsGo6eFRnnW3rkDtmfCNUD+2NY8Yc1jLoNBj34EtqjkN+VdfZ/fqwLVvVcYQ0rIcVUobvihw6eEGYI/o8ugq+lZ37drVWiedxlEVzGY1pzIXrrBPJo07deT5F/XiNDCDMEkT7ZQ2PTEAHonGbaAbMwXPURTnjvKvmWXf+QH9mXiOqqpvQ8/qtIk/ayu6u26rKzm/TOSsyGp0T4zryh7kc4zt6JuZ3luJ1fNW1Zm7catunGD+lU+aY8rChBhnHda4Rtl66CluX3HEdGcyciPkN/QuCd7wPfSS6kzJC9sIXPFAyl5InSvSD6KEg2gt6OYg5qfQR8TjbTyaEDN77V7Uf8ngdAFfcNrqQxbbc8TEymnmUPGCju8UTF/MndPFOzgHngHPAOTB8DjBOoQNgbcG4wB/HOq9aov+3wB3b/7ndqZrCn3MOOAcWCQ6ceuqpQXlDYVEMvfnmm0GxhNEY4QFhaGO3jBSsXHMAF1xwcg6UcyBWXvK0VQ6h9GfBTJjBFKFck5IIhdvUqVOzMZtRKF5wwQUjsomN5bqZKpc1hKJgRgkUE0pjPHAce+yxQREY3wdwc+WVV/YuozyUlw4ACYCVLKFMZMcTkxKUx8Q4H+uE8Z2FL2VOEYpcYotjwBGtssBgB39lbNB1/WIQJY1Vkgzie/A+lPHkjRIiRSzmAevKoEib4jvFBMAKD422nCirUAKkCAOIBV7YdITYwvgCybCPIYAy4IY3Jng7bdq0YAyx9+hbGPNS3wZlu4yi/UIQYvDq1ydVTvoEZV9mmWVsUcIxihBADiijLGEAwzCcI8ZcgBC0iyplyeXDdVvOo446qvcoPMcYnyLmAhhk+atC+pbUF0MUyswUpWLS1+0b/dpj6r32GsBCXMfTD1KE6+QTTjhhxC1kGIaYFMErjGO48RehwESuQfQTFOJciwklIwptDBkxIQ9pAxh/oUHkqXe2qR/KVuoAqC0mFqHUgT4iisckG6YbIALtNSbbX3ifeBs/p/McgIvvcOKJJ/bmtvfff38I32p5W7c+vLMO/2hHGhOldFa5AbtQPhHgZys7MCoh4yCAVIyn6t9N5KXeU+cXGSJgllWUk8chhxwSjHQc//jHPy54FoOXwHpchwCLsmsasspi+x0I3f3Tn/406dGLdG3HH/KoKxIoM/QAAB2xSURBVHtIw/fhO0GpMBP0ZeQLBIALGdpGZkm2kl9X8zLyalJ3GUOQeYCYBIQkPwBPAq9jtMAAoDUk90WkBbCAkagqMZ6zAUHjd5yOPOnzFiSOPGK+DMAoRQCfkNnIAJGdm/PtqB+Ks5iYs5911lnhsg0Rbp+L57MAhnPjIuBt5lKa/2EIJRSp5jGsEwBRiTAYysBC3ZEFxxxzjG6P+LVriBE3opN+81o2ayG7NS8kuWQ5IBcAeuoX3Ev1Da6LLK9t/erKM3QV55xzjrId8Ws3jQyifrYOANT5LvpmKgjfh7kq863UHBFwEG2I50SsE84+++wgJ3Ut/rVrrX5t0JbT8po8m/Cl7jeKy173XO9jHsGYYucDcV7IXubNOUqNMTyLrGKuzuakFCEvKAfg+6oE33PrEWQIc1HIArjiujJ/EsA9fi9jYO4e/c+GFdVYpDxYf+y9994LtVfdByQ2ffr0cAqQFeM4xBrv3HPPDcf2H20cAAOyTCAL7se6hqZjYZV0TcY0W4fUMXNG1gOpNTBgSnQc9PkYwFV3nd2vD6tsdcYR0lgAAXMRxjTJKOaPzMM0nx7EWkXzOMqiuUyTdSLp69adNCnqx2srL+nvqfUZ+XIPvU1MXZXT5lsnz7rzB+ZJgOqgMtnKfaub4FxUVZ6U5V9HdnfVVlX+pnNWa0NRXvaXOR3tTWTHnzprTZuuqz6Fbg0QMeuxmJjvIU+Z/6Z0y/HzOq8rg5E/qfczhmsdpbzjX40JzJWRxXjuIly01TNpzUJaC+CKx9l+MkHvrtMPSWPlL7px5m2Q5s52bWy/cdftO7w0+gewFyAbANVTTjklzN9Z38ZjeJTMT50DzgHngHNggBxwANcAmetZOwcWBw4wYWTyixIiRUwuY/AWzzmAK8Utv+YcGMmBWHnJXascQpmO0RpAFIYIvDlBGEMwlGB8wQjBdRRFMiZxjwUru0Hx+CCFHIs+DEGi1CKRe6lySbmmtPyyi4zFHN4L9A7dB0QD4JPdU3YBjiHp7bff1mOlvygP2EEMxcbs0oSjdBN+w1N4gcGF+mNg5zrKeH0fjJMKCRV/OxQTeI/CSMPOZ/EOWYtxQsCvQX0PKdphIYpUFu8YL+Shguu0L9ok3/74448P3xjPOhCGSQx2tEvaZ6qcgH15BtCh2jRpeVZAYJsuBeDieRGgMrVDKdJjA6k1apIOgwvtl12VMVinH4ALg0W/PikFkMpIewD8A+8AXTG2Qpx/61vf0mNhJz48VRtCUUY5aQvwWEZveb2sUpZe5okDlVPGLx4BnCkDK20AWQIAhO+FwUKknY06z/3ab6lnaOd8a+pkgYvUXcriJn2jX3vU+1O/yFjkqGQZbYj+y7dCmanr1gU/oaMXbGIJ2fGNMbhSfpTkGKyU5uqrrw4e13jQKuJUDtojPKY9AmqJCX7Rxm2eeA+hv0KDyJN8u6ofvKEdUU6+ufjSb0yCHwB1oLhPh4sL/mlnLecnn3zyCO8jesb+pgBcfF+Moepf7O4V6C7F26r1qcs/3o/iGf7YPkn5MfbKOyfneG+54oorOAyEgZk+yhjBeyH173Dy3r8q8tI+X+fYytnYOMF35HvS1vEkhtwT6Mb2ewtik/KYMtjvICNFv/6ekj1Vxp8msocyNgFw9asD+eYoVb+287KmdbfGEPqH+jjH7IbHIydePS0Qn3kA8pJ2awEsMaAgV3+ux++l/shSZDBzL4hxjDFbO+ytzABgjfcZ5KtNQ7kxXCC3IDs3DxcW/CMtHp14D3NdkYBRgCrxzKI5Mn2TeQD1Zq4IWYAn7xTYkjQCvnIdUBgeeiBrKLJ1Y74IgELzIIxCGAL7zVdCppl/eHmyHnqRH/RL5Di6Ac1rc7I8ztbKp/iezi2vf2YAanXlGXML+iR8VPtiLMVbFeA8vKAOqn62DqoX70X+2Pm07vGNabvMR9VuuWcNe5zTJmmnEO2ZNsF3Z15p02ks7NcGbTktr5vype43ChVp8U/v03jJ+IGnPNa+6gf0UQhvjsy5c5QaY3jWApOZhyAv+JYYpJmTQfRDxjTkSD/Cowheo0SUHZlAXnY+zP0yABde7RhDkZ/0Q8kP+hhgqtw99Ho5ABde8hRulPfTX5jXstbAAC5CzvCOLgFcTcfCfumajmmqa+oXMB9AdI1zzGtY81tZo3QxgKvuOrtfH+Y9TcYRCyBQWfmlHeHlFF2QAFy63+VaRfM48k7NZaqsE0nbpO6kS1E/Xlt5qfToZ/B6iPy1IM94Y1qX5dS7m+RZZ/4gUAleAulntHfaRz/dhMpXR57Qd7qQ3Xp327ZKPk3nrMhmvFRBlANQLmMHYzcyU3KDzQ7yxmrHn5BwwT943W/tbNN10adiXQhlYJ7O2IQstVQVwNVEBgPSYkzUuMPYyhwJfqQ8HNtyocfD26uAf4x56K/sHMeuHcoAXP1kAu9t0g+t/LXzvJRu3n5j1bOL9q284l9AlwAqNa9nPsV4p3lW/LyfOwecA84B58DgOeAArsHz2N/gHFgsOICSClfWKGwhDIcY6KXYjiupMAksaJn8OTkHnAMLcyAFlLLKIbvQxJU0YaigGGBy3HHHBfft3MPLgLxXcI6BBYW/FNkALqXMTi0SSZMql1WusZhHeSuPWSiJLQgFRTSLZSnK8QaAYVd54ymiH2EwwoAAsWgHzCXjW7+0o3XffiMLAKA8ViFiF8AoDVCkQ7ipl1GPc4z5AO5kIMCVPeBYaBDfw7Y9lCR42hLPUR6wg03KSWtg5Zo8TsTKnLicKINRDItoj/KiZRVZNl0OwIXygraOEQGCxxhmKSskAxbHMvJwbHfgcW6VOJzH/YtrKbLfO05j34fSDt4prA1KMMAMKJcgdnBj1NWxvJ/hZQHFv4h0tA8MfPQJyi0qK4ueSf2qnLZN2m8Sg7Sswik2SqTy55r9lsiOm266qaew5L52RnKMR0GM31DTvlHWHkPGmX8YufEWBcXgExR/yDRIuyIBo8AfFLEY7+irNgyNBaJgUEORSH+KFXG8C/CNvMxZcAHvmz59egiLwzEGAtoFxjqMG9qFOog8u6ofBmvkA8YtyHohog9bY2FqTCKt5A48tnNK+KFd7pYf4UWZfzGAi/7GLnMBEfBmw9grinlbtT5N+ScgFu/HcEzfhGw/4ZxxnL4q0sYJ23bVv3mmjrxUnnV/mWcg4+kTtGdkN4TMQlZzXeBTO2+w/d7KY3vdfgdrpCjr77HsqTr+NJU9TQBc8KesDtzPUVy/LuZlTetuvxvlBXQPvy1g37ZHQOz3GE9bhHs5eEH4JtpIPL7l6o8hDGM4aRhbkJ02LJ4Fu8hAwnvwBgchMxh/NVflmgVdMg8iBDJk50ecM/7Bb8ltjDw8A9k+yDl8oP0iO/hmIr67QAD0T+TbiwtAwyJkI+tpiLEFGSCy4wTGVMApdtNDXIamcwQbFlU8VBnQCeAJUOsLPPCiG4Akyznm22AIZt7It+1Hltd2HWTbTx15Zo3IFhBDOQZVP1sH3mM9EjJXhT8ac6gL349vBlnZaL0aAUIjHQSQjrYLb0V4Tdpxxx3DqZ2jcyHXBm05La+b8qXpN1Id6v7qfXYOSx6ax8IfjKtVKDfGiHfM81ibqs+Tp5V7ln9l70NuCKhv2wVpCM+O5zORba+5uqqvxTwgj9y9HIDLls2Ov+RlQQnaQNAlgIt3NB0Ly9I1HdMoT47seoE5DfM2+jGEvoNxhHEJsmsl29/qrLPJR+2wq3HErufIH4++eCyUHLL9gftc73KtYudx8Vym6jqxzRhKnXKU47X9fqTFw/KMGTN62dhxVnNdbg6inG3yrDt/YKxpopuoK0/gVVvZTR5dtFXykczluM6cVd6jkAnoaKVLIx90Xlq72Xld3N+qrjVtui76lNUpE6qPeanmGbR/6zky1vlRvxS1kcHSSVs5mnqHvYbOBp0Z4xRlZ67KWEWf1Hzfjt2UT99I3zweT3MyoWk/tPLXzq01ZkvXRL3sN+a8q/ZNXikCTAnYkDEK3Q1zVTYiwxOND6l0fs054BxwDjgHBscBxinsg6wvkMv8cazzqm/2EIpVOeXPOQecA84B54Bz4D0OaFFqQ65Y5ZBVBlulkAWLYCSWh5KcAXvy5MkhrB+vBTyjnUupRSLPpMpllWupBbuMx6RHMSGQGOfs3MNADlnvNeFC4p9VYKP8ADghgEvi8TFzyRrbCEeAAcUSux7xKMbObYyOLMj5Fky6rLHbpkEBgVEMsh5oBvE9eA/vQ9mBIRDlkSUW9Cg2IBS9KLAhlBdVAFy27YWEC/5ttdVWBWHSINuubP1yAC7aHKA2SwBaFNIABRqARgukscoym84aq2z/ss/Ex7k+yXNSAHHMN7agNa5ZBZl9H30Hb1t42WO3a0yA6gj7FBulysoS52HPVU6rqJKnHJ6zIDil49tggERuIaP6kf2WGHfVnpXOeiNQCLw2faOsPeqd8S8AE8lFjHQAZ+hvljBUyxspngTxoIChFbLgQ5uG5yZNmhQuXXXVVcWtt946QhHHd+QZKQ550AJeU+1VAB/bBqxyr6s8rYekNvVDTjA2WbL9Td7euJ8ak3Djjzt/6LHHHgsAy3Cy4J81WMceqfRM/Mt35nujhLz88ssDQEOGdBt6Suli3latT1P+2TAi1mBw4YUXhkU735cxg3aqUGlrrbVWkCmU+ZprrgkgSY7VvzmuKi95tg1Z+UHfZxyxfVzeiagDnpn4BdyKrIGsYQmvaAAVIPsdrJGirL9b2VN1/Gkje0YTwGXHz8CwBf/qzsva1N0aQ9jcI8CrykKIQOZAUGoc4Lo1iqfGTZ6xZD3jAAaTZ1M9Y8OOCnAA4FOeilLjG2mtfJJhx87NU3NtO4YwfjPGi3KGHwu4SvVP0ssASr8HPKJxCYMUYwH9B2IuxFwHYt7MXIhxXdRkjmA96lljtPLkl81dAO8gvEEBSIMkyzmOQSpcKyPLa7sOairPcgCuQdbP1iEGXFB3K+disBX3kY2MS3i+0Fxw9dVX7827AfYQys4SBjdAi1C8iSTXBm05xes2fGn6jWw96hzrfXYOS/q2IAA7xkiO0veQBxbAxRqcORzE/C5ek4Qb5p/lbUpO8qgdQ4cJ4LJrJdbcgDYs0R6RywA2ZVwe6wCuNmOarbs9Zi2MXEb2ImvxKsl8zJLVB1jgQdN1Nnnn+nDTccQCCKgHc1bbtu2cq6t1hc3T9jE7V0vND+wcUutEeNK07qQtoxyvrby0ehDlhTdEdA+QlfuDKGebPOvOH5roJprIE/jWVnZ31VbbzFnR6yAniaIg3QJ1E2k9x/qGdQ5k+wZ1qLrWtOna9imbV2q8o5xWz5Zad/CMJfKEB8jLurpO8pFO2spRm3/TY7tm0TyfvHJzipxMaNoPrfytA+Dqqn035Zuncw44B5wDzoHR4YADuEaH7/5W54BzwDngHHAO9BalbQBcVkEyd+7cEG4hZq1VKOEdi8UmJAOLFLFKp8WyLZdVrsWegUhnd5tZjyrcI7wDhibo6aef7nlMCReif3ZXMArR0047LevpL0o66qdWwUhhMHjdcccdAbxjlaIqKLuZZJgpU4KgCMJbE4t27STv+nugmMdgBBHCB0N0imT8tGWpakAnLd/fEoAYgDGQ9UBm65cCcFnwgs0PL3UYKiGBH6ZMmVJMXgBihKwxJFx4758FTVhAlX0mPi4ziEoBZPlk09ud7CqnvW+PAUuxm5BQffyhBIvzLSuLzSs+Vjmt8csq5/Ac9dRTT4VwbSjPmpD9lgDQ6BeWbPtRG2jTN2x+Zf3KlsG+L2W44lkU3gAw4Qnhl/CYgWzjW6AITPXx7bffvthnn33Cq1QWqyC1xlmVh7ALtA8oBSiyXmUEfhpEnhYo1rR+Fpij+vErBT3HeCqAp1BqTLLem1Ao2xBs1giOEdV6+wkZJv6hSCZPgLQY4gSAsEYhm8zytk59mvJPoQoog8bLVVZZpQdwwmgPaAJCduKxh3ZA6FMopYiuIy9DJi3+WQ9GUkpbBbctn3jE95fHCoG6Yi9M9jtYI0VZf7eyp+r4Y2WB+myKHalxeTQBXF3My9rU3RpDBJ62fLOARr47O/tjsuF7CbmIkauM1H54BuCQQgzaNPIIzbjDn8AY1rhqn+cYr1uAZSAM7hiUreE2Bxa1suXoo48O6fmXM/xoPqX230tgDshn7bXXDldUFt224FZd45cwrLRdS03mCHYzRW5eRGgcAG+Q5alkOeMjfZvfqmR5LVARaTVfqSvPcgCuQdbP1iG1cQWj7WqrrRZYwphOWEpLfEMMwChrkZk5Yu5O+ETGCIDOAie2AXC14UvTb5SrX7/rep+dw5JGc4x4rlyWX26MsYBO5iCAtO66666eV92yPON7FtgT92c9SxixPfbYI5zaNUuuruprMQ/IIHfPrrUFILTfPQWIJT/GCEKAAlLFC/dYB3C1GdOob4qsh7zcHMF6yxPwoM06m3J0PY5YAEFq/mv7wyDWKnYeZ+dqVdeJ8KTtGEoeKcrx2sp1vMlLX2bzkOxmjaF5wCDK2TbPOvMHWz8d99NNNJEn5N1WdnfVVrues7K2Q5dDeDo2F0A5AFedtabtp237lN1wK7kVCmr+WaB4Tv6Zx8OY0VTXST7qY7ny2HfVObZrFrsuzY2zOZnQtB9a+au1MuXXmG118/Ybd9W+6/DKn3UOOAecA86B0eeAA7hG/xt4CZwDzgHngHNgnHJAi1ILlLLKIWu4yBlfcGW9zTbb9DiYM5LISI3raJQSUGqRyPVUuaxyzS50eR6SMdMqrP7vTj0Alw0TxaI4Bvwoz7H4i0tTDIoYlGPCsEUoFHZoK5QYShyAQ9AVV1wRQDJxOs4tYEMeI7r+HtalO+/s1454BiAFxoyqBvRUu7Eh0KxyxtYvBeCyig3KItpuu+2KfffdN5zK8GHBAwACU+6/reE4Z6jUO/Sb65PclwIoBpworQXpqJy6x45+Qjyh4AO4p76r+/zGRqmysth08bHKaQ0/gJLgEwojS7wTz3oY1jFeyQ29fSZ1bL8lsidOZ73MqQ206Rtl7TFVPq5ZoFUchjaXRjLPytT4WQtepd2JrygCISv7ldbWfebMmcGThu7xa+VBCsDVVZ5d1E9eb2z5ObZ9Ei9SGOSh3Jhk68wYgWcyjAeEe4GsEjxcKPknkEX8SMqrDs9YxWmd+jTlH+9UWhnv8VzEbnD4BFCNfgsJSCIgW1wH9e868jJk3OKfDZFJGD3KJoC36qPsLXCW8R4vEAKYEyqT0B0i+x2skaKsv1vZU3X8sf2v7rg8mgCuVP3UjqrOy9rU3RpD8ByAdwpLdm6Xm1/wvMY7tR2bR3ys+nHdAkHj53RuwUZl+QOEwbgJKYyYnZvPnj07eJVSvvqVdxU7nnIvZ/iRFwaeyfFE/OCZVHu0fOUZQBUCVHEuajJHsEC21NitvGWwpv9qA4dkeW4OpLSpX8truw5qKs9yAK5B1s/WwRrnVF8MmoBLIDZmxN9fPI1lJs/juZZQiRMmTOiFr+S6pTYArjZ8afqNbNnrHOt9cZ9rCwKwYwwbpZhr2b5IGWnvzEUAVQPqr0J4JsSID6VkNtctuMquD3J1VV+LeUBeuXv2HQJw2e9O+4tBoOQX01gHcLUZ0+K66nz33Xcvdt1113Aqz7q6Z38FFta6ps06m3y7HkcsgMB6blUd7Jyrq3WFzdP2MTtXS401qXUi5exiDFV97W+O11au57695ru2Pw6inF3kWXX+AG/q6iaayBPe01Z2d9VWLW/i8ZlyijQuxHNKwFCE1kYPwKahFNm1q+0bddaaNl3bPmV1aKkNZNTBAlGrALjaymDppCVHU3xscs2uWexYnBtnczKhaT+08tfOETVm23W7/cZdte8mPPM0zgHngHPAOTB6HHAA1+jx3t/sHHAOOAecA+OcA1qU2sWYVQ5Zw0XO+GJ351dhp10QphaJ5JEql1WuCTRg3ydDWsrYYEEM8ihi09pj5RMbou0zY/mYHfMY21H44/ksRVdffXVxyy23FHZ3Ym4nNukt71MAri6+x6abbloceuihqeJmr0nhUdWAnipnUwBXrn1Y5ZMMH1YJlzKUUUEbBog2SKi2fpTrk6STAihXzhyAC0UXYeGkEFQZUB5ihEeJDVCQc3lj45mysiiP1K/KaRXNPIdRhrBIeJzifTEBJAFcgfGqH9n2m2oDKcV8m75R1h5zZbXvU4i33LO6LuOM9Tqie/q1SreXX365OPXUU0cAggB1EorFklV2Tp8+faEwSRbMJH7a93SV56DqR13rArgswFKh8KwXu5whxfJVxzGACyMschu6++67FwoL2o+3pEvVpyn/yA+QlkJvomTGoEHoVACUHMsjCOe0B4ytyIyHHnqo50mRfNS/c3IoJS9J14Yoh7xoIbPwVAewGYrBkXZugAcI5g+AEyCr0ObcfgdrpCjr7/1kT2r8sbKg7rjcD8BlgSQymlO3sjpwP0f96qf5VNV5WZu654whKrvarM77/eYMWDad+lhVkBBetTAqQmWGJzsPZ67GnM1eY46M942Y6gC46CeUvw7FfYK0tg1wngtX2GSOwBiP58l4vsF7LEnOpABcds1h05QdW17bdZDeU1ee2X6neSHvH2T9bB1S7cUCuDSOW56kAFwYgZGlGNJjgvfwhbkb1AbA1YYvTb9RXJ+q53pfPIdtCwKwYwxlYS7Mhim8Bsfzc+7D+6lTp/ZCnHItRQJbcy/13bm+ivG4adtrrq5ay8c8IK/cvRSAy3o0rOrRtAqAS2WIvWqmdA1Nx8JcujZjGvxLkfUyeskllxR42EuRDPsCHrRZZ5N/CkDQZhzJAQhUFzvn6mpdYfO0fcyOY6l+kVontqm76pj7TfGaZ/vJdZ6JAVyDKGdXeVq+U/bc/KGJbqKJPKEMbWV3V221zZw1p5tlLjV//vzeRs8cgCtVB3iTWmt22ae0WYh3peYtXK8L4GorgzVOSI5Shi4ot2bJjbMpmdCmH+bkr8ZLO3e23zjVNprojbrgoefhHHAOOAecA8PjgAO4hsdrf5NzwDngHHAOOAdGcECL0jYALnZ3oVCC8MbRbxcwIaYAHECpRSLXU+WySp6Ucq2uoZD3pEgGOULd4EZ/UaYVV1wxhLVCUY7xnYW+iDB1m222WS9U2qxZs0K4Rd23v4TJwmgACYDU9ffA25PCJgI0ufjii20RFjrGWISLdyinOOdev3KmDOhxupQHrjoGPKvwOuqoowqMHDEddNBBBSEwodECcOFRCC80aid4bAKQAZhMXtvw4ES4nNig2sQ4S11ziiruiTBcbbjhhsXEiROL5ZZbTpcDH+FnP+rXBlKKeQtwq9s3ytpjrqw2lIQ8ruSe1XUp6cs8cOFpAY8LEN+RtjUIRdwg8hxU/eCF7ZNVPHDRJzBoAyaUUnPatGmhPdIXUMSmQljyrpgE4CIdxlFAUAJ1cA2QLNdE/XjLc6n6NOUf+VnFLuMxYErqLuOKjAMAKTESUn8oDiOn/l1HXoaMWv4TAIBsCO0sUBZgRZTPlsQnysj8hLESig3I9jtYw19Zf+8ne1LjTxvZ0w/AZT393XDDDQUgLqisDuGBzL9+9as7L2tT95wxREW3QGoMe/36K2FCUmO18uNXbccCh+z9+Nh+79hbgn3WGrHoX3idrGK4rQPg4n0AHTGGAWygz/Qjwuwp3CzPrrfeeiNCynINGQao89VXX+W0R03mCDbMX8orijIXUAGva3hfg3LrC6Up+7W8HiSAa5D1s3VAhseAvyYALsl9eEf/AczMHJHQoYAYkZ1aN7UBcLXhy7DHHL0vBi+1BQHYMca2VcDe9LsNNtigwMMKXv1EVQzM1gMX8lvrceXBLxuAtEFjmAAuO35ps5AtV+q4CoBL4xBtlvWcKKVraDoW5tK1GdNUzvjXemi89tprizlz5sSPhHWcALpqF23W2bwgBSDgetNxxM4zU+BgO+fqCjRg87R9rN9cJrVObFN30pZRjtf95Dp5al5iZVLTb1RWxrZ5Vp0/NNVNNJEn1Let7O6qrTads1qgJvOxefPmhXGaDayscSABtMcagAtPz8x/oZynWXRCrHmhso0Q4YEF/9rKYI0TkqPKt+1vbs2Sm1PkZELTfpiTv6m5s5WbXbXvtvzz9M4B54BzwDkwXA50BeD6/wAAAP//JaFXhgAAQABJREFU7J0J3G1T+ccXka4KCXEN1zxkCmXI0DUWEkUliQYRKipSZioqqSgaZGxEyJA5YzLPQqZryJSM/2TW/35XnrfnXXetc/beZ59z3/fe3/P5vO85Z++11177u9Z61vA8e61pVl555f8EiQiIgAiIgAiIQGUCRx55ZAx7xx13hO985zvx+1prrRU+/vGPx++//vWvwwUXXBC/b7zxxuEDH/hA/H7YYYeF6667Ln5/97vfHT7zmc/E73/5y1+CxRkPvPZvmmmmCZ/4xCfC9NNPHyZMmDAU5/e///0w88wzh+effz7suOOOQ5dYHD5du+yyS1hiiSViGLvf0AUTv5CmN7zhDeFf//pX2Gmnnfyp8La3vS0ccMAB8dgtt9wSfvCDHww773985CMfCTPMMEMg3PXXX+9Pjfjvm2yySXjrW98a7rvvvnD++ecPS+/rXve6mMdvectb4vHjjjsu/OMf/whwRW644Ybwox/9KH73/8i7n/3sZ4HrX3rppfC5z30unm47P4j/5z//eYz78ccfD1/96ld9Moa+r7/++mHs2LHh3//+d/jtb38bj88222xD5feqq66K6bULuqVzpplmGioPt956azj44IPjpf46nplnR2A044wzhmeeeSZ86Utfisf8v3XXXTdsvvnm8dCvfvWrcOGFF4Ytt9wyrLnmmvHY4YcfHq699lp/Sfy+zz77hPnmmy9+9/VrkoDuQKlOEqRbOjfYYIOw6aabxtgsneutt1746Ec/Go95FvHAa/+snv3nP/8J22yzzdCpTmkZCpT5YukkP7/whS+E6aabLmy11VaBcnfllVfGeugvo/x+97vfDdNOO208vP3224cXX3zRB5nku8/LnO4YM2ZM+PGPfxyvs+dG13AdUrdudCqPMcLMv7e//e3hK1/5Sjxz++23h4MOOmiSUDvvvHNYeuml4/Gvf/3r4ctf/nKYffbZA3nxxS9+MdaJ9CJ0OTodOeOMM8Ipp5wS9fBPf/rTeOzGG28Mhx56aPxu/9773vcG9CByzDHHhEsvvdROxU9fVo0nur3tOL/97W/35fl4iF133TUsvvji8Xm23Xbb8Morr8TvpTaJk74cEQ7+CG3aN7/5zfi9yr+f/OQn4fWvf324//77w3777RcvIa4ll1wyfk/1Xze2XJR7nqb8iI/6hT6mHpJO002kl98rrbRSgBti519++eWw3XbbxWP2z+p3HX1p1/by6XUu6UKvUE8++9nPxk8ft2f36quvxmc3feTD+Xy47bbbwve+9714ulN992XG6oqPM9f+9KJ79thjj7DgggvGW6BPeQ4v9I2WWWaZeOi0004Lp556avze6Rn89en3bs9n7UXVflkvz05b8M53vjMmkefknl7Qkcsuu2w8RB+QPl4qc8wxR9hoo43i4csuuyygizsJ/WbYIdThp59+elhw2it0OfXonnvuCd/61rfCL37xi/i7VCeIAH0y11xzxbjQ9fTVfN+cPjL97VRox2jP0vKLbqb8Pvnkk0PtGtfCgTJI3aA+U/5TWW655cLyyy8fD59wwgnh//7v/+J37sP1xIsQt/Uv4WD6MZ6c+K9JH+GTn/xkWH311WMUcLv88sstuqFPyjvlHqHvu//++8fvnXR5DNDhn2ftx0FN9dmKK644pButv8Xt+/l8/hly5WW33XYLiy66aKSQ0030+9Gbvu5afaa80Ed/4oknhlH0fYeLL744MM4wKZVBn05j3QuXpnlk6az7afdL69yee+4ZFlhggVi3fF+5U/y5Noa2l3EFcvLJJ8d65uPwbfGzzz4b+4L+fPqdes41CGOodKzI8U9/+tNh1VVX5Wvw5bX0rFbXUgZcXzpHO2Rjddoh2qNPfepTYbXVVuOyYtpMZ73wwgthhx12iDrH2uK//e1vcXwQI3jtH+N5xl2IH8PyOzfX0LQtLF3XS5tGGnOCTv785z8fT/m+iA+71FJLDY1RbVzTyzibuEt12PKkbjtCe027jdC2nHPOOfG7/fP1oR9jFc+uW18mN04knU2f3Z6x9Fli7fVlTq8TH+Wdcu/rYz/S2UucdfoPTecmmugT+PWqu9sqq037rLTNiy22GI8yTH/HAxP/+XEHfUv6mEi3+kYYP16ysbO/rtc6tfDCCw+l5+qrrx6aV+DeJsxdMYeFpHN+FsZ/9qqDrZ0wPerj7uV7acxSamdLOqFpPSzpX2uz/dy8z+O2yncv7HStCIiACIjA4AkwJ8BYgnkt5or547v9rpqiaeTAVRWVwomACIiACIjAfwnYoNQ7SvnJIZtMJ7Q3vngnlFlnnXXI2YDJUQakTOJ5+eAHPxje//73x0OXXHJJOPbYY+P33CCRE7l0dZtcM8OCNzbEm0z8V8eBy64ZjZ84H80yyyzRCMfESpoP3rjyu9/9LpAXcKPTheGOyXBzVLLnX2eddcLHPvax+NNPjvcjPywPuZk5Clg6+Bw3blzYe++94yE/6eQnztMJn27p9BNZfnLGX9erA9cqq6wy5Oz06KOPht13390/VjSOegdGOJiD5LCAyY9SnSSYTQCVjMQ5By6cdigjyJ///Odw9NFHx+/2D4cXJu8QypY3SnVKi12f+7R02kSzdxx55JFHhgyz/lo/OZkz1PuwfPd5mTNU5ibmmfxuWjc6lcc0bfYbhx70KnURtpQHDFQmlFP0JedxWEPP+sk/nKyOmehs5QWWhxxySHQ45PjXvva18Nhjj3WdpPV6gjgnlwNXv54PFrlJaI6X2iTO+UlgJjZxGEYwdDOJXFVyDlyUwR/+8IfRYE48ZtDke7eJU8LknqcpP+JDvBMJv72DFoZ9JpMpjybmoGK/+bT6XdJDOYdXf33T796pw+J4+OGHo2HGftvnGmusEbbeemv7GT9zjt4+H7yRolN976Z7cu1PL7rHG95xwMMJ1oR+EA5ElmclB660DbXrc5/dns/a9Kr9sl6e3Zf3XLvgjYDeedI/l3dOxfGqmwMXDoET56BiFFdccUU44ogjfHTRQcEc5qxOm44hIH02+h1eKE84X5JP3oDi++Ylw203B66nnnpqyFGYe/q21Bx8fVqYpCMPKfv0EXHQNCcvjH4Y2xAcp3hJAmdg8hBJ26QmfQTv+ETbRRuWii+D3gnAOHuG6bWl3561Hwc11Wf+ObxDjD/e9vP5Z8iVlyYOXObURR8ldfijvKJf0DNIyYErLYM+nca6Fy5N86hUFrodt/tZH9bCmxMAv3P9TgvnP3NtzAorrBDHZoTjhRDKTyqmZ5977rkhx540jP32L12lDp2Eoc7TF6WNR3x5LT2r1bWUAdeXzuUcuHCwxWkBwTnQxhvxwMR/OHfhlIF4x/lOTrG0BaaD6zpw1WkLS/2AXtq0+KCZf29605tif5E6R13EWZu89/KNb3wjvuzEMT+2tbLC8TrjbMKbA0Fah5u2IyUHAu6F+PrQltOAj9P343w7kquvuXEiaWz67FzbSUqsvb7M6XXizDlw9SOdvcRZp//QdG6iqT7pVXe3VVab9lmp1/PMM08sXvTL7r777mFFjRdqx48fH4/5uTRfN3LPwAW5saa/rtc65ePixSrmQvy8JC8J0D+29qmKA1evOtjmpL0ejfB6/Fcas5Ta2ZJOaFoPS/rX2mzfd/b5kisbTeaNesSny0VABERABAZMQA5cAwau24mACIiACIiAEbBBaRUHLj9pxACdSUCbMGQlIt72RHC6wPBr53AU+dCHPhSNUUw0MtC0t7Zzg0TiyKWr2+SaTUpWNRRyn1QwptrKNxj4MEaMJuGNXN7MRW6++eZoTDNjG6s5wJ5nRODJ5D3OSe9617viMYzs++6779AqEqy6gFOXGXxtJQi7HocGJDfZ2SQ/3vOe98TVl4gT5xVWt8AwiLBCEc9nxkHe7MdAhLAiFhMeCPnP5BtvovPs3cpNzoBOPP66Xh24iM/KOt/vuuuu6MSIQwSrS7zvfe+LbzBwDoFdFQeuTnXSJoBKjhM5By5vKMJJCAPAQw89FPniyMcKfFYWqMt+NZtOafnvU+X/Wzq94cfeJuSK3//+9+Gss84aupjyTZnEOcnewB86Wfji8zJXVksT803rRqfyWEhiPOzfFibf0D///Oc/A8YanP7MMGrOJd6IQwTmIMB3non6bisneR3fj4m4fsTZr+eDT24SmuNWT/3EJcdNzDBiv71Tkx3r9plz4OIaX4eoX6QRHd2NLdfmnqcpP+JD/FvO/P773/8ecHAxYdULW3GHY7nVPKx+l/RQyYHLG/DT+9r9u33iwIQx2uTMM88MJ510kv0c+vTOk3YQ51WcWL34fPBGik71vZvuKbU/TXWP50YZpg1Fh9PGb7HFFkPOnDyXd+Dq9AyeQfq92/M16Qc0ffaSMcSnGYdW6gXCak70NalrlBPalXe84x3xXK4fGU8k/9CzOC3RHiEwPf3002Pfg/4MhjIzsrMiFfXAG00wUOG4aU5crKhIH8ycQ71DktcPpLvOClxWD3lW4qdM0D/yzmKc++UvfznUr6Ju4+RDmhBrd/junR6Jh2djZS7vlEG4Aw88MPZ3+O7Tn44hOF8SezGB81xH/wBuMGfFUPomSKqzu+nyeFHhn0+rORUR1Dj2os+8Qwxx9uv5/DPkyksTBy6fVhz0KC/kBY589K1Ywc4kNawau7QM+nR61v5edfLd7lM3j/z4o06bY/fzfVgYeN14/PHHx5Wn6S90klwb49snrkdfwAPBiM2KgfaCFIZ6W226032sbhCGa2BNfxrdSBvPi1kmvryWntXiSxkQR+mc1xW+7+pXNfR6gtXM6OfYGJAXwXgJyd+D7xdddFFczYnVqBlb2UpinKviwNW0Lex0XdM2jTSXBMcGWxmR8S7jNZxA0Yu0ZTYXwPXe8cCX8zrjbOKx/E/rcNN2xLeFvq3jXoivD205Dfg4KV+2epuvr3XGiU2f/b9PWP5fYu31ZU6vE6ONU3x97Ec6m8ZZt//Qy9xEE33iy0IT3d1WWSUvm/RZvb7hxRUcnujT0pfDGc70BvF7J15fN3LPQPjcWNNf10ad8rqNdOO49MADDwRehqStsz486Un7GRzLiWdC34B+sK2Y22muk7ioZ4jXo/FAj/9KYxar+77+cis73m/9a22271P7PM6VjaYOXJ4BKzDSDkhEQAREQARGJgHaZObMmNtivMEf3+131VRrBa6qpBROBERABERABF4jYINSb9z3k0N+Mp0BMxMJXsxgixELowoDPBMmSplcpkE3SZ0xcoNEwubS5SdUcpNrTQyFli77ZDKKyR6k9Da8hR2Jn29+85vjZDmdKYRBPhO05IO9rcZxv0Ubk+EYBDASmGCQSTtibCd01FFHWZBhxok288O/OcjNMFiQFm+Iv/POO+OE1FBiJn6xiQ07ZpNI3cpNyYDur2Pix94AtPvUNQ6xVQ0ONb4+WFr5JK/sXFUHrk51sls6cw5cMMYYbcZj0gV/X3aY0LHzlBMmW9iCpVNaiKcklk4/UeUna7mONMCf+xojjrOK3HnnncfXjuLzMldWSw5cTesGibHnsoRZebTfuU/0JzoRQ5AJxnGrzxyDP8Zy6jXy4Q9/OBqp4o+J/yhH5IvPM9jttddeUacRrh8Tcf2Ik7T24/mINzcJzfFSm8Q5xG8/xO9rrrkm4JBVR0oOXMSBw4MZwc2I3I0t15Wepwk/4kP86pr8xpGSNtzErz7EMYyGVi4tjNWDuvrSr7SBswnlt66kbQkOCzhE5iR1RmNSOd2a1edDWp/tOS1uO99N95Tan6a6B13mV3Kz9NgnusHaUu/AxfnSM9i1uc9uz9ekX9b02b0hILcCF+nnRQO2orV2BH2JfvT9H46xDR9O/FVkww03jC8pWFiuR+wefPcOCvz2K33xGz3PdZY3HEv7oL5vXjLcllbg8vlE3L69ZYVVc4LiHOmgzaW8m/BCBi9qwIoyS32xtJ544onh7LPPtqAxnL3QwX3gTblr2kfgRQG2F/Y8SR/3t2OkGSZ+i8VuunwowZkvnrUfB1kdqavPvGOld4jh1v16Pv8MufLSxIGLLcw322yzIWJpWad80PewfPGrKJXKoE+nZ92US9M88s9Wp82x+/k6BSD/XAaspJfsfKmNSfsdtE3oDHSlsSYv0CsPPvigRVf8ZLtm6rNdS0Df1/fffXktPavVtZQB8ZbOlRy4FlpoobjSnu/3pv3gdLVPv6IM90wFNjwr5ZPxnEluroFz9pwWztpz+136LF3XtE0r3YfjtPWw9W2X14v2zIRNHQ/SvhHXwcd0OtfkxtmlOkz4uu0I10wJDlxNn53rOkmJtdcrOb1OnDkHLo43ySOu6yR142zSf+D+TecmmugTz9ievY7ubtPBpUmflRe3WK2+pN/RDdR56+OhF3nxEYcmnKWQ3DNwPDfWLLVbvgzXmXtBpzGnbHNN3NeLb5+qOnD1ooOtnUj1qE9Tk++lMYu1I2l76nlyP3++bj3k+pL+tTZ7EA5cfoVM5hKxK0hEQAREQARGJgE5cI3MfFGqREAEREAEpgICtu2An5wcP358XDWAx/eT6fzGELP00kvzNQqrWbCqBYLTAQN7W/ElHnztH5PNbM/yxz/+0R+ORiDe8veDUALk0oXTApPOSG4iwAyFOcOK30Lx+uuvDxi5cuIN1qnxLBd+JB7jrVsmqL0Dh0/nX//61zgRxgSICXmHMceWXLfjfDLRg6MMA2sv/cwPvxKRvydpYWUqJpiY0PeCEXWTTTYZcnZhlSucIbqls2RA99d5By4mEd/4xjfGiS7CpFJaUYZwOAhSh+acc86hiTXygVXGeC6cLZDvfve7ge0qq0ipTnZLZ86Bi/vNP//80VjqnYg4jgGW+n7DDTfEFfbMuHLuuecG3lBFSmmJJwv/LJ2pDmD1HwxqfgLSoiDvMRjnVtOxMP7T52VOd5QcuIijSd3gulJ55FwnYSKUushqA6ng0IOeY8sDL7zNjAHL8sSfIywreTHgMkE3sBUSknNA8m9Soou9UZxrvPOB8exHnNwLafv5iDM3Cc1xc+RJyyPnEFYyZIVJE1ZGY1vUOmLGldw2bsTPSgpW7skndEOn/OLepefhXF1+XGNiE8n8xokKw7YJhgWMwIh/i9vO82n1G8NAHX3pt53NcfL3KH33K4j5iehceN/mlNLqy7jvMxFfqb530z2l9oc4m+qeueeeOxpjvGGX+Nj2iFXJcCRGUgeu0jPEwIV/3Z6vab+sybOzvR+OMgirddoqsGnS2YoZQ0jaxhGOvMdRndWm6gj3ZUthbwDnetp3+r2wTgXdyXZmObn22mujEdaf831z9AGGq1RKDlxsf8zWZBixEBwtcbg06aQjqH/UY+o4ssceewS2KEX89jvxwMR/tGGEt/Lnt2Nu0kcg3k55xrOwglC6TVA3XW7pzX161n4c1FSfdXLg6tfz+WfIlZeqDlzpmCp13DV+tIOs/sQ2rmxdhdBvxPkIKZVBn07Pmmua5HvTPPIOXOY8TRq6id0v7TPQr8RRhpWgTGgv4VmSUhuDXkHXwjAntG8Ymaus3GvXL7LIIrGvb0Z8O05c9HGsn+MduErPagbflAFxls75/kP6chf9ILZLxenTC2NAxvC0YzgdePH6344TnlW6mAOAXerAlZtr4NombWG365q0afYcpU9e2sLxgrGlF56bFwW22mqryDDneOD7POm1pXF2qQ7b9XXaEa5h5RtW20HshcD447V/vj70Y6zi+3Hd+jKdxokkt+6z++fMfS+x9voyp9eJy8YYufrYdjq5X504m/YfepmbqKtPetXdbZVV2CKd2sFSn3XNNdeMq+6m8wKUCfQ4831bbrnlf28w8T9tFeO7JmNNX0/bqlPESR+VOWd7BvQa/VD6uTioIelWzfFg4V9THdwvBy7fZvkxS6mdLekEe9w69ZBrSvo313f2edxm+Wb7X1v5uLRKtj2fPkVABERABCYvATlwTV7+ursIiIAIiIAI1CLARCFOWkxC8xZs6kgzduzYOOBmssXCXHnllbXuocC9EWCyH0cu3nBmgpfOFgaJm266KbCcekl83rFSCRMxOBJhhBm0YNjGCMTKVUy6k24matJVUXy6mHTjrX0MepTNkgHXXzO5vjN5xrYXZhRlIo0JNwRnDNtmtEr6utXJKnH4MEyWrbTSSoE3V0kjhmL/Vj95Q75g+MGhy3NuMy0YbzB8ci8MzzBhGX0mj7xDkk97v743qRu9lEcmGin/8847b2TPKok4lZYEPmwzyh/3xYERJ4TSikOleEbq8ZHyfDglYBzBwerxxx8fcoYZqdwsXSOFn6VnSvzspb534tFE9xAfRiv6AAiradAmdpN+PUO3+5bON332Unx2nPrLtm/woc1CT9I/glMvguMtBif6xRMmTIj9J+8sn8Y988wzx/CLLbZY7NvQ30JvY2Trh+A0Qf8Qpyzre9h90G28JMEf+oL2hm0a225DmvYR6JeQZ6QPhwwc33khgWeZEmQ0PR/1EuMfWzLhHIJTje+bk0f0EylD9EW8dCqDPpx9HyQX6gCOLzkDpaWn7idjYV5UwtnR96PrxkN4VufE6RMnXbjgNEc9oI+OcbuuoAvQgfSxGefhuFOlnah7n6bh0RWMZSlL9957b9Snnfr+jEkoXzg8MGZEP3QK3yldTdvCbtf1o03DoZbVVBBrQ6qM25uMs7lHpzo8qHaEdIw06cezd2Ld9Pn7kc5+xJk+Hzqv6dwEcdXVJ23q7vRZ6v5u0me1ORT0IXOAtBM4fJnQjjCfi9MuK26l87kWbnJ/km+UL3uBiLkpXp5C0tVtq6S1Hzq4yn3bCtNJJwyiHrb1HIpHBERABERgdBFgTMXYkT4JfTL++G6/qz6NtlCsSkrhREAEREAEREAEREAEBkaAySZW50BwQvPbDVkibOs0jDCscNDEGGNx6VMERKA/BPyqTr/85S/DRRdd1J8bKVYREAEREAEREIGBEbAth3Mrjw4sEbqRCIiACIiACEylBFj90V40wKE6la233jqu+sbxgw46KNx+++1pEP0WAREQAREQARFomYAcuFoGquhEQAREQAREQAREQARGDgHeyLZtO1mNgxW2/DYqbDmz6qqrxgTzNv3+++8/chKvlIjAVE6A+ssbymydyvamrE7DaiPbb7/9VE5Gjy8CIiACIiACo5/AaqutFthWjj46242ykq9EBERABERABERgcAQOOeSQoe1z2YKXlRRN/HbU6Za4FkafIiACIiACIiAC7ROQA1f7TBWjCIiACIiACIiACIjACCKw7777xu3wSBKra7G1DEuds8UlbxoiOInsueeek2xtFE/qnwiIwGQhsO2228atPFke2uTMM88MJ510kv3UpwiIgAiIgAiIwCglsMIKK4R11lknsLKmbdU0Sh9FyRYBERABERCBUUlgs802C+uvv/5Q2tnumy1hmS/jBSqEebRDDz00bn0+FFBfREAEREAEREAE+kZADlx9Q6uIRUAEREAEREAEREAERgKB6aefPuy9995h7Nix2eTIeSuLRQdFYLITwIFrpZVWGkrH/fffH/bbb7+h3/oiAiIgAiIgAiIgAiIgAiIgAiIgAiLQnMA222wTVl555eBfnLLYcN467LDDwvXXX2+H9CkCIiACIiACItBnAnLg6jNgRS8CIiACIiACIiACIjAyCMw///xh9dVXD3PMMUdM0N/+9rc4CfXggw+OjAQqFSIgAsMIjBs3Lm5xypZKt99++7DtHIYF1A8REAEREAEREAEREAEREAEREAEREIFGBGaaaaaw3nrrxRcfx4wZEyZMmBBX3GLeDCcuiQiIgAiIgAiIwOAIyIFrcKx1JxEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREYRkAOXMNw6IcIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIDI6AHLgGx1p3EgEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREIFhBOTANQyHfoiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIjA4AjIgWtwrHUnERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERhGQA5cw3DohwiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAgMjoAcuAbHWncSAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQgWEE5MA1DId+iIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiMDgCMiBa3CsdScREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAERGEZADlzDcOiHCIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACAyOgBy4BsdadxIBERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERCBYQTkwDUMh36IgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIwOAIyIFrcKx1JxEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREYRkAOXMNw6IcIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIDI6AHLi6sH7zm9/cJYROi4AIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiEA9Av/3f/8XL5ADVxducuDqAkinRUAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEahOQA1dFZHLgqghKwURABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABCoTkANXRVRy4KoISsFEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAQqE5ADV0VUcuCqCErBREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEKhOQA1dFVHLgqghKwURABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABCoTkANXRVRy4KoISsFEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAQqE5ADV0VUcuCqCErBREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEKhOQA1dFVHLgqghKwURABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABCoTkANXRVRy4KoISsFEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAQqE5ADV0VUcuCqCErBREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEKhOQA1dFVHLgqghKwURABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABCoTkANXRVRy4KoISsFEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAQqE5ADV0VUcuCqCErBREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEKhOQA1dFVHLgqghKwURABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABCoTkANXRVRy4KoISsFEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAQqE5ADV0VUcuCqCErBREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEKhOQA1dFVHLgqghKwURABERABERABIYRWHnllcOcc84Zj5166qnhP//5z7DzuR/TTDNN2HDDDcN0000X7r777nDzzTfngrVyjD7OWmutFbjnVVddFR566KEY74orrhjGjh0bnnnmmXDBBRe0cq+6kQySQ920dQuf4/fWt741rL766vHSK664IjzyyCPdoul4ft111w2zzz57eOyxx8J5553XMeyUcHLxxRcP/CGnnXZaePXVV3t6rEHzW3jhhcNSSy0V03z55ZeHRx99tFL6V1111ZjPTz75ZLj44osrXaNA3QmgXzfYYIMw7bTThuuuuy7cf//93S9SiKmGwHvf+96ob2adddYwZsyY8Pzzz4enn3463HPPPeGPf/xjePHFF6caFvagSy65ZFhkkUXCs88+O8W0OW9729vC2muvHR+Rvk7Vdrnt9twYT02f0sG95fbMM88cxwrEcuGFF4aHH364twhH4NUjfQy14IILhmWWWSaSO+OMM8LLL78cv+fGAIPCu8oqqwT0Wpvjx37E2SaP9dZbL8w444y12ibrkzOWOP300yuNz9tMM3FRTkjHE088Ec4+++y2o+8a35Q+vhg3blzgGZHf/va3kyWPu2bCKA/Qy1zN+9///jDTTDOF22+/PY7D+oWipKfbLP9N+5L9euY24u0lb3P3nxIZ5Z5Tx0RABERABESgTQJy4KpIUw5cFUEpmAiIgAiIgAiIwDAC3/jGN4L1I77yla+EV155Zdj53I/pp58+HHTQQfHUHXfcEQ4//PChYDiDMUn973//u7KhcejizJd3vOMd4ZOf/GQ8g1OMOWvtv//+cVLthRdeCLvttlvmyv4f6sSh/3fv7Q45fjhvbbrppjHik046KVx66aU93eTAAw+MjgX/+te/wp577lk5rhlmmCHMPffcMfyDDz4YyOPRIDvssENYdNFFY1J33XXX8NJLL/WU7EHz+8xnPhOWXnrpmOYTTzwxXHbZZZXSb+l87rnnwte//vVK14yGQPPPP390nsJw9dRTTw08yW95y1vCPvvsE+97/vnnB4yvEhHASXSrrbaK7WyJBgZf9Pcpp5xSCjJFHv/Sl74UMIjipLDLLruMqmcs9Z18u/yb3/wmOrJXeTB/XRvteZV7DiIMTkE4p/GywYQJE/p6S+ng3vDSn6BfgRx//PEBx/CRKk3LVdtjqLb7HYyfGEche+yxR3Qg4ntuDMDxQYj1ldscv/UjzjZZ/OAHP4gvIqVtU6fxzic+8YmwwgorxGS0MaZo8jyM3WabbbY4pt99992bRNHTNVPq+MKgbL755gEnUKTqHExTXWX3nNo+e5mrsXr7t7/9LfzkJz/pG7qSnm6z/Ps+YZ2+JA9d6p/2DUjFiHvJ29wtpkRGuefUMREQAREQARFok4AcuCrSNMNrxeAKJgIiIAIiIAIiIAKRQNvGBxy7mFBhJRBzPugFtRy4eqFXvjZnvPETV20YfG3isa4D1/ve977AH3L00UeHG2+8sfwgI+iMGZBIUhvGlkHzkwPX/woTDgJ77bVXPHD99deHY4899n8nB/RNzgMDAj2KbjPLLLPEcvm6170uphonFiZMMIS/8Y1vjA6zvJFuwkpcU8Pqh/a8o9mBq9R38u1yHaObv66N9twYT+7PL3/5y2G++eaLDlzkdz9FOrg3uqPJgatpuWpzDNWPfkfJMSA3Bugtt6tfjZPazjvvHC848sgjW1nFuR9xVn+i7iHNESR14Oo03tlss83CaqutFiM3Xt3v1G4IOXC1yzONrYkDV1Ndld57avndi5OP1dup3YGr1D+d3GWol7zNpd33m+v0t4lrpDLKPaeOiYAIiIAIiECbBOTAVZGmHLgqglIwERABERABERCBYQSaGB8wEDPpiBGZZeWvueaaoTjbnsAYyQ5cnTgMARmhX3LGG5aOZ9s+5KKLLgp///vfe0p9Gw5IRx11VLjpppt6SsegLl522WWHVrBi4q/XLRQHza+pAxdlhrLz+OOPh7POOmtQuPt6n34YUusmWM4DdYlN+eFZoWHeeeeND8qWmj/72c+GVlTh4Bve8Iaw/fbbx1WojAarZrAi5tQgcuD6Xy633Z7/L+bJ+22Qxmvp4N7yWg5ceX6lsUM/+h0j0YELKt/61rei0zHt2Pe///08qJpH+xFnzSQUg5sjSCcHrnS8s/766we2SsZRu9/OqqWET24HrilxfOFZy4HL0+jP95K+rXI3q7eTy4GrzfI/JTon9ZK3ufyfEhnlnlPHREAEREAERKBNAnLgqkhTDlwVQSmYCIiACIiACIjAMAJNHLiGRZD8mJocuJJHH1U/cw5cbT/AoB2Q2k7/5I5v0PyaOnBNbk79uH8/DKl10ynngbrEpvzwBx98cHScfvHFF8PXvva1opOo6XeIHHfcceG6666b8uFMfEI5cE352SwHrtGTx3LgqpdX/eh3jFQHro022iisvfba0TmJrbeff/75erAyofsRZ+Y2jQ6ZI0gdB6411lgjfOhDHwqvvPJK3F6v0Y17vGhyO3D1mPwRf7kcuEZ2Flm9nVwOXG3SkXNSd5pi1J2RQoiACIiACIhASkAOXCmRwm85cBXA6LAIiIAIiIAIiEBHAk0cuKaddtrwkY98JMZ7ww03xFW4xo8fHxZccMG4AhFvxLH60C233BIeeeSRcOaZZw5LA1tdvPvd747b4Mw444zh0UcfjWGvvPLKSSbx66zARbowViB/+ctfwr333htWXnnl8M53vjPQV2K1sEsvvXRoZSnSQPxzzTVXXKGE8Kwg9NRTT8U4uv3LcfDXMBHE/dn2irDE++CDD4ZTTjll2Kop/ppO31l5ZcMNNwxzzDFH3CoLIz6d5WuvvTZcfPHFRWN+Lk4z8LP11m677RaDjB07NmAwQM4+++xJOMw555zx/AILLBANCuTv5ZdfHjBIrLnmmtEQ86c//WkoD70D0n777RdWXHHFwCpVc889d+TNm/eEf+ihh+I93/SmN8VyRX7MPvvs8djDDz8cHnvssViGKEslYRKcssR2jSeccMKwYMstt1zgD8GRgTLrhS1MeHZWkDr11FP9qVh2LM2ceOCBB+KKYMTDW/FeuMdiiy0WtzMjj1MZyfxIa1MHLt4QxvBIPl5yySXxsSmrHMfw9Lvf/S7WAcoIZYd8Ik+pj+eee+4wTJwnHEIZtLLhA33iE5+I27SyMhur/1GullpqqRjkiSeeCH/4wx988PidejPbbLPFskp94d4loSzhPEVeIs8++2y4++67Y7nxjjDoFAyGpJnt69B5hL3zzjujHuF7TqrqhW4OXO9617uGVnz7xz/+Ec4444zc7YaOWZ5QbtlSbbrppgukZZlllolpZ6tS8u+ZZ56J56gXiy66aJh11lkDXHkutuTLrSzHaozkN3VlpplmCrQBrPpE/eWatO5aWuqWD3uYuvez6/hExy255JLxuUgf7c6tt94adQR6AL2UW/WvTrtFXd9ggw3ibU877bTIk3KFXuN+HKsr6MdvfvOb8TLy+4ADDihG8YEPfCCstdZa8fwVV1wR66APzLYjm2yySeCZyF8E/XfbbbfF/PJh+b7SSitFZuQp9ZnrKDuLLLJIbCeuuuqq+FwvvfRS1AWUBerFmDFjYt6zDSltRUloo6vq2VIcHO/kwDXDDDPE9oVnR1L9Qhu9zjrrxGcin3gWyjw65p577onX8I/yzdZWSKd6Z/qO+kabVNIH3fpOcN50003j/VjZkT7EKqusEvWT9Z8ov+eff34MY/9K7bnlpenKJZZYIvZTqJPUK1be5JnJs5ywyptx4ju6kf4W13GcY1xLOr1U1Xv+Gv998cUXj/1GdBL3QKin5BNc0CUmb3/722M9py9BGaTcooNOP/30SdJl1+Q+m+jgXnUb6aDu4vxEXw89jU5GL5144olD/SvCWRnj+/HHHz+sjLECG+0eQjtKefdC33f55ZeP+hx+9CerCNe85z3viW0kdem5556LOoC+3F//+tdhUaQOXE8++WS8dp555ol9YvoBlBX6ryWpk5evf/3rw8c//vHY/lj/II13vfXWC9zf6m6dcpXGxe+2xlBV+x3cs46+ZExEXiN77LHHUBnJjQGs7LbdR7C+Bv0CyjL6hTrMGIFjF1xwwSRtoqWlTh+hH3FGcC38M0cQc+CqMt6hrm211VZxTGHjNJLCmPLDH/5wHLvS98T5jfIMV/oXdYW2j60aqQu0fdRT+oP0lXfZZZfYd0aHsppnKvQFqo7n7dqFF1446ibGDbTLjGPRz7RhjMO95MYX/nyd/hzPSX1A6I/QdrHCGelhfMBzc3/mAUrt9cwzzxwdD3lu8oHyzHiCftY///nPGHfuH7qQeojuYdxNm02/CX3EPAHCCqu+HUvjqaOr6ujN9D7+N3lEHiCszE17u+qqq8YxF2WP+Zurr74627/beuutY5/CxlzoZrjB6dBDDx26DWWOtoq5AfpX3IO24c9//nMsgxbQt2n0L8455xw7NezT+qamx5gXSOes/AWM5Xgm+hZ8py9Du3TzzTfH1QHRUSUHrjq62N8z/V7S022W/yZ9yU79U9p++vlIm/2POnWMOt0pb0lbHR3RNqN0/pP0eKGeMgaHJfNP8LT+PeHou5133nnDxiAcrzPGbKoLmNOiXtCPpg2nnaFu0A8tzZNSv/vZHvDspOWDH/xgHKtRX5mTQKfcddddcR6Eei8RAREQAREYLAE5cFXkTcMlEQEREAEREAEREIG6BJoYHzDasNIWcscdd4TDDz88OgExyE8Fo9BXv/rVocMMujH+5ISwTOoxQWDChKdNuGL0ZqIfyRkffLomTJgQJ0rNWGvx8YmDB84ZGC5TYWKVbTiYlO0m/n7GgWswgmIoMcN4Gg8T+L/4xS86OpGk12y88cZDji3pOX6Tboz71nnOhfHHcvz8xBUOHji7mXCON8GZyEyFSU4mpxG/BYg5cDHxzzMzMZsKk9U//OEPo2MUBmdfVnxYnFNSA7U/78uxN1IRhrf7mfhFmGwmXV7YvoVJQAzacEEwBO6www6ByaicMFn0ve99b5gxlfBMACO77rprnIS2a0c6P9LZ1IHL8pkJSFgjvixhOMZIAeNUcIzwE/kYU9g2BsGojIEjFTOE2dY7GEQwfFjZ/P3vfx8n/u06M4Lxm4k96gnOKiWxVY7S8+iUQw45JB7GcYX8tnumYbkP29t5R7G6eqGT8wA6FF2KcK9f//rX0XiXpsP/9nmCQQUuafqJ60c/+lHUubn6itGEck84E4ym1BvqTElSA61PS53yQfxN7sd1OLvg4IOxKBV0FO0PBrmnn3467LPPPsOC9NJu8XyUa2ON4e7nP//5sPir/KD8UDYR9CbMSWtOYGROjRgpMUyaYBjfcccdh5xg7Lh9Yuz66U9/Oqwd9rqNOonTZCowRP+zhSNpTeXCCy+cxEG2iZ5N4/W/Sw5c8MD4bGUa/f3tb387tktcT5nYaaedAkb1nGDMO/bYY+Mp9Bh1gE/qQWn1GNMjhMH4XnKQ4VynvpOvK+gTjLg5wfkOnWPir/PtueUlZQgDaak/huMlhiMv8803X/j85z+frev33Xdf3N4TLp5XXb3n7+e/e6dEf5zve+2111DfByMxzq05IS8w3KdO3rmwHGuigz33uroNHfWFL3whWx5ID3mGodTaRd9m4zTuHaFw+iMtdh1tpBfrF8GEcznHXB+e7/RrMLKXhL47dcPEO3DRXlN+ckK7QruTrsJUNy99fuFMdsQRR0xyu7333jv2zW1sUrVcTRLRawd837ObA4bFkRs7mL6wMPbp+x1N9OUnazhw+bLbZh8BYzMrRiLeEchW08NZhn67F5+WOvWoH3H6dDX9bv1Wc+CqMt5hPIG+9nwwjuO8Zf2JND2MRWm/q9RnrqVtZNyVG68yrsSZhnbR55vds26/iOtw6rEXaiwe+0QX4ZRDfpvkxheca9Kf8/WONop5AOJJBd7MA/DMXpiP4AWSXP+GtDP2uOyyy/wl8funP/3p6KSRnuAaHNes/e+mP6rqqrp6M02X/+11OGN9+pW5ssdYjrkgyrcJY3sEZ0D6LTjrIXXKM/2Kn/zkJ7Ft8C8xdGq3vvvd78Y+CmFo52g30zmrmJCJ/+gP0/fDKSQVXvBAd/G8qQNXE12cxu9/l/R0m+Xf69SqfclO/VOY2NxPW/2PunXM12k/DwfbJjqibUalOS3Le/TJCiusEH+WxlaUY8ZP/sUfPzfaaYzZRBdQ3tFZlreWVvskPcccc0ys13aMz0G0B9RH9CT5nhPGxLS1JQez3DU6JgIiIAIi0DsBs0Exj0g/mbaEOSH++G6/q95pmolvN/xvtrnqVaMgnBy4RkEmKYkiIAIiIAIiMAIJtGV8wFjEZDMrBtBBY/KYNygxHDOZjPiJBCYAeGMYpw+ca1glAeE4hhxb8cJPUtRx4IqRTfzHZCL3wTCfm6ils8kENYYf0o3wFtePf/zj+L3Tv9LE0TbbbDNkPGcCHMMVE8FMFNokOROKTCyWjLr+vqxshlGP9MGHCU24YgDgrWGbTE6NZz6O9HsdBy7vAEM8MOX+vCWZOm3kHLjs3qSdMoGRjglrm4DhN8Yd8uBzn/tcNBbwZi/CZAznceBi8rgkfhLMG6sJ7w1jcGfyxwS2X/ziF+NPjNlMwCNm5OM71/AWJJxtRQyOky7CWR6aYZxz3oFrNPAjzd4YzBuWOUME4VLJTTD7SVALTz3DgY5xC2XHxDvnNXHgIh6cC231OPKLfMFIgMEAQ4xNzvOGKxOhnYTygKMHdQuhDjMZSPkjrcSJ3rSyj37BAE3dZnLRjBQ8L+XApK5e8MZonBe5N8KqKzh0ItQptsjDGNVNcnmCXuLZfLm2eIgbp0aexzu2pAy9UR/m6G6uQT/563CYsZW4cmmpUj5IW5P7cZ3pPL4jOOnSTlEWTfdzPHXg6rXdIk4TmLICoDkD2fGqn6y6Ze0YehgjO3qLlQGrCHqVsstkCsIkC9dixMUxw8o0ZZ72yYzAXrfZfSgb5Jm1+XacT67DSTJtI9iOiXuaNNGzdm3uM+fAxbPhHGDzJeggDHzoCSRlgl7HcEd4Vp+wsuEdKXCAw4kTyTk6eaNnt3a5W98pV1fIH+oS9Ys0mngjmr/Ot4m5vCRP0APoPHiZUFbM2ZV78dt4mH4gvLXXdp134Kqr9yyO9JOVLmw1CtPn5CWCwYZ88/0A0ke7Tfop2+hnSzsrcbFiVDdpooM9d4u/qm7bd999Y1+V67iGukle46Rs5ZfnovxSRllJ8LOf/Wy8TWq8NActSwP9IK8nzHGd9qvTan52/fvf//64whq/qTv0k+n7wNb3n1lV01ad8PXA4uF5yJe0fmFQtrECYZvkpc+vqg5cVcqVpT33SZ2wvOnmgGHX58YO3fodXNtEX5YcA6w9JD9sdadc2W2jj1By4PLlF2ePe93qS7m0VKlH/YjT8q2Xz9SBi7LabbzDOIm8YbzFKsaUM/INPYYeQLdRp9HNCy200FC7Tt+dPnwV8eXXdDqfvu0jHsqBX4GrSb8Ix+8tttgiJgsdgg5D/9CHQMeZHHbYYXH1S37nxhcct/LLd6RKf87Xu/9e9d8+NG0pbQrtn7URvr0nLCuEfupTn7LLYntJH4hrzDGckzhTeAc0P67iPOmkn8wz29id40g3/VFFVzXRm/+9e/5/TofT90SHM6/inx1HraOPPnooInPgGjrw2hd7WYqVh3AUMcGxnnhxqGdeyPqpPi/oy9lLGDgzpyu74ihGvUKs7+Xz3beT9Alw/rY8p0ySBuaj/HMRV+rA1UQXE09JSnq6zfKf06nd+pKd+qe85Ndm/6NJHSvlLZyb6Ii2Gfk+TS7vfX218+ha5sr8fALnGDfaWN/Pjdp1fKK7bYzp4+Z41f4weo68sPior6QHHW1jeuqKzbMQblDtAS/gWRrQpcyvokeZy7O5F9LKvI9EBERABERgcATkwFWRtU0aVAyuYCIgAiIgAiIgAiIQCfjJ226Th4as04QJbzlyPjWCMxmM8ZaJMib/WMnGG5PYJgJHF8RW1uG7n6So68CFUwUTiEz6I944xm+2gmLbA4S+FBPkTBhizCSt3aTEwRgwwYFjkt2f+PwkXbpSUOl+O++8c5j/tZWgeMOVCUgTjKdMUmF4txUF7FynT5vYKhlvvMHXOw1gmPPb3vmJHu5XcuAibRgMzYGDdMPbJlyYbGHSBWHrNv4QH188UPjnHbH8BC0rPvBGvhdvyGTpfd5oR8zBhK1EbIssJq4Iz4Q7Qp5jGDVHPD9Z7w3j3oFrNPDj2byhoW0HLia/WXXO6oJfHYR6am9HN3XgIv2+fuPkiMHMO1rYZD5huwn6iol9xDsj8BsnAtuaKrc9HW/c4ryEUM8oQ0hdveCN0ebAtfbaa8dtG4mPCVnqRyfHRsKZpBPTbIdi200yIUpazWCC7qK+kjeIN+J4wzhGDq5DMHxQj0mXiV8xwNeVNC1Vy0fT+7ENBKtWIDh6UFb4RNAROMhStxHfdrXRbhEnE82sukUZ7EUw4G255ZaTREF+oT9tq0+2/svJtttuG9hOA0kdJpgExzhmuo1tcVjZDfG6jfxlZRtWEkN82eA3xubvfOc7Q3y9EcGvqtdUz3KPkqQOXDi7YXS2CX8MxtRDc0wjHv9sqQESAwpbSJnDEMY0nIZYvXO77baLycgZC7zegSEsu4npB1/+uCatK6nO8e2l13H+Ot+e++clL1llyxxuuJ93/DG9w3FWVrNtZXG0oA9ndZ28RKebIdTrTHuutvpDtsIO9ya/TXAWpF6TBu5F+uhLmrAqF0YmhPxHv1u7bmHSzyY62HMnvqq6ja0tP/axj8UkMAmKLjUdxUFfj8wAzrPSP0Fvpw4WHPcOAmyPe/LJJ8f42TKMldQQn8fxQOGfrWoCW2+4I7jvZ/n+e6obKBc4HFu5QRdhALZyY32wpnnp88u3U/6RzPCe9pdL5cpfm/ve5hiqU7+jqb70Yw70u5X5bmMAnrWNPgLx4OhhL0qgL1lRx4S2gnEAbRaOOyZN6xHX9yNOS1fTz9SBy+Lx+jsd75hhGn3A2MmHZdtav5IgbRxlkbqU6gK7V/rp+0Vcg84wh13aPvSrjdF8nE37RVbHSId30uK3H+v7MVzOgcWnu05/zo/ZuSdtLSsA2zP7dp15Ctp+Ez+OS8dH6G62QIW91ytej6HzcG6y7bkZszPnYqtDc5+qczDGcRBtYKrDvTMVaUaH069ESI9f4dg7cNFu0BeiDTD9Txtnzt/piyE4xtEXsTbsyCOPjGMdrxd8WxMTMPEfzlu2Sqk5vPh892XLh2U+ij4D+Y6wKhJ9bWubvANXU10cIy78K+npNsu/Z0cyqvYlCWv9ON8/hU2b/Y8mdayUt011RNuMYNdJfL+OcOkLIYxbeRbEr3Lu9SXn0jGm1z11+sP0oegjkbfUU/Q0zvom6ERztiBEOloAAEAASURBVLU6O6j2AJ1gc7PpuIs+MCtn89wIjs/Mb0pEQAREQAQGQ0AOXBU5y4GrIigFEwEREAEREAERGEagTeMDEecmeTjuDZq/+tWvstt92eQNkwY4PjH49pMUdRy4iIOBvhkrSIOfKMlN/NmbnVzrDYNcm5PSxJFN1DORS5zmtEIcGDLM8Mu2YlUMu0zW4hDCxBn5lYotcV813VzfzXhjBl/vMJBjRlx+EtYbIGzikTDHHHNMfCuQ7ybeIOyv80YKf9yuK32awdIbGmxlJtjYRKw3ZGLEZfLJT7pjRMTBjGswCMPdC04BlFWESTMmuxBvGDcHrtHEr18OXAzozBkqgpr4D+MFBmHETwj34sDF5B71zfL5mmuuCfBHmJQnnygbVaSTIXWTTTYJTOBTPnBC9CsKETfnx48fH2/DSn42+VlXL3hjNEZ29KE5jnFv70QTb9bln5+Y9sztMso6+gnxzlZ23lZswdhleghHAHvbHYdUc4i1a5joNSOYNzj6tNQpH03vZ+0c3HgO7zxMWldZZZXw0Y9+NCbbs2mj3aLsYYjy7YDxafKJwyllzFbLysWBPiMvMAbYJDbtFXWO+kFa0FGpoPfQfwjX2cosXrddd9110QnDX2tlm2Pm5GTnvXPtlVdeGX7729/GU031rMWb+/QOXEzm47xlBkKcm2gjKAMmTPZTh5FcOeS4NwTxZjttGWIOLcRn/ZV4YuI/a4swnFD+/T0tTPpZ6jv5uuLbG7seAzsOC4gvu/46a88J4/MS508Mo158XTBHLN/XoVzQt0rLM/0a25bariNeKxtt9odwGIKp76f59suvAuWfzafR9yd9GP+9iQ723HNlqtT2mY7i/ubI5NPCd9/XMmcc6/9xnnKIQwOrveIMgMCJOo/jIXUTwVEMpwOE8o/TZTehjGFMv+WWW4bqgL/GyjyOvBjxEW/8L70Y4bdUs1VQm+alz68pzYGrqb4sOQZ0GwN4XWL53KSPYNeWPnF8mbhLRiyn6Ep0JtK0HnFtP+Ik3l7E9GDqGFRnvONfdMr10djCipWwGfdWWUXa6xzSx4sPXvyKOH5c1bRfhK4xA3tOxxEvjmj0mWmzEBtHeucFSze6rU5/zrdjXMuYAB3txetYXp5C/JbluTaTMN4ZiD4CfQWvx7wzJOERxpLcz8YsvTpw+fu11QZ6HU7Zpd2xPuV/n2L43IofX3sHLhzlbGV1rsNBijkZxDuexwOv/UMvUJcRriUOHDVsC2scsWFGXppYO+T7uD7fzYHLs0fn8OKN6R6Ly7dN3oGrqS62eHOfJT3dZvn3OrVOX5L0lvqnbfU/mtaxXN6S3qY6oh+MSE9J/LwkTrqUrVR8u0v5Zvzq50ZzY8ymusCvmJtbyXHcuHFD/W70NO3MoNoDr4ty85G8BMwLdggvDJpjbspTv0VABERABNonYP1pxvvMF9C3pc/GH9/td9U7awvFqqQUTgREQAREQAREYKogYJMcPGzVycPShAlxlCZ57D5MuOGQlBPexGf7C8TetvSTFN7gljM++HR5JwO71zrrrBPYBgbBeYr4vPg3y2zi1p9Pv/v72aQgYfwEMBOdOJKw1LytPpXG0+Q3k9wY6ZiwwEmFTjGTmN6g2SneHD8/cWUGX//2X/rWscXPdm5s64Z4hyubeCyly2/J4+OuY9CwNPBpbyXz3QyZtpoIxksMBzgomCETZkz+82n5xyCDY0inrYVsGXf/bN4wbg5co4mfn/Dz+RFhdPhn+ewNLL4sXXzxxYGtvVIxBwhvGOrFgYv411tvvbDBBhuktxq29cAkJzMHOjlwZYLHVQrYSgrjGROI9ta4d+Cqqxe8MZoVV2xlJO5v+jGXltIxnyfekcbC77TTTmGBBRaIP8kbtlvwgtGElYgwDmLEKQl1iK1/cLIgL+xFo5IDV53ykbtnlftRpwnny1oalxl+vOG6jXbLO7Ok9+zlN6sM4ITB1kkwRo+lwmQKk/8YprwjVac02eolJd2WrgLJPa0ucx9zHLG0sN2NlZfbbrst/OxnP4t50VTPWry5T3Pgop+BIwtGOoQ8hQPP5MU72JrziD/Pd+8gxTaDxIN4457vm/gtvKxdiRd0+VfqO/l6e9VVVwVWMUvF6qYv3/46a8+5zrdTZgzy8aFnzCnZVmnzZccbMv116D9zCvTlq67e83Hmvls778sn4ShjlDWOY4xNHcwIM36iYy3Oj0jOETGecP+a6GDPvY5uMwcPr39cUuJXVs3CiRWxNsD3l1iRBz1rjuvUA1blY7tPXzfJX/LZl5cYac1/tJP0Q+E6//zzx6tLDly5PjcX+LbW6kvTvPT5NSU5cPXSLy05BnQbA/SzjxALymv//KqafmvTpvWIaNuOk5VEKVupsBIhdbyKWP3uxYHLO9dyTwz5OMzgLJTTd93SZWnqpHOsbfe6omm/yDvQopvoD8CPel+S3PiiaX/Oj9lLzhLWvpAevpNOPzbCKYl8T8VvD8nW1r/85S+jcyzjTtok75zor/UrbFedg7E0DqIN9E4TtrKxTz/faV9w4kBMh/PdHLhy5cuvfu3nDbjOBOd7+g+Ij8O3g36FU59W72zv893S58OW+gJej1i/pxddbM+V+yzp6TbLv9epdfqSpLfUP22r/9G0juXylvQ21RH9YER6SuIduM4666xwzjnnTBLUHJI5YSv3+7lR39+2i5v2oew64kmdLi3u5ZZbLs6l4XhJGzSo9sC/YERaePGA/i4vK5njgKVRnyIgAiIgAoMlYHpYDlxduNvEeJdgOi0CIiACIiACIiACwwjYwJuDVScPSxMmxFGa5DHjOGGYdMyJN0Dj8MHErp+k8EbSnPHBp8ve1vT3WWuttQIT8YgZuvz5thy42Kpniy22mMSgzsQ9b40x2cDkWR1hEphVEzCSYVD2rCyedDLXjuc+c/z8xJUZfP2bdeStbanm4/QT134i1iYe/Uou/jrvUOcdhvyEoI/PX5v77o2ztj2mlTuMHGxVwRZQZsj0hnZbFc5vBcQ9qpRV3kDlGb1h3By4RhM/P4Hq8yPH2h+zfC45cLFNH2+fp2JvMXvDUK8OXNzDnPbsfmz1xvZ1dcQblXOTozgysdrBMsssE1f4YVI/J96Bq65e8MboNO6ScToN53/7+u11qYVhiyMcNRAcYdKyb04iOQcuVilie0f0lDmvWbz2WXLgqlM+LK469/OTvmyhZ6sVWVz2aU6Z3lBk+oMwKQ+7zuviXLuVbolh17X5SRrQbTh0saWNbXvEPcxpCidbnG0RS2f8kfzz7aCtkuF1GxP8fmVLLjenL1+XLdqcA1cvetbizX2aA1d6jryjr2Pbmdp5c3Sx393yGKcwnIMRnBRthTJzCua4Z5VuE8X5kpT6Tt3qLfHljOz+OmvPCevTZ47OHDfx9cWMln51hAsvvDCwdUoq3rDpdWZdvZfGm/4uGa+tDPo8Sq/1ZdFv3ZuGs99NdLDnXlW3eUN1upWcpYVPVmC0FQZs60PmASnbiNV18pUt0CiX9H1s+1j0Gc/NJzqjtJpMjCzzj/aBrTLhWFoBsOTAZX2sTLRDq7TZtjhN89LnV6mNNOc1v+oqaSqVq1x6/bE2x1Clfkcv+rLkGNBtDNB2H8EzS79bn807yDapRz7eNuO0vo+Pn++d+hNpWHOW6sWBi74Vq0pSTlKh38LqeDhKVll5xJe13HjZ4rf64tv2pv0i9AbOtYzVvdDuoqtwpEFfocNN0vGFb5868c/15/wcga0eY/exTz9mszkRK0uEKfUROGd9QdPh1qaXxsBcw7a+tJGI3S/+6PCvpKua6s0Otxq2imLJ6YfxEHUE8frfHLhweLPvdi/vhJXrh1g4q3u+3vDygq386+P2cVrflXh8vpsD1/rrrx8YbyI5XRdPTPxn9db6Qr3oYosz91nS022Wf69TS8+c60uSXivLfnzE8bb6H03rWC5ve9ER/WAEp5J4By5ebqEPlwrjOtte28bRfm40N8ZsqgvsOtKAnsGBtZsMsj1gW1NbWd2ni/kn9C7th6167s/ruwiIgAiIQH8JyIGrIl85cFUEpWAiIAIiIAIiIALDCLRpfCDi3CQPk5pMgtURDIQYCv0khZ9wyhkf/EROznjjHbjMwcenyRuue1mBizh5IxXjMEY0m9T19+LtBNgzsdtNMLzjeJbGw0Qyk+pMVjGxz++2V+Dyk2o5pw7S7rc58A5XNvFY2jqnbQcuP2mH086ZZ545tH0bE0xs52bbpPGb8sCbhHCzt6O9M1q3fLHz5tDgDePmwDWa+PXLgYsVY3IOi00duMwQkFtCnzzx+cDv3NYlHO8k3rjlnRG4hhVHWK0Ko0UqVp/NgcY7cBG2jl7wxmiuZSIVHWB6oI5zCNf7ielcnngHrpz+M0OKd+DCiE/5x3ErFQwu1H3bltEmnqukhTC58tHkft7gkmsXLN22UpAZKODcdrtl92rySXlg5UXKQc6R1sfpV/4z46Ff8cBW7/HX2Hdff8wI5o/lyoZN+vuyYfF5pxlzMOlFz1q8uc/UgYtVSegXIN7Jyq71q37asU6fxtLCmJGYNsRWfbJ6koa1a0qfub4TYbvVW8LkjG7+upIDVy4vfTtqRkvv6JbTHaSh5MDFuTp6j/CdpGS8NkOr1d9cHL6P2MSBq4oO9txLrFLdxqpaGJ+R0mognPNx/+lPfwqsWISY7rL6ZwY1jFn0g7gfYsatT3/60/E32/CiE6tIqa5Q9lkJwVaI9AZ8v8rJscceG2hLc2J5Z9fa77p56dvMkq63scNocuDqRV+WHAOMg9dTvnzlym6TPkIuv9Nj3kDNuAFd3S0txJHWIx9vm3GaTvfx872TA1Ea1sq0d0QhTN0XVuzlAYzY1s9M7+VXMkvP2W//AkunlxxsDGMOXL30i7g3dZRxGG1CztmfF2wOOeSQ+LIT4W0caS+INO3PEZfX/yX9kHPgMv1KHFXEnMMsz00v5671bWuvDlx2v7p6M5cuO+Z1uJ+DsfP2affGSR7dgthYLecgyDbMbHvfbd7C8t9evLL7WZ+TNhluCPWUMpU+v893c+DyjjNs682KgzmxttT6Qr3o4lz8dqykp+352yj/VXRqri9JGkv9U85Z/bBybszq9D8sDuKrIlbHcnnbi47oF6PSM/lyaHNJaVhW9mYMhti2hqW5UbvW6mNaF+w8n56d9YfturR/5K/z3wfdHnDv1VZbLa62zgp5ObE6njunYyIgAiIgAv0hIAeuilzlwFURlIKJgAiIgAiIgAgMIzAIBy5uaMuZMynAstzdhO27mJgrTVLkjA9+MiI3OTtIBy57PibbWaUHRyFWL2ClBZMqkwwY7Mkjc9pgyXIMfDfeeOPQW9YYjseOHdt1ItTuy2eOn5+4MoOvn8zmzW8m8VPxy7tPTgcu/1w4jrB1Jaty2cQvxg4mfREmFilbTADZqg8cHzdu3JATHBNaTOx2EowxtjWmd3IwB67RxG80OHB554KcA9eyyy4bPvWpTw3LMgwETEpjbKsqnRy4zGmDuJ555pnAdinUScoLOgtnS3QNkjpwxYMT/1XRC94YTTljYhWj5BprrBGjYUKfCV/uWUV8/W7LOOuN+jjLoJeodxMmTIjOqX6VojYcuJrcz5cZm/TP8bKJaz/h3Xa7lbtv1WOs9gRPxByrOl1rhhjC4PRCmbQthDutwGWr93CdOe163ZZz+jFjmhlwuNYk58DVi561eHOf3oGLLXTQ3xiGzNkyfW5WE6JeILQJOUdTfx/aFVaJMfHGV1ZbYtvdbbfdNp62bZQsbLfPkoGsW70lXstrM7JzzF9n7TnHu+Wlry9mtPQrcJVWlcIgT3uHpE6v8eDEf1X0noUtfZYcuIxBpxW4cL41Y+9NN90Ut3wu3YfjTXSw557Ts8SbOp74VSxs9RbCpcKqj+QFctxxx8V2h+/e+IvzAw7GCAZt+oxmHMUxhnaQvg/tBiyriDda055SLtDzbLFJnUDMaGtOWBzzxv/Sym2+327P3jQvfX7lxgCkyXQV7ZVt+cnxUrniXCdpcwxV6nf0oi992fAG4m5jgFzZ7ZcDF3yt/NCHOProo4fpr1xauCatRxzz0lacvIhjbYiPn/YO58UqYv2LXh24/L1YVZjVm1h1k76BjRMJs9dee3XcVgrnGZxoEL+KUTzg/ln59m1L036RizZ+pc1YYYUV4tbj5gDKCX+v1IHFt091+3Ne15T0gx+zmUOVObGRNpySyMNOghMTz2BteidnCD/usvt1iptzJV3VVG92up/X4aVtpnmxgnsj3qmxkwOX37K90wpcVocZa9nWztyHVc5plxCcg3EgxxkGOfvss+Nf/DHxn893m3dhe/f11lsvBsmtYMQJ75xifaFedHG8WeFfSU+3Wf6r9E2sDPk6SJKtLPvxkT2KT3vT/kfTOpbL2150RL8YGav00ztwUV/Qxank0lSaG7VrLR/r9ofturSdsnhzn4NsD/z9mU/FkRkdRb2kLJjYVpP2W58iIAIiIAL9JSAHrop85cBVEZSCiYAIiIAIiIAIDCNgk7McrDp5mJswsUhLkzzm9IDxh/vknA4YhPOH8KYnhqHSJEXO+ODTlZucHYQDF6s8mYMFKx94Yy/PxWS1TTKmE2ScTwUHpE022SQetgnENIwZpLq9yeqvy/Hzk0Rm8PUruWBUwbiSCkawueeeOx6e3A5cLDOPkwuCMX2OOeYYNqFs5R2nLQwYTNCy5QjlDeHtXSaMEf8mcTzg/rFdHMYTnGhwCkC8YdwcuEYTP29IaHMLxTrGNybUmVhHco4CflWF1IGLSVvy1ybxmGi2laHSyf94gw7/SoZUHCrRZQj1FwNYqsv8Nh7mwNVEL3hjtF9t5YADDggzzjhjTENpS5N4Mvnn63cuT5oYZ73uQaekxky/bV8bDlxN72cTzDnjA5gw0GJEQnyYttuteIOG/7beeuvoCMzlJUOaj9om4q1d8A42bLX0i1/8wgeP39GHtoqBd3Dwuq0NB65e9OwkiXYHzIHLGx/YbtO2kMOZFyMghnfEO6aU6hJMuB4DPjoH9ibe8QbDJXqGrSwRDG84zFSVUt+pW70lfstr36fw11l7TthueemNX9bnWGCBBYacgkrOWTgHsqomYmGa6L0YQYd/JeM1Dgvobco7jiqwSMU77J177rlxdao0jP/dRAd77jk9S/w5xxNz8KDfa44VPi1890ZOdBPOUojfUgqHLbib4zrnrW3nGGUfx3VekqDcVBHfpuX6Br4elBy4zHCe3s+v4GqrSzTNS1Z73HfffeMtcs4d3onf6zcuKJWrGFmHf9anJEivY6hSv6MXfemN6yPZgQvHexzwrdw2rUc+q/oRp4+/zner375t4vo6K3Cx7Rs6iZcFLr744mG3p4zQtlmf94QTTgj0uTqJpclW90nD0pdGR9AG+ralSb8IRwJWAUV44QHnTy/UXdJv25HzYhKOT6kDC9c07c91myMg7pwDl39xoLTNGeNJcwi6+uqr43Ze6CJbgZZno3+QCsfNea2q/ijpqqZ6M02T/+0duEqOfn5ew3Q4cXRy4PIvfv36178OMEvFO0vRrtE3NfG6khW+KP+Ep/3nZQfKjonPd2uH/HOZ47CFt0/frlpfqBddbPHmPkt6us3yX0Wn5vqSpLfUP+Wc59S0/9G0juXyljQ11RH9YkSacuIduEqOhNZ/43rr95XmRu0eTXWBXUc8OZ1F+4Jeo00wfTCo9oA6y/iKuo2DVirW3nOc9uWnP/1pGkS/RUAEREAE+kRADlwVwcqBqyIoBRMBERABERABERhGoE3jAxGXJnm8Aei8884LTFR4YVIMoxaTMThEsKUdn6VJipwDkp/ImVwOXH4FoJKR3ZwQOr0ZZ2w23njjgBMEwhL/6YpQ3ihvhnq7ttNnjp+fuDKDL2+3bbnlljEqv1KVxc3qYrYdD8faduAqOY3Z/dNPP+Fr57yDlp/gsfOwwFnLxPKH35RnjCVe/Coe3ljpDePmwDWa+PlJwpyR1jPw33MTzL4s1TFi+4ngm2++ObDVmwmGHfIKJyokdeDyb3Q/8MADcRsWnJ14Oxy54oorAlunVhFvHDBnBK7DUdFW7Xj88cejw5iPj7QxuWmrNZgDVxO94J0Hzj///HDGGWfEW6V1nolqnrebdMuTJg5c9nY8uid1zGWCl1X7WIEJacOBq+n9fDtneeJ5+Ulr78DVdrvl71n3O9tGbLbZZvEytr2ivUwd5ixOVuSwlaDM8IrzAteQL7StGLlwYvCC8zGrSiHeAcLrtjYcuIi/qZ7l2pLkHLgISznEmRfxesPXMVhgNKYse9lwww3DuuuuGw/ldIiVHa6DK32Zug6jRF7qO3Wrt1ybM7r566w9J2y3vMw5cPm+FQ4WODv6soNRBw6m90xnNtF7pLGTmPGaML4s+rY9l0+0HxiazPkVnYAO7yS+fFTVwZ57nbbP+mSk5/DDDw8Ymr3QJuHYRf31295ZGNON9tuvhMIKsDiAeuEFA5zYqohf/S+3SoR3VPd9Im8kp27QHpvTmd3X6g+/Kcc4lvWSl+aUwkQycXtBJ6IbkZIDF+d8ueJ3J/FtS1UHDF+fzKGAe5T6HZxrqi9LjgFW3nxZ6lZ2m/QRSHsV4YUIHBQRxjn023C4ROrUo3jBa//6EaePv853K5edHLi6jXf222+/6KBFXaKspW2Vd5hPV5vMpdUM75zzYzcL61f8s34E55r0i7gOHYX+4uUadEEqPl5zNsyNL3ydq9Of8/UuN0dAenIOXP5FqtSRyJ6BOQscZxHb3tw7pVibaOH5TMerVfXHINtAr8Np+ykzab9zn332iY6FPBPl/L777uNrRwcu3yblxlJc7/Pi1FNPDazi6IWxFg5ypIv2nbLl+3cW1ue76VtW7+FZuIZ6hO5hTsaLX43WHLg431QX+7jT7yU93Wb576bfSVOuL8nxUv+Uc0iv/Y+mdSyXt6SnqY7oJyPSlYp34OKFT3S81+v0xykDVk4Z4yCluVGLv2kfyqcnt4qw70PZSndeb1ed3yWdddsDxr/oDSTXD6Vvb+1KySkzXqx/IiACIiACrROQA1dFpHLgqghKwURABERABERABIYR8JMcTDD6iYNhAV/7gRMBHTQmcxCbDHvt9NDbssRDGLaXY7KZN0wxptgkBA4i9nYwE3Bf+MIXovGCeG677bbAW65IaZIiZ3zwEzm5ydlBrMCF0YHJPZ6TiXqeg4kEBOMmb+jaW7r29lo8WfjnDaC8dYazBkyZqGCiibe3uRcCcyZ2u+UhYXP8/MSVN/haWK5ja7Tjjz8+PhsrJ8DU3pjmvDcC2MRjaUUJVuuwLb28w5BPB+WLONOJVe5VEt7SNUMyYcgPjJlI6nCWe/P83e9+99Cb4hi3MBCYg8yiiy4attlmmyGnIP+WuzeMmwMX9xwt/LwDF0YKjCydhC2Ubr311qE671n6PKxjfPNvN1OOuZZJxIUXXjjgzGgrvZEuP1G/yiqrhI9+9KMxuegb3hyl3HlHFk6yvQNluJv4iUBWLKEsY8AiTdRBq3NW3kk3ZYsV4MxhjHvggIYjWhO9UHIeIF5f1qo6i3TLkybGWTMokiacJqjHGFNYsefjH/94XOWOcwirLrDtF9ItLYTJrVLT9H5+hQDSx9u7tHdLLLFEYDU9M7xxX+/A1Xa7RfyIfw7b5uy/Z8r/KXPoEhv38xysHEj5wgGV8zwHOtkmuYnNJtn57lfxsnbcVoWg/DLpb2Ub45Y5W/jylnNuMIMWdQXDq5fcFoqcb6pnfdzp95IDF6tjkC57tl/96ldxCziu32677WI54Ds6j/pt7Q1tFA5cXEfdh39qvKT8bLTRRlw+JDkjxtDJwhdrL7mP7ztVqSs5o5u/zrfn3fIy58BFknHUpowg5PMxxxwTnW1wUqZP47eHNmN1E70Xb9Dhn08/qzTiKE9fyxtkufyss84K55xzToyJZ8LIZPWcFTuqbOXdRAd77nXaPt/XpW7Td6T/g+DYg3MhTpgI9R7HdC/egYDj3nGd9gk9Y+Wf8ziD0UZWEa83WFUOdpQB0kW7bOWCuLz+9MZ/znENTtSsAEh+UabYWhzBUd7GFb3kpe9vsRoMrMhH9KJtucX9UgeuUrkibCdpcwxV6nfQp2mqL0uOAcZpEA5cfru+XBthfM0hgzJGve7VgYt4+xGnpbfOZ8mBy+uLbuMd30dnnHzEEUfE8TXpwFmNcbS94MBzpytAp+n1/SLKGA5k9CcQHCqo26YzvANX036Rryunn356YGVZE3QF/Q/Gk4x10XeItYt+fOHTXac/122OgPt5pyHvUGXb0BKGsQ+rRtFWo1tJ91JLLcWpqOOsD0RfjXpmDNHJzKHAev75549tkh+r+vvFyAr/SrqqF71ZuNWwbXAJQ7+R1Vtx0qI92n777eOzcM7nEb87rcDFed8PZp6EFXPIT8oAK4/bauZeR3GdyXvf+97AqnRebMzlj/l893NWvk9D+cahBGcy7k+eUiZNvANXU11sceU+S3q6zfLvdU2pb5LrS5JeS0faP7VnaaP/0aSOlfK2qY7oJyNj5T+9wxTHeXGGvGFMx9iJ8Rb1GvEvcfr+Yq4/2FQX0FfGCcrm9Rg/4uhvfRCc9W08xDwL+mBQ7YF/IZK2jTaV/ibC81KHeMENKa1mFk/qnwiIgAiIQOsEaA8Q5hboG9NW0Jbwx3f7XfXG00w0NA1/pbLqlSM8nE3kjvBkKnkiIAIiIAIiIAIjjICfUK2SNCbY7r777iFDi58M43o/schvP6Hn3+blHBNBGN6YgDHBcIqjl63uUJqkyBkf/ETO5HLg4jnSCRkmo3lWjJl0XhF+M1FmjkXxYOYfHWAmU8xwRxCY+UlfJjftvBn10+010qhz/PzElTf4YmDDMGBpT+PiWeycObQQxib86jpwMRHDRJ6Xk08+OVxyySX+UPG7X4kJHkyKmzCIsLf+OJYrJxz3K07wm3gQ8sMkNQL7su8duEYLP28csmfs9GmTiZbPvq77slSaKM456HA/jB+2alN6f1/WzIGLcRDGKsubdDLTO2iQRgzXlp9p/P63PZcdszc6vaMT54jLBqf85h5jxozha6znOLlhaKurFzo5DzDJSh2xZ+bNdN5Q7yTd8sQ/V85JxxwjvQE2dV4hfxDTB+hxdJX9ZtKVvOqWFuLIlY+m9yM+nmn+iUazbuIdEAjbZrtl9/aGCvSROYja+dInWz2im4ynhfP1wo7xedNNN0UHWDtGO0Gb750MKb/ERxk2SbcT9LotVzaaOHBxryZ61tKY+yw5cBGW7ZswvCGUS/QAbSd1CSa+H5KWW65JDc4cQ7iOttzyhLzIrebw39Dl/54xoUyfVqkrOaObv8635/4+ubyEB3UP8UZLygycrK8RA7h/lCPTR+bAxem6es9Fmf3qn8sC0Gaglz7wgQ9ERx07Tl5geLJ0cZy85fm6rb5F2CY62KevbtuXGkFJO8/g019arcQ7xJN2nhFnfxP0rm3nVeqTWdj0E8dp0mZlnPM+v0mj78vDmPaT6+hXpEL4NC70IKtvmTTNS78amMXlP+3epNFW0+S8zzcLb+XKfuc+2x5Dlfod3LuJvsSoyTgK8c/TbQyQK7tN+gjc129R7B2BOOfFt+8YYHGeRXJp4Xiuj8BxL/2I08df9XvJgavOeAfnLPLN2mrKMuNL9IMfE1pftUracJQaO3bsUFDiRKx+Wn1J861Jv8i/kMQ90CHUQ9oUux/H/ephVh+sPeQ80qQ/122OgHhLDlysDsxYwtIJF9Lu+1IcQ4/xAoyJX0WbY4TxbZLx5VxVB65Ouqqp3uT+OUmdcC2MTzfH+M0Ywa9Y3c2BC2cL+iPGlHgoE348Rbw4y+E0lwrs7YU5znnHPx/W57ufs6KvQ33yeejv75/R94WIu4ku9mlKv5f0dJvl35ebkk7N9SVJq+838jutj230P5rUsVLeksYmOqKfjEhTKmn/2M77sscxDON+RdHS3Khdz2dTXcCKw9b2Eg9pQXw99S8GcW5Q7YHvx5IuxnC0f5QDE9oqWFGXJSIgAiIgAoMhIAeuipzlwFURlIKJgAiIgAiIgAgMI2CT+MMOdviBAxdviDFRh/jJMH4zIccKRWbkSyfUWCkHA4tNQHONCRN/ODr4t4Z5s9+26PNOPPbmpp9EYgLb0nXjjTfGt4ktbj79Clx+BQ4L441nOcOmhbNPfz/PgcmEz33uc0Nvgll4+2TCgQlJjOtVhC37mDwzpxC7hskJ3uZlNQMMxsb0oosuCqxM0Uly/FZdddWYN1znDb78ZqUFlk5nGyqbxOH+OB7wHEwUIX47CxzPWFGgtEKQn/DzK3ARj3e64TfPyRZGVcRvNeZXdbBreWuQNwaR3Nu6Fo7VlFZaaSX7OfTJpBHPfOyxx8aJeDtBnjMBiXgHLn6PBn68cYyBpaqYA5fls6+LvizBCYN+KiXjG5PpGJVYtcELxhLKCVu8MfFuDlzmtEFYv32TXcvEHg4zNklv6bbzpU8mMXm72+oVK3exghe6jZXuWPHAC+WCuogTI6sXYZhDbCurunqBbcmop4jfvisemPhv/MQVEnhLHeHe3ZxGuuVJVeNsOpFcmoBmy1UMOFtssUVYcsklYzrRGRipuqWFwKXy0eR+8eYT//HGPgZdKwtwo9yyIhcOPpSrXBlqq92ydFA+bKWM1NHCwpQ+Z5pppoCxx1auyYWjjWGFy5xTHzoZh9y55pprkkvhQduROgB73ZZrG82BKy0b3MCvwJVui8r5unqWa0piBhucSWjPvdBuoausHfV9BJjQxtoKTf46+jCsqMVfSSjTtNMIK1NgrK8rpb5TlbqSM7r563x73i0vSw5cPA99HtoJVq4zvUiZoc+G3jNnab9dal29140b6aN9sDac8BhrbOKwU12lbpM3OHtVkSY6mJUf6OMidds+rkFf+pWiOGbiy6wds0/vCJIr/z5evxqiXd/tk/LEikiW7xYe/Um/HX1iz805+uLoKtvKlZW3aK/IPy/kBave5RzqmualXzHM7kU5vfzyy6M+oq6lDlzdypXFk362PYYq9TvsvnX1pWfBlmC2umC3MUCu7DbtI/g2IHUEsufiE11BuUFX44REPiG5tHC81EfgnEk/4rS463yaA1c6JiaOOuMdHGrIU++w5dNx++23xxWS0AFVBNaMsf1qQ3Ydq6AxdkHf5/KtU/3MjeeJl1WX6YPZONLuxSeOTbZKlR3PjS/sXN3+nB+zl3RpyYGLe9LGs+IU/YVUGOfiFAP/VNjakjFy+syMDWBserOqA1c3XdUpX+q2gd6BC8d+xog2v2PPSZnm2W+44QY7FD+tzHdyKCzNbxAB8bKd8L333hvjy/3z8za5Ld+4xue7n6vhHP1wxnO+P8Fx2otjjjkmjgtoW1MHLsLU1cVcU5KSnm6z/Ps+YUmn5vqSpLnUP7Xnaav/UbeOdcpb0lZXR/STkbHyn348y6qxpDftY9E3Yu6VMbVJaW7UzttnU13AKs5bbrnlsJcHiJMxPGMhHLhS6XSvttoD2iP6IfTNc4J+Y/VazyoXTsdEQAREQATaJWDzMFqBqwtXOXB1AaTTIiACIiACIiACAyVgBj7eRLUlri0BGM8XW2yx6OyCcwWTE1dffXVcMtzCTAmfOKC8613vioYtJmTYngkeOLMwOVhHuH755ZePW5MxkY4Bzq/eRV9woYUWim+hYSQ3I02de1QNy9vaGN0sXzfbbLOA0xSCUSjdYqpqvGk4JmpYwYEBAUZxJvcHLXDF+QRnCQwibO2CEZCJ5aYyNfFryojrmFiHPaug4KzFBDqTh4MUDCVsm0l+Y0Tw9YoJbfQc+gyHPowUVq/Razjz4SzCioXeON2mXhgki073YoUNDD08GwYSdJCvI7AgP+FQZQvLTvfiXK/3I19xRkUfm17BiQA9W3LAGWntFvqRMojTEdzRybQx8CUPugkMKb/zzTdfzBeuoawOuo6Rzn7o2W7PnzsPE7ZdhSkTUNR52tpu4p1fMR6wvVVT6dR3ahpnP67D4EmdsFWe5p+4up059/ntC+3ebes9DH2sKIXBxveFuB8GZvQ2f+hg6gR5wrY0o0FwfKIc0KfD0YiXJmj/6PtNTsFISz+U+sFqWfRlbQtW0oUTF302HEhZ3dR0q08zegtDH2Ew+HcztDXNS2uDSSv9NvhVcdzrVK78c/Tze6d+B/cdKfqynwymtrjrjHdwSqO/RVtFv4q2Cj1MnaOsNxHuj9Ge/ra9mFNF3zTpF5keYVxF/WbMiD5BH1Spo+nzNenPpXFU/Y0TFluEw542kDaFVXZZkbmT4PSF0wX6hWt4Vj8u6HRt6VwnXdVUb6b38g5cti0ZL6/gZIyOpW3FGS6n69O4Sr/pd9N/YJyAsyf9cvQ1n4OScePGRec08pf2Fke8Kn3hkaCLB1n+B9E/bVrHOpWVkcrIO3Dts88+cV6NcR0rbKF/qQeUx16kF13A+BBdR12gXSAtnZyDB9Ue0HZYe0V6aP8YvzIPIxEBERABERg8ATlwVWROx1EiAiIgAiIgAiIgAiIgAm0SYFKVt/AQVtXg7ehU2IoK4yzOK7zJak4sabip8bf4TY25rmceaQRwsMGxAGcIVvhIDTNMmLPyAnLFFVcEVouRiEAVAhibDjrooLjKAwaX3Xffvcploy4MK/zhOMCb9EcfffQk6d98883DyiuvHI8fdthhMqRMQkgHREAEREAEeiWg/lyvBKtfn3Pgqn61QvaDgMp/d6qjhVHOgav70ymECIiACIiACIwsAnLgqpgfcuCqCErBREAEREAEREAEREAEKhPgrUW2KUF4y40VtqyDzjG/LQ9v5x188MEclrxGQPxUFERg8hPwWxT57d1IGStP4JxiWxIdeOCBjVexmPxPqhQMigC6nRX5Nt5448AWSQjbiuS2FhlUmvp5H7ahZeUUhO1c/FZRvAnPFkBIujVdPKh/IiACIiACItACAfXnWoBYMQo5cFUENcBgKv/dYY8WRnLg6p6XCiECIiACIjDyCZh9SFsodskrOXB1AaTTIiACIiACIiACIiACjQjsuuuucfUaLmZ1LbZ9YOsEtu5gRQ6E7XAOOOCAoW0V40H9iwTETwVBBCYvgYUXXjiusMVqSQgrJT333HNxezW2tjG56KKLwh/+8Af7qU8RKBL4wQ9+EM9ZmcJxabfddutpK6HizUbAiY022iisvfbaQylhC2VWsqMfwLYpCP2DI444Im4pNRRQX0RABERABESgJQLqz7UEskI0cuCqAGnAQVT+uwMfLYzkwNU9LxVCBERABERg5BOQA1fFPJIDV0VQCiYCIiACIiACIiACIlCLAM5aX/nKV8Kcc86ZvQ7nLVateeqpp7Lnp/aD4je1lwA9/0ggsMoqq4TNNttsyOk0TdMll1wSTj755PSwfotAlgAOXOa89eqrr4Zjjjkm3HTTTdmwU8pBtlNeYYUVhp7bPxfOW0cddVS4+eab/WF9FwEREAEREIFWCag/1yrOYmRy4CqimawnVP674x8NjOTA1T0fFUIEREAERGDkE5ADV8U8kgNXRVAKJgIiIAIiIAIiIAIi0IjAvPPOG1ZeeeUw++yzx+vvvPPOcMstt4SHH364UXxT20XiN7XluJ53pBFgxcD3vOc9gbo4yyyzhH/84x/htttuiysGvfjiiyMtuUrPCCawzjrrxDJE+4fT0jPPPDOCU9te0ph3Gj9+fHToHjNmTLjvvvti/bnrrrviClzt3UkxiYAIiIAIiECegPpzeS5tHp1hhhnC8ssvH6O844474grcbcavuJoTUPnvzm6kM2I+jdXCkKuvvjq8/PLL3R9KIURABERABERghBGQA1fFDJEDV0VQCiYCIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIlCZgBy4KqKSA1dFUAomAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiJQmYAcuCqikgNXRVAKJgIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiUJmAHLgqopIDV0VQCiYCIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIlCZgBy4KqKSA1dFUAomAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiJQmYAcuCqikgNXRVAKJgIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiUJmAHLgqopIDV0VQCiYCIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIlCZgBy4KqKSA1dFUAomAiIgAiIgAiIgAjUIvOENbwgf/OAHwzTTTBMuuuii8NBDD9W4WkFFYFICq666aph99tnDk08+GS6++OJJAwzwyEhKS6fHnnvuucOaa64Z3va2t8W6+L3vfa9T8L6eW3jhhcNSSy0V73H55ZeHRx99tNL9RgvrSg+jQKOGwOqrrx7e+ta3NkrvBRdcEMaMGRPWXnvteD2/H3nkkUZxTW0Xrb/++mGGGWYI99xzT7jpppu6Pv7iiy8e+ENOO+208Oqrr3a9ZqQHQF+r7IzcXGqzzNFH3nDDDcN0000X7r777nDzzTePuAdnznSttdaKfYirrrpK/fnJmEOjobxMRjwdbz3LLLOE8ePHh//85z/hjDPOCK+88krH8FVO9iPOKvetGmbcuHFhueWWi8Gvvfba8MADD1S9tHG4FVdcMdDff+KJJ8LZZ58d46EvRZ8KueKKKyZLf6hu3yImdoT+6xfPddddNyy00EJh5plnDldeeWWcvxmhCAaWrH6xHtgDdLjRGmusEeaZZ57w+OOPh3POOadDSJ0SAREQAREQARFom4AcuCoSlQNXRVAKJgIiIAIiIAIiIAI1CCyyyCJhxx13jFeceOKJ4bLLLqtx9cgKijEZRxjkwQcfDC+88MLISuBUkpoDDzwwOkU899xz4etf//rQU88555xhxhlnDP/+978HZhQopWUoUSPgC0bm7bbbLhpdLTk777yzfR3452c+85mw9NJLx/vW0QmjgfXAYfbphtJ1/wP7rW99K7zxjW/834Ea34477rh47aabbhqv+s1vfhNwfJB0J/DDH/4wBsKB69BDD+16wQ477BAWXXTRGG7XXXcNL730UtdrmgQYZDuDoVtlp0kuDeaaNsvc9NNPHw466KCY8DvuuCMcfvjhg3mIGnd5xzveET75yU/GK3CSxCF1tAqOCRjkceKZMGHCqHuM0VBeBgm1Tp9lpplmCvvvv39M3umnnx7+9Kc/9ZzUfsTZc6JcBFtssUXAoQo577zzwh//+Mehs/1q0/bcc88w22yzxTHZ7rvvHu/n27STTjopXHrppUPpGNSXun2LQaWryX36wXOXXXaJzjyWHpyJjzzySPs5xX+W6kM/WI8UmPvuu2/ACfXZZ58Ne+yxx0hJVjYd888/f5h22mmjY+hTTz2VDaOD9QiM9v5QvadVaBEQAREYeQTkwFUxT+TAVRGUgomACIiACIiACIhADQJTkgPX+973vsAfcvTRR4cbb7yxBgkFbYtAyZEH4ydGraeffjrss88+bd2uYzyltHS8aMAnvcPUyy+/HI0pe++994BT8b/b+fTIget/XEbSN+m6/+UGhl6Ms01EDlxNqP33mrpG1jadaTqlepDtjDcYyvmvU65MnnNtlrnR4JAzJTlwffnLXw7zzTdfdOD60pe+NHkKUA93HQ3lpYfHq31p3T4LL3+wwiEOADgvtCH9iLONdBFHJweufrVpcuBqK/fK8fg+QhsOcawA6VdoxqGHlZJZqW5qkVJ9aJv1SOI5Why4cLrea6+9Irrrr78+HHvssSMJ46hNy2jvD41a8Eq4CIiACLxGQA5cFYuCHLgqglIwERABERABERABEahBYEp14DrqqKMqbetUA5WCViTA1g4YXljq/6yzzhq6qjTpOhSgD19KaenDrRpHiTPbW97ylmio/OpXv9q3lWmqJrCpA9doYF2VwUgP542hU7uuYxtgVvZLZaeddorby3Cc1QlYlTGVZ555JqyyyipaRSkFU+F3XQeuZZdddmhlP5yd+rWF4iDbGW8wlANXhUIz4CBtljm2xNt8883D6173unD77beHa665ZsBP0/12cuDqzmhQIUZDeRkUC+5Tt8+y0korhY997GMxid/+9rdbWbW3H3G2xXCkOHAxdqMvj1x00UXh73//e1uPWDmeun2LyhFPhoBt83z7298ett122/gkvKTGy2pTm5T6eG2zHklc5cA1knJj8GmRA9fgmeuOIiACIuAJyIHL0+jwXQ5cHeDolAiIgAiIgAiIgAg0JCAHrobgdFltAqVJ19oRTWEX2ApCbCnG1mKTW5o6cE3udE9N969rDJ2a2Nizss3I7LPPHn8efPDB4YEHHrBTwz7lhDMMR+UfI9XIOsh2RmWncnFRwAEQkAPXACDrFo0INOmz0G7jMHnTTTcFHNXbkH7E2Ua6RooDVxvP0mscI7Vv0etztXH9csstF7beeusY1dlnnx34m9pkkH28kcJWDlwjJScmTzrkwDV5uOuuIiACImAE5MBlJLp8yoGrCyCdFgEREAEREAERGFUE5p133rDhhhuGOeaYI4wZMya8+OKLgY7htddeGy6++OJJVqfYeOONA0uT33vvveGCCy4IGA6XXnrpMOecc4YXXngh3H///fFt2ZKRep555gm8fbzggguGaaedNtxzzz3hqquuCq9//evDjjvuGNnV2S6NC0jDyiuvHGaZZZYYJ1tdsMrJKaecEljWH1liiSXiCid8v/nmm8PVV1/N10mElQ1YReVf//pXOOGEE4bOV7nHm970pvCRj3wkzDXXXEMG+4cffjg89thj4cwzzxz25jbPvs466wQc1zDu4zRz5513xtUUYJKKcZ8wYULki4GMZx47dmx8M/kvf/lLfC6uI87VVlstjBs3LuYfby5fcskl4a677kqjLf7u5/14Xsrc3HPPHbcc49lh9Oc//zmWuzRRlpa6ZY63tymrDz30UHz+8ePHx3JHeWVVAlZeueWWW2K+kD9e1lprrViuqRdsFcEKOZRtyubzzz/vg8ayv8EGG8Rjp512WgxPOeI5r7zyysCxNC0Epu5x/JVXXgm/+93vYvldc801wwILLBDLIExYYePcc88ddj//Y4011ghLLrlkmHXWWQNljfvdeuutgcl1ygZpxuj0/+ydCdgtRXH+x7+JS/QB44qoeBEia1BBFNAoiIIgF1AERHABEcGIe6LgHhU31CCIgODCIotKUFBEFkFAEJSgKIsREVcEBfExmsQk/u+vue+X9/btPmemzzl3rXqe75s5Mz093dXd1TVVb1ePok022ST1GY5st/PnP/855fOHP/whld2fnTZfPO/8vBXAVeI14+3FL35xegXbfNx4443dtttu26299trdAx/4wO6OO+5IMo1IbZIZJEYu7bnnnqm/wMdStJNtttmmQ67deuutafsQzrkG0cdOPvnkJBvThYX/4DX9kK0qaSPGZx/qI4cmlXXwhPHJ2LnnPe+Z5oNbbrmlO++88xKPKOfyKOsYx89+9rNTX+ebnrb59a9/neQi277Q72dBrQAu5jCicq2zzjpJHvzqV79KY5t2qNGqq67abb311t28efOSPEFuIUMuv/zyVNfac6OuE2nhCU94QpIn5Md2JMyfzHP0YebKCxdEyMiJ55BPpEO3QJ7Qj84888xiFDI9v/HGG3dPfepTUzRA5NEf//jHtH3V+eef333/+99XsrnjUCcrshGeoq+gIzhNo4/Map7xcubnyIVddtklXSYC19C+AzgBuUmkKLYBZX6kvZhTvvSlLy2iuwCAYG6BvvCFL6QIl+mH/SMaHUAACJ3la1/7mt3tusc//vHpXegAEPoi8vWqq65qGoeM5/nz56e58z73uU8a28hxdKpcpi9SkMIP9FPmVOrPvMxYgr/oVeiW6KvMs+gtyEj4xpxNH6d/Mz6YY5xKfa51TuI59Ezo6quvTuObc+kTyDG2yKIvU+6NNtoo8YMoKcwzjGHu0Y6PfvSjk+5w++23J17R1qWIdEP6B2VpBXD1HX/oIbQLdM455xTlCX1i1113TWmY7/meEfWZR9ddd91uiy22SDyiP0P0UdqdMYbeJhoid73d0dt/vOBbCl2eMUGZkdcXX3zxXMQhygA/kaP0SdLTp+mLfWhJ9hdkPv0NmY1sYFxoDqOs6OLnnntu+u4rlX1oP/M86M8bbrhh6s/kw7x03XXXdWeccUbSs4boLJ4v53vvvXeSV7T56173uvx21/KNMos8FytYw4USgGvInFZ7Jf2Q71LGFd9G6NzIJGwNr3/965MeTv8++OCDUxbMMegPEOAg7+995QTPSkZzjp7C+H3Sk56U+gpzBXoV+kwus0k/SrcYMufIroBsZSu3kq4pecN7+Rb+wQ9+wGmS1dPQW2v8HNp30cnQNfhu4lsBgofM83wfuZ1h6Lc+8yR/2KD41uac7zR4fcIJJ6RvM+bbpT3HjRsPNV4nZi34Bw933nnnNIfx7Q4RKRx5xRyck3QS5mnkGd94zBnM+cg6bDzwHt1jKA3px+T99gXbyGJrQ79661vfmmQfuhH1oN3QPSkj47tEQ+vOWEVfp81LUd4Y3/RhCDnBHMN4I5o4z0GUFR0AfQkdsw8x5z/nOc9JbcT8juynfuhQl1566VwWLTYCPQwAkvZDBjLvY2tAr+G79CMf+Ujq/6PGA8+IeA5dge2esWEyJrFvoavmNivXQfraQySf0BnH6UMqUxyDA8GB4EBwYPocYC6C+M5hDsFmg1znj3P97vvmuy1QKGZjAe1bghmlQ8EJCg4EB4IDwYHgQHAgOLAicACjB4CRGuHgfNe73pWMFkrznve8JzmqACtgFMBYkBNGyiOOOGIxIz3GWwx/KJY5AVoC1AX1BXChtOIclwEszxNwxLHHHpsMI+T9yle+MiVB4X3zm9+cJ08gDAzJ0J133tmxpdyQd2C0Y+u5EgEQkOMdoybbauFQKBFGOIy8TuI7SjvtgvE0J5yVgHdkzPL7GL+OOeaYZCD067XzWb0PAxNOtVIfoCw333xz97GPfWwRg5PKMrTP6TmcSQcddFD3hje8ITnB8joDWlS7Yfg68MADi+l4DiPeqaeempy4ysedlRgQMTirfrQHfM/LwrPudOc5QD98fOXE2MCY50Q5X/Oa18wBBf0eThDqhJFV/djv5+f0RRxtJXr1q1+dLs+KL6V36lorgKvEa4zGrJaGGF8CxehdOmLoffe7350cpVzDCIwcgACPfPzjH0/n/g8jNjJI/QjnEnkAfoJwxBx55JFzj2AMJk+1NQAvjKyjaIgcapV1vB+DMgbjEiFDcJbTV5c3WQfAGMcr/aBEjJMPf/jDizgJS+larrUAuDDMYzAvEU6eo48+erFbyKEXvOAFac7Kb9J2n/vc5xZxPORpSr9r/YH8ABIgNzhHFjnhiNh000390tw56XEaOEBaN4n8J1CPrvkRR9Whhx7ql0Y6WRdJuPDHy1/+8gTK4Cfvw6ELTauPzGqeSYWs/PO5ZGjfwRlD9EUcUTUCrA8QGdp///3n+iZgJsAsOXmUG0AvilrDO+D/vHnz8kfSbxxXtG/ucComXngR5yF5as7N09LfGC/u7MrT+G/vH9QP8GJOzLHU6YADDiiONwBrAFhEnqf6XOuc5M/53OJ9AIcgQMicJ/Di8MMP716ln1hxAABAAElEQVS8AMwMUC8nnK/wn3Siof2D51wnot/Qf8bRkPGHDoljF8JJiZ6f04477tgB9IKkUw+ZR/35PO+3vOUtc99FQ+Wutx+gBwDfpXkJJzh6Cs76nPgGQMcAjDeO/H2z7i/MP4DTodrYoW8xPiRPVP6WfsazfEcx/+DILxHyne9YwDrS8/N0/n2W3+M3W6LxDQEB6gQQ7SSdc8g3yizy9DK1npcAXH3mtFHvo23hfelbnb5MG9GODuByeQYgFVAjNEROkB6QOd8SEIu3APnlcpF7fGehp2M3ENUAXEPnHNcBTzrppOICMucxQBXAbUPrqnKXjjV+Du27tBPjqUTwl23CoZZvfeYl5CkygoVLfEuL+E5CfxBQfWnOcd5WKh9HfQPWeE0aQFcsVmRMlAj71FFHHTUH4CWN9AfsD4D7WOBQIsBfAGT70tB+TL4CcFFXxm7JP4oNEP0+l5MtdUe2sxgRkj0i/Vj4j/nxZS97WfrFtyl/im7o6Thnvj3ssMPyy4v95hsaPbemE7O4jvkXarER6IWSL4x1vvdkM8AOgcwYNx5kNwDgWesTtBP2I9+C1nWCvvaQvvqQ6hbH4EBwIDgQHJgNBwLA1ZOvJQWl56ORLDgQHAgOBAeCA8GB4MAywwEMFABVMGRiLAM4w4o5HPOAg3B0QLnDVMY+rwhON4wEGKT1nAxZSscq7N13310/56JqYPzIDVl9AVz77rtvMsaSKYYkItlgAMZIJEMxBi8M75Tnve9979y73vnOdy4WOcKdDxjBMIYNeQcOAow+GDg5hwAG4IzEQYBxk+u8W+AN7hHlAh0TB4QMy7mhKec7bYazALBIyXFBfXk3fFCbwCMMj31oFu9jZf4+++wz93r6DaslAbTRd8STcXUngz59TnUQgAvDL6sHiaoFnzEyYoij32MwhWSc5BynApFB4BtONn0HwPv3v//9qd1I585KfotIx4pPwHh5WUjjRl49wztZOcm7APqIcgeTtjvUfRx51Idn1Ie41wfAtcMOOyQnB+1AG6hvkd/73ve+9IpZ8UXlLx1nBeDSu6gnEVMYQ8g88c37X6txFsO4IgryPkB8gPkgN/6zWh2w6zgaIodaZR1ABUXNQW4il5AxjBf6v+ijH/1oGn/Lk6zD4YRchhgrzBXIReZBGc3dKK+6TuPozru+Wyjqvcge+ihldzlP5CiPKEPkJCJ6iIhWwWp9+rUDNeTgULpRRzkPlAY5imOBOUWymnuMIwdw+TzKPWQ80QUAZvGsxhmRuIiqJUIOEZUSov8xNujLPMc41HM49jxiopwgJaCr8vajnGFcE5iG82n1kVnNM5SxRqW5pG/fcdAc7QsfGRPoghozvBf9ib7oAFE5mvJyAXBRf9VzpBHYlXPamL7BOFSkS66jE5GOth9HlBN9So425BUgJPoZzm+NbeZV6tmHvH8oPWOJPKQ76DpH5kn6N3OvysF1FgngjIU8T/U5d6KlRAv+9ZmT/LkaIEf5oQ8jC5y/use7qBc88nYGeOYR04b2D/J3nagvgGvI+IPX73jHO1JV6CclYI7rLOqDQ+ZR9FVF80RHgNDNIMC+9NMWuevtlzJb8A/QCH0X0D1g+Zww1tP/XA721R38fbPuLy77VQf6IHOrj0fuoRfjPBa19DOedV1YegtzFSBR6g4xPtGzxuksKXHln/oTfQB93kn6vV/r840yizy9DC3nJQBXnzlt1LuQ0f79hNxB/vg3L8/3AXANkRPk6QAufkOMN+YexpvrRwApPMpPSbdomXMAVwCygH68APiufNOFBf+QLywwQcehbFpINrSuyq90dB3BAXFD+y72GvQ9Ik6Kd8gn2g7QJjpd67e+65z0D+l8nJ944olJNgrApToujTlu3Hio8Tq3/6AfYGOAp+i50h/Q3bBboVtArj+o3jzL3I6O7za0km1Lz/ixpR/zvGSW8qJtkO+0A1EipXNxnbIw1qHWurcAuFisSd+ENxD8hFfY4LCljCJ0Icqtvod+wfciYxQgma4DngJU2GojoAy5HFC54Bnz2rjxAIjRF8vAc/QI5j9saoxRiOsA99HtIdcJ0oUF/0gzyh5ChNBx+pDyimNwIDgQHAgOzI4DAeDqyVt9ePRMHsmCA8GB4EBwIDgQHAgOLJMcYCUbBm6IVacY9kUYgxSRIXeOuLEPQyPGTm1rhOGDyDIymstpQr6HHHLInGOCbQxYXS5iOxhWa4r6ArgwePIujPZvfOMbk4NDebjhg8gjrFp0wzQh0HmPk+qGIQPjGQ6aoe8gP48+QZQGIlCI3BCXG4txcGC4lbOI8shhpLKRD8YoAEQ4JaDcaeKrYLnvBjfyp93G0Szex4pFAdtyJyGGNngusBmreKkH5GUZ0uf0nABcqrPaNAc3sU3BHnvskZLxcUR56QMi57MMbNxzZyW/AYgA2PEVj6WyuJGX52644YYUSQJHHYSRmDQQTmnKDRF5QlsDUT7aV+UkIh7ATI3BvI4pg8o/OcLciULSWfKlUpR0eZYALvjCqlSNIV/F6w6USYyz3l8YswB5aDs5cpCtOPk5jiP12VnKute+9rVzERUBaRHRT+R93J3Ay4OsQ7Yo4mIO0gKIxJwlsCTgOtpqmtQK4GL1OtE+RM7rHFjt82s+fzJ+2VIEx0M+nyvv/AgABhAOzzAf4mgA5A0hw4lmRhqI+wJwwUfkEc/RV1ntjvNDRFQuHA4QzineAQgIYk7DgcVzAHh0nXvINfonRH4f+tCH0jn/5ASZBMA1iz6iMZvL4FZ5Olfhwkk+l/TtOzi5kPsQgAPmPNpT5KvuHfyn+Yx0OJl5VkT/UHQC5kLaEiIC63Of+9x0zvwJmFFtzHzF/K8+5e9KD1T+4VBiu1corzPXAPYARIOop5yJ6ULln+to8IKoiwLf5mAAwN+AnDX/uswnMhkObcjzrAG4+s5J8Eq6gMvivA+4no1uTv0FvGSMMYbQKyCvl0eabO0fPl/0AXC1jD8HAyJnAF6LsNnihIVwdEv+a0z2nUd5XvOiyzmuQy1y19uPPGgDZJj0PuQnwBKRR3yiXgDXaEevl9KWjv6+WfcX7/+UJY8I45HTXC9v7WeuCzO30Sc0FplLkE0CRShqms+j+fdZiX+6RjQ35CHEe5BtIpeHQ75RZpGnytR69O9kLWRSXho/+Zym+6WjtxHfFsh96d1886I7CPTh3x4uzwQ4apETLtsony/Q4Dfbfu63336cprmPPqPylXSLljnHAVroPehPPs+yVSTbtUGKKNhS15RB5V+JnyRt7btE6iVCK5TrnK3f+m63IV++QQB6IusgrwO/l+Ycx/tr48HLqb5LevoZ/Q0iIqgWjvEb2wffCtKB2NaTaG2Q6w/0G8alL2JAdwKwAxHpfRxIiXQt/ZjnfH5izuI7GhAaxLzEfKkFPw5WbK17C4CLsgAOReZDisCZfoz5B8hX0Y/d7sJjLA7VFrp8J/K9OImNQPKFvNFJaG/KKtkwbjxQR3QbvreYd9CD1Bbk+cIXvjBFYuXcv5tcJ+BeX92TtKP0Ie4HBQeCA8GB4MBsORAArp78DQBXT0ZFsuBAcCA4EBwIDgQHlmkO8BGOY4sPdzk6vMCKEpM7LdzYx4rIb33rW/5YCmWuLT+0DQBbamDYh4jqoqg+/iBOCTnRc2Ogp/NzVsHLKY3hSw4Q0uAAUVh1tm/BGOYGHXcskh5eaBU/q9AAn0FD38EzNQeBRw1A+ZZxiWdEbuwmetOnPvWpdMv5njsd3ECNE4O2kwGIh91pgsGwz1ZC036f94EcfJAquODfgi3YE9CA3+6M97L07XPkoefcUcT1mtGVcSBd38GHPCNywzR9Dge0OysxomFM9b7Is6WyuJG31B9wPAFqgNxhonLSxjhh3WBHWo92589xbxTVAFx6H89Omy+jyjMrABd8wwGnD2CVwdtWWzVMYpxFNuEMEmgRZ8Raa60150TvOxYp31A51CLrXAaX2pmIYgABiPyBUwJaHmSdy0c3YqcKLPjHVmM4MyC24pLzLl2Ywr8WAFc+P1EMnJyaO31ce3SHHLyr4rtTgjmFuWUU+dgrgWkcqOU6gj+XR8rS+5iXpSM4uIO64cTCgal5T89w1LYogIUYVyI5QXzO0L3S0Z1hAtPMoo9Me54p1UXXfC4Z0nfWXnvtFJWGfAC651ve4IhTNJBvfOMbc9teOujer5PP/Pnzu6233prTtA2SHI3IFBYH0F9wAtKHnYg8BCAGKtXB0+p85513TsAw8kR+ydHr97fccsv0kwg8yK5x5P3jqquu6o4//vhFHpEs5iLzukD2/PboZIrQwHXPU33OnWhD5iR/rgbIcfnA+yF3vJbGNLoEDljkn74JWvuH60Q+xu8qyeL/W8afO6B9m05yd+AhkQqJWAip7dCT+3wz8EzNYdkqd739aHccsAIy8j4HQZXmK80nPCvgLM/VyN836/7iZffvKC+b90NFpGztZ8wD6CQQQAJFGNH7fGspLRqp6Sx6pnZkbkKmM0ZysKj0e54d8o0yizxr5e97fdoALv9+YPwJCK7yeBS7cQCuFjnhz/B9xvjJQfreb7/+9a93p59+eipeSbdonXN80RxzCnOLyIHGGhNe7pIcGKq3uo7goKLWvlsDcE3yre+AlZKtyOuwtOc42q6m43k5xWvkMN/zfBdiH0APyAn9CD0JEkCIc9cfSjq+f/P3BSu19mOX3w5Sp5wQdaBPUU/Aisyfk9R9SQK43OYCoOrggw9eTFawWBTgKcTcTd1YBAQ5+D1dWPhPYPN8AY3kC8lK89e48cA3OdG+odK8w3WBzNEXKDv9ynWCIbon+dX0Ie4FBQeCA8GB4MDsOSD7NTYPviOYb/k24Y9z/e5bkrstcMD839K9vk8tB+nk1FkOihpFDA4EB4IDwYHgQHAgODCIAxjCWWGGYZBw2SiAuZNAxj6MG1qJ5i9x47iAWL4KzCMr+XPujNFzfr907oALjBKAyS6++OIUBryUnmsOUuCcCAqQG3AVsYvrLe9wHjjYCp7utddeZJsigvGenNxJT8h3jGWQ+J63B/fQT+VwKxk9iVAhgMKpp57aXXbZZTw2kqb9Pnf4Ok+8EB65w42zKsuQPke+eq4vgEvOPX+3l4/zV7ziFR3OJkh92Z2VNeNpqSxu5HUnY8p84T9FpXHHhhytfs2f4Vxgh1F1yZ+pAbhmyZe8DP7bwSB9ZQLPl3jtBsuaY1GGSfLgHOPzJAAu8kGekhey1ImtD0444QS/NPK8RQ4NlXUOrKHu1113XdqmD6dvjZYHWedOEeqBzAd0ghNWRpBa/aZxXQ538pJzrpSvywMi9+AcyenQQw9NERp97Ps4wSHAqvecfHvMPn3Py4xzEcdDTiWQt55jnuK5HMhKHlsuANTgPIJKAJl0Y+E/gIiMIZ6ZtzBi6CwAXLPoIzXnXqs8db7k5619J8+H3xgA2QqHqGfbb7/9HKjZgVoORs8jAUnu0AfkMCJP5i2ILV4E1EoX7J8AGSU9x5KNPEWHYjsitktG78G4CbUAuPLotOSjObmkj7ANMWMAQn4effTR6dwdsCUA15A5yeeyGiDHwWOpAAv+vepVr+rWXHPN9JM6sIWYk2QLYCLVwe/rvE//cJ2oD4CrZfzhbAX0ydzqzm7K6U5mnKwC9rXMo9IL8j7ZKne9/RwsJ/6yjSzbyUIsPoF/TgAqFeFEQHO/n5/7+2bdX/w76uyzz+7OOeecvDhpkQaLNSD/1soT9ulnkqUCCuR50EcAlEA41wGG1nSW/NnSb43jvL9J5yzJBPLxd+a67DTzZI4EWFgigCQaB6X7ujZtAJfaaNS3iGSq6zU+pwkE0yInHAgFeIzy5ORbnvsYEcBiHDi8z5zjQDXPz6Nzef1b6prXy3+X+Mn91r5bA3BN8q3vgBUtAKzVYWnPcZSrpuOVeO3g7pqdgDyZ0+hPPt9IRnC/9A1B1C5AQlAe2Std7PmvTz/W3FqTdbwK2yB6O8R3AvoY2xpCQ+u+JAFc3kZ5tOZU+AX/iIyHvGDO4TuNxaKTArhqsnHceBA4tjb/UeaXvvSl3QYbbJCKL5uV6wRDdE8yqelD6QXxLzgQHAgOBAdmzgHZLgPANYbVAeAaw6C4HRwIDgQHggPBgeDAcsMBnHBsGYfRF8NNDjKgIm5E4reMfbkBm3uQr0qXodqNOTiGfLX5XU8tuo2BntO92pHtmDA25+VmlS0rVnHQawsb5YFjBAcJ5CtttUoNQwgOEo5QyzvcWO9gJbZIYKsEEbwtkerDViA4QKFRfHfgU8l4NwmAq9TOLe9z4BN10jYnef3lRKQNFfljVN15vtTnuK7n+gC4vE5ESmILuRI5L7VVgTsr8y1jlEepLG7kZUvRCy+8UMnnjopaIsO+G/VLYD09KEd4zTCodH4sAbhmzRd/f37uDtK+MoE8Srx2gyVRWHDm5+SrWZFZGKgnBXDxDt8Kk9/IP5zKtfFPmpxa5NBQWQf4ANANvHKinESZAWyDzPSxu7zIOoCzAGhzQjYw3qlXn+g8+fN9fgvURNqS80V5uDyogR5Kjk7fPmVUn9K8Mkq+qSySO6OcNHIM8E5FgpHzyecu5amjg1zyLUpwoDBeSIPzvUSzAHDxnmn3kZJzbxJ5WuKFrrX2HT1P5E+iZqETCvCkezo6gItrRBAFYAcxd7A9oQO7iAxJf4d8C0x+1/qp+ihpcPyhf4wjHOBE2tloo41StEOAHyVqAXCV9FX1cc3J/i7v20MAXEPmJJ/LHGwwrg/gQGV8QYzXvA2ke5UAXEP7h+tENVnmfOO8Zfw5mAmAIHo/C1EUoS+XFS3zaM1h2Sp3vf0cyCF++LZ6p512WgIb6x5Hr/O0AFylNmrpLw7gArzIGMjJt5DNZcqQftYSsY+y1HSWvJyl33wri+dsrQooDJLOWfpe4n7tG4V708zTdT7ydmJ8+7bufs/Ppwng8kiwpb6u9ypCjctUl2cCcJF+qJxwAFcNGM8cAn8glxk1AFfrnCMZ63qVby2cR3YbWtdUgcq/Gj9b+24NwDXJt74DVugTAC6dvA7TklnKf4js0TMlHY97Xk713a222qrbaaed0qNEhGTRVolcvqOHA7BxAFfJfuJ2gRtuuKH72Mc+Vsp6sWst/VgALhzIpSjyvMQXbRJZ6pGPfGRz3ZckgIuFIlpc8p3vfKcDRDiOJrERSL6w6Ebn/r5x4wH9Wvp6rs8pH9ep1e9cBxmie5JnTR/S++IYHAgOBAeCA7PlQAC4evI3AFw9GRXJggPBgeBAcCA4EBxYpjmAMYltRvzjngJjBMCIikEIwwC/5Zzlvox9edQF7kElQ7UMPtyX8ZtzJ1brKaLXELAGK+EARhHSPK8L+VNOVqnJEYjRn1X4pNU2PUSbwFkIlYzMQ99RcxDI4Z1e1OOfOwNG8d2dwu401CscdDQ0AlepnVveB2CFlYt5f1IZdVQ93cCta6Wy8Fypz3Fdz/UBcBFVC8MzNCoqjBtmzz///O7MM89cZAvFklGZPEtl8bxKWxHwnIAUcmy4I7wWrp/nFGliUgDXrPlCWWs0KwBXjW+tAC4B3/LtEVSvLbbYomNVuqi24lX3a8ehcqhF1mGM3n333dMKYxmGvTyMy8MOOyw5yrm+PMk6HGXbbLNNt8oqq3iV5s4dDDF3cQonLQCumjwoAbg01vsWtWaw9+flGKjJXNIqYpvL9D7RNtx54ACu2vxI/kRNI8oA5A5WfsvxUZq7uZ+TO8MUDUlpptlHSs69SeSpylg69plLSn0HkBz9E+BVToCoaX8iDEA52MKBJpdcckmKpuMgdY+g4RHg8vfUfpfAU3la9EYiS+GIzEk6H4sToBYAV0lfFYCrBHRqBXANmZN8/LjMGtcHHJBTqpfABV6v1v7RAuCijYaOP+9X2nLcQSwlfWzoPFpzWLbKXW+/Urv7uDrllFMW29bUHfyldoSPTv6+WfcXB3DVxi+R8ZDB0KWXXtrxvdfSzzxayk033ZT0Eq937byms9TS59fV7r6tnfT72nxZ+0ZR3tPK0/u+8tZxaQC4iPzCvA5de+213THHHKPiLHIUGFLfOdx0eSYQjB4aIiccwFWSB8pTugtAZHR6qKRbTDLnuI7DNtHILHQQIhRBHhU8XVjwb0hd9UzpWONna9+tAbgm+dZ3wEpJfngdSjrykpzj4HFJx+O6l1N91yOTKRISaXNy/bQE4CrJ/BYAV2s/lj1P9rO8/PxmYShAXQi9i6j+fAdDQ+s+DsDl/dC3hnbw6KioX6lQC/95G5111lkdi/TGUR8AV81GUJIv/r5R4wEbZimaoD+fn2sbYdcJSjoIz5XsIVyv6UPcCwoOBAeCA8GB2XMgAFw9eRwArp6MimTBgeBAcCA4EBwIDiyzHGCFOqAmAZ5YlQtohRVnbOkBEYll9dVXXwxw02Lsc4cDocYBleTk2wsOAXApHxx4RGDAmINhH5CRyJ0WXJOxmHNWec6fPz9F2uK3r6rmt1Pfd9QcBB6Jh4gveXQwfxfnOAPYRhEaxfcWQFXKdMS/ab/Pt+8prSBVUQQccOPgqLLwXM05ouf6ALh8G8pREWqI9MH7oOOPPz6Nmz7OylJZ3MhbMkbzjhzA5YbaUUAMOSMmBXDNmi/UsUbLC4BLDn22i8MZ4wT/MHjnYCgZUj1t3/O+coj8WmUdz+LoZvshHK4Cz3DdnWzLo6xDXjLf4NhjZTjGbNGoLZ2UZugRRxTADgj5RmSiEvWRByUQjrcxDgGAN6MIByVtOIokdxxEnKcvOepUvlERuBys/d3vfrcjSqUDMQBsEUWALZGJKMk8CGlumCWAS3WcRh8pOfcmkacqW+nY2nfcoYz8QgeE7wAiaHsHt+cALuQQdXQwvLZ0yWUh40wLAQDtnXzyyaVqzF2jDwN0HUeKNEk6dAa2B0WX5R1EUWWRAoAYKABc/xeBq+QELgG4WvtHH50oNUrlX9/xR9+j3MyvklWK5oMcqW3jymv7zqM1h2Wr3B3nPF1RAFzMRUQWyakkq1r6mUf7cyBw/r78d01nydPVfkum0L/oA8x10u9bAVzTyhPQvutqXgci9gLAH0fTjMDFoh30BIi+IMBCXgbNG65bej8RCCZ/ro+ccACXgMZ5PgAI0V0gj2ys8jo4fJI5x+dBvt+IlIT8Qo7xrc+3Qo361LX2LNdr/Gztuw6ccZvNJN/6owAreR1K38wtAK4W2SM+l3S8vJzqu741riIhKR8/Yh9hQSKEzoSccVBXae52u0DfCFyt/Zg+CqjfF9l5+Tn38rI4kr4HuBQaWvdxAC6PmjUpgIstt1nwCCnCevox4l8fAFfNRlCSL/6qceOBqKNEnGXxGJHOxhHbZqMXj9NByCcAXOO4GfeDA8GB4MDS4UAAuHryHaNbUHAgOBAcCA4EB4IDwYHlmQNu8KgZe2RwwHgkxxt1bjH2+ars008/PW1XlfPPjcZuDMzT6ffDH/7wue0Iv/zlL88BnXQf4AHvhdwozG83ZH7ta19LKwUxfOdOx9Z31BwE7qSubeWAIXfXXXdNziVWeGNwhkbxHcMuK7ihZTUC1/Oe97xus802S2U86aSTuiuvvDKd+z83bgMqxLANjao796cB4CIfgZ5qThjSuNMOAyhggj7OStXBwWTeD0vGaN4nIIX3YRntauAsjL8YgaFamnQz+6dVov4uksySL1kRFvm5LAC4MFTLqVICzBHdBVkJ5fKDa26Mpy0U6Qa5Cr+JLDSOWuUQ+XofGyXrcGYrShgACIAzTvABx7i2J8MxjtF4eZB1zAXrrLNOKi8ArZz23nvv7jGPeUy6XNqCNk8/9PesAVzugKptW4UjlehjELIXkOooUpnppzhgaOucBPhwHUHb6nGNPJAlOTmQ+atf/WrH/O3b7pTmfwc+zQLANYs+UnPutcrTnI/+28d5bS4RuM7lu+t5JXnk2/7kAC7ejzNx3oJtxSAcSDhQoTziAQAbAHiQRzhJF+wf2zg+5CEP6ZgncfSNIt8qjzoBFNDW13rO+1UAuIYDuFr7Rx+dSG3EcZLx587io446qtt///1T1h4hiQut82gNwNUqd8c5T1cUAFdtO3HX66RDt/YzydIaYJht0fieguTcr+ksKVGPf4Al0Of5Vrvgggs6ojpJv699O9S+UfS6WeSpvIce/Vv83HPP7WhHUW1O0/3SUW3k3z6ejvHA3AQ/fW7yOU0gmBY54QCuGoiMfGUvUFQ4ypgDLKYx56ivAIDBHpL3T/Gmpa56tnQs8ZN0Ks/QvlsDcE3yrT8OsOJ1KOk5LQCuVtkD72rjwcupvsuCGEAx0Pe+973u2GOPTef+jzEgQLJ/T/ocNw0A1yT9WAAuyl3b5loRBdHHmD8nqbsDuA4++ODFvin222+/bv31109slIznR0sELo8OWVvIp8jDvIP5i8UGrTaCXL6Qp9O48SAQHt9b7GCQ67/khfzjD2KuYpyP00FIGwAuuBAUHAgOBAeWPQ4EgKtnmwSAqyejIllwIDgQHAgOBAeCA8ssB3baaacOxxz0zW9+c7FoCG5scecs6VuMfQ5cKgFKcO5hwFWUmpIDl3c74XDH8Q7VVtXKMJcb93mPVr2yal9b7BB9gm1/RK3vcAeBbyPkK/UwzgGCgL9Ovt3h5Zdf3rGFCjSK78sDgMuNvUR5Y8V1Tm4w8ghFo+pOHjXniJ7LHQc1o6sATOR55JFHdkRuc8IgiJMYI6uiPXC/j7OyVBY38paM0eRdAnBptTr3S05pgSi4XxpvXC+R6u9OFNLpOufT5gt51sgdfX1kgvIp8XoSg6UcUHwww1snNx67wZ00vpoXAAzG5wMPPDBFfeJ+bRxwz6lVDpFHX1kHMAuABX371ltv7Q455BAvQjp3MATgHLbaWh5k3Yte9KIUmZFKYDDHiecEeFf1rRntPf3Qc4GheG4WEbgckO3AVy+nR8H86Ec/OhbA5WPPHZrK0510riM4GM7nLz1HP8PpAM8hZBnjAEcQ0Z6gUhvh5MQZD80CwDWLPjLteSZVvvKvz1xSAnApqlnJAYQsQGYpelwJwOXOcfQsgAgQDrzbbrttkdJKH+MivCFqjpNHZsvb2NPpnG2nFPGwJEtxUuJYA5wKleZK5eXHcc5S1cO3GtTzK9oWiq39o49OJJ5xnGT8+feK6/Of/vSnE5BQ72mdRwXgIh93nrfK3XG6yIoC4CKiEFvC+TeOA5V83mjtZ4xvbfFaip4sIAFtp7mmprOQpi+pT0gGSOccCoLx980iT8+/7/m0AVwCGfB+om0SddPJoxr7t4fPaQLBtMgJn6MATVGefOEEkcH5PofQ92+++eZ0ngMspjHnuO4kecVYYLEHv0UtddWzpWOJn6Rr7bv+Te/fZ369NC/zztq3/jjAiteh9M3cAuBqlT3Uo6bjeTnVd7Ez8T2PXgXQBp2Xb0anpzzlKR3bUEO+YGicToJcJW+otigz3Vz4b5J+7DK3tP3euuuuOwei1vfIJHV3eXTCCSekKKuqC/oWOio8hSYFcBGJD/2O/EpjEl8w9hDu802PDQ9qtRHk8iVlZv/GjQf/Js/BtmTD9z/9Ap2DPsd3IMdxOgjP+hgFHKbojZqnSOP6EL+DggPBgeBAcGD2HAgAV08eB4CrJ6MiWXAgOBAcCA4EB4IDyywH3JGBEYKIPmxXg1MVwxMGbhlEMGLwwc4RajX2OQgEBzpbB2CsBHyEUUHGU97hxkB+l8gNLayAI/KIoorgsCPaiCKO8D4ZSpSXh/nXNQwdvm1P6zvceAcICKM1zk3IV+8BlID3ukeIfQBcMh7BMxmaR/F9eQBwUXecOYpARFsRLQGjEE79nXfeeS6iGv0C46ZoVN1JMxTApfzo0xhgaXOMWu50pFz0KYG4AG9hrBPYj5WMrL6H/Dm/nm4u/Kd3OpjM+0nJGM2j9EmMs+7Y8NXilJOoQkQ8WW+99ToimBBpQjQNAJfXb9p8UTlLRweRYAhmvIwitv669tpr52SU83oSg6XLLsAstDHyCkcr4FSRA7gc7Md9ATkZq+QnsGqfbRpa5ZDK1UfWkdaBgWeeeWZ3/vnnK4u0ghdwDmPVDdfeh5dVWefb8+JUxtjOuIBoD4zkgAAgjxrixnNF2kiJBv6bNYCL4rijmnFAlEPkG/2Mdttwww1TqeVwHleFVVZZJclr6QEXX3xxR7QsHCYAqZBBIt6jKJ3wEwepnjv77LO7c845JyVFjuFwkHzybYncWfmrX/0qRXOirIwjAOdsjyzKZZrmds9PaUtHd4YBAGLctvaRUv66Jpk/rXlG+ZaOPg5rc0kJwOVzMoA7dC9k/JprrtntueeeHZHbRETmY9vgnJSvrufto+tbbLHFXJQ/5ngAVdpOlMgH++67b4esg0477bQOwNgooo+hP6mvCRxAn6e/7LHHHnP5kc9xxx3XXXPNNaOyTPe8f5QcVCsTgKu1f7jOUNOJvCEmHX9qE+XJNwHfFU6t86j3hzPOOCMtGCF/qEXujtNFVhQAF/wBhIA8IuoecwfjiTkC8sVDrf2MyCuA6CHkFlFtiEKMTEB+0a8gn/dcVuY6i4NP2G4K2VYiBwUx//Bdd+973ztFN9GWgf5c7RvF08wiT8+/77kDJnJQQG1OG5W3f6/wnYUuLDkMCJL5XTLcv3O8nQSCaZETzlfKieOLfgJIi++5Aw44oJs3b16qgn8zcCHXLaYx57DFJdFsnQR08WstdfXn8/MSP0mjNh0KPvSxkttsfDwP+dZ3nVuLRLweXoeSntMC4PKyDtWBxLtcx/Nyqu9SD9dz6YfYH9j6GUJnQVfXWECPBsgO+RxU0kmGArgm6ccO4KJsfBPzHcu3FXWgjnwnQoDjpOO11t37GXrj4Ycfnmw2XAf8qcUgvM8BXL4wB/lPWyFfkEGjCL3xiU98YkpCGyEDACIyb8F75jHIo9232AjII5cvXHMaNx6QJSwqoz3pg4xD6c0Am1kwxjcU5OUdp4OQvgbg8r6Y60M8FxQcCA4EB4IDs+VAALh68jcAXD0ZFcmCA8GB4EBwIDgQHFhmOYCDi4gjAqNQUBwSilTAbwwluo9hHCfMRRdd1GzsY1UeRm4Zp/J3+vtzYyBpS+RbM3IfYAFGDBw1eg+/McL/8pe/XCSL3KiLES03qvJAyzsw9ODYcdLWkRjaAEpgQBHhQIb3KjPXcwCFDIUlI6u/z400yt+jep166qndZZddplvV4yzeB0gC44/Xk76FsU/XaC/AB4AQRKPKQpqac0TP5UZ5N0DxvN/3SDXcw9hHmQS44Vq+qriPs7JUFjfylozRvKsE4OI6hkQ5Hfhdo5ozvZReRkh3oijdrPii/EtHB3CV7ufX5BAs8XoSg6VH/8nfyW/6B/3XAVzIEoyrUA4scSADz+J8duBoeij71yKHlEVfWefAXp5lbFIn5gGNT66ztRlzAeSyJ11Y8G9ZlHXudIDnzG+MaZfD9HuM4dQb8uhq1Hfclm7pocK/JQHgyudX6kjbCRBDsbiGQwVnYR/abrvtum233baalPzoFxwF4CLxjjvumMCNepD7yFGXoZQN2YYshYgIgIzxfkY76BnyQEdQe/E84xxgwDgniMqho8t/Abi419JHlGfp6O/h/qTzTOkdutZnLhHQyuU7gN/58+crm9SW/FA7wGfXTXDSwScnd8xx3UF7no5zj7TGb401tTPXcnnJtRq5w5Y05Of6BDwHWAHRhwD4EqlnFHm7lZylAgs5KET54dxjvEOui3me6nOtc5I/BwCFqJjQuD7gvCrVS1uier1a+0cfnSgV2v5NMv7y+TGPpqvX5On6fDM4X5WPwAUtctfbrxTBZEUCcIlfjD3JFK7hfPBopq39jLzc0cxv3gXpffxmjlBUpVE6C457HPgQOhlzVI0kBxiDRA+cFMDFe2aRZ638teujAFwux3je57Raflxn8cvqq68+l6TURrSXz00+7hwEM1RO5PqvCpH3SX4jAz0yZEm3cDlKXi1zjteBPPJogVyDPB3l66O33vXk4v9r/NT3Usm2QC6172sH1uQ2m9Zv/XGAFa9D6ZvZ22bWcxy8qY0HL6f3Xb6lsP+4Xk7/oe8L9ES+V1xxRQK+cg75e0r1GgrgIk/nFb/79mPvkzwnyscTIE1A86LWulM37GmuIypPjpRb9xzAxT31bc6hPhGWmZ8B9TkwjO8Xbx/GId/4HKEWGwHPleQL10XjxgPpPIIhv2kH/1biGotEmW/R56FxOghpfF71CFzet0kHSR+661f8Dw4EB4IDwYFZciAAXD25GwCunoyKZMGB4EBwIDgQHAgOLNMcwOCMUUjOLRUWY8hZZ53Vfe973+sOOuigOaPFhRde2LHaCuAXho0a4Klm7CP/Rz3qUWm1K8YDJ4wgRFLgfVBuDPS0fo7RZv/995+LnuL3OCdfwED5lg1Kp/D3/K45HVvf4ZG2yB+eskoRgn/wXpFI0sWF/3AoseqZP6dRfHeHRG40Iw8HcJ144omLgKP8HX4+q/fV+h3vpu44I3+8IGKa06iykK7W5/Rc7mTAuEy0DwEUea9C4ZOfOzD47VRyDLLqdJ999knJBF7xZzgvlYVINhj+oJoBvwbg4hnAFTi+ZAzGcEddici12267pchdfbaiIi9oFICL+7PgC/nWyLdjq6Xx6wJwlXgNCAHnDFRqQ67XDJbcy0EKXIPfgCFx2tOnMI7inCfy3/bbb0+SZFzGcIpjyskBcTkg0NPpvFUO6fk+so60O+ywQ+pTcn7qeY4YsYlEhSxzWh5kHauQcVgoAqCXn3PGCRH3fNs3ZIQiV/WJlJbnqd/Maw95yEPST1+Rrvs69pEHJRCOnke2Ek3CHQ+6x3yNw+v666/XpV5HIpwQyYQ5RoS8vPLKK7sHP/jBqd8zDhzARbrNN988yTZ3euh5eE0UNIAiTtR/l112mdM5dA+ZBuiGLRYlL7nHeAaMpu1L+jhneA6dAeAFJDAN5y19hOdqNO15pvYerk/Sd3JQi97DWMDBhNzfYIMN0mX0Q5w5Tquttlra/olr9AXmUjmKPJ3OPcKBrnHkWXQ15kJkTR9iDidCrMaXniEvdFiichHJQv0XID8giVHk/aPkLBXIIgehkKcDuFwX8zzV51rnJH/OAVzj+oA7bEv1EoArr1dL/+ijE+VtMMn4o/31/UC+OHxdlutdLfMoDmT6tADZ5MWcLgP6ULnr7VfSRRzAVdLXXXcotaPqqqO/b9b9xfsK31ToqPkcgL5D9N28ffxZlZ1jHznEvIFjOSfmGOY9gHJONZ1l0003TfMdaQHyoDfVSFvhIRP51mz9Lvb8Z5Gn59/nXGUgbf5dPG5Oq+WPPsk3EmCqnC655JI09xJB2AFcLs8cBDNUTjiAC2AMCxX07aeyoNPQT66++mpdSseSbjGNOcfB8fn3pxdgaF392fy8xk99Lw216YwCcPHulm99/85iS0lFJ1ddvA6lb+YlOcdRptp48HJ63+UZ5ASRkbRlONdE6C3Y2bRARtddfyjJ/BYAV2s/1naj6FLo8nwj5d+LtQUvLXWHB/CKbwzZOsQXFqixrSKRfaEcwPWMZzyjY6xpDrrpppu6ww47TI9Xj9goaaM11lhjsTREZkSvpO5O3nd1nfYs2Qh0vyRfdI+j51kaD0o76nuLeYzvJxZfiFwnKOkgpKvZQ8bpQ3pHHIMDwYHgQHBgNhzQ9yfAe75rBQJnruNcv/u+/W6bbbbZXUtf+j6xnKQLANdy0lBRzOBAcCA4EBwIDgQHxnIARW/jjTdO2+VgOGWLHI9Uhd6z1lprpdVaOKNyY9rYFxQSoGjiFCZfDN847LQyupC81yW2+cHwjpGHOrHVGo5dtpXDgFIjAVZIM87p2PIOjK9EFkHRpo65UxLHJ7wAyIUSDnCJNljRiTaat2DLCpzoODxpqxtuuKF3ZJhp8QeHAWXh/RgCndhCjPv0U5zRbAVDGXMgjj+ztM4xqAGmoB7qYwAiqRv9DiPhtGh54su06qx8MOrSZxmvbPNGf8hBKEo7q2OLHKIsQ2QdYAfmBQC3GPnZxhWjNc6tWn2XF1lHnXA8sQUmq5SJsnHjjTfObb87q3ZbUvlitGH7O+YVAAdEqCLqEFGNJiHmbfo9/NKq84MPPjjJnZoDkr7D1nj8ARTHeUJUIspUI/U93kWfYw7HsShijmdOpQw45SXvdH8ax2n3keVhnkEXwdGNfAHkgb5Hu4qQe/e9731TxDTacVJCtwQUBq8Zh8hTHF3+ziHvwJEKn3HwoVMC5pPuJ7lNH2SsK+rbkPxX9rRLsn+0jD/aHWAd8g/5wnw3ilrmUQAJbEuE49a/k3jPrOTuqDosi/cchIWjH72asUlENnRndCZ06RpN0s+Yo9ZZZ52kszNnIKd+8pOf1F6VQEOjvs+qD8aNJGtr306j2IOeKP2LrdWQ1a3fVH3lhAO4tEU2gE+2P2duoJ8AYhiqS0wy5zj4KY+2VOJf37qWnl2a15aVb/0+PJhE9ozS8Wrv5n08B0gInQS9C/0Eu9iSpEn6MeWkjQFF0kfRu9DxRwH4eaa17oxbvm2Yb+FVH9sh9hG+QdAtsbENsWMCOENP5btD34qj9EfpmkvDRoAOxPyHrk45KCcLbUZ9b9EWrTRKH2rNM54LDgQHggPBgfEcCADXeB6lFAHg6smoSBYcCA4EB4IDwYHgQHBgGeYA2zko6hJRSVgRHhQcWB44QGQqHE8YSYmakRt8MciyehK6/PLLu1NOOWV5qFaUcUYcCFk3I8auwNkS2Y9V3RBRYHB8OOEsYHsSnOYAGohyFBQcCA4EB5YGB4gc+fSnPz29um8E36VRzhX9nSUA14pe56jfss2BEoBraZfYo+h5NL+lXa54f3AgOBAcCA4EB4IDwYHgwLLLgQBw9WybAHD1ZFQkCw4EB4IDwYHgQHAgOLCMcYCw4azcI+ID4C1W1UFsU8fquqDgwPLAAd/+5Rvf+EZ32mmnzRWbFe5sJURfhwBZEN0kaOXiQMi6lau9p13bBdHUO7ZSgohoyRypCBWsuH/Tm96UIphw/5xzzklbLXEeFBwIDgQHlgQHiKyhreuI9gSotBYNcEmUJ97RdQHgil6wrHFgWQFwIa+I/uMLbIgQR7TkoOBAcCA4EBwIDgQHggPBgeDAOA4EgGschxbeDwBXT0ZFsuBAcCA4EBwIDgQHggPLGAdkyGVrHYBcENvOEcUoKDiwvHBg7bXXThG21IfZguSPf/xj2qaMkP+iCy+8sDvjjDP0M44rEQdC1q1EjT2DqrIFxyGHHJK24iB7QM+//e1v05aabKUn2fPzn/88zZ/arm4GRYksgwPBgeDAYhwALMRWv5JFJDjvvPO6s846a7G0cWHJcCAAXEuGz/GW/hyQLswT2kKx/9PTS/n2t7+9W3XVVReRVx/84Ac7tpIMCg4EB4IDwYHgQHAgOBAcCA6M40AAuMZxaOH9AHD1ZFQkCw4EB4IDwYHgQHAgOLCMccANuRQN0MsHPvCB7vbbb1/GShrFCQ6M5gDbmz33uc9NW5iVUn7961/vTj/99NKtuLYScCBk3UrQyDOu4mqrrda96lWvSsDQ0qsCvFXiSlwLDgQHlgQHHCzE+2IxxpLg+uh3eJsQFe3OO+8c/UDcDQ7MmAOuCy9tANf97ne/udrGN9ocK+IkOBAcCA4EB4IDwYHgQHCgBwcCwNWDSSQJAFdPRkWy4EBwIDgQHAgOBAeCA8sYB+5zn/t0W2+9dQK93Hzzzd0111zT/elPf1rGShnFCQ7048Dd73737qlPfWr3iEc8osMxwFZn1113XXfttdemrYT65RKpVkQOhKxbEVt16dTpcY97XLfeeut197///Tui/d1www1p7vzd7363dAoUbw0OBAdWeg6g92y66aZJ1/nhD3/YXX/99Ss9T5Y2Ax70oAd1RIiFrrzyyhS5cWmXKd6/cnPgnve8Z4rUBxd+8IMfdL/5zW+WCkPQoxgbd9xxR/pG+8UvfrFUyhEvDQ4EB4IDwYHgQHAgOBAcWD45EACunu0WAK6ejIpkwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQG8OBICrJ6sCwNWTUZEsOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAc6M2BAHD1ZFUAuHoyKpIFB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDvTkQAK6erAoAV09GRbLgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHCgNwcCwNWTVQHg6smoSBYcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA705kAAuHqyKgBcPRkVyYIDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHeHAgAV09WBYCrJ6MiWXAgOBAcCA4EB4IDKzUH0Jk23XTTbu211+5WX3317q/+6q+6P/3pT90dd9zR/eIXv+i++MUvdr///e+LPNp88827hzzkIcV7XPzf//3f7pZbbuluvPHG7je/+c1i6e52t7t18+fP7/7f//t/3a9//evukksuWSyNX7jf/e7XbbnllunS9ddf3/EXtPJy4AlPeELqt7fffnv3la98JTHiAQ94QPd3f/d36fzyyy9P/a+FQ3/xF3/Rbb/99qlvXnXVVd1PfvKTlmzimRlw4KlPfWr313/916lNaJsVjR75yEd2T3rSk1K1Tj755O7Pf/7zilbFqE9wYBEO7LTTTh36wFD6z//8z+7ss89OesjWW2+dHr/gggt6y/1pzRdDy90nPfx41rOe1TEXoUNdc801fR6LNMsIB6bZfswHD3rQg5JeftFFF6Uarrvuut3GG2+czj/zmc/0rnWtzz/qUY/qNtpoo5TPWWed1f33f/93GpNLuw+W6t67sks5IX1g9913T3rktdde21199dVLuUTx+uDA0uPANGXi0qtF+5sf+9jHdptsskmHDP7lL3/ZnXDCCe2ZLYUnl9f222GHHZIe9c1vfjPxfRqsm0We0yhXngd9bosttkh97r73vW+a13/3u98lHZl5vmSby/OI34ty4GEPe1i31VZbpe8OxsShhx66aIIBv7Chtny7DHhFJA0OBAeCA8GBpcCBAHD1ZHoAuHoyKpIFB4IDwYHgQHAgOLDScuBv//Zvuxe/+MXd3e9+9yoPAA/gED3zzDMXS/Pud7+7u8997rPY9dKF2267rTvuuOMWcaxiTHrXu96VkqPkvuUtbyk9OncNIweALwhnJvkFrbwcePOb39w98IEP7P7whz90Bx98cGIE4K1ddtklnX/+85/vLr744iYGARB629velp4977zzOgydKxKtttpqCawJ7wBZLk/0/ve/v7vHPe7R/fznP+8+8IEPLE9F71XW5z3ved1mm22W0r7uda/r/ud//qfXc5EoOLC8cuCf//mfm4v+6le/OoF2JfcBs1xxxRW98pvWfNHrZQMT/eVf/uWcfPvBD37QHXnkkQNziORLkwPTbL/3vOc93b3vfe/uj3/8Y3fQQQelar3gBS9IYAB+MAb6Uq3P8y2Asxd605ve1P37v/97N6oOS0qHKNW9b11L6ZZUuXm38+/73/9+9/GPf7xUpLgWHOjFgVVXXTUBMfguv+mmm3o9sywl8vGwss1pO+64Y/e0pz1trjn62DzmEi8jJ8tr+2kOYcwcdthhU+HmLPKcSsEWZsKCxwMOOGDkIkuS/uhHP+qOOOKItOBymu9fUfMCOP+yl71skQUnffSvmt7h+tiQb5cVlb9Rr+BAcCA4sKJwIABcPVsyAFw9GRXJggPBgeBAcCA4EBxYKTmw2267pVV5XnmiWfz2t7/t7nWve3XoUkTGEl155ZXdSSedpJ/pOATAxQMAEd761rcmxxC/A8AFF4JaORAArlbOdQkYgDH+zjvvnAOqtee2ZJ8MANeS5Xe8LTgwaw4EgGtxDi+vztLFa7JyXplm+8lZvCwBuABPLwkdolT3SXrUkio3ZfQ+EACuSVotnoUDr33ta7s11lgjRWV9zWtes9wxxcfDygbgeuc735nsKjTaf/zHf3Q//elPu49+9KPLVRsur+33nOc8p3vKU56Sxs0b3vCG7r/+678m5vss8py4UJbB29/+9g4QlwhANgu2AIITad/te9MEtul9K+rxJS95ScfiV4gopfAUu+Y4qukdAeAax7m4HxwIDgQHlk8OBICrZ7sFgKsnoyJZcCA4EBwIDgQHggMrHQfYeu75z3/+XL0J5c8KPAw8Igx1u+66a0da0ac//enuX//1X/WzcwAXRjEntv1hK7B11lknGc4IMw65oSgAXM6xOB/KgRKAi3D0z3jGM1JWF154Yfezn/1saLYp/YoegatmTGxi1hJ+KABcS5jh8brgwIw5gO0GnSMnonKiO7Ads6J15mnYQrfVCTKt+SIv0zR+U2+i8REhle2iv/Wtb00j28hjCXFgmu2HTkNfZbsjtgyFWiNw1fp8KQLXqDosKR2iVPdJmnBJlZsyOuAhAFyTtFo8CweWdwDXKHmyorfwhz/84aTLsH1dH8DHssiP5bX9iBSP/kj5v/KVr6S/Sfk7izwnLZOe32677bptt902/QRgBFCQiNVO2PfYnlh08sknd2wxGTSaA0Rmxz5EFMR//Md/7P70pz+NfmDh3Zre0frt0uulkSg4EBwIDgQHlhoHAsDVk/UB4OrJqEgWHAgOBAeCA8GB4MBKx4H3vve9KcoWFf/2t7/dnXDCCVUeAPQSiAvnEatIRQ7gGhVC/G/+5m+6v//7v0+PsfIRowcUAK7EhvjXyIESgKsxq8UeCwDXYixZZi4EgGuZaYooSHBgphyQ05PonWwnWqNwgtQ4E9dXVA60Arhq/CgBuGppuV5zSI56Zlm4tyTLHQCuZaHFV5wyLO8ArhWnJYbXRFFGf/KTn3Qf+tCHhmcQT0zEgX/4h3/oHvawh3XTBNDNIs+JKrnwYWxsq6++evqFve+WW24pZutz/ne/+93uE5/4RDFdXPw/DvzTP/1Tt8oqqyTgFu3fl2p6R3y79OVgpAsOBAeCA8sXBwLA1bO9AsDVk1GRLDgQHAgOBAeCA8GBlYoD22yzTbf99tunOrNyjMhZRLeoEWHWDz300BRunRVnGCwIGw71BXCR9oMf/GCKJMH561//+pTH0gZwPeIRj0jRmqjX5z//+Y6oYRhTNtpoo8ST73znO93Xv/71ZPDj3jOf+czu0Y9+dHf/+9+/I+rHv/3bv3Vf+tKXqvybN29e2qaSLS8IWf+rX/2q+973vpdWObKFgtOsy/K0pz0thX1/8IMfnOqJERND8mc/+9m0nYOXZbXVVpvrI1/84hdTeiKBPOhBD+p++MMfplWspL/mmms6ttYsEemp8+9///vutNNOKyXpdY3+9+QnP7lbd9110/vvuOOOjnYBeEg/euADH5hC2B988MEpP4yWbJUAsdKWLUGdaN/NNtssbS1A3txnZeq//Mu/LBKBbhyAa9NNN50Lo3/rrbd2Z511lr9m0LnaHpDCKaecksq21VZbdWuuuWbi4W233ZYisHz1q1+t5tu3r2255Zbdox71qFR2ViMz9umTGHjZnol3Qqeeeuoi/CBqx7Oe9ax07xe/+MViK5gf+9jHdhtvvHHK7zOf+cwiW1RMq++xOpj+OArA10QJmwAAQABJREFUhYFeK4/h54knnpi2bk0FH/GPKDdE+njMYx6TjLPwhpXLRCdkjOcGcPoOxm/osssu62688cb03rXXXjv1Sfrpj3/84xQtxSMbpgcW/mMbBvj28Ic/vGP72muvvba74oorOmQ0fRQCtEI9RtEDHvCARaLOIddZWb3hhht2rBJH7jBOKWdOL3rRi5JcZjwR5WfPPffs6Eu//vWvu4985CNzyRn7tD/8lfGafnnJJZeksTiXMDshPfMNR+QBAF6AwF/4whe6H/3oR1nqu36uuuqq3dZbb53KwRYgyCrKdvnll6dylR4iHSvKH/rQh6Y6I18Zl0Qtqq0q7ysLSu/za2ylwvtrcoD55PGPf3ya8xgbmj/Jg3nl2c9+dgfIGfsF4xHeI2eRKcxNOQ3tq/48snGDDTZIcxh9G97Q7x73uMclhw9zAo6cnOgTW2yxRdq+adxclj87ye9WABcyffPNN08RQFVe6nneeectUpxpzReLZDrix/rrr58A8byXfk1EU8Ym/RZ5wHxJ5EgIGcNW19DVV1+dxsALX/jC1GcY4zXgPfJ9ywVyHvrGN76Rnks/Fvwjz6c//empvzGmyQc9hnFSG48tfVTv05E5gD4EnXPOOYtFg+A6/Z8xDCFPL7roonTOP4AwO++8c8oD/QtCjlx33XVJPqcL9g/5R/RXxs8nP/lJu3PXKTJzp512Sj/QE5jTRlEL39kiLG8/fwdzBTKVstzznvfsMPQyz9BHmTucmJtIRznRSaEcwIU8Yw5mrqZdkfsXXHDBIu3Pc7U+787cN73pTWn+L/VB+lZNh0DuTtLOlC+nUt1bdKZR5f7yl7+8yGuH6CyLPJj9yAFc6FXIJca6viNoa2R9/k2grJAZyG1kBNtfoZfQT84888xFxhE6MjIaYl6m/zmN6sPIgvnz56fk9D/mgRKtt956qfzca9X/h/AWfRRdGKqNU8YBfGbeUpTCUd8w0iNTpiP+9ZV7k8i2vu9Q2/L9ea973SuVmvoyztEpXEccoj+5HstcQV9E90RfQR6jd1188cVzkYzpX+is9EX6IemJCJh/Z9XYWpInnnZaOtkuu+yS+gTzww033JDqhI5DP0c35xrjp6RfUZ6h8yQRj+AJegfz2w477JBkMXId/qEfSqdnnDOfM8bguWjoHEd9+GPe4Duec759aDd0A8a/vlGm1ba19nviE5+Y9ErsImeccUaHnKC+yGn0VSJhMzY9grvqzZE+Lb2Ec+Z/ysxzXOcaz+ZRpDyPcefwh+8dCJtWHp27pQ6zyHNcPfrcf9/73pd0Cr4nAH3WiO8ygZD47ihFuR0irzUvLws2tVqd8+t967fJJpskvZkjY5U6MpcgB7FL1Gic3oHMQ15ByPK+3y7+viEy35+L8+BAcCA4EByYHQcCwNWTtyiuQcGB4EBwIDgQHAgOBAeCA4tywFfm9Q0lj1MIpw9GYkAkMhYPAXBhMMNYDQmUsLQBXG44AcBAPQFuOGGkOfzww5MhFOBCThhbqFtuCMYp/9SnPjVPnn4DYgAc4QbEWZUF5/WBBx6YjMulwtCWOJUAjogw0MvwSx/BKCy+YLCXgw5nM1GwcgKQArgKuvPOOztCzrcQRlv6qxy2ngeAF5wX9CEMaAJwOR8B5WE8hzAi45Qs5cV9QBXHHnvsnLNzFICLdqV9Idr9pJNOmnMcpYsD/3mZ4TcAHgzlOeFgd1CN7g/pawA2cTTkRJ/EqYFjEQLQ5g50DIyUE6LP5NFwDjrooOQ4hh/cw3A87b4HAOOYY46pArgwHhMJkLaG6NMYRMcR/YxVtfe4xz2qSXGEu5HWHbM4FgALUN+cAG8hJ+mjTvvss08Civo1zuEfzlm1kWRlns5/02YveclL0iWcqgC3NF49Hf3nyCOPXARApKgAgCJxEuJwgig34wXCaQewopQn92+++ebuYx/72GIOaByvO+64Y/U5AFkAFp2QPThk1YZ+D9587nOf6y699FK/PLZ8OLKOOuqo1Cd5cKgsWORlhR8CFNbkoUexpJ/h5IJwMtO+9KUSITsBMLlztKWvkjd98zWveU1yYObvom8y/nEyluT1EPmS5z3p7xYAF05n+nKJcOgeffTRc7dc9k4yX8xlOOIE5yEOv5zo18yrgBU4p50glzH0YcbuIYccMidn6NPUNSdFaeH68ccf31111VUpCc7rV73qVWnOzJ/hN3KMbbKdWvqoP69z364HwDBzbU7ICpxpkJcFuU4EV/p+iRh38ML1KXjIFt5QKTosDuaXvexl6X4fPbiF78hioi5Aar/0Y8G/Wl/gPn0AEADlEr3nPe9J4B1A1sy1kAO4AOEBAi0RTka2RxLV+jw6H/IXEoCr1AdH6RCAEbUt09B2VvnyY6nuXoe+OtOocisqcIvOkpfXfzv/+FZgPJXmNoz8OM/Ra50AVLNYoET0E0DZWiAB6GavvfZKSekPbJslYixIrnCNBTU//elPdTs5kKXfoWOha5UI4N4rX/nKdKs239X0/xbe8u0BOAZCl/PvlHRxwT/NEQBiFNlo1DeM9Eg9XzoOkXutsm3IO1w25uVlm2E5iYbqT94/b7rpprSYgGs5AcpBx0Vu5kSfRccFkDyO/H0uE6etk6lPMCewaKmk28Ozww47bLFFAS3zpKLyAA6DD5p74Af8kV7t/OH9tB3UMsdJXiMHsMto4Qr5Ie+Z9zX/TKtta+338pe/PC1u49sQ8GjN9sFimHPPPZcizhEL3F7xilcU24hvC3jD97DrBHMPDzyRrp7rgWTTWodZ5DmwWoslV3/kxnHHHZfAtoslWniBBSaMDwDfPie0yGufl5e2Ta1WX10fWj90Zy1wUx46lnRM3RundzjPhny7KP+hMl/PxTE4EBwIDgQHZssB6eZ8L6HnYkdFn+GPc/3uW4q7LUDG/7lv4uUpXQC4lqfWirIGB4IDwYHgQHAgOLCkOOCgK7ZDJIpBK3leowwYy+oWim44EQ9wZuMwV6QqXeeIoRTnO8ZYgEMiosl87Wtf088UxUZOF54hMguONyIjsIIe4jrAMEW9mFVZ3v72tyfHPO8E8ISBDoMyThbpy5QFIyQRWSB3fqQLC/+RjpXDGPHlTC31IXcsYqzFaNtC5O1lhPeUgUgUDibpA+Dad999E7CFclB/nD08h3FYoC6MzzhHATPUAFyAUhS5g7LgIMewPAmV2l4RNKg/qytFRGrwKDLu4KM84/oaQCxW8dO/4SFAKyIp4XgA7PbSl740vcqdK1wQQEvlyB2AOM74IOX9OLuhWfQ9AAYymOMQlYMCJwAGVjlHWb0tx6bKXDtqGwzuA1xiTDLGAa36OPetKNyRoXzhP+ArgKpEhlMfxXmCo0gE2EpAOa7h8OG9tInKr7RDAVx6DkAi0VoA5TjwFKCWR6QRgEvP6chYwwmAcR+wmYhV2uSLkwt5RptDeR19ZTr3MV7gTGNc8Zzo4x//ePf9738//ST62d57761bSQ5TDnjpdXDABeODcsJr+M98hoyj3dZaa6258gH6IkoBNFQWzBWocqL+WHNo1wBcOOzVv+gDyCTaHye5HH6MTeZZUUtf5Vl36PCb9zH2kS3qp1zPAVxD5Qt5TJPkiC2BRv09JRmKnGc8wmPmDJGDU/05B3BNu4/I2aoyoA8w5pl7NIa4Rx8W0MJljOSxgwVKTkj6D4By2hQZICA1kfiYT/UuooAw3zN+fD7Nx3FLH1Ud/Ug/e8c73pEuMb8KMONpfL6QrM3LzRhjfKN/ED1CjnnamjmKPg1NG8DVwvdS+1E2tiRHJkD0a9qBeRP5j14mAoADEAcqgZhcz9IzjF9kYC4zkX0Cvtb6vPfRUQCuUToEwIGWdlb5S8dS3b0OemaczjSq3AAAIe+DffVlvb909D6g+2pzxiogHsnfHHTl7YtcYN6lben3yA09RySh888/P80dGvsO9OO9RIlUZFp+E8Xt9NNP5zQR7c2czvgZFbGFxIzNFv2/hbfTAHDdVcO7/sNHvmFyoKqn4XyI3GuVbUPeAThPkei0EArABcQciTxv0Z9K/ZN5A3mE7gjIISccUowNdDn1QSKGHnHEEXnSxX77+zSnkWja8630BhWAdqdOzH+um6MHvfWtb1WyFL21ZZ7M9StlyHiinq73Mv/xvYXc/9SnPrXYO/vOcS6vqZ/agnMiDxOhTd9HKs+kbVtrP4Gf9B6O1ANbCvyWvOC62wzQzfjtZUfnJz1zv9M0AFySqchgdGnpC7yntQ6zyNPr3XK+//77zy1ioD/QB1mQhc7I7z7UIq9L8/LSsqmNq+PQ+hFVj+925knkCHxEptCHiHhWo3F6R4lnfb5deF+LzK+VM64HB4IDwYHgwHQ5EACunvyUs6dn8kgWHAgOBAeCA8GB4EBwYKXggIAWVHYU6KoPM0YBuHCs4cTnDwCXgAlEJWBFIITxDiM25KtR04XCP7bV0jYfnk8haa9LueGErYtYaQxRNoyycnhi8IN3gEYgj3oDAAEgAoQzlKhUGCQxlgLc8FWNbGNCpC/IV4zPoixsCbDHHnukd8Ff2su3aZHhkQQCbHCeA7gwcrMqXxEuHJDgwAieheRww8CFU9XfeVeK8f+J4oDTFMIACGBIYEOcXjhoBXLoA+DCkI3xmXZ84xvfmBwQKoUbwonwwwriEoDL+x91+8QnPjFyZavyH3fM255IWEQowUkCYQAkDeSgpda+Rj7ihwM26LPwmT7vPCU91zWG+e0OQLaCYgU1BLgMkNms+h7vEGBGvJg3b16KCqGxChANMEYfAhjEOIcAJzFG3MDtkQ8cOOSODJ6Fj0RHUx/1CC8OpMDZh+EYXvMewFTasg6ZCWDLAU4tAK4chMEWTPvttx/FTO9E5qqcDuBibBBNDkeJeOAyPgeq4phhfKtf+Epvj1ijPpEKsOAfhnC2RYHc8efPOOCAdPQntmWFbw4AYVtb/qActOfOIe/P6vt9ZUHKfMQ/9cchAC54p+iFOUiLfkzUQvoKxApuDPqtfdVlKbKY/ieZDPCRCI30Z8jlwSTyJWU2hX9yxNJWjIUa5TI0j+7m/YR5DKAD5M85gGuafQSwBZE2NOYZc0SWgHBSUi+BiBl3owBc3gd8HKTMFvzztnawpjsn/TrPMZ8C9BIwgPkbcEBLH1U5Skcc5aonehFySoTtDkcu5OMIuYX8gohOIKANv5E7AE+UJ9GfkF/QtAFcLXz3OcLBCh4hzUFalNt1L39GOpUDc1x/41kAy8hAkc9dyA/0HvpXrc+7HjQKwKX8NUZcZnCvpZ2VZ+lYqrvXgWf66kykrZW7VWchzxp5HyAN44rvCNoDQv7SV5EN6Hvazsr1BGQf44XvBRELRADXQjiQkS+KmomDGSIqLfMelAPwKQd8FUnOSqfS9dKxRf9v5e20AFz5N0ypXrrWIveG9vmWd1A+yQ6fJ1TuFv0p75+0P/OTvj3QFQByiYiYytwKIbMBa6KvuMxW2tLR3+fyTWNyWjqZ+jNlYKwhz9DDIeZcyq35DiCjtqVtmSfJ0wFctA0LuvheYI4WSddGVhGxVtQ6x7m8Ji8AoAATaQvIec3vabSt5+nt53yj/izc8m1pXf7498ABBxyQIrtRPqKQIud4Hnryk5+cvn2RjdA0AFzoCgLsebuTf2sdZpEn5ZmEkC/+baa84C3gQfRPImSiC2pcKA3HVnmdz8tL06bm9cnPW+tHPhrr/k2Z51/6LRmX60s5z/p+u/COFplfKltcCw4EB4IDwYHpcyAAXD15GgCunoyKZMGB4EBwIDgQHAgOrDQccOObOwtaGeDO/T55YCjCkCrnxbIE4MqNKtTHjdcO3lBdBYYDDCHnI1v9aCsbVsF+61vfUvK5o4wuGNNwqsEPN+JMqyyUSTqxIlrMFWLhibehnHbuRKTNMATKoM9jOPa19UO+gpmoRYquQfQT3ttCXnaM8XJ4Ky9feeiGNOejO+Rl0MegTj29PjgotJ0SW+XhCM4BXLTRs571rPR62g3AXm2bGZWx79HLXAIyAuwBJAJ532jta+RTMyZ6uH/6JkAPopQJPEHdMai7AxCQIAZRCMcIBmJvv2n2Pd4hwAwOCdoY8JjAWzhOABr1JcBnrFaGAO/JOaXnfTsgBwi5LIUnOAX0oa5nfWwJLOvRt9y4rWeIeMBzcloMBXDlMlb5urPfwXdyKpEOAJoiAvJ7k002Sdt0ce6gF36LFkTzTsAqfvMsebhzOQcnkY62ov8BwGBc0ed8W9IaONdXlRO5gEgaDogtyWi2/yPiHM5tRYgYKgso8yhSf6w5Md3hjfEfsKwDgB3Iq/cA8iXiBgSYkzmmta9qLNJPmbMcUEz+m2++ebf77rtzOjX5kjKbwj+11RAAVz4nUQzAvlol7zLUZe8k88WoqvqYL/VRB2vQRqMAXLwH4B8OOog+DQhS5BHaJHfJX1GRSvMLzzrwi3HF+GrpoypH6Uh/1hwKaBUAtMjBRkSKIEoaMpaxlYNb9AxHInVIx5As4fq0AVzkOZTvPke4s5u2EDhTbUT+IuZ1dGPalT4JlUBMLtN9EYHy4ejvQtbjJK/1eQcESBes1YG8azrE0HYmr1FUqrvXodSnazoT76mVW3KSNKV24brP6eIR12vk/GNsozcLYKFn/HtAY99lBtujORhCz6Gzaks7tndGd/Vx5CBoAfApA+PJ5SnzI+AFqLTFmd6nY4v+38rbaQC4St8wqkvp2CL3hvb5lndQ1hqAq1V/yvsnMg5dSeQypqSnMAYADPq8pWdLR3+fy0TN832/z0p5+zXlx7Vcr+Wa81/fMq3zJPkJ1MF5Sa/nunRtB3DBj9Y5zuU10byk3/AuyHlN+0yjbT1Pbz8HP5X0d9cxBcTyvJi7KZ9/l1MHl3F6juuTEDIYGeZ2G/JrqYPKMYs8lXfrke92IttJ1yjlQ7/AjgGYTYt6SNcqr31edj1b716SNjW9s3RsrR95aay73an0jvxaTe9wng35dmmV+Xm54ndwIDgQHAgOzIYDsgvz3YfNk+8vbKD8ca7ffd8eWyj25VSkCw4EB4IDwYHgQHAgOLACcECGzSUN4CK6DVsGYtQRLUsArm9+85sd2784sSXbmmuumS5hZGULEyeieLCKVyvfuSfDEKvia1uRsE3dBhtskLJS1Bo34kyrLGrrkiFN9QD8AjAAUlkcwFUzmrpjkHNAO5Ab/BXNKt0Y+K9P2QWccEOa89Ed8u54w1gMsI5ITYDMSuQALgAXivJBWvGp9FzLNS+znNd5PqW6tvY18q4ZEz1SjSJ6aPsd+vSNN96YAIruAFT0AW+HPu3X2vecF2xJygcwVOur6eaAf3xYP/ShD03RMbbffvs5EGQNwFUDKsrZxqs5h38aNxjOiXoDH3MC7DVv3rx0eSiAC6AjvM/Jt7F1x4ucSiUZsdtuu3VbbLFFygqwhSKFed5EM2BsQcrDnytF6CMt2wQSaYxncCC4w1pAA9I5+dZj3/72t7sTTjhhEfARaQEnAVADhJI7g5TXUFmg52pH9cchAC4HnpAv8pP+BYBQBp/a+/x6n74qoLGPT8+Dczn41YZcm0S+8Pw0SHLE5U0pX5ehV1xxRfeZz3xmsWSar50P/twk88ViL7MLcnJzCXCzR+VQMgFn3RHuDk4fs+64IYoCAD+I9PRF5KE7gth+a6+99kppiC7JvJyTA9zY9ghH27T6qN4FqAYnM+VzsBX3eZ8iveDIZSwhI175ylemx0fJdvKk/M67WQC4hvK91n7ulGZOYFsj5n3auEYlEJPrWvAAJ35O2223XQcIBlL/rvV5BwQInFSrA/nVdIih7Uxeo6hUd6/DEJ2J99TKLVnjMjAvV0lnydP4b+efR7n1NA66lJ4gmUGfRmaU5rItt9yy23nnnVNWV111VdrO28FViljnAHzmF0DXEDKf+dIBxmr3lGDEP+kxJOmj/7fydhoArlGyo1TFFrk3tM+3vIOySqd0Wcf1Vv3J+2cOaCFfIqUSMRUCIAhQ0AkdVtu+apGC38/P/X0+p01bJ1N/Y67VgqK8LEojfaB1niRfgTpoF9qIY07StR3ANckc5/KaSL5EU3JyXk+rbT1Pbz8HP0mueFk8SpXkktfdeeLPsV2sohIOHceej5/73IGOK+BSSx2U77TzBIiLDSInANjMd0OICKsA9AH7kqciJnse6CHojrKHaGwMnQudD9OyY83iO6S1fvBMY11yw/k46rymdzjPhny7tMr8UWWMe8GB4EBwIDgwPQ7InhcArjE8VbSBMcnidnAgOBAcCA4EB4IDwYGVigNyhlDpPgbXUcxxoyvG3ZwwCuFUIrQ/DsWcfJV8aRV9nt6NyXLg52mG/HbDiVaw+/M4DzEyQjgEc6OsHMIO4JIjnGfy9FyDBDjhnEgTGOSmXRYHVsB/tuopERExFOlFWxs4gKu2It+3QfOIPoomQNtj3Oc4lNwJpag+pTxKwCHnoxyWPEtUIBxVznuuszqfleU4tjCeiRzApWs61qJd6P7Qo5eZLTxZwZ0TESFw+rjRsLWvkXfNmMg3FAZTCMfy0UcfnaLEYQRmpTptra0teT9RsDjCV628nnXfE2AmFdL+1QA0lqR6ioGbLTJZrVwycPNgDcDl2wD6CzxCmoBY4nsOYPDn2BqJ/grpOb+fn3s0gZoBGKCpto0DTKuta+VUwjGgc+XvzmpFY9M9P0oOMpYY8/4cW7rgeBlHvsVKTW6Sh8avZBptxVZRyIyccD4AcGFukpOGNENlQZ5v/lv9sdb/3EGO8R9HPgSoBqdhTmyTRv0Yax5dSemG9FV3FJeiNChP+gOAanfYTCJflO+kRzlahgC4SnM55VA7uQx12TvJfDGqnpLdo+ogUDd9X1F4as5S+jxjjrHgDmoHdJxzzjnd2WefnYolAK7KWBtfGltEXWS8Q619VO/Kj+7wB1jI3OtAfpdNW221VbfTTjulLKQn5fnx2/NU5KRZALiG8r3WfkSsAVTAfSfahTkWMA5jX9uckkZ6e20LxZJ+ynM+N2j+qvV5BwQIyFOrA3lrLnOZwXXI22RcO9/1RP1/qe5ehyE6E28plXsSnaVe8kWj4NR0WV8oovleoEQfi/l76Ee0E4QeRr0gPavvEqJQAj5E/hApBh0dkt4u8GSfb7D04IJ/Q/T/SXg7DQBX7RtGdSkdW+Te0D7f8o4agKtVf/LxXeqfT3va01JUN3ikBR3OL69zH3uCv88BQNPWyaQ31ECT1EH2C825k8yTAnX4fOx84lz6tYOVJpnjXF7zLZzbWJzX02pbz9Pbz8FPpW8F10NVfwdE1yIns0CB+QOaFoCLPJGVzOduR2qpQyrYgn/TzlPfVMpfx1E6vNKMOxIlHRDvRhttNAea5xnNF5PIa5+XS3r4krSp1fgwSf3IU2PdvyNq7/LrJb2D++N4RprSt0urzCe/oOBAcCA4EByYPQcCwNWTxwHg6smoSBYcCA4EB4IDwYHgwErFAUV6oNI4RnAYjSMc8muttVZKdvzxx8855GUA5UYf423pPTK0jnJu6jmPOFAz+Cltn6MbTojY4QAenndjU6l+MrLJ8IUDlPoMIbZ7oy7TLgtRtWg3SKvzS+Xy955//vndmWee2TmAq2SEIx/f6k3RPohaRP+CSgbjdKPHP6KT4dCGiM5zzDHHFJ+SAcsNaV4fd8iTAVGIMNIDRpKz2jMGgAF4CXBNDuACiMYzeg5AHCCLaZCXudQPeYdAAKrrJH2N/GrGRO5pXKtfC8iBw4+tfCgLJIDJPvvsk36zrSTgtln3PRkz00sX/EN2YIyHiKpGu/clQKQ4QUvbTABIok8oMowc4OTtjowaoK8E4JK8E29L5XRHkhy6pXS65k762nglrd7tTi05lUrjlUg4bNUmB5felx/lYJcMh584lyFF08mfyX+rz+XXa78dNAc4DSc1YCgi8ZQIuYZ8Ew2RBXqmdlR/rAG49t57744tXyEHcPH7yU9+crfNNtt0q6yyCj8XI3eStfTVNdZYYy4SZK2f8lLxX2CMSeXLYhVpvKA+q75Vy6aPDFU7SYaSlz83yXxRKxfXJT9r/YM0isrkY81ljPcD0rtuwhgGgKn5kDxwojKPQQKHpR89/uXg0iF9dFz2HkVPWzU6GMTll0fyGxX10h2vfQFcj3vc47oXvehFqbilbS1r9RjC91Hth37BtqXIIc1d/k76+2GHHZYAblyXjB0K4PLt8TR/1fq8AwImBXANaWevd+m8VHevQ1+dSXmXdJ9JdBblWzp6H6jJ3xKAS3JP8nhc3g7gkizhGYCCgPvQzbVVnGQ9v5GJAoM6mKH0Pr82RP+fhLd9AFzSYXyLvz7fMF6f0vlQudfS54e+owbgUpuW6lG6Jv1pXP90ANcpp5yy2Bbj0wJwUcZp6mQaPyW9VvzQtneacyeZJwXqyOdOvYuj+qkATFybZI4ryWvyFM2ibT1P10l8Di7ZSkoALv/OqcnwWQC44M9+++3Xrb/++imyoSJ8tdRBvJ52nrItef6c9wFwIZu1oIWIWqXojcqXb0e+32XXYN5nrmi1HY2bl11/KvUT1Vvfx7P4DplkPoJvGuv+HSF+jjqW9A7Sj+MZaUrfLq0yn/yCggPBgeBAcGD2HAgAV08eB4CrJ6MiWXAgOBAcCA4EB4IDKxUH3JmsiDmjGIBDHKeYDDw4BLQFkBsQSsaYUfnqnpyb/K5tA6O0clDyW8An3Ws5jjOcDDU2UQZtVwWPPvKRj4wtFtsyAg6adlk8kpKi1ZQKo9X53AOcB9irr/PD24MVwPPnz5+LHCQwT+md464BGgH4AeGYluE7f06h9d2Q5nzMHfJ6HsAHq09x4hJhjRWZIhmlHcAFkAeHwBOf+MTuKU95SkqKIxVjZ0uEMb1LRy9zzZCdA7h4trWv8WzNmMg9dwzgRMbBCGFc/dnPfjYH9sABiEGZ/gIfcC5Bs+57MmbieGE7MMohgzPX4BXX+pA7bTB0sw0J22vedNNNCQDhoEQ5wMnXHRk1x2wJwCW+j1qp71sjDAVw1bZI82iH7gTQ2Co5utyxXFpVL/5KhgvIyVwwb+EWkAJUKG3t6LKEMjHmRhEgNMZ9TmzLSDQHnDO0neYt0uGwkzFFz/WRBUpbO6o/1gA69E2cBhAyy6OBKU9kEAA0HCqPfOQjF4nOo61oW/qqO87ktNU7/SiHpwMGJpEvnvck5yrX0gBwqdyT9hHJ7lHO3RJY0mWM5iWVif6NExJCZjFv8B76uwMZuL/LLrsk/YJzQLc5UJ3rTvRjtlHMqU8fzZ/Jf1M+5hFAS+KHImkiu9Ev5XD0iKujInAhmwBlQ4pENS4Cl0crGwLgGsL3Ue3nfAG4sMkmm3SArXyrZtdrSiAmX9DAvAKIJycAIs997nPT5dNPPz21v+sbriP5vD8pgGtIO+dlzn+X6u51GKIzkbfmYJd1k+gseXn9t/eBmp7g86zme80poyJw+daIbG/MNseQ6+/wBqAg443IrkQr0zyCTGUbYtod4nsFPaAv+Zw9Sv+fhLfjAFw+v7nccx44KLRv3TxdX7k3SZ/v+44agMvbYoj+NK5/LkkAl3g+6XxLPtIbbr311rmIc8pfR8kV6RaTzJMCdWhO0zv8KF3bAVyTzHGMW/o5JHnt75tF23qerpO0gJ88AlctiiJzI99R0LQicJEX234CPoQUJbilDimDhf+mmSf6DOMgJ4BNbLc+ith+kXELKar6qPT+vXbiiSd29E9F4R5qOxo3Ly9Jm1qtzpPMR+Spse76We1dfr2kd3B/HM9II33A39kq88kvKDgQHAgOBAdmzwHZHLGr8B3GdwLAdP441+++JbnbgvCZi2/Q3ffpZTgdE3NQcCA4EBwIDgQHggPBgeDAohzwyAM4zHAcYhSqkW+xhyKKE1w0DQDX2972thTtiDzHOdJkACHtOLAXacbROMNJi7FJW1HBW5wxJYAPTnr+IJwLKPazKIuM2DVgAe93IxBlZwujvs4PLzNRxAA4sfoTB6xWtfKOFlLZPeKE54MhGaMWHz9u1PIyyTmJYVXAKyJI5c5pnKc4QyHl5QAuRSbjvraI5Ly2XR33hpCXeYgzsrWvUTaNJXdiqszrrrtut//++6efALbgn5wcXBTAiGvIDqIHAUSkPURqv1n0PRkzPeoE5aXckEeYUnlqR203xHjFMJsbx32Lk2kAuLRdEeUpbXui63Lky6FbKz/XkSW0CVQDPHofv/TSS7vPfvazKb2cSiUA1/Oe97y01QYJTzrppO7KK69Mz/g/wEaAJSD6CuCMPfbYI8kCrtWAF+q7cnjJqcwzbNvJ9p05AewkWhVEWXAubLfddmn+oC+wFa0TxhJ4rOhqbAGEk3eoLPA8S+fqj6pLnsbnSQG4aI911lkngaEBaOXkQOvrr7++O+qoo+a2xhraVwXEKo113ouzCBAM5GnURkPnspTRlP5Jjrj8KWXdR4aqnSTjycefm2S+KJVJ13CuEpEOPhKhUgB43eeoyAOk0XiqOUv1nOpDv2OO2n777dMtgBwAOkQeGaY2ZzGPsjUuTkPGCEDQlj6qd446uqOUfq25xgEYPO/OW7ZCPfbYYxfLlnILEOZ6hwO42GKVNndSBA6ujdM7/TnO+/K91H7wl6grEGB5xrYT0R6RWRh3IS2YENjA9SEHcKnvel6cezQmwNgAk0t9nrQlQECpDqSFRukQ3O/bzqQdRaW6ex2G6Ey8p1ZuyZqhOsuosjv/hgC4PDoQ8iPvv7zTASdf/epXU3RUrtN3AFUzNqS/cZ16M0+6LsDvhz3sYYvod6TtQ94G4/T/Vt4y30uulUAefHOgb0AuP/p+w5TqOYnc69vnW99RA3C16k/j+ueSAHC1fJ+V2s2vqb/VFkp4BLnbbrstLUppnSd5r0AdNR2QNNK1HcA1yRxXkte8RzSLtvU8JwVwrbnmmnOLg2rgLI/OWUuj+g49qs307epjt7Qg0cGi3ob+3lnk6fn3OWdxmqJijwIwKi/XlfTtpfEzdC70OaE0Ly9Jm5rqVzq21o+81Mb+HVF6R36tpneM4xn5SOf0d7bK/Lxc8Ts4EBwIDgQHZsOBAHD15GsAuHoyKpIFB4IDwYHgQHAgOLDScUBGYCqOcomDNwcucA+w1wtf+MLkCOA3Tn+c/yJ3TJcMXko36ugRoDC24qwplcUdUaOMpKPeld8bZzhpMTZ5tJVzzz23+9KXvrTIawEVECkDQyjgLlaBcpxFWWRoogBHHnlkh8HViTD7APhw9DhP+zo/qAvOUz2v7cuICPLJT37SXzX4XOABHswd0lzzfuNGLeejnJpsXwYgAqpFKBKQRxEPHMDlq1jd4I6znbHz05/+NOXd+s/LXDJ6kq+iuHhdW/sa+dWMidyDFFXprl+Lbt3gIFDdBxiHE1E0y74nY6YDuDCuI4/ok1Bfp7zqWQKp0K9x/ms7wGkAuNzoWnJIOCCKegwFcAF0YezkMtSBshivb775ZrKfcyqVAFzezkSN0qrs9ODCfx5lTFERfQtUwJKA1pxKTtctt9yy23nnnVMyAcH8Gc59ux5tYfqOd7wjAbSQofCKdnRyAB5gMsozVBZ4fqVzABc1wN0WW2wxB9jgWQG42L4N/kI49gDeOeFgBCwKaRV8a1/lnbKNHHHEER2RuJwEFuCaA7gmkS+e/yTncrQsSQBXy3wxqo4CvJLGwZN6xoGSQwBcDuBh3kIGlhzWPpcBcgIUlI8TB+pffvnlHdtltfRR1WnU0edQ9A7pDYqEoWc9+ivjG/CbonMpDWBMtmKCPMLc85///A6HPESUIbaHEyHPkevId6jvXKHn+/K95Ox2cE3NuerjTpFVSiAmLwd6PDIePonoD8gQ3unt7vqGdCSeeXEhokupDsp/nA7Rt52VX+1YqrvXYYjOxDtq5W7VWWrl5rrzbwiAywG8Go/+HtqUeZ55AtK8ojS+MINrLj8BEcID9X/uow8ga4fQEP2/lbe+mCCPGA0PyPe+971vKva0AFyTyL2+fb71Hf7t7t/crfrTuP65JABc055v6QzSGzjP5xWu7bXXXiniKefadr11niQP9W//jua6UwnANckcV5LX/r5ZtK3nOSmAy/NCPrGIwOd3Fl6gmyoSVel7yes79PyZz3xmxx+6EHYQxiRRMCEfW8q3D4BrFnnq/X2PXk7qRjT0a6+9tvg4spO+iyyFBBhXf+baENvRuHl5SdrUKHuNWutHfnrWbTG19/j1mt4xjmfkIZuHv7NV5nuZ4jw4EBwIDgQHZseBAHD15K2MlD2TR7LgQHAgOBAcCA4EB4IDKw0H2K4BI4TADhjPcBKwqhBHMkZoVvFhWBWVnOrTAHChs+GElwEJBySAGSJIYMwjugxbYrFKXHTBBRekyFX6zdHLAqiI8o6jcYaTFmMTjnyMjjhHMJ4BegP4ARFd4cADD+wATkFEmWHFIzSLsjgQizbmXQJxUQaMdXKe+jYj/pxfTwXN/vn2L7oF2OiWW27Rz6a28SgBOCUBhOHAgTBc7bTTTnMOKDdqOR/lnGT7OABatAlbs8EHQBEQxmFW+SuyD/0fQ7sb8x3AxTO+UlfbxnEdmkU/JN8SgKu1r5GfnKL0UQyLtJc7fx0sQ3ofc+644x6EAZzVuiLvQ9PuezJmOoCL93rbUy/kSh5tTeXTUQAgfiMDGa+Ul9Xhe+65Z0fUJxERU9hmFHLnQ80x6+AmAbGQd8heOU/h61lnnZV4P2/evLQVpBwWvEfPcV4jj8BFGgwGRKvBKcv4PuCAAzryhjyCC7/lVCoBuLjv/GHMEDEH/iCvAVwpmlXuuHJQEM8dd9xxHSCTNdZYI9WRMQmdeuqp3WWXXZbOfeywjSVRv2hH+hsO7Q033DClI+obwAbIwTHIU5wV6sdsp4i8lYMXIBmyYqgsSC8a8c+d7fTJY445JvEIoBoRBNTWZCFHO9sl4kSE6KM4HAFPQczPOOeYhyFAwICBvS2G9FWXpbQdEb9whq233nrd1ltvnSLspRct+OcArknki5e173ysMvhRjljKzViokY/9GqBDcmMW80WtXFwnQiH8UD/AYQzYFSDRk570pBQRR8/T3/tG4OJ5jQM97zJK1zg6AB7gEMBjxiPENk4AuCgf70c+AQBt6aMpwx7/BJhWUuZlbWmkaxwd6IBcY65izoXQURl74iuAFiKIQg4+RTYdfvjhaY7jOuBvAV9IOxTA1ZfvPke4s9sBlWeeeWaKnkY5IGQ5dUK+OhhP87XLbwdw8SwAFsY2R6LqIfuQJZBHEfWxIh2JNMgc5m1IwLFaHUijMtFnSjoEafq2M2lrpPd43b0OtfFe0pl4h/LLy92qs9TKzXXnX01PcB1a8z3tRn9W3z777LO7c845J70KBz0gPyIXQaW52yPXkEZRZjiHXD7zW9trcj6EvOx6Ltf/ud7KW+Z+gMsQ7UVbA8ZkS2K+Afy7cFoArknlXp8+3/oO//YgIhkLUrTddIv+NK5/LgkAV8v3WeoQI/5JbyAJ+iB6OzYF+tD8+fPntpjjNyBXonBBLfMkzwnUkevB3BNJ186jN7XOcSV5rXdxnEXbep4+p3m/HAJ+IlIU8ziEXv+pT30qySrGB9/kmr+4nwO41MY13YFnRpH3O3QyvhcmBXDNIs9Rdajd8+8i0rBwgO8s2cWw/2BXg8eyvTmIt1Vej5uXl6RNrcYbrrfWj2c11v07guvjqKZ3jOMZ+Za+XbjeIvN5Lig4EBwIDgQHZs+BAHD15HEAuHoyKpIFB4IDwYHgQHAgOLBScgDnMpEf5CAYxQSc0kQ8wWDh5MaDktHO044633zzzbvdd999VJK5ewBNcBDk5EZzDP59oiKNM5y0GJsol0eH4jdGYoyMGD9FOE8BOWjF6azKkgNxMGZTHoH3KE8eXceNW+MAXDl4JAc0kX9L2/AcALPVV1+d00SUG1Kf5TfnbkhzPrpzMnd24hzleQyunh+Gsl/+8pcjAVw40Oj74iHbxxB9CGqpq5d5qDOypa9RTje489udo/zGqY8TUJQ75QDDAEiEatsszKrvyZiZA7goC0Ayga5oR9pjFAFgwaEjyvsY4xNAlfqIIkq5I6PmmC0BuHgPjkciQ4l4J+NS/Un9mvty6Cpt6ZiPQaXxfLjGb8A08E0kp1LJCUwaQET0FdWfawJw6Rr5ArYCdCUCMIb8lHOA69TRf7vDgPtE28CB5vnCf8aoiHch3+WIyFeQc5+xDS8dCKcoVuQzVBbo3bXj+uuv37ElW40ok+okABdpfQyRBucf5fZ5AtnGPAHPW/sq72J+pk3GkQO4SNsqX1w36Dsfl8omJ92SBHBRjmn3Ebb63HbbbUtVTNfURzj2BXDxYA7EcBCTv4w5i77nfSuXbaTPAUVD+6i/c9R5zt9a1E4AqJTbZQB9gfHksiTfGjKfo/OykIfk7VAAF3n14bvPEe7s9ogz5EVZaAvqKjnBdd9+Vs4/n6dzHvIMpL50168uAd4oL++BXN9wHakECKjVgXzG6RCkyctYa2fS1qhUd6/DUJ1pVLlbdJZaubnu/KvpCQ6C8vl+xx13nAOakBft6noC1+g36Gbo8E6+AIHrDuDjd94uRLdj/hlKue5R0v+VZytvARMCmiyR9/VpAbh4zyRyL+dtrc+3vMP7vfghsGWL/jSufy4JABf1yHk27vtMda8dpTf4ffoK5DI2/75tnScF6mgBcLXOcSV57fWdRdt6nj6nuUwt2YLgq+w2DmBjXmd+hwcl8nk6B3Dp24U0o8D9pXx1TbKXeRWb0aQALvKdRZ4qb98juhF8dQAcz5bGANeR264jcK1FXrt8Ks3LS9KmRh1GUUv9yE9j3e1Oo96jez5GuCZdbhzPSCubR/7OFplPfkHBgeBAcCA4MHsOBICrJ48DwNWTUZEsOBAcCA4EB4IDwYGVlgMY+Vmpp5XcOSMwjBH9gygxJcJZqOgmJaNd6ZnaNQwRGHFzg5PS47gAIIBRqEQOnOkb8YPoF7vuumvKrrTNQl9jEwo6TnYnQGnk7U5G3QdAQaQYjw40y7L4dkIqg44l5wIrYlkZC/VZma/Q8KT3SAH8hlrahucwtFMOnEQ5sfKcVaREkXGjlvPRnZM4a/fff/+5qDZ5fhjeAaGwShti+wYMmlAegYtrWy6IAqYt3zCKsmUNoLyWunqZS/2Q92H8xgjudeU6NLSv8QzAnH333XfOcO7RPrjPOASEAZVWOHufqkV94VlPx2+n1r4nY2YJwMUqarbikJOmxk8vR+480j1W5eMkoA5sCwjJWQA4CDkDlerB9RqAi3sAuHDQqpxcgwCd0bcll9yhe1eKxf+7ExUgAwCB3CFC+yI7r7766kUykKPLAU6LJFjw4xGPeERy1t/73vfObyWwFFts/HhB5LqcaAtkaC7TGS9E02NrMxzQTryLiGEeIUf3cTJQh+uvv16X0pH6E8XAAVuegPREJFOkiqGywPOqnW+22WYJhJy3J3zBcU+EIwinipztyC/4g6wpEdGEiBao6BCkaemryhsAESAwgWFoB5wIRO3ZbbfdknzhnczrTi3yxXWDHPzpeY87B/xFe2nc1dL3kaGSGy5D/blJ5otaufw6QD+i+vl4YFxeeeWV3YMf/OAkk2kTAbhcxriz1PN0YJjApX7fzxlTOJFK+h7lIMobf04tfdSfr50jG5gzRcw13s91nSPlJprUQx/6UL+czuEXkWguuuiixe6RHl6qvysBIEVkD1GMoBYAVx++j2o/wNGMxVxeUB50XUVm5DdElBj4IKcf17QNGGODKI7kKVAa9yFALYcddtgceItrtT7vkWCYQ9FnRtVhnA7Bu4a0M+lLVKq716E2x9d0pnHlHqqzlMqsa86/mp4ghz/P5PP9KNmLrGb+JnJNicQ37uUymO8t9GEoB+2miwP+jdP/PasW3jJ+WcwhYL7yQ3cgYipbqKIbO4Br6DeM8tRxErnXt8+3vIN6wgtt2Ux5+faUk2io/jSufzqA68QTT1wEpM+7HQTRxwbg7/M5bdo6mfRagP7owjkAkHmDhTcAuHJqmScF6HX5nOcrwJEDmJSmZY4ryWvlx9F5XZI9LW3reXr7IUuQKVCpH9QAXKQnT6JO8i0vewntgz7ziU98Yg6c5VvY85z4Wfo+5X4fArCFTgShq2uRw9A6+Ltmkafn3/ecMcV3JIs1S3oG+aA7sL0iW2aX5pGh8nrcvLwkbWp9+DS0fuTZCuCq6R3jeMY7S98uXIeGyvy7nor/wYHgQHAgODBrDkg3Z6EzczJzMXoOf5zrd99y3G2BsfGupQh9n1hO0gWAazlpqChmcCA4EBwIDgQHggNLnQM4FDEusG0YCiZRrnB6azuaJVlAnItslYWBGsM9RjyAGmybgWNrFOH4x3iGURnD1NImyr/OOuskwyYrV3Hc46y9/fbbl3jR2MYJA+laa62VQBM//OEP05aZOLMnJRm0MLriXMhBGeQ/Sdvg6GDrIwCHrJIFZNVabpxAbB2Ac5cPKLaTwsjP6l7KPw2apK6t72/tazKa/3/2zgRu06n847dSoj6hkkEyNGWZIiMMKkurrEVpUWlDkVFSIU2rFm2UPZElSyUxWcLYksmWLYNMQ4Qo26dC63++h9/zv+bMOfdz3+d5nvd93neu6/N53+d57uXc5/zOOde5znX97nPAgEDeIGSQba9f+WXLKYhAtA8CExCMIDVICE5AVqUPz507V4d7+iRoQ6ARByw6AXKVyD1tErYELm23RxBxvfXWCyuAkF8CON30Z90z6SsEN8CBYBjthUAUn92E/otOZ35O/0X3pIIFSgeHBmMRhBfGAbAhyMAqYTlh3AIH7qGecJYwjkGe+vOf/5y8rd+6gHxPnIcRYxDPZwxtUp+rrLJKR78RjCLfc+bM6WzzGme+17ZKMA3CEHWnNsGWetRxvCqanl2qX3T/WP3sdxsBB9oqdg71rFVv9t1331An6BzG0EEK7Yd+Qh5opwQuIeHWSds2WpcW52hPkJ3pM/RvbIhuQr4Zr9iGlX6FnqafdLP10IWUl2dxPW18GAS7e8qUKRXYQjJg20q2umMcqNOPdXknLVZQpW2h80vTqXtGfK7Ohiip5zj9Qf2uy/cw2Sy0DcYU/iBRM56zXfBozCNSddHE/rf3lWLLuA6RnnkAZC3sj2593z635HuJ3mvb5kuegc3ICrjM0SH9W0HPtbWf7P2j+b1f460IXIxtEH3QtdjDjAXYoE10bMk42St2pWNcr88dlvvl92H8QibOs6dFpkq9HDYs+R7mfDDPZOxAZ9C+0JmQ5fGr0Q80B8iVoVRf59Lr5fgg5iEjXb46u6MUm7Gs80vL7Pc5Ao6AIzDsCDiBq2ENOYGrIVB+mSPgCDgCjoAj4Ag4AuMAAQhSvHFOQJRt1FxGBgG2OFTAGcLC4YcfvsCDF6a6WZjKukBF+4FRQSBF4BqVjPhDhxIBVjdYYYUVArGWVePioDfkbVaLQ2bNmhXexh/KgozRTLHSEivpIKxiQlDZCkEptomD2JVaAc1eO16+s1oU2/QirKJz2WWXjZeieTkMAl7PBoxx+LWJ/T8Oi11bJG/ztfCMyMmYwDUiD/WHtEaAVTixeyATHXPMMQvc//a3v71idVvkkEMOyb7UsMCNfsARcAQcAUfAEXAEHIFRRMAJXA3BdwJXQ6D8MkfAEXAEHAFHwBFwBMYBAmxNhf3HFkCsQuMyOATY8oA3/lgtBvKWtjaKt2lRDhamulmYyqr69c/RRcAJXKOL/7A/fZdddgkrF5HPeBsaVkcjiIZORyAS5VYsCxf4v9YIEIAkEImw4iPjpFY9YNWz/fbbL2wHzPlzzz03bEPM9/EmrPwGwZ4VIaZPnx5W4RqJFcfGG47DXh6v52Gvod7y19b+7+1pY+Nub/PDVU9O4Bqu+sjlhq2TtaU0L3/xEpiEVbfZJhJhVe+9995bp/zTEXAEHAFHwBFwBByBoUbACVwNq8cJXA2B8sscAUfAEXAEHAFHwBEYBwjsvvvuYZutmTNnjoPSDHcRRBhhy0GIXAjbcbG6S0oWprpZmMqaqms/NvIIqD/yZG2hOPK58CcOKwKTJk0KK2xJV7P97KOPPhq244JMI7nooouq008/XT/9s08IaFVGPhGIz2wNzdZobAumemEVCsbQfm3l26fs9y2Zd7/73WHLQJWXhM8///xqxowZfXuGJzT6CHg9j34dDDIHsjea2v+DzMuwpO1tflhq4ol8OIFruOojl5utttqqYoVSycMPPxxWiMUuYmVSBD1z1FFHBf+OrvNPR8ARcAQcAUfAEXAEhhkBJ3A1rB0ncDUEyi9zBBwBR8ARcAQcAUfAEXAEWiCgAI5ugQxw4IEHVg888IAO+acj4AiMEAK2PzqBa4RAH2OPYQu/7bffPmxXk8r6JZdcUp122mmpU36sDwhMmDChmjZtWiDNpZIb7+QtygzJYZ111ukUv4703bnIv4w5BLyex1yVtcqwtTe40e1/122tGtAIXOwErhEAuU+P2HHHHYNdYIndShry1g9+8IPqhhtu0CH/dAQcAUfAEXAEHAFHYOgRcAJXwypyAldDoPwyR8ARcAQcAUfAEXAEHAFHoAUCbHnAW7NPfepTqzvuuCM4V9niwMURcARGHgFW8pkyZUp48K233lr99a9/HflM+BOHHgH09cYbb1ytuOKK1VJLLRW285s9e3ZY2YCt7FwGjwDbAq2++urVc57znIqV0G655ZYwfj7yyCODf/goP4F2t+6661a0tdtuu22+7ZJGOWv++D4i4PXcRzCHMCm3/xesFG/zC2IymkcYZ9nW8t57763mzp07mlnxZzdAgNjdJptsUkF0X3zxxYNf4aabbgp2wnhdkbQBLH6JI+AIOAKOgCPgCIxRBJzA1bDinMDVECi/zBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEGiPgBK6GUDmBqyFQfpkj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDRGwAlcDaFyAldDoPwyR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFojIATuBpC5QSuhkD5ZY6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj0BgBJ3A1hMoJXA2B8sscAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR6AxAk7gagiVE7gaAuWXOQKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao5AYwScwNUQKidwNQTKL3MEHAFHwBFwBBwBR8AR6IrASiutVG200UbhupNOOqn63//+1/WekbxgkUUWqbbYYotq0UUXrebMmVPdcMMN4fGrrLJKteaaa4bvM2bMqP7973/3PVvPfe5zq1e96lUh3VmzZlX33ntv359RlyBl32GHHaqnPOUp1U033VRde+21dZeP23OjXQ8Au/nmm1eLLbZY9Yc//KG6/vrrB471lltuWT372c+ubr755uqaa64Z+PP8AY5AWwSWX375asqUKdXEiROrCRMmVE9/+tOrRx99tHrggQeCrj7rrLOq//73v8lkad/o9Jz861//qu68886Qzt///vcFLltmmWU64xZ9hL86efGLX1xNnjw5XHLppZdWf/3rX+su93OOgCPgCDgCQ4BAbg4wBFlbaLNgx9O2IGA/z507t2huM1Jt4XWve12FjXH//fdX5513XgH4P0YAAEAASURBVNsitr5+vfXWq7CnHnnkkWrmzJmt7x/kDauttlrFH3LGGWdkbbo4D894xjOqN7/5zRV1dtFFF1V33313fMmo/8b3QT0/+OCD1cUXXzxfflZYYYVq0003rZZddtlQhm984xvznfcfjsDCjgB94zWveU2AAb010j6yhR1/L78j4Ag4AqOBgBO4GqLuBK6GQPlljoAj4Ag4Ao6AI+AIOAJdEXj7299eTZ06NVy31157Vf/5z3+63lN3wZJLLllBuIEIhpO+V3na055WHXjggSGZW2+9tTr00EPD95122ql6+ctfHr7vt99+VSrI3+uzIW9tt912IZmf/vSnFYF/BCIPzl3kT3/6U/X444+H7/3+Z8v+u9/9rjrqqKP6/YgxkV6uHkYy89/5znfC4yBwHXzwwQN/9Le//e0QNLjllluqww47bODP8wc4Am0Q2HrrravNNtus9hb04sknn1z99re/XeA69acFTiQOzJ49u/rhD39YPfbYY52zBA222mqr8BtS79FHH905l/oybdq0auWVVw6njj/++Orqq69OXebHHAFHwBEYGAKQXSHkQ3J96KGHBvYcmzDk2iWWWKL6xz/+MdQB1tzcwdrBdg5gy7gwfh+NtiScP/ShD3UI0TrW9BMCF2Ow5nVt5jYj1Ra+8pWvVIsvvnj1t7/9rfrMZz7TtGjF133hC18IL2xgM33qU58qTmcQN37kIx+pXvKSl4Sk99577wpyfROB5LfbbruFS3/84x9Xl112WZPb+n5N3Xxd9cyLB/vss0/n2RDWdtlllzAH08E999xTX/3TEVioEMjZENY386Mf/ai64oorBo5Lzk4Y+IP9AY6AI+AIOAIBASdwNWwITuBqCJRf5gg4Ao6AI+AIOAKOgCPQFYF+E7g+/vGPVy984QsDgetjH/tY1+d3uyDnsB9NAtcb3/jGij/kmGOOqa677rpuxSg6b8veJshR9LAhvsk6CS2RbiSzLMKJE7hGEnV/1rAhwCpbe+yxR/WCF7ygkzXIupADCHY+85nPDH+suiA59thjF1g9UP1J13T7ZMWsL37xi53LnMDVgcK/OAKOwBhAgBcb9t9//5BTSK2QUkdCIMpgSz788MPV9OnTR+KRRc/IzR2sHewEriegHa22pIp1ApeQ6M+nE7j6g2Mqlbr5eo7A9YEPfKB62cteFpJjdW/s289+9rOp5P2YIzDuEcjZENY3M1IErpydMO4rwQvoCDgCjsCQIOAEroYV4QSuhkD5ZY6AI+AIOAKOgCPgCDgCXREYdgIXRADy+NSnPjVsk3XVVVeFMo0EgYvl4dlKA2ELiLvuuit8tw7hH/zgBwPbUs8GrhZmAleuHkJljNA/EU6cwDVCgPtjhhKBOHB75ZVXVjju7da7z3nOcyoCYFqlkFUdCVBCIJCoPxEY+/znP6/D4ZOVLyZNmlSttdZanSAaJ9gq9/zzzw/XOIErwOD/HAFHYIwgMFqkm1zwddhgywVmc3OAYcv/SOZntNqSysj2eKzqFss222wTxm2OsxrLOeecE18SyDDYBCUrcI1UWxCxx1fgquazw7D1cttixxU9LCtw1c3Xmd8zv+QFgbPPPrtTBIiuSy+9dLBrP/nJTzZedayTgH9xBMYRAjkbwglc46iSvSiOgCPgCDREwAlcDYFyAldDoPwyR8ARcAQcAUfAEXAEHIGuCAw7gStXgJEgcOWeXecQzt1TctwJXCWoDeYeEU6cwDUYfD3V4UeAoDHbCWl1LVaQSW2PSEm4hmu5B5k1a1bYTjH8mPdP/Ymtb9kCNyd2fPrjH/9Yfetb3wqXOoErh5gfdwQcgWFEYLRIN7ng67BhlCNwDVs+hyE/o9WWupX9bW97W7XhhhuGyy655JLqtNNOS94y7HMbJ3Alq63VwbFA4MoVSCuisV0k20a6OAILMwI5G8IJXAtzq/CyOwKOwMKKgBO4Gta8E7gaAuWXOQKOgCPgCDgCjsBQILDGGmtUa665ZvXoo49WP//5z6uVV1652mCDDapVV1015O/uu++uzjvvvApihJUJEyZUb3rTm8KhM844o1p00UXDSkzLLLNM9Zvf/KbimIRnvPrVr66WW265itU7WNXj3nvvrc4888zqT3/6ky5b4HPttdeuNtpoo3Af6T/22GNhlaUf//jH1UMPPbTA9RyYOHFicFCzTSBvIP/5z3+ubrzxxpAn7k8Jq4lsscUWIZi92GKLVRi+5I/VRG6//fYFbiEvb37zmyscoNh+vPH6l7/8pbrtttvCKiR2tZMFbs4cYDuAl7/85WHrq8cff7y66aabwhvSr3/966upU6eGu/baa6+KN6OtLLnkkhXBcsq91FJLVY888khYCYuAPHmSrLbaagGXl7zkJRVvZyPXX399eHOVt3ZtulOmTKk23njj8IYrjnzaBnhfcMEFFStNWXnKU55SERRArr322vBsvtcRuPqF3/LLLx/aFc/jTXK2UiAvtDPaIXLPPfdU999/f3XWWWeFOg0Ha/61yVsc5DjllFNC36EuWeXmgQceCO2HlWlyba9p31D9kfVf/epXFVvVWHnPe94T+iAO7eOPP96eClhstdVW4RhtGqIF0qas4YbMv7ge1Dd5259AFn1o5syZFc5EsEF30MbJByun3XnnncmUaadvectbQtumndNG6ZsEni677LL57hHhxBK40B3oMfojW2nGQt7II0L7QddZoW+TBn2G76zwBiHmhhtuCEQVSDC33HJLddhhh9nbwvdXvOIV4c14rXJEGelv11xzzXyrIS1wY3RgxRVXDKvMUfaTTz459PFNN9006Gn0G2375ptvrn75y19Gdz7xk5XxeIud1ZKe/exnB+IO+pd+8Ytf/GKBPqHngRnbYdJGqDfGCPQc25GCP3qGc5AlwUft/fe//31IN7cKAHqKAGIb/ZwsWM1BxjAwQlL1yvF3v/vdYdsq6kSrBnIcaTsecA968LWvfW0YE9A99EOwIO147OT6zTffPOgpxr9zzz232nLLLSv0LuNPHWGKe3PClrgrrbRSOE1b/cY3vpG7NBxfZZVVwnaL/Ii371J/6kbgoi9/+tOfDunZa0eCwNVr/6ZdM7YyblJ/6C3q42c/+1lFWVJC/8BWeP7znx9smX/+859BJ1199dXVxRdfnF39ApywgWib9GVskssvvzyMWbRV+hvjazxOtG1XqTw3OSZdPXfu3KCTsUXABt1OW/r1r38d9B5pYfe88pWvDG2Nfs55dAL2T0ralmGQeaFvUn/oZfQh/RQdyphKHcZS108ZB9DBYABZMmX3dRu34+cx3tEe77vvvmBLxufRw4wt2DrYbXxKSsbztuODnsUn7Xny5MlB9zOeYPdjt2K3024Y39GvsYzEGKBnNtXlEFFZ0UXzHvr/nDlzgk3LmC3BDsCWoh+zHS11z7XoelaIifUGWPCH3cK8he9veMMbgj3BWPqsZz0r2ETYEqSFXmDugb1aOo5h19iVaWnj6MqXvvSlIc/MiViZEf3TRNSGc3MH2n1qDrD++uuH9oG9ge1CvaNz0R/oWlaAos2QP+WZMjNHBANsrbo89svGaoIB1zR9XtO21Ma+bZrHbteVEriazm3Q9am2oHw17Y+6PvdpCVysELreeut17G3aG7qH8TS26ZVe2zFJhCHmLZ/61KdCMmqz/GAe07afoeff+ta3BjsQXcLYj97HZqRfNBV0CnqLvGG7xMJ22vRF7D3KjT1K32O77d122y1cjm6K51RtMdK43XS+h+7rNl9Hj4Ez9YiNsc466wT9wSfzb3QPWFHnrMaGjkGwqVO+JfQ3mCPod2y2JtJW70v34QM4/fTTq9VXXz3YU9iPjLnYTNSzfcECvdFPe6KX9vne97435BObiPnlu971roAtPqWDDz64Axl1sO2224ZzzAERVkubPXt2mAd2Loy+lOi+pr4u+6iSPlaio9r2FfJYYq/ZsvF9k002Cf0av0rKhmC83W677cJt2Iv0Cfl35R/FZtPKyXH6/G7qD+xmJ1gfY+o5fswRcAQcAUegPwg4gashjhh3Lo6AI+AIOAKOgCPgCIwVBAhk4wxDcOrhCI0FJ9mFF144HymLAB8kHYQgOUEJHAgIDoEjjzwyfMfxs+6664bv8T/SxUF/6qmnzneKdN7//vfPtz2TvYD7jj322EAmsMchVUE8SgmBVhxPOM6s4KjCAZoSnoMjkPJJCMRCpMJxlRIC4d/+9rezBLPUPZSVwFwsPJ9ABoQkJCZwUQfUHw7BWLj3Jz/5Sccpu/XWW1ebbbZZfFn4vf/++4cAEz94m1Wkk9TF4GeJAZbEBKno0EMPDbfRNsgfAiFBga1+4medU5BNcMiynUJK7PZeqfMca5s3W3YcY9yfqgsmUl/60peCg90+u03fIHi04447htsJFh5yyCGdpCBtQN6QfPOb35yPFIUDD6wQ+iX9s21ZlXbqM66HSy+9NFymIAtBCYITkHZiIWj5ve99bwGSC8GGXXfdNQQZ4nv4TdD9y1/+cueUCCeWwGUJLXvuuWfnWn3Bqb7LLruEn/Rx289xtE+bNi04WXW9PglWgx96KiZwERT5yEc+Ehzaut5+4gCn/8REDXuN/W6xJX8QOnEWx2LLrXM4yQk8kaecQKyzZFv7PJz3OG+l15UGuuW73/1u0P+QIGKhL1BGrrNSop/t/U2/MxZBvEBwWjOuxYKOplwE+rRqFNe0HQ+4B1IIbYWAVEoI0kDwsKKA4IMPPhjIcCJecU2qrdp7c9/p99I/X/va1wJJL3etjhMggzRGPmwAT/0JvV1HKCNIxPiB2O2MRoLAVdq/wYgyKeAkLPQJKeb73/9+h4ys4wQoRQzUMftJABU9L8eZztGnIObE/YjzkIAIviDxdr8l7SokVPBPupq8U47nPe95C6TCuMPYAQ6x0NcZWwjcWSkpw6DyAnGU4G2qHsjzHXfcEci4VjfX9VPGIMqHnHjiiYEUE36YfwT7Zb9BIoa0Uydf//rXg762fcle/853vrNjo5M3AsRIyXheMj7wLAJ/9D2VnWMSAunY2gROY1Io14zUGMCz2uhyqzu5VwKh8aCDDgo/IR4xtufaD33giCOOmE9vyA7mHCRrxiYJJCZwigX8sGNLxzH0CVvkIugXiFupPGMzYK9bEmCcF353mzvQX7Tdnp0DgBWkLyQ3t6S9oPc+/OEPd8aucMOT/5h38mKRlX7bWDbt1Pe2z2vSltrat6l8lRwrIXC1mdvYOZFtC+S1TX/sVjaNEbQf2m/KDoUwgB0TvyBSMiZpHLAErl76WbexCOwOP/zwLCHc4mP7GfN3iGQSiNbM/3L9n3aIxASuEoxUJ03ne5B8u83XlSYvke2zzz7BzobkmRLsV4iqCERYbLhYrC5L2eXx9fwu0fuqE9og5PCcX4qXaHhBEsEm1ZjaD3uil/Yp+x97BWIOcwTEzgeYI0MAxI5ICTYMbTj2uZXovja+LuWlpI+V6KiSvlJir6lc9tPal/a4bAg7n4eIR12mBLsd2yWWNv5A27fidKyPMT7nvx0BR8ARcAT6i4D8UIzD+LywAfEd88d3/W761EXmvdE3v0e36Z1Dfp0TuIa8gjx7joAj4Ag4Ao6AIzAfApbApRM4RQlO4WSQ44ZzdksmS+DSfXwSqGAlJq61aXOctyh5Ow+CEMFTORVZiYu3ZSXve9/7wtu0So8gFfnhTVIFyXGMffazn+0QgywZhmfhRMTpt+yyy4Y3upUWxAMCFwhkNQJiCOlBzOA+VtfgWRLIMgQvEQK0ygOr0EAAwDjGKSWsYnKJ0kl9EmhRAJfzpImTjDwoGK/7LIGLFXXASUIwCJwIvFpntogpEID0Nitv/iG8iY9AZiAIwyowrCKDgAcrauAIor5YnUD1RSCKFQKQnMNegSuusQSufuJnnVMQuHDaQvqhfnijGSGISNkgcBHMqpO2ebNlV7pqR9SdSD6ci0lXbfsG6UGKoQ7kzNYzIQewGoaEN5Xt1ihyDEOWYhscpG1ZlXbqM66HmMBl74HARJuiX6p9y9mo66i/L37xi532xnn6Ge0WoovaIW9en3TSSeE2OZwtkamU4IFuwuGo51Cn5JuVIWzf4sExgQudJGII96HzKCf9Wf2O9sh1lKubWGx1LQEa+i7zTt5KlsQkRet8RaeADTqKwIl0GPd+9atf7azElXoe4wH6xZZBz0TXondI16ZJ0JXgq6REP+vetp+lge+S8QA9Q1sVqY66ZRyhbiA3qQ1ZIgDlUUAwLpvto/G5ut+0LRFraR/UfS+i/mQDNqn0IMQoaHb77bd3tl4cZgLXBz/4wUBqoDwEZNEttHECUrbvEjBUH2V8/+hHPxrqkzYP2QfSG32JMVe6LCY4Q4BkdUQJwWbuo9/GxEpL4CptV3pO208FSnWfbCjaFe04FnBhbAUvld0Gt7m+tAyDyAsEeYjyEvQ5upngG2OR+m+bfoqOg5CE2LavZ4AdxBZ0APX+iU98Qqeyn6UErpLxvGR8IOOx7sJmRW/RpqXvuC4mcI3kGNBWl++xxx5hbBdxkbbMmIfNyLjK+IaeV59lngCxhfZv50mx7rV2MH1K+PAdkgYELsZVjoMhcwf0A4Hv0nHMBu6pB4T2R3vnedaGIUCfWp30ibue+N9t7oCd043ApfSwFcBIZdZxPik/88NYN7LVL0EISb9tLKWb+2z7vG5tqcS+zeWt7fESApee0WRuY+dElsDVtj/qmbnP1BhB38H+gjBLPhB+a4VQfpeOSdJ5dowr7WfYhqRHn0cP0OYhmdEuXvSiF3XGIghJEKu6ichCXIdOp38hrLSzww47hO/800razOVj0o0lcJViFNcJz6yb75GPbvN1pak5Lz4KcGfcZsyWnYLuQGeyGhuCfZIih33uc5/rkGbtvCfclPhXqvdtnShZdBhjCmOMxZ9xhTYAyauf9kRp+yS/sv+Vd32iv2m7cRuhbLRhyoW/SOMk/QU7mvpBSnRfW18XzynpYyU6Ksah6fyvxF6jXLFAzoQkrfE0tiFS83nqhBdDqQtr27N6n12Rrq0/sJudADYujoAj4Ag4AoNHwAlcDTHGWHBxBBwBR8ARcAQcAUdgrCBgiSTk2b4RyG8boJUTjeMxgYsgDisw6G07nPA4y3BS4vjlLXYCpRJW5SKgg+B0gDBBsBinHg5zOTchT9mteQiCiVwlkgBOCJz83EOggmfZt24JoBJIRciDVlyBzKJVgSxJi+ts+eSIxvHGc5CYpIUzcfr06R1CBW/G4SipE4sRjkgCKdpyBgcYhC2CixJL4DrggAPCSgics85XfrN8P8vxg0fsyFSZeR4EFysKHlJf1AH1IQEnkX8shjmHvQ1cicDVb/yscwoCl4hDbOvGH2KD4ipL6rMkb7bspAmphralegczMKYebGDP1nubviEiFs/ad999A+mA7zhIbTshHzi+JVptiIAjgbaSsiqt1GeuHuR85x76Je1bW3DiPKS/KNBinek49fWmqPKs50KyoB8g4KztTORw7geByz4fPYI+If8IqxWyEhp1ilgCF2+7b7/99uE4jm5WYlAfopzUkwgiIlaGi2v+WWy5jOfxZrmCNHZ1NYsVQVoc7QhBFFYro89L7NuyNi/x8y6atzUMW4Ag1BlpiuxA26W981zEBgyo56OOOiocL9XP4eaCf6WBb+lGHtlkPOA6G6iJg+EE9hmvRNyjP4g0C44KpFMvkN2oBxGGSLuNQDAicIwwFqO/exH1pxSBi3wT7KC+7cphJ5xwQmc7ymEmcKED6Y+0X4K76kvgZcctVrBk9QSEVdEmTpwYvrNqDTaBhKAV9cmYHY+3dpyG9Gy3OrVEddKyY1Vpu1Ke2n5aXY1exRYgqIjENiKklqOPPrrzCBsUpb1LV5aWYRB5Qf8RbENkN6oAjIfoZhHRKJvI3nX91BK0sGEZl6yOhVQNuRrJrQiiPOhTNhgBUdmaOsdnagWukvG8dHyArKktqAjIUfcKzGHvQHLUmG4JXCM9BpTocvKo1QTj1Vl4+YGtNxG2J2dLQCuQBSBzIrQZxn/E6hN+Q+Tn5RZLSJI+snhxbek4Zsdh0olJiWzbvfPOO3MqtFeCyerr4WDmnzCN5w7UdzcCF/dgD7CCHxLnEdIaq0aqLVmdY1fRHISNlSluOFz6vLq2ZO1La7PxwJx9W5fHNudKCVxN5za5tqC2Q16b2lZ15bJjBGMudiikBITxGCIP5BsE3c98HSkdkzQO2DlH3Iab9jM7P2VrYrsCuSW4QCpnntdNbJksgcvaHtaOJz3bDvhtfQg2vTY2ra2TNvM9i4e1gciX0rS+J46rPmKMLNmSeRt1IiFOB1kKyY2vulafpXrfYojuY5UtvfRG2nbOzvZ1kIT7bU+Utk/yJ/uf79jJrAjGmCj7hvGDcQRhZSfIcxLsKHwVmuuyXS/3IyW6z7Zj205JL+frsm2qaR8r0VG2npv2lRJ7jbLWSc6GiOfzse1icYpfPpEtSv039QeSR+EY2wl1+fdzjoAj4Ag4Av1DwAlcDbF0AldDoPwyR8ARcAQcAUfAERgKBKyjHAcoRIpYbIAOUgKkBktwwlmHQ8oGQu3KUnbFJps225exjRnCNl5s52VXx0i9gWq3i4PYxfZrLOPOMveIDSKHA0/+kxMIpwJBWxyxOHkh0yCWQPLkLSFdHKo8B4KQdYhZEpOuhySGww+BZNEtKGIxih2spMFWNTifRRYRgcu+qRkHU7kPsY6yY489NqyKxvE65woBFJxvBBu5JxZtDQIhhKAPknPY28CVCFz9xs86p3olcJXkzZaddkXwzwbmwMe2OxHmbL236RuWdGMdmaoX8kBbweEmkhNvZ+JkRETOLClrSCDzL1cPcr5zW6pf2v6v7aUgQeA4RCgHAQz6qhX6L+QYRKszyOHcK4HL9jmeT2CWTyt2mwVL4EKHEDyiHtCZBGStkDbtAWlKsrHYMiFXgFnpWrxsAHjSpElBB3AdRBQct1YgwWpFGOvgts+z6eleOxZY4pfOE0iD4IXuU7CkVD8rzbafpYHvtuMBY4fe+E/VDfm2hAdWppReVQCKa1K6n+NtxAaaY0d8m3R0rfqTfnf7jEmjw0zgEqGVwC9jk7VbWKFG26pijxB8Qhg3IWjQJ9SuLSbayoS+Lz1vt71N2Qvcb4lFCl720q5sntp8t7pa+dD9drwAM8pKOSWW5E8gj4BeL2Xod14g3WLnIrm+MW93gEB65xo7hnTrp5bYd9xxx1XXXHMNSQSxpB7ZzTqX+1TQLBdgThG4bP2k2lnKLi0dH2j7+Dupf3S9fVGCMtlVX+z4MdJjQFtdTt7rSDfbbrtthY6l3KQd23mc32STTUgmzEn00om1g1mZERs7llzwtXQcs+2BuRk6Lrah7LwvXrE1zp9+5+YO1g7Wyy7cY4Pb9Av6hxXpYY7R50Vu5rclJNuVVgdhY/G8nJQ+L9eWrL3Wxr7N5a/tcUvcqat3W6e0+aZzG3ufbQsl/bGubHaMwKbCtrLClpyrrrpqOKTxrJcxSeNAjsDVpp/ZF8pSNjSrMDFvgzyPf6Ob2H4mApcd83J6x9aJ5pO9YGTrpOl8j7JZEonqSmVWmk0JXJZwxctwpCex82dWGmLFoW5SqvdtnaR8NHactGThftoTvYwD1v4/+OCDO6vWgxd9HDslfjHNYsk8GN2JqM+U6D7asVYlS+FI+ilfV0kfs/2BvIsQyjMQbBjrDyztK7ZemtprT+Qg/z9nQ9j5fMrnAMlVNom113hSiT+Q+3J2AudcHAFHwBFwBAaPgBO4GmLsBK6GQPlljoAj4Ag4Ao6AIzAUCFhH/tlnn12de+65C+SL1ZwIcCFalcISuKwDSjcTNGCZe5y/BLJskFTXEPDAQYbIwa/7OBY7jjiGrL322oEoQTCOAJKCSqyCgPMgJR/60IeqyZMnh1NaYcESSLh39uzZYQlxu7qGTcs6pTjOW+OQHyBHyFi213f7LocRGEGmiIki3G8deiJwWQIQjrbb521dFYtdDv7qq6+ujj/++HBJW+cKgQjeCqeutAJJKYGr3/hZ51SvBK6SvNlghbYWiOvBblME9rQztfG2fcMGhfTWq31jn3aofqqAsQ348lyCAiVljctlf+fqQc532rUIZfY+67hXAMEG7uJV7nQvb7BC2ATLK664IugYOZxt8B0ihVYIoh/FAnlUZA0FUqxzVTopvs+uXiICF6QlAtoI2yuJqBXfq60TqHsRPeJr7G+LbS7ooKB//Da6TYfv5JHtZVgp5U1velMIxHM8R+CygVOuQ6ZNm1atvPLK4TvPZVsmK2zjx5vkduWmUv1s023zvTTw3XY8sAQdVmpibIzFOsjZPgUCHKKAIO0AvcBnL2Lbcioo0DZt9acm90H4hYBpx69hJnBZ0hTBpauuuiqs3hgHbLqVnWAO+heCDG2BgJbt15bYJP0Wp7nNNttUm266aTis4GUv7SpOv+lv6Wqbf91rV65IBYNZmUjE9VNOOaW6/PLLAx6sVIi07Rv9zoslLQhjlU2frM5Fu0BsIKtbP7Xb+9ixx66m0U0vKw98Spe3IXD1azxvMj6IoFtXJhHKLY4jPQa01eVgb+2r1JyGa6yg29kuCrIF7V8ruEG6SBG4RFK3afA9F3wtHcesDcNWrxClYsF+IiCNWKJNfJ39nZs7WDvYpmVJDPGqhaSrtp6yD5k7Yq8izMuOOOKIYL8MwsYKD0n868Wmy7WlUvs2kb2iQ1YXNiVwtZnb5NpCSX+sK2DdGMF9dvsvjb29jKsaB0RG4Rml/cySd0gHHwZ1AQkt5Sfhmjqx/UwELktgkb8jTsOSnfqBkeok1Z95dmq+Fx+Px2el2ZTABUkI4gm2mK0rnoPtDUEf0Ys/4UfLf030vq0TzcXtY1idSqvkai7P+X7aE6Xtk3zI/rdjOMcRq8PqxknqAaxkU9r7ms7t2faaciBtfF0lfaytjirVJ/2y1wIoT/7L2RDWf4CvhNUsY9Gcvc6m0z3d/IFcl7MTlIZ/OgKOgCPgCAwWAcWk8CUwN8UmYk7DH9/1u2kuFpkXXOjNS9n0SSN8nRO4Rhhwf5wj4Ag4Ao6AI+AI9ISAJXDhJMdZHgvLlL/jHe8IhxXstwQurexj75Pzhi0xWDEnJdZJr60kdB/Xi/CSutceU8CIY7lAOMaqhDcvIUPwfMhlOJ2tkAZvg0PgwLGqbT24hqAkjptYcDCyPQrXK3gTXxP/ltMldjTa69hmku0mERG47PL7ufJyvcpMvti2AunmXMHJxrZsYIMzNCWlBC7S6id+1jnVK4GrJG82WGGDtxYzS3hR/amNt+0bpKt7RZDh7VRWZMNpzlvyIg4pOCOndbxC0EjUg5zvufZtV5JTAGETQ+qMtySwuMbf5XC29VBC4Np8883D1kWkr1UB42fxW6tHiMBltxjlfK5fqk9yTZNtVm0bZytDVmuKRatEpBywrAAFmYa3hRVgju+XTue4fV6q/GzTh45AwDcup5zBap9cV6qfubdESgPfbccDtkdjmzRJjIWOq85tf1dAMN5yT/e0/bRBgdSb1m3TU38ioKitaW0aHIccMGfOnAVWd+E6245yb87b9KyezAUc7fV8L+nf3Md4CrFV9cIxhFU0IL9BhCXYEQt9CDto4rytFAlMxfdzPW1AxEy76hBjvbYatelaorWCl720K5t2m+91utqSm2ywUemnCFy9lKHfedl9990rVpxCsEWtPacy8CndRTvQ6oRN+qnus4FruyIe7Snecs8+134XqaUNgYv7S8fzNuOD1TEpIp/KIZKyDf6O9BjQVpeT9xzpRuWClIe9teaaa4btOHGEpyRH4CJgj26ORfMAixfXlI5jNnCfC9pSFtotYu35cCDzLzd3sHZwjsCllwds0rJlU3YL9RcTuAZlY9k82e+9PC/XlkrtW5uvXr6XELisTW2fbcdszW1ybaGkP9pnxd/rxgiufe1rXxtIXHzX3KKXMUnjgJ3LlPYz7HBWFqaNxIIOgBDP6p/dVvDWvZYsJAIX9QG5HEn1PY7brVT7gVG3OknN98iHJXbJBuI4ojSbEri4h7GbFYYRCJ/YdJDttWp4U30XEpj3r0Tv2zpJ2Rx2LNUcUs/rlz1R2j7Jh+x/Xg7Ud+WPFw548QCRL03n7KetB+aoq622WueFzaZz+1JfV0kfa6ujetEnpfaaxdd+z9kQdh6Wms+ThmzO1Djc1h9Iejk7gXMujoAj4Ag4AoNHwAlcDTF2AldDoPwyR8ARcAQcAUfAERgKBCyBK+fos9uwaVtDS+BKOQZEcIiDErbQ1tkrApfuaxrYJoiaesPcPif+/vOf/7y68MILw+Gll1662mGHHcKKPimCA0G5gw46KDgBlQ7Bude//vUVK/GkxAYxUud1TGW1ZAed06d1EslJblcQ0XV1nxDKtBVDnXPFrlJm0yMozWpjvDWKWAeorUNbbrt1TNyu+oWfdU71g8BF2drkzZb9d7/7XXXUUUeRxHySCnKo3tv2DRK2b4lCPoQswKpK2sJMbYPfOOZwBtNH7CpsymCbsuqe1GeuHuR8zwWjUw59G2CaMWNGdf7556ceucAxOZltsKkbwYOV/NgOEdEKXFYfnnTSSRWrUKVEAWk53y0RI3V96ljcL1LXWGx5ezZFLEkRuCBfkj6kk1ggKFAnehs9R+BKPc8SuFKrmin4IJ3Wq36O897kd5PAt9pLvFpVm/Egpy9zebTBv1RAMHdf0+Nqk5aA0u1e+ijBJMSuAil8VI/d0onP2+CRdFN8jf0toinHcmQnez3fS/q30mAFGsZWtmJNEbHoH6waRJ0hBK3Ygie+lrGRwAcYYj9YApcNPpFXzsVit+5T8LKXdhWn3/R3na62BC6thmPTTRG4eilDv/PCahus2mjrxuZf3/VcS8Rq0k9tWbWdl115k5VWsZ+aiIJpuTHzfe97X1ilg7TIG6vjSNqM5yXjgyW05Owd8iIbRPbNaIwB5KONLuf6HOmGc5AhsOUI5sciHQGpE8kRuHLjfS74WjqOWd2bmpsp/7JDcyss6Tp95uYO1g62cwBLYkjZCiJwpcaYFIFrUDaWyhd/9vK8XFsqtW/jvJX+ts/XSx6ptGyd5vp6am5j77NtgWe07Y+pfOmYdHVOT6YIXFZPK526z272Wi/9TKQgXgaT3ojzcuaZZ1YXXHBBfHiB37aficBl7alU3yMRu3qzCFy9YNStTlLzPfLRbwKX7bfattyuyFanE8mPlVK9b+skhX8dgcvWQS/2RC/tU/a/nU8LF6tD6l62sBgwR+Vllw033DAk03RuL1tCz+72aX1dJX2sjY6y9dQtX5y3+oTfbew1rq+TnA3RxH8gmzMmcOXKhx2d8weSx5ydUJd/P+cIOAKOgCPQPwScwNUQSydwNQTKL3MEHAFHwBFwBByBoUDAEhZw2vDGXSwpJ0A3ApecAnbVkThd60C8/vrrKwKYuq9NEFrbukD6YtvFbsK2X2y/FgtB3XXWWSdsiSKyEtfEjg3dR2ATByyOMrZqw3kt0VaT+p36lNOljqxmt0sUgcsGhqkzsKoTAjSUAck5V6zTEwcNxBS2lmLFDZzkiAgCvRK4QmLz/vWKn22X/SJwtcmbDVa0CXKojbftG+TN9jsINpAPIQ6wMhMrNMnpRhCabTN32mmnUKTcdqScHFQ9lDj07bYekLdw9DaRlMO5G8HDroYgAhdbC0LORFIrC3LcBqRF4KL/8zwEMirkrzqhzzbZss228RShimekCFxqB5xntSTeeKY/z507NziSIf2xAhgySAIX6fdLP5NWE+kW+LbBk5jAZdPvNh6wUiH1gxAMTZHrbHroUbZRRJoQQ+y9Tb4rTa494YQTQn3X3cdKMmxRgsSBUPWnVHC9Lk2dYwUErWJkCTE6H39qbOE4q1NovIqvs79L+re9n+8EecABMidvm6MLJQpAs3oDZC6Rt9i6mdU56VNaJQMy7fLLLz8fSciuwJUrk92eWgSuXtqV8t72s05XgwmBNKQpgauXMvQ7L5ZokFoNQ1ipDdoV7NSn4uCb7uHT6n4CiIcddliHOG23TbX35L7LNoj7o663q4nRJtX+dJ7PJuN5yfhg9aYNlNpn813EIBG4ODbSYwDPtNJNl3NtjnTDOa0qxnfaB4R4dABjPXMJyJ2bbbYZp0eMwGXrw45jNnCf274UAh9tDalbTS1c8OS/3NzB2sHSmdxiA/gpEkNbApftZ/20sWwZ7fdenpdrS6X2rc1XL98t+WKkCVw23036o70+/l43RnBtisDVy5iUGgf61c+WXXbZsCooK2Jhl8vOoBysqqxAHL9TYvuZCFx29aPp06eHbYHje+0WcCJw9YJRtzoZKQIX+PEiCfNijdusfohPB98GtlrTrSpL9b6tk5Tus7pbc0jVj9U7vdgTvbRP2f8pApftW3UrcGFr8YIEgq3OfZD9kaZz+158XeFBT/4r6WPddFQvfcXmrYm9Zq9PfZcv0dpcXNfEfyCb0/o5S/2BPDNnJ3DOxRFwBBwBR2DwCMhuxJeALYRd5FsoJnB3AlcCFD/kCDgCjoAj4Ag4AkOLgCVw5QgLlkSEQwsCjyWSpN5oxPGIExuHGW+epwKy1gHyy1/+sjrrrLOCw5L7kNSWI6wow9ulGKOQzXA0ycnGsyA5pchZOLP4Q8gv5Bmc2QiBGIhKVlidhudrmxScfty/6qqrVhCuIGjFYldHSG01FF9v35JNlZXr5XjkuwhcNviW2/aSFSdERLnyyivD9o6kkXOu2MCgnLlcL8HGJWCIlBK4IMf1Ez/rnOqVwFWSNxu4akPgKu0bYE97JNBM+4dIoK0icOAR1KIc9GmE3yussELYXpG2Iykpq+5NfebqocShb1f7s1t/2ufaVcikj1IOZ0vwSBEodt5557CNCGmLwGWd3rnnsxXErrvuGrIk5zsTZOoFqVvRgu0McSazJQjO725isW1D4FJwFJ1I8CleAcZugzFoAleJfsbpUCroPYh4SGrbSbslsALfEHnajgeMgWzFh+S2qqKfvvWtbw1EIZ5FQB1JBQTDiR7+sZocRCTEklBySdogX7zNofpTKYGL/qDV/3h+jmzCORvsbkL24h6kpH+jL7XtJfaGCHVPpFjNpz8V0LBET/V3Xa9P29/IF0K9s0UdcswxxwTCV/hh/tmVmkTgssGTtu3KJN3qa52uLiFw9VKGfufFkuROPPHECnsoFhs0ZVyl7SJN+6nyTPs97bTTQt1zv8YVvjcRBdMUeI7vsStSqE+VjOe2vbYZH0TEigOFyicBWwK3iL1mJMeAEl2OTW/10G9/+9vqhz/8YSiH3X4LncCKbvEcw9rP/VqBq2QcI8PWhtEcKRTE/LN2olZVNqeTX3NzB2sHD5LANSgbK1nYeQd7eV6uLZXat7k8tj0+GgSu0v5YVzbp2xzR1ZJMNJ/tZUxKjQOl/Yxt2lnph/nZxRdfPF8xaXPM+bVy7qmnnhpesJjvouiHJQuJwGX9OoxHkPViwXYFE6QfGHWrk5EicFEei8nhhx/ema/J3ueabtKL3rfPb0vgIl/Cshd7orR98nzZ/ykCF8QmXk5A2O7z+9//fvhu/zHvEYkOshztskT3sWXx5MmTQ9JtfF1t+xh29kjN/0rsNYtt6nu/CVzWnpFusM/N+QO5Jmcn2Pv9uyPgCDgCjsDgEHACV0NsncDVECi/zBFwBBwBR8ARcASGAgHr6COYyXYvBP0lvCmIMwmHDMcVnOxG4LJkplmzZlUnn3yykgyfEFEI6iyxxBLht4JRNj+pbd9SpAvrbDjvvPPCyjn2YThFWaWGQAOBF60MIiLMfffdVx1wwAH2lvDdpgsJbfvtt+8EyHFwERyxQlmUTo78Ya+3RCwbMNI1NqjIMRG4bEDZBht1H582OH/IIYcsQODiGutYZDUe3v5FUmWzwehSApclGKSe0RY/S27JEbhygfNQUPOvJG82cNWGwFXaN5Rd+1YqxyzpgWAJzjz6q+SOO+6Yb5vRkrIqrdRnrh7khM4FWVIOfVaGILgsfUMwmGC2hLkWwRTO25XrUg5nG6BgNTL0iYQteiB1CScF2iEroJf0fLCG7GnFvllsCR0KinOtCHX2PrvioO1D9pr4u8W2DYFLq8mkSK2UjbKDATJoApfVo031cxwgj3Gp+20JdjExiXGH9kNwBlFAxxIjm44HtFVWN0AIUkDytWMnx+32cnYcTAUEub4Xoe8z/pAvRG/vox9isfqcPNPm7Wo+6k+lBC6eZ/sJ2yjSP+J6xb6gLWo74lTAKM67fpf077XWWqtC/yK51WnUj7VC4jbbbBO2UOQetlSNV9ezAS1rI9kVLu6///7OKlakg9gV0PgtAhfB3dJ2RTolUqerSwhcvZSh33mxW+XSxkVEtzjZ1dLsFttN+6kliTFesS0WbSEev+wzU99ThH1dx/ZDCjJyTDZzyXheOj7wTPk7LVFJeRQ5nd+WwDWSY0CJLkfP5Ug3kOAJQCOp9sNY8rl5L5WgfxGLy07zVkBlroS03UKxZBzjOTZwj+5Ht8cEbvQLfRRhxTRsxG6iwCzX2bmDtYMHSeDiudLNfO+XjUVaOSl9Xq4tldq3ufy1PT4aBK7S/lhXtroxgvtSBK5exqTUOFDaz/CxQNDCFmJOH9uM9uWKuhWOhI8lC4nAZclqVg/rHnwikIX5RETS6AWjbnWSmu/xbLuFYjxfV5q88MJcTKL6EMlex/VpbTKNx5yDlIuvpYn0ovdtnVhdqefWrcDFNf2wJ0rbJ8+X/Z+yx7Ft8Kcxj6QN4zuKVzTjJQm2KEe0WmeJ7iv1dbXtY5deemnnxbhBz/9K7LUAZM2/fhO4Sv2BZDFnJ9Rk3085Ao6AI+AI9BEBJ3A1BFMOjYaX+2WOgCPgCDgCjoAj4AiMKgKWMEVGcLZAFGAVGQL8OJ8I4CE2eNmNwGWJENx79tlnV+eeey5fK5xXBHS0epB1EnGOIDROXwRiBatz4SgigEXgWeQKgl0YqSyNT+BIx3FGQkpAWEnrox/9aAjO8NtuAWSDUWeeeWZ1wQUXcEkQnF8EesmHiCI2IAvZjcAHzlGE8hKswXGI5FYzCyef/GfJKByaOXNm2DKOsk6cODFgpKAQ50Xg4rtdiYGt0VhZAkcwDlny/dKXvpTLqjgAbx2LrE5DEJvt3KxTiWA7W+4pqEUAm2CzxDqEc8EbsIgDV/3Gz5JbLIHLHiegRGA8JuGoLPosyZstexsCV2nfUF633HLLEKDQb7YE1XY4HJPzUufjN7BLyqq0Up8Wb1sPcr63IXCR/jve8Y6KVZIQ+jfOZAKn4IY+EvHI9uWUw9kG7nHif/e73w3bFnKcN3tFHuU5InDxnW3l1N4JEBDs5vnoAvoWukFiCVw2wM7zCOTeeeed4VLePv7gBz/YIdc0ebOeGy22bQhctg1AHEInEsxdeeWVq3e9610VK/RJWIHwuOOOCz+7PW+PPfYIW81xcSowwVvX6Cyrd0r1M8+weo60Iax2E3QgdYagE8EN8t6kSZMqdBmBGYkIXPxuOx5wj10NDsc/K9RI12jLEI1LBJ0USFcAinairSxJD7HtKO7bT1yR/8/b6hCDJfQ96p8VIRnTabv0f8iEktR2JupPth51fdNPayNwD6TFCy+8sGK7ZNoEeWWFKvq1pG6rV12jz5L+bYNIjHu81Q/ZGqHdsuoNfwgEbXCwpC9sAeqY7U/RH/QXApDUMUJ7I4ChoKzqmXNsX3rKKaeE8Xbq1KlhyzXZOZwXgYvvpe2qtO3U6Wrqp+0Wir2UYRB5sfqQ+mZVDvQh+G+77badVdni/qj6i49TPiu0Z+xRKzlyvb0m/s74QntDWKHlyCOPDPlkPGTcVzvjvAhcJeO5xaPN+GBXbgI/VqIlIL766qtXrC4pm578WTtxpMeAEl1uXyBA79EOGf/py/R5Ya9+yjiDnYC9ItIs5T766KMriMNIyg4OJ8w/tXeeQyAW3cIcoHQcs4F7HoMNxSopkLQIvn/4wx8O8wvOxcQIjuUkN3ewdvCgCVxWv7WxsahbdBh1iN7XSzy5sup4L8/Tyzy2LVGvJfatbUfME1lFukRGg8BFPkv6Y1351Gdyc4sUgYv0SsfV1DhQ2s/siubMYY466qjQ38kfq/Pir9ALBhBD41VCuc6K7ZcicHFeeeY7tgxb+9JnGM9p/yJwcl4ELr6XYtStTnIELjvniOfrSjPWUypbjsBFOSz5kt9t+j3XoytK9b6tk9Q8qRuBqx/2RGn7pOyy/61vjuMS6y9ifGHcYtVfhDERO0bjJQRibH+kRPfZOWBTX1dJHyvRUSV9pcReC+DV/FM/iW0I27dy/gOt+mr7kq3fNv5AsmjbvvUx1mTfTzkCjoAj4Aj0EQHGZQQbmbkk4zH+Dv74rt9NH7nIPKfV/y/t0PSuMXCdE7jGQCV5Fh0BR8ARcAQcAUegg0BM4NIJHAFywHAMYxCSlMQGZ1NbKHLd1ltvHYKUuoc0FZjQMd7c420+CBKS173udWHlEv3mPsTmxxIuOAchAwehhHtw2BFckBBYpwx6W9AGZrmGgBTnCHLYZ9m3YHGoQgpDeAYOUYxj+xwcITyH9LqJXd1DaVqMeIbyYglcvJ2P80jnuI6820ASxyAyWNKDdegob6wMQJlw6io9zpF/yoaQlsWTZ+E0kvOOa2zwxgYc7MoD/cTPlsUSh2zAmXwhMYnpiaPz/2+bNxu4akPg4qmlfYN77VvS/IZ4CAFREvdpCCK0Uytty2rvjb/n6kFOxVyQJefQB1cCzJZgRZ+wZAfKQ8Bc5Uo5nHGS4/xVG47zbdu31SfcR4DA9iWu1cTX9klL4CJ9++Yqv6UDbB5yTnGuj8Vim3PAoj/Js3XAEkzfaqutOsmRZ0T9m/4LYUW/CRLRJro9r4TAxXNL9DP32SAMukyEOM7VCTpHRL/4Olt/lsBVMh6AO45/q/9jbHl+TBBWAIr2GxO4bN0RzKd+28iOO+4YSFpN7slt0af+1AuBi+dbQkq3/PAWPHq8qZT271g/QsqiTdDf1R/4TWDjnnvuCfqDYDx2gYSx0JKrqUedp89jE7E90iqrrBICskpX9+vTtkURQzhX2q5K206drrbjqSXNqgx2lTkIapdffnk4VVqGQeQFYj2BJVsPVqeTYeoCIjxBQkldP9U1+rRjKsfarPahNNZYY42KVWZzYtsLekd2s30219Ae6+xS2054FvcgwifWYRofuIaA9MR5Lxh0E0vg4tqRHANKdDl5VNvjO6KVdO24x/G47UAwWHzxxTkVsLzpppsCMSNnB4cLn/xnA54csmSFknEsDtzrWbbtcIzfkKIhCjYRaxvoevKH/iSAj9g5gC1XisSgsT01xjB2kzYS65wSG8tuh0bdMZdqKiXPI+1cWyqxb+3K04wrTbbfTpVvtAhcpf0xVQaLbW5ukSNwlY5JqXGgtJ/RFklP8xn6IX0IfW1tCumeHAY6bvuZJXDFPgKut3aL/W4JXKUYqb3n6iQ337P2hcqk+brStDqRa1Qfds6je/UZ23nXXXdd2Mpa55t8lup9Wycp3QfGsuvjOaTyZcd0jrW1J0rbJ8+S/Z+bq2LrYn/Ec2TsB7Vr0onnFyW6L27H9BfsE/tsjllfV0kfK9FRpX3F1i1572avgWWd2PbGdeovdszO+Q9SBC5ecirxB/KSkH2m8sxYzjjv4gg4Ao6AIzB4BJzA1RBjJ3A1BMovcwQcAUfAEXAEHIGhQMA6uVgl6w1veMN8DhgySZCIFQvYBkjCW3asVIPI2aZz9nODDTYIq2ZZp47O81Yeq1ilJvasrkEg2hIfuA/nO9twQbiIpe5ZBCl40zV+m5VVDQhmKXBl04Q0olWxdJytMXDqsQVCSigTq3pYrFLX2WNsmQChJ84DwWNWyGLVMcQSuPjNKiq8TW+JLhxHeBsShw0rr1jB4cRWX7zhKYFshrHPaijbbbfdAvWPMwjs2GJReeFegj8QDPhEbPDGvsHHNkJalaaf+JFf5ccSuMiLfTOS3zNmzKhYaaZO2uYNZ7vKnnMOT5s2Lax6xHPj+qtrr3V9g7QgFKjecQRTDxK79U4cRNU1bcuq+1KfuXpQHmmL8eokpJNz6HMORy9vor/whS/k53zCqkSQHfRWLyfRI/SfOOhBm2XbV+vs5XpwYVtFVgJELIGL3ziAWUnH9hOO42w99thjw1ZWBB5Sznf7ljH3SLiX1YdwxKNbmojFNufATxG4SNvqdvssdBMOeragYxUkREHNbs+zAY1UYIL+QL+ICb88o6695/SzgrzcT9qWjMqxnFDf6Dm70hjX4vgnWMX2HuhCS+DifNvxgHvohzjQ7eozHEcIzDFe8WdFK+DI0W7PWXIFYwAYtBUIK+hg2nFKeC59iP6SEgVwUsH11PV1xyBkY1fY4KS9HozYtu6yyy6zhxt9L+nf2BS77rprZ7XM+EEEUyDy0FcljLXUsUgaOk6/YWy58cYbwxY/snMuuuiiirfPEXQtAfjnP//5nTGe+yAj8gzGfsRuvcbvknZV2nbqdLUNsMZbkpJPS+A64YQT5iNAlZRhUHnJ1SFloA0eeuih1e3zViqxUtdP7XV833zzzUM75zvpoX9KhNXZdthhh05bURrkDaI4eCOWwFUynpeMD8oL/Zm2pnGVsQ2dwopckETQrYzPrL5hZSTHgBJdjq6iHtWPWTXvoIMOCuRM7AFWyLFCuen76FLKSl9BpLdzdrBNA13N6pwigNq2UzKO2cA9AXQC00pbz+UZzA+uvfZaHer6mZs7UO+yg+0cAB2LLYqkbAWN7SlbwRK4UjqnrY3FOKi2KFuna4HNBW2fx625tsS5tvYt7UOrKqdWzCTNJoLdw/ZmyCWXXBLm7qn7Suc29j7bFnhGSX9M5Y1jdWME53MELs6VjEmpcaCXfsa96IacTcS8nVXzIFl1E9vPLIGL+yCQ4yOgvVnBxmF1KW1LaAlcXFeCUbc6qZvv5ebrSjO2k5sQuNDVKh9l4mWeNn4Z7kFvluh9Wycp3Ycu7Ubg6tWe6KV95ubTYCKhjTBHxwaPhXER+xeyaSxtdR/3l/i6SvpYiY4q6Ssl9lqMo/2dsyG6zedJI0Xg4jj3tvUHMj/P2QkiFJC2iyPgCDgCjsDgEJC+9RW4umDsBK4uAPlpR8ARcAQcAUfAERgqBGwQZ/r06YHYgDOAFbZ4uxGCAtsq9iI4wdhCjD8CoARFeKuat7W6CQQOViVQwJO81Dk1CXisuuqqIXCAowjy2ZVXXln7LIIuU6ZMCc5O8so2V5BECG6kyGXkGccoJDNWQyI/EGjmzJmTDYh3KydOIEhxOKrAhWdrdYW6eyGtrDxvWzQwgmzCvbz9z5uTdcJzWHWLIBvBJomwgIwABmyPo6XxuQZnHW/n4QAmmNiUiKL09dlv/JSu/cRJRl6ZyLB9TNO8jkTelM9e+obS6OVzJMtakk/6BSQj2p36WJN+ET8LZz59hP5CP6U9NJGVVlopBEC5D91DYAVd1E2YE5Jv8EU/sA0BK9MQOB1JmTBhQtg2DyITATWCoTYPBFgJboIpennQUqKfCUAxdvBGcBPsbRkoG/WAnoasxXjWLQ3pQOqu6XjAM8GaNobuxGly+zzCBdtTlgg44VjPEUObpkm9M56TJ9oh7R4McuNa03RLriOgMnHeyj3UBQEexlnaJH+9Skn/Bpt111036BYIG2yBSQCCMY/8xcI12AmMt9hG1K0dO+nzL3rRi0KwlH4m0rJNZ/nllw/YQyBFtt9+++qVr3xl+E6QWFtshgNP/mvbrvrVdmweev3etgy9Pi93P3VIG0TvQRKhvukPTYmhuXQ5brf0jFeeqLsvdY7xhnyi99AljDtNxr2243mv4wPBOoiJ4Cf7CmIAOKNrCATHMpJjQIkup0zqMjQfAABAAElEQVTgzjiJDrf9GF3KdpGUAfIlBFjpCuYbtCvmONgYTerLYkO64AaW0g8632Ycs4F7beWOflxvvfWCbmKcZ1xRfekZTT9zc4em9/fruhIbC0II42DqhYJu+Sp5Xl1b4nn9sm+75X1Yzpf0x0Hlvdcxqdd+BpGcNLAZ6d/oeeY4zKuZL/RLeA7PwDbB9kVvNZ3/9IpRmzKUztdzz0BHQxJlLMUvAumrVAap93N56tWe6LV95vIVH6eNMHbhq2PMw55n/Os2z2qr+6jHtr6ukj5WqqNK+kpbey3GPv5dZ0PE1zb5LSxK/IHDYic0Kadf4wg4Ao7AeELACVwNa5OJnYsj4Ag4Ao6AI+AIOAJjBYEUgWus5N3z6Qg4Ao6AIzB+ESAoTtAVwuhnPvOZ8VvQqGTaYpWVoCAgu4w9BCDgsIoo8utf/zqs5hmXgjYNkQwSCCs9iAwSX9fmt7edNmj171oIpgS6EK1q2r/UhyMltkSFFM9Khqy6FAdpCXTvtttuIbOzZs2qTj755J4zvrCOAaXApQL3pWmNp/sg47MaKyumsbqRiyPQCwLez3pBb/D3spoSK7Eh8Qpjg39670/o1Z7w9tl7HXgKjoAj4Ag4Ao7AWEPACVwNa8wJXA2B8sscAUfAEXAEHAFHYCgQcALXUFSDZ8IRcAQcAUcgQoBtwphfswUhq4ksDLL++utXbNnESiH77bdfIK8tDOUeb2Vk9RVtk0NdssKWnGqUlS1MWRUHYTvFb37zm+F7L/+87fSCXvt7qWNWaLLEpXhb1vapDu8ddqsrSImnnnpqJ7OsoMKWVdoW7Ctf+UpfVpJZGMeADqgFXzxwvyBorPbCFmqs4sJWqf1Y9XHBp/iRhQkB72fDV9uMx7zsQX9nRXlW4WIlxdLtjEe6hP20J7x9jnTt+fMcAUfAEXAEHIHRR0C+Jt9CsUtdOIGrC0B+2hFwBBwBR8ARcASGCgEncA1VdXhmHAFHwBFwBJ5EYPfddw9bws6cOXOhwWSttdaqNt5440COYEsdl7GLwN577x1WLKIErK7FFjOsKMSWSWzvghBwZJW5eNu0cLLlP287LQHr8fLPfe5z1ZJLLhmIIUoKIh6EvPEokyZNCitsQYRB2Er00UcfDdsGEjSXXHTRRdXpp5+unz19LoxjQC+AeeB+QfQgRkybNq0655xzwvaRC17hRxyBdgh4P2uH10hcjT+Lba41PvHM888/v5oxY8ZIPL7nZ/TTnvD22XN1eAKOgCPgCDgCjsCYQ8AJXA2rzAlcDYHyyxwBR8ARcAQcAUdgKBBwAtdQVINnwhFwBBwBR8ARcATGEQKQtfbaa69qwoQJyVJB3mKlooceeih53g8ONwIEXJdaaqlOJi+55JLqtNNO6/wej1822GCDavvtt+8QEOMyLgwYxGUept8euB+m2vC8jFcEvJ8NX81afxa5u+uuu8JWv8OX03SO+mlPePtMY+xHHQFHwBFwBByB8YyAE7ga1q4TuBoC5Zc5Ao6AI+AIOAKOwFAgsMwyy1S8VY9ceeWVYdumociYZ8IRcAQcAUfAEXAEHIExjsCKK65YTZ06tcLeQn7/+99XN954Y3XPPfeM8ZIt3Nlfe+21g/384IMPhpUC77777oUCEFaPY5VA2jUEtvvuu6+aPXt2wIAtq1xGD4HFFlssrEJDDtgqkFX/XBwBR6C/CHg/6y+e/UiN8WjdddcN2ybedttt1c0339yPZEcsjX7aE94+R6za/EGOgCPgCDgCjsDQIOAEroZV4QSuhkD5ZY6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj0BgBJ3A1hMoJXA2B8sscAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR6AxAk7gagiVE7gaAuWXOQKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao5AYwScwNUQKidwNQTKL3MEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcgcYIOIGrIVRO4GoIlF/mCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKNEXACV0OonMDVECi/zBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEGiPgBK6GUDmBqyFQfpkj4Ag4Ao7AqCHwute9rlpmmWWq+++/vzrvvPNGLR/dHrzRRhuFfD744IPVxRdf3O1yPz/GENh8882rxRZbrPrDH/5QXX/99Y1zv95661WTJk2qHnjggeqcc87pet9qq61WTZkyJVz3ox/9qOv1g7zguc99bvWqV70qPGLWrFnVvffeO8jHjdm0F1lkkWqLLbaoFl100WrOnDnVDTfcMGbLMtIZf8YznlFttdVW1XLLLVctvvji1cknn1zdcccdI50Nf15DBF784hdXkydPrh5++OHqwgsvbHhX/WWDSLP+id3P0pfR+U996lPDxdgef//732tvfMpTnlJtueWWFZ+I68xauAZ2ciyOW69+9aurF7zgBdVf//rX6txzzx0YNk0SXnbZZavXvOY14dKZM2f6uN8ENL8mILDSSitVzIWQk046qfrf//4Xvtf9K53jjeU51zDoqNI5TV1d+rmFG4FhaNcLdw146dsi0NZH0zb9hf36sTxO5+oOvxi6rkSwqfF1lNjY41W/pmyRXFk33njjaumll67++Mc/Vtdcc01JFfg9joAj4Ag4AkOEgBO4GlaGE7gaAuWXOQKOgCPgCIwaAl/5ylfCZPdvf/tb9ZnPfGbg+YCks8IKK4Tn/OlPf6oef/zxRs9UPh999NFqn332aXSPXzR2EPjOd74TMguB6+CDD26ccdrs8573vOof//hHte+++3a9793vfne1zjrrhOv23HPPrtcP8gKcVNttt114xE9/+tPq0ksvHeTjhj7tJZdcMjjtCErOnTu3k9+nPe1p1YEHHhh+33rrrdWhhx7aOedf8gg8/elPr770pS9VfEpOOOGE6qqrrtJP/xwyBKxO+OIXvxgIJ71mcRBp9ponSJlf/vKXqyWWWCIkddttt1Xf+973apN9+9vfXk2dOjVc85///Kfab7/9qscee6z2Hj/ZfwRsexor49bnPve5aqmllgokQdrNSMiECRNC+8Y2seRsix8k8iuuuGIksrNQPyNnW4w1UKwO3GuvvSr0YDfR3KntHE/39TrnKp3zdStX3Xnbx0ZLR5XOaerKNRrnRqP+RqOcI/nMiRMnBiI6Lx499NBDjR89DO06l9nSMuXSG+vHS/tN6X3DildbH82wlmM081XXJvo1To9m+eJnMzd85jOfGR9u9Pu4444L98q31sbGHmb92qjwmYtStkiurF//+teDzwj/vPxumWT9sCPgCDgCjsAYQMAJXA0ryQlcDYHyyxwBR8ARcARGDQFN/ts690sz/MY3vrHiDznmmGOq6667rlFSymevwYRGD/OLRhyBlIOhSSbaOgedwNUE1dG55uMf/3j1whe+MKwq8bGPfayTCSdwdaBo9QWiIu0dgRSHjj/xxBOrm2++uVU6fvHIIcDqUjhNWZnq6quvro4//vieHz6INHvO1LwEWBlst9126yR1yCGHVL///e87v+0X3hZG10P8Qk477bTqkksusZf49xFCIOf4H6HHFz1mNAhc9GPGLlbTmz59eiffFr82waVOAv6lNQI526J1QqN8w1gkcJXO+XqB2vYxJ3D1gmQV5uslc/benjp+78aW2X///UMBf/vb31Y//OEPGxd2GNp1KrO9lCmV3ng4Vqr3Su8bVsza+miGtRyjma+6NjEefaNf+MIXqmc/+9lFkDuBa0HYUv7V3FjiBK4F8fMjjoAj4AiMZQScwNWw9pzA1RAov8wRcAQcAUdg1BDQ5H80CFw/+MEPGm+XxzYgbD3DFjhnn332qOHlDx4MAikHQ5MntXUODhOBi/ZMu0Yuuuii6q677mpS5HF7TS7ICmmDwCWkFshHvoJUsyaw4447Vq94xSvCxQSJCBa5DD8CO++8c7XGGmtU//rXv6q99967LxkeRJr9yNguu+xSrb766iGpOhsEHLRy53333VcdcMAB/Xi8p1GAwFgct5zAVVDR4+iWnG0x1oo4kgSufs25bPC5zZyvl7oZBh1VOqfppdyDuHc06m8Q5RiWNHshOw1Du07h2EuZUumNh2Ol/ab0vmHFrK2PZljLMZr5qmsT/RqnR7N88bOf8YxndFZntuemTZtWsZoqcvTRR1esEhXLI488Um2wwQad1e3bvCQxrPo1LmPb3ylbJFdWJ3C1RdevdwQcAUdguBFwAlfD+nECV0Og/DJHwBFwBByBUUNgrBC4Rg0gf/CIIJByMDR5cFvn4DARuJqUb2G6ZrwEWYelzt73vvdVa621VshOv7bjG5ayjed8QFQScYu3ia+55pqeizuINHvO1LwE2N4TMtaiiy4akjvnnHMq/qy8/OUvr3baaadwiJXk2BYUIreLI9AUASdwNUVqfF43XmyLkSRw9asl1AWf+/WMYUyndE4zbGVZWOtvUPUwHslO47FMvdZ/ab8pva/X/A7q/rY+mkHlYyynO97aRGldsP35MsssE27/5je/Wd15553JpOzqUm0IXMnExsHBNraIE7jGQYV7ERwBR8ARMAg4gcuAUffVCVx16Pg5R8ARcAQcgRIEVltttWrDDTcMt/7qV7+qbr311vmSec973hOCoazeEW+/xMR3q622Cteff/751R//+MfKErg+//nPV+utt14I+hPw/cc//hGuueCCC6q77757vufox4orrlhtscUW1fOf//xq8cUXr/75z39WGAps/3TxxRdX//3vf8Olz3rWs6q3ve1t1XLLLdeZgN9zzz3V/fffX5111lnVvffeqySTn7xlhpOQfMRbJy211FLVW9/61pD2M5/5zOqxxx6rWKWDlXp+85vfJNNbGA/yVttb3vKWauLEieEttv/85z+hrsDzsssuS0LCSjCvfvWrA7bUL22CujrzzDMXePuNYPy73vWusM3V9ddfn1wp6fWvf331ghe8INTPjBkzOs+sczCwBdgrX/nKirZPG37wwQfD1pu0sU984hPV8573vJCvfffdt5Ne7ktM4MLRM2XKlLC6G33mz3/+czVz5szabebAjz7Idn9LLLFEuOfGG28MbY2211SWX375gC3XQ1p46KGHwq3bbLNNaOtz584NK3NBYJg6dWrF9azS9etf/7q64YYbwrVsQwY2K620UuhrnKc+b7vttnBe/7bbbruwndPs2bOrW265JaS39tprd/DkGHUKQSIlTft5fC965E1velNYPQes0A+QL37+859Xf/jDH8Ll0mkveclLKtooQvuhPnC+kSd0B3Lttdd26mb99devJk+eXD3wwAPV6aefHlbyASfyympdYIEOyK08xbNe+9rXhq3c+D5nzpyALfdxnGPcm3rLM2TG/Nt4441DH8GhSP1QZ+hS2gjbvt5xxx0hj5pEmVs7X1ktC8IVmCGkBQ4QeOJ6oe74I70f//jH4fsb3vCGirkHbYk0Vl111WrppZcOaVGOxx9/vKLPsdKRpE3/5p73vve9ne39WA2N/k5/+Mtf/lIdfPDBleoEPXHyySeHc/Qx2int+4orrgj9hLpFn6PXV1555TB2oFfI5+WXX67szfdJ2Ri/uB49z9jy97//PWy7x6qMfLeivPTSPtrqS94Ofs1rXhPKzbjE28DgNGvWrICRzV/dd20hwXiHQ9UK+lCEJrCi3VL3kyZNCroQ/Xj77beHlSotJoNI0+ar9Dv95J3vfGe4nTol4EP7QSgrNspiiy0WfmNT/OxnPwvf+ccWddtuu23A+znPeU44jn5Bz/3iF7/oXKcvG220UegX9Ce2cI6FNon+RehHObsnvo/f6Iu27aWtXtXYQP0yTtG3Xvayl1UTJkwI/Rub7qJ5qznmAhs237yBje2GoOPOPfdce7rzXf0XzNiODFsOmwCx41bnhnlfmuoyCCmMC+ikU0891SYRdBo6DkEHovutENRiTNR4Ys/F3y2B67Of/WyoY3QSbQYdSvkZQ+g7KWnTzjbZZJNqlVVWCfXC6pG0aewD9Bu2LngyHiOMbzybFQPQ17InbrrppgobvUTA5aUvfWkoG+Mg2NIfKN+///3vTpK0VXQUtrK1xXTBmmuuGeqRe8in7t18883DWKc2s+WWWwYbij5KgI22yJiPnHHGGWE+Qj1ju2GPc0zSRl+W6L1utgU2cBPBRmSMZzylLTCmM54xN/rd7343XxIl+ZwvgXk/6NPYENjKjNu0B8ZO7GdsHGSvvfaqmuS/dI5XN+dirEF3oC+pd/oQ7Zs2i25C2sz5aBukh93C1knYB8wNmeNi58fSrQ3mbGul08Qu1bX6bNMGuKduTqM0U5/oB+qYvklbop3R1xj3NJ6D0fbbbx9uz/VfTm666abBXkJ3o191f7/rj3zKlqYuqT+2Q8b+lp1vyyrbbBB2on2Ovmvc7OecSmm3KTt6EB2CrkeoD+w3xjbGuDq7Hn8O9mTd2Nu2XffDd9KtTMKpzRiqe7p9NukrSqPXOXBTHdVG7ylvfDa5r5ves+k1tcG4BzsUfc94wryNdiHdgU2CLmYe88tf/tI+ovOdPlDio2mihzoPafClaR2V2r70P0kb24V7us2dla79bNImUuM09aE5Iv4IxmTGFNoE82jq8tJLL+2s+o4/C3sD/yw6meuZU8snZfPE97Zlj+8v/V1K4GpqY+fsBo1XvfgSND628TW10W91mKZskVxZ6whcjC/4GhB0xQknnDCfDTpa7aKu7H7OEXAEHIGFHQHFHvAH4RfCN4WdwB/f9bspTovMMyjS0aKmKQzpdU7gGtKK8Ww5Ao6AIzCGEWACztZYCA7SQw45pFMaSBwf+9jHOr/jN5RwYDEhRI488sgQFJBzn0k7ARocw7EwUWMCGAcFcYri5MkJgQdWy8BwYLL4yU9+MnkpwaNuwSrlk8DJPvvs00kHxwPkLYyPlEBwO/zwwztEstQ1C8MxAoq77rprWPEkVV4cdF/+8pfnOwU5Y911153vmH4QEIA8YIOuOKanT58eLiGoddRRR+nyzifBU4KmEHlse0g5GLgJZwfXKTjfSWjeF9oXgQKcXLTftgQu+g9B3JQQaDzppJMWOPXmN785BPIWODHvAGWCxAIBqInQFxXIJTCOQw1RW6ffUEYIarGQd4J6IhvY89QN/ZugreTb3/526CPkDbIlZLtYeN5BBx20ANGkTT+3aaIbtt5662zfhNSCs5hrNttsM3tr5/v+++8fCJkHHnhgOEZ/PvTQQ8P3j3zkIxWkL/QTgT4CrCmByHHeeefNdwpi1e67757EAbIVDm0mNhCK2H6wm4ggQDskKIMjORbySVqQsqxQF5Rl4jwiVEogR33jG98IOOg8zlkcrtQ1TnU51Th/3XXXdVbe0vX6xOGmLSjb9m/SUD/lGQTHRayhzDhXVSdcS8AZck4sYMRWSh/+8IfDRDI+f+GFFwaCnz1OPyXtnJ4HhyOOOKJD7uNe5aWkfZToS+oDgiiT41jI309+8pMsUTa+Xm9ccx+EJgVeuY5AmPoD7VPEjzgN7kGngzcyiDTjZ5b+tlskWruGsR3SFaI2pmfQR3fbbbcO6VPH9YmjgrHf6mPsI+wkZM8999SlnU+2c2RbRwRyEn9NpKS9lOhVjQ0E7SELo8digTD0ve99Lxk4t9cybmKfIbQzCCHcG4sc+lyD7YVNkBq3uK+tLmNlQPlL0B+2nfMsAm0IxGrKbuVb3/pW0NEEVSAn1on0M2M0Y6qeae+h7NgzjEtW2razT33qUyEQZtPgu2weO+4TSEOPpoTxG53WVKhP2jekmpRgK1HfDz/8cDiteqWfoGNigVQp/Q2+4IyICArZjYCm+hPn6FPoQcYnhP7D2CS9jc2CbYK01Zcleq+bbSGnZshQ5p/VTalL0C+Mz5KSfOpePt///vdXkOdiof9BkiLIirQlcDEOtJnjSdfEcy4C0SJWpvIIEZR6bzrn6zaHwx477LDD5rN/urVB28esbU1+m9qltmxt2wD3ylaCwMS8oJtgN6ADU3Md7qXuvv/97wcbB9uUNsen9HLq5RF8AKTLNegldFC/6w9bl62t0D8pSdnQss24vp92Yur5HFNb7uecinTbll31wb1WIJYx90Jv5ux65qOMr7mxt2277tbvmvpOupWJcrYdQy02qe9t+oru72UO3A0rq6Oa6j3lS59N7uum90irrQ3GPVZforshCqNbYknpslIfTVM9FOch97tNHZXavtKxbW0X8qzxIDd3TpWrSZuQbrPjtLVB0C0QwTkWC6R+5o/axt6ex05m/mhJa5wvKbtNt5fvJQSuNja27QfWbtB4VeJLKPE1lei3OlzV9mz/zZVV8wJIb/IzkDY6HPuevCGM27zYIRnNdqE8+Kcj4Ag4Ao7AggjI1+EErgWxme9Iyjk43wX+wxFwBBwBR8ARaIkAkycctwRE7ISdZHijXm9n8puVeE477TS+BtHkl0AVW4ogmvyHH/P+4eiFzIOjgkCBJv38/vSnP63LwgoDH/3oR0M+uAcHFgEdHA4QTjTJU3ADcg8EIhwnrJ6CEEgiXQhcWlGo84Doi/Jpy8w4i0MLLMgDKzFAMuMZL3rRizoOKFaXYpWahVXAg0Cpgmg48Vmpg22rCL7puCUt2ZWqwJaVSMCXN7AIMOgeVm1iFQJkEAQuG+AlHwQR+SRIqTzwbIJTbQlc3IfQDikb7dYSGGkzdmUyS3ghDwTRaY8EmVmdDOH4d7/73a7Bc67NOVDU1rkG0bOor1RwlvqkDNSL+h3ONwI2Ejmv9Vtp4qSl3MISZx0kOwnEhDb9XPcR4MNBK2HSgC6gjSgozzlIfmCnt0i1lRoBe4R84zSTIylF4AoXPvmP5/DWKGXCsSyxWwjG/QEsaFdcL92k+1LBJ52znyII2GPkg/ygR1UvPAtSGsclIjXym7LS17geop3wQE9yHXWN7PRkoIfvpKn64/tF81bgYWUt3qIWwUp4nnLKKaFtlvRvniVHIN+tiEQhR2d8DvIA5VE+dZ6xiL7Hm5sEHiQQCoQRZaD+dJ5+h3OR9s5qLyojz9DWg6STyktJ+2iiL1k5jS0rJdQ9mMQ6heAIf92EcRdHKnhB7MSZLLHOeR2j3gnu015sf1ZgkOsGkaae3+sn9U8fUvsg4I09QZvXMQKclAehn9ImFGSiXhn76cOMUWor6EGIQCImDYLAFeuTJu2lVK/GYwNYQPDkmehV6Rl+W4I016VENiHn0A3x6neQi7DbENlyuXGLa9rqMquHbMCEtGxgGL0IYUUCfnvssUf4CXkXcmSdxPqZ/oKdi92AfpYO4Tjtir6LlLQzAuwQi6XvaHs8i/YModDipzzTTum/tCU7xrPqDqvONREFd7kWvFjhFvtk4jxisOx4dC3lQxSooe+UELhCIuaf5hUEchifYgFbVpqBxFyiL0v0Hi+81NkWjKt1wupirJyAgCkrm9K30DHYMtJNkKhZXQ0pyWe4cd6/D3zgA2H1Lf3GHoPUSFtS39a5tgQu3ae2T9nr5njSNXbOZVdLVBtjPCZ/BIklvFiEXuo254OoBmFNwj3YPxBj0GfS73Yc41rb1nUvn2qDto9ZvdLGLtXKaiVtgLzIVrJBU47n5IMf/GBYOY/z6APmaOgnAqcidYE54xltEPKyXkBJvaTAKm60J0S6u9/1F+tH2hR6h7m5naPF9ZeyzdC5vdiJoaCZf2rLOk0foN32MqcqKTtjFnNMbDSEesZWxP+BH6TOruflC1ZDShG42rbrfvpOupUpxqmpraa6Sn227SukUToHbqujSn1dTe7rpvcoZ1sbjHusvuQ3Ql9k3khbwT6XxC9cYk9wDUK/oh/zafs/56yPpo0e4oWObtK2jkivxPblvhLbhfs0HvDdCnhRrylp0iak2+w4nbJBIACj8/ALoEdiIcBLnVu7BnuHl0EkpWXX/b1+2jpjfhC/VKz0U+25iY1t77N2Q2q8KvElqH908zWV6DeVPfWptmdtkVxZNS+wBC5IaJC0ZYOyqpt9eXe020WqzH7MEXAEHAFH4AkEnMDVsCXImG14uV/mCDgCjoAj4Ag0QsBOYiGt4BhB7IoF/I5XLZADy07MNPnnepzCrGpAIAlhksm2igpu8TYWgSiEN3EIDCGsiAOxQsJ9OCQIosaBRK0CwrWswhKvRqM04k/l0zopbFrxhNIGVa3jKE53YfhNEEWrPNi6p+wEBxQYxcEB6ccG0wkYEDgnmCBhBQ7ITAgBEwgpBJlw/PRzBS5WXmEFFoQ6xGFDABKBuEEwXm2zaR3bgDHp4ISg7Ujsig3gAWlRzkiCnATscISBiXUesXUpW6wgYEU/6iY5B4raOveTBxwqKnecfxz/Rx99dOdRNlDN9pLkFVHf5ztpokN0Dgc7/VxkIUif2qa0tJ8fcMABHSchq+vh9JXYgJh1EEIqxVEE3tStxDojcwQu7mGVLQVRudfqQ5sHVn7SFiZsFUBdcj/CNhAESBSYLSFwkRZtlaCZxOpsu8IQz9NWODhyuU+r0FBuyqDgnSX/2EAPzyBNAuM4FSUQinCsITbYW9q/SUeOQL6jG0488cSwSpnws45OjkHQY9UVxAYU+Q2h4Wtf+1pnZQ3btnmzkzc8EQLw2upNq7aFE0/+g6gCcRhh3AFHJM5L0/bRVl/yLNveY+In2y+wzQxtKh4PuTcnBMYgqtgxj2ttf+A35E0IT9IRdhUp+jh6QDKINJV2r59sD6OV5CAu8CdSQKzndt5550BS5Jm8YQ0xRoKTmf6mfnPllVeGdsr5QRC4StpLqV61YwN1yzaQIhpg8zD+0j6Qr371qx1bLhxI/LNjUGrcsmVDv6AP7T02yFGiyywRy+p2xgG9ZKBs24ANW+qy4gLSpJx2XCRIRX/R+A1JhGeprTEmSM+VtjPyBemYuqB/yi7iuMWP37FOs3atiBdcVyfWVkLHEMQVOQk7nJW3RGrEXsOZp0BNLwQudDwrJjI28VwkJnDRj1l1y46HJfqyF72Xsy3qMOWcMGKsA1ONzZyzbdT2ndJ82nEZXOnbmh9Rd4zhlvxux3TykxOrM6ijpnM83WfHH+HIsyBp2eC6rXfbl217jud8zCmxPxG21aYtSSC5YP8oaIidqxd9LJEh1QZtH7M6yrY7axPyzJxdWtIGSE86xAZNOZ4T6QvaGvMO9JTE2nuQVSGt2nGeeTlYWrEEL+w0xsF+15+1sVhZxm5LzBwN20PzCtqTXiSw9/XLTrRlj7+rLXO8X3MqW4Y2ZYfYgg5G4vmFrWfOx3Z9v9q17ZP98J3UlamXMRQMUtK2r5BG6Ry4VEdZjGO9lyqTjtXd103vldhgPNe2K37fcsstYbU/6SDmxFyDWB+StTva+GhK9FB4eOZfSR3ZMtvxW49I2b6cs2NIm7mexgPSSM2dOZ6TujYh3WbH6dgGoc54vurT2sM8k5XQtfIs8Uv8QdjFsW1YWvZcudoet34UOx+I07F1y7mmNra9z9oNVtczXjX1JZT6mkr0W4yB/a22Z22RXFll76if4+fHbyAyffxCGc8Z7XZhy+rfHQFHwBFwBOZHwAlc8+OR/eUEriw0fsIRcAQcAUegBwQsycQ6EJjQ4uxmgkmwGCeByDmsCMAkFLFv62ryz/Fjjz02vCHPd4mdgFonFA4YguYEp/RGv+7hU9vIkBdLxqhzRNj74+/Kp3VSWNKMJTfoXra7o9wEXexbZDq/MHwS+GFCjtAeIPzhwLZCsABnOwJJaYcdduisAmBXFrD3sM2Ull0/44wzqpkzZ/adwGXf7MT5yipvVuxbXyUErtw2jziv9MYpjg8CujYYYrehs/mRE4M2D6YxzvZavuccKGrrXGP7HL8tCYZgHP2M50nslmOQGiA3INZ5TfAaR44Vm64lfpb0c0vwSwWWcAThoEJXgZFWCpNTN9YZ1hlpA4PWqRYTPCjbBhtsENoy3xUosWnxbNq7nJpch9i2rfueOJP/bx2irJjCyilWIFfgfKfsVi9DPoDwSplJA31qhTdlaVcIgXCCyIgN9LDiAUSoWHIELrvKR5v+TfpyBPI91Y5snVxzzTXVcccdx6Udse2Qdq6AHhdYModdDXDbbbcNxDowom9akhr3cX6TTTbha9DzkAIRm5em7aNEX66zzjoVYw2Seg7HbTAgNc5yTSysroHeQSxJ2rZhMKFNaHKuNGxAA7KQZBBpKu1+fFrdq/TQczju1U8pP2MaNg7H7Kpruoc+Rd9CrI7pN4GrpL3Qfkv0KmWxY0NqHLK6i2A6geU6QR9pKy7I2NiLtCmJbEqLc27cKtVleoYdw7WaLHkRmdauKEsAnKAxbaPJSmNWP1tyqMpJewFbnqVVfHppZ6SrIEwdgcvqdOUFUrr0eXyvrok/IWgxxiApvWy3fhZRRoGaOEintLttoch1F81b7ZFteKxYIg8kQ0g46rtcx1bHJfqyF72Xsy1svlPfqQfslBtvvDHMj+Jr1HZZOUrbkZbm047LKVyxBdDr6g8lBK7U2JOb40nX2DmX1c8p4iTjFe2QMZgAKJKb8zFuQtpGckTFqVOnBvIz19jgoyUypLBK6ahSu7SkDZBf2Uo23xzPiWyjeLzjelZN0da+zLUgYyHqw+jJeM6htom9CZGKa/pZf8yRSA/B/hApKRx48p8leLD6Hu0PsbZZv+zEkHDmn9oyp/sxp+ql7HVkp252fb/adb99J7ky9TqGZqqzM59t01fUv0gzNUam5sC96Kic3suVScfr7uum90ptMNuuUn3Z2rnWJin10ZToIeETf5bWUYntW2q7kGeNB3xPtT+O56SuTUi32XE6tkHwc1jyuX1ZKkVeE1GKMUP+217KnitX2+PKF/cxvukljDgd257b2Nj2vhyBKzXH77evSbqqjX6LMbC/1fasLZIrq2wKCFxgsPvuu3fIWxDsmT9YGYZ2YfPj3x0BR8ARcATmR0A+Yvw9+DTwI2AD8cd3/Z7/rvyvReZNzv/fU5i/bsydcQLXmKsyz7Aj4Ag4AmMCAess0+oTdiUl3jbC8Y1okmuDMUyCNZnX5N9O1C0I9q1kSxaz1+g7jnvywUpEbF2CQRCnW+eIUDqpT+XTOinspJl7mMwT5MNRbANGqfQWlmOWFJEi04ADb7oT2Cdwyao3kLzYwoS6IziawhLCBMQJRA74fq/AJSeGdRiGB5p/cjbY4K85vcBX67giMAP5JRa7EoycOHJUKrgb38PvD33oQ9XkyZPDKbtaQOpajuUcKGrrcd/hHmxLESZTxB1WKmLFIsRuiSUs6wLeuqYblt36uV0dJbd9Ke2S1SSoW63QlAuyWmdkjsAlPRcK/uQ/VuAR4Ul60vYH3jI+7LDD7C3hO9sjiRRSQuDCOc3qUrGIdMBx+hXBba3UxtYGImrF9yk4b9uDDfTkiBo5ApecoG37N/mSIzDXJ21gzpKOVCb1V0ti0zl0DnlDZs+eXR1xxBE6tcAnJAfqCYIu7Z0JKQJRN0XgKmkfTfUlW0AR/EHAB8JnLHbLkKuvvro6/vjj40uSv6UL7BZEtj+wWqaISjYB9SWO8R29JelnmtgZWs1O6fOJDsFeaCusggQxAdtBAgmQMUZi+3Bd/0S/005sv+k3gcvmpWl7IT8p6aZXuUd1l+o/nLf2VTd7jesRHPSTJk0K37VSCz9sQNMG31PjFk6gUl1m2yokBFaO0uqJEDwJlkOwErGYtsGz+LTjQShA5p8IXDncuI12h/2KQCpGv/DmOdK2nXFPEwIX9haEslgg1bFyTbexWPdp7M7ZJwRgCXQiENcJakkXlxK4aMfUXdyeLYErhZslKrXRl73oPbUxqwuEXdtP5l+0E2zgifNWJ0ByBK42+llBbfII4Ya2Gotdua8tgStX9twcT7rGzrksQZS2xjgNYZ1+mBOrkyx5xtqJ9rhNh9W5IK0h1uYQkYEypdpgSkfZ57WxS21+9L1bG+A62Uo2aKr7U5+WdA3p+KqrrgrbJ2tF7NQ9bFPO9nmIXqThO/MQ5iOI1ZH9rD/m+DvuuGN4Rm4bW0tGZatA9DAyUnZieNi8f2rLqT5QMqfqpezWfxPrx252fb/adb99J7kyWfsoLqvqhs+UrWbPx99L+orGyDZzYKsz2uqonN6LyxL/rruvTu/1YoPZdpV6AYk8yl6wNokwtbo5Lk/qvhI9FKer373UUVvbt9R2Ia8aD+qwUpniz7o2Id1mx2lrK9lts5Uu20Iz7iMQghk7rGB/aEVavQDUS9lt2r18l++CNFJzeqVt23MbG9veJ98fadrxKvXcfvuaSvSbyp76VNuztkiurLa/Lr744p25eE5/D0O7SJXZjzkCjoAj4Ag8gYATuBq2BCdwNQTKL3MEHAFHwBFojYAcXhCxmNTqDXuc/hAFRAbQqgUKYMVv12nyb1epsJmxE/04IEhg7R3veEcIYuCotUFXpRE7TOscEbon9al8WicFQXvIRjgPY8FJwlvzOCe0tVR8zcLw2xKt4m0ecuVX2yKQSkA1JZZsoaW2+0ngsg5h63SI8wJBB+eJdSrG19jflsBFMJ/2GYsNXGt7CZw2Iomk7iEN2/5ZgQlHaJ3kHChq66k+aQNaIiXZZ3QjcLG9HE7glMhpFPfZtv3cOkW17VbqefGxXJDVOiNtMMo61RT4t2naVXhE1rJvC6beJuR+6wjPOa3sc/gu/RpjZ6+zW4lANCKQQJklTdoVxALahQ300AcIyMeSI3CV9m/SlyMQkpK+2+faOmFcElFY1+jZqf5qdUpM4ILMwBi35pprhi2XqKOU5AhcTdtHib4U2YT85OqQc9IPdgtNjteJti4hXeqeNmP7g92C1KZjVwyMA/39TBOiI0SXWOqIMvG18W/bbjW22Gs23XTTaptttgmH6vSsDUJotZh+E7hK2ovK0lavcl/d2MB5q99ie43zKWF7ZVaIQ2y/tnpc+HFNatyy28lxTa4fqA9wjXSZxVBbuWi8xX6F6AtJUG3KkhNSq5CRdizSz7H9a6+zq5KwQsJKK61U3M5ItwmBy5IubF5sEAUbt05yqzTW3cM5PaOUwJULhFsCl13tV/kp1Ze96L2cbaE81X1CQkBnMj5BhEtJjsDVRj+rvaTsPj2TrctZSQqJ9bquiT+76YzcHE/32TkXGEA+py6s0N8hWEK0pc9q+06uyc35rH5Jjc9KX2RGux2wiAy5NpjSUfZ5bexS8tG2DXCP7KO6+QvXSahXXrSyOpJzlJtVUngxS9tK657llluus4KtCK6cs3aY3eqyn/WnVRKVl246384nbf76aScqL/Gn2nKqb5XMqXopu53bxvOLbnZ9v9p1v30nuTL1YqvFdWh/l/QVkY3azIGtzmiro3J6z5Yj9b3uvjq914sNZtsVK2qyqmEsWt1Lczdb53U6LuWjKdFDcX70u5c6amv7ltou5FXjgbWxVYZun3VtQrrNjtPWVkrVzWabbVaxkwNy6qmnVvi5rNi5kwhcvZTdpt3L9xICVxsb2/aDHIErpQf67Wsq0W91uKrt2baQK6vmBXF6uXnCMLSLOK/+2xFwBBwBR+D/EXAC1/9jUfvNCVy18PhJR8ARcAQcgR4QsG+w4VAnMIkzV05ckTH4zYQMJziO4Xj1D03+c5OznHMfxxwOgNjZjBMXBw8TWpyE/CZvkjpHhK5JfSqf1knBdQrs80YsJLKUnHnmmdUFF1yQOjXuj9m3E2fMmFGdf/75XcssR2fdm4LWQaQgexMCV84BGTsYbJCWFZqOPPLIZL7lPJBTMXmROdiEwGW3G8WxRSAcTNqItimquyfnQFFbT/VJG2yISS48qxuByzpw4rxphSjbZ0v6uXWysXQ/5WgiuSCrbWs5ApecjPY5KaeaDbykttLi/l4IXKkAkfIE2XX99dcPP2nPrLhDwK6NKNhlAz06FqdjiTA22Fvav0k/7qfxM21gLlUnInCJeGzvx6lPWRDbtlntZNq0aUHX2+v5Dt6IdH+OwJXKS6p9lOhLjbUhIw3+5YL6qVvp76zAxjirbYJtf8htA1tH4OpnmoMgcNngQmyvgJGto7qVDm1bFAGpG4GL1UxY1QQR3uFH5p/NS9PxlaRK9Cr31Y0NnC8hcHGf+iWr6qArEGxGbLjYDkiNW3aFuXBzg3/SW7YfMtafddZZYQUikoDIxUoAbOus37QP6olxKrdSUbjY/BOBK7Wdii6z+hk9wkqyG264YTjdtp1xkwg5dfjlxiAFUZrYNXaVE7tSn8qV+9QzUnYG99jxA7uNwDciGy431lkCVyp4Vqove9F7OdsiFKjmn11V1V5G22OVTV4eQHIErjb6WeNyamzUs639Ysd0nU99dtMZuTme7ovnXNj59EdW7dVLDfa5EC0POuigQDrieG7Oh23I6r/W3rTp6LvyIQInx7u1wZSOQt9gYyBt7NKSNsAzutlKXBMLmFLHbGkfz625lr7KCryyezhmV2jViskivaX6aL/qL4cLeUqJzYsdm1O2mcajVF/I2YmpZ+qY2lBK15XMqXopuyW+1BG4ND6qDHz2s13303eSK5O1j0rGUFv2+HvbviL92mYO3IuOyum9uBzx77r76vReLzaYbVc5myQmcPXqo2mrh2Kc9LuXOiIN6Zomtm+p7cJzSsYD7kPq2oR0mx2nu9lKdo6llyaeeNIT/1MErl7KbtPu5bsdw1MrYSntJu1Z9q+1se19OQJXaryycxi9LGhttVyfqvM1tdVvKnvqM9X2cmUVLkoHu0t23qWXXtrZGlvnh6FdKC/+6Qg4Ao6AI7AgAk7gWhCT5BEncCVh8YOOgCPgCDgCfUDABkmYHOJQZ5LFm3O8QScHI5MvtmvaaaedwlNZWQAHlkST/5Rjk2tSzn2IBziR5WC+6667wpvXrPCk1a5wJC+//PILOOfrHBHKU+pT+bROivg6VmrgzaU11lgjkNmUP66DnCIDJr5vPP9mezFIPQjkLYLM3UQTePvGdHwPpAoFeq+//vqK7QWaELjkLGNbRm1TR9qxg4HADo45pO6NRW1taJ0w4ab/Y+/cY3Yrqvu/m/zahLYhtrVU8MLBGyIqKWgBJQjeUbkoKCioCIhgVUKaeqEYS1tpTaotXrgLyKWIwUtFpNy9oSBoURDQqkiJGiXR2sTYP3/nM/B9+J45M/syz/O8533PWSt5372fvWfPnll7Zs2atb6zpvLPAVw4WAGf5bTXXnt1hxxySLr86U9/OkUUYMsmjC2s+KcPDdFPf/rTDbYtK6WvGVDU1kt9ssXZwLtlvO7bqk/vlaOstZ9j4Fr30NZCAk6U6p9fqzlZ3Rg5L4DLAQ61lcYYzgDAQLmDJS+zfgsgUNvCinTutKLt4UARuJV2eOmllyq74pFIDNpOB3nOGACVHD1cdwe8O3tb+zd55v2Ua05ex5KhU/1/imNODkreAwADUA+RPuAZ/AZIjEEamhfA1SIvBSLl/fCH79RHACGQV2NJY6nAJ94fpgAE/H2LyhOZj0zKydtqfm/otzsXSgAu10n6InCxUhpHOEQ/E1CAyEpQqX16NKgxAK6W9tIqVymzZHRpbOC+y7exEbh4zrfYJjoNYx1jJZTzoTRuwdNWWcY75JikXmwdxnfQOOTbcBHdB7m39dZbd7UtK8kvJ8ln5Znf57fLLqKDUU9tMzO1nZHfSgG4iOTG9nuQwPTpx8A/jQO1tuSRLdCzpNvrWzkgw1/lc5MSgKtVXs4j92q6hZc7P3eHOLIDxxxtk8in8AxSpLhFALjUXmpRpXifb1fjYzr3ajQkM1yeuszQc31zLnQltuZk0YPAbJTD9fHanA9Q9g477JCKXYpqofqIxxr/uD7UBksyqkUvbW0DlHFIVyJNjdANiTYKUBWApo+xrgPzvDuK0WnR8Yn2CpXGz3TjoX/zfD9F8iQr5HIeHczfwzl9hm0UIZe1pXG4RU9MGVf+qS2XZF3LnGqeutfAThR9SK9fVLvO2TSv7aRWJ5ctLWNoXs7S77F9pWUOPI+Mqsm9Uh38Wt9zfXJvHh3M21UNbJIDuBZpoxkjh5xHfj7PNyKfKbpvq+7Ce+YZD/rahGSbj9NDupLPscYCuOapO/VfBK0lAJfPxeaxNY2Vb338LbU97/MOVtO8AJ338ssvT4vCmQtAXEMOsDBctBrahcoSx+BAcCA4EBzYmAPyfzIHwleMfxQ7H3+c6/fGT5av/M4ee+yx8f415bRr6moAuNbU54rCBgeCA8GBNcUBBl0M2wy6AKiIUgAJlIJRXQ44nDpsc1RyXmnyXzJskp8b4GTcx7l20EEHcTs5Nc4444x07v9kgJXTVPf6DBFKUzqqnG6k2G+//RJoiPrl29WhoBC6HecWVAoTXnrP5nbNo0nVtu7yaG6AJY4//vi0LSXfDoNJCWzgBuxrrrkmRc14xCMe0eEohUpRZtwROwTgIg8ZXP2bc12EkQpjA33AHUa6Xzo6gMuNFp7W+UEkAaJaCEQCT3CcARzJia0X+YNwXNKn+qhmQFFbL/XJFmcDZRAva85BtmFiJR28lGO8tZ97JJOa0V78dCdwzcnqxkh3Xg05gUqrInEYYvCFauAsnPbIvb406ab9E0CAS+9973tT1Bq7nU4FNhTICxmFDIf6tvV4/vOfn7YRox/AT+jIOQBcHmltSv/mvSVDINdFQ99E48JYABdgF9oKRB8H1Jn3PQcazAvgapGXbO3IanTorLPOStHD0g/7h7PjRS96Ubpy6623dsjiseRRoRjfcdByhFoBXMvIc2x9htK5c6HkgHaAJdskn3vuuRtliRxTBCkfazwCF1vT5WObb3OaA5c2esn6Cy3t5WlPe1qz/tQ3NlA+dxpIXyuVO7/mDlgA/sgmHIKMd9q6U8+Uxq15ZBn5+phB+95mm226n/3sZyk6AvclOxmb6Et8X7bHZpwdQy6ftXVj/pxWkks+z9POyFuAnGVH4OJdGt9roPvnPOc53ate9SqSzgB5ctT4GJwSPPRP/ODnIgFcWlxCvlPkpesBU+VeTbegDDXycaXUl7DzwRdoEQAub6O1bZG1HRXvXGkAF45EIupAAKgBsjmh/1M+5qYQIGH0zdqc77DDDuvW24BT2ksuuaRjXMzJQQnMc5HpUB+QgfslGeUyZqxe2toGKMOQrkQaEfP3vffeO/0kAqGATrrv8/l8ruPtEJkJ0I0tZyHGCzlbF/39HNwGeAvgR07IaeQO72YbyK9+9aspyaL1xPy9+e++cbNlTjVP3X2szecgQ3r9otr1om0ntTrNO4bm35HfrX1FY+SUOfA8Mqom90p18mt9z/XJvXl0MG9XYwFclFk8nWKjaZVDziM/n+cbkY+33SHdt1V34T1TxgPSO/W1Cck2/wZDupLPscYCuOapu9dlnvO1BOBqtTW1yrc+vpbanvd5t4VqXuCLQY477riO7Uah3Fa1GtpFX93jXnAgOBAc2NI5EACukS0gAFwjGRXJggPBgeBAcKCJA77yhQwcoIWRBAcSBlTRfffdlwwu+s1Rk/8SWIT7JQDXgQcemLYA4v4tt9yyUeQYN9r1AbjOP//8jqhdY0jldCMFUQcAaOFww5nBu5zYpoiyQjWDvaffHM9/7/d+LzlBaQfwh5XuOOxE6CoYBbkvw6ZH7rn55ps7DDxOOGgAVQD6gdy5J4MeyiJAESd3jrtTnTQlA4NAPtwnwheRvpwATuCwhnKnhqfzcwdwUUbANg4IAfRz6qmnJicUZcQJBd/ckXPttdd2V155pWebHN6sTMNwRn6EoPd8N0j80I+aAUVtvdQnW5wNvE7fhXMirOA4cDriiCM6tiGFFCa9tZ/71go4ov72IVCf3sc2gjjTIJw6RDeD5GTl3FfluzFyXgCX54W8pD/wnUXIE9ot8hPKHSxKlx+pIw5MqPSM80Tb3JJWgCbOkdcYzZw80p07iYccPeTh/didvX59av8u9VMv76IdcwCPFamPCDBymOudALzgvb7XvACuFnnpQEd3MquMHH1Lio9+9KOTAFw8LzAIW8wxbvIbmgpkSA899G8ZeXr+refuXCgBuAACI2sZs5CxAHK8D/NenOFEJoEcTOyr7YlMSv4itmYC1CWdaQyAq6W9tMpVytk3NnC/FcDFs5JhyEWtynP5TBqoNm61yjLydKAGvyEHaLnMevDugyAObeuna7Wj6sb9Up/BOYGTAlIfnqedkY/610oAuLx+55xzTqojZRCVwFh9YCC2jhRYhzxcx+tzIpN2KAJXq7z0sbv0DXl3bevYmm7BMzVCrrAtPcS4RyRYJ4ApAOMgH5tby+mOsJIOkfcRH9O9XPn5kMwozfHIQ89pzuWLhmqRXF1PxtkKUNudzz7ncxBxaWynDP49fVvyoTZYklGug43VS1vbAGUf0pVII9pll12SvsZvQE5EvshJ8rUE0nRQPmMiYA6PWEZei/5+HnHZ50pebt/S3XXNReuJ/s7SudryouZU89TdASN5Px/S6xfVrhdtO6nVad4xtPQtW/tKyxx4HhlVk3ulOvm1vueG5J5kBPmhf4ydT3q7mgLgarHRtMoh55Gfz/ONlI/0pyHdt1V34T1TxgOVS8e+NiHZpnGaZ4Z0EJ9jjQVwzVN31WPe41oCcPk3mGJrapVvfbwttT3v80MALuyizCHQKyCfG6+GdtFX97gXHAgOBAe2dA4EgGtkC1gUgItJEdtQYTgS80cWIZJtZhygTaF041xgtRtbNK1GQtHDsU452c5t2eWkj6CIQhiItMXQauRNlCk4sEgOeKQY8qWvsXpGJCOdfms7OP3mqMl/ybDJ/ZJx3yeYgH4AYNDvAPTQFzE2yAkK+AXnicBVPmkEjAEwB4P0EKmcbqTw7UTuvvvuDseVQDNsCfC2t72tw8EPYaDJVzUPvXNzue8rz9EjmMzjMAEMBFAGxzUED4mIwHUMc/qGV111VXf11VenNMh3nDSK+MaKRd9SUAZGEt90000pQgbGbsYuVi2LMPgLmMG1koHBV53zXXH+3HHHHSkLjAY4wlXGFgAXGeGgxlnCke22aDPUH7r++uu7K664Ip2zPQxOEt5HWyYiw9e+9rV0D+AOzzEWQeJj+tHzz/uCG1DU1kt9chEALnh54YUXJkAcddl///1nW9DxGwAbkU7m6edyKFF9og197GMfS/38cY97XGo/AB+gyy67rPv617+ezt2pQ8h5HFlsxeaGsHkBXLzoqKOOStvTcI6D8YILLkiyEwAbUZL0/bmfO1i4ViIZgHWP7Ur5Q7bBxze84Q2zyBQuh91RDrASANL999+fsiG6zzHHHNOJVx5FcMjRQwYOenBn7zz9u9RPVWeO/g0dhKc0cjCMjcBFf2N8UT8XkBNDIlsMIdvEH95BO5OMGCoLsgwgEMT2WIokOVVe8rwDJNhmi4gi9CXKyXcg6hKU19u/owNWUmL7J+AphmCA2xrnpwIZLMsUoRMZu8g8Pf/Wc3culABc5Et/wnEDMabhrMJhDdEu4LnaDGMZAAvInT30tw9/+MNJd+E68xaBkknrRmp+12hqe5lHrvaNDZRvHgDXi1/84o7IHE7en3S9Nm61yjLlS3QdATG5hqwgogzEN0Vui1wP1LW+Yy6f0U2Qz+iE5O3ymaiIksGt7Yyy6FshB2if6MiMvc6/mrNUq+DH6jVsG65t0+jPRKVDksxcxwAAQABJREFUD6EPHH744TNwtssfHx9w9J599tlJFgCwZm6h/kNdFgngIr8Weel6wFS552OB6xaUpUb+7QFdo+fCP/Q8dE/ajchBeq3l9MUU5Mt4wJbntJl167ekRu/2/uFjuspROqodlvRJ0pfmeFzXc97XaAeyb6IboyOLiD5Lm8JBr8Ug3PP2ns/5fI6KnnjmmWemNkgeRHlWVKo8SpzmGfl1lcXf6br1VL20tQ1QjiFdSWXl6EBgdF7mYYrSyTdHL1UET+zBylt5EKUVPd6ptNBk0d/PoxUD6kNX03yedgWACzmCDOSb/epXv0pF9P64CD3R6106V1su9YHWOVVr3dExmGNByBPKhpynn7s+KACk12dR7XrRtpO+OnkfmqKreb39vLWvOIBr7ByY97bKKP9Wudzz+uTnfc8Nyb1WHczfWdNJmCsxZ3KdpNVG0yKHcj7579ZvpDzG6r6kb9FdeE4yO7eZcW+I/PvkbUmyzcfpIR3E51hjAVyUsbXuLtf65rlDfFhLAC7q0mJrapVvfbwrtT1vU64jae7hEbjI29MzntPnZFNvbRd9ZY57wYHgQHAgOLAYDghDxBwIuzTzMubZ/HGu32PfFlsoFjiF45lBn20ChHYmGQMmE3qcTSjQQVsWB4ZWta4Wbnj0ndK2A4supyuVroQu+j19+RG1A6MufZTttjYlLbosgBowFEMYDzQITK0j8gxjHg58AA8y8jIBYBsUDPJB0zjgq0B50gEn/JbTl3OotHWMJv8lwybPlIz7jEsYIFldKcLo7M4FDOu6j0OJb8w2h24s1bMOaNC1/KhyupECcBZtkrYE0f9wHFA+LwvjJlFPtlTCmMNk253TGDDFN/jC9yIiA0fogAMOmIF6+A1vecZ1EkBYGPUAg4k8KoGu+ZF8UBTHALh4jghYgNhFPA+RB6T83KiYblT+5X1CyZSPfgMGgGe0XZFH/OIazwhgpDQ4L3ASUb8hqo1dauulPun9pwQU8xXvDo5y47XKlfOS6/RTDGzQPP183XqH49vf/vYN2lje5vKIgM6PVID1/zDY0adxgEOLAHBhIMOALPmUMrZ/fHO181YAl7LL25VHA1Iaj/LANbU5lYFrudHXDaIlRw/PuIM+d/a29u+SIZB3iZbhmKMdPf7xj9crZg5eyQDGhK222irdh99EqQLMO1SWGoCrRV4SwQeHnspEOZABtDUR1wCHEOFH5FEJGR+1Rabu64h+iVyBACBoC+OpQAblx3EZeXr+refuXKgBuOi79GHnL/0G/vu4lm/txDfH2Ox9y8vpfX8sgGtqe5lHrvaNDdRjHgCXOw7Iy0EY/Ba5nM7nXC2yTPmytS3bjkB8B807+M031Zbh/K61e+6VKAdwKU0unwF/AloTtbYznnf5w2/prs6/mrNUTpSxeg35e7QiflM3yGUS8ptxF3LQV7qQ/XPe0Nek5w05kcfYKlrk5ZBTkuI7D3zMc56rmrVxU/eJ/kjkRPGP6y4f4I/rf8h7+qdACqSvtdNaOT06H8/zDte7/Zt4/UhboyGZUZrjkZeeU7vlmoNP+Q0/qDf9xPnkEY9dZ+UZSHM+7EX0E3+WPOnvukadAUQDjBYNtUH/3i6jpuqlrW2AyIBDupLqomM+N0H+UnfksvMC2SBgq56lb3Dd0wH0FphK6Rb9/RhPkQ28X0R7YP6tsnA9B/u5bFyrAK7WusMP9S3OIdkohvT6RbXrZdhOanWaZwx9kDsb/2/pKy1zYN7cKqP65N7GNXr4St9zQ3KPXFp0MG9XNZ2kBODifS02mhY5xLtq1PqNlN9Y3Zf0LboLz00dD3hG1Ncm1O98nB7SlXyONQXA1Vr3sfNc1bd2XGsALtpVi62pRb7VeMb1UtvzPu86kuYeOYCLfE4++eTkm+bct7hvbRfkE7Q2OYBvENmPrn777bd3F6zHJkwl9EQipbOADn0KnRH9n8UA6IzY8kS0VwDKU4gFsYDkRVPep2fy4wtf+MK0mAL7PD6CIZrqQ52afuj9897nm1Bnok2jNzIfZYE3i17xFbT4hFvzbH0u5wEymfZGVGdt656n4TeBClg4wxG/NfMx5lS0TxZU5/Mw5UE7w1+E3Ro7K+Mx8yJ4deutt6bFoUqbH9k5CP1om222SXZt+E0bBltw8cUXj/It5XnyW98pAFwl7tg1rVCzS6NO+dBs6eKT4fxBhBuGzGVHNsrfG783LQfGGEU3bQkffPuWCODS9gwI9xNPPHGTfoZFl8VXz+EE0YqLKZVkGwqMzQxqNRrrKK89v6VeB0glYA4GFo9Ax6RK28L4CnHnlZ7Pt1tQmppxn629MMLKca70jE+sGr/zzjtTpBA5U7/4xS8mxzPpfOUqv0lPNIQ+UjndSEF6Vn3TRmtt65577knRCHC0bMmETkGUKACUOaFLENlGUUp0f8899+wAZOkb6jpH0mIQ9cmR7rvM0DVkI5GWiPbFGIEy6xG4ZFyVIVvPobADaOc754Tijc600047bbAqNE/nv7VVoNopkSZyZz7RuE477bQ0kfRnOe/jCQYPgCNjZSQTIvgLuQFFbb3UJ92QlzubyccBXCj7cnqJv4BHMKgr6hrPQHyfG2+8MQG4Hrzy4P95+jmLEQDfUGYn3kXZ2cKMdiBiQoVBgGhnIkAr9Hmis0AO4EK2IeOgkhOI/JCJkEdY4jfyAoATbUftm3Lx7egLAg8QZY2J2hAxNjLBw+mG0YEJGG3XibZNZIWSLPIoPv4MZWLrULa9ZCIn8j7GnCF31JGuD8DF/b62XOvfakd5PyU/aOibKAIXk0kBkh58skttEgMt5G2b9opeQ3tygjeMM3wvoiypncmwOFSWvvbRIi/pK8cff/xsPPay0pdxjDAeORFhTdG5GAMZC2sEv5B3RIMRL9iCmMiEOdUAAnm6ZeSZv2Pqb3culLZoVn7oPYxp2uZM1znSNgC6AYrLifTo6RjXndCRkElEuoHGArhIO7W9tMrVvrGBcswD4OJ53+qzBp6rjVs8D02VZQ8+1XV77bVXd8ghh6SfJeeBb/lXigymfEpHtkpmwQOyAdlGn8vlcw1A2drO0HPo38gvSIA4519pO2PSyokyBcDFcwcffHBaKc+5E3oa8idfKLPHHnt0hx566Ea8+PH6KD+kRZ+AHMCF0ZqFQrkurvd5tDQBdXTPj1PlJWO29ICpcq+mW8io6eXyc74VPJWOoHvUHV0PWSIdjnuUjzlYaznJAwMuAOu8fdJ20Xn1vrEAriGZUZvj6bn8O6MzE/EpLx9lR0dR5DB+i/rmfDVZyLP0mdNPP72jPToNtUHvY65bk8dUvbSlDaBnD+lKXh/OmYegsyA3SsTiGoBs+VbySkt7gJdQvjhCaTgu+vshH7EFKCqzv4vvRyQw/pyGdLMWPdHzz8/Vlhc5p+IdLXXnORxiRLuUXME5w5xzSK9fZLtetO2kVifxqUVX49kStfQV9cepc2De3yKjeK5P7nG/RrXnhuSe8puqg3m7qukkNQAX40CLjaZFDql+pWPrN1JeY3RfpeVdU+d6an+1ubPyrh1rbUKyzcfpIV3J51huJ9K7nRe5baWl7lPmuSpD6QgoWXNfj5Sbpx3Tnks6tj/nesPQeNVnS2ixNbXIt5wH/rvU9mp1FV9KczB4j71Jup/LipZ24WWM87XDAb4/c0LtcOL2urG1wF6H/Ym+UyNsOPiOII8ami6M+AfgmcUU0NT31bLXGKz5fC2drrtOh514yD8wNb3es4wj+ik2WeyOJcLuTDso+YFK6bnWmmfrc3k53D7xhS98obvmmmvyJOk3iwnRWaWjlxKxMw071DhhV0RGuh/D73OODQo5Sxty8m/v13VOenYNUIR4XR9zlK0jAFwD3GoFcGnSOpB9QtJhzMRIHbRlcCAAXOXvjEKJ4QBioPfoBuUnFn910aCpeUq4yLK4gk+ZxigfedlxaqOkCSSBERKABIY00Mk4VUQYJ0EnB60NDqDY7LrrrilqAo6mb33rWxsg0hkHn/CEJyTnJgq+gwxQCFndjFKBsdnBCVNrT9vCGElEASYUKCg4UXBA4ewOepgDGJx33nnn5HiCRz/84Q9nkRUeTvXwGc5HtnLjD7AehmYiP2lS9HDKDc9waAOuwajPNwBAM0XJ3zC3B6PFsEoHeYHyihODNrcIQg4xvsIPHIND5URB33HHHVP9qCeRKZBbQzxZRFlb85ABB0cYq/GYzLKlJeMn/GQFVa3e8/Rzyktfx2GOPOBdRKGqvYv0GIIYN5jk1Fa4kG5RxESLb8r3h9atWzcDhPn2oX3vY2wUgAsQGkY7QFz0G9r/GFkEf+ibtEdAXjwH6DGf5PWVY+q91v499T2LSI9jE8Ad34r+jyFccyDJG2TUkEybUpap8hLDFhN/xiLaFTIBQB8R1IKWwwFWLNIuACcjiwF50gYA6fYRso/vxDcjvSIT9T0zdG9Ke5lXrg6VZVPe3xSybEp94T3yGVmLHEGncTBxKa/WdiaQMHNTQILLJvRh9BN0b+ZZ6GzMuWpE+1+3fsxjrEJ3BmSqaFu1ZxZ1faXlZYtuga7EPAddlsUOLDaCryJAXMxlmNsyzs8zl1GeyBEMzZSXMQT9bKW+icrQdxRP6D/oEGyNB2/69Mi+OR/9kTbInIHFBfQV5gzLtOdM0UtV35VoA0Qrf9aznpXmaPCF1d7wgXYnfaf0bRywz0IBZFqNVJ9FfT/eg3xkPIVHyBHmGtgEtgRqqTuOS2Qu+j28cvvIPDyb0q4XbTsZqlPrGFrjx5S+Ms8cmPe3yqg+uVerF9dbn1OeK62DUd6pNpoWOaT6lY6t36iU19C1ldZdKM+8bWKoTmPvb4q6jy3bak431dY0Rb6thnpHu1gNX2H5ZfBIobytBcDlC7LQgbCTMdfBl6MFxsylTjrppKQbvfa1r0168VDtaIMij7I89X3KQ0dsjJSBMQ6izNh6+2iqD3Vq+r53L+IeIGVsCRBzD+Z4+OyYqwl4hy0foJx2bxl6b2uerc95ebARAsriW0I1ANfWW2+d/N3oExC+JuZg2J7JQ9fhCdvGO6BKiwV5DhskQFh8C+wkw9xIlPcZBRfgPvlq3oceh+1b7Rr+41OfSgHgGskxGD6VHKDDs3xAGheNBrR8viqrFPZ06jsj/drhgLcP3+ZotdWAdsrqf2gltlBcDfVfJGhq3vrMWxZWG+NwACgh0JXK1ALg8pVgGKDZvsYN3H5/6mpzlSuOwYHgQHAgOFDnQG68rqfc/O+wipOxjYlVKXrRYYcd1hGZBGLrVRz8Q5QDuIbSx/3gQHAgOBAcCA4EB4IDwYHgwDwcwLjPNuMsHAg7yjycjGc3Vw7EHHhz/bJRr+DA6uTAMmxNq7OmUaothQMsfGKxgFMORvF7pXNAUEQbgtBX8Qv6ol6i/gESgqZEQt9///1TdGCe851d5nkf/mwAZUSdFoiG/GsArqk+1KnpefdKEGBhAYXAo2ALZ/G1yAFx7N7BdoRD1Jpn63OUZ/fdd+/22WeftKBW0dBVzhqAiyjYBx54YErG4tsPfehDeiS1gRNOOGHWPm+++eYOPA7kux6xCJvtjX3hFeBEIsmJBDDEH0HUbtoXvGYnAF/8QtuD3/LJsysIu4NMoQBwjeRWC4DLV0/xmi+ujyZE+EARH1qoVK6xNQ6hT4PWNgcYWEDdsooT4wursADtAX7yUItDAC5WarLdgfZNZXChw7INBltDOGgGjrHvMBEjQHmWtm1h9eczn/nMFAmCrRfybX9AkiIYWcEHGhUh941vfCMhVPsAXKQlZD9AL9ozq55xkNKeW6IUgGrde++9UyNgoBfPEL4IfVa2EdKf/ZMRnqwAUxQo+pgjZ9WS+A7sX0sZ6cvwjmggDF7wCgELIazZk5lVdEIjExmCOsEzjwDACl62NwEYBfqXcMeU9frrr0+rdvVujvAIhDtEBA6iA7z4xS/unvjEJ6b9x1ntSr2IDiKlZ0pZUsaVf2xFpLCoeRKc1OJvfq/2G2Q2aGYIZUCDiKdn4ILnEAC0vK162jgPDgQHggPBgWkcCOP1w/zCWMDKX+jMM8/cYFs7n+gzjvs2nw/nsPEZY6NH4No4RVwJDgQHggPBgeBAcCA4EBwIDszPAexO2Ayxd+F0gKY4vOYvQeQQHFgbHIg58Nr4TlHK4MDmwoFl2Jo2F95EPdYeB7Cb4tMDSIIPk4j30FQA18knn5x8mTzLjhD4M52IcoQvECKSLBFlh4goyGz9CggGnzKAI9E876N8JaoBuKb6UKemL5VlGdcOP/zwWcSzW265pbv00ks3eA2+bKJNQUQW1/kGibIfrXm2Psfr3/SmN6WdLbKipJ81ABcYBgVNYjvEPDquA8rAa+DDhhzHUwu0g0+BtgppC1r3O+SAsZRw/T+2nAY7AbFTTGnxebpZ+Sffe2yhWGGQLrcAuHzPafIhLJuHvQdQAgBE5I1G1+K4djjAIFPbN55aABK64IILUkfldx+Ay403pM0JwBKDhDow97XPNZ2ZwS0nQkWyxRLk+wjze6+99upADTsamesQwgdQF5RH4AKwBXK1BhAiRDwCbQoBzKIskO9ZDvIV5QKAGsIXhSAngEIf+chHNgCOAfD6q7/6q1mIxfwZBioMAQCZDjjggBQdL0/DbwcrucAupc37MgAvVlNC8IQQlmypkBPgLSYIINjHliXPI/9NPh7mcd16FDxhI6EWABf1oD4AAJFxJdKe0tyjPg888EApWVwLDgQHggPBgQYOhPH6Yab5Ki2uMqYDtkYv0ViH/nXOOeeksN4PP1k/CwBXnTdxJzgQHAgOBAeCA8GB4EBwYHEcQK+HZItj0cE73/nOWAS3OBZHTpsJB2IOvJl8yKhGcGCNcGAZtqY1UvUo5mbIAcAs+Eixj55++umz3ZamArg0FuOTVpSnedhF0Iu///u/Twtz8XcT8MaDjszzvmOPPXYWYIIyAuxB364BuKb6UKemn4dPU571CFvvf//7u5/97GcbPf5P//RPs+AlJaBT/kBrnq3P8X62o+dP9Gd/9mcpmhq/awAuAW/BDlCvnAgco+0z77vvvoQJIA3YAYLpQGAQfv7zn6dz/0fkOQBb0LXXXttdeeWV3X777ZeCtHDtuuuuqwbVATMC5ZiBdHHgn/AfAeAaYFQLgAtwFo1CBBgFASE67rjjUsQf/R6LSlX6OK4uDjhSk8GQvX8BrgCeEcAJpyKCC6BODcAFWIo9XRlQyAdhQoQm2hL7ZyvkXt7hWwFce+65Z3fooYfOmKlIUqBxFYVKNx3ABXKbAZaBFkIwMiDQV0CzygB17733dqeddpqyGDwOAbg8A6Jo0acQ4OJLPgg7Gvp///d/E4iStPBZ4Rf5Tgh4IpS98IUvTOVXBCkJbBQG6uioWb4nUbx4JwhceKZ6X3PNNWkwobwO4FL5+bbso8t7+K56TvwaUxblNeXowNIWABcoevhGuyQqWU60B0Be8Jg6kp5jUHAgOBAcCA4shgOawLLSqbaiaDFvWhu5sN/8brvtNhtHvdSMP+edd15aVebX+84DwNXHnbgXHAgOBAeCA8GB4EBwIDiwKA5Iryc/FiSy6JMo8EHBgeDAhhxQX4k58IZ8iV/BgeDA8jiwaFvT8koaOQcH6hxwX+YVV1yRdk0i+AI0BcCFL/Xtb397eo4dotgOjm0Z2UWIXQzwC5PfPffck9KM+eegGPyM7AYlWvT7FJQi9x3rfflxqg91avr8fYv6LX2pbycKviP8hUiPn7ePWvNsfa5Ulpe+9KXdi170onSrBuACpAg2AbBTCbj21re+Ne2IRSa+W95rXvOajqAn8ExRufIysK0uGAToYx/7WGrr7MKmncQuvPDC7ic/+Un+WNo1TWnoN2yzOIUCwDWSWy0Arr6sAfWA7BNog7QRJruPY6v7HsAdgFl8T5yF+d6yLsDZV5b9ZWsALt8rGET097///VnlAVQRPYuoEvlg0wrgOvXUU2fRoFxw8dJXv/rVaUtBFcABXG95y1vSVoPcy8P/ISyps0BQNeSq8vXjGAAXSGzCDX73u99NjwKQI9wjQCkIFDHgKIBRikYmkFZKsP4fQCOeYS9aiFWOIL0hQEdE+OJbnnjiiema/onPDsbTPQ8TSrQ9ou5BOYCLCCHswau9dHfaaafuzW9+c0qbR7bqK0t6YOI/b4stAK6h13l4SwBz9Iug4EBwIDgQHFgcB1jxgT7AOAfoN6hLE7R99tknrSgjWieTz7vuuiuBrKeCiNnemAijRMNEvwkKDgQHggPBgeBAcCA4EBwIDiyDA2yp4U4vbChBwYHgwMYciDnwxjyJK8GB4MDyOYBPeFG2puWXNt4QHNiQA0QVwreIz1rBQPCFtgC46AcHHXRQesHNN9/c7brrrrOdD/yt2KrPPvvsFNzEr+fnAL8IcAPhIyVQiNOi37elALi00Lu2Sxc8PvLIIxM2gXOAR9/61rc4rVJrnq3PlQoyBsCVPwco62lPe1oKjIPfXkF2StHe8mf99x577NEddthh6RI+BnAEYDOGCGzCSSedNAuAw0Kd22+/feixDe4HgGsDdtR/LBLAtcMOO3Sg/RQxiLcCBuHDe4jAemnizmrjwDHHHJOEAeW66aab0laDXsbtt99+BgQiYhPb/NUAXAyqRNsC5JMPXORJO9l22203AhcJWFQTzqUtFIlY8brXvS4VFVQqYRVz8u3wBODygb4WMhME6qte9aqUHYIJATWGxgC4Lr744oQW9/wAQAGEggB34XR9+tOf3h199NHpmgOq0oX1/1A0iLgFnXvuuTNAVR9oCh7Rd++8885inT7wgQ+k+0QHI/oX5AAuhDygJgnflGD9P4V45DcgPlFfWZRmynFZAC62hOQb0NZFgNTYgjMoOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcWB4H8F/im2SRK5gD/JEsVHW/7pQIXPm2oio5UYsIlOFYB/zTbLHYt6CW++ziBJWiQC36fVsCgAsMi/AEJVCcvtkhhxzS7bXXXunn5z//+bT9n+7lx9Y8b7nlloWWpQXA5ZgN1Ys2SVCXMYtmAD7CK3AOohtuuKH73Oc+p5/V41/8xV+kXc/UL37605924DemkjAE9Cnyokz0N/441++x+f7OejTaZrlX1qIAXIR5Y29MGOskYIxfi/O1wwH25yVaA1QDrWi1EGjn+++/vwrgKtUadCiIaQBHbK1H+0HYeHSoFgDX61//+pQn71Tov/z9AJxe9rKXpctqp5SBMLLQV7/61e7yyy9P5/6PbfYECPuf//mfjmhP69ajXtlatERsMYogGgJwAXYkel1OL3nJSzr+IJWTCCVE4xKxFeXXvva1DpS4hJ/u+XEqaAplg++zz3okOnWEagAuUOheppR4/T+9k9+cEz5f56VoYCB/d9xxx5TG/6GIUf8aLQPARRuB9xqQaJtXXXVVxzaSQcGB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8GB4MByOUB0K6JcQeedd95si+5WABfbzO2+++6zQuNbPfPMM2fbxj35yU9OwR3kH8QHyzaLJSLAzQknnJBuKdhJnm6R7yPvLQHAxRZ/bPUH1YK1cO+Vr3zlbOu/K6+8srv22mu5XKTWPNkSfpFlaQFw0YbAMQjspAoCaPz4xz+etkHUtfz41Kc+NeEfCFoi+q//+q+085p+l45E3TrqqKNSgB7dBywGTuI3v/mNLo0+CsMQAK4Bls0L4EJwEXUL4eQE0IG9Z0HuBa1dDtABASxBDr7pq1EtAhfPMJBq71XyzQF/pFkEgAsgFMAjCBBaSYggrI499tiURsAoF/LcoCwlUrn/7//+r3vXu97V+Z7LeXr2lwXcNgTgIsQhUchyAhj2ile8Il1WOfkB0AxBndNvf/vbDqH75S9/OW2x5PcFpsp5rDTsEXzwwQcn0B7bWZaoBuCqKSV/+Zd/2T3pSU9KWfFdAKpBtbL89V//dffoRz86pfF/NYCb0iwSwEV52ata4Sd5B20IMGBE3hLH4xgcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcWB4HiP7DbkwQQBoAXKJFALgIPEEUI4FLlLcHAumLAOV+TQJdEPAiJwdwzfs+8t4SAFz4qRXl6YEHHkg7PuV85Tf+XILNQJdcckl36623pvPSv9Y82R1rkWVpAXB5fZ74xCd2r371q7ttttkmXQYQdfLJJ3uSdE7/eNOb3tQ95jGPmd3D307UrS996Uuza/kJILHDDz88BcsRJgJsAdtTwmMFa8mfG/qtPhYArgFOzQPgAtzwjne8o9t66603eAvhBdkPFhBJ0NrmAGEe6Zjsfcq3HkM1ANe+++7bHXDAARuBtujwRFciqhSAwBxcNBSB641vfGO3yy67pKL93d/9XdqHmKhYj3jEI9I137bPyw/ASxGvBIxCiO28886erPdcoKtFALhqwrUG4KJghIQk+l3eB1Xo73//+93pp5+un1XQFAlqded7EOHrj//4j1M+NQDXd7/73e6cc86ZvUsnaw3Axb6/RAETMZDdeOONHWE3g4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYGV4QBbJyrgAsEk8FuKCBbCbj8QPtv//u//Tufnn39+8j2nH4V/z3/+8zu2NYTYXeoDH/jARqn+4A/+YAYaAvsAUCunRz3qUSnQB9d//etfJyBYnobfi3qf8t4SAFzUVTgFoj6xbWaJ3A9d203Mn2vNs/U5f7fO5wVwkQ8gK7aYpJ1CeUAbIswB8lIUOfrNXXfd1V100UUdAWpqBNjrLW95S+fRun7+85939KkSOLGWT+l6ALhKXClcawVwAbY55ZRTZtGZlDWCEcBI34dX2jiufg4IPEX4PSIcjaESgIuBFSEilCYRqUBpglgFtQwBENtuu+0mA7iIAAfSFOId5OfRmEBNM2jm5NslCsBF9CkiZUFEsPrGN76RP7bBb0BXbKOIEBPAaYME638Q1hEQ0FAErhYAl96FcKY+T3/607vtt9+++93f/V3dSttAsh0kVIt65eh1BPj3vve97rbbbuvuueeetP0jz6K8IOSXDeD6oz/6o9lgw3tFtMG+gcG/OQA+vstUOvrooxMP9dzdd9/dXXDBBUnp07U4BgeCA8GB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8vnwKmnnroBmGTMGxXwo5bWd2mqBajgWe1URcQhfKw5OXiob/u+Rb1P799SAFyqZ43/8OOkk06aRaLKQUzilx9b82x9zt+t8yEAFztmEVkMuuaaa7qbbrpJj25wdL/2Zz7zmVlUrT333LM79NBDZ2nBThB8CSBWH7HFJHgNgb4I8APg64477uh7bPS9AHCNZFUrgIuoRuvWrZu9BdAHgum6666bXYuTtc+B97znPd2f/MmfpIqAbAXh6kToPcAyALN+/OMfd//6r//alQBc++yzT3fQQQelRwEHnXHGGZ5NOtcgSFs68cQTZ/cFIlO0q9mNh07e9773zQA/AnC97nWv63bbbbeU4tOf/nQCY+XPEW4T4BIkAJcDmQBv/du//Vv+WKrrq171qu7//b//l5DcAkdtlDC7sEgAF3XbcccdU2S0yy+/PHtT13lUMkBY7NsM1QBcDoITLzxT5AS8hZYN4PL3TjmfF8D1nOc8p+O7QigCbJeI0hYUHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOrDwHjjnmmA22gPMS4J/GVw0RCIJgGdA///M/z87Theyfb71I8Ai2PizRBz/4wRTpCOAJPvOceA/+YnzbROiiDCVa1PuUt8BEY3fQmupDnZpe5Vr08Z3vfGe37bbbpmxL0bXYEhF8Ae2A3b4Acw1Ra56tz5XKMwbA9fa3vz096n7+PK83v/nN3U477ZQuE5Dk9ttvTwFnaKsKqgN2Z+wuUw6WvPfee1PQJqLPLYoCwDWSky0ALg8HqNcQZai2ZeIPf/jDFGlJaeO4djjgQKhvfvObCWXppT/22GM7UMPQf/zHf6S/EoDrwAMP7NhCEbrlllu6Sy+9NJ3r35Oe9KQOlDKUA7gAjim6FVseEs1K9OxnPzuF/9NvAbgciFUKWQlyFGCYEKQCLRH9iYhdEAIJlCnlcXrZy17Wse8xdPPNN3ef+MQn/Hb1fJEALt/PF9Ac4DknIoIhZCH65Uc/+tF0LgAXP3xrSR90SvkBbALgBC0awEWeXhZ+t9C8yoQ/TxRBtp8MCg4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB1YfB9hCEQAPRJQggjOMJQer4DP+6U9/usGjL3jBC7qXv/zl6Zr7WpWIXaXwI0PsPAWYq4/mfZ/nvaUAuBwHwM5LBJVxYhtMtqeE/vM//7P7+Mc/7reL5615tj5XKsQQgAtQoNoTeIV3v/vdG4EDwQIQaY60EG0RQB9tlrYLfelLX+qIzDWGHP/TB2ock1ctTQC4apzJrrcAuBzAkmVX/OmAj2KCuLhqOcBWmQwo7KMKAdIiVB8RihBUAHtAcAJyAmhFxysBuHbZZZcUFYo8EB4glun8CBeATS95yUtmSFDyAmgk4JRHk/rJT36SQvwB4mLvVoSQEKTkLQAX5witrbfemtMEcCLqF1G82G4QsA5gLZEAXPx2tOovfvGLVFZtCYrAo/2rzrzjV7/6lbLpPS4SwOXbPzJgse+utomkfkceeWQHKA7ykJ3sWfvkJz85Xf/sZz/bET0MNLgDwgifCIr5N7/5TYq+BvjuGc94RnqGfw6IY6tGlASoFmLUw4c6AK9WlpRZwz8HYDGA17ZQ1B7F1JtnRK7sADIcIgY8+kFQPwf222+/tNXuj370o+473/lOSkxUP/oDBAiyb2vMlGiOf4A5UaKJHnjDDTfMciqVa3ZzSScoUYQt3WGHHbpHPvKRqS+x7TAKFHIxKDjgHEA/e9aznpW2CKYNM16iqDPmMJH83Oc+V11BRDsj1G2NkF30OwD22sbY0zLGMfFh7EeHG4o0+YhHPKIj0ibEahD+hoj6Pe95z0vjKREv88nx0PMrdR896BWveEUq5xe/+MVVW84x/EAesuX0L3/5y6TP8cyi5DHyjUknbYYtspFtyyJ0yp133jllX4qUuqz3rpZ86Z+Ev4bXd911V1pVtVrKNk850Fv5ruiZN9544zxZzZ5dRp6zzBdwQkTdxz72sSkn5liab7Rm/ZSnPKXjD2KMWGk9daXl+mqWz6yqZc4I0Z5Z7LbaaFO3l9XGj7VWnmV9PxaqPeEJT0ir15kTo/ts6bQsXi+Sr6U5b03HY6xlfMTec+211y6yGGsyL/Qq5DW6LHOjRW0LshqYwXyQeSG0CD1jNdRpbBlWWicZW65lpduc27F4hv0fGz+2Buab0FqQzyr/Io+M1X/6p3/aPfDAAyHHF8nYyGsyB+iXjJ/ojIua7ywjz8kViwfWJAeGAFx//ud/PtuKDhs0QC0R9vfDDz88/cT2TgQj7cxDUItDDjlk5g/+h3/4h41s6diNn/vc56bn8Z1/4QtfUNbF47zv80zdpykQmd/Pz8f6UPXc1PR6btFHbKDUVQFZ+D7svIXPj3HxRS96UXolmALwDIyRolNOOWUWnQ0wFCA7qDXP1udUHj8OAbhI698A/AVbIN5///0pG6JuHXHEEbMdyhxwhS9eNkd8O8Jb+Pv9/Gtf+1qS5fhNX/ziF6db8Bk/Qh/94Ac/mBzAKQBcfRy1e0xqppJH8RnzbAC4xnBp9aZBAMoATSnV0ZkgihR9i98lABeCFcHJQCoCQIOSJwJcpfsAtHA8AGzwfYGV1o+UR2VxABcTOcBYuscz/k4/dwAXzgDyAZwkYuCmrJ7XFVdc0V1//fVKMnhcJICLl/3tepASTnMIHsA/+OzlJlwkYRIVtczLkB5c/4/9gMmHgcDrxzMaEMkffilv+PGP//iPCbDXCuCqlQVDYgv5QAZvagAuIoxB1I9BDCLCGwDEKUTUMnge1M8B8RsAF8BAyL/9pz71qe4rX/lKuk7/f/SjH53OAWsugr8CcpIX30xUKpfuLeMIcJK9qCXj/B30rU9+8pMdIJag4AAcePrTn56AuJLBJa4glwElMhbl5FsL5/fy30xoWJXkQMo//MM/7JiQQijUpdDQng8rXAB8QWNXOZV0Bc9ztZzj4FKEUNcVVkv5ppTj5JNPTuBRDyVdk8dT8iWtRzCdEpK57z3r1m/VzqSYiaKP6Q4AX0QEz74yrMZ7Y8Dzq6XcgFhwICOvCLndR94WXZ/ve2bo3jLyHHrnlPseZRi57QamWvvvy9/7BiH70ddXklZarq9m+cw4jt4HXXbZZd3Xv/71lfwUo961qdvLqEIuMVFLH1ticSZnvYzv5/NpCjRWp5tc+FX6QK1NLIPXi2ZBac7rY6DPuU888cRu++23T/YdvvmWQjWdxPUqorATjX1zIRwvOGAgtgPy+d7mUsdaPVZaJ6mVY6Wub87tWDzUYtzvfe97HQu0oUXKZ6JNsGiOufJq7yvY47faaqu0oI85/mqktcTP1ci/RZWpNvYtKn+1Reb6p5122kKyXUaeCylYZLLqOYDPpS8CF8FAXvOa16R6OMhFFfMdiriGHQtyf+mXv/zlBBpKN+wf/lWAtRBlGANonOd99uoEakIP2Ny3UKTO2IAY+/2bOC84x8/I3MfJ/SQf+MAHZuAn0rTm2fqcl4vzMQAuAkHQxvrqTV74u/GLCxwFSJGtJccSAUvAY2i+OPa5UlS6oWdVRrY6xf9F3fAB8Me5fg/lo/u/s8ceezzYY3VlMzm2ALiY+AFyGUsB4BrLqdWbDoQySM7cmYxQYNUeAC4R0ZqOOuqo9BMULAMbBNoTAcskw4k82Hv1zjvvTCEA6aQQqz2JEgWt738p2kAupH784x8nNLQAZrnD5/GPf3x3/PHHz4BHKbP1/wBzEAWMkINQ7pRl0kZZH/OYx6T7/o/BkDpPXa0IWpuIZZAb0BQyE7RwCUQEehsUN+TlxCHG/rco4yWi35111lkbOIPotyCxtSUlz+GYR2BSvoMPPjgJSc/vt7/9bXfOOeekPYZVfu6DVkbZUQjHb3/7293555/vj6bzWgSuvrJslMmICz6wjAFwAZqRwdJ5POJVKUkAuMZxqgSUqhmTicTHH0Rbok3NSyVjNnmWyjXvu2rP01cxqkh+IfPo70TLkxLFpACZJPR8La+4vvlz4NWvfnWKcOk1ZcwCwILcRG/TOEmaW2+9tbvkkks8eecTkw1uVH7QJhl/BKANANfDjFrNAIGHSznubC0BuJCbAg7moa/dUB8Aru8mHW1cC1j5VFr0wxiHntZHyDWtpitt2973bO3eMvKsvavleg3A1df++97jfSMAXH2cWv69AHAtn8fzvKG1j83zzkU/u+j+zmI1zespKzohwEPsNFsC9bWJRfN6GfwszXlrc27ZTdwesowyrbY8azrJ5gx8CQDXkakZsjjYo7Gvtra5iPJszu1Y/Fk2gEsRTHzXCb17tR0FcMHpuVoBXGuJn6vt+y6yPLWxb1HveOUrX9ntvffeCeiCn2QRu0ssI89F1TfyWd0cGAJwedQrAgcoKIXXqmST5z7RzVl8z04yJQIUhN+cdPS7sdT6Ps9fIJ2xAC7NBcijz4eqd0xNr+eWdawtesfuCHgLPEJOLFLH1wGVFjW05Elerc/xrIjIYYC4ICK3EcGtRACj8bPX8DzshHHeeefNFkDjQ6KuU0gArqn4nwBwTeHyxLS1Dz4xm0i+hXDgcY97XIqIhbMXoAHh8TD8jCUcKbvuumvaQoxVLYQ9dkQy7ZGQ/Uw+WfHpW4kAfli3PhoDkWyYpBA2ubT1U14WBk+ieJEv5WYbt/vuuy9PVvyNYORZgFy8E8CYQjUXH9gEFwGpAbAj+gXfAlAVod8RnDUCTEfULUBezn8AJXwf6ksoUZymAE1E2267bYqQBJiA8ImL2JqlVha9M45rmwMloBSh/InsBwHUVNhSB3ChcGjLxXk4UDJmk1+pXPO8p+9ZX1GRr9R43ete17GFEhRg5z4ubhn32P7kta997ayyyOePfOQjM2AVNxgfAdOSVsS+7shrkQO4aH9OOOhYdb/jjjsmQ4uAhb5iLgBcD3Nscwdw1eTxwxwYd7boCFxjHakB4Np8AFy0tGOPPTbp3USOAoC0CFpGnosoF3ksGsDF9qIYkCC2F12Enp4yG/lvpaNdrGb5zHdY7RG4NnV7GdmslpKsb4xZyguXkOmiv59HPa8tzFpCNVZNln1tYtG8XkalS3Pemo4nJ0wAuB78EsyFDjvssOR0w8Z42223LeMTbZI8A8B1ZOL7lgDg2pzbsTpPCcC1SPm8lgBHAeBSq4jjEAeWDeDCjwQwAhnku/MMlavv/jLy7Htf3AsO5BwgsAdzI+wNBL3A/z3VB57n2fd7pd/XV5a1dA+cwjOf+cwUIAH/Cdv/gTeYh1rzbH1ualmRtfh0wAOAW8B2il+V9ukR/afmuynSRwSukVwPANdIRkWy4EBwIDgQHJjMgSlAqc0VwCVDExGUWNngRAQuQuuigAEw1baenibOtxwO+AqHoQg0AL0E4gLMTARKkQO4+gAu7vz2lToB4BInHwylvDlvofhwTec7CwDXfPyb8rSvsAdMT5TU1UpTDcZsoyzg1oUXXriQRRPLyHNR/F40gGtR5WrNJwBcD3NuLQC4Hi7tlnfWB9bZ8rjxYI1ZEPaGN7wh/ViUA24t8XKtt4kSgKvG/wBwDUcFrfFurV0PANeR6ZNtCQCutdY2W8oru5pvodiST+2ZAHDVONN2fS3xs62Ga+OpqfPxlloxf2fOXdtZZrXk2VKOeCY4EBwIDgQHlsOBAHCN5GsAuEYyKpIFB4IDwYHgwGQOlABc2223XYr8Q2Y4CFj9S8hYIrxpv3CQ8yDHCR1KVDkRUfxe8IIXpFUQpAVpTrQ5Vsr+6Ec/UrLZsWbMLpWLvdB33nnnhNb/xCc+0a1bH/GPrScAuQC++sY3vtHdcsst6Z0Y+YkitsMOO6RtYSkjEZDYasQJNPy73vWudImogh/72Mf8djpXGfkBgAsg16Yg9IH9998/1YkVT0TuYPsU+HvVVVdtEAXq9a9/fUckJ/h/0UUXFYvLaoB99tkn3WMVBCuaRWz9+vznPz/xmGiATPS5TyhiIpHlhEOJaIqAmkh3+OGHp2dJ+6EPfWiWfEodZg+tPyGsrNoV50QxpMysYuA61/i+hFjOidUerPzEYAERnZLocURrJHzvWPKQufCVyFl90VPoC2x1w5H3YDRRRMyxAC7KpjDPnLOtLHmsBgAX/ZvtkeHr1ltvndoaMuGrX/1qageUt0REdeS5bbbZJvVNgGlMCmg77ONe4ymRJ5EBtFt4ijyhzwOybAVwTa3Dfvvtl+Qg7ezqq6/uXv7yl6eomIQBZ6/7MUTZ99prr+4pT3lKkqe/+tWv0na01J/v+8hHPjLJuJNOOilll8tjZJ2IPs4WzshA+ha8o8+xsoctlbx9DwG4CFWuyEC/+MUverdkIgoC+RElDkIO0Sdvv/321K98KyMAisht+iGRVikT/ENW59vw4pzmj/bAltSc49iibsgxl1GstmMrAMYltv5mFRdy/oorrthIDtBGkEkAcen7pcgN9G/aWK3uvA9AJt8DeYi8YXtU3g/fiAL7xfURK6EcwHXZZZd1e+65Z0rHFtm//OUvU8RYvpFHstW4RR7khZxh++ynPe1pacXaz3/+8/TOfBxrqR/t79nPfnaKmov8hOAN7yQyVN84pzGRSLCEgneifR955JHpEuWkXfANn/jEJ6a2TXsnWm4+Zi0jTy9X63kJwDXU/vveRZum3xApl9DnUCvP/D3kS1uhPSIXaFeMj/Qjlxk1ABfbYDDW19r/M57xjNSHGX9oHxrLVIZW+UzdNbaP0Rl53xS5p/KVjjmAi7bJlu3UhXIxntHPGZdqRFRknkEe0u/Z1h5+X3/99SkScuk5+EyUTr4VuhzfCr4jl5CLTqX24vcZfwnlz5FVwYynAMb//d//vahz+7P5OX2UsRk5xJiGHEamXnfddanPkp6x/pBDDkmP1toKN/fdd9+kqyLv2cZC2z+PrfuUPjZFx1vGPCIxo/Kv9v0OPPDAxGdkIVuGMZehPTInQTawtQJjgLZtp20dfPDBSYbynSDGAyKz0m4Y80WL1muWybMpc4KhNlHjdStfpn4jvafvqHGOb6zouzUdrw/ARf9kXk67gJirMx6Lps7h9Fx+bNEtyKNFlxnSSZAl1BlC15Q+yJyC+T73P/WpT6Xxgf7EmIVOjJ5JdG30NsYOFqOxW4B0MebQV155ZXXuga0BXYkV+8hY+t2dd96ZZLXrb6lgjf+GAFzM/enXELYKdE/R1DF0aB4jfqIDYmtBZkueU3/GRXhf2zaGck3hWU0nUf38SLQ6xihI8zC/r3PZhtQm+PaiqX1jjH1jik7C9yq1Y5WPPn3QQQclHtJGIcb0u+++O7VTpdNR8pl5xWc/+9lup5126vbYY4+O74hdBj2QMQJdZpGE7EbnpC9xznt4B+3zgx/8YJpvOYCrTz6P5d8+621W2AAYK5nP0b/pi+gp2CMZP7W1EWBA8mXcoO+gW3FNNHWc1HNTjx6B65RTTknzSNmkmLcy1qMvuvz2d7SUc1H8nFcW0P6QzdQX3ZFvRp2xISNz+W5Oep/6LfVYa7Lc6zNWDg6NfT4fpx+Nmet5Ofyc57UIABsl/dapRZ4sI08vU5wHB4IDwYHgwKblQAC4RvIfhTgoOBAcCA4EB4IDy+BACSjFZBlHAYQxFAfsO97xjuLrcUDj3IEwMpxwwgmzPavzBzDssJWcU8mYzf1SuRwUAHBD0Y08PwwDbO94/PHHJ8OV3+P8xhtvTI4tXcfYg2MfwqF811136VY6YjxgZRpGBze4b5BoBX4A0KD+lKNEGDvOOuusmUH71FNPTYZm0p555pmz6/6sVnpxzSOpYExl60gMLznxnssvv7y76aabNril74WhHEMEzgUIp52ALVProBdgNH/rW9+agDq6piPb7WLwwSCaty8cD/AMA0qJALpgvBhrgKcP4GiBxkY+wLnLMxhfMHbLCDMFwEUZaYeQAISbGsCFMwMHdK098l3OOOOMjXiLQwxHQI3oY4R31yRB6QA8IZNK7wPIhUEXAqyQt03lkR9b6iB5haMfhwBhkUV9kdSUBqAM7UhGeV3nSN0Bz/BtkWMCcOXy+Ctf+Up6DNlFe5ADz/Pi/Ne//nXHKmiBN/oAXAAQAIJB9PFLLrmkCHJKCdb/c1ChrnHUNp8uq4lABYCrRABYHBwB8Af5QxnoLzi1RJdeeukM3IDMBnBWIp4FOARoQOR1r0XEElDHI93peYydGChz4l044AEKc47DFeKbMG5AOJn4ViV5SjunvfPtIQeU4AQBuFVr86effvoMRNNSvwMOOKB73vOel96b/3vPe96zUR/0NIrGSZ1PPvnkGTiDNF53ZDJgJZx+OTE2IAdp69Ay8szf2fJb7YJnKS/Oy6H23/ce7xuAeunzrTzjPbSPo446KrWd0nv5RhdccMEMLFlzlgLEY8wEiMg3zckjSiIHcRaKWuVzi844Ve6pjKWj9zecaOgaJaIPf/jDH95oPNNK9tIzXMMxwvjtNDTufP/73086m4DMpfai/BhL6cclGUEaAPc44MdQTcbxLG0IwDJ6D7oWdeLI9Xe/+90b8YVn1EdIA1AFuTql7nqevJw0xnCtRcdzfi5qHuHly8/9fervpJFTFxAcOmip7dEG2KIbHcd1vvwdvvhkiMcl3WxIr/E6LJJnU+cEQ23Cy+m8hl8tfJn6jfLvUvotXvt8sqbj1QBc6JHohzjEIeYyRAcWqLZlDlcqK9dadAuec9k6VpcZ0knoJ9KrkJPoQJDzj4UQzLtymYgcQoajY4pv6eGH/iHjkWukc0IvRj8uETKNBUq5A7yUduhaH4BLW0yTB/oCYzV6CNQyhqoN1uYxzk9kPosbkPc5IZd8gZbuT+VZTSdRfn50Oci3Yg6ksdLTSZ8hjY9RLX1jyL4xVSdxfc/bMeXHnsGCJC2q8Dpxjn6GTcfbnOQe9gUWUNXaK4CZa6+9Ns+y6TflxN4n24RnAjgHntAHHcClcpLW5fMU/qFLAHzPSfM2b0u0XfqVZAH2vbPPPjs92jIe5O8c+1vjCHMdZHRJ/vDtaGcCbCvvlnIukp/zyALaMLIGPa1GgNcdVOfvW6uyXHWdIgeHxj5sBbTjKXM9laN0lHwEFIrt2En9dKo8WUaeXq44Dw4EB4IDwYFNxwH5ZtBDsWkzJjE34I9z/R5bwt9Zv9JgwxnX2CdXeboAcK3yDxTFCw4EB4IDa5gDMky5Ic4n0AC4WN123HHHJQcC0QIgwAEYUwFwYZzlOlvEycjHPYw4jGGsxGVQh9z5wm8ZEt2YzfVSuTSp5L4IJyIGTaL56B26h1GPVYustnQDAs5JlI8xhCENJwPEKt2PfvSjYx5baBrAUPBWdcDhg7EZEAiGGoGl4ANGMQhwDSu0oNIEHcULYzU8w6BE5B+IVXJvfOMb0zn/AH/AY6ICudEpBzDpe80efOiEZ/nGLXUgC4y11F3fFmMseWIYUlvUO3MAlzveMUSwupF601ZkdKSdkg7j3xA56Ioy0bZayfPqA/7Q9hRdSgZK3ulGbBRqABd9RDQ1VnBD7uzre8YNob6tBavaMSKJcB7BWxwJrI6WDMj7OiCrt73tbelb8h1xJOJAAOBG+xLAJXd6E7no0EMP1etmEU5wLOVG7rEArtY6SF7NCvPQCbIGQOQQ0W6k16stc3QZSR5jAFwAf2gHEGAyAAjwED5LJuDkoa1B7ogDdIvshgABAKyDKAtgTvpSH7397W9P8oDvBiG/kRW0LfItyWrGDL437cT77t+u37ZWILMjHwJwkSdl8X5/8cUXpyhtgEt32203kqQ0tD36IhFokIl6hkhcrGqGvO5TAVxeJvIiwg7gI96lts51ylsCcHEPQgYxJvKN5Nzguo8r7vTkHoR8po5EQXAZDFj2/PPPT2la6kfUGlZG0/YkD4ksAQH8QzbWCAcUxlp4DaAQPUHkzildgzestOY9tBl9I5cRy8hT75/n6OOIAFxD7b/vfd435MRq5RnvYbxm3IbgM+Mj/Z4oUpIPtD3qQbutyXUZ36cCuFrlc6vOOFXuJcZU/pX6G7KM/pbrrkQcwXEqIvoikcMg+EsEJMZo5BD9UW0cICqRISDyZAzhHt8KuYXDju9EdELJE0DIjGVQqb1w3Ve885vvxtjJuxmHRWzfiszrIxZDANCDJKfQM9GVaEci9F/klevFJaew81Xj+dS6j+lj3jcpN99tSMdzfqpe9JllzSP8fervvFdOXZWBI7oUbYjvJ31Ieh+6DuML0SY1DqD7oSsAqmKsW5Ze43VQeeflWcucYKhNeDmd1618mfqNxJu+o3RIn/Pmc26B9EsALtoBi2KkRzJmI7tp/1DrHK5W5hbdgrxcBijvIV1mSCehjkMALr2LfoFe6fM93dM4SRvUGMk9ohayyEvkCwV4BpmI/kf/pB9CXAcYhg1lHqoBuLC9sCgKQha8//3vn809W8dQtcG8vJrHeHtUGuQjbY12h01F5Iv4uNbCs5pOonfkR9o/8wiI6LZ5RFr4Bd8gjT+ct/aNIfvGVJ3E9T0HcOXfkzEd/YA+j14hGxCyA1CagGsu96gnxLO0f/RtnyfPa22PaTYAAEAASURBVLsgb+Y+2Byk49AvGbt8bCIdNAbANYV/LOQi4pdsfvAAfZe5JfqZt6UHS/Dgf/opUftYRNo6Hnh+U87zcYSyUGbmWIDRaA8Qv7UjAL9by7lIfs4jCxiDabcQcw9kJDIXm4/LXcDHisRVet9ak+XUd6ocHBr7aBtT53qUo0ayo9B3+U6SJaRvlSfLyLNW/rgeHAgOBAeCAyvLgQBwjeS3Jugjk0ey4EBwIDgQHAgOjOaADFN9AC4ZkxUhg8yJcsV2SyKf8Lljmfs4qwEIyUmMMUOOYhkS3ZjNM6Vy+TswgOCYUsSs3FiMMQdDp5zQmliSN9v/4PAYIkAqGFAgDKeEPscIsdKEk11bBpQiKnhkKPiJYwUHD+eQHEBebsBdgLwg/14euSsHwxBWm1D0GO3yPPW9yA+DABF8AIHwnaCWOvAckdS0TRuRbk477bRZnnnUDwdwcU/b/MAPVu7r22Esw/iJERLKwWjpYuEfWwLIwdoHuio8utGlPgAXRlqcs/wB4JIjz4FXGL8w0kErDeDysucOD4zF8FZlZktSyg3Bs3UPRUNj5TyGaxEGZtordc/blrfJL67fUogtIkRsQ8EKVVHeZnU9P7bWQfKK/GjbOHtoP5R5iLzPYZCkTQoEiIzEWSfgFff7InDBZ0XJcZAWZaCNvve97505WVixjHx1R5wAXA7soz7IdX2vofoA/BFw0Psez+WyGqM5xnORR/9zgEMOlgIswLM4IyAcRwC+kEHIGeQBwDURUbkwnEIYJCkf/d7rPgXA5Y4K+IOcA3gI4Wxh9b/kCPdrAC7GO+QH3wEi2gppqYcDb/NxzAFOPMc2jkRjgHgfMoA21Fo/8tG38PJzfYhwpgMWxKFJnxe5c4prAPeI0qC2zvYub37zm1NyBw9zYRl5phfN8c9BIgJwkV1f++97nfcNgQxaecZ3p3y0I74fABvfSg29SwAcyWp3cDkwtxXA1SqfnQ+ug8C7ms7YIvf6vkXe35BjAFjhJUR/e9Ob3pT4y293NIlfyCGB40gD0b8F6EU+0fch16HZAtqjBDKmC6zu8t/5pPZCXs53yXOuQw4uoz0QxamPJANII5CW0nt7kbPZ+3A+/vCcA7zQBdnuq6XufX2sVcdzfvKdlz2P8Pf593OnLnIQMK6AdrQFxnA5dr3dOXAv13eWpdd4HRbFs9Y5QV+b8HI6r1v50vqN1HdKR+mQPud1pzVgaM25cwAXkSzRC+X8BhAOoMmdry4X8vbRN4crlZVrrbpFLlvH6jK8U/KItiadius+TkoWcd35x2+fJ8AreK65G/IaecwiKMjL6boh7Qwdm7GV/omu6dFxXv/616dIX+ThMp7fLVQCcLkcpb0gB7BtiLy9jx1DeVZtkHN4nM9jcn4Cwjn33HOTrsozgGhIA8FHgepaeeZjjOsk6QWFf16+Eu8d9Mb8gXEdau0bffaNFp2k1o490loOGGdODXBN8w3GVMZWyNsB35MoWwKNcx/9XKDuXFfg/lRy/tIn6Bv0EYjFNUccccRMZxoCcLXwj/fQ5uAj8wvGSpG3Ja6xuImoWwD5RK3jgZ6fevRxBFsB8keAJWwf2BY19/c5Rks5F81P72vUe6wscBsk4D7qQtsUecQpt8Hl71uLsrxVDsKb2tjXMtcTr0tH5AjzFujTn/502mJY6VrlyTLyVJniGBwIDgQHggOblgMB4BrJ/wBwjWRUJAsOBAeCA8GByRyQYWoeABdOdQwQUA1M4uAFHPkXXHBBSi9DohuzuVEql08qv/WtbyVHW8rkoX9EDcHYCjlIjN84mnEOQ7fcckvHdlx95MAQDFOAwXBUbQo66KCDOpxVGD/gs8AMKgv399lnn/QTR50cuBifMeZAfp3fODe0Mk7OId9GzcFCpBe54Y5vKFCGvhfpcNbnq5Fb6uBGTtoH9QHs4AQQAGci5CAS6oRhDJ4B+MDI54QjBGMuhIFPhgxP4+deFgdceJop526YG/McbRDjLXyAcEpsCgAXxlnAkJCvbE4XHvq3PiJuAvrx0+UKhilWXvItcFTnpK0R+GZy2vj7cFbRD3OiT2hFeO6wytPy2/OcWgfJK/JxwyK/h8ijbyGrBATSc7463B34btCUc8+dTiUHBlvI4CCFcLzkIB+M+LQlAUPhuTuyVaa+41hHqst75efjgfdbB3CVvvfRRx892yrOgV/Kl6PLBDmDWp2Q/j43Mut9DijzduvygusAyXK5LWeSP+ffNe/zeqeDkb/85S8n42tr/cizZjDW+2pHjw7ogMy87shWTfqVl8s/B8MuI0+9s/VI+eU0o9zSA/raf9+7XI8RyKCVZ8ccc0zaZpP3edQmvZ8tXiVLBeRxB5f6B+kFSKKdChyqfDiWtlB0WVrqrzxXks+tOqP3j7FyjzLUyPOr1du3FmR7IraQhhiLcKoSoVb6rL9H273hvNJY7U7/kjxhyxWiSwA6Feiq1F4cqFoCUAFWwMFJ+XLd2suoc/9G0gd1jyNAAnQO2pCi7am9IL+IGiHdhPSqO2AJQISkaal7Xx9r1fGcn8ueR8ALf5/6O9fdqUt0ydtuu43LM/JxDHAXAA2oBuDyvrhovcbrsCietcwJqH9fm/Byitfz8KX1G1HOGkmH9H5Z0vF43gFcgBQAbwEch/jG9DP6lmieOZzyyI+tuoXL1im6DO+v6SQ+TtYAXDmgg/yYAxLBFCrJXQAVyEz0ZM1NHDxV6p/k5TpcLgO5P4VyABcLkJ74xCemLGgrvMvnsa1jKBmqDXJemsd4eyzZdFhog/yHnN+tPKvpJOkFhX98K23jC3iRhRTeDzT++Fx9nr7RZ9/wdj5WJym1Y67B03xRh1cfmwbjHuTyw+VeyXbjUVJ9vuV5jz3HdoIeTDkZ31lAyNHJdaYhAFcL/3jXGAAXcgfwmtuN5hkPvI5Tzn0ccbuZ8vCFiloc21rORfOzVRYguxQFD52ZxadOLCxR5H9fzODvc9miZ9eCLG+Vg9SxNva1zPXEs9oRuwT6lI97pJ1Hniwjz1r543pwIDgQHAgOrBwHZMvFXoZ9CT0QfZw/zvV7bIliC8WxnIp0wYHgQHAgOBAceIgDMkw50MIn0AIMkNxX0MvIwHVCP7PiDnIHV7rw0D9Wlwl8QVh3JuGQDIlujOJ6qVw+qXSHMekhOZQwJmHQcyLcPgAYqLSloKf1KCfkhaEQ5+RqIvgJAAtnH0ANFCnIgVpusMTJCJADckOhg5ccrAD/iXiVk2+1881vfrO76KKLUhJ9r5LBJc9Dv4fq4KA7NwLqeY7wAEcNJMMkiqQiXrDlhYBaKZH9w6mKUxLDr5zcdnujUwEE3Si8UaKRFxzAMPQIDmC26HDj/aYCcDmw0WWA1wEHE/WDhtoD9XjsYx+bVrIjR1D+/Xu409ejefn7PJLDGADXPHWQvKKMGNrcaeBlKp2r/fTxRDJsCMDlhnzexap8jKAYSTXBysvgjjii0gmUQroab/M8/PdYR+pZZ52VZK4/S4QdbRXhK82PtC0U3WmtZ7V1CnzHceCGeaXZZz2YFecwJGez192jLOgZjgLqeAQ4vY/7vK8Uaa0EPHQHDbym3eSE3BKIlraEI8qN7wD8aDM5OchJTszW+pF3zWCcv7f0W44Jj67hdWeVuRxO/rzeyTXVXfcXmSdgUkVwVP4c6V/IijGkdkHalQBwTeGZt88ScJoyA/hAXuDwJ1pCzVkq2VMDMpUAXK3yuVVnbJF78KBG3t9uuOGGDkBbTi7n1N/yNPpNWsYzZNC6h6JNOoDLHak8w/cAhAnItSTLSON6r0ApPoaVgHs8h/5E1A3GG0Wq5XqJHCyEHEJH/tKXvrRBlMz8OXfSOhBw5513TlHLSO/8aqm78176HfnOo+M5P5c5j6CckL9P34/rknOl+Qr3fb7lek0NwOVtYqpuNqTXeB2WybOhOQF8qbUJ7nk5xet5+NL6jShLjcRrn/PW5twCcNEniSYNeAKiTzOHzvXPeeZwtfK26hYuW6foMpRD+gH187mZ6xYuW5x/pQVaJ5xwQrfDDjukKjLOsdWqE3N8IoQDnJWdQAsu4D3lKRHRGZF3UIsO7Xk6gIsxQ4uveD/gvVyvbx1DeafaIPylbnk7cn4yDnzmM5/xoqZz6Qs+V2nlWU0n2eilduGtb33rDOCmKI/c9nbni0fm6Rt99o0WnaTUjt3e4WOdVTmdYkdDTnrfcLkHeM0jxfGQR8bx+Vae95jfzl/Nr/LnPPqS2268nJLPLfzjfWMAXCU+zjMe5PUc+1vjiH8zf9Yjpmqsby3novnZKgu8fjpHb2PLSCLUvvSlL51tA1wDcK1VWd4qB+FTbexrmeuJ77Wjf1vKrCjZ3k+nypNF5klbZqcI2k1ObJuLDhAUHAgOBAeCAyvDAc1DAsA1wO+IwDXAoLgdHAgOBAeCA80ckGFqHgDXK1/5ym7vvfeelSE3BuoG4AwIQ7Sc9zIkujGbNKVy+aSSyay2xCM9JMOWGxQfvNN1UwBcHm6+BD5Qnit5xLhMdAa2dAQgU5rQUh4HcAHqwjAN3x2UgHNRAIerr766u+qqq1JVvN61b0hCfUe2N2O7HUjfC9CXztMN+ze1Dg5AY4sHtoDKyR15Mtb59kWkr9VF9SCNtpnjvEYywnHfo8bU0vdddwAXjuOcMNoDGoTHgOxy8hXQKNSseuujF7zgBWlbJdI48K7vmZJR3Y3m9GFtUZrnI4cIK2C1ypI0rBp/zWtek5zbGKH9GygPN3ICxMQhDpX6PNcdcCnjJ9drNE8dJK+8P9Xe49fd6eiy1tNwLrCIyzA3iDmgFtAsTpyc2NKOdgMwQNH4SOOOuPyZGqgpT+e/vU7qe7o/JKuZ22AwhNzA7wAueJG3fcl4H0P0Th1d1mt7F697ra7ivX9bRXmpOdl5p5x43m7dQVP73u5UpJ3zDneOsM0v2/3mhBylf0ECh7TWjzxqBmPuDZG20qHuyFB453VX1Kc8H1+hrLorzSLzxEkkkJzy59j3PT0d52oXnK8EgGsKz9QfKBvfkXFjiEpynWfkkJ0C4GqVz/PojFPlXh8/vL/VoqzwvMC3ebQrnK60V2QOY3KJ1Ee5h05GFB1kZ06AMgDaow/IkUIal6VyevoY5ltE5XmO/U35AajSd53o12z/iqOW8cTHexxx9HmINOhHkJfXt2NsqXttjJlHx/PylXQK9Skfg1PF1v/zsWVoIYie8ffp+3FP+mQ+99Fzrv+6XlMDcHmbmKqbDek1XodF8mzqnADe1NoE97yc4vU8fGn9RpSlRuK1f/eajicAV54X/RL9CXC40zxzOM/Hz1t1C5etU3QZ3l3TSVy3qAG4HEyqemhrZn7DU/jnpPmKA7gUxYl0eXo96/MXQE6AnVrJAVx5HkQMB8zgNM8Yqjbouq7n7e2RLeuJ0pWTdGOXk608q+kk+Tv991Oe8pRZhB+3O3h/p4zaqm6eviGbhr/HyzJVJym143333bc78MADU7Z9bcm3xVb9XO6VZL+Deny+5XUYe77ffvt1tFWo1NeUj3Qmf5+XU/KZ9FP5xzNjAFxXXnll2k6S9CJvHyVeKZ1kQm7D0P0px6FxxO0zGuvnKeci+dkqC8Qfom0///nPT7YfLTTVPR1rAK5S+1oLsrxVDsKP2tgnvVRpxsz1xN/aEfupIvW6XdD7aamP9MmTRebpC/3yOjAmMTYFBQeCA8GB4MDKcCAAXCP5HACukYyKZMGB4EBwIDgwmQMyTLmT2SfsDhjwFeG+wlsO7LEvd8O1DIl+jXxK5fJJZQlAowmuG2FVpimOF+VTc2Qqz5U6AmDB2Y+zIyf4BgGGgRzAxW83dsBTDJAyZGKUZnKuPBxUxLND5I7m0vfy51vq4AZqgAw4AXIqAbg8Slievva75JTK0yrSDtcxyOG0HCKMcE94whNSsgsvvDBFCeOH87rUlofy5b4MpGPACL7tWg0Ml7+zZFTXtpy0HV8Znz8rg6WXDQP1AQccsBFoi7xwAmAUwsDnef/t+igD2vqkxifaFkACSMbPvDz+e5461OSV518696gkREM5++yzS8lmfdOdIjV5TAZsrfqiF72oY8VzidzJ5Y440mIAxAElJ5Q720t55dfGOlJL320MgKvUJ9Xm+6KYuXNkCoBL39adWjLE9o0Fil7j7dbLUAOMDQG4SsZrfQPxQdG9/NvW3leqH/nVDMZ6V98RMDGRDGlD2ppoTN37AFyLzBMn0VoDcNW+X4lnagfeZvu+F/dKcp3rQwCuN77xjR3bvEK0Jdpeq3yeR2fk/VPkHulr5CCDPiCU+OxgrFodkANERFSEQ3+Gcgi0AvhWelteviuuuKK7/vrr02XXe+X0RDai00KMZ8ineQkZcuihh3ZE+Cs52hjLTzvttI5tokQexVQREeX4zHV6npla99oYM4+O5/wsjU3S/+edR4hH/j59P+5JR6qNLVMBXMvUa7wOi+JZy5wAvtXaBPe8nOL1PHxp/UaUpUYah71/1HS8HMBFlD7GV8hBk3qXzyt0re/oc7haulbdwmXrFF2GctR0EtctXLd1/pXmij4PLrVfySz1efQZZP4UYoERc6tWygFc/q2Rvch8B9DWxp/a+729ldqgPzfET9LmAK55eFbTSbxMpXPJauYymgPyLRm/8jnCPH1jyL5B2aboJKV27BGX+qK5uYwrAbhK7bsPcFHia981tyWUgIV6VnOnMQAunpnCP9KPAXCV5M484wHvbaGhcaQE4Jq3nIviZ4ssgEcsaEBmsWgvJ0Bx6D2y7dQAXGtRls8jB+FTbezTHGTKXC/ne+n3sccemxZBMt6gM0EuY1rkyaLyDABX6YvFteBAcCA4sGk4EACukXwPANdIRkWy4EBwIDgQHJjMgZJhyifsYwBcipjBy1mlXwLaeMGYuLONIlQzJJbKNTSplDFPRlh/5xQAlybKbDmEMXBTk5xklIOINKyUIiIC4AQMl4Binve856Vi5gAuj0707W9/O0V0weiHkQFHnLYa5GEBuziH/xhZ+ggHLkATqPS9/NmWOrgDq7YK2LcTUxSg7bfffgYugkcYGfuIemqVbl86d2DfcccdacuMvvQ4Z8Vr0vkWbG5MLhlI+vLVPRlI+U3b79vm07/tWEdDyajuoJPSyry8bNqik60SiRhAu4PoW7Rh2qSijcCf7bbbbgMAl682ZhsRjPI5+VYiYwBcrXXgvTV5lZcp/812KBhkIUCU6i95OniE3j8WwKXnAb3AB5xmtH85+bh/+eWXp61t3RFHm0fO7b777rPoiUTuwuA6dmXnWEdqqX23ArgEMumLwOWAvu985zsdYGOvew0go/HDDZlyUrnzSzzXsWRodwdN7X3eDhWFyp2ete2IPfoefZ5yt9aPOtQMxqrf0FH9Vn19TN1LYCR/z6LyhC/0jZzGynyeW+kIXLX2UuKZ+gP18UiHeX39d0muc1951QAlHhEAOYXcbpXP8+iMXpcxcs/T5+fe32rAZm/PijrqACIAWzgpb7vtto7tieAfpPE5B3B5Gdji8FnPelZyoBDRSuMjaYiqibHM9V6BUpCp69atS1nJiZt+LOgfetVuu+2WtugWEI2sfVzit4Ps0dHYshrnDeQr+tOF7N+YutfGmHl0POdnaWzSODDvPELV9ffp+3FvyKnr+q/rNbUIXD6eTNHNKMuQXuN1WBTPWuYElLXWJrjn5RSv5+FL6zeiLDUq8bo253YAF9vBMZdh7qDFPHmkHtfzp87hauVt1S1ctk7RZShHTSdxWbxMABdlYG7MAiEc5mxPPERsyzhWdy7l5QAu5tbIIfiAnIQ09ujZecbQUhtUvhy9PZZAFKSRbuxjQivPajoJ7+kj39YZADbfC4ARpAUFen6evqH5mi90VL75cYxOUmrHDuLJ+7W/A9kOsAFSNDmXeyX5vEgAF1vfsWgIKkW44rqDWMYCuHgOGsM/0rUCuOYZD3hvCw2NI/7tNdYvqpzz8rNVFjjAlDk1th505HvvvTctGvUIrssEcPG9WuVSy7ee9321sU/zsylzvTHlf8xjHjObO2oRy7zyZFF5AsSlnZSIRTLo6EHBgeBAcCA4sDIcCADXSD4HgGskoyJZcCA4EBwIDkzmQMkw5RP2MQAud2bVtkrAoMNe9hieAQ5h0IVqhsRSuYYmlYtyvBDGHic5zjiAOpuSAL7g6IAwlAIWyI3E7ljNAVw8p4k/IASiOmCAgzyKGr/d4FLbOhIgiox3t956azIq82zpe3Edaq3DDjvskCKPkYfAWZw7vfzlL+8wfkFKw6Qf5ymkCDXpR/aPsO4YxwGuYDAdInec4bTlW/QZEF72spd1L3zhC1O2KL2+zeEiAFwAmnCuQLmxOl20fzJ2col+0gf20mMlo/phhx3W7bHHHinJJZdc0tEGcnLnqkCQvm2nG3T9WfVfj2Tkq30//elPJ4CoP8O5G/Fl/MzT+O/WOpBHTV55/rVzAUNpbzgTcsKoT19FVrpTpCSPca7vuOOOybkEQCsnBxsix84888wNQD7IAaK8QKeeemr3+7//++m8Jr/TzezfWEdqyaHQCuCiD/Fe2ghgM/iUkzu2rrnmmu4LX/hCWulLtCCoFHUCsCXtD3IAF+8A/Mv7tEVgSmT/FL3B2607aGqAHDfQlwBcNaAf315OqptuuilFnWMlc0v9qEbNYGxV7D11uYicAcTBEarVvQRG8pcsI0/Pf8r5agZwqT9QH8qZbznK6nfaBTJF7akk13ne9QRtjcd1kY9ZAnC1yudWnbFF7qn8paODDBwQ4GkZ7xgzIPU317lKY47LNwdwsf0QYzbA8ny7LfQWvqEiFnzyk5/scGy53itQCtsQA76Fas5eAWT6wKc8j05O9A8IUDXjhROyhXLhHIcciO71RKeg/TEuQTgtPUppS91rY8w8Op7zszQ2SQ9ZawCuZeo1i+ZZ65yAdlVrE9zzcqqvzMOXIcd7DWRHWWpU0iFLOh7PC8DlTlu2pGIuDRGZib6pecg8c7iUYeFfq27hslVjT559SZchTU0ncb3K5bXzrwQ4mhqBizJIfqLXoZ/l827SUEf+IKL9CLybLkz85wAugXJp68wzGb8hj8zUOoaST6kNcl00xE/SlQBcrTyr6SQqT+3osgBwFeMC88+Svj5P3+izb7ToJKV27IvR2Er53HPP3ajatANFGPN5isu90pi2SACX9+scVKgC+/aWPt/3cko+t/CP98imkUdaG2pL84wHqt/U49A4UgJwtZZz0fxslQXSo+iLyBsAN06+ZeiyAVytcsnLO+V8nvfVxr6Wud7YMms8AITMPND7aas8WUaeY+sT6YIDwYHgQHBg8RwIANdIngaAaySjIllwIDgQHAgOTOZAyTDlE/YagOv8889PK6p4oa/QxaiEg4dJu5ODWW6++ebuE5/4RLqtSV7uZCqVa2hSKYPBohwvXv5Ndc72Txi6ICJe4Dh1whGCg1YrsksALneyErkGY14pDLeDbAS88Xdx7hE3fMu10vfSs611cCMnjgpWntK+RDg6MWqo7gJwcV9tgXMMfThMnTxKjztYPU3pXMYV7qHIsrIvN0xxDwDC61//+pnhPXfyujO8ZCAhjyF6xSte0eFAgvieGAlLZdEWb6TL+xnXalQyhDqwotQeycvBGYr2BSgSgx10yy23bBQVzY3XyA4cV5A7KXJDLfcx2GNw4gjlfE4Xs3+tdSCbmrzKXlH8KaMeN3PwJNf8ew4BuN7whjekNsZz9D0cZE4AsgBmQTKyu5y+7rrrus9//vPpfs572vT999+f7vX9c+eJ9z2eGZLV7vh3A/+RRx6ZtngjD8BTckzyG3Jgmo8jD97tEsgAPguQJqAJ9wWgy8GU3FPIf87dMXL00UfPHHQCb5BG5EZ2b7cuu2ogpiEAFzKPuuR92oGb1Om+++5LxWmpHw+6TGuVRXKmsD0ousG8AC7KtYw8yXcq4SBXFCLk9gMPPJCy6Gv/fe/wviEn1pj24nJVgD8f20sRj7xdC+RbkuuU1+up/FWPZz/72TOQD9fUr1rls8uiKTpji9xTHUpHd0bioEdmog84ueOEsQYnh2+nXJK/ACwAWkCuX5xyyikJoMW74HGuJ7tTS8CsUnvxLXmJZosO6AS4C5AXlEda9XScA8wC8I5zGPClxg1P54C1XC6LP9SFejEWKxqf59FS974+1qrjOT9L8k75Lmoe4e9Tf4cvQ07dGjjI9RfXd/z6FN2MsgzpNV6HRfCsdU5AWfvahJdTvJ6HL63fiHLWqMTr2py7BOAi35NOOqnbZptt0iu8f88zh6uVl+stuoXL1qm6TE0n8XFy2QAul3nXXnttijbkPELOAWKiTMg95sYcW6kE4CKvQw45JG0vxznzPL49gL7WMZR8Sm2Q6yJvjyVAHOlKAK5WntV0EpWn78jYB8iQNsZYxjjmfULPztM3+uwbLTpJqR17xG7aETqG2zuox957752iXnLuC1Fc7pXk8yIBXER0Yl4CnxnzWYzkW3tSNo8S5vM7L6fkcwv/eIfmB7ldYKgtzTMe8N4WGhpHSgCu1nIump+tskARaGkjOQCWtoMc0zbgywZwtcqllm/NM/O8rzb2tcz1xpb/JS95Sccf3wrAMG3oyU9+cnq8VZ4sI8+x9Yl0wYHgQHAgOLB4DgSAayRPA8A1klGRLDgQHAgOBAcmc6BkmPIJuwO4/DrGU0AIMtw4SAQnECAA3cM4AYBLBh+Mh3JK1wyJpXK58ac0qVyE44UxFwc5ZQXEoHJMZuyCHqAc8JIjJOAHxuNnPOMZyUlHtDCRr9DVNd8+UteItHDhhRfq5+zowCLCnRNliUk97wM88bSnPS2lzZ1b4lNpi4F56nDUUUelevJS3nnBBRckByrbxREJDGOiyEEk7nAGtASwTaAUDBPHHHNMirLGs4p0oXz6jryPNgs/IIzGAEkwUgKiAQzDd9lll11m2ZTAcM7nUluePdxzQlvFKarIGBj3AeawbRyGXxwobM+Es0x0ww03pJXi+t13rBlC5YjlWcBBRHeS8fyggw6abcnnYDH4QfuBKCdtmm0rAdogVzD2qI3T3jBicYQkIziHx2eccUYCovEtcJrgyBC5Q1PXSseWOpCPyuJ1K+VfuubRBjDQA3RRhD+cC4DcxIMhAJdvG4kDHwebtpeEL0euB0LRFiFtc+EOHwdwkcZla8n5TpqcHCRG38RITbmpm+dXat+tAC7qJucB5bnqqqu6q6++OhUNJwWGU8L3Q7ks0rfjHmAsIibAE7afBYgicgDX1ltvnfqYvstXvvKVjqheyFQAGnxTEe1VwEN30LQCuMiXyTIr8QFp4eA5/vjju3Xr1qVX5pHcWupHRv6t2AaN6Jg4CCHaEXIA6pMdMi4jB3DoAHSBanUvgZHSA/ZvGXla9qNPHdjkAK6+9t+XufNbTqwx7aXEM9o8gBuNAYC0aJ/0QcZAgES0Xdom9aA91eS6gyMBPJ999tlJrgMGItKl+gB1E4CLc293U+Rzi87YIvcoY40cZEAa5BiLC4iAgaxB/3j84x+fHocnAia6k4woU2yxxbOAS5Dj6AAidzA6IPTuu+/uzjnnnJnTn2igb3vb21LEUp7FMY1sL7UX7gs4xTnjMLofOvfjHve4JAelF1522WXd17/+dZJVie8pexORGYnQKIJHtA3aGGM3CzSciGS6//77+6WuBHhoqXtfH2vV8ZyfpbFpEfMIZ4a/T/2d+0NO3akALvJcll7jdVgEz5AlrfOavjbh5XRet/Kl9RvxLWokeek6pM+tfc5dA3ARCRkgpWTyxRdfnLan4p0+t5gyh6uVl+sqM+djdadctk7RZfw7uk7i4+SyAVyAtpGxGj+ZWwA0gAAMIauR9xCynKjVIv8GRExiDjhENQAX7yc/2j2kbcE5bxlDeU7f09sg10XeHqcAuFp5VtNJVJ6+o/NN6Up2EO75d5nSN/rsGy06Sa0du15Bf0HfUFRVdArGYfV55kECm3t/KcnnGoCLNgVPyNOj/ImPtaPbZZjzAdYBNIyOQBnp+6IhAFcL/8hbshndFj5hT0DvHdOWWscD1zkUrUj17DuqrLXtyUsALvJrKeei+dkqC7zs2MiQn8wPiax/+OGHd4xhIrdHDr2vJZpiq1yifC4zxsryed7nfdnHvpa5HuUX+LqvfzNXQO9FDmDjYC4yL4BrGXlSn6DgQHAgOBAc2DQcQC+F0GXwgzFmoPfxx7l+jy3d76wPcb9huI+xT67ydDKorfJiRvGCA8GB4EBwYA1yoGSY8gm0G5NxaDGZddK2ZkwucQJhmBLhCCc6EgO6KHcO1QyJpXL5xLZkpFqE48VXd9dWsasuK3V0gwXvFFhGfMWRv9VWW6XiYMwiCgqOQSc3pnDdjX+ejtD3GIWVN/nxHeUMJC3XMNi5Ubr0vTzf1jrwXtoV4IUSwQuBqRzARVqPksFv0kJKz3kO8uDaEOHQJvKOeNSXHqcvkcowcjq5UajUlj1t3/mee+7ZHXrooX1JZvcwcLJqeizVDKEAg+iLXv+8TdJGAP9hJIfgOUAD/44YlBQ9jTQ4E3Sf/ADYsM1U3iZJ68/6+VgAV0sdeG9NXnFvDOH83m677WZJ4RMkXvKb8yEAF8/g4MeRBPEc/IPPLoPJBycU/OwDcCG/aZPqGzfeeGNH9LQhkmFa6RTta0hWtwK4eM8BBxyQQFd6J3XHcK+ycx2ZRVtHhos8Ko6u+VG8dwAX99n6CydRjfQcx0UDuPROvcN/Y0z2yIKt9fPxXvkryo5HcKIv1raa9XETozNATmgeANcy8lT9phxrAC7yqLX/vvy9bwhk4A69qTxjm14A8iLaCiSZwrmib3Fek+tPfepTUyQ60pTI26ADuFrlc6vOOFXuleqiaznIQNe9rlxDfqLz4LSDAEUDHnYek0YyiOcZlySLkSm0FQBQjCEYuyDScY3nfCyUHCVNqb1wfd16ICd6lfLiGnLQfwP8xIEzRA6wJi11ocyMx15HRQXz/KgjgE2lo06lqBxEi51ad97T18dadDznZ0n3WsQ8wvnj71N/93rVnLotAK5l6TVeh0XxrHVO4LwTn9VfvJzO61a+qO1N/UYqV+lY0iF9DPY5dw3ARb5sewqgAKKvEjkDHTCXx/RH7g/N4VJGlX8tusVY2Ur5cl3G+aEioZMgKwWiXTaAi/d6VFx+U1aX61wDNIuODY9FkiH8ZtzQ4iHdLx0diITuynxNxDc97rjj9DMBhpm7to6hpTY4y3z9ifN/CoCLPFp4VtNJvEy1cwcKkKYEMtazrX1jyL4xVSdxfc/bMeMtupX3VcZixlYf1/Ot7l3uleQz7URzfwdU+Va2vIdISWOI/GhDeTkpI2Wln0gf8Pd5OV0+T+UfZfS8+K0FLWPaUut44GD1KfaUoXGkBuBqLeci+dkqC5xXfB/aBKR2gbx0+7AiyQ69z/WGUltnLCFfFlQwZoha5BLPtshynmt9n9effCDNx6fO9XhWsmuofysiOP2I8WpeABfvXkae5BsUHAgOBAeCAyvPgQBwjeR5ALhGMiqSBQeCA8GB4MBkDmh1jgzgZEBkEQy2kBuT+e0rPvnNNlxEc4FYzYdRRRFQ0sWH/mFUY1U+f04CFsn4onulcmHAxAAHlSbummijYGBQdfIoVES9YYVmiRzksFoAXBj1iEjEqignDCJEiiAqF4AsAHbQz372s2R08LQOQpChxO/7OdsLEulFK379HqtBMejec889fnm2ysvbkSeYpw4YY1jVudNOO82MmNSdelB3GR09DLvezTZCAK5y4nlWMn/84x+fRcDI0/T9pp0QUaLU1nkOYwmRj4haUyK+F8ZTqNSWS8/UrtEniFaj75+nw6kLkIrvNoVY8ctKW0hATT1PG6GvCzio6xzp66effnr34/XRspxqz8Ar5AhtGcevDNVf/OIXO8AgEFFQaJNyiCtfHFZEcuA5aCyAi7S18nCvVoeavOKZMYTxEp76CmU9R+QjQCu0cwdw1eQxaTFmspVoiVidTVQAbflGOsoP5RG4uLbP+ihgAt7QP0pOeNI5YVBEtuib3Xvvvd1pp52WnE19sroG4PIV6GzDoSiO/k7OAS4yRum9fp96M35gwM3J89c96kqEGsYIDOYYlnEuOAFuYdWw9zHayK233pq2MuI58hGAC5mFIRn69re/naKteX6cy7jJOTKMfuBOT5w0ACuQnU68l758++23++V03lI/nDEAC1k1LGL8ZBwlUqGiLpbajNJz5BnaJBGJNFbV6l6KJuV56XwZeSrvsUeVgfQ4rX75y1/OHq21/1mCwonrMXJijWkvfTxju5UjjjhiBiDSa2lT6FwAuER9cn39YrgECJaTRc8gywGWCSjmAC7StMrnFp1xqtxTHUpHB60ReQv5R39wQo4wxjgYlPvI5YMPPngjGYQuC4B+2223nenRpEceAHqnj9NP+eYlQrci6h5AAajUXvQc/YwxwOUS95BF6LkXXXTRBsACPVc6EmUNp1v+7UmLDoEuo2138+eRX4ynUB9obGrdyW+oj03V8ZyfJd1rEfMIyi3y96m/cw9AO+2/FvGyBcBFvsvQa7wOi+LZPHOCWpvwcjqvW/nS+o14X41KOmRNx4PX69YDNZEFAEad6KeUTzq4j7W0galzOM+7dD5Vt5hHl6npJMhW6VUOfHH+MadjMY/TWKd/yXbQp2sCoEfWMxd1kgzhmuS+3y+d06Y1vuYALtL7+O8yo2UMLbVBL9MQP0lLGflOPldRHlN51qeTKM++I31Dc/HSVtL+bEvfKNmjPM+pOonre96OyZPvSXQ39IecGNeZE7OYwsnlXkk+850WCeDi3dgvsEv5vIHrlJFI6QBM0UscwOXldPk8lX+8hzkX8xPNj5gXMY8Z25ZaxkkHJZXsbJSrREPjSA3ARV4t5VwkP+eRBYqinPMEmwTAote+9rUd24FDzFXQI4fetxZkueo7VQ7yXG3sk+N8ylyP/ATgKukQ3BcB2MKmBzHfQ++ApsiT9ID9W0aeln2cBgeCA8GB4MAKckDjUETgGmB6ALgGGBS3gwPBgeBAcGBFOYBxgAgEDOQ4a3DuOD3qUY/qcIxhUGOQZzJIiOyg+TiAwQpwB6seAR8BlsJYBgFsATCBMf+HP/zhRs7GqW/GOUCoc74jBjqc1kT2YtXvPDRvHSgL9dfqZIwMMjD4dmpeRvQojEQ4mTFiADAAsIHBb17CQEmd4BVRNCgXDlhtbTBv/lOep78BthCPcCrgYMCgnffRKfnW0gKegf+0O8AvOKcx1npktvxZntl1110TvzD8IxcwhIr4Vk94whNSe8YB7QAe+Et75D7GPvoA8mceaqnDPO/Ts8hQDHEAAVntSF3gRwvRrpUX7Zs2iAxAPqwEYXDEUEd/Qtb7N1vm+zHc817+kHuAx9jKxgE2pfdLVtJfkAW02RLYq/QsbZDn4DHgQeikk05KIC45EErPjb3mTk9tfQlIgy0eKTd1xFHb159b64ejgIhuyC7vk2PLvqWm21Ttv8Rvts9DRiIfkSs/+MEPZkCgUvrSNcZ+5Dr9Cv2N8SwHL5Wem0c+t+iMy5J7yGYcL/RvQJICwJbqzPjPeIZMIDoXwAFteUR6nLDoyuQFAE79Fl7R1/lWOELhMzKFNMikqUSZGfsZP/Xdx8o0f5fqA2+Rr2x1Tr3gQ19+AOy1bTSgYeRwjVrqPtTHlqnj1eqxmq9vKr2mhSetc4KhNlEqy1riS6n8U64tYw43RbdYhC6zWnQS5pw77rhjmuvAA8ZDwPt9uiYAOsZQwEWMx8umljF0mWVq4dkyy+N5L6NvkP8idRK+J/YedDraG0Av5nXLaEsAjJg7Em12Km2//fZp7Ien6Jvoi61lbOGfFvZhd2C76inUMh7Qrok46oDZKe9sSdtSTt6z0vws1Y12zFjAlom0Yew6bnvDfoQOTBtnfrtsapFL88jylvfBg6GxbxFzvWXzOvIPDgQHggPBgc2HAwHgGvktMUoFBQeCA8GB4EBwIDgQHNjSOEAUIBx+AJHOP//8jarPVoZEDYHYqnClQCsbFSQuBAeCA5sdB1hxDZgDuvjiixNAzSuJcZbtMZBRAJ+IrDcPlZye8+QXzwYHggPBgWVyAMct25oRVaQUjWWZ7468gwPBgdXJgS1ZlwHkBSgG4C5bWwYFB1YrBwBgETmYCNhE/gzq58ABBxzQPe95z+suueSSBOLsTx131zoHQpav9S8Y5Q8OBAeCA8GBRXAgAFwjuRgArpGMimTBgeBAcCA4EBwIDmxWHHjf+9432x7ozDPP3GDrRqIOsa0HVNr2bLNiRFQmOBAcWHEOAA4FJAr94he/SNuQKIoOq6L/5m/+Jm0byP2rr766IwrgPLQlOz3n4Vs8GxwIDqwsB4iARLTFAw88sNt3333Ty9mq07frXNkSxduCA8GB1cKBLVmXYYth7PdsX0wk1aDgwGrkANs1YmMBhH366aenCEmrsZyrpUy77757x5bNRCtj7qcozKulfFGOxXMgZPnieRo5BgeCA8GB4MDa40AAuEZ+swBwjWRUJAsOBAeCA8GB4EBwYLPiwP77798RBUdEiHzC8xNyneg3ENtInnPOOWmLR6WLY3AgOBAcmJcDWn3LEcJwz9akbC+GDMLxAREh8J//+Z9nW9qmiw3/tmSnZwO74pHgQHBgE3HgX/7lX9KbJQMB0b/zne+cbRO5iYoVrw0OBAdWAQe2ZF3mrW99a5qP3nDDDavgS0QRggNlDgDCPuGEExLomi0Bg/o5wDbRz33uc7tPfvKTacvr/tRxd3PgQMjyzeErRh2CA8GB4EBwYF4OBIBrJAcDwDWSUZEsOBAcCA4EB4IDwYHNjgNHHHFEt9tuu83AEl5BwFvnnXded8cdd/jlOA8OBAeCAwvhwKMe9ajk5Nhqq62K+S0KvEXmW7LTs8jcuBgcCA6sSg4A4BJ4i6iEF1xwQfed73xnVZY1ChUcCA6sLAdCl1lZfsfbggPBgeBAcCA4EBwIDgQHggPBgeDAojkQAK6RHA0A10hGRbLgQHAgOBAcCA4EBzZLDqAL/X/2zgRst6n8/4uUqMtPopA4SubMcVAyy0wqKpUQSmhWpyQNmjSojJE5UylDOKaSZJbhZCqhTCVDrlKh+p/Pcr7P/z7rrL2fvdezn+d9zzn3fV3vu/ez99pr+O617jXc332v9ddfP0CmgEhx3333xS+cf//73w/s9WaWBMwL5Qg4Ap0iwJatyy23XFhggQXCU089Fe68885IHH3yySc7SwfPXquttlqM76677gqPPvpoZ3F7RI6AI+AIdIXAxhtvHOaff/7w0EMPda4Hu8qjx+MIOAJjg4CPZcYGd0/VEXAEHAFHwBFwBBwBR8ARcAQcga4QcAJXQySdwNUQKA/mCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKNEXACV0OonMDVECgP5gg4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjRFwAldDqJzA1RAoD+YIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao0RcAJXQ6icwNUQKA/mCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKNEXACV0OonMDVECgP5gg4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjRFwAldDqJzA1RAoD+YIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao0RcAJXQ6icwNUQKA/mCDgCjoAj4Ag4Ao7ATIbAG97whvDSl760KNeXXXZZYEC94447hjnnnDPcdttt4aabbiqKq4uH5phjjrDllluGueaaK9x9993h1ltv7SLaMYnjhS98Ydh6663DIossEuaZZ55w2mmnhfvuu29M8tJVossuu2zgDznnnHPCf//7366inunjoQ3SFv/3v/+Fs88+e9yUR/kiQ1dffXV4+OGHp8vbKqusElZfffWoQx566KFw0kknhcUWWyysscYaMdz5558fnn766emeGcsf/cozlnmbndJGD6y22mqxyD/84Q9np6IHdPv2228f6K9+8YtfhAcffHCmKP9Y9K9bbbVVmG+++cIdd9wRbrzxxpkCp0EyudRSS4UVV1wx9o3nnntu7A8GiW9WeNbHDbPCW/QyDBuBddddNyy00ELh8ccfD5dffvmwkwtrrrlmQF899thj4cILLxx6ep7A8BB40YteFDbeeOM4JmGe+Zvf/GZ4ic0CMY/FWCgH28tf/vKw0UYbxVusx6Tzs9wzw7o2M4zpV1hhhfCa17wm/OMf/wgXX3zxsKCY7eJtOk4nHOtz11xzTWC9oAsZRpxd5MvjcAQcAUfAEegOASdwNcTSCVwNgfJgjoAj4Ag4Ao6AI+AIzGQIfOlLXwos3pbIiSeeGElSX//61+Pjv/3tb8P3v//9kqg6eeb5z39+UF7uuuuucPjhhw8c74QJEyI5DSPFE088MXB8TSJ4wQteEL74xS8GjpKTTz45XH/99fo5Ux4/8IEPhKWXXjrm/eMf/3h45plnhlKOhRdeOMw777zhqaeeGtMF7TaFe9e73hWJUJCdPvGJT7R5tJOwVZhBKtthhx1iGj/+8Y/DFVdc0Utvm222CRtuuGHvN5PrAw44IOy88849AtdBBx0UDYq9QCM6KSnPiLLmyUxFQPUdMD70oQ/NVphgQNp7771jmc8888xw5ZVXzhTlr+tfh9VPfutb34pG5TvvvDMcccQRMwVOg2TStgv1kf/3f/8XCbKQe++5555Bop8pnx3VuGGmBMczPW4QGOt2+uUvfzl+7PHPf/4zfOpTnxo6Lp/5zGfCggsuGMfZkyZNGnp6nsDwEIB8s9dee8UE7r///nDIIYf0Ehvret3LyDg6qRsLDSObTeYzfAhx7bXXDiP5RnHasct4HdN/+MMfDksssUR49tlnw8c+9rFG5fJA/RFoOk5XH8U49tBDD+0fcYMQw4izQbIexBFwBBwBR2CECDiBqyHYTuBqCJQHcwQcAUfAEXAEHAFHYCZD4POf/3z0cFGS7VmdwIW3HggpCF8kn3DCCSUwtX4Gj0YshiIYbf/+97+HU045JXohaR3ZOHpgVIZYSHwssP/tb38LBx544DhCoDorBx98cCSd3X777eGoo46qDjikO1WY1RG4vvCFLwTNE//1r3+FP/3pT+Gwww4bFwSukvIMCVqPNoPAzGDsyWS7k0uzGoFrmP1kU8NQJy9mHETylre8Jbz+9a+POZER9CMf+UhYfPHF41gAA+TsJqMaN8xuuHp5u0VgrNupDNlO4Or2vc4OsdURuMa6Xo9H/EdN4Goyn3ECV/+a4gSu/hiVhGg6Tn/zm98c1ltvvTiW3X///Tvxzj2MOEsw8GccAUfAEXAEhoeAE7gaYquF+YbBPZgj4Ag4Ao6AI+AIOAKOwEyCANs54a0olf322y/w5S1y7LHHhgceeCANEp588snoHYPFRWSsPXCxrcFOO+0Unve850Wy06Aeq4ZpmJ4BTHPBejCCNDarbGex8sorh9e+9rWxpCw2D2sLxarFbgPxuDq19ezoo4+OW5GOOoNVmLFFxyabbBKzw3ZvfJ0v0aIteuCzn/2sLge2qVh11VXj+z3jjDPi1869myM6KSnPiLLmyUxFwAlcM58Hrqr+1eqvronO0jGziweuzTffPGy22WbTkbVmdwP6qMYNrpgdgUEQGOt2yjiN8dqjjz4aLrjggkGK0uhZ98DVCKaZIpATuNq9pqqxULtYmoeums/YD2ycwNUfTydw9ceoJETTcTre/vEuT/th290utt4dRpwlGPgzjoAj4Ag4AsNDwAlcDbF1AldDoDyYI+AIOAKOgCPgCDgCswgCn/70p8NCCy0US/ONb3wjetfJFc1+iTrWBK5c/ga5NkzDdF2+3vve9waMlghejjDIuDRHoGqxu3kMow0pryv/+c9/wkc/+tHRJj4ttRLMvv3tb8en//jHP4ZvfvObY5LvqkRLylMVl1/vHgEncM18BK6qWjDMfrKpYagqbzPbdbwT4FHA9gVjTQyZ2TD0/DoCY4HA7NZOncA1FrVsOGk6gWs4uHYVa9V8xglc7RB2Alc7vJqGbjNOZ2vwV7ziFfEDUPvhV9O0cuGGEWcuHb/mCDgCjoAjMDYIOIGrIe5O4GoIlAdzBBwBR8ARcAQcAUdgFkGglMB1+umnh7XXXjt6WlpggQXCY489Fu69995w3nnnBbZZywmevjbaaKMwYcKEMP/888eFnTvuuCNcffXV4a9//Wvukey1OeecM7ztbW+L92666abeloNrrbVW9ApEXn7605+G5ZZbLkycODG88pWvjN668CqEty7r6QpPXi95yUvCMsssE+P7xz/+Ee6+++5AvDfeeON06a+xxhqRcMWiFMJWcrfccksMxxaIVvBMxB8TkTPPPDOe43GD8TZfIxIHaZI2Qp7+/e9/R/zYSlGy2mqrhTe+8Y0xHCQ6tk154oknwqWXXho9oSlceiTtddddNyyyyCJhrrnmiu+E8pMXns9Jm/Llntc10qZslOcnP/mJLscjedl+++0D24uBBd65ePe///3vY9lTHKd7eNqP9ddfP7zqVa+KdY8vPIljypQp4eGHHw7nn39+WHjhhcMWW2wRQ59zzjmx/LxniIrXXHNN4JqEurHllluGl73sZWGeeeaJrv55ZzfccEO4/PLLe97D5ptvvgABC/nLX/4S86o47HGDDTYISy65ZPSsglcq6pPkoIMOit7u/vCHP4TvfOc7sV7iTQEj/mmnnRbbhJ7HW94jjzwS6/ZFF12kKGY4brjhhhEH8g+2eMmCaMV7tu2wH2aLLrpo3PKABKif1JE3velNMU+0IYT4aBfE/+tf/zq2LxEQec+8bytt6yD1Yeutt4748bUt7xX8fve730VPExbLkvLYvFEXeO+0Q97tM888E/H+1a9+Fd+9Dcv5tttuGyCwoOMuu+yygEEFL3PUNcoNJnguQye0laZ1UPHivYd2jbfEyZMnh6222iqgJ+aee+6APreCrl1nnXXi1mzUqT//+c+xrdAObP2wz+DZkHrJuwUb2thTTz0VHnroofCzn/0stjMbvu48JXCBG3nFgwiYkx/wpB/ISWlellpqqfh+eWfgQptGP1xyySXxHebSol/ZeOONo26ifpA/6h59Bm22ShZbbLFA34NOIg7CXnvtteEFL3hB2Hvv9gSuUh25/PLLxzZM3UCX8c4o87nnnpv1rFlVnlz/2qafrIqX9k2ftPTSS0fdT39Ev3frrbdGYij1LOeBi35vu+22C9RlxhoIZGe2oaU+1klJn9am7pTqBdrAu9/97qg7jjvuuNhGwQVPqQjjCuof3jboH+rkPe95Txzf0GfRjt75zndGrOhX6WeslODBe9t0000DuKCXGYNQvxmDvP3tb4917brrrovvkbR4x/T/9OWULRXaJLgh9DUPPvhgPK8aNwy7fDHxin9t+qSKKHqXGcfRNukj6D9XWWWVsOaaa0bdDKb33XdfHLdq8br3oDlp8/7Ak7/cGPSkk04Kd911V9hll11i7OTn3ql9G309aVBu6tIVV1zR88ZJP0KeKQO6hfB4gKoaT7YZu6JzlJerrroqjsEZL1PnFlxwwfD444/30rPjAANNHFs1nV+UpAf5BQzatlM9R17fWcfnAABAAElEQVQZX4C7FfQAOp/2znuxQj/EmAih72KcQd9MG6Ld/PKXv4z3GEMMMpYED7Z0Ja+kCd4333xzHAt97GMfi++Adz5p0qSYnv3XdAyqsS3PMn+075HxAOMxhHKlHluod9QnxoToxKeffjqG7fevTZ8IpmCIMJbjfaDLVlxxxaj3GKug56ifVdJmjltSB6vSbXo9R+BS/Wxbr5VmP/1c2h+UzOeVJ46MNdFnrDWANXqKMTNzFVv37DPpOc/l1hoGbW9pOuv3mdOmBC7KwfoL/azG9bfddlvUEWnc+s34qe1cQM/a41iN6ckDc1LaI+NA5gaslTAOZL3n2Wef7WWzjsDFfIB3yrgSsWMQfqMz6JeZEzOfZoxKu6ffY85HmugHSb/6r3Btx7Fj1W5Kx+kqp46MPcAGOeSQQ3rjCN0vad/DiFP58aMj4Ag4Ao7A2COgOTB9Lf08a0OMxfjjXL+b5nSOqQPB6S00TZ8c5+HorF0cAUfAEXAEHAFHwBFwBGYfBEoIXCweQlxgYJ0KA29cp6dkDhbgWfjLPYOh70c/+lG48sor0+iyv1kI40tVBGPI4YcfHs8/8IEPROMKBk8MJRjMcoLR9+KLL4638DqWy9M999wTDj300BgGQzxxswCaEwylLFBZQgTGKMpM2SDgYIySYBgR8UXXdDz55JMjYYDf+tpQ99IjBnDStcLEZtddd40Ljfa6zsnP8ccfH40zulZSPj2bO+o9cI8yYAhBqDN4ntLCabxo/v3tb38LfOFZZRBU0P333z8upuq3jhh1PvGJT0TcwR9hcRbswQVhkZvtCxGMyBiVqoQ6TF2mTjNxBGuOYPipT31quvetOFSfCEM+ZWh68YtfHOMi3CmnnBIXhO2iPPnESE78qYjwZa+zaL/PPvtkcSAcbQAjGYZ2pB9mNi8//vGPo+H2q1/9aiS/xAjMP/A44IADgt0CFHIaRj+kpA5C6KPe6D2Z5OIpeB511FE9ok9JeRQnRoy3vvWtlWlhSD/iiCOme79f/vKXI1EB8h7tfPHFF1d0vSPGxe9973u1ZJ9e4Gknbeqgnv385z8fiVXgjYFhiSWW0K3woQ99qHcOUbJKB1IvIXegQ6xAICF+dEKVQLiyJMiqcFy3xh7IULznnEAoO/XUU6e7VZoXjAYs9OeEegTpLTUQY7RmO2HaaU4w4rDNbSoYvXfYYYdsXaLdQupCIFQ26d9KdSSknde97nVp9uJvyozRGUJpE8n1r9Jr6fO2n0zv2d8YPMEXokIqEAMpN20/JXDxHCQ4EZvSZ1ngO/LII2eox6V9Wtu6U6oXMJaj7zAkU/chQeQEPavFzNx9rslDIuMKjPEYKBHiFqGzFA/w33fffbN9NuRAiMP0Wfa9yXhKHqw+4jcCsX7PPfeM57RDtcWqccMwyxczUfGvbZ9UEU3v8uc+97lIJoAIw7tB56RCv42egcBnpeT91Y1B0bV8oKBxNO0YImpubIZxHKIA7y0Vxkhf+tKXYj9k77Udu1qdg64VMcHGyTm4kR4YWmk7vyhJb5tttilqpxDiGCsh9IGHHXZYL+v03bQXSeoFmb6FsRmibbelcyD9MQ5F7Pit7VgS3cq4WeTYGOG0f7xfxu/0iymBq+0YdLfdduvNSyDR8IGExJaTNpB6qKWckLzoy7jHWKuftO0TIWeQRwRSMUSR3HiUfp05pyWM8Mwo6iDpDCI5AldpvVY++unnQfuDNvN58sR8nn4vV5+5z3s75phjenMJrlWJ1RN2rWGQ9pZLq818BmIt7zEnkJmYJ6VSMhdI49DvsRjTo3+oR5Asc4KOYq7OGgKiOse7hoAqQddBQuXDFIS1m6985Su9tlw1/kPv3Hvvvb2Ps6zO7lf/SadkHKsy8HzJOKptu1E+S8bpPJuTr33ta3EumauXGu+1zecw4szl3a85Ao6AI+AIjB4BrXk4gasP9k7g6gOQ33YEHAFHwBFwBBwBR2AWQ6CEwCUIWHjB+MqCqQyw3EuNFJCV2C5QAkEHL1l8Wa+FNO5ZY57C5o5Vi6paELLPMAEgPdKyRmBtWYhxkjxwH8FgQXgW8PEmhuD+XYvBlJkvxCmzvB4RBlIH4UTYkfGMeyz+yRDAOV9v8pUnXwbL2MrX3QikGwwEeNXBGwxCmnioIm48BuG1S/FBDsPrlMRuy0ha4IwnJ4xzIicQH3nFGIaUlE/p5Y72PVgCFwusyoM8RYEjJAfhQF4x0NUJxh4M4OAPDhhzeA5CC8Z8DCngnwp44EEKAylpQoDiea5D2uF5vrqlLpAvxJLkIBKIgGJJgErHGoDsc9zHuwBf95OWDFDWCKA4WIimLjAv42t+CXURDwwSGYP5zTN49aDu8p41pyMtFjtpo/0ws3kRgWuvvfaKdQ3DGUL9AyPigwRYReBqWwd597RHkYYgSUESpc2hV1Q3KCf1CSkpD8+ttNJKkeDIOcICPu0ZYzrlFIEuJabIaPrcU8/951kw4TnVF35jDG0iJXWQeEXgStOgHbC9E2KNl9QDMMXgS17xzoRw/bvf/e50hDNreEc/oIvAn3ahtsuzGD0gcPQTa+xRWIwseFBK9X9KcirJC1/Ov+Md74hJqX+i7OgK2oYEIzr9FIJXIeqf3j26nDpOO8JgJF2b1gk8IOy4446KsucdEf1s+xoCpGXrPZSclOhIizHvlPoMvvQVtCHlH09ceE3qJ7n+tUk/WRUveYCIpHzwXmg71EPb//O8JQKl74W+HD0HtpRN+gK9h4HfGvVL+rSSulOqF/DmgNEWfUpfIq86IrhpPACh2RLDcxjLgJjeo+9HVyAleKT4gzNtiTYBecOKfW+DGh6J144bhlU+m//0vKRPSuNIf9s+W/cYa1KvqQ/qQ2jDtBeuS0reH2MgxkIIcar9cc6HApDEROBSOhi8eceMTdN3TBgW1umH7RiUsSnEZUnJ2NXqHMVDPuljaBP0Fcp/qodL5hcl6UHEKmmnvFfI/+Tfkq4oJ9uosp2qBI9aZ511ln5GIgpjE9u3S+fYuOz4TQ83HUvS99kxI3oD7G3fR5wpgcvW5yZj0BVWWCG8733vi9mzZBguiKAVb079lxLZ2Lqb/pm6efDBBytY5bGkT7Tjd0VMe6A/pT3YvgqyrPUuOKo6qHyVHnMErtJ6rTz0089d9AdKq998nnC77757JN9xTp+F1zrqLiQaO4+nzjFerxOrJ2ydHaS95dJrM5/R85QN/ci43BKbUnJk6VxA6aRH27Z0b5hjetKwcx6N69F/EyZM6JGOGfOiyxDVOUvgYtzIepd0HWMs5sfEh9j+kt/EzxyIOqO5AdfRjcQv6Vf/03FU03GsykA6JQQu5Y9jk3ZTOk636aTnqitgzJjOjtHtOpGea5LPYcSp9P3oCDgCjoAjMLYIOIGrIf4azDQM7sEcAUfAEXAEHAFHwBFwBGZyBEoJXCx+sajOIiKCRxoWnDBSWKIF91hwlzEoNWTjRp3tmXiuKfmhalHVLgixyIaXLUtuskYCyDAiaLH4icEMST2t4GFFW+dh2MCwIOIT+SBOLQpbAlq6GAhZAOIQC1QSS3SB1KOFRO7rK0OuYbxTmtwDaxE1WJzmPSAY1QgLlpQfkgLGNQlfoorEcPbZZ4ef//zncduUkvIpztzRvgcZYjHAfeYzn4nBU5IWi6MHHnhgj7CEYVv1Khe/rmF85B2weMzzkpTABVkM7wXW2xALohOmLv4ifE3P4ryEhV55IbJ10noOScvAs5bgJS9bipOyg4EldqVGAIzgfBlO+0FY1CcMAqFJxlbaDFtXIUx0IbxZI78WOLlvjfj8rsLM5kUELsIjWqC2Rnqu5whcJXUQo6i2z2E7VbaTtAIhCgIRwnuhTJK25QErFtQRtQHFxfuhPcuQfuyxx/a2BZPRlLAYBTDe/fa3v42PYkCh/lEXkabkppI6SPzWmEE7px2je2SMQp9R39AD5BVPgnZrR7ZsYjsixOoPDJTEjUCwASvil1hPDVbX6X7uaOsi9/ECxXZdEhsnbf6Tn/xkTLM0L+hFeUezJC3Ss3rBGuOsvkoNsxAI0Zsi1lAPRK6x/RqkXDzVSNiiBU9vkrTf03V7LNGRkDwxpPOu6St417xTCV65MOAhGE/o52xfonD2WNW/1vWT9vn0HCKoPEZQD8kj9RJZffXVox4h/4jVMXvssUckOnMdrxOQcyW0Uetlgy1u0LlIaZ9dUndK9QL5h8CJ/hYRUunT5qyBUGWuOko/c586AA6MY9R2S/GwHnMYv9BPKs6UvGjfWxeGR40bKNOwykfcVTJIn1QVpyW8gCNjSTsmsWNx+xFE6fvrNwa17Zw8M8YAa40/bH65T79M/4ywbozXTcZujGk1tuNeydg1zQtjOjxEYpRH7NjLGuW5Z/Vwqmer5heDpFfSTu27xQOMPIjZOQlloW9Bp0ggcKIb7fhPOqeOwNV0LMk2XXgjRcgTdVKY0/fRlkWgtwSukjEo5SB+6oyNi7S5rnEXvy2RjW00P/jBD3I5fsSgeVu8kPlX2iemBK6UKMhHN/RJCO0XsrWwGnUdzBS70aUcgUsPltRrnu2nn7voD8C76XxecwL6QsaT0mfk1epEvH7jqbtOrJ6wY0Y7X+L5pu2tLi3uKe/pnDZNL50nsbUgf4idY5bOBWJEFf9GPaa3Ooo5Dmscmu9C4qcdiszP+JYxleqc+grWn9C7+giFDzTAWoQiS16irlGn+agLYb7I+ozWebhP/JJ+9b90HKsykE4pgatNuykdpwuH3BHMeF8IxGRt+ctvO+9qk89hxEl+XBwBR8ARcATGHgEncDV8B07gagiUB3MEHAFHwBFwBBwBR2AWQcAaFlhEt4Z+W0S7kMliS+ohgLBaxLYLXGzhhft+BK9WkCJSsQtHePbBS1Kd2LzYRVW7IJRLyxodLVGrzjANGQNCD2XCoMXCqhUWBik3AlFIi1V2oZjFQrajS6WOwEV4DBpTpkwJYJKKDB4QLVjAROyXx2zXhTHLit2uRV4TSstn403P7XuQIdYaRyxpRM9CKMFoikBikmFE93PHqsVuS9RgARcjmV3EJy6MFZCCeJ/6atemoS0tbF3mvoyTXMc4YIlmeicYDiB9EAbBAAbOGLDs19F2UZ4Jq0iE8aGp/1iUJj3ELuhbjwlVZCFLVKKNi7RRhZnNyyAErpI6uN1220XSBXhhFLYkR8rO/fXXX5/T6OnDkhLblAeyCMYHxBo54oVp/yZOnBgJpfzE+xRGZERGU87tNqf8RtgSTFtMQe6CCNRPSuugJXClxCHStETCXF4JY3W16jGGUnQxgnFLxvp4Yeo/yJ/ajgQSVpMt+ayxB8Lb97//fUXXO/LO5W0OY8i9U7cqKc2LjSvXNsAGIw51iHpOujyD5Nog160Bib4JfWzrUpV+t3lJiQXEm0qJjrQkm9Qbo+K3dZOtL9kCs06q+te6frIqPvpHdBG6D70IGZOjFbttjYhA5AHdx3PobvqRVOiXeccIehidjXCtpM+276tJ3SGtLvVCFwZ09BV6y0oJHhZ/+lDem4ydihtvOnjVQfTeOO/C8KhxA/FZA2lX5SPeOhmkT6qK1xKi2D6OsYAV9BK6HYILbQSjMVLy/niu3xjUtnP6XkhYGifwvNXduTGb5g48aw3aJWPXNC+Mo7WIT14QO6aRQbt0flGaHvkoaaeWqGz7Ao0ZwVA6Uu9d26ySpvX4Kp1TReDK9WNNxpKQxURaIE3EepaypKvSMajG1cTNuAMiBp6RVGbhYIlsfKwAYQxBR+O1sE5K+0Tb/6LzqN92fE+atk2IZDYWdbCu/HX3hk3gyunnLvqDNvN5kR4h+/AO7dwPT2ravpdxEMTvOrF6wq412PlSm/ZWlxb3msxn7DqD4mOOqTUGO1csnQso3tzRtoFRjOmtV9pc/bLbQ+qjHNU52jEf10De0oc7zP3Qu+gaidUZuY9TLCk07e/qxifUn9JxrMpAHtXfKb8cLanZ5tmuvzRtN6XjdJufqnPWNZg7sKZj11pK8qk0hhGn4vajI+AIOAKOwNghoLkfa8HYQZgbMS/mj3P9bprDOaYu7P7/3r7pUzNBOCdwzQQvybPoCDgCjoAj4Ag4Ao5AhwjICEOULGo1IXClXn2UHYxubG2EYOTA4GcXxmScV3gd7dZFN9xwQzjppJN0K3usWlS1C0K5stgv96xHjyrDNJMFebeq27pDC4x2Yc8az6oIHXUErlzBySfGjvWnklkmTPMeZQlc9l3mFjqJc9VVV43GbRYx+aq/tHy5/OmafQ8yxFpjO+EwwkACgSSiyZqeb3qsWuy2BC5L1OsXLwZU8IVMxrYiTBLtO+V5SzawZIi67WE22GCDsO2228a4MITLoGCNADmDLumJMGaNZzJQ2IV6wlrBYwEkGMR6kqrCzOZlEAJX2zpYpW8wSKBLMGJC7JN3BrZqKiVwWa9IP/jBD+IWUhYzzlnkx1CMWHxlNLWG9Rho2j/7Bbw10NowTc6b1EERuKib6FmOVmRcRf9yPyeWfGHrRxoWHcj2Xni12mKLLXrbj5QQuDAwQXZKZfPNNw+bbbZZvJzWPRu2SV4sWYny33777YG2hfEtJ7RzPMkheGOAuJaKNY6x5RlEDOvFrAo/68WnSZ0o0ZFqb9QBq1tsGegvIKQgN954YzjxxBPt7RnOq/rXqn5yhgjMBWsUr0rbelsTEQjvVGzbiNTpcOoU70d6epA+u23dIW9d6oUSYgh5kAHR6iuuI6V4LLnkkmG//faLcVThb8dtem880IXhUeMG4htG+Yi3rTTtk6ritQSuKiKKDJPEQXvGAF06Rus3BrXtPDWukj5beLMdIgLZgfGOFevRNWdgtmH7jV1tXvBIB2ktFbUPrnM+yPyiND2lTX8onZPmM/fb6k7NPSxxiXEw5HFEcxe2AqaNIeh5keukc6oIXF2PJbscg9pxkrxxahtJ3ufdd98dtym34yzIfMzd7Bg4glLxr7RPtH0VRDbG2amwhTqkGESEntI57iB1MM1X09/DJHDl+h/y1UV/oDZhy1k1n7dETwh4119/fbjiiit6ni5tHP3O7TvS++YZO19q0976pddkfnbttdeGH/7whzNExTateIq17aTLuYAStASuUYzpNd+tms9ATuWDCgRCGQQ31TmegSQKQQmhjtIPp3Mm6QzC0O/KmzG/JSKfpnq/bnxSOo4lTZWB81z/2oTA1bTdWN3XZpxO3vqJbSvUR32cZ9eJmuZTaXUZJ/MuvFAyVk7lqquuqpw7pmH9tyPgCDgCjsDgCMgm4ASuPlg6gasPQH7bEXAEHAFHwBFwBByBWQwBu3CVW0RRce1CpvVMo/scMfhh+EP4oppFeLtFSLpoFgNO+wdZBrFbx0y7NcPB5sUuqtoFIX3dbR+2BnJrcLTGFWustFsVEk9V/pV3wrDIx6KxNZ5hgGBRMZV+BC4W/9hGb6GFFuptEZDGYQlcMmYTRgauNLz9PUj5bDzpuX0P1hBrt9yzz2CI4r3zRbsl59gwufOqxW5L4LKeC9I4+KqWr/shw2Gcte9RYdPFWsgs8vRivQTYMqdbt2nhN9120S5Csv0a3pRSkfcNLcpbglFdW2FLQnk0s9uFVmFm85KSaLRAbdsM+bTvUwbptnVQ5cXwwJfUK620UiRR5RZTCTsIgcuS2nL6QXmRIQTjuTxOyWhqPf0oPEfrhaEJWUfPltRBEbgwMmBsSAU9LsJbE51lvcIRFx6nNtpoo+idSvGkaZQQuDBI5PJjjQdpvG3zgq4EE/oIK6RLe8U4gZ7RFiwyHitsLn/ck27gOeoO/RsGeMQa1+OFaf/slktN64RtUzauKh2p9qZ82Wd0DibkEbFbcel+eqzqX6v6yfR5+9uS8yzh1YbhXEY66RiRXrmX1k+uSSyJBF2JMY++T9LvfRJOfXbbusOzXeoFEVTIM22lqUg/3zvVc53O9WxpHw+ZRJ5TL7jggjB58mRF2TvmiHfc7MLwaMcNKlOX5Us96/QKZU5K+yQTxXSnGI7xAFP3fu12S/R16PjS+txvDGrbeW5Mv+GGGwY8RyEi29gC2baXGpjbjl1tXhgDUvZUrDeZQecXpemRp9J2Kl0NEQt9LM8xzJMg7smTrzw7qb6kHn6kc6oIXE3Hklaf596/8E8JVIOMQbE3yAsL5Oqjjjoq9qds10j/TNm1pSPjGPorjvS/OW8yyqM9Cue2faIdh1SRZNAJjA8Rzb1K57iD1EFb3jbnwyRw5fQzeeuiP8iN16vm82wbDflRYzbhw3geT4KQJXm/TcS+I7vWYOdLTdtbk/SazM+qxlE5ouWgc4Fcni2Ba9hj+ioP57l82Wu2ztnr9L3oHz5CtKJ5tiWO2vuc66OXtP+uG5+UjmMhMdsypP0r+WlC4GrabkrH6eSjnzCXp14zl7QfaNo1k6b5VFpdxknfQ/o5YU2G9u3iCDgCjoAjMBoEnMDVEGcncDUEyoM5Ao6AI+AIOAKOgCMwiyBQQuCqcpufI3DZr2GbQFZlvLHPVi2q2gWh3IJX1YKvNWRYApf1MGHTrzuXMd8az3Qtfa6OwKXFwvQZFg/xXsXXx4iMCJzLCF5F7CCMlUHKZ+NJz+17sIZYwr3+9a8Pm266acD4mxO7SJ67b69VLXZbAlfVYjcLqxgn00V+8IUsRV1hwTFdrCV963ENsggetUT6SQk+xME90rHbHBCPNQLwRXXOqKCFZRG48KoFEQmp+lKWezbuSy+9NJx77rlcbrRFxyAErrZ1kDxBhEF3YBxLRYZ2CHbIIAQutohacMEFs+/UpisDqV3M1zW+CCOeVEoIXKV1UASutK6RJ+oZ76CNaNsRyC/oKkhlqWD8ouwQEJCUaJWG1+8mxh67VZTiHSQvL3nJS8KOO+4YPXnkCGi810MPPTQa86r0rPKfHoW5jOvcz/U1XLceVpoSuHiujY5Ue6vyfkF8tr8cNYHLvv9TTz01XHPNNWRpBpGhUQQu6y2vysMZkdi+Bl0JYQnjbRux/XObukMaXeqFUmKIDIg5EkZpH49Bj3qIVBHoaFu8N0TvjfN+hke8gOLJErF9on2XdtwwjPLJm1HMRObfIH1SJrp4STpDOiQXzm4Xd/TRR8ftXkvrc78xqNULuTG9JXCddtppM2ypW0XgqtKpdWPXfnkBqxyBq3R+UZoe+Shtp9bDH+NG2gkfBOhDAJWF35AxNG60Rm/Sl86pInA1HUtar7G33XZboL7lRASlrsagKqeIbNL9kLfOP//8nvc1fjMf3HXXXWO22H6ZetpPSvtES+CqmjeQtuKXJ2qVp1++dF9z3EHqoOJqexwmgSvX/5C/LvqD3Biraj5PmnhKg5wPOSOd43GfsSxEHs0vuJYT+47s3NTOr5q2t1z86bWqOW2T9FIC1yBzgTRf9rcd0/Fu0eupdDWmtx6s7rnnnjhuT9PK/bZ1jvvM0XmXiPRt/DHtn3RQ1fyOYNLf6ZpA3fikdBzbhMDVZBzVtN3Yd9pmnG4xrDsXMZ33wNgOseO9pvm0aXQVpxO4LKp+7gg4Ao7A2CLgBK6G+DuBqyFQHswRcAQcAUfAEXAEHIFZBAGMl3ieQFjEqtrSzC5k5ow9PJ8jcGnxn/ssdEEEqBMWxTEU1InNi11ULV0QqiJwLbHEEnHxmbxg+GZhq04oGwtvSD/jGWGqCFzW6MpiIcZRtoFg6xUWGBEtOFoClxZwyYc8B8XAFf8GKV9FlPGyfQ/WEGuf4St+ti/DaEI+tLhKGLYxYzuzflK12N2PwMU2dSzea2Gf7SQhQ91888099/4Y2BZddNEs2cd67eHrTLbXZDERSY1ta621VvTyxT1LFOB3k0X5lMBlPRjUeeCSdwfSYcs0yodUYWbzMgiBq20dJE8ixHGOpzowJL+0ObbAgGiHMRkZhMBl9VPui9+YwNR/alvkBe8TiIymVQv8bQlcg9TBOgIXeWXLLb5QhsjJVqr95MEHH4w4W8M7C+20B/QOhhMMXdb7nIhW/eK2hgHqHu80FYgib3nLW+Lls846K3rh6CovGPLYXgWDkkivJCRjNB4OqfsIBuMciTLenPaP9882ipa8cOCBB8atWWw4zu32jG0IXIqniY5Ue6vzNmKJZLfccktg+9A6qepfq/rJurjYdhPCLlLlDdEaGkUEstu3VRGIiJN2jAEGwWgHgYsj0rbPjg+Zf/3qDkG71AulxJA6A2JpH48Hvq233jqiUdXW7TaLem88YI2nOcOc3dJzUAJXafliwWr+DdInVUUrAlfVVlA8Z8dO6EtIzaX1ud8Y1Lbz3Ji+hMBVOnbtlxewyRG4SucXpemRj9J2aselkD4gGUOClJcP9XkQjNlGnveHpNuhS+cMSuCCzC4yepX3JNJnrMy4U33moGNQygUWCERqxmUIhDXG4iJEQbRgy2XC1rWZ+LD5V9onWgJX1XbKEMuJHyFvePsaizpoitvqdDwSuJr0B7l+pI7AJVDQn3j1hewCGYgxlcSuHehaerR6woa386XxSuCiLKVzgRQH+3uUY3o+JsG7M8J4jj6xidgxyE033RTXbtAr+kgoHVNqnl1Hrq76AKhu/FU6jhVJjPENkqv/Xbab0nF6k3dBmMUWW6y3JnXCCSfEbdHtWCdXvn7tu6s46YOZ2+aEjyX7kf1zz/k1R8ARcAQcgTIEnMDVEDcncDUEyoM5Ao6AI+AIOAKOgCMwiyAwbAKXjBLAxXYZbJuRCoYEGXmvu+66uJ1eGsb+rlpULV0QqjJMWw8T+tra5kPnGDtf/vKXBwwqLAwi1kiREnf0XBWBy271ljP8WwOKJXCxDQtlQXLbNrIYihERgzkGm+9+97s9Dxpty6cy5I72PYjABYlimWWWiaQSCFqpWCwgqh155JFpkBl+V5GRrKEs9yW9Xfi0hmebgLZh0UKqvWfxx4gD0YeyIRjXMDxJWJicMHWLRkgfYG+l1AggDwBVZCLSsEYljNHUE6QKM5uXQQhcbevgMcccEwlc5A0DIYvkGOus2PYwCIFrp512CmwNhpxyyikBXZOKJQRgTMSoiMhoWoV5WwLXIHWwH4FL5APqLltNpXhSHoyV/CG0Ecpl6zxpsHhtxW4HUkXqsOE5t8aetF4prL5s5zcGXQhjJXnBOMMX7wgEQPSIFbyHoRe1PSckTXSFPNxUbZuEvmRbJ+JnCx6Mu7ZcIp3ZtDgnXggNSE6PxxvmX4mOVHvjXdPP5MjPlqR20UUXRS8nJtkZTqv616p+coYIzAVrFK8inFqjsvQx5CkIG8iUKVMCeiIV3gvtk35aX/aX9tkldQeCZJd6oZQYUmdALMXD4l9F7rAkL7033pE1nk6aNGmGOinPCYQdlMBVWj7SrhLItehQpKRPqoqX/l8eDKtInyLLiLAySPn6jUFtO++KwGX76pzOs2MnO3btlxcwzRG4SucXpemRj9J2Sr8DORy9xdgCAzTCmAxyAvqffgXh9yte8Yq4DT19uBXpnEEJXMSpsaSNy6YFThCWyLMIXPa5qvEQYarGoFbfCwfr7XS33XaLYxOuYTzHWy8kcxGnbP5y56V9ou2rqnSefUdXXnll7NfHog7myt3kWoq9xrc8W1qv6/of4u2iP2hK8KBNrbfeeiQbxznMu6zY92frsw1jz62emBkJXKVzAYtBem7HvsMe05O2dFTVRwpss65tVzWeUJ2zH7PZcOgW5gIi52gdjHE0W2oztkuFtsI4MV0TqKv/dhzVZhxL2ioD56XjqKbtxuq+NuN08tZUNG+VLrfrRE3zmaY1jDjTNPy3I+AIOAKOwOgQcAJXQ6ydwNUQKA/mCDgCjoAj4Ag4Ao7ALIKAFq4ozjA8cFmigiVEWPisN5PDDjts3BC4yKOIBJzL0MK5xHo3sQapfsYznrekJYw0LCoiLCDqi0AWBzEmWGGxksVIxKZpF1ZTT1CEzRlOS8tHfFViF+ZE4GLLJL6CRnJlmnfeecPBBx8c71ctIMab5l8VGakfgWvbbbcNkFEQtvRKPavZRdd0sVbJWyMRxlYMrdZjk8LJmxPefSB6WLGkqTZfcWvRkrgOP/zwgGHBCkQLiFAY3dIviqsws3lJF+W1QG2N9KS38847Ry9DnPOVNISftnXw1ltv7W2p8Oijj0ZvD8QnwZiO4ZuFc2QQApfdciKXFvFbI7G2FuS6jKZVBsu2BK5B6qDef/puySdijegXX3xx9Hz03J3n/lNX+eIc4xR1F/3LUXWVOp8Sv6hLGBHkrbGEwMWiDMQF0pLwlTXtHuM2JBxIVaRfkhdrIMcrnvSJ0uJosaHvw6MGeUJs+vHCtH9bbrll2GSTTeKvq6++OrCdmPU0k9u+EIwxOHNEcmSGeMP8K9GRtg9R3kyUEVeMeOhXBIIIdb9OqoyWJQQuPF6QPvWH94pRH0OcFetFSzqGLVOpozxHfaFP5P1YwVCLN0RE21JxXtKnldQdjH9d6gUZ0ClDzqDF9ZxIP1dtYVWCB/jzHMK4hHpjjeHghV7WVsh6b4S3xEU8CTEWkaA/0CO8V0QGV85z4wauD6N8xFslEGe0zU+un+jXJ1XFC14icNmtuhXebmlnt3kqeX/EuYvxdISuk7Fa6dl23hWBq3Ts2i8v5Nn2zRovl84vStMjH6XtlGctqYnftC0RtBjjMD5T2+D+fffdN8OWyNI5lnRlx29txpIieJAWnhnx0GjFenK1hBeNQQjbdgzKM+rfOUfkzYpzO07jN8LWipCPm0hpn2hJDLwXsElJ7IwV2GYXgVjC+xmLOtgEh1yYJgQunuuy/+miP8jlJ+ehZ+WVV47zaspQ5UVN+rSKEMSzEqsnZkYClx3vtpkLqPy5o53jDXtMT/q238xtoyqPfYTV+FbkJ0vg4j5jj5e97GWcxo8x8FCGiDTKuYiZnEvsxz/pmkDd+GSQcewo203pOF34NDm+6U1vCvyBH+sTzHfwjIw0bd9pOsOIM03DfzsCjoAj4AiMDgEncDXE2glcDYHyYI6AI+AIOAKOgCPgCMwiCAybwAVMdoGNLbnwfsMiDsZtFttXXHHFiCYGJvLTT6oWVa0BsM2CkCUPySiLsQLD8TrrrNPz6gJhAgKJtplk8Wn33XePJADyfMYZZwSIDUg/4xlhrKFBBimuW0M+hjy2UCFfGNAhfrAlhMSSBywZgvsYRzF6qBwQv2RI5+tTJkml5VP6uaN9DyJw2e3EMARj/CDvCIuH4AVxCqnaaiveNP9kyKIuYfhi+0rK2o/AZRf5+dKWRVyepR5gCGNRUEY04sZgx9GK9T6i6+kCuTUIYfDCy5mVUqObLR+GJjzbicRFHYEEw8Ixknogq8LM5mUQAlfbOgghCvyFt4yI6Abq+dvf/vZe+6I8xx57bID0JWlbHohmeKJDIAri6Q0MISRst912va/3U3KU0umKwDVIHZTxNM2jMGGrQAiGauuQh6SXIBDss88+PU99eESk/iAWG8hAPAc2bJX2zne+M+ApUYKHK7bm7CfW2ENYPFjhgY8jW9+RF21rc+mll4Zzzz03RlmaF3mwIRLiIk4J7RGdy7um3dNOEOsBDOIX9VEkI7ZAgcAlLMFeRl29B+K4dyrJ9ogjjoiEScoDKU6GXu43IXCV6EhreCGdCy64IEyePJnTQFvEgCdPL1UEnxjY/KvqX+v6SfP4DKe77rprr8+iX8V4DzGG98D74L1ILBHI9oP0Veh4SLIIuoFnpTcwtsvLYGmfVlJ3utQLtt9ka14M0Bgg+0mdAZFnS/HYa6+9AgZ/BNyPPvro6DmIPgZ9wJZUEvveLAEDHYWnT/pXrkMIEZmQZ7sgcJWWT3lPj9SpQfqkND79toZorl1yySXxD11Df0B9p00g1qtfafn6jUFtO++KwGXbbJuxa7+8gEmOwMX1kvnFIOmVtlPyutVWWwX6FIm8kei37fe4ZuuBwkjndEHgsh6JGDsfd9xxvfHV+uuvH+cb0rGWwDXIGJRy2I92+H3ZZZfFsSrnjP3wdqN0uYbBn7FXEyntE+14nXToc/D8CEmL8fT73//+MGGqN13EYs/vUddB27YtduSlTuoIXKX1ul//00V/0HQ+DyEfghZ1h76TsS3jfASCJJ6+5e2bMZvyXoWZ1ROjIHCpbadzWjs/qyJo8sEA5bfttHQuUIUH10c9pl9++eXjx2ekzXyENsm8hXfMOIRxM2LXj6oIXMxhWGOSbjn55JPjNvEQ0dG9un7FFVfEtRMI53wsh56U8G6IX6I6VDW+tn1im3HsKNsNZSkdp/OsvKSlhDnuSWzbBF+81g9K4BpGnMqvHx0BR8ARcARGjwD9JMKYn/kA/TJzY/441++mOZtj6tYL06/gN31ynIdzAtc4f0GePUfAEXAEHAFHwBFwBDpGYBQELhaNMZJrcYwFMLxpsPgi4RqGXbx09ZOqRVW7AN10wVdpaeFUv60XKOtVgPssIiJMLCTp4p1dYAdjFhdTqSJw4QECA4fw4jnSVHpgxUIZOCBgSf4hCOEpBsKBhLCIjcsaTblXUj6eqxL7HkTgIqw1YJIvjLuUSeUgDIvPkE+EMdeqxKZDGBlVrHEpJTARjjTxziOSE9fAU16e+E3edJ+8EM/ll1/OrSjkWVvLcIHypN5lMPiwtaJdWH7u6ef+N1mUxwsNRAy7KM/TqQEM4xt5UB0hTM57SBVmNi+DELhIt20d3HfffacjA4C3JurEx3udZ555OI1lvO222wJfYiNtywNJkGdse0jTA0dIppBNJdIPXRG4BqmDIg5VEbjIs/WcwW/KZHUG1yAO0Nbk2SglJfIMIqwIRxvR79y2oPEB8y819ugWcSserkEQwYCidl+aF0uMI17iI9+0ZZseW92qPdO+IO9YPZSWlbhSQljarxHG6hF73oTAxfMlOnKbbbYJG264IY9HAVv0gdUFlAdd0s/7FhGAA2QpxBot+a12wDli+8nnrsz4H3yps7a/t23O1gVLBOKd8V7S53iPIrqQWm7ry5I+raTuCI8u9ILVwUKxauyg+xz7GRAJU4IH7w1inO0XqVcWe+JG7HvjOQgNtv49F+q5/7x73bNjEavL7bhhWOWzeUrPB+mT0rj027ZtXeNo6z+/rTc5fiMl76/fGNS2864IXKVjVxm2KWsuL1yvInClehg80XdWb3DNzi/6lb0uvdJ2SpyQeuXxkd+WtMzvtL/kvdPPW5HO0XiXezZPVQSPqrEkROZFF120lwRYIeovVT+7GoMSNyQ2yGwS8gbJU2LbSpVuVdjcsaRPTAlcilflt78hmLHNpWTUddB6U2Ycw3imiZBPiLlI6hHb1iHF1UX/00V/0GY+n7YhyPq8Q/SBrdPM3/D8VidWT9ixkMWqbXurS8/2gYRTG2+SXo7ARRwlcwGeq5IUX4VL20lXY3rit7qf36SF2PfJOAGyJVJF4OIe26xDikboJyCHomM333zzsNlmm8XruX8qH8c2BK7Sceyo2w3plYzTwUpjNMZ28iiZw3C//faLHwRRr/kQclACF2kMI85c3v2aI+AIOAKOwPARcAJXQ4ydwNUQKA/mCDgCjoAj4Ag4Ao7ALIKA3dIDA4e8S6XFw4jHojVy8803xy+10zBaSOG69SjFb7YahNBivS9wHWGhj0XQO+6447kLff7bvNhFVesxos2CL8lBOmEBT8bJe+65Jxx66KG9nOAJaK211ur91gmLeWw7csIJJ0Sjua7bry7ZIkoeXXSfYxWBi3t89bnDDjv08sM1hIUvyCtssYhHLQnvRuQ3vtxkezsZSRWGxTW8RGE0TaVt+dLn7W/7HqwhFs8dGEblAck+wzkeVPhi+pFHHklvZX9DxsEDmohW8qqDdxa+JkVyHgy4Tn1ksVzEIK4hYHTeeeeFKVOmREKW6sMvfvGLgFcUK9Rx4kFyW91oq44cwYBneMd6h9QftlVKpcroRji7xUL6XFUbrcLM5qUpgcumDxFIk27y0qYO8v7wcsYXuVZoW7wHvHJBJJCnJrvdTkl5qt49aVOH2BLo3qlf51uB8IfuQlfhvS6Vtlso8nxVPvrVQXnpkHEnzYt+r7322rF+qQ7rOkeMj+gRuy0a16uMM7RJFul552z1hfRbrCeMttlUmTDapnoJb1zoWsJYKc0LaUAAk3HHxgkBBY8VtHErvFv0gbxV2XvUCfQmf6nghYh+DUOfFQxCePGhf0WaErhKdWTdu0av8oV8jkRs86zzqv6V+/36ScWRHtl2jjaORwgrtPHjjz8+GtVo35YIRDjeC17atKVw+iw6WUQ8e4/zkj6tbd3pUi9gQINMYTFK9WpaRn7L+0E/Ml0JHnimwIubth1S+rw3PI7uuOOO8VL63nhfGDktiYaAeN1kW0XiRCyBq2rcMMzyxUxk/g3SJ2Wii5dESkGfQEKGMJjqKN4hYyDIn6m0fX/9xqC2nefGC5BCIcIg8lRi82RJ5Ha8XTJ2hbzTb35hjfiDzi/6lZ1yVqVX2k6FnXQGv1PikiXYWO+6epajnrf9vx2/tR1LUgcZL1tPiEoPL4D0Scstt9wMHxEQxo4B9YyOuTqle+h6SJ4IdZ26ZMXG29Tbp32e87Z9oiVwMWanfWpuobhpu8xVb7rpJl3qHRnPtZnjDlIHmffIczWe/NLxTC9TyQnjZeo1ks5ZSut1E/08aH9g9YuKRH5pP4jtfxhf0pdQ1pwwNuMDjXS70FxY+47sWsMg7S2Xjq41mc9Ute8qAhdx17WFqrmA8pQex2JMTx5YD4HIlgrjWtokxF8J9WXCVG95Od2CvkOHau5v9RTevvDqpbkm8dHmr7vuujgG4v0w9rEErib1v3QcO8p2Q1lLx+kicOXwJl4JhC3mWQhzbN4R0qZ9xwfMv2HEaaL3U0fAEXAEHIERIqC1ZPfA1Qd0J3D1AchvOwKOgCPgCDgCjoAj4AgUI8DC2ZJTt+NikQwjJR6jMGThvWo8CAvCLAaxYMfiUkq6YqwMeQGjPQtVbBFz1VVXxfDDyD+LiKuttlokFbDVCgQfCCQSFvfweMCCNIuXkBOsLL744hFriBEQ8/DsQL6rZFTlAz8IPngjID8Y7+6+++7eVhdV+au6jnEJkgoENm3LWBXWXucZ8KVO4l0AQ5H9Ihs8Xv3qV0dyBtv2pfXBEvAwurKtw6gFAzvlJ598Tcw7xphBeeqkFLO6OHP32tRBFsfJF0Z/jCsYs1ksRyDIYNxk0Z26knoSalse3j2Lx8TJNhnUHXATCTJXlmFcG7QO9ssTWOIFjnKCIbhhjEi387TxsLUhxky2G8FoRd1HJ0qIi4V+4oLo2lZo/3jJo91jPKkjFpXmRbqTtDDCsu0hOhTDa7/06J8gcrF4RD+AXqgTDIY8QxtE11J35Qmg7rm6eyU6knLSf/FHO+HdoJPq3nVdHqru9esnq57j+hJLLNEjrqCrIG2nxL3c89QD2jj6hHpHvUQP9Hu2pE8rrTu5fJdcgwjAVqcQ72x/VBJX+kwJHsShdgg2jNl4d+hmjJeINaDHC9P+QcqlbTD2430N2i5s3Lnz0vLl4uLaIH1SGqclcEHUgxwASYT2yliSMRzHOum6fHVpDXJPbQg9WjJ2LUl71POLYbbTkvIP+gxELY3LmS/Qj/UbR5Jm6Rh00Pw2eb5Nn2gJXNrGHf215pprxrET/SnjlXSeZfMx6jpo0+7qfJj1epT9AePX173udZH8zTib7bEZ3zOP1ryiK8y6jqftfKZJ+iVzgSbxEmYUY3rSYazNfIaxNmshtEk+AOlaSIe+izkK6yvIpEmTIomLuZC2X2+bbuk4dpTthjKVjtPb4uHhHQFHwBFwBBwBi4ATuCwaNecsCLg4Ao6AI+AIOAKOgCPgCDgCjoAj4AhUI4Chhm3GMMKmW8tUP+V3HAFHwBFwBByBmR8B+sB+BK6Zv5TdlSAlcHUXs8fkCDgCgyKQI3ANGqc/7wg4AuMfATz24qUMwdskH25YgfzG1rUQuyDW4xHaxRFwBBwBR8ARcAS6RcAJXA3xdAJXQ6A8mCPgCDgCjoAj4Ag4Ao6AI+AIzHYI4H0GT1zbbrtt2GCDDWL57TZQsx0gXmBHwBFwBByB2Q4BJ3C1e+VO4GqHl4d2BEaJgBO4Rom2p+UIjB8EJk6cGHbaaaeYITy1sTWnPO3hve3Tn/503EqWAJMnTw4XXHDB+Mm858QRcAQcAUfAEZhFEHACV8MX6QSuhkB5MEfAEXAEHAFHwBFwBBwBR8ARmO0QkMcRjNcI2xbuv//+vcXe2Q4QL7Aj4Ag4Ao7AbIeAE7javXIncLXDy0M7AqNEwAlco0Tb03IExg8CbC9/8MEHx61SydWzzz4bnnjiibj9OtvFa77/wAMPhEMOOWTcb8E5fpD1nDgCjoAj4Ag4As0RcAJXQ6ycwNUQKA/mCDgCjoAj4Ag4Ao6AI+AIOAKzHQIQuLSYyxe6xx9/fLjllltmOxy8wI6AI+AIOAKzLwJO4Gr37p3A1Q4vD+0IjBIBJ3CNEm1PyxEYXwgsvPDCYb/99gvzzDNPNmNO3srC4hcdAUfAEXAEHIHOEHACV0MoncDVECgP5gg4Ao6AI+AIOAKOgCPgCDgCsx0CG2+8cZh//vnDQw89FG699dbw5JNPznYYeIEdAUfAEXAEHIFVV101Gjzvv//+8Mc//tEBqUFgqaWWCgsttFB46qmnws0331wT0m85Ao7AqBGYe+65w2qrrRaTveuuu8Kjjz466ix4eo6AIzDGCDCmWW655cICCywQ++o777zT5/pj/E48eUfAEXAEHIHZAwEncDV8z07gagiUB3MEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcgcYIOIGrIVRO4GoIlAdzBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHIHGCDiBqyFUTuBqCJQHcwQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBByBxgg4gashVE7gagiUB3MEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcgcYIOIGrIVRO4GoIlAdzBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHIHGCDiBqyFUTuBqCJQHcwQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBByBxgg4gashVE7gagiUB3MEHAFHwBFwBBwBR2BICCy66KJhtdVWCxMmTAgLL7xweMELXhD++c9/hsceeyzcfffd4fzzzw///e9/h5R6+2hXWWWVsPrqq4eXvvSl4aGHHgonnXRSWGyxxcIaa6wRIyO/Tz/9dPuIZ9Inll122cAfcs4554yrd0We5phjjrDllluGueaaK9anW2+9lcsjlxe+8IVh++23j/n5xS9+ER588MGR52EsEtxkk03CQgstFB555JFw8cUXj0UWxjTNl7/85WGjjTaKebjsssvCww8/PCb5Ge/tFFA233zzMPfcc4c//OEP4ZZbbhkTnDzR7hB4xSteETbYYINAG0APH3LIIQE9uOmmm4Y555wzXH/99eH+++/vLsGKmNr02V4HK0D0y+MOgVxdrdLzO+ywQ9Stt99+e/jNb34z7sqSZmi8jNvSfPnvsUVgzTXXDEsttVScH1544YVDz0zT9NZee+3YzzFn7WqOMYw4uwJs4sSJcb5OfGeffXb43//+1yjqddddN84HHn/88XD55Zc3eqYq0HgZW1flr+R613pv5ZVXDiussELMyg9/+MOSLPkzY4SAt5UxAn7EyebGcSPOgifnCDgCjoAjMEYIOIGrIfBO4GoIlAdzBBwBR8ARcAQcAUdgCAhss802YcMNN6yN+d///nc47bTTxoXRKc0vg+4DDjgg7Lzzzj0C10EHHRRYnJ7VBIIdRneIdU888USveB/4wAfC0ksvHX9//OMfD88880zv3ng4ef7znx++/vWvx6zcdddd4fDDDx+TbL3mNa8Je++9d0z7zDPPDFdeeeWY5GMYiUK6gayBPPDAA4E2K/nyl78c5plnnvD3v/89fOYzn9Hl2eb4hje8IWA8RzCgXHvttWNS9vHeTgHl29/+dsQGAtd3vvOdoeJUpc+GmmgSeV27SYLOdD8hkuy5556RuKXMf+hDHwrLL7982GOPPeKlCy64IEyePFm3h3Js22ePsg4OpcAe6dAQGA86wxYuV1er9Py3vvWt2BYhcB111FE2mnF5Pl7GbeMSnNk4U4whF1xwwfDUU0+FSZMmDR2Jpump3TH23X///TvJ1zDi7CRjUyP5whe+EGRL+ehHPxr+85//NIpa8wE+0vrUpz7V6JmqQIOMrflYbN555431aKw+qsiVq2u9pzpEWoy/2sh46+/a5H1WCFvVVkrq7iBtZVbAcjyXITeOG8/59bw5Ao6AI+AIdIeAE7gaYqlJR8PgHswRcAQcAUfAEXAEHAFHoAME8LK17777Rs9Vio4veFmUh+jxohe9KP7xNark+OOPDzfddJN+jsnRLlr/61//Cn/605/CYYcdNssTuPA2BlENwXvDCSec0MPfLhA7gasHywwnszKB601velPgDznuuOPCzTff3Cu/FqGdwOUErl6lqDgZ1UJ2nT6ryNpQLte1m6EkOMJId9ttt/Da1742pvjss8/Gvv2zn/3syAlcbfvsUdXBEb4KT6oDBMaLzrBFydXVqvGYE7gscn4+syLQlFDVVfmapgfZRQSZY489thMvXMOIsytcbL86MxK4+KgHstTf/va3cOCBB3YFy8DxjBcC13js7wYGdyaLQHPnlOxYUnedwDV+X35uHDd+c+s5cwQcAUfAEegSASdwNUTTCVwNgfJgjoAj4Ag4Ao6AI+AIdIjA+973vp5bf6K97rrroncauw3DAgssEDACy7MPX/h+/vOfjwuuHWalVVQygj355JMBY7SELQpWXXXVuH3gGWecETBYz0pSt5jLFg0y1ONhaDxtd8k7gAS40047hec973nhjjvuiNt2jcW7mV0IXD/4wQ+m2/5Oi9BO4BpbAtd4b6e0yVEtZNfps1HqBkvgStvNKPMxjLQwir7kJS+JWyt94hOf6HlmBHu2DEHYSnbYWyi27bNHVQeHgbnHOTwExovOsCXM1dUqPa92MLN44Bov4zaLt5+PPQJNCVVd5bRNel/60pfih0d//OMfwze/+c1OsjCMOLvIWCmBiy3V2frw0UcfDXjgHEQGIaWUkGAGyWvTZ7vWe5bQK4Jhk7yMx/6uSb5npTBVbaWk7g7SVmYlTMdjWXLjuPGYT8+TI+AIOAKOQPcIOIGrIaZO4GoIlAdzBBwBR8ARcAQcAUegIwRYGGRRXN618OaEV6ecEIawPINcffXVcTvFXNhRXNNCS5cL9KPI96Bp+GLuoAiG4AQu30JxLLdQHLwGDz8G6ddhb6E4XvTZrEzggmw933zzReIWnhnHSlSnmvbZCj/sOjhWeHi6ZQiMF51hc9+mrs5sBC5bTj93BIRAG0KVnhnk2Ca9rbfeOmy00UaRtMz2gHhpHlSGEeegeeL5UgJXF2krjkFIKSUkGKU7Mx2dwDUzva1meS2pu4O0lWa58lClCLQZx5Wm4c85Ao6AI+AIjE8EnMDV8L04gashUB7MEXAEHAFHwBFwBByBjhD48Ic/HJZYYokYG943DjnkkNqYX/WqV8XtFgmU2+5goYUWCltuuWX01CWD8SOPPBJ+9atfhRtuuGGGuLfddttICLv33nvDZZddFljYwoPUwgsvv8949QAAQABJREFUHP79738HDL14BmF7RAmG9vnnnz9MnDgxXmJhnu0cCfvrX/86LLfccgHPB8hPfvKTGE/8Me3f8ssvH9Zcc82w6KKLBrx3QVjD69giiywS08Y7EWkibC/5zne+MxLcbrnllqzHqE033TRuP/mXv/wlnHfeefE5/r3nPe+JnqYoN96miIdtOP7617+G73znO71wjIExDCy55JLxi3G8Zv3jH/8Iv/vd7+JX0ZxL8F6FJ5VlllkmXuLe3XffHct/4403Rs9j3AM7yp4KZV9vvfViWeeZZ564ldbDDz8czj333PDAAw9MF3zOOecMu+yyS7x21VVXxXQ222yzsNRSS4UFF1wwPP7444H3xpfbNo/TRZL8IM63ve1t8SrvDFysEDf1BwPt3HPPHZhIkb9LLrkkpmXDNjlfbLHFwlprrRWot6QNEeDaa6+N73XvvfeOUZx55pnhyiuvnC66UdRjmyB523jjjSOxjLSfeeaZ+P6vv/76mGcbtur8xS9+ccSWekwcyEMPPRRof+eff37E0XrgOuigg2I7oK3gWY8tU2lDl156aXjwwQenSwYvPcRLHZk8eXLYaqutwmqrrRbf0ac//enpwm644YaxHb3sZS8Lc801V2xjxAvOqRGNOr/BBhvE5y+88MIZ0uXGu971rri9SlX7oz7jdQ8vgZT3mmuuCbfddltsC7Rx0uZZJF04pzxrr712bE/zzjtv+POf/xyfpb61lTZ1Fw+BaTu17Q09RttCx62xxhoBHUFbueKKK3pektZZZ52wyiqrxPfCuyM8bfGJJ57oZZ12xJfjCDqNerXuuuuGFVdcMeoayovuo32n0m8hm3yp7vAsOhqc0UPWe2Mar/3dT5/ZsE3TQ5eq/uPhJi0b3v/e8Y53RJ2Od0ZwoU+pazc2H1XntKEtttgitiXq0tNPPx09W5x99tmVbbhNW3nlK18Z3yXeL0877bTYB9J2aEOkRzunjlx00UW9LK6++upRp3BkOyDeC+2D+nLOOefENkP/hVDnaCtWXvjCF/b0Euf0NYRjrIC+4hr9Z9p32Dg4L+2z+9VB+lPaweKLLx4xoD5PmTIlljHVNcpTm3aqZ6qO6Lftt98+Ykwbpe+mf//9738fxwJV7aBNP1yVNtetzijto7vKi83n//3f/0XyBO+HsRrjLOompH/wkdA/q/6B3amnnjrDeI26y5iQtkr9RKf3GwOhXxk70OdwzriF93PSSSdNN+agD3vjG98Y46N9sD0S+pM+8Le//a2y2eiYq6s5PU9klsD1ox/9KOoNsGK7csZVjFPOOuusQFu30rQfbjOmJH7yWYfZXXfdVTtuI46m+pmwEvpj+jjqCHUZ7NEljF2bjikVF0d0JONHxh6Mb9HB1APG4JdffvkMHmmb4kncJeXjuVQYjzJeeeyxx8JPf/rTOF8BA/JO34RuZdxX9SEN8ZXmpURf8l5e//rXh2WXXTb2q9RPtuUG04997GNxLkB/MmnSpLSoYdTp2QzQNzHe5cMj5pb0d1ZK+tNhxGnzVHpeSuBibMgYkfH+L3/5yxmSH/bYev31149zM/Q774k+gP6bOR+6mLENcvrpp0+nD/AaRjtHyDtzByuMi9HtxMeHGugBSdO+ifDU/br5KvVB8zbO+42PUgIXeoC2/OpXvzqOzdB9jM/stvdtxsgqo47SNbRPxoy0R3QuHzCha5kLkx7zAs0VwBzdyTtAB6XjZ8XNsUQPte2bStqpzWNX4720rdTVXebbdTLoPJR+gvww/2KdjbbDO2b++7Of/Sy+u1z69LNvfetb41yHsQZjZNat6G+oB03k3e9+d5zXU2cYT+WE9RbwQZgv2HWeNu2P5+vW0cg7cx/aOR+f5sbb9FvMERDWIRnL1EluHGfDN52zaW7Gs13qL5sXP3cEHAFHwBHoFgEncDXEk8GciyPgCDgCjoAj4Ag4Ao7A6BD4xje+EY0GpPjVr341LgD1S52FI8g1LORb4guLJCwOsZiUk/vuuy8cccQR05E4RChhIYbFJAyxqbA4873vfa9nBCefpJ8Kg+4DDjgg7LzzznFhkfuQVMinhMUgDFWpsPADAYLFS84htiEYCtl+CsGg9/3vfz+e239s3wh5hEVitqeSaCGIxVgWkZRnDFMivbCQyqJuFWbk5aijjuotgNn3pXQ43nPPPeHQQw+NcS299NLxFp5WWGSTQCB73etep5/THUmHhVq2nJRg0OTrUoSFXAgnLJalQnnYWoQFxH5i42Qh7fDDD+89UvVuCED+IA6lC/W9hzMnGJ122GGHLLYYSFlkRFIC16jqsbIM2WS//fYLELByAvYsTvYTDNu2/tnwEAshJam98a4wiLP4mwpGY+quJU3Kgw9tCWO8SJ88q61AqBv77LNPXBxO4+Q38bKQidFAgmFdW7hVecSSsRvjvd0Kh/RopyLrKE6OlI/2yIK1JZrahXMWlWmXOWm7tVXbumsNOWqntm3QniE3cC0VjL60RYiqqUDcpC3yjhAMY2x9i9x6662RuJXTNbQH2qLdblb6i3uWcAqplfxjDMoJBA2IwFUEGvtMP31G2Lbp0T/JuIfeAA9LGoGUioEPgaxLn1TllUrtJgau+cdi/TbbbJPVNTyWeqssaSu27qIHIb5gYEzFvi/0ioygaTjaLeSdPfbYI96C/IeOldAXf/CDH4z465qO9OUY1ki/iX4q7bOr6iD5gDgF+SYntH3qbLolZNt2motb1yCZf/SjH822UcKgd9BdllDJ9bb9MM9UidUZJX10l3lRHmlbkG4xMqZCe4SwpHEjBDjap8ZG6ZgAYx/jL9VzCF4Y1HNxawyk9k1akBnpYyQ8L0MlbV5bguu+PTb5oMGGz9XVnJ7nGfVpbF1GH5UrDwSGww47bLo63KQfbjumJD/9MIOUq7Fg+o7a6mfSo7yMgRk354R+6JhjjumNe3Nh0mt8DCIyeHqP3/SNX/ziFyOhS/eb4FlSPsWfO6pOMB7CoFylwzDEX3zxxdNFMUheSvQlpBTGlLn3BJ7MLxi3Mt5KCVyjTm86oKb9+MhHPhLnlHbOpXAl/SnPDiNO5an0WErg0nwAXYOXMsmoxtb7779/dr5A/33nnXfG8St5gtAJAVPCvI73h9CO6IetUBZIXvQB3GMNAWnTNxHe9q+p3isZH6ntEzfrCRC4cmLL22SMnIuDazY95l18uJYKbZctw9///vdn+6Gf//zngY8QrJTqoZK+qbSdkt8ux3tpW6mru1XzcGFoy9R2HopOpt/iHVRJjrDab12D+n3kkUf22kpV3AcffHBvHYjwlpylZ6Qj+X3iiSfGj3o4b9v+eEbjqtw6GnVX8/9TTjklfozEM1bsezruuOOmI0facDpXenYexb22czbm3cy/Edue+V2qv3jWxRFwBBwBR2B4CDiBqyG2TuBqCJQHcwQcAUfAEXAEHAFHoAMEMJ7J4xYL8VVG7CZJrbTSSmHXXXftBcVgzpexLK6wkCoDnIxsCqhFMf3myLMs4PKcDFv81qLYXnvtFYlV3Ee4B7GErw+PP/74SgKXjFTxoan/WLRmYR/jhPLHPRZ9uyRwKT0d+fKeBTiMliy8ayEOEhtf4JIfjMMyatp3s++++0bSDR6wEIwoGIghZ0A2sAu2IoYQDoMqniwQyse7wXiIAZP0ROrAExfeJxC7eB0vTP3Hs3yZS90hD3oufa8Knx5tnHZBnIVlvOIgLMjzLsEDTwoQWSQYNPFM1k/wqrTjjjv2gvGuwQlCHgugViyBa5T1mDzwFSx1QPUP0gtlZ17EF9Ft8KVstA0MasSLQCIgTuoGdSRtb7xPPPcQBg9EvB+E35/85CfjOf9k6OxdmHaCYYTFWuRzn/tcNEZzTp2FAEb95P1pnkd6X/va13pE0UEIXGmeIC2RH4z+wo28VBG4uIeQR+o0uIG5JF101fX0WFJ3c+3Utg2lgSGbdoCRnwXkVFhoAGvevcqM9x8Ir4glcOlZ4qT9E6cl8LFAzgK3pGohW4RVwtFWiQs9LY9rXKf+EA7dXCf99BnPlqQnAx7Po+doYwhf40NKQlQXyWu/dhMfqPgHIRhDkQRSGOQP3on6KO5B/pVXn5K2Yo0+Sot3j9cp2hf1XiLiGZ7yqAP0w+gYykx9op1AqqoicNEWwEx1iufot9Cd0i1KqwmBq7TPrqqDlnikMqHjwRvvEQjXv/vd7/aI3yXtVGXMHSGDgBOC7oFgSjuAGKy+G90KQUlS0g/r2dwxpzMod5M+uuu8kD88Qrz3ve/tZZU+l3rDWMHqGgiIImNj1JU3TB48+uijoxdEzq3xTXqtn86w4zywsHX45JNPjp6DaBd4TkHQYcSNrmI8ZHUpBLB+3jRiJFP/5epqTs8TXgQuPasxD3ll7Kdxb0qMSfs8Pa9+uGRMSRz9MMOzYhWBq0Q/77777pFMTNr0v7QdygopVGQhMEGP9+tDiIM2B3kc/HjnEEyZE0Bqp+4Jz5SU1w9P4i4pH89Via0TCkOfQVshr3Z8ig6m/5KU5qVEX5Im6duxG20ZfO3YlHBpPR11euQhJxBk3ve+98VbtE8+0pGU9Kc8O4w4lafSo31PEJZoO01E84GUwJW2i2GNrSEy8MERY0faLnqMPpO2i7dZvTs7V6RcdnzHb0hO9oMTPvRgvMNYB8IJUtI32f7V5qF0fJRr+8xPKC9jNDu2YoyITujX38XCVfzLpUcbZtwozO2j4I++YSypdQnus3UpOkpSoodK+6bSdtr1eC9tK3V1F2JTneTK1HQeaonnrF9BNAJb+jqNR0n7K1/5Ss8TFzqcNq3+kXdMeyE83t+0/gCxnvWQOuEjTbw4I7kPnehrWVckLeaaeGlEStofz2lcxbkV6jGkTojCCLo9Dcs6FeOWNC82nvRccaQErrZzNttPWN1BeiX6K82n/3YEHAFHwBHoHgEncDXEVJPDhsE9mCPgCDgCjoAj4Ag4Ao7AAAhg9GBxEGGBlkW5UsFIqcVHvtbkq00JRgkWLGREOfbYYyOZhPtaFOOcxR4IBDJys7iE9wWRSuyCFOG10MKXunhRkeQ8cGEUwjuXFrB4FiMPQr5Z9JbhCANF1wQuFtT5QhBjN/Ej1lNM6qGF+xDWWJRDWHxjwQrBeEJZkNR4bhdsReBiMZbFJ8pOPvDUhcFMglcuDC4IC7jEzcKgXbzmHgvNeDWRQQkPQHvuuSe3pluoixcq/tk47aKW/WIzJWnx1SYGRsQ+Ey9U/LNfibJFGl6LJHjwkEt9rlkC16jrsX1fKYEGQy6LnyxCIrQVyBr9hO3K+EP4slrbB/LbtjcMoxg6MPYjGA7xWCfyAVhgTEGsQYf6S/vGAC/jKlt1vP3tb49hmfzyLMQYiSULUI+JDyklcLGAzEIyQjrUb6XHl/EYc6U36ghcabuz2KXG3phY5l9J3bXvXe3Utg2SgcyJnsLYglBGSFcStkQh/wjzaN4dC/EYWjC4ICmBKyVaWgIP7xVSitq39KtdyMar3Vve8pYYN+8Rwxm6AiH/6HnpUUvSiAEq/tXps9L0IIvQd6jPwbsU3kyolyL48BsvJxL77tN2ozC5o9U1eLmDPCWxRBERUErbSmr0od/DS43qh/2qm7ojwgV5UftNDe32/VsPXHhjwMsbgmGEPkP9VurZMO2D4kMV/1SnmvTZRKHwtg5SX6jfMsqQN2u8ZYsXtk9CrNe+knYaI8n8Y0yjNpaStGiD1D0R6iAhYZwr7YczyfcupTqjaR89jLyQKdsWbL/KPeo920Hx3ug3RMjnnu0fwArvTOh4GecID97qb+p0hiUjETeEbzxYWgM0JGIM1IyHGPdKhxGe/kOkZFt/uFcnubqa0/PEYQlc6Fv6YeUBHY8eVT9sPTGpHRNHrh8uHVP2w8zWMzsGK9XP6CbiBH+I4tJhlMvmBW9teKnqJ3gTnDDNIySeJMmjhHENuPG+03rXD8/S8int3NHWCd4h/ZAlCVrjru1PSvNSqi/tGIt+g75eYwPGpsyRVEdtvzLq9HIY22vy/ogeYG4hGaQ/HUacylfJsUsCl33voxpbSx/YsTr9BHWO/tTWL/DhusZ2/Gb7R7acRSxJ37afkr6pSu+Vjo/Stk+/dNNNN8V888+OUSx5uK6/6z2cOUnT4yMCtpdH0rkBJDLqteZRtk+23pFL9VBp31TaTi2WXawpaO6ckh1zdTfzKqa7lJap6TyUeQ19FsKHjsxnNC7nGl6A2eIPsfMvO7dhS0Prad2SEdN2FiNK/tk8pP0pQa3+sGsaJe2P+DSu4jxdR7MELdauWMezeLAF7Jvf/GYejVuzMl/qJ0rPzjlK5mxd669++fb7joAj4Ag4AoMj4ASuhhg6gashUB7MEXAEHAFHwBFwBByBDhCwC3FNyQq5ZPHsxGIfUhXPxIkTo/GOMHZhRItiXMc7wvXXX89pTyAIaauw1P25FlqaGIOtO3O7sKWErEGTBaCuCVwQnyi3le222y7wDkgP8oU1MBKO++uvv358BI86EACQusVcu2ArYogtu10UjpFN+2dxPueccwIu+O3iNXnE0KmJjZ5lAVHEPW2lp3u5o43TGgIpvwzeKVGPePDQwUIjGPz4xz/ORd27Zusj3qxYlE7FpidDs31uFPWY8pIPBFxFyrN5tYuhLPQfP9XDXD+xi7UpEcW2N+KyxgPitYYJ+6w1dKaEOJ6zBqTc+yOMrSsY6DFYlxK4lB71EuO3JXCQlvXAZo1CduE8R1rFIKn6Yp8jziqxdSlX9lzdzbVT2zYoF4QFGfVJ2xpUcsQCMOUrfp6V/rJGGgiyhIEgYcXGaw1h0q9WX1M+DOKkAaEMjKzgJYyFeiSHrw2r8zp9Nkh6tu2QX/oJbZmZkm7IS127UV7ToyW/5uLE+IiBB2MjuEPmUd0lrlx94Xqurdi6m9MXEBQgpiBp3VX7TQ00OQKXrYfkmXpoCRbEb/uLURO4aE94bUJyYwauy1jEe4cgQjlK2ilx5cS2q1xbhECGwRLBaATxobQfzqWva/ZdUdamffQw8sJWcCJc4e0Ron4qeGJTG7T9D8YuyKMaS0C2sJ4h0q166nSGJQBV9f/oeNrklClTsn2qCAIYSclXE8npy5yeJy4RuKr0qPVWYb3IqB0TR64fLh1T9sPM1jM7bivVzyo/BmD6JKtfILDpwwDGoddddx3FrRWM9XzsgN5Dv6YiT27grb6RMP3wLC1fmr79betErp3YsYvVraV5KdWXtp/ifemDF5XF1lHbr4w6PeWn6ghplPkn754PIiABIIP0p8OIsyr/Ta7bdzWoBy7FBV6jGltXkWDUbsGAfhxyEV76KCNCHuk7+LCFuQ3ChyQQLhD6fIhJpX1TTu/Za23HR7bt5+Zzdsxq235dfxcLWvHPpsc2uGxnZ0V6mGvgZz8Qsh/4se0w2w8jpXqotG8qbaddjvcot+bOXRO4cvOkqnko5ETGUAjkZn3AEy9M/Ye3a3m8skQt+1FDbv2LcRue8Jhvynuz4swdmRPwEQNi16b4bT2EUVf4QKy0/RGfxlWc59bRLHnbbtdIePsRJGO6dJ2AMKkoPTvvlU4krMqUPpebs3Wlv9K0/Lcj4Ag4Ao7AcBCQnQObDOsEjDFZy+OPc/1umvocUycgz3223/SJmSScE7hmkhfl2XQEHAFHwBFwBByBWQIB60EpZ4RsWkjr0ciSPuzzGOZY4ECscVmLYiyqa1HWPmeN6iLa6L4WWpoQuERu4FkWdeTJQXFx1GKLNfKwnQ7eNBA8g/EFayoYTfE6k36NqPzZ8qbPpr9ZuGMbHxbTMADrK2O7SFa3mGsXbEXgUtkpF2W3xjKlD1GMBVZEC712oZpFOBauUrFfuXLOV5B1YuO0hkBLCCAOXPPjHt96UqiL196zi5XW25sNY78GVr0adT1eY4014naf5AsvEyzIpmIXctlOA9JMP7FtJm2Pam+2jtv4rMcg4cJ9GTp5jvfM0YoMAXV1na3rWIBG9F5KCVwYlnJf5ts8yQhv82SNAddee23gy/JU2P6BL3utUTINY3+X1N1cO7VtA8IHi8ZW2PKL94Ng2IZoaYWFe203KjKlJZpggOU9pWK3MLNtUvpLC9ngDe6IJRWk8Wlruao6loav0mddpIexfokllpguSfoajDsYTqzUtRsbzp5bnVG1/QiGMLb2ox7i/aC0rdi6i25ki89U5Fkorbtqv+n1HIHLGu7SvlXp0UfRvyDW0Kj7VUfVqTTenNdM4lB41UGuyZhCP4EuyglbL7GFCSJdU9JOc3FzDRKj7Q8xEmMww6Cmxb/02dJ+OI3H/rY6o00fPYy8WFIY7w3PbanYbY1uuOGGcNJJJ/WCYJjnfbI4aiUNx70qncE9S0ZKSf/crxLiJA+MhSZM8+Y0bAIX28+KdJnmS6QGazBWO67qh9M4+N1kTNkPM1vP1EcMop+tsRMSBB9usGWavIHmytH2GoR/3idkSsZa1Ku0T6rDc5Dy1eXV9v05ozJzCeYUyB133BEgLw6Sl1J92aSfyvU3o06vDmvuWY8xdov4QfrTruPEcw5zzVTQofT1/USYE25QAtdYjK2l6+xYnbLYMRleg+hj8aiDZx36/7vvvjuSue36gebjdqxT2jfl9N4g4yPb9o866qg4z6WcEjzbaet6tX3u1fV3ejZ3tOmlngkJr/Zr8VM8fAzCOAHRVnmD6CHFa49N+qbSdtrleI88a+5s+2OuV9Vd7lWJLdOg81DeySKLLBI9h26xxRa9LW8tgcuSgskTRCY+1oFEmFsPqsq3rltCFiR4ebaivVCn6GstMa20/ZGe5gCpblBeLJHYzhOsdy6rC/Rc1VHp2bia9IW59Y2u9FdVXv26I+AIOAKOQLcIaA3HCVx9cHUCVx+A/LYj4Ag4Ao6AI+AIOAIdImCNkHaxpW0SduFCX8nm4hAxAk8w+kpQi2IYcSBQpWIXiiyhhHBaaGliDMbYSnlzC5VKU0Zfa+TpgsDFIrzyqrR0ZJGJrx9XWmml6H2CxbicDELgwtsEC6V8vazF4TQNu1ir7bfs4rW2/0qfs1/bNzEc2DhlCCRO0odcxn0rvAu+CoZUxoKjtnewYdJz8oHhDmEB2noxUlhLXFC9GnU9liFCeaKsOZFBu+792efsomEVgauqvVmSkHAhbhk6U5Ii9yw5M92mhvuSLbfcsueVRlublBC4rN6q8rBCmiIS2YVfu3AuT3PKn44yajRd9C2pu9awIqKlbRt28Vj5YmsMDH2IjFm6x7EfgavKUIAOQjcjlrAgnaW82K3FCNuvvhIGnU5dq5Mq41QX6VFXqAeUUXL66aeHq666Sj97x7p20wuUnFidwXY4kJnqZJC2YusuW8LigScV9XNp3VX7Ta9bPagtFG2fy1apbImcijXkjZrAJWImeWpSByG6YQQvaadpue1vSzqz1zHwoQfpr+Q1k/ul/bCNOz23OqNNHz2MvNit36reC/lXf5brK+w2oISl78bbQxpflc7gGUtGwpifEjUJg2CIJz3qBd7rcmL1Ye6+vZbqS+7l9DzXZRDMeWDiPiIPIpRdXqPUjnP98HNPhajr2o4p+2Fm65nGbYPoZzwXvuMd7+jVBeWduQEfk0CEpL9qI3g0xfsO5DvGu6pnNg6LJdfr8BykfDbN9NzWidx8yY5vNLcZJC8l+hKjvDzCqv9Py8HvHFlm1Onl8pVek26yH0EM0p8Sf5dxam6c5rtufGvDdkXgsnWvLu2ux9ZVJBjsQ5QNEYmINgPRiXkhfay2UqfeMXflSNu3ulXvinjSvoRrEukM9U05vTfI+Mi2/dzc1JZXbZ+81fV3ynvu2C89jQPScSFx0S+mBK5B9BBxlqx3lLbTrsd7WqvqmsBVMg/FU9tGG20UvZbrIz/wtWIJXISZNGlSrEc2DOfMj9H3fBSkLXLTMOlv4kNn0V7sWATyuz4EnDx5cmBOgZS2P57VuKpuHU36067v2R0G0i0qibdKlJ76vUHmbLY9D6K/qvLq1x0BR8ARcAS6RcAJXA3xdAJXQ6A8mCPgCDgCjoAj4Ag4Ah0hoMV2S6rqFzULWSz0IpAGWJzBnXpqHEnj0QKYXWTRNb50wFCXil0stYQSwmmhxS50ct0aVrWFgspZlQ7P6YtNW44mBK4qI5Dyp4Ug0rACyWi//fabjlyg+yI9YIhCBiFwyVhoiSxKR0e7UJ0jcFV5H+uKwEU+wHrHHXeMX1TnFiWpN4ceemg08CnfuePnpnqpYhseRJ6I0nBgL49vqlfaFsC+//Q5fqvODlqPRRjMpZG7VkW6SsPWEVGU96p20I/AlcsDXrUgsiDy3pbmid92If7SSy8NeENoQuBSO5KXQGtEqKqXpCcvH7be2zzgfStnJG5L4CKttnXXGlZyBK5cuSyB67TTTpth+4x+BK4qQwH5l4547LHHolGba8Jd+st6z+F+E8kZqdLnqoxTXaSHgYF6wNaOktw2HNyrazd6Nj1SPgxFCPqDdlUng7SVJnW3CwKXJZZWtZGxInDxPqmrbQQCGkQ0pG077ZcORqJNN900ennJhRXZhXtqY1Yfpc/k+uE0jP1tw+d0BmFzffQw8iJ9a/NXd54jnK2zzjoBr3aSKq9iVTqD5ywZqUr/VPW99P14U8MLEjJsAtfNN98c8BKWEwg0lNOORzTWzPXDxFE6puyHma1nqtOD6mc8P6JrIGOIOGFxQJdC3tBY2N5LzzfYYINIbk7jATvICcxXGFNaLImjDs9By5fmUb9t358bn1oSjeY2pXmh7CX6Eg+btBEEr5FHH320sj/dUYZ5EUBK9XNpetNlpuYHW+pB7kMYA0P+GaQ/JZ4u4xQBgXit1JGobLiuCFxjNbauInBRRvUrkHnR55rLQ946//zze54wRZjeddddIzR4y6ZPtHHEHw3+qW/K6b1Bxkf92r4lfKjtk926/q6uOP3SE4FL2Nq4cgSuUj1E/KV90yDttMvxnubOXRO4qsbYuXkoRHPaAGTlVFjDo8/U2oclcBFW5Dm8UWpdKY3DeihM76W/991330iC5zpzRQhW6g/oZyFaqu9WG07jqPqt9qe4OWoemnvGjue0NTfzajwFI1oHzD2bXkvnvYPM2YhbZVcbK9FfaR79tyPgCDgCjsBwEHACV0NcncDVECgP5gg4Ao6AI+AIOAKOQEcIyIBBdCeffHLcyqQuajxFaYFUJBBISEsuuWR8LPdFueLTwoX19qVFMcWlsDp2ReCSYbvK8EV6OQIPC4D9tlDUIiiu6LWtFPGlC0Fcs6KvmLkGJmwVBAEGAhXbQ+BtB9IGMgiBSwuBdR6cLKHplltuCXhusovXbYzDMcMV/2ycMgTmgmLcW3311eNWkjKmEk6GotwzumaJLLw7DOap2O0LReAadT223kYwPuTIRDbftBE8CPSTOiJKv/ZWQuCyRgd9uZ7LI15BaM/IiSeeGOt6PwKXNWaKwGWv2YXeNM0cSaGJMUDtpUldS9Pkd5O6aw0royJwVW3TiVGAMiPWYJjqL7YilCcYdNSpp54an6n6h0GhyZZYVcapLtKzZF7lEwMMuh4CppW6dmPD2XMM8BOmbbdGH9OvvIO0lSZ1V/1cWnfVz6fX+3ngqvL0ZbfdHLUHLm3xxJf/kPH6CdvU0Z+m0qSdps9U/eYrffoUtiyl3tLPSdgal7YnvdK2H1Y8uaPtT9v00cPIi4x35BPdQfuvE8ii1EcJbQPydUretgQ8ha3SGdzvR0ayRmgMjRjK2cKPLavoYxGNVYdN4LL6NiZs/h188MGReGoNxmrHVePY0jFlP8xsPdO4rQv9THExLDOvWHXVVaNBmLYkUVr6nTuyVSIEFpG37r///ji2gBwnjyJ4d1100UVbEbi6Kl+aZ9v3NyVwDZKXEn3JmJs+EsEwr7FAWhYRh2y/Mur00jxV/VabFmlykP5UaXQVJyRG6yVU8WP0h1DaT/QeCNfEE7Li03xAOmasxtZ1BC6rm/iAh3kaAumNti6CBKQ89Okqq6wS+3u7vXJp35TTe3ZNou34qF/bt+PD8UjgGkQPlfZNXbRT6sug4720rRAnUld3nwsx4/8mZdIYzepWS1RivQldxtjlnnvuiWQptlKUJ/uUwGVzwZbueMFk/M8z6jsJA3FcBmz7THpu5w7kAyIa8w/i0lxdz5S2P55X31NH4LL1kjWBI444ouchzHpdVH7qjml6tk22Xd8gnS70V11+/Z4j4Ag4Ao5Adwio/2M9gPUI+jQ+WuSPc/1umuIcEydOzO+v0TSGcRqOztHFEXAEHAFHwBFwBBwBR2B0CLznPe+JhhNStMSqqhxYcoy2KNhpp53C1PFpfOSUU04J11133QyP2wUWFl1ZfEW0KDZsApc8pWCwY4EL428q+grafqXP14wYFZEcWYQvGSFwIW0IXBidWNBEWKDDWJIame32XIMQuKwnB3CwRtOYgan/LJnooosuil8228XrNsZhxZk72jhlnMNwIa8bENgwpFrhHbBVC5MnBGNc7v3pmXe9612R/MXvs846K26zoXs6sn0PhlxEBK5R12NrSK7a3o6JIluEgBGLohAB+kkdEaVfeyshcJEfkaWq2jFh7CIudR/jON5rtthiC26HnDHEejmwi8IyElZ5s8EgBpkUsWFKF85jRJl/pXXXGnJGReCqMsRClKTNIFdeeWVsD5ynC9ksZGCwRKynrnjB/GNrDwwEGAXZvq6fVJExBk3PkozQ6ZAeMFAi6rts3urajQ1nz/HqQR1FtFWfvc+5DFciXZS2lSZ1twsCF2RsGUmryFlbbbVVQFcgVWHizeSf6pQ1ThLEEu3s1/IKb403wpN3irE67TeJDyIVfwie5yBNddnH0GaWWWaZ2A9B0Erlve99b1h55ZXjZfqzI488MhrGqOvku00/nMZtf9v+tE0fXTomsGmn59a4eNRRR8Utr9IweGpF5yOMEzGISbQ1Fr/R2fIyAV4QlyyRoUpn8Kw1mIEzJAgrdlylvt/etwa7YRO4RJyw6XMOiYl6Tv//yCOPRJIC1+sIXIOMKfthZuuZxm2l+nmxxRYL6623HsWJ48yUlG77I2u8jg9k/tltm1K9ouD6yIK6JBIy9+rwLC2f0qw62r6/KYFrkLyU6EvGceqnquoodQKSAXXUvqdRp1eFc3pdOlmecwfpTxX3MOJU3G2OXRG4SHMsxtZ1JJhll1027LXXXhEO1g7QH3qHXNxtt91iX881dP18880XIG1TNyWlfVNO7w0yPurX9m3fY3VZXX+nMuaO/dKTXpR3IBtHzgNXqR4apG8qaael8zJb/vRcc+dUH9bV3TQO/W5SphyBS+8rNyYibjxRbrvttjEZS+DafPPNo/dZPrxhO3ErvFPWVzTeOuOMMwLPNhHlkbkNXrU1l+cDQD4ElJS2P57PzQEUrz3q/aAHWPfR1qoXXnhh4K+p5NJTX9h2fYM0u9BfTfPu4RwBR8ARcAQGQ8AJXA3xcwJXQ6A8mCPgCDgCjoAj4Ag4Ah0hwEIXX/rjhQXR12ssgqTCgsi6664bL7OAxEI9X7jz5TxEMITfLCanYrfxsR4VtOhStTBiv3ZNjW1aaLELnaSbMwZrkZf7lqTAb8SSd1IjjxZvGNRj+LSyxx57xK8YudaGwIVrd3nrymHGgufnphLHeD/IIAQuGRmI5+qrrw5sv2YFYhTvUtuM8f7Ik128bmMctnGn5zZOGQJJH2IIhiC2UqE+pmKNrjmjrA1viVGWvKMwLFiy8MgRUb0adT223t2oOxDTqHtWttxyy7DJJpvES7l3Z8Pq3BJR2J6Jr2Ml/dpbKYFLhlDSOfzwwwPv1goGCEiKvGMRWbhvFzdTUg31gnhpC4glcFmDlW0bMeDUfyIo8NvWgdKFc8WbHkvrrjWsjIrAhU6nnVsiBOXBSx11EUHX3XffffFc+tWSZ2RAIACGCwwCVqwnv6bkhzrjVGl66E08M2irkMmTJ4cpU6YEvDJQBxG2hWJ7KEldu1GY9LjCCiv0tprKfemdIyCWtpUmdbcLApfV0dQZiDXoJwmGHtqX+qZRE7hsX3DxxReHn/3sZ8paPKLXwYFyQO6CdI502cdY4jvtBHKkFfpS9WP6ar+0H7bxpuf2XbXpo4eRF0uksSR9m2f7AcBhhx3WI3DRx9HXIZCzJ02aFPbZZ5/ozYxr6RipTmfssssu0QsLz+XGChD48TqB5N6dHec21WGKi6PVlzk9TxiNKTnPbYdriaEi1BNWusP2oVxHBhlT9sPM1jON20izRD9DbKT+IVVeIRVvnbe6GMHUfxisMVwj11xzzQyeIVMib1MCF/EpH5x31d/ZOtGUwDVIXkr0JXqTsYLGXqlBnvxYj6qWwDXq9MhLE4FUzkcECN5DmfPy4QpStY1ZVX8aH5r6bxhxKu42RzseHsQDF2nauEY1tu5HgpGnM2FiPRfaeZvus7UielNS2jfl9J691nZ81K/tj3cCF3iW6MRB+qaScW/pvEz1JXfU3HksCVxqB6wTpB8vMK9h3KQt3S2Bi48iGLej13kuXWewxK+qD1FymNiP5eir+UCG8RtrGVZK2x9x5OahNm6d2zU8xkfM/Sgn8xd+N5Vcehp3EUeb9Q2lqfem3231l57zoyPgCDgCjsBwEXACV0N8ncDVECgP5gg4Ao6AI+AIOAKOQIcIWCM00UKmgiyC5wg8reDJgu2BMM5LLrnkknDeeefpZ9ACERcwWOJxgsVNFtK222673hf3qeFJi2LDJnDxVS55lPH+iiuuiAu8LHZBSuOrfwmLPjkjD/chf+HRA8ID2xvKixP32hC4yAdfOis/MpBgfGY7GQx4ItUR97HHHhu9xnBujcN8OQuGGFBYnLMLxCKGWG8OPH/BBRcECA0IC24YXPiqGbHGR7tQ3cY4HCOq+GfjtIZAazQ499xz49ecioL6h8GPupRbHFQ4e7QLbhjXcalP3QMLjMgirPCMCFycj7oe77nnnmG55ZYj6Uheo06wEIpApsKoTR2hTlKmlHwTAyb/7II3GFO3FGe/9lZK4GLbEozACO0eDywicWFsZ0FXZBraz2WXXRbDUt9Z3EQoI4Y0thJdaqmlomGWhX+JJXBZLx2khxcciCRgiQco1WeeHSaBi/hL6m6undq2kWtv6Bu2VUVyRn9LjpBhmLYDeVXCwsQxxxwTSVq8j/e///1hwoQJ8XZqmMgtZK+zzjo9T0a0Jwx8f/rTn+LzSy+9dNh99917eqvpl9x1+qw0PUustcQqu8iPLoFUyBGpazcxQMU/Sxak70NX094WX3zxqFulx08//fRw1VVXRXJJSVux+WtrcJY+tIZ2imO3QbH9Atsk0w8h9DHHH3989GrBOAAPSnabs1ETuNjeC8ylF9Hf8hiAt0aIP+gc5Pbbb4+6iPOSdspzObFb8FK/IOSgZxCw4f1CHEEgmEE0K+2HYyQV//rpDB77f+ydCbhvU/nHl4hu9SCZh1xcswgZK/OQ4aKUhOaJBjSHRCUaSCpzGTJHMoZKLpFCkkskEuoqyvQUadD/fhbv7/+eddb+7b3Xb7jnnPt9n+ec3x7WXsN3rfWutdf73e/y5Hkz7g8iL6Rl21lxzNY+eGRFr6PnGcNXW201bsU2BbkKoa6M3Mu5kY7JI+2WZxE/5+ymM8Ce8QjJEbg8+Y5tt9iGkzZOPiADWbvneT92cN5Ncvoyp+eJwxO4wAePEXjhZA4HkZQxDGEbSsZOriPWj9N5NPd6mVPWYebbmZ+3lehn9CEkAPJL+ZgrmCc2SKHoF/PSxtzNcKWMOfGEMHQ5cyi2sqWNoDPBk7QQsIbEyy/SDU/ul5SP57qJbxM2Tvvwfhs7/3FKaV5K9aWfY9H+6JeQ7BEM8vQVw9WPK8NOj/z4ebttrcf1VPgohjGCfg95sFcCF/EPIs4033XnfmxjPLb2XfUc7+6829v7gJ/7+Xof1tza8kG+IXPRf03nUQY/t+Wc9wfeIxDGB+rc2iLXGE9YU/BSMjZV6b3S+VFd368icHUb73wZ0+O69IyM1dQDF/GX6CHqpnS9o3Te6/tEP9YUrI36vgIedr2q7RImlSZlMu9WXrd6PccaHXNf+ihe4XbfffeAh1MTvJl/5zvfiaf+A0bmxCeeeGKnf0FCZc5sZF30WeoV0+JMf72XNrvn07Vr/Jb0P56z8d+vTXE9FcYdPIl5qfqQwIdJj3Ppla5vWNz90F8Wl36FgBAQAkJgcAiIwNUQWxG4GgKlYEJACAgBISAEhIAQ6DMC3mtVXdS57d4wVrJY6BdRWViCdGPXWODCoIdhz8QWvwZN4CI93MhvvfXWlvSoX/JHXvn1BC7vkWHUQzMv2HNtCFzEs/fee4dll122E2WKFwuFkyZNivdJA28xLLwhhls8mfnPvHz4BVsjcBEG4gcEEBPiY4HcjKNcJ/98cY63C8QvXucIJYTJGYe5XiU+Tm8I9EY4ngUL8gPJxNoP15t+HYpnJ8hR/lmMheY1xh97Atew2zHGOhaawcWEcpNPn/d0AdrC5n4xfLNY6sW2krR2U9XfSglcpJUuUtK+aGe+jaWeVHgOI7t9Ncy5F+tbXPMELs4xfk5+jnzEeZV4I3zpwnlV3Fwvabu5fur7Rq6/9YPAZeXwuHKNc4xg3qNWbiGbsN6DDef0VcTXc91ie3zA/bN2aZdMn3HeNj0IGBjYEMqFIYQvnhH6FJ5F6COIT6dbv4mBK/7RBtHljHUmtH1/jlczSBsmJX2lSdut8hhiRAVvDCIvVQQuSBboJSNdWr7tlzq3+h42gYs8eO8vnFPP6HSvRyHRQfRCnyIl/TQ+WPHPDPfcJn1INWDi8wDe5MH6SMk4XJF8vFynMwhUNUb3Oy+klY674AL+RmIkDNcg7WJcQzC6YXxDUr3hjcU8h8EZ4z5SpTPqyEiQgul/fnz17Zl0fFsi/6QF4aGb5PRlTs8ThydwdYsTYrLfNtn6cY7ARTylc8o6zHw78/M20myrn3nGe+7gHOIVuNNOrF4497qbcDmhz+HtzusqP7/jGfCy+9Q15A+2kqrDk2dLysdzVeLbRBsCF/GV5qVEX5Ie5MHFF1+cwyjUCeLriON0XBl2ep4cgG4xUnnMrPsHMXLq1KnxCsRa8/rXlhDtooxky37H6eNvcuzJKk3C84EVH2iZDk1JKcOeW/s+Qf7T/Pj3Eu4zz7FxgHM/Fle925SMTVV6r3R+5MuZ6/tVBC7KaHXFMeLnrs9eGf2/Lr0SAheplOih0rGpdN7b7/me4Z+2TY8x2KT3uZZKkzLlCFxefxFnqo/TtQP7eAVyFmOdvZPwHGMuY6etiRBfkzZFOC+eVMZ13q/wXJpKSf8jjty8Ko3bzr0e4Nqpp54aP+yy+01+q9IreWez9Pqhvywu/QoBISAEhMDgEBCBqyG2InA1BErBhIAQEAJCQAgIASEwAAQgr+CdwL7ES5NgYQpvPizy5AQPXSxmGenIh2GxCNfjfFHvBcMLX5c+8cQTo76eI1zJFoq77bZbxzMWxlObjBMfBmu+VDQDPtfI24033hgWXnjh6DWDxS1P4CKM99rAOUI4PKpAPgE7Fs9sW0Tum5GualEMgxLeAPgC0gvxst0XWHuygXe7zpZDENJsQe7ee+8NRx11VNhzzz2jEZX4PIGL8w022CBARrNnuGbCghv55UtcExb2IHUgbMPHF/ipVBmH03B27uNMDYHbb799NIaYccie4RdCBF9de69v/n7uGHIcHoa8QZ1wGPP4Gti2U/EELu4Pux3T/uk33msU+UBom3hv4a+NeM9ePAdueDCp629+odHjYou0dYvTvu+l+a1qQxhDMBT6r4d5lv5EHl7/+tdHT3EpgYswEDJZ1DaCAH2HPGL43mWXXeJzfhssvO3RB5Cqxd3cwnl8oMu/tm03109938hh5Qlcp59++ggiLFnzC8xmHPIeuCDeYtQwQ7YVhzaGAfOWW26xS/G3m/7yW3z5h8D/1ltvjdjSZ5tKlT6z59ukZ/XHs7ltR5deeukR+t1vpVjVbywfVb/ocAxUflwhLHjgteS0007rEIksjrZ9pUnbbUvgwqhCW0RSkijtEa9JeLWzMYPyYBRibMKbE+K3aokXuvwz44j3LENwj4Ufs7u1wW7jGUREyM6pJ4G2/bRLUaLHJuqcrWlygt7Bu9DDDz884na3fOfG4REPJyd1OoPg3cbofubFssb4ybjLuJYK8zx0DeQBBE9L2267bTyGWEPdQwbx4vWaJwBX6Qw/V2P7HPM+6eOkL+F9x9q13WPsoN2wxaKNE9zr5tXHns211ZyeJ7yFRSfjPSMlMKM78XqHLvVSNw6XzinrMPPtLJ23kb82+pnwGI7BhnlzTpij8bFHWv5cWK5VzdloU8x9mE8z37P6njZtWrjgggs6npvq5jVty1eVT677NmHjtA9f5YHLwpTmpVtfr9KXzMUhQzOPSAViIV7rGB9SAhdhh5me3+4xJff4fNPuzFsT72XW/qrmglXj6aDj9PE3OTYiYpOwhDECl70P5Nr/MOfW1APeW21uypyUdwITT66HnMmY4MXPH6o8ABG+zdhE+G56r2R+VNf3uxG4qsY78lkldekZgYs1EsZeL967Urq9PeHa6qHSsamXeW8/53tVfaWu7XpM7bhJmew9JtWtKfnZ4mSeyfyavoBXfYTxz+bp6HDGedptTpiT4Z2Z/tVG/IeRRhirer5t/yMemytVraP5tHxeUh3iw3U77pae1zNpHLl3dgvTL/1l8elXCAgBISAEBoOA2Yz4GIB3Bt6DeHfkj2M7b5r6HOuvv/6zn740fWKchBOBa5xUlLIpBISAEBACQkAITGgEIFKwKAWhhMUcvIdgcPXknioAmOBOnumRBMMwC4B4WeBZ87ZQ9dywrzMpp3x8xYuxCNl///0jiatq4QciEOXiObbgaIpJXdnAGiMIJBQMVixUYShHLE1Icffcc0/HOxb3MPSwbRn5hRiXM1QSzguLqDzDH3FC/MKlfp13CR/HII9Z6FprrbWiZzLyypaBM2bMiOSSJu0vzRv1DGlvueWWi4uZ4Et7rpNZ0Y4XXXTRmFfaFy+O1CnGiFLBuIa3EV5GKXMbQk1pmjzHdqW0ZzCHhHX33XfHvpIa5tM0II6y8Mz2lpC16F8sQDcR+gIETPSMlROSHvWYej9qEl9JmH633ZI8pM94Apdt5QbZiK1f0S30fxaeDbP0+W7nvLtTXxAlGSfQiRBa0UclUqfP+p1eVR576Tc8yzZx5BUvILT9bnqrtK9U5X1Q1/GQxPhkXi8Y44184LdeHFT6VfGSpxVXXDGOy7RnCD6QsbuNZ/3up7T/NddcM+ot+gEYMVYzjlfJWBqHB5EXFjghJjH20naoDzyI4mGrn1KnM7qlZe2A8ZY5Bp7kIJiZQOJi/GR+iEfEEh1pcdX9Ug4IL+gNyEa94lQ6p6zLZ7f7JfqZd4111lknEuYYqx966KE4hjfZBi7NC88zd6TdMddg7mSeFwlL/piToCcgJDSZL/s0Ssrnn+/ncWleSvQl+WZcMx3HuMY8um4+x3PDTo80Jf1FYJhzayOLM4+3LYn7W5pnvcf1e2wa1vyol/Gu3zgSX4keGvbYZOM887R+rClU4TiMtmtps1bAux3jJ4RqxjP/3sU6Fe/TzId5xzNhPYTnmJdxn3UG5qvMb3h/G4YMcm7IGAVJDcntFNCP8o2Xd7Z+lFVxCAEhIARmNwRE4GpY40wAJUJACAgBISAEhIAQEAJCoN8I4KUHAxmC9xrIMV4wNOAinwUu77HHh9GxEBACYwMBPANhXIcghkeFlOiFkQDPM0jOC9PYKMXgc5EjcA0+VaUw3hHAWw1jIZ5Zcp4Xd9111zDzo8JYzKOPProrWWm8Y6H8CwEhIASEgBCYHRDQ3Lq+ljU/qsdIIYTAsBHw3lq9J99h50PpCQEhIASEwPhEQASuhvUmAldDoBRMCAgBISAEhIAQEAJCoBUCGJsxOiN85c/WGOZRgS/3DzjggPiVOfevuOKKgFcRiRAQAmMTAb/VXbqFG94iMLDYVhEQM4f1dfFYQ0sErrFWI+MjP1/4whc620HadkuWc/+Ve7ptsIXRrxAQAkJACAgBITC+ENDcur6+ND+qx0ghhMAwEMArHd40/UdbePDGA7dECAgBISAEhEAbBETgaoiWCFwNgVIwISAEhIAQEAJCQAgIgVYIsG3KoYceGrdP4UG2OnrssceiS31cyePWHcHjCB59bAvDeFH/hIAQGFMITJkyJXrYsn7Ldj5PPfVU3Bb0hS98YSev06ZNCxdccEHnfHY7EIFrdqvx/pR36tSpAa+VJmxnhJc7xkq8VSKMkSeeeGLcGs/C6VcICAEhIASEgBAYnwhobl1fb5of1WOkEEJgGAgcfPDBYb755uus4ZHmEUccEbewH0b6SkMICAEhIAQmDgIicDWsSxG4GgKlYEJACAgBISAEhIAQEAKtEVh00UXDPvvsE0keuYdF3sqhomtCYGwiwJaob3jDG+JWb7kcXnPNNeH888/P3ZptronANdtUdd8Luscee4S11157hGHEEoG8ddJJJ4Xp06fbJf0KASEgBISAEBAC4xwBza3rK1Dzo3qMFEIIDBoBCFzzzz9/Jxm993eg0IEQEAJCQAi0REAEroaAicDVECgFEwJCQAgIASEgBISAEChGgC2gVl555bDAAgsEPPf89re/jYboJ554ojhOPSgEhMDwEZhzzjnDxhtvHJZaaqm4iMv2qHfccUf0CvSvf/1r+BkaYynOM888Ya211oq5uuuuu8Lf/va3MZZDZWcsI8D6zCabbBIgP0+aNCncd999sW/dfffd8lI5litOeRMCQkAICAEhUIiA5tb1wGl+VI+RQgiBQSLAeh5eAx999NH4bjJjxoxBJqe4hYAQEAJCYAIjIAJXw8oVgashUAomBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEQGMEROBqCJUIXA2BUjAhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEGiMgAldDqETgagiUggkBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASHQGAERuBpCJQJXQ6AUTAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICIHGCIjA1RAqEbgaAqVgQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACDRGQASuhlCJwNUQKAUTAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCoDECInA1hEoEroZAKZgQEAJCQAgIASEgBAaIwCte8Yqw9tprh5e+9KXhwQcfDKeddtoAUxs7Uc8xxxxhu+22C3PNNVe45557wvTp08dO5mZxTtZdd90wZcqU8Mgjj4TLL7+8dW6e97znhfXWWy8sv/zyYckllwzzzTdfeOaZZ8ITTzwRHnroofCjH/0o3H///dl4eWbVVVfN3rOLjz76aPj9738f/vjHP4b//e9/drnzu9FGG4UFFlggpnnRRRd1rucOaAfbbLNNmHvuucNf//rXcO211+aC6ZoQmGUIrLHGGp0+ceaZZ86yfCjh8YkA6y6bbbZZQNfdcMMNYcaMGeOzIAPO9ViaE4ylvAwYdkUvBCY0Ajanffzxx8NVV13Vl7IOIs6qjPGOuOGGG8Z3xBe/+MXhP//5T5zL//nPfw6XXHJJ+Nvf/lb1aF+vSyf2Fc7ZKjLeZ1dbbbVY5uuvvw/9HugAAEAASURBVD785S9/aVT+V73qVWGhhRYKvHNeffXVjZ4pCVQ1R1t22WXD6quvHqOkr9H3hi3qdyMR72ebGA/YjoU2OLIGdCYEhIAQEAJCoHcEROBqiKEIXA2BUjAhIASEgBAQAkJACAwIgR122CEadi16JrIHHnignQ71d/LkyQHiD6Shxx57bOBpP//5zw9f+cpXYjp33XVXOOaYY3pOc9FFFw0vfOELw5NPPhkwboxX+fSnPx0WXHDBWI7999+/VTEgAn74wx8OGHq6yZ133hlOOeWU8M9//nNEsPe+971hlVVWGXGt6uTpp58OZ511VrjllltGBPnSl74U5plnnnht3333HXEvPSGfhxxySLw8K9t/mi+dCwFD4P3vf39YYYUV4mlde7Zn9DscBMaSzq/KCwb4t7/97REQCK0/+clPhgPOOEul25xgIsxPxll1FGeXsX+JJZaIz//pT38KzBMkQmBWIfCa17wm7LzzzjH5z3/+830hPA0izhSf+eefP+y1115hkUUWSW+NOOdjim9+85vxg4kRN/p80k0/9zkpRTfBEHjXu94VXv7yl8dSnXvuueG6665rVMLDDjssTJo0KTz11FNhv/326zxTNdfqBGh5UDVHY97GPeSAAw4I//jHP1rG3Htw9buRGFa1iZGhmp2NB2zHQhtshmZ9KD4mZI2KD//uvffe+gcUQggIASEgBCYsAiJwNaxaEbgaAqVgQkAICAEhIASEgBAYEAIYE2xOBpHmgQceCEcfffSAUquOlgUVI4796le/Cqeeemp14D7dGcTCGYQw4uVL+4MOOqhPOR1+NKUErle+8pVht912i0Q8yzVfDEPI40vTeeedN+Jj9yDrfe5zn7PT+NuGwGUPnnDCCeE3v/mNnQYRuDpQ6GACICAC19itxLGk86vyUmUcHLuozpqcVc0JJsr8ZNagOvxUX/va1wb+kJNPPjn8+te/Hn4mlKIQeA4BPkxBN88555zhl7/8ZV+8HA8izrTCDj744ACJywTyCB+nQGjhQxXyYIIx/KijjrLTgfxW6eeBJKZIJxQC/SZwVc21SkGrmqONBfKM+t3IWhWBa/gkwpE1UH72kY98JLzsZS+LBC4+NJQIASEgBITA7IuACFwN696MhQ2DK5gQEAJCQAgIASEgBIRAnxE48sgjI7GGre0+85nP9Dn25tHNCgMphKJdd901GlXwBnXTTTc1z3BFyH4v6lYkM/DLJQQuyFmf/exnY3sigxACIQNCCvSy8cYbhx133LFj/EkNWp7Add5554Ubb7zRPx4WX3zxuL3j5ptvHl7wghfEe//617/CJz7xiU44Ebg6UOhgAiAgAtfYrcSxpPOr8lJlHBy7qM6anFXNCSbK/GTWoDr8VD2B66STTgq33nrr8DOhFIWAQ8Dmtf/+97/Dxz/+cXen/HAQcVpu2FZ86623jqeQtpjL483Oyxvf+MbAdmImeMP9xS9+Yad9/63Sz31PSBFOOARKCVxbbrll9EDHNqGXXXZZB5equVYnQMuDqjnaWCBwqd+NrMyqNjEyVLOz8YDtWGiDzdCsDyUCVz1GCiEEhIAQmF0QEIGrYU2LwNUQKAUTAkJACAgBISAEhMCAEPja174WY77//vvDV7/61QGlUh/trDCQ1ueqfYh+L+q2z0F/nighcH3gAx8Iyy+/fMwAi91f/OIXA8aqnKy++urhne98Z7yFK3sW1fhFzCjF8dlnnx1+/vOfczhKXvSiF0XvXXg1QPDkhUcvRASuCIP+TRAEROAauxU5lnR+VV6qjINjF9WxlbOJMj8ZW6gOLjcicA0OW8VchgBbehpx6zvf+U64+eabyyJyTw0iToueDyL4WAJhLl+1Jbw37kOUhDApEQJjDYFSAldVOarmWlXh665XzdF8/5pVWyjW5V33JzYCE6kNisA1sduqSicEhIAQaIOACFwN0RKBqyFQCiYEhIAQEAJCYIIhsN5664VVV101bsUAQWPy5MnhNa95TSR/sNXaDTfcEL/ihfyB4Yyv3ZZZZpm4bQOLyGyxd/3111eiwjZua6yxRmBxG8EDEAvLLJgbScQeHmRecDu/0047xfItsMACMUmILXfccUe49NJLLQud3zXXXDPwx2Ty3HPPjcd8Ac2cadq0aWGppZaKYadPnz7KK5FFgkcptrb4+9//Hr773e/a5VG/GLjYGmP99deP9/CWdMsttwSIXD/72c864duWwR5s8xx5fslLXhJWXHHF+DjbdNxzzz0xP9QZHpsWW2yxWI/kjYXOddddN7pBf+qpp8J9990XLrjggoibpc/voosuGrbddtt46aKLLgpzzTVX9Li10EILxfZ1ySWXhF122SXep+x44UKsTUAGIt6VV1454gT+kIX++Mc/Rm9dtEOTTTbZJCy77LLh5S9/efRA9cwzz4TbbrstGj1+8IMfWLCuv8RNW6ft4s2KLzP58v3BBx+M7SU1oJAfwv/3v/+NRCfqc9NNN419hTbw8MMPxzL98Ic/zKbL9ievfvWrw0orrRTA5NFHH41bDeER62Mf+1hYcMEFY/r7779/9nl/kfr55Cc/2bl0+OGHR5w6FzIHLEaTLuK9ZDQlcPHcpz71qVjPHJ955plRd3A8aALX3HPPHXbfffdYR+iWnPe2rbbaKiy55JLhoYceCrQ1L9QV3gvADSIa/Y9wxFPlwQA9MHXq1Fi/PEMbo6/87ne/i1+Gc5wTvJRtscUWUb9yTN+iH9GOuc412nLqXYG42ujSXNq5a03jfOtb3xr7LOPAaaedlosq9jn6HkKZrA9zTvu2ctPOiAeswPj3v/89QUYIXuEYb/7whz+En/zkJ3FMoj+jR55++umoG9HDqUe5EZE8d0JdUb8IeF999dXP3Xn2B/1rxtGLL7449lUf4C1veUvcapRtQSEwpgQuxk9wXG655eKYRt3RbrptFdYUd/JRogN9/rsd095e//rXx3Fxvvnmi/qLMe+aa64J1113XfbRVVZZJWy00Uaxv7B9E3oRfQh2abul3jE4IMwTwJ9xdMqUKVGnoeeoY7wp5PpM075Ju2ui84eh1+vykhoH6QNveMMbIkY5/RRvzPxn4wnzJuYTObwsrMedvgjGzC9od/QH+uZPf/rTzriw4YYbxrEcHUh9Wp0wB8zJMNoAZUjnBHXzE8trmzkPz3Sb76Hv7rrrrlF54ble+2Yv4wHp54T5O3VN3wFD6pB++f3vf7/TZpjT9NLm6L/bbbdd1NHzzDNPnO+hA3784x/HtkO+XvziF0fMaFM2t2D+xFyIeZifQ5HPNuOD4U5bHcR7S4prXfvwYx3vUPQntgZi7veXv/wlzj8ZE5hb5KQJnulzbds43pmY16M/2MoyFcZbxl3k8ssvDzNmzIjHbcrOWLraaqsF3rHQtbz78I7F3J3tu3PSZiy055u0cQtb9ctHBvQDyvnlL395RDCvP9uMW4OIk4zZHJp5JkbvKvEksr/+9a/hkEMOGRV0s802i+9GCy+8cJzT4fGZd03ec6va56hIZl7I6WfCWd9s886Wiz93rWm9l74T+Hrv17g5iDg9NmuttVZ8L+e9HZ3Aezg6/8orrwy33367DxqP3/a2t3W2D0Vv8e6EzqK9MP9AZ9HOTj311FFrNETAOyr6Dbn22mvj2BhPWvwrJXDxjo2eos8yR92k4byPrLXRy+kcjXcQhLks9xAjcA26fmNi7l9Vv7MgJWOJPZv7ZW2DsR59wXwfD9u8I7A2wbsUbaWp2HsdW7zy/gaWzFV4/+IdmD7HmhrCB2isiSy99NIxDe5T53ffffeI5NI2wc3S9ZgqbE2nDWK+0XYtIdcGRwBScdL0XYrxgb6CXHHFFaPe6bhe9U7Nut7rXve6WHeEoW2gV6gz1l1szdd0yAorrNDx3M7aDesCrB2xjmbCeyke3skTZWC8Qm/xLk7cXobdF33aOhYCQkAICIHeEBCBqyF+DLASISAEhIAQEAJCYPZDwBujIWtBxkmFRQtIHXvttVdc+EvvX3XVVeHCCy8ccZkFVOK2hYARN2ee8OINscQvGA8qLyzm4JEIY1lOMDIcd9xxHWMmYWyRhAUHSDe2fQX3MFJhbEJ4Fg9JqUAWgXiDPP744+Gggw5Kg3TObXG+c+G5AyayBx54YDwrKQMPtn3uiCOOyNYxC25HHXVUOPjgg+MiCm0CA7IZ5p7Lcvxh8YXFXxZkTPyCKMYh8IQUhUCMwKDEV7QIxtJjjjkmHlubIE4WiyGQ5QQS3o9+9KN4C/ISRsNU0q390vt2TjvBEEMbrhIWdCGimWBY2HnnneMp5YMwxGJSKpBVvv71r4+4THp8YW/EQn8TsgqLWhhDwbwJgWuHHXYILMIhTb25sUgKQQbBYAW5EWlD4KK90+6RM844o0Ns9O173333jfer/lFOMzr59l8VnusYLqx/YbA48cQTRwVnS1LwTdsAhgjIPdYW0wdpi+gGv0jNwjLtsuoZdMbxxx8/gsBEvBh0P/jBD2bbFcRH+iptBgIX/cekRJfas1W/beM89NBDo2GH+MDDG6wtDfualnPv1QIdsc8++8Q2bGH9b1pe7h122GHRWIBBiTEC7FKhTr75zW9mCWA+LGWlDVJfxAXR0AveDq2vYiA4//zzO7fpFx/+8IfjOduH0q5NJ3GR9gaBKycQJlKyWFvcidfSa6MDc/lJr0F42nPPPbPtkbAQLb7whS+MeAxj3zrrrDPimp3Q7tEdnqyMQdH0OvUMgQADYSqMJaSFjjNp0zeb6Pxh6fW6vPixkDFk2kxDFnMh2iAY7rfffiPmRYaHjc2EIQ10WZV43Bm70ctcSwViBXUCMToVxh7qBIOJl2G1AV8GmxMYBj4/HNv8hOO2cx6e6TbfYysyyOvWji0vPNdL3ywdD0g3JxBmMCrn5hGEh0DzrW99K+pu2lppm4MAAKknJ7RNjH7MgTDK+q2UfXiMecyjkZLxwXDn+X6+txBfTurahxG9MV5WzVHpr8z9MER7aYqnf6akjTOOMZ4huXkYOuB973tfvE/98Yc0KTvzNuKHXJET5rDM63gXMikZC9u0cUun6tc8w9FmeYfzhFive9qMW4OIk/wbMYzjb3/72x2SA+ep4FEXbCEOeoI74+6HPvSh7LsRcTC/OOecczofXqTxpuceo37pxDQNO29b76XvBL5M/Ro3BxGn4YIXOfs4zq75X3QNet6LefrmAwOIFBBwEdo/8y97p/fvcP55P7/hvb3bhwr+OX9cSuCy9wJIasyTfF58/Ol7Xlu9nM7RuhG4Blm/vkx27NPz/Y77JWOJxZv7hXDFhwNVwhwRvW5G16pwdt3qj/A8y4dpqfBhD2tCRib299HVJ5xwQiQF23WL09oE10vXY6qwHdR8o2QtwcZjymkkQo67SZt3Kb8VLx8+MmdMxa8x2fs7H1d99KMfzb5n8Dxj/5FHHhnJpf75NG7WPK090Q/5gAr9nwpt4bzzzhvxoZGvv37p7zRdnQsBISAEhMBgEDDdj20Nvc+6LesV/HFs501Tn2MmQ/x/TQOPp3AicI2n2lJehYAQEAJCQAj0DwG/MGGx8uUsC+58ccdkyQtGc8gdfBXFIrEJC+BMuEyMMME5C8N8MclkzL765TrGdMKZIXIQecE7zuc///mOgZ48sqiNMZeFTysDi0ksCBpRwy+SsFBgOHB8+umnR9IHcSDEb4SXeGHmPxYd1l577XgKsSjn5cvCYkRnwXmRRRaJl8ADzyR4KzjllFOiV6CSMpSUfe+9945fpdvCGrjwNS9fRWJ0MwKX5Z1f7oMrpClbaAEnFmKsTfgFUf8s4fC4xVd33Qyk/hniJE3yaHXAfasHiFR82WftlzqFjACmkE/qxC+Ks6gN6YpFbgySGKpM/FYmfsHQ7tOHMKIwz6a/mHjjJdfIt83FwYP+xy/GMGt3hGtK4MIIZwZ5b4gjjrbSlMCFcYhyWP3TTqgjZKwSuMAcwxgYgzd9GN1AHeNNiRdGBE9EeCdAaAeU0/QGBCO8m2A0ZwHTjCHUvW3Rw3PEyXNWn1bPtF/6qRdbELVrJbrUnq36bRunX9TFowYENS/UO4YiygdRwMirqQ5C56PXwN63b0/AIF5blPdpQPpFN6InrZ2lhhof3h97veUXvSFReI8W9FfSNvHlxvD1h5mejHLjFIvT6BeMX74+fT8gzra480wuvTodyHPdJG2P4AjZky+oMfJbO4WYAIEF8WMa7ZcxnT7DOEr7t2fwxIX3B8QvqMcLM//xLN53SAsdbs/5NtC2bzbR+cPS63V58WMhBC6Mg37LW09GNswg12L0RHJGWQtnvznc6ZfoK75iz5HoWDhDbzEXsTrhy3lIkibDbAO+DGaorJufpPqGftKP+R5k9H7NT8Ay7X9txgOri/T33e9+d/R+xHXmbfRn5gyQfYzUxVyceS79vaTN8YHHbrvtFpMmLnQ5bYq5lpG3uXn00UfHjzSY21JW04noScYA5kDMKdP6ajo+5HRiP95bYsEy/+reB/BG4omN1Ce4YFRmvMJzCcL1b3zjGx3ScRs8MWwjKWZN23ivBC7SJv+mGzjmXYiye4KRtQvKPnny5I5Bl7GCOZBJyVjYto1bWrlf9AuetygP3gi/973vdYJ53WMXKW/duDWIOEmffgTZBiEf6EPI4czFOG8ifg6Enkcvoifot/79A0zo13XiMTL9zDO5vtnrfKVtvfeDwGXl73Xc9Dj1K07i2X777TsfktHnGKvR68zH/BjOB2je87QRuCwv9ov+pE1BdkKY66ZhmbMxDtJn/Dzf4mj62y8CV91ci/yU6OXcHI24/Dhg7xGDql/Sy4lPz/e7krEkF79d4wMPCJ/UNTqGj4x4x2EdhHm7vYM1mY9anOl7HfEyTtKucuRf2jNzBuYvlh46C+KeicVZReCycE3WY6qwzem0XucbpWsJuTZoZcz9tn2XYq3qs5/9bIwK/HMkfD+W2DoYRD7meoh5daTOaEe2LmIfBeF10zynUfcI794IJC/mgXi/f8c73hGv8Y/1JDCn7eE508Svcfn6s/u96m+LR79CQAgIASEwWARE4GqIr720NQyuYEJACAgBISAEhMAEQcAvTLCYgvcavn5DvNGQcxZvIGPwco14Y57fMg2357Y9Cy/ceE2wr5t5wcaAZAYl//I9iLx4AgpeYzyBh8UF77HAPKxQNr9IwjnGE7zisBCNYMQyb2We4BFvzvxni0pgSnkNM7uf+7XF0t/+9rfh2GOP7QQpLUPpcyykmeevlEziF24oG3XrPQqAp33BC2YY8hC/IMo5Czx8SWnP+oUXvyiZtgnIcH4xGmyN+IZHBwyDJiw0Ey8LgOahye5V/bIwhDEKgTCCBxLKaeK/HPRtNyVwUYd8uciiIcJCM2EQCD9mDGZbG0giCMZW8DQyIIQgjG62+NWUwOXrAAIgBLlS8W2IrYpwW2/Cwi5GJRbj8EJkZDpewKz9EHasErjMWwJ5ZOsG7znIG9g97iw6sp0EAhZg4oXFTha4EdoR+g/Be6FtS4phBG921q7Ql7QP8ER8nyvVpTGiin8lcfp+kVvU9e2YL/Jtiybff/11skb7huhlC7joTFvENf1JOBZgic+2g6Fu6M/0bcQWkONJxT+2YrNtX/hqF29+CNsHsh2gCYYwviI2gZjMgrE3VvkyUYeMC76PeU9k3nhWgjv5SNNrowOtHOmvNwh7fUQ4CB+GgRlLWNRH99NGwYj2C0HEBK9cGMoQCLP0f8Z8r9e5hy7GC43pOO/1xWNc0jeJv0rn+/Y7DL3eLS9+LDQCl8fBjBzEYeLJNlWeMSwsvynu1DHzCxuP/DhOeK/bWRfCgAOBlfmOeRgddhvwZfBzgm7zEz9e9XO+V5WX0r5ZOh5QV1VibZ/+iZdBq2vC+/ms6b+SNud1G3M7IxWRhm/Xvr58X/bbM/OMx6/N+OCfQwf3472F/FSJx48w6fsAbZJ+gn5Ej6EfIciYsAUxW50h3itqCZ6lbbwfBC7yn5bdj/3MDSBm2fsORHcMu0Z4Z1xgflg6FrZt4+S3m0AIxcDsjf+E9/2d86bjFmEHESdzEN51jMRAOghtn/dyyBV4S6EP0f5SYQuwN7/5zfEy+PNeY3XERf8uz5zV3oHSePy5x8j397Rv9mO+0rbe+0Xg6se46XECv37ESTwQ7ehX6Hv6nK2xcM9/mOD1DfdsnYFjnmU+wTsHbckTtJjHMQ/kuglzZebMSJV3Hgvb7bdfBC5Lw9pH+q5fqpf9WGZzNNLy40AVgatf9WtlS399e/L9rmQsSeP253hpnDyTgIvgEZ20THjXR0fQ/nLvgxYu/fXvdbxb0IbtXcDrIJ6D4I23QRM/Z+W90fScxel1eOl6TBW2qU7rx3yjdC0h1wYNo9yvn381Xefw5GrmMnxcY8L7gRGx7f2A8cneE9L3F94jeF+3jxgh31H3iLVZdAzzEy/e6zcf0LHOasJ4xnbqzLd8+/P1R9hB90XLj36FgBAQAkKgdwRE4GqIoQhcDYFSMCEgBISAEBACEwwBvzDBNjFsfeWFr6GMWOAN7IRh4ZvFasR76sCgzgIPL+UsurCo5gXvD7ycIxB5WCxA+p0XXubtC2uMWd4jTkxw5j/ySX4RM1Zz7BdJ+BoZEooXb0T0ZSCM3zqGL7Ytfv987tgWVj2Bq7QMpc+RL182Tybhnl9E42tdtgnzArmChT0WbVgcNiKAXxBl4Q1jhDcw+oUXvyjp20S6oEe6G2ywQXjTm94Us5DmtWpR1+c3PZ4yZUr82p3rGDo9YYlrfK1u3oX8YphfMEwJTDzHQidtEfGLzCyE2TycvoYhxov/CtETiXyY9NgvfGE8g7BQKt5Y2DSOdLF3rBK4vFHVk/GsnHyJjic3DCPmhWannXaKxkd0GyQHI3TaM9zfZJNN4inP8FW8b9voGBY6fdsnsPea5ttxqS6NGaj4Vxon+WahFrGyWRLeuxHxo/f8l7y5PsGz3vgLCQrCIWKL8hzj5eOmm27isCMerybbuOCVwPS/J3YYAZT6tHEOEiXGd87ZXpFf743A6ySfZ8ucL1M/6tKn11YHWp78r9dF6Gi2ZbUFdQsHAQSCHUK9o2Nti1VPSrPw/Po6MaOXb/tgzFhvCzT2LMZk89BjW3uV9E3iq9L5w9br3fLix0LDifBmkAUn8Pd1QpvEeE99Mf4QppukuFOH3sDrjWWpgZd4jQRMOmZY8UbXYbQBXwY/J6ianxB+UPO9qryU9E0fV9vxoFud21wdgxb158cYvK7RPxE8vvGxAtK2zTHmmSHO9HyM6Ll/EA2ZAzLumUcjb0D0BK5exgePe7/eW3w50uO69wFPsMyNV8Rn8zLfv9vi2Usb7weBK/cu5D1vQM7FY60Xv33ZhRdeGK666qr4TlTyjljSxn1e0mO2sKLuED9v9X2U+mo6bhHPIOIkXojVeKKy/se1VMgrH2CxDbQRIwjj3zNy/ZYwfhw2cgrXq8Rj5PWz75v9mK+Qftt67weBCyz7MW56nPoVJ5jwXsWcACKVzZu5bmJzBt7/6KMmts7Aea6/euKO3wqd8P4DFZsnc72t+LlESs7oFpe9F3iyDuGr5n2lerlqjubHAesjg6rfKhx8er7ftR1LquK365BrWMtivcIIO3aPX9u+kjZtc0R/P3ds9cc9Pxfg3H8wyhyG+InbxHtD5kNM3uMQi9O3idL1mCpsvU7r13yjZC2B8ubaINerpORdypPL8D5LXZn4jxhtDdDXXe59AvI6cSJ82GhjUxWBi22ozRNgbgwhHv8REvqPd3Fff/3UtaQnEQJCQAgIgcEiYOuDrK0zv2X9FXsSfxzbedNcaAvFpkgpnBAQAkJACAgBITAuEPALE34B2zJvBh5PxrF7eFpiEQuxbbWYZGH0RnCNbkSteMH9s0V/v/jT77x4gpk3pLtsxEMjmPi8+EWSKoKAXzDjmC+hEW8cNW8H8UbNP1tY9QSu0jKUPkcWqwyk3PMELl9m7pnwlT1xICz4shjnF0RzdeEXXvyipG8TuQVjPLkZAdATM0i7alGXe22ENs32kHzVvO2223YIV1UELlvUStOwvuSJWGaY8KSuJs+lYfw5eJiHOxZebbHMh2l63IbART2zOEf9ebH+xTUjaPj7/hjjrxkbqkg/PjzHpcYaT/4jHkg711xzTVwI9MZv7nUTPKRBEILsxSKleUkwkpPvi75v+zg9wcj6Ry+61Mftj3uJ0y+q+q/v6btGnPBkVjyz7bHHHjF5PF6hC1MBO9oHwhYJ6BfEFuVz4w73PSmgqQEo1//MyAWhAS9SCG0AA6j3UOP1uNdJbCXJ2OcF4hMEHMR0Ui+4+/Ta6kCfLzv27TH9WtrCQNTDEI0XhhtuuCGSvBjvGSPR6bn+sclM4iJGCcSMHF6vV5GZbRGf5zgmzdK+2UbnD1KvU5aqvPix0BO43va2t4U111yTR4O/jnfD97znPfG6HxvjhYp/Hnf0f2qA22KLLeIWTDwOoYf0vEASsy3xTGcz1xtmG/Bl8OWump/4dm061JfJjm08ajPfq8pLSd/0+WwzHlj+q349AQNiGKRXtoaj31VJ2zbnSZr0U3Qf8510zPfpeV3tjba9jA8e9368t/j85o7r3geMIAMm6LCc0IfpywieRTBMtsXTt522bbwfBK7cu5DNYavKDmHYtpPHiyYL86XviCVtPFcX/prNNfwWvr6/txm3LN5BxGlxM7+AJM78hLmvzTftPr/UBfMd6/tWR93eMz74wQ8GiM6Itc94UvHPY+T1s++b/ZivkHzbei99J/Bl6te4OYg4K6okvndD9GMuNvk570lVBK6qtuA/GoKMCckL8d65/DtsVV66XR8WgatUL1fN0fw4kCNw9avNdMPOtyff79qOJd3SqLrH+zntC0IO4zfGVD+PqnrOrptezD3jPTvliMJ4vjYS0DnnnBOuv/76GK3FWUXgarMeU4Wt12mDnG/UrSVQ4FwbNHxzvyXvUozZzJOpX+aSfstKvwYIwZXx3H8ISx5YC2V9jA8gzSCfy5u9+6XtwesH1kX5gCoVv2Uo2zifdtppIwhcw+iLaZ50LgSEgBAQAuUI2HghAlcNhvblf00w3RYCQkAICAEhIAQmGAJ+YcIWpHwRzdiVW7DLEbi8637i4cU8JywMmJhL7X7nZdNNNw077rhjTAZPUSzk5MQbK+3rZL9IAiEGYkIq22+/fcAQipjRn2P70p5FdOLmt4nkCFylZWBxv7TsVQZSymCLN+mCiy+fJ/0YicUviF566aWBbTW8NFk4gxTht/3geb9wlBpDqwzoPt2qYwwkm2++efzaPWcg4bkqAtcFF1wQpk2bNipq2hb5tb7kcfYL5emDRsiy59L76bnvR/ZlYhqm6bmvSxaLbctL/zwLZffcc0/HWOTvcWwLrBwbGYDjnPhFXE/oyYW1a6XGGuoV70NGNrT4+MW4AUkJcgPl84Ihg69DV1999eg5CCJITqzte+IT3ifwQpGKJ/iYYbYXXZrGb+e9xAlehx9+eFzUhawHkQfBWGTEnSuuuCJcdtll8Xq6PWHdWEDfNuKTtZl08ThGPPOfx7QpgcsbKBlzFl544Y6HQAzKfNGLd0i2cSR9I+KSb8JTZsT3r9yY6duw6aRecPfptdWBMcPJP19f6bZlSdDOqc0DfB11bj534OcDbF1h+pdfBK889IlUvJcEPDZC2ivtm5ZmlXFyGHrdyleVFz8WeqIWJGEzlFgbJC5f/+m2dZZW+uvH09zYstlmmwW+pEfYOpaxzIufE5nOHnYb8GXwhko/bpquJO+lcyVIDnXzvaq8+Lpp2je97mozHvj6yR1DQGVrbz+3JhweT/GKgDENMqaXtm2OPo7eBw8v6EjaLMRN5sJ+nlZF4OplfPC453SwtdXcnMnrKfvwxJcld1zXPowIzLN14xxh7H2kLZ69tPF+ELjSd6Eqb8qUsUp6GQtL2nhVPuy6bW/ux3nf39uMW4OM0+JOf/GSs/7668f5KJ72TPC4SN/AuyUEKMRva2/h7NcTJNLt6C2M//UYef3s+2ZTnejjzR23rffSdwJfpn6Nm4OI0zCC0En7RY9AushJFYELUoStOaTPMc/nPcd/QOG3Pc1tH5/G0e3cEzSazt+Jz94LPFmH61VzrVK9XDVH8+OAjTuDrF/KlopPz/e7tmNJGm/uHG9/bL06eSYZEHJROrfgGfRmWw9cufc6r6fswxefJ6+f2hC4mq7HkFYVtl6nWb37vJXON9quJZBmrg36vKTHpe9S/h2A92PmkP4Du1Sv8LEWpL5U6KuMO8wLGUu9VBG4zDM2YavmU9yz9mjjmq+/fulv0pEIASEgBITA4BEQgashxiJwNQRKwYSAEBACQkAITDAE/MKEGep8EW1hwhaD/b2cIcR/FeXDdju2BZF+52WXXXYJG264YUy62xfFPt0cgcvyl5YB4wUL4ywimOcZbxDLLSCkcfhzW0w1oz/3Ssuw0UYbFZe9ykBKfozAlVuA4z7Cgt96660Xj0844YS4pUfVgmgMNPOfX3jxi5K+bnLts98ELhbBqe/cNiUYQvkqxIwkVQSuM888c5ShlHKmBC7vWYVtT8AqJ7aYlTNG5sL7bfyaGGKIAyIiC6QIRnHbStUTuM4+++xRW0rGB2r+eU91thBY9YjHJPcVbu65JsYatvWcd955IwnHiEfEZQuoLDyyQJ2Tiy++OFx55ZXxFl8f77PPPvG5NCx9ArF4jMDlDdVVbSNH4OpFl6Z5s/Ne42TLXIxGiH0Za+2ThVYMdoaD9zhi6Xf79TrFDDX0N77yTcWTIJoagDBy7rrrrjGqs846K3rVg8xkRir7gt3Ord2mRMI6nZQjcPWCe1163XRgihvnfky55JJLAjqiTpp48PA6PEfgwvvKiSeeOCqpHIGLQG37Js9UGfKGqdfJB1KVl25jId4HMZLQl8zTmRlTff94NoXq/74ucrh7AldOr3vjjY27w24Dvgx+TlA1P/Htut/zvaq8lPTN0vGgurb//w5e84gfLz1m3Pr/uyHOX/BKYjqae23bHOMtW6qSFsbBVNCfRx11VDT4ca+KwNXL+FCHe9v3lrQM6Xk3oyk40zfaiG0lyDNt8OyljdcRuPD+h0c2xG8p3a3s3iOY92AVI6n418tYSJQlbbwiK/EyxAH6APVo5fb9Pac/ebBq3OJeP+PkPdM+MoBsmvN+SZoIW1m9853v7PR93mV4J4W8jphnzHiS/PPbjjHfZd7bTTxGXj/X9c228xXLQ5t6L30n8GXK1XvJuDmIOMGkSn8yd8ADjnlhTokWts7QbX3Cx20fAflt0qu8b1td1f0Og8DVi16umqPldOGg6rcKQ5+e73eEbzOWVMVv1yELQ/JP5xG0L9Yi6MeM/5y3JXDl3us8gStHrC4lcFW9c6frMZS7Cts6nVYy3yhZSyCPuTbI9W5S8i7lx2m2J0QP+I9W/ccnljYkz6222iqutdg1/5u21yoCl/e46J+vOjaSta+/funvqjR1XQgIASEgBPqLgAhcDfEUgashUAomBISAEBACQmCCIdDvhYmll166s5iDERdDeTeBFGPbPPQ7L36bIPviPZcXSAcYvBAWoliQarpIYuQFnuXr9KlTp3a24sJYzSJCU7GFVU/gKi0D3qNYbEHalr3KQEpcRuCq2i6FML4eMWLTDqoWRAmP+IUXv8jj4zJD8rNPPPu/mzGgyoDun0+P/cI1hhI81LAVEcYpjJ6eoNcrgYttyoyc0u1raNsGoimByxvieBliS8s6MbIK4fyCZz8IXHiVATckt11XvPHcP2/ctkU5fz933MRYYwus1CmGiJwsssgise+ussoqMb9+4RoMwdKM3TwPaRPX/RjFaOP0CRa8MfIgRuDyZKOqr4ExTmEQRMyrTC+6NEaU+ddrnGBDm0DoG7QVFsLBii90bWsk7ptnC475+jb1/sJ1LyzqQ5ZCBkHgYhEbnUBe0csYuGiXRhT0C9ZsG/OhD30ohr3uuusCJDGTOp2UI3D1gntdet10oOXZ/7INiZE1mxI8bfvJbh64MEjgQQu59dZbA9uleb2eW1AnbDdDOPeRJn2TcFU6f5h6nXwgVXnpNhZ6/YeuYBtq62+2TcizsXf/X4d7iSF62G3Al8HPCarmJ6VzpSbzvaq8lPTN0vGge42PvIuuw0MkcwFINhhHTTyWXOulzTFusT0e2wcbYYA4/VylisDVy/hQh7uN900/PCHP3aTufYBxDxI2Xhpty7Fu8c2YMSPrmbcOz17aeB2By3tmNCITZehWdj50YO6IGGk3nnT518tY6KNt08b9c7ljyLJ4srIPcXx/Lx23+hWnn1M2Ga95T5o802MOcvrppwfeJ3mHQMxTSTxJ/uFVFt2E8AEH89pu4jHyOqWub7adr6R5aFLvpe8Evky5ei8ZNwcRp5+rMn5Rx7yn4rmIeTRi3qdKCFy+j/Ieduyxx3a876YfNKT10+R8GAQu8lGql6vmaDldOIj67YahT8/3u/SZurEkDe/P+YgAnWHvwHjeRh/wzmceqU2/icAV4naDfLjVZr5RspZAHeXaoK+7uuOm71LUPR+PQNKzj0fMGzx1bh+Y5NJjvslHcRCK0SW0WZPzzjsvXHvttfG0isDl11VZF2WduJs88sgjcc7p+0a/9He3dHVPCAgBISAE+oeACFwNsRSBqyFQCiYEhIAQEAJCYIIhULfY2tYQwss+C4cIL9V4vskJBCMWEnCvDcEI6XdePCmC7dC+9a1vjcqKX6Tw5I6miyT+q2W2w8HzFF9M+7hGJVpxIUfgKi1D6XNkrcpAyj0jcHF80EEHxa3mOPZihCNP8qpaELXn/MKLX5SsaxPdjAFVBnRLM/dr7Z0FKtouXzJ78VvY9ErgIl7zapJuCWFpgguGc9qpN4ra/dyv//qf+6eeemokBeXCcs2T0tKFuX4QuHbfffcOqTEl+aR5sgU9rjfxAkA4PKLRLpEc6YuFVeoV8f1ym222iV8sY3hMt1dFj7FYaZ7Y2GYMUgqLrgh1Afku3R7Vb9NnBK5lllkmeu3iOSNncezFf9lqYXrRpT5uf9yPOI3IwaIudbTtttvGJCDsgJGJNzJB3oLslQrt+o1vfGP0tkTbsIXdQRC4SBsiHvoNI9ekSZPi4rRtY+Z1EG1iiSWWiNmlvBjcTep0Uo7A1Qvudel104GWZ/8L2YI4kSqDrnkjIwxtfq+99oq4oR/w6kH7T8UTMn74wx+GH/zgB8UErrZ907YArNL5w9brYFOVl25joW87EAshFKy44ooRavoE29Q1Ed+W+2XIsL4zrDbgy+DnBFXzk17mPHXzvaq8lPTN0vGgW70vueSSAa+rCP3OiLD2DCQrtoRF0nlE0zYHcQIPUAjGXMgCXhiHGTNtS2EMfBCaqghcvYwPdbhbf29jUPVlSY/r2ocZY+kbkFjTeQHxYczkD8F7BWTYtnhiDDWid9t3Gk/gYuvoVIf7uV5TAhdlsTlsFbmXbWsZ4xHiZfv0knfEXtp4TLzLP//RA3ob4iy/SE5/cr2OeNyvOCFi4lULIV+HHnpoPK765+v5+OOPD3izsTrKeb6xeLzRnPYM8aeb9FMndkunpN5L3wl8mXL1PlYIXP49I+eB1uv0EgIX9WFzcDwqnn/++SP6MP24FxkWgatEL9NHquZouXFgEG2mG7Y+PZsXlY7NVel4Mq//oNCHtzGWMQ+d00SsTeX00OzkgctvRdhmLQGMc22wG/al71LE6edZxx13XNhzzz1jUuk6DvNL3lOY70HQSuUd73hHWGONNeJlvz2mrfekbch/cGNjWBonH0Di7Qu58cYb47us7xv90t9pujoXAkJACAiBwSAgAldDXJnkS4SAEBACQkAICIHZDwH/gp7zcGSLNG0MIfYMaLIIjjHci/fU4RcX+50XiBvmHQaDCp6AIHB4weiFBwLEkz+aLpJgmOcrNYgIEBps+zS+VDz55JN9UrXHOQJXaRlKnyOTVQZS7kGUsS0EjWjCdRO/BR7GZhbskKoFUXvOL7zYoiT36tpEN/JClQHd0sz92lfLOUMcdYzhi61DkX4QuGyBmfhSAgzX/FfxqeGV+1XijaYswLOlFNs0psIiGB6prN36xTXCeqNebqutNL7cue/v4Eq/8EQfe8Z7JuFaFUHQwvtfM1DlPI75MngCl3kdQzdgdCVvXjxZD5Ip+sG8d/EVsnk1sGdYlKV/sJiOGIHLt23qAo9/Xg9BEoMcYc/5flWqSy1Pud9e44QIwIItgtGWPsjCLQZ7L94LAuXlfoqx3xbj5z//eaCNId0W+rnv20rOgEWYnHiSkd33BC3batPu5cpVp5O88cwbP0pxr0uvmw60cvhfthMkL+gz6sNve0k48g8O3Lfy+wV4X08WL6QNdBnkZYS+QR/xbT+3oE7YnCG8bd80AmaVzh+2XqdcVXmpGws9UQrdxBzDPMMQbxOpw73EED3sNuDL4OcEVfOTXuY8dfO9qryU9E0fV9vxoKruMY5RPwgk2JwRzfRPjmjTpM3Rx+lH6IUqIoknFtjW434uwtjP3BjpZXyow93K2ua9JWaq4l9d+/DlhqB06aWXjoiJPsy7CHVPn2aLUqQtnniiKH2n2W233QKkOeS0006L3kPjycx/zGmZ21K3SBsCl38nyHkd9tsh2bhg9UNaTd8Re23jpNVNTF8zT6adco60GbfS+PsRpx/fGa/BODeXJ23moIzdKYnSz2uOOeaYgD71gk7lgwTq37yt+Pu5Y6/HvH6u65u+PH5+lEuDa6X1XvJO4MuUq/eScXMQcXqvxqwb/OEPfxgBH4RJiJOIX2Ph3NYZum2hSDi2G2fbccTWNnLzxRig5b9hEbhK9DL6uWqOlhsHBlG/3eD06Vm/Kx2bq9LZcccdA++/yC9+8YtRnvQ9WZ42IQLXl+IaStP5Bh8HlawlUB+5Nsj1Kil9lyI+X8+mA7iefhTI1ssQlpGcPuK90IjH/qMhI3DxnF9/9gRCvL+xxpqK3+b96KOPFoErBUjnQkAICIFxhoAIXA0rTASuhkApmBAQAkJACAiBCYZA3WKrLbQ3XZgAng033LDzZTkv/RAZHnjggYgc3j/e/e53B4zICJ5tzHvGIPLiFxaYGLKgjjEU4ctmjF5mtMAAbV8dt1kk2WeffQJeFbxgZLGtIf31bse2sJouapeWofQ5v9hCvUOmgDzEwqY31lAWtvTgD6MgC+2kacYDvtpl6zSkakE03pz5L7coyb26NtHNGGAkEBYYqXfqgzJ0E1vsIgxEBcghGFmpXzxJQXgywQsFW40g3hOb34LQwvJLmyC/nojlPWOQNwxH06dPj4+xiMVCqrVP/5yPt+rYjLJ2n3gxSkDSYsvQ1VZbLWywwQaRIEAYyBoYXT25yJOfSglcxO3bDfXBtnS33HJL3L4O0h95oT+a2LZ2dl736w1UxI2HCwzEGFzMaEkcnsDljQh4KcAwZu0D74BsoYdBDCH/jz/+eNySw+rDCHcYZ8n7m9/85o5e4xlIc1aXeFCw8tGnTjnllOjViW0G+IrUb3HlCVylupT0q6TXODH20k68+L7gr3tPThj92dIEXYGwHRQELvCkTVCH5vHO+m7uS22eLSVw0e4hLJnQt23bP655ncm5X2zmHKnTSVUErlLc69LrpgOfzfHo/7RVvEUijIuMPRCuaIcspBtJlX7BF9CpV7/LLrssXHHFFfF50sdQhqcMxBsGvV7PGUQJnyNwte2b5nHI2k2q84et1ylXVV7qxkK8k7IVs5ccKcTfT4/rcC8xRA+7DfgymKGScnabn/j+28/5XlVeSvtm6XiQ1rOde1ImJB/6LLoLgRjMGGPeCjD221zTnm/a5sy7Ks9dfPHF0QOjxYF3KebTzP+M+Mk9PzeiHhk3bQwoHR/qcC95b7Fy5H7r3gfYPpL5lo1lzBvtvYYPHphLQJJBTKdyXIJnaRv3HqF4L/vGN74R58Rc50MBI9+SrzYELr+tMuMpno4pI1gwZ2aOg/h3yJKxsLSN+7rrtoW3EdMpA96oIHYjbcat+ID71684/XhI9Mxxr7/++oBxG6FtrbPOOrGP2/vXfffdFz1vcd+POZQP/UBfRHgWcr19xMHcGZzqpN86sSq90noveSfwZcrVe8m4OYg4vQ7gQym2baV/UZe8M9q7Bpjy3sLHMCam+/08ze75X3QaHhW9pGQK34/xUmt9xj+TO/btmTh5N+gmbA8JadHmVKnHaruezvtK9bLvL74/eF1iBOVB1G83LHx6fl5UMpZUpeNJk4zlvLexfsIYwXgOKdveg8EcIg6/dWL1lHuvY34J2RfxY6TF6T/2Oeecc6L+457F6duEn3O0WY+pwrbf8w2wA1PDsM1aQq4NGka5X9/XwLVuncPepSwum0vZOfNLI6DbNcb4PfbYI57yPORZ9A5CvZJnyGAI5HbeZxCPK1vG8/GBbZfoid/0/zPOOCO2MdZbmGeyZoT4eYWvv37p75iI/gkBISAEhMDAERCBqyHEInA1BErBhIAQEAJCQAhMMAT8C7T/AsqKaS/v/iXZ7nlDfrrg4r8QJTyLxggv3ybpAuIg8sKiNAtbRhgjbfLCwoktdHMt3eKLBQcW8RBbqIsnmX8YrlgkMWnrLcOes4XVlMBVWobS58iPLYpZ3ozI4Ik4do9fFu9sMYpz782M86oFUe4hfuHFL0rWtYlu5AX/LGn4BT7Oc5IaMm1R0soGAQiDqJ2zWAUmpQuG5AHjyeKLL97JTpqmYduWwAXZjLh92+8kkhwQ97HHHtshWtrtfhG4yAtGMd//LY30lwVjjC8s8jYV/8V57hnD0BO4Um8FhCFt8mjesIjL2j7He++9d1h22WU5jIIuQY9Ye6CNsTUfQnwYHVgwpQ7QQ2YkiwHcP+IxbDyBiyAlutRFnT3sNU5PiCEBT371CdI/KTf92yTtQ1xPyQCmf3IL/YQvJXDxLEYm6xN4hzRPG9zzHgQ5zy3+e72SGzOrCFzEV4J7XXrddCBp5oT6oA690R7yoh8TMfJjwOMX2WGHHSIh0uKjffOMtVuuU7cQVSGDIV6v5xbUCZMjcJX0TeLyWHFuOn9W6PWqvDQZC2mjplPAGd1ppBfKVSd1uJcYoklzmG3Al8HPCciH6QeOEdPRpXOeuvleVV58Hed0QVXf7GU8eLbEo/8bWcTuMJbRdkjLtyXaFgRpL5SvSZvzhl2eZ9yiz4O7pcF1PFaaVzxvmOUeYuT+0vGhDvfS95Znczf6f1374AnvLZVzsMcYCbYm9GGIXmCGlOBZ2sbBGqOo19eWL379HKQNgYtnvQ7nnLIj1iY45/0GUpFJyVhY0sb9HJY2SdvMCeQX6gbBmLzTTjvF4zbjVnzA/etXnIzLzKPoS15SnO0e76CM79SpifdYwjXGbp737SHnWdaeT3/7rRPT+P15Sb2XvBP4MuXqvWTcHEScePChPq1/gZXvv9Sr1z3oG8bMRx55pEPeTddfPN52nL7vp553/LwKgg9zvybiSSVNwpsXKBv3bV5nz/rxgGv+folerpqj5caBQdSvlSv369Pz86KSsSQXP9fQCXhM8u+rtCf/Xsx7gd2n7UF0szG/Kl6rv9x7nZ8npOuJxDeRCFyUp3QtIdcGia9KSt+lLL5U91btLuB1BfqH9kE7or2asM7EGGvjkl83szC23rrSSisFCP6m44gTPWbv7oTnGl5Mjcjs+0a/9LflS79CQAgIASEwWARE4GqIrwhcDYFSMCEgBISAEBACEwyBPffcM/CijOQMUGYIYVJli9sGgSdw4WkGjzNevJcPf52XbrZQYzHQPN5wf1B5wUjNF/CLLbaYz0Y8Ji8s1qcLT/4LVzy21BlPbasMIvXeSUYl2OVCFYGLR0rK0MtzW265Zdhmm206Bv177703HHXUUZGshEcBjIOQU1g0tAUWKxrGVL7wti/puM4XwXidQMx4F0+e+8fCoLlJ94uSdW2iykBKtHzxh7c3W2QkzxCa6iRdsLLwDz/8cFz8ZisaiB4IC1F48GG7CgwGSLrIHS/O/JfzwMU98AMbiICp8EUiRqCVV155hOeuNFzVObjSD9daa61R9cQzVX3R4gM/+9KxFw9cxMdWgSy0492qSvDUB5GMhb624vusPUv58FSArqI9sABoWxcQBsx5zi9M27P84q0MjxLWlmlLfG2cloF0brvttuhdBDKTGdq8JzHS4MtR6tKIMjwHCZAvcM0TlN+a0/LSVpfac91+e4kT3bD11lvH6I3EWJUWuot6Nw9NPhx9kq9x7Ytcu4fxgOeqyLC9ELi8sQfvgZdccoklG/uIfRlN3WAks8VmC1Snk7oRuIijLe516XXTgZbn3C+L3YyLL3vZy0bdxpsCbdI8UloAPPah56z92nV+CcuX15C9Tbxer1r498Z/+oDh3bZvkmY3nT9svV6Vl7qxkHKAA1vPIt6TSrzQ4F8d7t4Qffrppwe+cPfijf3pvHBYbcCXwc8JyGfV/IR7JXMlP3bk5ntVeemlbxJn6XhAOVPBWEZ+aHc5waCGF4Pc9sWEb9rmtt9++4DhPp33EQfzebz3eJ3Kde9pi3Puo3uRkvGhDvde3ltippJ/de3DgnfrG5CFIXMzXnopwbOkjZMm70Bsd+WNoFzHWwbbKuJJEfEErqZlz21PTFyMBxChMaim0nYsLGnjfg6bjvdpfnjHZb6NVyOb47UdtwYVJ2Vn7MVzZq7vkS5jJ+9lzNX9OGx58tto2jX7rSqn3U9/B6ET0zTsvKTeeda3XYur2zuBL1MOj5JxcxBxUhbeOelz6VwM8hJ6hr5u76SE5/0aooNtLWmkZ+5ViZ/n596fPYHLv+tUxWfXGfdYO2gqRuCy9wJP0CKOqrmWxd9WL1fN0Xx7snnCoOrX8p7++vTSeVHJWJLGb+fMP3lXsg+S7Do6hvGb910+LLD2N23atLiWZuFyv1Z/ufc6T+DKrSd6Apefs1qcvk2UrsdUYTuI+UbpWkKuDeaw9tdK3qXsecZB6tkEEjhrYakwbkJKY50nJ7wfsi7on+XdlTU5POWZMAabEZ82uNdee8U5ot23X9oQ8wrWaEx8/fVLf1vc+hUCQkAICIHBImC6H5I3837edZhj8MexnTfNxRwz9wGv9w3aNLYxFE4ErjFUGcqKEBACQkAICIEJhABzDMgueK2BBMHCOIQKFgOHLWyfBXkCgzVfGrP4dc8993QMxr3kx7ZrYHGYBQn7wr6XOHPPlpah5DkWV9jykrr6w8xtdyCxHTzT25QRuCgnCyYsxBKOusVIw+9YEiPMsHhtbt3r8gdeLHrhOYp2woKib7OQHvmykXYEua0fwgIY29mw9R9EJgytJUSmXF4gayy33HJxK0j6JFvVUa7777+/4y0h99wgrs0777xxWx2wZdGWFzW20Ljxxht7Li/lpG4gC9EO8WaXM2T5cvGSSF2zFRB1Sn74mrxbW8ZgQLvCEEo9YQyh7yOWBxa+0S/mjcinyYIlz5IOMnny5A6BtooAOghdOog4fTn9MX0KjKkbMEansPXi7CjDxL0OX8gAjNEY/GiPVW3W4sHwgL7njzaO/uOLebw79FtK+iZ5qNL5s0KvV+WlG1bewInRA3zHkgyzDVSVOzc/8WFL5jz++WEel4wHVfljXGU7Nfozi56Mrcx98OxoY1Tu2TZtjnEbUjjzetoC8wlIn2yLXDXeMr/BcwwLtJAS/ccb5GeijA+M6yuuuGKchzAXYPxnbtNNP5bgaZihX9q+02CUZSxmMRx97z1j5dpG02voa8rOXBMDK2MDc8xuUjIWlrbxbvkYL/cYrxl7MW7TZyBUYBSHIEj/S/tVWi7m37QZ6oj3VLwlM0/u17tGml4/z0vq3ebjbd4J+pnnQcZleoOyoX/R8fQ7E8YAdC41DLmOAABAAElEQVTkXd5n6tqGPWe/ftvV1Eu5hUHf4b0xR5iwMMP67TbXKtHLw8p3P9OxNtFmbK5Kn/kD4/wyyywT9QPva957J7obPUIfY42k7kPHqnRm5+u9riU0xa70XYp+AyGeuQJzGNY6uwntztaxWPO1d0rWSKqEsYx1RUhevn0RnnRpf8xXmCeTB0jKeBCUCAEhIASEwMRBQASuhnXJ5EsiBISAEBACQkAICAEh0B4Btr8zz058DXbccce1j2ScPJESuMZJtpVNITBLEbDtIzGynXzyyaPysuuuu4aZHwrF60cffXQkhI0KpAtCQAhMeAQwWODRE3I0RvX9999/wpd5divgWBsP1OZmtxao8goBISAEuiPgPXF6zzj+KdtWGe+OEFUlQkAITBwE8Oq2xRZbxAKde+654brrrps4hVNJhIAQEAJCYMwgIAJXw6oQgashUAomBISAEBACQkAICIGZCGBcxejFF2aQt/jSFWGbPPOqEy9MsH8icE2wClVxhoIA2w7wZTQCwdO7/fdfuadbPA4lc0pECAiBWY4AHqXwYLDjjjuGTTfdNObHb2U2yzOoDPQNgbEyHqjN9a1KFZEQEAJCYNwjYGMCnoHY3hrBix5bi6fCdp5sg8o6yAEHHBA9faVhdC4EhMD4QgAdgNc+vD0edNBB0Vt4bgvV8VUq5VYICAEhIATGMgIicDWsHRG4GgKlYEJACAgBISAEhIAQmIkAW669613vilvSQORC2Kbm8MMPj8cT9Z8IXBO1ZlWuQSIwderUsPnmm3eSYCtPtr5hy0a2KEDY3urEE0+M2wN0AupACAiB2QKBI488MpbT5hOQOT/5yU+23vZotgBrnBdyrIwHanPjvCEp+0JACAiBPiLAO/58880XP1CzaI844ojwwAMP2Gnnd4011ggbb7xx+O53vzuhP1zrFFgHQmA2QOAtb3lL3DrT3kUo8o9//ONwySWXzAalVxGFgBAQAkJgViAgAldD1EXgagiUggkBISAEhIAQEAJCYCYCRuAyMJ566qm47dEjjzxilybkrwhcE7JaVaghILDHHnuEtddee4RhxJKFvHXSSSeF6dOn2yX9CgEhMBshAJnGDCbPPPNMOOWUU8Ktt946GyEwexV1LIwHanOzV5tTaYWAEBAC3RCwd3wLc80114Tzzz/fTvUrBITABEcAAhdrFSazw8epVlb9CgEhIASEwKxBQASuhriLwNUQKAUTAkJACAgBISAEhMBMBNgODY86c845Z7jvvvsi8QKPGRNdpkyZEhZaaKHw5JNPhl//+tcTvbgqnxDoKwK8c22yySZh0UUXDZMmTYq64ze/+U24++67oweuviamyISAEBg3CGyxxRZh/vnnDw8++GCcTzzxxBPjJu/KaBkCs3o8UJsrqzc9JQSEgBCYiAiwpTvv+Y8++mj0BjxjxoyJWEyVSQgIgQoEllpqqbDOOusEtk1kbeLOO++sCKnLQkAICAEhIAT6g4AIXA1xFIGrIVAKJgSEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhEBjBETgagiVCFwNgVIwISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBBojIAJXQ6hE4GoIlIIJASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEh0BgBEbgaQiUCV0OgFEwICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAiBxgg0IXBNnjy5cXxzrL/++v9rHHocBRSBaxxVlrIqBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIATGCQJtCFzPe97zakslAlctRAogBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBB4FoGmBC7IWy94wQtqYROBqxYiBRACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACDyLgAhcDVuCtlBsCJSCCQEhIASEgBAQAkJACAgBISAEhIAQEAJ9Q2CllVYK/CEXXXRReOaZZ/oW97Aj2n777cO8884b7rzzznDzzTcPO/nwqle9Kiy00ELh0UcfDVdfffXQ0x90gksvvXQsI+mcddZZ4X//+1+jJPlic+rUqWGxxRYLkyZNCmeffXa47777Gj07kQPtvPPOYZ555gl33HFH+NWvfjXLi0o9ve51rwtzzDFHmDZtWpgxY0ajPJU+1yjyikCzuq9XZEuXxykC6667bpgyZUp45JFHwuWXXz5OS6FsC4HBIcC4sN1224W55por3HPPPWH69OmDS6zHmNuOD/PNN1/YZJNN4tj3u9/9Ltx+++2NcjCR5s+NCvxcoFmtL2dX3OvqaI011girrrpqDHbmmWfWBdd9ISAEhIAQmM0REIGrYQMQgashUAomBISAEBACQkAICAEhIASEgBAQAkJACPQNgfe///1hhRVWiPF9/OMfD//+97/jMcSSJZZYIh7/6U9/Ck8//XTf0hxUREceeWQ0wP32t78Nxx577KCSqYz3sMMOiwSlp556Kuy3336V4cbrjV133TWsv/76Mfsf/ehHw3//+9/aosw999zhkEMOCfyanH766eGmm26y09n219orBK7jjz9+luOw/PLLhw984AMxH+eee2647rrrGuWp9LlGkVcEMuxmVV+vyFbry+NRz7Yu5Dh44NOf/nRYcMEFw5NPPhn233//cZBjZXF2R2Dy5MmBLW0gHT722GMDh+P5z39++MpXvhLTueuuu8Ixxxwz8DRLE2g7Przyla8Me+yxR0yuzZhSNX8uzfd4eW5W68vZFfe69uFx2XfffeuC674QEAJCQAjM5giIwNWwAYjA1RAoBRMCQkAICAEhIASEgBAQAkJACAgBISAE+oaAX/D3BK7Xvva1gT/k5JNPDr/+9a/7luagImprtOt3PkTgGo3o2muvHd7ylrfEG3js+vvf/x7OOOOM6CVtdOjZ64q1VxG42te7YdfG2N4+lcE/MR717OBRGX4Ks5qQMPwSK8XxjMBLX/rScOCBB8Yi4L3x1FNPHXhxROAaDXHV/Hl0yIl1ZVbry9kV97pW5HERgasOLd0XAkJACAgBEbgatgERuBoCpWBCQAgIASEgBISAEBACQkAICAEhIASEQN8QYMuNl7/85TE+ttywLRQ9seCkk04Kt956a9/SHFREs5rUseWWW4ZFFlkk/O1vfwuXXXbZoIo5y+It8cCFVwu8WyAYmcfCVoGzDMAkYWuvInAlwDQ4NewmEoFrvOjZBtUz7oLMakLCuANMGZ6lCMwKAhdbKDIHmHPOOSMBeyx70Ww7PpR64KqaP8/SxjGExGe1vpxdca+rWhG46hDSfSEgBISAEPAIiMDl0ehyLAJXF3B0SwgIASEgBISAEBACQkAICAEhIASEgBAYKgIicA0V7nGRWAmB6x3veEfA2IZ8/vOfj+S2cVHYIWTSjMwicLUH27ATgas9dnpiNAKzmpAwOke6IgSqEZgVBK7q3Iy9O23Hh1IC19gr+XByJH05HJzbpiICV1vEFF4ICAEhMHsjIAJXw/oXgashUAomBISAEBACQkAICAEhIASEgBAQAkJgjCCw3nrrhVVXXTU8+eST4eyzzw6TJ08Or3nNa8Lyyy8fHnvssXDDDTeEX/ziF+Hf//53wOCGh6ZlllkmTJo0Kfz5z3+O3oiuv/76bGnYrmannXaKcS6wwAIxDJ6dIHtceumlo56xvDzyyCPhggsuCCuvvHJYf/31w1JLLRU9Jvzxj38MeExIPSCtueaaYcUVVwxPP/10+P73vx9e/OIXh1122SUstthiYaGFForpPPjgg+Hhhx8OP/jBD2K+LXHub7fddmGJJZYI8847bywn4a699trwy1/+0oJ1frfZZpsY75/+9KdwxRVXhO233z6stdZaYZ555gkHHHBAJ1y3A9ZPXvWqV4UVVlghcEy5KNP06dPDV7/61YCXiCpSB0Y6yETkF3nggQeiZ7Gbb745sL1fTqZMmRLLSP2RTxa6qLsf//jH4Q9/+MOIR6hfws2YMSNcc801I+5xstFGG8X2Qn2CKW3jN7/5TaAOFl988XD//fd3PJ1Rb8T33//+N7at+eefP2y66aax/bzwhS+M9XHnnXeGH/7wh6PSsQu0xw033DC87GUvCzzzl7/8Jdx2220x3X/+858WbNQvHtle8YpXhCWXXDK2C/JIW95qq61im+KBj370ozFvox5+7gL5p55oWy95yUviVeqJdnbJJZfErRTt2VVWWSViQ5ujb9CfwPjiiy8OtJVU3va2t8U2TRsDg9133z32k7/+9a/h61//eho8e/685z0vbLHFFrGv0o7po7/73e9iH/n9738/6pkdd9wx1u29994bpk2bFvGhf1FvtMGf/exnsQ3yIP3/1a9+dVh66aWjRzvu0x7uvvvuEfGakZk+fd5554Vtt902luNFL3pRePTRRwP5OP/88ytxblsGS5x6RV8su+yygThIh/qde+65wwc+8IEY7Nxzzw3XXXedPRJ/S58rzWcvfX1Ext0J+pl6oz+RL/Q0bQzd949//COGRHdusMEG8Ri9cuONN7oY/v8QQiP9im1Bv/vd73ZuNEmjjZ5ti98g2iqFm2uuucLrXve62L6pG7w10udo1/TpKh3aASY5wJMPOg6dzPiB7qbvoxsZ49ABXnrVieBIv1xppZXi2EYfY2tg9MjHPvaxsOCCC8b0999/f59s9njRRReN/ZWbF110UcSG9oAuQa9zzaRt/fFcKdabbbZZ9Ki58MILxzieeOKJOK7Qn1OdT39Hd4I7HjZzHpXQ+fT7hx56KNaxlampDmasRa/xS1/517/+FUm8F154YdQ7Fp//nW+++cLmm28edSH9lDKg53/+85/H9ubDtjkuyUubsYl6fvvb3x6zxLzunnvuCVtvvXVgDkHbor0xZ8A7p+kan3/K+sY3vjHOkRgDqC9wp15oUya0M8ZUxlaEuEjrlltuCcxlmE/wx1yFeueYfNBnTzvttBHbFzP/2njjjWN8zDmfeuqpqBOvvPLKcPvtt1uS8ZfyMTdESIs6QUrnnvHh5/6VzM36OT6UErjA1s+frUymg6nvn/zkJ/G9gHkVeoM5EHM95hHMQb28/vWvj2NT2t8szOqrrx49mv7nP/8J5jGXvkgfRqi3++67z4LHX+ZVfJDBM7QV5hRNpBd9Obnh3JN8MYdC0Am846Tyghe8IOy2227xMvOvq666KrbpHO72LPXC/JOyo0vpS8zD6A+M+Tlpmmf/bJv3A/+cHdOG6fMIffjqq6+2W/HX48N8mPcrL2xPTr9ljo5+TAlcvJ/Stpdbbrk4PjPXQZcw7lVJm74Izvw10TWD0utV5dB1ISAEhIAQqEdABK56jGIIBmyJEBACQkAICAEhIASEgBAQAkJACAgBITB+EPCL5RAg1l133VGZxyDN1lh77bVXJJ2kATBGYLjwgqEaIgWGi5xAGDjuuOOiQcLuW14g/ECgwiiXEwzjP/rRjzq37DkufPzjH48G6E984hOd+/4AIz3EJQRiEIYHMxz5cBxjRDr22GNHGI0/97nPRUM9hkyMshBcTPbdd187rPwFl3322ScaZNJAGP0xjpGflMCFoZpyYqDJCSSEww8/fEReCYdhDONETiArQEK7/PLLO7cPO+ywSEDCCLrffvt1rmO4/vCHP9whxHVuzDygfWDUxnj7+OOPh4MOOijehgSy8847x2PSwJCOQS0VyDc50hJEi6o2QHo8g0ErlXe+850BI2EqlBdCBQYxpI7AhQF+6tSpaTTx/PTTT++QBSAQrLPOOtlwpIkh3JNjCPi1r30thscIBRkDYh2CcbIJERCSBe0IEk1OIJqx3aMXq1sWOjHAYpBPBQIYhjSMt6lQlhNOOCESMO2eEbgwWlL/EFpSoS0dffTRo+qqpAzEDYGFdpXrt7QlSF1ISuAqfa40n6V9PWY+8w9saRtGhk2DYNz+1re+FUkJYLD33nvHIOhavI2kAqkF0g9i/bZNGhitm+jZEvwG0VbRrfR5jMU5AQPac5VxPH2GsY3xAN1cJRAfPBGqF51IeuCdq3/6MwRO9AH6uAmBC4KrkXXQzxBkrE+hA+jrSEn9lWDNGPOhD32oo59TTJkXnHPOOZGoafcgAdl4A1nnxBNPtFud38985jMRM8YM316b6GAIxzvssEMHl06kzx1AOID47gVcISXkdCE6FKJrSiz1z1cdl+Sl7dhE3/jKV74Ss8AYAsGEekmFceoLX/hCbGt2r24+ddddd8U5H6TJI444IosP5JajjjoqtktwBC8I3rRNk7POOqtDBmO+Z2R2u+9/mR8wLzLx5SM/xxxzTLxlc8i2c08eLp2b9Xt8KCVwWdkpC3iiRxDTwRCxIA9Bok+FuvzmN785gsj45S9/OWJSNe5AZLL3DPQnH2wwd7a5LfqLcY66N7F5N+dV/dzC2m8v+rLN3HPPPfeMczjS5R0KUloq3hswRFPep6pwRwczh7Vt4dO4wOWUU04ZRWBqk2eLs+37gT3nf2n/X/rSl6KOpJ186lOf8rfjRyk27+cjAE++4/2J9woEkvkZZ5wxAhfqGgJXTiCsp2Sxkr7IGNhE1wxKr+fKpmtCQAgIASHQHAERuBpiJQJXQ6AUTAgIASEgBISAEBACQkAICAEhIASEwBhBwBsRLEsYVDDi4AHDDLp2D4MNZA2+RPaGawgCGGwQPC+w1Zwt2nOdr/QxqGBss+cwOkMSIk4klxeexaAO2YTnTfxWdv45DFAYsTGq8EteEIzzGBcgcOGRBoIPRhITCFB4ncJYvcgii3TybgZFC+cNSXaNX8rwkY98xF8adYzh/cADD+xgirGQdPHYhPcWLymBy4zQhOE58oqB2LyUcJ3yEQ5DNYKRzL765xkIYhjjeAbyhgnkGog7iBntUgJXWm7Ia5SZduDbiBFBiMuTFThHaFd40WINiWdNPLGOa97wjMGKfJMn6ga8EK5/4xvfGGE8fNe73jXC8EU+MTZT5tSgXkfgwnCE8R5ikhGsyDsCkQCyEIb6tddeO14jP9QL/YN2Tn0bNngewLOEiZEH7Nx+6Xtg3U3S/kW9U7dgivc0SzNtu1a3FrfhincHnkuFdkR9Ug7Djj77yU9+shPUCFx2wdoZeYDEYc+lpJLSMuBV6k1vepMl1/G2ApHD6wcCeAJX6XOl+eylr3cKlxy8+93vDquttlq8Sj3gAQVcIQKQHgL+6FTq7otf/GIHE68vY8CZ/3zbhRALMbZNGmBTp2dL8RtEWz3kkEM6hEf0AvjRPiG7Wf/GOwjElCbiySPoGPQB8UBs88RK6sE8cfWiE6lDW3un76Ir+PV9nnynfa2qLBikMV6nQpx4J4IAWlp/JVgffPDBUdeSH8YJ5gy0c8YqX24IIug7pB8ErhiR+2c6GOIzBAcT5iIQgkiTccgE0ph5esITG1vumjB3IT7mL36MhzDnidMWvuq3JC++f1OnTcYmT3CyvPAs7ZdxgnLkxhfqh3GLe4RnDKT+6Ad4zrG5IMQ19DLkUvAwEjH1DFbMzZgL0C5pnwjxWZocQ57G4xzeT/FAiaD38KKH3mPspY7sGQhgeF5FfPlyBK4Y6Ll/TeaeBC2Zmw1ifBgUgctjwnwVjGn/NrZz7omRJQQuxm50Bm0MwcMSRD0EL76bbLJJPGYOyLsG9V0npfqy7dzTk6WrCPjM+22OZeNB+t5ixDm/ZTftHf3BuIQetHGF8tPuSA9pm2eeKXk/4LmceN0N+c7yBenPvxcxf2ZsN+EDGryMIcyJ8fbmcbFwzEP5aIb3M8YkE9L1hOuSvthE1wxKr1s59CsEhIAQEALlCIjA1RA7e5lqGFzBhIAQEAJCQAgIASEgBISAEBACQkAICIFZjIBfLMdYgDES7xsIX4BDiDFhAZ0vrSGMIN5AyFfnfH2OvPe97w1s24OwRQ2etkww+ngvMvbVNffTvEAqMMMb9yEmmOEUL1oY+hD/nPcg4L9654t3vnw3wUhvhgC8h+FFzASjImmZgerb3/52Z2s5T2QCL57DEIsRq078l/oYN/E0gdccBBLQHnvs0TE6egIXnoPe8IY3xHAYc/BeYQYSDJLk1Qgc3jCM4cS8JniSFhF54703ZBpxwhO4MLDYFinUPUYTawPEj9cU8oF0I3BRJjwEmaEKL0oQGhC2RTHPHxi6MNJhgAUfcAIvk7e+9a1x20rOIWGw7SQCIYy8mRH55JNP7tQ5pEEIW9Z+CF9H4CIM4g1q/hmfHgY18kl+TPDKhWENgeyGEc/qzRO4eBbPA3g8oU3ViW/vePCinCaQpvCqZIZQ6tNIZ1a3hMVgjqEVQzvi+zLnGNJp9ybgCpENIX5rt57ARVzUhZWR8LRNI8d4z3mlZTj00EM7HmGmzdy+ia1WTdgaCy8wJp7AVfpcaT5L+7rlPfdL/6Cf0V7wcmH9iLDeCImHHzwYek8nRpzw8Vp7oM1RT/TptmkQXzc9W4qf5Y34+9FW0enmhSwlaUEuwYuTEUohKJJmN4F8YkRLSA2MJ77vQvxkK0DE6+SUwNVUJ3odDEGLMcD6Ln0eLybWz0oJXJDa8LrlvRqW1F8J1mxh9+Y3vznihSEGPG2M4aLXT0aw4nq/CFw5Hex1hp9vkK4nD0EcwgsR4p/x+od7lJGtAxmfUtIL97uJj7dJXkrHJk9wIj+M53i6tLbG1qzve9/7YlYZA8yDn9cBbMXrPU5COoFMQ7l922ScZ0xEUo+RXp9xH4I5hEJIVSZGFKLuPJmF+5444ucIvnx+3uPbOf246dyzdG42iPFhkAQu6pp5hhEVqVN0ps37jJQE9lYv1JXpXK6b+HEJHUp/RvyclDpAx5Eu+tjmdMyxIPnUSam+LJ17+vEKIhpjggnvGUYKRsfSVhHf5uy9BX3GfSsv83a/bTX9zT6+sPeW0jyXvB9YmdJfP/ey+Qdh2E6TbddN6KvMoU1oH4wXXpd4XGgH9HsIxSY+356cWdoXm+gar3/7qdetTPoVAkJACAiBcgRE4GqInQhcDYFSMCEgBISAEBACQkAICAEhIASEgBAQAmMEAb9YfvPNN4fvfOc7I3LmCRoYKYwMQiD/5bl9MY9BBwMOBggIBhgmUuFreww+CIZy8+jj85ISSAjrveh4g59/zgwhhPdGRU/ggiyFQRhJt9iJF2f+W3/99aOxlXO/xZ8ncKUEEns298s2RBhxwAUjBh4L+PXitzPxBC6wAjOMGRBpMKp6IW4MDIg3EH32s5/tkBK8gc2eZYtLDHEYiL73ve/Fy2aI8gQu82RA+hB0PJmKh3y9VBG4WFwzY62lD6mKtoL458jX8ssvH6/7rQrjhef+mUGFPEFmoR1571u5uvF1QDSejOXjTo+rCFw+PW9I8s9j7MbojbCVGluqIZ7AhYGcNtZEMMxTr0gOU6574yWGr1NmbreDWN1y7PsD556sCbmAPgm2Jt5TAoRMiJmI6Yeqtuk9F+BFjXorLYPvt3jggUyaim/zZmgrfa40n76dte3raXn8uWFN/UCC9QQuyHJGrKCNQYz1BAmvF4jTb3+Idx3Tx23TIK4qPVuKH3H2u6369u0JHaSFrLXWWmHLLbeMx5BMjbASL2T+TZkyJXof4xYGa7bS84KR3cgtntDiCVy5/lulE00HkwZ1xPa+Xnw/8yQZHyY99oQJDOiQ+HybKq2/Eqx9+XJjFXln/DTStXl56ReBK9XBnnybEv7IC6Q/yI6QvG0Ow3a/bGOG5OYvXPfEHfSyJyZwPycleSkdmzzBCZ0OmcQMY5Y3Xw+2bbQnVXvCoj0DLiussEIk9xrZzesnP5/jGU+qqNL16H/wv+222zpjnKXHr23TCJkGUg3iy1dF4MrVnZ/j+LyWzM0GNT4MksCVm4f5uQ3kLsjkSCmBi2d9/2DMgvhuxPEcCZlncuL1SRt9WTr39AQmr+/JG9twsx034ueJufcW7wEzV16/5aARR0vz7OdKOZ2bez+Ihcj8w+udvev5j3YYU/hoAl3Cuw9Cv+QdgnPeJ/iFlGdzYo+Ln79asn5+22tfJM46XTMovW7l0a8QEAJCQAj0hoDNUyGOMy9kXGGezh/H/E2ePDmep966cynPMXMR8P9XQHIhxuk1EbjGacUp20JACAgBISAEhIAQEAJCQAgIASEw2yLgF8uPOeaYgFHLixlj0i+nCcN2FhhSkTvuuCMcf/zxI0hdfnE9BnL/ML7hMYSFfbyHID4vtsjvHolepuzrdW8k8M81IXB5Y0tKZLH0/FfznlxkBC7yzZfg/DYRb9TOEeWIw3t1MQIXi0/mYcoIMLn0bMsqj6c3sGEIo46uvvrqUXXs4zPihCdwkT756EYMMGOpx8qTFUj3+9//vk8qHlv78nGb8Y08g3FO3vOe9wS2OETMQ5oZpMAA8kRKkCMsBmcW8ZBeCVy0ffoA6UHI8+SHmMDMf5vM3PqHLYAQX+9mrPJ4xUA1/7yRFi9LkEdSoV8ZuYntZSD9IVa3vo3EGzP/saYH7kjOYL7ddtt1CC5sH3n99dfHsEb4YXsuI+PFG+6feXWyNlVaBk8SsDp3ycRDSDjkFTECV+lzpfks7esx013+eeIEpJGbbrop/PSnP+1sz5d71PoE9zjGiyLiPRp5jxklaVQRuErxI3/9bqueNEz84IChHeKVLfxzvRdBRy622GLR+8+2227b2fbPG/RLdaL1s276IqdLu5XHE7hyY3Vp/ZVg3aR8H/zgBwPEOcT6fz8IXDlM/RwhR6QgDxDYISfwPF5LPWkK/Q4pIRW/bRnbAJ522mmRLL7iiiumQeN4iw4ryUvp2OQJTp7Y6TPnPeBwzDjtCU6EhaBxzTXXRIJablwkTFMClycH8Vw3IU62lGXctXG+LYGr6dyzdG42qPHB91ebP3bDyu5VzZ9NB+fm/jzr9b6NtVw3PdTWAxfP4jmUMci8CXIN8fOYZ690/99En1g++zH39GTXtNw2BjPvso8NyH0Od+u33E9JpVxD2E4VHcvHJ/Sz0vlyyfvBsznI/8/hae8FEMohoiLohfPPPz9+2GCkcz8H8bjwTsl7ixc8ToIjYu+BpX2RODyBK6drSvU6cUuEgBAQAkJg8AjYe5wIXDVYi8BVA5BuCwEhIASEgBAQAkJACAgBISAEhIAQGGMI+MVyjAe2/Zll04hW3shh93IErk033TTsuOOOMQiEHYg7OfFbgdjX3z4vLND7LZSIwxuGvYHKP9eEwOUNwbl0LL+HH354NCj57T2MwNV2C6RtttkmbL311jFq74nJ0rJfMzxZ+fxWQITBCJQTvi40sS3AqB+IRRhlvRAHntQgFGFM8Tib0c7INh7zHLHH4jUCmTeGe7ICW93hFSsV82Dh25cZfQjbpLzWzowoZB5R0rQ4Z0tDMyT1SuCyvgF+ZlBK0/R9xG8TaQQujPx2nD6bO0+3pKnDx+fN6jaHjycsmlHMp19H4Mp5LbHnvQETsmZpGagvjPNITldxna1b2cIVMaNy6XOl+Szt6zHTXf7Rbtl+yvd1gqOf8CoFGcm2sbVo/FZvZjjlnnmwg3yBLuYXKUnDG/I9IbYUP/IxiLbKFrWQHFJB17FFG/j47arScLlzvIHgXQUDvm25m4arInA11Yme6OK9QabpQG5mK12vS9Mw/twTuPz2phaml/prg7XXPdQD24blxOsg20awHwSunA72cwS28ILgVifmbYZwVXqZe9Z/razMWfBgk4oRZ0ryUjo2eQKXefhJ8+U9/tgYStvff//9IykrDc+cAC9ZeAb0nu18u04JhJ5UQbvGG1NOINGxFTPjLN7rctKWwJWbE/p5UK9zM7ZWLZkL5srmrw2KwJWbL5Cu90xkYy3XjciTEpm4h1Rtofjs3RA9tTGnN6EvMb/0bcfu5X59u2qrL0vnnuQDD7OkjdjWkJ7YBdmK+E1y7y3Wbwlj5EgLX/VbmueS94OqPHDd6yneQRZeeOHOdol8BIJ3NbzP8e7B+G4kcuqX8LxTIR6X3DzPf2zQa1+kbdfpmlK9Hgujf0JACAgBITBwBETgagixCFwNgVIwISAEhIAQEAJCQAgIASEgBISAEBACYwQBv1hu2+H4rJlBAWIXi+lePDnFPHB5bxHmJcM/Y8c+3RyBK5eXnBGN+HxcTQhcn/70p8OCCy4Yjazm/cvy5X+NRGCGVO4ZgavKqOWf98dmrODaWWedFdhyMidmjDHDhPfYkQufu+aNHhi43/SmN8UtCXMEA8p21FFHRQIIcVmZjcDlCWS33357OPHEE3NJdra3qiJwnXnmmaPIJUSUErgwbkNiayMXXnhhuOqqqzrb+eXaqsXnSQlmfLZ7Vb9VWyga2c6XOY3DG8RzBK5uBsY0Ls6957Hc/fSab6dWtznDqidRWF/2cXnyRM4DF9sn4b0gJ2bYxFBHfystA57EbDulnH4gbQhe1CtiRuXS50rzWdrXY6Zr/rG1KG0YLxRGBPGPULd45KDeEb9dl22jiJcoDKZIrv21TaOKwFWKH/kaRFsl3le/+tVhq622it4OOU/Fb6uW3vPnkEXQsxjoU4FQRz1YW60icDXViXgaBEsET08nnHBCmmQ8N0NzCYErRyrupf7IUFOs8aqF8R/xXgrjBffPE4KvvPLKcPHFF4cmBC4bs1PStRFnc32AumV+gzBfoD7rxHuvqwvLfSNI1RG4SvJSOjb58apqvM8RuCgP3pPYKhEiUepBifsIdUbdIZ5o043A5ecz8cHn/lW1T8YZPOxBZkTaErhyY0tu7lk6N0N/s60v0mYuGB/o8m9QBK7cfIFslBK4/HzKiE6+WLQL+pyNb+hT2oCNaT5s7rhUX/Yy9yQfEPN22GGHmCXzjurnm6l3p9x7i/XbVFflysm1XvPc9v2gKh9c99vO066ZZ+AJz96fzOOXnRuxP/Wu5nHJ9cUcgau0L/Ku4AlcOV1Tqte7YaV7QkAICAEh0D8EROBqiKUIXA2BUjAhIASEgBAQAkJACAgBISAEhIAQEAJjBIG6xfK2BK4tttgi4PUFMc9IuaLi5QASAgKpA6NbXV5yRjSe9881IXDts88+YZllluHR6DnJe6CKF5/7Z2QqIz5w2YzBnhjjn6k6ZkstiANIztsJ170xxghcSy+9dGeLSQhAGEa6CcYutj7KCcQMDIcrrLBCx7hJOG/wN+KEEbg85mZwzsVthidPZvIG96ZkBeK2LRsxYrGNTJ2wfR8ehMwDVzfjl98OpVcCl3ma8F6u0rx6QtGtt94a8FCEdCMPpHH4c7yNgCuCx6DU45IPyzHGVwxkiNVtziDbK4Grm3c28/Zkbaq0DN5r30EHHRS3LosFc/+8EdsIXKXPleaztK+7YtQeQpZYffXV43ZKeKGh/kxSEpIRe7iPN5upU6d2vNBByISokZOmaVQRuErxIy+DaKu+jOBFW2E7M3Ss91Lot3Pyz/hjTx5hizgIjP/X3p2FWlX+fxxfafRLiixoABu0jCwIpCDSopkQioaL5oQGo+EiL7roIiL+ENVFV0aZhdEMJiVCYXNYRAkSWIZ1UYoUDVZeBZEN/3/v9e97+LrcwzrP3tvfOfp+QPe09hpea61nrXOez3kehrTcsmVLHTTIIblBA1yEjQk0UDr1FhXrFUNp5fo8Puv0mHvg6hTgGmT/5eX1s8Yuhm+NXqny9+M54SBCI5TnnnuuDnu1CXDFPQz7ifuDKL3qYIIDMQRfBMzje90e83nGvLkW9yrbt2+vr71sQz5/4ztxLS9Zl9Jr0yABrlhvHhlakt786BGRcyHCOHxGoJfGtkECXDmwwb0j90ucf/QeGWG7uH8bVYCr9N5sVNeHfO2L+0e8+5Vu98+96mDmWRrgyj01cd43e9bi2s45kUuve888Hc8HqS9L7z1ZLtdL7kE51uNnhqiTm3UP03dyj/OWc5/7ljZlkHXO82/z80Gevvk8bz/3FAQoOffj3jCfs9zX33nnnbVVc4ja7NI2wFV6LrIN/QJcpfV608fXCiiggAKjETDA1dLVAFdLKCdTQAEFFFBAAQUUUEABBRRQYIII9PtleTR+durVqFMPXDQC0EMDhWFzli9fvsuW0sDB8IT0CJUbNvqtSw4T5Qaq/L02Aa5rrrmm/mtxVuzFF1+s1q9fv8s65gaBb7/9tl5fJioNcBEUIDxE6dZIfeKJJ9bDjDBNbB9GNERSaPBl+Z0Kw3jRaEpIhuAcjSn0hkahVxMaNnOhdxjCHFOmTKnfZqhFgk/RaBdhGz6MBqIczsrzIogXQwjmaUoDXDEcIw2zhKxieLe8TDz5RyF8QKPt/6QemroNu8T70TPHoAGu3LMUf7lPcKJZcgjirbfeqtasWVNP0is80JxHfp0bwQhvEYxrFs6vK6+8sj4GGFqP3iAosW9HEeDKx0teH4IJ7E/W6aeffqp7aivdhtyz1apVq+oAW14Wz/PwTBHgKv1e6XqWnuvNbcmvjzrqqOrss8+u3+IYilBeTEMwk+2kNAM8+Tykp7rTTz+97pkr1718r3QZ3QJcpX6sy7CPVXzmzJlT13EEtJol9wrTaQjR5vRxXaSOok6mx59c8lDCgwa4mG8EZLudZwRvaPznPGvu/7xe+Xm/AFfp/iuxju3rVDfFOueGdOoUgjlcx6j3KZ1CHvQExb6iNI/3XnXwtddeW58nfK9bED2uUxHmzqG+J554oqInw2YhXBJBbu47uBfoV0rWpfTaVBrgYthYQjeEzJvDZnMPw3U3eqtbuXJlxTkxSIArh4Cins+OuZeeUQW4Su/NRnF9YNsnWoArzou8X3ieezRqBrhySJNgPOcvdRql2x8B1B82/ov6ZLz1ZZzT4733jMXnsCUhpcWLF9cfNXuY481OP7fEecvnne5hOYeo7zCJMG/JOmNb8vMB69WrxPpTj0+bNq3+GY97DnrJzXUL9UQMG8t1iz/CiJJd2ga4Ss9FlnnjjTdWXAspnXrgGlW9Xi/Q/xRQQAEFBhYwwNWS0ABXSygnU0ABBRRQQAEFFFBAAQUUUGCCCPT7ZXk0VLcNcNHgQo8VNDAQvGGoLhpOcyGIwNAilNzo2m9dBglw5eFLTjnllOqGG26ol89f/0fvH/Ub//6XhwmKIfr4qDTAlYMsNA7RGN3s+Sv3ShYBLpYZ+4Dn/IU/jR+55F6eorGSYBbBL/bDtm3bKnpBapbcCBoNFxGcyA1f0YsA33/00UfrfZbnFY02vDeMAFder7fffrvusSwvj8YajjEahDjG6KmAx9zQ0qnBLIfymN+gAa4c+li3bl21YsWKvJp1OI7GNYaxo+TGyl7hgZ1m0niRe5zhvCJ4x/GUSx7uMK9X7NtOIQmOTxpWKSVDKPI9tp/l5ZLDBxFgK92GHCjJx1ksj+OCxkAeKdGwX/q90vUc5FyPbWk+zp07t+J4o8TwTM1pop5o9giHB4FZ6gIa1WN4s+awl6XLyAGuXM+W+rFdwz5Wqe+p9ymcezR+58I5GnVkt4Btnj569+nU0I/zPffcMzYE3zACXNFIzzrQix+9+eWSgw/DCnCV7r8S67iusk1Lly6t6EUulzysWjMYEmENGnC4FuVy66231r1A8d54Alx5GLbmEF/MixAkdRuFkCwhZ4YLu/zyy+v3cui7fuPf/3JvgI899lirAFfJupRem3LIYjxDKMZwaFyHua42r0k50BiBuEECXNxX0rsPpdP5TID5zDPPrD+PeyJe5O3LPRWW3ntGncu8296bjeL6wPInSoCrV0j+jDPOGAsOsc75nogAMccO9SeFOpYe3AgHUhh6j3nzs0i/Ulpflt57xvrkcB7XYX5moXBvRYA9l3zMxR+e5LD5J598Uj3//PP5K1Wuz954442KfyXrzExLfj7YaWU6vMh/sBAf54BWruf5vFNvudmlbYCLeZWci3zvxj4BrlHV6yzbooACCigwuIABrpaGBrhaQjmZAgoooIACCiiggAIKKKCAAhNEoN8vy+OX4m0DXGxWbsDllyo0bDGkCIVhv2hcjEYaGlpoYKP0W5fxBrhyzzM01tHwHaGpaHBkuTTYL1u2rG4gIvhEI2z0dtNsLI4GiOb7zKdfufnmm+vtZzoa2WlAIUDGMjGJHqX4PAe4cqMXyyVE9c033zBZPRziLbfcUu2333716+jdghc5ePXqq69W7777bj0N/7EslsmycyNKBCdygCv37kMjGj3YEJA66aSTKnr+ouEtSg7WZP9uvScQxGK/5tABPWTREM8xQkMwQRxCEBR6XGHoFRp/KTlsxO+l2D9xbL333nvVa6+9Voe7Zs2aVTd00TNZlEEDXLkhlnm+/vrr1ZtvvlnPnm2iYS1sNm/evNNwkKUBLmZ+22231fY8J5xHeCCOa4YwJcAVdnhE70Cxb0cV4GJf0TMWPYPRkE+wh+ODwpBAhM14n1K6DXH+MQ9COI8//ngdSmJfEI4gdBIlAly8Lv1e6XqWnuux7s1Hzm/qYvYrlvTwE733cEzTq0/07INLHF8xnzxsbLzXHBqudBn5PG/Ws6V+wz5Wc7iBQA6hH+oqCscOjbj0HknpNsRt/eG//+XrB6FFjjXqRobmvf766+thvGJ6ekBkyD9KthpPnZjrYM4hgnIbN26s50kD82WXXTZW7+W6tJ6gy3/9euDiayX7r8Q6rwuOHN8R4qKup+6I4GFzuMd8bjMcF59TD5x//vkV4c0o4wlw8Z0cTOZce+qpp+p69phjjqnr9rjmvvTSS9XHH39cLyb3LsSQfvTwSb1IiJLr7cknn1xP1+l+qv6gy3/jXZfSa1MOOI0nwJWHJuaazNCsUdfTMyjX7AMPPLDeOnoQ4hzMoUk8OOc5dvke52OvXnHyPeaPP/5YX1uZB8cK5wL3mVHyPUnevmEEuErvzYZ9fWBb83nHNT7On3BoPhIy5B4p33dHkIhpe9XBfN5tCEWOc8LAFP7Y4Mknn6zrRkKPDK8e92d8HgEu3uNnAY5bSg4v5WOf+fHzRL9SWl+W3nvm9SGwFHUD7+fjL0/XyZ37RoLE3JdTCGgRfOec4FgjmBj3doTZ+PmqdJ1Lfj7I69/pee6Nl8+py7nPjpLPW97rFJbOLuMJcJWei/3qGtZzVPU687YooIACCgwmYICrpZ8BrpZQTqaAAgoooIACCiiggAIKKKDABBHo98vykgAXDa00DuRGDH6RT8NDNEyw+c0h4Pqty3gDXDQGRa9CwR1Dr9FYz/JyYxLryPrFezS80gBLQ2yUaCwuCXCx/ny/6RLLZHmx7BzgYtm5xwles66U6G2I582QUO5Rh8/5Do3Y7J9YDu9Hjxg8j0a7HODi/Tw0DK+7ldxYVRpWYN65RxleY0NwhQbYKISWaNxjm6LQeEtvH1H4Ho1f4ZSNBw1wsYxLL720Dgp0Wx7vs36EZQjrRYmATXOfxee9HjmOOL+yBcsgyJP3azO0F/t2VAGuXutM6C+GcmS60m1gmFFCJXk7OS4imJef5wBX6fdK13OQc72bY+6dg2kIXnI8U5+EB69pQP7+++93mk3uGYQPCNTSANwsJcvoVc+W+o3iWCU4QgCUghN1OPVCPo8IkFCnRP3a9InXBBMvueSSeFnPjxexH5rnY/TiNEidSIhpxowZXZcZddswA1yl+6/EOvdOxUZSb7NNUXfzXqceM3NvS0zTLOHCPiGgEqVfHUzolyHQ8j0L65Rfb926tQ4Dxjyb9QzLZrn5ms97hLcJ0LQtJetScm3KAafxBLgIZ3FvEzZsI/UT+y7qZra1GdiI8zwc4vN+oQqGX+N4ifON73POxrHC8vP9AvuAZcUfFDD9MAJczKfk3mwU14cc4GK9+pU4l/J99zACXPSaRU9R3Qr7JvZbBLgIvZ522mn1V6iX6aGWc41CKO/ee+8d+87q1aurtWvX1p/1+q+0viy994x1aYaUcrA/puGxm/uFF15Yh/BjWrwoYcbz6H2L55SSdS75+eD/l9b7/xxgawbucm+CzKVTiDm7jCfAxfxKzsV+dQ3zHVW9zrwtCiiggAKDCRjgaulngKsllJMpoIACCiiggAIKKKCAAgooMEEEbr/99vqX06xOp1+WR4CLX47QsJ3LYYcdVjH0HoXeQOihIgq9K9DrQgxzE+/zSIMEjTDvv/9+frvqty40ehGEoeSAU/5eboBiutyDCK/pkemdd97hacXQgzQWTJs2rX6d/6PxkaGc6M0ml+h5pRlwytP0ek5D51133VX/1XyeDpNnnnmmHl6GQETevpguD0cX7/HIdxlS69lnnx1r9IrP6fGAsEFu/InPaCCLXqriPf76n33XafsWLFhQzysao1ku0xHOueqqq+pQTh6uiCGMaFynsG702tUsnXrgimnmz59ffz8aheN9HmkYopcPghHNQoCLxuvmNhNqIUQU6zSMABfL7rWeeNDbD72D5BLDfkWDdf6szXP2Ecdu9PCVv8Oxy9CT/Msl9m2n8E4O4TTPZeaRh2V84YUXxkKNsR2EMel9iDohF44xjuvmkG9MU7INfO+4446r7rjjjp2CN7xPwy+9kdH4S8kBLl6Xfq90PQc511nfZiGcQF0XPUU1P2f7CZx2smZaei6JsFK3RuXSZfSqZ0v8RnGsEgQgkDN9+vQmXf2ac5Wen5pDXXWc+J83m2G3mI7vEw667rrrKhqsKYRLqG8GqROpz+i5J/fUGMukXmP76BWxbYCLXoqYHyWCzTG//Fiy/0qtMcu9ZuX1aA75mT9rhib4jOsTPWNRJ3HONANcUXf1qoPpPYpjhvoxF+ZNPckQZ8w3F+4rqJ9waxbqXkILX375ZfOjvq9L1mW81ybCVgy3SunmnYeXztdQjkv2Qw5s5Y1im5cvX14Hq+J9wioMkRfX+C1btlRLlizZqRdXhpaOHibjezxyLjFkW3w3PuOehHsD7j3jWs9nbNcPP/wwtn05wJXvITvdB3e794xlltybDfv6kIcGj/Xq9RgBrrzt+f65Vx3MfLv1wMVn8+bNq66++upd7sG4nyYYyP0EJYLo7OMo1MH04pbLRRddNNbDJPcUBHWa512enueD1Je9zpte954sN/dCRT1BkKzTunZzZx7sy4ULF44FEnmPwnWE+zoCXM1Sss7j/fmgucxOr3MAi5+1+JkrCvuEezQesSGE2QxLZ5dO5yLtzxw3lGH8nJSvHd3qGpY1qnqdeVsUUEABBcoFDHC1tDPA1RLKyRRQQAEFFFBAAQUUUEABBRTYSwRozKBRmWGHaDCi0ezrr7/e5Zf2o+SgMZkeG/gFDz1mxF/2s0wa/2b909MGf2FNQy+9YtAoMJ7eMUrWfebMmfUwMzRkfPXVV3WDbrMho9N8+d0LoQDCKPQwwfBBNFIT2ulWaHw+9dRT6+/Q+xZD6n333XfVhg0bdgkWdZtHfp/GzMMPP7w2CksaZbBs9kiSv1fynLDYnDlz6v1D+IRjaP369dX27dt7zo7GcwIKNLowLdvKd0dVcD3hhBPqfwQCaYimEbLfeg66Ppxf9HhBkIuetWggZci2/1bh2KAhkeP0888/r3uF67cuJdtA0Ijtnj17dl2XEFri2OtXSr/HfEvWk++Vnut8t1M59NBD695KCChwzjGMJvUVAUkaRbuV6D2wV6NyfLdkGb3qWeZb6hfrNKxH6k4ayBlmjzqUUAfXJII84y1sE8EVvLi2EerJdTHXFYIa1D3UCcMoOMf6M5Quxz6hrVGXkv1XYn3QQQfV9wyc2wQfuD5yTe63jVwf8KYu5LrId5rB2VIjzBn+kHoNc9ap17y5rhNopY5iiDOuA5s2bWpVH/Zbx/Guy+68NlG/cj6w3Rz3XJM4vwjtsE86Fa4ZXDs5b7h+dQprdfoe78W9DfucexrqwBium8+pI7n3I9zKOsT9Cp8Nu5Tcm7EOw74+DHu7SufHOTDrn3tr9i3HAQG+Ud6DdVvP0vqy9N6z23qUvM/PTZxL/GwQ9Q7XrG6lZJ3jHKKuHsbPB93WbXe+X3ou9lvHUdbr/Zbt5woooIACnQUMcHV22eVdLo4WBRRQQAEFFFBAAQUUUEABBRRQQIE9R+Cmm26qG0FpTKcXi2bQjN5N6JWDsm7dumrFihV7zsa7JQrsAQIMvUdPIBQa0pctW7YHbJWboIACCiiggAIKKKCAAgoosDcKGOBqudcNcLWEcjIFFFBAAQUUUEABBRRQQAEFFFBgkgjk4dE++uijauXKlWNrTu8GDFkXwyY99NBDXXvaGPuSTxRQYOQCnJP0GEGPHYS3Yjhbhi2lVxyLAgoooIACCiiggAIKKKCAApNRwABXy71mgKsllJMpoIACCiiggAIKKKCAAgoooIACk0Tg+OOPr3vYIgxCYSir3377rWKoQIYqjLJ27dpq9erV8dJHBRT4LwowlNmiRYvqYRXj3GWoRXrRsyiggAIKKKCAAgoooIACCigwWQUMcLXccwa4WkI5mQIKKKCAAgoooIACCiiggAIKKDCJBObPn19dccUV1dSpUzuu9QcffFCtWrWq42e+qYACu18gAlyxZEKXDz/8cLV9+/Z4y0cFFFBAAQUUUEABBRRQQAEFJp2AAa6Wu8wAV0soJ1NAAQUUUEABBRRQQAEFFFBAAQUmmQDhrXPOOac6+uijq4MPPrjatm1b9cUXX1SbNm2qduzYMcm2xtVVYM8WOOCAA6oLLrigDl1u3bq12rhxY/XHH3/s2Rvt1imggAIKKKCAAgoooIACCuzxAga4Wu5iA1wtoZxMAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFWgsY4GpJZYCrJZSTKaCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiigQGsBA1wtqQxwtYRyMgUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFGgtYICrJZUBrpZQTqaAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAKtBQxwtaQywNUSyskUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFCgtYABrpZUBrhaQjmZAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKtBYwwNWSygBXSygnU0ABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAgdYCBrhaUhngagnlZAoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKNBawABXSyoDXC2hnEwBBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQVaCxjgakllgKsllJMpoIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKBAawEDXC2pDHC1hHIyBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUaC1ggKsllQGullBOpoACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAq0FDHC1pDLA1RLKyRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUKC1gAGu1lROqIACCiiggAIKKKCAAgoooIACCiigdw4mPAAAAw5JREFUgAIKKKCAAgoooIACCiiggAIKKKCAAgqMRuDXX3+tpk6dWu2zzz7VlClT6n8859+sWbPq1/vvv3/fhe8zb968/+07lRMooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgqMCRjgGqPwiQIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiwewUMcO1eb5emgAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCowJGOAao/CJAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKLB7BQxw7V5vl6aAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKjAkY4Bqj8MkoBQ455JDq2GOPrT799NPqr7/+GuWinLcCCiiwVwtY3+7Vu9+NV0ABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUECBSShggGsS7rTJtspHHHFE9eCDD9ar/eGHH1ZPP/30ZNuECbm+Rx55ZHXuuedWU6dOrX7//ffqlVdeqf78888Jua7DWqkFCxZUHE+UDRs2VJ999tmwZj3y+Zx11ll1iDEv6Oeff67WrFmT32r9fN99960uvvjiavr06fV32Pcvv/xytWPHjtbzcMI9T8D6ds/bp26RAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgrs+QIGuAr38YwZM6o5c+ZUmzdvrrZu3Vo4l8nxtdmzZ1dbtmyp/v7776IVnjt3brV48eL6u3g98MADRfPxSzsL3H///RXHYZQlS5ZMqkBTrHfbx5kzZ1b33Xff2OS//PJLdffdd4+9nshP/vOf/1RLly7tuIqLFi3q+H6/N88777xq4cKFO032yCOP1L3c7fSmL/YqgdL6dtB6fq9CdmMVUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUECBIQsMK8D1f24uqiazdlIEAAAAAElFTkSuQmCC)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "O3ENsWYB27Mb" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Pk55Mjq_3DiR" - }, - "source": [ - "## Example Use Case 1 - Code Generator\n", - "### For this use case enter your system prompt and questions\n" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "id": "_1SZYJFB3HmQ" - }, - "outputs": [], - "source": [ - "# enter your system prompt if you have one\n", - "system_prompt = \"\"\"\n", - "You are a coding assistant helping users using litellm.\n", - "litellm is a light package to simplify calling OpenAI, Azure, Cohere, Anthropic, Huggingface API Endpoints\n", - "--\n", - "Sample Usage:\n", - "```\n", - "pip install litellm\n", - "from litellm import completion\n", - "## set ENV variables\n", - "os.environ[\"OPENAI_API_KEY\"] = \"openai key\"\n", - "os.environ[\"COHERE_API_KEY\"] = \"cohere key\"\n", - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "# openai call\n", - "response = completion(model=\"gpt-3.5-turbo\", messages=messages)\n", - "# cohere call\n", - "response = completion(\"command-nightly\", messages)\n", - "```\n", - "\n", - "\"\"\"\n", - "\n", - "\n", - "# qustions/logs you want to run the LLM on\n", - "questions = [\n", - " \"what is litellm?\",\n", - " \"why should I use LiteLLM\",\n", - " \"does litellm support Anthropic LLMs\",\n", - " \"write code to make a litellm completion call\",\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "AHH3cqeU3_ZT" - }, - "source": [ - "## Running questions\n", - "### Select from 100+ LLMs here: https://docs.litellm.ai/docs/providers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "BpQD4A5339L3" - }, - "outputs": [], - "source": [ - "from litellm import completion, completion_cost\n", - "import os\n", - "import time\n", - "\n", - "# optional use litellm dashboard to view logs\n", - "# litellm.use_client = True\n", - "# litellm.token = \"ishaan_2@berri.ai\" # set your email\n", - "\n", - "\n", - "# set API keys\n", - "os.environ['TOGETHERAI_API_KEY'] = \"\"\n", - "os.environ['OPENAI_API_KEY'] = \"\"\n", - "os.environ['ANTHROPIC_API_KEY'] = \"\"\n", - "\n", - "\n", - "# select LLMs to benchmark\n", - "# using https://api.together.xyz/playground for llama2\n", - "# try any supported LLM here: https://docs.litellm.ai/docs/providers\n", - "\n", - "models = ['togethercomputer/llama-2-70b-chat', 'gpt-3.5-turbo', 'claude-instant-1.2']\n", - "data = []\n", - "\n", - "for question in questions: # group by question\n", - " for model in models:\n", - " print(f\"running question: {question} for model: {model}\")\n", - " start_time = time.time()\n", - " # show response, response time, cost for each question\n", - " response = completion(\n", - " model=model,\n", - " max_tokens=500,\n", - " messages = [\n", - " {\n", - " \"role\": \"system\", \"content\": system_prompt\n", - " },\n", - " {\n", - " \"role\": \"user\", \"content\": question\n", - " }\n", - " ],\n", - " )\n", - " end = time.time()\n", - " total_time = end-start_time # response time\n", - " # print(response)\n", - " cost = completion_cost(response) # cost for completion\n", - " raw_response = response['choices'][0]['message']['content'] # response string\n", - "\n", - "\n", - " # add log to pandas df\n", - " data.append(\n", - " {\n", - " 'Model': model,\n", - " 'Question': question,\n", - " 'Response': raw_response,\n", - " 'ResponseTime': total_time,\n", - " 'Cost': cost\n", - " })" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "apOSV3PBLa5Y" - }, - "source": [ - "## View Benchmarks for LLMs" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "CJqBlqUh_8Ws", - "outputId": "e02c3427-d8c6-4614-ff07-6aab64247ff6" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: does litellm support Anthropic LLMs\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
6togethercomputer/llama-2-70b-chatdoes litellm support Anthropic LLMsYes, litellm supports Anthropic LLMs.\\n\\nIn the example usage you provided, the `completion` function is called with the `model` parameter set to `\"gpt-3.5-turbo\"` for OpenAI and `\"command-nightly\"` for Cohere.\\n\\nTo use an Anthropic LLM with litellm, you would set the `model` parameter to the name of the Anthropic model you want to use, followed by the version number, if applicable. For example:\\n```\\nresponse = completion(model=\"anthropic-gpt-2\", messages=messages)\\n```\\nThis would call the Anthropic GPT-2 model to generate a completion for the given input messages.\\n\\nNote that you will need to set the `ANTHROPIC_API_KEY` environment variable to your Anthropic API key before making the call. You can do this by running the following command in your terminal:\\n```\\nos.environ[\"ANTHROPIC_API_KEY\"] = \"your-anthropic-api-key\"\\n```\\nReplace `\"your-anthropic-api-key\"` with your actual Anthropic API key.\\n\\nOnce you've set the environment variable, you can use the `completion` function with the `model` parameter set to an Anthropic model name to call the Anthropic API and generate a completion.21.5130090.001347
7gpt-3.5-turbodoes litellm support Anthropic LLMsNo, currently litellm does not support Anthropic LLMs. It mainly focuses on simplifying the usage of OpenAI, Azure, Cohere, and Huggingface API endpoints.8.6565100.000342
8claude-instant-1.2does litellm support Anthropic LLMsYes, litellm supports calling Anthropic LLMs through the completion function.\\n\\nTo use an Anthropic model with litellm:\\n\\n1. Set the ANTHROPIC_API_KEY environment variable with your Anthropic API key\\n\\n2. Pass the model name as the 'model' argument to completion(). Anthropic model names follow the format 'anthropic/<model_name>'\\n\\nFor example:\\n\\n```python \\nimport os\\nfrom litellm import completion\\n\\nos.environ[\"ANTHROPIC_API_KEY\"] = \"your_anthropic_api_key\"\\n\\nmessages = [{\"content\": \"Hello\", \"role\": \"user\"}]\\n\\nresponse = completion(model=\"anthropic/constitutional\", messages=messages)\\n```\\n\\nThis would call the Constitutional AI model from Anthropic.\\n\\nSo in summary, litellm provides a simple interface to call any Anthropic models as long as you specify the model name correctly and set the ANTHROPIC_API_KEY env variable.9.6981950.001342
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: what is litellm?\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
0togethercomputer/llama-2-70b-chatwhat is litellm?Litellm is a lightweight Python package that simplifies calling various AI API endpoints, including OpenAI, Azure, Cohere, Anthropic, and Hugging Face. It provides a convenient interface for making requests to these APIs, allowing developers to easily integrate them into their applications. With Litellm, developers can quickly and easily interact with multiple AI models and services, without having to handle the details of authentication, API calls, and response parsing. This makes it easier to build and deploy AI-powered applications, and can help developers save time and effort.13.4796440.000870
1gpt-3.5-turbowhat is litellm?litellm is a light package that provides a simplified interface for making API calls to various language models and APIs. It abstracts away the complexities of handling network requests, authentication, and response parsing, making it easier for developers to integrate powerful language models into their applications.\\n\\nWith litellm, you can quickly make API calls to models like OpenAI's GPT-3.5 Turbo, Azure's Text Analytics, Cohere's Command API, Anthropic's API, and Huggingface's models. It also supports additional functionality like conversational AI, summarization, translation, and more.\\n\\nBy using litellm, you can focus on your application logic without getting tangled in the details of API integration, allowing you to quickly build intelligent and conversational applications.8.3243320.000566
2claude-instant-1.2what is litellm?litellm is a Python library that simplifies calling various AI API endpoints like OpenAI, Azure, Cohere, Anthropic, and Huggingface. \\n\\nSome key things to know about litellm:\\n\\n- It provides a consistent interface for completing prompts and generating responses from different AI models through a single method called completion().\\n\\n- You specify the API (e.g. OpenAI, Cohere etc.) and model either by name or by setting environment variables before making the completion call.\\n\\n- This avoids having to use different SDKs or APIs for each provider and standardizes the call structure. \\n\\n- It handles things like setting headers, encoding inputs, parsing responses so the user doesn't have to deal with those details.\\n\\n- The goal is to make it easy to try different AI APIs and models without having to change code or learn different interfaces.\\n\\n- It's lightweight with no other dependencies required besides what's needed for each API (e.g. openai, azure SDKs etc.).\\n\\nSo in summary, litellm is a small library that provides a common way to interact with multiple conversational AI APIs through a single Python method, avoiding the need to directly use each provider's specific SDK.10.3164880.001603
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: why should I use LiteLLM\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
3togethercomputer/llama-2-70b-chatwhy should I use LiteLLM\\nThere are several reasons why you might want to use LiteLLM:\\n\\n1. Simplified API calls: LiteLLM provides a simple and consistent API for calling various language models, making it easier to use multiple models and switch between them.\\n2. Environment variable configuration: LiteLLM allows you to set environment variables for API keys and model names, making it easier to manage and switch between different models and APIs.\\n3. Support for multiple models and APIs: LiteLLM supports a wide range of language models and APIs, including OpenAI, Azure, Cohere, Anthropic, and Hugging Face.\\n4. Easy integration with popular frameworks: LiteLLM can be easily integrated with popular frameworks such as PyTorch and TensorFlow, making it easy to use with your existing codebase.\\n5. Lightweight: LiteLLM is a lightweight package, making it easy to install and use, even on resource-constrained devices.\\n6. Flexible: LiteLLM allows you to define your own models and APIs, making it easy to use with custom models and APIs.\\n7. Extensive documentation: LiteLLM has extensive documentation, making it easy to get started and learn how to use the package.\\n8. Active community: LiteLLM has an active community of developers and users, making it easy to get help and feedback on your projects.\\n\\nOverall, LiteLLM can help you to simplify your workflow, improve your productivity, and make it easier to work with multiple language models and APIs.23.7778850.001443
4gpt-3.5-turbowhy should I use LiteLLMLiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience.12.1098810.000881
5claude-instant-1.2why should I use LiteLLMHere are some key reasons why you may want to consider using LiteLLM:\\n\\n- Simplifies calling multiple large language models - LiteLLM provides a unified API to call models from different providers like OpenAI, Azure, HuggingFace, Anthropic etc. This avoids having to deal with different APIs from each provider.\\n\\n- Easy to get started - LiteLLM is very lightweight and simple to install with just one pip install. It has minimal configuration needed with just API keys. This lowers the barrier to start experimenting with LLMs.\\n\\n- Standard interface - The completion API provides a standard way to send messages and get responses back regardless of underlying model. This keeps the code interface consistent. \\n\\n- Model agnostic - Your code doesn't need to change to switch between different models. You can easily compare performance of models from different providers.\\n\\n- Open source - As it is open source, the library can be expanded to support more models and providers over time via community contributions.\\n\\n- Cost effective - LiteLLM helps avoid vendor lock-in. You can optimize costs by testing multiple models and choosing the best suitable and cost-effective option for your needs.\\n\\nSo in summary, L12.6102590.001568
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: write code to make a litellm completion call\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
9togethercomputer/llama-2-70b-chatwrite code to make a litellm completion call\\nTo make a completion call using litellm, you can use the `completion` function from the litellm package. Here's an example of how to use it:\\n```\\nfrom litellm import completion\\n\\n# Set ENV variables\\nos.environ[\"OPENAI_API_KEY\"] = \"your_openai_api_key\"\\nos.environ[\"COHERE_API_KEY\"] = \"your_cohere_api_key\"\\n\\n# Define the messages to be completed\\nmessages = [\\n {\\n \"content\": \"Hello, how are you?\",\\n \"role\": \"user\"\\n }\\n]\\n\\n# Make a completion call using OpenAI\\nresponse = completion(model=\"gpt-3.5-turbo\", messages=messages)\\n\\n# Make a completion call using Cohere\\nresponse = completion(\"command-nightly\", messages)\\n```\\nIn this example, we first set the ENV variables for the OpenAI and Cohere API keys. Then, we define a list of messages to be completed, which in this case contains a single message with the content \"Hello, how are you?\" and the role \"user\".\\n\\nNext, we make two completion calls using the `completion` function from litellm. The first call uses the OpenAI model `gpt-3.5-turbo` and passes in the list of messages. The second call uses the Cohere model `command-nightly` and passes in the same list of messages.\\n\\nThe `completion` function returns a response object that contains the completed messages. You can then use the `response.messages` attribute to access the completed messages.\\n\\nHere's an example of how to access the completed messages:\\n```\\n# Print the completed messages\\nprint(response.messages)\\n```\\nThis will print the completed messages, which should look something like this:\\n```\\n[{'content': 'Hello, how are you?', 'role': 'user', 'completion': 'I am doing well, thanks for asking.'}]\\n```\\nNote that the `completion` attribute contains the completed message. You can use this attribute to retrieve the completed message and use it in your application.32.1924530.001821
10gpt-3.5-turbowrite code to make a litellm completion callSure! Here's an example code to make a completion call using the litellm package:\\n\\n```python\\nfrom litellm import completion\\n\\n# Set the model and messages\\nmodel = \"gpt-3.5-turbo\"\\nmessages = [\\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\\n {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"},\\n {\"role\": \"assistant\", \"content\": \"The Los Angeles Dodgers won the World Series in 2020.\"},\\n {\"role\": \"user\", \"content\": \"Where was it played?\"}\\n]\\n\\n# Make the completion call\\nresponse = completion(model=model, messages=messages)\\n\\n# Print the assistant's reply\\nassistant_reply = response[\"choices\"][0][\"message\"][\"content\"]\\nprint(\"Assistant: \", assistant_reply)\\n```\\n\\nMake sure you have the litellm package installed (`pip install litellm`) and set the necessary environment variables for the API keys before running this code.9.3771550.000686
11claude-instant-1.2write code to make a litellm completion callHere is an example of making a completion call using litellm:\\n\\n```python\\nimport os\\nfrom litellm import completion\\n\\n# Set API keys as environment variables\\nos.environ[\"OPENAI_API_KEY\"] = \"your openai api key\" \\n\\n# Conversation context \\nmessages = [{\\n \"content\": \"Hello, how can I help you today?\",\\n \"role\": \"assistant\"\\n}]\\n\\n# Make completion call with GPT-3 model\\nresponse = completion(\\n model=\"gpt-3.5-turbo\", \\n messages=messages\\n)\\n\\nprint(response)\\n```\\n\\nTo break it down:\\n\\n- Import completion from litellm\\n- Set the OPENAI_API_KEY env var \\n- Define a messages list with the conversation context\\n- Call completion(), specifying the model (\"gpt-3.5-turbo\") and messages\\n- It will return the response from the API\\n- Print the response\\n\\nThis makes a simple completion call to OpenAI GPT-3 using litellm to handle the API details. You can also call other models like Cohere or Anthropic by specifying their name instead of the OpenAI9.8399880.001578
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from IPython.core.interactiveshell import InteractiveShell\n", - "InteractiveShell.ast_node_interactivity = \"all\"\n", - "from IPython.display import HTML\n", - "import pandas as pd\n", - "\n", - "df = pd.DataFrame(data)\n", - "grouped_by_question = df.groupby('Question')\n", - "\n", - "for question, group_data in grouped_by_question:\n", - " print(f\"Question: {question}\")\n", - " HTML(group_data.to_html())\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bmtAbC1rGVAm" - }, - "source": [ - "## Use Case 2 - Rewrite user input concisely" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "id": "boiHO1PhGXSL" - }, - "outputs": [], - "source": [ - "# enter your system prompt if you have one\n", - "system_prompt = \"\"\"\n", - "For a given user input, rewrite the input to make be more concise.\n", - "\"\"\"\n", - "\n", - "# user input for re-writing questions\n", - "questions = [\n", - " \"LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience\",\n", - " \"Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!\",\n", - " \"Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.\"\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fwNcC_obICUc" - }, - "source": [ - "## Run Questions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "KtBjZ1mUIBiJ" - }, - "outputs": [], - "source": [ - "from litellm import completion, completion_cost\n", - "import os\n", - "import time\n", - "\n", - "# optional use litellm dashboard to view logs\n", - "# litellm.use_client = True\n", - "# litellm.token = \"ishaan_2@berri.ai\" # set your email\n", - "\n", - "os.environ['TOGETHERAI_API_KEY'] = \"\"\n", - "os.environ['OPENAI_API_KEY'] = \"\"\n", - "os.environ['ANTHROPIC_API_KEY'] = \"\"\n", - "\n", - "models = ['togethercomputer/llama-2-70b-chat', 'gpt-3.5-turbo', 'claude-instant-1.2'] # enter llms to benchmark\n", - "data_2 = []\n", - "\n", - "for question in questions: # group by question\n", - " for model in models:\n", - " print(f\"running question: {question} for model: {model}\")\n", - " start_time = time.time()\n", - " # show response, response time, cost for each question\n", - " response = completion(\n", - " model=model,\n", - " max_tokens=500,\n", - " messages = [\n", - " {\n", - " \"role\": \"system\", \"content\": system_prompt\n", - " },\n", - " {\n", - " \"role\": \"user\", \"content\": \"User input:\" + question\n", - " }\n", - " ],\n", - " )\n", - " end = time.time()\n", - " total_time = end-start_time # response time\n", - " # print(response)\n", - " cost = completion_cost(response) # cost for completion\n", - " raw_response = response['choices'][0]['message']['content'] # response string\n", - " #print(raw_response, total_time, cost)\n", - "\n", - " # add to pandas df\n", - " data_2.append(\n", - " {\n", - " 'Model': model,\n", - " 'Question': question,\n", - " 'Response': raw_response,\n", - " 'ResponseTime': total_time,\n", - " 'Cost': cost\n", - " })\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-PCYIzG5M0II" - }, - "source": [ - "## View Logs - Group by Question" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "-3R5-2q8IiL2", - "outputId": "c4a0d9e5-bb21-4de0-fc4c-9f5e71d0f177" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
3togethercomputer/llama-2-70b-chatHi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!\\nHere's a more concise version of the user input:\\n\\n\"Hi everyone! I'm [your name] and I'm working on [your project/role involving LLMs]. I recently discovered LiteLLM and I'm excited to use it to [build an app/simplify my code/test different models etc]. Before LiteLLM, I struggled with [describe any issues you faced working with multiple LLMs]. I'm looking forward to using LiteLLM's unified API and automatic translation to achieve my goals. I'm eager to learn more about building impactful applications powered by LLMs and to be part of this community. Let me know if you have any questions or need further clarification.\"\\n\\nIn this revised version, we've kept the essential information and removed some of the extraneous language. We've also rephrased some of the sentences to make them more concise and easier to read.18.3006200.001200
4gpt-3.5-turboHi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!User input: Hi, I'm [your name] and I'm excited about using LiteLLM to simplify working with different LLM providers. Before finding LiteLLM, I faced challenges working with multiple LLMs. With LiteLLM's unified API and automatic translation, I believe it will help me achieve my goals of [state your goals]. I look forward to being part of this community and learning how to build impactful applications with LLMs. Let me know if you need any further clarification or details.7.3854720.000525
5claude-instant-1.2Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!Here is a more concise rewrite of the user input:\\n\\nHi everyone, I'm [your name]. I'm currently [your project/role] and came across LiteLLM, which simplifies working with different LLMs through its unified API. I hope to [build an app/simplify code/test models] with LiteLLM since I previously struggled with [issues]. LiteLLM's automatic translation between providers will help me [goals] and build impactful LLM applications. Looking forward to learning more as part of this community. Let me know if you need any clarification on my plans to use LiteLLM.8.6282170.001022
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\n", - "\n", - "1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\n", - "\n", - "2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\n", - "\n", - "3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\n", - "\n", - "4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\n", - "\n", - "5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\n", - "\n", - "6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\n", - "\n", - "Overall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
0togethercomputer/llama-2-70b-chatLiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenienceHere's a more concise version of the user input:\\n\\nLiteLLM is a lightweight Python package that simplifies API calls to various language models. It abstracts away complexity, integrates seamlessly, supports multiple models, and handles authentication. It's ideal for rapid prototyping and has community support. It saves time and effort while providing flexibility and convenience.11.2942500.001251
1gpt-3.5-turboLiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenienceLiteLLM is a lightweight Python package that simplifies API calls to various language models. Here's why you should use it:\\n1. Simplified API Calls: Works with multiple models (OpenAI, Azure, Cohere, Anthropic, Huggingface).\\n2. Easy Integration: Import and start using it quickly in your codebase.\\n3. Flexibility: Supports GPT-3, GPT-Neo, chatGPT, etc. easily switch between models.\\n4. Convenience: Handles authentication and connection details, just set environment variables.\\n5. Quick Prototyping: Great for rapid prototyping and building interactive applications.\\n6. Community Support: Actively maintained and supported by a developer community.9.7783150.000795
2claude-instant-1.2LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenienceHere is a concise rewrite of the user input:\\n\\nLiteLLM is a lightweight Python package that simplifies accessing various language models. It provides a unified interface for models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more. Key benefits include simplified API calls, easy integration, flexibility to use different models, automated handling of authentication, and support for quick prototyping. The actively maintained package saves time by abstracting away complexity while offering convenience and a collaborative community.7.6975200.001098
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
6togethercomputer/llama-2-70b-chatTraceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.\\nRewritten input: Traceloop is a platform for monitoring and debugging LLM outputs. It allows users to track performance, rollout changes confidently, and debug issues in production. It uses OpenTelemetry for full visibility into LLM requests, vector DB usage, and other infrastructure.9.0604440.000525
7gpt-3.5-turboTraceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It helps track performance, rollout changes, and debug issues in production. It is based on OpenTelemetry, providing visibility to LLM requests, vector DB usage, and other infrastructure in your stack.7.3046610.000283
8claude-instant-1.2Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.Here is a more concise rewrite of the user input:\\n\\nTraceloop monitors and debugs LLM quality. It tracks LLM performance, enables confident changes, and debugs production issues. Based on OpenTelemetry, Traceloop provides full visibility into LLM requests, vector DB usage, and other stack infrastructure.7.9761580.000538
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from IPython.core.interactiveshell import InteractiveShell\n", - "InteractiveShell.ast_node_interactivity = \"all\"\n", - "from IPython.display import HTML\n", - "import pandas as pd\n", - "\n", - "df = pd.DataFrame(data_2)\n", - "grouped_by_question = df.groupby('Question')\n", - "\n", - "for question, group_data in grouped_by_question:\n", - " print(f\"Question: {question}\")\n", - " HTML(group_data.to_html())\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb b/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb deleted file mode 100644 index 338785ea5e3..00000000000 --- a/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb +++ /dev/null @@ -1,406 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ZwuaylskLxFu", - "outputId": "d684d6a3-32fe-4beb-c378-c39134bcf8cc" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting litellm==0.1.363\n", - " Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n", - "Requirement already satisfied: openai<0.28.0,>=0.27.8 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.27.8)\n", - "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (1.0.0)\n", - "Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.4.0)\n", - "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (2.31.0)\n", - "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.65.0)\n", - "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.8.5)\n", - "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.363) (2022.10.31)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.26.16)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (2023.7.22)\n", - "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (23.1.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (6.0.4)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.0.2)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.9.2)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.4.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.3.1)\n", - "Installing collected packages: litellm\n", - " Attempting uninstall: litellm\n", - " Found existing installation: litellm 0.1.362\n", - " Uninstalling litellm-0.1.362:\n", - " Successfully uninstalled litellm-0.1.362\n", - "Successfully installed litellm-0.1.363\n" - ] - } - ], - "source": [ - "!pip install litellm==\"0.1.363\"" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "W216G__XL19Q" - }, - "outputs": [], - "source": [ - "# @title Import litellm & Set env variables\n", - "import litellm\n", - "import os\n", - "\n", - "os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ff1lKwUMMLJj", - "outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\n", - " Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n", - "\n", - "\n", - " Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n" - ] - } - ], - "source": [ - "# @title Request Claude Instant-1 and Claude-2\n", - "messages = [\n", - " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", - " {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"}\n", - " ]\n", - "\n", - "result = litellm.completion('claude-instant-1', messages)\n", - "print(\"\\n\\n Result from claude-instant-1\", result)\n", - "result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n", - "print(\"\\n\\n Result from claude-2\", result)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "06hWKnNQMrV-", - "outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " Here\n", - "'s\n", - " a\n", - " quick\n", - " overview\n", - " of\n", - " how\n", - " a\n", - " court\n", - " case\n", - " can\n", - " reach\n", - " the\n", - " U\n", - ".\n", - "S\n", - ".\n", - " Supreme\n", - " Court\n", - ":\n", - "\n", - "\n", - "-\n", - " The\n", - " case\n", - " must\n", - " first\n", - " be\n", - " heard\n", - " in\n", - " a\n", - " lower\n", - " trial\n", - " court\n", - " (\n", - "either\n", - " a\n", - " state\n", - " court\n", - " or\n", - " federal\n", - " district\n", - " court\n", - ").\n", - " The\n", - " trial\n", - " court\n", - " makes\n", - " initial\n", - " r\n", - "ulings\n", - " and\n", - " produces\n", - " a\n", - " record\n", - " of\n", - " the\n", - " case\n", - ".\n", - "\n", - "\n", - "-\n", - " The\n", - " losing\n", - " party\n", - " can\n", - " appeal\n", - " the\n", - " decision\n", - " to\n", - " an\n", - " appeals\n", - " court\n", - " (\n", - "a\n", - " state\n", - " appeals\n", - " court\n", - " for\n", - " state\n", - " cases\n", - ",\n", - " or\n", - " a\n", - " federal\n", - " circuit\n", - " court\n", - " for\n", - " federal\n", - " cases\n", - ").\n", - " The\n", - " appeals\n", - " court\n", - " reviews\n", - " the\n", - " trial\n", - " court\n", - "'s\n", - " r\n", - "ulings\n", - " and\n", - " can\n", - " affirm\n", - ",\n", - " reverse\n", - ",\n", - " or\n", - " modify\n", - " the\n", - " decision\n", - ".\n", - "\n", - "\n", - "-\n", - " If\n", - " a\n", - " party\n", - " is\n", - " still\n", - " unsat\n", - "isf\n", - "ied\n", - " after\n", - " the\n", - " appeals\n", - " court\n", - " rules\n", - ",\n", - " they\n", - " can\n", - " petition\n", - " the\n", - " Supreme\n", - " Court\n", - " to\n", - " hear\n", - " the\n", - " case\n", - " through\n", - " a\n", - " writ\n", - " of\n", - " cert\n", - "ior\n", - "ari\n", - ".\n", - " \n", - "\n", - "\n", - "-\n", - " The\n", - " Supreme\n", - " Court\n", - " gets\n", - " thousands\n", - " of\n", - " cert\n", - " petitions\n", - " every\n", - " year\n", - " but\n", - " usually\n", - " only\n", - " agrees\n", - " to\n", - " hear\n", - " about\n", - " 100\n", - "-\n", - "150\n", - " of\n", - " cases\n", - " that\n", - " have\n", - " significant\n", - " national\n", - " importance\n", - " or\n", - " where\n", - " lower\n", - " courts\n", - " disagree\n", - " on\n", - " federal\n", - " law\n", - ".\n", - " \n", - "\n", - "\n", - "-\n", - " If\n", - " 4\n", - " out\n", - " of\n", - " the\n", - " 9\n", - " Just\n", - "ices\n", - " vote\n", - " to\n", - " grant\n", - " cert\n", - " (\n", - "agree\n", - " to\n", - " hear\n", - " the\n", - " case\n", - "),\n", - " it\n", - " goes\n", - " on\n", - " the\n", - " Supreme\n", - " Court\n", - "'s\n", - " do\n", - "cket\n", - " for\n", - " arguments\n", - ".\n", - "\n", - "\n", - "-\n", - " The\n", - " Supreme\n", - " Court\n", - " then\n", - " hears\n", - " oral\n", - " arguments\n", - ",\n", - " considers\n", - " written\n", - " brief\n", - "s\n", - ",\n", - " examines\n", - " the\n", - " lower\n", - " court\n", - " records\n", - ",\n", - " and\n", - " issues\n", - " a\n", - " final\n", - " ruling\n", - " on\n", - " the\n", - " case\n", - ",\n", - " which\n", - " serves\n", - " as\n", - " binding\n", - " precedent\n" - ] - } - ], - "source": [ - "# @title Streaming Example: Request Claude-2\n", - "messages = [\n", - " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", - " {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n", - " ]\n", - "\n", - "result = litellm.completion('claude-2', messages, stream=True)\n", - "for part in result:\n", - " print(part.choices[0].delta.content or \"\")\n", - "\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/Evaluating_LLMs.ipynb b/cookbook/Evaluating_LLMs.ipynb deleted file mode 100644 index e27e8934f76..00000000000 --- a/cookbook/Evaluating_LLMs.ipynb +++ /dev/null @@ -1,579 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "Ys9n20Es2IzT" - }, - "source": [ - "# Evaluate Multiple LLM Providers with LiteLLM\n", - "\n", - "\n", - "\n", - "* Quality Testing\n", - "* Load Testing\n", - "* Duration Testing\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZXOXl23PIIP6" - }, - "outputs": [], - "source": [ - "!pip install litellm python-dotenv" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "LINuBzXDItq2" - }, - "outputs": [], - "source": [ - "from litellm import load_test_model, testing_batch_completion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "EkxMhsWdJdu4" - }, - "outputs": [], - "source": [ - "import os \n", - "os.environ[\"OPENAI_API_KEY\"] = \"...\"\n", - "os.environ[\"ANTHROPIC_API_KEY\"] = \"...\"\n", - "os.environ[\"REPLICATE_API_KEY\"] = \"...\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "mv5XdnqeW5I_" - }, - "source": [ - "# Quality Test endpoint\n", - "\n", - "## Test the same prompt across multiple LLM providers\n", - "\n", - "In this example, let's ask some questions about Paul Graham" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "id": "XpzrR5m4W_Us" - }, - "outputs": [], - "source": [ - "models = [\"gpt-3.5-turbo\", \"gpt-3.5-turbo-16k\", \"gpt-4\", \"claude-instant-1\", {\"model\": \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"custom_llm_provider\": \"replicate\"}]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompts = [\"Who is Paul Graham?\", \"What is Paul Graham known for?\" , \"Is paul graham a writer?\" , \"Where does Paul Graham live?\", \"What has Paul Graham done?\"]\n", - "messages = [[{\"role\": \"user\", \"content\": context + \"\\n\" + prompt}] for prompt in prompts] # pass in a list of messages we want to test\n", - "result = testing_batch_completion(models=models, messages=messages)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "9nzeLySnvIIW" - }, - "source": [ - "## Visualize the data" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 403 - }, - "id": "X-2n7hdAuVAY", - "outputId": "69cc0de1-68e3-4c12-a8ea-314880010d94" - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Model Nameclaude-instant-1gpt-3.5-turbo-0613gpt-3.5-turbo-16k-0613gpt-4-0613replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781
Prompt
\\nIs paul graham a writer?Yes, Paul Graham is considered a writer in ad...Yes, Paul Graham is a writer. He has written s...Yes, Paul Graham is a writer. He has authored ...Yes, Paul Graham is a writer. He is an essayis...Yes, Paul Graham is an author. According to t...
\\nWhat has Paul Graham done?Paul Graham has made significant contribution...Paul Graham has achieved several notable accom...Paul Graham has made significant contributions...Paul Graham is known for his work on the progr...Paul Graham has had a diverse career in compu...
\\nWhat is Paul Graham known for?Paul Graham is known for several things:\\n\\n-...Paul Graham is known for his work on the progr...Paul Graham is known for his work on the progr...Paul Graham is known for his work on the progr...Paul Graham is known for many things, includi...
\\nWhere does Paul Graham live?Based on the information provided:\\n\\n- Paul ...According to the given information, Paul Graha...Paul Graham currently lives in England, where ...The text does not provide a current place of r...Based on the information provided, Paul Graha...
\\nWho is Paul Graham?Paul Graham is an influential computer scient...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist,...
\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - "
\n", - " \n", - "
\n", - "\n", - "\n", - "\n", - " \n", - "\n", - "\n", - " \n", - " \n", - "\n", - " \n", - "
\n", - "
\n" - ], - "text/plain": [ - "Model Name claude-instant-1 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is considered a writer in ad... \n", - "\\nWhat has Paul Graham done? Paul Graham has made significant contribution... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for several things:\\n\\n-... \n", - "\\nWhere does Paul Graham live? Based on the information provided:\\n\\n- Paul ... \n", - "\\nWho is Paul Graham? Paul Graham is an influential computer scient... \n", - "\n", - "Model Name gpt-3.5-turbo-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He has written s... \n", - "\\nWhat has Paul Graham done? Paul Graham has achieved several notable accom... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? According to the given information, Paul Graha... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name gpt-3.5-turbo-16k-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He has authored ... \n", - "\\nWhat has Paul Graham done? Paul Graham has made significant contributions... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? Paul Graham currently lives in England, where ... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name gpt-4-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He is an essayis... \n", - "\\nWhat has Paul Graham done? Paul Graham is known for his work on the progr... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? The text does not provide a current place of r... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781 \n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is an author. According to t... \n", - "\\nWhat has Paul Graham done? Paul Graham has had a diverse career in compu... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for many things, includi... \n", - "\\nWhere does Paul Graham live? Based on the information provided, Paul Graha... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist,... " - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "# Create an empty list to store the row data\n", - "table_data = []\n", - "\n", - "# Iterate through the list and extract the required data\n", - "for item in result:\n", - " prompt = item['prompt'][0]['content'].replace(context, \"\") # clean the prompt for easy comparison\n", - " model = item['response']['model']\n", - " response = item['response']['choices'][0]['message']['content']\n", - " table_data.append([prompt, model, response])\n", - "\n", - "# Create a DataFrame from the table data\n", - "df = pd.DataFrame(table_data, columns=['Prompt', 'Model Name', 'Response'])\n", - "\n", - "# Pivot the DataFrame to get the desired table format\n", - "table = df.pivot(index='Prompt', columns='Model Name', values='Response')\n", - "table" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "zOxUM40PINDC" - }, - "source": [ - "# Load Test endpoint\n", - "\n", - "Run 100+ simultaneous queries across multiple providers to see when they fail + impact on latency" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZkQf_wbcIRQ9" - }, - "outputs": [], - "source": [ - "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompt = \"Where does Paul Graham live?\"\n", - "final_prompt = context + prompt\n", - "result = load_test_model(models=models, prompt=final_prompt, num_calls=5)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "8vSNBFC06aXY" - }, - "source": [ - "## Visualize the data" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 552 - }, - "id": "SZfiKjLV3-n8", - "outputId": "00f7f589-b3da-43ed-e982-f9420f074b8d" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAIXCAYAAACy1HXAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABn5UlEQVR4nO3dd1QT2d8G8Cf0ojQBEUFRsSv2FXvvvSx2saNi7733ihXELotd7KuIir33sjZUsIuKVGmS+/7hy/yM6K7RYEZ4PufkaO5Mkm/IJHly594ZhRBCgIiIiEiGdLRdABEREdG3MKgQERGRbDGoEBERkWwxqBAREZFsMagQERGRbDGoEBERkWwxqBAREZFsMagQERGRbDGoEBERkWwxqBCR7Dk5OaFLly7aLkNtc+fORd68eaGrq4uSJUtquxyNO3bsGBQKBbZv367tUtSmUCgwadIktW8XGhoKhUKBdevWabwm+joGFVKxfPlyKBQKlC9fXtulyI6TkxMUCoV0MTU1xR9//IENGzZou7TfTuoX3PdcfleHDh3CiBEjUKlSJaxduxYzZszQdkmys27dOul1PnXqVJrlQgg4OjpCoVCgcePGWqiQ5EBP2wWQvPj7+8PJyQkXLlxASEgInJ2dtV2SrJQsWRJDhw4FALx8+RKrVq2Cu7s7EhMT0bNnTy1X9/soXLgw/Pz8VNpGjx6NLFmyYOzYsWnWv3fvHnR0fq/fVUePHoWOjg5Wr14NAwMDbZcja0ZGRti4cSMqV66s0n78+HE8e/YMhoaGWqqM5IBBhSSPHz/GmTNnEBAQAA8PD/j7+2PixIm/tAalUomkpCQYGRn90sf9Xjlz5kTHjh2l6126dEHevHmxcOFCBhU1ZM+eXeXvCACzZs2CtbV1mnYAv+UXVXh4OIyNjTUWUoQQSEhIgLGxsUbuT04aNmyIbdu2YfHixdDT+9/X0saNG1GmTBm8fftWi9WRtv1eP1EoXfn7+8PS0hKNGjVC69at4e/vLy1LTk6GlZUVunbtmuZ20dHRMDIywrBhw6S2xMRETJw4Ec7OzjA0NISjoyNGjBiBxMREldsqFAr069cP/v7+KFq0KAwNDXHw4EEAwLx581CxYkVky5YNxsbGKFOmzFf3hcfHx2PAgAGwtrZG1qxZ0bRpUzx//vyr+6CfP3+Obt26IXv27DA0NETRokWxZs2aH/6b2djYoFChQnj48KFKu1KphJeXF4oWLQojIyNkz54dHh4eeP/+vcp6ly5dQr169WBtbQ1jY2PkyZMH3bp1k5an7g+fN28eFi5ciNy5c8PY2BjVqlXDrVu30tRz9OhRVKlSBaamprCwsECzZs1w584dlXUmTZoEhUKBkJAQdOnSBRYWFjA3N0fXrl3x4cMHlXWDgoJQuXJlWFhYIEuWLChYsCDGjBmjss73vtY/48sxKqm7DE6dOoUBAwbAxsYGFhYW8PDwQFJSEiIjI9G5c2dYWlrC0tISI0aMwJcnitfUa/Q1CoUCa9euRVxcnLRrI3VMw8ePHzF16lTky5cPhoaGcHJywpgxY9L8vZycnNC4cWMEBgaibNmyMDY2xooVK/71cc+fP4/69evD3NwcJiYmqFatGk6fPq2yTlhYGPr27YuCBQvC2NgY2bJlw59//onQ0NA09xcZGYnBgwfDyckJhoaGcHBwQOfOndMEB6VSienTp8PBwQFGRkaoVasWQkJC/rXWz7Vr1w7v3r1DUFCQ1JaUlITt27ejffv2X71NXFwchg4dCkdHRxgaGqJgwYKYN29emtc5MTERgwcPho2NjfT58OzZs6/ep6Y/H0hDBNH/K1SokOjevbsQQogTJ04IAOLChQvS8m7dugkLCwuRmJiocrv169cLAOLixYtCCCFSUlJE3bp1hYmJiRg0aJBYsWKF6Nevn9DT0xPNmjVTuS0AUbhwYWFjYyMmT54sli1bJq5evSqEEMLBwUH07dtXLF26VCxYsED88ccfAoDYt2+fyn24ubkJAKJTp05i2bJlws3NTZQoUUIAEBMnTpTWe/XqlXBwcBCOjo5iypQpwtvbWzRt2lQAEAsXLvzPv0/u3LlFo0aNVNqSk5OFnZ2dyJ49u0p7jx49hJ6enujZs6fw8fERI0eOFKampqJcuXIiKSlJCCHE69evhaWlpShQoICYO3euWLlypRg7dqwoXLiwdD+PHz8WAETx4sWFk5OTmD17tpg8ebKwsrISNjY24tWrV9K6QUFBQk9PTxQoUEDMmTNHTJ48WVhbWwtLS0vx+PFjab2JEycKAKJUqVKiZcuWYvny5aJHjx4CgBgxYoS03q1bt4SBgYEoW7asWLRokfDx8RHDhg0TVatWldZR57X+L0WLFhXVqlX75t/e3d1dur527VoBQJQsWVLUr19fLFu2THTq1El6DpUrVxbt27cXy5cvF40bNxYAxPr169PlNfoaPz8/UaVKFWFoaCj8/PyEn5+fePjwoRBCCHd3dwFAtG7dWixbtkx07txZABDNmzdP85ydnZ2FpaWlGDVqlPDx8RHBwcHffMwjR44IAwMDUaFCBTF//nyxcOFC4eLiIgwMDMT58+el9bZt2yZKlCghJkyYIHx9fcWYMWOEpaWlyJ07t4iLi5PWi4mJEcWKFRO6urqiZ8+ewtvbW0ydOlWUK1dOeo8GBwdL21KZMmXEwoULxaRJk4SJiYn4448//vVv9PnrePHiRVGxYkXRqVMnadmuXbuEjo6OeP78eZr3nlKpFDVr1hQKhUL06NFDLF26VDRp0kQAEIMGDVJ5jI4dOwoAon379mLp0qWiZcuWwsXF5Yc/H1Lfk2vXrv3P50eawaBCQgghLl26JACIoKAgIcSnDwIHBwcxcOBAaZ3AwEABQOzdu1fltg0bNhR58+aVrvv5+QkdHR1x8uRJlfV8fHwEAHH69GmpDYDQ0dERt2/fTlPThw8fVK4nJSWJYsWKiZo1a0ptly9f/uqHU5cuXdJ8EHXv3l3kyJFDvH37VmXdtm3bCnNz8zSP96XcuXOLunXrijdv3og3b96ImzdvSl+Onp6e0nonT54UAIS/v7/K7Q8ePKjSvnPnTpWA9zWpH4rGxsbi2bNnUvv58+cFADF48GCprWTJksLW1la8e/dOart+/brQ0dERnTt3ltpSg0q3bt1UHqtFixYiW7Zs0vWFCxcKAOLNmzffrE+d1/q//EhQqVevnlAqlVJ7hQoVhEKhEL1795baPn78KBwcHFTuW5Ov0be4u7sLU1NTlbZr164JAKJHjx4q7cOGDRMAxNGjR1WeMwBx8ODB/3wspVIp8ufPn+bv8eHDB5EnTx5Rp04dlbYvnT17VgAQGzZskNomTJggAIiAgICvPp4Q/wsqhQsXVvkBs2jRIgFA3Lx581/r/jyoLF26VGTNmlWq788//xQ1atSQ/hafB5Vdu3YJAGLatGkq99e6dWuhUChESEiIEOJ/f+++ffuqrNe+ffsf/nxgUPn1uOuHAHza7ZM9e3bUqFEDwKeu6zZt2mDz5s1ISUkBANSsWRPW1tbYsmWLdLv3798jKCgIbdq0kdq2bduGwoULo1ChQnj79q10qVmzJgAgODhY5bGrVauGIkWKpKnp833x79+/R1RUFKpUqYIrV65I7am7ifr27aty2/79+6tcF0Jgx44daNKkCYQQKnXVq1cPUVFRKvf7LYcOHYKNjQ1sbGxQvHhx+Pn5oWvXrpg7d67K8zc3N0edOnVUHqdMmTLIkiWL9PwtLCwAAPv27UNycvK/Pm7z5s2RM2dO6foff/yB8uXL4++//wbwaWDvtWvX0KVLF1hZWUnrubi4oE6dOtJ6n+vdu7fK9SpVquDdu3eIjo5WqW/37t1QKpVfrUvd11rTunfvrjIzqHz58hBCoHv37lKbrq4uypYti0ePHqnUrenX6Hukvg5DhgxRaU8doL1//36V9jx58qBevXr/eb/Xrl3DgwcP0L59e7x79056PnFxcahVqxZOnDghvYafv6+Sk5Px7t07ODs7w8LCQuU9sGPHDpQoUQItWrRI83hfzsbq2rWrylicKlWqAIDK3/y/uLm5IT4+Hvv27UNMTAz27dv3zd0+f//9N3R1dTFgwACV9qFDh0IIgQMHDkjrAUiz3qBBg1Sua+rzgdJHhgkqJ06cQJMmTWBvbw+FQoFdu3al+2M+f/4cHTt2lMZQFC9eHJcuXUr3x9W0lJQUbN68GTVq1MDjx48REhKCkJAQlC9fHq9fv8aRI0cAAHp6emjVqhV2794t7U8PCAhAcnKySlB58OABbt++LX2hp14KFCgA4NMgw8/lyZPnq3Xt27cPrq6uMDIygpWVFWxsbODt7Y2oqChpnbCwMOjo6KS5jy9nK7158waRkZHw9fVNU1fquJsv6/qa8uXLIygoCAcPHsS8efNgYWGB9+/fq3xIP3jwAFFRUbC1tU3zWLGxsdLjVKtWDa1atcLkyZNhbW2NZs2aYe3atV8d25E/f/40bQUKFJDGFYSFhQEAChYsmGa9woULS19an8uVK5fKdUtLSwCQxmi0adMGlSpVQo8ePZA9e3a0bdsWW7duVQkt6r7WmvblczA3NwcAODo6pmn/fOxJerxG3yN1e/1y+7Szs4OFhYX0Oqb61nvjSw8ePAAAuLu7p3k+q1atQmJiovS+iY+Px4QJE6SxHdbW1rCxsUFkZKTKe+vhw4coVqzYdz3+f21L38PGxga1a9fGxo0bERAQgJSUFLRu3fqr64aFhcHe3h5Zs2ZVaS9cuLC0PPVfHR0d5MuXT2W9L98nmvp8oPSRYWb9xMXFoUSJEujWrRtatmyZ7o/3/v17VKpUCTVq1MCBAwdgY2ODBw8eSG/Q38nRo0fx8uVLbN68GZs3b06z3N/fH3Xr1gUAtG3bFitWrMCBAwfQvHlzbN26FYUKFUKJEiWk9ZVKJYoXL44FCxZ89fG+/BL52iyGkydPomnTpqhatSqWL1+OHDlyQF9fH2vXrsXGjRvVfo6pX64dO3aEu7v7V9dxcXH5z/uxtrZG7dq1AQD16tVDoUKF0LhxYyxatEj6laxUKmFra6syGPlzNjY2ACAdKOvcuXPYu3cvAgMD0a1bN8yfPx/nzp1DlixZ1H6e6tDV1f1qu/j/wYjGxsY4ceIEgoODsX//fhw8eBBbtmxBzZo1cejQIejq6qr9Wmvat57D19rFZ4Mstf0afe/xYb53hk/q9j137txvHlgutdb+/ftj7dq1GDRoECpUqABzc3MoFAq0bdv2mz1n/+W/tqXv1b59e/Ts2ROvXr1CgwYNpB6t9KapzwdKHxkmqDRo0AANGjT45vLExESMHTsWmzZtQmRkJIoVK4bZs2ejevXqP/R4s2fPhqOjI9auXSu1fe+vH7nx9/eHra0tli1blmZZQEAAdu7cCR8fHxgbG6Nq1arIkSMHtmzZgsqVK+Po0aNpjnuRL18+XL9+HbVq1frhA3bt2LEDRkZGCAwMVJma+vnfGwBy584NpVKJx48fq/Q6fDnjIHXEf0pKihQ0NKFRo0aoVq0aZsyYAQ8PD5iamiJfvnw4fPgwKlWq9F1fNK6urnB1dcX06dOxceNGdOjQAZs3b0aPHj2kdVJ/MX/u/v37cHJyAvDp7wB8Ot7Il+7evQtra2uYmpqq/fx0dHRQq1Yt1KpVCwsWLMCMGTMwduxYBAcHo3bt2hp5rbUhPV6j75G6vT548ED69Q8Ar1+/RmRkpPQ6qiu1x8DMzOw/t+/t27fD3d0d8+fPl9oSEhIQGRmZ5j6/NrMsPbVo0QIeHh44d+6cyi7mL+XOnRuHDx9GTEyMSq/K3bt3peWp/yqVSjx8+FClF+XL90l6fT6QZmSYXT//pV+/fjh79iw2b96MGzdu4M8//0T9+vW/+gXwPfbs2YOyZcvizz//hK2tLUqVKoWVK1dquOr0Fx8fj4CAADRu3BitW7dOc+nXrx9iYmKwZ88eAJ++uFq3bo29e/fCz88PHz9+VNntA3za1/z8+fOv/j3i4+PT7IL4Gl1dXSgUCml8DPBpqu6Xu/RS998vX75cpX3JkiVp7q9Vq1bYsWPHVz9837x58581fcvIkSPx7t076fm6ubkhJSUFU6dOTbPux48fpS+E9+/fp/nFmfpr+MtdC7t27cLz58+l6xcuXMD58+elcJ4jRw6ULFkS69evV/nCuXXrFg4dOoSGDRuq/bwiIiLStH1ZnyZea21Ij9foe6S+Dl5eXirtqT1SjRo1Uvs+AaBMmTLIly8f5s2bh9jY2DTLP9++dXV10zynJUuWqLzXAKBVq1a4fv06du7cmeb+1O0p+V5ZsmSBt7c3Jk2ahCZNmnxzvYYNGyIlJQVLly5VaV+4cCEUCoX0vkj9d/HixSrrffn3T8/PB/p5GaZH5d88efIEa9euxZMnT2Bvbw8AGDZsGA4ePPjDh7Z+9OgRvL29MWTIEIwZMwYXL17EgAEDYGBg8M2uQznas2cPYmJi0LRp068ud3V1hY2NDfz9/aVA0qZNGyxZsgQTJ05E8eLFVX4ZAkCnTp2wdetW9O7dG8HBwahUqRJSUlJw9+5dbN26VTouxL9p1KgRFixYgPr166N9+/YIDw/HsmXL4OzsjBs3bkjrlSlTBq1atYKXlxfevXsHV1dXHD9+HPfv3weg2sU+a9YsBAcHo3z58ujZsyeKFCmCiIgIXLlyBYcPH/7qF/P3aNCgAYoVK4YFCxbA09MT1apVg4eHB2bOnIlr166hbt260NfXx4MHD7Bt2zYsWrQIrVu3xvr167F8+XK0aNEC+fLlQ0xMDFauXAkzM7M0wcLZ2RmVK1dGnz59kJiYCC8vL2TLlg0jRoyQ1pk7dy4aNGiAChUqoHv37oiPj8eSJUtgbm7+Q+c0mTJlCk6cOIFGjRohd+7cCA8Px/Lly+Hg4CAdQVQTr7U2pMdr9D1KlCgBd3d3+Pr6IjIyEtWqVcOFCxewfv16NG/eXBrMri4dHR2sWrUKDRo0QNGiRdG1a1fkzJkTz58/R3BwMMzMzLB3714AQOPGjeHn5wdzc3MUKVIEZ8+exeHDh5EtWzaV+xw+fDi2b9+OP//8E926dUOZMmUQERGBPXv2wMfHR2V3ryZ9z+dnkyZNUKNGDYwdOxahoaEoUaIEDh06hN27d2PQoEFSD1PJkiXRrl07LF++HFFRUahYsSKOHDny1WO8pNfnA2mAVuYapTMAYufOndL1ffv2CQDC1NRU5aKnpyfc3NyEEELcuXNHAPjXy8iRI6X71NfXFxUqVFB53P79+wtXV9df8hw1pUmTJsLIyEjl+Alf6tKli9DX15em7SmVSuHo6PjV6YGpkpKSxOzZs0XRokWFoaGhsLS0FGXKlBGTJ08WUVFR0nr4Ymrv51avXi3y588vDA0NRaFChcTatWulqbWfi4uLE56ensLKykpkyZJFNG/eXNy7d08AELNmzVJZ9/Xr18LT01M4OjoKfX19YWdnJ2rVqiV8fX3/82/1teOopFq3bl2aKYu+vr6iTJkywtjYWGTNmlUUL15cjBgxQrx48UIIIcSVK1dEu3btRK5cuYShoaGwtbUVjRs3FpcuXZLuI3Uq5Ny5c8X8+fOFo6OjMDQ0FFWqVBHXr19PU8fhw4dFpUqVhLGxsTAzMxNNmjQR//zzj8o6qX/DL6cdp04VTT3mypEjR0SzZs2Evb29MDAwEPb29qJdu3bi/v37Krf73tf6v/zI9OQvpw1/67l9baqwEJp5jb7lW4+ZnJwsJk+eLPLkySP09fWFo6OjGD16tEhISEjznL+1vX3L1atXRcuWLUW2bNmEoaGhyJ07t3BzcxNHjhyR1nn//r3o2rWrsLa2FlmyZBH16tUTd+/eTfM3FkKId+/eiX79+omcOXMKAwMD4eDgINzd3aXPgtTpydu2bVO53fdO4f3W6/ilr/0tYmJixODBg4W9vb3Q19cX+fPnF3PnzlWZni2EEPHx8WLAgAEiW7ZswtTUVDRp0kQ8ffo0zfRkIb7v84HTk389hRDp1IenRQqFAjt37kTz5s0BAFu2bEGHDh1w+/btNIO+smTJAjs7OyQlJf3nVLps2bJJg+xy586NOnXqYNWqVdJyb29vTJs2TaWLnrTj2rVrKFWqFP766y906NBB2+X8sNDQUOTJkwdz585VOfIvEVFmkSl2/ZQqVQopKSkIDw+X5vd/ycDAAIUKFfru+6xUqVKaAVn379//4cFw9OPi4+PTDIj08vKCjo4OqlatqqWqiIhIEzJMUImNjVXZ7/j48WNcu3YNVlZWKFCgADp06IDOnTtj/vz5KFWqFN68eYMjR47AxcXlhwawDR48GBUrVsSMGTPg5uaGCxcuwNfXF76+vpp8WvQd5syZg8uXL6NGjRrQ09PDgQMHcODAAfTq1Svdp8cSEVE60/a+J01J3Vf65SV1n2tSUpKYMGGCcHJyEvr6+iJHjhyiRYsW4saNGz/8mHv37hXFihWTxlB8zzgH0rxDhw6JSpUqCUtLS6Gvry/y5csnJk2aJJKTk7Vd2k/7fIwKEVFmlCHHqBAREVHGkGmOo0JERES/HwYVIiIikq3fejCtUqnEixcvkDVr1t/q8N1ERESZmRACMTExsLe3h47Ov/eZ/NZB5cWLF5zVQURE9Jt6+vQpHBwc/nWd3zqopJ6M6unTpzAzM9NyNURERPQ9oqOj4ejoqHJSyW/5rYNK6u4eMzMzBhUiIqLfzPcM2+BgWiIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki09bRdARETy5TRqv7ZLIC0LndVIq4/PHhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki3ZBJVZs2ZBoVBg0KBB2i6FiIiIZEIWQeXixYtYsWIFXFxctF0KERERyYjWg0psbCw6dOiAlStXwtLSUtvlEBERkYxoPah4enqiUaNGqF279n+um5iYiOjoaJULERERZVx62nzwzZs348qVK7h48eJ3rT9z5kxMnjw5nasiIiIiudBaj8rTp08xcOBA+Pv7w8jI6LtuM3r0aERFRUmXp0+fpnOVREREpE1a61G5fPkywsPDUbp0aaktJSUFJ06cwNKlS5GYmAhdXV2V2xgaGsLQ0PBXl0pERERaorWgUqtWLdy8eVOlrWvXrihUqBBGjhyZJqQQERFR5qO1oJI1a1YUK1ZMpc3U1BTZsmVL005ERESZk9Zn/RARERF9i1Zn/Xzp2LFj2i6BiIiIZIQ9KkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWz8UVB4+fIhx48ahXbt2CA8PBwAcOHAAt2/f1mhxRERElLmpHVSOHz+O4sWL4/z58wgICEBsbCwA4Pr165g4caLGCyQiIqLMS+2gMmrUKEybNg1BQUEwMDCQ2mvWrIlz585ptDgiIiLK3NQOKjdv3kSLFi3StNva2uLt27caKYqIiIgI+IGgYmFhgZcvX6Zpv3r1KnLmzKmRooiIiIiAHwgqbdu2xciRI/Hq1SsoFAoolUqcPn0aw4YNQ+fOndOjRiIiIsqk1A4qM2bMQKFCheDo6IjY2FgUKVIEVatWRcWKFTFu3Lj0qJGIiIgyKT11b2BgYICVK1di/PjxuHXrFmJjY1GqVCnkz58/PeojIiKiTEztoJIqV65cyJUrlyZrISIiIlKhdlARQmD79u0IDg5GeHg4lEqlyvKAgACNFUdERESZm9pBZdCgQVixYgVq1KiB7NmzQ6FQpEddREREROoHFT8/PwQEBKBhw4bpUQ8RERGRRO1ZP+bm5sibN2961EJERESkQu2gMmnSJEyePBnx8fHpUQ8RERGRRO1dP25ubti0aRNsbW3h5OQEfX19leVXrlzRWHFERESUuakdVNzd3XH58mV07NiRg2mJiIgoXakdVPbv34/AwEBUrlw5PeohIiIikqg9RsXR0RFmZmbpUQsRERGRCrWDyvz58zFixAiEhoamQzlERERE/6P2rp+OHTviw4cPyJcvH0xMTNIMpo2IiNBYcUSZndOo/dougbQsdFYjbZdApFVqBxUvL690KIOIiIgorR+a9UNERET0K3xXUImOjpYG0EZHR//ruhxoS0RERJryXUHF0tISL1++hK2tLSwsLL567BQhBBQKBVJSUjReJBEREWVO3xVUjh49CisrKwBAcHBwuhZERERElOq7gkq1atWQN29eXLx4EdWqVUvvmoiIiIgAqHEcldDQUO7WISIiol9K7QO+aZK3tzdcXFxgZmYGMzMzVKhQAQcOHNBmSURERCQjak1PDgwMhLm5+b+u07Rp0+++PwcHB8yaNQv58+eHEALr169Hs2bNcPXqVRQtWlSd0oiIiCgDUiuo/NcxVNSd9dOkSROV69OnT4e3tzfOnTvHoEJERETqBZVXr17B1tY2XQpJSUnBtm3bEBcXhwoVKnx1ncTERCQmJkrX/+uYLkRERPR7++4xKl87doom3Lx5E1myZIGhoSF69+6NnTt3okiRIl9dd+bMmTA3N5cujo6O6VITERERycN3BxUhRLoUULBgQVy7dg3nz59Hnz594O7ujn/++eer644ePRpRUVHS5enTp+lSExEREcnDd+/6cXd3h7GxscYLMDAwgLOzMwCgTJkyuHjxIhYtWoQVK1akWdfQ0BCGhoYar4GIiIjk6buDytq1a9OzDolSqVQZh0JERESZl9pnT9ak0aNHo0GDBsiVKxdiYmKwceNGHDt2DIGBgdosi4iIiGRCq0ElPDwcnTt3xsuXL2Fubg4XFxcEBgaiTp062iyLiIiIZEKrQWX16tXafHgiIiKSuR8+hH5ISAgCAwMRHx8PIP1mBREREVHmpXZQeffuHWrXro0CBQqgYcOGePnyJQCge/fuGDp0qMYLJCIiosxL7aAyePBg6Onp4cmTJzAxMZHa27Rpg4MHD2q0OCIiIsrc1B6jcujQIQQGBsLBwUGlPX/+/AgLC9NYYURERERq96jExcWp9KSkioiI4MHYiIiISKPUDipVqlTBhg0bpOsKhQJKpRJz5sxBjRo1NFocERERZW5q7/qZM2cOatWqhUuXLiEpKQkjRozA7du3ERERgdOnT6dHjURERJRJqd2jUqxYMdy/fx+VK1dGs2bNEBcXh5YtW+Lq1avIly9fetRIREREmdQPHfDN3NwcY8eO1XQtRERERCrU7lE5ePAgTp06JV1ftmwZSpYsifbt2+P9+/caLY6IiIgyN7WDyvDhwxEdHQ0AuHnzJoYMGYKGDRvi8ePHGDJkiMYLJCIiosxL7V0/jx8/RpEiRQAAO3bsQJMmTTBjxgxcuXIFDRs21HiBRERElHmp3aNiYGCADx8+AAAOHz6MunXrAgCsrKyknhYiIiIiTVC7R6Vy5coYMmQIKlWqhAsXLmDLli0AgPv376c5Wi0RERHRz1C7R2Xp0qXQ09PD9u3b4e3tjZw5cwIADhw4gPr162u8QCIiIsq81O5RyZUrF/bt25emfeHChRopiIiIiCjVDx1HRalUIiQkBOHh4VAqlSrLqlatqpHCiIiIiNQOKufOnUP79u0RFhYGIYTKMoVCgZSUFI0VR0RERJmb2kGld+/eKFu2LPbv348cOXJAoVCkR11ERERE6geVBw8eYPv27XB2dk6PeoiIiIgkas/6KV++PEJCQtKjFiIiIiIVaveo9O/fH0OHDsWrV69QvHhx6Ovrqyx3cXHRWHFERESUuakdVFq1agUA6Natm9SmUCgghOBgWiIiItKoHzrXDxEREdGvoHZQyZ07d3rUQURERJTGDx3w7eHDh/Dy8sKdO3cAAEWKFMHAgQORL18+jRZHREREmZvaQSUwMBBNmzZFyZIlUalSJQDA6dOnUbRoUezduxd16tTReJHa4jRqv7ZLIC0LndVI2yUQEWVqageVUaNGYfDgwZg1a1aa9pEjR2aooEJERETapfZxVO7cuYPu3bunae/WrRv++ecfjRRFREREBPxAULGxscG1a9fStF+7dg22traaqImIiIgIwA/s+unZsyd69eqFR48eoWLFigA+jVGZPXs2hgwZovECiYiIKPNSO6iMHz8eWbNmxfz58zF69GgAgL29PSZNmoQBAwZovEAiIiLKvNQOKgqFAoMHD8bgwYMRExMDAMiaNavGCyMiIiL6oeOoAEB4eDju3bsHAChUqBBsbGw0VhQRERER8AODaWNiYtCpUyfY29ujWrVqqFatGuzt7dGxY0dERUWlR41ERESUSakdVHr06IHz589j//79iIyMRGRkJPbt24dLly7Bw8MjPWokIiKiTErtXT/79u1DYGAgKleuLLXVq1cPK1euRP369TVaHBEREWVuaveoZMuWDebm5mnazc3NYWlpqZGiiIiIiIAfCCrjxo3DkCFD8OrVK6nt1atXGD58OMaPH6/R4oiIiChzU3vXj7e3N0JCQpArVy7kypULAPDkyRMYGhrizZs3WLFihbTulStXNFcpERERZTpqB5XmzZunQxlEREREaakdVCZOnJgedRARERGlofYYladPn+LZs2fS9QsXLmDQoEHw9fXVaGFEREREageV9u3bIzg4GMCnQbS1a9fGhQsXMHbsWEyZMkXjBRIREVHmpXZQuXXrFv744w8AwNatW1G8eHGcOXMG/v7+WLdunabrIyIiokxM7aCSnJwMQ0NDAMDhw4fRtGlTAJ/O9/Py5UvNVkdERESZmtpBpWjRovDx8cHJkycRFBQkHY32xYsXyJYtm8YLJCIiosxL7aAye/ZsrFixAtWrV0e7du1QokQJAMCePXukXUJEREREmqD29OTq1avj7du3iI6OVjlkfq9evWBiYqLR4oiIiChzU7tHBQCEELh8+TJWrFiBmJgYAICBgQGDChEREWmU2j0qYWFhqF+/Pp48eYLExETUqVMHWbNmxezZs5GYmAgfH5/0qJOIiIgyIbV7VAYOHIiyZcvi/fv3MDY2ltpbtGiBI0eOaLQ4IiIiytzU7lE5efIkzpw5AwMDA5V2JycnPH/+XGOFEREREando6JUKpGSkpKm/dmzZ8iaNatGiiIiIiICfiCo1K1bF15eXtJ1hUKB2NhYTJw4EQ0bNtRkbURERJTJqb3rZ/78+ahXrx6KFCmChIQEtG/fHg8ePIC1tTU2bdqUHjUSERFRJqV2UHFwcMD169exZcsWXL9+HbGxsejevTs6dOigMriWiIiI6GepHVQAQE9PDx06dECHDh2ktpcvX2L48OFYunSpxoojIiKizE2toHL79m0EBwfDwMAAbm5usLCwwNu3bzF9+nT4+Pggb9686VUnERERZULfPZh2z549KFWqFAYMGIDevXujbNmyCA4ORuHChXHnzh3s3LkTt2/fTs9aiYiIKJP57qAybdo0eHp6Ijo6GgsWLMCjR48wYMAA/P333zh48KB0FmUiIiIiTfnuoHLv3j14enoiS5Ys6N+/P3R0dLBw4UKUK1cuPesjIiKiTOy7g0pMTAzMzMwAALq6ujA2NuaYFCIiIkpXag2mDQwMhLm5OYBPR6g9cuQIbt26pbJO06ZNNVcdERERZWpqBRV3d3eV6x4eHirXFQrFVw+vT0RERPQjvjuoKJXK9KyDiIiIKA21z/VDRERE9KtoNajMnDkT5cqVQ9asWWFra4vmzZvj3r172iyJiIiIZESrQeX48ePw9PTEuXPnEBQUhOTkZNStWxdxcXHaLIuIiIhk4ofO9aMpBw8eVLm+bt062Nra4vLly6hataqWqiIiIiK50GpQ+VJUVBQAwMrK6qvLExMTkZiYKF2Pjo7+JXURERGRdvzQrp/IyEisWrUKo0ePRkREBADgypUreP78+Q8XolQqMWjQIFSqVAnFihX76jozZ86Eubm5dHF0dPzhxyMiIiL5Uzuo3LhxAwUKFMDs2bMxb948REZGAgACAgIwevToHy7E09MTt27dwubNm7+5zujRoxEVFSVdnj59+sOPR0RERPKndlAZMmQIunTpggcPHsDIyEhqb9iwIU6cOPFDRfTr1w/79u1DcHAwHBwcvrmeoaEhzMzMVC5ERESUcak9RuXixYtYsWJFmvacOXPi1atXat2XEAL9+/fHzp07cezYMeTJk0fdcoiIiCgDUzuoGBoafnUQ6/3792FjY6PWfXl6emLjxo3YvXs3smbNKgUdc3NzGBsbq1saERERZTBq7/pp2rQppkyZguTkZACfzu/z5MkTjBw5Eq1atVLrvry9vREVFYXq1asjR44c0mXLli3qlkVEREQZkNpBZf78+YiNjYWtrS3i4+NRrVo1ODs7I2vWrJg+fbpa9yWE+OqlS5cu6pZFREREGZDau37Mzc0RFBSEU6dO4caNG4iNjUXp0qVRu3bt9KiPiIiIMrEfPuBb5cqVUblyZU3WQkRERKRC7aCyePHir7YrFAoYGRnB2dkZVatWha6u7k8XR0RERJmb2kFl4cKFePPmDT58+ABLS0sAwPv372FiYoIsWbIgPDwcefPmRXBwMI8cS0RERD9F7cG0M2bMQLly5fDgwQO8e/cO7969w/3791G+fHksWrQIT548gZ2dHQYPHpwe9RIREVEmonaPyrhx47Bjxw7ky5dPanN2dsa8efPQqlUrPHr0CHPmzFF7qjIRERHRl9TuUXn58iU+fvyYpv3jx4/SAdvs7e0RExPz89URERFRpqZ2UKlRowY8PDxw9epVqe3q1avo06cPatasCQC4efMmD4dPREREP03toLJ69WpYWVmhTJkyMDQ0hKGhIcqWLQsrKyusXr0aAJAlSxbMnz9f48USERFR5qL2GBU7OzsEBQXh7t27uH//PgCgYMGCKFiwoLROjRo1NFchERERZVo/fMC3QoUKoVChQpqshYiIiEjFDwWVZ8+eYc+ePXjy5AmSkpJUli1YsEAjhRERERGpHVSOHDmCpk2bIm/evLh79y6KFSuG0NBQCCFQunTp9KiRiIiIMim1B9OOHj0aw4YNw82bN2FkZIQdO3bg6dOnqFatGv7888/0qJGIiIgyKbWDyp07d9C5c2cAgJ6eHuLj45ElSxZMmTIFs2fP1niBRERElHmpHVRMTU2lcSk5cuTAw4cPpWVv377VXGVERESU6ak9RsXV1RWnTp1C4cKF0bBhQwwdOhQ3b95EQEAAXF1d06NGIiIiyqTUDioLFixAbGwsAGDy5MmIjY3Fli1bkD9/fs74ISIiIo1SK6ikpKTg2bNncHFxAfBpN5CPj0+6FEZERESk1hgVXV1d1K1bF+/fv0+veoiIiIgkag+mLVasGB49epQetRARERGpUDuoTJs2DcOGDcO+ffvw8uVLREdHq1yIiIiINEXtwbQNGzYEADRt2hQKhUJqF0JAoVAgJSVFc9URERFRpqZ2UAkODk6POoiIiIjSUDuoVKtWLT3qICIiIkpD7TEqAHDy5El07NgRFStWxPPnzwEAfn5+OHXqlEaLIyIiosxN7aCyY8cO1KtXD8bGxrhy5QoSExMBAFFRUZgxY4bGCyQiIqLM64dm/fj4+GDlypXQ19eX2itVqoQrV65otDgiIiLK3NQOKvfu3UPVqlXTtJubmyMyMlITNREREREB+IGgYmdnh5CQkDTtp06dQt68eTVSFBERERHwA0GlZ8+eGDhwIM6fPw+FQoEXL17A398fw4YNQ58+fdKjRiIiIsqk1J6ePGrUKCiVStSqVQsfPnxA1apVYWhoiGHDhqF///7pUSMRERFlUmoHFYVCgbFjx2L48OEICQlBbGwsihQpgixZsqRHfURERJSJqb3r56+//sKHDx9gYGCAIkWK4I8//mBIISIionShdlAZPHgwbG1t0b59e/z99988tw8RERGlG7WDysuXL7F582YoFAq4ubkhR44c8PT0xJkzZ9KjPiIiIsrE1A4qenp6aNy4Mfz9/REeHo6FCxciNDQUNWrUQL58+dKjRiIiIsqk1B5M+zkTExPUq1cP79+/R1hYGO7cuaOpuoiIiIh+7KSEHz58gL+/Pxo2bIicOXPCy8sLLVq0wO3btzVdHxEREWViaveotG3bFvv27YOJiQnc3Nwwfvx4VKhQIT1qIyIiokxO7aCiq6uLrVu3ol69etDV1VVZduvWLRQrVkxjxREREVHmpnZQ8ff3V7keExODTZs2YdWqVbh8+TKnKxMREZHG/NAYFQA4ceIE3N3dkSNHDsybNw81a9bEuXPnNFkbERERZXJq9ai8evUK69atw+rVqxEdHQ03NzckJiZi165dKFKkSHrVSERERJnUd/eoNGnSBAULFsSNGzfg5eWFFy9eYMmSJelZGxEREWVy392jcuDAAQwYMAB9+vRB/vz507MmIiIiIgBq9KicOnUKMTExKFOmDMqXL4+lS5fi7du36VkbERERZXLfHVRcXV2xcuVKvHz5Eh4eHti8eTPs7e2hVCoRFBSEmJiY9KyTiIiIMiG1Z/2YmpqiW7duOHXqFG7evImhQ4di1qxZsLW1RdOmTdOjRiIiIsqkfnh6MgAULFgQc+bMwbNnz7Bp0yZN1UREREQE4CeDSipdXV00b94ce/bs0cTdEREREQHQUFAhIiIiSg8MKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbWg0qJ06cQJMmTWBvbw+FQoFdu3ZpsxwiIiKSGa0Glbi4OJQoUQLLli3TZhlEREQkU3rafPAGDRqgQYMG2iyBiIiIZEyrQUVdiYmJSExMlK5HR0drsRoiIiJKb7/VYNqZM2fC3Nxcujg6Omq7JCIiIkpHv1VQGT16NKKioqTL06dPtV0SERERpaPfatePoaEhDA0NtV0GERER/SK/VY8KERERZS5a7VGJjY1FSEiIdP3x48e4du0arKyskCtXLi1WRkRERHKg1aBy6dIl1KhRQ7o+ZMgQAIC7uzvWrVunpaqIiIhILrQaVKpXrw4hhDZLICIiIhnjGBUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLVkElWXLlsHJyQlGRkYoX748Lly4oO2SiIiISAa0HlS2bNmCIUOGYOLEibhy5QpKlCiBevXqITw8XNulERERkZZpPagsWLAAPXv2RNeuXVGkSBH4+PjAxMQEa9as0XZpREREpGVaDSpJSUm4fPkyateuLbXp6Oigdu3aOHv2rBYrIyIiIjnQ0+aDv337FikpKciePbtKe/bs2XH37t006ycmJiIxMVG6HhUVBQCIjo5Ol/qUiR/S5X7p95Fe29b34jZI3AZJ29JjG0y9TyHEf66r1aCirpkzZ2Ly5Mlp2h0dHbVQDWUG5l7aroAyO26DpG3puQ3GxMTA3Nz8X9fRalCxtraGrq4uXr9+rdL++vVr2NnZpVl/9OjRGDJkiHRdqVQiIiIC2bJlg0KhSPd6M5Po6Gg4Ojri6dOnMDMz03Y5lAlxGyRt4zaYfoQQiImJgb29/X+uq9WgYmBggDJlyuDIkSNo3rw5gE/h48iRI+jXr1+a9Q0NDWFoaKjSZmFh8QsqzbzMzMz4BiWt4jZI2sZtMH38V09KKq3v+hkyZAjc3d1RtmxZ/PHHH/Dy8kJcXBy6du2q7dKIiIhIy7QeVNq0aYM3b95gwoQJePXqFUqWLImDBw+mGWBLREREmY/WgwoA9OvX76u7ekh7DA0NMXHixDS72oh+FW6DpG3cBuVBIb5nbhARERGRFmj9yLRERERE38KgQkRERLLFoEJERESyxaBCREREssWgQkRERLLFoEJERESyxaBCREREssWgQkRERLLFoEJERESyxaBCvyWlUqntEoiI6BdgUKHfko7Op0337du3AACeCYJ+tS/DMrdB0oYvt8OM+COOQYV+W4sWLULz5s3x8OFDKBQKbZdDmYyOjg6ioqIQGBgIANwGSSt0dHQQGRmJuXPn4v3799KPuIwk4z0jyrC+/MWqr68PY2NjGBgYaKkiysyUSiXmz58PDw8P7Nu3T9vlUCZ26NAhLFiwAEuXLtV2KemCZ0+m3050dDTMzMwAAFFRUTA3N9dyRZRZKJVKlV+sd+7cwerVqzF79mzo6upqsTLKTFJSUlS2t+TkZGzZsgXt2rXLkNshgwr9VgYPHoyUlBSMHj0aOXLk0HY5lAlFRkYiMjISjo6OKl8KX355EP2ML0Pxl969e4fTp0+jYsWKsLa2ltoz4nbIXT8ka1/maAcHB2zYsCHDvRHp9yCEwKhRo1C+fHmEhoaqLOM2ST/j5cuXePHiBd68eQPg09iTf+tH2Lp1K5o3b47jx4+rtGfE7ZA9KiQbqb8EhBBQKBTf/EXx/v17WFpaaqFCymj+61fr19YJCwvDuHHjsG7dugz5pUC/3tq1a7Fs2TI8ffoU+fLlQ+XKlTFnzhyVdb7WU+Ll5YV+/fpBT0/vV5b7yzGokFakhhHg0xtQCAE9PT08f/4cO3fuRNeuXWFqagrg0+4eS0tLTJgwIc1tiX7U5wHk6NGjePLkCZydnZE3b17Y29urrBMVFQWlUpkmIGfEbnb6tfbt2wc3NzcsX74cJiYmePToEebMmYOKFSti/fr1yJYtm/SZ9/btW4SEhMDV1VXlPj5+/Jihwwp3/dAvkZqHo6OjER8fD4VCgUOHDiEkJAS6urrQ09NDWFgYSpUqhRcvXkghJS4uDvr6+li4cCEiIiIYUkgjhBBSSBk1ahS6dOmCefPmoVevXhg2bBguXrwI4FP3e2JiIiZMmIDSpUvj3bt3KvfDkEI/6+LFi2jUqBG6dOkCNzc3jBgxAoGBgbhx4wY6dOgA4NPU9+TkZPj5+aFixYo4deqUyn1k5JACMKjQL/Tq1SsUL14cx48fx8aNG1G/fn38888/AD7tzilatChatGiB6dOnS7cxNTXFiBEj8ODBA1hZWTGkkEakbkfz5s3DX3/9hU2bNuHWrVto2bIl9u7di3HjxuHs2bMAAAMDA5QqVQq1atWChYWFFqumjOjx48d4+fKlSlu5cuWwZ88eXL58GT179gTw6XAMjRs3xvTp09P0qGR4gugX6tq1qzAzMxM6Ojpi5cqVUntSUpLYsmWLSElJkdqUSqU2SqRM4vXr16Jly5ZizZo1Qggh9uzZI8zMzETv3r1FqVKlRK1atcS5c+eEEKrb4sePH7VSL2VMgYGBInv27GLz5s1SW+r25u/vL5ydncXFixfT3C45OfmX1aht7FGhXyL1sM6enp6IiYmBgYEB7OzskJCQAODTrwU3NzeVQYvsPaH0ZGtrixEjRqB+/fq4evUqPD09MW3aNHh7e6NVq1Y4d+4cPD09cfnyZZVtkbt7SJMKFy6M6tWrw8/PD0eOHAHwv8++kiVLIjw8XDpVyOcy+u6ezzGo0C+RGkAcHR1x6tQpuLu7o23btti9ezfi4+PTrJ8Rz1dB2vOt7alUqVLIkSMHDhw4ABcXF/Tq1QsAYGVlBVdXVzRp0gSlSpX6laVSJuPo6IjevXsjMjISCxcuxJ49e6RlOXLkQJ48ebRYnTxknkhGWiH+f/Dry5cvkZycjFy5csHW1hYVK1ZEQkICunfvjnXr1qFx48YwMjKCj48PateuDWdnZ22XThmE+Gzg7KpVqxAeHg4DAwMMGzZMOv1CYmIinj9/jtDQUBQsWBCHDh1C06ZN0b9//3+dKk/0M1JnjVWvXh3Lly/HmDFjMHLkSAQGBsLFxQVbt26FQqFAnTp1tF2qVnF6MqW7gIAATJo0Ca9fv0ajRo3QokULNGnSBADQtWtX7Ny5E0OHDsXr16/h7e2NmzdvokiRIlqumjKaiRMnwsvLC+XKlcOFCxdQvnx5+Pn5wc7ODnv37sW0adPw/v176OvrQwiBGzduQE9PjzPNKF2kblcBAQFYvnw5Dh06hLt37yI4OBhLly6Fo6MjLCws4O/vD319/Uw9FZ5BhdLV7du3Ua9ePQwePBgmJibYtGkTDA0N4e7ujo4dOwIABg4ciCtXriAxMRG+vr4oWbKkdoumDOHzXpCPHz/C3d0d/fv3R6lSpRAaGopGjRrBzs4OO3fuhI2NDfbv34+QkBDExsZi5MiR0NPTy9RfDqQZqYFEfHHsKF1dXQQEBKBz585YsGCBtNsR+LS96ujoqGy/mWlMypcYVCjd3L17F9u2bUN8fDxmzJgBALh58yYmTJiA6OhodO3aVQorr169gqmpKbJmzarNkimD+Dyk3LlzB9HR0VixYgUmTJgAJycnAJ+mhdapUwfZs2fHrl27YGNjo3IfDCn0sz7fDt++fQuFQoFs2bIB+PSZV7p0aUyYMAG9e/eWbvNlDx579BhUKB0IIfD+/Xs0btwY//zzD5o0aQI/Pz9p+Y0bNzBhwgTEx8ejbdu26Nq1qxarpYxs+PDhUtf569evERAQgAYNGkgf/I8fP0aDBg0ghMDp06dVTu5G9DM+DxhTp07Frl27EB0dDWtra0yfPh01a9bE8+fPkTNnTi1XKn8cHUYap1AoYGVlhZkzZ6Jo0aK4cuUKgoKCpOUuLi6YOnUqkpOTpTcvkSZ8Prtn3759OHjwIBYvXozly5cjT548GDt2LK5fvy4dKTlPnjzYt28fSpYsyfNHkUalhpQpU6Zg0aJF0vR3a2trdOjQAevXr0/Ti0dfxx4V0ohvdU8eP34cY8aMgZ2dHTw9PVGzZk1p2e3bt2Fubg4HB4dfWSplAgEBAThz5gyyZcuG0aNHAwBiY2NRunRpmJmZYdWqVShRokSabZa7e0iT3r17h7p168LT0xPdunWT2nv16oW9e/ciODgYhQoV4u6d/8AeFfppqW+yM2fOYMGCBRg/fjxOnz6N5ORkVKtWDVOmTMGrV6+wdOlSHDt2TLpd0aJFGVJI4+Lj4zF+/HgsWLAAt2/fltqzZMmCK1euICYmBh4eHtL5fD7HkEKa9PHjR7x9+1bqrUs9wKWvry/s7e2xcOFCADy45X9hUKGf8vkUuwYNGuD06dPYs2cPxowZg+nTpyMpKQm1atXClClT8O7dO0ydOhUnT57UdtmUgRkbG+PkyZOoXbs2Ll++jD179iAlJQXA/8LK3bt3sWLFCi1XShnJ13ZOZM+eHXZ2dlizZg0AwMjICElJSQAAZ2dnBpTvxKBCPyW1J2XAgAFYsGABduzYgW3btuHy5cvYsmULxo0bJ4WVUaNGQV9fn0daJI35fEyKEEL6srCyssLGjRthaWmJuXPnIjAwUFpmamqKV69ewdfXVys1U8ajVCql0PHixQuEh4fjw4cPAIBJkybh7t270sye1IMMPnv2jCe5/E4co0I/JPWNqVAosHz5cly7dg2+vr54/PgxateujcqVK8PMzAzbtm2Dh4cHxowZA0NDQ3z48AEmJibaLp8ygM+nfi5ZsgTXr1/Ho0ePMGjQIJQuXRoODg548+YNmjVrBl1dXYwZMwb16tVTOcIsx6TQz/D394erqyvy5csHABg9ejQCAwMRFhaG2rVro2nTpujQoQNWrlyJqVOnIlu2bChWrBgePnyIyMhI6aCC9O8YVOi7pH4pfB40rl27hpIlSyI6OhpPnz6Fs7Mz6tevjzx58mDNmjWIioqSjjDbpUsXTJ8+nYPG6Kd9uQ2NHj0aq1evRq9evfDs2TOcPXsWzZo1Q69eveDs7Iw3b96gZcuWePPmDdatWwdXV1ctVk8ZxYEDB9C4cWOMHDkSgwYNwoEDBzBixAh4eXnh3bt3uHLlCgIDAzF+/Hj07t0bN2/ehJeXF3R0dGBpaYkZM2bwoILfK13PzUwZyqNHj0S7du3EP//8I7Zu3SoUCoW4cOGCdErymzdvikKFConz588LIYR4+PChaNy4sRgzZox48uSJNkunDCYlJUUIIYSfn5/IkyePuHz5shBCiJMnTwqFQiHy588vBg4cKB49eiSEEOLly5eiV69e4uPHj1qrmTKepUuXCgcHBzF16lTRr18/sXLlSmnZ06dPxZQpU4STk5M4ePDgV2+fnJz8q0r9rbHPib5bQkICTp48iS5duuDatWtYu3YtypUrJ+0GEkLg48ePOHv2LIoWLYoNGzYAAIYNG8ZjVNBP69SpE2xsbLBgwQLo6OggOTkZBgYG6N27N0qXLo1du3aha9euWLVqFV69eoVp06ZBR0cHPXv2ROHChaXBs/wFSz8rKSkJBgYG8PT0hImJCUaPHo2YmBhMmzZNWsfBwQGdO3fGoUOHcOnSJdSrVy/NyS252+c7aTsp0e8h9Resj4+P0NHRESVKlBBXr15VWScqKkp06dJF5MuXTzg5OQkbGxvply7Rz4iKihKTJ08WVlZWYtKkSVL78+fPxevXr8XLly9F2bJlxfz586X17e3tRY4cOcSiRYuEEELq+SPSlJkzZ4rw8HDh7+8vTExMRMOGDcX9+/dV1mnTpo1o2bKllirMGDjrh/6TEAI6OjoQQsDe3h7z58/Hx48fMW7cOJw6dUpaz8zMDPPmzcPy5csxceJEnD9/HqVLl9Zi5ZQRxMTEwMzMDH369MG4cePg5eWFiRMnAgDs7e1ha2uLly9f4v3799L4k+fPn6Nu3bqYMGECPD09AfBYFfTzxGdDOtevX4+pU6fiwYMHaN++PRYuXIgrV67Ax8cH9+7dAwBER0fj8ePHyJUrl7ZKzhDY70T/Svz/wMWjR4/i+PHjGDRoEJo0aYLatWvDzc0Ns2bNwpgxY1CxYkUAn046WLduXS1XTRnFiBEjsGLFCjx8+BA2Njbo2LEjhBCYOnUqAGDy5MkAPoUZXV1dnD59GkIIzJo1CyYmJtKUUO7uIU1IDbtHjhzB1atX4evrK3329erVC8nJyZg8eTIOHjyI0qVLIy4uDklJSZgzZ442y/79abM7h+Qttat8+/btwtzcXIwePVpcvHhRWn7jxg1RpEgR0bhxY/HXX3+JSZMmCYVCIZ4+fcpudtKI69evi6pVq4qCBQuKN2/eCCGECA8PF/PnzxcWFhZiwoQJ0rr9+vUT+fLlEw4ODsLV1VUkJSUJIbjLhzTr2LFjonjx4iJbtmxi165dQgghEhMTpeWrV68WWbJkEaVLlxYbNmyQBnBz4OyP4/Rk+lcXLlxA/fr1MXv2bPTs2VNqj46OhpmZGe7cuYOePXsiPj4eUVFR2Lp1K3f3kEacPXsWb968QZEiRdCmTRvExsZKZzh+8+YN/Pz8MHXqVOlkb8CnKfMKhQLFixeHjo4OPn78yAGL9FPEF9PhY2NjMXfuXPj6+qJ8+fLYtGkTjI2NkZycDH19fQDAggULcObMGWzbtg0KhYI9ej+JQYX+1dKlS7Fz504cOXIEUVFROHr0KP766y/cuXMHw4YNQ7du3RAeHo6oqCiYm5vD1tZW2yVTBtG5c2e8ePEChw8fRmhoKFq3bo2YmJg0YWXatGno168fpkyZonJ7fjmQJi1btgwODg5o1qwZ4uPjMW/ePOzcuRPVq1fHjBkzYGRkpBJWUgPOl0GH1MfBtPSv7OzscPnyZcycOROtW7fG2rVrYWRkhEaNGqFHjx64f/8+bG1tkT9/foYU0qhly5bh2bNnWLp0KZycnLBp0yaYm5ujUqVKePv2LWxsbNCpUydMmDAB06ZNw+rVq1Vuz5BCmvLmzRscPXoUffv2xcGDB2FsbIwhQ4agcePGOHPmDMaOHYuEhATo6+vj48ePAMCQokHsUSFJ6psqNjYWWbJkAQC8fv0aS5YswdatW1GzZk106dIFf/zxB16/fo2mTZti3bp1KFq0qJYrp4wmtTdk8eLFuHr1KhYsWABLS0vcvXsXnTt3RlRUlNSz8urVKxw/fhytWrXibh7SiC+PdwIA169fx+LFi3H48GH4+PigQYMGiIuLw5w5c3D48GEULlwYy5cvl87lQ5rDHhWSKBQK7N+/H+3atUP16tWxbt066OnpYdq0aTh//jx8fHzg6uoKHR0dLFmyBHFxcexFoXSR2htSvXp1nDhxAvv37wcAFCxYEH5+frC0tETVqlXx+vVr2NnZoU2bNtDT05N+zRL9jNSQ8urVK6mtRIkSGDhwIGrUqIHevXvj4MGDMDU1xYgRI/DHH39AR0dH2u1DGqalQbwkQ6dPnxZGRkZi+PDhon79+sLFxUV4eHiIkJAQaZ3g4GDRq1cvYWVlleaAb0Q/KvWAgl/j4+MjChQoIO7duye13bt3Tzg5OYm2bdv+ivIok/h8O9y8ebPImzevykxHIYS4du2aaNasmciVK5c4duyYEEKI+Ph4aXbZv23L9GPYo0IAgLCwMAQFBWH69OmYM2cODhw4gF69euHGjRuYOXMmHj16hLi4OJw9exbh4eE4fvw4SpYsqe2yKQP4vJv9woULOHPmDI4fPy4tb9q0KcqXL4/g4GCprUCBAjhx4gT++uuvX14vZUyJiYnSdpiUlIR8+fKhUKFC8PT0xOXLl6X1SpQogebNm+Pp06eoW7cuzpw5AyMjI2lMype7jOjn8S+aCS1duhR///23dP3evXto06YN1qxZAyMjI6nd09MTHTp0wO3btzFnzhxERkZi+PDhWL9+PYoVK6aN0imD+fyDfcyYMejSpQu6desGd3d3tGnTBtHR0ciRI4e0/z85OVm6raOjI3R1dZGSkqKt8imDOHDgAPz8/AAAPXv2RM2aNVG2bFkMHToUdnZ28PDwwKVLl6T1c+XKhbZt22L+/PkoX7681M6Bs+lE21069Gs9fvxYtG/fXjx48EClfdSoUcLW1la0bNlSOrBWKm9vb1GwYEExYMAAHrSI0sW8efNEtmzZxPnz50VKSoqYMWOGUCgU4tSpU9I6lSpVEh4eHlqskjKqdu3aCScnJ1GvXj1hbW0trl+/Li07evSoaN68uShWrJg4cOCAePz4sWjevLkYOnSotA7Pyp2+GFQyobi4OCGEEOfOnRPbt2+X2idMmCCKFy8uxo0bJ16/fq1ym5UrV4rHjx//yjIpk1AqlcLd3V34+voKIYTYsWOHsLCwED4+PkIIIWJiYoQQQhw4cEA0bdpU3LhxQ2u1UsZVsmRJoVAoVE56merkyZOiU6dOQqFQiAIFCggXFxfpRxuPfJz+OJcvEzI2NkZkZCRmzpyJ58+fQ1dXF82bN8fkyZORnJyM/fv3QwiBgQMHwsbGBgDQo0cPLVdNGVVCQgLOnz+P6tWr49ixY3B3d8fcuXPh4eGBjx8/Ys6cOahQoQJcXV0xZcoUXLhwAcWLF9d22ZRBJCUlISEhAc7OzsiVKxe2bNmCnDlzom3bttJhGipXrozy5cujZ8+eSE5ORrVq1aCrq8sjH/8iHKOSCSkUClhYWGDo0KHIkycPvLy8EBAQAACYMWMG6tevj6CgIMyYMQNv377VcrWUkdy4cQPPnj0DAAwePBjHjx+HsbEx2rdvj7/++gsNGzbEwoULpZMJvn//HpcuXcK9e/dgaWkJPz8/5M6dW5tPgTIYAwMDmJmZYdu2bdi9ezfKlSuHOXPmYPPmzYiJiZHWS0hIQJUqVVCzZk1pbBRDyq/BoJIJiU+7/FClShUMHjwYlpaWWLx4sUpYcXV1xdWrV1VOa070o4QQuH//PmrUqIE1a9agd+/eWLRoESwtLQEArq6uCAsLQ/ny5VGhQgUAwIsXL9ClSxdERkaiX79+AIB8+fKhdu3aWnselPEIIaBUKqXr69evR8WKFbFw4UJs2LABT548Qc2aNfHnn39K6wM88vGvxCPTZkKpR/2MioqCiYkJbty4genTp+P9+/cYOHAgmjdvDuDTYaNTd/0QacLKlSsxYsQIJCQkYPfu3ahbt650ROQtW7ZgypQpEEJAT08PxsbGUCqVOHPmDPT19XnuHvppERERsLKyUmlL3f62bduGoKAg+Pr6AgB69eqFY8eOISUlBVZWVjh9+jSPOqsl7FHJZD5+/AhdXV2EhoaievXqOHToEMqUKYNhw4bBxsYGkydPxr59+wCAIYU0JvUXq6OjIwwNDWFmZoZz584hNDRUmtLZpk0bbNiwAVOmTIGbmxtGjhyJc+fOSedPYUihn7Fo0SKUK1dOZXcOACmkdOnSBSVKlJDafX19sWLFCixZsgTnzp2DgYEBj3ysLdoZw0u/wrdGo4eEhIjs2bOLHj16qEyrO3bsmOjUqZMIDQ39VSVSBvflNpiUlCTi4+OFt7e3yJkzpxgzZsx/bm+c+kk/a8WKFcLQ0FBs3LgxzbInT56I4sWLi6VLl0ptX9vmuB1qD3f9ZFDi/7szz549izt37iAkJASdO3dGjhw5sH79ely6dAnr169Pc4bPhIQElYO+Ef2oz484GxERgZiYGJWBsF5eXpg3bx66d++Orl27wsnJCU2aNMHYsWPh6uqqrbIpg1m5ciX69+8PPz8//Pnnn4iMjERcXBwSEhJga2uLrFmz4sGDB8ifP7+2S6VvYFDJwHbs2IFevXpJJ2978+YN2rRpg5EjRyJr1qzaLo8ysM9DypQpU3Do0CHcunULbm5uaNGiBRo0aADgU1jx8vJCsWLF8O7dOzx58gShoaE8uRtpxKNHj+Ds7Aw3Nzds3rwZt27dQt++ffHmzRuEhYWhRo0a6NOnDxo3bqztUulfcG5VBnXr1i0MHjwY8+fPR5cuXRAdHQ0LCwsYGxszpFC6Sw0pEyZMgK+vL+bOnQsnJyf07t0bDx48QGRkJNq1a4dBgwbB2toa169fR0JCAk6ePCmdBZlTP+ln2djYYPbs2ZgwYQKGDRuGQ4cOoUqVKmjWrBmio6Oxfft2jBs3DtbW1uzFkzNt7ncizTh69Kh4+PBhmrYKFSoIIYS4c+eOyJ07t+jRo4e0/OHDh9znSunq6NGjomjRouLEiRNCCCHOnDkjDAwMRJEiRUT58uXFtm3bpHU/PzUDT9NAmpSQkCDmzZsndHR0RLdu3URSUpK07NKlS6JgwYJi2bJlWqyQ/gtn/fzGhBC4evUqGjRoAG9vb4SFhUnLnj9/DiEEYmNjUb9+fdStWxcrVqwAAAQFBcHb2xvv37/XVumUAYkv9iLnzJkTffr0QZUqVXDo0CE0btwYvr6+CAoKwsOHD7F48WKsXr0aAFR6T9iTQppkaGiI3r17Y8eOHejRowf09fWlbbVMmTIwMjLC06dPtVwl/RsGld+YQqFAqVKlMH/+fGzduhXe3t549OgRAKBRo0Z4/fo1zMzM0KhRI/j6+krd8YGBgbhx4wane5LGKJVKaUD2o0ePEBcXh/z586Ndu3ZISEjAokWLMGDAAHTq1An29vYoWrQoQkJCcOfOHS1XTpmBqakpGjRoIB1MMHVbDQ8Ph7GxMYoWLarN8ug/8KfLbyx1P76npycAYO7cudDV1UWPHj2QJ08ejB8/HjNmzMDHjx/x4cMHhISEYNOmTVi1ahVOnTolHRWU6Gd8PnB2woQJOHv2LIYPH44aNWrAysoKcXFxePnyJUxMTKCjo4PExEQ4OTlhxIgRqF+/vparp4xIfDaTMZWhoaH0/5SUFLx9+xY9e/aEQqFAu3btfnWJpAYGld9Yao/IoUOHoKOjg+TkZHh5eSEhIQEjR46Em5sb4uPjMWPGDGzfvh3Zs2eHgYEBgoODUaxYMS1XTxnF5yFlxYoV8PX1RalSpaSZO4mJibCyssKpU6ekAbPv3r3DmjVroKOjoxJ0iH5EWFgYIiIikC1bNtjZ2f3rEWSTk5Ph5+eHTZs2ISIiAufOnZPO3cNeZnni9OTfXGBgoHQiN1NTUzx48ACLFy9G3759MXLkSNjY2CAmJgbHjx+Hk5MTbG1tYWtrq+2y6Tf3Zbi4f/8+mjdvjtmzZ6NJkyZp1rt48SLGjRuH2NhYWFlZISAgAPr6+gwp9NM2bNiA+fPnIzw8HNbW1ujfv7/UU5Lqy+0sKCgIt2/fRr9+/TjL7DfAoPIbUyqV6NChAxQKBTZu3Ci1L1myBCNGjICnpyf69u2LvHnzarFKymhatmyJMWPGoGzZslLbtWvXUL9+fRw/fhwFCxb86kEEExISIISAkZERFAoFvxzop23YsAGenp7S4fFnzJiBR48e4fTp09K2lRpSIiMjcejQIbi5uancB3tS5I8/ZX5jqb8QUrvYk5KSAAD9+/eHh4cH1q5di8WLF6vMBiL6Webm5nBxcVFpMzIywvv373Hr1i2pLfX8PmfPnsWOHTugo6MDY2NjKBQKKJVKhhT6KZcuXcLUqVOxdOlSdOvWDcWLF8fgwYPh7OyMM2fO4Pbt24iOjpZ2i69fvx59+/bFX3/9pXI/DCnyx6DyG3rx4oX0/4IFC2Lv3r0IDw+HgYEBkpOTAQAODg4wMTFBcHAwjI2NtVUqZSDPnz8HAKxduxYGBgZYvHgxDh06hKSkJDg7O6NNmzaYO3cuDh8+DIVCAR0dHaSkpGD69OkIDg5WGTfA3T30sxITEzFo0CA0atRIaps0aRKOHDmCdu3aoXPnzmjbti0iIiKgr6+Phg0bYtiwYRw4+xvirp/fzPXr19GvXz+0b98effr0QVJSEmrWrIm3b9/i2LFjsLOzAwCMHDkSRYsWRePGjdOc1pxIXT179gQAjB49WtqV6OLigrdv32Lz5s2oWrUqTp48iYULF+LmzZvo0KEDDAwMcOTIEbx58wZXrlxhDwpplFKpxJs3b5A9e3YAQOfOnXH48GHs2bMHjo6OOH78OKZNm4aRI0eiffv2KmNWuLvn98KfNb8ZExMTWFhYYPv27Vi3bh0MDAywYsUK2NjYoHDhwmjevDnq1q2LRYsWoWzZsgwppBEuLi44ePAgvL29ERISAgC4ceMGChYsiA4dOuDEiROoUqUKpkyZgs6dO8PPzw9Hjx5Frly5cPnyZWnAIpGm6OjoSCEFAIYNG4bz58+jbNmyyJ49Oxo0aICIiAi8fv06zVRlhpTfC3tUfkMhISEYM2YMXr16hZ49e6JTp05ISUnBvHnzEBYWBiEE+vfvjyJFimi7VMpA1qxZgwkTJqBt27bo2bMnChYsCACoWrUqHj9+DH9/f1StWhUA8OHDB5iYmEi35cBZ+tWePXuGjh07YtiwYTzp4G+OQeU3cOXKFbx8+VJlX2xISAjGjRuH0NBQ9O/fHx06dNBihZSRfT61c/Xq1ZgwYQLatWuXJqyEhYVhw4YNqFChgsp4lK8dfItIHZ9vQ6n/T/33zZs3sLGxUVk/Li4O7dq1Q1RUFI4ePcoelN8cg4rMxcTEoFGjRtDV1cWIESPQoEEDaVloaCjq168PExMT9OjRA3379tVipZTRfOsYJytXrsTkyZPRpk0b9OrVSworNWvWxOnTp3Hu3DmUKlXqV5dLGdTXtsPUtoCAAGzatAmLFi2Cvb094uPjsXv3bvj5+eH58+e4ePEi9PX1OSblN8cxKjKVmh+zZs2KOXPmQE9PD0uXLsX+/fuldZycnFCjRg28evUKR44cQWRkpJaqpYzm8y+HM2fOIDg4GNevXwfwaWDt+PHjsXnzZvj6+uLevXsAgKNHj6JHjx5ppi4T/ahTp05JJwwcMmQIZs2aBeDT+JQtW7agc+fOqF27Nuzt7QF8OqHl48ePkTdvXly6dAn6+vr4+PEjQ8pvjj0qMpPanZn6CyD1C+P8+fMYNWoUTE1N0adPH2k30NChQ5E3b160bNkSOXLk0HL1lBF83s0+ZMgQbNmyBbGxsXBwcECuXLlw4MABAMCKFSswbdo0tG3bFu7u7iqnZeAvWPoZQghERUXB1tYWDRo0gLW1NQICAnDy5EkUK1YMkZGRcHV1haenJ/r37y/d5vPPToDbYUbBoCIjqW+04OBg7NmzBxEREahcuTL+/PNPWFhY4Ny5cxg/fjwSExORN29emJiYYMuWLbh+/TocHBy0XT5lAJ+HlEOHDmHQoEHw9fWFhYUF/vnnH0ycOBGmpqa4dOkSgE9jVjw8PODl5YV+/fpps3TKgMLDw5E3b16kpKRgx44daNiwobTsa2NTvjaWhX5/3PUjIwqFAjt37kSTJk3w4cMHfPjwAX5+fujTpw8iIiLg6uqKefPmoVq1aggJCcGjR49w9OhRhhTSmNQP9j179mDz5s2oXbs2KleujGLFiqF169bYsGEDYmNj0adPHwBA9+7dsXv3buk6kaYkJibi1atXMDExga6uLtasWSNNjQcAa2tr6f+pR0H+PJgwpGQc7FGRkUuXLqFt27YYNWoUevTogbCwMJQuXRrGxsYoWbIkNmzYACsrK+ncKV9OASXShIiICDRu3BjXr19HjRo1sG/fPpXlY8aMwenTp/H333/D1NRUamc3O/2sbw3gDg0NhYuLC2rUqIEFCxYgX758WqiOtIU9Kloyc+ZMjB07VvolAHw6RLmrqyt69OiB0NBQ1KpVC82bN8e4ceNw8eJF9O3bFxERETAyMgIAhhTSiM+3QQCwsrLC+vXrUadOHVy9ehVr165VWZ4/f368e/cO8fHxKu0MKfQzPg8px44dw8aNG3H9+nU8f/4cTk5OOH36NIKDgzFixAhpAHeLFi2wZMkSbZZNvwB7VLRkyZIlGDhwIGbMmIERI0ZIb9A7d+6gYMGCaNasmfSFoVQqUbJkSYSEhKBRo0bYsmULz5VCGvH5l8PDhw+hUChgYmICOzs7PH78GJ6enoiLi8Off/4JDw8PvH79Gu7u7jAyMsK+ffvYvU4aN2zYMKxfvx56enrIkiUL7OzssHDhQpQtWxY3b95EjRo14OTkhKSkJHz8+BHXr1+XTsxKGZSgX06pVAohhFi5cqXQ0dERU6dOFcnJydLyp0+fisKFC4t9+/YJIYSIiIgQ7dq1E0uWLBHPnj3TSs2U8aRuh0IIMXHiRFG8eHFRqFAhkSNHDuHr6yuEECIkJEQ0bNhQGBkZiYIFC4oWLVqIevXqifj4eCGEECkpKVqpnTKOz7fDoKAgUaJECXHy5EkREREhdu/eLVq0aCGcnZ3FlStXhBBCPHjwQEyZMkVMnz5d+tz8/POTMh4GlV9MqVRKb0ylUin++usvoaOjI6ZNmyZ96IeHh4uSJUsKDw8PERoaKsaMGSPKlSsnXr9+rc3SKYOaMmWKsLGxEYGBgSI2Nla0aNFCWFhYiNu3bwshhHj06JFo1KiRKFmypFi4cKF0u4SEBC1VTBnR+vXrRb9+/USvXr1U2i9evCjq168v3N3dRWxsrBBCNdwwpGR83H+gBQqFAocPH8bQoUNRpkwZ6Rwqs2bNghAClpaW6NChA44fPw5XV1ds2LABPj4+sLW11XbplAF8PiZFqVTiwoULWLhwIerWrYugoCAcO3YMM2bMQJEiRZCcnIw8efJg/vz5yJ49O/bv34+AgAAAgKGhobaeAmUA4otRB7t27cKyZctw7do1JCYmSu1ly5ZFlSpVcOrUKaSkpABQndHDc0hlAtpOSpnRjh07hLGxsZg6daq4ePGiEEIIX19faTeQEEIkJiaK27dvi6CgIPH06VNtlksZ1IQJE8SsWbNEzpw5xb1790RwcLDIkiWL8Pb2FkII8eHDBzF27FgRGhoqhBDi/v37onHjxqJs2bIiICBAm6XTb+7zHhF/f3+xYcMGIYQQ/fr1ExYWFmLZsmUiKipKWicwMFAUKlRI2hYpc2FQ+cXu3bsn8uTJI5YvX55m2YoVK6TdQESa9vl4ks2bNwtHR0dx69Yt0bFjR1GvXj1hYmIiVq9eLa3z/PlzUaVKFbFhwwbptnfu3BGtW7cWYWFhv7x+yhg+3w5v3bolSpUqJUqUKCF2794thBDC3d1d5M+fX0yfPl2EhISIkJAQUatWLVGtWjWVgEOZB/vMfrEnT55AX19f5QiLqTMvevXqBVNTU3Tq1AmGhoYYNmyYFiuljCZ1ds/x48dx7NgxDB06FEWLFpUOJFirVi1069YNwKeTYfbo0QO6urpo3749dHR0oFQqUahQIWzcuJGzLOiHpW6Hw4cPx+PHj2FsbIy7d+9i8ODB+PjxI9atW4du3bph3LhxWLJkCSpVqoQsWbJgy5YtUCgU3zzWCmVcDCq/WGxsrMrxJ5RKpbS/9dixYyhTpgy2bNmict4UIk159eoVunfvjvDwcIwZMwYA0Lt3bzx8+BBHjx5FqVKlkD9/fjx58gQJCQm4ePEidHV1VQ7mxjEB9LPWrVuHVatW4ciRI8iTJw8SExPh7u6OmTNnQkdHB2vWrIGJiQm2bt2K+vXro23btjA0NERSUhIMDAy0XT79Yoylv1iJEiXw9u1b+Pr6Avj06yI1qOzevRsbN25Ey5YtUbhwYW2WSRmUnZ0dAgICkD17duzduxeXL1+Grq4u5s6diylTpqBmzZqws7NDmzZtvnn2WR47hX5WSEgIihUrhpIlS8Lc3Bx2dnZYs2YNdHV1MXjwYOzcuRNLly5F7dq1sWDBAuzZswcxMTEMKZkUfxr9Ynny5MHSpUvRu3dvJCcno3PnztDV1cW6deuwbt06nD17lkf4pHTl4uKCHTt2wN3dHT4+Pujfvz9cXFzQtGlTNG3aVGXdlJQU9qCQxoj/P1GgoaEhEhISkJSUBCMjIyQnJyNnzpyYOXMmGjduDC8vLxgbG2Pjxo1o3749hg0bBj09Pbi5uWn7KZAW8Mi0WqBUKrFjxw54eHjA1NQURkZG0NXVxaZNm1CqVCltl0eZxNWrV9GjRw+UKVMGAwcORNGiRbVdEmUSN2/eRKlSpTB+/HhMnDhRag8MDMTKlSvx/v17pKSk4NixYwCArl27Yvz48cibN6+WKiZtYlDRohcvXiAsLAwKhQJ58uRB9uzZtV0SZTJXr16Fh4cHcufOjTlz5iBPnjzaLokyiXXr1qFXr14YNGgQ2rRpA0tLSwwYMAAVK1ZEixYtULRoUezfvx8NGjTQdqmkZQwqRJnchQsX4OPjg1WrVnE2Bf1SO3bsQN++fWFgYAAhBGxtbXHmzBm8fv0aderUwfbt2+Hi4qLtMknLGFSISBo7wKmf9Ks9f/4cT58+RXJyMipVqgQdHR2MHj0au3btQnBwMOzs7LRdImkZgwoRAfhfWCHSltu3b2P27Nn4+++/cfjwYZQsWVLbJZEMcDg/EQHgtGPSro8fPyIpKQm2trY4fvw4B3eThD0qREQkG8nJyTzyMalgUCEiIiLZ4qg5IiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSL6rRw7dgwKhQKRkZHffRsnJyd4eXmlW01ElH4YVIhIo7p06QKFQoHevXunWebp6QmFQoEuXbr8+sKI6LfEoEJEGufo6IjNmzcjPj5eaktISMDGjRuRK1cuLVZGRL8bBhUi0rjSpUvD0dERAQEBUltAQABy5cqFUqVKSW2JiYkYMGAAbG1tYWRkhMqVK+PixYsq9/X333+jQIECMDY2Ro0aNRAaGprm8U6dOoUqVarA2NgYjo6OGDBgAOLi4tLt+RHRr8OgQkTpolu3bli7dq10fc2aNejatavKOiNGjMCOHTuwfv16XLlyBc7OzqhXrx4iIiIAAE+fPkXLli3RpEkTXLt2DT169MCoUaNU7uPhw4eoX78+WrVqhRs3bmDLli04deoU+vXrl/5PkojSHYMKEaWLjh074tSpUwgLC0NYWBhOnz6Njh07Ssvj4uLg7e2NuXPnokGDBihSpAhWrlwJY2NjrF69GgDg7e2NfPnyYf78+ShYsCA6dOiQZnzLzJkz0aFDBwwaNAj58+dHxYoVsXjxYmzYsAEJCQm/8ikTUTrgSQmJKF3Y2NigUaNGWLduHYQQaNSoEaytraXlDx8+RHJyMipVqiS16evr448//sCdO3cAAHfu3EH58uVV7rdChQoq169fv44bN27A399fahNCQKlU4vHjxyhcuHB6PD0i+kUYVIgo3XTr1k3aBbNs2bJ0eYzY2Fh4eHhgwIABaZZx4C7R749BhYjSTf369ZGUlASFQoF69eqpLMuXLx8MDAxw+vRp5M6dG8CnM+devHgRgwYNAgAULlwYe/bsUbnduXPnVK6XLl0a//zzD5ydndPviRCR1nCMChGlG11dXdy5cwf//PMPdHV1VZaZmpqiT58+GD58OA4ePIh//vkHPXv2xIcPH9C9e3cAQO/evfHgwQMMHz4c9+7dw8aNG7Fu3TqV+xk5ciTOnDmDfv364dq1a3jw4AF2797NwbREGQSDChGlKzMzM5iZmX112axZs9CqVSt06tQJpUuXRkhICAIDA2FpaQng066bHTt2YNeuXShRogR8fHwwY8YMlftwcXHB8ePHcf/+fVSpUgWlSpXChAkTYG9vn+7PjYjSn0IIIbRdBBEREdHXsEeFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhk6/8AHoK08GWUizwAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "## calculate avg response time\n", - "unique_models = set(result[\"response\"]['model'] for result in result[\"results\"])\n", - "model_dict = {model: {\"response_time\": []} for model in unique_models}\n", - "for completion_result in result[\"results\"]:\n", - " model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n", - "\n", - "avg_response_time = {}\n", - "for model, data in model_dict.items():\n", - " avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n", - "\n", - "models = list(avg_response_time.keys())\n", - "response_times = list(avg_response_time.values())\n", - "\n", - "plt.bar(models, response_times)\n", - "plt.xlabel('Model', fontsize=10)\n", - "plt.ylabel('Average Response Time')\n", - "plt.title('Average Response Times for each Model')\n", - "\n", - "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n", - "plt.show()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "inSDIE3_IRds" - }, - "source": [ - "# Duration Test endpoint\n", - "\n", - "Run load testing for 2 mins. Hitting endpoints with 100+ queries every 15 seconds." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "id": "ePIqDx2EIURH" - }, - "outputs": [], - "source": [ - "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompt = \"Where does Paul Graham live?\"\n", - "final_prompt = context + prompt\n", - "result = load_test_model(models=models, prompt=final_prompt, num_calls=100, interval=15, duration=120)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 552 - }, - "id": "k6rJoELM6t1K", - "outputId": "f4968b59-3bca-4f78-a88b-149ad55e3cf7" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAIXCAYAAABghH+YAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABwdUlEQVR4nO3dd1QU198G8GfpoNKUooKCYuwIaiL2GrGLJnYFOxrsNZbYFTsYG2JDjV2xRKOIir33EhsWLBGwUaXJ3vcPX+bnCiYsLi6Oz+ecPbp37ux+lx3YZ+/cmVEIIQSIiIiIZEJH2wUQERERaRLDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNEcmSg4MDunfvru0y1DZnzhyUKFECurq6cHFx0XY5GnfkyBEoFAps27ZN26WoTaFQYNKkSWqv9+jRIygUCgQFBWm8Jsoaww19tiVLlkChUKBatWraLiXPcXBwgEKhkG758uXDDz/8gLVr12q7tK9Oxodidm5fqwMHDmDUqFGoWbMmVq9ejRkzZmi7pDwnKChIep9PnDiRabkQAvb29lAoFGjRooUWKqS8QE/bBdDXb/369XBwcMC5c+cQHh4OJycnbZeUp7i4uGD48OEAgOfPn2PFihXw8vJCSkoK+vTpo+Xqvh5ly5bFunXrVNrGjBmD/PnzY9y4cZn637lzBzo6X9f3t8OHD0NHRwcrV66EgYGBtsvJ04yMjLBhwwbUqlVLpf3o0aN4+vQpDA0NtVQZ5QUMN/RZHj58iFOnTiE4OBje3t5Yv349Jk6c+EVrUCqVSE1NhZGR0Rd93uwqWrQounbtKt3v3r07SpQoAT8/P4YbNdjY2Kj8HAFg5syZKFSoUKZ2AF/lh1t0dDSMjY01FmyEEEhOToaxsbFGHi8vadasGbZu3Yrff/8denr/+yjbsGEDqlSpgpcvX2qxOtK2r+trDeU569evh4WFBZo3b46ff/4Z69evl5alpaXB0tISPXr0yLReXFwcjIyMMGLECKktJSUFEydOhJOTEwwNDWFvb49Ro0YhJSVFZV2FQoEBAwZg/fr1KF++PAwNDbF//34AwNy5c1GjRg0ULFgQxsbGqFKlSpb79pOSkjBo0CAUKlQIBQoUQKtWrfDs2bMs96k/e/YMPXv2hI2NDQwNDVG+fHmsWrUqxz8zKysrlClTBvfv31dpVyqV8Pf3R/ny5WFkZAQbGxt4e3vjzZs3Kv0uXLgAd3d3FCpUCMbGxnB0dETPnj2l5Rn79+fOnQs/Pz8UL14cxsbGqFu3Lm7cuJGpnsOHD6N27drIly8fzM3N0bp1a9y6dUulz6RJk6BQKBAeHo7u3bvD3NwcZmZm6NGjB96+favSNzQ0FLVq1YK5uTny58+P0qVLY+zYsSp9svtef46P59xk7M44ceIEBg0aBCsrK5ibm8Pb2xupqamIiYmBp6cnLCwsYGFhgVGjRkEIofKYmnqPsqJQKLB69WokJiZKu10y5mi8e/cOU6dORcmSJWFoaAgHBweMHTs208/LwcEBLVq0QEhICKpWrQpjY2MsW7bsX5/37NmzaNKkCczMzGBiYoK6devi5MmTKn0iIiLwyy+/oHTp0jA2NkbBggXRrl07PHr0KNPjxcTEYOjQoXBwcIChoSHs7Ozg6emZKWwolUpMnz4ddnZ2MDIyQsOGDREeHv6vtX6oU6dOePXqFUJDQ6W21NRUbNu2DZ07d85yncTERAwfPhz29vYwNDRE6dKlMXfu3Ezvc0pKCoYOHQorKyvp78PTp0+zfExN/30gDRFEn6FMmTKiV69eQgghjh07JgCIc+fOSct79uwpzM3NRUpKisp6a9asEQDE+fPnhRBCpKeni8aNGwsTExMxZMgQsWzZMjFgwAChp6cnWrdurbIuAFG2bFlhZWUlJk+eLBYvXiwuX74shBDCzs5O/PLLL2LRokVi/vz54ocffhAAxJ49e1Qeo3379gKA6Natm1i8eLFo3769qFSpkgAgJk6cKPWLjIwUdnZ2wt7eXkyZMkUsXbpUtGrVSgAQfn5+//nzKV68uGjevLlKW1pamrC1tRU2NjYq7b179xZ6enqiT58+IiAgQIwePVrky5dPfP/99yI1NVUIIURUVJSwsLAQ3333nZgzZ45Yvny5GDdunChbtqz0OA8fPhQARMWKFYWDg4OYNWuWmDx5srC0tBRWVlYiMjJS6hsaGir09PTEd999J2bPni0mT54sChUqJCwsLMTDhw+lfhMnThQAhKurq2jbtq1YsmSJ6N27twAgRo0aJfW7ceOGMDAwEFWrVhULFiwQAQEBYsSIEaJOnTpSH3Xe6/9Svnx5Ubdu3U/+7L28vKT7q1evFgCEi4uLaNKkiVi8eLHo1q2b9Bpq1aolOnfuLJYsWSJatGghAIg1a9bkynuUlXXr1onatWsLQ0NDsW7dOrFu3Tpx//59IYQQXl5eAoD4+eefxeLFi4Wnp6cAIDw8PDK9ZicnJ2FhYSF+/fVXERAQIMLCwj75nIcOHRIGBgaievXqYt68ecLPz084OzsLAwMDcfbsWanf1q1bRaVKlcSECRNEYGCgGDt2rLCwsBDFixcXiYmJUr/4+HhRoUIFoaurK/r06SOWLl0qpk6dKr7//nvpdzQsLEzalqpUqSL8/PzEpEmThImJifjhhx/+9Wf04ft4/vx5UaNGDdGtWzdp2c6dO4WOjo549uxZpt89pVIpGjRoIBQKhejdu7dYtGiRaNmypQAghgwZovIcXbt2FQBE586dxaJFi0Tbtm2Fs7Nzjv8+ZPxOrl69+j9fH2kGww3l2IULFwQAERoaKoR4/8fDzs5ODB48WOoTEhIiAIg///xTZd1mzZqJEiVKSPfXrVsndHR0xPHjx1X6BQQECADi5MmTUhsAoaOjI27evJmpprdv36rcT01NFRUqVBANGjSQ2i5evJjlH7Tu3btn+uPVq1cvUbhwYfHy5UuVvh07dhRmZmaZnu9jxYsXF40bNxYvXrwQL168ENevX5c+UH18fKR+x48fFwDE+vXrVdbfv3+/SvuOHTtUQmFWMv6QGhsbi6dPn0rtZ8+eFQDE0KFDpTYXFxdhbW0tXr16JbVdvXpV6OjoCE9PT6ktI9z07NlT5bnatGkjChYsKN338/MTAMSLFy8+WZ867/V/yUm4cXd3F0qlUmqvXr26UCgUol+/flLbu3fvhJ2dncpja/I9+hQvLy+RL18+lbYrV64IAKJ3794q7SNGjBAAxOHDh1VeMwCxf//+/3wupVIpSpUqlenn8fbtW+Ho6Ch+/PFHlbaPnT59WgAQa9euldomTJggAIjg4OAsn0+I/4WbsmXLqnzpWbBggQAgrl+//q91fxhuFi1aJAoUKCDV165dO1G/fn3pZ/FhuNm5c6cAIKZNm6byeD///LNQKBQiPDxcCPG/n/cvv/yi0q9z5845/vvAcPPlcbcU5dj69ethY2OD+vXrA3g/rN6hQwds2rQJ6enpAIAGDRqgUKFC2Lx5s7TemzdvEBoaig4dOkhtW7duRdmyZVGmTBm8fPlSujVo0AAAEBYWpvLcdevWRbly5TLV9OHcgjdv3iA2Nha1a9fGpUuXpPaMXVi//PKLyroDBw5UuS+EwPbt29GyZUsIIVTqcnd3R2xsrMrjfsqBAwdgZWUFKysrVKxYEevWrUOPHj0wZ84clddvZmaGH3/8UeV5qlSpgvz580uv39zcHACwZ88epKWl/evzenh4oGjRotL9H374AdWqVcNff/0F4P3k5itXrqB79+6wtLSU+jk7O+PHH3+U+n2oX79+Kvdr166NV69eIS4uTqW+Xbt2QalUZlmXuu+1pvXq1UvliKpq1apBCIFevXpJbbq6uqhatSoePHigUrem36PsyHgfhg0bptKeMUl97969Ku2Ojo5wd3f/z8e9cuUK7t27h86dO+PVq1fS60lMTETDhg1x7Ngx6T388PcqLS0Nr169gpOTE8zNzVV+B7Zv345KlSqhTZs2mZ7v46PYevTooTK3qHbt2gCg8jP/L+3bt0dSUhL27NmD+Ph47Nmz55O7pP766y/o6upi0KBBKu3Dhw+HEAL79u2T+gHI1G/IkCEq9zX194Fyxzcdbo4dO4aWLVuiSJEiUCgU2LlzZ64/57Nnz9C1a1dpTkjFihVx4cKFXH9eTUtPT8emTZtQv359PHz4EOHh4QgPD0e1atUQFRWFQ4cOAQD09PTw008/YdeuXdL8gODgYKSlpamEm3v37uHmzZtSCMi4fffddwDeT7T8kKOjY5Z17dmzB25ubjAyMoKlpSWsrKywdOlSxMbGSn0iIiKgo6OT6TE+PsrrxYsXiImJQWBgYKa6MuYRfVxXVqpVq4bQ0FDs378fc+fOhbm5Od68eaPyh/3evXuIjY2FtbV1pudKSEiQnqdu3br46aefMHnyZBQqVAitW7fG6tWrs5yrUqpUqUxt3333nTRPIiIiAgBQunTpTP3Kli0rfdB9qFixYir3LSwsAECac9KhQwfUrFkTvXv3ho2NDTp27IgtW7aoBB1132tN+/g1mJmZAQDs7e0ztX84lyY33qPsyNheP94+bW1tYW5uLr2PGT71u/Gxe/fuAQC8vLwyvZ4VK1YgJSVF+r1JSkrChAkTpLkqhQoVgpWVFWJiYlR+t+7fv48KFSpk6/n/a1vKDisrKzRq1AgbNmxAcHAw0tPT8fPPP2fZNyIiAkWKFEGBAgVU2suWLSstz/hXR0cHJUuWVOn38e+Jpv4+UO74po+WSkxMRKVKldCzZ0+0bds215/vzZs3qFmzJurXr499+/bBysoK9+7dk36pvyaHDx/G8+fPsWnTJmzatCnT8vXr16Nx48YAgI4dO2LZsmXYt28fPDw8sGXLFpQpUwaVKlWS+iuVSlSsWBHz58/P8vk+/uDJ6uiP48ePo1WrVqhTpw6WLFmCwoULQ19fH6tXr8aGDRvUfo0ZH8hdu3aFl5dXln2cnZ3/83EKFSqERo0aAQDc3d1RpkwZtGjRAgsWLJC+jSuVSlhbW6tMyP6QlZUVAEgnPztz5gz+/PNPhISEoGfPnpg3bx7OnDmD/Pnzq/061aGrq5tlu/j/CZnGxsY4duwYwsLCsHfvXuzfvx+bN29GgwYNcODAAejq6qr9Xmvap15DVu3ig4mm2n6Psnv+nuweGZWxfc+ZM+eTJwvMqHXgwIFYvXo1hgwZgurVq8PMzAwKhQIdO3b85Ajdf/mvbSm7OnfujD59+iAyMhJNmzaVRs5ym6b+PlDu+KbDTdOmTdG0adNPLk9JScG4ceOwceNGxMTEoEKFCpg1axbq1auXo+ebNWsW7O3tsXr1aqktu9+y8pr169fD2toaixcvzrQsODgYO3bsQEBAAIyNjVGnTh0ULlwYmzdvRq1atXD48OFM5yUpWbIkrl69ioYNG+b4JGzbt2+HkZERQkJCVA4D/vDnDQDFixeHUqnEw4cPVUY3Pj5SI+NIifT0dCmcaELz5s1Rt25dzJgxA97e3siXLx9KliyJgwcPombNmtn6cHJzc4ObmxumT5+ODRs2oEuXLti0aRN69+4t9cn4Zv6hu3fvwsHBAcD7nwPw/nwwH7t9+zYKFSqEfPnyqf36dHR00LBhQzRs2BDz58/HjBkzMG7cOISFhaFRo0Yaea+1ITfeo+zI2F7v3bsnjTIAQFRUFGJiYqT3UV0ZIxOmpqb/uX1v27YNXl5emDdvntSWnJyMmJiYTI+Z1RF5ualNmzbw9vbGmTNnVHZ/f6x48eI4ePAg4uPjVUZvbt++LS3P+FepVOL+/fsqozUf/57k1t8H0oxverfUfxkwYABOnz6NTZs24dq1a2jXrh2aNGmS5YdGduzevRtVq1ZFu3btYG1tDVdXVyxfvlzDVee+pKQkBAcHo0WLFvj5558z3QYMGID4+Hjs3r0bwPsPu59//hl//vkn1q1bh3fv3qnskgLe7zt/9uxZlj+PpKSkTLtHsqKrqwuFQiHN9wHeHxb98e7GjPkIS5YsUWlfuHBhpsf76aefsH379iz/YL948eI/a/qU0aNH49WrV9Lrbd++PdLT0zF16tRMfd+9eyd9iLx58ybTN9uMb90f7/bYuXMnnj17Jt0/d+4czp49KwX6woULw8XFBWvWrFH5kLpx4wYOHDiAZs2aqf26Xr9+nant4/o08V5rQ268R9mR8T74+/urtGeMfDVv3lztxwSAKlWqoGTJkpg7dy4SEhIyLf9w+9bV1c30mhYuXKjyuwYAP/30E65evYodO3Zkejx1R2SyK3/+/Fi6dCkmTZqEli1bfrJfs2bNkJ6ejkWLFqm0+/n5QaFQSL8XGf/+/vvvKv0+/vnn5t8H+nzf9MjNv3n8+DFWr16Nx48fo0iRIgCAESNGYP/+/Tk+LfqDBw+wdOlSDBs2DGPHjsX58+cxaNAgGBgYfHJYMy/avXs34uPj0apVqyyXu7m5wcrKCuvXr5dCTIcOHbBw4UJMnDgRFStWVPkGCgDdunXDli1b0K9fP4SFhaFmzZpIT0/H7du3sWXLFum8Hf+mefPmmD9/Ppo0aYLOnTsjOjoaixcvhpOTE65duyb1q1KlCn766Sf4+/vj1atXcHNzw9GjR3H37l0AqsP/M2fORFhYGKpVq4Y+ffqgXLlyeP36NS5duoSDBw9m+WGeHU2bNkWFChUwf/58+Pj4oG7duvD29oavry+uXLmCxo0bQ19fH/fu3cPWrVuxYMEC/Pzzz1izZg2WLFmCNm3aoGTJkoiPj8fy5cthamqaKYw4OTmhVq1a6N+/P1JSUuDv74+CBQti1KhRUp85c+agadOmqF69Onr16oWkpCQsXLgQZmZmObqGzpQpU3Ds2DE0b94cxYsXR3R0NJYsWQI7OzvpTLKaeK+1ITfeo+yoVKkSvLy8EBgYiJiYGNStWxfnzp3DmjVr4OHhIU3oV5eOjg5WrFiBpk2bonz58ujRoweKFi2KZ8+eISwsDKampvjzzz8BAC1atMC6detgZmaGcuXK4fTp0zh48CAKFiyo8pgjR47Etm3b0K5dO/Ts2RNVqlTB69evsXv3bgQEBKjsitak7Pz9bNmyJerXr49x48bh0aNHqFSpEg4cOIBdu3ZhyJAh0kiWi4sLOnXqhCVLliA2NhY1atTAoUOHsjwHT279fSAN0MoxWnkQALFjxw7p/p49ewQAkS9fPpWbnp6eaN++vRBCiFu3bgkA/3obPXq09Jj6+vqievXqKs87cOBA4ebm9kVeo6a0bNlSGBkZqZzf4mPdu3cX+vr60iGSSqVS2NvbZ3koZobU1FQxa9YsUb58eWFoaCgsLCxElSpVxOTJk0VsbKzUDx8dRv2hlStXilKlSglDQ0NRpkwZsXr1aukw5g8lJiYKHx8fYWlpKfLnzy88PDzEnTt3BAAxc+ZMlb5RUVHCx8dH2NvbC319fWFraysaNmwoAgMD//NnldV5bjIEBQVlOjw0MDBQVKlSRRgbG4sCBQqIihUrilGjRol//vlHCCHEpUuXRKdOnUSxYsWEoaGhsLa2Fi1atBAXLlyQHiPjsNM5c+aIefPmCXt7e2FoaChq164trl69mqmOgwcPipo1awpjY2NhamoqWrZsKf7++2+VPhk/w48P8c44LDfjnDiHDh0SrVu3FkWKFBEGBgaiSJEiolOnTuLu3bsq62X3vf4vOTkU/ONDtD/12rI6LFsIzbxHn/Kp50xLSxOTJ08Wjo6OQl9fX9jb24sxY8aI5OTkTK/5U9vbp1y+fFm0bdtWFCxYUBgaGorixYuL9u3bi0OHDkl93rx5I3r06CEKFSok8ufPL9zd3cXt27cz/YyFEOLVq1diwIABomjRosLAwEDY2dkJLy8v6W9BxqHgW7duVVkvu4dLf+p9/FhWP4v4+HgxdOhQUaRIEaGvry9KlSol5syZo3IovBBCJCUliUGDBomCBQuKfPnyiZYtW4onT55kOhRciOz9feCh4F+eQohcGiv8yigUCuzYsQMeHh4AgM2bN6NLly64efNmpolv+fPnh62tLVJTU//zsMWCBQtKEw2LFy+OH3/8EStWrJCWL126FNOmTVPZfUDaceXKFbi6uuKPP/5Aly5dtF1Ojj169AiOjo6YM2eOyhmgiYi+Fdwt9Qmurq5IT09HdHS0dP6FjxkYGKBMmTLZfsyaNWtmmpR29+7dHE8IpJxLSkrKNCnU398fOjo6qFOnjpaqIiIiTfimw01CQoLKftSHDx/iypUrsLS0xHfffYcuXbrA09MT8+bNg6urK168eIFDhw7B2dk5R5P4hg4diho1amDGjBlo3749zp07h8DAQAQGBmryZVE2zJ49GxcvXkT9+vWhp6eHffv2Yd++fejbt2+uH4pMRES5TNv7xbQpY9/vx7eMfcipqaliwoQJwsHBQejr64vChQuLNm3aiGvXruX4Of/8809RoUIFaU5IduZtkOYdOHBA1KxZU1hYWAh9fX1RsmRJMWnSJJGWlqbt0j7bh3NuiIi+RZxzQ0RERLLC89wQERGRrDDcEBERkax8cxOKlUol/vnnHxQoUOCrOvU7ERHRt0wIgfj4eBQpUgQ6Ov8+NvPNhZt//vmHR8MQERF9pZ48eQI7O7t/7fPNhZuMC6Y9efIEpqamWq6GiIiIsiMuLg729vYqFz79lG8u3GTsijI1NWW4ISIi+spkZ0oJJxQTERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs6Gm7ACLSLIdf92q7BNKyRzOba/X5uQ2StrdBjtwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrGg13CxduhTOzs4wNTWFqakpqlevjn379v3rOlu3bkWZMmVgZGSEihUr4q+//vpC1RIREdHXQKvhxs7ODjNnzsTFixdx4cIFNGjQAK1bt8bNmzez7H/q1Cl06tQJvXr1wuXLl+Hh4QEPDw/cuHHjC1dOREREeZVCCCG0XcSHLC0tMWfOHPTq1SvTsg4dOiAxMRF79uyR2tzc3ODi4oKAgIBsPX5cXBzMzMwQGxsLU1NTjdVNlFfwooWk7YsWchuk3NgG1fn8zjNzbtLT07Fp0yYkJiaievXqWfY5ffo0GjVqpNLm7u6O06dPf/JxU1JSEBcXp3IjIiIi+dJ6uLl+/Try588PQ0ND9OvXDzt27EC5cuWy7BsZGQkbGxuVNhsbG0RGRn7y8X19fWFmZibd7O3tNVo/ERER5S1aDzelS5fGlStXcPbsWfTv3x9eXl74+++/Nfb4Y8aMQWxsrHR78uSJxh6biIiI8h49bRdgYGAAJycnAECVKlVw/vx5LFiwAMuWLcvU19bWFlFRUSptUVFRsLW1/eTjGxoawtDQULNFExERUZ6l9ZGbjymVSqSkpGS5rHr16jh06JBKW2ho6Cfn6BAREdG3R6sjN2PGjEHTpk1RrFgxxMfHY8OGDThy5AhCQkIAAJ6enihatCh8fX0BAIMHD0bdunUxb948NG/eHJs2bcKFCxcQGBiozZdBREREeYhWw010dDQ8PT3x/PlzmJmZwdnZGSEhIfjxxx8BAI8fP4aOzv8Gl2rUqIENGzZg/PjxGDt2LEqVKoWdO3eiQoUK2noJRERElMdoNdysXLnyX5cfOXIkU1u7du3Qrl27XKqIiIiIvnZ5bs4NERER0edguCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWdFquPH19cX333+PAgUKwNraGh4eHrhz586/rhMUFASFQqFyMzIy+kIVExERUV6n1XBz9OhR+Pj44MyZMwgNDUVaWhoaN26MxMTEf13P1NQUz58/l24RERFfqGIiIiLK6/S0+eT79+9XuR8UFARra2tcvHgRderU+eR6CoUCtra2uV0eERERfYXy1Jyb2NhYAIClpeW/9ktISEDx4sVhb2+P1q1b4+bNm5/sm5KSgri4OJUbERERyVeeCTdKpRJDhgxBzZo1UaFChU/2K126NFatWoVdu3bhjz/+gFKpRI0aNfD06dMs+/v6+sLMzEy62dvb59ZLICIiojwgz4QbHx8f3LhxA5s2bfrXftWrV4enpydcXFxQt25dBAcHw8rKCsuWLcuy/5gxYxAbGyvdnjx5khvlExERUR6h1Tk3GQYMGIA9e/bg2LFjsLOzU2tdfX19uLq6Ijw8PMvlhoaGMDQ01ESZRERE9BXQ6siNEAIDBgzAjh07cPjwYTg6Oqr9GOnp6bh+/ToKFy6cCxUSERHR10arIzc+Pj7YsGEDdu3ahQIFCiAyMhIAYGZmBmNjYwCAp6cnihYtCl9fXwDAlClT4ObmBicnJ8TExGDOnDmIiIhA7969tfY6iIiIKO/QarhZunQpAKBevXoq7atXr0b37t0BAI8fP4aOzv8GmN68eYM+ffogMjISFhYWqFKlCk6dOoVy5cp9qbKJiIgoD9NquBFC/GefI0eOqNz38/ODn59fLlVEREREX7s8MaFYThx+3avtEkjLHs1sru0SiIi+aXnmUHAiIiIiTWC4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZyVG4uX//PsaPH49OnTohOjoaALBv3z7cvHlTo8URERERqUvtcHP06FFUrFgRZ8+eRXBwMBISEgAAV69excSJEzVeIBEREZE61A43v/76K6ZNm4bQ0FAYGBhI7Q0aNMCZM2c0WhwRERGRutQON9evX0ebNm0ytVtbW+Ply5caKYqIiIgop9QON+bm5nj+/Hmm9suXL6No0aIaKYqIiIgop9QONx07dsTo0aMRGRkJhUIBpVKJkydPYsSIEfD09MyNGomIiIiyTe1wM2PGDJQpUwb29vZISEhAuXLlUKdOHdSoUQPjx4/PjRqJiIiIsk3tq4IbGBhg+fLl+O2333Djxg0kJCTA1dUVpUqVyo36iIiIiNSidrjJUKxYMRQrVkyTtRARERF9NrXDjRAC27ZtQ1hYGKKjo6FUKlWWBwcHa6w4IiIiInWpHW6GDBmCZcuWoX79+rCxsYFCociNuoiIiIhyRO1ws27dOgQHB6NZs2a5UQ8RERHRZ1H7aCkzMzOUKFEiN2ohIiIi+mxqh5tJkyZh8uTJSEpKyo16iIiIiD6L2rul2rdvj40bN8La2hoODg7Q19dXWX7p0iWNFUdERESkLrXDjZeXFy5evIiuXbtyQjERERHlOWqHm7179yIkJAS1atXKjXqIiIiIPovac27s7e1hamqaG7UQERERfTa1w828efMwatQoPHr0KBfKISIiIvo8au+W6tq1K96+fYuSJUvCxMQk04Ti169fa6w4IiIiInWpHW78/f1zoQwiIiIizcjR0VJEREREeVW2wk1cXJw0iTguLu5f+3KyMREREWlTtsKNhYUFnj9/Dmtra5ibm2d5bhshBBQKBdLT0zVeJBEREVF2ZSvcHD58GJaWlgCAsLCwXC2IiIiI6HNkK9zUrVsXJUqUwPnz51G3bt3cromIiIgox7J9nptHjx5xlxMRERHleWqfxI+IiIgoL1PrUPCQkBCYmZn9a59WrVp9VkFEREREn0OtcPNf57jh0VJERESkbWrtloqMjIRSqfzkjcGGiIiItC3b4Sarc9sQERER5TXZDjdCiNysg4iIiEgjsh1uvLy8YGxsnJu1EBEREX22bE8oXr16dW7WQURERKQRPM8NERERyQrDDREREckKww0RERHJSo7DTXh4OEJCQpCUlAQgZ0dT+fr64vvvv0eBAgVgbW0NDw8P3Llz5z/X27p1K8qUKQMjIyNUrFgRf/31l9rPTURERPKkdrh59eoVGjVqhO+++w7NmjXD8+fPAQC9evXC8OHD1Xqso0ePwsfHB2fOnEFoaCjS0tLQuHFjJCYmfnKdU6dOoVOnTujVqxcuX74MDw8PeHh44MaNG+q+FCIiIpIhtcPN0KFDoaenh8ePH8PExERq79ChA/bv36/WY+3fvx/du3dH+fLlUalSJQQFBeHx48e4ePHiJ9dZsGABmjRpgpEjR6Js2bKYOnUqKleujEWLFqn7UoiIiEiG1Lq2FAAcOHAAISEhsLOzU2kvVaoUIiIiPquY2NhYAIClpeUn+5w+fRrDhg1TaXN3d8fOnTuz7J+SkoKUlBTpflxc3GfVSERERHmb2iM3iYmJKiM2GV6/fg1DQ8McF6JUKjFkyBDUrFkTFSpU+GS/yMhI2NjYqLTZ2NggMjIyy/6+vr4wMzOTbvb29jmukYiIiPI+tcNN7dq1sXbtWum+QqGAUqnE7NmzUb9+/RwX4uPjgxs3bmDTpk05foysjBkzBrGxsdLtyZMnGn18IiIiylvU3i01e/ZsNGzYEBcuXEBqaipGjRqFmzdv4vXr1zh58mSOihgwYAD27NmDY8eOZdrd9TFbW1tERUWptEVFRcHW1jbL/oaGhp81okRERERfF7VHbipUqIC7d++iVq1aaN26NRITE9G2bVtcvnwZJUuWVOuxhBAYMGAAduzYgcOHD8PR0fE/16levToOHTqk0hYaGorq1aur9dxEREQkT2qP3ACAmZkZxo0b99lP7uPjgw0bNmDXrl0oUKCANG/GzMxMukinp6cnihYtCl9fXwDA4MGDUbduXcybNw/NmzfHpk2bcOHCBQQGBn52PURERPT1U3vkZv/+/Thx4oR0f/HixXBxcUHnzp3x5s0btR5r6dKliI2NRb169VC4cGHptnnzZqnP48ePpXPpAECNGjWwYcMGBAYGolKlSti2bRt27tz5r5OQiYiI6Nuh9sjNyJEjMWvWLADA9evXMWzYMAwfPhxhYWEYNmyYWlcPz85ZjY8cOZKprV27dmjXrl22n4eIiIi+HWqHm4cPH6JcuXIAgO3bt6Nly5aYMWMGLl26hGbNmmm8QCIiIiJ1qL1bysDAAG/fvgUAHDx4EI0bNwbw/sR7PEEeERERaZvaIze1atXCsGHDULNmTZw7d06aH3P37t3/PIybiIiIKLepPXKzaNEi6OnpYdu2bVi6dCmKFi0KANi3bx+aNGmi8QKJiIiI1KH2yE2xYsWwZ8+eTO1+fn4aKYiIiIjoc+ToPDdKpRLh4eGIjo6GUqlUWVanTh2NFEZERESUE2qHmzNnzqBz586IiIjIdCi3QqFAenq6xoojIiIiUpfa4aZfv36oWrUq9u7di8KFC0OhUORGXUREREQ5ona4uXfvHrZt2wYnJ6fcqIeIiIjos6h9tFS1atUQHh6eG7UQERERfTa1R24GDhyI4cOHIzIyEhUrVoS+vr7KcmdnZ40VR0RERKQutcPNTz/9BADo2bOn1KZQKCCE4IRiIiIi0rocXVuKiIiIKK9SO9wUL148N+ogIiIi0ogcncTv/v378Pf3x61btwAA5cqVw+DBg1GyZEmNFkdERESkLrWPlgoJCUG5cuVw7tw5ODs7w9nZGWfPnkX58uURGhqaGzUSERERZZvaIze//vorhg4dipkzZ2ZqHz16NH788UeNFUdERESkLrVHbm7duoVevXplau/Zsyf+/vtvjRRFRERElFNqhxsrKytcuXIlU/uVK1dgbW2tiZqIiIiIckzt3VJ9+vRB37598eDBA9SoUQMAcPLkScyaNQvDhg3TeIFERERE6lA73Pz2228oUKAA5s2bhzFjxgAAihQpgkmTJmHQoEEaL5CIiIhIHWqHG4VCgaFDh2Lo0KGIj48HABQoUEDjhRERERHlRI7OcwMA0dHRuHPnDgCgTJkysLKy0lhRRERERDml9oTi+Ph4dOvWDUWKFEHdunVRt25dFClSBF27dkVsbGxu1EhERESUbWqHm969e+Ps2bPYu3cvYmJiEBMTgz179uDChQvw9vbOjRqJiIiIsk3t3VJ79uxBSEgIatWqJbW5u7tj+fLlaNKkiUaLIyIiIlKX2iM3BQsWhJmZWaZ2MzMzWFhYaKQoIiIiopxSO9yMHz8ew4YNQ2RkpNQWGRmJkSNH4rffftNocURERETqUnu31NKlSxEeHo5ixYqhWLFiAIDHjx/D0NAQL168wLJly6S+ly5d0lylRERERNmgdrjx8PDIhTKIiIiINEPtcDNx4sTcqIOIiIhII9Sec/PkyRM8ffpUun/u3DkMGTIEgYGBGi2MiIiIKCfUDjedO3dGWFgYgPcTiRs1aoRz585h3LhxmDJlisYLJCIiIlKH2uHmxo0b+OGHHwAAW7ZsQcWKFXHq1CmsX78eQUFBmq6PiIiISC1qh5u0tDQYGhoCAA4ePIhWrVoBeH99qefPn2u2OiIiIiI1qR1uypcvj4CAABw/fhyhoaHSWYn/+ecfFCxYUOMFEhEREalD7XAza9YsLFu2DPXq1UOnTp1QqVIlAMDu3bul3VVERERE2qL2oeD16tXDy5cvERcXp3K5hb59+8LExESjxRERERGpS+2RGwAQQuDixYtYtmwZ4uPjAQAGBgYMN0RERKR1ao/cREREoEmTJnj8+DFSUlLw448/okCBApg1axZSUlIQEBCQG3USERERZYvaIzeDBw9G1apV8ebNGxgbG0vtbdq0waFDhzRaHBEREZG61B65OX78OE6dOgUDAwOVdgcHBzx79kxjhRERERHlhNojN0qlEunp6Znanz59igIFCmikKCIiIqKcUjvcNG7cGP7+/tJ9hUKBhIQETJw4Ec2aNdNkbURERERqU3u31Lx58+Du7o5y5cohOTkZnTt3xr1791CoUCFs3LgxN2okIiIiyja1R27s7Oxw9epVjBs3DkOHDoWrqytmzpyJy5cvw9raWq3HOnbsGFq2bIkiRYpAoVBg586d/9r/yJEjUCgUmW6RkZHqvgwiIiKSKbVHbgBAT08PXbp0QZcuXaS258+fY+TIkVi0aFG2HycxMRGVKlVCz5490bZt22yvd+fOHZiamkr31Q1VREREJF9qhZubN28iLCwMBgYGaN++PczNzfHy5UtMnz4dAQEBKFGihFpP3rRpUzRt2lStdYD3Ycbc3Fzt9YiIiEj+sr1bavfu3XB1dcWgQYPQr18/VK1aFWFhYShbtixu3bqFHTt24ObNm7lZq8TFxQWFCxfGjz/+iJMnT/5r35SUFMTFxanciIiISL6yHW6mTZsGHx8fxMXFYf78+Xjw4AEGDRqEv/76C/v375euDp6bChcujICAAGzfvh3bt2+Hvb096tWrh0uXLn1yHV9fX5iZmUk3e3v7XK+TiIiItCfb4ebOnTvw8fFB/vz5MXDgQOjo6MDPzw/ff/99btanonTp0vD29kaVKlVQo0YNrFq1CjVq1ICfn98n1xkzZgxiY2Ol25MnT75YvURERPTlZXvOTXx8vDSJV1dXF8bGxmrPsckNP/zwA06cOPHJ5YaGhjA0NPyCFREREZE2qTWhOCQkBGZmZgDen6n40KFDuHHjhkqfVq1aaa66bLhy5QoKFy78RZ+TiIiI8i61wo2Xl5fKfW9vb5X7CoUiy0szfEpCQgLCw8Ol+w8fPsSVK1dgaWmJYsWKYcyYMXj27BnWrl0LAPD394ejoyPKly+P5ORkrFixAocPH8aBAwfUeRlEREQkY9kON0qlUuNPfuHCBdSvX1+6P2zYMADvQ1RQUBCeP3+Ox48fS8tTU1MxfPhwPHv2DCYmJnB2dsbBgwdVHoOIiIi+bTk6iZ+m1KtXD0KITy4PCgpSuT9q1CiMGjUql6siIiKir5nal18gIiIiyssYboiIiEhWGG6IiIhIVhhuiIiISFZyFG5iYmKwYsUKjBkzBq9fvwYAXLp0Cc+ePdNocURERETqUvtoqWvXrqFRo0YwMzPDo0eP0KdPH1haWiI4OBiPHz+WzklDREREpA1qj9wMGzYM3bt3x71792BkZCS1N2vWDMeOHdNocURERETqUjvcnD9/PtOZiQGgaNGiiIyM1EhRRERERDmldrgxNDREXFxcpva7d+/CyspKI0URERER5ZTa4aZVq1aYMmUK0tLSALy/ntTjx48xevRo/PTTTxovkIiIiEgdaoebefPmISEhAdbW1khKSkLdunXh5OSEAgUKYPr06blRIxEREVG2qX20lJmZGUJDQ3HixAlcu3YNCQkJqFy5Mho1apQb9RERERGpJccXzqxVqxZq1aqlyVqIiIiIPpva4eb333/Psl2hUMDIyAhOTk6oU6cOdHV1P7s4IiIiInWpHW78/Pzw4sULvH37FhYWFgCAN2/ewMTEBPnz50d0dDRKlCiBsLAw2Nvba7xgIiIion+j9oTiGTNm4Pvvv8e9e/fw6tUrvHr1Cnfv3kW1atWwYMECPH78GLa2thg6dGhu1EtERET0r9QeuRk/fjy2b9+OkiVLSm1OTk6YO3cufvrpJzx48ACzZ8/mYeFERESkFWqP3Dx//hzv3r3L1P7u3TvpDMVFihRBfHz851dHREREpCa1w039+vXh7e2Ny5cvS22XL19G//790aBBAwDA9evX4ejoqLkqiYiIiLJJ7XCzcuVKWFpaokqVKjA0NIShoSGqVq0KS0tLrFy5EgCQP39+zJs3T+PFEhEREf0Xtefc2NraIjQ0FLdv38bdu3cBAKVLl0bp0qWlPvXr19dchURERERqyPFJ/MqUKYMyZcposhYiIiKiz5ajcPP06VPs3r0bjx8/Rmpqqsqy+fPna6QwIiIiopxQO9wcOnQIrVq1QokSJXD79m1UqFABjx49ghAClStXzo0aiYiIiLJN7QnFY8aMwYgRI3D9+nUYGRlh+/btePLkCerWrYt27drlRo1ERERE2aZ2uLl16xY8PT0BAHp6ekhKSkL+/PkxZcoUzJo1S+MFEhEREalD7XCTL18+aZ5N4cKFcf/+fWnZy5cvNVcZERERUQ6oPefGzc0NJ06cQNmyZdGsWTMMHz4c169fR3BwMNzc3HKjRiIiIqJsUzvczJ8/HwkJCQCAyZMnIyEhAZs3b0apUqV4pBQRERFpnVrhJj09HU+fPoWzszOA97uoAgICcqUwIiIiopxQa86Nrq4uGjdujDdv3uRWPURERESfRe0JxRUqVMCDBw9yoxYiIiKiz6Z2uJk2bRpGjBiBPXv24Pnz54iLi1O5EREREWmT2hOKmzVrBgBo1aoVFAqF1C6EgEKhQHp6uuaqIyIiIlKT2uEmLCwsN+ogIiIi0gi1w03dunVzow4iIiIijVB7zg0AHD9+HF27dkWNGjXw7NkzAMC6detw4sQJjRZHREREpC61w8327dvh7u4OY2NjXLp0CSkpKQCA2NhYzJgxQ+MFEhEREakjR0dLBQQEYPny5dDX15faa9asiUuXLmm0OCIiIiJ1qR1u7ty5gzp16mRqNzMzQ0xMjCZqIiIiIsoxtcONra0twsPDM7WfOHECJUqU0EhRRERERDmldrjp06cPBg8ejLNnz0KhUOCff/7B+vXrMWLECPTv3z83aiQiIiLKNrUPBf/111+hVCrRsGFDvH37FnXq1IGhoSFGjBiBgQMH5kaNRERERNmmdrhRKBQYN24cRo4cifDwcCQkJKBcuXLInz9/btRHREREpBa1d0v98ccfePv2LQwMDFCuXDn88MMPDDZERESUZ6gdboYOHQpra2t07twZf/3112ddS+rYsWNo2bIlihQpAoVCgZ07d/7nOkeOHEHlypVhaGgIJycnBAUF5fj5iYiISH7UDjfPnz/Hpk2boFAo0L59exQuXBg+Pj44deqU2k+emJiISpUqYfHixdnq//DhQzRv3hz169fHlStXMGTIEPTu3RshISFqPzcRERHJk9pzbvT09NCiRQu0aNECb9++xY4dO7BhwwbUr18fdnZ2uH//frYfq2nTpmjatGm2+wcEBMDR0RHz5s0DAJQtWxYnTpyAn58f3N3d1X0pREREJENqh5sPmZiYwN3dHW/evEFERARu3bqlqbqydPr0aTRq1Eilzd3dHUOGDPnkOikpKdIlIgAgLi4ut8ojIiKiPCBHF858+/Yt1q9fj2bNmqFo0aLw9/dHmzZtcPPmTU3XpyIyMhI2NjYqbTY2NoiLi0NSUlKW6/j6+sLMzEy62dvb52qNREREpF1qh5uOHTvC2toaQ4cORYkSJXDkyBGEh4dj6tSpKFOmTG7U+FnGjBmD2NhY6fbkyRNtl0RERES5SO3dUrq6utiyZQvc3d2hq6ursuzGjRuoUKGCxor7mK2tLaKiolTaoqKiYGpqCmNj4yzXMTQ0hKGhYa7VRERERHmL2uFm/fr1Kvfj4+OxceNGrFixAhcvXvysQ8P/S/Xq1fHXX3+ptIWGhqJ69eq59pxERET0dcnRnBvg/TlqvLy8ULhwYcydOxcNGjTAmTNn1HqMhIQEXLlyBVeuXAHw/lDvK1eu4PHjxwDe71Ly9PSU+vfr1w8PHjzAqFGjcPv2bSxZsgRbtmzB0KFDc/oyiIiISGbUGrmJjIxEUFAQVq5cibi4OLRv3x4pKSnYuXMnypUrp/aTX7hwAfXr15fuDxs2DADg5eWFoKAgPH/+XAo6AODo6Ii9e/di6NChWLBgAezs7LBixQoeBk5ERESSbIebli1b4tixY2jevDn8/f3RpEkT6OrqIiAgIMdPXq9ePQghPrk8q7MP16tXD5cvX87xcxIREZG8ZTvc7Nu3D4MGDUL//v1RqlSp3KyJiIiIKMeyPefmxIkTiI+PR5UqVVCtWjUsWrQIL1++zM3aiIiIiNSW7XDj5uaG5cuX4/nz5/D29samTZtQpEgRKJVKhIaGIj4+PjfrJCIiIsoWtY+WypcvH3r27IkTJ07g+vXrGD58OGbOnAlra2u0atUqN2okIiIiyrYcHwoOAKVLl8bs2bPx9OlTbNy4UVM1EREREeXYZ4WbDLq6uvDw8MDu3bs18XBEREREOaaRcENERESUVzDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs5Ilws3jxYjg4OMDIyAjVqlXDuXPnPtk3KCgICoVC5WZkZPQFqyUiIqK8TOvhZvPmzRg2bBgmTpyIS5cuoVKlSnB3d0d0dPQn1zE1NcXz58+lW0RExBesmIiIiPIyrYeb+fPno0+fPujRowfKlSuHgIAAmJiYYNWqVZ9cR6FQwNbWVrrZ2Nh8wYqJiIgoL9NquElNTcXFixfRqFEjqU1HRweNGjXC6dOnP7leQkICihcvDnt7e7Ru3Ro3b978ZN+UlBTExcWp3IiIiEi+tBpuXr58ifT09EwjLzY2NoiMjMxyndKlS2PVqlXYtWsX/vjjDyiVStSoUQNPnz7Nsr+vry/MzMykm729vcZfBxEREeUdWt8tpa7q1avD09MTLi4uqFu3LoKDg2FlZYVly5Zl2X/MmDGIjY2Vbk+ePPnCFRMREdGXpKfNJy9UqBB0dXURFRWl0h4VFQVbW9tsPYa+vj5cXV0RHh6e5XJDQ0MYGhp+dq1ERET0ddDqyI2BgQGqVKmCQ4cOSW1KpRKHDh1C9erVs/UY6enpuH79OgoXLpxbZRIREdFXRKsjNwAwbNgweHl5oWrVqvjhhx/g7++PxMRE9OjRAwDg6emJokWLwtfXFwAwZcoUuLm5wcnJCTExMZgzZw4iIiLQu3dvbb4MIiIiyiO0Hm46dOiAFy9eYMKECYiMjISLiwv2798vTTJ+/PgxdHT+N8D05s0b9OnTB5GRkbCwsECVKlVw6tQplCtXTlsvgYiIiPIQrYcbABgwYAAGDBiQ5bIjR46o3Pfz84Ofn98XqIqIiIi+Rl/d0VJERERE/4bhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkJU+Em8WLF8PBwQFGRkaoVq0azp0796/9t27dijJlysDIyAgVK1bEX3/99YUqJSIiorxO6+Fm8+bNGDZsGCZOnIhLly6hUqVKcHd3R3R0dJb9T506hU6dOqFXr164fPkyPDw84OHhgRs3bnzhyomIiCgv0nq4mT9/Pvr06YMePXqgXLlyCAgIgImJCVatWpVl/wULFqBJkyYYOXIkypYti6lTp6Jy5cpYtGjRF66ciIiI8iKthpvU1FRcvHgRjRo1ktp0dHTQqFEjnD59Ost1Tp8+rdIfANzd3T/Zn4iIiL4tetp88pcvXyI9PR02NjYq7TY2Nrh9+3aW60RGRmbZPzIyMsv+KSkpSElJke7HxsYCAOLi4j6n9E9SprzNlcelr0dubVvZxW2QuA2StuXGNpjxmEKI/+yr1XDzJfj6+mLy5MmZ2u3t7bVQDX0LzPy1XQF967gNkrbl5jYYHx8PMzOzf+2j1XBTqFAh6OrqIioqSqU9KioKtra2Wa5ja2urVv8xY8Zg2LBh0n2lUonXr1+jYMGCUCgUn/kK6ENxcXGwt7fHkydPYGpqqu1y6BvEbZC0jdtg7hFCID4+HkWKFPnPvloNNwYGBqhSpQoOHToEDw8PAO/Dx6FDhzBgwIAs16levToOHTqEIUOGSG2hoaGoXr16lv0NDQ1haGio0mZubq6J8ukTTE1N+UtNWsVtkLSN22Du+K8Rmwxa3y01bNgweHl5oWrVqvjhhx/g7++PxMRE9OjRAwDg6emJokWLwtfXFwAwePBg1K1bF/PmzUPz5s2xadMmXLhwAYGBgdp8GURERJRHaD3cdOjQAS9evMCECRMQGRkJFxcX7N+/X5o0/PjxY+jo/O+grho1amDDhg0YP348xo4di1KlSmHnzp2oUKGCtl4CERER5SEKkZ1px0TZkJKSAl9fX4wZMybTrkCiL4HbIGkbt8G8geGGiIiIZEXrZygmIiIi0iSGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYb+mYolUptl0BERF8Aww19MzIuwPry5UsAAK88Ql/axwGb2yBpw8fboRy/+DHc0DdlwYIF8PDwwP3796FQKLRdDn1jdHR0EBsbi5CQEADgNkhaoaOjg5iYGMyZMwdv3ryRvvjJifxeEdEHPv5mrK+vD2NjYxgYGGipIvqWKZVKzJs3D97e3tizZ4+2y6Fv2IEDBzB//nwsWrRI26XkCl4VnL4JcXFxMDU1BQDExsbCzMxMyxXRt0KpVKp8M7516xZWrlyJWbNmQVdXV4uV0bckPT1dZXtLS0vD5s2b0alTJ1luhww3JHtDhw5Feno6xowZg8KFC2u7HPoGxcTEICYmBvb29iofJB9/4BB9jo+D9MdevXqFkydPokaNGihUqJDULsftkLulSHY+zut2dnZYu3at7H556esghMCvv/6KatWq4dGjRyrLuE3S53j+/Dn++ecfvHjxAsD7uTT/Nl6xZcsWeHh44OjRoyrtctwOOXJDX7WMbxxCCCgUik9+c3nz5g0sLCy0UCHJzX99O86qT0REBMaPH4+goCBZfpDQl7d69WosXrwYT548QcmSJVGrVi3Mnj1bpU9WIzL+/v4YMGAA9PT0vmS5XxzDDX01MgIM8P6XVggBPT09PHv2DDt27ECPHj2QL18+AO93RVlYWGDChAmZ1iXKqQ9Dy+HDh/H48WM4OTmhRIkSKFKkiEqf2NhYKJXKTKFajrsA6Mvas2cP2rdvjyVLlsDExAQPHjzA7NmzUaNGDaxZswYFCxaU/ua9fPkS4eHhcHNzU3mMd+/eyTrgcLcU5VkZuTsuLg5JSUlQKBQ4cOAAwsPDoaurCz09PURERMDV1RX//POPFGwSExOhr68PPz8/vH79msGGNEIIIQWbX3/9Fd27d8fcuXPRt29fjBgxAufPnwfwftdASkoKJkyYgMqVK+PVq1cqj8NgQ5/r/PnzaN68Obp374727dtj1KhRCAkJwbVr19ClSxcA708zkJaWhnXr1qFGjRo4ceKEymPIOdgADDeUx0VGRqJixYo4evQoNmzYgCZNmuDvv/8G8H5XU/ny5dGmTRtMnz5dWidfvnwYNWoU7t27B0tLSwYb0oiM7Wju3Ln4448/sHHjRty4cQNt27bFn3/+ifHjx+P06dMAAAMDA7i6uqJhw4YwNzfXYtUkRw8fPsTz589V2r7//nvs3r0bFy9eRJ8+fQC8P/VFixYtMH369EwjN7IniPK4Hj16CFNTU6GjoyOWL18utaemporNmzeL9PR0qU2pVGqjRPpGREVFibZt24pVq1YJIYTYvXu3MDU1Ff369ROurq6iYcOG4syZM0II1W3x3bt3WqmX5CkkJETY2NiITZs2SW0Z29v69euFk5OTOH/+fKb10tLSvliN2saRG8qzMk4J7uPjg/j4eBgYGMDW1hbJyckA3n8rad++vcrETY7SUG6ytrbGqFGj0KRJE1y+fBk+Pj6YNm0ali5dip9++glnzpyBj48PLl68qLItclcUaVLZsmVRr149rFu3DocOHQLwv799Li4uiI6Oli4z8yG574r6EMMN5VkZocXe3h4nTpyAl5cXOnbsiF27diEpKSlTfzleH4W051Pbk6urKwoXLox9+/bB2dkZffv2BQBYWlrCzc0NLVu2hKur65cslb4x9vb26NevH2JiYuDn54fdu3dLywoXLgxHR0ctVpc3fDsxjr4a4v8nAD9//hxpaWkoVqwYrK2tUaNGDSQnJ6NXr14ICgpCixYtYGRkhICAADRq1AhOTk7aLp1kQnwweXjFihWIjo6GgYEBRowYIV26IyUlBc+ePcOjR49QunRpHDhwAK1atcLAgQP/9bQERJ8j42i7evXqYcmSJRg7dixGjx6NkJAQODs7Y8uWLVAoFPjxxx+1XapW8VBwypOCg4MxadIkREVFoXnz5mjTpg1atmwJAOjRowd27NiB4cOHIyoqCkuXLsX169dRrlw5LVdNcjNx4kT4+/vj+++/x7lz51CtWjWsW7cOtra2+PPPPzFt2jS8efMG+vr6EELg2rVr0NPT4xF6lCsytqvg4GAsWbIEBw4cwO3btxEWFoZFixbB3t4e5ubmWL9+PfT19b/p0w4w3FCec/PmTbi7u2Po0KEwMTHBxo0bYWhoCC8vL3Tt2hUAMHjwYFy6dAkpKSkIDAyEi4uLdosmWfhwtOXdu3fw8vLCwIED4erqikePHqF58+awtbXFjh07YGVlhb179yI8PBwJCQkYPXo09PT0vukPFNKMjBAjPjq3l66uLoKDg+Hp6Yn58+dLu0SB99urjo6Oyvb7Lc2x+RjDDeUpt2/fxtatW5GUlIQZM2YAAK5fv44JEyYgLi4OPXr0kAJOZGQk8uXLhwIFCmizZJKJD4PNrVu3EBcXh2XLlmHChAlwcHAA8P4Q3B9//BE2NjbYuXMnrKysVB6DwYY+14fb4cuXL6FQKFCwYEEA7//mVa5cGRMmTEC/fv2kdT4eKeTIIcMN5RFCCLx58wYtWrTA33//jZYtW2LdunXS8mvXrmHChAlISkpCx44d0aNHDy1WS3I2cuRIaVg/KioKwcHBaNq0qfRh8fDhQzRt2hRCCJw8eVLlAoREn+PDUDJ16lTs3LkTcXFxKFSoEKZPn44GDRrg2bNnKFq0qJYrzfs4243yBIVCAUtLS/j6+qJ8+fK4dOkSQkNDpeXOzs6YOnUq0tLSpF94Ik348KioPXv2YP/+/fj999+xZMkSODo6Yty4cbh69ap0xmxHR0fs2bMHLi4uvF4ZaVRGsJkyZQoWLFggnWqgUKFC6NKlC9asWZNptJCyxpEb0ppPDZ0ePXoUY8eOha2tLXx8fNCgQQNp2c2bN2FmZgY7O7svWSp9A4KDg3Hq1CkULFgQY8aMAQAkJCSgcuXKMDU1xYoVK1CpUqVM2yx3RZEmvXr1Co0bN4aPjw969uwptfft2xd//vknwsLCUKZMGe56+g8cuSGtyPjFPHXqFObPn4/ffvsNJ0+eRFpaGurWrYspU6YgMjISixYtwpEjR6T1ypcvz2BDGpeUlITffvsN8+fPx82bN6X2/Pnz49KlS4iPj4e3t7d0/agPMdiQJr179w4vX76URgUzTloaGBiIIkWKwM/PDwBPWPpfGG7oi/vwcMamTZvi5MmT2L17N8aOHYvp06cjNTUVDRs2xJQpU/Dq1StMnToVx48f13bZJGPGxsY4fvw4GjVqhIsXL2L37t1IT08H8L+Ac/v2bSxbtkzLlZKcZLXjxMbGBra2tli1ahUAwMjICKmpqQAAJycnhppsYrihLy5jxGbQoEGYP38+tm/fjq1bt+LixYvYvHkzxo8fLwWcX3/9Ffr6+jzjJmnMh3NshBDSB4ylpSU2bNgACwsLzJkzByEhIdKyfPnyITIyEoGBgVqpmeRHqVRKQeWff/5BdHQ03r59CwCYNGkSbt++LR0RlXHiyKdPn/JCrNnEOTf0xWT8MisUCixZsgRXrlxBYGAgHj58iEaNGqFWrVowNTXF1q1b4e3tjbFjx8LQ0BBv376FiYmJtssnGfjwMNuFCxfi6tWrePDgAYYMGYLKlSvDzs4OL168QOvWraGrq4uxY8fC3d1d5UzDnGNDn2P9+vVwc3NDyZIlAQBjxoxBSEgIIiIi0KhRI7Rq1QpdunTB8uXLMXXqVBQsWBAVKlTA/fv3ERMTI50okv4dww3lmowPkg/DyZUrV+Di4oK4uDg8efIETk5OaNKkCRwdHbFq1SrExsZKZxru3r07pk+fzolz9Nk+3obGjBmDlStXom/fvnj69ClOnz6N1q1bo2/fvnBycsKLFy/Qtm1bvHjxAkFBQXBzc9Ni9SQX+/btQ4sWLTB69GgMGTIE+/btw6hRo+Dv749Xr17h0qVLCAkJwW+//YZ+/frh+vXr8Pf3h46ODiwsLDBjxgyeKDK7cvWa4/TNe/DggejUqZP4+++/xZYtW4RCoRDnzp0TSqVSCCHE9evXRZkyZcTZs2eFEELcv39ftGjRQowdO1Y8fvxYm6WTzKSnpwshhFi3bp1wdHQUFy9eFEIIcfz4caFQKESpUqXE4MGDxYMHD4QQQjx//lz07dtXvHv3Tms1k/wsWrRI2NnZialTp4oBAwaI5cuXS8uePHkipkyZIhwcHMT+/fuzXD8tLe1LlfpV49gW5ark5GQcP34c3bt3x5UrV7B69Wp8//330i4qIQTevXuH06dPo3z58li7di0AYMSIETyHCH22bt26wcrKCvPnz4eOjg7S0tJgYGCAfv36oXLlyti5cyd69OiBFStWIDIyEtOmTYOOjg769OmDsmXLShOI+U2ZPldqaioMDAzg4+MDExMTjBkzBvHx8Zg2bZrUx87ODp6enjhw4AAuXLgAd3f3TBdg5S6pbNJ2uiL5yvimHBAQIHR0dESlSpXE5cuXVfrExsaK7t27i5IlSwoHBwdhZWUlfaMm+hyxsbFi8uTJwtLSUkyaNElqf/bsmYiKihLPnz8XVatWFfPmzZP6FylSRBQuXFgsWLBACCGkEUYiTfH19RXR0dFi/fr1wsTERDRr1kzcvXtXpU+HDh1E27ZttVShPPBoKcoVQgjo6OhACIEiRYpg3rx5ePfuHcaPH48TJ05I/UxNTTF37lwsWbIEEydOxNmzZ1G5cmUtVk5yEB8fD1NTU/Tv3x/jx4+Hv78/Jk6cCAAoUqQIrK2t8fz5c7x580aaT/Ps2TM0btwYEyZMgI+PDwCeS4Q+n/hgWuuaNWswdepU3Lt3D507d4afnx8uXbqEgIAA3LlzBwAQFxeHhw8folixYtoqWRY4vkUaJ/5/8ubhw4dx9OhRDBkyBC1btkSjRo3Qvn17zJw5E2PHjkWNGjUAvL8wZuPGjbVcNcnFqFGjsGzZMty/fx9WVlbo2rUrhBCYOnUqAGDy5MkA3gcgXV1dnDx5EkIIzJw5EyYmJtLht9wVRZqQEZAPHTqEy5cvIzAwUPrb17dvX6SlpWHy5MnYv38/KleujMTERKSmpmL27NnaLPvrp81hI5KfjGH8bdu2CTMzMzFmzBhx/vx5afm1a9dEuXLlRIsWLcQff/whJk2aJBQKhXjy5Al3AZBGXL16VdSpU0eULl1avHjxQgghRHR0tJg3b54wNzcXEyZMkPoOGDBAlCxZUtjZ2Qk3NzeRmpoqhODuKNKsI0eOiIoVK4qCBQuKnTt3CiGESElJkZavXLlS5M+fX1SuXFmsXbtWmsTOycM5x0PBSePOnTuHJk2aYNasWejTp4/UHhcXB1NTU9y6dQt9+vRBUlISYmNjsWXLFu6KIo04ffo0Xrx4gXLlyqFDhw5ISEiQrtz94sULrFu3DlOnTpUuSAi8Pz2BQqFAxYoVoaOjg3fv3nHSJn0W8dGpBxISEjBnzhwEBgaiWrVq2LhxI4yNjZGWlgZ9fX0AwPz583Hq1Cls3boVCoWCI4efieGGNG7RokXYsWMHDh06hNjYWBw+fBh//PEHbt26hREjRqBnz56Ijo5GbGwszMzMYG1tre2SSSY8PT3xzz//4ODBg3j06BF+/vlnxMfHZwo406ZNw4ABAzBlyhSV9fmBQpq0ePFi2NnZoXXr1khKSsLcuXOxY8cO1KtXDzNmzICRkZFKwMkIRR+HI1IfJxSTxtna2uLixYvw9fXFzz//jNWrV8PIyAjNmzdH7969cffuXVhbW6NUqVIMNqRRixcvxtOnT7Fo0SI4ODhg48aNMDMzQ82aNfHy5UtYWVmhW7dumDBhAqZNm4aVK1eqrM9gQ5ry4sULHD58GL/88gv2798PY2NjDBs2DC1atMCpU6cwbtw4JCcnQ19fH+/evQMABhsN4sgNfZaMX8SEhATkz58fABAVFYWFCxdiy5YtaNCgAbp3744ffvgBUVFRaNWqFYKCglC+fHktV05ykzHq8vvvv+Py5cuYP38+LCwscPv2bXh6eiI2NlYawYmMjMTRo0fx008/cRcUacTH56MBgKtXr+L333/HwYMHERAQgKZNmyIxMRGzZ8/GwYMHUbZsWSxZskS6dhRpDkdu6LMoFArs3bsXnTp1Qr169RAUFAQ9PT1MmzYNZ8+eRUBAANzc3KCjo4OFCxciMTGRozWUKzJGXerVq4djx45h7969AIDSpUtj3bp1sLCwQJ06dRAVFQVbW1t06NABenp60rdmos+REWwiIyOltkqVKmHw4MGoX78++vXrh/379yNfvnwYNWoUfvjhB+jo6Ei7pEjDtDSRmWTi5MmTwsjISIwcOVI0adJEODs7C29vbxEeHi71CQsLE3379hWWlpaZTuJHlFMZJ4nMSkBAgPjuu+/EnTt3pLY7d+4IBwcH0bFjxy9RHn0jPtwON23aJEqUKKFyhKgQQly5ckW0bt1aFCtWTBw5ckQIIURSUpJ0VN6/bcuUMxy5oRyLiIhAaGgopk+fjtmzZ2Pfvn3o27cvrl27Bl9fXzx48ACJiYk4ffo0oqOjcfToUbi4uGi7bJKBD3cBnDt3DqdOncLRo0el5a1atUK1atUQFhYmtX333Xc4duwY/vjjjy9eL8lTSkqKtB2mpqaiZMmSKFOmDHx8fHDx4kWpX6VKleDh4YEnT56gcePGOHXqFIyMjKQ5Nh/vzqLPx58oZcuiRYvw119/Sffv3LmDDh06YNWqVTAyMpLafXx80KVLF9y8eROzZ89GTEwMRo4ciTVr1qBChQraKJ1k5sMPg7Fjx6J79+7o2bMnvLy80KFDB8TFxaFw4cLSfIa0tDRpXXt7e+jq6iI9PV1b5ZNM7Nu3D+vWrQMA9OnTBw0aNEDVqlUxfPhw2NrawtvbGxcuXJD6FytWDB07dsS8efNQrVo1qZ2Th3OJtoeOKO97+PCh6Ny5s7h3755K+6+//iqsra1F27ZtpZOlZVi6dKkoXbq0GDRoEE9ERbli7ty5omDBguLs2bMiPT1dzJgxQygUCnHixAmpT82aNYW3t7cWqyS56tSpk3BwcBDu7u6iUKFC4urVq9Kyw4cPCw8PD1GhQgWxb98+8fDhQ+Hh4SGGDx8u9eHV5nMXww1lS2JiohBCiDNnzoht27ZJ7RMmTBAVK1YU48ePF1FRUSrrLF++XDx8+PBLlknfCKVSKby8vERgYKAQQojt27cLc3NzERAQIIQQIj4+XgghxL59+0SrVq3EtWvXtFYryZeLi4tQKBQqF2bNcPz4cdGtWzehUCjEd999J5ydnaUvejwDdu7jMZCULcbGxoiJiYGvry+ePXsGXV1deHh4YPLkyUhLS8PevXshhMDgwYNhZWUFAOjdu7eWqya5Sk5OxtmzZ1GvXj0cOXIEXl5emDNnDry9vfHu3TvMnj0b1atXh5ubG6ZMmYJz586hYsWK2i6bZCI1NRXJyclwcnJCsWLFsHnzZhQtWhQdO3aUTolRq1YtVKtWDX369EFaWhrq1q0LXV1dngH7C+GcG8oWhUIBc3NzDB8+HI6OjvD390dwcDAAYMaMGWjSpAlCQ0MxY8YMvHz5UsvVkpxcu3YNT58+BQAMHToUR48ehbGxMTp37ow//vgDzZo1g5+fn3TByzdv3uDChQu4c+cOLCwssG7dOhQvXlybL4FkxsDAAKampti6dSt27dqF77//HrNnz8amTZsQHx8v9UtOTkbt2rXRoEEDaa4Xg82XwXBD2SLe78JE7dq1MXToUFhYWOD3339XCThubm64fPkyBM8LSRoghMDdu3dRv359rFq1Cv369cOCBQtgYWEBAHBzc0NERASqVauG6tWrAwD++ecfdO/eHTExMRgwYAAAoGTJkmjUqJHWXgfJjxACSqVSur9mzRrUqFEDfn5+WLt2LR4/fowGDRqgXbt2Un+AZ8D+kniGYsqWjLO/xsbGwsTEBNeuXcP06dPx5s0bDB48GB4eHgDen3I8Y7cUkSYsX74co0aNQnJyMnbt2oXGjRtLZ8bevHkzpkyZAiEE9PT0YGxsDKVSiVOnTkFfX5/XiqLP9vr1a1haWqq0ZWx/W7duRWhoKAIDAwEAffv2xZEjR5Ceng5LS0ucPHmSZx/WEo7c0H969+4ddHV18ejRI9SrVw8HDhxAlSpVMGLECFhZWWHy5MnYs2cPADDYkMZkfDO2t7eHoaEhTE1NcebMGTx69Eg6fLZDhw5Yu3YtpkyZgvbt22P06NE4c+aMdL0eBhv6HAsWLMD333+vsqsJgBRsunfvjkqVKkntgYGBWLZsGRYuXIgzZ87AwMCAZ8DWFu3MY6a86lOz+MPDw4WNjY3o3bu3yiGMR44cEd26dROPHj36UiWSzH28DaampoqkpCSxdOlSUbRoUTF27Nj/3N54mC19rmXLlglDQ0OxYcOGTMseP34sKlasKBYtWiS1ZbXNcTvUHu6WIon4/6HW06dP49atWwgPD4enpycKFy6MNWvW4MKFC1izZk2mK9cmJyernMiPKKc+PPPw69evER8frzIZ2N/fH3PnzkWvXr3Qo0cPODg4oGXLlhg3bhzc3Ny0VTbJzPLlyzFw4ECsW7cO7dq1Q0xMDBITE5GcnAxra2sUKFAA9+7dQ6lSpbRdKn0Cww2p2L59O/r27StdYPDFixfo0KEDRo8ejQIFCmi7PJKxD4PNlClTcODAAdy4cQPt27dHmzZt0LRpUwDvA46/vz8qVKiAV69e4fHjx3j06BEvQEga8eDBAzg5OaF9+/bYtGkTbty4gV9++QUvXrxAREQE6tevj/79+6NFixbaLpX+BY9JI8mNGzcwdOhQzJs3D927d0dcXBzMzc1hbGzMYEO5LiPYTJgwAYGBgZgzZw4cHBzQr18/3Lt3DzExMejUqROGDBmCQoUK4erVq0hOTsbx48elq3vzMFv6XFZWVpg1axYmTJiAESNG4MCBA6hduzZat26NuLg4bNu2DePHj0ehQoU4WpiXaXOfGGnP4cOHxf379zO1Va9eXQghxK1bt0Tx4sVF7969peX379/nPmTKVYcPHxbly5cXx44dE0IIcerUKWFgYCDKlSsnqlWrJrZu3Sr1/fCyHrzEB2lScnKymDt3rtDR0RE9e/YUqamp0rILFy6I0qVLi8WLF2uxQvovPFrqGyOEwOXLl9G0aVMsXboUERER0rJnz55BCIGEhAQ0adIEjRs3xrJlywAAoaGhWLp0Kd68eaOt0kmGxEd7xYsWLYr+/fujdu3aOHDgAFq0aIHAwECEhobi/v37+P3337Fy5UoAUBml4YgNaZKhoSH69euH7du3o3fv3tDX15e21SpVqsDIyAhPnjzRcpX0bxhuvjEKhQKurq6YN28etmzZgqVLl+LBgwcAgObNmyMqKgqmpqZo3rw5AgMDpV0FISEhuHbtGg+tJY1RKpXSpPQHDx4gMTERpUqVQqdOnZCcnIwFCxZg0KBB6NatG4oUKYLy5csjPDwct27d0nLl9C3Ily8fmjZtKp0gMmNbjY6OhrGxMcqXL6/N8ug/8OvONyZjXoKPjw8AYM6cOdDV1UXv3r3h6OiI3377DTNmzMC7d+/w9u1bhIeHY+PGjVixYgVOnDghnR2W6HN8OHl4woQJOH36NEaOHIn69evD0tISiYmJeP78OUxMTKCjo4OUlBQ4ODhg1KhRaNKkiZarJzkSHxwBmsHQ0FD6f3p6Ol6+fIk+ffpAoVCgU6dOX7pEUgPDzTcmY+TlwIED0NHRQVpaGvz9/ZGcnIzRo0ejffv2SEpKwowZM7Bt2zbY2NjAwMAAYWFhqFChgparJ7n4MNgsW7YMgYGBcHV1lY54SklJgaWlJU6cOCFNGn716hVWrVoFHR0dlXBElBMRERF4/fo1ChYsCFtb2389k3BaWhrWrVuHjRs34vXr1zhz5ox0rSiOZudNPBT8GxQSEiJdbDBfvny4d+8efv/9d/zyyy8YPXo0rKysEB8fj6NHj8LBwQHW1tawtrbWdtn0lfs4kNy9exceHh6YNWsWWrZsmanf+fPnMX78eCQkJMDS0hLBwcHQ19dnsKHPtnbtWsybNw/R0dEoVKgQBg4cKI3IZPh4OwsNDcXNmzcxYMAAHp33FWC4+cYolUp06dIFCoUCGzZskNoXLlyIUaNGwcfHB7/88gtKlCihxSpJbtq2bYuxY8eiatWqUtuVK1fQpEkTHD16FKVLl87yxJDJyckQQsDIyAgKhYIfKPTZ1q5dCx8fH+nSCjNmzMCDBw9w8uRJadvKCDYxMTE4cOAA2rdvr/IYHLHJ+/j15xuT8U0kY/g/NTUVADBw4EB4e3tj9erV+P3331WOoiL6XGZmZnB2dlZpMzIywps3b3Djxg2pLeN6UqdPn8b27duho6MDY2NjKBQKKJVKBhv6LBcuXMDUqVOxaNEi9OzZExUrVsTQoUPh5OSEU6dO4ebNm4iLi5N22a9Zswa//PIL/vjjD5XHYbDJ+xhuvhH//POP9P/SpUvjzz//RHR0NAwMDJCWlgYAsLOzg4mJCcLCwmBsbKytUklGnj17BgBYvXo1DAwM8Pvvv+PAgQNITU2Fk5MTOnTogDlz5uDgwYNQKBTQ0dFBeno6pk+fjrCwMJV5ENwVRZ8rJSUFQ4YMQfPmzaW2SZMm4dChQ+jUqRM8PT3RsWNHvH79Gvr6+mjWrBlGjBjBycNfIe6W+gZcvXoVAwYMQOfOndG/f3+kpqaiQYMGePnyJY4cOQJbW1sAwOjRo1G+fHm0aNEClpaWWq6avnZ9+vQBAIwZM0bazens7IyXL19i06ZNqFOnDo4fPw4/Pz9cv34dXbp0gYGBAQ4dOoQXL17g0qVLHKkhjVIqlXjx4gVsbGwAAJ6enjh48CB2794Ne3t7HD16FNOmTcPo0aPRuXNnlTk43BX1deFXoW+AiYkJzM3NsW3bNgQFBcHAwADLli2DlZUVypYtCw8PDzRu3BgLFixA1apVGWxII5ydnbF//34sXboU4eHhAIBr166hdOnS6NKlC44dO4batWtjypQp8PT0xLp163D48GEUK1YMFy9elCZtEmmKjo6OFGwAYMSIETh79iyqVq0KGxsbNG3aFK9fv0ZUVFSmw8IZbL4uHLn5RoSHh2Ps2LGIjIxEnz590K1bN6Snp2Pu3LmIiIiAEAIDBw5EuXLltF0qyciqVaswYcIEdOzYEX369EHp0qUBAHXq1MHDhw+xfv161KlTBwDw9u1bmJiYSOty8jB9aU+fPkXXrl0xYsQIXhjzK8dwI1OXLl3C8+fPVfYth4eHY/z48Xj06BEGDhyILl26aLFCkrMPD6NduXIlJkyYgE6dOmUKOBEREVi7di2qV6+uMr8mqxOqEanjw20o4/8Z/7548QJWVlYq/RMTE9GpUyfExsbi8OHDHKn5yjHcyFB8fDyaN28OXV1djBo1Ck2bNpWWPXr0CE2aNIGJiQl69+6NX375RYuVktx86hw0y5cvx+TJk9GhQwf07dtXCjgNGjTAyZMncebMGbi6un7pckmmstoOM9qCg4OxceNGLFiwAEWKFEFSUhJ27dqFdevW4dmzZzh//jz09fU5x+Yrxzk3MpKRUwsUKIDZs2dDT08PixYtwt69e6U+Dg4OqF+/PiIjI3Ho0CHExMRoqVqSmw8/UE6dOoWwsDBcvXoVwPvJxb/99hs2bdqEwMBA3LlzBwBw+PBh9O7dO9Nh4kQ5deLECemilsOGDcPMmTMBvJ9vs3nzZnh6eqJRo0YoUqQIgPcXXX348CFKlCiBCxcuQF9fH+/evWOw+cpx5EYGMoZaM75pZHzInD17Fr/++ivy5cuH/v37S7uohg8fjhIlSqBt27YoXLiwlqsnOfhwF8CwYcOwefNmJCQkwM7ODsWKFcO+ffsAAMuWLcO0adPQsWNHeHl5qVzSg9+U6XMIIRAbGwtra2s0bdoUhQoVQnBwMI4fP44KFSogJiYGbm5u8PHxwcCBA6V1PvzbCXA7lAuGm69cxi9nWFgYdu/ejdevX6NWrVpo164dzM3NcebMGfz2229ISUlBiRIlYGJigs2bN+Pq1auws7PTdvkkAx8GmwMHDmDIkCEIDAyEubk5/v77b0ycOBH58uXDhQsXALyfg+Pt7Q1/f38MGDBAm6WTDEVHR6NEiRJIT0/H9u3b0axZM2lZVnNtspqbQ18/7pb6yikUCuzYsQMtW7bE27dv8fbtW6xbtw79+/fH69ev4ebmhrlz56Ju3boIDw/HgwcPcPjwYQYb0piMD4Pdu3dj06ZNaNSoEWrVqoUKFSrg559/xtq1a5GQkID+/fsDAHr16oVdu3ZJ94k0JSUlBZGRkTAxMYGuri5WrVolnYYAAAoVKiT9P+Ns2B+GGQYb+eDIzVfuwoUL6NixI3799Vf07t0bERERqFy5MoyNjeHi4oK1a9fC0tJSulbPx4fbEmnC69ev0aJFC1y9ehX169fHnj17VJaPHTsWJ0+exF9//YV8+fJJ7dwFQJ/rU5PYHz16BGdnZ9SvXx/z589HyZIltVAdaQtHbr4ivr6+GDdunPSNA3h/ens3Nzf07t0bjx49QsOGDeHh4YHx48fj/Pnz+OWXX/D69WsYGRkBAIMNacSH2yAAWFpaYs2aNfjxxx9x+fJlrF69WmV5qVKl8OrVKyQlJam0M9jQ5/gw2Bw5cgQbNmzA1atX8ezZMzg4OODkyZMICwvDqFGjpEnsbdq0wcKFC7VZNn0BHLn5iixcuBCDBw/GjBkzMGrUKOmX+tatWyhdujRat24tfcgolUq4uLggPDwczZs3x+bNm3ltHtKIDz9Q7t+/D4VCARMTE9ja2uLhw4fw8fFBYmIi2rVrB29vb0RFRcHLywtGRkbYs2cPh/5J40aMGIE1a9ZAT08P+fPnh62tLfz8/FC1alVcv34d9evXh4ODA1JTU/Hu3TtcvXpVungwyZSgr4JSqRRCCLF8+XKho6Mjpk6dKtLS0qTlT548EWXLlhV79uwRQgjx+vVr0alTJ7Fw4ULx9OlTrdRM8pOxHQohxMSJE0XFihVFmTJlROHChUVgYKAQQojw8HDRrFkzYWRkJEqXLi3atGkj3N3dRVJSkhBCiPT0dK3UTvLx4XYYGhoqKlWqJI4fPy5ev34tdu3aJdq0aSOcnJzEpUuXhBBC3Lt3T0yZMkVMnz5d+rv54d9Pkh+Gm6+AUqmUfpmVSqX4448/hI6Ojpg2bZr0QREdHS1cXFyEt7e3ePTokRg7dqz4/vvvRVRUlDZLJ5maMmWKsLKyEiEhISIhIUG0adNGmJubi5s3bwohhHjw4IFo3ry5cHFxEX5+ftJ6ycnJWqqY5GjNmjViwIABom/fvirt58+fF02aNBFeXl4iISFBCKEaiBhs5I/7Kb4SCoUCBw8exPDhw1GlShXpmj0zZ86EEAIWFhbo0qULjh49Cjc3N6xduxYBAQGwtrbWdukkAx/OsVEqlTh37hz8/PzQuHFjhIaG4siRI5gxYwbKlSuHtLQ0ODo6Yt68ebCxscHevXsRHBwMADA0NNTWSyAZEB/Noti5cycWL16MK1euICUlRWqvWrUqateujRMnTiA9PR2A6pFQvGbZN0Db6YqyZ/v27cLY2FhMnTpVnD9/XgghRGBgoLSLSgghUlJSxM2bN0VoaKh48uSJNsslmZowYYKYOXOmKFq0qLhz544ICwsT+fPnF0uXLhVCCPH27Vsxbtw48ejRIyGEEHfv3hUtWrQQVatWFcHBwdosnb5yH468rF+/Xqxdu1YIIcSAAQOEubm5WLx4sYiNjZX6hISEiDJlykjbIn1bGG6+Anfu3BGOjo5iyZIlmZYtW7ZM2kVFpGkfzo/ZtGmTsLe3Fzdu3BBdu3YV7u7uwsTERKxcuVLq8+zZM1G7dm2xdu1aad1bt26Jn3/+WURERHzx+kkePtwOb9y4IVxdXUWlSpXErl27hBBCeHl5iVKlSonp06eL8PBwER4eLho2bCjq1q2rEoro28Gxua/A48ePoa+vr3KmzYwjVvr27Yt8+fKhW7duMDQ0xIgRI7RYKclNxlFRR48exZEjRzB8+HCUL19eOjlkw4YN0bNnTwDvL9jau3dv6OrqonPnztDR0YFSqUSZMmWwYcMGHp1COZaxHY4cORIPHz6EsbExbt++jaFDh+Ldu3cICgpCz549MX78eCxcuBA1a9ZE/vz5sXnzZigUik+eC4fki+HmK5CQkKByfhClUintPz5y5AiqVKmCzZs3q1ynh0hTIiMj0atXL0RHR2Ps2LEAgH79+uH+/fs4fPgwXF1dUapUKTx+/BjJyck4f/48dHV1VU7QxzkO9LmCgoKwYsUKHDp0CI6OjkhJSYGXlxd8fX2ho6ODVatWwcTEBFu2bEGTJk3QsWNHGBoaIjU1FQYGBtoun74wRtmvQKVKlfDy5UsEBgYCeP8tJiPc7Nq1Cxs2bEDbtm1RtmxZbZZJMmVra4vg4GDY2Njgzz//xMWLF6Grq4s5c+ZgypQpaNCgAWxtbdGhQ4dPXlWZ57ahzxUeHo4KFSrAxcUFZmZmsLW1xapVq6Crq4uhQ4dix44dWLRoERo1aoT58+dj9+7diI+PZ7D5RvHr1FfA0dERixYtQr9+/ZCWlgZPT0/o6uoiKCgIQUFBOH36NM/0SrnK2dkZ27dvh5eXFwICAjBw4EA4OzujVatWaNWqlUrf9PR0jtSQxoj/v5iloaEhkpOTkZqaCiMjI6SlpaFo0aLw9fVFixYt4O/vD2NjY2zYsAGdO3fGiBEjoKenh/bt22v7JZAW8AzFXwmlUont27fD29sb+fLlg5GREXR1dbFx40a4urpquzz6Rly+fBm9e/dGlSpVMHjwYJQvX17bJdE34vr163B1dcVvv/2GiRMnSu0hISFYvnw53rx5g/T0dBw5cgQA0KNHD/z2228oUaKEliombWK4+cr8888/iIiIgEKhgKOjI2xsbLRdEn1jLl++DG9vbxQvXhyzZ8+Go6Ojtkuib0RQUBD69u2LIUOGoEOHDrCwsMCgQYNQo0YNtGnTBuXLl8fevXvRtGlTbZdKWsZwQ0RqO3fuHAICArBixQoehUJf1Pbt2/HLL7/AwMAAQghYW1vj1KlTiIqKwo8//oht27bB2dlZ22WSljHcEFGOZMyF4GG29KU9e/YMT548QVpaGmrWrAkdHR2MGTMGO3fuRFhYGGxtbbVdImkZww0R5VhGwCHSlps3b2LWrFn466+/cPDgQbi4uGi7JMoDeEgDEeUYgw1p07t375Camgpra2scPXqUE9xJwpEbIiL6qqWlpfEM2KSC4YaIiIhkhbMAiYiISFYYboiIiEhWGG6IiIhIVhhuiIiISFYYbohI9o4cOQKFQoGYmJhsr+Pg4AB/f/9cq4mIcg/DDRFpXffu3aFQKNCvX79My3x8fKBQKNC9e/cvXxgRfZUYbogoT7C3t8emTZuQlJQktSUnJ2PDhg0oVqyYFisjoq8Nww0R5QmVK1eGvb09goODpbbg4GAUK1YMrq6uUltKSgoGDRoEa2trGBkZoVatWjh//rzKY/3111/47rvvYGxsjPr16+PRo0eZnu/EiROoXbs2jI2NYW9vj0GDBiExMTHXXh8RfTkMN0SUZ/Ts2ROrV6+W7q9atQo9evRQ6TNq1Chs374da9aswaVLl+Dk5AR3d3e8fv0aAPDkyRO0bdsWLVu2xJUrV9C7d2/8+uuvKo9x//59NGnSBD/99BOuXbuGzZs348SJExgwYEDuv0giynUMN0SUZ3Tt2hUnTpxAREQEIiIicPLkSXTt2lVanpiYiKVLl2LOnDlo2rQpypUrh+XLl8PY2BgrV64EACxduhQlS5bEvHnzULp0aXTp0iXTfB1fX1906dIFQ4YMQalSpVCjRg38/vvvWLt2LZKTk7/kSyaiXMALZxJRnmFlZYXmzZsjKCgIQgg0b94chQoVkpbfv38faWlpqFmzptSmr6+PH374Abdu3QIA3Lp1C9WqVVN53OrVq6vcv3r1Kq5du4b169dLbUIIKJVKPHz4EGXLls2Nl0dEXwjDDRHlKT179pR2Dy1evDhXniMhIQHe3t4YNGhQpmWcvEz09WO4IaI8pUmTJkhNTYVCoYC7u7vKspIlS8LAwAAnT55E8eLFAby/IvT58+cxZMgQAEDZsmWxe/dulfXOnDmjcr9y5cr4+++/4eTklHsvhIi0hnNuiChP0dXVxa1bt/D3339DV1dXZVm+fPnQv39/jBw5Evv378fff/+NPn364O3bt+jVqxcAoF+/frh37x5GjhyJO3fuYMOGDQgKClJ5nNGjR+PUqVMYMGAArly5gnv37mHXrl2cUEwkEww3RJTnmJqawtTUNMtlM2fOxE8//YRu3bqhcuXKCA8PR0hICCwsLAC83620fft27Ny5E5UqVUJAQABmzJih8hjOzs44evQo7t69i9q1a8PV1RUTJkxAkSJFcv21EVHuUwghhLaLICIiItIUjtwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs/B+XLE52CERTBAAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "## calculate avg response time\n", - "unique_models = set(unique_result[\"response\"]['model'] for unique_result in result[0][\"results\"])\n", - "model_dict = {model: {\"response_time\": []} for model in unique_models}\n", - "for iteration in result:\n", - " for completion_result in iteration[\"results\"]:\n", - " model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n", - "\n", - "avg_response_time = {}\n", - "for model, data in model_dict.items():\n", - " avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n", - "\n", - "models = list(avg_response_time.keys())\n", - "response_times = list(avg_response_time.values())\n", - "\n", - "plt.bar(models, response_times)\n", - "plt.xlabel('Model', fontsize=10)\n", - "plt.ylabel('Average Response Time')\n", - "plt.title('Average Response Times for each Model')\n", - "\n", - "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n", - "plt.show()" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb b/cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb deleted file mode 100644 index 7df1c47eb11..00000000000 --- a/cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb +++ /dev/null @@ -1,422 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "BmX0b5Ueh91v" - }, - "source": [ - "# LiteLLM - Azure OpenAI + OpenAI Calls\n", - "This notebook covers the following for Azure OpenAI + OpenAI:\n", - "* Completion - Quick start\n", - "* Completion - Streaming\n", - "* Completion - Azure, OpenAI in separate threads\n", - "* Completion - Stress Test 10 requests in parallel\n", - "* Completion - Azure, OpenAI in the same thread" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "iHq4d0dpfawS" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "mnveHO5dfcB0" - }, - "outputs": [], - "source": [ - "import os" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "eo88QUdbiDIE" - }, - "source": [ - "## Completion - Quick start" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "5OSosWNCfc_2", - "outputId": "c52344b1-2458-4695-a7eb-a9b076893348" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Openai Response\n", - "\n", - "{\n", - " \"id\": \"chatcmpl-7yjVOEKCPw2KdkfIaM3Ao1tIXp8EM\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1694708958,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I'm an AI, so I don't have feelings, but I'm here to help you. How can I assist you?\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 26,\n", - " \"total_tokens\": 39\n", - " }\n", - "}\n", - "Azure Response\n", - "\n", - "{\n", - " \"id\": \"chatcmpl-7yjVQ6m2R2HRtnKHRRFp6JzL4Fjez\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1694708960,\n", - " \"model\": \"gpt-35-turbo\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"finish_reason\": \"stop\",\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello there! As an AI language model, I don't have feelings but I'm functioning well. How can I assist you today?\"\n", - " }\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"completion_tokens\": 27,\n", - " \"prompt_tokens\": 14,\n", - " \"total_tokens\": 41\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "# openai configs\n", - "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "\n", - "# azure openai configs\n", - "os.environ[\"AZURE_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n", - "\n", - "\n", - "# openai call\n", - "response = completion(\n", - " model = \"gpt-3.5-turbo\",\n", - " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - ")\n", - "print(\"Openai Response\\n\")\n", - "print(response)\n", - "\n", - "\n", - "\n", - "# azure call\n", - "response = completion(\n", - " model = \"azure/your-azure-deployment\",\n", - " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - ")\n", - "print(\"Azure Response\\n\")\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dQMkM-diiKdE" - }, - "source": [ - "## Completion - Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "uVvJDVn4g1i1" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "# openai configs\n", - "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "\n", - "# azure openai configs\n", - "os.environ[\"AZURE_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n", - "\n", - "\n", - "# openai call\n", - "response = completion(\n", - " model = \"gpt-3.5-turbo\",\n", - " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True\n", - ")\n", - "print(\"OpenAI Streaming response\")\n", - "for chunk in response:\n", - " print(chunk)\n", - "\n", - "# azure call\n", - "response = completion(\n", - " model = \"azure/your-azure-deployment\",\n", - " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True\n", - ")\n", - "print(\"Azure Streaming response\")\n", - "for chunk in response:\n", - " print(chunk)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4xrOPnt-oqwm" - }, - "source": [ - "## Completion - Azure, OpenAI in separate threads" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "V5b5taJPjvC3" - }, - "outputs": [], - "source": [ - "import os\n", - "import threading\n", - "from litellm import completion\n", - "\n", - "# Function to make a completion call\n", - "def make_completion(model, messages):\n", - " response = completion(\n", - " model=model,\n", - " messages=messages\n", - " )\n", - "\n", - " print(f\"Response for {model}: {response}\")\n", - "\n", - "# openai configs\n", - "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "\n", - "# azure openai configs\n", - "os.environ[\"AZURE_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n", - "\n", - "# Define the messages for the completions\n", - "messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n", - "\n", - "# Create threads for making the completions\n", - "thread1 = threading.Thread(target=make_completion, args=(\"gpt-3.5-turbo\", messages))\n", - "thread2 = threading.Thread(target=make_completion, args=(\"azure/your-azure-deployment\", messages))\n", - "\n", - "# Start both threads\n", - "thread1.start()\n", - "thread2.start()\n", - "\n", - "# Wait for both threads to finish\n", - "thread1.join()\n", - "thread2.join()\n", - "\n", - "print(\"Both completions are done.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lx8DbMBqoAoN" - }, - "source": [ - "## Completion - Stress Test 10 requests in parallel\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "pHYANOlOkoDh" - }, - "outputs": [], - "source": [ - "import os\n", - "import threading\n", - "from litellm import completion\n", - "\n", - "# Function to make a completion call\n", - "def make_completion(model, messages):\n", - " response = completion(\n", - " model=model,\n", - " messages=messages\n", - " )\n", - "\n", - " print(f\"Response for {model}: {response}\")\n", - "\n", - "# Set your API keys\n", - "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n", - "\n", - "# Define the messages for the completions\n", - "messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n", - "\n", - "# Create and start 10 threads for making completions\n", - "threads = []\n", - "for i in range(10):\n", - " thread = threading.Thread(target=make_completion, args=(\"gpt-3.5-turbo\" if i % 2 == 0 else \"azure/your-azure-deployment\", messages))\n", - " threads.append(thread)\n", - " thread.start()\n", - "\n", - "# Wait for all threads to finish\n", - "for thread in threads:\n", - " thread.join()\n", - "\n", - "print(\"All completions are done.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yB2NDOO4oxrp" - }, - "source": [ - "## Completion - Azure, OpenAI in the same thread" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "HTBqwzxpnxab", - "outputId": "f3bc0efe-e4d5-44d5-a193-97d178cfbe14" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "OpenAI Response: {\n", - " \"id\": \"chatcmpl-7yjzrDeOeVeSrQ00tApmTxEww3vBS\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1694710847,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello! I'm an AI, so I don't have feelings, but I'm here to help you. How can I assist you today?\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 29,\n", - " \"total_tokens\": 42\n", - " }\n", - "}\n", - "Azure OpenAI Response: {\n", - " \"id\": \"chatcmpl-7yjztAQ0gK6IMQt7cvLroMSOoXkeu\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1694710849,\n", - " \"model\": \"gpt-35-turbo\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"finish_reason\": \"stop\",\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"As an AI language model, I don't have feelings but I'm functioning properly. Thank you for asking! How can I assist you today?\"\n", - " }\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"completion_tokens\": 29,\n", - " \"prompt_tokens\": 14,\n", - " \"total_tokens\": 43\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "# Function to make both OpenAI and Azure completions\n", - "def make_completions():\n", - " # Set your OpenAI API key\n", - " os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "\n", - " # OpenAI completion\n", - " openai_response = completion(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n", - " )\n", - "\n", - " print(\"OpenAI Response:\", openai_response)\n", - "\n", - " # Set your Azure OpenAI API key and configuration\n", - " os.environ[\"AZURE_API_KEY\"] = \"\"\n", - " os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - " os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n", - "\n", - " # Azure OpenAI completion\n", - " azure_response = completion(\n", - " model=\"azure/your-azure-deployment\",\n", - " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n", - " )\n", - "\n", - " print(\"Azure OpenAI Response:\", azure_response)\n", - "\n", - "# Call the function to make both completions in one thread\n", - "make_completions()\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/LiteLLM_Bedrock.ipynb b/cookbook/LiteLLM_Bedrock.ipynb deleted file mode 100644 index eed6036392d..00000000000 --- a/cookbook/LiteLLM_Bedrock.ipynb +++ /dev/null @@ -1,310 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "fNkMBurtxawJ" - }, - "source": [ - "# LiteLLM Bedrock Usage\n", - "Important Note: For Bedrock Requests you need to ensure you have `pip install boto3>=1.28.57`, boto3 supports bedrock from `boto3>=1.28.57` and higher " - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "htAufI28xeSy" - }, - "source": [ - "## Pre-Requisites" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "jT5GbPjAuDTp" - }, - "outputs": [], - "source": [ - "!pip install litellm\n", - "!pip install boto3>=1.28.57 # this version onwards has bedrock support" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "H4Vu4er2xnfI" - }, - "source": [ - "## Set Bedrock/AWS Credentials" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "id": "CtTrBthWxp-t" - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ[\"AWS_ACCESS_KEY_ID\"] = \"\" # Access key\n", - "os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"\" # Secret access key\n", - "os.environ[\"AWS_REGION_NAME\"] = \"\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "ycRK9NUdx1EI" - }, - "source": [ - "## Anthropic Requests" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tgkuoHa5uLOy", - "outputId": "27a78e86-c6a7-4bcc-8559-0813cb978426" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Claude instant 1, response\n", - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" I'm doing well, thanks for asking!\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-4f2e64a1-56d2-43f2-90d3-60ffd6f5086d\",\n", - " \"created\": 1696256761.3265705,\n", - " \"model\": \"anthropic.claude-instant-v1\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 11,\n", - " \"completion_tokens\": 9,\n", - " \"total_tokens\": 20\n", - " },\n", - " \"finish_reason\": \"stop_sequence\"\n", - "}\n", - "Claude v2, response\n", - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" I'm doing well, thanks for asking!\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-34f59b33-f94e-40c2-8bdb-f4af0813405e\",\n", - " \"created\": 1696256762.2137017,\n", - " \"model\": \"anthropic.claude-v2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 11,\n", - " \"completion_tokens\": 9,\n", - " \"total_tokens\": 20\n", - " },\n", - " \"finish_reason\": \"stop_sequence\"\n", - "}\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(\n", - " model=\"bedrock/anthropic.claude-instant-v1\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - ")\n", - "print(\"Claude instant 1, response\")\n", - "print(response)\n", - "\n", - "\n", - "response = completion(\n", - " model=\"bedrock/anthropic.claude-v2\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - ")\n", - "print(\"Claude v2, response\")\n", - "print(response)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "HnM-HtM3yFMT" - }, - "source": [ - "## Anthropic Requests - With Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "_JZvg2yovRsU" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(\n", - " model=\"bedrock/anthropic.claude-instant-v1\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True,\n", - ")\n", - "print(\"Claude instant 1, response\")\n", - "for chunk in response:\n", - " print(chunk)\n", - "\n", - "\n", - "response = completion(\n", - " model=\"bedrock/anthropic.claude-v2\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True\n", - ")\n", - "print(\"Claude v2, response\")\n", - "print(response)\n", - "for chunk in response:\n", - " print(chunk)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "zj1U1mh9zEhP" - }, - "source": [ - "## A121 Requests" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "6wK6MZLovU7r", - "outputId": "4cf80c04-f15d-4066-b4c7-113b551538de" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "J2 ultra response\n", - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"\\nHi, I'm doing well, thanks for asking! How about you?\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-f2de678f-0e70-4e36-a01f-8b184c2e4d50\",\n", - " \"created\": 1696257116.044311,\n", - " \"model\": \"ai21.j2-ultra\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 16,\n", - " \"total_tokens\": 22\n", - " }\n", - "}\n", - "J2 mid response\n", - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"\\nGood. And you?\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-420d6bf9-36d8-484b-93b4-4c9e00f7ce2e\",\n", - " \"created\": 1696257116.5756805,\n", - " \"model\": \"ai21.j2-mid\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 6,\n", - " \"total_tokens\": 12\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "response = completion(\n", - " model=\"bedrock/ai21.j2-ultra\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - ")\n", - "print(\"J2 ultra response\")\n", - "print(response)\n", - "\n", - "response = completion(\n", - " model=\"bedrock/ai21.j2-mid\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - ")\n", - "print(\"J2 mid response\")\n", - "print(response)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Y5gGZIwzzSON" - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_CometAPI.ipynb b/cookbook/LiteLLM_CometAPI.ipynb deleted file mode 100644 index 0a7ab581ae3..00000000000 --- a/cookbook/LiteLLM_CometAPI.ipynb +++ /dev/null @@ -1,474 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "iFEmsVJI_2BR" - }, - "source": [ - "# LiteLLM CometAPI Cookbook" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "cBlUhCEP_xj4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: litellm in /Users/xmx/.miniforge3/lib/python3.12/site-packages (1.78.2)\n", - "Requirement already satisfied: aiohttp>=3.10 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.11.18)\n", - "Requirement already satisfied: click in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.3.0)\n", - "Requirement already satisfied: fastuuid>=0.13.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.13.3)\n", - "Requirement already satisfied: httpx>=0.23.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.28.1)\n", - "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.6.1)\n", - "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.1.6)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (4.25.1)\n", - "Requirement already satisfied: openai>=2.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (2.11.10)\n", - "Requirement already satisfied: python-dotenv>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.1.1)\n", - "Requirement already satisfied: tiktoken>=0.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.12.0)\n", - "Requirement already satisfied: tokenizers in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.22.1)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (2.6.1)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.4.0)\n", - "Requirement already satisfied: attrs>=17.3.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (25.3.0)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.6.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (6.6.3)\n", - "Requirement already satisfied: propcache>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (0.3.1)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.20.0)\n", - "Requirement already satisfied: anyio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (4.11.0)\n", - "Requirement already satisfied: certifi in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (2025.10.5)\n", - "Requirement already satisfied: httpcore==1.* in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (1.0.9)\n", - "Requirement already satisfied: idna in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (3.10)\n", - "Requirement already satisfied: h11>=0.16 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.23.0->litellm) (0.16.0)\n", - "Requirement already satisfied: zipp>=3.20 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from importlib-metadata>=6.8.0->litellm) (3.21.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jinja2<4.0.0,>=3.1.2->litellm) (3.0.3)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2025.9.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.36.2)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.27.1)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (0.11.0)\n", - "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.3.1)\n", - "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.67.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.15.0)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.33.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (2.33.2)\n", - "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.4.2)\n", - "Requirement already satisfied: regex>=2022.1.18 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm) (2025.9.18)\n", - "Requirement already satisfied: requests>=2.26.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm) (2.32.2)\n", - "Requirement already satisfied: huggingface-hub<2.0,>=0.16.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tokenizers->litellm) (0.25.2)\n", - "Requirement already satisfied: filelock in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (3.15.4)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (2025.9.0)\n", - "Requirement already satisfied: packaging>=20.9 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (25.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (6.0.3)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken>=0.7.0->litellm) (3.4.0)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken>=0.7.0->litellm) (1.26.20)\n" - ] - } - ], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Completion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "p-MQqWOT_1a7" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['COMETAPI_KEY'] = \"Your_CometAPI_Key_Here\"\n", - "api_key = os.getenv('COMETAPI_KEY')" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Ze8JqMqWAARO", - "outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "ModelResponse(id='msg_017L3DDDit8AkEgHRe2DQBc9', created=1760589916, model='claude-sonnet-4-5-20250929', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='I\\'ll create a simple Python script that says hi.\\n\\n\\nhello.py\\n#!/usr/bin/env python3\\n\"\"\"\\nA simple script that says hi!\\n\"\"\"\\n\\ndef say_hi(name=None):\\n \"\"\"Say hi to someone, or just say hi generally.\"\"\"\\n if name:\\n print(f\"Hi, {name}!\")\\n else:\\n print(\"Hi!\")\\n\\nif __name__ == \"__main__\":\\n # Say hi generally\\n say_hi()\\n \\n # Say hi to someone specific\\n say_hi(\"World\")\\n\\n\\n\\nI\\'ve created a simple Python script called `hello.py` that:\\n\\n1. Defines a `say_hi()` function that can optionally take a name parameter\\n2. Prints \"Hi!\" if no name is provided\\n3. Prints \"Hi, [name]!\" if a name is provided\\n4. Demonstrates both usages when run\\n\\nYou can run it with:\\n```bash\\npython hello.py\\n```\\n\\nThis will output:\\n```\\nHi!\\nHi, World!\\n```\\n\\nWould you like me to create versions in other programming languages, or modify this in any way?', role='assistant', tool_calls=None, function_call=None, provider_specific_fields=None), provider_specific_fields={})], usage=Usage(completion_tokens=290, prompt_tokens=26, total_tokens=316, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=None, text_tokens=None, image_tokens=None, cached_tokens_details={})))" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from litellm import completion\n", - "response = completion(\n", - " model=\"cometapi/claude-sonnet-4-5-20250929\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-LnhELrnAM_J", - "outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "ModelResponse(id='chatcmpl-CRA9Uo6nsQ9C7kMJv1J4kyNDFJym7', created=1760589916, model='gpt-5-chat-latest', object='chat.completion', system_fingerprint='fp_2da73a467a', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Sure! I can help you write a simple code that prints out \"Hi\" in different programming languages. \\n\\nHere’s an example in **Python**:\\n\\n```python\\n# Simple Python program to say \"Hi\"\\nprint(\"Hi\")\\n```\\n\\nExample in **JavaScript**:\\n\\n```javascript\\n// Simple JavaScript program to say \"Hi\"\\nconsole.log(\"Hi\");\\n```\\n\\nExample in **C**:\\n\\n```c\\n#include \\n\\nint main() {\\n printf(\"Hi\\\\n\");\\n return 0;\\n}\\n```\\n\\nExample in **Java**:\\n\\n```java\\npublic class SayHi {\\n public static void main(String[] args) {\\n System.out.println(\"Hi\");\\n }\\n}\\n```\\n\\nWhich language would you like me to focus on, or do you want me to make it interactive so the program greets the user by name?', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}, annotations=[]), provider_specific_fields={})], usage=Usage(completion_tokens=174, prompt_tokens=12, total_tokens=186, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None)))" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response = completion(\n", - " model=\"cometapi/gpt-5-chat-latest\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "dJBOUYdwCEn1", - "outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "ModelResponse(id='02176058998406949c23b3bf3d52941de23f13565a086747738f0', created=1760589991, model='deepseek-v3.2-exp', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='Here are several ways to say \"hi\" in different programming languages:\\n\\n## Python\\n```python\\nprint(\"Hi!\")\\n```\\n\\n## JavaScript (Browser)\\n```javascript\\nconsole.log(\"Hi!\");\\n// or\\nalert(\"Hi!\");\\n```\\n\\n## JavaScript (Node.js)\\n```javascript\\nconsole.log(\"Hi!\");\\n```\\n\\n## Java\\n```java\\npublic class Hello {\\n public static void main(String[] args) {\\n System.out.println(\"Hi!\");\\n }\\n}\\n```\\n\\n## C\\n```c\\n#include \\n\\nint main() {\\n printf(\"Hi!\\\\n\");\\n return 0;\\n}\\n```\\n\\n## C++\\n```cpp\\n#include \\n\\nint main() {\\n std::cout << \"Hi!\" << std::endl;\\n return 0;\\n}\\n```\\n\\n## C#\\n```csharp\\nusing System;\\n\\nclass Program {\\n static void Main() {\\n Console.WriteLine(\"Hi!\");\\n }\\n}\\n```\\n\\n## PHP\\n```php\\n\\n```\\n\\n## Ruby\\n```ruby\\nputs \"Hi!\"\\n```\\n\\n## Go\\n```go\\npackage main\\n\\nimport \"fmt\"\\n\\nfunc main() {\\n fmt.Println(\"Hi!\")\\n}\\n```\\n\\n## Rust\\n```rust\\nfn main() {\\n println!(\"Hi!\");\\n}\\n```\\n\\n## Swift\\n```swift\\nprint(\"Hi!\")\\n```\\n\\n## Kotlin\\n```kotlin\\nfun main() {\\n println(\"Hi!\")\\n}\\n```\\n\\n## HTML (webpage)\\n```html\\n\\n\\n\\n Hi Page\\n\\n\\n

Hi!

\\n\\n\\n```\\n\\nThe Python version is probably the simplest if you\\'re just getting started!', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}), provider_specific_fields={})], usage=Usage(completion_tokens=347, prompt_tokens=10, total_tokens=357, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=None, audio_tokens=None, reasoning_tokens=0, rejected_prediction_tokens=None, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None)), service_tier='default')" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response = completion(\n", - " model=\"cometapi/deepseek-v3.2-exp\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Streaming" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Streaming Responses" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "I'm\n", - " doing\n", - " well\n", - " —\n", - " thanks\n", - " for\n", - " asking\n", - "!\n", - " How\n", - " can\n", - " I\n", - " help\n", - " you\n", - " today\n", - "?\n", - "\n" - ] - } - ], - "source": [ - "messages = [{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}]\n", - "response = completion(model=\"cometapi/gpt-5-mini\", messages=messages, stream=True)\n", - "for part in response:\n", - " print(part.choices[0].delta.content or \"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Async Completion" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ModelResponse(id='chatcmpl-CRAAkmfczlmCEnCM55D9CKexbRenn', created=1760589994, model='gpt-5-mini-2025-08-07', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"I'm doing well, thanks — how are you? How can I help today?\", role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}, annotations=[]), provider_specific_fields={'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'protected_material_code': {'filtered': False, 'detected': False}, 'protected_material_text': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}})], usage=Usage(completion_tokens=26, prompt_tokens=12, total_tokens=38, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None)), prompt_filter_results=[{'prompt_index': 0, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'jailbreak': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}])\n" - ] - } - ], - "source": [ - "from litellm import acompletion\n", - "import asyncio\n", - "\n", - "async def test_get_response():\n", - " user_message = \"Hello, how are you?\"\n", - " messages = [{\"content\": user_message, \"role\": \"user\"}]\n", - " response = await acompletion(model=\"cometapi/gpt-5-mini\", messages=messages)\n", - " return response\n", - "\n", - "response = await test_get_response()\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Async Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "test acompletion + streaming\n", - "response: \n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='Hi', role='assistant', function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' —', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' I', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='’m', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' doing', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' well', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=',', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' thanks', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='!', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' How', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' are', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' you', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='?', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' What', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' can', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' I', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' help', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' you', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' with', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' today', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='?', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(provider_specific_fields=None, content=None, role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None)\n" - ] - } - ], - "source": [ - "from litellm import acompletion\n", - "import asyncio, os, traceback\n", - "\n", - "async def completion_call():\n", - " try:\n", - " print(\"test acompletion + streaming\")\n", - " response = await acompletion(\n", - " model=\"cometapi/gpt-5-mini\", \n", - " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}], \n", - " stream=True\n", - " )\n", - " print(f\"response: {response}\")\n", - " async for chunk in response:\n", - " print(chunk)\n", - " except:\n", - " print(f\"error occurred: {traceback.format_exc()}\")\n", - " pass\n", - "\n", - "await completion_call()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Embedding" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "EmbeddingResponse(model='text-embedding-3-small', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.018048199, 0.0047550877, -0.013976435, -0.021936804, -0.038773336, -0.03708264, 0.03854791, -0.0007172257, 0.026473511, -0.0027438616, -0.019823432, -0.011947598, -0.013426959, -0.0059914105, 0.020485623, 0.04269012, -0.028276922, -0.015216281, -0.03325039, 0.045057096, 0.0037477135, 0.015793936, -0.005188329, -0.0071713766, 0.008446445, 0.0070938864, 0.0027632343, 0.025656339, 0.022091785, -0.026797561, -0.029840818, -0.0542714, 0.017907308, -0.03454659, -0.014582269, 0.0429719, 0.03575826, 0.007939235, -0.010123054, -0.029587213, 0.018921727, 0.022556728, 0.019598005, 0.008819807, 0.01655475, 0.043310042, -0.034321167, 0.004441604, 0.032686826, 0.047226828, -0.0043253684, 0.006681779, 0.008995921, 0.06593721, 0.0066078105, -0.023190739, -0.00777721, 0.049875587, -0.028023317, -0.019386668, -0.0013252605, 0.009214303, -0.0055828253, -0.007432026, -0.01199691, 0.0054384116, -0.024247425, 0.047255006, -0.013743965, 0.012799991, 0.009883538, -0.005579303, -0.012616833, -0.014709071, -0.004473305, -0.022204498, 0.010454148, -0.02202134, 0.034631126, 0.0097567355, 0.041478455, -0.010750021, -0.0005340668, -0.041450277, -0.04516981, 0.014709071, -0.06142869, 0.021640932, -0.0008752884, -0.0012671428, -0.045254346, -0.004216178, -0.028220564, -0.011074071, 0.010693664, -0.0029481545, -0.039590508, -0.030178957, -0.019062618, -0.03539194, 0.019978413, 0.019161243, -0.025924034, 0.0058329077, -0.029305428, 0.028840488, 0.02731886, -0.0008048426, -0.056582022, 0.003043256, -0.08459125, -0.012560476, 0.00081100664, 0.0015771041, 0.015188103, -0.0049206354, 0.046888687, -0.027107522, 0.019245777, -0.011412211, 0.039336905, -0.018611765, 0.0429719, -0.04302826, -0.018132735, -0.0074883825, -0.02035882, -0.073038146, -0.042380158, 0.04485985, 0.03361671, -0.033842135, -0.0017259207, -0.0017091898, -0.049199305, -0.024148801, -0.044803493, 0.040943068, -0.03093977, 0.045057096, -0.0186963, -0.014053926, -0.009848315, -0.0070480965, -0.0060054995, -0.02813603, -0.022105874, 0.010235767, -0.004476827, 0.027220234, -0.026290352, 0.0216832, -0.05559578, 0.042464696, 0.0253182, -0.0031594916, 0.02266944, -0.030798879, -0.02896729, -0.0005891025, 0.004061197, -0.042295624, -0.008383043, 0.017146494, -0.025797231, -0.046522368, 0.044127215, 0.021105545, -0.04302826, 0.024430584, -0.014934498, -0.01556851, -0.075743265, -0.022415835, -0.028586883, 0.032743182, 0.03539194, -0.0034483192, -0.04669144, 0.051932603, 0.021626843, 0.03127791, 0.015610777, -0.016470214, -0.0056744046, -0.012743635, 0.060132485, 0.017428277, -0.0039942735, -0.017118316, 0.025106862, 0.008974788, 0.018513141, 0.0016035212, -0.0049593803, 0.0017514573, 0.031221554, -0.034856554, -0.011461522, 0.04621241, 0.044239927, 0.034715664, -0.0121941585, -0.012053267, -0.08526753, -0.011813751, -0.025769053, 0.0125182085, -0.0046670306, 0.038266126, 0.1187997, 0.005787118, -0.030038064, -0.054553185, -0.041506633, -0.024078354, 0.0071854657, -0.013391736, 0.03192601, -0.059625275, 0.0023458432, 0.027924692, 0.09163582, 0.030967949, 0.017639615, 0.01489223, 0.029559033, 0.042943723, 0.0003306547, 0.0047198646, 0.029897174, 0.00012603184, 0.0046811197, -0.0427183, 0.01789322, 0.018175002, -0.0081857955, 0.02581132, 0.009890582, 0.03840702, -0.094453655, -0.01076411, 0.06858598, 0.041647524, 0.033532172, 0.007467249, -0.008235107, -0.030967949, 0.0151317455, 0.027361127, 0.011834885, 0.008707094, 0.008178751, -0.022458103, 0.02844599, 0.003605061, -0.02399382, 0.05212985, 0.06041427, -0.023317542, -0.013335379, -0.044099037, -0.040802173, -0.0047656544, -0.023909286, 0.017315563, 0.017428277, 0.00736158, -0.0016070436, -0.055454887, -0.038012523, -0.020626513, 0.018273626, 0.03260229, -0.016991513, 0.038463376, -0.022458103, -0.0109613575, -0.021810003, 0.04846667, -0.042521052, 0.008601425, -0.019259866, -0.0040048403, -0.03308132, -0.02499415, 0.026783472, -0.032884073, 0.021824092, 0.013145176, -0.009186125, -0.01769597, 0.03240504, -0.015343083, -0.012539342, 0.03578644, -0.012299826, 0.011898286, 0.035730083, 0.058441788, 0.032010544, 0.048720278, 0.012926794, -0.0015207474, -0.03313768, 0.014540002, 0.020189751, 0.00029058868, 0.011531969, -0.022514459, 0.019752987, -0.037956167, 0.005272864, -0.042295624, -0.08521117, -0.03494109, 0.053313337, 0.029981708, -0.008150573, -0.053200625, 0.059681635, -0.035476476, 0.034828376, 0.00087881065, 0.025712697, 0.018668123, 0.03212326, 0.008474623, 0.017836861, 0.004910068, 0.016174342, -0.059681635, 0.04004136, 0.00753065, 0.008150573, -0.038012523, 0.0051178834, 0.012525253, 0.022119964, 0.030235313, 0.008242152, -0.01835816, -0.003150686, 0.010714797, -0.0033162334, -0.028882755, -0.06836055, 0.056159347, 0.013624207, 0.0077349427, 0.0066183778, 0.018583586, -0.008883208, -0.046550546, 0.046945043, -0.07393985, -0.017343743, 0.029530855, -0.010982491, 0.008129438, 0.009700378, -0.024613742, -0.0030097943, 0.0078053884, -0.006438741, 0.04770586, -0.008221018, 0.01654066, 0.02498006, -0.015793936, -0.010827511, 0.02399382, 0.03192601, 0.022923045, -0.029192716, 0.006724046, -0.04601516, 0.038519733, 0.031531516, 0.019443026, 0.000109466084, 0.03525105, -0.027248414, -0.038125236, 0.011771483, -0.007467249, 0.010285079, 0.01670973, -0.007861745, 0.026868006, 0.052327096, 0.026374886, -0.03905512, 0.031193376, -0.053566944, 0.04257741, -0.004670553, 0.0168788, 0.035166513, -0.057878222, 0.07095295, -0.009749691, -0.0137510095, -0.02151413, -0.02431787, 0.010073741, -0.05176353, -0.02083785, 0.003959051, -0.02682574, 0.062104966, -0.011461522, 0.04170388, 0.0076363184, 0.026177637, 0.0144413775, 0.014821785, -0.00046890447, 0.0050544823, 0.00032228927, -0.038970586, -0.011355854, -0.056300238, -0.04302826, -0.003545182, 0.04021043, 0.0051108385, -0.048438493, 0.00252548, -0.07692675, -0.0012433673, 0.0054278444, 0.029305428, -0.016188432, -0.003263399, -0.046156053, 6.031917e-05, 0.060977835, -0.016611107, -0.010637308, -0.012602744, 0.016442036, -0.051509928, -0.016991513, 0.0019407802, 0.019161243, 0.045282524, 0.031869654, -0.036941748, -0.035814617, -0.017850952, -0.027192056, -0.049734693, -0.020964652, 0.0228526, -0.025050506, 0.023472521, 0.025740875, -0.017738238, -0.009813092, -0.030883415, -0.012405495, -0.03277136, -0.029502677, 0.016780175, -0.04421175, -0.0020816717, 0.010341435, 0.059230782, -0.041901127, -0.04119667, 0.025924034, 0.02334572, -0.0008435878, 0.020654691, -0.022753974, 0.010700708, -0.013856677, -0.0121941585, -0.011391076, 0.006590199, 0.0050227814, -0.007960369, 0.0008418266, -0.0198657, 0.10781016, -0.0384352, -0.019147152, 0.0057237167, -0.0038181592, -0.047424074, -0.009341106, 0.018499052, -0.016906979, 0.005642704, -0.01837225, -0.038125236, -0.024895526, -0.010285079, -0.055708494, 0.014173684, -0.019724809, 0.00024215724, 0.04500074, 0.048804812, 0.009777869, -0.006572588, 0.008277375, 0.012328005, -0.012609788, 0.026079014, -0.012990195, 0.017963665, -0.007312268, -0.0015682983, 0.05446865, -0.01258161, 0.00035376972, -0.011299497, -0.036321826, -0.0071854657, 0.012969062, 0.026558045, -0.051819887, -0.0029146927, -0.044606246, -0.010383703, -0.03919601, 0.013624207, 0.0030978515, 0.0121941585, -0.0022225631, 0.0512845, -0.0029780937, -0.025191398, -0.015751667, -0.021006921, 0.0039520063, 0.04418357, 0.020570157, 0.00083390146, 0.020541979, -0.004807922, -0.0114263, -0.036152754, 0.018428607, -0.032658648, -0.002035882, 0.013828499, 0.03144698, -0.0003275727, -0.029756282, 0.008488712, 0.0041879993, 0.027826069, 0.0007273523, -0.018949905, -0.0029023646, 0.007861745, 0.011398122, 0.0125322975, 0.014976765, 0.006318983, -0.0066536004, -0.042915545, 0.025867676, -0.015272637, 0.034602948, 0.050241902, 0.014582269, 0.005987888, 0.015244459, -0.050213724, -0.003212326, 0.01315222, -0.022866689, -0.004772699, 0.035673723, -0.024796901, -0.00699174, -0.002072866, 0.022077696, 0.021147812, 0.005093227, -0.039618686, -0.0049241576, 0.012264604, -0.062104966, -0.0022613083, -0.004339458, 0.065486364, -0.0033320836, 0.02944632, 0.017498722, 0.0033039053, -0.020260196, -0.0154980635, -0.05460954, -0.03626547, 0.0072629564, 0.0028900367, 1.2954037e-05, 0.01769597, -0.0045930627, 0.022260854, 0.0027192058, -0.0010566862, -0.0005212985, 0.012158935, -0.0017312041, -0.035110157, -0.0036032998, -0.02317665, -0.01639977, 0.010327346, 0.018259536, -0.011095204, -0.00061904197, -0.023134382, -0.011989865, 0.0025924034, 0.0056708823, 0.03110884, -0.013462181, -0.021105545, 0.010376658, -0.010017385, -0.025106862, 0.026093103, 0.018456785, -0.02134506, 0.0066993902, 0.011891241, -0.010017385, -0.012687278, -0.017132405, 0.04717047, 0.012475941, -0.018752657, -0.008657781, 0.005276386, -0.02582541, 0.02913636, -0.0193444, -0.01101067, 0.029305428, 0.011736261, 0.043140974, 0.02135915, 0.00089422066, 0.009827181, 0.013638296, 0.013884856, -0.014004614, 0.010285079, 0.008108305, -0.04035132, -0.02978446, 0.008481667, -0.022289034, 0.01621661, -0.0057941624, -0.019090796, -0.01852723, -0.022923045, 0.0077208537, -0.039985005, -0.017428277, -0.009460864, 0.018301804, 0.0014397348, 0.04815671, -0.012187114, 0.018879458, 0.021739556, 0.018414518, -0.013462181, -0.06368295, 0.0057096276, 0.013088819, 0.0061640027, 0.031193376, -0.008728228, -0.019245777, 0.010735931, 0.012454808, 0.0397314, -0.017597347, 0.012278693, 0.0130465515, -0.025473181, -0.03215144, -0.0053292206, -0.0027068777, 0.014068015, -0.028079674, 0.016498392, 0.015159924, -0.009207259, -0.02334572, -0.0013710503, 0.008488712, 0.0012231142, 0.0020464489, -0.025149131, -0.021063277, -0.014427288, -0.035222873, 0.051030897, 0.016103897, 0.0063401167, -0.03093977, -0.004684642, -0.0070199184, -0.008495756, -0.0038674714, -0.012222337, -0.022556728, 0.0036015387, -0.040943068, -0.011362898, 0.016794264, 0.017766416, -0.014194817, 0.0011755633, -0.039759576, 0.011384032, -0.0006318103, 0.008298509, 0.04449353, 0.0004838742, 0.016935157, -0.011341765, 0.016864711, -0.00027892113, 0.0009140335, -0.031306088, -0.049452912, -0.0068367594, -0.00011216283, -0.005079138, -0.014420244, 0.01803411, 0.03984411, -0.026276262, -0.0011077593, -0.00063313113, -0.006301372, -0.019992502, 0.0064316965, -0.024289692, 0.0120039545, 0.0068649375, -0.017724149, -0.015667133, -0.0036490895, -0.007953324, 0.024627833, 0.024402406, 0.021810003, -0.015977094, 0.010524594, -0.0060054995, 0.0414221, -0.048551206, 0.01472316, 0.015427617, 0.0029217373, 0.012666144, 0.0048995013, 0.007326357, -0.04187295, -0.0064176074, -0.00674518, 0.0047762212, -0.053059734, -0.09541172, 0.022063607, 0.029530855, 0.01556851, 0.011292453, -0.0038709936, -0.0055370354, -0.016005272, -0.0035170037, -0.0572583, 0.038632445, 0.007981502, -0.005434889, -0.023895197, 0.0021380284, -0.0015084195, 0.016117986, 0.005434889, -0.014694982, -0.007104453, 0.011595369, -0.055229463, 0.0036455672, 0.0027104, -0.010052607, -0.023697948, -0.016315235, -0.002757951, 0.039505973, 0.011095204, 0.0002681341, 0.058948997, -0.0074883825, 0.0050122146, 0.040604927, 0.012912705, -0.025078684, 0.040464036, -0.008925476, -0.00876345, -0.040633105, -0.009024099, 0.024796901, 0.03592733, 0.03626547, -0.029474499, -0.00055431994, 0.0010839839, 0.016737908, 0.013286067, -0.005441934, 0.0059420983, -0.0121941585, 0.015089478, -0.010186454, -0.03477202, -0.0076363184, -0.0087141385, 0.0018439173, 0.028065585, -0.022331301, 0.0029516767, -0.045789734, 0.0010672531, 0.018287715, -0.015948916, 0.04849485, 0.0057589393, 0.0066219, 0.002196146, -0.047255006, 0.012116668, 0.02085194, 0.025924034, -0.0036737456, -0.02877004, 0.016906979, -0.037336245, -0.016258877, 0.010883868, -0.003765325, -0.0049523357, -0.002613537, -0.03263047, 0.023204828, 0.0049946033, -0.007692675, -0.034236632, 0.034095738, 0.020133393, 0.019259866, -0.014103238, 0.024599653, 0.005889264, 0.02430378, 0.0111233825, -0.018780835, -0.00040550332, 0.020232018, 0.03806888, 0.009890582, 0.032376863, 0.031052483, 0.01871039, 0.03891423, -0.0009739124, 0.002759712, 0.017498722, -0.01158128, -0.0045578396, 0.02744566, 0.06497915, 0.024853257, 0.004709298, 0.016667463, -0.00066263025, -0.018132735, -0.013138131, -0.01124314, -0.0125182085, -0.0038111147, 0.03361671, -0.007270001, 0.0012011, -0.01771006, -0.00039999973, 0.024021998, 0.0027896515, 0.0024744067, 0.0013965869, -0.05939985, 0.0014150789, -0.0052517303, 0.052524347, 0.015779847, -0.03327857, 0.042633764, 0.0059420983, -0.023387987, 0.0039097387, -0.028023317, -0.011863063, 0.004378203, 0.02052789, -0.063626595, -0.014864052, 0.014293442, -0.00015938349, -0.007932191, -0.0010954313, 0.023528878, -0.007467249, 0.0059667546, 0.017132405, 0.005730761, -0.00020495309, -0.032038722, 0.0036631785, 0.042915545, -0.029925352, 0.015667133, 0.018935816, -0.0072065997, 0.01556851, -0.025473181, 0.017625526, -0.0026698937, -0.007446115, -0.008622559, -0.043422755, -0.020133393, -0.0039801844, 0.01489223, -0.021655021, 0.015357172, -0.03640636, -0.005663838, -0.028530527, 0.0022648307, -0.00043015933, 0.043591827, -0.015526242, 0.011870108, -0.02530411, -0.016315235, -0.00032316984, -0.030150779, -0.0052552526, 0.020372909, 0.0075024716, 0.0104330145, -0.00055608107, -0.026248084, -0.015202192, -0.03341946, 0.031559695, -0.0012046222, 0.07185466, -0.039590508, 0.022979401, 0.05810365, 0.014025748, -0.029756282, -0.022866689, 0.0073897582, 0.037618026, -0.004180955, -0.0051566283, 0.009728557, -0.03604004, 0.040633105, 0.0026963109, -0.0054172776, 0.034095738, -0.00595971, 0.040943068, -0.031390622, 0.055962097, 0.02117599, -0.012912705, -0.019626183, 0.055877563, 0.017343743, -0.0035416598, 0.013257889, -0.0186963, 0.01656884, -0.06396473, -0.0055405577, 0.020767406, -0.0046564634, 0.045085277, -0.009221348, 0.013645341, 0.008777539, 0.004730432, -0.018625854, -0.011067026, 0.021500042, -0.015047211, 0.004600107, -0.0014344514, -0.0023740216, -0.016188432, 0.006209792, 0.0011993388, 0.004180955, -0.017160583, 0.014497734, 0.015371261, 0.018259536, -0.028333278, -0.008390088, 0.041929305, 0.003923828, 0.02550136, -0.003300383, -0.008058993, -0.010418925, 0.058216363, 0.01885128, -0.02020384, 0.002858336, -0.009806047, -0.022274945, 0.0070445742, 0.026670758, 0.008213974, -0.035307407, -0.027713355, 0.042915545, -0.039675042, -0.0029217373, 0.012053267, -0.003853382, 0.01133472, -0.010073741, 0.005878697, 0.0070938864, -0.035673723, 0.024205158, 0.005896309, 0.030573452, 0.02416289, -0.0072911344, 0.01738601, 0.017005602, -0.02846008, 0.0030344503, 0.018794924, -0.0148076955, -0.0344057, 0.025430914, 0.033503994, -0.0050580045, 0.0077138087, 0.03243322, 0.01372283, -0.005441934, 0.0073404466, -0.0007832686, -0.04767768, 0.0070480965, 0.015145835, 0.026233995, -0.01670973, -0.019513471, -0.014849963, 0.007953324, -0.0032176094, 0.006572588, -0.0012477703, 0.004230267, 0.004476827, -0.021810003, -0.030009886, -0.019273955, -0.0030414949, -0.002918215, 0.060639694, 0.024641922, 0.010327346, 0.026558045, 0.018921727, -0.025867676, -0.016117986, 0.023881108, 0.025360467, 0.009770825, 0.03792799, -0.022429924, 0.033363104, -0.0018914682, 0.04040768, 0.018484963, 0.0070199184, -0.017583257, 0.016258877, 0.010954313, -0.008939565, -0.024148801, -0.02498006, -0.007889924, 0.02748793, 0.0307707, 0.029756282, 0.0051425393, 0.0045719286, -0.03046074, 0.013596028, 0.025684519, -0.0033197557, 0.006967084, 0.03677268, 0.0120039545, -0.0032792494, -0.0032211316, -0.02399382, -0.026924362, -0.013920079, -0.0042197, 0.025346378, -0.0027015943, -0.016991513, 0.0031594916, -0.007579962, 0.018978084, 0.017681882, 0.0126591, 0.028939111, 0.008833896, 0.10183637, 0.0059632324, -0.05196078, -0.023697948, 0.011045893, -0.008777539, -0.013807366, 0.019273955, -0.025346378, 0.0074742935, 0.009961028, -0.010813422, 0.018597675, 0.009636978, 0.014948587, 0.024064265, -0.008693005, -0.020570157, 0.014194817, -0.026219906, -0.02299349, 0.011067026, 0.032066904, 0.013391736, -0.05148175, -0.009489042, -0.03062981, -0.0012847543, 0.07286908, 0.026529867, -0.00025008238, 0.013638296, 0.016089808, 0.018654034, -0.0020394044, -0.024543297, 0.0147795165, -0.009601755, 0.0018791402, -0.040520392, -0.003360262, 0.02216223, 0.0137650985, 0.0059914105, 0.0048361, -0.0009844792, 0.016977424, -0.00934815, 0.024233336, -0.013088819, -0.017555078, -0.0050263037, 0.010595039, -0.027516108, 0.0071537653, -0.023247095, -0.0017655464, -0.015948916, 0.058160007, -0.025966302, 0.0121941585, -0.012384362, -0.0015612538, 0.009946939, 0.00628376, 0.011327676, 0.0109613575, 0.008601425, -0.018329982, 0.055680316, -0.012778858, -0.0100807855, -0.011067026, -0.0036490895, -0.01356785, 0.0073193125, -0.014272308, -0.027403394, -0.030742522, 0.02862915, 0.03062981, -0.014596358, 0.021697288, 0.0042408337, 0.027572464, 0.0019601528, -0.037138995, -0.031306088, 0.041929305, 0.017738238, 0.004857234, 0.008256241, 0.0118278405, 0.021753646, -0.00160176, -0.0018333505, -0.0047374764, 0.042239267, 0.0058329077, -0.026459422, 0.015075389, 0.021147812, -0.005212985, 0.01281408, 0.017738238, 0.008242152, -0.020372909, -0.011081115, -0.011017715, 0.007706764, 0.01834407, 0.01954165, 0.037477136, -0.010278034, 0.015808025, 0.00031590514, -0.017681882, -0.008967743, -0.020612424, -0.025416825, -0.0037970257, -0.029868996, 0.01720285, -0.0144554665, 0.026727116, 0.00414221, 0.0040154075, 0.05838543, 0.0005622451, -0.025219576, 0.004180955, -0.002932304, -0.0090663675, 0.011574236, 0.02450103, -0.012553431, -0.020612424, -0.032095082, 0.015526242, 0.008974788, 0.0053151315, -0.0003112821, -0.017935487, -0.0076222294, 0.03358853, 0.029474499, -0.011496745, -0.012835215, -0.020739228, -0.012482986, -0.037871633, 0.0052517303, -0.012926794, -0.0025237189, 0.0020323596, 0.045113456, -0.04835396, -0.027755624, -0.0079955915, 0.007896968, 0.0072559114, 0.015047211, -0.0014573464, -0.014032792, 0.021091456, -0.0046071517, -0.0065232757, -0.02582541, -0.035870973, -0.015343083, 0.03254593, -0.028431902, -0.003286294, 0.014328664, 0.008840941, 0.015948916, 0.012835215, 0.019400757, -0.012342094, -0.010693664, 0.004772699, -0.03254593, 0.010707753, -0.016822444, -0.0032827717, 0.021246437, -0.04485985, -0.04384543, -0.015906649, -0.009707424, 0.02299349, 0.019513471, -0.010151232, 0.018963994, -0.0057976847, 0.05739919, -0.019922055, -0.029108182, -0.0106232185, 0.021077367, 0.0036455672, -0.026614401, 0.04497256, -0.04446535, -0.0004556959, -0.004578973, 0.003962573, -0.004910068, 0.015089478, -0.0301226, 0.007664497, 0.008375999, 0.031982366, 0.006135824, 0.02152822, -0.015469885, -0.007210122, 0.034715664, -0.01233505, 0.0004490916, -0.0144413775, -0.003150686, -0.02003477, -0.027924692, -0.0015850292, -0.009376328, -0.0035997776, -0.03240504, -0.010912046, 0.0031999978, 0.022303123, -0.008988877, 0.00024633997, -0.0035698381, 0.0070974086, -0.002599448, -0.042267445, -0.016935157, -0.0002481011, -0.041393917, 0.014483645, 0.019006262, -0.02813603, 0.0072030774, -7.3032425e-05, 0.01802002, -0.017188761, 0.015991183, 0.020401087, 0.03542012, 0.04469078, 0.04071764, 0.011095204, -0.031390622, -0.03254593, 0.014187773, 0.016272966, -0.009721513, -0.026388975, -0.014849963, -0.005642704, -0.022556728, 0.0064457855, -0.043450933, 0.010834555, -0.015977094, 0.020880118, -0.02385293, -0.054806788, 0.03789981, 0.0013516777, -0.026431242, -0.015540331, 0.016695641, -0.037167173, -0.021190079, 0.023881108, -0.0045860177, 0.0064105624, -0.007763121, -0.013053596, 0.024472851, -0.0004962022, -0.00976378, 0.060019772, -0.0057624616, -0.04384543, 0.010313257, 0.0076715415, 0.0025888812, -0.03589915, 0.008791628, -0.012785902, 0.01042597, 0.015653044, 0.04767768, -0.009869449, 0.0064457855, -0.010947268, -0.0077349427, -0.032715004, -0.023867019, -0.011327676, -0.00046274049, -0.036998104, 0.013913034, 0.012250515, -0.009996251, 0.021204168, 0.020091126, -0.003740669, -0.0049769916, -0.0140891485, 0.024064265, 0.0038815604, 0.025684519, 0.041788414, -0.013553761, 0.006681779, -0.0050826604, -0.018175002, 0.008228063, -0.006230926, -0.018907638, 0.0154839745, -0.028713685, -0.015047211, -0.019682541, 0.02516322, 0.040802173, 0.007213644, 0.011743305, -0.015963005, -0.03818159, 0.01191942, -0.031728763, -0.011863063, 0.023881108, 0.0053116092, -0.020992832, -0.017991843, -0.00405063, -0.017780505, -0.0057659843, 0.02978446, 0.031165197, 0.0014221234, 0.021316882, 0.026008569, -0.0018544842, -0.032658648, 0.028474169, 0.013109953, 0.018076377, 0.0007991189, -0.0042373114, 0.028910933, -0.0029358263, 0.021866359, 0.024472851, -0.002576553, -0.033532172, 0.01920351, -0.0095665315, -0.03093977, 0.0034817809, 0.018654034, -0.0074038478, 0.021443684, 0.0038604268, -0.02745975, 0.031587873, 0.0061146906, 0.022711707, -0.019795254, -0.016991513, -0.04471896, -0.007875834, -0.0034941088, -0.043789074, 0.021091456, 0.024909616, -0.013194487, -0.0042690123, 0.027896514, -0.018414518, -0.023303451, -0.025797231, -0.009524264]}], object='list', usage=Usage(completion_tokens=0, prompt_tokens=3, total_tokens=3, completion_tokens_details=None, prompt_tokens_details=None))\n" - ] - } - ], - "source": [ - "import litellm\n", - "\n", - "\n", - "async def main():\n", - " response = await litellm.aembedding(\n", - " model=\"cometapi/text-embedding-3-small\", # The model name must include prefix \"openai\" + the model name from ai/ml api\n", - " api_key=api_key, # your aiml api-key\n", - " api_base=\"https://api.cometapi.com/v1\", # 👈 the URL has changed from v2 to v1\n", - " input=\"Your text string\",\n", - " )\n", - " print(response)\n", - "\n", - "await main()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "EmbeddingResponse(model='text-embedding-3-small', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.018048199, 0.0047550877, -0.013976435, -0.021936804, -0.038773336, -0.03708264, 0.03854791, -0.0007172257, 0.026473511, -0.0027438616, -0.019823432, -0.011947598, -0.013426959, -0.0059914105, 0.020485623, 0.04269012, -0.028276922, -0.015216281, -0.03325039, 0.045057096, 0.0037477135, 0.015793936, -0.005188329, -0.0071713766, 0.008446445, 0.0070938864, 0.0027632343, 0.025656339, 0.022091785, -0.026797561, -0.029840818, -0.0542714, 0.017907308, -0.03454659, -0.014582269, 0.0429719, 0.03575826, 0.007939235, -0.010123054, -0.029587213, 0.018921727, 0.022556728, 0.019598005, 0.008819807, 0.01655475, 0.043310042, -0.034321167, 0.004441604, 0.032686826, 0.047226828, -0.0043253684, 0.006681779, 0.008995921, 0.06593721, 0.0066078105, -0.023190739, -0.00777721, 0.049875587, -0.028023317, -0.019386668, -0.0013252605, 0.009214303, -0.0055828253, -0.007432026, -0.01199691, 0.0054384116, -0.024247425, 0.047255006, -0.013743965, 0.012799991, 0.009883538, -0.005579303, -0.012616833, -0.014709071, -0.004473305, -0.022204498, 0.010454148, -0.02202134, 0.034631126, 0.0097567355, 0.041478455, -0.010750021, -0.0005340668, -0.041450277, -0.04516981, 0.014709071, -0.06142869, 0.021640932, -0.0008752884, -0.0012671428, -0.045254346, -0.004216178, -0.028220564, -0.011074071, 0.010693664, -0.0029481545, -0.039590508, -0.030178957, -0.019062618, -0.03539194, 0.019978413, 0.019161243, -0.025924034, 0.0058329077, -0.029305428, 0.028840488, 0.02731886, -0.0008048426, -0.056582022, 0.003043256, -0.08459125, -0.012560476, 0.00081100664, 0.0015771041, 0.015188103, -0.0049206354, 0.046888687, -0.027107522, 0.019245777, -0.011412211, 0.039336905, -0.018611765, 0.0429719, -0.04302826, -0.018132735, -0.0074883825, -0.02035882, -0.073038146, -0.042380158, 0.04485985, 0.03361671, -0.033842135, -0.0017259207, -0.0017091898, -0.049199305, -0.024148801, -0.044803493, 0.040943068, -0.03093977, 0.045057096, -0.0186963, -0.014053926, -0.009848315, -0.0070480965, -0.0060054995, -0.02813603, -0.022105874, 0.010235767, -0.004476827, 0.027220234, -0.026290352, 0.0216832, -0.05559578, 0.042464696, 0.0253182, -0.0031594916, 0.02266944, -0.030798879, -0.02896729, -0.0005891025, 0.004061197, -0.042295624, -0.008383043, 0.017146494, -0.025797231, -0.046522368, 0.044127215, 0.021105545, -0.04302826, 0.024430584, -0.014934498, -0.01556851, -0.075743265, -0.022415835, -0.028586883, 0.032743182, 0.03539194, -0.0034483192, -0.04669144, 0.051932603, 0.021626843, 0.03127791, 0.015610777, -0.016470214, -0.0056744046, -0.012743635, 0.060132485, 0.017428277, -0.0039942735, -0.017118316, 0.025106862, 0.008974788, 0.018513141, 0.0016035212, -0.0049593803, 0.0017514573, 0.031221554, -0.034856554, -0.011461522, 0.04621241, 0.044239927, 0.034715664, -0.0121941585, -0.012053267, -0.08526753, -0.011813751, -0.025769053, 0.0125182085, -0.0046670306, 0.038266126, 0.1187997, 0.005787118, -0.030038064, -0.054553185, -0.041506633, -0.024078354, 0.0071854657, -0.013391736, 0.03192601, -0.059625275, 0.0023458432, 0.027924692, 0.09163582, 0.030967949, 0.017639615, 0.01489223, 0.029559033, 0.042943723, 0.0003306547, 0.0047198646, 0.029897174, 0.00012603184, 0.0046811197, -0.0427183, 0.01789322, 0.018175002, -0.0081857955, 0.02581132, 0.009890582, 0.03840702, -0.094453655, -0.01076411, 0.06858598, 0.041647524, 0.033532172, 0.007467249, -0.008235107, -0.030967949, 0.0151317455, 0.027361127, 0.011834885, 0.008707094, 0.008178751, -0.022458103, 0.02844599, 0.003605061, -0.02399382, 0.05212985, 0.06041427, -0.023317542, -0.013335379, -0.044099037, -0.040802173, -0.0047656544, -0.023909286, 0.017315563, 0.017428277, 0.00736158, -0.0016070436, -0.055454887, -0.038012523, -0.020626513, 0.018273626, 0.03260229, -0.016991513, 0.038463376, -0.022458103, -0.0109613575, -0.021810003, 0.04846667, -0.042521052, 0.008601425, -0.019259866, -0.0040048403, -0.03308132, -0.02499415, 0.026783472, -0.032884073, 0.021824092, 0.013145176, -0.009186125, -0.01769597, 0.03240504, -0.015343083, -0.012539342, 0.03578644, -0.012299826, 0.011898286, 0.035730083, 0.058441788, 0.032010544, 0.048720278, 0.012926794, -0.0015207474, -0.03313768, 0.014540002, 0.020189751, 0.00029058868, 0.011531969, -0.022514459, 0.019752987, -0.037956167, 0.005272864, -0.042295624, -0.08521117, -0.03494109, 0.053313337, 0.029981708, -0.008150573, -0.053200625, 0.059681635, -0.035476476, 0.034828376, 0.00087881065, 0.025712697, 0.018668123, 0.03212326, 0.008474623, 0.017836861, 0.004910068, 0.016174342, -0.059681635, 0.04004136, 0.00753065, 0.008150573, -0.038012523, 0.0051178834, 0.012525253, 0.022119964, 0.030235313, 0.008242152, -0.01835816, -0.003150686, 0.010714797, -0.0033162334, -0.028882755, -0.06836055, 0.056159347, 0.013624207, 0.0077349427, 0.0066183778, 0.018583586, -0.008883208, -0.046550546, 0.046945043, -0.07393985, -0.017343743, 0.029530855, -0.010982491, 0.008129438, 0.009700378, -0.024613742, -0.0030097943, 0.0078053884, -0.006438741, 0.04770586, -0.008221018, 0.01654066, 0.02498006, -0.015793936, -0.010827511, 0.02399382, 0.03192601, 0.022923045, -0.029192716, 0.006724046, -0.04601516, 0.038519733, 0.031531516, 0.019443026, 0.000109466084, 0.03525105, -0.027248414, -0.038125236, 0.011771483, -0.007467249, 0.010285079, 0.01670973, -0.007861745, 0.026868006, 0.052327096, 0.026374886, -0.03905512, 0.031193376, -0.053566944, 0.04257741, -0.004670553, 0.0168788, 0.035166513, -0.057878222, 0.07095295, -0.009749691, -0.0137510095, -0.02151413, -0.02431787, 0.010073741, -0.05176353, -0.02083785, 0.003959051, -0.02682574, 0.062104966, -0.011461522, 0.04170388, 0.0076363184, 0.026177637, 0.0144413775, 0.014821785, -0.00046890447, 0.0050544823, 0.00032228927, -0.038970586, -0.011355854, -0.056300238, -0.04302826, -0.003545182, 0.04021043, 0.0051108385, -0.048438493, 0.00252548, -0.07692675, -0.0012433673, 0.0054278444, 0.029305428, -0.016188432, -0.003263399, -0.046156053, 6.031917e-05, 0.060977835, -0.016611107, -0.010637308, -0.012602744, 0.016442036, -0.051509928, -0.016991513, 0.0019407802, 0.019161243, 0.045282524, 0.031869654, -0.036941748, -0.035814617, -0.017850952, -0.027192056, -0.049734693, -0.020964652, 0.0228526, -0.025050506, 0.023472521, 0.025740875, -0.017738238, -0.009813092, -0.030883415, -0.012405495, -0.03277136, -0.029502677, 0.016780175, -0.04421175, -0.0020816717, 0.010341435, 0.059230782, -0.041901127, -0.04119667, 0.025924034, 0.02334572, -0.0008435878, 0.020654691, -0.022753974, 0.010700708, -0.013856677, -0.0121941585, -0.011391076, 0.006590199, 0.0050227814, -0.007960369, 0.0008418266, -0.0198657, 0.10781016, -0.0384352, -0.019147152, 0.0057237167, -0.0038181592, -0.047424074, -0.009341106, 0.018499052, -0.016906979, 0.005642704, -0.01837225, -0.038125236, -0.024895526, -0.010285079, -0.055708494, 0.014173684, -0.019724809, 0.00024215724, 0.04500074, 0.048804812, 0.009777869, -0.006572588, 0.008277375, 0.012328005, -0.012609788, 0.026079014, -0.012990195, 0.017963665, -0.007312268, -0.0015682983, 0.05446865, -0.01258161, 0.00035376972, -0.011299497, -0.036321826, -0.0071854657, 0.012969062, 0.026558045, -0.051819887, -0.0029146927, -0.044606246, -0.010383703, -0.03919601, 0.013624207, 0.0030978515, 0.0121941585, -0.0022225631, 0.0512845, -0.0029780937, -0.025191398, -0.015751667, -0.021006921, 0.0039520063, 0.04418357, 0.020570157, 0.00083390146, 0.020541979, -0.004807922, -0.0114263, -0.036152754, 0.018428607, -0.032658648, -0.002035882, 0.013828499, 0.03144698, -0.0003275727, -0.029756282, 0.008488712, 0.0041879993, 0.027826069, 0.0007273523, -0.018949905, -0.0029023646, 0.007861745, 0.011398122, 0.0125322975, 0.014976765, 0.006318983, -0.0066536004, -0.042915545, 0.025867676, -0.015272637, 0.034602948, 0.050241902, 0.014582269, 0.005987888, 0.015244459, -0.050213724, -0.003212326, 0.01315222, -0.022866689, -0.004772699, 0.035673723, -0.024796901, -0.00699174, -0.002072866, 0.022077696, 0.021147812, 0.005093227, -0.039618686, -0.0049241576, 0.012264604, -0.062104966, -0.0022613083, -0.004339458, 0.065486364, -0.0033320836, 0.02944632, 0.017498722, 0.0033039053, -0.020260196, -0.0154980635, -0.05460954, -0.03626547, 0.0072629564, 0.0028900367, 1.2954037e-05, 0.01769597, -0.0045930627, 0.022260854, 0.0027192058, -0.0010566862, -0.0005212985, 0.012158935, -0.0017312041, -0.035110157, -0.0036032998, -0.02317665, -0.01639977, 0.010327346, 0.018259536, -0.011095204, -0.00061904197, -0.023134382, -0.011989865, 0.0025924034, 0.0056708823, 0.03110884, -0.013462181, -0.021105545, 0.010376658, -0.010017385, -0.025106862, 0.026093103, 0.018456785, -0.02134506, 0.0066993902, 0.011891241, -0.010017385, -0.012687278, -0.017132405, 0.04717047, 0.012475941, -0.018752657, -0.008657781, 0.005276386, -0.02582541, 0.02913636, -0.0193444, -0.01101067, 0.029305428, 0.011736261, 0.043140974, 0.02135915, 0.00089422066, 0.009827181, 0.013638296, 0.013884856, -0.014004614, 0.010285079, 0.008108305, -0.04035132, -0.02978446, 0.008481667, -0.022289034, 0.01621661, -0.0057941624, -0.019090796, -0.01852723, -0.022923045, 0.0077208537, -0.039985005, -0.017428277, -0.009460864, 0.018301804, 0.0014397348, 0.04815671, -0.012187114, 0.018879458, 0.021739556, 0.018414518, -0.013462181, -0.06368295, 0.0057096276, 0.013088819, 0.0061640027, 0.031193376, -0.008728228, -0.019245777, 0.010735931, 0.012454808, 0.0397314, -0.017597347, 0.012278693, 0.0130465515, -0.025473181, -0.03215144, -0.0053292206, -0.0027068777, 0.014068015, -0.028079674, 0.016498392, 0.015159924, -0.009207259, -0.02334572, -0.0013710503, 0.008488712, 0.0012231142, 0.0020464489, -0.025149131, -0.021063277, -0.014427288, -0.035222873, 0.051030897, 0.016103897, 0.0063401167, -0.03093977, -0.004684642, -0.0070199184, -0.008495756, -0.0038674714, -0.012222337, -0.022556728, 0.0036015387, -0.040943068, -0.011362898, 0.016794264, 0.017766416, -0.014194817, 0.0011755633, -0.039759576, 0.011384032, -0.0006318103, 0.008298509, 0.04449353, 0.0004838742, 0.016935157, -0.011341765, 0.016864711, -0.00027892113, 0.0009140335, -0.031306088, -0.049452912, -0.0068367594, -0.00011216283, -0.005079138, -0.014420244, 0.01803411, 0.03984411, -0.026276262, -0.0011077593, -0.00063313113, -0.006301372, -0.019992502, 0.0064316965, -0.024289692, 0.0120039545, 0.0068649375, -0.017724149, -0.015667133, -0.0036490895, -0.007953324, 0.024627833, 0.024402406, 0.021810003, -0.015977094, 0.010524594, -0.0060054995, 0.0414221, -0.048551206, 0.01472316, 0.015427617, 0.0029217373, 0.012666144, 0.0048995013, 0.007326357, -0.04187295, -0.0064176074, -0.00674518, 0.0047762212, -0.053059734, -0.09541172, 0.022063607, 0.029530855, 0.01556851, 0.011292453, -0.0038709936, -0.0055370354, -0.016005272, -0.0035170037, -0.0572583, 0.038632445, 0.007981502, -0.005434889, -0.023895197, 0.0021380284, -0.0015084195, 0.016117986, 0.005434889, -0.014694982, -0.007104453, 0.011595369, -0.055229463, 0.0036455672, 0.0027104, -0.010052607, -0.023697948, -0.016315235, -0.002757951, 0.039505973, 0.011095204, 0.0002681341, 0.058948997, -0.0074883825, 0.0050122146, 0.040604927, 0.012912705, -0.025078684, 0.040464036, -0.008925476, -0.00876345, -0.040633105, -0.009024099, 0.024796901, 0.03592733, 0.03626547, -0.029474499, -0.00055431994, 0.0010839839, 0.016737908, 0.013286067, -0.005441934, 0.0059420983, -0.0121941585, 0.015089478, -0.010186454, -0.03477202, -0.0076363184, -0.0087141385, 0.0018439173, 0.028065585, -0.022331301, 0.0029516767, -0.045789734, 0.0010672531, 0.018287715, -0.015948916, 0.04849485, 0.0057589393, 0.0066219, 0.002196146, -0.047255006, 0.012116668, 0.02085194, 0.025924034, -0.0036737456, -0.02877004, 0.016906979, -0.037336245, -0.016258877, 0.010883868, -0.003765325, -0.0049523357, -0.002613537, -0.03263047, 0.023204828, 0.0049946033, -0.007692675, -0.034236632, 0.034095738, 0.020133393, 0.019259866, -0.014103238, 0.024599653, 0.005889264, 0.02430378, 0.0111233825, -0.018780835, -0.00040550332, 0.020232018, 0.03806888, 0.009890582, 0.032376863, 0.031052483, 0.01871039, 0.03891423, -0.0009739124, 0.002759712, 0.017498722, -0.01158128, -0.0045578396, 0.02744566, 0.06497915, 0.024853257, 0.004709298, 0.016667463, -0.00066263025, -0.018132735, -0.013138131, -0.01124314, -0.0125182085, -0.0038111147, 0.03361671, -0.007270001, 0.0012011, -0.01771006, -0.00039999973, 0.024021998, 0.0027896515, 0.0024744067, 0.0013965869, -0.05939985, 0.0014150789, -0.0052517303, 0.052524347, 0.015779847, -0.03327857, 0.042633764, 0.0059420983, -0.023387987, 0.0039097387, -0.028023317, -0.011863063, 0.004378203, 0.02052789, -0.063626595, -0.014864052, 0.014293442, -0.00015938349, -0.007932191, -0.0010954313, 0.023528878, -0.007467249, 0.0059667546, 0.017132405, 0.005730761, -0.00020495309, -0.032038722, 0.0036631785, 0.042915545, -0.029925352, 0.015667133, 0.018935816, -0.0072065997, 0.01556851, -0.025473181, 0.017625526, -0.0026698937, -0.007446115, -0.008622559, -0.043422755, -0.020133393, -0.0039801844, 0.01489223, -0.021655021, 0.015357172, -0.03640636, -0.005663838, -0.028530527, 0.0022648307, -0.00043015933, 0.043591827, -0.015526242, 0.011870108, -0.02530411, -0.016315235, -0.00032316984, -0.030150779, -0.0052552526, 0.020372909, 0.0075024716, 0.0104330145, -0.00055608107, -0.026248084, -0.015202192, -0.03341946, 0.031559695, -0.0012046222, 0.07185466, -0.039590508, 0.022979401, 0.05810365, 0.014025748, -0.029756282, -0.022866689, 0.0073897582, 0.037618026, -0.004180955, -0.0051566283, 0.009728557, -0.03604004, 0.040633105, 0.0026963109, -0.0054172776, 0.034095738, -0.00595971, 0.040943068, -0.031390622, 0.055962097, 0.02117599, -0.012912705, -0.019626183, 0.055877563, 0.017343743, -0.0035416598, 0.013257889, -0.0186963, 0.01656884, -0.06396473, -0.0055405577, 0.020767406, -0.0046564634, 0.045085277, -0.009221348, 0.013645341, 0.008777539, 0.004730432, -0.018625854, -0.011067026, 0.021500042, -0.015047211, 0.004600107, -0.0014344514, -0.0023740216, -0.016188432, 0.006209792, 0.0011993388, 0.004180955, -0.017160583, 0.014497734, 0.015371261, 0.018259536, -0.028333278, -0.008390088, 0.041929305, 0.003923828, 0.02550136, -0.003300383, -0.008058993, -0.010418925, 0.058216363, 0.01885128, -0.02020384, 0.002858336, -0.009806047, -0.022274945, 0.0070445742, 0.026670758, 0.008213974, -0.035307407, -0.027713355, 0.042915545, -0.039675042, -0.0029217373, 0.012053267, -0.003853382, 0.01133472, -0.010073741, 0.005878697, 0.0070938864, -0.035673723, 0.024205158, 0.005896309, 0.030573452, 0.02416289, -0.0072911344, 0.01738601, 0.017005602, -0.02846008, 0.0030344503, 0.018794924, -0.0148076955, -0.0344057, 0.025430914, 0.033503994, -0.0050580045, 0.0077138087, 0.03243322, 0.01372283, -0.005441934, 0.0073404466, -0.0007832686, -0.04767768, 0.0070480965, 0.015145835, 0.026233995, -0.01670973, -0.019513471, -0.014849963, 0.007953324, -0.0032176094, 0.006572588, -0.0012477703, 0.004230267, 0.004476827, -0.021810003, -0.030009886, -0.019273955, -0.0030414949, -0.002918215, 0.060639694, 0.024641922, 0.010327346, 0.026558045, 0.018921727, -0.025867676, -0.016117986, 0.023881108, 0.025360467, 0.009770825, 0.03792799, -0.022429924, 0.033363104, -0.0018914682, 0.04040768, 0.018484963, 0.0070199184, -0.017583257, 0.016258877, 0.010954313, -0.008939565, -0.024148801, -0.02498006, -0.007889924, 0.02748793, 0.0307707, 0.029756282, 0.0051425393, 0.0045719286, -0.03046074, 0.013596028, 0.025684519, -0.0033197557, 0.006967084, 0.03677268, 0.0120039545, -0.0032792494, -0.0032211316, -0.02399382, -0.026924362, -0.013920079, -0.0042197, 0.025346378, -0.0027015943, -0.016991513, 0.0031594916, -0.007579962, 0.018978084, 0.017681882, 0.0126591, 0.028939111, 0.008833896, 0.10183637, 0.0059632324, -0.05196078, -0.023697948, 0.011045893, -0.008777539, -0.013807366, 0.019273955, -0.025346378, 0.0074742935, 0.009961028, -0.010813422, 0.018597675, 0.009636978, 0.014948587, 0.024064265, -0.008693005, -0.020570157, 0.014194817, -0.026219906, -0.02299349, 0.011067026, 0.032066904, 0.013391736, -0.05148175, -0.009489042, -0.03062981, -0.0012847543, 0.07286908, 0.026529867, -0.00025008238, 0.013638296, 0.016089808, 0.018654034, -0.0020394044, -0.024543297, 0.0147795165, -0.009601755, 0.0018791402, -0.040520392, -0.003360262, 0.02216223, 0.0137650985, 0.0059914105, 0.0048361, -0.0009844792, 0.016977424, -0.00934815, 0.024233336, -0.013088819, -0.017555078, -0.0050263037, 0.010595039, -0.027516108, 0.0071537653, -0.023247095, -0.0017655464, -0.015948916, 0.058160007, -0.025966302, 0.0121941585, -0.012384362, -0.0015612538, 0.009946939, 0.00628376, 0.011327676, 0.0109613575, 0.008601425, -0.018329982, 0.055680316, -0.012778858, -0.0100807855, -0.011067026, -0.0036490895, -0.01356785, 0.0073193125, -0.014272308, -0.027403394, -0.030742522, 0.02862915, 0.03062981, -0.014596358, 0.021697288, 0.0042408337, 0.027572464, 0.0019601528, -0.037138995, -0.031306088, 0.041929305, 0.017738238, 0.004857234, 0.008256241, 0.0118278405, 0.021753646, -0.00160176, -0.0018333505, -0.0047374764, 0.042239267, 0.0058329077, -0.026459422, 0.015075389, 0.021147812, -0.005212985, 0.01281408, 0.017738238, 0.008242152, -0.020372909, -0.011081115, -0.011017715, 0.007706764, 0.01834407, 0.01954165, 0.037477136, -0.010278034, 0.015808025, 0.00031590514, -0.017681882, -0.008967743, -0.020612424, -0.025416825, -0.0037970257, -0.029868996, 0.01720285, -0.0144554665, 0.026727116, 0.00414221, 0.0040154075, 0.05838543, 0.0005622451, -0.025219576, 0.004180955, -0.002932304, -0.0090663675, 0.011574236, 0.02450103, -0.012553431, -0.020612424, -0.032095082, 0.015526242, 0.008974788, 0.0053151315, -0.0003112821, -0.017935487, -0.0076222294, 0.03358853, 0.029474499, -0.011496745, -0.012835215, -0.020739228, -0.012482986, -0.037871633, 0.0052517303, -0.012926794, -0.0025237189, 0.0020323596, 0.045113456, -0.04835396, -0.027755624, -0.0079955915, 0.007896968, 0.0072559114, 0.015047211, -0.0014573464, -0.014032792, 0.021091456, -0.0046071517, -0.0065232757, -0.02582541, -0.035870973, -0.015343083, 0.03254593, -0.028431902, -0.003286294, 0.014328664, 0.008840941, 0.015948916, 0.012835215, 0.019400757, -0.012342094, -0.010693664, 0.004772699, -0.03254593, 0.010707753, -0.016822444, -0.0032827717, 0.021246437, -0.04485985, -0.04384543, -0.015906649, -0.009707424, 0.02299349, 0.019513471, -0.010151232, 0.018963994, -0.0057976847, 0.05739919, -0.019922055, -0.029108182, -0.0106232185, 0.021077367, 0.0036455672, -0.026614401, 0.04497256, -0.04446535, -0.0004556959, -0.004578973, 0.003962573, -0.004910068, 0.015089478, -0.0301226, 0.007664497, 0.008375999, 0.031982366, 0.006135824, 0.02152822, -0.015469885, -0.007210122, 0.034715664, -0.01233505, 0.0004490916, -0.0144413775, -0.003150686, -0.02003477, -0.027924692, -0.0015850292, -0.009376328, -0.0035997776, -0.03240504, -0.010912046, 0.0031999978, 0.022303123, -0.008988877, 0.00024633997, -0.0035698381, 0.0070974086, -0.002599448, -0.042267445, -0.016935157, -0.0002481011, -0.041393917, 0.014483645, 0.019006262, -0.02813603, 0.0072030774, -7.3032425e-05, 0.01802002, -0.017188761, 0.015991183, 0.020401087, 0.03542012, 0.04469078, 0.04071764, 0.011095204, -0.031390622, -0.03254593, 0.014187773, 0.016272966, -0.009721513, -0.026388975, -0.014849963, -0.005642704, -0.022556728, 0.0064457855, -0.043450933, 0.010834555, -0.015977094, 0.020880118, -0.02385293, -0.054806788, 0.03789981, 0.0013516777, -0.026431242, -0.015540331, 0.016695641, -0.037167173, -0.021190079, 0.023881108, -0.0045860177, 0.0064105624, -0.007763121, -0.013053596, 0.024472851, -0.0004962022, -0.00976378, 0.060019772, -0.0057624616, -0.04384543, 0.010313257, 0.0076715415, 0.0025888812, -0.03589915, 0.008791628, -0.012785902, 0.01042597, 0.015653044, 0.04767768, -0.009869449, 0.0064457855, -0.010947268, -0.0077349427, -0.032715004, -0.023867019, -0.011327676, -0.00046274049, -0.036998104, 0.013913034, 0.012250515, -0.009996251, 0.021204168, 0.020091126, -0.003740669, -0.0049769916, -0.0140891485, 0.024064265, 0.0038815604, 0.025684519, 0.041788414, -0.013553761, 0.006681779, -0.0050826604, -0.018175002, 0.008228063, -0.006230926, -0.018907638, 0.0154839745, -0.028713685, -0.015047211, -0.019682541, 0.02516322, 0.040802173, 0.007213644, 0.011743305, -0.015963005, -0.03818159, 0.01191942, -0.031728763, -0.011863063, 0.023881108, 0.0053116092, -0.020992832, -0.017991843, -0.00405063, -0.017780505, -0.0057659843, 0.02978446, 0.031165197, 0.0014221234, 0.021316882, 0.026008569, -0.0018544842, -0.032658648, 0.028474169, 0.013109953, 0.018076377, 0.0007991189, -0.0042373114, 0.028910933, -0.0029358263, 0.021866359, 0.024472851, -0.002576553, -0.033532172, 0.01920351, -0.0095665315, -0.03093977, 0.0034817809, 0.018654034, -0.0074038478, 0.021443684, 0.0038604268, -0.02745975, 0.031587873, 0.0061146906, 0.022711707, -0.019795254, -0.016991513, -0.04471896, -0.007875834, -0.0034941088, -0.043789074, 0.021091456, 0.024909616, -0.013194487, -0.0042690123, 0.027896514, -0.018414518, -0.023303451, -0.025797231, -0.009524264]}], object='list', usage=Usage(completion_tokens=0, prompt_tokens=3, total_tokens=3, completion_tokens_details=None, prompt_tokens_details=None))\n" - ] - } - ], - "source": [ - "import litellm\n", - "\n", - "\n", - "async def main():\n", - " response = await litellm.aembedding(\n", - " model=\"cometapi/text-embedding-3-small\", # The model name must include prefix \"cometapi/\" + the model name from CometAPI\n", - " api_key=api_key, # your CometAPI api-key\n", - " api_base=\"https://api.cometapi.com/v1\",\n", - " input=\"Your text string\",\n", - " )\n", - " print(response)\n", - "\n", - "\n", - "await main()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Async Image Generation" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ImageResponse(created=1760591151, background=None, data=[ImageObject(b64_json=None, revised_prompt=\"Generate an image of an adorable baby sea otter. It should be floating on its back in a calm, clear ocean, playfully grasping a colorful shell in its small paws. The sun is setting in the background, casting a peaceful orange and purple hue across the sky and reflecting upon the ocean waves. The otter's fur is a deep, rich brown and appears silky and wet, with glints of sunlight catching on it. Its eyes are bright, expressing joy and curiosity as it examines its newfound treasure.\", url='https://oaidalleapiprodscus.blob.core.windows.net/private/org-OKnsK88id12jfvnKByup1O0l/user-3GxuMyEg9YMU8LFCPHi31prf/img-7PUEF8Wb6thGDAuZWLJjSnfP.png?st=2025-10-16T04%3A05%3A51Z&se=2025-10-16T06%3A05%3A51Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=38e27a3b-6174-4d3e-90ac-d7d9ad49543f&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2025-10-16T02%3A51%3A01Z&ske=2025-10-17T02%3A51%3A01Z&sks=b&skv=2024-08-04&sig=IZKG2VE%2B6VdOe5Tq0Zk/5bVyGK/oK/yO8g%2BDX4krpug%3D')], output_format=None, quality=None, size=None, usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0, completion_tokens_details=None, prompt_tokens_details=None, input_tokens=0, input_tokens_details={'image_tokens': 0, 'text_tokens': 0}, output_tokens=0))\n" - ] - } - ], - "source": [ - "import asyncio\n", - "\n", - "import litellm\n", - "\n", - "\n", - "async def main():\n", - " response = await litellm.aimage_generation(\n", - " model=\"cometapi/dall-e-3\", # The model name must include prefix \"cometapi/\" + the model name from CometAPI\n", - " api_key=api_key, # your cometapi api-key\n", - " api_base=\"https://api.cometapi.com/v1\",\n", - " prompt=\"A cute baby sea otter\",\n", - " )\n", - " print(response)\n", - "\n", - "\n", - "await main()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "base", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.8" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_Comparing_LLMs.ipynb b/cookbook/LiteLLM_Comparing_LLMs.ipynb deleted file mode 100644 index 0b2e4e8c776..00000000000 --- a/cookbook/LiteLLM_Comparing_LLMs.ipynb +++ /dev/null @@ -1,441 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "L-W4C3SgClxl" - }, - "source": [ - "## Comparing LLMs on a Test Set using LiteLLM\n", - "LiteLLM allows you to use any LLM as a drop in replacement for `gpt-3.5-turbo`\n", - "\n", - "This notebook walks through how you can compare GPT-4 vs Claude-2 on a given test set using litellm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "fBkbl4Qo9pvz" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "id": "tzS-AXWK8lJC" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "\n", - "# init your test set questions\n", - "questions = [\n", - " \"how do i call completion() using LiteLLM\",\n", - " \"does LiteLLM support VertexAI\",\n", - " \"how do I set my keys on replicate llama2?\",\n", - "]\n", - "\n", - "\n", - "# set your prompt\n", - "prompt = \"\"\"\n", - "You are a coding assistant helping users using litellm.\n", - "litellm is a light package to simplify calling OpenAI, Azure, Cohere, Anthropic, Huggingface API Endpoints. It manages:\n", - "\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "id": "vMlqi40x-KAA" - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ['OPENAI_API_KEY'] = \"\"\n", - "os.environ['ANTHROPIC_API_KEY'] = \"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-HOzUfpK-H8J" - }, - "source": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Ktn25dfKEJF1" - }, - "source": [ - "## Calling gpt-3.5-turbo and claude-2 on the same questions\n", - "\n", - "## LiteLLM `completion()` allows you to call all LLMs in the same format\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "DhXwRlc-9DED" - }, - "outputs": [], - "source": [ - "results = [] # for storing results\n", - "\n", - "models = ['gpt-3.5-turbo', 'claude-2'] # define what models you're testing, see: https://docs.litellm.ai/docs/providers\n", - "for question in questions:\n", - " row = [question]\n", - " for model in models:\n", - " print(\"Calling:\", model, \"question:\", question)\n", - " response = completion( # using litellm.completion\n", - " model=model,\n", - " messages=[\n", - " {'role': 'system', 'content': prompt},\n", - " {'role': 'user', 'content': question}\n", - " ]\n", - " )\n", - " answer = response.choices[0].message['content']\n", - " row.append(answer)\n", - " print(print(\"Calling:\", model, \"answer:\", answer))\n", - "\n", - " results.append(row) # save results\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RkEXhXxCDN77" - }, - "source": [ - "## Visualizing Results" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 761 - }, - "id": "42hrmW6q-n4s", - "outputId": "b763bf39-72b9-4bea-caf6-de6b2412f86d" - }, - "outputs": [ - { - "data": { - "application/vnd.google.colaboratory.module+javascript": "\n import \"https://ssl.gstatic.com/colaboratory/data_table/881c4a0d49046431/data_table.js\";\n\n const table = window.createDataTable({\n data: [[{\n 'v': 0,\n 'f': \"0\",\n },\n\"how do i call completion() using LiteLLM\",\n\"To call the `completion()` function using LiteLLM, you need to follow these steps:\\n\\n1. Install the `litellm` package by running `pip install litellm` in your terminal.\\n2. Import the `Completion` class from the `litellm` module.\\n3. Initialize an instance of the `Completion` class by providing the required parameters like the API endpoint URL and your API key.\\n4. Call the `complete()` method on the `Completion` instance and pass the text prompt as a string.\\n5. Retrieve the generated completion from the response object and use it as desired.\\n\\nHere's an example:\\n\\n```python\\nfrom litellm.completion import Completion\\n\\n# Initialize the Completion client\\ncompletion_client = Completion(\\n model_name='gpt-3.5-turbo',\\n api_key='your_api_key',\\n endpoint='https://your_endpoint_url'\\n)\\n\\n# Call the completion() method\\nresponse = completion_client.complete(\\\"Once upon a time\\\")\\n\\n# Retrieve the generated completion\\ncompletion = response['choices'][0]['text']\\n\\nprint(completion)\\n```\\n\\nMake sure to replace `'gpt-3.5-turbo'` with the desired model name, `'your_api_key'` with your actual API key, and `'https://your_endpoint_url'` with the correct API endpoint URL provided by your service provider.\\n\\nNote: The above example assumes you have a valid API key and endpoint URL for the OpenAI GPT-3.5-turbo model. Make sure to obtain the necessary credentials according to the API you are using.\",\n\" Here is how you can call the completion() method using LiteLLM:\\n\\nFirst, import LiteLLM:\\n\\n```python\\nimport litellm as lm\\n```\\n\\nThen create a LiteLLM object, specifying the API you want to use (e.g. \\\"openai\\\"):\\n\\n```python \\nai = lm.LiteLLM(\\\"openai\\\")\\n```\\n\\nNow you can call the completion() method on the ai object:\\n\\n```python\\nresponse = ai.completion(\\n prompt=\\\"Hello\\\", \\n model=\\\"text-davinci-003\\\",\\n max_tokens=100\\n)\\n```\\n\\nThe completion() method takes parameters like:\\n\\n- prompt (str): The prompt text to complete \\n- model (str): The AI model to use\\n- max_tokens (int): The maximum number of tokens to generate\\n\\nIt returns a Python dictionary with the AI's response.\\n\\nYou can then access the generated text using:\\n\\n```python\\nprint(response[\\\"choices\\\"][0][\\\"text\\\"]) \\n```\\n\\nSo LiteLLM provides a simple unified interface to call the underlying AI APIs. The completion() method works similarly for OpenAI\"],\n [{\n 'v': 1,\n 'f': \"1\",\n },\n\"does LiteLLM support VertexAI\",\n\"Yes, LiteLLM does support Google Cloud Vertex AI. It provides convenient wrappers and simplified functions to call Vertex AI API endpoints for natural language processing tasks such as text classification, entity extraction, sentiment analysis, etc. You can easily integrate LiteLLM with Vertex AI in your code to leverage its capabilities.\",\n\" Unfortunately, LiteLLM does not currently support VertexAI. LiteLLM focuses on providing a simple interface to call the APIs of services like OpenAI, Azure, Cohere, Anthropic, and Hugging Face. \\n\\nVertexAI is Google's managed machine learning platform. Integrating VertexAI would require additional development work to wrap the VertexAI SDK in a simple interface like LiteLLM provides for other services. \\n\\nHowever, LiteLLM is open source, so it is possible for someone to contribute support for VertexAI. The maintainers would likely welcome a pull request to add VertexAI as an option if implemented well. But out of the box, LiteLLM does not have built-in support for calling VertexAI APIs.\\n\\nThe key services LiteLLM supports are:\\n\\n- OpenAI (GPT, Codex, DALL-E)\\n- Azure Cognitive Services (Text Analytics, Computer Vision, Speech) \\n- Cohere\\n- Anthropic AI\\n- Hugging Face Transformers\\n\\nSo while it doesn't cover every ML API provider, it does make it easy to use the most popular natural language, speech, and vision APIs through a simple interface. Adding VertexAI\"],\n [{\n 'v': 2,\n 'f': \"2\",\n },\n\"how do I set my keys on replicate llama2?\",\n\"To set your keys on Replicate Llama2, follow these steps:\\n\\n1. Open the Llama2 dashboard in your browser.\\n2. Click on the \\\"Settings\\\" tab in the top menu.\\n3. Scroll down to the \\\"API Keys\\\" section.\\n4. Click on the \\\"Add a Key\\\" button.\\n5. Enter a name for your API key to help you identify it later.\\n6. Select the provider for your API key from the dropdown menu. For example, you can select \\\"OpenAI\\\" for OpenAI GPT-3 access.\\n7. Enter your API key in the provided input field. Make sure to copy it correctly.\\n8. Click on the \\\"Save\\\" button to save your API key.\\n\\nNote: The actual steps may vary slightly depending on the platform or interface you are using to access Llama2.\",\n\" Here are the steps to set your API keys on Replicate for litellm:\\n\\n1. Go to your Replicate project settings and select the Environment tab.\\n\\n2. Under Environment Variables, click Add Variable.\\n\\n3. Add variables for the API keys you want to use. The variable names should match the ones used in litellm:\\n\\n- `OPENAI_API_KEY` for OpenAI \\n- `AZURE_API_KEY` for Azure Cognitive Services\\n- `COHERE_API_KEY` for Cohere\\n- `ANTHROPIC_API_KEY` for Anthropic\\n- `HUGGINGFACE_API_KEY` for Hugging Face\\n\\n4. Set the value to your actual API key for each service. Make sure to treat the values as secrets.\\n\\n5. Make sure your litellm code is referencing the environment variable names, for example:\\n\\n```python\\nimport litellm as lm\\n\\nlm.auth(openai_key=os.getenv(\\\"OPENAI_API_KEY\\\")) \\n```\\n\\n6. Restart your Replicate runtime to load the new environment variables.\\n\\nNow litellm will use your\"]],\n columns: [[\"number\", \"index\"], [\"string\", \"Question\"], [\"string\", \"gpt-3.5-turbo\"], [\"string\", \"claude-2\"]],\n columnOptions: [{\"width\": \"1px\", \"className\": \"index_column\"}],\n rowsPerPage: 25,\n helpUrl: \"https://colab.research.google.com/notebooks/data_table.ipynb\",\n suppressOutputScrolling: true,\n minimumWidth: undefined,\n });\n\n function appendQuickchartButton(parentElement) {\n let quickchartButtonContainerElement = document.createElement('div');\n quickchartButtonContainerElement.innerHTML = `\n
\n \n \n\n\n \n
`;\n parentElement.appendChild(quickchartButtonContainerElement);\n }\n\n appendQuickchartButton(table);\n ", - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Questiongpt-3.5-turboclaude-2
0how do i call completion() using LiteLLMTo call the `completion()` function using Lite...Here is how you can call the completion() met...
1does LiteLLM support VertexAIYes, LiteLLM does support Google Cloud Vertex ...Unfortunately, LiteLLM does not currently sup...
2how do I set my keys on replicate llama2?To set your keys on Replicate Llama2, follow t...Here are the steps to set your API keys on Re...
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "
\n", - "
\n" - ], - "text/plain": [ - " Question \\\n", - "0 how do i call completion() using LiteLLM \n", - "1 does LiteLLM support VertexAI \n", - "2 how do I set my keys on replicate llama2? \n", - "\n", - " gpt-3.5-turbo \\\n", - "0 To call the `completion()` function using Lite... \n", - "1 Yes, LiteLLM does support Google Cloud Vertex ... \n", - "2 To set your keys on Replicate Llama2, follow t... \n", - "\n", - " claude-2 \n", - "0 Here is how you can call the completion() met... \n", - "1 Unfortunately, LiteLLM does not currently sup... \n", - "2 Here are the steps to set your API keys on Re... " - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Create a table to visualize results\n", - "import pandas as pd\n", - "\n", - "columns = ['Question'] + models\n", - "df = pd.DataFrame(results, columns=columns)\n", - "\n", - "df" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/LiteLLM_Completion_Cost.ipynb b/cookbook/LiteLLM_Completion_Cost.ipynb deleted file mode 100644 index b8f5eb36a5c..00000000000 --- a/cookbook/LiteLLM_Completion_Cost.ipynb +++ /dev/null @@ -1,241 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# Use LiteLLM to calculate costs for all your completion calls\n", - "In this notebook we'll use `litellm.completion_cost` to get completion costs" - ], - "metadata": { - "id": "BgWr0PsUR3vV" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ViczFTjsDzSI" - }, - "outputs": [], - "source": [ - "!pip install litellm==0.1.549 # use 0.1.549 or later" - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Calculating costs for gpt-3.5 turbo completion()" - ], - "metadata": { - "id": "k_1CWUwmSNtj" - } - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion, completion_cost\n", - "import os\n", - "os.environ['OPENAI_API_KEY'] = \"\"\n", - "\n", - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "response = completion(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=messages,\n", - ")\n", - "\n", - "print(response)\n", - "\n", - "cost = completion_cost(completion_response=response)\n", - "formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n", - "print(formatted_string)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Tp0fyk-jD0pP", - "outputId": "ce885fb3-3237-41b2-9d8b-3fb30bba498b" - }, - "execution_count": 6, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "got response\n", - "{\n", - " \"id\": \"chatcmpl-7vyCApIZaCxP36kb9meUMN2DFSJPh\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1694050442,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello! I'm an AI and I don't have feelings, but I'm here to help you. How can I assist you today?\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 28,\n", - " \"total_tokens\": 41\n", - " }\n", - "}\n", - "Cost for completion call: $0.0000755000\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Calculating costs for Together Computer completion()" - ], - "metadata": { - "id": "AjDs4G-uS6PS" - } - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion, completion_cost\n", - "import os\n", - "os.environ['TOGETHERAI_API_KEY'] = \"\"\n", - "\n", - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "response = completion(\n", - " model=\"togethercomputer/llama-2-70b-chat\",\n", - " messages=messages,\n", - ")\n", - "\n", - "print(response)\n", - "\n", - "cost = completion_cost(completion_response=response)\n", - "formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n", - "print(formatted_string)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "jMPsUV-KEa6a", - "outputId": "7a69b291-f149-4b9c-8a78-9c8142bac759" - }, - "execution_count": 7, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "{\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"Hello! I'm doing well, thanks for asking. I hope you're having a great\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1694050771.2821715,\n", - " \"model\": \"togethercomputer/llama-2-70b-chat\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 12,\n", - " \"completion_tokens\": 18,\n", - " \"total_tokens\": 30\n", - " }\n", - "}\n", - "Cost for completion call: $0.0000900000\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Calculating costs for Replicate Llama2 completion()" - ], - "metadata": { - "id": "vEa4s6-7TANS" - } - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion, completion_cost\n", - "import os\n", - "os.environ['REPLICATE_API_KEY'] = \"\"\n", - "\n", - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "response = completion(\n", - " model=\"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n", - " messages=messages,\n", - ")\n", - "\n", - "print(response)\n", - "\n", - "cost = completion_cost(completion_response=response)\n", - "formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n", - "print(formatted_string)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Xf1TKRDuS1bR", - "outputId": "cfb2b484-a6e5-41ad-86c5-7e66aba27648" - }, - "execution_count": 8, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "{\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" Hello! I'm doing well, thanks for asking. How about you? Is there anything you need help with today?\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1694050893.4534576,\n", - " \"model\": \"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 24,\n", - " \"total_tokens\": 30\n", - " },\n", - " \"ended\": 1694050896.6689413\n", - "}\n", - "total_replicate_run_time 3.2154836654663086\n", - "Cost for completion call: $0.0045016771\n" - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/cookbook/LiteLLM_HuggingFace.ipynb b/cookbook/LiteLLM_HuggingFace.ipynb deleted file mode 100644 index bf8482a5f11..00000000000 --- a/cookbook/LiteLLM_HuggingFace.ipynb +++ /dev/null @@ -1,252 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "9dKM5k8qsMIj" - }, - "source": [ - "## LiteLLM Hugging Face\n", - "\n", - "Docs for huggingface: https://docs.litellm.ai/docs/providers/huggingface\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "BVDdmCp-o97j" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yp5UXRqtpu9f" - }, - "source": [ - "## Serverless Inference Providers\n", - "\n", - "Read more about Inference Providers here: https://huggingface.co/blog/inference-providers.\n", - "\n", - "In order to use litellm with Hugging Face Inference Providers, you need to set `model=huggingface//`.\n", - "\n", - "Example: `huggingface/together/deepseek-ai/DeepSeek-R1` to run DeepSeek-R1 (https://huggingface.co/deepseek-ai/DeepSeek-R1) through Together AI.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Pi5Oww8gpCUm", - "outputId": "659a67c7-f90d-4c06-b94e-2c4aa92d897a" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "# You can create a HF token here: https://huggingface.co/settings/tokens\n", - "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n", - "\n", - "# Call DeepSeek-R1 model through Together AI\n", - "response = completion(\n", - " model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n", - " messages=[{\"content\": \"How many r's are in the word `strawberry`?\", \"role\": \"user\"}],\n", - ")\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EU0UubrKzTFe" - }, - "source": [ - "## Streaming\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "y-QfIvA-uJKX", - "outputId": "b007bb98-00d0-44a4-8264-c8a2caed6768" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n", - "\n", - "response = completion(\n", - " model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"How many r's are in the word `strawberry`?\",\n", - " \n", - " }\n", - " ],\n", - " stream=True,\n", - ")\n", - "\n", - "for chunk in response:\n", - " print(chunk)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## With images as input\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from litellm import completion\n", - "\n", - "# Set your Hugging Face Token\n", - "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n", - "\n", - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"text\", \"text\": \"What's in this image?\"},\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": \"https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png\",\n", - " },\n", - " },\n", - " ],\n", - " }\n", - "]\n", - "\n", - "response = completion(\n", - " model=\"huggingface/sambanova/meta-llama/Llama-3.3-70B-Instruct\",\n", - " messages=messages,\n", - ")\n", - "print(response.choices[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tools - Function Calling\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "\n", - "# Set your Hugging Face Token\n", - "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n", - "\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", - " },\n", - " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n", - " },\n", - " \"required\": [\"location\"],\n", - " },\n", - " },\n", - " }\n", - "]\n", - "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n", - "\n", - "response = completion(\n", - " model=\"huggingface/sambanova/meta-llama/Llama-3.1-8B-Instruct\", messages=messages, tools=tools, tool_choice=\"auto\"\n", - ")\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Hugging Face Dedicated Inference Endpoints\n", - "\n", - "Steps to use\n", - "\n", - "- Create your own Hugging Face dedicated endpoint here: https://ui.endpoints.huggingface.co/\n", - "- Set `api_base` to your deployed api base\n", - "- set the model to `huggingface/tgi` so that litellm knows it's a huggingface Deployed Inference Endpoint.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import litellm\n", - "\n", - "\n", - "response = litellm.completion(\n", - " model=\"huggingface/tgi\",\n", - " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}],\n", - " api_base=\"https://my-endpoint.endpoints.huggingface.cloud/v1/\",\n", - ")\n", - "print(response)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.0" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb b/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb deleted file mode 100644 index 8fa7d0b987a..00000000000 --- a/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb +++ /dev/null @@ -1,97 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "iFEmsVJI_2BR" - }, - "source": [ - "# LiteLLM NovitaAI Cookbook" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cBlUhCEP_xj4" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "p-MQqWOT_1a7" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['NOVITA_API_KEY'] = \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Ze8JqMqWAARO" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "response = completion(\n", - " model=\"novita/deepseek/deepseek-r1\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "-LnhELrnAM_J" - }, - "outputs": [], - "source": [ - "response = completion(\n", - " model=\"novita/deepseek/deepseek-r1\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "dJBOUYdwCEn1" - }, - "outputs": [], - "source": [ - "response = completion(\n", - " model=\"mistralai/mistral-7b-instruct\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_OpenRouter.ipynb b/cookbook/LiteLLM_OpenRouter.ipynb deleted file mode 100644 index 6444b23b294..00000000000 --- a/cookbook/LiteLLM_OpenRouter.ipynb +++ /dev/null @@ -1,179 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "iFEmsVJI_2BR" - }, - "source": [ - "# LiteLLM OpenRouter Cookbook" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cBlUhCEP_xj4" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "id": "p-MQqWOT_1a7" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['OPENROUTER_API_KEY'] = \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Ze8JqMqWAARO", - "outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"id\": \"gen-W8FTMSIEorCp3vG5iYIgNMR4IeBv\",\n", - " \"model\": \"chat-bison@001\",\n", - " \"choices\": [\n", - " {\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"```\\n#include \\n\\nint main() {\\n printf(\\\"Hi!\\\\n\\\");\\n return 0;\\n}\\n```\"\n", - " }\n", - " }\n", - " ],\n", - " \"response_ms\": 7817.777999999999\n", - "}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from litellm import completion\n", - "response = completion(\n", - " model=\"openrouter/google/palm-2-chat-bison\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-LnhELrnAM_J", - "outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \" Here is some simple code to print \\\"Hi\\\":\\n\\n```python\\nprint(\\\"Hi\\\")\\n```\\n\\nThis uses the print() function in Python to output the text \\\"Hi\\\".\"\n", - " },\n", - " \"finish_reason\": \"stop_sequence\"\n", - " }\n", - " ],\n", - " \"model\": \"claude-2.0\",\n", - " \"id\": \"gen-IiuV7ZNimDufVeutBHrl8ajPuzEh\",\n", - " \"response_ms\": 8112.443000000001\n", - "}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response = completion(\n", - " model=\"openrouter/anthropic/claude-2\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "dJBOUYdwCEn1", - "outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"id\": \"gen-PyMd3yyJ0aQsCgIY9R8XGZoAtPbl\",\n", - " \"model\": \"togethercomputer/llama-2-70b-chat\",\n", - " \"choices\": [\n", - " {\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"*gives a sly smile as they type*\\n\\nHey there, handsome. \\ud83d\\ude0f\\n\\nWhat brings you to my neck of the woods today? \\ud83d\\ude18\"\n", - " }\n", - " }\n", - " ],\n", - " \"response_ms\": 9618.775\n", - "}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response = completion(\n", - " model=\"openrouter/meta-llama/llama-2-70b-chat\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_Petals.ipynb b/cookbook/LiteLLM_Petals.ipynb deleted file mode 100644 index aacc22dd1b9..00000000000 --- a/cookbook/LiteLLM_Petals.ipynb +++ /dev/null @@ -1,568 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "dwGtLi_tvM6N" - }, - "source": [ - "# Using LiteLLM with Petals" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "bdlgaWQqDpzj" - }, - "outputs": [], - "source": [ - "!pip install litellm # 0.1.715 and upwards" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "5Id2QKwOEH8X" - }, - "outputs": [], - "source": [ - "# install petals\n", - "!pip install git+https://github.com/bigscience-workshop/petals" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "k42fldw3veSN" - }, - "source": [ - "## petals-team/StableBeluga2" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tIHcEHdSDqju", - "outputId": "485dbf54-395c-433a-bbf4-8eb70a9fa624" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "You are using the default legacy behaviour of the . If you see this, DO NOT PANIC! This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n", - "Sep 19 18:39:50.634 [\u001b[1m\u001b[34mINFO\u001b[0m] Make sure you follow the LLaMA's terms of use: https://bit.ly/llama2-license for LLaMA 2, https://bit.ly/llama-license for LLaMA 1\n", - "Sep 19 18:39:50.639 [\u001b[1m\u001b[34mINFO\u001b[0m] Using DHT prefix: StableBeluga2-hf\n", - "Sep 19 18:40:13.920 [\u001b[1m\u001b[34mINFO\u001b[0m] Route found: 0:40 via …HfQWVM => 40:80 via …Zj98Se\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"Hello, how are you?\\nI'm doing well, thank you. I'm just getting ready to go to the gym.\\nOh, that's great. I'm trying to get back into a workout routine myself.\\nYeah,\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-f09d79b3-c1d1-49b7-b55f-cd8dfa1043bf\",\n", - " \"created\": 1695148897.473613,\n", - " \"model\": \"petals-team/StableBeluga2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 45,\n", - " \"total_tokens\": 51\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(model=\"petals/petals-team/StableBeluga2\", messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}], max_tokens=50)\n", - "\n", - "print(response)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "J8DubRnHvh_j" - }, - "source": [ - "## huggyllama/llama-65b" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 538, - "referenced_widgets": [ - "2fec5cc400424671a3d517327117d18a", - "3687c76fe84d464baaf35366b21e83b3", - "c29d4460dbaa441cae110b58e0014151", - "6560449a38bf4a7bacd97ccaacf01c4c", - "5fbd6ae281984d28ba59ebfd0279eda7", - "323e30e275434aeea241163e5f1f9031", - "48f4adec51c94f9da6e4c4564daeff84", - "2a672981a44b4a7fb30674f97f4c10c6", - "d75ae8d22ea74840b4c80c8f386384c4", - "54c06312ecff4e7588665e8b0cb7118b", - "300078a9d1a6483fba81a4be63793ff7" - ] - }, - "id": "IlTCJwDsNvgF", - "outputId": "2e84d125-d982-48ed-8a92-6ca438a50d0c" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Sep 19 18:41:37.912 [\u001b[1m\u001b[34mINFO\u001b[0m] Make sure you follow the LLaMA's terms of use: https://bit.ly/llama2-license for LLaMA 2, https://bit.ly/llama-license for LLaMA 1\n", - "Sep 19 18:41:37.914 [\u001b[1m\u001b[34mINFO\u001b[0m] Using DHT prefix: llama-65b-hf\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2fec5cc400424671a3d517327117d18a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Loading checkpoint shards: 0%| | 0/2 [00:00 JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" Good morning!\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1694030351.309254,\n", - " \"model\": \"claude-2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 11,\n", - " \"completion_tokens\": 3,\n", - " \"total_tokens\": 14\n", - " }\n", - " },\n", - " JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" I'm an AI assistant created by Anthropic. I don't actually have a concept of the current time.\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1694030352.1215081,\n", - " \"model\": \"claude-2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 22,\n", - " \"total_tokens\": 35\n", - " }\n", - " }]" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import os\n", - "\n", - "os.environ['ANTHROPIC_API_KEY'] = \"\"\n", - "\n", - "\n", - "responses = batch_completion(\n", - " model=\"claude-2\",\n", - " messages = [\n", - " [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"good morning? \"\n", - " }\n", - " ],\n", - " [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what's the time? \"\n", - " }\n", - " ]\n", - " ]\n", - ")\n", - "responses" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb b/cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb deleted file mode 100644 index d89f2b67c56..00000000000 --- a/cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb +++ /dev/null @@ -1,568 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "kccfk0mHZ4Ad" - }, - "source": [ - "# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n", - "\n", - "Covers:\n", - "\n", - "* /chat/completion\n", - "* /embedding\n", - "\n", - "\n", - "These are **selected examples**. LiteLLM Proxy is **OpenAI-Compatible**, it works with any project that calls OpenAI. Just change the `base_url`, `api_key` and `model`.\n", - "\n", - "Set `LITELLM_MASTER_KEY` in the notebook environment to the same key configured on the proxy before running authenticated examples\n", - "\n", - "For more examples, [go here](https://docs.litellm.ai/docs/proxy/user_keys)\n", - "\n", - "To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n", - "\n", - "To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nmSClzCPaGH6" - }, - "source": [ - "## /chat/completion\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_vqcjwOVaKpO" - }, - "source": [ - "### OpenAI Python SDK" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "x1e_Ok3KZzeP" - }, - "outputs": [], - "source": [ - "import openai\n", - "client = openai.OpenAI(\n", - " api_key=\"anything\",\n", - " base_url=\"http://0.0.0.0:4000\"\n", - ")\n", - "\n", - "# request sent to model set on litellm proxy, `litellm --model`\n", - "response = client.chat.completions.create(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"this is a test request, write a short poem\"\n", - " }\n", - " ],\n", - " extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params\n", - " \"metadata\": { # 👈 use for logging additional params (e.g. to langfuse)\n", - " \"generation_name\": \"ishaan-generation-openai-client\",\n", - " \"generation_id\": \"openai-client-gen-id22\",\n", - " \"trace_id\": \"openai-client-trace-id22\",\n", - " \"trace_user_id\": \"openai-client-user-id2\"\n", - " }\n", - " }\n", - ")\n", - "\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "AqkyKk9Scxgj" - }, - "source": [ - "## Function Calling" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "wDg10VqLczE1" - }, - "outputs": [], - "source": [ - "import os\n", - "from openai import OpenAI\n", - "client = OpenAI(\n", - " api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n", - " base_url=\"http://0.0.0.0:4000\",\n", - ")\n", - "\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", - " },\n", - " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n", - " },\n", - " \"required\": [\"location\"],\n", - " },\n", - " }\n", - " }\n", - "]\n", - "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n", - "completion = client.chat.completions.create(\n", - " model=\"gpt-4o\", # use 'model_name' from config.yaml\n", - " messages=messages,\n", - " tools=tools,\n", - " tool_choice=\"auto\"\n", - ")\n", - "\n", - "print(completion)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YYoxLloSaNWW" - }, - "source": [ - "### Azure OpenAI Python SDK" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "yA1XcgowaSRy" - }, - "outputs": [], - "source": [ - "import openai\n", - "client = openai.AzureOpenAI(\n", - " api_key=\"anything\",\n", - " base_url=\"http://0.0.0.0:4000\"\n", - ")\n", - "\n", - "# request sent to model set on litellm proxy, `litellm --model`\n", - "response = client.chat.completions.create(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"this is a test request, write a short poem\"\n", - " }\n", - " ],\n", - " extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params\n", - " \"metadata\": { # 👈 use for logging additional params (e.g. to langfuse)\n", - " \"generation_name\": \"ishaan-generation-openai-client\",\n", - " \"generation_id\": \"openai-client-gen-id22\",\n", - " \"trace_id\": \"openai-client-trace-id22\",\n", - " \"trace_user_id\": \"openai-client-user-id2\"\n", - " }\n", - " }\n", - ")\n", - "\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yl9qhDvnaTpL" - }, - "source": [ - "### Langchain Python" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "5MUZgSquaW5t" - }, - "outputs": [], - "source": [ - "from langchain.chat_models import ChatOpenAI\n", - "from langchain.prompts.chat import (\n", - " ChatPromptTemplate,\n", - " HumanMessagePromptTemplate,\n", - " SystemMessagePromptTemplate,\n", - ")\n", - "from langchain.schema import HumanMessage, SystemMessage\n", - "import os\n", - "\n", - "os.environ[\"OPENAI_API_KEY\"] = \"anything\"\n", - "\n", - "chat = ChatOpenAI(\n", - " openai_api_base=\"http://0.0.0.0:4000\",\n", - " model = \"gpt-3.5-turbo\",\n", - " temperature=0.1,\n", - " extra_body={\n", - " \"metadata\": {\n", - " \"generation_name\": \"ishaan-generation-langchain-client\",\n", - " \"generation_id\": \"langchain-client-gen-id22\",\n", - " \"trace_id\": \"langchain-client-trace-id22\",\n", - " \"trace_user_id\": \"langchain-client-user-id2\"\n", - " }\n", - " }\n", - ")\n", - "\n", - "messages = [\n", - " SystemMessage(\n", - " content=\"You are a helpful assistant that im using to make a test request to.\"\n", - " ),\n", - " HumanMessage(\n", - " content=\"test from litellm. tell me why it's amazing in 1 sentence\"\n", - " ),\n", - "]\n", - "response = chat(messages)\n", - "\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "B9eMgnULbRaz" - }, - "source": [ - "### Curl" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "VWCCk5PFcmhS" - }, - "source": [ - "\n", - "\n", - "```\n", - "curl -X POST 'http://0.0.0.0:4000/chat/completions' \\\n", - " -H 'Content-Type: application/json' \\\n", - " -d '{\n", - " \"model\": \"gpt-3.5-turbo\",\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what llm are you\"\n", - " }\n", - " ],\n", - " \"metadata\": {\n", - " \"generation_name\": \"ishaan-test-generation\",\n", - " \"generation_id\": \"gen-id22\",\n", - " \"trace_id\": \"trace-id22\",\n", - " \"trace_user_id\": \"user-id2\"\n", - " }\n", - "}'\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "drBAm2e1b6xe" - }, - "source": [ - "### LlamaIndex" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "d0bZcv8fb9mL" - }, - "outputs": [], - "source": [ - "import os, dotenv\n", - "\n", - "from llama_index.llms import AzureOpenAI\n", - "from llama_index.embeddings import AzureOpenAIEmbedding\n", - "from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext\n", - "\n", - "llm = AzureOpenAI(\n", - " engine=\"azure-gpt-3.5\", # model_name on litellm proxy\n", - " temperature=0.0,\n", - " azure_endpoint=\"http://0.0.0.0:4000\", # litellm proxy endpoint\n", - " api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n", - " api_version=\"2023-07-01-preview\",\n", - ")\n", - "\n", - "embed_model = AzureOpenAIEmbedding(\n", - " deployment_name=\"azure-embedding-model\",\n", - " azure_endpoint=\"http://0.0.0.0:4000\",\n", - " api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n", - " api_version=\"2023-07-01-preview\",\n", - ")\n", - "\n", - "\n", - "documents = SimpleDirectoryReader(\"llama_index_data\").load_data()\n", - "service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)\n", - "index = VectorStoreIndex.from_documents(documents, service_context=service_context)\n", - "\n", - "query_engine = index.as_query_engine()\n", - "response = query_engine.query(\"What did the author do growing up?\")\n", - "print(response)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xypvNdHnb-Yy" - }, - "source": [ - "### Langchain JS" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "R55mK2vCcBN2" - }, - "outputs": [], - "source": [ - "import { ChatOpenAI } from \"@langchain/openai\";\n", - "\n", - "\n", - "const model = new ChatOpenAI({\n", - " modelName: \"gpt-4\",\n", - " openAIApiKey: process.env.LITELLM_MASTER_KEY,\n", - " modelKwargs: {\"metadata\": \"hello world\"} // 👈 PASS Additional params here\n", - "}, {\n", - " basePath: \"http://0.0.0.0:4000\",\n", - "});\n", - "\n", - "const message = await model.invoke(\"Hi there!\");\n", - "\n", - "console.log(message);\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nC4bLifCcCiW" - }, - "source": [ - "### OpenAI JS" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "MICH8kIMcFpg" - }, - "outputs": [], - "source": [ - "const { OpenAI } = require('openai');\n", - "\n", - "const openai = new OpenAI({\n", - " apiKey: process.env.LITELLM_MASTER_KEY,\n", - " baseURL: \"http://0.0.0.0:4000\"\n", - "});\n", - "\n", - "async function main() {\n", - " const chatCompletion = await openai.chat.completions.create({\n", - " messages: [{ role: 'user', content: 'Say this is a test' }],\n", - " model: 'gpt-3.5-turbo',\n", - " }, {\"metadata\": {\n", - " \"generation_name\": \"ishaan-generation-openaijs-client\",\n", - " \"generation_id\": \"openaijs-client-gen-id22\",\n", - " \"trace_id\": \"openaijs-client-trace-id22\",\n", - " \"trace_user_id\": \"openaijs-client-user-id2\"\n", - " }});\n", - "}\n", - "\n", - "main();\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "D1Q07pEAcGTb" - }, - "source": [ - "### Anthropic SDK" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qBjFcAvgcI3t" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "from anthropic import Anthropic\n", - "\n", - "client = Anthropic(\n", - " base_url=\"http://localhost:4000\", # proxy endpoint\n", - " api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n", - ")\n", - "\n", - "message = client.messages.create(\n", - " max_tokens=1024,\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Hello, Claude\",\n", - " }\n", - " ],\n", - " model=\"claude-3-opus-20240229\",\n", - ")\n", - "print(message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dFAR4AJGcONI" - }, - "source": [ - "## /embeddings" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lgNoM281cRzR" - }, - "source": [ - "### OpenAI Python SDK" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NY3DJhPfcQhA" - }, - "outputs": [], - "source": [ - "import openai\n", - "from openai import OpenAI\n", - "\n", - "# set base_url to your proxy server\n", - "# set api_key to send to proxy server\n", - "client = OpenAI(api_key=\"\", base_url=\"http://0.0.0.0:4000\")\n", - "\n", - "response = client.embeddings.create(\n", - " input=[\"hello from litellm\"],\n", - " model=\"text-embedding-ada-002\"\n", - ")\n", - "\n", - "print(response)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hmbg-DW6cUZs" - }, - "source": [ - "### Langchain Embeddings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "lX2S8Nl1cWVP" - }, - "outputs": [], - "source": [ - "from langchain.embeddings import OpenAIEmbeddings\n", - "\n", - "embeddings = OpenAIEmbeddings(model=\"sagemaker-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n", - "\n", - "\n", - "text = \"This is a test document.\"\n", - "\n", - "query_result = embeddings.embed_query(text)\n", - "\n", - "print(f\"SAGEMAKER EMBEDDINGS\")\n", - "print(query_result[:5])\n", - "\n", - "embeddings = OpenAIEmbeddings(model=\"bedrock-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n", - "\n", - "text = \"This is a test document.\"\n", - "\n", - "query_result = embeddings.embed_query(text)\n", - "\n", - "print(f\"BEDROCK EMBEDDINGS\")\n", - "print(query_result[:5])\n", - "\n", - "embeddings = OpenAIEmbeddings(model=\"bedrock-titan-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n", - "\n", - "text = \"This is a test document.\"\n", - "\n", - "query_result = embeddings.embed_query(text)\n", - "\n", - "print(f\"TITAN EMBEDDINGS\")\n", - "print(query_result[:5])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oqGbWBCQcYfd" - }, - "source": [ - "### Curl Request" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7rkIMV9LcdwQ" - }, - "source": [ - "\n", - "\n", - "```curl\n", - "curl -X POST 'http://0.0.0.0:4000/embeddings' \\\n", - " -H 'Content-Type: application/json' \\\n", - " -d ' {\n", - " \"model\": \"text-embedding-ada-002\",\n", - " \"input\": [\"write a litellm poem\"]\n", - " }'\n", - "```\n", - "\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/Parallel_function_calling.ipynb b/cookbook/Parallel_function_calling.ipynb deleted file mode 100644 index cb7fbaface2..00000000000 --- a/cookbook/Parallel_function_calling.ipynb +++ /dev/null @@ -1,478 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "This is a tutorial on using Parallel function calling with LiteLLM" - ], - "metadata": { - "id": "gHwFJ-srdnku" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "RrtHuVHlZmUe" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "source": [ - "This tutorial walks through the steps doing parallel function calling using\n", - " - OpenAI\n", - " - Azure OpenAI" - ], - "metadata": { - "id": "sG5ANaazjU0g" - } - }, - { - "cell_type": "code", - "source": [ - "# set openai api key\n", - "import os\n", - "os.environ['OPENAI_API_KEY'] = \"\" # litellm reads OPENAI_API_KEY from .env and sends the request" - ], - "metadata": { - "id": "l4GQ-M5yZ5UW" - }, - "execution_count": 3, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "\n", - "# OpenAI gpt-3.5-turbo-1106\n", - "## Step 1: send the conversation and available functions to the model" - ], - "metadata": { - "id": "AxgR2fCgaRoW" - } - }, - { - "cell_type": "code", - "source": [ - "import litellm\n", - "import json\n", - "# Example dummy function hard coded to return the same weather\n", - "# In production, this could be your backend API or an external API\n", - "def get_current_weather(location, unit=\"fahrenheit\"):\n", - " \"\"\"Get the current weather in a given location\"\"\"\n", - " if \"tokyo\" in location.lower():\n", - " return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n", - " elif \"san francisco\" in location.lower():\n", - " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n", - " elif \"paris\" in location.lower():\n", - " return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n", - " else:\n", - " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", - "\n", - "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", - " },\n", - " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n", - " },\n", - " \"required\": [\"location\"],\n", - " },\n", - " },\n", - " }\n", - "]\n", - "\n", - "response = litellm.completion(\n", - " model=\"gpt-3.5-turbo-1106\",\n", - " messages=messages,\n", - " tools=tools,\n", - " tool_choice=\"auto\", # auto is default, but we'll be explicit\n", - ")\n", - "print(\"\\nLLM Response1:\\n\", response)\n", - "response_message = response.choices[0].message\n", - "tool_calls = response.choices[0].message.tool_calls\n", - "print(\"\\nTool Choice:\\n\", tool_calls)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Y3qteFo8ZrZP", - "outputId": "ee6c1183-55c1-4111-cdc0-967b8fed9db3" - }, - "execution_count": 18, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "LLM Response1:\n", - " ModelResponse(id='chatcmpl-8MNdPbrhtnwiPK1x3PEoGwrH144TW', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')]))], created=1700344759, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 77, 'prompt_tokens': 88, 'total_tokens': 165}, _response_ms=1049.913)\n", - "\n", - "Tool Choice:\n", - " [ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')]\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Step 2 - Parse the Model Response and Execute Functions" - ], - "metadata": { - "id": "tD4lJQ40cU44" - } - }, - { - "cell_type": "code", - "source": [ - "# Check if the model wants to call a function\n", - "if tool_calls:\n", - " # Execute the functions and prepare responses\n", - " available_functions = {\n", - " \"get_current_weather\": get_current_weather,\n", - " }\n", - "\n", - " messages.append(response_message) # Extend conversation with assistant's reply\n", - "\n", - " for tool_call in tool_calls:\n", - " print(f\"\\nExecuting tool call\\n{tool_call}\")\n", - " function_name = tool_call.function.name\n", - " function_to_call = available_functions[function_name]\n", - " function_args = json.loads(tool_call.function.arguments)\n", - " # calling the get_current_weather() function\n", - " function_response = function_to_call(\n", - " location=function_args.get(\"location\"),\n", - " unit=function_args.get(\"unit\"),\n", - " )\n", - " print(f\"Result from tool call\\n{function_response}\\n\")\n", - "\n", - " # Extend conversation with function response\n", - " messages.append(\n", - " {\n", - " \"tool_call_id\": tool_call.id,\n", - " \"role\": \"tool\",\n", - " \"name\": function_name,\n", - " \"content\": function_response,\n", - " }\n", - " )\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "af4oXQvicV_n", - "outputId": "abf6ac3e-4a21-4a4f-b8d7-809b763d0632" - }, - "execution_count": 21, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "Executing tool call\n", - "ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n", - "Result from tool call\n", - "{\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n", - "\n", - "\n", - "Executing tool call\n", - "ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n", - "Result from tool call\n", - "{\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"}\n", - "\n", - "\n", - "Executing tool call\n", - "ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n", - "Result from tool call\n", - "{\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"}\n", - "\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Step 3 - Second litellm.completion() call" - ], - "metadata": { - "id": "E3OL1fqUdFdv" - } - }, - { - "cell_type": "code", - "source": [ - "second_response = litellm.completion(\n", - " model=\"gpt-3.5-turbo-1106\",\n", - " messages=messages,\n", - ")\n", - "print(\"Second Response\\n\", second_response)\n", - "print(\"Second Response Message\\n\", second_response.choices[0].message.content)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "8KYB2n-jc1_f", - "outputId": "6c6448ae-1c09-43ae-eb90-208b118e6179" - }, - "execution_count": 26, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Second Response\n", - " ModelResponse(id='chatcmpl-8MNhat166ZqjO6egXcUh85Pd0s7KV', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.\", role='assistant'))], created=1700345018, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 28, 'prompt_tokens': 465, 'total_tokens': 493}, _response_ms=999.246)\n", - "Second Response Message\n", - " The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Using Azure OpenAI" - ], - "metadata": { - "id": "1cIIFEvXjofp" - } - }, - { - "cell_type": "code", - "source": [ - "# set Azure env variables\n", - "import os\n", - "os.environ['AZURE_API_KEY'] = \"\" # litellm reads AZURE_API_KEY from .env and sends the request\n", - "os.environ['AZURE_API_BASE'] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - "os.environ['AZURE_API_VERSION'] = \"2023-07-01-preview\"" - ], - "metadata": { - "id": "lG9mUnModeeE" - }, - "execution_count": 32, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Step 1" - ], - "metadata": { - "id": "17S-Ysksj-E_" - } - }, - { - "cell_type": "code", - "source": [ - "import litellm\n", - "import json\n", - "# Example dummy function hard coded to return the same weather\n", - "# In production, this could be your backend API or an external API\n", - "def get_current_weather(location, unit=\"fahrenheit\"):\n", - " \"\"\"Get the current weather in a given location\"\"\"\n", - " if \"tokyo\" in location.lower():\n", - " return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n", - " elif \"san francisco\" in location.lower():\n", - " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n", - " elif \"paris\" in location.lower():\n", - " return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n", - " else:\n", - " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", - "\n", - "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", - " },\n", - " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n", - " },\n", - " \"required\": [\"location\"],\n", - " },\n", - " },\n", - " }\n", - "]\n", - "\n", - "response = litellm.completion(\n", - " model=\"azure/chatgpt-functioncalling\", # model = azure/\n", - " messages=messages,\n", - " tools=tools,\n", - " tool_choice=\"auto\", # auto is default, but we'll be explicit\n", - ")\n", - "print(\"\\nLLM Response1:\\n\", response)\n", - "response_message = response.choices[0].message\n", - "tool_calls = response.choices[0].message.tool_calls\n", - "print(\"\\nTool Choice:\\n\", tool_calls)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "boAIHLEXj80m", - "outputId": "00afcf09-5b6b-4805-c374-ba089cc6eb43" - }, - "execution_count": 33, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "LLM Response1:\n", - " ModelResponse(id='chatcmpl-8MOBPvEnqG7qitkmVqZmCrzSGEmDj', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')]))], created=1700346867, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 19, 'prompt_tokens': 88, 'total_tokens': 107}, _response_ms=833.4319999999999)\n", - "\n", - "Tool Choice:\n", - " [ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')]\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Step 2" - ], - "metadata": { - "id": "hqh1y1IMkmGO" - } - }, - { - "cell_type": "code", - "source": [ - "# Check if the model wants to call a function\n", - "if tool_calls:\n", - " # Execute the functions and prepare responses\n", - " available_functions = {\n", - " \"get_current_weather\": get_current_weather,\n", - " }\n", - "\n", - " messages.append(response_message) # Extend conversation with assistant's reply\n", - "\n", - " for tool_call in tool_calls:\n", - " print(f\"\\nExecuting tool call\\n{tool_call}\")\n", - " function_name = tool_call.function.name\n", - " function_to_call = available_functions[function_name]\n", - " function_args = json.loads(tool_call.function.arguments)\n", - " # calling the get_current_weather() function\n", - " function_response = function_to_call(\n", - " location=function_args.get(\"location\"),\n", - " unit=function_args.get(\"unit\"),\n", - " )\n", - " print(f\"Result from tool call\\n{function_response}\\n\")\n", - "\n", - " # Extend conversation with function response\n", - " messages.append(\n", - " {\n", - " \"tool_call_id\": tool_call.id,\n", - " \"role\": \"tool\",\n", - " \"name\": function_name,\n", - " \"content\": function_response,\n", - " }\n", - " )\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "FGu7DY7PkOiG", - "outputId": "96d39ae7-7fc8-4dd8-c82f-5ee9a486724c" - }, - "execution_count": 34, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "Executing tool call\n", - "ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')\n", - "Result from tool call\n", - "{\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n", - "\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Step 3" - ], - "metadata": { - "id": "4MjYyeajkpBl" - } - }, - { - "cell_type": "code", - "source": [ - "second_response = litellm.completion(\n", - " model=\"azure/chatgpt-functioncalling\",\n", - " messages=messages,\n", - ")\n", - "print(\"Second Response\\n\", second_response)\n", - "print(\"Second Response Message\\n\", second_response.choices[0].message.content)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "qHgXyZq1kqGn", - "outputId": "61a30470-d7f5-484d-c42b-681c9b60b34a" - }, - "execution_count": 36, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Second Response\n", - " ModelResponse(id='chatcmpl-8MOC90vwZ2LHX0DE796XYtsOxdGcc', choices=[Choices(finish_reason='stop', index=0, message=Message(content='The current weather in San Francisco is 72°F.', role='assistant'))], created=1700346913, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 11, 'prompt_tokens': 69, 'total_tokens': 80}, _response_ms=824.882)\n", - "Second Response Message\n", - " The current weather in San Francisco is 72°F.\n" - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/cookbook/Proxy_Batch_Users.ipynb b/cookbook/Proxy_Batch_Users.ipynb deleted file mode 100644 index bd18de5e988..00000000000 --- a/cookbook/Proxy_Batch_Users.ipynb +++ /dev/null @@ -1,208 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "680oRk1af-xJ" - }, - "source": [ - "# Environment Setup\n", - "\n", - "Set `LITELLM_MASTER_KEY` in the notebook environment to the same key configured on the proxy before running the cells" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "X7TgJFn8f88p" - }, - "outputs": [], - "source": [ - "import os\n", - "import csv\n", - "from typing import Optional\n", - "import httpx\n", - "import json\n", - "import asyncio\n", - "\n", - "proxy_base_url = \"http://0.0.0.0:4000\" # 👈 SET TO PROXY URL\n", - "master_key = os.environ[\"LITELLM_MASTER_KEY\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "rauw8EOhgBz5" - }, - "outputs": [], - "source": [ - "## GLOBAL HTTP CLIENT ## - faster http calls\n", - "class HTTPHandler:\n", - " def __init__(self, concurrent_limit=1000):\n", - " # Create a client with a connection pool\n", - " self.client = httpx.AsyncClient(\n", - " limits=httpx.Limits(\n", - " max_connections=concurrent_limit,\n", - " max_keepalive_connections=concurrent_limit,\n", - " )\n", - " )\n", - "\n", - " async def close(self):\n", - " # Close the client when you're done with it\n", - " await self.client.aclose()\n", - "\n", - " async def get(\n", - " self, url: str, params: Optional[dict] = None, headers: Optional[dict] = None\n", - " ):\n", - " response = await self.client.get(url, params=params, headers=headers)\n", - " return response\n", - "\n", - " async def post(\n", - " self,\n", - " url: str,\n", - " data: Optional[dict] = None,\n", - " params: Optional[dict] = None,\n", - " headers: Optional[dict] = None,\n", - " ):\n", - " try:\n", - " response = await self.client.post(\n", - " url, data=data, params=params, headers=headers\n", - " )\n", - " return response\n", - " except Exception as e:\n", - " raise e\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7LXN8zaLgOie" - }, - "source": [ - "# Import Sheet\n", - "\n", - "\n", - "Format: | ID | Name | Max Budget |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oiED0usegPGf" - }, - "outputs": [], - "source": [ - "async def import_sheet():\n", - " tasks = []\n", - " http_client = HTTPHandler()\n", - " with open('my-batch-sheet.csv', 'r') as file:\n", - " csv_reader = csv.DictReader(file)\n", - " for row in csv_reader:\n", - " task = create_user(client=http_client, user_id=row['ID'], max_budget=row['Max Budget'], user_name=row['Name'])\n", - " tasks.append(task)\n", - " # print(f\"ID: {row['ID']}, Name: {row['Name']}, Max Budget: {row['Max Budget']}\")\n", - "\n", - " keys = await asyncio.gather(*tasks)\n", - "\n", - " with open('my-batch-sheet_new.csv', 'w', newline='') as new_file:\n", - " fieldnames = ['ID', 'Name', 'Max Budget', 'keys']\n", - " csv_writer = csv.DictWriter(new_file, fieldnames=fieldnames)\n", - " csv_writer.writeheader()\n", - "\n", - " with open('my-batch-sheet.csv', 'r') as file:\n", - " csv_reader = csv.DictReader(file)\n", - " for i, row in enumerate(csv_reader):\n", - " row['keys'] = keys[i] # Add the 'keys' value from the corresponding task result\n", - " csv_writer.writerow(row)\n", - "\n", - " await http_client.close()\n", - "\n", - "asyncio.run(import_sheet())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "E7M0Li_UgJeZ" - }, - "source": [ - "# Create Users + Keys\n", - "\n", - "- Creates a user\n", - "- Creates a key with max budget" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NZudRFujf7j-" - }, - "outputs": [], - "source": [ - "\n", - "async def create_key_with_alias(client: HTTPHandler, user_id: str, max_budget: float):\n", - " global proxy_base_url\n", - " if not proxy_base_url.endswith(\"/\"):\n", - " proxy_base_url += \"/\"\n", - " url = proxy_base_url + \"key/generate\"\n", - "\n", - " # call /key/generate\n", - " print(\"CALLING /KEY/GENERATE\")\n", - " response = await client.post(\n", - " url=url,\n", - " headers={\"Authorization\": f\"Bearer {master_key}\"},\n", - " data=json.dumps({\n", - " \"user_id\": user_id,\n", - " \"key_alias\": f\"{user_id}-key\",\n", - " \"max_budget\": max_budget # 👈 KEY CHANGE: SETS MAX BUDGET PER KEY\n", - " })\n", - " )\n", - " print(f\"response: {response.text}\")\n", - " return response.json()[\"key\"]\n", - "\n", - "async def create_user(client: HTTPHandler, user_id: str, max_budget: float, user_name: str):\n", - " \"\"\"\n", - " - call /user/new\n", - " - create key for user\n", - " \"\"\"\n", - " global proxy_base_url\n", - " if not proxy_base_url.endswith(\"/\"):\n", - " proxy_base_url += \"/\"\n", - " url = proxy_base_url + \"user/new\"\n", - "\n", - " # call /user/new\n", - " await client.post(\n", - " url=url,\n", - " headers={\"Authorization\": f\"Bearer {master_key}\"},\n", - " data=json.dumps({\n", - " \"user_id\": user_id,\n", - " \"user_alias\": user_name,\n", - " \"auto_create_key\": False,\n", - " # \"max_budget\": max_budget # 👈 [OPTIONAL] Sets max budget per user (if you want to set a max budget across keys)\n", - " })\n", - " )\n", - "\n", - " # create key for user\n", - " return await create_key_with_alias(client=client, user_id=user_id, max_budget=max_budget)\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/TogetherAI_liteLLM.ipynb b/cookbook/TogetherAI_liteLLM.ipynb deleted file mode 100644 index d4700914910..00000000000 --- a/cookbook/TogetherAI_liteLLM.ipynb +++ /dev/null @@ -1,1006 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "WemkFEdDAnJL" - }, - "source": [ - "## liteLLM Together AI Tutorial\n", - "https://together.ai/\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "pc6IO4V99O25", - "outputId": "2d69da44-010b-41c2-b38b-5b478576bb8b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting litellm\n", - " Downloading litellm-0.1.482-py3-none-any.whl (69 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.3/69.3 kB\u001b[0m \u001b[31m757.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: importlib-metadata<7.0.0,>=6.8.0 in /usr/local/lib/python3.10/dist-packages (from litellm) (6.8.0)\n", - "Collecting openai<0.28.0,>=0.27.8 (from litellm)\n", - " Downloading openai-0.27.9-py3-none-any.whl (75 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.5/75.5 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting python-dotenv<2.0.0,>=1.0.0 (from litellm)\n", - " Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n", - "Collecting tiktoken<0.5.0,>=0.4.0 (from litellm)\n", - " Downloading tiktoken-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0.0,>=6.8.0->litellm) (3.16.2)\n", - "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (2.31.0)\n", - "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (4.66.1)\n", - "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (3.8.5)\n", - "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm) (2023.6.3)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (3.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (2.0.4)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (2023.7.22)\n", - "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (23.1.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (6.0.4)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (4.0.3)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.9.2)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.4.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.3.1)\n", - "Installing collected packages: python-dotenv, tiktoken, openai, litellm\n", - "Successfully installed litellm-0.1.482 openai-0.27.9 python-dotenv-1.0.0 tiktoken-0.4.0\n" - ] - } - ], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "TMI3739_9q97" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "os.environ[\"TOGETHERAI_API_KEY\"] = \"\" #@param\n", - "user_message = \"Hello, whats the weather in San Francisco??\"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bEqJ2HHjBJqq" - }, - "source": [ - "## Calling togethercomputer/llama-2-70b-chat\n", - "https://api.together.xyz/playground/chat?model=togethercomputer%2Fllama-2-70b-chat" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Jrrt8puj523f", - "outputId": "24494dea-816f-47a6-ade4-1b04f2e9085b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " 'choices': [\n", - "{\n", - " 'finish_reason': 'stop',\n", - " 'index': 0,\n", - " 'message': {\n", - " 'role': 'assistant',\n", - " 'content': \"\n", - "\n", - "I'm not able to provide real-time weather information. However, I can suggest some ways for you to find out the current weather in San Francisco.\n", - "\n", - "1. Check online weather websites: There are many websites that provide up-to-date weather information, such as AccuWeather, Weather.com, or the National Weather Service. You can enter \"San Francisco\" in the search bar and get the current weather conditions, forecast, and radar imagery.\n", - "2. Use a weather app: You can download a weather app on your smartphone that provides real-time weather information. Some popular weather apps include Dark Sky, Weather Underground, and The Weather Channel.\n", - "3. Tune into local news: You can watch local news channels or listen to local radio stations to get the latest weather forecast and current conditions.\n", - "4. Check social media: Follow local weather accounts on social media platforms like Twitter or Facebook to\"\n", - "}\n", - "}\n", - " ],\n", - " 'created': 1692323365.8261144,\n", - " 'model': 'togethercomputer/llama-2-70b-chat',\n", - " 'usage': {'prompt_tokens': 9, 'completion_tokens': 176, 'total_tokens': 185}\n", - "}\n" - ] - } - ], - "source": [ - "model_name = \"togethercomputer/llama-2-70b-chat\"\n", - "response = completion(model=model_name, messages=messages, max_tokens=200)\n", - "print(response)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "GIUevHlMvPb8", - "outputId": "ad930a12-16e3-4400-fff4-38151e4f6da5" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[92mHere's your LiteLLM Dashboard 👉 \u001b[94m\u001b[4mhttps://admin.litellm.ai/6c0f0403-becb-44af-9724-7201c7d381d0\u001b[0m\n", - "{\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"\\nI'm in San Francisco, and I'm not sure what the weather is like.\\nI'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1692934243.8663018,\n", - " \"model\": \"togethercomputer/CodeLlama-34b-Instruct\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 9,\n", - " \"completion_tokens\": 178,\n", - " \"total_tokens\": 187\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "model_name = \"togethercomputer/CodeLlama-34b-Instruct\"\n", - "response = completion(model=model_name, messages=messages, max_tokens=200)\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sfWtgf-mBQcM" - }, - "source": [ - "## With Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "background_save": true, - "base_uri": "https://localhost:8080/" - }, - "id": "wuBhlZtC6MH5", - "outputId": "8f4a408c-25eb-4434-cdd4-7b4ae4f6d3aa" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Com'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'bin'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ator'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ('}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ')'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' two'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' popular'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' gained'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' recognition'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' effect'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'iveness'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'urt'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'uring'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' scaling'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' early'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'stage'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ities'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' also'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' distinct'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' differences'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' set'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' them'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' apart'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' In'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' this'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ess'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ay'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' we'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' will'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' explore'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' key'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' features'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' discuss'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' which'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' might'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Com'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'bin'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ator'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' one'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' most'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' successful'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' world'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' port'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'folio'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' includes'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Air'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'b'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'nb'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Drop'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'box'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Red'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'dit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' F'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ounded'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' '}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '2'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '5'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' over'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' '}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '9'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' start'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ups'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' combined'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' valu'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ation'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' over'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' $'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' billion'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' The'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' known'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' inten'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'se'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' three'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'month'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' boot'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' camp'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'style'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' format'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' where'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' work'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' closely'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' experienced'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ment'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ors'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' develop'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' products'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ref'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ine'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' business'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' models'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' prepare'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ra'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ising'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' software'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' technology'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' internet'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' start'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ups'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' strong'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' track'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' record'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ident'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ifying'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'urt'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'uring'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' successful'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' these'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' spaces'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' other'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' hand'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' relatively'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' new'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ator'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' was'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' founded'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' '}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '2'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '7'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' While'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' it'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' not'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' same'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' level'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' brand'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' recognition'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' as'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' quickly'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' gained'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' reputation'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' unique'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' approach'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceleration'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' The'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'es'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' supporting'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' under'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 're'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'present'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' particularly'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' women'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' people'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' color'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' provides'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' range'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' support'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' help'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' these'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' succeed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' designed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' more'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' flexible'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' personal'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ized'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' than'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' traditional'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' connecting'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ment'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ors'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' tail'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ored'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' needs'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'One'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' key'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' difference'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' between'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' type'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' support'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'es'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' primarily'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' software'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' technology'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' internet'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' start'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ups'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' while'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' bro'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ader'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' includes'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' range'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' indust'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ries'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' such'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' as'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' health'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'care'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fin'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ance'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' consumer'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' products'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' This'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' means'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' non'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'tech'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' industry'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'An'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'other'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' difference'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' between'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' two'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' programs'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' approach'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' provides'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' seed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' all'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' port'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'folio'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' typically'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' range'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' $'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' $'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '2'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' In'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' contrast'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' does'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' not'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' provide'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' port'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'folio'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' but'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' instead'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'es'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' connecting'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' invest'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ors'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' help'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' them'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' raise'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' capital'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' This'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' means'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' looking'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' option'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'So'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' which'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' right'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '?'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' It'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ultimately'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' depends'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' needs'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' goals'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' If'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' non'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'tech'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' industry'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' bro'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ader'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Additionally'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 're'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' looking'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' more'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' personal'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ized'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' flexible'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' approach'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceleration'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' choice'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' On'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' other'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' hand'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' software'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' technology'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' or'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' internet'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' space'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 're'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' looking'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' seed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'In'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' conclusion'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' both'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' excellent'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' provide'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' valuable'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' support'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' early'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'stage'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' While'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' share'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' some'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' similar'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ities'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' also'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' distinct'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' differences'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' set'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' them'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' apart'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' By'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' considering'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' needs'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' goals'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' determine'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' which'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' best'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' business'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n" - ] - } - ], - "source": [ - "user_message = \"Write 1page essay on YC + liteLLM\"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n", - "\n", - "\n", - "async def parse_stream(stream):\n", - " async for elem in stream:\n", - " print(elem)\n", - " return\n", - "\n", - "stream = completion(model=\"togethercomputer/llama-2-70b-chat\", messages=messages, stream=True, max_tokens=800)\n", - "print(stream)\n", - "\n", - "# Await the asynchronous function directly in the notebook cell\n", - "await parse_stream(stream)\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb b/cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb deleted file mode 100644 index 0c3ff97a373..00000000000 --- a/cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb +++ /dev/null @@ -1,157 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "eKXncoQbU_2j" - }, - "source": [ - "# Using Nemo-Guardrails with LiteLLM Server\n", - "\n", - "[Call Bedrock, TogetherAI, Huggingface, etc. on the server](https://docs.litellm.ai/docs/providers)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZciYaLwvuFbu" - }, - "source": [ - "## Using with Bedrock\n", - "\n", - "`docker run -e PORT=8000 -e AWS_ACCESS_KEY_ID= -e AWS_SECRET_ACCESS_KEY= -p 8000:8000 ghcr.io/berriai/litellm:latest`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "vOUwGSJ2Vsy3" - }, - "outputs": [], - "source": [ - "pip install nemoguardrails langchain" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "xXEJNxe7U0IN" - }, - "outputs": [], - "source": [ - "from langchain.chat_models import ChatOpenAI\n", - "\n", - "llm = ChatOpenAI(model_name=\"anthropic.claude-v2\", openai_api_base=\"http://0.0.0.0:8000\", openai_api_key=\"my-fake-key\")\n", - "\n", - "from nemoguardrails import LLMRails, RailsConfig\n", - "\n", - "config = RailsConfig.from_path(\"./config.yml\")\n", - "app = LLMRails(config, llm=llm)\n", - "\n", - "new_message = app.generate(messages=[{\n", - " \"role\": \"user\",\n", - " \"content\": \"Hello! What can you do for me?\"\n", - "}])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vz5n00qyuKjp" - }, - "source": [ - "## Using with TogetherAI\n", - "\n", - "1. You can either set this in the server environment:\n", - "`docker run -e PORT=8000 -e TOGETHERAI_API_KEY= -p 8000:8000 ghcr.io/berriai/litellm:latest`\n", - "\n", - "2. **Or** Pass this in as the api key `(...openai_api_key=\"\")`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "XK1sk-McuhpE" - }, - "outputs": [], - "source": [ - "from langchain.chat_models import ChatOpenAI\n", - "\n", - "llm = ChatOpenAI(model_name=\"together_ai/togethercomputer/CodeLlama-13b-Instruct\", openai_api_base=\"http://0.0.0.0:8000\", openai_api_key=\"my-together-ai-api-key\")\n", - "\n", - "from nemoguardrails import LLMRails, RailsConfig\n", - "\n", - "config = RailsConfig.from_path(\"./config.yml\")\n", - "app = LLMRails(config, llm=llm)\n", - "\n", - "new_message = app.generate(messages=[{\n", - " \"role\": \"user\",\n", - " \"content\": \"Hello! What can you do for me?\"\n", - "}])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8A1KWKnzuxAS" - }, - "source": [ - "### CONFIG.YML\n", - "\n", - "save this example `config.yml` in your current directory" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NKN1GmSvu0Cx" - }, - "outputs": [], - "source": [ - "# instructions:\n", - "# - type: general\n", - "# content: |\n", - "# Below is a conversation between a bot and a user about the recent job reports.\n", - "# The bot is factual and concise. If the bot does not know the answer to a\n", - "# question, it truthfully says it does not know.\n", - "\n", - "# sample_conversation: |\n", - "# user \"Hello there!\"\n", - "# express greeting\n", - "# bot express greeting\n", - "# \"Hello! How can I assist you today?\"\n", - "# user \"What can you do for me?\"\n", - "# ask about capabilities\n", - "# bot respond about capabilities\n", - "# \"I am an AI assistant that helps answer mathematical questions. My core mathematical skills are powered by wolfram alpha.\"\n", - "# user \"What's 2+2?\"\n", - "# ask math question\n", - "# bot responds to math question\n", - "# \"2+2 is equal to 4.\"\n", - "\n", - "# models:\n", - "# - type: main\n", - "# engine: openai\n", - "# model: claude-instant-1" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/VLLM_Model_Testing.ipynb b/cookbook/VLLM_Model_Testing.ipynb deleted file mode 100644 index 0cacac66133..00000000000 --- a/cookbook/VLLM_Model_Testing.ipynb +++ /dev/null @@ -1,404 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "machine_shape": "hm", - "gpuType": "V100" - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# Set up Environment" - ], - "metadata": { - "id": "vDOm5wfjdFLP" - } - }, - { - "cell_type": "code", - "source": [ - "!pip install --upgrade litellm" - ], - "metadata": { - "id": "Bx6mAA6MHiy_" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "zIYv7JTyxSxR", - "outputId": "53890320-f9fa-4bf4-8362-0f17f52c6ed4" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Successfully installed fastapi-0.103.1 h11-0.14.0 huggingface-hub-0.16.4 ninja-1.11.1 pydantic-1.10.12 ray-2.6.3 safetensors-0.3.3 sentencepiece-0.1.99 starlette-0.27.0 tokenizers-0.13.3 transformers-4.33.1 uvicorn-0.23.2 vllm-0.1.4 xformers-0.0.21\n" - ] - } - ], - "source": [ - "!pip install vllm" - ] - }, - { - "cell_type": "markdown", - "source": [ - "# Load the Logs" - ], - "metadata": { - "id": "RMcoAni6WKEx" - } - }, - { - "cell_type": "code", - "source": [ - "import pandas as pd" - ], - "metadata": { - "id": "zchxB8c7WJe5" - }, - "execution_count": 4, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "# path of the csv file\n", - "file_path = 'Model-prompts-example.csv'\n", - "\n", - "# load the csv file as a pandas DataFrame\n", - "data = pd.read_csv(file_path)\n", - "\n", - "data.head()" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 81 - }, - "id": "aKcWr015WNPm", - "outputId": "6e226773-333f-46a2-9fc8-4f54f309d204" - }, - "execution_count": 6, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " Success Timestamp Input \\\n", - "0 True 1694041195 This is the templated query input \n", - "\n", - " Output RunId (Wandb Runid) \\\n", - "0 This is the query output from the model 8hlumwuk \n", - "\n", - " Model ID (or Name) \n", - "0 OpenAI/Turbo-3.5 " - ], - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
SuccessTimestampInputOutputRunId (Wandb Runid)Model ID (or Name)
0True1694041195This is the templated query inputThis is the query output from the model8hlumwukOpenAI/Turbo-3.5
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - "
\n" - ] - }, - "metadata": {}, - "execution_count": 6 - } - ] - }, - { - "cell_type": "code", - "source": [ - "input_texts = data['Input'].values" - ], - "metadata": { - "id": "0DbL-kirWUyn" - }, - "execution_count": 7, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "messages = [[{\"role\": \"user\", \"content\": input_text}] for input_text in input_texts]" - ], - "metadata": { - "id": "cqpAvy8hWXyC" - }, - "execution_count": 8, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Running Inference" - ], - "metadata": { - "id": "SugCyom0Xy8U" - } - }, - { - "cell_type": "code", - "source": [ - "from litellm import batch_completion\n", - "model_name = \"facebook/opt-125m\"\n", - "provider = \"vllm\"\n", - "response_list = batch_completion(\n", - " model=model_name,\n", - " custom_llm_provider=provider, # can easily switch to huggingface, replicate, together ai, sagemaker, etc.\n", - " messages=messages,\n", - " temperature=0.2,\n", - " max_tokens=80,\n", - " )" - ], - "metadata": { - "id": "qpikx3uxHns3" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "response_list" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "QDPikHtwKJJ2", - "outputId": "06f47c44-e258-452a-f9db-232a5b6d2810" - }, - "execution_count": 10, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "[ JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \".\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1694053363.6139505,\n", - " \"model\": \"facebook/opt-125m\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 9,\n", - " \"completion_tokens\": 80,\n", - " \"total_tokens\": 89\n", - " }\n", - " }]" - ] - }, - "metadata": {}, - "execution_count": 10 - } - ] - }, - { - "cell_type": "code", - "source": [ - "response_values = [response['choices'][0]['message']['content'] for response in response_list]" - ], - "metadata": { - "id": "SYqTcCiJbQDF" - }, - "execution_count": 11, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "response_values" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "wqs-Oy9FbiPo", - "outputId": "16a6a7b7-97c8-4b5b-eff8-09ea5eb5ad06" - }, - "execution_count": 12, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "['.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is']" - ] - }, - "metadata": {}, - "execution_count": 12 - } - ] - }, - { - "cell_type": "code", - "source": [ - "data[f\"{model_name}_output\"] = response_values" - ], - "metadata": { - "id": "mElNbBehbkrz" - }, - "execution_count": 13, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "data.to_csv('model_responses.csv', index=False)" - ], - "metadata": { - "id": "F06NXssDc45k" - }, - "execution_count": 14, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md deleted file mode 100644 index 538a5f8a404..00000000000 --- a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md +++ /dev/null @@ -1,114 +0,0 @@ -# Use your ChatGPT subscription through LiteLLM - -Sign into ChatGPT in Codex or Pi and send model requests through your existing LiteLLM proxy. The client keeps your login and refreshes it. Each request uses that client's ChatGPT account, while a separate LiteLLM virtual key controls gateway access and usage attribution. - -An administrator adds a `chatgpt/` model using the usual model configuration. No ChatGPT login needs to be added to the proxy. This uses the same separate-key convention as provider passthrough, including Claude Code subscription forwarding. - -## Add a model to the proxy - -Reuse your existing proxy. If you need one, follow the [proxy quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start). Merge this entry into its model list and restart the proxy: - -```yaml -model_list: - - model_name: gpt-6-luna - litellm_params: - model: chatgpt/gpt-6-luna - model_info: - mode: responses -``` - -Use a model available to the caller's subscription. The virtual key must allow the exposed model name. Leave saved ChatGPT logins and API credentials unset for a client-only deployment. Explicit cross-provider fallbacks can incur separate API charges; do not configure them for subscription-only traffic. - -## Configure Codex - -Use your existing LiteLLM URL and a virtual key assigned to your gateway user. Export that key as `LITELLM_API_KEY` in the environment that launches Codex. Keep the value out of committed configuration. - -Add this provider to `~/.codex/config.toml`, preserving your other settings: - -```toml -model = "gpt-6-luna" -model_provider = "litellm_subscription" - -[model_providers.litellm_subscription] -name = "LiteLLM subscription" -base_url = "https://litellm.example.com/v1" -wire_api = "responses" -supports_websockets = false -requires_openai_auth = true -env_http_headers = { "x-litellm-api-key" = "LITELLM_API_KEY" } -``` - -Replace `https://litellm.example.com` with your proxy URL. A local proxy normally uses `http://localhost:4000/v1`. Launch Codex, sign into ChatGPT if needed, and send a prompt with a model available to your subscription. The selected model must match the name exposed by the gateway. - -The `lite` CLI can set up this provider each time it starts Codex, with no `config.toml` edit. Export `LITELLM_PROXY_URL` and `LITELLM_PROXY_API_KEY`, then run `lite codex --subscription`. Run `lite config set codex_subscription true` to make it the default. - -For a managed workspace rollout, use OpenAI's [gateway configuration guide](https://learn.chatgpt.com/docs/enterprise/sign-in-with-chatgpt-through-a-gateway). - -## Native compatibility and Pi - -For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials. A ChatGPT model keeps its upstream name and needs no deployment of its own. - -### Mix ChatGPT and LiteLLM models - -On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. A name is routed this way when another provider serves it as an exact `model_name`, through a router `model_group_alias`, or as a team's public model name. A ChatGPT model, a name that only a wildcard deployment matches, or a name OpenAI's catalog lists for the caller's ChatGPT account keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. The proxy learns the account's list from the catalog Codex fetches, or asks OpenAI with the caller's login. When it cannot get the list, for example because OpenAI is down, the request times out, or the client is not Codex, every request keeps the passthrough for the next minute, as it did before LiteLLM models were added to the picker. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit, and the proxy logs a warning with how many it left out. - -Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it for a model that does not support `none`, and the provider applies its own default. A model whose cost map entry or `model_info` sets `supports_none_reasoning_effort: true` keeps it. Without `drop_params`, the effort reaches the provider unchanged. - -Codex also sends its ChatGPT app tools with every turn, which on a LiteLLM model adds a few hundred thousand prompt tokens to each turn. Set `apps = false` under `[features]` in Codex's `config.toml` to turn them off. - -Only a key that may call `/v1/responses` gets these models. A key whose `allowed_routes` covers only the native routes sees OpenAI's catalog unchanged, and every request it sends uses the subscription. - -Codex sends earlier reasoning items back with each turn. The proxy tags the reasoning that a LiteLLM model returns on this route, OpenAI and Azure deployments included. When a session switches to a ChatGPT model, the proxy drops those items before forwarding the request to OpenAI, which would reject them. Messages, tool calls, and ChatGPT's own reasoning are kept, and a LiteLLM model still gets its own reasoning back. - -When the catalog adds LiteLLM models for a key, its `ETag` is a digest of OpenAI's ETag and those models, and that key's responses carry the same value in `X-Models-Etag`. Codex keeps its cached catalog while the two match and fetches it again when OpenAI's catalog or the key's models change. A key whose catalog adds no LiteLLM models keeps OpenAI's ETags. - -### Restrict native models with an access group - -A key with a model allowlist needs a rule for ChatGPT names that OpenAI adds over time. Put the subscription wildcards in an access group under `chatgpt/` names: - -```yaml -model_list: - - model_name: chatgpt/gpt-* - litellm_params: - model: chatgpt/gpt-* - model_info: - access_groups: ["chatgpt-native"] - - model_name: chatgpt/codex-* - litellm_params: - model: chatgpt/codex-* - model_info: - access_groups: ["chatgpt-native"] -``` - -Then add `chatgpt-native` to the key's models, next to any other model names it should reach. On the native route, a key that holds only the `chatgpt/` form of a name is checked as that form, so a request for `gpt-5.6-sol` is allowed when the key may call `chatgpt/gpt-5.6-sol`. The key can call every ChatGPT model that matches the wildcards, including ones the proxy configuration does not name. - -On `/v1` these wildcards match only a request that asks for a `chatgpt/` name, so a request for a plain `gpt-` or `codex-` name still goes where it went before, such as a `*` catch-all deployment. A `/v1` request for a `chatgpt/` name without a ChatGPT login falls back to the proxy's own saved login if it has one, so keep that login unset as described above. - -### Configure Pi - -Sign into the built-in `openai-codex` provider in Pi. Merge this override into `~/.pi/agent/models.json` and export your LiteLLM virtual key as `LITELLM_API_KEY` before launching Pi: - -```json -{ - "providers": { - "openai-codex": { - "baseUrl": "https://litellm.example.com/backend-api/codex", - "headers": { - "x-litellm-api-key": "LITELLM_API_KEY" - } - } - } -} -``` - -Keep Pi's subscription login as the provider credential. The header value names an environment variable, not a key to paste into the file. - -## Verify account usage - -Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person. - -The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key needs the `chatgpt-native` access group above or that exact name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions. - -The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate. - -This flow does not connect accounts through the dashboard or pool subscription allowances. Use a Codex build that supports ChatGPT authentication with a custom model provider. The integration forwards HTTP and SSE requests; it does not add native WebSocket or other Codex backend routes. diff --git a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md b/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md deleted file mode 100644 index cd27eed18a3..00000000000 --- a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md +++ /dev/null @@ -1,294 +0,0 @@ -# Claude Code with LiteLLM Quickstart - -This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code. - -> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls. - -## Video Walkthrough - -Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56 - -## Prerequisites - -- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed -- API keys for your chosen providers - -## Installation - -First, install LiteLLM with proxy support: - -```bash -pip install 'litellm[proxy]' -``` - -## Step 1: Setup config.yaml - -Create a secure configuration using environment variables: - -```yaml -model_list: - # Claude models - - model_name: claude-3-5-sonnet-20241022 - litellm_params: - model: anthropic/claude-3-5-sonnet-20241022 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: claude-3-5-haiku-20241022 - litellm_params: - model: anthropic/claude-3-5-haiku-20241022 - api_key: os.environ/ANTHROPIC_API_KEY - - -litellm_settings: - master_key: os.environ/LITELLM_MASTER_KEY -``` - -Set your environment variables: - -```bash -export ANTHROPIC_API_KEY="your-anthropic-api-key" -export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)" -``` - -## Step 2: Start Proxy - -```bash -litellm --config /path/to/config.yaml - -# RUNNING on http://0.0.0.0:4000 -``` - -## Step 3: Verify Setup - -Test that your proxy is working correctly: - -```bash -curl -X POST http://0.0.0.0:4000/v1/messages \ --H "Authorization: Bearer $LITELLM_MASTER_KEY" \ --H "Content-Type: application/json" \ --d '{ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 1000, - "messages": [{"role": "user", "content": "What is the capital of France?"}] -}' -``` - -## Step 4: Configure Claude Code - -### Method 1: Unified Endpoint (Recommended) - -Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here: - -```bash -export ANTHROPIC_BASE_URL="http://0.0.0.0:4000" -export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY" -``` - -> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI. - -### Method 2: Provider-specific Pass-through Endpoint - -Alternatively, use the Anthropic pass-through endpoint: - -```bash -export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic" -export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY" -``` - -## Step 5: Use Claude Code - -### Choosing Your Model - -You have two options for specifying which model Claude Code uses: - -#### Option 1: Command Line / Session Model Selection - -Specify the model directly when starting Claude Code or during a session: - -```bash -# Specify model at startup -claude --model claude-3-5-sonnet-20241022 - -# Or change model during a session -/model claude-3-5-haiku-20241022 -``` - -This method uses the exact model you specify. - -#### Option 2: Environment Variables - -Configure default models using environment variables: - -```bash -# Tell Claude Code which models to use by default -export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022 -export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022 -export ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-3-5-20240229 - -claude # Will use the models specified above -``` - -**Note:** Claude Code may cache the model from a previous session. If environment variables don't take effect, use Option 1 to explicitly set the model. - -**Important:** The `model_name` in your LiteLLM config must match what Claude Code requests (either from env vars or command line). - -### Using 1M Context Window - -Claude Code supports extended context (1 million tokens) using the `[1m]` suffix with Claude 4+ models: - -```bash -# Use Sonnet 4.5 with 1M context (requires quotes for shell) -claude --model 'claude-sonnet-4-5-20250929[1m]' - -# Inside a Claude Code session (no quotes needed) -/model claude-sonnet-4-5-20250929[1m] -``` - -**Important:** When using `--model` with `[1m]` in the shell, you must use quotes to prevent the shell from interpreting the brackets. - -Alternatively, set as default with environment variables: - -```bash -export ANTHROPIC_DEFAULT_SONNET_MODEL='claude-sonnet-4-5-20250929[1m]' -claude -``` - -**How it works:** -- Claude Code strips the `[1m]` suffix before sending to LiteLLM -- Claude Code automatically adds the header `anthropic-beta: context-1m-2025-08-07` -- Your LiteLLM config should **NOT** include `[1m]` in model names - -**Verify 1M context is active:** -```bash -/context -# Should show: 21k/1000k tokens (2%) -``` - -**Pricing:** Models using 1M context have different pricing. Input tokens above 200k are charged at a higher rate. - -## Troubleshooting - -Common issues and solutions: - -**Claude Code not connecting:** -- Verify your proxy is running: `curl http://0.0.0.0:4000/health` -- Check that `ANTHROPIC_BASE_URL` is set correctly -- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key - -**Authentication errors:** -- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY` -- Check that your API keys are valid and have sufficient credits -- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key - -**Model not found:** -- Check what model Claude Code is requesting in LiteLLM logs -- Ensure your `config.yaml` has a matching `model_name` entry -- If using environment variables, verify they're set: `echo $ANTHROPIC_DEFAULT_SONNET_MODEL` - -**1M context not working (showing 200k instead of 1000k):** -- Verify you're using the `[1m]` suffix: `/model your-model-name[1m]` -- Check LiteLLM logs for the header `context-1m-2025-08-07` in the request -- Ensure your model supports 1M context (only certain Claude models do) -- Your LiteLLM config should **NOT** include `[1m]` in the `model_name` - -## Using Multiple Models and Providers - -You can configure LiteLLM to route to any supported provider. Here's an example with multiple providers: - -```yaml -model_list: - # OpenAI models - - model_name: codex-mini - litellm_params: - model: openai/codex-mini - api_key: os.environ/OPENAI_API_KEY - api_base: https://api.openai.com/v1 - - - model_name: o3-pro - litellm_params: - model: openai/o3-pro - api_key: os.environ/OPENAI_API_KEY - api_base: https://api.openai.com/v1 - - - model_name: gpt-4o - litellm_params: - model: openai/gpt-4o - api_key: os.environ/OPENAI_API_KEY - api_base: https://api.openai.com/v1 - - # Anthropic models - - model_name: claude-3-5-sonnet-20241022 - litellm_params: - model: anthropic/claude-3-5-sonnet-20241022 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: claude-3-5-haiku-20241022 - litellm_params: - model: anthropic/claude-3-5-haiku-20241022 - api_key: os.environ/ANTHROPIC_API_KEY - - # AWS Bedrock - - model_name: claude-bedrock - litellm_params: - model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID - aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_region_name: us-east-1 - -litellm_settings: - master_key: os.environ/LITELLM_MASTER_KEY -``` - -**Note:** The `model_name` can be anything you choose. Claude Code will request whatever model you specify (via env vars or command line), and LiteLLM will route to the `model` configured in `litellm_params`. - -Switch between models seamlessly: - -```bash -# Use environment variables to set defaults -export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022 -export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022 - -# Or specify directly -claude --model claude-3-5-sonnet-20241022 # Complex reasoning -claude --model claude-3-5-haiku-20241022 # Fast responses -claude --model claude-bedrock # Bedrock deployment -``` - -## Default Models Used by Claude Code - -If you **don't** set environment variables, Claude Code uses these default model names: - -| Purpose | Default Model Name (v2.1.14) | -|---------|------------------------------| -| Main model | `claude-sonnet-4-5-20250929` | -| Light tasks (subagents, summaries) | `claude-haiku-4-5-20251001` | -| Planning mode | `claude-opus-4-5-20251101` | - -Your LiteLLM config should include these model names if you want Claude Code to work without setting environment variables: - -```yaml -model_list: - - model_name: claude-sonnet-4-5-20250929 - litellm_params: - # Can be any provider - Anthropic, Bedrock, Vertex AI, etc. - model: anthropic/claude-sonnet-4-5-20250929 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: claude-haiku-4-5-20251001 - litellm_params: - model: anthropic/claude-haiku-4-5-20251001 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: claude-opus-4-5-20251101 - litellm_params: - model: anthropic/claude-opus-4-5-20251101 - api_key: os.environ/ANTHROPIC_API_KEY -``` - -**Warning:** These default model names may change with new Claude Code versions. Check LiteLLM proxy logs for "model not found" errors to identify what Claude Code is requesting. - -## Additional Resources - -- [LiteLLM Documentation](https://docs.litellm.ai/) -- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview) -- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration) diff --git a/cookbook/ai_coding_tool_guides/index.json b/cookbook/ai_coding_tool_guides/index.json deleted file mode 100644 index 3e71670d623..00000000000 --- a/cookbook/ai_coding_tool_guides/index.json +++ /dev/null @@ -1,134 +0,0 @@ -[{ - "title": "Claude Code Quickstart", - "description": "This is a quickstart guide to using Claude Code with LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api", - "date": "2026-01-15", - "version": "1.0.0", - "tags": [ - "Claude Code", - "LiteLLM" - ] -}, -{ - "title": "Claude Code with MCPs", - "description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.", - "url": "https://docs.litellm.ai/docs/tutorials/claude_mcp", - "date": "2026-01-15", - "version": "1.0.0", - "tags": [ - "Claude Code", - "LiteLLM", - "MCP" - ] -}, -{ - "title": "Claude Code with Non-Anthropic Models", - "description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.", - "url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "Claude Code", - "LiteLLM", - "OpenAI", - "Gemini" - ] -}, -{ - "title": "Cursor Quickstart", - "description": "This is a quickstart guide to using Cursor with LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/cursor_integration", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "Cursor", - "LiteLLM", - "Quickstart" - ] -}, -{ - "title": "Github Copilot Quickstart", - "description": "This is a quickstart guide to using Github Copilot with LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "Github Copilot", - "LiteLLM", - "Quickstart" - ] -}, -{ - "title": "LiteLLM Gemini CLI Quickstart", - "description": "This is a quickstart guide to using LiteLLM Gemini CLI.", - "url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "Gemini CLI", - "Gemini", - "LiteLLM", - "Quickstart" - ] -}, -{ - "title": "OpenAI Codex CLI Quickstart", - "description": "This is a quickstart guide to using OpenAI Codex CLI.", - "url": "https://docs.litellm.ai/docs/tutorials/openai_codex", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "OpenAI Codex CLI", - "OpenAI", - "LiteLLM", - "Quickstart" - ] -}, -{ - "title": "OpenWebUI Quickstart", - "description": "This is a quickstart guide to using OpenWebUI with LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/openweb_ui", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "OpenWebUI", - "LiteLLM", - "Quickstart" - ] -}, -{ - "title": "AI Coding Tool Usage Tracking", - "description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding", - "date": "2026-01-17", - "version": "1.0.0", - "tags": [ - "Claude Code", - "Gemini CLI", - "OpenAI Codex", - "LiteLLM" - ] -}, -{ - "title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)", - "description": "This is a guide for using Web Search with Claude Code via LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch", - "date": "2026-01-17", - "version": "1.0.0", - "tags": [ - "Claude Code", - "LiteLLM", - "Web Search" - ] -}, -{ - "title": "Track Claude Code Usage per user via Custom Headers", - "description": "This is a guide for tracking claude code user usage by passing a customer ID header.", - "url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking", - "date": "2026-01-17", - "version": "1.0.0", - "tags": [ - "Claude Code", - "LiteLLM" - ] -}] \ No newline at end of file diff --git a/cookbook/anthropic_agent_sdk/README.md b/cookbook/anthropic_agent_sdk/README.md deleted file mode 100644 index 4521aee310a..00000000000 --- a/cookbook/anthropic_agent_sdk/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# Claude Agent SDK with LiteLLM Gateway - -A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK. - -## Quick Start - -### 1. Install dependencies - -```bash -pip install anthropic claude-agent-sdk litellm -``` - -### 2. Start LiteLLM proxy - -```bash -# Simple start with Claude -litellm --model claude-sonnet-4-20250514 - -# Or with a config file -litellm --config config.yaml -``` - -Generate a master key before starting the proxy: - -```bash -export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)" -``` - -### 3. Run the chat - -**Basic Agent (no MCP):** - -```bash -python main.py -``` - -**Agent with MCP (DeepWiki2 for research):** - -```bash -python agent_with_mcp.py -``` - -If MCP connection fails, you can disable it: - -```bash -USE_MCP=false python agent_with_mcp.py -``` - -That's it! You can now chat with the agent in your terminal. - -### Chat Commands - -While chatting, you can use these commands: -- `models` - List all available models (fetched from your LiteLLM proxy) -- `model` - Switch to a different model -- `clear` - Start a new conversation -- `quit` or `exit` - End the chat - -The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured. - -## Configuration - -Set these environment variables if needed: - -```bash -export LITELLM_PROXY_URL="http://localhost:4000" -export LITELLM_API_KEY="$LITELLM_MASTER_KEY" -export LITELLM_MODEL="bedrock-claude-sonnet-4.5" -``` - -Or just use the defaults - it'll connect to `http://localhost:4000` by default. - -## Files - -- `main.py` - Basic interactive agent without MCP -- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2) -- `common.py` - Shared utilities and functions -- `config.example.yaml` - Example LiteLLM configuration -- `requirements.txt` - Python dependencies - -## Example Config File - -If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`): - -```yaml -model_list: - - model_name: bedrock-claude-sonnet-4 - litellm_params: - model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" - aws_region_name: "us-east-1" - - - model_name: bedrock-claude-sonnet-4.5 - litellm_params: - model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" - aws_region_name: "us-east-1" -``` - -Then start LiteLLM with: `litellm --config config.yaml` - -## How It Works - -The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic: - -```python -# Point to LiteLLM gateway (not Anthropic) -os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000" -os.environ["ANTHROPIC_API_KEY"] = os.environ["LITELLM_API_KEY"] - -# Use any model configured in LiteLLM -options = ClaudeAgentOptions( - model="bedrock-claude-sonnet-4", # or gpt-4, or anything else - system_prompt="You are a helpful assistant.", - max_turns=50, -) -``` - -Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically. - -## Why Use This? - -- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc. -- **Cost tracking**: LiteLLM tracks spending across all your agent conversations -- **Rate limiting**: Set budgets and limits on your agent usage -- **Load balancing**: Distribute requests across multiple API keys or regions -- **Fallbacks**: Automatically retry with a different model if one fails - -## Troubleshooting - -**Connection errors?** -- Make sure LiteLLM is running: `litellm --model your-model` -- Check the URL is correct (default: `http://localhost:4000`) - -**Authentication errors?** -- Verify your LiteLLM API key is correct -- Make sure the model is configured in your LiteLLM setup - -**Model not found?** -- Check the model name matches what's in your LiteLLM config -- Run `litellm --model your-model` to test it works - -**Agent with MCP stuck or failing?** -- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2` -- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py` -- Or use the basic agent: `python main.py` - -## Learn More - -- [LiteLLM Docs](https://docs.litellm.ai/) -- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk) -- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start) diff --git a/cookbook/anthropic_agent_sdk/agent_with_mcp.py b/cookbook/anthropic_agent_sdk/agent_with_mcp.py deleted file mode 100644 index 8a7513c786c..00000000000 --- a/cookbook/anthropic_agent_sdk/agent_with_mcp.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Interactive Claude Agent SDK CLI with MCP Support - -This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy, -with MCP (Model Context Protocol) server integration for enhanced capabilities. -""" - -import asyncio -import os -from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions -from common import ( - Config, - fetch_available_models, - setup_litellm_env, - print_header, - handle_model_list, - handle_model_switch, - stream_response, -) - - -async def interactive_chat_with_mcp(): - """ - Interactive CLI chat with the agent and MCP server - """ - config = Config() - - # Configure Anthropic SDK to point to LiteLLM gateway - litellm_base_url = setup_litellm_env(config) - - # Fetch available models from proxy - available_models = await fetch_available_models( - litellm_base_url, config.LITELLM_API_KEY - ) - - current_model = config.LITELLM_MODEL - - # MCP server configuration - mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2" - use_mcp = os.getenv("USE_MCP", "true").lower() == "true" - - if not use_mcp: - print("⚠️ MCP disabled via USE_MCP=false") - - print_header(litellm_base_url, current_model, has_mcp=use_mcp) - - while True: - # Configure agent options - if use_mcp: - try: - # Try with MCP server (HTTP transport) - # Using McpHttpServerConfig format from Agent SDK - options = ClaudeAgentOptions( - system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.", - model=current_model, - max_turns=50, - mcp_servers={ - "deepwiki2": { - "type": "http", - "url": mcp_server_url, - "headers": { - "Authorization": f"Bearer {config.LITELLM_API_KEY}" - }, - } - }, - ) - except Exception as e: - print(f"⚠️ Warning: Could not configure MCP server: {e}") - print("Continuing without MCP...\n") - use_mcp = False - options = ClaudeAgentOptions( - system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.", - model=current_model, - max_turns=50, - ) - else: - # Without MCP - options = ClaudeAgentOptions( - system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.", - model=current_model, - max_turns=50, - ) - - # Create agent client - try: - async with ClaudeSDKClient(options=options) as client: - conversation_active = True - - while conversation_active: - # Get user input - try: - user_input = input("\n👤 You: ").strip() - except (EOFError, KeyboardInterrupt): - print("\n\n👋 Goodbye!") - return - - # Handle commands - if user_input.lower() in ["quit", "exit"]: - print("\n👋 Goodbye!") - return - - if user_input.lower() == "clear": - print("\n🔄 Starting new conversation...\n") - conversation_active = False - continue - - if user_input.lower() == "models": - handle_model_list(available_models, current_model) - continue - - if user_input.lower() == "model": - new_model, should_restart = handle_model_switch( - available_models, current_model - ) - if should_restart: - current_model = new_model - conversation_active = False - continue - - if not user_input: - continue - - # Stream response from agent - await stream_response(client, user_input) - - except Exception as e: - print(f"\n❌ Error creating agent client: {e}") - print("This might be an MCP configuration issue. Try running without MCP:") - print(" USE_MCP=false python agent_with_mcp.py") - print("\nOr use the basic agent:") - print(" python main.py") - return - - -def main(): - """Run interactive chat with MCP""" - try: - asyncio.run(interactive_chat_with_mcp()) - except KeyboardInterrupt: - print("\n\n👋 Goodbye!") - - -if __name__ == "__main__": - main() diff --git a/cookbook/anthropic_agent_sdk/common.py b/cookbook/anthropic_agent_sdk/common.py deleted file mode 100644 index 505b5ecd91c..00000000000 --- a/cookbook/anthropic_agent_sdk/common.py +++ /dev/null @@ -1,164 +0,0 @@ -""" -Common utilities for Claude Agent SDK examples -""" - -import os -import httpx - - -class Config: - """Configuration for LiteLLM Gateway connection""" - - # LiteLLM proxy URL (default to local instance) - LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000") - - # LiteLLM API key (master key or virtual key) - LITELLM_API_KEY = os.environ["LITELLM_API_KEY"] - - # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.) - LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5") - - -async def fetch_available_models(base_url: str, api_key: str) -> list[str]: - """ - Fetch available models from LiteLLM proxy /models endpoint - """ - try: - async with httpx.AsyncClient() as client: - response = await client.get( - f"{base_url}/models", - headers={"Authorization": f"Bearer {api_key}"}, - timeout=10.0, - ) - response.raise_for_status() - data = response.json() - return [model["id"] for model in data.get("data", [])] - except Exception as e: - print(f"⚠️ Warning: Could not fetch models from proxy: {e}") - print("Using default model list...") - # Fallback to default models - return [ - "bedrock-claude-sonnet-3.5", - "bedrock-claude-sonnet-4", - "bedrock-claude-sonnet-4.5", - "bedrock-claude-opus-4.5", - "bedrock-nova-premier", - ] - - -def setup_litellm_env(config: Config): - """ - Configure environment variables to point Agent SDK to LiteLLM - """ - litellm_base_url = config.LITELLM_PROXY_URL.rstrip("/") - os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url - os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY - return litellm_base_url - - -def print_header(base_url: str, current_model: str, has_mcp: bool = False): - """ - Print the chat header - """ - mcp_indicator = " + MCP" if has_mcp else "" - print("=" * 70) - print(f"🤖 Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat") - print("=" * 70) - print(f"🚀 Connected to: {base_url}") - print(f"📦 Current model: {current_model}") - if has_mcp: - print("🔌 MCP: deepwiki2 enabled") - print("\nType your messages below. Commands:") - print(" - 'quit' or 'exit' to end the conversation") - print(" - 'clear' to start a new conversation") - print(" - 'model' to switch models") - print(" - 'models' to list available models") - print("=" * 70) - print() - - -def handle_model_list(available_models: list[str], current_model: str): - """ - Display available models - """ - print("\n📋 Available models:") - for i, model in enumerate(available_models, 1): - marker = "✓" if model == current_model else " " - print(f" {marker} {i}. {model}") - - -def handle_model_switch( - available_models: list[str], current_model: str -) -> tuple[str, bool]: - """ - Handle model switching - - Returns: - tuple: (new_model, should_restart_conversation) - """ - print("\n📋 Select a model:") - for i, model in enumerate(available_models, 1): - marker = "✓" if model == current_model else " " - print(f" {marker} {i}. {model}") - - try: - choice = input("\nEnter number (or press Enter to cancel): ").strip() - if choice: - idx = int(choice) - 1 - if 0 <= idx < len(available_models): - new_model = available_models[idx] - print(f"\n✅ Switched to: {new_model}") - print("🔄 Starting new conversation with new model...\n") - return new_model, True - else: - print("❌ Invalid choice") - except (ValueError, IndexError): - print("❌ Invalid input") - - return current_model, False - - -async def stream_response(client, user_input: str): - """ - Stream response from the agent - """ - print("\n🤖 Assistant: ", end="", flush=True) - - try: - await client.query(user_input) - - # Show loading indicator - print("⏳ thinking...", end="", flush=True) - - # Stream the response - first_chunk = True - async for msg in client.receive_response(): - # Clear loading indicator on first message - if first_chunk: - print("\r🤖 Assistant: ", end="", flush=True) - first_chunk = False - - # Handle different message types - if hasattr(msg, "type"): - if msg.type == "content_block_delta": - # Streaming text delta - if hasattr(msg, "delta") and hasattr(msg.delta, "text"): - print(msg.delta.text, end="", flush=True) - elif msg.type == "content_block_start": - # Start of content block - if hasattr(msg, "content_block") and hasattr( - msg.content_block, "text" - ): - print(msg.content_block.text, end="", flush=True) - - # Fallback to original content handling - if hasattr(msg, "content"): - for content_block in msg.content: - if hasattr(content_block, "text"): - print(content_block.text, end="", flush=True) - - print() # New line after response - - except Exception as e: - print(f"\r\n❌ Error: {e}") - print("Please check your LiteLLM gateway is running and configured correctly.") diff --git a/cookbook/anthropic_agent_sdk/config.example.yaml b/cookbook/anthropic_agent_sdk/config.example.yaml deleted file mode 100644 index eb1984fc4ea..00000000000 --- a/cookbook/anthropic_agent_sdk/config.example.yaml +++ /dev/null @@ -1,25 +0,0 @@ -model_list: - - model_name: bedrock-claude-sonnet-3.5 - litellm_params: - model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" - aws_region_name: "us-east-1" - - - model_name: bedrock-claude-sonnet-4 - litellm_params: - model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" - aws_region_name: "us-east-1" - - - model_name: bedrock-claude-sonnet-4.5 - litellm_params: - model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" - aws_region_name: "us-east-1" - - - model_name: bedrock-claude-opus-4.5 - litellm_params: - model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0" - aws_region_name: "us-east-1" - - - model_name: bedrock-nova-premier - litellm_params: - model: "bedrock/amazon.nova-premier-v1:0" - aws_region_name: "us-east-1" diff --git a/cookbook/anthropic_agent_sdk/main.py b/cookbook/anthropic_agent_sdk/main.py deleted file mode 100644 index 506c6fa07b0..00000000000 --- a/cookbook/anthropic_agent_sdk/main.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway - -This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy. -LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.) -through the Claude Agent SDK by pointing it to the LiteLLM gateway. -""" - -import asyncio -from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions -from common import ( - Config, - fetch_available_models, - setup_litellm_env, - print_header, - handle_model_list, - handle_model_switch, - stream_response, -) - - -async def interactive_chat(): - """ - Interactive CLI chat with the agent - """ - config = Config() - - # Configure Anthropic SDK to point to LiteLLM gateway - litellm_base_url = setup_litellm_env(config) - - # Fetch available models from proxy - available_models = await fetch_available_models( - litellm_base_url, config.LITELLM_API_KEY - ) - - current_model = config.LITELLM_MODEL - - print_header(litellm_base_url, current_model) - - while True: - # Configure agent options for each conversation - options = ClaudeAgentOptions( - system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.", - model=current_model, - max_turns=50, - ) - - # Create agent client - async with ClaudeSDKClient(options=options) as client: - conversation_active = True - - while conversation_active: - # Get user input - try: - user_input = input("\n👤 You: ").strip() - except (EOFError, KeyboardInterrupt): - print("\n\n👋 Goodbye!") - return - - # Handle commands - if user_input.lower() in ["quit", "exit"]: - print("\n👋 Goodbye!") - return - - if user_input.lower() == "clear": - print("\n🔄 Starting new conversation...\n") - conversation_active = False - continue - - if user_input.lower() == "models": - handle_model_list(available_models, current_model) - continue - - if user_input.lower() == "model": - new_model, should_restart = handle_model_switch( - available_models, current_model - ) - if should_restart: - current_model = new_model - conversation_active = False - continue - - if not user_input: - continue - - # Stream response from agent - await stream_response(client, user_input) - - -def main(): - """Run interactive chat""" - try: - asyncio.run(interactive_chat()) - except KeyboardInterrupt: - print("\n\n👋 Goodbye!") - - -if __name__ == "__main__": - main() diff --git a/cookbook/anthropic_agent_sdk/requirements.txt b/cookbook/anthropic_agent_sdk/requirements.txt deleted file mode 100644 index 1e810bb7d99..00000000000 --- a/cookbook/anthropic_agent_sdk/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -claude-agent-sdk -httpx>=0.27.0 diff --git a/cookbook/benchmark/benchmark.py b/cookbook/benchmark/benchmark.py deleted file mode 100644 index b38d185a166..00000000000 --- a/cookbook/benchmark/benchmark.py +++ /dev/null @@ -1,90 +0,0 @@ -from litellm import completion, completion_cost -import time -import click -from tqdm import tqdm -from tabulate import tabulate -from termcolor import colored -import os - - -# Define the list of models to benchmark -# select any LLM listed here: https://docs.litellm.ai/docs/providers -models = ["gpt-3.5-turbo", "claude-2"] - -# Enter LLM API keys -# https://docs.litellm.ai/docs/providers -os.environ["OPENAI_API_KEY"] = "" -os.environ["ANTHROPIC_API_KEY"] = "" - -# List of questions to benchmark (replace with your questions) -questions = ["When will BerriAI IPO?", "When will LiteLLM hit $100M ARR?"] - -# Enter your system prompt here -system_prompt = """ -You are LiteLLMs helpful assistant -""" - - -@click.command() -@click.option( - "--system-prompt", - default="You are a helpful assistant that can answer questions.", - help="System prompt for the conversation.", -) -def main(system_prompt): - for question in questions: - data = [] # Data for the current question - - with tqdm(total=len(models)) as pbar: - for model in models: - colored_description = colored( - f"Running question: {question} for model: {model}", "green" - ) - pbar.set_description(colored_description) - start_time = time.time() - - response = completion( - model=model, - max_tokens=500, - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": question}, - ], - ) - - end = time.time() - total_time = end - start_time - cost = completion_cost(completion_response=response) - raw_response = response["choices"][0]["message"]["content"] - - data.append( - { - "Model": colored(model, "light_blue"), - "Response": raw_response, # Colorize the response - "ResponseTime": colored(f"{total_time:.2f} seconds", "red"), - "Cost": colored(f"${cost:.6f}", "green"), # Colorize the cost - } - ) - - pbar.update(1) - - # Separate headers from the data - headers = ["Model", "Response", "Response Time (seconds)", "Cost ($)"] - colwidths = [15, 80, 15, 10] - - # Create a nicely formatted table for the current question - table = tabulate( - [list(d.values()) for d in data], - headers, - tablefmt="grid", - maxcolwidths=colwidths, - ) - - # Print the table for the current question - colored_question = colored(question, "green") - click.echo(f"\nBenchmark Results for '{colored_question}':") - click.echo(table) # Display the formatted table - - -if __name__ == "__main__": - main() diff --git a/cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py b/cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py deleted file mode 100644 index daa38dda585..00000000000 --- a/cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py +++ /dev/null @@ -1,30 +0,0 @@ -from dotenv import load_dotenv - -load_dotenv() - -import litellm - -from autoevals.llm import * - -################### - -# litellm completion call -question = "which country has the highest population" -response = litellm.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": question}], -) -print(response) -# use the auto eval Factuality() evaluator - -print("calling evaluator") -evaluator = Factuality() -result = evaluator( - output=response.choices[0]["message"][ - "content" - ], # response from litellm.completion() - expected="India", # expected output - input=question, # question passed to litellm.completion -) - -print(result) diff --git a/cookbook/benchmark/readme.md b/cookbook/benchmark/readme.md deleted file mode 100644 index afa59aa91ee..00000000000 --- a/cookbook/benchmark/readme.md +++ /dev/null @@ -1,181 +0,0 @@ -

- LLM-Bench -

-

-

Benchmark LLMs response, cost and response time

-

LLM vs Cost per input + output token ($)

- Screenshot 2023-11-13 at 2 51 06 PM -

- - Bar Graph Excel Sheet here - - -| Model | Provider | Cost per input + output token ($)| -| --- | --- | --- | -| openrouter/mistralai/mistral-7b-instruct | openrouter | 0.0 | -| ollama/llama2 | ollama | 0.0 | -| ollama/llama2:13b | ollama | 0.0 | -| ollama/llama2:70b | ollama | 0.0 | -| ollama/llama2-uncensored | ollama | 0.0 | -| ollama/mistral | ollama | 0.0 | -| ollama/codellama | ollama | 0.0 | -| ollama/orca-mini | ollama | 0.0 | -| ollama/vicuna | ollama | 0.0 | -| perplexity/codellama-34b-instruct | perplexity | 0.0 | -| perplexity/llama-2-13b-chat | perplexity | 0.0 | -| perplexity/llama-2-70b-chat | perplexity | 0.0 | -| perplexity/mistral-7b-instruct | perplexity | 0.0 | -| perplexity/replit-code-v1.5-3b | perplexity | 0.0 | -| text-bison | vertex_ai-text-models | 0.00000025 | -| text-bison@001 | vertex_ai-text-models | 0.00000025 | -| chat-bison | vertex_ai-chat-models | 0.00000025 | -| chat-bison@001 | vertex_ai-chat-models | 0.00000025 | -| chat-bison-32k | vertex_ai-chat-models | 0.00000025 | -| code-bison | vertex_ai-code-text-models | 0.00000025 | -| code-bison@001 | vertex_ai-code-text-models | 0.00000025 | -| code-gecko@001 | vertex_ai-chat-models | 0.00000025 | -| code-gecko@latest | vertex_ai-chat-models | 0.00000025 | -| codechat-bison | vertex_ai-code-chat-models | 0.00000025 | -| codechat-bison@001 | vertex_ai-code-chat-models | 0.00000025 | -| codechat-bison-32k | vertex_ai-code-chat-models | 0.00000025 | -| palm/chat-bison | palm | 0.00000025 | -| palm/chat-bison-001 | palm | 0.00000025 | -| palm/text-bison | palm | 0.00000025 | -| palm/text-bison-001 | palm | 0.00000025 | -| palm/text-bison-safety-off | palm | 0.00000025 | -| palm/text-bison-safety-recitation-off | palm | 0.00000025 | -| anyscale/meta-llama/Llama-2-7b-chat-hf | anyscale | 0.0000003 | -| anyscale/mistralai/Mistral-7B-Instruct-v0.1 | anyscale | 0.0000003 | -| openrouter/meta-llama/llama-2-13b-chat | openrouter | 0.0000004 | -| openrouter/nousresearch/nous-hermes-llama2-13b | openrouter | 0.0000004 | -| deepinfra/meta-llama/Llama-2-7b-chat-hf | deepinfra | 0.0000004 | -| deepinfra/mistralai/Mistral-7B-Instruct-v0.1 | deepinfra | 0.0000004 | -| anyscale/meta-llama/Llama-2-13b-chat-hf | anyscale | 0.0000005 | -| amazon.titan-text-lite-v1 | bedrock | 0.0000007 | -| deepinfra/meta-llama/Llama-2-13b-chat-hf | deepinfra | 0.0000007 | -| text-babbage-001 | text-completion-openai | 0.0000008 | -| text-ada-001 | text-completion-openai | 0.0000008 | -| babbage-002 | text-completion-openai | 0.0000008 | -| openrouter/google/palm-2-chat-bison | openrouter | 0.000001 | -| openrouter/google/palm-2-codechat-bison | openrouter | 0.000001 | -| openrouter/meta-llama/codellama-34b-instruct | openrouter | 0.000001 | -| deepinfra/codellama/CodeLlama-34b-Instruct-hf | deepinfra | 0.0000012 | -| deepinfra/meta-llama/Llama-2-70b-chat-hf | deepinfra | 0.0000016499999999999999 | -| deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1 | deepinfra | 0.0000016499999999999999 | -| anyscale/meta-llama/Llama-2-70b-chat-hf | anyscale | 0.000002 | -| anyscale/codellama/CodeLlama-34b-Instruct-hf | anyscale | 0.000002 | -| gpt-3.5-turbo-1106 | openai | 0.000003 | -| openrouter/meta-llama/llama-2-70b-chat | openrouter | 0.000003 | -| amazon.titan-text-express-v1 | bedrock | 0.000003 | -| gpt-3.5-turbo | openai | 0.0000035 | -| gpt-3.5-turbo-0301 | openai | 0.0000035 | -| gpt-3.5-turbo-0613 | openai | 0.0000035 | -| gpt-3.5-turbo-instruct | text-completion-openai | 0.0000035 | -| openrouter/openai/gpt-3.5-turbo | openrouter | 0.0000035 | -| cohere.command-text-v14 | bedrock | 0.0000035 | -| gpt-3.5-turbo-0613 | openai | 0.0000035 | -| claude-instant-1 | anthropic | 0.00000714 | -| claude-instant-1.2 | anthropic | 0.00000714 | -| openrouter/anthropic/claude-instant-v1 | openrouter | 0.00000714 | -| anthropic.claude-instant-v1 | bedrock | 0.00000714 | -| openrouter/mancer/weaver | openrouter | 0.00001125 | -| j2-mid | ai21 | 0.00002 | -| ai21.j2-mid-v1 | bedrock | 0.000025 | -| openrouter/jondurbin/airoboros-l2-70b-2.1 | openrouter | 0.00002775 | -| command-nightly | cohere | 0.00003 | -| command | cohere | 0.00003 | -| command-light | cohere | 0.00003 | -| command-medium-beta | cohere | 0.00003 | -| command-xlarge-beta | cohere | 0.00003 | -| command-r-plus| cohere | 0.000018 | -| j2-ultra | ai21 | 0.00003 | -| ai21.j2-ultra-v1 | bedrock | 0.0000376 | -| gpt-4-1106-preview | openai | 0.00004 | -| gpt-4-vision-preview | openai | 0.00004 | -| claude-2 | anthropic | 0.0000437 | -| openrouter/anthropic/claude-2 | openrouter | 0.0000437 | -| anthropic.claude-v1 | bedrock | 0.0000437 | -| anthropic.claude-v2 | bedrock | 0.0000437 | -| gpt-4 | openai | 0.00009 | -| gpt-4-0314 | openai | 0.00009 | -| gpt-4-0613 | openai | 0.00009 | -| openrouter/openai/gpt-4 | openrouter | 0.00009 | -| gpt-4-32k | openai | 0.00018 | -| gpt-4-32k-0314 | openai | 0.00018 | -| gpt-4-32k-0613 | openai | 0.00018 | - - - -## Setup: -``` -git clone https://github.com/BerriAI/litellm -``` -cd to `benchmark` dir -``` -cd litellm/cookbook/benchmark -``` - -### Install Dependencies -``` -pip install litellm click tqdm tabulate termcolor -``` - -### Configuration -In `benchmark/benchmark.py` select your LLMs, LLM API Key and questions - -Supported LLMs: https://docs.litellm.ai/docs/providers - -```python -# Define the list of models to benchmark -models = ['gpt-3.5-turbo', 'togethercomputer/llama-2-70b-chat', 'claude-2'] - -# Enter LLM API keys -os.environ['OPENAI_API_KEY'] = "" -os.environ['ANTHROPIC_API_KEY'] = "" -os.environ['TOGETHERAI_API_KEY'] = "" - -# List of questions to benchmark (replace with your questions) -questions = [ - "When will BerriAI IPO?", - "When will LiteLLM hit $100M ARR?" -] - -``` - -## Run LLM-Bench -``` -python3 benchmark.py -``` - -## Expected Output -``` -Running question: When will BerriAI IPO? for model: claude-2: 100%|████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:13<00:00, 4.41s/it] - -Benchmark Results for 'When will BerriAI IPO?': -+-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ -| Model | Response | Response Time (seconds) | Cost ($) | -+=================+==================================================================================+===========================+============+ -| gpt-3.5-turbo | As an AI language model, I cannot provide up-to-date information or predict | 1.55 seconds | $0.000122 | -| | future events. It is best to consult a reliable financial source or contact | | | -| | BerriAI directly for information regarding their IPO plans. | | | -+-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ -| togethercompute | I'm not able to provide information about future IPO plans or dates for BerriAI | 8.52 seconds | $0.000531 | -| r/llama-2-70b-c | or any other company. IPO (Initial Public Offering) plans and timelines are | | | -| hat | typically kept private by companies until they are ready to make a public | | | -| | announcement. It's important to note that IPO plans can change and are subject | | | -| | to various factors, such as market conditions, financial performance, and | | | -| | regulatory approvals. Therefore, it's difficult to predict with certainty when | | | -| | BerriAI or any other company will go public. If you're interested in staying | | | -| | up-to-date with BerriAI's latest news and developments, you may want to follow | | | -| | their official social media accounts, subscribe to their newsletter, or visit | | | -| | their website periodically for updates. | | | -+-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ -| claude-2 | I do not have any information about when or if BerriAI will have an initial | 3.17 seconds | $0.002084 | -| | public offering (IPO). As an AI assistant created by Anthropic to be helpful, | | | -| | harmless, and honest, I do not have insider knowledge about Anthropic's business | | | -| | plans or strategies. | | | -+-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ -``` - -## Support -**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://enterprise.litellm.ai/demo) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. diff --git a/cookbook/codellama-server/README.MD b/cookbook/codellama-server/README.MD deleted file mode 100644 index 82a7e62f40a..00000000000 --- a/cookbook/codellama-server/README.MD +++ /dev/null @@ -1,153 +0,0 @@ -# CodeLlama Server: Streaming, Caching, Model Fallbacks (OpenAI + Anthropic), Prompt-tracking - -Works with: Anthropic, Huggingface, Cohere, TogetherAI, Azure, OpenAI, etc. - -[![PyPI Version](https://img.shields.io/pypi/v/litellm.svg)](https://pypi.org/project/litellm/) -[![PyPI Version](https://img.shields.io/badge/stable%20version-v0.1.345-blue?color=green&link=https://pypi.org/project/litellm/0.1.1/)](https://pypi.org/project/litellm/0.1.1/) -![Downloads](https://img.shields.io/pypi/dm/litellm) - -[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/HuDPw-?referralCode=jch2ME) - -**LIVE DEMO** - https://litellm.ai/playground - -## What does CodeLlama Server do - -- Uses Together AI's CodeLlama to answer coding questions, with GPT-4 + Claude-2 as backups (you can easily switch this to any model from Huggingface, Replicate, Cohere, AI21, Azure, OpenAI, etc.) -- Sets default system prompt for guardrails `system_prompt = "Only respond to questions about code. Say 'I don't know' to anything outside of that."` -- Integrates with Promptlayer for model + prompt tracking -- Example output - -Code Output - -- **Consistent Input/Output** Format - - Call all models using the OpenAI format - `completion(model, messages)` - - Text responses will always be available at `['choices'][0]['message']['content']` - - Stream responses will always be available at `['choices'][0]['delta']['content']` -- **Error Handling** Using Model Fallbacks (if `CodeLlama` fails, try `GPT-4`) with cooldowns, and retries -- **Prompt Logging** - Log successful completions to promptlayer for testing + iterating on your prompts in production! (Learn more: https://litellm.readthedocs.io/en/latest/advanced/ - - **Example: Logs sent to PromptLayer** - - Prompt Logging - - -- **Token Usage & Spend** - Track Input + Completion tokens used + Spend/model - https://docs.litellm.ai/docs/token_usage -- **Caching** - Provides in-memory cache + GPT-Cache integration for more advanced usage - https://docs.litellm.ai/docs/caching/gpt_cache - -- **Streaming & Async Support** - Return generators to stream text responses - TEST IT 👉 https://litellm.ai/ - -## API Endpoints - -### `/chat/completions` (POST) - -This endpoint is used to generate chat completions for 50+ support LLM API Models. Use llama2, GPT-4, Claude2 etc - -#### Input - -This API endpoint accepts all inputs in raw JSON and expects the following inputs - -- `prompt` (string, required): The user's coding related question -- Additional Optional parameters: `temperature`, `functions`, `function_call`, `top_p`, `n`, `stream`. See the full list of supported inputs here: https://litellm.readthedocs.io/en/latest/input/ - -#### Example JSON body - -For claude-2 - -```json -{ - "prompt": "write me a function to print hello world" -} -``` - -### Making an API request to the Code-Gen Server - -```python -import requests -import json - -url = "localhost:4000/chat/completions" - -payload = json.dumps({ - "prompt": "write me a function to print hello world" -}) -headers = { - 'Content-Type': 'application/json' -} - -response = requests.request("POST", url, headers=headers, data=payload) - -print(response.text) - -``` - -### Output [Response Format] - -Responses from the server are given in the following format. -All responses from the server are returned in the following format (for all LLM models). More info on output here: https://litellm.readthedocs.io/en/latest/output/ - -```json -{ - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": ".\n\n```\ndef print_hello_world():\n print(\"hello world\")\n", - "role": "assistant" - } - } - ], - "created": 1693279694.6474009, - "model": "togethercomputer/CodeLlama-34b-Instruct", - "usage": { - "completion_tokens": 14, - "prompt_tokens": 28, - "total_tokens": 42 - } -} -``` - -## Installation & Usage - -### Running Locally - -1. Clone liteLLM repository to your local machine: - ``` - git clone https://github.com/BerriAI/litellm-CodeLlama-server - ``` -2. Install the required dependencies using pip - ``` - pip install requirements.txt - ``` -3. Set your LLM API keys - ``` - os.environ['OPENAI_API_KEY]` = "YOUR_API_KEY" - or - set OPENAI_API_KEY in your .env file - ``` -4. Run the server: - ``` - python main.py - ``` - -## Deploying - -1. Quick Start: Deploy on Railway - - [![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/HuDPw-?referralCode=jch2ME) - -2. `GCP`, `AWS`, `Azure` - This project includes a `Dockerfile` allowing you to build and deploy a Docker Project on your providers - -# Support / Talk with founders - -- [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) -- [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai - -## Roadmap - -- [ ] Implement user-based rate-limiting -- [ ] Spending controls per project - expose key creation endpoint -- [ ] Need to store a keys db -> mapping created keys to their alias (i.e. project name) -- [ ] Easily add new models as backups / as the entry-point (add this to the available model list) diff --git a/cookbook/codellama-server/imgs/code-output.png b/cookbook/codellama-server/imgs/code-output.png deleted file mode 100644 index 67e298bd3be49e64f5cd199a6967e910468937d2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 238123 zcmZ_0cRbr)`#;{Qw$#>Esa3mH6s^5#Zxw3qT`@|88lhT6YscQT_ZBK@wDvAdj1XGX z2#MOO_<4Ws-~IW1?sEU(A&+>S*SXGhp69xr*Errk(^et7b^q3tD_6+WR26|&u3RrY z|JftDasG|{^X#?rUsu6E75OWb!wjoet~|J+rubCg7R@RvhIdUW*Oco2 z{mlQJ!U{lG2+IZ=pVf8=cI~EV>a$5F+xmNECM`%%aasMMPwcs0v6mAs@x$*ac z{~2Zw-p;DDa?9Fu5=i3-l+54$}XOAYpRJf@rzqWIoV1`!*Z{uu=y52#!b3u z!s`@2yL0F+8AVA+O{*tzOCjaC@T*>do|JOsyC~Ied;}@8DD$UYX7c(=cSnIJlA9^4 z^9jWAlb2<%;6Ws3XmNkHEIKXGa$hsQBqqYSEI=hhk+3 z5dkKt*u99L5Fxh9WLwdB01sJb?)&WbwZK;zxTVTdN>z)azsV_nTWoMGNB!~t0nGn? ze2^rk&P^w|pMmbkfj$8~Qe!QRi^ZGTH+xQ1u4R3{7H@Qk-e0$~66I4SM1%|PGE74- zEYS*|NNG^2YV1$^A9zI^&4rl`=OMJAhn7O>$|(>KjwP%&)XExml?F#}eMzz=x^yt{)sn|%`Y*u|jw=X5S7`;` zbG%T6;xy{iHJ9p>#Jgbgp?0K~jua&T!%D#)>{lQr1#nV(-M!>T@B53DnZ}bHo*NC? zGlJi_Xc0u z1I{=p^t|qR_GTySf)&bVdqCy9>X5##3bqzZ7CW-2u#aimK;Qyd$XRuxR87gBN-wWH zUY-PETyi|DC|Ns?FNhI*NKK^HaHyOGs1+AZ0lWOVYZOZR9Y?dTYEDXtx_ShLMSwqW z)W1mP9{*-yGdowg7HzF)dO;KIrBsuZ1fHqm(nUtgcLs7dx}N6xNxp9suY^PkHi`VM z?W6;5SIno`jO$z+e1))^t@=Z)r}6mX9V=$?O5lDd9XTBk|U?jVQuJP}4{buOD^dxzIn@q#4nTepok zsAv)Q!N$BEA(i^uCr!UUM4;zPBCzldl^EB;`!C5C_}*U0kmxR17TE7In^%RqpCh#7 z(UBQq${54QEKhcVjPB}68%~KSnduE0yM5wb#-n$@Q5t@EW8@bM z&$(qDe6FlbH*p87BUog&Kk@{iwvHH5PdH7<3&~(D#0hc9oA~)RPHWYu9I0sNTu`Z; zw$;@2naK$cw}8TM$L2vgw1(bGqAqr5(5lkzOy~5SO+L?IrQy5i&U0knS<9UZ8_Inz z%>1`DHJ@6g7R*e3Ug~s+tKsvk+`6wV=s0njYAQ9>(u&OW(-w03U_>Lh%(@ioHZI-; z?3U_D$(NeOH5~aAaQ*!fpoAK_EPyI5Pwk`44n zlqj|D_84tPKdJOmNW(2r517rbbbQ0yWXX)?V0-$1?8ZMI+Ybm0RIES-V5)HNds|IT zy$X*~-{tc?8+l)F2L+}K)9Z@$n?)|DiDY6N$OTtYmZ(*jLx*I-SALSxU0@qYVeQW0 zE{hH=3&1*6>8zdE^KBPy74LkoR1g#gK3J*IdJ$GKt;K)4C{4EOGk5oj|4C6sbELQy zX&Fm!qk%0Qw2|%PZ>MFYMpRxA`gZ)t_0gs{Q4hDRs-VvY!6UQYk|loIOus5h;==hLY~MS>1@Nsz^|&ZLKF%nlJHk3^abS!4>x;-qkaJh7%SF8F=e+ zXv8Okda+ZE`U2Sx$Xvf1og{$L*Nj;1cPvET8&flLgFTh^>JF04nmrH{ymkRjAKL;m z>6)!+Buq8iSVF1H^5wL|C(L@V=tOSYB>XA&QbMN)NLSj-MxFBUbfmN+c2h4h*(Q>5 z^(S~oue1f~xe?7mz0fij^X=8b|g*I901?f{GCcZDB|R9h;K39T;SM!pgMmN5<&ybf!e|3%MD(hs#|i=@#y} zkBXpiv>g`ho`iOh0NcoE z$3YpWeELJT!9YY~k*NVsu5DXyn%BHhNbi=+1-)G%D3g2e?oWK#&sV{{TEX7ly`Fr=4iKv><1G(I-@uy+NiXrT7uYa#Hzdt zCzlTKWxi+}gRKdVKBzV^;RULW)m(h-pc8o3S;z8o84dl7yVp&6XhB3Cpc4WcK(2&# zXg8n^6Bv02jG6NJ+;N zTLFl0+Rd;3lh!3e<3#bd=cF_WB+%1nL=qOP#c|u{czNE=Bzxcvh)_o;8}2}l_i&95 zturNBzqOj}1UN{=M_{?_O#8)`9~sQiE}nelFM_RvHCI!l%^ zI>c%Od(mNcYkDO~?!R-qHK$zrc#V&0Z>vYP^? z%Nse>q-yq>efI%dvS?E38x~!+a(a4*fY-E3f)=Kmdh%`j9>#xDX7>N{?T_GwfL^)a zS}y;^Y3PXffW2hD1ij10Hs8*hSQT_=&D5#d@)`FuG**owPxLDCHfHG&fqkDzZlm+z#Px5}ehvvEa{^n{Ho@-5M7Tm1g!u(y zGk$WfhE%@Ud*zScpF6f8^Ik2od!I?DPjV<+x3n+_xO>(p$VUi))N|z;vU#{g$d~fMQz5pSb8+d2ftZhA+Yeb z)4xKvq-u3*^IiuFBp`ypo#91PHIpe*49fx3KM)y-RqAg{0Piw_m`-LmWR8fil{PHU z2?@ns4dxjITLpFvweOd(tzcS$lCIyAFi)q?<|i%l2)0D@nZbQ)=@3Ut1L!e7@PUAH z<*4E=ZS6~!;7QRX-<^m%NG7m!QU-qD;ZC*A6Sq$ZkN`X2fBiy|90 zUbP0+7>`F#j{8;AtVO!YCKS1!?L@KFGY{?S4aZ64NmRTB*{Vqg{%c|ZL;QxQ#SKWd z(eX}AmD>lwuQswv2K49DA@8YFSS9dB`N%V$*}r^PwJVVqvudO3BSM-qZ5II`ZQ-2| zs9iaz0&3&AOQ&Z z8|5*t^M>C&-fxCwLG4(5x^Ho9=Zyr;k0uCbzd2Cg?)TH|Bu$>&F}@5Duh3t) zmsdpj@s(vEVilC-*DXl%UXlvFsRu3HS=gadKs%yB;9i(YkVyUpYqthRS$w#2Dp6z; z_cBMZZHO;v1ioQ_tJroN5WnN$uN;{96)0g58`=|T8P!U0w1ZZLUa5})h-mz0gp*5T zl`H8H6~@JvHq>tC5z?c{Ep!}xcUlJ>c+0w?D(-P>N`s&+K>iU$6c;4&trEmid}_B| zReI^$X8y_ddQ-qtesi#Ii|sVU92ch05HB7^XL~61#$6Id&Y9v)oKD(oqVdT}{y&{+ z1AlP!*O>+Pn_r#;U071>aD3N23(06}+Taqra@hPjTh%Xj*UnTcw9Nu%v zjNRC~(#`YN#u^cREd)=q&{w*W*--`9Yjupi>miPbBY_w<=Li9XV)C^N2f>u%uqUc$ zRkz-zDBErZdQ$;iMqYl7XL?boHVmt@Fz;!#XBw|k{GX1?WbpEBClhxzxa5v|Q&h=$ zhS`;@ehr;2qtkDg*mZ#ahf$DAk(2#J%jC0ax|2}DFFt)Q#Om~A{BIOlOx%mvA3V7n zc`oxV7JZX25irfT!UBcGgS=p`%5Nzp-K)H1c}j3iKxh&sK(;G|yZf82T2EX(xY4+R z)R{PJ%Xt9JN!ia`TX7xus`|m*K`KQ2jULK0E8@9iTpfoQEz4gP@HZu`0)Ss~y*l)6 z`6Kzy2<6*d4^r4>*roK+1ix-4Fy#9K$F#>BZdvoc*BDdg!C#j&*dclSO>8VhVPPi@ zrJL(VdNsjjP$`BbwF=|l_mrP%^JIK!F0F>{*OIDr4E=ch7eNBDuflhzZgSn4Yn^ZU zs!*Xw(r01z{$1_gsjk~pv*X`5;b8YNel4PS(`nKj{*ta`>>6N6*e)UYVu)qp*K_g4(SFtw;0f4BoYmLoM zN;*uDcq>sq{?uMeN=ZZ5zO!pbJjFDO#*xU=RK)T-&0Vce7dtS3aUKT2{j zS-ig_G(pR!Kv$Rv+MrCaKAT}_JJ}Uv5R!ONgVW%v_#VZK;2=YpYqpH))=;Ke-~^_(nO2J$h&2iPiy|{KXR) ztZQPOd^>@@=Il7duZXbnq+TbrfIX(Q?_8l_ARUTSVXv4;TS=we6xQ;M$%u8XRNe%o z%mI%zPon;So%CktlwRa*6I*|Ad2#7C3HY%@YMNGhN7cE?bituds%V%;b)fG`U+>?U zQ-hH9*SD_n%D0!J0%=Fe#0?5C9?qieVNR{yX*76~zA_Nzo~5v~5TdF4_s;KEBI#z_rhWd%oY)+xP9vs`Sw)^ zSW-~3W`lrLxK{T5!o)iDp%4}02lw!qq~F^ZNeCH@g)-viDz-p41O6B`2k5^W` z!n0)_@5Ie0+0(boGloUa_zD&3xlHe{jxs(Slp1SbQNty?4=$hoHlr{bvtc?g@a<=~ z!gWY?r021E*!VJ<4qBUk0u$5TD~ucR0ITCi9($-c!WP&W1%K{;PFlUoyeF0 ze#VmpP2Q?wBq%!hlezuV`9(L!q*p&|-Yu4$1Uh9liQ#9#IpVw?^fjaTwrO$vr7=69 z=>^5w0&MAG>ZD zx4orq!6+U=*S}p1Q64;?WLYK56JN~9pLp5F$eVX;XW4VBCkLlc+pu<0-RWA4?7nW4 zAB(3(N9ytw*T=0UhoA znls85LM6-WRg%Jy6>`V7)~W~b*&bOy@0;(Stj7F_Xh9r8G2kFjH89yh571H^9hZLY zW`WF2w2gp?@B*2mTkGE>(pY?Y*DB#rpWG(MsM7fvOzsh>)YLy3L@Lkq&Q}28XHp zH07GnRen16>GLB-E0PmAWg+C9F~iTw`~Y;UzG9Z-V^5a85f%FsQ{&)k#^Q)IwR91W zj+>@PXjZFDz1dyOH@C!Y4d;vC(Ecm$&PEyjJ%Np|x|ro=?=PXwFXd8#e086b`Z3zv zt{Ydcsm>|fEAVJ)u$*fZgtemeh_MDeL@Ox`HLRa?{QGdS&dP8_^5m!8^UPJH)c2DYF{RebG)1go2NqT#oU-dYP(fa#&qMCa0!){k+OxK zqnRZ?2bRp3(^&JH>rB^}-jqRTy;`WtfpJn_WnlYgi&kF}xp!N#VX^zZmVVgc%W_ST z>7GbN=X7~sGokhUDYGgGx6cx}NDdhrtWHl?#>vr_rgLR_vcFiP%hw|uQF_c3uCVpv z?@U3Y3i&|c04?T}Ur@7HCoOjW`(w4odq>Bf^D;So11#!s9r znh*GL-5`40CYf#wD3=RtipR^{Oa6;B1M z3f1U(eS))yBNRfN;om@bwz9 zgxOJfbTe(TSW?wYOC+tYwAqz4r=c~T;adE2YDF4%o@_X2{D?oUZM`JBb>8P;Cfd_0(sUGMR3reNc%t~BLE8>E z$~adUvyMw>YUKk(Xl++PW?Dz4bIo|%p=%_#vY%f~IHw07A@gyS6SGOSQS^wnFh~8K z@H#=XdUr;1KAtTQN{3Xq4wp)jVU^Z9sE|fH;>M*sRJLf&qri9`9>bt$>9LnX((N?^ zG-{Vrw673cyPY198m6{EYdsPBrWHbwFy_XLk<~B>;~v28R#nCzcO8sl*vs|8cVTi) zI(#Mu%M;HUMhb8?-I1=-yQ(Rg=i*4bFjZ?-QpY}qO>l^GJSnE9uFZ=KGX{0&XLks< zcL*lYgJV;YdL7i#@m3iHPuPzupxb1(2VX31Rny)Gh>|}4b^dCUW>!=3R(0lFF-N;uGzJiT=rdty5f7>#lJiy;?9)^QcEB9WT3kF*v~yb zFjqXv3@^GL>qu}3Cs?4OyUdY zx-JIeUMubWkuQ!L87eTCqNrn&u3ImfuHd;cel7T5woaCkWEE?ZNK)B_IMegzEJUSC7J7RXN}3wI*^LHkpjWv13%$)>TXn+~Pt&fMdNkE`K%X>o zq#~nAj(V2(Mp?rLlxg)iJGp)`CfC6Q*r&e1L=Mer{pHY=zG`;1(~^bB|11D3^v?p8 zcuMR!L=YO#e86TLYvGpPKqtiX|1b3mRNbjyuMrE2l;(4mzT&lmxjEexl329AyoEg2( zgXl<=St-skbfPZqLzARl(-U zdi5TEDaV^h2#MOU8hRJ>9rDdmFE+HxvGpsrI<(VHH+?O(&uIcNMuIikWXKj|{4@@G zOJAnNU`pOOc|rvBf}9vF@We|4*t(iBqA`^S~z9Nf2oqWA=qZ^ln*K5AlKV+E0H-#)hfsskIiPpsk{ z4YcYq7;p8Z1z1a&d>Tt+WB6z+5axY?=1b0}NgR53|6OhI_gh%i7r3(I3eoo*&!KC* zX(s7B5_&TF^pfgk0raW2cvs1~s?&b=15M{XxU4E+0R#VL} zpU)x@@yHsw@i8Ft7e410O7E`Qrx_Z`T+~Qt7hKpMb5GAyqLI->@jZnaH?{u)NkNs7 z!?An$m$_EbXzQ1Rq!+ukoNg1{vLdbz9y4W7uwq&k(a-XS9Q_LCTjWa98{k04=3^eMh%p0AGH&$}}q^@Et+`I~1!^EQc6o-+; zD%zp4%q*djwZ#hdGD{DNZPf!6Ej28))?Ly=)PspYw#lZI3W^b3RnHun-JCZW%q{D0 z4sOtUYZx_}ZnY5p9iGWq9TJs4iJ;9lZkL;yfmPJ#5-)Egpk;zrMSLqpLN%Z%<~9Wl zaY`;D>%3QMNx!1++CFGT^MAL8H7SOb<6FPL!kyo{?DdyDdruhrD1LCm?Ry-gSrdl7 z3##SA66som_gaPexZ%tAsd#jN$8!d8F@BK^zua8JEt~G}CVbX0oT38j&j~Oi@Zx=e z(I>e^*Ra#yKhdZ?_W9!RQtYB$%QWROU<{$Lif>2?>KLP(=ooGNk2S7)D`MuYfu;u0 ztVV7M*SyrN>*_<*#LqtK-w6}SG~X1Lo%my?PwM-5LDhxXk~!~&+7~8@zWGXcB3P~a zHwKj%ro~er0n*oCuW`3go4ezhqy!XQ7<`wbQH@Z1sk#>`Tu9P5$zVp)5&j0j3r3XL zsUZDX1R2@#!yNQT6dRzj9DVxQi$(FB)TgI!Y5sOS)i#8%@TI|zWH-O_(!9U2erssd zlBfDDB#-ecoaO^r)KvI4L%$B1#8Df>3+=Kt)OJ${WCiSMV*XWvw$3O&-J{Q@`tXTI z@8VuctqE(Y9()r+ZP7Q!QOZF*C7P4({y}8=Ij|?`XoaKCp9mZKY~)u!+u)0u3_SJs zv9#}kTZqrZ@@IYz8XhYXG-ass?f+iuXaidjUNDy81O#)uCSZXnmly`_mbYNe{!;JI zU%ZWRrn+II!?WR(-_7Nx^_bZ8Eo`rTHOVk%`~Wwxxo;g^PCnN?3=&N&wg|}|E-1e0 zkn71j3NtdovdKi4m@IX~;mkYPsYY9CnE~V#Af&MlB5&NQ2fmxc@H9++=EL*nz|8?d5Tb)LPtXeg1EpKfy-uV_7HvOR(T&{17YDJG`jv=@! zo7F=Ogenxm?lmA^t&5$`4%>ai|J6K&EOOTRBBS6lgZtiTSh z`2O`x(|bLh_vwPSu)17>x#NPf0oy}ZOPRwFtZ?>W+}C(Ilog)!a>Q1vy+V~ynowqS(waD4z`(jQyTkudl%Qn<|=JFGQFWOHLcU7k-J=#zZ(GGR_NL+NBRCcp@~@|O8o z_Sv81B7eU}!hd_c|6f7MtiM0rlvb7A^N2B&JznXlw}bQh4^W!?lJj#k>^UcRhUc_q zC}$s9UXY9daDV4Hp`z|Gzy0p0FsMxd=;(l^>`?w;YY&`QxV83l8iuB-Vk{FVJ03T6 zf6n};oeYZrRq4{x(~q6VN&XG+{~uC(jKRCR5Xes@o#!$Bf39O^z$15wLAp4sycGra zb=Hak42K^F|jMj zHuv=RevkkjU-0C=0PMVOfH~k7vN+r6eNT?8g8vPdH^WC#0b28cag@!`YXO zv#g`|m_WOCOuT&8|H~QkAPih`WUdI@Ob!%Z(?+t6{@d{7OV06@}V(>*D2-4>XR0nH^1I{(yplzxO#oYp)@_iM6)JS>xlpW-aIo^U1gT``kn zx?eO5K{Mm);KM&q(H0Znl9L@P?w9qYR-)BN`wS4%hs8&W^7N|hw(P_ zC-OCwM-LZUlovBKE~*50L&A!gbb(==2iIeQ)<4LY7}5NfL$yfgcF-QS)t$MUcZ`B1 zC0r#Y*$SjKVXZzMFT=3y(KB~?` z^ThY0-0YbFe5C&cMnHm%wx4f}MIh#f3Vy{l;vFVsJIZC9%G~*70O%4)#5{YgiBXjw zxiQK{w`J1B*|;ypm8(Shml7pvOyw@_lIEU&F4m;VahC25aKy7=4UQMRaD^mSTkUEQ z=BEgnC8!;oT@UTBG#%8Zv|Bud%+hpj5mO7Tug}b!Vl1U{P5Sdt7SFTKGK@4Zg>1c2 zAzSBy>9+Kb7>-|hhv&fq_TR1aGLyrFB6gm*>Vnkqz_{sRV;to|sDP+?gk5%p!ENgK zL)xD}mP+p^i*DMXpP}!aEdweC?!wTly3)@@Ab9|YTWZ;py}rWev^{k&=o>C;R?5A;#ZQxd0!}MfWO3*4xcn`ev_+P4GSBO?=4P}t#Bf~)O%Og z63&{2V>M5dnfxX>5}MV$9;?4TSTbid#&Cfp^vCn|_FTCwTkcV#&g00`r3yEg@)lUJ zFXmv2=`f)d#xO+9Y|l+?XpX%DLzhD_A8ixqe9!Hw(Nq}8y`jhz>pnkP+A;`bY$;lV$0p}5bsBP#k7$w;lAE+i}W-?vGH zW#Tf{e)h+w7l@V%G2kVT4x}EbNNqJ-23SM-@V!p#Y}sW!^8eGp-B11L6uL3;xSn@l z$~dcoB~Gv3*{6wW`%1?aFd>+rqYzobe`> zPMCde9o9UFd0!>HOEDUJm*aQVerub@E9;{{Y=frv`^6@PN~dm-{c>l-s8q&Yibj(IoNCqx#ggoE8jCTLo z<5t#m#JXODy&e+Mg1Flj-M!hsP_A=YLQiYpbp|cZ)oIH6F>>md-FsRDL{9%4D z{F*t3$RKWtD_vsgcPAeMTj8_5m>Lc^e=8MgE9@@+#UkQA9>5@34p1t5H|^gj2dcI~_g-(6$nhzXdDMOncv9afX-KY5~ ztmQRWO6tTvJmru_--~Zk?D-92#@9%8)}u?~yT~?OKdjQKRS=sWEOHHPao2%dwWRGP z5QvE*ZOBseABX9-rh-<42N1f~87eVa6GT(<(OfdouUA?W)iOK_Jl@7OC}fH?g+G*7GBR{tm^H;NHmSSf`lc%wb)Am8AgfksM zSvs`7^k=KFu4H=aylIdM{LzeF^bjXL}yMuE6R{6k!kS&nm3(QA0anylR)#O|H!0cS7kkc((4vnsLgmuBzLn< zG1Py{bf-OzAF_T>h;9MvL66ELQO&1*MMuaeNt*j}-6xI6nrtJzo{hQ^?<_MzNuMfS zxQ`=K$Ip1|;wBlyb2$_ZT_A__XS+o`ywu-ta=yg*&rZi_Wxl}heYLJd zb5vrCQMpYt`UG}v#wgbFy5_frj}Tnj%NzB&ZxJu@%YFhK_4Z3OG6h6R7sK>4O;eFm z)m@X7G0>Y_i0-V2Vj0I}@?BoX4Lyn(c;&F{HYw!jMNtW(5+}SxMMSg_nUkl2(mJLn zJv&!Q;C{n3tSOhM3N#@TdX#h&1eknAArYjJ8H7hj*pB!)2HcEsDNc98JUjFFo_2(x zA4j={b>ID(?)VV0!&)GM&p}o;+sM85CvDZM=>RX*NUiF`4^17G-2 zd_c=^8Ce+2dQF3BVi-Q1JD7Z4+u_r0!#jW$wD8aBO1Kg?UdSX_CgJ5|vvQJ={blg^ z`PnW#Zk=17;4ihUE9tQLY`ix0vNKgkL^zYz!{xo54?p%L*=p@;HIcBmOn&Zt-7u?8 zr!N1|cRE>rSaRx$gBCKN&LdghkzSF8`2JzCsZPS~|1C`^($g_p7x1C2` ztYKSy<{0nJCAWE)F^VH{PA4GE|9T({pzFM!rH8`%XHeap+pJ>N?}<5wSetsBIipha zW%RN9elV!2xwRgtW)Cs-Y*c!s@ma)K&Y#@9+_jO-0vzo?P@rfKhMp9fB`TPtIpy2* zoC~q@8a%GJk76OtjL-{m#a8a)_^dmwErW!+8Pl|+vk^#T?pRed{=UKSJXwg1ZUx8g zMn;_uW_P{0xTPo)y>{q*u#$;Qn#MnClXz1;4m)ZD$W-FJhqw46m zmeBilp+10GGz}_MN`$Yzxl&(M1F9X$Kihd0ZJ zB!n9VNbd617jVdR50|z18ep zDy*Xue_{skJj=O_wLSCmd}#nl(mDxmao(qUp>(wQRs}WAMG^lz)5}nZ_7eOI$o;@f zm>D|it-ZdUiW>6!t^{4lb>9mU*FN;HYjJibrtF#QMjEtW5rqxD{5D-)Q;k%5CwZW*P{2Nv0N~ zWio>MYx)V)1HleIo$0zAjx`5?ut&|=6?pKeuX9hdH#F^AAl5Gk4A#KB`aXp1oBO1d zN%Y6FQ*^j8Q{8-jz^C*W10H#}30(!LAm6q5luCHiE~`%@B2FemP^>v9_EU$5lhXyC=zDho!V3R81t z!oToz>8h;Kpjy-eUy%a`d$v^Z(P_wnYJmOzom>gtV~@r<udq+4^qlwJ!#~k~CS09I?s9RT3Encl@t+;!|6*d5g!RS4 z`A%-iZ|&v&)J}~~Fw`md-5W9iiYGjG?KY3mg*H$wxanp#x^u>>3sw9wMqz%*nr%<} zv+J`ej#LN!_!f8DJRJn7E#cS2Q}w1P`G`P5%!Y4*67Le*+c%P1*V{bN2sisvA3<~K z@@oCuxr}7{G$7{09ixQylUpIpez>8=5(>tqx`~4rXs6n4ZI4UI@!Ud!#fd^@>%p!q z7Q$?y$9%Ln{7i58xaYKrAMWvm8=K&dTSLhrS0@cAig)RwOY}}~Z<5Bbl!54G`U?TqA%fl~Du$tO)2#+)Mbn!5dUj*e3D`sbj!rM%OdHVi zn8fFqiuwZfBJjM$uyO#rHN-x&p?opSJXylzoSux=HjB?D`5k@FLi|2HgMHW06KL)+ zM@=0M)7s;w$r<#kj*s}Ur3xzkQD-~Mv(E(l2h${$0nPrq6p*8F#vas~%;$pHUD36i z!r}dmfW;(m7(CmUK!UNaB9gk&0owKvZ~qYMyk#*7@P z{Kl@ku0-p>9CAXdE=DirCL!^t!BX-M*yra_{yX=|>}|t`>1bmGM8ZTXpKD&Hs_|By zbTmtA1&kekgfun&rqf2N>nf{Fg$l=_4ygy!FF|}P^{WZggxz>!0xBnR#d|ddNU(hL zDb97he;BatA){Sa#hqksFT1rLA@hr%MW`jR$TDOLY1se=Rm z>DujWiy@gTG_s|xgiSq=5DP4VA^nE*!kyKFo*gM1U4P?!sfE`i6w=L<#N^ka{Xg2c zMUcG6(3yF@k68RsVThxgz`>Zi2qM17*O~y!BrHQs)95vJn3kL@=B)Ug%k0#;^2?MV zX)eyviOnKjC(-;V+rQ;+a(YL&Wb2FcJ1xd9f+2$w(APE(NDwJ`Ol@?A8b-~|x{Wf1 zi-C^BC{rvfBDcOL9e=76r!PS7zrLR;(V6tzgd&{-GUgxSlx)D7|D@RV z#0@{6q|po#Y`|4AW5)6)7>*1~t%Sj=WTO^il<_xm4-Ah0tP0>hy90(U!o{^U(nN7^ zur|1Pr>}G`>68|xRq^&daL@MBrzA(c0W8pwmq)6Wiw!Nd{>U9OIO8^6Kptbw+@8W* zI60U<%!p6&>;5i~wOF$mg!j9*yRe*9u!qCMd!1ejiz#mVwClY)=hkHV?Nb}Lk9fED ze96O7(20*8>%+$sna`MyC*rY&Rrf{kKbN`Td;jAHi#Vb-4!-0$uf>O<&glpFfYqho z+#{?Em_R*0jCyFy$T!DQY{*aBk?^H8#`GqyL~Yvid5d{nDeiEnZ8}#XsOs|{H*q(& z=-6WKVcfuZ^~Cw`7+=fm0NAOfW)^?LGKDnXE)qMaW?+jEJ`OVV^LdZw`7R+VRWq0m z0FM<>ql^z@z$5w;tv$YK%)sp9qake2EW#3sJ$V0gLFS)$PZWTDHs{e+&w4P|_PBgq zrRkyh>B4W}apdgH3`wU<@8u55PtS2Jo*ETFg06-BV-GH4Cs&CgwlRw5H6)xM@%sb+`M=FLMjp-T-RNrc zI7mTk7hZFoD*!q7e0i!cN_SbP%EuARTj^(*sO9RwBx(mQ(}tfkv4W=(8C?e!4Q|xe z5vuB$G_qI38M`u(npbWoAr6bOj4&_ZZZLJ3W!cY%Z$s!C5NQUVePxyx^#^E~&S?RWq2gFIPT zYrbQSIp&yimAG(sr%|5{dwL5gybPv%EZ|MNPXFxk##}WISZR^5JM%O%)NlDzk|>Xd z)9Xe}=)LSI{;aydOZOqYV0xmN&D=OQ^cKqdeO_>nAKa?9NH5}eMD7R!IoszZTISun z!9XgXm%r1r7~LieJ)bg<{?&w@mMh)Mz*-rF?^^d_s_ojWe*^J-ZVZ~P?~{vn*~Exa zUE}Y%aJBLX%jRuYxTrl|-ub69D{>_=!1l08s1Pmk7Cc~TZPr5P%WI70Dc?|M0jdNh zXVYN4rzSGZDgCqjs()MJN*?XIm_`##KrC3E`#FO~b)Wr$%lEHh3q0I#GZ0jnaE&$g zxtFuqU|rb)NM#Rp?3%$_1A+{J^hH4SL;TQv^TH|=WYWhHI-*d^Es#~dFmAnBI ze5t%Z{g8B{Y;d;Bi9~Js`D)5{D#(wG5wf|^H+G5CQ8k@zV+7lNVi~&1RP4VcCyoA3 zxp&-`L)a^4eiM|{{)|cZ5B$9UJO(a5QFQ4E1a{Il*FXNUjbBloXiBde!FSJM%)A_( zgSD~u8(heI#*NiUs^x8dzL`7No?jr3J?9|;LIFm>k&v(C0AbbVLA#(Hy^5$((g_3l@fl*t}e zqd+#b~dtA?wQXPIzx)J2#<5oGQ)gIkmyjQ;oMfVnG~uquwNMkFl>upTGrP_c0 z|4OuS{&bPsxb&%}gADn=DkJOQod$R9w5h;yBch$9v75t>Uzz`&eE`aWp@feLbAe7$ zdQ-fy{L#W0Y(wqq>+;#nVmv=a2S$b-n#S=RSzoq3@lm)krCcw;UGOlE%ZiqduRPQe zTd?jKthWcOeYsr*z2~g8t>L5+RZy3{ST=H)_t}Fm^~vh2fUa(sWA0R{nkVd% z;EU{XL@wSp?&%+h^>0!By{aHmzs%i516G9#Y=YkT zE!qPmUa?ylG>wvox^g2I2N)E=9`3AP%i*^5J$4zrE|E*Aa2Z)oRsG{yvEM5X6eAlu z7UK3##a?Q*g3Y^P%Nsy{i*$L^(ZvY54cA;OAikZPv%;d*#bWVnlkHYmPdp^i@c9z@ z1UlC+Kd%l&LL9QqRmu=agAH5;CfU;Y^c~g}%}Opv54UQ*Dmve+%VGk!?@Bsj;$`K~ zDf~>-OxQ?WnAdiIlGMM4kpHQdo$2QGlLAr_@~dMR zZT_=v<~{yO&h1Y`~)yZ+Y;>XoDAwqLZaR ziJ0cs;rF?`UV{*Zn)EF zeNAEO;JwRRuu}iqVV-G=EcD zZFA05&jZhMyRmi1qd52bW>7<|fLM?FT2L4w7r7p7`A*%=H4h<1G!Yldp0bwgGPaXf zQK`w8AX%Xq9#!z8HmNf!H;|>ze@gs&L)L$uOY6B-lIPYPN#u0xlrrm>*wSTl+OF^q z_Qap+M-kmg73(w6jf4JoC9?;&t#(?CerVoLT@s`g@IvOpfBuZE66L>!#coGE^jWW# z%^95|wXZU2?&g2~{EHA{^9G$entmy#ExY;?m{53ZQ@0pBru{{ho40l`8hbnXH6jq50gP!?M(kdAPd#_yo!-9%rbFWmdT~e@cRQE^&R?<9 zTld%TRwsk$o{Py!{h;~$4tHUUn~T4%s)ox5)-qY%4sLA{kj5ra1SNP7oHP89rt&mv%YRUv{&SN&&e@zaG+MtvGh&N}P*e^>+9Kj3O_zONcm=_% zq2Y1yv;>wZ5I~gGRiTBJ@ri|aw?G+a$y@p|n=U7(d%cb(+T8Hf{t_Af0BZ~Sy0Trh z>(Np@j>fVK(25@%nfj2li(_g^Tg@>9yyB(vbbsQ!s*faTVI8E6q;c-kwaCyHEyu0= zXm=M6jtW9 zRa-Xf@yyx~7Y_KdP|D#azIjsk;nA)?6^g*G&!mrR1VUspfsn?pDlb(yY;;pojxl+C zpPsOubFr-2j0CSkva|hwe$xs}m0#w82jNbDPQsDQgduoCe&`Ng$;N>`U>l0DFCO4@ zQdxeZuT&%k?de9iL%6zP5V?beDfj+M4DfxsijC49CYo833Q6;4VKimEt`}DKZPLf3 zmp%6d&o5izvGWP1S*){$Ni<1Q7;M5zqDgiQyDnNi-tpnmKS4VeqN6_4Dq= zp#BnnCnMN1%9+vNO~qU~jeiXY{PVOT&!2BS+-Y=nl421p-$?MVmVp^HZ@Y{kMAJ9c z*PYo?jPB^wA%{=1I4sszd!afWLS=%WBnP5NfU`+5Kk|qZ`IUK~IYf+VPQOj~qRty{ zWwzg$GXMJh$g<$AYzG-s5(8N2`$Tfte}PW*M?&CP=85JHl}jkolR{?9C9vm#U@Cbq z+{DTv1Oc~xo_)>W*|#4R8c^fAIDQqOK#vf^!P%gGpdmoXO;__wU%lh>jO3$g0>k$~ z5mDzr&c-WJclDw$p&O@}2cI`I$Ve?zN);A|LM$;=61KlWB#|z=clj9qATk2J(Ef9+ zYogwsnCypEw2q$IIpQJ}zMfoh0{uvtx;*_c$P@R%hfscG9q0m^ZHNsOJh0~WN~<}! z6=HLk>;;~UK}y5g7<>x>y0j0Nh=y06OXNhmW~}v5b15Cqt1Yy$C^oWvR!sxZI)gwL zMuQKJ!D7s^QiNI*jzXNv3N@iDzGI{KaCg17pJzt1*Oy)F|9G7Mql@~OJr@!v=9$=e z*}7K#E!u$-nHfK||2-iTWa{CpHfWz#JT=5ul zn)D@YN7Qor;4y~@PL&=l>)MoqCJ(o|2RZ`TR-@ZDC6T2eCla!M=4=nnwQdi?R)l2I zOo7s}_IUH-{HB<5^##B+xg18&m$&$#IPAYqlMnpp`GH2=UDd03ZsQ_Fh@B-7nw@Up7?icCcny7p_ogDqyk0@LzcX$10qHowNR zV#s~l77raSxY7&sruzlr>sB$c`Hxl!TYQJn0RC)^CC1cS_)6d*Cc zwf~zJjH9eBOIjw+V`NVFI_9j@YogvXWv7!l+eu|W#3#I&F+hgooC=xSY%I1t{(S8L z<>I5T{}Qqf9)7#z9prnI-}SfIR-dC^)>Tg`-qc(ETHmlbTjhQ+s~FWU4V3C)3i~Y2 zK9qv$B4ra)diP1GAE^*(g)D!-gt!mtW>s@b|ND8m`#V~&zVZ<#ut8o#WV}% z^^PJLxRI`05w6?>*Ek5w(Tfvl;jG-K8<$3_=$YfznRrcsN2Hvc$ZBj-ZG;Dv7Kw=j z%<#lPqQx8P%B0o9vr=0r@pc+oq>H&PU;WdU`M=+JM+;9F|NF)Nb2*v0ADz5IjMe`(w;X?C zg>_fhfwLA)BE2TdMoM;v)-q-%0Huqw{oWO@XU6UwnSAKS$7p~8tV$2~Ip&G!`ulOg zR~ft0{rq7R*9@lfxKb|{)0V+7#|Q7D#jWs|s00QlVC~SO@*bTBc;oGe zz5Xu#)rwTI%@8p`8 zO*=F%e{tBD0!XBfFMG3_iL6SaE7%i*Y;bIPV8V|7+Fs#m9CYnwpcx*M?R;sY1QXUD z-;c;Wk`ctCDcO772NVhvcCJVVY1R{o!m?XqyGt z^Mb!`vfm`awo5Ipb_%QCZ1a5a$71JkN++Bl+UMKj_96dUjX9TdQ16x*yI()`;znnt zA2B)_uVmo^%Ti$Nzch_iPG|mMRpfuP+P7DpuY9!4`Cl&pg?-Kfc7kvN2fLZl*CF*k zaaxy8v5b)B<1=+rJ*bvxLW0T=Y+plNzNe+9^F3K*83vQPz4d2m`QoS6d4~}VD-yE0 z={#P)=c@_mNng=K-!n65-Clt<(FOSA2(p%xWi5>3ifm{wWe(n%pUew>*heN0;tc7 zEzb9fBea%f^{)#Ej^m11QbJ@F zefYCm+ziA&9BUsjYM8U7m;mB$Z(@Zimp7ExGq1~nz^*M~C=#%2m2lM1s*rXNcW`>S6Xwd`{%hA1A?qhiF^&k&8Z zfu4|Wlb-)CkWm^J++x&WaCx0m`~D?!!Bl`e=5TrrISj8<%0hJ94?e}8c(~PX8jrHVSsj7(##pz zx_r%OAv4SGPE_41F5FE$A5^y7VesSLzco=U7;OeeFsV$qdb2_T(j%dmjjpO}D{hd9 z51LAp+kL|osG01H2UXJ@I6mShAt(~W_IeQsn^gE|r8ESQvL ziS^flelNH71)2xT&o`21mo)g8f+fMTI;}~4h<(l=i+Hzi8R+CK@632l;nOp)98P)6 z7Y> z`b8d_TCCzoDsx|ZUp;rb)B)h?e>WdD1+N<&FIQUgOYeLWI)qd`3`v2fb3z6kMneFZ zOr4pX?d#_0Yr_vbG)N-3FX{dZwU&s~HCYkFLN7fTRf~0g?YC;!jr2kldesB@&lF(P z-e|G!V@QLI%QM}jFPBv8HK95oxTtxHa`D9AMBf-r_23qcI z>n(5M%;)ZaDI8Ju^5 zXJgT5-;A4Gx@?J5mhBKV5SI4CN(5<~M)^<#%?1~KcA@zZsGbZIwpNyVyIUe^+WO>dAw`nI_{3rutOW>9}7 zF1?+o9EGueyn`V{pLAXaHgVPf$FA5DFvwy{{m;3Z3nuBlKRVOcaU5N~uAK@I6*w9j z=o*o%qh#UyWk{3D89|5QtsVpyn%(~f2+wG%4-0XEkN~u$)RcG?GQ zy`y=K8%4`>T>Zv-Cn!=+jgtjiOjD&Bw&>h^Tn^SMa>EO@yMty+AEH!E=6$|>@xh59 zojxqw=mtP^|Gu{i8$(-P1cD)B%QbPVdR<$rz0QG)_=cxSomqY9?*>NfTa{Dif7d_% zu1+4Y0M;up%?k1IsVu=Z-a(kx1s5VsPyax;1+;iW@Pk+1C^sfIIOJYE~E&1om{6bh*`3u`|Xvc&JdYbF;RVd>i3LUxSP4kmL@bOf zz%q6awa9XHPFdkt$l$t_NA`~7XJyPvSGUB+jJ?nD=E_I~^;It{TX2GIq)=ml)r&+C|1<E=2f{OVnht#+T3sCG$39AdC=lOSA%bPV?>LkQKIn=gI3HO& z0l47Pbjlsg8lJ9VQd#`weWGzT+jtdv5>U*&zRwKp?>pJHFo1`c$r(m5g#O&n2Y;S9 zzQapvWnykW1x3K?>a~I5!d#zd!y#MuVOqM1lkIEqD=~ulG^wPM4U4BqT=xdDJm6tM zmAQ995{<(L#9LZ4P=~g~&Fs*+^1h3Kp69o1;X$8YGPl$tqC;^DfIdH zSkC-#6Tya!Vl{xK*jo;$R~K;A6)Hm)Xu4}J##pwi+wOM`A*(c?g{?7mhCmC`wSY1J zqkb*|(cJn!Ym@&M1wV3}Yb8sEr=`z6&7d-Jq*-Jv^$9;uTVRbAf9G(mq_L#|1*Iu0 zG>}q9yV^C>QT?3lS7@8Sx6#1REms{Y+5~6jPqKgmEea`S>z2!25N5AfljL8D;7_W$uEO4%1d5>YBLNx3Db55abm9iv8D{F#hRPxzQFQk_4$^ z18HI)`@PA=k#29?_ZX;5!@|=pv;GvLySUHILqT4YC$WSE4}aL^ok5?w%2iaOTb5^w z0|FDR;dl`#BiR9Ew_A3N4k9FIz7|TQ5FJ)(`Bom*=G;-0>;#`m2M**I3pR=A#z-gQ z;Juc*vW=I_3PfWCke!FieOZP_erc3*xH>Tn(qu-^l5ig)m(0_5{3kr5p`Ez$oKzfc za?Wuq;z6!AwD|SF{v~(*HZ1UydP&V}O07Gqg@m0&jfXc{6`{y8BJMRjHI2pT;6rPn$LwY*Xrp(&bzI=~w<{Vk+^YBoz z6HcnF9P5SNvHzvo?X16MMcV=>wL9p{My7sK=YW%}FZTii0`8|w1}~Yd-ZKQ7kP8KK z^iSCgJ;Eex3L{$vp{(|DyC1U}E;&9)2KZBXS~O5Fu(@lqfdVWqYlpf8Uo2mb{_Exc zKOMNEiK;%eq*pFG;*_ybw>2C$=m+RWJROq@n|^#t+5q|63;MqC5FHj=NV|+A{Dw^YfIz#AT6d z;~7)_{`CN9RhLaNZOPCh-v2>>jIh4IEk>Jgc`c(`?Ks_e^8<*`VkZm5g!y@Z~HQWXsHw7-2+V z%S3iQ#1|c@pfy$F5RE=)+sZ*TZ8S_{KD|7voqf=L(B>_(&lxaNu)j4wpXoB1t3;>@ zXmj6ht`4UV30<*Q3cJUof6i`&1waP`OY7f>?7Um;H&CJYK}Y0P9DQc|CiDrvwijLo z&TM!E7?mO0Zg1)7y3`$@h^jFkeL~xAQ!vFOOAsAbaYls??F{ zrb*-Tw^yt6r`M!m)nnyDHsJ<{4%y=#+4h1YPKAa6Jtxn9PD&&noh~@X=|OhZc>50b z$kcBWwpFw^p|4Kzx|vGdCKTUuG9k}VQ{}sH?^Vsa+eaUTu&Lo<-ZvEuYusG+J(C@h z@FX7xuU$8p<+5Jy#NsppP-bu!ANt~ahgjR55p-vfoWS$&^DXiVDYWflPsUu~~?%v&SN-4{{xm_r<)}oig^GrOM7V!ntGKm%`3Utq?xZ@4XOLy zR2HQH(8)KD5G60HZAEG8{ATm&`Bj?8bF>F7^Ayb6+&w;*WMWmz!_XyRyI5Aw!|ahI z_dT2Yge}yFJW+&H=^a{b9zzZ`q(JX?oNR8LJfsZIAM029tkV5sL;U&>I3L*R8U|y` z4}WdbQncU)%96^Y|6QVf<3Lms-`)$NiLARWv3$lS#a=MJ&N+%sT&2puTx78^{C%rl z(?jU&&AjT&fv<5%awS)>Coh?y7<>ybrhTFo7no5eHUxH&Y#azT!H+r3084eEhGdDh z0hqcf4HWQ*U6T=92R=ubQf1TG-Q0trTWTf-yvXqN-Y}6Pp6ZY%+fYJTDbkb?TOEWg zuXP&MG$Dpxy-ao*Y7afa>;l_cL^i{z2jS-2CqJXxWD8i1tbW2gRd!O7hm_g<)3*+N z@-jBZ#M^(_Lr)G@%)M>H4wov!+b+iuqr+CCF`(m*AjjdVj*}qLl6bfrChFi4ExElN z*f^6%p=ZlUEUO#uI2`u=jJSOY@WY(^&;S*LH`{D(ZwMe)cBIiOOB|@_gRrdwBBey=fdd?L*tJwA7o{o>U|9g0; z3@Y%ey><>k_aQqJIXpF?^&6E?miy|R4U!+Som4-wVY?aOFBvyZUv-xSS`Z^z`xn;m zu>^S>I!mn*#xsi8k0-c=d{>M6LB<3|XblqqJ3X{4xg=zSpvjK^Hr{!eoxNQ-LzsEv1P8*ot~T4pVN z$?D;ztZMhp4ke9|X68>e=5w7Y@&e`pEuUP0jBB`rt#Mb7K;826lOgvtQ5o0BXZjELol@;NgzHt_mmmTEG745#+A9E=s_gKpyxHIu zMV69roM5%lqHVlOUL_axSFbRo^l+SNq-{sv5;FbZvn}$@gVZKYO;l)23HiEGuYO^8 zS$T)^>#(xvt1ayTD8!`7MZgJQ5Kf-(wGoo|pD}J{&5z935dgJ}xV4;7S%Q$570$~5 zHL7jKS5qV7u0JieMVU(Sxf_RyZK`(rJNqjaE;4G?>2tl52xyi9R-mw_E5=Q6K=B|Y zJL+|8?gLlec(h>pE*7QPL{!x>-n^cLKXpGR*}LCEd)XJE3@z8t2KRK%j^+f;3SG00 z-}u{x`a{@GBL3`WWX4G*tr#4gO*T=XD;1+ibk9kyV0H`!9dmMQ+xNsB{gl4#&<}1j z&Q}4ujzH1_?{zd#BVky}59DJ0Rm;tDth~dliiY|s-)idl zmXYPQ{VDfroC6K-4P1;=7T8An!cG-SAyjs=Mg<@*A+Uw910^TAaSwV8=-UMoyTS>Z z;rhZNM(h(_`@8>ekB|E|ex9vvuZL#Zn;K7%nG@>M1*IZvMYE1CLy(oC=acfzf#SjV=wM_0On4~{E$`i%Q`a=-$)Mw0}5x|&d}7oiB;(xIwPMB4-O zs?XXNz6gZZPA0*9kK=$J9+YGrmH7_=R$vs+M3u9ksuVl{c@~=^1F{pTa0^tm$5!)K z#H%xx?>m|Kh=mfv7|J~N*lr=~BHxpWH!4=O`jhQX$e_9mKXQVb0~%1L@`RT;g^&=c z<6VHhh0Uj*=M+ZvRvSQI)R=OE**Bwr6q=6rTMQ^$g83voJI=Nqic;vsl<+xaweDMmonX~ zpHju{MFd*B=jDH0zxurn5YL$cbN8+TZ}=qg&g$FC1>F? z(Zeby3o6d){;`B;rS-?Uu#qp~Z5^af0c7R zLtgXJ522$(yA#E4r$BwnSf90KQYc)nY*qbsD2|_`RE?toE5ZZ@bFn_FN5)&)p&e^& zRSwsWig737u=cwS`P61I@Cf8ctt(36*=9I5ES0&PSjy7MP-UX@WoZ)( ze;?j=aDfa{Clgf<$z^%uvi0Cg6cQx74P{3T7NCSyQl?r0s&^TmP^4|CJNK50d&lSN z*6a+|LaTcPIt~bAuG&^{QX8UqZA*#7N)855Y!BNiPPme;{ZE$jq5~Ln$~Rn`P-OX#*QLsI(^>ICH0ywMX;2Jeq&I`R!eTQ@OKDyHxQ-Og)-mHBr3y_(luW!0r4scTFu0PR9Irc`bXVWhkE^gL3E9er=3f z&f_vEQ4-M`U`=(@b=b*Yv_3(=1iY$*oE)b-Jmp~KE5u`~HgDatALarv7; z=|qnTau6d{l#Zo8+13{Poag~z|g zkqxMO2H1`hVn;*9`yaWuT3x*$n2qRd%x5#0HS2C}co#2M>a2HrTh*Do_5;*M_T6Hs zGN=|R3=+7Mju*$}9?h%>iQY!VU=BiXdRx0&0Mqo6j+J`=q&U{MV_zNI!`>})91nIl zj-CcSiX^q|09?9Z7k1yF3lSbzK-KI$2pK)NOTeZ3t_2?&=aEg!)ThJJkKn@WGa+Wb zS(tyA=ly0`M(l7oX2rdpiw6sYFflHS1r`h1=Nj+8V1oxwmJ+>W_kyEf9mh*}@s%R+ zdGU>*t&X7@dyp|^VRc8ZYLT8Ys7K-0r@U!x#qAy3#@YBLqQAYMuT{6)F!y+1B=f&} zW-jvNYoesG<@LO)>fTfEn8{CZ5A+2~Z_ZVLh}worIlbU0aJXDl0MPwn?^=wp(-MZQH-V6oj^t?ix1-iCBF?sK%%q zcdFdWlPjqom-ZxqE|T*bcb>}TTq@-6wRFizcQShN_;H6(LIH7~AYw`x&$?GE0E=@I zjUs{EH*e%4X8PFsfIRO30P=; zeg;i)$Rcmucc@kZ%K-W8l^oLwDf{|+LUrNla4YVxlHLitN|30$l&#)_9Kj+xt7k+B zo9dlBD)FGc39GmVigV&1?~=FPMHw(=mS+i7x_f$9y`x^F zgMci5j}L*st+rB26_*6rKsl8~{3u~PUFru!xQ}7`9tsm`uYo#Q($5;JsxGjNZ?CdS z$qrVx^M?3$hOD&}fy^d|t;aiq)>-TFETYwW$`d$Cm)C3Dtt3()g=m*|X^9CdA8$c5 zQg6f&aVi+yT3h3p>f)#4HEZjJp{r>b-9cdh=MM!DjxfuI*`l;u1Z(eP&ZKiXB;|G^9n)( z>Ri5S7b831eRsgeFP&CUvN*ssbc(+zn_tN<@N!FnHm5rE^wrsDK{vF2fAWqm!)1@O zuLwzW-Lk(YAtd3ra#J4bpXdfG;+Uw4f^5bubLjCPTcQHcFPVqcnf2CYeoe59WT`d~ z^c5t_00Gc?WB7raOQ>7#uZ}&WxfnS&U}Q!90odCt2MdpS8Mc-nM(K>)a?%y1yo_RT zwu&H7heGq0YEJ&b^S2r&JnnIs+)0mf+l@Axe@Bu%wAv$Z8?D1~pylBaUw|IFiYb;! zZBQ$x24Uya0GCTT`Lr4(9`g9s$R6mHd<&Mj)Tso5?@eFwa@qUZ;rNAi^GCe+NZ6w> zNrJw2kg=ZP6SYzx_2^Mk_JCD=`Ke3gQL!hFb+)dGF8wC3d`)>r$BY##VFK%AA8F#Ho$0+v2O-uMA)dvWvB&n?}@n3CHBMF(RFLapi zMBI3BR7CT>soTFd} zTR(hbHM(JD>v!(W@kDd{V6pogc3$zOkfnD1_M_30kEdC@Xs|RyMw12aM35M$LNVb= zAm_`y8hVAE=h*yrfv0cisa)W0WfND-EVMSxavY+cxnY$R#lTYVnfAHrO15`cA~xHz zA?vj3x|A3ghwX{QFx%^i@BSh`<6gW3`x+Xy8gK@( zV!?&o3f&5xJt(Tmf9Lz{>+v95(R*xFwzY}dXMQ=Dp5930LX>wuq0#Eg+cOVUQOtx# zwj~geke%kM#vhsQoOy7{_>nF1uP@OmMo|-L@Gi4Ff_-p$pP{BbFowj#joLC- z*KYr*YmLNLvRa&Ei?^!fzf&pO<#)_XrYJo3tCbQpY0%kL^u-&PsI!4090H07Q6H_f z$BW>%PxUd~pZN$iSi6*}GQ>ivjQ`mei=YOyEJXMB3E zret?qf%boX`B9`toLg!tYOO(4y z)VUJQJOjz)z;7*o< zD$y?dp{WLIX5HLVvQ2TSqXzKl(!bp!nm>QbLK68tT~;BXV8M%l310 zjDJhA;sv>C{=bm%WMLW3lL(`5C7uO8H7GWyP%--Eg%;YTa>Hhw7=foV_-LB0+k2%9 zZv;;UkCw&Z*GF4$t;^D4&yu6P+G!d!C7y*!*VD#4*8h_B^Onr{Bye<90F!I6sO+ss z=j&F*tv07(-<-QVI(qk~SGo8hmhYA~u4;wF;^gg?KI=@2mSj1r42ZZm|MP0nsXD6u-`J}+?d!(U?%kPu{UpLIi8pmZ z!}!Y8`r7C5(_ev@!MlGJ$HencA|p~d_yDy&nL ztXTcF&=e*KJmo@fzbEB$ubWTl-RQ|>hF`b*a^Z@*{Z$n89p9}d|T9PpyF{DmcJnEPCsXh^^GgHqjXjzIF-e5P8l+ z*!v1t#hde?-mWsPw?Xih;D!*A-^hcRxW$Wx8qN8cWqlwOf2VG_#d-AVmwnw?U^ zBt)Q#I$5EiIh*>078x&EILUo7AwvW;nt}{B8e#2LXR+=lYq>`@2xo_znjMklPPS^$ z^Vss#k;iF%TQP);t{BL=N_{o7nQ|xGE%rmMtYIjzvJu_>s`3oKQz2uvvUEM+bc-Lv z0aHNCrn5T#G@t9y#Z&>BPO+20^}+(v3HMx$TRZuYIcF{t(K_RjA(b_hn5#FPV~dLG zBMBJdeycJ)bMdRoax^3AASF!b^9UdEd1^r(^N40^ zN3uKt@y_wO$u64AV(sz$n}}ib*-y%;7^6c*Lw!D}2#t|50=J@c0?C)F*M4x`|77=u zE-z>-Lg&f1Te4R@Xg;6QReW*EGmRPX*T|3iiNhe(R9cZFOPg2lBnew-gfqt-uGeYz zds@TlXf9v`5)<1cyhn;`s zt-}|TcL(PVM2*|sEdM%@e=gtEx;|tgdv0+qE^bL-ET8PaePpqx4EJ1ItSoQ0n%*yo z5PWnYleXmJU1k~+TwB@MUnXbIGYRM7F3Ip{-i6+Mrod!b`9|h->X>x6+biRZf$+p% zroEvhE0;`;_Jj3r{e`W;9^U6g()Wp^_Sb)K(IDMbg`P|2zUZm`*DWKAKUaYYO2Ld# z38L}1yknFd!El@H{*R)zZKErxugz)%zKdP?$_iawctBhE)$V4mA4Ku}>BQ$xKdQ){ zk&&*yDWw#9(Lpohlf&G|o4|I#H(AlFT+RZX`PaWP@7AALr!5vBUkwjCt3z{3;F?`X zP5fm1RZh;irBiiV>UVkydg#YcUVB`DQ=IwtkN1{B-Ip#~0RaL4PI*!%Hz|UY*88ToMVPxe$}z zP0Qy@f=|o-ra0HRAKp}t2BJc7pW!(aUMi+h=1>g*r<(6m!x|?&bj#B^q!>Y_;644 z^hZX|H_A0YSJAmOZr+JueVSqC5)EQ3RU&@luU(1Ix7!0QA|E94zNlENKhgEWulx$K zj`Y{3M|eUFPMd%>*z0Jq4hc93!ntb8`QSa)`Kx=`$~9Y3o)leW?uC}WD@F~jhwwm2 z+X&|oxt%`V*{THJ_VD7kMd=t(u0-icw43*I7VZB8iGz?~1i|$#D!KNr9m47=dLz2_ z{3$rY1B69l*FJb)dokv1ph;n#hLOQ_Fr>PZ;slRzvomMtU;1GNN5`wMz}mSHW?VOz zfHl8{mGDM~pVn}=96M*F$o8Cm?Byd36VNHFsli%CSNs*Sv2Q|n=zMT5$LJLt|Ch82 zG#`HHb&#>u{-CvwmT>85+w+(&NYQP3WWjHI$)&UEKe5PT^!%Hd6QqKsp%-ff7VuPm zh8J8_uKP9IqkF-Gh;wxqn>4z1xk*(=mmSd)aE&_q;clc`PW}hc&t@Od4M-ix0)Q%* zLf)6pm$(eoU$y*IrlCCUtb5IHnD&&_Lgzysuw&-OYEM_l$7P6`YaA=raoxDUd$ELICegdY1`VISS=ui|$eEebV= zMJD4F7$ZkuDOF>g4iN+ivX(jI>%*HxmPMMk1!<&N{30er_?U+2MI6D&a)Cp1uj?Zb zUdxx}{F&TAkzW#2l7zkA=&-A^=PR$w)>jVg?x`&g-XE~|=`j@OV3z-mDC5_f1lE4e zc{Eqp`>QH{-qD>rhnZaI0sc;wh>vmW*H%+F=l3l)PiTkb;n0ppMa^VMbEwyE{P+zc z57Z;6N`y|VW1Kv^`B(F7=~O2%>HO8xdZ0@Uk!=%aun1OKuS08f4b*gzV#l$$sX2;) z8rWiovYGHYf3=*%e-o+a3*uZ8IeVSn1a~uEE@CIhze6juY(W`!zk|rt&+IP4rJ4QA z?n%JToK;c$Qgma1*^w5LUK*X*>$t5dnIU2RXR0I!(u1DyzO0!MTDzs`i_QI zc2^HVo!j9YA|oiKf_I#W{R~&%^WitN7K(?9Z#f+?P8VJI8*b>&WqV%M(fAIWH?OKE zZzJ4jlt2|}C0$zAD;!KeHs}{`segF5&JwQ-b2#6(bRc6W8v89mU-fpqUc#1hsVG8H;Xko{k#p>gUglD)9Zw|X@u8agXk#8Y6OU93GW z$;YD48DF%Vz%+*26C}W4R{^U8H7E&&@ox#_BTRR{tY*`U&UYd`khif z34j)UnCsUzThP_);U)61Kt_=$`t)d^uzGYO;_O)u_6XKAi^X58UoJe?H7Wfo{jma+ zZv5^|BnjmjboJWPWP_V$wp`X}-|5;lmGdelE0b3pKdVQ|aAX&$elOp8EU8)Z_bK@H zqY=KuHrlY^8rg^TD}1??k6(0R4&Sz3PTnZdF_}_Zyy-2qN)!jX2PM?T?2H7oPjE7n zGq7a(jfED`Bb@EXoS=?9=v-C)94nv0v_uav^JA<`qSA6CMx!z|JV2y2AVFeY2K4>Y z$k$V=KW}}i>E)Vk(Ayf_4(Z)|p{%p`d|mXSkds0P#@P4K^GBCb$Qm5Uv$EuKpA`)j zn?&5Aj+nNSI4@tWd>*s#Bxy9<-0uJ3>#L)p?AE?#NC5%qZV;44=^7eDkQgO&NKqOD zh7=gOK|lcs8OdSj?vPYadgv0QTe`p7=RD{Ao^#f_-oIF^#e(79`@Z&d{c0b`IYc<{ z)`KOO=PDL0`|#1{GCm;J`6t#>TJ^$2l+uPRwl0UKTn zMq|wXnt21{2ydYr4-4=>rGqV)jnz<`Xmd9=j0h*9SAdn_kp8YyG{RO}2sAC4xe*Ri zZ+YICHswnm@-1XLoUyVfr;M~q>vL&u+Jl7X@SS^yjwrVMo zSJe5@A@C{xvajQwNzX{|GfRY^Ecg+Y)V2F!1h>0(_{75q+rAjXUjwv*k?Pj$SpmhM z0NTsJ`0No7hCao%^p_3;F=CqCdD3Gw!^8bs8_1kH!%c7Vijast7X0uEU3`Hlsp9q|#wQMQg z-K)p*wB&0F=@}hoe_VpDwMyR~!;hj9hLDdHQj~}_YYdKl3aPnX&?0LicXiem(Oh9R zY0M1ecGJ(iW4e$MRt!St*-hlOhD@({1xNkE3=pV!rRDG;C(Pl~h+Y^UH4>Ld zw|o~d)xdu&mKLxf*Ze<2Ns2h`^@-Blc}KtoNXp z*PtcAoN4G8j&@GX2wCT8;vz?JAFy3`+_NU{3Ad_($d2z=D2Wj3{3!RB0B$Jn67(y# zQ{!Xfpdcf4@vOp12p|;ReNh|-G4E!XD}~JERk$qo-X3@r**50vL%gRqEgE3kkG*Ss zPPE;A9Xh@~9mz*sMp`=CeHjI3`W>pcB?|jE;bq88o|<*_#ws8f-vAVc5N7TB1I=B& z@hao4NIduncSo>0U^g6rG11_XLeKfUg+t(C zbydtAM5hJYp2ZeB(aX_3EF%gY@^^~zoDI)?io&*sSqq<3d?~-^p78!rA;-Q&cT$cE zv~eF3y`*tiF3Sm%BxXEP3=EYyhU^FzD;?~0pI^|KI)B0KUwuxqKPC^&Gc*w3t5%4V3^zhQ($~?(ZUV7Z-%7w%!Qku_0A=Et@r4)zv^v^c`EWDO7V{ z6S|FOf98$v|4gh-(kyGb2Pd$*^J5wbuY(zB>e`sRF1LCkNwi6djfuD38O&4i(K@Ka z8&LRF9ff-4&K3ia#>!Um->|jlMl*fAxLNjf6;{%)XnoHu;un9as;75MVhJe}-WUuN z0eP)0|D;({Bo|?S(xvp?zXKnyWriY#5P^68twyGkJcE$33lDMSS$z(u1&dyE2qUSe z2<<^Q`;dX^&mc~ZS-3$hg4N0xNq@^5*f90@o=EJ{ZEPN%Ho`VKIkeUl`__VIiHjC0 z5uewr70=)NWtJa*S_pYo4Oav^2D%=OL)#{d}BLpl%aj~W%BWr!)jdY(r=w} z{juo+^z{sdSkGAIW~(U5zKMoPpA6Dr-Dwb<;>deZHe1 zZsV#SUrU?KZ86odD{H)y2_V3OLx(T0OF~S9mOceFQ}{A%io&|j z0oY~Ksl2%m5hT4uhf`vdUJ>pNA`qD&rV z_fOgWs}sI+e@~u83aup4RrLfeOE3KiP)c6t#=YOct&xlXAePCym$l@&+j1}bkWw`X z^RF%fpzuvns%EALR4j(BFni@`Ms%$E=U%TWPgmvYnstzjK~tWR&MYk(OsmPTnq0$E z$IgEP3WZ;vbYY>rP&iqj_S50VInQXDh4HHBvtykTu-C|`I+8ESIO)_9Vi`jB@S55)A9d{%`UC9n4# zu$KHXJ-C0p6=Hi%q!V5~O$BweZ$q5|Jy0ls_>LgreS4btK+gS#4o3E7=8sm0s{mG7 zZWVF-u)9h7uIkkbr4qUWf~yyVpo{1S{XEomSxioW@*A}hdmmpbR1^qZ8%=zE$nUAn zG~qvZBAHp1Q{?yNP zW^5)dSUQ44!G~lk>a0>+%S`g!9=N_4Mnd4W5%JtI1Rs%IQvpp2)tyc5^Wx$?*KSfL zjgXN2nCj%W_%EfYkLWC;Md_s9o)8}B?|lr|Sbk`+{X**xuxIE^D4{d^?r}L*mcAV3 z1CL#Xj&O8rtk-j&vSJ1Jn#1;IH@*V)aZ3>Z%6mPE$H`0uE5H8gA#lKv$K%}FYm09@TwvKg?r?(;&+zjK7;pi$kZ^mE!>GPasg1Zut`2H z$3?UkHND}jA&8-!2D#C>#Gb#g~bS#QPQoKS!-t;O4*nUh~&s1L7xUOeZ5cNK+SL=f_ zE;V9ZrHG$~zl!CzFHg~AiTlw5?5SH9ybefyzLg>IBag3gpH_b3P2e%u>wH|&N2aJz*mhQ^6b8_E8S+f?d6AaF~OE-dAIUvzdn#w)P1{XVDFz4i#A z=x2L%RdfZ4w*E!5NO!8P&+cz-w`yJ6ah5MK25eNs_vSZ{0m$(^y>SVov5(mwV5b6%tj2=Uay!1bIWm1Eypy~w{81*n zmn`Kg%_Wplh?E22z{(_#tq49;t~v~aBNnkbKRml^pJN$Sv|g?lBgZJ!NV_ zvz809rcI`trAbyYxNED&lN;?tZ&s&axx%>45DlMFc<(`|&ey%!^@5!!;Ba0nEu+gb zw(s+gCMz7i*YUAuQD3zk{6}-ZL^MrcQV+whe`^8SuP6n#1rFvFZEUY9p$!AR4qLl~ zhh*n8t|Ct4@aEMg2aQ-nb-$Ss>I)qL1jb&Uk#HT09p2RJ3fuF}tI?VA;Z6Gbe}086 z9s4)l%13#cd2{tn(~wp(Z-PKhO}5b0FkNZ2J2+CoMcIFMfTNs>l{Ar#x>YRz&UxN| zg_bQUQHA!+haxs?nXcb$4`EzpiRA5O$s#+WKgc(jvNK~UaWP#(yx&NDJ8-wo{Fjgo zx2_!@MW>ACa^w`6Nr5oEc2m=#!QV&i(2@bVMt2ELTn5hb0lyIR~Ue5ncR==JNo`O>3ffZTnFGzPt)8`{3ns|MSI zjnJD3-a-kG)f}e^GU8(! z?49OI!&7y&Vwql^HOJ|0iu)TpTl5GLJ{dIG1Vx~&n(NcinoDM_thFJh?`S1%m&50| zS(PL3Q`lns*s#L=%w&Hz`2v_`+;$f!nq#mrOdbxdx4c-pwSJeE6n}uljnkA*zBlvf zFL>0l+E1HV>Nnf%^;g1YLy)5v194*#`ewTK5OLgX^oW>J;UP@o>1zRWVoi7Izk%d` zE(RYRKy&tIC*>#V2-hy}$+7hwXEWa;#woqfw=0Yf$5jVq@RDH&p>`K>b>c_lSeV@6 z&^X7uhbqu8vaugxtmGfJ>|)Ly_3ck_pnGsJN1c((gz~ggp~^)vZk+HR`BJsaZ{954 zUzEC>WXfNv!9yp1XSC(E_%~+xH+1>WFG7J>qB^s-;0o2eM2;Hor(w=g#{RP-^fEB5 zcG`*HTfrvkI!uV&%W;wRdTY%_wd(oW8G*~@(;V^;ej1rtZVeTv;9X$jS~tnqWb3}`R9lEL(XU_fWa84 z&lXOnjMYTnWV2j7(*mcL-?*m5=HROHrAj>xNBKKFX+tSg#&!xgJIB8jF15Rd*bwu# z?H8ujgSv62)Z*Vmpv`h}=P2pRmE8n=4U3P)k(3Trh`BiP? z&ljRG+W+N$+g$(NV|7@Ct!-dhZ+`nx3kQC!6H?tQsA zo1d}h2wG)NmHG%3#>X)6DW)&E^?Ed*Ou3}LXBWt}@i!OyrcJfEQblj#geP2p>fVIFv#d^V{@i!THqViqWqhK+yv8qr5 zPzBMrKz5BNYA4mRoa^(VEq-;HZxJlGR=ieBr}-=Q6T6mwmHhuKJ^!vio`jqVaEfpf zGRSsp%Lp^`ss>e*?*?*8X+r~2-O;7u@|7MbVB^A>FtsZ!2FwQrE<0#7xYe{v_M(NL zx-#6AT}3UHh&Lvk!+=7J+N}QZyrCAFRs)FwRmcb80g)}0QYFXfvCsVmIhiK;-0!w* zKh4~5k90Oz*G{)`ctOK z8b_Lnc1u(IbfvW|=Zd9vyCU6r0US*AcFtUu--*&_cQ&=rY+c+aMOeC zq~2zqJ|6WfW=It;|6lLQUmO)9z4(Q`rt%?uJuDo*g3E18zDEJhG3Z!9xG%lZqfgDK z@Vo1N8A%pk^Y|(!=>Y)Z0k+&vnT!pxKfB7!NM|(=KsT*7@{}XlP-D8o^iXL;QFgih zrxA0vDo%5^o0{j|Re>9inDEd&>77La|Hp0&=qiO!LTkOB^6JcX0T#D+w765Bal?OC z;YpP`UyqUp6u(7RzlQHnYzL(jMv&4%;MO>;lyyXd#PLxA<9BK^NP~$)dSsk2uu;Z??4RMci@XhdJTfn|~ z^^KC|=ArHDzc}9ibps;q2|)u$uTO5M#~7_TD;}X!erG=0le8gY*SG`BoC`L^#ftO_ zaP6e7XPh=yY||k(&yem+&KExl%Z!k<^TzgCzX)%!CO1qez&&qY`E)f)IjQ-_ZAju{ zs=wN1Ea-skNWT}3vDiRhfg4z?YDo8bf7vs0Mn`qP)vUebCj*{Y^y+(;lwiBGU(?J`1Kop@@WVhe=`p2 zA(2_hm*qNNbuPLrh9~#gjWF>w*Vc0;J}SvQgxQ}8AGjiCN`b$b@JjNgqU ziGn4^@pFX_qEIHr2L%~1|JS=4Ob9{;3^TJw;a71=+3#icZ%bqijfObEKdS4pe_Hx9 z7Le@58~~foCxB|=xXEwE-PI{Juh#BeyEfUO<$QCP>8_DS+rHPke|VHW9=`DgQvJKx zF$P6Ke*bRmc$=Y_7Z%h*t7&3UKLZCdbd|)6@|zm@lvE736sv940`Qr?I<{UN6XfT2 z)lYMsDK0WBXZJa-%-j}hJm{CM^x18$b)dPJp)_y1I&P_ z?)AyMzyI;Y{EgpG^*p@{6YeS2A77y|@6AZ&_4`@hli3l;JuFP`Oh4ld;71g=*fhOq zfT&e>{P+13r4&Gq()Q9eNVr?#>^x>}zsv@^k;ESa$}JXJ$=}wBioMJrc8R0uX`t#8 zI5ttN$Z+F?{EW7rV3X)#{DjL9jsyd$WWtFx4q_ScxSxwi25l)f4%`-_E(=E7)st*^ zavM4hRBEWlet|$Nl}3|c?Y{G)mxZ%6aE5q8cGc$XCU2J^ASo-QvLkT2Zf7G z``=tWFoPR9Hg4B!Rc2nE^(COdoqsiFj~G&AaDWyGTrbkJe%%4AoIn~gVCMSq z>FoRKR|B5i4=VEIjtU$v<}X*o9gAi>=6y5zZK9>txCgw)Dql!{I_>0^n&920Mf1Z; zTd&V|UPuV81eW(P%%7@`?_^$Wy`DT<&6IaPq8L}p^lYd2^zLdu&Of~_b|UsWah%W0 zQbjA^U!05)pOdhi)dU#CZeUq<*6zXJn5UB;3``^|N0_F>X-hA*D&y>KFb1X>us2;) z52Ta5B=!K^$AtYXd>pP-|1*@K!vAuKMc-)*aMq>_^%#P#&j$SEj+Q3D|Nj{MbJ(BX zV?|CWtWwDZHqZfJi`4wH=fR`fitNY+pzodtrPQFsF3Z5eSaG`3+h^;^o@HIwJ*00S zLY0h3sxf0!vO35T?v1_8m9)c+o-N&wz(Q-9F)6F?PLoa=jOSlv*%45Q0_A@bjZvNH z&9=q-q;(~!=NdA%ksbar%z=fTMv7(LT(mZ~&WFR0#q)ci;xCn@P5iHpCJg{7z*x3p z3Z>>^p**CYt%XnPB z{MWnfS-hZP`M3ODbhcN*l+bau2gFfoiWg5F0T(VH8?5sqdt{8rO4Te zn+xpZFe=h6IvHg*dwwHwSyBY~K)1%sBrq*NrNg9cbFtG{8YKaXMM{k-21P8BpiK|6 zXso1nn<^960J&(YBVE%Me^I6(V{p=CfU$9F>&^$^o9v*lp|EZ}An$`8Ame z#L0jK0j%8>K`_2R&~t0-8j(X$cCq?5zrMju6~mGr@&mkap|9p ziNqn#e>nM7MtK9TvA@iH)SN{We+7T-jEK+P7kt>fQ`A&uB1om{l4lHsRkYe?g1 zXLaoMR!p&!o1!vqK1KZCXc``-H?9FRt6-sQ5P;ueCA-dJKdG%|9)+vMPwzvnGrOpz z#gUDas?V$yj$d=Y#%THnis`^imb|brO@X8nRtg~@RF5c7)DvIb=Wni0DxKagF?4+c zm!3>JwMJX(unLncm5Zn{Aq4h?|Kh88lin#6F6xdgLMQp@l4 zXU-mH^8peV#aj!RAtB}9(}yK}GGp~n$dQb#bf4G{EU0Ts<5q>{aXU@R;Bqu8Iu%!y z_n(g`1?%850vjXAbF9$I`+~=57)zP!(cw@pQ!|gk8kodU5sg&j^N#(|xF+s%&CMyn5pt_U2dzd`F@8xu_fQ2+|urN;EBWWU+ z)%$jG+f^ek+=>L~6*J*uMw5T67RG`M#`r@BWp_Qjp?<82a(2m`x+U+k7Obg)o1r+r z!_nsKWFpj96#ik_pPT@qhZlSq9RL9!HLjQWtq3CO^o7yoD_hAu)0^}0O1qN!S2l$9 zudV(uZ?>^RS#dDTP44?>0ScO$3uA*?tqun!HW=0dFg8}z5}~Ne7(uVVV`rJfP)&L5 zY4REMnNW^|LD%b1bljg>GeYZ@f;o5um%Rx&c4FNxWR4IS)6Z`uy zr)m2uHau4SZCWBIMU2L1si8EW%x2n=xjr9nt(o&i#GZ!AFW-CeP2EH#xM*$C%AW*; zAP;Q1lsES( zjM*Dic_{(t!z+Qz1FvEv{dRfP=bQxOkW)bE`#+BKNrRT(AA0Mfb;Ck`p6c`*dVkY2 zNof8f{_5#|{gG#`WEo&q-_&%FNZ0)skXX9?`rdG^bMgM`*G)qPYov$enb(K;lUFD6 zEXC|Nm=~noOup%Fdg{0*jnb0O-t9+P5mAHmYUcfX##X}aGgbH*ip|NNbd@x-I`B7=cJIbFnKut!25B>3HB&nb5TOGbP9 z>I;ezc~_j({)yG8VValyYZP5ll+Y2u^~~=9hKGYfyYea*AL8O1jQVC^#e7s3HM{cXh0s$L@-MQ&3;Q3!=G$k zpbG8bgSd$(^_zl0Ga^v}U=P{zdJCGY1=qxrs@mVy8g+(M zngO;6_x#LkIQ4Jm=W58Jy*#^nK5#mgV?pZxZ{19gmHyA4bQ9f5&P96-g1S)gIZw-N z%PhZg?K^kqr&%Oihg4|bzdtfzL$SBiC2%i?aD@;@N)%g$z$p&c=47#hvp65R8v`&A;tCg|@_Jml@%W z37{urI>f?Zas4h#LW@;e*S(tVs@+`;qNJ$TTDKJaa*Dlxoe?3@v(!)wdZ^KKT9Mq$ zHzjX94;ZGf^K}g$qOJjRV^Vx46{>(pD6#x5@?nwXnx`)i*a6$;agv^ISzwun`hnOH zcjz%IN91Ie3-`5vR1(~~sl!6Qe-?3~fSm`@G_@(%-kX?l=1(EYDYO21+3}p9fzj{u zr=zJ%3b?)0ciYa+m8LACd<;|Qqo)%S`hE4gX?eRMAwiHOk%{FI@yddbB&WlA;^Q}8 z;md}9m|G#$M+TKB1Dbdk;E#F!U@-MXQr}dii=HJ@}A#7Cs&;$t^7(W zmXG`u**k80p#`vzgfC+~p^e#8@smd{BU?lZa-y$C?jH&`$SLxY)fauH3NMwDcdw7X zC~zabCvSRC(zxr}=46ofm~Tt8Q6{mJ|MePC zNu7H?)PZs8JVtDUJAAbr-yweo%nAL@jk_&AKtbB9m!AWM6oCX%6rXO=WyZWbFxhZ} z@y$}goC9P)%IvF%3Xp8p2I$Q=%maqUNJKP9tV4ohVY+b7fz3ABHv*Ck>Wh%jQi*!h ztOVE#>Xk*X^+3=z_KjaJc(e$3#b@30@tb(INOcz9h9j}>favH*P0xpu6Ih*riNf_x z0+vudN&iQba)$EWT6UcKJ17n07Q~Pc^ubPpjsC&RC-3TA*X_lZ9~(=Ye5R)f>%MyZ zv9x4t^M@bLSjNfD9CfIrZUVBvL>zLSNAa#yU$Mfu6w}##X~R$A1?k@%(Y2P}{;B!u7stT#Sv$cGPU7LS;R8>&x zi2R?NqZ1EU_Xfj@2VRN6=T_SGLG2CvN+rBUw@|O)`xQu5SJ zt%7Pg88w$YvgpP@Tc&#%$JNI*%u^jxSB%&ghfiaYsHF21D(Hb3>22_PkSCxhXGeqE zl=Jlv^sAZvez zh9Ik4r1fbMd9}vk%RZz}55A$bd@r?FQSJQ|epDjn|B11UjnFI&?A5w0J8G?rYoFVR8UAJ@>zZ=riOkNN-s|8{l; zA3d2cgYiNc&lj<(?5HREhQ3QcH(x6K%0NfAuKs5W+d&0_`dp`BZvY)y zMAZ%+T8VTW9is+9!WeL_ZLMcW&?6=ow01$GMCiq5@`Ch8)NvyY{OQIJ{AqT%mFZRK zi~4Wu=mX0x2>fGF*)1ahi{gALU0A1!!Q0%cd(ZeGXx;uZYFQ3+8b?{Rfg7DJY*tMT zdV+^x%F2OvRz+owz4M6&>_Ym0x{Y4_2ezcHgW=tWDn!49qEwVmr|R&YH}(5o9gTB8 z3Hd_9KmqHA^_LRiFcP8z2$d6U7c>~a2z+Kwq@4gn4hOsNS#j@aTteVWsYR(*HlQjT z6TOEM+gZByX~GcEBdq#gPp_db-)@~MC%g9lEL?{{juh~je#yY3|CpJ`enfL*IZa3INUN^Jz{e5Uu~d)_7-=rHZKzQCE(cp9LWil8Lx1d$L%Y=jq$sVp!e z@!!c7ukm@U;j{mNah%@U@u0NVKZEu7>RJ3+m9+r1;c1+L7l>7iepKa`q)A%?=v6== zC5Q@PsZIxId+;E#<22F<0wLQ0J8RBUHu+EAHa8+{1Dup|6Fs8>yKd`#q=bmJ8D|xp zJwGM98kG997tAd|Y7^;K2=XLe`y~ooxhON@TjIbN4sv?_e$uYAIh`vgHqT-BDMIFm zJm$&6m-9e9kR#-brV41DaCp1&AamX4R;_R5z7%!z)}}u%S#MRcwawhxntpeRQngz3 zF(nNH-5fKSyl1mDxxiO-`6piEt|n8jR-#NVE!+=&h>L%6G}Vc5z$e@IP6ok17gZf( z733X%U$7K99fC2IFAQYyP9@==EV4UNuGiG$jT`+fz#&wWAzpb;hCAhay*}4sPPo3w zmkm0UE&Y$3<*zsVJ>e*%I;zA(H=;sYaZ?hQ%cvPp)Y#S@xg9uR-k{*UEPC*EEnh7) z@95gUwEz*GkBL~185%5eH*ayEefv1?S*EGMX>?#(wrc||SZH3eMqpPQRmZC6iYsdv zWTAL3R*=S5KOoh+mh^3+U|kvAa$jVyuFLI9kLFESe+4HV>_YnzLE9$LQh$j*`XSIE z+*6qaKB^n@loS`ERT|a`biN+jShnC=H!{PVY1XOBQrR)?74qg`ZZIdL+ za#(oaUL>pl&)g&k?`Z)UGDY3p!b0Qft#d{sypONBg`#e?z7FLUb|=sa9aE6+91hAV zzW2-6PKTd}N;w=WBQ-L%-^qtUVS|i;g54$pm5~D|xqmxRUxd~ur&-2~mz(7%-_9l_ zVsMChb?)#E;MU7s6BLGgqa~pbxPk`d+@iy~Ah!t4j!rd-zcO7nEdYU&+h4fyG#4AZzS$4(Pn%uKT#Ua^g>It6nv|V@ z*~NO(jq=T{Y%mGrZf|YtmBYzAXNct;qtKIL&X@M+uwXB(JGMN)6V>LTryp1utYqlJ zX^OsQ9g$bS4&laS-2XKPJSxH`)*C-RIFV~DzIo&#x-CDQgDHYkBdl2;BoUgsS<&e5 zFBqr|IRV(G09kg;^@QKWb|nA_PkaL*M_U$@HV4O_VpH*%A%2`YDVt!|Hz0n^x=;U5 zsl&>%c1;_XSiK8L#>vbCp^WoAVYxx`%jhMBmQQr<30dgJ=YdX;_J&W_`Z!S(hDHnmDQC!quS{6&zFLllf+ zRrO~~T8Y!fcXYG+i+|E-NflT(XJ2jvt_rxwC$FYILErc(RM!yO9H3SKrl728#*0F`|!3O5=cuBZlm)J4SuT}9EMx=SWlTyhlSLR(Cm3+&DlG<*jj$( zS_tt1o=V$&c}t?~&!@ET@E>6{?PhENavIS4S0U`a*a$Zutcz2SI6^I8YUWTQ^j6N6 z-b+T?(k+3Dbq5@%IA#doC4p6JQ`Zn7D8oL@wdby){^P;dRLqwhyjYV2mJ%sdMV?9t z757RWnFrOphq}D*z4nDvn+*Pn3A{UR<9mBWc%=_Ym+hqE!J;GE4PQ>N46xm3&kJ4K zd@XnI@dNH-Iqcvi1D5k2C+mHc6G<1?PKA3bZG+)wMpg5%Hva^>^zwu)}P6&>VI zwd`Ei@E`6fE}e@`_l2`ZabLr9z14_tP85c(4R=z!+S|LsH|~~h%-~NOaQA2i*_K$Y zYi#@JxAw5P@?|xc#EoJ|&{r+X?&l}HrfCZB_$kgzyrepl)b*l>_ zxZyg{Z;S;7fp`7R=0=rFhA1}`iAo6cmkOdBEAa2h=_93X(gJMfFFY4;6LeCG@DDdv zI3K-yeviICFF~bRYtB=O;^>i0Ah(pQJF|jlA|V+DlvbncaVnPhkSxwTD<+Nb2wbe; z9U=Zeu1U}0+g^LaD=_*Gb|tMNt1&eG%+%G3-^8SE{K{ds`J_KQD=y$hc;lVAjYA07 zI^VG#?*+HLbGWyI&j;AlsAxgP8qYhU>6k-|ClBfqAh0F`u6AxyMc56#?-U%3RDZ^9 zeD9uJ=w08Q6r~oDkIZq0>ozuel4k8|H7$@%v(4b5?F+;jjDJ{)B|tCB z7*_q73#gTaFcIA&bR{@R7?jG{y(}A?t~At$4S-gxggC}A4g3QE2v|k~IBoA@HV8vD zO-lN{(Ag^$2~7DkV=PPx@cLb#Dahc$-^8fD2G2SM5hBKp+00D=e8^PO#1=W_pTw=7 z?xe$XP*(!lO@Edzg9dtv%%(v#vE&6KWPM~iGd8qviINOMtUIm}Tpcv;b&({2qBb1= zC0mFecs%D&vAv)`drLV&#s5*&J$K4KleYK3W}0JzO$}hjYq*M)JGVY*W8zKlj2+m! zafqB4L9)G7k;6c;8a&|{A{xq~7x3y=Z9(+AyFj?Bs5AiVc-!Emx8;A-D2iIK474?q zY_KC;GhFJy!b~v-hbvHnc$A-7!W24X%~IsP21dqKB)J1t59K!fe2{CbnsiCyj#Mox zpeI}B+0Hdb|k{>mB~(LM9Vuw&waI2GmY z7c_l|!X>;t8F;9%^D?0zD)Ot8tgG+bsf@46DC%hq7d-!PNh^B&U;gXZYv;WaXp3Lm z!5Kk3G>nVV4&0W}^9~H(phO99v=p}0F*EVU|NP?4?#i{!?$5`8o^`@yq@Ipp^u8b^ zircs}sBRq<<;$8HPe@)SLi=L1i$qF0UJc)X-h2^g!kfvp6(fzOp~2vkn^Ka=2Uv>UiU@tZO--q>hQ6Ab*SaY z=84RAEWS>xYyxXn`|XS?Ad_`c<+S>PFr@mAVF;3FP|=~NsBXr_6|LiBm;F|NY!ZgJ zqdNi8Ce#=M@b}n~umXGTPraArKZ4RC7$-)c;p%rkGU2kAG}Ydd(x4S$30Lqfu(Xx` zZKyX0zH=hyAjv01Jl`mNm0U@;#Q-_@+aY-nXR-XRGuBIkpruIS@Cv4VUu!%-Vd$Ub z>GsQp)Hm^!)m8j)9SO41>E$}{w%;}g&3loNiS*V&f0{Ec9T4hkKe@6`oh*RF5S8WH|m|mOf~GGz8%)FXlFx0wCLL zbhv)+pJ=JAVNF4;{&nTV&^;_`{HAP>VU=1rl#}4_*U6~+Z^`&qAkORU#jajIpx2Oz z+84;nA#bqpp9esrLh6j~?F#x;65}{)Skb$`BV|GF?9^=@rB1qWtTd3}3h+i$#f8t& zhVObS?It{qZ#bTDeX=^Dl_QIkeawDus9y`=Xw3xCDSEs&hj-`?j2;D?@-Nv!)Sn_x zGd$)#l4UUPZjoMEe#zy%9654kDq}OhF_dn1utYllwfY~_-vGpy+ptFU#Vh^H1TQDO zO_0$cxWr{v@8IrutV4Ubq#Fqn12mheWxx=*);_@n<~abbhWBd`*i3q4`X0YYQi_(e z=dzF#I}}byTr-VM7Q5g$j+!NHx0B6rT5I5(N^@Pko^TgXD&_3~YGUmy=^?`tSk|w` z{sE%;*_<(1qDfWhq^laZ{QPKLO&leyVdIWXT;Vo;naznVD6Fcb(|;^dBSz#KWv>V^vy zKZp7(?0lE!%C#EJBPZfiair*~z3?oj=~&bId$+m1AgJm6aSyDjbGhhmvz zF^_G2Q`5+X*I|obPaz)u3IC2d-cH?$_C^K$a=g?u9|{cLmxBk7=kucOMoie5UKrO( zk1+%2Tg&EnI_Xf?sFKYi5(!l%3&AhtKti`FjjyJ>)opNfcz^E0$!;OMy6Z#PHq%?P zjc?C}8b0HX>kA4AtFEzZmbUt-zVt8*4cxGaHYnj(N)HspzJYr^2)e#?`ua1nFH7sO zX~vx-KP@-;CG8db?rng2SmmQ0d{9M2Cl#iR4SjIp%zUU%xR4^(t4N<)sL~0@M&54& zRm_oPg(JbQ7p{;=yGA`=eypGt3kMnm>fsUUxxfXj3~t$Vjh*7aJQ17uTbrX%;e6!a z-)a3;Yw+b$;7!?Rd!8KCznXB@kahPs#9a?&P0qk>p`MyQ^(g?8ZE zl>I*oiG?03A=gElKu@Boh6gh#S`C`e+J&Dgc47vL0lr% zU-XN+0qvJ#j8xpH6l*jgL8VJC*!!O4@ z45bi=&uQFgu;UtHi627fGFJqwy?>9-rTZ02H{7`M^#@CA5d;R}1`;Flqs*!Sb8F!a zLU7pb@bW=6i3IA9zgGdkiVDjdREU5QBI#Ja)q`w!)&f7JzcvJ6tk=VS*(N*3AK>dz z1n}Ud;cT%?v13n~ToCqMyYNR?$O6=_@$TnQwlQixb{SayXJl{*105QA=e!>XQ#?*znvBq+8_r;2ZP9ZBBre`5^xdttdE zK-bs9>|;NAAD2`A^F1xfc5K?+OK`zKQL z7Npp2@7rs>KZrJRAm}{(vhlg#@sxPGlw|VN@w6ZZowZ>~cAVr&%G7+=3xoR%*wD@3 z%wfTgt79fU+ZI2QEW7OE8V;|#CRLwXl;8UL!uEecr~Z{$_LCBY*kJ7UK5b%GPp8Em zCeUGOx7SwNlJXAP0KJVm3M;8RoxIYiJOEgMNwM-L*Q=bUV>O*5jv?P>qOZcnpzyAUSz5t zK^xW<_`ciNVBSmfra7%&j2CO|LBc~ZoX_pCvlxj;q3jEY7IQZG+ii3QJGhP@1u!9orV5Wo*xxCbK~4EKYy(i)!w8B@5AIoUtj-N&6E zDdTo&)=Uc>1B3G9$%>0EC|F}vQU)Cr>~fmt03tei>v-CUg_F&l$Fp(Rv>odu7ue)` zn(uAEd%NG8{$*w#u&j;8ump6cKmY#vlU)EV$+Y&O1Cwoc-JccO8O{HaZ z!TkF6VD=_4-wRb+vu_Mip8cC%oVe(M*Tb8=Eq{jk?m9uNi4MPS!nB)*ni>zKt@S?c z_gfsaW8t0T$X&U0Yt?q7{X?Q(wg!OSVeuoe5!7;nUN7G~bdkcxXpnBUaY)s!@?@hj zD&d;SN6tFi>@8epYzB|H6mN5&58{AKFCfR`j)|%wM~X6}(O)eRwS$Akmbcq=Dwah* zRj$^KP4O|+RP0bd0wSf@b3KEFnp2|-v0mY1DO3_2c^fi20!-vU>Zj~Ou0b5!@eHB9 zAO>MdiwDyy?*q#%bGq)XQKaGnb|NR9fVT>G;Pz`V9|oixA>61cSMuK)I>0e9InMauQWWJ;Fc9v5 zGkJTgE-ekndLr@(Y%g7y;LY zolNC#6hD8|F%2ixv<#^856Rk(Lv2B2VW64`usVz$V39@ z2JCc1R7`u`h1*gYGqqdJzEN5;2pjR+%z^h%7R%=mqB$VXmF!r)ZkxN<%E+xjFcCm{7d%dJXU=Q0Fn~42j7b@|Kw4=*J81uLa}L{(TT1VaUfk;0qYTw_4b$YQ=@T6^_(Nn zgAjkteQm84yyukb75;m?|K_W_-5lJb^BjmtBdgxZHO%lrGJ*%``Bvq_kPY;2GSuuf ztK4p(6P>3aM{l2mj7jO6dGR2)NXQzLyOy}@tWvO)2`sUd*vuCUhekV$MW>sIp##tS zrOzb@?>yzdUg6y3w4M2{f$ETI4Hs=m%|f69d(vr*l)}v;hVl}T#(nUm^e9w=2OX*L zlUWi%dCOi*s;NMG5;<1L#_-Vj@|u%me+%=dW3cehG(U1HLjT+I;+o&KhLl=2fEm9z z4R>%_`pN3B{QC9IPd*hl3+3h~U6$VQM*f3fMIi^IHx}^svr^PFSD7*NQsXr-%CeQu z-)0U!X(+pJrlW+VyE7NeR>Ta{t!)|*P$dI)F=>PW! z1#lHZz-X$6B*pR@LqoYCVV+QNhwQ6@fB^t58IfZ{k<=1Xa0Q6bHmy)i&kg#vg| zMi(CY2MxNeq>h_v*(`589oUXRST{hh;13Y&ZqUC6j1D=k^`s^L-l6$zb3VHnvVZRC z_%m5;0fuv-r~j#0{Oh~5A_xacTUE`j)CTuc;-@h$kq*X%JO~_|?mD#zt_*9@BE#UJ z5Bgw+`hwVT0Qa`-p!DcYNi7B`b@(ZEXXKsE8y?ik>CNiA{GeCaROmnV+JB3b^V;;D zF5Fy?<456yS#;E>*lU} zvOmS?bF77QykreRBD0;%k}KW;FhP5^O}72V}AXnmwK|FW{IR%_JwkTRjSXS@^ch()queEhvC?J6e%K0{)N^)bVABCeZAu!Kg9pX{L|&e zb-@DHYrS#z<_Cp3U(yEAGF*MI;G822R1V%psR?x>^gdFq-RVh9BjWH?3P^!r-k)Nd z+MxC)l_;GT77E|VED3C74lYROh7n$yyw${zRaZT^W2bM{+hYIgx%;m{NFsH}5W)K1 z6ueOf#7@^9850UVzb?cOdGhZz+uu^u)-sIH({X}*)MigOsF&qqeubuvcpg3k&!zWP zP1mm*4IH{(Va{T~Z4dtK_xS(#e7u-d9K0A1ivFkMnzk7e1}Le_e8Z z?D~<S~HV zp%banqn9;mS*UJtQ+rI6I zqpFIFNq+%tqIr7x{17AP>ow#{3D&SS3c~!TR5_Y5pj*Pa8W+B3EQ*bYf!UxVmOmjg z3HZFX%;=NF&*nbKoongdADn=&A6 zP)L4%xm|8SD5g=vKN3b zE}jt^)arwyKNe!*Sr%@E_>+?@J$bD@63D6>9fExm9_7jc1*c1@x70B1pJO_;8!yIL zliw}SO(AbFEaInV95Wz8s=~PjsQZ8a%CE9kN-vNQwnL$|uPM450?@q*_QoZ7Ln|^} z;B;0y?SNEB2;~y_X)Ma8oK#Lu>u+oAzdoK!s{_4kF~&=f?C|urV7RMs#*h&J_qUMX zukH)hP0x8eqeO9l^`@^q1@yoD2|)edLw#&P?n#3)Xqvr?5 zIZ|!$>u}>&K`$~ZOrDFK;WkoqC-ID?A5D!N*%J5-bOY)4Mg^io?Meoojmy#<5V@x> z`i^G5t2 z1OHWa)0X@}`!6*n@{a{TfD&Y;`z$$QYv5-X5;xk=vT#WzxX}U6JyQg+Zji&p#DT2u zX_j#LCQ%sS`D^2IMskMpI^SP4hUf@7y3y;rU*t5EnWfmhB(G@E`{WHoE<_06t{6vI zBlLvuwe4PFgsk-O24ej7Jc@8^>IRrNddNo9kP`3q70ZC zW$DVTr21a0-{*Cj>H3jumsEST%;Vm$*#*Ar8X=)^337rm(3M0AQhL-S&X+>S=`Yf* zOoC`vnE2Q@{OM7m$+R<+P#Ly>hgPUsa3NNjCMzaFP8-xpX7VHdEtX*CK_jA1yZ{}4 z5z(8oMK*~bzyk_-BYA@pBkK&WRM(LXFyndL?T|6$&??K%3B!LBRmXV+1`I%P5&h!N-GqU(g7eo&dwZf<%^=tAxoo+Lp7Nb7uBI{Y0sEEtm z`uZX6M}$C!!he(5uV6}?^yqG(qG=21TBiT!?zj)ukMdCv-P&^6w|ha7TSK2J$K8;q zd@y<4Zrx=uz-CPr+_npw2*!~$u3X8)@`TtP%+YTW=$7I@93YZTeYh?-`EIeyLfW?S z*-0-NI#rM@?GgYNC!JF(0pIR>nlXc4Y& zeg`$g$w8hIz{Q(KF2Nd5_hpnzEbCT4*As&n@m1sl&X$KVUb8?Mk^D0IcS|?F{SD8#y#L-8Y05EHoI!1H*`O1G|c%3*~ z-9;!+!U-sLxH3rGVA;71l;B>o_Lv-I_|WUqY5*4kiMiceeJyy;atUz%63BB`>s0|f zjR-Od`^`ce@HLdb1+Ast;oiQ7f`2<8ss@yxfeKi&U)5e_TEj&qiRSbj;G;|VTB`K( zWJ{m061RWZRa(>X)o92A_~vDGJ?S0d>(%|WjlIW}dh&Aa=h&xl5LsJr)j1WH7o?|M z@BU1(O%tYZ=axL|lbh>OJmrmr(Pn?}hUqQc2)&YqLBPlhzcpwrxMZ}@SY~)>>i_r? zk%9ra7wFN>7*wJ3B}ll5-MftpQ15SKZ*pWzu7kLQvOB_{mBPbNMsu|_=WfFVV zOo}7^o&j#hvZZ7h=@^m2yIxe$4-y@u!hX;|VTyP79%PBm>w^UE`NvlY{mwqBJXQR| zgf3~~GXK?vRr=+WGirY5~h#9B!Ft>U5sH3o7$&zF~rrv?k5a&n`Z?D_ z$RC~fYyb0Dh{nakiD;r2GiUCiK=qpM=|@>0Z5Dmc5OhDQW_du)CANjjcLRfaA^6*V zK2idsQHR+XlVIH%^6*%}$WIuFprtele~i1@an5gOzGM1+-m}wnM~k&KKc^~$;GJkW z_x8r`o_^skMbBuR44z_N+#j90x!iRJiikwz6p<5w`6)I`1pkc8HD%^h!Ug#S*J1*2 zJ@@DqA8zrmRq5=M=zi6GJvB)=yA~3EG!5aGUB4WB*(&PgaLZu&^wClI@cT~RA~_gy z1 zZhGW-VpG7P{_7Qvd=`;*-8-{Vp$MUlTFq9h(Jwb`?gIJvKA(l7eOo;p3fAMqJb_DX zS^YAXz2m8S9fbj~nzJ38{0~%6GUvDXkd%igKf}1IVEZN>P546HC^06~ zPj5ifVQTHRt-UkC2$`hxLdSqioRrYFB(TafFbIa!oksY-g1YU?)?RG<`km@F!J$X* z;HapTn@^j?16vWH0g69PqW{QpZf7`L z`+r3O9^MS2f!O;%@5_UmF>}4+K3Zlka-ZbUf)OpJR0z_upFlSdb4<40G`G2-?_#qc z9cgoGCq$B1yJgPPg=~)Mi8IqKFit`iUpA#-JqE>;m?!7evk+t}VDk6b*Fo$fuir zg*V?>7o}wOtW~HOY@pm;S9~i6KO7$ntUS^MyLx(VPx^eF$OoaT@XLYHQNzins+B?q z1|N6tn4r;&#|U46HjPTJxa>{@jch*0EMNaMzP*QRUO^G8`SeG=5{m^U9Q+RZ*g&V- zr5Tp6tD!`mJoSGnb1=?AT`5;f3a03>*U?`ZbQRs z=+98}Jrv0`VnX4|0TI~NN)^})nFMop30s+M(Hw&{I_?qcy zNV_QL@W#bBpus_XoXVYhO_kQ)t2bhp@r2>AhF7r4LZt5GYPUXa1gtTYw3Hv}O zS{IP!S63kt%CkRE9Ayi8C{bv*HO{L`t|I^%M)Y@^G1RR5@Zx`W-*J~l$Er|ZNF-x;J+X* zSPinL3wKp3(FMn)i;p#qQa=1{!01`(@hLBEO!pDXPE@!z`GcnfhxhSnYskLiorZz% zk*@1IYTs@v=`$Gh_$VB~T=D1OIZWbyq`nR_ifpM^t!i#~aQTx&zPMff3s;~UdMbwj z=p3!gv{&-q^7ZO=&l$FQ7B@)GbA27FdEBa>MkPm+?9F{B)*l}jUnY}$Z2DwX{>_tY zJlIM#)~qo56gDMWINplWk$K(9YG#y#2yRc%47G4R@Nt%}tKOrZt)(nDqHZM!^hV#S z>zaHGZewKJ;XA)4|6?x+Zu zR{52KUxb%3&uidt^Yz&S%=qboK!YHU*O%%P&wo(vR~Zsyhy-^FZbmtol4$f8`HRXu zI>>ks7&HVrQ+@c|yim=uff%I*xrT7xulMJQR@1zzu=VI*DC#DjK<2jq&GBp}pg$Rp z|K!VjlDVhMQ*g4;_z|D*=`bqJ8R(zr1hAVN4FkAxJN&khi7r7b_+H@sXCr9jV_4#K zH^&7QDfYQIuocw!Lj}>fY+sF~KR)1a|C%?Ey^OdayA1m!-I(xmW#wPoLT4|2y)?g4 zxHHe`k7dkp!c?UA7GDU4{ud0>XGp6)gkShcp1Mr zR6oFw>^v`W`$U|^*nm-{C`V&92S^ahBVB{;4$^NrPjztZ+4O9)hDRlLDI!g}-A{>j%OF*u9US2I^k1oFoEI@)Y!d7Hh62TDLbR?&v8 za6eP7=`-=^P-2%nCxe(qdfB`h?dx_sYMm+{TM*^6n7aG^=j%6$;}8e6Z{8^Re3wz* zPoMQFD0B?f2XBm49bSL9BnfQ!n22}*!y6^xs)$s3`O?-){oqrPhsSK)&^0- zZ2h9RmDEh|RKgPF3*E4)NL`Olnq$inh&2o7_vzba8<8Vecw*UY1Tkj6+>`42br6=bwsyoCJ?5hRh$;j9 z_6NgVVuwe(d&b?e`y4-0v_jw`dnhP(a^>6&T#kfdX8i+x@ZW)rXfPhXMHM29ecsvl zsVpZ$oC53^?q4IvWNBL;#9QPSIMGsVIFAPdt zaWoHfKdte%y5jRN#lRo!pu%;D7Vx7b=!%9?*pgN>B7aG{cA_aTg_~(6nd!&vU4hN& zTX{t}A2#YO@|xWm_@LE~zQ97b56IvLYYn7BYta)PbI3ELhi{@_a( zqRYQG(;8mRs96CHo%R%k4N@ip#lP}C>v!?(L=0z33q&Wuc80IYb>F9lT=*daMxcwA zj~=Qy&y-aCy^sVfLs2r4aIE*!PF_;s6C;Rk*R4DuL3JY%?XSg(OQ#j4E@Yw5Fm_AZ zb4H3W`G`Z6Yok*g^K-^}inUi8hxDY!OZBXo#jdsHkCiiMZ`9}uawplG^pP4u4#kkN zL1Wpds9ys;TApV_+RuGqr!NNjOs~Jsxe&jqjRtE_-HGj3qZz=PIvVwrMZtJhE8Uh+lIR*gX?>c>~##0SPFxq13B5Pj-?;1HfQ1F>1{M)J` ziWv*eN$?dz0;~g2w!FLf+L-TsHgS|WqGZYbhK11m^n~SZN>rMLEv9d7gI^~z;lkEW zSL>>k3tbNr8zW_WLM_4xe^sT}@#WH8AX_5rt#OKK!60YJVJc8-BX@lQ)EfiXom_4j zc@(myi;A+|3?`J*KfkjNmz z63`#nwr=BJ7r0Zp_Lw!vWs6!fZ}uUho``u4-_gYSVhPtmFD|63X2hj$@$nWEW#kcM z5e(M#WW{NNzH79C5XS06qr%zAJR)H>MJ-(Bzwtw#;!#peE?=33P|2>k&263U;*fO^ z!shQ9$Ri=l6@18Oo){cO7#HrU)k5V$ei_@4tjB$Xk&$F@ZQ>-wExFp(>nETQK}15l z?dNuDbgkp>TqnI6ydjK*pbNH<+(66-y;d+=t?!eS6Cf5?Sf4USt6Be%Y(1c7lUMq3 z{?EiqY)x3v8#OA3AEz6JND|*d3wVHMu0_yqM?50tY7bt5{ds3;n0qkqV4jdi7{rmpFF~ zU~fWfN9%W2CoE1I@p!tC2gIX1rb&;*2!mn{+<~SwDo&9JiSU;rdl`T<-?sOw-sDv4 zXl}F_if?~j##S8mDf|faBHwqDVQaC$tO(bbPkvIvvLQil_(0pc<`}RA! zbJEZNy3tl43zo-x2f)h|3*?g%J{JL%I-f9ksk$8IvTWLRou`*oilce zkk_AK&yiC@C-L-n=T%F_qbYS4q?<(n zNy=%GdwT3aF{wsvA~f;Hb1^TZT7z!2QroN2IcWhU2DGd43951JEs(fg zWBLcu4Np8{$d=Vf{QltOapR!wXlYm^UJ-Ij7sLhRIVakuzw*U?lAc5%hjIvM7csj> zu70=bq%v(dpye)Jifv#ynkimq@htc^--9z-^v!Qw+VVUOpPg^OBU>9RanzFy>}(I& z?6KV5hKq94e|093`T%kNR9Ir5MOQJ$C_KXiLbyU|(UI0yRODyzyiZB2(_^OB6e%QZsi<7ol zs#M3+RD-sNSQvQq0wSgak`wVnIN;}@uL_44{j=fq39kD4MLNF3bL~Ddz=ZNy&%l)s*pk0Er&ArFhA#1*JHGj863) z#73i)GJL$VIiF`>USuAMgxh4=|B>+TkqKxE>YbeCifR-6jPlHOv7%Sx?6DJ>7d18jx5=VMp)_X z-d%rOR`Bm~+c2pmCW7y|b;#%5%n5^rq!NYc;J9-Td z@`5Ow=opTg$UNJx0loR*AIU3+_i^FwpfqeD?J(Dg|%eR}Bp>fWiWK5IWaD@0TkWhp=@YcMvM{x2@gSE2d zJ;4+Gd{-5gitWcMRb@Oc*__Gysqhkzwj;gw^pqcU%u}8|+b{y)*cm9Z z-v?EHj(#;%bVLqw=$gd{ePEkQ46KL|JX?&J9+1(KD1U3E8yuKhTJQICf%X?+qc zs=-x}^R-2)YYICv*tPnY1s9}?I(}&?tbr$PI4XfZty8ws6dNH$-XN|KYB{nPDa!3> zpe15ugZsW|A22Bqf)lVkS5{5NQ4M7s4_-A5wIw>@OUnarqb*<1oc)KNlTF9nQHLKdj)UwyO;6I}@N2VBH zy-G{#^;`wx_`@?u*!E}a&fdL`xAI$JIN?8fT&bf3`04+oPI@>jfK%vso2JILB;GrP zth(iRRn3p)L0?sVBk`A zL5S$@yz-feRIj*F8D(nx#Kv}&+NheO?%5x-fo*&k4o}YaKCd!gBHeZf(%h61$wv4D zVIyWd?|bU({b;MM0kc)J_==SAAu+vEi1NGN_xm+SMFX|fg;~6)7F;gb zY)lQ)kWVbcZ3`3qbyo3wD;g7Q8A&%iQ0aFB!*h}tZQxR%!A5&N6K`CiWEL`2kF75QxuPORwNbX6#Y?cV?OY#>??-CDfe|K>Jg2oEBW7)M7 zP=1o*QPzi>wLT^;tdY-D(IJ{LA0SpC&wa&ihT5IKy$L@Djg)8;8#45DaIUm!(*j?` zBJeC+i~Sr`$}n(zG4A&F7&bT|4Uxz$BhBx?*D*ypt2RJ5!i$UvdoH^17W>ZKiCvz{ z7GDBMymAg}qW-ko0Kb0$F|@eqxz}mv{`z*Wjo+UB1m3N;bmcd?4?$S(=$;srrvbC| zQOp2A7Pg=B@lwnbx4)k6bB|ZQK@-Gjf)*EkKe2%hy0uexX-5%AaTxdYL2NKyVs80> z6zS6s5sJD;oGtIk?S9hho82Ms1p98k?C&ylH6SCl$-%6@MyYLgzpY|DSd!g565}SWHFCC)~bn>MSfF0VDfa18#`uX-j7f)ZiZAMq3^Et4`TO==OE%~%I zA<-Lh`9hlxZ#$hf*}b(wXfSda1%6B;{<)UO-?&PA6&fb+)75mxKxm9NI*Smy z(yF+v1r*3BcJx|X31kZZEbvV)*@;%PEE^u7I!^;OSckNl;y4hBJNL*E z-H57bA@Sm{VkQipDmwon^R*I8(XTK_;=cxyDqZxz9m9 z*MZI$@?)=-IaHMiE*pju>4Q9k8~36wVN&R8$hj{PD-UyPNGaci9Da+WS$tt@JP!KK zueY$9TOWUA4rq8~ES>uZ6LiO{P?WxicwKGi)IUTKpG(K-Lb~vws(%>FOVexQJ=hEgzQTTE&$*D$LGu9}g@Q={Ir3FxuDzCs<)kV!>XA(5mr)bnexR^mDQznm@0?pQx zk!1G$eebJ0JeLt?VkP&0J^r0>$#~`$O>bw;qzQNMoWyo)LA%qiRpz0arq}n#{f?UE z`YGA9WG~+=@wE=e<+_oj(n!{^1)>18dBY9?W5=Ie=Qe5r?DFU8&N*6dOLb!{7)cdmuA52t>i5yc2gz&Z|C;C{tCo}EQ=s8 z4QjN#h#~al7SF$)T^K;jTMQDKdxybB3=!IUNX<;!JMjr7I6g$@=M+#72pkxffg~5< zL}}EA_qLlqZh|}LGE-4jGt-~u$ynQ9)y06cKR9=>UfzqrYanOvUC&Oa3X(JM@W*_Q zruI_f>TecchkdI__NlAG`x+)x(-J;R_<4A6J5Xg5!jeEag5@INO_-KHp;&ZVrxq$A zh1xfXx%i6>=Y*4i|8FhRf^uhkR7#}BWZ1TDx3N)(-Reiku$7KSKQ?5D&2ta)jWTKl z4h-!XYGb1egb|Offs+Hp^y0oxN|`e1jv7#vJ;D!O9z)w(``s`e$PV_b>rZPwt&}V) zn^b8*pP_Pb=uvB#`uA3d{9{Nnn28&U_>ZbSyE+jZd9GwSIKE99HT|plEfp-$Ly0K{ z&FOAQK~Fb3Can{HJR%ShAd~U&>== z%2}3v#T_A}ngIlLd^?%7@IG18eUygs^P6+wrLphw%;2)V+pLc}c3ZYa5QR(svnq^{*{yqOv%e&5`{%KYuc z8ka9TqhEJLrmz#8k9;t_Ir8n_F`BH^#<)Gg29IU&S=T9lXRdAj58)__ z<+&y8v}=J5u+r7Z%&LXJ?R#AD9TZ=W6)->j+SVGOzf%psf*r}y2+-!?C$tiCKu0qr ztY?;Aluj8vmu*1BU?3r$!)*Q{lRp2#jIPd;h!iCS?Ug?XLyVlJ7Gc{$MT4y|_O@z( zwJ%l7=#=en%yKK0_7#uRzKt=o_ePYm0J1oBVEa zr8~b$g6iOmcQUA=e-oLr!+u1ZeRrn&0+xo+^)bj3(w2))=K)0pmz3AUvo@{P+V`Cd zw2uedTt01=6f{vHot;`AJ2M+=&3I&7OAP?^(~K2Wm=xV%(kCxe<$xD*QFt_Oleryq zHe!Wb1c9>BRP$3G@as%}Tvk!;xV1OaD)4sTOG|Vj>mx6;s$&^)SCaF-sWvgjfzIxr zvm+sSMP~wk(GBw$d7FVzsz7<671x~vF9HzqGSMwKMIl^P_x$jF+Cg_J2hR@Qiqpdl;>qaG#-6Hb9Sr$Vfld~g!)m~X5{=O`A)$}@Lv*S1>5XG2% zlpV{`bhX{JWtN-^m6}iT;FYE9m(m*brqXRE2`Q5cEEKgFhG@mES1IUiZ?^78ndT3T zF#opJ9$h>T#K6N2cb|b+R%0=eV&*&12847!I4V%eZxb^d&8I4!juwV|w4ReR``}5+ z(5b61^rO;VZyyplR9bIeSTZ*%z2~`9;Q#tn%xP%JeuU5+$kASCx~@^v=ThrXqiye zXq(tpqy^OnPOJA%fNj{!#X{XPKd8acoh{u9!ywOEp)#vNjdc66@wax4{zaie5+bkp z{()s2$=1-|o5aJNbXkC`_a<6>QY&+=1|;M7km|5^BlHsoP1-9o7x8st*^(zm0Kgq- zk~*%q7lU~F7tAQej)KFI!zyHaRli?v%gM(+Q5NMhlPiS-zuYh%%_=Z0^U=ZdI1Cg< zQ|z4-J@bruRdRX^y7Aj$4SC&BXpb-$;eae>;SxrrTBO^gtRkfWfT^d@ra~G}?r{9sPWF-WJ7IJ-CsJNH(ueU{n*=+O; zQ+Z3-!})t2C8@18>~<9kerfa8`%ySrpJ%Ugs6O^tkGqop&;~Y`bGDP^T)en`C;hvMp8Xv_@@{3+oDS29Z2CZAL%k(^7eWn7Bh<*_ z5-*Wg#A&2{Ck~Hivf|l9puz76G65*6RnFQ139W@`qe2%^2y}Vouz5OE%#_%*EdVZb zWn-V=^f{Ey~p@-6nY@h zZeq-7Oo(?yZ6oF-+RI5M_;|l<+X*mGF|53^n^V4fkpiL&ttNrJr{{J%t@Cu|XG*VS#$Ly|6h!HncR4~@&F>KY#=D;-G z%lcj$IoHOK2XC)|_<|Z{c=oM3@-H~~e`_TE^}2;Hvc!k58^5j8(*!R0%0iU`MvG{6 zX?`XKVQfr!DSjMEO^l)I^%>V&yeo}L|2Oi|AFx?+G8DBmqtTT~BIP<96hAZw&wAln z=MD{d@9xyvoOGvoTIco{&_&?uCYsiMGAO#u1>^SLSQQSyUP~KE57IE9>xB4cu zDNR*V)43#ADMH(dUta$B3vKO7ea9Adg^+g&;d-B+8-Fv8zXLp>bjCodrHLu*5&ww) z_-~7#O6OIAdTBdNG?Fz%V}J)otga6|m!zc^dqk3mgVx1WuCEJ>2b%y9LjDU`N^Cmp zlawS*P8xCbs#D)=63r29t9^Xh50II+9-y|u_Rc3!H{n8>S+?W7TSd0zXO8l~}qZ*>FGQ8wDXI7`{_de$<1^8n!z z0Cm$lW@-!i=aaKHsQqIvfrh)ekJDdzrRNpeyv74|jS)Z`6Puz;@`jGFq%}*;h@gj5Dp&WoopK0Bv`0`zV|61v5PycL?c6q$F^}CNiqiI-|tg;*b z5)@`A(>xYJ>Zp!^`8of)s`Vd^(+P%N98UABz*~Jc^4RU6n14aZpt7d1*wtQ>V-Yd~ z0O`EBzbfp!a)hAMc_`~2#CB5)p!vLqDco(YAuRDY$+l&$rra;suD=wYuOwqLsS?#@ z3Sv3f(Rwb$I0Dc4Hp$OOq7qT0hWFrUF#H^#^V6dj`A>ZAIQVXXK*s&5*aYAu$e<5K zZ^*sP<|KEP@5zf78}C!O`#t}(iZ|i$)bJX(S!tbqw2`iw6#*bo6L4%5<=FlfzIXv@ zBTs2u`lTKH{DA$W`h-de@E8^B+_3gM=wNF?f~IM0A1MNmWhm_oaj1}K>2spsN z63uR0kX>r-hGwn6B#CL3T& zBLDtm+=9@@PsM%@;-FS$XK5fVBmw4-wSE+JYC3>s@giuaA6_BDpUcKf z!`Rz-fE9Yh8klQjC$oOHx^3=thp8jmj)8yf?9KfKtjjY-Z-5>LBJ((2WDhM@I62>B z^Z0P=h+aRgZ5Lk8ifb(H;-+TdDR%#YOUSA~<0$v;0L&9&JK%`_vWM*=vaaY47WF3} zerF&mGLMmBDgSUTdD&k8<4;2-d-Xw=^$OGR6 z)r!?cUP(?k+xBoMj}5jk7>OAO#F?i{e0$gL_VO&s39w+T#^HvsqV{;#5p&ERQrN|? ze=-11z_*ov^XCFpBLcpP5O1Y0j3g2b2q)akNE7@_CZLL|oGO$*M%L8|0h8Z4)g^iM zhmh5IH0+(XhGuQ&!(X2Ok<3@N3~)t4fd3gSx~WloANL4Scq1(_ehacWXkH_8N%S~qyzMyz=i&FDI%rnseo z7KBjQ;Ud4Ui%>$EZNZwfUXb1M4H#3EGV`iofmbrlReQz5^r zh+vSUN{<%c8j`mZoSxNuWv9`>ZxHzl>h(VA1thBTmDrVAEWswH(Yf-JgQSt^-TIb$ zUo!<9(?G~7S z7o(1s&2^7C?g*P$vFXDSPWt;2kD~e%Mb+MYyD;Akaa~@w{NX=KFd)@|s5-Sl1>A*&AL7v?V0S9T}Ci zESVBYoEdXHxX_q>qZTeY%_?dxDi+CiO2a4;>u!!0)bwv7ik~^*Sj9F_WFpgH3VdZIO~`Huy+g>`RbcLklmXqT`-VC zyM=C4q*-AxHxqCZ;gVfHxNFiNkS;8Hu&F5V+4()HQ+r=mhk9v3KLje**HlGuhr_*)qkYXNh2G2%zJu2d^Y8F>2q56Fn#ETw0F%W zp^TJf9xU=&0sN$U!@yC>Ei{8o|K6cy97YZ;^FSE`EuO`6;9^|j*hPd|0TgC!K_%*e z*z_~Fqj%(?ft5vedhO@R)$qXkapi!=4y3DrcQG*f9VXAJCg$c$21(1iN_dX&K7fZz zEwX-j-d(wW1(XW3n6`!;>?)71f<$8m1ucmnWzCC7Be-w-B6GKyw4swjpiW$k|?xq~>fAejT6`K@U z41c`v=Ixa6YT)v>_P;W4D%|$aRskh#Gtsi>v}Cc%jHw*DfyCFHzF(`NXpi2!#J4YU zkc4^_)SlpiqlE1jJJ8>JxtPZ%%^0V=@9-X+%eYGnL!YXwlakC@FN>87tL-I8Mz3x@ z`KB7v(UnLFvy!%ssjkvCp%2N*N&EFE>T2QfpdjI~+T@E3eU(YsBL@Q${;{e?74+6H zR>CH=Q$`~KVOMZCX-z0d#f5&7a0dI^@$ZndO$KvI*`b35biz--rPgg~0qJ|WKp8}I z0TLuMUr!_Bn$8*{9gz!&f#v-jVk9MZR?V4T!PF_-s+MJFxR#7`5bjHOJZ0k$xzB5; zvz9)p@7yCZ_+yDur2RZ;LgjW0u!OBU5%9|ZF*ACPS4KL6d?v-Z`36&LymTvpH!VK* z+H9F{t=_=f3MbtXjbpA5uGzCTE;-j!M^vd3F$x=T)bZXla%7h}T(!OB8wYA^+B7Bh zU&Qi1(xk&-E9Rr%tS6DLYMQ;X4toHcUmq74s%vVRClu!Ji&@eP~3V1RAVG$2dtwYpv@?ZOMLi$~gr) zsQ59uLej6iHY0+W_ygP=emB=KXvT&w{WwGJ%^7P{1(Glr>p|kTOb%N%1f`rMt0un6 zn4_3+Q2Sgffo4~eNaNl@7|#y-i$e$&4iRh8;#o1BPncu$g{M>vSTYxzo*AwY<~E@V zeE1YB+AQ2PgcZpXh{OBI)i73U(ftVTjld?$c_7Z*TT#ZyneD${j4`# z=Q5J3`ri$47f?OFQk+oa#NZNBQ`?qkUKug{Ui*b;#QUmFVi&!bWFLebj%01kia%^K znzB9ID65LP4z_+PvzmP;@Mf4Jsje!|g>Iyy>c~I9VM%vyFf4%t=W#24B$v;Zo5*)C z@(bC}4=eUaYeW645ZFq909T*L5!ps2QGkw*$7)T^iE}*5M0kQP2UYa7BC&KyDurQ+ z?2%{dykX4DqFZqrk8-NbyU)HctsySGj&+WzgJVs*<3Y$>asMeim$n#^mZvuA2@8e*e!z*F7wbb$)nE2yF<-|GDGR*L3S_n@gQJ$me-BA7tjY zK)=e_X6hVkX*pzQruo@I6;Fv-@k!Y5Q4rSAA@8!AjWMsX5HY zeK=Ex{e48E>o%QsaU{hw0l2wM;tZ$X!`r6nvkJB>IRn^TjZ9viW*U9}HaZjq%a^dv z=Gm{{UNvK{Up1@`ak2LbSDqExmq|&uCC$gAxtAYx-kwOKKnvVSH9<>v2NWb60 zrkSmVXJ1=+hjlRjA-4zn%I@U5-)gy$>~mi}L#?i_x!ceMg>1!2n0hDUuPA|!o#*0` zT=|8E?!uTu=2yD-^V=h$mXA6=W90<=R$0(vFe^2dQ3*_NCq;g zIF!>C;|JTBIqo)XB+4YmY?4A1LaaAmjP)W3l0Mnibn9UcFWm*yK-&kw>zTsXOZQ_| zU1KY%klLfk{TfRq_^-{A{+^2L(Q%n|{THy})>EO2QSR-A0&rwJ1MWb-1=jWOo)~DdG?QCb; zr<1RspTLeo1sy(2@SeKo$UP`2+VR^(750&WyGE?9z<^iki3X+fhL+_?ai`xIW{gc1#3-HuN`aV^M{mx{CD(n0YpAYHfE zr&VUWkDc`^Ki}JHJnL~b@y)Z4tguifUaU!6=ImAv+@x-csY}_p-P}PW(@0*38$HuV zs`Jko>Ec_h`+5lrISbA2@U7^!{Ga<=-wSY`V{^t?lxbH(qj+rBUis%&{8d1BJtAa- z^)lLyfhZGsof>^{L58TE_;cgCZDSNxjC(HCyaC~vVqknuOd9OX^h^2KE0@VzeTaLN zmAa<+G_=lXzx4zOL0O5xSaX`K-3 z{WIpclb7~)4}<2^b0`knhUH(ZUcS~UelkC1`%SE;1(rgg$}bC{mLb<1|FzWiBmXu# zh-G759*6MJ$1i0IoU<<_p-I>>iF>nl#ChkV_toguCVAOi^OuiZ!ipcXT_UgcMkohP z=1fqy_qN~6h&6^of$Qb@C4>l3qq-mh>FG&LH(Pqic2TD1vBowL>6mcoZ>M<;IPKA@ zSToDXLxGTSJ0AAI(xM&Ahx$&M5n0bXRs$6+^8cg^)zrvuJa<`Fuq5z`;4Xchoj>4N z54CmW5u7aX)X+45+8;UVPi~q0GkLh1R%U8jUkqjpKu9Wq*bZH^amf7OMB|>DA&UCPh2r=bC;`mWciG@OGDPEnp zc%8Lsf?k`h-r6hpit*_(U%pwo-J=jJW)|t6T3W?l*3hkrRhU8*zB5;gLRD)ChCeT$&t!(3@O*PQ>4y{`_+a_!z#HY$j8OLs|2H@tKS zynvDlqO{V|A|Tz}N{4_b-6E0_(h?%w-F@x{Htzj>?%&Lr^T(MpbN1{R2KVsBbH|En zUF%x4-@2?UURKXh9-KVNT2gcy%f|i&$pKDjzUVh|wVTQzM~px>{=_tzmh*bG`~0P+a> zeiwDAwuJ4TXnX`6l%JiN*Q8|k7B^07^e-?A0sqA~rd&s3`3hLVEoyY3?e6k~tCTk} z6%=oNQ=aZ^mnYS2o>S2ATDlpbknU1B2Ber9KNIpC9?KsH)X^+lpBXap~GK`Yu{7n1}A_>@Cp>zNVO? z%TOE+S2C9ierwS=cpR^(XcSr&rCjS5oZjb37z(rFxf-MTb30zN#N=$DL`Ov2rrXT6 zkelno`TT&$Ro^27!=V?6PKXCHN2{1)0v456?>Talzw}ZKsy0fbIk`F|Jjm2p2G1Sj zDo7Z`zK8UBVf$4?T^Vk?-O-Q+eD37qh^F^}&4ZEFUT2bz;Fv6Ye>3PJvOptIR#+|f z!?{GLU?J~LGSTVy*^xU*-Oq_@OkHK*j1ViU2Cz%3*iBA6swPL!ob<;g&N^A2H>1TGMlKYZz}JrDj6>X-yv&|C%OE z=K`<|Wvt!9+d*!@W@@Wr(e09US-$J^5ZKSL%YNOSaUsKIN50wR;KDA*N%k|{6d4mi(}q)J znNWSw#HmH1vAA^=S4S;&{~M);ANl#VyJehjJJEGoMsux2VUcOhGsJ9C1ba) zr*x&4znS~yz0A4n>?$>cw1|NxKC~|GNniU&=w0TRTx$H3>~L6R1Iks>O@<`elC+?Y`(T?>U~%beF5mYUrr?iFphM1$}`jL*-tmVU}kvrE%u-0kmpv_;LOwJ z#w`afv1Y``EPnB*iy@X6Nh~~=?oR5_&Dh08ZL5(TYyTleS8m>ry}|(^IRI^0j0~9^ z=)ggm)On;{S$eCd3EJuyzckuhfuyZ5n4O#A#5LQpD>26en0})Su`g$aZ+28pjcdTM z2|l^f(I`u3<|nW^l&{{Sr8KqgKFV}aZPK&xVtY&ZiF-^{IJ$bz)L93V{e}pNMwbF` zgKsn&4W;t?*&ZV3b8HxOlAHKmCv4Vnc!GHj(T8ZlyL429a9>voQdVi}4Y+(L$<6K* znOouJv?7fX!c654L0iL{iTn{5;-8(FG9BS)8_-BUmKB~|kol>lEMmOa9*fJ<*HmwI z{e2G;jl?O3w_pGLWL5EG73EV5g@SG+llwiNkqia&#cZ)GXlUdYri2-HG9qCNTI{ZV zpZFT)<^`{$0O|H3^yiSDrYCG)45+RNA zi?}pwR!Aa=KLaW}NMIVYvPJKf&cL0|9~|RYQk{VH9hv%Ja-cdmU#%b2GF6x@Oi%g& zoRK=#S*M+xQt^)kwWX;}SxNS{QSL16QeGQTIS%#|;`7q>8pget{+z*eUIm)n8WWI2_VNj{yF`D9Z zlG3+_W3)dHfyx55Fc>)gB~t_7fFC&{BKu{@2LwRv?=#S)!Cz`oW9wUS7s(Mp>=?iD(dtJ(>zqN7- z&}H=FJSLX{#bAv#P?o+;HMz;53)=w^lp1?nr}3 zukI-{(!Tg!@S@moj9$U5P2x~JODG-23!6+e;mS@-3ni?r1q1*C9%_A1im(q=D za>@lg79!!k~Wpm!PGeVNTnTY$Cv5TO$E0rROFIp&_CJCenp09 zsiUmCB>d)6V@)!*KuHi9t97GG6L zA*5ID3L;(Zr=H&RyC)*Jh41TlJHb+L%g!l-DmHsc72vB=A}QtJo4J;waoMgiJG1)! zwh_6Gxha<7RdDAc-2Meg%d8l#YWM_ySNUC))cER+7WbxBtikb%fH&spSkpnIp1+|! zKH>N2Ol*TfpWdWr^GN zad{l+xHT(@%yEQ$NXm;!WK_v>#5_duEGuqUf*n~ug^ce9-Y5ibnlc6 z9;Cx`eyJ?mT19kEoE}aE@u0!rYW$Khb0^G_=DSp!@;lno!YZsLn<|}W&@hDEPicMJ z^8FGi(70qnwEL9yvD0@#;FB3wr*6CY#u$wWr!gPAZ#>P&e%Oi96vvG|=s0cCdjlt! z41L>WqJF3eA7jj#FtfUyW{V`6e~?$t_ja&Zp>n_@MKTuVj}r$_;#2xI!dLVO6))!d zNXwi;xt#+yNjs;v3c;EMIsFIpv1@*g;J_*PsrzOo^K2&n~`%E*q6H zogDV!m4lm1u_}u_nKN@#tQ!@2aGU0s=rEeY`*JAt1PwdYt>s)Ggn?D%r1f!C7H4-# zYWB8n$bl;wZ00wmDh{7x@T|-jmR+&17Lr-mo*&mqA>0D9j6(!D)@g2kNGQgu>BMb9 z3}%Ypx2efZaPXZsEMTbSUB>esEzLjJ1V>;qhBb+Huw#y85sy&s+_=u%{(QnZc%THd zuZ~|&JoZ~$q)ziBb{~uq#lEnubPxj|;;JEKOzVtRybcpGFCxRYXq`s29REc@xtBX( zVha=D3Cxt8`+b4@YJi#3^?d-)!U;%Z-zrfKmlH1AQ^kVOwIpsM7G*)QYF-FA6u#L? zE21qMgii_++Wj7n7q5ai0FzkoaTt26c*@U5AQhG?%l<3|~}s=puZ^ z^z^qOlmUxgq707FB8a_c!p+r-xp_^8{I1c}uaN4MSY30Q1zIr=Db3@KKdLPzX1Zpq zSU;x<=V;|7z@q52#i}YDVXAR)x}UB$bTWQ+GCt&(vPX3KW98a|Rv((uD3KNl{y0RR zxS3wNx{>sI#3-c6su4z5=pSt8*St?+WdLSzWeY|gQNFv05Gk{L?2XJC;Q*GeK88ub zayUQow!}zsVMU@*bqXkC1!85r4{1TIwSLo=aq@F7!|1V}aC8>NyqEhRGG0F}Kf%2?naC?=IK{bu|;xy3hsbIO$QUV_fC&HJ2-# zOBW07c(bSIn38$6DW1efL(eu77s8&{BVQgfviiiHF} zXQpP8rVm!E(qDK9_j82I`RAlTVB;7Vo@=O9zVK5jgy#rJ&vMB3F-@+ADsKNV%gl}0 zr=+~CCJHr3h>y9@tpy3_{CXb0ir>z$o|LHS>7c^Q?pNN&#E0QzCJ0)dhH@=S9FkzG zuK$#`CA_BcnM17xgQ2ZaFVeE2QGQKY?&MPtEziqQ87dA;!iAwsB9$IgTmMdCIJ_7C zyL+kZTe9xv7jLLLq!t|n6X9Q^7#uZD-o@UaY>+lMRwdhy^)E<+Y&Ttf-F1{P@^MJbO;8MV3ahtih6f9?pN!A} zuGA1g+YV0j@(qJ153nXvBfn8$cPP0f=uK{Ykr$|I6sA)w_hDd()UuKb?Iv9Myn zXx>UctI*a=f*=vRuIKCEoYt2Zll}(&>Yvj;sBx3U%A20MC&`J}Xm1aXAFlzER{mq_ z8UY99tt-brQ1F2_|8ww#UpNy0_5kNnD}4iq0&Zjg&G zKWlTml137Vh0Px&jApkpgN=u_^_s@dRPPqsBb*PsvhKsJpP4G=_O}bQz2@Jf##?-A zmboF}-f~3XPPN9UvuOERzV+iV1;Hoj$G1`lXVH^KBYO*>c)w}vl_8)Y!86ZhcXs5$ zdS6hX;!Ri2Il(z_VV)gm)kv5-Fbb926m_bfP`!3Y-*@+Ih%I9VS@e+7VX9Zm%LU2n zb;O}8Psl$=sD#~p&w!Gc1VM6%57=1?z zZ9y85#gM6oTSRzlRphWnYR^=;w=SkqO4%#>30dg7(Ucy2o6OG`{pNM!Q^S$&W7ZkK zYfw27rj^rfDyTA7Q&Wn*T)M4K&c%?v_ExxaT=3x=KhR|zOvU;ddAhamTN2F96q^y|RY7-I&q2c< zC)v29)Zfv)xG<9w-4l;K6SzOTA$@8^Vld(xsv<+AF4Vb#;K*Q==M%>S>l^!DdP0*L*OLSEn#%W{aHpKD~}QbF}U* zVVAtQKw;X76?4sr*y7ZWfTgRfFMBm5V4rpcXgLe!HWTJv>zz~&pOzY4U*g=|`1*w_ z=xAYa^r|P@?b*t0nkJEC)cschhU|f6HifR{&N>g0ot+9Ks}aeXK1yb}w)CcK*cm0q zXAZg@IUwKqv-l_jaZPO5tWM&@$Q56I{y~Yq@AI5;*fG`eVQ#9^lhA_kc77G12*>iZ z3y!Zxj(EMX+&=)kk}7VxG^Nq20x*6rpmcNvvhcZ z0bv%Sp&~v8@{@(P4B(j1D0z|b8 z9DcI93K3T1%Is2!nwNJJEX5&mp0@I9o^LB3-7dJ_^$IjUxtuad9rv`)-%9Sh00w<> zxS_wD>_0HY0pS}wU+oD09)^Dp!@q~& zKYNepwlwUpoNZqDY61Qu*Hi9;xU{FiY9*s`*TmCK}YH z9xLMR_TqS>ISZCx+<24e9R}3?zCN4q>@^rF^`x!1{q?r;PtKGBg?%tJ)h*A_xTAVGrhFXVyBS|W>(;45D^1KTsFmdS0L6K;yYMIVPjs#Lj;1_oCBR z-MkI^*n;%u&x|HqN8)>#eLkGwop<&4^csGX*PxU*CH-c7@iWaX&7`2G|N zDt+NoV*KEvqyayRnKQF9y|Uw@5#qeG%CF8lUtC*Dqt1@&&bG!+63ehZbmD27V+wEh zDK}OsgD$##eM-N&b#KB|yzJzD+hpqQ@bgMaRbc9&>vhl= zMTXOVMkpIrL}Xe1Na4ilnI?1Oeg29uD`B6`_OX zUXH2VZlc_5QhsY5`rrOI)(XPfFScB0>h-C;z`JbA=JpJlziUujyvkS^;xj-blz;$J zYwg0?!^KDa>YAub@7{~Q05g&rKzA%YsmuDObsz@U7Yy$w?kC%?ztLctmybucoR%D; z@5VPPV;N893B|LGtGA)y zaW9Pv@Vqt>`D_|MPtM)dXBFAPTVFXpf4V_N45?IEwU2>j%ZUN)XDhrPm`nOpCtm)6 zmaQ8T1(j;ofp`OV+m_M{nCYi;AE*pBpH!?o_HVJKW}U1ps0QTh2_O!!K;IQM3|jRa z6xvlaG0u2KpG&*|A`$$n?b0HL-`;}>Jcy6&bmPE92ZR5ZCgoo$Wq_WlSV%q|{o|94 zo1is*{4?mu7usGbPIa1Z8GG{^?)wIy(%=HwW6`PEa(of9fn!rR1C4_a+3$c{^2Jx5 z@~z(_4daU-t^p)4RL7tk@V1&!l`679ACAxGCkYmKq@X7c{7+V|nHYUx|2fdDf49Kl zU?!%yP923~cYi7*q+Kl;T=n(o2<7XD{3<=N!dpjRCR)0o?KY|NGA3l7x^ zfiI}M~_%GA7=Uyl`Bp~;VtCBDNxq)x`8weQi-f$~?G_2KUC(3_zqvjwau>EIC<%}AV<>^IPX zvcLq5)EK58rp|qEx>7bCym>d~TY8tMHKkiv?s1J47>PGmFVY6FVxNN|U7XZ?saTC;%4R9KsQ<)ydJxzXKtleUBpS%#WrGIll z*`41UbXkW@fSHr+Ru+=tA8!fz-m3#O*QWlkbT(ELX!k<`s-NhVej2aJfV~B#hnL0I zQcl->0h!Yah=ZLmb~}UqUhBBa z#uxk$ZL$qFpz!-e=1Gqw#&_MpXqPXbbqL5p;-y99904+*zX~8wlc1;8`DJFpy`A;C zz@RX)(qYrQbFT%=qq7ct2WY_T#P%LS6E_tcIB;*SjDbUw`q2IKV9xOFi$cSDm1F*k zJ+OS`i!#gKj1?R2g|DCl2i_@k!b`AiT0E1C!ac0sLY^fzCrxP;>w6&Y69E{{9Oj7c zBSiKYV66fx-Y>O5^;IypsA@k8nr$T|=)hzl$(4E>six$oj#Z>MZ9gJDy$3D9-dZ4M zr*GI-JDTx1{wKwryI?QQkU(Q;*D$wXLo}UuLiSeWHZykVw|53$;pwYTe_wcki-l*0 z7QX7O;Y5ZS;s(%CxDE8E5^HnYBzGL@j>d-XQb7%R5BY9lHcsm*83C&h999`==@`=L z>I!LL%<#`tb~?Pt&!&}ZWp5_N&fD)Vbi+M89IPvy4O6vG*BN%ac0e69U3A)jUyE@j z=)Akk=CC^IT^dE%eL&N53?A8!L8da}xyjfNrjD43(5o9AwEyjZ>x=2miP%a7!1$wc z42N0&VRG1rt?UIX)4RglO(2|4S-?bg&?4OcxY$HzUKrAWSs%bceo5?t^rWh!_DpVg0O zXbiK%W~9t<0J=H6CI<0$I157z?ST-Gg92;Mgn+hg&~APgi0 zZm-6p(Y-LJ;6oeZgfCSvl00jT?YV+er{LCCy*!*FfrdnUqUpX)y2d-D9;CPoxjvG^ zE58d}#bck7!JnP$wV)GC#f;hw>i!Yd90{iIGJ|Bj6E;=1@a~4~@%#nRZ^?y#YjNo`bP3jWh8IwJp{aS88YD%^Ez>h3{993|IH5 z%75{lt!87VITHWzOf@1jHC;Q$P;Tsa39m>2kw0vHU3?flE&VRLiT1~*UBO}fpYQlD zKI*a8{_kT!?(yV;U!evy-9M3#n#=J0D%q_XD;kP?2rNQH5iXc2_pHWlAIMJWBs4qS zQ|WlUIX_32;=-Y{980|pr5PBJfUp)(Om-&*<+*CYdwpdv)hPCg6rv{8tM8hPLI`nn z?B6Cp2XRFxuf9zOm!Vu3(2DYcygTub;2byRVmhYdPQPYzw1h!n)oP^`f^5$VjFlvO zRVQLFV~~9ZY3JN`xo7GU-=Hk}UPsH}(tR2>KsSc8qU7ysqi1>Rf6J>Upb{paD&A59;6c1y!h0)7*!>y4H8~@ zzSY6Q^TUXij2fM1r7(K@bhqr>C6|v!_)i19hrkHROq)PBJ|qN2ZgBg7LIohQnMrbe z-36EA{TDp#!2Tfm6rim_&LP;}PKG%I&2$yF5QV$%GefQwwWRc7@59ma{ z04=G8yv+D8)d;FY*K(j1Xb^e$J|NX;sBtX$KP?T2U*GURuFgpzHDi@SPm2`^Bu6JZ zH7WRaKz7vcEHc=-y5|-%;QlOVZ_(+@AtzbRb&F{!D4+A}WJO~@`~5>DL9deuG4!M467pGp(_Z< zAki=kJ*S0?o`eN0GL^o({8BB^w(!^KUZIj7u=2mUGqh;z7HYm_ma#1F4h9)C?J*dy zx>_i_NOsM)M9voojZ>nucf zJj9TwRdn;GzH)XEYZa`D4X;)He!CZBF7vdksi9e6**8=kkZj63kgrXhx&a?nCf7`9 z@Y|}rL~Oext1Mbm`E&gqE}wh4H|Q771DG?--4l$zef&jUMK68%9omEF2dIBbw$3Y! z9>VAT2?~M8Bz*FIcM2CDNDQ5F2*hULNxdKO+X7!WyTkKlG04%m-XbZW{2xB>*K$^l z`Wh(y58r*>^Ac#;W4A5${~tc^|75W^lj^~3Z-DIQKI<9Ci;C7kOHhaHoz=R3_!QS) zu(hxcx@LkOV)TP2aqmGU>>b4OgZIvd7D_{;Ctb-Tvze|L{0CzgFeiYplFj z21H>n>HYP2BD)GaHBPeY?)USs8&b}z-k%KMyykiIdukji`si}gbzj^ss$7tJ2ub`w z3U#A3(xi^m$3Q*+9R=@BY(dqM+dmu}HrVCsg&HZE6@xfDpu9d2(m}v4s5?D`X2k?; zt`6D%!>P=~xZM9_afxunxHiOJ>Swedu!@=hBTkaqI!Y}5;rRo9y_WPXMSG?atp_Ui z5IHe~IUxf#?+77hbRYRFFDO?38OfQzE{|~5x6s)F1?sDkJFuP-%iq3Y{a0EjK8DU7 zr!$W28WHvoxY(a5e2B4Pcv36bx$*xAXuEp(RF-V;a$iL8A@h$N4PbBufy$+%Th{&m z_(FdkX;I9}rPO`QRusOV{1D`nw}1;Z8~q05EngFOo*k|gJO2*#gx24?yl*@At`qTa z3?oM0H}HdU#q+ej8pzg80QZCpHj-c^>BG)HEU3bt*b<1mjjmG@ao8c<0fSc_MJRG3 zgTchcKj*-l70B9hrfFGc0rTxS8;Ecb#rd{)5-ARRXqW&QrJmZ3QfJ7pDs*@)wHxd9K9z)1f z>bqnChl2TkTu%01%V{oJl)4}fNtd!q?{Ku=!G=!P^VNzurST-Yu77wj0spTjIIN$# zE*(*Sp^za4r+UZy4^J%f2J!L*EbP(un{SbeGKNCI_v(T&5T97z?p5x3S!0j?57-;Q z^~+b1Hr=1%&b;aY1pnEHty?Iplv(PIH`f6KbM`&!+a(u3*|WwP{~x|`?9E>eg)vaF z^bsNw+uKQy&;xkT!_Wx(_fJlfy1WI_7NZ|(BEY2(lKnaM?zatjc>NbOl8~*hB7g?z z9oWSQpvV}Y4h`6Xt|_xhzTH1rd5_EcgXWE840{NHigbz3#dUTsk(0C?oj>`Nf6Mg2 zuiKQDudHc8K4bBac?iMOS&7zi#|Pl(-2xg~L684o8AAW-7R8DE1mHUwq{1}%9di4p z&LHsn4265^(EO2GNUIsYJ+UalmvhAm$bW5ke7Tv$>H(o+%XMd`kARM*wlUq)KdrN? z!>CZiZz+V#L-EVk?|7kcXb1}-zXJ%LhdY!*#d4WxyqJIM)4)_%%^~D4P!s+SuW$J4 z_2_pke+QE<7HjPC)|aHE|HDH-2bDJJ@-8Val214Nehkh}^W|Cxd;Qbl1|J^_zr0LB zIi8_Jzh_Msn}6=i0VfyhAMkJJD?e2I^<3#_;+8tUg+&+71=byk3sz+Q!PK4S;n?5x@U9O!7P$fZ)-6&KYlfJtQc|RThMrD|F@- zvQp5V)$C|=&CnTmfV7n3z8!&iTb^Xc4}l@4X&`!5jR_@i zZ_Rsy{`;QQH~zYxeuTR3)De0^uc+FBp~V5tyX&9PeFpX2g-)v>H~>;)Zkv3X;##_q z%nFR@Bgnx(De`?=kg#g4UA8X|RssMp>G^b+bd%99J>QK|d6shEhi;sz6oA9L4wNXP z-qjlFLW+0|ZA^v4Pexo0GtgG^+`Nwl-Cn;y7QAG{*;IvYlJ4%-Yd42yoEI@;oKu7^n(>1tz zHy)IeE3}7h( z7@8LpoeuyLsrg!(%(d9~B#BWCj(%;c#WH>SfIp5LlKfrwQ+A=`o&e(!0?u@;zTkew zuU+@v-^z&5yPfQ`nDh4Th6LH+y&-^^e1<$%CzzD<*8JnPRHoWEeMi&bFB4e*_(DhN zObuSw1x$I|RM<##eI(sIRm-pOTud%AO!|H%0;Df~daSqWpK??K0qA@J48_3g)A}32 z7&Hf#DeVr@lyTV#R8x~?(@p=F^}q|OOiQ~nk*br2S!ia zN~b;igzm=n-S}|$)OL}`_3Cq)zqU~F&#MlEV!utaK9PO6h)K`D(^BXH;UG!GPwr0l zr473W;Pjt-Oh7h^7SyLph)Jzn#*9^RO}j&A_Lo(SkiT?PbPhPyOIQRH2sAU-D+LEUVHfG$I4dAEfSx9zlz zGXSPF75NeNB6JDu>AV12*z}Epbr}?O_*VBotx)v^fW)196qX`ToL5IHtPXHB?o+U_ zPk|OsT#)0sdqcivwz;!cT%AeJu3vr1secooNSxqZiG)VNSfNP*H%`G6W$Q;d8PPW( zTY$4SedXz^Hl+;4+_=PW8M|HY#EEqBM`S)5STFr%j76}`Gr4>9Ur>B1^5JPlD%hIs zSSnXee>JgR05J9qvWoTvl;=h#dfwdc&&;u}v&lB&;6dO;A!E~FBS6B5_e{XtpW|wPv^Cwx{QQdH*)vmC zJ{M5TIg1q$WJT(P#1K_-)lslEB2tbYuLK|!^a2GM=ft}oSFa(A@a+y6eg^pSH!IDD zVk}!ANY9(%@ZAKqft6e;&qA5$8yzT;zYG2lAA@>>__-*kC!(5`NwDldfd|Q;a&wfQ zEHp8qHpI+?&=EufO|Fwhjo^~wL-jbZD#fXt8vxlQ9xNVLBx>I{0VhhgYMCSq&*UJt}+nC{B?J*E^42_LNJhJ+rJ0tiW4>g%#ysE z8+4Es1(DPE!P6n>cXoa@R`-)7eKSddrF0qdXl1)ys@yYx5}undse9OdGL4j*bdt)O z4%5cEVvU*eQqgn`;XQCTE0||{4&16nn_;|);l@x5`YK+oDiRh4&^Rj#qVR4Q0S6z= zWI~GHyh2#?JKy_404XyV!&VQ^&a9EC1iIDy??Yh9R2eZKJt9!)s?`O;*X$g8tk37J zAvRx$>j(fdko@i_VW!e^4BtxgF=4HEs!E<|C2d@NwN-tNZ}c z=r1w&IE!ycU>LnD!^$GFbCAY(2`l?h*Hnmz+p@Qq|8p9~c;N=bH#lzSKKu*G!Ax`4 zEavq23t+}swF67#Xx6nF-`^|JvM63-fr>vsJtZj5;U1_-Sd9TadBevpjDcc-`o0fq zF|zR}SC?q*Ns6CMr^avj*mn<-xo&^sb+ZoSU1NZ$p8;{*l@~5@yFqX55P1GVOP-u} zM)BL*Krr9+K!?>-#Ot&#Z^5GPg}#+#p&IuUmYffJ@>aI>ORwWx(jXTJ5NoUjIzP=8 zdsMkOP^}6XDBY(taf_LT;>3NeJ^pz^#MgwZ5?6Ri`m1WZ|Dw(@CaU7Jc?o$0Mmf-W z$Qg6Nia!o3rt|$QsN04nu#9038mG%wAq|b^AdlkayX(pv0pnW0z{uW4rR>>iF=^yd znfEO@-jmUxl*C*CoB-2(pm~d_PnzDaVI<<)mPNiQ`!)>!a2C?QZ7E{u!JlviMdfDv zKZuos%}NS;g4L9TxbESrP1xW4T@)vDK&j}FnGd1 z#8m703K0vDPk7_+esvuS$?hxz)0xGRY$iqu>x1&(6V zu8=nkYfNf1%Q1|fCfV6&col=*Pk*2^(su0!NEP&$;=}r(8gQVDR4v8CaP;v^J%|}d zNC7_KolEY20@RNK>Lj8ECEM*sXgPM|^$%48Nu|>VB^zuMATdmWtIC}ghnifuPK+dj z#dW@l!%>?-5lXWFMlg0pqBS}Ib0tc?o%k+dxl-ZxZ^waR98c}gf^DY=(0O2&^p*T8 zvVLfLQ7bBA1mSQRr}=kPWCe5tAVJ6JXQ*3j&2m@3ulJaKn<`h9Rg53u=7;K$bibtF z7g+Jrw?M4%(F~ueCt+8yxENF}A?;EwblZ}*P?TrL=&h$x2~)?W2}YumwjFu({%m-# z7EiMZM4F!=69pc$<_{AA`hJ-s+7T@!t6im@1^Ixr;la8A1b{k8Nr4>v__s)VJJ#R)j9isuM8k$k4UTZOFG!xdWU z;nD#`5QU6Kr_L<7-_b_QRDIb_n|Nw5d&|g!F&lH>bez;i*GJyEtT~S_5fmw|>ZKaX z(ugzSmxkSpFi~w0QZfy$4yAl${M3uvgjyy_oSNUDlqJU2?yMB(Z}VbOWsJ1@02#{PzKvfaKov1(9* z;5UK~<{5~Y=pH~B!hJi9>`Uv-JnwC+7V(=5XaGvDE%*tZ7Y@mSv)*N=u9MJ;NQt{k znx0HPXOk7KT)77r57qYUo7~Snw9=})j9`42D#wzmEON4)>{z2Fuo;73F|0xN6Qd8t zlqJhw)xQVEDzJ|NOEHD#fr}Lf$IoFyrfK#{tWQT-pq@}`32ZV`z$TI8MeP<$$-C^*lO2c1ume}4MP~-;$FFi z!{vq@UZzUS2giw~o2eEsw3Lp&z(CGd=QTuuN~nfN1qcVK2x`7BQ=KL}JuoC6763*a zJ3@S7MnE>VD><(nXD6c2s4i;O|tN`?}LJ*hJwYBPHk+(E%4$Uu34%)jx#_L9>*Tg)5?*W16#3{+O89sW1s<{#eF(G z7@UqKQLed!;Y>xVV8tqbRa+#r5;oo<$)*2CU(KDj`GI8BZ&Crf5ZA+H0Pk(PeA+gq)AYjgZ#*Xp<-nwG(@8Mx#Lq5zzmGJm z6ijhMq}53We_DqOmUXY(Qv^IB(EFr+Y7Mu!)!Lghmu=ZQqRclFNua2xw=jOP>QIt5 z+q?iu`bc1f%n}q~SNEa8HM706*II!2tm!_S?iT*}@{h?iC`DTbOohIIIxzJkcxJeE zI$Yjh+CeX~$EsuG3%mG-2$rWmsvYx(6lc7hNd`?vz{Q|Blw!#xL?^&sY=ih$u^PTT zwED+zVX4wTdwK!-@D=C6tSmuhMS*a8R2f+T7X-gMfP(Oa!1cJS?u7XmObuDCaK@&{ zjRDy`FEW`ocXzbdIcKdF#Qx2u{K%s@)OUf@(egMXK}8hgULB?1={8p5aJn~c3)#*y3ur*(?wn7j?K3}WZ3)VJgMD76EbIG}H zv{%OvU*eEXCo5e$l{o>0(>vC$G)u@oPWZvo*gPmcaIw||BaggA>xmi#oXQCR3~C#@ zo_4Z}(f0@+8mePaI^}p;2tf=%05T-@G;?vWh)5j=AOEa|6L@pXgX2sUe*1yq0=LPN zT0O|zm@b|Imep-2^YD&uuGpAib?2L{DqKp#wyK6i7uJhHT+QE^cA}J8g#87*U2q_S zYovfBvJ`PCe~n1ZhUB4|>I8rk6hNDLXnr$%6PRZSa%Jn=BY6>9ia=^XTej~*4dBi) z%#wh(uuP!T5XwJV)P|2C;+!I-??Xz1m_D*;*z6&Muw0|0&oip@HQil;2o(wl&8Jvm zN-*om=J6OU)13Mu!w=${%9{Gu(|0D^JGcfLTXEDp=oa)azV`XOquq=+u5lD=qNJGn z8i!2^W-al?88IrnZyksH%zmDdxX@aN3=B-*cqMz2l}vamt_v2Xm?MAmX)UYAO7SQ~ z8gWZtUk9Y3p0}<_XqPowbr7{XIs&d>v;Mab-${3nLp#?}6BR#$OL9#SSM(fUdAv%e z)jWbqyk~W4rfEGt3EjV!!58%qqJCuV{Hg_cEc~e>pbb_oxxh2=h{hvF?oDJittzGo z>~z9Z>~1m$pioYnQej+SAJX3g0Cb?-o?H<#+7Yt1X|@>{b^*S$QT3oKKpK0W9oL$h z%5tdO?WoLrBQA4>mU2x@kv**dw)DyVUw2Lu|}dQv@M;!4-#TQ4X);B z`wV1`1BtT4MJNbyY3b@X_(M~5R1ic_YIZLe+;s8;oLVS+yZ|Que0i{%TvlHu^e~IX zeftd*cbR*hwrL^~qc>q3nlM(suL9{Gn4SINQ)56XyfTA^P|kDI_t)(rIIpzXE!RSp z1u4XkTNi;!7A(_~xv-bhWIuqQPqX#>=Fua6(^8+WFwsIF-(_V`2VaEkKvo{2uh9?r zr54~lM*rZ~vaDnTT`)Dm)fT) z-;!Nv*+`Jir85p-X((kkb>iItOLjY;Cdh5Hk=%$b{ zrDZ|X>^;RMmoW_vP)F?N0NN&Yy${oK&V{~{Zb!hFI;x1)&1T5_Xnyhi3qDElqu6Ur z=?@LCq$#Tb*>v)e<&VFpnI0w*=dM2K0d!2b)8YjvJqVSqJc``J;4Gc12=wuLT}&z~ zioG%qGbn5~r z{t4sNO;HF<$afe&{+J(D@e0u&Bi27P-cRYA#1Bet&3#mWgCY6TDEe=i!5fiFP=9QE zenJZh%cNqjRHU>9jFpBu<3E7R<7@ecjQ*yy!9rjHR|CYAhH~lHuD?g9-6EG2tW|eJ ztG`GaY={We_xYzoB_^YxK_lw`i$;HxSHttEF)M=0#A0or!t1ZgajsYif&M~TY0>%r zO}@LRkjC0W?g^?5I1QtkuZI45^*QI@&njxY@h`5!x7#J=ztc9LVoH56)n&<}*Ycr_ z&o319*D9YcM!;q5BxE4l$n4i4_>VuzB)NS0Wcxo71pZ3#|NfqATED(|u@tj5)1Tk< zKi-I%ZRN6@vr|koG)nPTAojOE_xSd!FqXz;`d#av|H$MQNh52leirSQt^V5^T}2MW zyZq*)`6}%Lau07~-gn*U{|pfs+F9#}XFC#jjCRc-c+%`$k(7BCG)EEmntX;{Zlb?s zC>J%G-iXV^QttZJQ4?N|h;d%IfH>x6=WKI8m?L%n;j-278DjTmtO7#th;jRQBSBg1 zPX7AY_bT-D?6%^dDdpQ<)OW)uNyM(1-w#zc_bovB;EyXNiD9>5`Eg~ntTM04QXO9t zzI^S=E7T%Q93pNU6;TWvu>}knnYe?3BN8RvxUULNXd^gU&uUm$#Ag%t61Nf}rdEZN zlQrE7?-w;>Dkk6!y4e+}?5&Qicws9&_i+Nlf6NayXY8zo(nH?v!PYuLL{8^Vjcu0( zI)+Ov%)Gt5OJ8hErq2~Dk?eeu94yeyx0&lK{fs6sl&hY%>_g`E9)K7>JxJbslYb zb>Jh76dI(|mrgm2Xc7y&Vc^Pp<~GiLJW^FS*0fq#<2qWgIF6|wQhZWssPPrER?%44 zfw|W3psse&zkO-hC+QE0m!cilv`S2kuxS;ZV+(;fhb)49vAkBPxHvdN%iljjDUk1y z=*?^h#V%tOv(tI3!qelNx;G0-05FrX=jL(3qVY1eAOj4dM2ii8S*iei_g_x=QU`N znZMI6>-1jD#d`X;)dcGrr1GPGlU$pT(cFJ885q-9qQGHAzg|R zag!jB7-p(ggKsrs8t zF*^;7=pC@~ZG3q6_{}8`AGx{J+Aa3AB6z%He(<5Oqqov-$^XlqzrVjum9Jy~p1SD* z4r+#BbFk&N+TE3rs!}YiyDBFi)V~w08ThZ|j+R_<0kj&s6C?u8EQX6)nXhD@j`63+ zNaWvv2EHxuQ(9zQV^kl@e`}BbL)>p-B$Gc|qi%y)yR>{OFM!>+n_OVM{*SS@eLha# zKNzW_RRLG`9aEZkq81wNF$9K*s^e!hvHxvnMXz}9W0PmoX4~@+3FR)R=sr~@LGu#5 zjna2FTjOj3R0a$EMXLiCRzI|}a2qrtm7S%_Mwo+mxCk7$@-~-6MbU0ar6XeY;duw0vG6b(XoAnF_`y*NXa4-S3@%W`_huD|Mr4 z>lrzAKz%kJ4^;~ne?;G|a|5$S1$Ziyllc2PcwVeOB;tMk=BWPBr&rfjo$+;^ zdn}Nuz9^8yCH%XIdMRp%b-uTIoMK#Jc7{&xt(cMeqF*wGjg(p}Wsa9yrA0+YR~F)~ zK6(7O+#d7Wur7?Bu5j%+hgiw1Ge~_bIurSD)tHEdLFBCHQQ0{K>Pxo$6{khLN&*!Y zckx*D%%MWs7w*dI>g;PEn#HYxMz9tH`M77Rx5Tm=j&d>GRktf-46S)vN}7ADv7Wy+ z4p2>XC&oNf|4s>yePEZC&-pA@p7mx11(660c(0inau_xR#y-pWfnCE>`K0E?ZPW3$ zTiwp-dyL5wh1&zW8?~$(UrEdoz$o0I=?U~V%EAYamwGY+KtBF-7cgF{#%e^JJL0)^ z0pquC`~7mre;Nh)3u7jM1;>W-WYb~Od5U6VX3O5ozc~#7%>I2K^nU6?Z0&n=da^gZ z#+Z3B1#klI6+lC0NlfN2^3S;BU&?Dsq#&5uROZ15-dtgwBq)ES*{FHR|78ab6n8x3iX}Uxr%L6vZkS zYt&H-wsJe(nu&nF8lyOVTdhg6TDT|-u7L%p3fX!bgEl6;gsd;4|GCXcC>Y>bsuF^M z`K;_r5hljCil~!-T$12hW<5B5I8`#J_YsUx)==3lE4=579Nc+?qb*_ zd}Q9}Am*BNw6{jao~;pZQ+?Oe1%Up$V7Fnebhx~Eue9@-`Wh&E*7bWq46&Tsi(ASh zqbVP1j6y#!utG&em}~l7pGHr+S&ge4!8Hg3>v6a4;^;9o?z)pts6SZRfm%5Vu5p?z zRmb69I+b(xB2#h&9Lt{a>@Rp+4NSTHol}t0FBAf+RlrligR^<)gZBr#(8PUh zMy8pF{}+3I9uM{U{(<8s`Rdp*~-|LP-wDxbzj%>dS1`#desWa%`av}n?1%cI?`!?HuLnl$3f(L zCUD}YTt~Ux&u<@%eP&DOZMylGD*kisO0HNJ^Vo{_PYz$QTFOm*evC9kSxu76g$7qS ze>qYCL?xexf1cIbw3Gwx|Ab=Jf*KedSGBkCUWn7s4HXpe9}^Nr0EM3m*hZNHcK!*(KukGuNCP-niaU}Odcj24Q8_**BQ~Tc#O0ux^oJ}yEFp|$iZvwY`WjUM%|ZeU zbP)o-P<*W}OgI74e7`qpqDIG>{s^dC23Yk9;nu`m{JycD$z8c6K%w51_!HC`oPnz) z08wir!QA*ItFlvGe}n<;{BZ;IN@A1O=Z#?*889<;>AioUqsN2aDr`US2o`Ldu4@q^ zraFRuM3q%}d0B@bIBrVC>*j>=vfBTYm%iSW_>;(7I0)LE_+!;jv~E^6YbIbio?Biv zzE(;*X_%nUOfA7Tr5?B<@Ew7D62YsMZQ1r&lLat(2eKgGOtb2?MtYr3$=`vNvq+tMA0$k8X#V-O6bvzy%N0VDo(%UoqyX4f~(+G?=L zukQG%V#f=U-hz;soK4lvc^r`fi~Wpw*?AulJ5Y{v5$g1lP(gPt#+)75-Un&|CwRBN z6hvATI_P(NF>8u&XpXn=^z?k~w!(GHy{Di)F_PJ76#0Y|{M-E6({vO-r1*A+zuyJC z+LJI`7j8ZYHLprRCW2~oG}k0l6R$=jkty7$vnkJ|IoklhcVQhZ!F((znc$g|^o?ln z{?E<6RCN2=RquPqy6U@<6!)`j*;c!44<}YVzt#%+p&Io56uS?UN8mb`wdy8(kn!>H zzI!|Pt#a+4H?a{t%9$;C#12}Y$m0+MdlsPcpa1Y+Z^K|fx}FQaLE(m8!gdk-q+63z zEB?0s`RJgwD_~Z=6Uh4^#uRZs{XVC6q^!io1k9xuGOQQLm|rgQowfA`??yGbeIM;5 z3UV%mt8e~Tq^&H^QCI;cE$z_L(Z_mX7RJN)Pyr$>y?j{u{e)Xq;|=Kj)0=@_nB>}} zoYG&z!?Z^umAp=qYHfupQ)a(gkE&fV3mKxIdK5| z5Pk5B5$khfFYPXmXYty&on_^Wkjs{o{j{$d#Qt*Y>6#|+88s~;fbn2@SpedPji~b)emj@jT z?{)gSnUpZ(vvYM}%B3V<+0jGX2T${P*CpXrLwPCSbt$lMYp>ZiNFDYJ8rCaFl-@ikhvXx+wCopN?-5{;Vp%_63S)PD7 z3n-P#(x*GB4+R{XgqPk(CELTVT$!-$l*e!odV|q~Fu}|7=C<_lnUcyjE5vCNAqJ-E z@7eO|D2PRFu_R}sJ(-(uzCA4hB%XmY35du-{ z2;Vavs$sg@s2lsJL)=K0%hU9Gnu0i3>xiqmBWyoi zyYPvYMkZUlkX21~qMG6t&~k8`MH1-SePin;}a!i84j&Y?rrzlI*))P!(hJb?ZVo1b3X8us&1Z40SdY^+$}Fn zH${)k6D&hzeGNQI4z^MTAJaCmt%D=r!=r

3Tet1VpTUvf2R1t%L`VH#Ik2;I`5= zgues836U<-C@jm<#+PbO5+{*x^bPq zf1c%SSYLC0ZQO(3ASuQA_otAl2j9Kr`!_59_dAvyu)fAF-v2MlxT8_U`Rh}<$lCYs zZhY$BKmTjXo1X&kxas8A{x{g~_bKAL!g)FLSV!!%#D;?r3PM8Inwpvl0kzZzx-^{M zUdQ_`%b2dxlAhMWI7yiHD8icNTyJr{it2NbYLAdGn_0_r}--Kxw`H`RG&Z z5w@t=!4Hj%1ptr7VY21Ta;{thkh}2nXA_oQT$*=##>a{}=Z$Blp|s}ySqG9;R~P{T zvF7$;c?l=4g($yh5D)4H9d!PS)OQc~;g{GO5@Oxj6c@nV+6gtmx zoO9_s!HSY3!PxYJzmFPRmZ$$*q9bT`6CPC=!QlAqXWpMII3hM#0Z%zKC<2VLgnOUd zJNP%a&wxE}jU&=)JN^5t5x-cHl=4oolb0WA*zr3(ymsBrKZZp16CSNP^-pdnk$A@W zN$iUkT|(eJs2i?h@R0+*PGVr27}DwxTQ(o-4PKdR@6%1@ z77Y8>28iK1pGtsq`U$s$Ws%dhL}j1iCr@?*EH~}S3dJBo_Ry$gHvlAyJx6t$T3daF zhQQAgG4*$>IL zetvxkheo4&or^q@)lmIRz*yl)W?kuXMm*C61qEm?r~GQ5%KfP1JvreQ;1cAPE6A@@ zJ$B#m`t|EqUdX`vtw4(IHC}c7<8I4h*EoBS0!+I&RbLsGpzjh@I%_B$eVju&dSC19 zzx+Mkm8&%$Z=wl zQzPE&(0w5B^X&uD7|E8u4^D%h7_J?iNxiYe$_X@)OKl5)*~hb#ReX!oV)2t#?}!4# zm6@mXdHXgt1;B|!uuIgH7#peGnlK^{)zln67JX*VC)Nh?)eVDi+{5bTrY^;M0CIBy z5U=ldx(J--5mERQ`h8e?7wzg-Fa^O=b^yHCk#8K)nG>fh1izG#fAuY!KM-gS(1qN< zgPxB!^m69Y4xQp`O7m_X4Y z%DIolxpy_O_YI0sqR!E6i^2?xj=2nCANlI3y*ql#1a^3rbomhSg zN|($rV=oYTh!KVzoB*<(DM^e^kG)XV8NiT95-VAm4xfZ>X@pJa$Du$v4rg7}yQdg% zn?y=(YQ!X~J3P&Mda%mv+lR+f^6V**g3>p);Af-`$2YeMlpGa+v|efT<2k@;xdQC$)0HDe{EwtmVJ$KY zFm#Nnn(w5PDO``U07{h!?{=80bNjAZCDKI0h}C z;-a$r?J0gB^gZZVm|}B;ZUb69x_~=IOia=J71v{duPrZ6)rD3Lww-g@QVCd)A(RWL zBO8y@Su!fcn*GHYF2ELU&gmisJw`@C=fijs8`y=czJ|C#MOt5-D{NHYY+3BK;}L4q zLzLtL60!_I0!-_nv^dz4o#JZ!&2&KY^?;HDX1{iDwU&PrkjH#xpBoHny%p`JA6%w* zh@A+OCOH*mp-ez%KLG^{^VsNgBb&Yt-#|6A2oOGhji6plR`EH@L4B|z10tB{Adnl8 zR>t+VR8I}m)@oG%>XN* ~7J4QBDr*V%?qR?tr9Opn3pfFQJ@Nd*_h5kv;BqhFyd zqX_V~Y4&Vo9w?prSX%xU;p0~}LW{O->xSWG$U;!tl7p<|*pNrQ@|-_vm}F)AHiub5 z+M3ho%1#`d=|vp-!;E(_YId}tLQnvqpIUsNJq&~4*tLH`vXUsF6z~c_L43RgMX6); zYiuSj={6W$qNxv1~WxE^1SRywo; ziwl#CB^tu$0ie|s&wgJrSwfn*mkc+#S`7@U!utp<{ajV5b=M%<)8WYF;}9@L!E_)K zv7AbpyHHoy;WDgA=wh=6AOso`RFh3JOh$HExYM$d1ACvJuy`V-XwBxhZ&1jxg0%JV zc#?k$fq8W^rX)zSqBCXc6e~yT4eUOt&ZckiMjYZ|tR!uam8#av8nhHE_QqOgo=1J< zDvLz!GfTz6{Qgn`dU4H??$jW3Yq3_Zpfn5;U^$)<*r}8$SteleUYK-H)wTO%S5~hG z!Zh`brSwHX8LPd8@X!U*vyxk(vu27+&^lkP_Ia=yXz(XlacUI;cCAi=o1Mvtm9M=^ zVig0PXd=f`x`>iG(x$?FZK@ayqDrV-eO@!H){kmyu0vq{N9}}geEoT3*PbV!?W7`` zJ{&v`eQVeG5X9}8c6Ml{(G^^%csil3Z5Lw8 z%{N;(Uc&6|g*VftBmC5YKqa5gu+@c}DntyjKs^!&qY0%?_Lbxw=k40ggCU-VRi(@C z$-wC7W(-4dWceT?T*}ImOER7n$1)bxRM1n_M0p)cqx^EzmmkQW`f@=`ht|{o)4l2& z!4r6d$8=M&xiSu+&FBwy@EgQW^HX9JJ>VZk|pUx_}+F7%YL7>pB`RvKcm_Q z%qoIiym;|lw&B^~ZREkK3jgLIIh|FrQZ@Jfvx~jJS6dBDUC@ShVX8a94p%J`(`kTq zc(C}BbX}QB>Lkd*%ZqGho0ZPxO&=JveTw;N?t$D~d@)K`ub46qfm8+G%)#v?(bmCQ zYSRrsSK)73RAsOvY4?O=9xkG0WHlyiR!$8v2m`>GI*2NJ7hO1*N4troT=ewJ2ZP?+ z__ClzSk9ZduP^wf;NDoPW>Wo%QY8=)yvhktyf+6fg3`9&VptRx%sI3vz0Bir!{Pi6Ll>n$O;EVjMSZxEHxi;!0 z)6b5Zf?#(La851pK`TqMncjyV_aa!$rz}-U#qH_JIfIgy*TD>z@9o83_Qzk+t|}$m zPY;?9z9l`lJr~`%oCw~5B*SYKGYK$|+O^i+Qe5SMp#)3f2u3)UJRHQc{r^O4hB_d3 z`r^OhS--s_Q3Df#NX+lO>7=GwLS#E1DL@lafgJt`w3j22&jH$@2%ewkAz?r&!hLCeSUnd2d`eb z?$d4G^t|j&x8-@`gG-tmqSrwwDcNPm2o%C1OhFcA{2c~CQlMh2zVQW+Zj`%qNYR0k z#AiG+R06mf1j>+sB{l%`lyt9KC9XXqmpM5(`)wl!(O4aJq!s+BV4_uZKw0z4QxYR| zqoxuZx0p)zpCB;r;vbZ#ERJbj*%b4Cq4TwvKQ9&pFf{V$qsA1NBTmCypaJ2Z%1N3@ z9c<-%0F}ovo0+Ls&EC2J^4p46M6Khq9Z0+ICdww4cDdzhzz$~Oo3OqMt?4%`1>PhF z{CvQbD#4u?E*VbTYVo#6aegQrw4l9*DoEv+)fd$+Ux<|RqaW)xgBFFr*}rqP0Gsyr z2bdLSn5-sqzw@Qa?VBB;JN&7n?~)${7Q%VpciHU9PF%@LX>s3=^5y+;H>6xC7*z91 zF}I?eq+ukI)!AJeUEG<#bz=DiD=SxIHi(+v*BISnbP}6lmRbn2KbAqP^=FRz6-k-@ zk<(@ojAlbXja`BWP4^vr1=?_yCEfzfy@s(LKOg;52V#iR02#;ki+Bx*N0U zu%qkQbhB*)_w>hi`&bFn@!e-ZBN-e@WrO0{^?GSB~}w)k&5!gsi?^|%;9~XChc(9pv(qAijumm6F$5qIsx`0i}U;}wl2~c0xvl4kh z4G6*G+O<@`PY%xmDb9t%ND0(**@+E!p4Z9h_lHN3v)>apgQ@ikl99-YkZb%=uWISp zYa4(6A6g~&{o&uFNNEznihHB^{qr|9de>@zYs}EV-AMk8`}-@$_}_9^yxIEh1cx02 zmrv0JyZK9I>hn7&6+_V z*AE{)yhDEwFnmB262}0!_Eb!mPt6`d!CsK6^T+2_D|E_Fz|P!?AzOE`6cpdTed}(F zm7)NkjmiBoTDrkCo{{{g+|P0MZG=b#sY|nu`E!lA=(g+YdOWrr)!L_Cr00asyIniR zb&q16Z+NxOvx*`WE!y@ho~T$ey_ZQD=yJF785mTXvZ7Ak&(-S<>^mhjwteWxndAz= zq5Ksa8)W_WC{l23J)ro$>|R=K4RUAD7fV>;G0^(y=`#yzA>pDx-aTdE#~M*4&3n2J zsHELMno@%eE&%>;IBT)%$74yRgCa zr4;G}D+i2|zS_<~45!4BJy_OLNPc9_-^o&j}?%cFXrb(3^7s|KvGXd z3s6w#g+DLe-0BB8x}(8;gO&bua4k=A7x9C9NmqI>&QSp@iC^e~j?qAY{iTv6TM()b zA%N1X4@fDT=f+!Omi*k^@#NPzZ_a)OgP>$5!H}u@d%dNesR1O}@BSsr(C6SZ=u`3N z>FJ5h{HztysxtQw2d^k@JSYR2dnNlC%B73mq|wavggKwboEzIKbT*ua;X?y+@CYEY zJZalj31%I11K`r{{`81`0lUsA`9XV1qA-cezRfveN)(*}v{ zx{`NU8C=hs;H?8lmuv4qetK+bF6T$sHln=1^`H%|cVx{FynVP>y#7iI_gWOxVw2~_ zKncn~S;e~e35EhSn5zJq6jH!kn$A>h4%64uV9xtT9J}u}1R#5ur z(FO(gekY)E(2k{yw-=&fU=kfS-xxpY&aWc@(I+I8?9I^&X<>|`81Uq2wG8baxEAN;=zr;%=rCvq#mwINBe(EPE z^gP^m+Umn3H{rlO72iAxKzGKib%2pb2W5i3F_7_|3Wwc&1$1gM9p)mBz@Pg-2_V79wKD4GALL5@>y+M=!|5>EJ|{g*E%wuK+? zv$^F~uDdC7?66wbN9eFBY4Sm@PHqM9nYZQ+?A);cqA%bEDS!rxDKBJuJ5$}r`A&e7 z>=D|)`M55;_D1L{&J5zvGLZ4*o{3RMoBeScR43wKQA^>N@`i`|Myp?+gbTFi>jK4_ z`i&rvBTs^cOYF_<`;K*+gBh@nghR7DV9O*)EIzJrKXXB>;C@HryS{870A2CXqFVaJ z(tG7#NEEd&k9{L*ICE<~?8L08%;v)d#>0Wc_0m@$X!N0+fV{>;zzG!>uA{T4Xyf#L znS8cyzqt330oBERJCKH;&puQ~TZ4`S^1yyIh_OGIl~n4n#(YRsFzA2rLt@0$2c-kh z?EJ*Ezj}@Ga_`tcD_N@B*Uc-M$TK_yxpTAc_y+>p**rZyhXL~W)qHU96dytR+D4cJ zp!8Z9*++q8ZKDD2JfQ#`^0Grj16Y!p>N>>YEbuQ!0^)(*EsI-Jse_V5x(4Ob69#G! z0ub!z>~v%Rxz_OjK)@bqnFVu5J{LKq3pxaV@fx;CE@>d|swNyrzg%=71?KImJv7G? z+JJCOn`M!#Y22Yq3J(V47#G*nUULvG0jRypJ6MNtu3}M&(y+)q(SPY%5ELluF-)Ho znG(GXWexTM`Hb0{8jS%^Idh~votVKy5{7^igSSg42Z#j;&A#t6i|Q;o&?3nD94h&`@o(Jl@N z5APnN?uHx>&AD5YXR+*ObY3GXpJyyKtRXTA?tXSLk2D8 z6ACo|YV_Mac&rPAw5&E+FIkovaRKniis+PH8@QPDtY~g%u>{)Se5Bh(ke_n!cL7=P zog@D^i1ATYUxDnpgjMsc4dtSMNjjI75DeUL7TlqO>n72X4-ky`JhD?O6qeT}!kwB1 zL!pW3uqw<`Zc2id(E!24TuN3pcb<7T;7i`7=Q`uGc6V5m;h&60Bi}%&*prjxVT!?i z#r49`4fD^XEq9l<$-v9If#i2`VHu1Ab6uG1;ae03YzzeuWmAOXrkUMfc`ov3F=g)@ z7EAyXtS#t^zGc<*jWP$$rKgTZ-6~tu+={1&$Ojae zyaD!mFQ5cOmnryJ&{5Y5I3TgP=E!f4NuA=HeF7;i$M%|OScz+QDS3{ZUkr3XV`!id zeZTij#-N0jjtMuLujDOt7=?cr1W*M8Vjmx0+w$koU#iO5D+K}!Xl^u zu={8h>-zTbpmG&LNML5(n_Ws12YfC|_oLNkA(6>ol4NG601}7Tk|yaHbU%(&LyI8X z`9>mQDPW#e4tTBJZqQmpC@Z;Tfd>I7r_l3XcIGd1*!^wPJpUlpOk`A5fHKj+MEz$ zX;6#N7g=Ngw(x{4@b;F3&X|NGc%(oFC#EVj&ll*^p&Bj{XtNYQFi0y-GPo+$ zNS=3lX^`YA$T4xXB6QjbB_s(l`(!^vT81MLt2ox0Y(v&0L=(WZ>jjPl_5)HiV7?vgps{@^IrGJHsjqEfg*qHL<+dsG-CKWvD3)zCYx4oTjpo@B+PwA8 zohPA(=yg}7tw4vEvXB8LUa_|uxInj=uW!LZ3;#|)Ke3S30cC(ao?X4$F~|eWSn`ja z!Yr4$M~R7W&lOf;4kpZE?~hAFf^uW)0+7X-vLly_2*HrL_~>+xwiNp6G7N}B|h2M(X5fPFkX`y9gu!-$Z^uBNphP!^VQVj_jf~EP%l5{+v#;-jwI0!%u3{OY;cV~q*$M> zZ{()Q^(QD8EuMNGx0!udH;{~qUpWV|T)pA!Mtu6~S%S~}ld%Bx(pNgAVgSmfnWuxo-VHg(Oq0}sH_eyQUVt-vjjq-n3W{^YEy&leONV*pogBcU%&fr()9=151I9c2SI77^;C=aFLvU$h6KdX=HIHKFDs|s|M=hL{BLsr ztoWbhgpmGst$w+_Uw-3%KIi|94!L;Jwa_l?In2zg?RoBpA|CaR221Uo)$q1Ic_fi^ zl!9#E^ApWEFV5{&JEr_^O7ZV^9lv8Ep@PF;zlE*2m6fZ#y~Fs5UwEiRr#bvUIgGl)R|q zG+||{%S1vIsJtv04tt7`TETv5T&Y%8$*aJo$ze9Mem_~C#f)?&`xLB1ly~R6vR~f}03gnuZ^D%5bVl(sn6mypl>g65lQ+ckHSok@kk-DA z9{exR>kl^ST5|*oGs1Iu4(^Kny;Hxolx4$fRU}YJGSvDn$Gv463Hs|(%B{t<_V54Y zKsT+0g@;*RN%#I6cH7?ZuW47Q-8*~szb=-=gR#C;cKkP}$%{7s4HErO+dXgm*Tw$U zuJ8Vzfv^th*U`oQ&p_7g<(GkAQM-!~`-%c9o>x$+SCpC*1RNk0hLVa-UzD8S${Z)u zWhXMFrJM6&#vNY&;S64i-YMI>DsV<{E zY{3fTaze*`X|52nN8&{S>hFb$4kI7sC1&a$FS1r;TX?VBTXrrYn~yPkFfesC8HN z&ds;DHTUjR4z#+|S5D}KOS-k?quyi=nNK8%WF?wk9g@LTB=@{)d4}8MNP-`+Tpgi; zZ#$-BA1_2NK%rC52#0vnFnzqJQZSWge;_Q;IiVPe!}mS9CS+}_9&C?VY0%dE;x8Mf zg_j+3?74S|tcGrJ6DzvZI)JE)ktB$CxVRfOyP7%f;Xa5q)1klnQgSpD^A1YD>Q-tZ zCCAXG=a1Y>0S~d88$)2!l?cIFr?$|$aAgYE8?Z~Aoh72_JEmxXr@w!!q;JCPMjM#h3gfb z;TWluHPLbKYOBhXiD-XDl=S3}aWB&3eqk*m(ZGB<1@7>4bsr`#S`m$pI!Bt+cRIxJ zcRgMBfn<9ME#vWF@8$aXsjcPw?**=0biV1KK^MKd@U1E#$KrJ1C45Wa;uyqcIl2FF zH`l*w1mB^{enk-^!SD%fJLc#>Uj3gGO9%0QS8hV^i-)fgJWQlLG>qW;&I$xOfQ|WH zQuR^a#&Y`NAx)%1X3rO%J@GJ~cYR0x>J*{Akcym~kocSOMYdqF-=>SL|2bSkS&J84;V=umbyb6<0TlGnPL)>^2{Tn_BwBnf+u@%)76$>&Y z_Ct=nN0S}<#^(xMSGUZ{X1+`oDys74X>P|_z6rF7b>^!6SOgqLg5M=Q5l%AuY-P9|>)vpGkTNQ3-lu>$WK6U0a-BL|c zr=4huh)B1I#9u2>``($Y4H}_8=H0dl?|pu2YERxZy=UcrR2QA8tF}yX&aFyxYchv$ zN8c=H-lde1!`by-f1jtcLl1z)^MTJ_Bvdg1KVX^+w)FYr-t6;Vl(Vck_+?D?Ls8dI zOa7Y$0z@i0ONXC;Qv`snoA^SYB1nztdfI&Q+c8fvX{<$*l;{RWL5Q)kG0vSFDVd;536A=JdDD} zAXMzAy*owJ%I#GWj#M^((o-m#OAhexvdlZy#Pn@`D}+n7O)KkTaJ>}n}Zd2=Q!=vYS3Y`#7;GsfpseQNmg-XW&Z zH^DR&-}6$+ly^Aq_MSeWt@B6Bz6MdL(ynvQ8xKCyn({yDX7It|-OJ~M-44rlNi~Lu zYd!--J)uly{_6{i=6J5bbZ+*vtSHHRt`-?@aou}7?1Sd;u3GkLVpRN}@$ofBymqQ) zbTDTUY`<`558DyrchVWu+jye3WOxY4JgNKo%@ulvmH$Z3Qs7kvondE!r>E43;J`rS z_@>CoubNQxXYC5GBGbqn?`eN%as^n=RUGPS4V;-VD{gA}?oF>LDa}&~=)d8G*AJ4R z-R}CPdTwF-wKvgT2bv_rIVM{!H)Q=%kajzo*rGjt!Y;JQRLp6LnPL0k%3D-gYs=fa zjgvs7e;sGZL7`fln;O3xya@<*{L?^DvK^D)ZNzAGDek_;5gL-_@xs3R26Xfesxikr z<_?N*tckX;1KOp_g*lp3O>8-5hvmwLP_a6zkm25Z?KivnYd%4Sv2F9cL4>?GS#$jz z0+X1L&mv06f@P_Hf0&WH0F7kMJ%5K`8%5ps(lT^EEBngy-t6hnFFi8^HKc{J zeZUq&Pm<~MY517rA$|~Qrr|vDC4~*{pdSIFW$lHng+iYUl1qln&+(etPuw0Y=IHa2 zt@QsEnOktESfb6}L?pJ-+DLW$i>u2Q$%(0KkXIMSV&z;*2Db<;A&%v!d0!3|aL2=z zvw7^?%iSY?)a~IONbpi?@)nq`D(9GN+j?n$<1pr#xQ}r){oH+d#BL|Qxg*Nqg8PWJ z>B2wF4p#(j)!IU7U648reZ|4%V9q?;I_P_pP;)y)N7a9EXKC`V%ZOEnY1J6f%i8Xf z>>QlEGA!^&@|8I=oeko1t6V^J1*RCE`qF3M^ODM>m);J#Cre zGJc1?4_^9LQ^*!Cuk4?+ARomNy4V9T<`H_*D{rj_3`C!Qfjeqqxdfq~Ng=93mZyWl z-g9E07jY4YD2Pm;Z=>hog+4Cr#ZMRcIr|yATT4gP6Wrd2^#v8$)*fD-LfLLkTnX#q zmMN(aQlMmR6i zz9(EVt=J9|t<-K-!VM>P9F6bMFVWlesS3;<_{M1z8PSZdmT48F)9NHk%>*jc_5y+R zt`g%Vg&WCBeOr75h=VFmr#=xl#0IP{Vv6~EhW2yr=<@iG$CpLO@g0(=lI2d-BeanF zrsXJ~u~O!RF(Oq&nOD4=L~7s!YKJ&8Wor0G^W<}7FDZC#$Yts*5m|jsxi8R1f0AA0 zDfb!1cgxzVUixy)P{QV?@(IX)tJEVGLO1A4sc2s#sP!=sy`M4vATURKkip&QeS|gJ``9JA8OP0#|Ad&UPi-gunJvu_IAJ zJF~hL47cB^(LvDpTdn>Bi9^;98D)xZ?c7(%z0%$j!S*smrpnyxSY|4(!wJ+WRDk?TaK4&NhQ{6>mb{wCif`z{oOC*0IcQh|$^p zx`Z4gHl=FCA)3LYHHy0)D@qLXNagZSUsh=vsN-a-YiQSd)@y@5adSv*t~S7QVxQYz z{7zx5;JlbO78O@tI2I0KVW%j$rP1CzNBzYy4YcIkorGr(6fZDZZHh}f#c0jjs)B?$ zdy{XU+*69}UHZg@8BF#wn?SjJ@w1!=^wxYANMeF90x>mbYtm!`(z@b4`Hw6{FC_W4 zUPv^|Nr6c}hfs5QXwet1`AewQ4EZW`_?^d@cZ3{H^6WnaJo)R?=Q@~s>Wr0@`_q~# zUiq7~szz&KS3-pzaL^7$ILz0Q`K!>#kDe!-`h;9ZWGhZCCrrB2~1ydPZS=e z79ThM^m58R_k>UTF(Kx}BAQXm{ysTdLJ=E%T)gL7Al{TAvIFy`t2HyMrreC8G?fT( z!rWs{i}*{ z8JFtFz>?${FP4HXy=FvflG_RWjN;1~Jia5UYJD3zVF4pVT%QNze=nun`A;HA1Q$_H zs^t$_CC;S4C@Stfok0ucK{-|wOdxTvtt)5=I$^#H&BjC!ucTMsC0!ZsPIo~oS!r`s zsnQ@s-fnS7^4aPX6Z#{P*r&(VxfhK@rC&jFuTl4RE{$?aPu6cIjt-WcxtF437BnTo zymDP{zX|EpcZ&FUGJ*I$riY{jhi;)u_0>(cAD!Akcy)VX*1>yDv>j{kbjO*xp|0y= zGq5j9*ryT$vX}$r$zvDZqU08ngOc}|a^dQtx0u>^QPsOiX^CUuSGM^?Ue!8fGApY4%M2vcz2N;FY<6^3t*FOg zpslt|I6s6mp6M>fUPqX%b_Va}D~mxU$yX_MZ!SclAFSt?LF<;qW)n0u@0bdD*L00=>(+~0Z@5)c?LX<0mC=FY3|G?`F=w<81^Y#N%j(@2`NCC$ zY+)xBQsCtgYnOCb9ryi&*bO0rk}=wDX`)B^YRsSjCm-RYb%u^i6^fPNVPOljA)y}Q z@YwQt#TfO%n?{Thb-|uC?czm?m(Ts^j@cWej_mVETy@XGBgNnV+MmDmONM3VVov&Wdmf zUG7FWM(tvhhX`s|&Dwn{KF!1gQ4<3f>M=N9(1EIBKGwt0CwMn%9EJY@<~G`P7K#jL zy&|Dd$SZyOJeZU#o~XN|Z-7~XdSn-0cmf<}oz!THtHF|Ika`Plt`(_U-x$EjHH_&}E|#kYQ;gDy0%F0P#TqJSFw}Oe8KxKS`zyEgAhbiR&r}T+2dhAYtaYp-h@_&IE1Vy zKQAI;BimST`p`VZV4=Jyd95!lhi4cer%vCH#DaS*|IbJjUeGVK6GNZp?C$8(79c1N zK6l|Ui~%o+XLgWGaeQKs7?tvlJ&g#bw{NC{E~x$WQ@xE;kp_dtVOk!ZSqV<{o5C!* zFU^D?q%+Xg;6*Rnh+8fxpsUAxQlFnz$F&sMHDqL#pKaGT{h6LTv7edrGhgjOd;U~n zQO};AG7PmZ21$?NT^8_fhu&+U(l1?V?$~u6I;bQoy&XLy*XAMOt4gQ~n)^DLZwYD# zsXV0y@kUphs$*9Ajk@t|@KK&1w`;Ohd=+Wsw?QA;A5`?ZR>AfYPxdtG|4*kRcG$yn{+Wvrol!AfNc z8T1{*^@s<;ud`AxhpGfSS@rynP$Tl~8QK>Sed$ZJLZVc2&2V+om2q0+wb$)M~wMXu8D7=-w!-lcYCtyZf}wKx2=krpJ|*o&?Zu>yWjDO& z_K-Q|{emBN)E#pd5sKubjNpncBaJAlMiMZtu=9gM=Zm4153E(6*f@n%O5jzh_+nKuU`;18mnCxPUp(G;YFmIn#p~6$+S9MF+4UQ zt@WgexynRFT55QL+&>52m%@Fnv?E$6`?Yd%5M0C$px`;zpX%CE2);D4gM)kX^vbgR zu1NHH9G)Q5FsnaAtDNYtS$Wwm?WTV(xdF)fKUN;K=9zzDQfOxo)&Kq-J@0%MA2yA) zGUw>%nQqQ7uqH5p zFT!>$Qh7R;Bv&~DyIQvRk0facL7dKG0xL&GY?VE)Kfo4?(3{vS_owT*nmv!xYULKO zGbeQV7~x#17}aCmUD~lyJWzeVRa}BTg=dd2cHm-$WF3TqXx;W&t?6it?2T089H3Hz zHJDJV1}n(c@s?<8Cq>wL1)oG9!zp%&=>UiZsT zJMEeIqB!OH*k14R*^K2zqpy2$@YZP=Nd@Vfm{Y@_1I!0r0i&BI8L9_qK1cQ(|C%J? zr{MHZ*xJ?{Zst1O{LwoYnXw>2I~VV5jutOe+bJxMw~t-S0I=kLPd<3vo7U|Gk+PU(D2%`z+4wz(ris*)58F@V&p^K&H2Eg`qgOdfhZ@&b&e@h2U2ctHT;+~ z^kw$##zwh*6;9X6)R~f0F!6GE&n~Hxrc3y?0zje)O}I=@hgOcuB8YClQHjpg?yz+0 zpD^0KPpy`2)TL`Q;L&x{^y~M5qe>ivlo__4Pu~l5W8~$hSN8|HWpBsGk2@cI)*Z?p zF~G4)n6sw_{@nTYZB25y=^NDcuTi6|C(sILv!NmSYDo`DgI~@1_UQ(Ebv5FwNHtu} z>)6?qhUAFLu|vWMoTvK_fl&>Q`&8ytQ4<-Yt}8VIytBFya|ue(l9Q^Zsb1zbJ|4br zaZKDvICN=PJGBDlzu)u1L{A$1!gTncIAikmXlg@3TyOJv5ipth>21RftWBhv-}D36 zvmA>T&Q<=;D|mRV6JzZ-#ElTm3FZLT%PY-ayF1l-Eo)ClX_lWN!|YR4D$lOKAQYGX za&x!%ZdxMMYPBvm5{Ig|(;Oh*fYPYnR4(3zxWr;n{VX-|RGqSgVH~K~_`d30f8uOo z+-&qXD}bJUfrrA!NvIv$ks!-kY*Vim?AAan&OPg`VTv#tTB){MD)F(Yh=OZO|Nk7k zV{NJ4x1$xSR&5T4T~{s|2$VgRQnmPaY@xsD+`>qDs*aGIbU@~#G4q-e#S%PH*zKQI zj;7@;5-fV~FQ|rCJ9BKOj}8HE5g84J3gu!O@`aX51vM7gyaI&UKuYZ$2?pl1Hsi;t$Vf-W~A#S!QzU8Dff@6T4>WAg{^2x0?f z2}^gsUzlDz=jbZWuqaWh(|zdtBk9_Mov4$s$>KRw`tiqJww4=#fYYmWfZmdsR_+W& zk&Am^4h-T9%g9pn<|h=~;oY~lNp??En!fP^oZlV0y0D$*ZrK{x)R?#!S2=>Vu@Q=* zZdrQLH+P^(|7}W$H>1~IWs8HGO%}$05nIWbx|=%%;TK)-v3WpDMo_>b7mxqZPpAqi z++R2~(C9WkwWWj?E%{bRrsXDa?qhVZTZn+bd&?4-iPvGJ@IX^daWOQf?+>52Vqc58 zQgwcLNH=~#qhfM@-e;3S_Q*n4BI(;u4v@DhB&4c+L<*ZlOsKwp~^{c3Q z1w4=67mI5mNrUY+JEaJdIT%I)R=Kvqs{kHjuS;SFW!14jjVQmW4Ncz+~5Fo$B}VO)8Ls6V8uX&s+FaMc_MbeQ}Pay)wdgTYf!RN$X) znT3!#syqvm1||e!F5OyLFuc9cSy3I!&7XLOlFr$Giy>%wC2``VQdQi*Ez5a!@b57c ztdLRb6ewSG7Zijn#9Wl#ZH+FPmig|z(2c{hKM>U1`4=$-*mskRUGU{(iKej;!lB#m2vufaV{IHg`h&Y z-q5+GRL&09vm0>!sa|r0Y_=tV>K?6Mt@z;P*Y})t$jyAn+G+^Le(8?kkuiHS|KrcD zd$pAHJ_8_tA(4{zxNhFSe=wbf^zUwyt`Ui_>`kTVU&U1PT5*ZB%5vIKcFWCh@Sn{w ziu_%bBrM&(hS+%jemhyrZ+$nOdGYP$ef?GY=t%p0veIg{FNJ?C;`bi`lCNOW?=!## znQU$b`@OpV*dl{ijUjj=4p5TPHjkZ$Rv*P-oXPe{f@PZI5|=k2nuUfnA8rP9Ep%gp z$_TGLRoQ-N0i5`X2xK|e;gOtPHLG{aeMP%^-BFDa)d~PEy7fOZrrbu7cjtlph_3`x zCMOY#j(0n_GDc8Q<6vBMN2~q>bNMG36cw*90DE7VU;Lv-^{R5wj@}roTz_Hs=2Pj& zT!(fwA=MdZ&^~SRsE9(o+9p}>8U&ztiFs9z)i$wI?a*d^1)Bq|H4!MbzG$+WjpyW@ z%#NJuWmLnA$8@!~NCkOe!E+oFq!*z2d7&@b;*{Q!Lh5oqVJZ6iDh(qgliQr^G3Ndk z+ka{${3;5bvujFy;OUJJf69ZCNxI_d_{zlAqot*I2>$fN4B@&%e3Y>9$O8Rqx+`5K z`fFhCGc7Vv;_5$Qz@=0CY9F^N??03G$-O-A(z|S#QkB-R02r!mvPUQWo1wQ2M z>(fLSha9ERt^8xYs#FGbOZYLyaNKTdzr1@EVP2Q>7m1aP1GSYmh#BQV+1wCQ+!-@= zfC0E{Rww$qS~~o&@yAkoO=Z;J-64-kVOlbqNFG8qPBrs)y~C6Xawq)=-NUlQbKNM> z<r}EG)btRfAt|($ z0g?g|jRYhGW2n8hYLZ`5Z)wbr2FCPk1~{423ZNf2d>lkn5g->DpYiP1?DLuhz(%7& zWgm1zkDa5<eHvz)=PbpF5r&6=7eIF_5?$E=hdqWdo$yXKRo{Y z*6Tej)J6}%{ZFC_S9-f&Pq}e%orn-~92RyAE}n{Bii(q^67#Vx(^L%kGqW|Z-s!+Q6daeGhu?; z6ON}+4gKT9DN;B6!8~9NPL{s1cG2xzh)Y6ha_lLL22 zqY|-<<@+y)Tq6H!a(y8}>*lzJz$+*HPRaut@vw>I09zsZHk(2>KcYxxugj#7S)Zrg zAnZzGjD{6X>AX)#Ka+`AMX{w7Uob}y(sfQBMO4__U2Rv_EVh%YP2 z=z`C=c-W;?yQw164pUO3sD0@b>sIB%D4%k7_S2;o_~%}iap{S6P8PS}(Gu7$SLAc6 z_l#G`-AdM5jNYY$pMvRSI1^iLms4lNDH#;WmIl*$8|v3`?Y^^FqXn(=dB9s|ka`H2wCF~n z#1fpbBkdC5jZ#9{`F#ReuI#M%m@WH#BJLtY5T&z(R^kv8=njMGJBbpw=NW!}HOKr26Fxq> zvBpwUW}FkK!mP?lMMhPTRBFv5hbX73Ngj4zY_+h~6WKnYh0{{EERA~2rHgOFIlVe5 zLKo(2m*!@zi5c4eb>ZqdH(4QklY^pesCi9$W61bebk1%fM-&_lX`l`Z5s7fOk47m+ z#lvxHNVySj#Sa0fojKM^uJ-Gg;Kp&6Yt_(qTKbgTdP@z1y{0b7pK&0 zJ*L#n5M4#z@S-k@3~YP-bcN!+7S zv}Ixu>dAbl>r+00c5X*mCgN4_H84q zaK~BWu7S7+Za7K=aQ=mNhV$ce%87?>_9sME5E8FxZ{1c8HSi%yBfIOIFom-LJ)|bW zJoki}t)B}EX=cpqstdu54Z;x}B?^aT`zH6D>WXupn;9q8jQB%pEr)U;m6oJQvKPU6 zn#(zvZKMcNzUa89-AJg4!;@?>Yhq!RJ}+jg;6g5SO#gg4N3?-3L*YXXt$I2GI z^h8aS=uK2-G=9!2mGjB46^VAf&giNlD}Zf8mN(wZ7%!*lVCN3CO1^!f5A&EhARg=w z6o6&%aoNzzz&NOoW?KLoX^(N|8O}tnD+`>zL8D*(_dQCnr@>Z3B4wHEc!g5i(*%sv z?@2FST?(v|J>qPRTA`X@1=Gunp%TSkA7`h&??7O!_2+d<8zwr!^Yeq*eYmIj3fc+R z8E*CSBPu0Ihq<-#I&7S;Ghhjp70M*gaV4BCk+Sf-`r*OFh%;Zb*)wtmf(@tYUPNC)A4dyT z2colPtJ@1Ky=rGTqw28HUhaWo!8qmoP+VZXr_n71{{MU^A+T062$5y=v@7zpUnIL` z%D`;oXJ@Un;3)KVFTZ|&ZKa78>3rM8%LMPDiEymTeAATnOwAJN5h@>++2Uf#cj)Z{ zvCw2jW%n;{MZbN1Uhrw~t;0?7Z$+l|wK+?~$@{bfpk0J?RZ0(#EG?B^WI2Hu@u6y( zEbT%&Zkjun0@;c_PJSq4oq_GB2}arBdGWV)ATXQVyQ}Ie`!!T`Y{R4XGsUQgBP&?H zD8SOTVTGj2aH!Lbkgk7x)Vx00;)=DtQIoZB!99i5;;yh

HZCU-(p=v}?KFxQWRr zLkK&=W#8tm@x`s}V{_%|z0LL0;~_WIh%Bx?i%VqkSIheGT}#^f*A|bQ^bkU-EX8`W z@hLgh)&|*QE2vYkNPP9z5`6%7~K8O!I6CfMI%lI#ZJ9@iek0)>V!zb zV>|Al$IFF~*(;PZ$Nt<&jvkQQy%q*WzI{#=bKj3}FGIb_J972{CC$Lytx$4SZg8L2 z*kwVlU-K)O@x9+iG)T}gY&SF1y=6^S5djXTL=aQ!;aqB_M;2H%YHn=Nsp+C)@eFQKOJnn9WpG zG+)O#=L%@+jZvh{SxfgFsd*ZuE?8@A;`|xcM-XL}-kb*XqU9(x5R7+n{T>S)LM(Z9 zA7b+r5>g%+5osPr*qW8hfrcuo@QULNwbqMbCAiV9?T8g*nbz<~?-PxvR;c^#FZX|q z#$PUd_qBpNVESak${4~#VUz*OEPXcb9Q)Sr)KQGHUypW<;)pIqp|r& zPd-^vRXL4>x$E*H2>dpd8M4&V_sLp8=L~cCeLh*|M$};|Tw9aLNhA1?Hp+@dN92ubTNx;6hWXc~bS$HsZ8Jgq=q%ELU4B1XDU8Dnt?uw2 zKl!kVZIor<+2axVAr0mm-b@66w zRpP9SlD^-fM4WRdRul-0!*M_5e+$v+p~+k#N@wMhErQsz1!mn9epU4D=W1C}2<36z zwUaGWN9%|l&gngs<0nA3bhPu?HY}h0x37svgH-Bhh_wNVs93LehdQKvRdhLb9(KKE zd_Z?e+RVK$@kWqlhNZD(N8~OJPUr#}_t~c`7pKYLIcg7&EE}3mv=oCYfdIRr zOovOqsCu>DS9&$2n=7JuSxWW0m_>ep4x!1xJ4{50ufBxV#rWlYQcx?!9@s_xA`lOz z4;L#D3D*Ty#eyi*yAr>KcD1VCw{3(LxQ~pmhEdf$WJxbZaJVVzfg@kXhDf%yG>f&EHH#@X% zsmkO%%Tdrt;k!ChHxO^9E%M0HasVlU##S^NroJ(cJLFUjCY!j9pI(?!;f(&eFF`LJ$kvkkx-0pAU*e@Z-GuKbFEK`&8|`Dy#*WT!(fhIJ7sSE<{sh)ZQWGwkppBXamx z_wn7?hvJ%lDeFuwmwrOquN6PyI5EspF39~-$`hwQB4GG;t@NhNgjE^&@Ve2dh{_I0osiHattzo@zM03fnwM;V7R-TU;7+l7F|kN( zG_9x^gE8e}Owj5*-0`SfOT<6kP?Ds@0M^`0FO#*PO^`x(Nc;FDkOl%aj1? ziL381IV;Ab^67JUeuN16nvIcVlr8RdGuJ=&zLi}UNsXDN@NwjNz>(@+rbnnV(VuK2 zI`F$zcl2DNr3m%z{0@@iiYtNAsA^USa%DPI1$|i|J^9-%zJY%|RGz_CXTLl-N2o-e zK!tM>lAerNF@278yB+9k-Zx4}49^o@BB~-lX8nm+rH#jFw6+(pwMkQzJv=7kvB|G_ z`h^4~2~E(YfktD-{L0`|XD=k2G_adIxjft_s5s)6bR`41YLa-dPYc#@bGKmHIgq^? z+=q-p%}%6pxq!gz*zPH(UfwzXU*mGrTI&PXGeZl@QI>=N&pbmg#^bij+-MM)R5xtG zr4XoNE4|Vz-ti1#!0)-sc5G)^TmC~)1{*{r(!C@LTI2`wd2=AmQ!e-1%7QkCSwM%t z)S5%$-o0xY#K|x#5IDF$gBzM^YZ$vMcLwEsjhe6Xxgv4Rr@qgehdY5}|G6KN?4hhI z0@wEB^w961=G2jo=$MVeaGmPB+MT$%;Uz0P@_p%pTSRINuB$KOrcYrQk_sI%j7t^o zjdHTijbCcV#q&Bj4J=Sh;`$(4X>45ev7dC<4EqNA=pE6Ny7 z2N@K8vNl8lcX3nfn3K0*NFmyA*PJQMZ=BZxgBlboyfQ-u13hck0Z(CBN&M#|QnAf6NBojX~W6YmkR_4}7 z+qb2Nl(xym|{zf#r=-nboX1HSRl%Ue4NIL**;^9AToU zg6K=iV4v6Wj6U*89+1FZhuLk#Bgq&D6o$TV_3h$N?;FiyapV z|I=Yhy}QX}f>h_}y6Nw`2|a^iYlS5?qU;V(^GJ|gqSkd#ZC(SQvZQ)#ZCsE<8(w1e zIWao+0~K+ElTXk=6f-i#|X%b)(^3w$uz< z(B(_H;ndeoELS@Ve;*!{bablsx+UuE2P=@xX*&$&@Kh=Dhl~@PS$uqgBDNzk zs2-ujQV+PEy4!+J9dSYXMt1=Ds|Az0mV=xuz7KsmhjDXV(uY>nZ2Vh523J5m&mmL- zy}MeL{JCtIuq2~?EDypLYxmh5WUbo(0*zef8j4oi!5lgUVW&gb*y!9n=oo3 zxZO$mw2b!4v`ligPZ+*?p$ej%A0O=Q$Z~O6S!fGZ3UAf(C1@$dC%kytfD$a%-m03O z#U04(pB!;|sXwsF=##F`#s1{;#iW>~5sRG3UFf-|9b7TJ?SiYev$!vdXAQ}p)JD<& zkix?;7fbi3E1#m}C=D6{x4KwX1bbpTu4GtJYy!zm*6}h1Ebla7RmeirY!kt8rPWe^ zQ*(S%D8y^ryVx6+rd)QS;PUnCSXJ_`$FOPZ_vtcHJ7|{S=Cfm)gi;|gDRw&jO#~5Z zD-cBF_2iKR6>qgp`7Aym?=7dsOnym52)B1QEwj_M-#JwnRQ^PUK<1Vlnr^7E zuwCe=bpPk)sXXUO#PM0M6K3H03%K;g$QEvR%Nl_l$Jmxz8pjIppGS3T?Xt10LT;Yu zPpR6CVEfYNpq%79*;aC(uC*~AW3^RR~=Wq6k9Im+RUNa zx~CnD6Gezw{UdRi5_7J-)Obm`c!-YlofVNs#Ex9m%Ey4s`XXnstN}vXXmzd4Ed8U+ zT#fKyS@vooa|Cs@(aQYP?m}9bf1$;QnviMs=JI6;P+plx%X1kJNJ_TD>b**9ZDwj? zj5Rk~EzCQcIrzl2Zftj1iMLkB{0nuFJm=uYftv@y021JF8D6v?w2BFZA!za2l>$e= zv_u%MjUV=vXAOcBvXxb;29{YD*}_4Uu=ma5ors*+=zR1UnYmysXOF%}y{|XiPMZg% zoEfAJUBxD!=YkdzKoSPCE>~K0U;4N(uAhlyXK@z*+t+8CrR%pc!;Vc3Uv=A5Of|eS z$eaOVaExHi6z-b)Vw~blvN3VLpP8Z6V|?-Qn?quv?u8s!m3FebSYr8@N%HyolS!i; z$M@3`KqQy==pDs;bYJWfuK$!vk!=@v>kG^RB}&6~BeV<^C?lW4l_uE3lPFb@5hdlL z9KW`gzWwllS?Q^!SDn3;4I(&D(y3(=Mni%-dreZ@*$Ea}q>A9@j$e81LeCM5{E?xh zOwD@PACuE3mSj-wJrpF;uR2~$&)J2PT{xcnAMQVi*<#NJa?Mhu9QwBnihH|PW|~u~ z>Io-#9h8e)EUch+(%CQay}kL?!avLKra6a;UUB=)4jw2d@r`oHn6piP<+)0pF)OqW z=5^@BLGBb!qCjO-#~Rhe$jx3ua`<>9IOH?=nW45|-m0cZc-6zvZ?)g2e~ z4{~fq$$rU~M&A>;Z{?{4oQwbxq+trKxN|&1bW2|#&y#Dqi>1>f*L+$oE5W+r^ zr;Kzhwz#C{Tb>HX)w>Q+5QNW`_FK1R^TvtyQIvO=wtf50kzaqs_VBMta`fdH`n~#_ z^+Il4LbA@oLQ$}N!IDYU5R`?n&w_1X#y0Qa;QGB_#FF=>(o1vij$uSyy`S~CtImcy zRQK)I`yFqZMei9JH?pG4nI94$PFZ4Fzh)9^J;vHY^e0D0ymYv8v||NX2kIJ`>_*m> z9m+{(uL|n#ul{vZ!S@&Ur}qF1=7l+br)2w)9KUQ`i@<4yR7~>vxl4aNo9^{!40D3Z z5kbPgOzoxn##)P?Zy59T;BOoi>wVpR`g|*P>dl%%zi-9&*BQ*E8KSje-72cLcJBUF z6@En0antloWA;fN*tt=A*a1o_=&tp!Q+Us8qu~hXJ=h<=sm;^Z?JwPG|M;&DfZY^Z zZ}_NL^YEBnRF#f|@xyo0_c}F?vj`e0_8(Rn+>k;Hoo*(_IGTl~q!!B2n&#(#0oN zE&cE2sl41zUxIQ-GtWqR4ycU9IyH5pQC%Xd>A4232v8C_lNGy~jv`xBy=UPEZ+l%Z{A5HWP;>!tW4utyZ-r8HzIfc%WrwQ6*yxFV zzwF;;H?E&`JADGl#mEOYly!B6Q0}6Uk1ld3{>5tjDTl6qnTOy_y5#7d#L0y-!5^)IOvLYB%HV z)O^RrRM4Kv%POblcOO2yKQgMj8ocRDV)m~i;?@sIMu{$!`Ew*Cj`xGqnT z7PKLJzc&8Y{^^@e$1IpR@B1I99)G2xIfZnU|L5@`2X=;=u44fn!b~?FN0Ln+ zX#&Sy|DRYNZ~jKtNZ%X?UDtBNcl-~NL?SkxMpBEQt2Bi7f10c!yzz7vT{`h#1L5TV zK!Ry{Y9sq3a}-He7G3!NG+5>H#=}|e(FeSUBSU_0VE!ozbU}&uje2ZdWd08e%KsIP z?;HPp>intH|0BYI>)B#Z?1G7r^eBo2BgMz5x=<)=Ws&44EdTki5x=vcF8Nlh z&0?lGS{I7UDs_c6_uNt1$i9kgGS;BTDJ#pkrYM%iW(4R7b?-Dhie~%Uu|1IaskA!~ zIpVtq{>B*X&{g(+0las4d;~&6-x+ zjoWBJ{%f>oYp{b)2b53m|r}I->1V48L=KM~({9W*CIq0rE zS5=2b(+b8x>$w z`-u2PhW7N%pAT#_9jFP;2Q3H?oUy|I#Gt z#{G*DYvScMT>hSQ$!Vyyq7g&mX$-UP9&8fzAX|CNRIv*?91;Ju|G0iywLbKBd{KXo z@rd06VjkXx45&a0!ueX(P4)Vx*Q9kjow&E5c0vc}^FVcA4j^BTI&bn^OH0dS@RHN> zUqyEP6jy)Q#g8{`+?ePz4yIWK+;~R0O-9ML8@OQECmV{$PsZf-LnuJZV6gp|t4JPoe_ee@S?w zV=no5%mE*t%Wf1qv7+fQMt_4{^r4NcqTx^NuDVxsbVKddrfavI*` zE~vA2v+us#Xx;ulrVoQzas|Ri%r@G-m+bWIb33PdPI04kSI~nV!R|vr!nA}rVf%mK z;ds+1>jMQT+XMo4?v*e~zy>q*Z&8YpfrW==ei z3ERqAMC0KBzR0>c;i?TQ_@W$CelKq2G9czT8+f*?l04aiS9o!7Z@~;8)Uf*anrFuO zzhuSh=D~4C`-&aB1!yCm6=x4H`2W29n}Wp&K+mG3 zt&Imn&;vLr?Tfd-g-$@<>KW3s2amBdlb(Oi66u9@0gMGSmLQLp(}017r;V56XYBuW zZIm%_GNHPZR)&Obt+KY# z5}B7gzLmQrJ}*+HY-4?>g@8%G%Z3$<+s>5VE%u>4N!!rPBuwEz-yKY(J5VcS4oiO$ z;2<5NN`jWpSrddy6`OW}j|(8es|HM|-erK~)2b`mWCpGumS0ElII-W8Y3f?gk+C9g z8aKdDxKYE0I|<~pC5zP+O1^b|PrFUof+>wf8|t}pO#BP@#fC;5S*nov^u5U>5fta{w@KFdQ-chRQT-bk1Z}$)3OrERUwFO?@nu)44nITFQ1nN51HuoJ;J%Jflui(t?Ms8 z0|x6P&r8_BE-?OE13vor@wO#k54#V}eUm92O!uHnLT=rEr618RYaVmP2N34uISkc35)6>ypfO;LU41If9%|kCNyIt$LsZ7>+L^v)2@GV2XFM33c@taM|J{= z6#T^0r@A`*dINtQS!L%3fUJ5Y%WsU%n30wCwajr6@R(60#D`4ps-$@9OFrECs zeSW$heNW+SGUIU?-NT1C4;tmNO1|N(OJ)s6I7DEYIaJpF#*4h?ny=OGcBIjq*h%yz zv!6wHJFA^Y4bpx^T#o^K!@l1LcRfOD_Oww zJ`EeXXdCiS0M+XWtgv!_O!=AG%YYp?ssc@8w??5g-y}UyUqXyc0Xl%mK&l@5V62rI zs_7F)@gkOBv7-J@2HShQRZgHpG^D8)EPAgtJXtuE+Rx%+4hH+Xwpx_UwJ4E?qipj! zt;6d|`vVPF*mLLv?MF4TbE4x3xF2p9#NC`$KRMVd?FJ+7>rp8u~L{R|?8 zWFaMm76_G0%f%^)FE%~DW6O}Uy9Lk!xuydqUetE!;Wx(J-cVVTqx$J|Y}I1P90B{u zOM%OK{>~EMbGRtZF8OXID9y;+Bk;O19&J?aok%n}G#KwOPh)$)6F4?Q_oBxW*yU-| z>nLc){lTSoTUahIf5ybJYtr%_FH@2ka3hoYb1!eiN{a+qw7k7u&qX-mJ{q4KN;Ygg zM?x_W%Yn5t?Xa8dF7)-u*ag~`R(xN8IDSu1WaPd}>f>WHuz>mSZDZgz-+lHnFp`af z$hr^l%uOwNhx<2Lp+RbPhud13uiY;X*LQeU%(AbLix=+nIxRQw;T^crJi?QdvZPub ztVKB0br_p{>pBzu4k#ztFIi%9rTU&-sAoBx6ImT6c2k)pe=zNx)M&5L;6bYZdc^wj$NgPP z8hmnptW@U~V0jgl-*HE#r4&yVeL{JIu1T$?T9S_fQ#I?iAiz9S_vup)QH4(C#QXa}PL@`}Dfu#mwwFRlv z5BpV}Ft17f4gx|~YtbvAlFhKYsL$mf6n`!5?=-j9`^&wD1ZKg4cc>%YfLloaP?RP# zTh{q^SwrT#gQ)wsM>>JlD! z;Yn{1lRDOXKCxR$r&8MyM7Rs?BPzKJP%=kCOdoSsN*gi^KPyudvN_oInCT+hFyX4} z4Elr0@TIO!v(cvbym54~_>?8OFu7}wCZ9Zi7lsdFa`LsezO{8?J)FHcVd+2H}ZS3UMa zVB6~*h|=5jNI&9~9?#a2y`8mX z=3p;rWr_M1%R+lKOYK6pH)Ddy=RZxW(1bdchGeU=Atjw>cTY=&x$^EWOF68GyhmeP zGw9lBus>8~X428i@ja5;8!+1xxzA9C8L&lMh(p|MCFjLo84^bOp2QVlRdz+M4?t$k z{C;w!G*AQc{g{C^&|qXbI?HbG2Fn4z6HqR^aoMX4rDt*B=L$ zMdB8_9UGmKuI{q{+IxO??(I$f@dy?a6w7C@4{6|x)v~!-T%Pvg5!dlilQ3%Xtz+(~ zaqK%_{@I39_OztL;OVlE9SA@yroT*ys7A4Ug1V)L^YbkSrX+zWGbXW!9T`sqv!G2Q zU90D-6#HV-i1yf{9V5E2T1^&G0F%-e1i!Go|RJs=kA`R@udXGb+)-VwN=XmJeRcmD?X^P7d0ga_yyxH3@b2XHAgZTJa?ai|9d*;SkXGymPKNsY8 zJ17Bv{U1j#q}-CtQ%$3bJ=8}%fK_veGTs#Lmu8D)(td}^J`RH#pjyt&Jzh;f&wO^)c$^4vbumcgnMGINO4k+@ZlBYcSyeQx<%{GzJQ8l5Y(<~;l6 zzP{hDdimVlrUPxOH2Ov0(Us+V&6LED$vM?|_h-I7f(hrh%|8rEWH%x(ynz8;DC^rt z5q)lxi*6N%Zyxc8GT1OpD&@!X*+r-(e zmGygi)+$g1t!p`IA>Ydz0RJ4ip2BTbZRO{QGg67Z)C|(Z6Ar0lG@%z^;hYP(UIU}m zO!$T3+40H|=XqeOS&KJDrq~1#X#4BepK;&;V`u*jkwN#_8%3!l9{|4?_a%LX=^_=w z6@t01%iCE}MlOxV&Y+ZVMen>_#&zCqiN;AZ_w^bEnNcQA zP&$Cwy!6I6$R1CKQG<``DDyXw#BLRn+VzxqG#*A$?y9c{o^DccRr4aGr4+pubU&A7 zsCMILhxtdn$CaH1VjWp^sy+nhh{|@n-KUA{_I)T6Dyuf1{!+&`y*w=}3E!G{oEK4X7PW&pys4v=5c$c_laVvO!0aQKg<&vL%CUQ8jy7 zEFkl0CHJ@*QO2Weiavs98*ioI^Dl>mgu$%2NaYIPr&WO;)=3fP^m@e7tG5Fki7zr+ zSzY3yhLr1 zYH=9@*pmYO`$ia8=-A=v>=({4%Yb0@N8oyN9qqj452#KbwgVD13~j}qZU}vp`1+`pD!@X?yfY8KiI5LSZy^?`k28&rR`8W}%YXv|OV)IEt`onSV%nX+oHN*`q`A zeU@Q+3bqpQY66wIyC{zd5-y-O45?bu-i2()Mpqqsa^!2B$?5IdN4dU-L{twKK>8^P zXz?g=a#KD5{){b$6RXdG*qufa-=oA+n*4;z{bM|Y#Q!;HMD|&O&+_~XdimL63-dv( z6bWc*hF62w$Pw*CCb zR3#E&yw7##DtXc@KZ_$LKbQkam7J!?;%zUPYH;`CA1ykLe3b}W9Z~XG2$=BvCdNlA zIc(VsaEMGJfJ)62a4K4hbA;xr#ihmJET5&HYG<;*4rsYOn@(<8q}dZPg((5^u+V3f zs$8ydJ2^B4ibaJBxGT%{y}cfC9-T2zry#UvC(Dem_cSX@+!GciX>9Hd!}wG*M$X$n z4|N3cz>8L>v5o;AGfKjBQsEj&PJJb*?&*CYe7EOpea~|@5e6i}7A+%@O7nZ(&Rl%2 z;>Y*(!eFooz^jKJ&0y2yKS}PBF3JEAApA)Ot9YN=2wXYdJ6GvT#+FHIZ9TnlZ_U2^ z9v8iwo)w^}8 zZ_f;BO{Zf^p>70>qAZ5u5emj~wF8pC3lh{v#B_ASzA6fE zdTk{UQ9Gvq^R&myeoR~(^nm=6-55X@Yn_YA4}PfIqX8{FqIrs^rEttpD}t>DNsZH% z#oTzkRYdZsdKX)d1~r*&d@=gtEg?azR8(V3-OKq^iU~lg2Fe;Y*AaS4J&GUb8yW`W zH+YIpIR^#OSfaRuZO}p zkQV-Fn{W9)I2-^e_Ff@%s2e%3BOBZ2Jy5Wzc2txer;sZ{!n1Y?8N$WCryXW$!0ey( z>`@VDm$#5w{q~@ClBxMR>ET{=@&hxI{3urKckmk;c9W8OnpU~yAli(iw4*x4YS%df zW%-egV?6NeHfKodhzQhrO7n9k`%iNu{G;5-QRlC=ctSX2XX!>|+@pKe&`xW(jrPm$ zr`D@Y^zWg;O=?r=wO$3@L*g5;q{$pzn^U~j?MZig*heou{nT`zcIx!f^9z;pP8%|Q z$eg8rMrU?w`Nq#mq?d9J!S9W3$ok={L;pOvKIuav%Q_Z-4#)Y?Ve@_Q6PDogI^J>JJ^c2$yK50C%{A6U^3F?c_}oME zT6DwcN%@gq_mTd7ALuQTAQ9DRj}4!7{wF4(`TFCAy{B1y^Q`gUzM`#-h=>RTG*Ylw zY`p_m1|K7K2>OLb(!!LSYtWNAQ6BP~T4Zr|)0i zi9!1AhQ!Kt?)Qb9Y)aGMPD5J0Rw5u8ga8VJKL*(Yd0AP44k9o_%jp?fOptp#yM;hmBv)ZT5TvAlyU zz5(73s?63#o-#CFx3^B5S)Ik7&77BR zbgBK^c;Uz<-_5L}php@A*s|d?HfJl)L1MTc7QOG!qZXj|oO;gZ?7Ai!_YQ~~^Tb03 z@jsUHvmyH(k5LO4X9%994ljPgE`kh=o&NHU_BMBKZXYa>0WpvjAeXj@Zj@dvDFtz! zn6&A(KRb{zXu9Bz5SBj_?5VZzef@f0@pJ`q&%r*ADM=3y={dNU{Dlzn)u(gLHw=kA z&pHP1Z6K-1vgs+(m6!LUWxSv!8reEvV)w_Iwh#UUs-PQGUoiVdvybBQGj?%^e#Yfm z7zYLh()y)%KIo!t^9WMtFly=2`GyJgFtovownbTxInBY1b`_CeIU1A10>t{t&| zG&b~{N(O|=xQVWuS#rt2^#qqL>;m-OaL;SU-8FJ?aa&ncxB;&-Ek<@V1ZqMCeUd^p zEc1c0nSf-Ry=}qa{GV&NcY`i&8netR$JM4cGo^Wu2ty6CfG9pwH^)@hz`)>r>_y(c zcD{&yWUZ1JP28JNa&f7twSYAKxOS{Nt$=6p{P#ONG-IOpQ@jhi{7Uzj4{hcYcgh5{ zTKt(3P-+9(4GO0}ef!#Puh)3aX=n!(8}e4SA8Uec1n>7In2SqJmw_E z{^tomHP^+Y=qfMg?#D7^Oiok1Jt`8!+`z}>?j)E8jS?NyFKatm?{@#Wxr2}Bd#iF& zVdrLFefHGU)XY*c?o&HT&kVH1^I%v$y^{Oq_|qI$D$Ys${&?U2+t{TycK^_fJ+;hq z6JGKr{ou(zMF1RyOh4U3vQO@;iAXxDrmlX6hVUQ|U`PZa+j+b_shgLRZdiUEUq3w> z`TS!071o27?BY1@fALi=H_9;$td9_`F>Q>o2KKJ@PtO8;y~SFJw5GJ5?#c1`pJ_@_ z<;H^y00nZFypN}%bU+C}qS-9Qt-2RTaK&Yd4Nx*rIvZ|L6))HBH?fT3Wvg$FYi_O8ThV88Di@!KWcEt{rk7n8Tf*lcc-BT(ZtyqK-{s~Ba|81D|DE>4CShq(jdapo9d&|U2`Q7$rmn~XOwtiy! z+tNIcLD07~t>MH(-*eyu^qQ-N|B55bW$A}|All)QW*M*`BTH=emVchm_umG8f|hI= zR9^7$GSK?cPdHuPaNvpgeK!1%^xBX1=THBxSF{*ei2j7-NX$WN2!lIi()xtL3p?XC zfTvqPC-W$Bp#RIC&pWDikiG=}b5ZsO1dca>HYSI(cL~O#wYpT1MpvDSKp+l?i89Lm z7DNI8_Wc8}@agHhO^iow)9R1o#d&${2L$!a`#fg@nbX;Y46T44J0vCu|0@mnQ>;yH z-mc+v&Gp+YUW?1KE^a=TTj9R&ZRn7MAm?9Uz094TDlt4!t!r{KPBm7_W$@KCQU5kt zLzu=Q&V~h@lE|^XjZW}5efiHmAF{&NG0+e@pc$Lj(sB!#k&!>vnt}&WS7)8pC9l6W z{V$=BX^fGy9;|&e`NGJHqy@T85PoqF6}G?p;Q60uowNmiZ)UYP&|O1MJi(PxdM%a0 z1qI&a4F310g^-C$qC09m_ul^dZfF0!{q7%$!I$#ghi=APj&N^t^AtEF{-@FJ*b`+vXQvbHMd+9>!nkIo-4d8yuM`_97` zYRO$@+?G~WxjXis(vg>!7cWX;`D@ETm_fU^K>H8nd3bPc|K5Y3JW87AY6ZnN%Zoiu z8pn&X1^(QE?ML^hulv2oGy8m>Zl=LsxFp?m04|O>m2}OzAzF+K8V1KUxH$0k>iVCG z3xw=|vnIcL=evanTAkgag8*y5>@gbu#0q?$zgx=&D8t;fYVz7r15_^``PXM_Uaz|) z-lmwCUQ~nwnq>uW33;9i-_yq+;9}+4GtD|@PCFX-a2U1RcRusGZn$~5!p`^3};W&Tz z+}}a`kJb$he6tLH549WIiWPWTAOF`&#eJv0Sepn@^h_vXKM3Fz0=@qF+&{ka^gL)F zN&vb50cyIu?aSO>dvo-+8VXaA<3oa6;cLKu^rwd!{5gqHut8bu;;Dy(=;D(*x6_T# zx>I7J-`FU6kevG&&IImiS&bd~>vME$q3Nj?^1!g|J}LT_W81X#H653&;;?9LP`p<& zA3a{*^|$l-4!v62nThm}lxrta8(NubGXGxiZ!p3-@=<`)_F}js_1A(~v+04$u||6f ziHi>jx#}%@|N7j~rkm?_DKhQcZjg>&dfB=4@8v4qTK|~%$FW3bBAC8hguZC@6A7>$ zXbI3kj6K;oDluH3kEtqjr1IAhb8FxJdlb)y_q)lH+Dog>0%R0M3km7^BvfuS-jUHr z9<%fO%Ye%qO`(g3BkWck_Wel?P0DxZVY3r-cDOz#CDvP9TQ$V~FUR!HW?gzL+E6zb zV#}y3n%ZlT0-pelYin@>6e8xl$&E6&)W8N9;NJ>S$!g-NqN#!c0vP*&>Imu>V5=-? z2F94l|MBdfUJeM@EqaMT#qGGoE>7^upOR_OFBYO#7>Z>|&nO%|j6|WVcUeWB`1_>x zgwu~yKjf-q-)VrFa$!ovUNbRCQjWfqg>iKFr6sYh0q+#(uI^^Qrp-)YpzW8F+_#fk zsvE>bmQy9OLG{sM!gq22G@?stX~H$n#wPO4p{p<>2%^dW0w!Gw@m}Um(A` zMIrDo)@l{N5v05p0pzMO9gS{CR)NDnP40Sd0Owy){Ex33T`>ed6bcME5^xwK5#Yn> z7eNNEY6aRj_M|?vfdt3-dryhkp9J%~SKNyMrotM8aQ4Jc3hS$*7q}nDSklXgHp-5M z0WrvjgqeXFJiy@C2$jyg2I)ce)!@eW8(169pW+a{z$aAG1!7mq=Bq!Ek-t2JzI@p5 zM&R2`TmS!jdrBA&IzcE9uF=x?PvJr}e;X30+PWPnVn~Tz7Hebip59{Ffrc}mEp5s+ zYm8~fSZeDPyP&{$h?)h*^J1bc4R8rPXJ-t`s9Y*V!KLLoZx=+*+C(YrY?ei9GDDCu z<!rC(TJs<8R39lq{>U5q0f!o+ z01RtZy(s=O!^&c7k`lYCbvbp>5|kdYyr>H!>Pv*qtV95>b)hAwZzcnQT@tG?`4rX# zKn@(veE`RgcWPDcna7kj$aW}>OoM`KVdL0yHvYRY0o6T7ZJ`cr5E^3J+l!2S0GJ@p zu3rwM1Hqlw%Dq?8b8{~YpIhem4423sGg7*7BONs7b`Ud`R47Iz%2MJNMZCyT%hQ!_ zifsT)F|X=?Ucq>phhE3kQ?nzDt}KpsF`8)0^fHZw(rOIAK8ZS?v+ccXX>R@$moyn{ zo!3>}W4u^08q*?K<6S=$2bx4PM3~|k4i1hesekHyY;vc3 zO8ZBu6)bVam5s7qkEjiJ>D+6n8JXS{ikFeh6avOZr4z8O##Y@jM$?x5gPQz;L zzVziEcEKkX>#G_iDbWk^J=4}I02<29rC>Q#zFH=^x;*Ran@|BRnmj3DRC|RZV39P$ zNH&Yq{876Q;f6~m2vpe-XMk)MUBPTF2X##eL0j9zE5k>RG;EAhWcL`X2;hu;o>O2H4L~&>(Dk363q26L7hG4`=1mCfn&I# z2KWA2;L+yXMA{CE^gZDyalMVrz`-! zoa}|kb{eV+wb8;nv|Z6qKGq8W(uXb`5jqKZmuxF_TC<&?6KFk-%C~`zrH5ufdAQ3Q zTmV8Viw4>&!$fM4TGJtU6 zu5o*{S?wGvK-Mk zWO0})&&p-eql8atq26_v$;a-^7!yuuG%OK#H4ZrCoIO`phiAD}`G;&laq8W%BLGa7 z2jceP#%EHN?oFvn6-x?_Desh4KvRI&@a&g1j!eRu=bnsNgRMG!J3U9-foS#d2S02h zt%3ou=I|Axjt)M>&~pcAFdo6)IOs8qwj8Z3BtWOk1(6UjcrnD)+f)bMhB2fSR_+ui^H&{FN|azCyrn2Y$R* zq7gt@($Xew=6VfYbQn~m{1dO2ZzFgw=zu0SC>fs85LwN?USgaYtf4B21>G?C6G-1; zPph-Reo~;Bhvey|I)^}YN@D)q%e0{yu?}@-YEAC%>hn_Fo1Lhlp(*5<{9`uw89C!qriSiBAzeoVOEU1GgRCXP!5xZCZeiZ#zO zvT#`eY;--NG>gAK6UT}& zQU>!)7#etYwGL};ax)>YhMC0X(06|z(BA(QmFiTg$#sQ4W}r|Cr|p{#xrQ8U0Ov@Q z(?U6EXZipf>$ct6@6#a*u+Zr>V3bPvlN*f`|u;E*SAf zwdL#v!Sv)E#rqup-hRLzl924c_pvR7B7Nj(O7K0jrT9a_{m-O5^6yJFuUKZ3ipCgp zZu=SS`39~<*FC#r!KL~7?t(0tqK7blrrvSNz`WlLFGy29Y$6iH3ipP@74bN=xSA%X!4b9CmgrrE7o@WwIhOiK#OKveU ztsXDcQN}NgGg`96T+IsrAiD&MC%!(siJII6B_6RZS(+@)pryMuA?7Bx zOTK6hU9l-8wZGk->!3$TfNfHZyl|#ps^r@)+&S}5`}Ez;UX1`n$R377=Rjh3aV~(L zSb~t6*liOmN!HO9lGn60LD_i$ROm!nN=!CD7auiMqlE~?*{rW1+sEf=w{1Xa)!XK5 z|9dYb!;ULEwL+k>rNhO*Le~dLEsnmCC4rb@Vt7Y=M^TMm5xQR{%VY0PER?J&UzxZJ!^v z$ZgkM#{#5229PyfAxJ9%H$BXf`&C`LUc*1^Rb(C4&O_%vgmP}TMewKpBLtmB&o*^T zlr9zFu|;N&=EGPIx|kEFPkH4F=815U+PPKaf`Dn43ko z8t5LMiaQZ}`EI}gf#(HoU`#YPm--Cal^u>%pjF|T5z+TE_pjM2TbT3>aD#b%l5|?9%s~#1Rz1h+%JV#Uxj62Kf?#Y-7FQAvD;+ zcgG83#Qzpdb8}-maz;(_4eIqKrqEo3o>1hh;lmMB!T)0KEyJSxzOYe3KuJX!De3MW zT0x|{L0Y^ddEh5#+Bpd9z@i#^k-lIU7hfyEzC}_eX|JoIZ%#MsFZ}Y>E^n+OpjKbe=I*@al zg)A7iwe1y`ymZ_SOv#|^edSa5e0S(()9Qf`;%vPz>-4+B;K1A@9AW$gZG6Rm>G|9t z;1~>)-2g4$#>C2}7Ut#HGqXhWAD4FH!ejvB&#=wUZoS1W0Aa42^fctF>{A(Q5T0chE2} z*9d+xtB!r@dZTo0iv~HzTy@tKPHdpv_A zEwsVrnQFmzBE@b=SEPRo&Bws;S&fhR`UYr1_7wAK(t`PU5{IhfrW0m6!NHZzqE8zY zAYN&9n$pykTJuyDT?=`>2+XR)pai!sZGTQZ^3DkF&Gwa|C)CV*1$kCllkD)FhktxX zYgH(xs0-#wxnwT$93}Xu5t3&325HQiCVOl zbO7|s&-UZ&$d{t#<-THGBUX@N)Q0a>X?D^z8q4m4`3jweEk{9c#mUj+K#7d-5xKxq zy$Kn?jHnTX`_f<5{5KsoEHj_||C@HukNYQJS$J?ygVCDgOJ1#=BCVz?!=f_BF-#LD z0!;Ror55nE8b#zn?FH|S_dlvhK;VkqxYPmtM`*($b)6)k>iTOyG=|r=S2uTjTk^qt zYD5VT!hdp@BAJ%Z3cmuh(>O13+eYlb-wGt(pYDu1w$(5D0HsxRE3DI)10kPZ9%RK^ z1CU<{(F^=z`fx%wHK`Or0QYLE?PO142S#U0ssy@A*z~ldq6|OO;mzG{b9!ZkGfCmu z5u_H^Ra4>?^;-iXoM)qhtgy8e$?TLfHSg3{C|2cof$f5T?p23{@$*(vr>bhPr1< z%?-6&Z(>+KT$wg$dNXqmfDNZB50F%?PVdY4+6e_xK;_h(AE1T0gw>ThcM-^_5~Jki z{Qc{m2V$kqM>jXzW{>w~O&yx&a6>N+*DM_x%jer$jev&1u%U)<7oqno>6OE zc%JEJT+iE;Hzs}aY0vR}9o%r_eQf!?a!+RePIES2D`#FVPfOEBM(K8+92HhptQ}N< zPhvdm(4xqx?w#;vrq5(Szc04VOhBE=IEzfhhNApSH06uJq1P85wXy6tQAM2eBIXk?F-AAcr{cTM zi7k+lN@68_Ub-6`a)f&8CAm$90Tei1PHey08vk&vyRr74iXSAD(!Xs5=9g~&x`-E< z;&ICLJzkq}J0ZdEZr@jdsuZPXHsHWC`>IIR-Oul_^l=J(>G3t`Z8%fSbO2D&GboEk2SkYG z-n-{MT)F2sg+_qPdf3>LyaZ2gcX_NUIE&b!cOLS;vR&%GMt&~;(D1|Rtzuk|v2p`Y zN82=#X^K*lHCUzuGFj(9q3)C(Fv; zQa;Z5YGS5+iszXo>%}Zk7Jdh5sYa^K{dsprI;!}g*}!>*8tcG}=8f~?4jMWkszHvX zK}^<)AGfd`qmRup@fAW+M30p}$OO?P!Y(uG9ryC{3$Cv&jzOcK6h$u`o!jE3qr-aT zjE${A!wt8E? zdc7P(!~!pq!-cW+WPZI}zh_8U@ow)##vz+)_V~{FE^PRZEJ_FV(M!`W3tsV*qwFvfbGmJjb{cZ6 zRJ&0xNb9QmBSXQC<{_HiK<(E=!PguUH&VOw=RTd)AA_6)FZD^5=@xJ!{065l)g>|C zATv{=eGejtM?sZA4z>-p75s%(j8=@%`(*}EN{2j$D|)c%NU;6+n+gu7aD+H01ok!R ziNw0tXJozZr?CqIF&NtqjR;bw2(!qX^g)$#%CS1OH0V1P*Pu)Yv5~u)*|Uda&_@rH zsT3&>q1GyIX4K-~w#N6E)SHXWQZFBwoEnv7i*_pG4*q=P_rxHmfIU(*_;dGCFKzu( z)R3xX6k2f}?w@de2}(VZm+~h0XpB_L3@g4Nqd)MtrASlOO*gc}fFfG268p^M^a-uo z)X8bCIcsqrBuL6OPYRwlL2vUJ77cjmc~r|&xM{d96BNi%O}zdBtV*d0NihR+hfQ6T z^p5ckic+2LP6MzI|5+8udu_=q{jB*IXJBFW?!=}%O!hm85saDOPUFB^#0Mv1rb2XV zBc6o($k zSLTqn;;Q^BB6JxlQ~;~v%2D%ZVbgcuPo$bsM%+VDa_b4pgYMS6B<7rBjiEyNd6bXS zg(8h?l?_qYJ#`0lBzmswwOllbDH|Vh2$1DLP6LX3>T=BEs!@Y`$g_M2KwJ7*=a^eJ zhCwFGK@6}2SaDB+`#Om}Q1|6>v|`_{pEz&uDL{#(nN`Z;G_%`=!IZ&2!(p63V?_f! z+c;+fN^qOH{|(E0fVhb~;wdwD)BLyfyKGH{f2`%-i}XkBpCuxGSRy|L+>EpzNY1$J zQC?OY@UZ%<^L_@#!^fRrXk;g2g`urJ4Q+uTvgpo=7@Du!ez;l^#|;4M!^B3@zs3r`#* z514!3f}o?{vpRDC1s>JWz5UHdg;j2y9Rkye2&KUR6Lp>lv_iSo7~(}zFSsGsWwq1# ze>2aU>!tJP7zMRx^yVHNL(-?1c=d&>&`bC$`qMqW!(w;A6H z>EvRnpUR@#2G3vWX5S7R;dpvkU?GR#?tJe5X5ioX2DI$I6Zalv!2VDlD6lfb4rH4V zX4kjc$yNpvL+@IWa!jggJE*qx@|?h5^$4(AdfnJRb~pVdY>(g^eW}2Zi}yIK6NS#a zEPhP9aaw6x{y&ZhEcGc4+9SX#m);N-5)~eiYiO?&Za&EI?Q?r02Gbt31bqbm?&*2G zId3NELGs}@P!>}yxD2Mv1xsc<{iP}hHAOZUZzyV|ekYJMg<54#>7HFLEPr?+Ib=17 zx}#jh4T=qZn*uHN2OXZIkL`0aYX;+SOZzurX@>iE`(q(mhx1K2`rv3&A$`<**k1{v z1QgNV{;*CM*|d$8>vTpM35p2uj|{s-`o?3$8-uZK{M0=2VztQc&fp(v z7}7yVQqoSZB#|kd!9nYmq=@L#8Pp4;BS~W?I*M+$Sa8pmC*OY(pnupG!a%|hLJo0} z!1uI-FB-V^(>nk(hkJeR2JWaiQl=nuCU`Z7t?m|eHs{G?r^DfnBqR$f$g>Ma(7pc` z)PuOt*esIX9+1VoT{-WwV9*X51})KE8%Og~X%iZRYZ8b`2O<(qKdm1?AObsP72> zjG?2wTkLsRjKU2uipV?(F%jJX)_TwLL7{$WC)4>*FV9w@Ud$28&&)G_s)KC@OtG`b zdpl9i&qwi_P)>KrGlhv}A3u8*n%r|{S0iSs=pIY>9KAO=gSC*s;wfBs!wyjCJC>n{ z^OonJVo`&rz=eJE&2;Hc-3X(o+U?)5$)>QiKAV56;)zW$&TF&AQv|6-ux5wqMwr{f zqt!8vUN6aiQy_m5_Yp?=I^AKOj0~bPq~)g#lSu;;aS^fY!j%N(y0?I?K&P{&E-ZQMa-y{wH+HX!^zr~yjI?8N>Igh{t$U`^0=+q}%xT+DKD@scw`Gdlr zqDZ1UNm5dy1-)FPq^b|4qW&13u0q{gd$Y>Dy^nXA7t;;NHw3-m)oajNhHS41Q%(e> z3ZdqMxw>6~=(x}>lnjupm&+Q~KPhLCeX9a?KvXEiMI}q{T_{h3AOff;>1iI$P4#(7 zOv)8e*1oLxwB`$~2hg3Q{xhxImL{FPk#h%H#)##MtRkw7(ULL!Cj+H$ZCzPa47;nzzziX;CC|MCKgib$jR>PXzxHMi6q>;`Qp#Wqf- z!T{q^6T_u)kQGS=q3I^Xi-Fsy)T9IA%w}f|N(*aSJgj)OS3ZL^sLUot?ydjVy}P3M z&!uAcaH+Hkha!&jH~8grk)eLMobaG0NC+T7me`Qdq#I{D*@1MLPw9{1AO6$7;wGQY+{r-=2CqNyj*4wg{=@p_i8TqO24h z@iUQ+T&y+p92Us8!PYeA=QF(L$lA|dYYd`4AQ&&I;tTAzj9^+VoKUD z26yBG6B=qs1_?DcV55%F?}zNRh(Q;mD=E^xtCouC{(Pn!v1Erd(ND}iC{&8#Fk#nFdqd9Opg*?y zXw!}ZwP-0|qx>p)Xg{<0OJFr0BmGM|Y;~ z^e}<+X$JoHBXP_@_2MqYbDJ`MUgLuY{=)Bn11j`76qKodzEFLE@uYbV*!=IG;iwh# zs1Xo7RYq~D{5S^g_gsVH^0faOCn*2L2@eR~cl+SG{(t}TRvj5s6v?b&rko)BfB(Yo zNPo{lEa3jL6ZrDqQ&O^?Sye};z@lI&zs`~Ff8Q(jKkv1YQmLu&pZEITXyWf*98mP1 zC)?5&Cz<}ApKSgAf9795`~PS?ClXk=&J7Ri<_Aj4hU!;JD6 z8jTYL$#VwB+Fx!0SX428=So`U&7tqtG$@GQ)hxE&G=U(49%X1c7i8{^ z=6_w7#X^mmPOmt?nsxWcQrzt|keKs$Oos8nBBpOm5A0 zIP<@~-1V(jqsv3z#5jLE>ebeqXw-bmUsiS_aRy$ztUgZRp!HEM2F-k3HC{U|$n;Y; z$q>ErZ}FPVpWE9*YS_eGHzNm_8wJieE0aJCtLF@7x5WLX`?d3-+mp?86ams1Ej&f1 zs~4$QdkzRKbxw!umMhuNn9D#9@0i@XUyL3ehjm>7%<+FFtCP-#y^4tEGV1JrR;@$J zJ_U<+=f+3=9UoAmkRgF{qQQ#B8HOJcbtvbpj$0vVMDXDS$GByIt|de|mhD0QLWzPZ zD&e>Z;X70Fl)U+ak8!T#CjRo_-rF|y4t0vd3gu8^Q>uW3#u$*OUD>>CNm;i_+7zyB zy;Cb|z2GuOeP3o-m6P?T-BPL4>(DtYJQc6{mWyJt=F|CK5))cUR?P=v2S>2Y8OyUF z_+=oK5D+O))Tn6u0FgWLCf9lxuN*ouUoYP7*)9NYI&WV(rSDKgKW`s=xd8MkT*~(| zc7z0IJDy8uTdt_EV!tSF)GvX}eYic5Nf8sFX&t@Hblob#If5<$+&;u_n%50Y`Eg0< zi?*luhqI6Xmg;cGP$2gCB+~9E+?ghoqY<2@be%n9HN7oZcRlE}C(Sp(4_Lx!z&}w9 z6JcRyY4F3rTH90iD^@AbJ#0sm}7Y1(0^4$Hw$lWh#W=AwD; zrF>CI#1lEbCSqBq{U77D@FAe3VbC71O+eYE)w!X>ggnn#vd7tK#dAv03oknK_ibGCJ?^Z5B%-;=lhJQi@J!pHI3|`vvdgs!|-5D%@*1?Uzoms2ZWDw(N8|c&@}SuljFE zpwDHTbWGzg09a8yE0htz@}Yl0cn0_|tIT zn<6k1?PRBGx$%>V2!HV2z%$6-D!~cl7d9&;Y zl6Zw#7`x}&-4Ww?fy``#Rt$&eoMZ-EOi!*OQ%)ZHs~&-FSXmbw`g`Ubx;zYi_zCni z@AcY>9#^HNJGu6jli4RfW5)CO!5OrNdM?lY!@K0i8)UC&XF5dn79Umlv7cANCYy1j z$^l@Xt?HGti10*>jIk?0G zJi@l&mKfoO@j@$EBcc>%+NSpYCUB59~X4a$L3b#_r$iWkC$?H(mvv3fy>0 zAQn7r!DF%iwXk{CqtoaBH}K!oHn2lzzN7|izDd34T_CJm6KQ1B_`M0~)?>v}nhMp8 zqx+F|s{K0W+{n-g!Js-Upu4U)@-4*#CDlyMG&?KR;c~xRjFFojmJ|pd{NhEM88zCr zC`@_a#;M(Ocq!lOx$80WzLrqueQghsjaTM-`;>sgVazmfT0x^Qjv1@{vMC^`J-OFD zx$mSsORJ*Ys9SGWK5iRS5hu<*N&@$EHfkDdk#LSx???u{=qR;ew;|s8JN;5^<8jAh<+2Lk zz8F%#!yke{pLG$7!P?VKJ$`7tWCb?J@U#CwH%i}ReDgI9Ea{ApdSN{jxN|-& zN~mqL?+^{0Aoy{lD?;X$>)oS9BRHI!Mihq3gP0-tF#~|Um z|FwTg%Vis;?`|!oAYKn?yn8M}J&BI_-7iD=lqs#lvnj#Nvj94&^)2#x5_o3)MBv=2 z*(6V(u|zBHYEB$!y}b6p3;lu0XVWe-6sd>xjQLxiQR9lM>#Z2^V(UIuh_IoX(lODg znPzKiTNRCdC!Dv>Bh)V~O4)8%yc2AEx&5k8fo*zgtBYUT!DEzRBZ4kIM#<8fUc=&U z;M|-2b@X1LXQ+>6F1>@U^|1CfyJ9A4CNI%5l9GG>#$UrdMwd`^fzcQKZ}BDeD%x)x zrpHvROahD?N87$E-019p2M{8yfns0<7SK=sIL$dyYs*;vz{%_w&)EPa&-0I3N4%T6 zzSh5Sic|Vh>Ms;chgx|X7mfXMe+@e<*@S{Mc_seL4Q->D(%ae&)mq-x8W>W+Q%(*7 zS=~Ox3;|OoA+@ug25aYEV$~ZcDK+1o&_b-E$3Jyf5ay+aN2SPImemZ5O`ezQto*3B z`qmxlw>&XQ0!b2A^Hi5eY_psqT(A$_RPUGYHeSB2SfmVq4z$KyE_Vrs==|yRx=?E_ zXkEQU+@pJ3#R9dFDYxfIzmF+Gq1*1;-=ePFcdk-vty3=IHyac(2cX!MOGFj0hwM6lAJ_pp}VKu zqFv~%)^kFhGRJ*i)g>#aZ)cEVhN}#J?w)F9C%sdRTV&vvDZd#2iX?vb&ojGTq?bCvwZ$w5PIItA#lsWdljxY0}b}G ztP`GXpMN(ebJ$*kX+CbaDG0fA3V7J)e!cy>qXm=;(j3$#ws>CgkyYm)(z#@NTV==j z!|N&jV-eT;4oBqdr{=*YaRq!NNGv07Ft6+37?y_bHb~9as6`lb?WxDPZ$BYV1n;?M z55eaRB>7jK-Q8~MjSPzQ?-WSfnya!^uuQ?5HtL2we^tgfO0o9qzPVV^`zR9niP-|1 z&<7_fT=0f5X=W{Kq1Xt2oz6V?I4sJ#N!YIbBYHF1gr5|munS|MGF-5!r?HTmLw@fk ze3C-@abYXsQ%c%QRb>g-}sy4n*^V#Z8YCH=;Zejr!HPQ-)tkdLyncMkH!hoZW3QF!n|}ARycQr z#Qh`k4g0WG3?hs&EC5IN?cK1e6Y}fE9`2~qmsJp!CEpCze<_d(*55KkzTv6wR*Jim0gT+A%ZekrhZZt@uBtNIFb;kHAG@`JN_rBEK zWGoz8U|Xwt{ZA2bTr*{1mBgJ`wHN+20$mm7n?Lwwh%cm*9B(JNAKvg~qghpTSYi2d z_>kUByKmn)CuKhPNbb;SdrAmzqwuiaIuSL6q>r&x(-fl2>~-7NkIwG6Sov6rB#1FH zG7y;__pswSe-Cvw^TI3-?rC|jzWehVnzXrmR_brl4cxcEj(acw_LfnC(H)!Aq85VN zqi&Axr!Vv3Ua19PIDvh#)y#9}Hb&h8q`FZBQ=NGprMADV3%eI;8S=Ig;QLnErd;By z%_XUz-pMRE9(c#idiyeVhWp8ue(JEcQPJ?p??0n!t>Jje*paIy--DSn#5vtg!jq@t zB$T{$IjdBg+1nILl9JLKs%{N4t_Uf-kLnlsJS6xmkuih%cn6_fBag4V(!|2LeM;`( zL&7s|D$pe+jIlMbD?Q2C`=3q96~8`m9u2RfxO|esb@FT)!y#=3LP84agtj7p_em z##qN#+;UqjD`R1e{bTym|~0ipgkdW}jc`c|8SRu7S|#7a%iPi#~$eFj60QMo}cSiRv&178Eg;ck;)MPyp=Q>`*0DvOx0zM)6E#_Nc;4lE2xs zkhj@KcPxi$4v;%Wtr>yLCRb-)>$abH5YIHfiXGbUgIw$+x+eQ2T?T&i&|lmJ|MHqs zAMTp0{w7?bnUxwkK6qcg%#Lg3*IU)~c}dytcukbVJnZ>zl4bO3ML`(bp|YKD#*XXfRdJ{|tWVu25)1sIpr zFS)&`4@ELH^ya)AZDUdh=WHj(-~_K4s5eAah=np}JR*X9K(Sw=eBJfqUk9iw^IEq9pdK5>(vd3^>){^ zK}|zEi6o-XnmsN2~j{2%ECGayk8)vg-+Q$|)q|hiF$yu2*2OiLuT%s^*B$4$^ z@+Il$0-h(ZlcU&J-mMo~l+mw0CI(0pe%$gsTC``RT3Nk17{Mq$82Y*X1;YgFdKfVF zu72=C28j*?!t&h7o@k4Uij$}`uyxRoqVqXeb=Z4IOfFq_#?)8Xc&km6Su0McT@6!} zVyZ5lWGXLRCW?aGWngN0LSS)3>Z_8dAih_d3GNi<&4TPR}8Gef|;`~9{oQ?VLYVCu+w;TlCRc~8q za8&Gy^;nHIc(s;zsyTzu}lqjm% zMYlY0r}Md5=9UTtrnSyBndGAPg*1wt2J554L4AimF076U5sPu|Z_nfh9!H z$jVBSbQ<7J2va|pFjbhES#~O#UG(qhp&qpjU!)a`3wzMXPDf?7|Ja9POC*@R_D|5k zAe$8~J7M8j^Y@F9p;;jyWLD0uL7KvdvmFqUwb7{LE>=Kfh!{Y2|7sz!p8k_2Xm|#= z2!{pAI6np9V=4Z4^q=A0PPAFWTcWt~AF1jjsx1YY4nZNW<&7EEh^8Cj(U)2AIeXW9 zakxR$G8>`y*#_1kDmH+Tg{uZ1PI7j4H=#n8d@dDwU?GED73uJ-bq`t2#_{OmR-JxU0982WyH_&q9>nQ(9~s?>-(qVYwca zQ19bxuAAq08?mGSf-QO4dv3xNd+Af*oWH*GtWUZWSO#QD4^uETS>+?!1V3SWXZh9& zV<<9gALjo7gTd5%Ju4AnfP`lsi64j#?$MCD<=G(TBb7@nrW`8n(#DPOTJY6pJ)rU;w>7*d-~s3w%U&KhN{T$kv@8;J zDi3Vzi&dz6P_f$n8AFgPyR4$r>-a)3dCz+|kMhpATND8KouEH%jdi0bP zzIoZ7C=Bx1CAHgMTyB(=NRxPhxXJk893D&hCy0R2Kk)BK&xa+MMS&^mE0%A_(l3(j z>NXu(C(;|L+C-Yh&z4&HDbrMUrP;t(GAtE6A|o1~+{)V;mP3!)bM3>`+y*wWj=F*M zmkbFXh_vpO30;9q#>WNIZ`+C~^)@=e8X9=Fh zeXxDkek+&<;)`^O)lo}$M*KXk`j>bdxC;~D1~*kG-nSn2MYjeSDl-f`d?Z}!Z~IH`=zbPMvCkVf#)ou(+S z4bBU6d~>kx^Rqq!61YHgx$s&h=}5_QmA4=kZqPLZ%F2&^M#(!HA$&Oe(H4J#rvQw1 zUV%d+^ez^w1?GcQ$g%lCe`q#h{~Bww$g7z!{ajR1NZ9WO7JU~An&-sfl&3PCr)^zj z4@^nG-bI+Kq5+M{r1y!O!2De`aYsbxVd5v(7ZtegtQp%+TEZJBUaMiPDZMD<=z-5C zOY)0~oly7XM19KhF?QPx>?B#6dT~qnWw7=#Gx1TeJe${s!&Id(-i?^w&zgqO7C$ME z{7?lV3B5-dSC)-2i{;>;G^JcXyON*&IHZy~l0)*oVnZ}g<c@42VPt< ziYD=&OMOh$+x=W}qdzi4E4t~7-X7cxsz%X15A0>|@;{6ldF}&$*5Y5ahlRkBLgb4M zC55pDkTY}6bcZ55Z)Rm_g)uo*y2r}wm){=(xx<-f!$%67 zFk6^&cjpi=q-@JlY@|~cqNW;CFjS4F=w+~9-soLN{M)|ZfUAS7ZG&)RPy#(7uD2iS zcqt7a`$#lW_H>yF&z{CV@&wD%&l$~(@FldrHs_t#MpA0GS{#IO;&HM-izW3uxhnt8}H>mx@r}tWTr1Nzc zQv_^ve=n`%a8Y4vNHlIK^G`yL+fzL0=n4|Lflgep)-oACYYACigz&;wr#2R?6}ORc zm;P2L5H=X?3D^OjWK zBq^+D!#><0)#yD-^zYd2ZiHtae^GZ_N@rZi@`_7V6TzsR6Y3OO{;j=kRFp!?j`X3x z{f)yeodXC-4iFYmN?%A@thuv9601Mp<;pNodct0@D^2s zl=ko09C>pvwkpXi=|KZ2&qJDG)2x0sd_H4Ko!N{9*nq8Ygh2G_GmO^|d-(k~%DqDxdk%+_?bXkptZgT6 z6u|DvKlALVM$kuRW9lKS(+hkW!r6UZB<2!6N+bgt%i$%_ZY|y?9-BB*o_CvWlC?8& z-=@3kdP$-VZrXJV%{xZYFsA54&1q;yIve_Dmen2Z??%(F|7$K&#|OMtjTWsz4H;*H znQw@k(dl}54uAFYwcd=7kIdPeuHsfYcI|h@)2%_Dal}8%4{@eK-B4EK&M4;&@ExtHW9s%n63D0f~({MrsM&18dfVj6Qx67SO z%%fsAQf?7BVEO9$78Hn&z1SqgBF6bjs_DC#Rif?s^Gm}F|*Q={1t@nBi6J4 zxEwJZ9r7s+$8NVqOEg4VzJGwp?$UDpV479knH8{_*F{o>GM@By5C~)Wo z#nJG`a#qK}I;^)y#k@T*I`O-xxbKpJYjTfe)=LG0_S8qSd!l11?!f_(*CURk(nl4E z)M=quhI7(V^hrHt?2u4h5;9hwyFJ$Wd4A%2GwbkdGh17AQQ?XPD0XZ5ywu^@kUuP| zNfS5zaV;f11G#{Mt zB<1b91%`3oo2{MGnh<-+W!Z7ab}lrelg^b{VlVk0Vw-c*?jawzMXpkF9d*yo9)s$Q z_$NqAx(xs_o8Uzz_XIYcz0+E3kbf_>GB!KWQ4o!lU}ba5VUtWX1tA^n$Yhz9MB&c{ z-T@N>?81Qcl()%>q9awowq>R?A1VB3>l~GRo5C%ho$Yq7Of_-6n)^Wj#Au^4>Op&v zc;7v9KG$5%w2exP1Zt88@7d?R?gJzoJA?lMIIIr>ovI2f-M~C%?D-{mW#3~+gSz!d zsy@a)3yD3^wpFFTi6WS0G<0vgxw36O`E(;z+(K-fUf@@Ac!`qmFQZ^DfQ-EhSwkv2 z~U$VV*7oGrnW&VkN5S8&B71q6bz zwOXH9>nuv^GiS~$BNfYoE-x~84W%uJYUKezqDJ}LiAl|@NkHPSYF0`9OVnAku*!F@ z{B9a5UFM(Q)b%MBT{7cpZi~cz5*zJP%unaiOi}SnbGjS{=$)O^iUTep-~7-tzx_n_ z0F@7W70PV*TvoFkU7cuaW?a(9pRwo{pb5vI7t77xg!zb7Th{+LcuwRM9S^%SC!EYO z-rzXU->Ch+pZTmnXw@FWe$!RAPtwda`9UZXN>FL(I`PacuWdD)b3A4Jh!RxXA1ekz z!_CPe%aV__1?}>Ldi|_-x`W60puTIp(+;KZF@%{+!dn8t;x3|*c=AS9IfQ_!t{_lv zutCO@y`Y6+n2JM_ z$TT->6{?WQ~L#Aq2!M-~GIstH2JgLXUW=5Ib8#AggJ`t^a z|2WdHSMRabdcye?&(CJbJG6s%^L>DNbKBY`zvG>Vkjm&U@r}_kINXD4xG5|OJ>W;e ztg92$)PsmQY6;?a^4@jvHNjc|Q!_ud-!gK7bK}+KGrV!UkmNt!i2fYpXu8ChkDH65 z`yolPj3kh^55vsH5#R)C+tAJYEVoMEpKPAVxHpHm`L zifM{BN|acxpm&XajtGLnWxoxg>+^$ja=#I{H&w0kXLel8>(>d#flPx9qHHsFgoPto z%n!+3Nt1sm5)B?$$yT9YM4d=7re=7iwY+2y`_TE4P*;yev=1~VDy}Q#NoFW#tjlV< zOt9AqXZ>v)$s#jMmFDVNsL39-$j<;%rm-Y)9iD4Tvs|$M{DEW6jGtINz{~J?XM%H- zU!AXS+-+Dd4|%H&RL+v5j&_fqTpFXZ)4pvqezyCKI4;MwoD_fM8`9hmr=LV0=nQ;s zV9(=h-4~+Z-Z0MB5sw1_|JV$Qt(0{uZI&n}A$h{Kozh!IZsKs?#3UYRtDdogwVu3D zl)@awXuJS+GFOY6QU7R5v@aV?hMy+H9=$;(s3Pt^icQQZXyX@3!{CZM6(AlRCfYyQ z_fLr08rv7+Q1kK3#x*KU^q9EZPTg1AByXy$5s=8jT=S3Nm1E`-d!b}s-KL$a=jy6u zUWJH;3iXXHOa@*U!{Ew~tu`b*0PikCnp)#Xe0GgpS>85$uuB=XXUZM^IztHFPb*A> zHV*`{{%?}00$Hj{J)D&grF$aKCG zhH1j5KiPFAo8#q<12rAnZj@|qR3LiOYjDof!q_mnKM3<@`^)4SSRHhScY zXj$g&`W3x{3wqU`U$H$sq&*sBaW-rm$}f#Sdl@$nnCSmo3&1-;Kh?DkT>MDygDf%8 zR+Fp@g!&tKNaZ2D6qE0fn54y@;#D={ly_n{zLipsp6y&XB*hbJYIeL_J}zpuZfMB7 zY(E>D5N766Bysz4*Z}4VtDBQ$_dGJuFyE-!F9wt_qSE7eU*GcvAyHoH5@^B&sfu%I zrwrN7xMZz0F^H_*Gjh@OOUJD%_Rn|jDxqS(J=^}5_Pu&W0TP%R_v(|U^n975lE}+Lr)VOme$XToKeewa`9F^V z5MrKev@lLfHs-d}!+fm2^^~ZEL@xVM_a6N9IjTTY-l8v`s}e`nyln7wEH<_ZSBmSK=Rl z+b%+Z!{{xlIQ)k-CdD(K{kAi!PzoZt%8@*dO9l-KM>^=pDJjJQS?63^hK{-5N{X1W z9|vO4U0EgAHq)}zoPLgjHuW%iPL#=0)VU&xBC{VeCEVY;XDHjWkQx6=Vj+mHHMe3S z;u0+ib;|D?x>20sh@3P!{V|ThUdfFz3U_n!kH5&G)@Np9*GhB71q6I3wYR=@$%l5?Aio(bB;k1tuq$B35VGu!ah%yXEET z$a@nHjoOc!D}Dx6`#eNt+khOsgv^uEJ?Nq>xY=3j`MofD3P5MeknBan=s6aDG zH&PFs`8C(_N0KL9BjWYF^wze6t8_Y;s>8RP4%aGTIyUVXZF%V%k_R|54wWr(0t;1x zHFH02*Tr@|d4Jv#Z!aksDe=Nmjkd$b#>WrZCe96nb&AA>{s8W{c{~baOrI4OrM=5X z43u8_Mr7J>SN|<$oFgALva^qE9%zMDAg%x;`)D<@@2Hb7!|oArulOZboo$zAy(iY( zn-V%x4VCQ;FWe*ts=sA-yhy<+6t{0RNL?})55*^uV7YoZW*H?zEqjK)(KB}6svakrFkqobqEs` zKXZXj5x2mY#O7n2@5K0E<0q(nUQ3tT4ql>~7+YR&2&Z0kZY&u4r9t<7W< zI_D;ppXM5rA9T}2ei&VC7lXa!Qa8i#K$qi$F_NPfZlMwBM#FNXqea=CXu{XeXIA)x zkiMp}4vF6vFSg#d4xOYTJYbuas8i`)ozIGhXM`5u@f~O51xCk~E%%v9o)^JNN1=!h zthU&ADXs5IN*DK_@BOJJ(2ad=2PrAvuK3Md(tjiFb_VEb8?QhLDYi9nKwPyLp#w|SD_Z4Rx=XoAS{m}(qUHPX}+>uD& zdV&wKjZ?g$^xV|C8snT+zUN_h=#vP|Rg@L!n0BQ&=*0QA1$LX!l@!!7*%~3{MCV{k z*0gsQRe6{tBVf&hC}&x+Bd~JTtI^#>H6E1Y^uc52B1k%zV)?Xv;|gQasn2QCD=$k8 z`wW$nlRHFONb=S{ZZ24yA59F{%1MB~6HyhY?7*cq=5VH|; zbunl;t_slRpK(hfN@6VRtMB$HtK3Pt0uO-cJdz{TY>yV2KMTTlq%pkZ>og{0GFRpJ zLQE)21XEwIQ}8%_(mNGlae=4zw9WQ74u@DP+V6Sju!2B={-lYZyGJwbSI}Y^<*b6Z zm33CdwZSTbq=UT^*^e-^{sWAE%*QNO?eVpp7SR>ZlKzR{Pg#{t^3@(fqA_ffChi*tItoUp~vMcMU<0PTjM zY_2UuRK0Nnkay^(o zxn0eo<2#nn?HYLaV)Dd0&p-T5LSN+#p>@W!{4sb=){Tntav6&|nkPK02@5sjKi0IL!Q( z8%7D`NeAn69T%Gju|?Op#^DTQJmfcWcOu(!uJ$d~c{JfW&0b3sF*!GsvsnLD_!fb9 zemcKFC1;|)H-H!mR0ngW0*EKJi_)WmNj?$#Ru*|lY`^2Ly`Cr%BAac;hgNKfzfAq2 z_q}vEz`Qgq-tTjCrPlJ@0pq9_)11lUBT;$#r7g_Yyf+Y@)}A_NOzr6@&!cG-G54Rl zO?Q=1LH8ji239j6b39cR`W(gb4wCECedYZmbF+S(@wvSWBwaG4 zlIT}E%0sU{&MY~OzZ6=PKJ>R`A{e~4D6V#g-viNoon`0RXzhHfSN)E^H_a>vbGn<@1ojdAka+GYuGfa` zMDLoa{=?|t-9Z=A&~ArA#HR!6(z64r)otyojy)#r>dP1}3&%tB@kV6N4d()=Z44bw z@?tFy5@Lg}@ZD_aou>l|R7RbmhDSB824A=eb1v8nkS@J+!a-?S!XL@T35du#GnH$v z>-e+}S}D3^f=K}g#HTHKvRH-BFhz1FDxSqOnBIGQ949I~R<24DAwF4~(pf&%gjO~d7-O{mE&SzjOT@iek<99y*L=Me1v)(^ zRvPMJ+u?Z$$Y# zsF$O)RU@aOnpN`Y>VF zX;pF_9?U~@$GM0beBVdx$C%pJxY2YNLyNW#x*fjm@7H{_dQTsRsJP9gYwE2;5Ym1} z>6tV+!F?RP7K1BEJ{C>~b4A5de9pBM`~~`TD-~CfRybf{9gnzsqL)EC?ogz9@zjRo z9D5btE1Jh^ucP?Dz?l}K-flcncFn+Pyg2D0$wPJz(&-FWlW(YoWJc%kTo#wLsh9Y{ zmdS)@8m=@JW4)SCT$T6Xl$$a*p+(`^Y zv+&IVFW{?#4vG8ai#%z4-%GTYA2n^bZO^8Tzuyq#Wdiubg(@^L$-AeICzsRHP;p(( ze1B_WgQjD0=OJwWt=i`*r9tn4 z{nJt8%L@XH>M76*_Ylq7hJ`IeE1GGaqx`%%1eZ4lnu-Jcuy%RiHX$n^hKgF&7?n(2 zJc+2=SMf$eesfD~5j5|YAJgYC1@B}Ry(wu`xRZf|W=am#-^Wa`p0Wa2bllNX-9cj0B#A?9pb7E!9=ZTe`&Y8gwpx zv9|mksJoW`pM`d~rqr1sjL9Ge+TDdt#(FJ+IEM>kV`J?sb=;1OTmajROXaKTI>@Q~ zkCDwyMaYmqf2vQ6;|1-qlvUJQ;5j)7syPQ1+GZazEb5PzN(kAFMl+`bP;J6fy`eo)cXwN7&V?gs-B(r0N4gE#*BfiI5<4;?gQ4E;&?q{O-{ z!vn~mEjO!{%K=@=^qC$|MXN)Uf|!GWKGDRrL^^@V`SVfQH1UvfZ+x9mW1xSmstYRv z49(-_IV0e)fy5yb&7P)polyw?_a`bh?Oy-u zqMZLM?LiZo?g=ygW~%O9sgF_n(U>-^VG2OctBuShS8*Q@LYJB(!OHQobLs!*20>XL z;XhZ|#d!bQSBDL2xzec41jW5$%_Jpd<*_f~cQ(e)XD-f1Xk_A}!!URy{$;1rh21Z~ z=*w3U>al!W@j;HxKZ;QTzX;IWO}l~Z9j$(Wu^bw`1J+ME*BX$b=M^^HD4%_eHm+sc zZ4?m^!Srp~VFqRb@kyTnnmxja4o)eis`PHYty->UpWpr1qw+lk9L>2YDHODMm60!j zE+Q*(p~CXT*_L}g6B=Wo2cU7O;_fWvX@4S91r+ZmM9}U(1Ii^0(n*3-;Divzu^IU@ z(8l48(@K0D9vrNE1eD6gtfr{juEJ|XRdDJ5Gf(`_QHCk~?4P!qn3x!NRJpsg!L_`- zJ~J_KAJ(!yv(WM6tXns!`mVA{di)b7%lB*jBcJF|5I*;PfdM1YUWth1aqLNk)3d=a+0`^f@r#02DRSYCpPETq6J5Vc0Ai~V zgBBt{+x(^`#sCMoq+-mNI1Y%TVPACzhjAU8wH3Qsz_bsf7mNZJi=WJwb$55S?iJJ@ zo3&zS8^;(~03v)2VcX{(fRv?v>@DV<%~}50;}jn<|N8-KfH$0vnBVb5)J^QofiD^AVS=}qO*V)c#i+N38z16ifZBE1h4?_=2g(o4PY%2X=YLILEIiK&(oct|mrQ?2sL)yu%*dW_!WF5hP4Ft(uf|%O z7#C1sQ}!bf)v@{fP4?@6zGw3Q5QX&8$R~u`TB4c*m~qA(Qa~pf={`Pv$7!-+X}=bp`B(N zZCEG9SL98^A3zD)-n{{6ega7b(E74BJqWx3)cbZ32xc17W;sBcNFaH z$Q(zL4Hkav$rRAO@Oz8^M;w-ylYW9F@@CP{<1m1!nl4G|xT}gP0$l(OSUPo79q>@p z=-&N@_~8#1Pw@HYqwHL=-n^r2QYfzjt@ONzoFm!}+L}fY%7N)#^2i!9;P>tWiXl7R zYgYg*I!yb!xZXb4sxAH!3AL3uE}eQWf{*jUdl2y(e___s7=wkj5ZQ23;7!G9(vrt% zy%w7?Gc)6#(G<;5M`?u)RKbp@j`2499tCULwBY)o@}9ipQ+D_t<_%)HsQPtF{2^zX zANIDVGgml2`54GI+hHAn2%}h>%!w)|7r~G}4Yj4HUy`B9yok!$`Srnx^v>+EFz^(h5@VPXzY zxGlhn2uP{P><8slw6!gPn}SP4;&aZ+;6x%bTv4(mrJ(c}QoD^b@{5YC=r^OC+J{el zBsRh>X+d0UJ$^Ml%q)SANUsjPxptGS>GiB-Li%IbAWy&__r>*YJD^{mAOpG(Rs}Z< zy^3Dnllk9pTEK|ta6Q=jwn_q22|&-^E+6x9LPVjc965Kc5 z<`37hv0)n^%A3sg?yc&#Nr& z=>ssu{-JjB->glY{Ld4>bjho-aAin!t@LOqZUQn^aM|@`KOEX>6ZAuVi&2&dy*D@e zFJzI1(!_#_fp+#JAh9Jq0pB>Ffbbwn!|dJ!AQ3|!Afz4-_S7uPhAg1)9DHhiry=Kn4u6k8dX^5v z!`ABUnNJXaG^CvZ@A!lH48Z*LqxCe!1O>YhhiR9l4o*x>z0QG^( zKw&V17t0xWYq>A`di%+?$2A2^pzTE(EY-16kS55GTlks*O&Eh@*_pq7N2axgPm#n7 zTNtTSRl}5e8Ai%Ue!SQorcDSOvk&6um=g^Rjp~J0MF-sTzLfD5PANqlF1hp7MZgRa zh9f)u?gjY6g8yOy!e+P(CEkWZM$&H?57%U=HU_cVQ2~ePP4v(akxlA7WM)ePdSp9y zAnX*c&onYy)`jU2z2-SF$DSmcK!+^1VW_+?1p}c*b+rIc0mEoAVe6qBY6B)ikcl0`gz^eR^Z#@BAu7=vq%;R%E0WqQ~y}$<|g! z-CBN)7Kyp~=8aq+iPJMcf~6QZ_y6TxS)tmns zWWGzkEF6GUD>Hs_?_IV>NSQRh8NaK<>XVn^B7U$B8$EVua!Mw#TV-T zv5%Jlej~yqbO&W~&K|OF%-m?Mnwgnd2YU z;GZ<5<CdS1xFv4zPR%ly)wD~3Kc!w{vv4exr z{R}KXhIvdu>i*Xoi5#Y(C-^K?AZn?Z5jvbj1{Tx!oAqR`q!ut%mLPZwvK#`!X~w`8 zs)v+CpD!+hYe1ik0g;96yqN-8NBbakz*CZOugb2iopiKMab+9!gKP?fI9p>xK&W-J4Kw6s78D2P}G7~^6|~}8rw|SBrD*H z4E^EAf zAlU%TI4Zd*c>oh6PGAW9kUMAtI*5GM?iN6N06-~NAS?d~APfuAFB(H>HS%T`e(61q zM9#!25eq-+$ilk)aR%5okmXqbl@dFO$KQzlYyGe1ia5^*DixyAWdhyS#st1K9I|K74#gItkg)yvIExS zR`%xH`6_d=GbAX*danm^ay1|u-c^akVm;ZYNa_~4I7J$A%E?0eV=Kw8R!jae&EDrL z@T13H_i@0()nrnh00tQ%?_=W+020ocImXG>0mPP5J-8yaSjEo(HLuKy$ip(s5qA4k zrB5Y9VdC~%gf(!#KhU;oKN-O;(P#T}#Tr`xa`B3%eCL-2nx-F*KYxQhIPDlMyY-~1 ztc>%%g}QnSDKOMm-M$5=m}pM9vQGr9>Q3WLzrMqiKq^grZ`^)i zAD05aSB;NSEd@)q2`q^tQi$j2e@ZAQDVnB{={47Y+6W~Mnq*{+OpTvr* zJJK$hwY)aN-iM5XsX%s^>{~97c8{99V|mZ(LMgM)HDR%E4h4yAG!Ln+F`eD!obmb6 zu&@&&rTcaSx4%FuLUK&lU>seKEwlN8NBC7!xZG>NbJ(-7e=)OE= zeheV?FyK6`)I(>%^MLagoU5lW(@~C8?mM1*e}f02|Cf7vM%!IR#h_&~kYOCyw_)W+ z^yXD9uUA15Ke4y*2vcA&k#s8oz_`lJ34dMwzLuXkCq*W~kiYy?i*1nq)~6fcNJ_|9 zkn*>t!<0Q~r9aWEV}H5?Q;KdpVz%$VjoX3jp$O20OYks5v3nBV`pN+l7g3t(b-D~d zv0he>#;}`=0}yDjF}?yA)LsnU74ie!e09hgg4|_Q3D@HUl%4Rmnn{bCQBm5D(7cnh^yVf8Yy$7= zGzfd(c`JbZ$W-LqMlz1$?L*y7>}-6fS0H<>b;cSvEsR}t6uanUk2z^+X$%=#0W6#s zXOL1v@e$v}^<>2UTJPl8>V+~XzHP2d^i{;|kK5^2`;1CNz@)G%02}ReiW*-T49@eQ zPyW!!nLPQG$frNzM3@9nqQtRv_?$q`7v`SbLqFO?-@R&T&(s69tPoSoNQP{tOkEf< ze|9Th;?76RZLD6$T1QyI$}TXN{5j$;%LN1YFw|%`-aPmrs=A7`%-KZ{c*2K3_%3jc z^Jzo>9sWBWhOG>Shrs1Yd2PlB??HN|=6Pk23>PsbiA5j{x>sKH>FrIakXy;61=oA& z4SPSC>+M~fA2i4qfmw19OjuC|eNLtO&SZu=LY+=QI#nvk3{Y{K6hXJV_Jm#Vi+9V- zCA3yDvS|n$GN_2aR`Tg(c)-ZPWk+Gi=qwHRXX+PX^z(%u{J_a(dK_|D!u!XjdMfst#O38yd%t>zISC16cj6@I+7Uy^jzg^T0->il$BsKS0&z0H%8 zJhlQ0H2Vh607w)=!G+SRmB>7I?!2=IUdURmSjA3W}8W9o_l!Z}-dp#Jh88eO} z-1y`n?4MR25}+3tST2T7Q?7IAH+hZ9 zk$9Ir7~u471{Ampyyr%USD5EmPSMdQ9e0sTQLi?=mN~52Si`vvC&USq#sehIWZQxv z?6ZQ?Wm=yImYtehMv3pJV9cK zy{jU~J%|?)t*4D80jx}?yB+g*2ee+|nJhH=RRHfS0}`Y6&S>#+)d<#58Ez{LI5=($ zT&(y5E;muk)x>{-Qz-D2_-V9?C_pfqXR2_+&KrrM(e+@;L3=)*3b|f%P7`GtA>U>? zOn8^ou$5pZG7XFQ3NP@)(U1b>=DtTcz|E%4FeaP%u#uM#>peA8wQL?&!zr>egGtaH zy?JqWj~fVnkUsq}vC3|$fszdx&_f^4TMEv4AjfApVWa%mTfHZzZP9Ri?32v9F7D31 zj?+|(#haFshb2e4Z{B}be8nc=wjk9FIAR&&?OWq-Z8{CEpm9l*gDju7z?J_47=|xu zMQmn_OtvIB#eB#K_4^L9r=&tPvZ z{*IUd=cXHoL%)${&!e3DYWlnn!uGO1k~e?X@GldAUnvzaG32;oX)=UW>1GAOV+;{= zB_z&6H>E$nnR+$RP3i{nP@Fp2Pop)CPR`bYSSynNV`TEYi`Nv>w-#M^vw869pf-pY zzL|L^^D1ghV)Yv#TL3!8M;JWK)G1v{-G2D2)6Ej(eGhW8gzbyYR@b(GA*+KxjCGdp zehV+v`RQ8NIYAH9Hzu6ET5c`b~Q z1Z`|y{N%Xx0S z|42^yFcbvzyyf@wVFfjty!d>)?cc$CS7B&n2$#H1pCV^$a;O}g<6A>~trrh{u&DK6 zAf&~gD5$?lreuP2;AM_5jJ-uD+n-+!W(gAyn=yOv4n~=ks9GK#B*%_^bIr#gLhj@+ zt|OacdIM5*hPqCVg)ci_KW#K`yhM4p2V&+sacbdI@$o)Ijac6G&6#AJX5oB2lb$AA z?Zlr7nY5=u>Z(p%AC~E|1iHvvfxFns=2-o4p?!5Nk9aJy<#C-*;$mBJ4ovt$@X_uQ2p9;w zn){r9^n7#9OpR;#Rv2kAkVfCCNx($@@*QzV*qwj22+Fe*yPQFE$^ClJ_>buQhtAK? zedRXP4^lS}ZZ;f++lU-}#Y31H`{Jv4BV#gN? zlVeLEPzZ|RLy69-ADUSg4Ee%tW62(6dnXh5c9wVyH@;{^Xuk+;4Z>`D7~`>_6=*{c z#HtOZQ_j>65tEsVnz5M{C7ye{Fql3${9ZL!(GU-$HueLE!)RrLNi~5KXYxk%TD}aC z!C_b-9%ZCGD8aeEf4cD-3%QK_Rw#d(&y>sVX7MFP<&uzlvk&jG8vAj^72G7|>f#!E zhT|{>c*qk77RG(nL9sMK_T?~`0|RMP3BHaaMfp9)O(d}*tiZN}cxGt$ak^TKqsOAY z`gqsn=g~etLsViH+sVGxY%ImyVSpaM*6h6OrI=`Z)HK?*p>|*h+(368P1PGaVbzzb=Zi30jkUkkI`EHrTu4?_)Z?1pr zB0v(~1^mPh;~|-dWj6g4l6&t8kf>;=9_p8;yGeY-Hg^Jv)Fw;HBNPg36N+b_%)`}k z)xek>lsP9l&ijoz=a%&F0)lxl{%7-dX$f^wLDe~mFCOi7>V3!T1yZT5X%;d2`m_07 zX54I=TgjNr@9r<69zeT$c7mLpdB5uIXV7&KDCH}TZYDAYrqeSC-imxQ9B#|OFf8*K z@qtC#$uKSCG6>r{>Ux}mI`k$2e{0ES_A%1dlD5=QfT4%621=@qJWj68!BzQN`bXd= z_(1G(Sv=rG{Z@ab>B$n`2%MND$U5|OZer*&>2x`=x5Z#?H5Re(Xo5Gptgh$joo%?vO=>@0PbMhjAx*0)>RWCv ztZpYFs(HMpVqYYnKuz7g^5Tjx64he?=^7-Zv;`0Xn<-|=Y>?mE69OTX7RvQx_HCXp z8#1!~(-Qz{@@z+RH}&0(F)%uLInNO^jVyPwuqW~}v*MJ9we|c{T(+2c&zX->}1jH+ZSLqO)Ew&A!V-ra^_ zl5#a#dgUFJ;pBHMaU7yZB&A(O6aNr&s;@C^px7dRU9D!k$t1+()^1!&YGR?QuzmS4 zZtPQ=!EAZ4RIxx-nc|7c^ysn8f`vZ}Jun00m(3}>XQ6Y~#Y~=0S#4~5OZxztI4paT z6dKw-ZVt|z?tNfTFzP;!W2RNm(Oj(l!D(5tMO>9%R~K~m9w(hprMP4L$yCDn>Ux2V z{nv}4wuPP~Sg#%77h8p|RpnO>Znz8WErxMWN_e^zG(t6)@>N5i=biT^+9KiL+PVfP zKwGxpss;ubl8W%pvPck0{P0H1MKMSC+?t9|07N~e-+GB7)EgqqL7~aEOfyhy=N%cg z5L3fx(nGigA=5yc;UG}diNHPJP5kspq@iQjyy<4a9527&mr$y3ovo*Pqac3PUe|3T zFC^*xtQGdn`zxp?my4m`t=_bto-QuHw-2J(VaU`ygSk*5ZyXZMS< zg!3LwFW#N$z<6PSZEjjqBF&w;uk%yrhoAV^&@qMMPM48%K2}lXd@4yO#GgBZ2~4|R zz$zNCG+578JHG`b#}?jh)R|3vfQQJxZp7+QA4HTQghKkpfv=TaB+xbyvo&_hq{)xU ziPr>qDEyIbNwNrRlyG*T>f~4sv$h6y_JMD2Ru|9fc~V8?Ha=kam@bJsVZn2nR_8~A z7Z-jF;@)5!wu=fPU~RgOv{PFYy$ykhoq18ZZbx>G!4?mm*{p#2$>&!Z$AEOyc_HWl zms_oLEJr&~n%arq%E9@&G9?`U)1_!ZF2&qG97#K`k*$-_5Eu`m>)C5tsJ>0?+=}r_ z0Fan)ig=sszqX@t^&P^aG~g$Rgs)b%q_O6rk>A7|%=R(m(nhTH;Nj;INMDSllIvz2 z_m0}QMr?tFR<-k%T8vF%3Fg{ARgjUy5vo2~v<4EYcg|j!rLj~8azuQk z)L{G>3GJJzyQTG&p~0JKAts!t=zh1>AF{WxIVHYi(%E0ZCz2z?)MzT^XgjnoWba=De1-T_GDhk18p%8TO6&x%Tn{Eef-q2fjSFWEBi74F%Q76Ei%B8gcg!%9pW2xD^FZ z9A-a~B&L5kY_J(Y^7(?(h|Vb9X|RfP1+&f=4Ha2LVAoPg_J=vE;oT(jF9R;Q0|j4L z=)M$lC$%8gF2mf7B`u<4wZHR3@hVRNQJ z>j^_Tdx*$C31LT+nLPS&Pjc`5u3~LpjPciZO@Qs)q2seQOI z=x7flw@M=h%|`vxbv~#fE#E}Ed?@fyt+EK`2(L1+WeC4Rvk% zA?2maI4ccflh5CvcUTb@4&tQF8bXgk{O9M%5j<_Mg~5;C8uDGWnO2NO`{xOvANlpL z;LArizrQQRnq(ckz~BG;^E?0l&-%H!|2NjFG4<9cXtwAAjS|)&4xsW94eA4WUxK?m zp;PncqU+UO!5Z>41IN<2)&bX8^VJoyvla0;TA`!%{aUvlX?MnwfKkpE18Vde3vpJ} z?NC(+C?Vv=zY|0|(`;}%3uSTw-k%)+L^G)!^B`*<$knbHPCNhzqbU>@P#dw_yspdC zP%vRtljpRmm}EwFao83obQC7Uy1Tn;JXLMK1fmeOhSRUiE;F_qA)Cq}&Vy-zWY*)Q z?O(^Ub$<5(Oaf!O1cH06i?f*vftz;-&Hr<-pp(P@`z|cKMe&qD0_b*)QwxKRe#Oy> zxR9B< zYolrSq|XzY#OqVf8r&4M5BrgOQ7=gYJ!U~Y34^v2h#-=ymSWAjKrUyww6vrF2rod; zzYgjGOSeKCMQw=NS3&QjU3SqN&kDhtalt<=%j; zsPcoh+@;SIHoIfCe!;jx=RTP3&&RiJjk%szbsT!%L z5e2Rp+HFuzWwy>dn$@to?%oKo59uqlJ@>ekfDq9BFr~7Vqj4%B>l~_HW)=av0GTg4 zsIeHmKio|`;4c9e<93v2_49XAqC7&?(Mi0&p+k|34cgh4mKpUh^+ie{G zI&hGs$v;o}xV`3)&E#hx%30AFA*FZuoQqgPt#a~HwbdqP2g<3(?;c?ku|C57Lj1t$ zsf>a=Ii#hE+e4&#*o5D;<#tf>ERNHBc$C>jr9j8>;9}iGXd+tNo%*f{(Q$y+o%3y< z&OMcnP7nXEKYGR}nQCm`2QI>7GNFj9DNoaZ?LEa zFR1D6H&1^K;O=49HO(2xa%J@HHJtB0|nN!v)!|cqoj)>>F#TshT%%b z143ttapEUyx)(Vymt64fD8XUc{?#GI0BvX#iFvC*s z`FiiVl-YddfxV1BTb~=yNA>x-13>lqO1Ff9@|%y=ctKay#sCo5t{)}XtY|pPb6QF9 zX?6By2y+Id(t@Si0icH4DG2&uJBgDrdTiwes`Z7!UTh2E4$X~!5v=ZMuC!dS?K4R= zf0Ps@D5XuTJre#s#9`c&mM??eg<5TIY=sDu6*qRF;e4%u_p7;Dn4pQal$i`+EQ;Bb zMu4E~V?Dbd@6-CoC1vIBAUCvh_sKF^IoKFA49Sbmf%B5}!ZvNYrF4%KPWq!7PK_qE z%Q&XN5gqMSKgfjsZQ!MI?teTcuzd`X2*3whGP|+C1(4a`s_WhFaI9$yv3B-WRU5|F z!9iD1q8C4swssD~9ocy|deWg!9Y)bsoKaJzn!(q#k90VWJws29MjRyb;#({|?>F3RKYc%dD^y74n- za5=dKc_d5S_i}VMcHhgo62o}5rw{5!$i#K;(i!rS$-0aWbeZey5}ky)MY(}`3+He; zPq5IrDXs3Y8$DdcF^;v(}V#F}&yAk5hlt zz#N(tx(0uaw~)pE_1=ESql`kq>VJ6+DZYPI3yKHNqjZqo+s%T=Mrno&py({X}gw z09~14S^B7n1YR3aIv7O_uMZM(v843OwN!G_G^@rBFtY^gOe1oK34MDRXZDt6D?}rl z@gd~id){vh=)SL7lIpGvo>;PS#@bRe6dc7im67O#6ztA3uimXYERL${;$PvdU&1mS%tjV9#DxjY!JOA4jW57P%_m$Uh{d(dlTbN-;Rj#?B`jRfV|Bl zDFq8EQc<=_>Dzgs??qr%A`TgYyRkkOKB92t2_w90iX|ce+d-VvJ{;H4l1*#=EZQ)* zs7Y(6BI&5h1b$+`2=C=yKtHK(1)~oJ#8W9J%WZ=DMUqeHbwt;?83YXF^I%GZZXmX& z;x5H7+LA0hpTl%7yhUMY82NZlL@2B=oeHizmB)O`4&4Y-D&K0n^n)+!xy3Fs%RFfl zK1%HJVwkK991cE*nN&t#Z?JV|{)XOH#~ga0I~2@ELKnM2mRGtt&|#)`Qy8ik<_i43 zA71FF`E#yz-hVv1g65Hb!#+>iK(`?)B~@EjWu>}rD<-{cQ{os69p=~icZ3$1cxibH zM4)il=GG-@rKL-Gq0J#eD2~XY3F>QLETGy zR+V1WYMPaxVno=+wu_T=D60I8=t|9Ex~M$?@L$GOPK&~faAiceWM98;&f|rJuM<7R zAkQ+vR$ENfczsb;0L!y{mu0>IEhD8r)l#oy?(Vf{{RzjqV{FQvnBd1{ z^heBKYN)l1{=;$oL0xZs{^~RoZv=E9q1$YXrV_L zpF>{MLlKC@t`e$Q3(kj>R0G8yEz$LkC`AHL5N)}LraaX7RdSC1kDSCf=I05~g-%GM z^|M@y1`PE|Y}=0IZgEEV1e@su-8$&Phr0fXtQoA*@{ri^`fY5>4)<_m z`HQeM9)`jnCZ#_om~(Wp13y6hfjUQu(&~Gp)ECE-Ukl-*rnJn6Taa?&IDaSKp{}#F z4xRmWXNqGmi-Ej$MU-EP8GX=v%-sqDnc@jLF25lSa`C6Tt&F*dDL4^LNY)Ti=e zYzbW&Q*<*ox=3sHlZg#o(`;<52JxzGs7a#B^E%n{tKp^*@8)t{YHfIIfce z4#8gTWU1P+#VwRnXV8hkHBn*d)uEy+NdJoRPEmGG8veCg>NT(O-uDfz`1i{p3%1;N z-h7t_Z-_Sj#p}GKK{em(dI-&Gu`n&=wC<`%Fp{t9>!m9E{J4y&kRA>?j8tpdRqMY; zh<)lVv?ghzpqQ*kRgq@HmB$fdgxmU&x*NX{Eh`_=g5Ma4!cFls<@b43M&w+6X4I5C z7kvF73xms_ zzHS|<-{(wwf)FI(Yi4wtA*Yh#w)*z6`}y*IUjFLNuYIdfQ(~|_1mjGFJiXQ~6BB{+ zyy1qqhvBuRMxz&Ckcn$PJ39yz_MfNEB-r?`IwHbvxXt4vJzo%pauiL{TpQ3j6b)?` zWu85rgMUFp{^$6^$tK-Ie?FXm&)d#yMYE#K_mgR>_5dOct*W^g}#_>f4?%Gm=VKzd_+>{Q+- zL;WK2PhNw;hN-31$X3RHRc|*MPx0xeJ0QR>v6k3|W=d2xS35&MAVPbzRaJOFJiUD_ z&M7z0>}ymA8x1vlq09nwL@j|<$C57;PBh1K=48fqP;Fs;=~|oHBUe7HH@vfagi*Wn z+`LpHR!FqE^vqjqborNi1~aNNr~4HR+Mi#J;s5z=SZd@y52OfmAd$AH*D*Zr0*j>U zIOn|trU9N?9E`RJ4re1NJW_6jbM>@|i0%Pf_&mdCCNz+h6aLEjW#rC-aT%sRh}*lpD8Z9mg6Q-_xD&A zGiaZB9(3*RY}K-T#Lekqh|JE65uJ3r9`qo`62T;!nv7iR2PqqwNg5YN>9stgJ~iN@p!kvZZEI^NvT9i%vmC(!X+`CpUdUzW=HA&RwS_LG5kv#~n{+Ti1L(AlQaxG3I`2jc{=Cr73peM0kgm_uS>e%3HR=7^Dal3P@9K!u@AGc8xvd!s47(gu<06VV(EUAFI_gSG7G1IzJnEknREc_?Ag8Z#U^T7Auq@$n1S2qXj%vo-|`43~|JeSh057jS~yv`Xm?L&Y-h%L(@L*%YGEePP`4rJDoeX8H%y4 zt)u-3_e4nkv>t-MdekXczm7k62K?H6iR>r7pdooog?$E$aV^P*IEP2X5nloQ zmu4Mss#az(EI5aA2k000Lp`b_FB%cstst}tmfx%Nn)?n2rkk`NvmfAHDf%O-{zt~M zd&lU*`P$~MKx^EpJtm*?X1hWA zsv}+S!-&C@rmE%_DXF|&_fX405p?XvgyRz~ZMUPZGzDFQTwz#4{-56o8Q`5AVchPxoug|c4(e3IV;Mv|>mUr09iGKc{XSct6fQGTq3nf^z}#x|td$yGre>7FFu|uJ(?*+o5>$_+Q2Lo%f^=W;YYmaTAaDD&NU3{T=|jHzT!ZfM=h}?Jr;?1+ zM)ojCwYB5EC`J%UxtJ?<+FY33%NHHm!=9Zx^lw!2p9!-`no#JbjXdVJ50a3}Db`3@ zqiZlv@%v_A?9Gh(Ao#B{-a-RGnw0SF+L$D9CFcEZpM^YhBx?COww_w2Nbp|;^%v7v z(aJ~^1GTqtIC7Sot113Kqgn3%gG6g=isu>^i%3P7`mV^q9OUekuR@f`+}Qg3nmt003(LIdOR`hpPs_g^zP4hyEQamCu?xO zGY}!wxmvWWI4YuuZ(WY8AoeaBc_B9H6To%T#VQu0+ie~k(w7YNuT0&KbHQkb22`naSeL_|E|IH|heG_Xqj!Cl|27wLtpADE;xJXZx++g_-s#{ph~}pFdVs z;1_->Y9FF`HSUjJ^5>U&FaO!cmktERZ~w}B{<-DCn7`LzYC+{A%^$zy&o84%enY5< zlcaBB{yZ*!oSeQlzxVr~Qn?WI&#Uy$??brq|LSsOqgVi{fhF0CAF=_W5VTujI*^I` z*gVd%oD~{k6<7vUlokk7|9t`GkvY@15YNLz2m!krBTIR?w4q6HczGZEFXCi3B$p^Nsu~&QOX6oYJWfts8{2)mnYDH~l!ToUC zr}Du=y;an8(A&O*Tj1!ey(}(tEPt{K`A{CfMX|8}pr+j=O!vc2jk|q)+FBM zZ53!XyuQiqcatg*LZ+Z1LUfp{pcA$aUC%w%^Hlu0Y-d^AVOhr}ob}xd>>;p)F=tN` z0T>etdHp4*G?uy}tn@%WWX-|yi(oSaM(IK@Uo059s2yMUoq7|oKau-gDk7Z+Y#JOR zTDCz|fOVpKa&i(YM^EvuGoTmybG42|m0Yg*uA0{C7L`-qWnprnAv}hHy~Gw!tAR>4 z*z4-O&XH9BM(Vzo6<;i>6ohN@e-QSb0ZnCF|L}}suYjT`RRK||NRzHYsEP&?dO$&X z6GG_4hCt{@l_o(T93U7Wl+YXjBVE8yLQ^9x5R@(;@b1jLbMMS^o%jDuzmSuYz1Lpr z*EZm71`@DNNQh+=_daB`z1|NnR&Tkg-&ju=aU<8`ySB<__6b+_fONlh8@^k0{bwQi z*n!Qlv+Gx907qCE5}Jcu6j7blwWC5v?<~;mc3J?Bey=yVc+cNE!T5i5g1oFwuqj|~ zP024i=n3}Wj3w(Uh@+eFV`^`g!78d0-V}bYwF!I#3Q%$16bCho$1^<2ed~)+;1m>dg9Jvw#&!@$NUx&XySYGuTOxu7gwGehjtB3+-kf6J2F)Z{Ab*Ph53FTJ~I0oww7;t zoYjZJdHeVc%#ejF`&^L!=5qguwQVqx$sGayxtYl`?A3rnOJxCk6{mBpPc_}B9!IVH z4*C(CZX&ZJ=Se1j3f!Z;8k4lvNSkaB;cQP@(0SS6fPhmF`0SO<40+GqU!VsDga zSJrCnB)BVitYP()(0n4`D>&Y?RvHPu?YVjMHH$cbon9mN*AXxc71fFT8L=sKv$l3D zy<qTUh-WE2h-!HsEt!lVp8#Ye8!J)v7)2P!hdF8#)1eC}Qi<6@X2^=L(m#W=H$e%5Mn5}XUNS~vc&}u@R4eQiWn$*#S z)8{u?h);bWIPXn~1e1OHCVZ!8KYCK??~eNny`MZk-2jRr$!!rV&Hf)B_ozDrg?tzz z^*rYIKweqS!`5H=4(w6`-#s1h2ktzMUf%G0e`U|0l*9)wq?J;X24V&s{54?A+4#lG z>}**V-yO{C-_PA^-2ZCup0XOe@{tUat|c}uWR|YzS3o{dBXm9%en!%b8`hKjK0er! z@Dy>}wNUyQ`oxw@N2y_ zJ72!Vihc@~Bukd7p+um*?cqtaezp!Go&`laq@QG1U=BHCe@9d0d^!~;kEt7>(#cww zk|#5IhUUm^vRacyg7?kE26#xnUZg*W_{j*2_{MY(83Ku2a*rh2h;^ll!5c$k+NH<; z)B=pTRSZ}k2sosZ9IKC@EKarM^jK*{F*Q^7{;gb@$o^}I4gj2$0jrt0_dHEdU@hNd z*P+en`X{2NIGOB1w#KrWmp;K|APqtJjztjiHD&=cFPs_?t$5DhO2ZYW8Br(X0BF7oPN6RQ#!AFUXFGT%nj1F-V(D~(=&ug?km~v<>7cV!WCzR94igkoN+D2 zLOICk422H|oH^WiwKZ_27v;BHgo@-+Iv0Ty=c3itLdw(L2pDPf{fQIgCvTREXJ+I| z{jLjt5w=(shN(R~W(2Ev?0a76tmbL|W0*p`g)q!;e1}yDE``d8gx9Xti0+2$ER~I+ZlzItJ1kYavU)nQ?vqer6}S zFVO;7u^tN=skfe2YonUh0S+^GC8ZTtTMBt?a^4qxh7^{3pmt6$z~aCy;Tp9N)QR<3 z4cOKtP12Uq)p@yfj+_}Ze}DCbBY%GzzB^Ch{Hdb3+uhp)%e-uk>N&a1a(B6`(muOK zmEDe^YYWVxHD>oFF+;P0Li$=`QzchiF*LY_%PunNA9@nBB$E+2B0JVqcYD%DFu8vY zR=FDt!tHm})MK+pyPnRS=wc6Po&T{Wzx?Wr*&EU5!36?lrfBm1Rj?$?l{$T^33T!SQCSiK*#ZG zlXRAfgI$!PEMX*Q>&T%!R3k|wPdys zGtQ@vfO=4Zu`b=tfZ`Ks1`W+s^JvK&NR(Nj$9OH<X2qQ&v`jT04n#L*bP-| zm#J^a3@sU2-DO)N4ws(oJo4X4cRP=r3y^%(v)I%C1Y$C&uSE^Mt32gnO9ld=AX)cq z+x`~3E)hrO+F*xX)X=Z6QN@_vt!szRfbTyFPRDitQQH)3EfuJbn5mxBA2VEbtF3^; zNKG@KwQ|pzkFuAocWv)NasHmY{bMpE4U@xyz{m%}luwUXM+b z48RX!~oeM+=>w21c<&NjLjTbxs4WldkRo`-fBX@Ujo>NbtYMB{a0>1qR5#jYA z<7JWJ^D|eQ2**MbBAs+^yVihqnN}^FD4aa>L@ibXe87s7U82;8J$^^!bwmiAd+Y*B z++&(y7l-dN;8^?v7Q2Rw@l+EkX?7-h;x@i& z&94#8@LYsxE-%!$dTQlzaKR8_hpj;iMG8})N6EO9HTC$iGZPN$cemc7MGzks@gS1^{RAcE23 z4o_aJ2|Re|B^`t=I!DikfBn_ZtL-J_PQYKc|K;1{eFw#$q9<6t;ypN;f=v`R-+I-t zkbO7d%v<=U!_((Esf2yj)xY9`RNe9i&Xh%LrOu`E7nuj{$PH)sW=jQrI#6-m!ck?$ zwCmM%4i+urB8DA$dk$xIed`0yP55s6OpK9-)5sU(m13@NHJsA%dKAtrAv^J6MXcJo zL%?2JDX~X>@55KufC|QbsZ=&`d^5(Uwy_=wJ?uz7Kk7Z?>TQnH)qfmWa0UU|9G(F6 zPRaqxZ#!}`oT33?>-fU%!BYLRBf+D@AV&q%Ny08Q)&!+?rbmM+}?qKx?zfx?BOO9;Xmv_89cuSmfy6!>-gh9&i0ZLhx#wcuKR0YZ zKRy0sog+8V_IM|vhmAfEZHM(Z3Nyc!5^;AU0m@o#RFJ>6 z;*r!vx*5Y?Bt+|+5-reQym1*ZIWG{-7!P_hK5Mw6m5D5Vq-7kaRh#1WIH3vK8yBd? zEbl#aRsZ^7WOb3A9Wo{>WNR&RDhVTITGCB@E}f-WO;%=>KaJ*t@6=T*H;{|JC8HRP znduuOc^|SprT?8_Z~nBtx$21!BwS2A2G7>d_5j(z>8wu0yBA1?dLKJYy)5;eDoais zE_ZyY2!a>66?*{nv7!G;R83=b3OI@@;n(^3HEOqkCq5(-NmbGw7f=T&an$oM`iL3Y zOsh$Z*cJ#`S%S3`#w?Ct-t@AicUPu;?#)_}&9zHgkJ^y!n?-lLI`p?L=fdVrYj`Hb z|C$AY@{k9PTH^EX4vnaCTAB(A3TmtAD;YaBTDopul+r!<||t6jL_4Paa7?AS#<42|xwg zXIu7Dv~PnPO9cuKIw}b_(_gBPtzL&?|j@wDSiLMhPRK3AA_S9X>2+DXG0Jz?P*176x#f=q~c(A_!;3SUjfc{#^ zIHuh#SFmzOvF({YXD%zhK-g=U_I#5vKaJmS;oSdBs&2_}NXT%qTMluEkh(|LJLti^ z-*=rra4sl?WTuE_$N3kDpTS z5VZdAE*NoN_QzW*crinzCE=UNm9@=p6Y-*ITl&AE7QqH4md@Ng?MG`2?wgk@3IcwA zJv7KgJ*3=WQu$R7B1P9P#sBC(p|OtQUJ7_m^+FemyuSNatn~hC&3(sU805Z@2An^M zM5{0&hrlp73Cfy}5gw=O4V4LRDk!ETatsIiX?_U~W0$a?rQVXR$We!Yn(Q{AeeT+^ zGf~qjR!U8&RjL{x_ zhTdcQ%rR~TqC9<>EK#?@sBbP@xJWBo+`46;vVOTl%k<4!TzNf+?-ki9RNI=5Rp2O@ z50NfpTzYo}`AZ58v1B^eu)ReMcJeOJj-h0;m4u7`57h4`Si|gmhjjb%SNmZ_O2Zae$YEIA9e03A^L5}lknI8;m8UxihbuXGJxGw+zN9iD!pf&5p zyUgU-zL`f)CX$AY_w*>yIx$=<&VhW?aPq^Le-!NlPURisdGrdFzIuvKt=1fpvQN*}tHa-q*}iuT_Bcw{bNAFufVp86gj2ycI|)VuRue2 z+iWlRwXaWS7@LGSM6a|u1C*0yhg3dkyItnyHD#6IpoMw&zLQ&M;v`kxpEh|T_6(%I z0Sec6Fh%Kaec6Ay)BMEOH*sXaKEc>6{1MHJPKSq z-XRrX|HH0;BOWxq=lgzxqx5})+(>Z+Y1Sy!Ivz1d86xb7aQgIpjDPE+^^Ytxn67T0 zWV-~;i0He^FX{gl_>aG=A*%lE3}7)Wx8eJKWN&ZWAeiDxOgc}Atb@a|zl?d;mgV?) zamUCZYdmC#mtnKm^3tjCpkBcK*I|80}F%* zkx$GJQG<_BAvd-xe2f%~w`?$B-lC1w8O8kCG8P$YGZq;G?_yexu_kYY(ACUe2xQLn z^>Kti|BzA1flbNfMS-*tga@oVD9`EPR2CIS9<5iio7oDr!>$FkW*WTk&i$;YRB_jj zdX8vmkK!Ha5$fJ$o?bnrLP5bxp1xe%(2ADS9#REtXzrA%*O%p$yG(b3}>`Y6Qvn)@4fgGAw+j3RE!PY<6rA32g?#HJ@#aABAW9Be_ChLNhb@0`yVHK~Kdb$(d*CI{-h;=ed9#cw*H0Wk z4j@;`OO;||AbM52o!0VBQAne&4AUL%B$81y9M?mS6RR7*(X{U-dX@x9dcUK1s|WoE zD;w#{tAvFBk6a@&68(2A>7O5EBDvGf0Ac)DC3@@Ls~VxqvWX~5gCYKK#`?vqVtmX_ zpDRi=K4rCzQeC}KGvsGO0M9oKQicT&CRU7SY*DDUI}>}M09p-Fw;5g1t#9Z;Lye(7)*wd@20%`aDFPBN3pP`jb>5zOU9J6rJHbqK&9= znmPT%cC}6bkxL+A7c3B0~DLc^hPXNNNEcSnk#~*9(sbE|< zL+7d>)Rs^Ub*{X<^u?mUQz_6g9P9zt63kgL3RamkCTN7{hx6SzY)o<~N}ec-G{I_iJCL>peBv@JCo*Pp3phe9j%|#twgUlfMV#aRk%eiv)j~YsKsv=3p4n+?tal5ZiMV@25TbL%U30MqhlgRKv4k<_MVDBlgpWu< zHuJ*?aMbv%eoy7-UmW%9M4=+RaUHJ6>=SQ0To~6bQZ~XUzllKwmk}w}s>Q$DC@Vm@ zd1t{??Zqg)!%nLiSjLq~d|t@5UspiZAh4t}uS(W_yNYf+q#(YDGx<9f{BQQ%#BL{9 za~VPwp(HUG>P-5T1Qmu#E;-Jo?tK)a1CD4p032dD>PABBD<1#P>BcINQ?T3lFpN<2qYK^% z0}3~z_yck*Tu)?exH^qx7+LR8`WiO4V$M6L~W)Mq;Nxhb;nlp)4AWGm!0z+ zR=H4!je%&Tth=4fXw8d?Zm+o_zlyg023Jk@sgf>P2d*a25*ACdj4vG^P*Ex0G;Q}! ze|rHo6Hj=i&NnJDQH%Os8jA6L=&uCB-13TiSu}gtW>EH4u_pQoLSYvs3(orjy@Yu1s)-6jP0al@?Zdb-V?@P|7cz+pe) zu_Zx|wR6&Nz5Vs`x5MXNZd3<3MexX>Ekxn2{q&-;x^q&-w+p)YJ<0%1prv5jST!+8%l zIVX;sx|)Syqs0%zl2UX9iltb9_x{?#4l= zfr^CDzU97oK@806U4BearE5vWP<@pZ8|7&sKIVa*!|G(^qgxg#<0cjiEYs{IUuI9($36F?X;GsjD@tXvu@`SYFGy`VBXh z<@XMLZGh|NyGi(h4kCrF_Swmqe-jRSGVq!qUPew+&Ae;2!83jHWHi2Wrvi=!VT@0z zKEX18uG~6{bYcu(LYSHnmZcLYJX`>?Qt+iIcyq=o9fU>Q_RAEv|GOmS-!J8O+Uu*J zMfV}gQOqVeVg3S8(MPt82ny$iwMB3}f{oWkR2hyY`tcl}kP5-72u2)|{}C2#;oW^| z>vP8{rmJzwQhv`gkF_GCI}#}QN^sQuCEg&<3oE-=pbR5rKefX~ z22!}^@BL9=zr<;@-^yAzrrZdST$ewLxN7JCZrvS8J9{4s`8C95vQC{jHhd8h7`-fGe4k?q{CHD!kKoNnc(>Gp|y>h zoCODO?k3g!Un+#PIVV#Mfp`zPidnF)^-(VoPZvWn-o?IiKgQUAK&Pg^I z1%_3{e<>)v=Qda|T59?$s{J+C`^uUa$sGEmS<%rUwZ6w##>r|J*_O|viG6~_E>(kI z*A5T|rhLoUp7_(V==k15O;S-@j2JHSo@?X`UKA3MfQr^g98zWGFTCT1<_fk?sIe*E zi2Gz9T4Ozqkey7Z_UGeQPN%462E$82y_+A%ESt4vy*3^g$te4aAKEN%YUE-LE#_40cY` zKDO&~S#hSlBqj8VUvKk7q^U+TA&8AS*P2xPuW1swt`Po_w%2^xdE9-;tS-nlhI{B* z063L0m@Uy=@cB6Zhn&<$nwU5W>JUdJ+RYIhvJVoxGy`tORO=K4jzeBiQlodc(<jutqdf%5_%^%GL(H{~>bFw41NJWtr-%v`b3^E2N$f&=! z?>bwN&11&I!&z?%pSw(Roc@RlG$r$*?mhvhUMoac<$j8;64THQ z(XTdmAUK2_@mUchg0D{{^e4S>J3(D_*{`HDopOneGrL;d&DM>%=5@Fp@~z0P>b^#V z!$(d-y-rhefQk2{DNeb;nJZ~!Go9QQ4dJSpl`TLsHrC$Jkp}YYo`Iyq)Uh8E3RwPz_Y9-_7rKcq2RKwsL&aT&7*1 z4j%6|m#t@iKqjFRV3%cu))qj%_c`XMu@$A8zYajXD_**Y6r0Pn_|6G{H%+{KTm19I zUj`Fkry}dUgdlU7f;tPy2!>Fbe-4gD33*%e;II6=8 zq~d1IMjSXnCBBc@h31ejZ1EK7pko;?$Vw-`oDul_QHXlln2TNLPY1gDuMe_+Z`fYy zk{=a}@9}%o)b~BMeDF^7G%oTT7r^HPSqM5ao)^Pek^buBCh-F&)uV~zo^N80kh%e9 zj=OUz7~=8Z3vf%HK1Ad_bA6(twiG|dK}~osp9S2{`c$eVg0cYMcMTu?zbfVB73c*)6IL+j3BTCqbVnsIkABzoq`^G3uj* z!k}@CFtoW6&NxXne}(w@F&@GG5~svLNy58vc9E)*M@m3%ApoJ4w93+~`QfOqLx%7= zHh8Ao-K8uQj?s}<%G(#{6A^Bb{E8gzsm1+W?olkI5_SuZS-OCut$kv1V%qN;?CYM% z5gpw4g23*%!_zWxM-wAwud5=GG@z=9$z98#8|uzor8kS-A`3K*1mu|7jCyse_Xmfb zjUYTE6EX;sdRE}jogIDK^z#iS!!pPinV2i^oRPsn3_}Bqac}db9>F5-m}cM20qw^F z=i-$%$Uyj0BB||nEauaS>j^4mbTN*Tnud4lq{BRqoy(j=h3N~WeDLm?K1RK=Zf6+S zt%*6#-y>QvnIT|ct%b9gx!ZtW`*aDgYGj)3jH3CxTXXe^LH_lRd+esXe>ePD8D%Z4 zlRLJ*-{9B1n4c+PIr2;yG&?F)MN zkaR-Ne1S3_+%@P`>~HgBlhj_fr}doFMo`?gJhd>gldrLiV4RYx3)jRA?h8iL4&u#~ z&jVWMrnNpxQRAe3k6Iny()U8rr5_A5Fv~gjuEBUr~xUX}GRl6wYH*GyTw_E*b+=zKDbpBF&o(u{Siw=9kR-)Nh;oops-Im1xvd&4Yh@~{ z>Md#K#^W+avO;pxKI^ejEA(h@UBxXBlUEgEwsdSXaT5{p4oYrVt_Vh7pi_!IuO?<7 z#7uJ;+<#%ix+E11BsqT^eAm+hV><`Q*+jFS{Z_v`I4LXuV4rDA4?al>m_=e_3WSa6 zQ?cKxI;LVj0l~Bm*hWi}&!ZUkc%(vtG<6Ht?&M0-y&{~PU75Bf%?472OWK$(c%rqL zZsxdt4RRfG*a{9n+XFB~cUoy)oLuHtnVSf0;u$0ilN>P8UXkT9vw#Z#k7^mfv^@-( z2Zp>iNSjCnkwBvsAjmhGXYpiRKhfpqk9++NIO+nF z_}(aRH(}b^d~Apcr}JI*pO=xgl-KH8i`o|9O``Oo%*+>uOG7Cg=h#HDwMXTXKT+vA zSGA@2euH787aGm(Hq?us5n^vAX|75cZ&XjFy*#|}ME{JEcWsE{*msiusuG`2h=hAS zmv&-RnLbW^spq>W#B>o1XS~FQA`oFVGXP*RT_%N$W+7Eg;h740UiwSb1C|NSu2)jS zC{+L4fjCEY>b!)(t91!?zerXG+5ya>j%tf(UE*WP0`|LqzMVON-M-?@V7~na99g+Q z2^0;Np{yWPMSu;%MAG^UR+WBLsRHsQN(=qXRygCD=blfqLgl@KEnWCiHOciOIHgfZ z_xyqI`FH3Nn|O=xnJKaCBKr5WXK;Q@GU@0rdZqSJZI%|iu|1RJ*Q7 zrYxs|*zuqDYcB)$flh0%cjWpp4H70`bm;=Mf0~$q@3c`Ni$aSBX9Y56vV#ryoej@V z_leF5l%{v{%Sb_)LWiaK=5TZVqeVDc`sDk8_vfeVQ4P1Ybb==7l%(0$A-H&Ne-hJ@eeL zA?O)gl^u@S_fZ!pZ!ano5%)CtYHmy%;?S_rDgbX^K{T(IE)SJPn)bw`9&L5pS`t_r zcIY&DHU?sNHhyo|uF>6elX9@sTz>eCDD&5O{Xf6N>$sg;)7jg3G3(zJs+HYoqIbEw z;uS1h^q6*uVIysC?|t_R6w*fd@#;^SxmCJ-W!{rn5Ij15r-pX6vXz-&;{B%fV)ryX zCd#=|Um~|g6i;_O8kD)$qb6xHF+ky#7mXSnYJ{j!9F9VIcxOqWT$%(xyx74SEzC=32JW8tJJt!J*DIG zm|fn-#-*e1l3Mq#HQ+}e94H2OnH@qLvnO=a&y_yH>NOeNqpm?HK!L_f5-kQ5auRkfG~HRRT44c z9MXN%sClyzi2&+{-h&rEf=Xi&`KhP!Ef6e0Di3h&u1q}F!m@1b?I4yQWhN+Y z@!?OW6%%WqLkESGrtMjdofY2_OrdFfLDcAQC|GeUvp1o&1)G2G=D*@L)N(4C&E5*p zf3`|4m*&dBQM^ahdnnW)$^@Z|$(YouDN=SV=CU<6R_q^v^I^0|%6bX$%yI8qCd#Hf zk&KGn5yT8W7ywu!&S;-PpQR3=2{>ANI*@8ArJu(xu@dhVSw5TKCoF6KTC-H}9hR?4 ztQg2N$DV;WppXc6D;~a|_d3;V-wR@YM0P#%j^}gieY1j(F%dW-NTNT9%-bE|kwrS* zD+-w}6oy4_?Opq@t};{XG**1U`C%%;euMWoKx-6?yl(YHb1>zSi%(l%g3`|`>CfIJ zjw!h3PsvYv(qye z4$6LmR7|JWs4tP7{WHw1>?AHI*D0S%!KJ(kND#Khozz!VI-ofbzz-`2wxgms={82% zmN7V5%RItAtat{vFC)~V@Ptr|cye#xY}LBbajIhmhQjz#Z1|Cs1DTEa5rDqwCJ$pSEy7UB6ccQR90^1gf!730x^Z zLH$ioeK$gflGI71LlM@f&`gBpee6u z5K)0L_tDgmXf7~E0N3pedEj(SturbmGelWNt?VI@pPnVa>#g-1iSU8>f7hGb6~S-{ z9y;57V1cY_gZCtiR}T?}Z+6#z!l_wzk5mT@h-{j2wCTEev5ntx3sKoFM|x8hpBzE2 zg*&;YY1ozp(M;1uit#JAOw3VGoMdx|Ecdwy+>o)pv9w^}Fk$ru3^V$e=vtF7%y$kd z+RIj%+_3$6%wRSp;Pg?Jolky6gPwuzly2DG{CeUd@5aw?Sy23*KYgtp^2I81n3Pa? zavL7C^m5n0ql)Xz9<*c9qYa%2hX=b2KKAv{L>Y#{7~x2pE3Q_>3xz|?-UE``X%mJu z8os)GF_R&E>Q{GFq>IfS-EQYk$r9NJz2Q21lKM=+9Y3&KJLo`U+x&F4WM^5BQX<|v zV5tP?T9k}9`(VFk0x`J{E&1}$k15&Y#sQO<7Y}Q+bDNwte-kU+^e8Yv03~vlJj=9N zcaX(RKh^FCiT`==zLp5Y5J!p=EuZ5J&=4A)sW)jB4y6C)*6QmK41^>lX@De1$U_eg zUqE;(6J54@=X&I{b+1hVxy{)XJ$CA-hm;PY|Io2Vz63$l1%`uIwrldz{Yyuz)`pGM zj43TLrRI60wqcgV!vw*~Cz;3!m`j_A4sZGS!E7WC4(6n3mjfw#CJ?0yX?>d&nE8JA zGR7mAf15}qyy%;CG3#J<^W z8ah&W1*b}>u$ad02)LZ&bp|T(JyP>VYl~Dgs)Z<>HJ7qK^0)Kv&lk&f)FGU~7tcq$ z%x4jVQh;{ z9IE)?AFSfDI~d^{u1>NQgfNDtmKe5~w3m{$h$Arvba#1;Af#h|DXfeXyu4r%Z@Jc6 zVO9e(m@Mt;xZw(025;4o6$(>I2KM+tko){V`JeY2fFl6Fls^Ma1rHGj-2CglF|~$6 zx-%2rKzrh%>>gJ17ON@-iRoH|AJ7d&Yr`;cZ+nHHua+TS^T`aqc`>NO$@{?vNrT1) zHhV-(%5Y8Ly^7MR&7y2S4b94~H!zI-I9uC+fe2+Oga%!#7msHDweO6o~0;M}NxH?7jR^aC@P&9ye$UI})|c1=LAA z9p_>_U+5g4gzHY$?pNlZS#qob$;a*}k`18<-`n3~7d}_#pSj{^qJdhO6(f(-1mV;d z0vY!*?!i_F_&G5sbHBHsdsb>gAMbA*kI}01b2u7rZ-+Ek&nG_^BrAFFGKV-$Sg#g6 z6L`YR&5v#g?hu`l1jngWRBUh20xik;MhRXGT#3^DH2cS0G`NZ5*)0RbP-WqNs&jhJ zIat*=&=%drKkUdfKlKCX*Zl^8! z{Ccjz0+;)eHmVneafs5aGPnQUB~buGObn$q{9(`Pt+duTfyW~o9RR~MwLV+Hozxwi z0GV13ACq>E<9ZP!Toxo;UK*H%jwCumvAw)A($^MY`_d;Y@_qk}lot3&MOA)c$s{B|h%P;Y*?2O8FZifgIHcRuzO&zONG z;0uM15@SN1P(}d$o40rj%h2Q8z|tX?LCa}+A>Lai>5C^O>H$tEv$5dehs7A$O=4Ac zDHJ36a-%620q4u{SB5&>it_o$Ik+?h{_jH#Y1o?gf1(<9K{%0TWc9~~2io(m!y7sa zO(Y`!dvyO5E--Q1skTtckK`pr{@^k})`7r79qIHkD6WFkrsniQVoUHS?TTGk{^QW% zX@R$Ne2kuh_QhIAr&F|O@<>UdD2TyJHXwGRxFnByANA_^VA(dxg5rEcmSKfWp5 zjH6{cC&luPU!_pHjml4W7T74F)ZN@`FU4dq03dL~zvpSYWX-HK5QnRkdC;}cApiD{ z)SBv%ub23L`hp*v7Y2fsOW?pbC3GC-3(_A_*&@D_?^iCW?R^P753_v*5rEvcAiSo!@W5PY1g(7kxrDwm3bD zKFR+4s1SH>@X=~(dF&aH-<6#gH7~%+*%PsspKI;Y&U>iPIvVM4)TuLlYadpkMGpj| zPQ}^^L$~HFjP;CRy{1c)3jjH(I6eqsyDD?qeg}eo+T2ZB%OgYAh-Enc`}m5PYke8= zdc_wD_Q)dIWr|$eb@U%;`KXMQdj#{rJ4XH~b>}U>3@Zj=Wh=?5*-B#sAHinp(Ismv zBknx!W1j$FFUht#nZR~w03`esjfJh|FhBJR;OgI6Urr$n`oJw~u0 z2X=L@;r8A8-QPOmOB0Xy6B{DO$CqhOkl4T)=@Uxy`8I~ z?bP`R!C?f z&!TpBpLr%N;1?BUzER}~0rTu4!=Xl!dy>7$Tc%5?Gdl(iAh#Ki0I-?9>cRi>LHGV^ ztjM=ge%0Tb{f;%G9h@#{s$tG%E&64a8*mGW8VM0REt%x=>v(plIzb}y%QE1*y7Igi ztEPdt#kZ?8cZ@u;o5VWu)HlH&p$A|nYnnz~7)~VN;=zkhKN+s#@TY<9DHd8#MXFy3B z(&h6mGbESgSf4H6P8vWR;+mWUDbG*-p`d((Va(Mq!Vw2J`SYd9EmUDipHfCU7(ZCUfpfb1d9YvXYvjmu)z*)vPw}$&0=l zTE{U-&LezBAL1v@{+#;!(bGZqFEj@o(4z$-?6gDsJkmK!!55AoiwR5asd%Lp^yXI_1eDJly~riSXo zUUDZ73c#I9Y_6`$BE(y#vlOYcRrk?t%Q=m|9_fEM=uPH#s;#?=Xhk2O`yL)!dg3qZ-YxAdXZw9fTj8+nu z1-=Jd_~k#C=-=R{C~3gR7Wwl@XDe}bZk{^tR3zI^n=}f@owY91M}D}%b%c<=a!d0K zd9?Dns4=BP;asbmg3m34kDOuDYT;_JjQVHs*|+O7(4*>jCtMk zUW>3nHM}VzhMloX71SVO9rl&(bgaLjxbR z4@Wjdq7F=cS2Y%3rXMj{qmrHz9;1&7_lRb1$QmA8doMBgo>axhv&mjjl8J86!g_B2 zopP??aSg58}54Mpv*j?gKip}q@gJ(pJsg>xnKKcNEY>b-xkFU^>{Rr;Xp=_2YE9%7#g6 zY&op#=*KFT=T;QjL|Ll{`^^VuR-gMlDa+Fi^(DT1lrs=2D&e{nw0&;EY*qD zZ+Ioo(+#a6k=Ox$+<5HCb3;9Q+K*jO?4RK^iPqBfrgrrBim(T*4hmgp2j_5T4BFG* zJ5?TAKP+m=wMO(;8)||jv|1Kk1=Y$%!XbXI#1Y9IW-qcxJ(?cDh}KQ*+#h{H(*#oz z{0VFOon2}L|cSkKcm z<+1qq)t1yE-1yu4V6o;pNI{P%agM8amQ{A4F2OeQb*P?TbKI@Z;^FIHwQpKHTMs<(io*>GpaoUNPk{Q@O(qo#DixKX# zjy6D-gC1hyjYw@aUl!Cvt>HaZdU|#@(J?+b3e8T%{yMgRZusVjT_z3$}+GU z8m|^hm9L5}3G+`y?_zr&%4D6pjcDfzKSs&3?X*nUkVhTb>sd7^#gdCNw;1T_V;ZrU z>4$%lnOdq-A$)h0-@GlZ^u1uxSN86S=p?@%Zt0@x_OMd>&GL~<*K~YVooM ztQh}Ti3xg75nP_XGmU;u(RoYQiRZhUH>+X4D%<_&U+oD&WW5iR> zp1^e>{aT@`?2R?&O+E~-i7kI;Zu@K_8gPhh4*mamCjD`7tCF)&NIui{lzqaex&SnR z_tR?MhOT5``ld-e@j4bQEQBi449n9MS2J~p=$7eu-j}`4JL00r4j%NFvEJ`-HWFz^jmbmdM`gP186uY4b{dgb!PfJNuH!$)rCircM|szw z>EsL@jqJi>Z<0M(9i+XmUE%MZVW)*{d9Z5h6(3Ixk?maM+jRBNKQ4xwZ}MY>opm@s1YV|fU_iv)CeOMl>f+k>;(QUq@}I&-XMEtCp|*Et(ATslmxNKd zCgQFQTafDf{+-unm%S9uNj*Uy_6UV6VnFH24h7R32{!H-QWe>+8OB>13);Ril2g_^ zja`mXi?e7m0wh#Zd#mU$xLCaGrsjg^<}%spL`J%YzYgvl!`2q=u?QQ++^N>>o5hv>nxdu=^Mq#QlD92Qx>{)V|GEyUL}@*54%OX)prL7n6znlm-%d z?wIOzZY;yi7GKOg$?x0Noo@+%=(Ti`D*CHraLH`wG{B8sAJQ zyLFus$XZ`1r<s0C0kc}*3*sPin7py$u zRNx77H7h`WdO!d8@R8O2mEGg6ja7Z=5UlTKklo95-8XHaauCw!XoJpMfaxTO2+8**6Fe=MDNKdfY5HzHF7 z&GzLuI|A)|ZdGrmD!II`eB(oWTsmJ0d#8tO5qe^g;b~Zw4_%eoX?X1awysupG;me6 zUw`HhyHXgGFt5gZn@GC(z7E_5L7y{4Vz~!ztAE2iq{auXBpEAooZU^(`3CYfQ=#6h zHmYWoH=@J##wb0h{K%`Xi{VrV1ar)TLgx&O=j#BA0q8{Rw2bj@VHLcmVh7^QSOM_% z4ORyizUhJ`x_@9Z3VEaA>PIj!*ZAwLsN3J0rV<9GF_Grp>nr*MzQDCek=k2HSFGJf z{5yY-E!*3wN0y5-ealb+odC2gL3Un(4udRq?1QnN53|7W({FWrI@N!9-T8+K6H?X67T_qc- zSIeh!6**Ev@v9N5m#=f4R+hcJZ1Db%9Mhpz0Ax%`AQsb`CuKk>YBMND;rr4VrfqUa7zMOVs~C;jPx=qZ4bI=^bTQuM`Sew5?0aLt%rGVcZhm58%q`#7&j5`WP`N7DuZ`D@?WQKE#KtF-TuDc%tN?Vxu~Hi z>}~UpR6p+BfUBfzbfPx1oreFLOrckv)>Fa;>Z_6liZs)?{n(CUBkD=~4ZpyIjON0^58r0B_ z(Zf|DkkWZ86KqFN-~2e4K%ckEP%(wZgT2Rj!Y3sjXLSwz{S^ac*HSK;-hSc7rKq}{ zbgj$-)`}Un9m#xI9R@Qhg!z#K!LCvp7Rl~)Gu$Q)BN$efM>&Ut@sB>Ig#+)a6Rjrz{Bgq@{I#{Ud*m^?` zmH$lO$+s7jWAqA!`baFB;gw<|c=#v2LM6VgC~7#()SX4J>Y?oB=1^O}TJ}(^=t7mX z>#Gd0e{;C3n-ja#Hcb1XWT@mjVX$FG7X*Us$aqGJ8B=gWzVdlg&8o4@Ou8i#uZEZm zx(NEbuDYkGJ^yvw<+%2hpVcAjHKhDmy6)b*r@~tzD?nv+J_J5(Bv4-BqG#KK@$B6! zcw`e_h6c$5*EL@K7-)F-Z}YbMM*gGn8^1+c#6PcWL5h-xG#@^qD1or(=%qYXTG@(Y z3x%Q`T)j~?6121!?SWMx)dDr8_!p4Kq2jt8fo{DqR_qwN{;T9w*~`9dqfhaoH~ER) z^Rzw7BQ_a_tnvn;FUu7;y!~~%Hd>7wQra6S+HZ*!Ur+;fAavCK)X$Y$JkMo zQ;ujITTc`0aE;gQzgv^yuH9bLYdz`0I+DxYEjREtk1|Ym&{|L5<}S956DrLij!4uS zCh(_P#j+V^TgDNd)T}>@4q3rpm3bhUu5r+6)2VG}Rpv>>dE<2)YKWmYQ|XH6{GZ=p zD9szWYF1_JGRBQ+l&lC1LIh(Z&Q_SQeHMeR3N+L|VaFndI2YHsEtxrqY`8vCj|O$p z#jl%hPc?s3fWFU7SqdyvRAQSlqxi-fa@Vl+b+bE>a0kQEUFDO43YEA;?8|MoVdNQ) znXgjLR@a0{R%7dff2ILPV%f!&p8LZ&6=~wbT|lQwh|G1Z?$Ez>{5*D5$`JZTn`-Z8 zulkoZ=*gn{$&#ZZ=K^9JQmFKa_bmVZ#Be8VVRfBxNdZM(Jdb4Nz~n=Lpb$LC+cUaY$ij{5D&O>=6XYd8gFZFM(~0!*Ku5@@qfpM zw{0z;nd)+5$nhC2OZ)y4o8!AGMGhu)l175@j^;R`)q_tgZN@|enJA1RqlJ$d?=d9a z^EHM}E%;yEFNSOk#(ZM>4nj;BhNr&8A(6vcX zn`~6*<%37VU#)eeJU7vFnz1qYwf%Lt_sy~geiIo57X{jnnuW;cVB}G&U6%bsxkl)Z z4+ES-PfOm%IbR5NQAG79XrMeMn3^-7|Toi;*qke$m2{9y?B4Oft3C9=5gFsDv-Rw6>3`_$j73$C>XP3H$ z(y!tiiBpx6&75!Lam2je9h{X8BkHmyR0k3hpzvWXD$~{v z+O3v{!M9x&0hny;2D!z5&5=JsSzLA9UY`A{N4EH~{g-v_KV9{I*+dW-LC{Wp&<^V0 zYPjv<4Apjx?U>vu;>Po>72}&&}t~d>;C3`oU(4KoF7sxiaC(LW>=~ zxOe{JJSQz#f;qluAg{;sb=H??tHQW|ppVMW+N4&cEf~~f54zM8QRVHY%f{%W8CUAF zW(J$JO18jZe#gsnHJOp@OW?kL>9Ovq0^oldT|h92k!F{c>1T^)L{=u(Vc$3vy6IOl z+UZFS?32IklpnZ^D5LnKtcJ3;qks3ykq~gOl`RRA^HkLZSzs9J15OD-g=x_t;3pvSN7mhwa6h=4{X!>Lj27hI>&VcUm)jn)3WTITzdTV(~2xa-bOB z;lh3VCci8AT#cp>I_v&s#1iIoor~|f+7*jOzaw`kSjLO$^QZ>i!AT0twaX1dGCtAb z`(uo=gR6QjR~A9mXhz8(3tAyND|&fVDZG1Y$c9sLaqAj-HJ}y9qWWfcfIsPWOeOt` z;kyL};i`|rHtG+!7w-6&QB8wsAl1!!@nmKInLgFT`=`EPP74%>sYox;dHgBR)6lvxi{=MyF1ZromDV z=2Bje)Ptz?2BvLXw}tY;#nK`JT-TF^w=&{8zsl8;ZSg2x$sokjc72Bch%${HtF;MZzPN?T#DS*!Hcn2eQF$B3s`oEp+|9f)3g9rlNUPYyOWsLlY zOQJ`ct*_(^3)Hr*W(C#K112)2-2~_^XVb zGH&`Km-@02n_g@wW{5YR>?IQx`txB__l}R_HMyw#Dqh&rwx>xz90t!?#M0kCOt?%> zTz8J)FSRqeLa!bqJsH#`8(=>;Enxv5SW)<|Nc8_M z2Zx|l6PI2b1PzBnQRf_SA-%pV z?wS(^MErnI$frX21CfHx1^e@#jv!O8I-3WO`Mv|g(PtNQ-Um;ZTCe)t;uX^=9c$XC z5opajt*ilKlQZkL{1|**jNl1FKiY@NfeBn+(r1RU5(F|Om)G5ndQAPvS1@(oFn80R z5P2{U&;TG3=>&)Uv^Mp1>`r$TW9~n<2uDm-@4f<3KnF-9c6lGxPm-{+vLH@(@qi=R z0Bhv&*PueOQ4@ICjJ*U$uolYMXirXsHR5Dv>tuu>xbNAQcbXGTaH>2gE2e2T!Tk*Wg3jTE8B?G$zTu*!(Ic5~zQvuL59qI0b}E22t*8re3{rp0{e16ClWJEUihOMr+J_-K3?%cSA_Z zc6?VtjHM;{95IO_blz_DFRwUrE;&m=IdJg;=`&Mo6r3s85mHLS0`*f8uWBcsN9Q_A zous@d3RRVi^Tot3{Cb@&i9a&th_70LjdWF`Q^}>)l-j)g3aPw<R@jAbgqJ z8>bMQJjzaQGxL7SAULXaUuG(w(MiF1q)b9=;(N=< zTl@(Ury@JG(EeHue$-;H`z2~XUh1zbs=XTTkz7EA?fs>Q-;M9V)tI8B%gb|-U(q=sPAM$VJ`Akv7Jbj@G_mKwsPmc6(^;4ZGwJcih8a#7?^gqu9UfUJ zQE=crYj1sJ^N%VPSw_^zOBFTCa(D^msx^~I1bOY?uYV4mSQGk|{hdB__?TKnbJ!qW zS|+z9rT!z0&7g!etBEr^4u>kLI!6Z`t@mcf{&l}IPAu_6GRLh*m?xR248+{tCpg)5y zk{P_-S!!i@KpoiKxT_knv4At~LRLHotjFJJNhS^bJIAyrZBNnNZuX&LDwaRi=8MjK zwIhHM8}v{=rm69z-a>$u8%sZn)OINkKQ{3stEy%n%cvEsz%Md2C7(g(cM{{SJ|0q= z4NREMrF%R)H(CCzr+HmDd~!;SFv$5vk}w>F_6MSD_tpmkEh?U71*Tpa^iHzh;23~&iJVw2Q)hb7g$|Kry1)=SHI zN0;>0%^y!Ej`8JCQT%>8Kez9lIu7a+Oh0zLZceZOq%wsxyO42R?4QbMH026V{vEG$ zIe)*ex)%JyEjr|Mdk=S9i@$?r{)m3m1b>2)iNV4qwH-cFG@(Lcz>0^aH59A0Jph&? zh%zN35Q^Q-DjFNlQjVWBmU5L;IKS#=@h%o8K;)-OP#Wfli9cj=qC-4%GGrQx3vXrZ zK)1T|r7FOurqmKge*HPd>51&wjm3JPD}UG9L}bEEeV{Ya%K~DAH|M3VrVPHmps6zX zb(lAL$Goxw&61?BLD+JPGoGvvxh$ml{Yy6IkgdfT{6I2((K=D?2hXR$pFu^TFeEwQ zPuN{u*(ZZ^^uvG3iEDt9I9K3MPc}m1PK4|HY{h#8{=Xm6d9mEx`)!WKuI^eWuG)#p zaNvNzwMvgh0j^l)Q+;JLNzU-ha}}2heajgeN=JCN;_#hofL6$Ps(Atn=p0Q>Bng;)*#v>Oc{I7?MDK_X3hk{{_VWlo8I*S#~=!~M&VHu~h_Q)fwoIfN}; zT6AM(w}$2X3QS?Csch!}?2`3#B;u;eH4G@MlU@#57XJ}jwN@{;XL!9D*&WM$#tdd7 z`rEh8lQhq8mh~-CDIcaVMvT$K9wL}Rw;3G~(i*CzmW{}2q7snGn8a&`TY-j> ziOx4#z->=nm~zHsZIPiITE{Q^^4N=`(jH?4`~9i0)b{$C;}e2rD6>DjrjdxPD2Srv zm*4F+_?^D5e32EvFihjj zH;(>#=!@fV+?G$&1GObx>xZKN_XhJ{{`nn z2Nx&e;GI8=Y7rg23CGr*sY!k%@$he-^Di*O2$(pA0 z8#>&H>r~{eYlugLc)c-AA@y}5%)IJPj0iJA>ouuf{JEzg*wwiF=O(Y(NnunbfY@!X zSx*}u;Y+?9{6=jVk@8~v!n(rECOZuMWqWvl22^z~qYd96dM*&)?k$YS3$$2)9QME0 ztp_jO%2a3Z5tj`yo!*_0tsN}}v>Ob#&;P25k2(s+b=qV)XHuDvmErkXNZ#T#1P-zo z_#pQ`!IE4kt6Jz??$(wSY&N#0_^G)?snonrU$t-e6!&@0Y|!iZ$?FDm zh12ES2%A@&f}Ak^%IdNGPkTn?pbe?9;N#nyp~aioOMVFggq9h(!AQGjTE0XXX9kNr zctDW}W|9%4v8|<%3nNLZ(P<|56wtUbE%>1cdH{!jq**cI@7KN)8~JU`qLuk|-V7Kdc>jM&akDX=~!H9H^b-&FDc z&CTD!UswKa7&=geRB+xoUY`P|R=RS27W<#~6mUTD$L$Y-OUhjaIm_Gm>DSh?LQ#>e zADX%LNc0afQlp6}7Uy;Fc3e1oUB{#3qE~E0QNyMw4*i1Ff#%ne3q{>p0m~8JE63oQ z`DE7XyW1}4$MO!vIw(lY@MblFlxSf5LR}s|NlcALa`o=V#3JVgQVYsE3f`wv`Vn)SK zC!l<=IIZpglgRBC&m-TPupsz4gU@e6-F_(kolFh20-prb7$EF5U{!mQuv(kVoc9pF z_0c7BX*cr&Xb*Wf?*F_5EXTk`?ZY=3MDCNE?)YAS_9268jFqONhauU?@`-LBL82`O zwPrC&HJxCfkcVkfT;f1$UbT{9@k<9td%}qN)v=Fw%6dbcm9WFPHv9Mb-4gNFDAEy) zc}G;pq{cIf@1XvBn&;)Z=6TO=aH@BTT#A(?pJcKZhj-zM$(9s5%f1V^RB!FnzGgQ9 z>x(vcjuxA26!LBb@Lju_w51)80ubd-2NGLmpxF6+UJ}p>w9ep(;^%P(fbi*T$XA{! zE1}@NytA;^@Tb?9YH#pE<>T|!QK2uZegVuzNmNg3ijw(ZhKvWxf9D~K;dnAkL5LTxsHFIN0ih3`;N_JPT0Os3w~q14 zndIaK8>*cPMKnL21IuNu$a1KvsOFsGK-IL9^V51H!zVSkkiPSez!T~rE1YY*uY}so zf}bWx+R~$cn*@EOA#?9#Wh^^mb)tw*8d$BsDh=md91YY0?LSVj=3`7SG2ii=Do=YJ zxY6isiWgGj(*xaYD1f-doMj?rh^c5s<&?7mL*dvV^Qn?SZ498kIi(iV7V_HHql)*E%8|sLKya#QE zhEIN5YvJa+$tqF^6eKXz_vft)a~R<$4}8%J#GP&a@B#nuv8@%^Uoi=XB%lEgcgf6f zDQwfuk}d1uX1n}65U~t;q3v*xJM?l{$ar&9DVG3}P!nTsaXIpy(rKlF#cYz?08}sR zU;1Q-qczyWK!w?XR1lA zuGZ2W?wKPRNuY+prS4<;^GHE|f&3K+FM;%uF4*9Kw`Zy5HeWf6y;Qmq{H&fKdh&^t zaAwU}%K}9~SC04&p*CF&&PsrJPgTotPp7Z61Eo$j)qWQ32sx1S*Fov=b^`Goil1vZ zuUl!9EoK%FZ#DdL1^$6&(0#syw}5&v7c#7+QS7en=KTD=n_BUY=T$8IWtIt_Ahiy? z?zI9WIR=(ssKOj+{l#;F*UShq9W0L+_&I;&Z3T)kY_e8~NJm$?sNL6oM*BT)Q|Y$M zgb3T3r-{mUmN4ZoEmklzjtk8d2DWzq1U<2!n?@?vGJL=ao2mTP537~&?sr&w6a z4B?9hQbJWYnNMsLvEcF_yyTHPP4K6PTO@?mTq&r1`N6fu;%+b@Zn|i3XNn^jTEAr$ ztXL+$V@11NEaMHRxjLbnr2jIn&phMVNnQ^dea$t(2@+YqX~Fr_mQ8){ZxYI)L=I=I z6%OaWLpg=+2BCufP?BulEb2}O`w?!&aAL&q_kxLi;}DZrVRud7Dn~(< zIAlA3IA)^7K0W@2Ir7a=cc1x86u$8`Ps{jGenoB_$+pEnKaUvTsR7YBeep090lFjDcbGjo3uCy7yc|xh`-?vm{)bS^*Yyb8Bqm8P@ zjgl(A-E?V?V#;dWL?xVBY)?|-(x(_+mco^+_`#Y}8hU)v*wGZe+v-}Y(Q|VyV5tFg zTCB%^j8WPQ%;ix=6NM9(_k9Q2e7{cd4@Jlz!7B+di>YA_Js!KSoNbwZa2HDf|15xU zHl^qJ|F#Y)eYU%?`OCHLylxdqcrY?H-`hNS$uT}2 z4nya-xD45HS)F59yNJSW*4@I-<>tg$$K(R75^+5#yqE5$Uaaqfj*dEYqGkj;(Q4(x zPziQ&jGtK54LcUwOnB*Ta&+z2Yp3V<5tc$1Qshq`oUxSEVGQD{{5A`tN@vwQCVGgx zg*RdRqo#z~cQrE?#%boQd0{w@e7^-Dg}oVU>wa%#XyI?%;Z5N|g(14lYU&^sp%8yQ zqrvQx`%iTf%#bW-)A{%^pPAgONU~?rc4%TqTZFwVqevMUTe;HH5MMg=o*bc4#2*pr z;^AxSs+cLZn;`2u|I^FJ*5O$_zk0!dSJhbLm3#b>zbeNegW1iw_P0HFBK7{hHlQ#` zqP$qlv`>wUNLnQT%B-CHvEjW2((B1MS&%X+p!SQy{3EtZL*d$mR2a!4cSK_${r2mu zI*L-znjzu$rZ^WUm4QO3POM^%1nL|QGmsIahUa>zeE&w8pi4sNY<^O_GD~40Usjn7 zCXy{ndmITDhh)ETN1puod`7${d&H>fOhc9ueeFRoPEB3cRG=Teb*jGlyt)x8K`!F2 zxY<-U;7jf$x&oC*CrUXls)3h%lEaX}cTRJ!m5mENECybsVN+y>#R;Z&#~tl(=MSrp zsqTBvC2_{Lq`#HTHXv5oAEgF#I0INQ1FyyOZvP~PbNxdo|NWTY6_qcCH;x$6&1GMX zo2xq`KR$Hk56;(pAlnL4`}VV_^@<8-v7^+ZY@B|NR>lE0!(-aVcs^RWf|ZT1T{b?S zT8pYwq!r$3>!`y{0*lR^?!bo*vfFPo^PD#Z0?hT>@H|fyBahZ237;QSD}6Hi=~14$ z*RX18NS_ma!Ew%KHMABTRqj&QM;!+#)-WsoR@F+=7Z@-go)4dq>o24%@jVU$&mE4@|2KwB=Olk+H#paQRYZO zEGwSvpB4>gw`H4YN&dONv|`|=HzDzGiI?C_dznP3G0#V+vD}J5I`YK_ggjv2iXh6_ zf<1%wSlQCtz!hhmupP+cPFpx3wO9c@(q(i$9@Cw}MYy2i)3HE$Az z-BCL}{XsA_Cm$SQYBuvLf6hk-&Sx;`XU9dcNv$LCgLrrN9C@ip$)n+wf#7h5mYK?%o(C;Wc(Dl?J_zgBg|P1EYf^kf&=w3maW|}}uHI$aIHv!IJWa|8%B-FI z-Q+y@rb4!j4n&}Y&pj~THS(7!g&rt`;=Dbhy^lLA+|48$|@CSkYejm-V~YFPC{X^fhop_JcYy)wk!s}(r*#$kdf z`t18a9Qem(+~RDb7db_mn?ld+z$m}3)w5!0c5X6U$JTLZQ8`DGbb^NHg!}x4fkPrs zoL>?*(7NolDvy}-RXnhC4nO((YrMms=_#dE;1m;z_xDXy8nj3D8$|Fr6+OTYpI7x-2DkHEe`Um*Do`PgvW@xf zpB4)?8}}$9Dna=y2h>%d;?UzySJn!nIUVAY^eDPsS_l2R}D$r7n zgr9*Kb_x5_ML9v9uYt$D8TSmljh_3xr{vKbwgPKgZh2=?xHzxNoA`N#xVF7Z?AB{5 zu09U!-F_+!HO+WxUt0|R49I@{B|Zo$59F~Uw~4g_sg$FoOmrVs5!d1$fAgp%IRV<; zN$+Z=e_<}G^y;q~NsgQ@TYJQ+qcfo-E_Pa_16gs!PxlJ>_Z*4BBX~2X5?ql#55K(I z$jNFf&E6*TWU*@4vH)yo&La#^WQm}u$L~JL`flg!tkBA(;BVYLX&pU!vi)>r5=h9M z-6AKTrN1N@JtOK@xUL+l!D?R~DTKey!FOSO_A{B4I`L)_%yEE=(5gFV*UubKe=dn7 zpluU?CbvM3@@^yV*AKisUdSC7&O5&Jf8R!FuqntX^!c+t5!JiIa zb<#GSSh6gGXw&PbN6jmyaa|d;+V>d>cxS%(l%0L~DiO)CTg29YES9WB)zP<<`j)j~ z@e$=l&i=~D6u+ly^sFzv>~xc?ny4`6@{ateu6na{@BA67$^Ki?1j7ahNZ_jQ+kn9pL!=v z#f66&%$k}2iC(SLFiFc}gvCIBdinK-tf~VHS9ab(;-KT_tMfoAMiI3BwgAiD0f&Wv zR_>^bjZP4lgx^R+R1>qxZvMRuzbxCs05IXHX);`7W0IcEK94(DNHM{Xxi5KLq8P*c z3_PLajGNbxD+yrE6@@C}9Kl(BR0Pzb2dFSesdqAMA08e1mNDkDSP(;q+$ts!pc92v zd57=bXU9-f(CC~N??y?GOSgROjrO}=Ha*(q>v8q?_ugiR=Mn1-zk&3nlq3AB^Je_Q zyV-#m;x(q+#7zWxb_quAJ{WiFDmb@%`}r7y^vPF)x$)+r#(atOZxuTM0@vPnWvBo? zmd23{A#F#VwPvm*zl?!wO3JrQZ!abtc&18aTh`(Ur*IA=mTHQr_W!OU25}SMS^Cyl z(lN>=;di29Yw#<1>T63rZ+M}63mQ8W2Y6;QAfFUw!)Me&;AMRthQN?UUfT`7VJnb- z;ys@Sz1&MbhQYcBMMF$}w}MYO93mPx^)`xXR~wpk>p4rKYly?5)6c{hDz z3Z^#BfdVR-M+6pG)9)T9h|eA6vs;%-s0<`7#KT>?$@ENJ&~O$Ot$5>X&JnA<*4Nd~ z^iARhSvyn0(ZzM&cTB=orCL>Lri)Dmgd#i9aQwleiGvf^4rbSsjelyIkNNwcz3rI z*9i#g1Lm4M3DxX2+fnn-xwG-N&s@L)!s%8xBi&tPlDlOcLm+V_YE{_*n^Hrj3F{ws z)?S8W6zlZtzlx6kf+U#iXn+~8mMaYqJByX3uz^*rVVhK%Xo=*YZ%7hL-W21V_<(YS zqq@LxE9dU#QDyMb8Af2trRDbUt_fG*)q6e-r!g~8ux#y1sy%r}LiFFxDR3e6=;B>7V?}{&y`D*LTA|obVDj^Hj z1m-afOBF@JumjUWTubSgb$?d8sZ(Nhw+Y9wdriZf+wJPle#28%bnGrKE;ZwhszSYK>5SG+|H|2`YfK7Pw{Kq4#}z7$&ZWE=#V9jNT!D$M_?`_!?CHSce><+ z#9hln$+vBFU4VRcyS|-Iq_($rVg3|9S8?bmy1oYfq7@_@EPyl*-};L)Cj8S)Z5;1#h{U*3mpA zot57nN(?ob!oJDh!Vl`MEn#Q6PV2QEz>y$&J=r8_qie{&0*|jNx_L6m93i+l98IGe z!&LHQ@Tg@9uYk;!NAA6V$J{^_oIRthcFf%_n*Kr=6}X27WUM3>+|dV|W!q!N(m$?0 zd--(tY2$Rj4jiW)UkrQ@e`m(FpQ7=s9qV!SbYC@X(9)87F5AW)>d_Pbbt55obkx6d z4s@b$-X$MCzL-B3m%yL=L1@XNL@tXyH*7ztIfJ@M5UJmmy4#(P7f?89B$#;&{)E|o zdnCsjtpQAH5lcjfhOJk#Xe(q|P-aY|NKnT2JYOa%VG5g6brv$eZFQ|eM#87GvMax@ zrop>?Wh9H~P))9NH$G4Oa{NR-jxYm;U{(EXQ*xeJ%)!+wbNZRq8X5CgFAL&PB2Dn9 z=_!WkMS?3r@aMj(VQ>uU{d!4hqj4bk@OF#9z)%A=B`1inap274WkjWb>mQ$y^nrti z{B0-y%PH|EvG+PtJUxN$_YSToXS_VkN^@i{_i>J2ivR25cuvjMP_d1LvOqB>GxEMablXdLz(`YB* z4U{~;rQ0Z*qI6(96_{~WsCoWofRYf_=WFwOwuF;%dFgrZ0p8i{G|K;liB_^aTAb0r2TiFU&q@S)gNyw zcb2@O1UC6_w(S^R<}l+1{*AL9?2ee5RKMDKg+ro z_6z}G!p?W;H;`g%BAE!Gg%#5K15rJ~lt7f5VG@20?oL`03pH`U`gLLUw>jGh6zKJL z`@3cX_|C9%Do^kIPX1|CQOlw$e{FvXA z*6q4T_mfAoR_d7*vUaMj!=B(Vu#}QEa=d>gvzA2LX0$^w%bbQ#%cz1ho=&XlVRTTx z{UYDm(W~+~*$RIShVH);lOP;|YK|k zD$f{-kOeQ()?)DYaB>WorgeKW(4&CK_2lt3i37igt?-Bb_mk>=`LrTu1BD#kS2*JP zAr;2hU-9@WR^O1$_}sbG(adKuQD}(l!nGDrmAnQqdyTGwcYz z>7r%9H?wdze31C$_YD4&;%PIGFkxzLbPUKHv{;tGyArcNubbTiC>9RPO8L2Oi_?;^ zh~fVg_?O56hx(5WK{Adg(^1eS>R)ps1YQ%2JqJ6iL`s%C%!4+& zt>2RsnwTw$vBXX^tL4W2e&}ox7sduCPb_|x{2GCO{MxO^U;p5L^auWuIQk+ zwm?j5+Xbo(Lo_0At^T{)DHP2lhENcn)U0&GX1}-^<40F1CLy(d2y==OE+oo$t!3og zKlByoe_lRu;d5knM-!4$H92R~c{_1^=@9iZ)9DO)&z1`R*(V<*2P`M_(qK|2HKPQf zAJB$OOIB=CaWCLB@hP) z!SGYbX+;s3CgP}jz}v6Y#Iv3K4mq*w3Y&2R9O+t}9TVKW z7T~}5Wnxg!Vf6QZPvLaC8{&#+x$~vGfC%_~T{JeUn_M`VD1Rrdvw_BPFYcSm=Ck6E zDBZ}j>ZZ$zXo+uHNd7Wq{I=`$Xy4Y=UW)EP%`LuDo?0pF=Y&LU_8J}650mWE}vqk3DnA|GEs>mw!5`QQQ{~}Zm%m!nljcVK-Qr~ zo2&66PYD_riL8g0=1oa%i?(c44Es(oO(lwZs^2lGNxfD%ne7z@It7jz_^DOKFdvOS zL;0GEMnbZgS#fi5b6Q5g|6u=HBusKkreCstIThF7@PPEa+2gm ziq&JQ+)z&^lq?sHVMAs-V6v2VCM>@nSqaoc|4anfe5<^Urf6*f`i_Scd|N@P|zL(oFf{k9-HiW&MD)Za&gI1{q>#Y z)I)V~mY`@qY2f{a_R@lAZZy$++L;yauvUcSy(E=$wrO0>KpN8*M9NdgQTVR; ztuSY3qu5t`GU1HYu`^bM8*}1d`rcEjgq9^aiuK&n)+3Bo?sBEs6YIY@Ec4vws< z<1|$)Zl5edr$&B#GaH-6XgZRJ)?6mmu{dQA_LBdvw!ZqW$uI057;}UY0@5W4DkVye zECB8#y{fLXnWzXpm;4z-aK{^L$_5Ump1n?)%*5KG*fG z>nNeWt};f|$3FFHl4h>Jf$;m^>)aaN&0lpQrV(qi5MDZVOm)T|MRU} zIZX@C0f!M!^QVqF)s3OQ@NeiJ-Uw`9^WmB$wW?% z9ImrgIU5KXh>$61#DA6kXZyE$=eN?6H`fiSW^qEgs7x?WS(x4d-Os+s=kv3^03F4T zSB<@zN8F-iB7Dhj)_iqUnwpE?9N{3V8B`UP*-PZ75gP;o!v%E+3RRe2?!Fg4rH?u* zP|MqtFM6lBj!my1W;(h7EEJIokTN^-kJEl06NXBvJI;o7|UxA_YjW>Y=}~&|>S})aG!C9AAgYBrLS3+P_pWKh+3vc|98Fdm z)fPv{sW#^L(YBJnM8Z@pQ=Y~sI%DdpUaX6(`REy2WoaJBcF6-qN-paApW9i-ln8)P z>3%V8$I{(!x%H{A(vQ#K6C;T%#jOWFP1ASw7vi?~&`KWSHLj0`1aLHYYT8}XQ0hTk zJMNpm>5z|%BO>pM4)lfzx1gQLY(3&W3Y=x!MGGx{w~E(xJQ17qtwd;3nO}y2kz|oP z(sb_7ZpqQtguf3id<6!u38WlBOtpIJQ=h?TUKQx&)~eUT*$j;AcgU-S$h(q&F+LJ* z5;P#N{jBim86x1CNh!Ff$C)*W|LYy=`YCLdpVHquh0)auhvHK31f*hPzwX4vD@uD> z&XA1U)fk&HX_OlcAKE!rX>tC2&vL^!eT^Vk{~Kjwcg1qlM2*0ch@8r2R@o=(M2x;D zdilzUH12j8e|A@*ON9~d$pAP}v-{48BrM%qc(vv2hs1LW&=9rW0oz;4?mD7q+-x3n zjdH2|GSl>zR>{Mrrh%tC0Z#I9Y2IzvCHqA(w~$oqth7j8eTZYbY+bB zQ}Nu-L!z5AghBq@y+Xv5wCcY}9WX0YAyG)u`90Y>g98?q>wBdgoxo`tx7XrC``QM} z)#W%+K2+UaJmg`V_~udi-uZ|rDf|7cT9oNekw2eG#@~xISaS1UOJi$HAufDrX*NG3 zDA{^9@5g+Qdb5l^u1aygb_uaQUEF&8mIB%Z;L3l!o#SmOr-bMmu|9QG7`+E+7Qm2%je=hi|vYJ67- zA#HM+iuOEVDyv`9Uc)voZ#-FedSi{vPj|iCHDaOx%=}r>!BOh#=2$DZe{rDx{HO93 z#lc!pEHbUbr32kifR=8DA6MO-#gkR>ck_>b2i2AO2HzLG8h@lfCBltrAEd|s>$g%2 zBV*q_Fs#22sd z9S+FD`le|N9UpzididdYA-1%u8o`7D7~U`)bo<(!%=c|uLQ~Gvbe}#oWC&pFOMsgF ziovt&ea`ZmN$qfGV#g!%fnsVoh<}frY$29e6xXh(0$rqK4+jJxx99GMjU#?8b+hi3H!90`@+r2Xx;&{rR&9 zB?ZR4>DN6HN$Lw354z46Ibjg7K5NY%*WNnrpIo5C~DS$pdWO!9(3x0ZL$6fm3!oKo(cP-nLrYBxZ?>nKnL7qu*xoiH~uhqi-0cjVpY@zm9 zjKAFx4AbZc1g$^4}^ZfAxR9NVXyQM($qCeSDrAk`nK^Iwp+;$7f5IRNijnj2cPcmJw2{N{AeT9_c^H)4fsjH0b`z=v4+OxM9b- z{@GO@?n0fh6qZsK(j0V8_=sJ@yI;X+JqT)4-@DJuT5L$aF`{|366ueGO4GojxqD^& z6R(;B^w~Wr(4O7FMhBj?9;JN{ckP)sgTc~GjbUvIx%ZAYBFZ)$>sUaSFxnARYT19> zQs`VTZGe}3dW`@OTr}+0%j(nG9>1w;vLQpd^pZlGw=pCH#<*5IedAK?-KE(={^;eh z$a*z{^zV)EM7cQr{T{OHJUDu``FY#n^-t0D%#E%QNvUN^B9zYPEu7BuUu!DN9QP~tH7-1%>5O8km2@uZ z!w#@qyX1A*sX~=DA59K?CUL~Le}!MM)0yW)u1N@B$-ZO)OJMq*i_tt_?XA#S*Y=-N zlM*5`re#u8g`V`QEmu(hy&5L{-=4HN?b|GQn*phv!tq7h4bzpp6*${aAsN{cf%4rO zF%d4z<~=?_9wg5-IL3E;6~cxMOD{NC);$Hi^a?C|v%VOeGDBM8AY+T!_|-P`oU&>7 zV_zo8fzWlpj8-6VnyM@VmAimpmj@4b;W!3gq(jrA3?Q{t0qNV74$jutuWWK5cvX4v zohPYu4jHF(D{NODd%}|9H5$(KvyOmkX1~bn{ynm+z9`XkHs?0qSF*gzU)!8N8Jtw= zddAyYFl@JSbb56ZIb=W1VM6Vbw#2HGtb{KLf%IzIK0*>1xr~2-^SG3}*ln@DyVWsB z`)fS9y9kX~@T-P%SYyS8hWtJJh3_7=s2*{x!u{qHS5I~r^3T^s$3D##=jL$Ay=yXQ zyU)B5iC1r)#CnG|h~uHQ{%85d9Jt4c2$Bk&J%DL77cbxf_33_uY-?3MlpOvn6?Zmn z;;vm9HG(h}CH>7)X7RpmhnC`%OPPY`)l1~>tky^-iq?xeo$0RhQ4_=L6sW;4z2()L zNS9R}b#{ot04ZOZr7xu&xlZNnr#@+Jk1Tq36^37C0C?Ppxo&kdcFT)H@&oB3s{|vF zna9G=0R%>Xe=srSGvv$Inw@|UkEs&a{!DPfiWda<(Ydn4l$$Tb)TWQv^K5x*DjDdnS1+X!T5Sob|wbgwVq(;0ArB z5)^$!T9`{L_iiO+ut4l4C=G{dLJd9jU;*AqAuX0^pUXP-N63Gq(WT=)5snSQv3I0S}y z(ViauMhw0@SG7hvCWbBeJ(Kcege;%&oD4jP_vhp0U2Y~T(9UR}s#wl9>Gx~kBCEF| z$y9~cm9&%m0~9XE(tALa*u&TEq$`c)#ym@vQFq>Ao*K35qz}qapXpY#7O_LL@aDMF zaSE@9ae=0?7)U@Kgm?;gZW-Dw^4Tpis_J0nz0X-Twt4s%a{TB68)4H0!EYPACL~5W z+u6qEyX#O}RPu!PBIA0n-gX%!EUm{w0i%6TYoIzHW8!cp!RX`2bZfCCt71s7L=&{vMs;4~G={3^`+j0ahBARebYJOFXV^uO zYVOX(E%NCdEL-$Ez3;?N9BY9k%iH4-4Scn|FZ@K^^OOQ`%Q9W}p3>HqXgJhtVG?9e z5=B8mCgeDw#Qp2q8?@IqB|wmZytBirNTc98zhTp@bTFE)50V9|7%N3fjp~*Vi}N={9nDvj&(t!(1w%;6vX{3L z-Q%FYZ*W5_AsSh|AWCHjJQgcdS5^7UPs{JsrUx!<*!8uKcJov4_$FUz7wp+?$0tdF zz$T*CW}_llEcgF=0q%&AW%B)*NOFuJfO)A(@JO#9Yi|fru2SBF(_ODX8uBzkC&D26 zp4RKuKUAm$87&`9R~>zvwf$J(71Y< z_J|7%Im@Lc~;v4sO5^XHrYm9p>UFr^$C4m#`22C>VO|%Aw#;o zQd}W#no~;6=%4|O;q2tpbR8)SyrmYr|LH>T+q2Y5e^5=lrq%06YCGafQxa$_QQ}i_ zO*5u!5kmvmg?kfSTm@;?I5dKP{epUY@NGYl$<`%rz%-zpXiwaUvIVO(l7{%1-jrh6MXId*-ETlGW6r$T~*3*7;tJraic(Hg;%oqBL~4 zva)rJ?UBxe_qvJ;LP%BD8{G4oYeb^0pQ5DZVl2a(Kp@I!r_;~4;GI)y4JU4BlJCwJ z3?$j0B*bc$R3fDGMwJ^V^)F)FiF)uR%P`)PKJy`jTPkzer*X)$8crEm8~&@1v` z=V{u|;UXRP0k=2MHQF08qVfe`BwlAnmKQ$N6$2R2wb^nZeX!JHW>>Q&9a1OYs^noo zTzLx?sU}Na;+5*o zXyE%@8*Y~fY6}*Y(pS89!v9s&vRdIlIi$b54K)j(UzRA06&L%F)Ty^5q`ImRK@s|e zy^OTLDaA38o-)@U3KBtDc;tbqW*62x=AX+k^89NZo8sJ7)4*M2Ty=Ef8doiA~SZvNlyL~m9t*l;ue zAg)1Zn#)04_uY^mWclq84}6q`{he}A$(+Ck`5)qpZl;77hmEGrj)u%^ITPEeVH0<` zJjJ%8QQo|81(2lLbdyGmTa^?%efN1Ku=pX%o>Q{a%Z>cVa>NLu66B9A$!iuKD9sEl z6qp0Wj<$fg!@a)TO0#UZ;dAaH#}Ck+8&l9IMBP-~fr5cK)+ zz9g3g2fDYL@lnBc2PgX0N95#R({Cx?f|&CpZDi2Bz&}4S1d-BIMV`*;g3scrmv=<@ z1$e+@S$uX2xyedI(IDc%-xbmq8_CgqrXj!N3~GVh4||axtu8W+776W#RZJ9C$0g&z zc|uN}Uz+j?Hk~SJ;@oA1{IlRz@zTG4aNFD@X?P?PHrQ1rMUfk`i#n~?np^+*mE)3Q z`=aFDQ2K+u-oAZOR;`!8>4Of|)K2S`F39gMHXUE=r^&HqtzMc-E!LqkTLVb7L2@Om2ISTX_!S3MP;rH4hl3 z?@k~;P9Ia5^b$PB-F?bZpiL`2(XDnvtDH`4+0s)2ZaY;8^{%7^l^}GNr?t%1d~|>j zpS-O;5{-%pOO~jvMwEblSEe-sbEQ0|jzN@j|H9Ra@n1Qu;GS`B?U|nv(Bv#v3v$hd zGJS6iGt4R@=B7i%>jU72yy&vr6k08+N%CP|;l&mL|LgRht|~=c5uN&)oTNMY5nf*A zxX)>%etq_9Nj62E(*3F%s_V__?6hWM^9vW9p?X&3K1zLQ%G0pmi79K%iJ=+f=R}xG z?T2(p9tfx9#7DV;AC)T@{x~Rwtkn#t&yA^!lXS5W*4%#eWsJ_6`S*KiT#r%5Qi)9W zT~B|#{1E^MT1h?|vS(FkAyE>=Z^xaQQTK%Y3s6y1q2@nI=7c;-=h=*uy}rC96c3j; zuYp{T+0tJ7>qLSj=+eW7pUG)pksWf4M*eWF#L{3nccm#oE4co0Juzkd^iP1e!qkhN zy7jiTkE&+Np&KusY+x6p;RgPH&>DY}@f;)lSrjnb8}%h$JKR>+WO)O|`Du)VNqWw$ z?`$zw_dNC|MOVa;{S6=cLkqW%`pF-X%IyA$@5d)KH-nUC1$JLDsWu z`I)1C4tRD}1GTrhca4|D^v=xpkmXEr5Bk|Fm+CWWi+5EZMclL{kDGGiWJKp#V%b~e zIxtUUHXi$MvTc8`G$S82@{hI241~#hO!6cdLEu`++Y3t%N-RSA_gF3ZZuhC|wUf_Z zVmu_Fa=BYZnWSnxcF$34SsZ`Wa?Oa4T})AHN91vk_UUO9;%{aoC#`S88!2t@NWSI2L?`*beHu+OJGnkBY(Eqx&@f;1Loa z!WkHid|>ECFM==9I7;Ld7xWoDpom5Ae|Jx?1Almt2MKPwnFW;TT|0ebW!IZ(NzDx z=LevY%1-qTYP8Z}#u(-$Wp4pIHnlu~$4Nz&ZOGhdIMA^-_X)mwvl;`03g6E`D+MY8}Q8Oid2np zU4`szUsGs1l)ZC=Gwi9_!$YYI>xtvPNuk=~mZ%KN>bK^X&jog!Vf!^6PIsuas$A z*%4HNfBj73zPq_5W?Kfy0Sn@Pz=K@MgddU6X;@3(1(N(>LEh5qZL6a_f>N_yS?ZdQ z66I$gAz4pzJj;(U@$9PhK!^LXTlx!$2$GKDo{&N zEqGg=8E)nJ4LB98(kj-wczt$i&|}gCfooI^$bN;z(*6QTT`kT;(wn&4-e2-N;5#C+ zSY5c>QkH3!#Rb3LIWanrtxrAV3zjgW0Ckt>K6PZ=`g`LDp96>f%;Ma| z!oFA-_jC27X_ommf(<1@)@nmeU8CT9q78f+`B~4CdI(By-4q`*R~9|Y@K^9^PO)W@5L}Mc%P!ah(rV(^3j1GbOxM8RhI4-Zmzp#CQOn1(&e;F%VaVVaQ*Q} z-I`dKKbQ&$8Deiu!6iHqKUcbsYa;#P`R4VeLr@wE;X3wLgagpJ#2qW(9&%_I+uqF`p{sG(s2=Dsd!}bRn;C#``gzH^s`#= z(NSB&wY^cQYe|v5xJWR%?&X!d38gT+hV`Zf9hL=_-6T3KX`@lRwQjDnLM$y zmax(-^Y+?Sk4!M~e9MXAb*&()7LGV3z~Py81K`rsjIWPR65Y$D;^%=F0w(9g-;;yR zho-U6Sb6vM;l@bR$z^Mveh)@^PYNUb+RBpoHc2WrEBl4Hhk@czS2XAKn?3i0HHa+2 zgfy;~kCeoP1+TvG=SNpNrd|W9!E5-(KpLK}?&ZVz4TVk=|E#tukf+p?Z(y9H7)*ilUZZ#hl_=hB zgzf|?6OJq)0&-+YuRi>WIXLZC5WNV>=$8V1&3|%0-|`T)?PK>dGGp6=2rt71+Bn=w zvbVzierop(48gFw5iWo`8k4gCn9A_tmyNBY)DsFG-CO2WUO4E(%3ph^wrmG`mSDO zH11OaS?10xF8zoxk%h{y=0)?JiWGE!OWFs-J)rH|pC4~gBs%CP5GWQZBTP2U(De^t zbqgto!4?vo7@PiVSY+Aod;86tJ~#&83CI6beD-a&rGlwF(?(@m%1g&iK=IQnb!wT{ ztk2i1@9&S*fs%-${^gtPgk&y3SW?#*6r0^fU`*6pX{bN3p0Vl8EhU4Rn*HE#!n#vn zK8QnwmESn}CUc7FvSh}TlpU$am+OL}-u#0Y{&!X{iCiRsUe!gXs>5%t{UCVHz#YY; zIVqNI>-_BJY}Fn6{HCtjaG9*W`Q@xbOk};_b-1GOkx>^ET8YcqYB~)G3);!GmU$rZ z;E^H(k*+QG})|&O|sTI>Rl&4Wo$tNgQI3 z#tU(+R)>kEKf2f6xYIdi0Ss>b8c|Blimz-=Sy2?eyvr)n%ep+vO%`70IQhz*C z&vLidN)URTkvs30UE*2IF(P3@&e(G;vae2N)4+z9`+>GR@O}WhNf+Ts?)ZLZtJmbO z>Aq1PWH{{#Y1$R@vE!pb`gN+-YQ*k_*u1Q!kLbaiBF*$C`*QxPmpMES1nEL81;VDD z(gVCW8qv?2=f3E4!zeaYR>gbWpKnDcw+Hq7 zRnZy)9U?jB+HV|kT$|!-nDL!AS&0En#i8MWS9<*nRj4uDo39BE1KMolhxSpzZ=Z_` zidUx?rTyHYyOQPA9WO7hQ9@$o{|uM$5%JOpL7Jkw@`2|q=_PeuZ&sDj1oX3FJ8+b< zziB!XLRyzH_X;qi>oMl`=7tx(D6K;MSiAE2O$if;uJoyj>OXR0%U{h6Xnv@Ca9Dy) z1p2JuVjNCcS-mDNbR~egwOBd>XOQP)gEg_T@8O-PcnI*TjFkRyB$B8XvD%EmUKxj~ zW$h)IZ1KBFBPG&YGTS5$173nHd-4PI(iZ>521RCFy{r~GF^xv_Jg)$}t(KFZQRy8> zq4v$oYfsjjuhSE^`7U;&M!e?aVPXAv%Mn-B614Ug;+DENP3AJ&l|0LnTCnq3c6%9% zUL@JdPtL85M#_?=@z-g1vDT(h4?7&__*1*z@Ym9EUT3&JU=|WQxcW&hNx#+gn}MB3 z+EYuNbs@7D_icTc0#{3&j-B!RuTXc*U9hQ4zRSx(AGN5Vv!&`- zH1saS&)g&LOamnWLVG$`q;RRe#E+d5Hxyo;6sI3eE!#^o*Fb^>TIo&9yMjhWbt6V% z6x?Gs8WMGbi4gfSaTDvNt(Mn~xq4u*+eIl3qft*bIYjPfY-=BstML+_exF*~#YmE1#Jr-Neh{zitZ0IF zaQNNHme;!3BPyy&U{X4zv}=l~TZ<;7Y?qLTTa+1xfJ!n=rm4?w%}u!TEXzp>HBzD? zB!GS<1u|-*=@6oX2yzLqjIEDZ>Uj6mFS`_bFk5~CDr=B`^^}s>Hofb?3skv;TOwh+ z&tyXCWE9I+YaYS{sVl85%7ln(U^xR~Z&Brcc>W`j5pVHbM$%ob=N z(ZKRbvHaV+)E2@0HFh(|ks|)`LW$Z*fth5fkc~KhWou!4gPErVRz3)5c_G`r7BbF# zu$(*}NGuxa45C_TF2C{Z-+}iRYq<@iYGf1FNEAoU58OT*wtjfz_;;02-9m@_`J>-N zl+^Ctv`^&-*xNW$wmJXKFKo*D|D@+k^r1o3gV&2=M;-smh_a-YJ=l!jmH~A5yqfAZ zDP__mw{8X)`9m{_VN1|MIctl4JSQyu^Rt-s_fKWlT+>d&9u}&<&7A!n=WWvdZrG9Qd{UIj)_(ii8WlvWvEWJ_praa%btjw3{o z<7Po#O@z&`a~^uAp$cs$`fQM9;)#oH!<5Ehel($=kb>t+M~g?mt!hxoZc6CnLuL=9 z5DTfD@x}u%iqD^WEM|1fM=Z)Rd@8u>51Dy0jhpB&qlrxTr8{nb$pra@o^qbz_4SRc zFK|bf5qE0a(z|oPJ@L$-=C8^dZG%h$jbtIlxOC|A^7{=dP8(>$mv6tBneIBBcJHR( z4mT!h5-$eGkRMp*HRU-*SGqd{mNZR52a=)Xk_W(6@GCDt=MF<1<2k2Wx~;`)kk5C& zP6ba*hj@npttbZtn`j#4>0#WLG(uUeYQYj|Y_U5elK7PqUN_~XG4R2J&p@Gmtqc%b zF%l+}+AQZ@+0n(|!K^ZqAq&`+!Sl41eQc;V6S&Ft%3G5!seHMUz7#^S&6U)RP&KX`HkB#6JWVHjNQ=>DZd0HkpOW_wSYvBbG_x;q;g9zQuE}W2AN&o+HbGk|=_mBx18t=+1kQ39p8vny&zN2&D?%0S zbZXrHuJho+AmU>2c5M;y_pCR8=f|A9b@~?;(08*M$&GUk_N!f_TQcSsH;yR8FH5LD zwBfjPrB0~6&030YUD+~J>2glFAd?!YZ$W(b_J4d`6f*p{p=dcg#|QXbcV4sQAA4-1 zNa!9fT%QJG9p)@NbNcNdySqXEiAP(RudQdE1A4rJcbe*rGJK-`^9QRJHLs2l{D(4p2 z1Jf~`({veG=)0vqlOrLa@4v_frZycno@DU%tX+5XEdV}|lImJ7U*6r! zJs4k>uD*F$BYp8qB0yWrlBR^W^rQt(6E$Cxul}8>*&;?g`X11LD&6bf3%B1*`|%xM z#dwX6zDaQPHax&=d|wKlA{~+SyxGrTK@Aw`|3I;A{#d1JvlD@Dbj*Mkvac+23@ALC zSxrJ?eFyviYG!?FUXKJo_bU*t#|_Iww!Z8gi|(cqub{z3Pi;jfiNb%!xXuc~XorvM z6oS6-H!x7k4tX&Xnrx@NZZnkU@JxgZv9|v3apOF#$)B~tD)hK6r_TQQe9yAT*EyjK7Cz8_Cx3=_=pC+*qF4HNxk-AsCa@ z$BhcWUO8Z-G+Gqb{+L&5=wZ{RMm`w35 zmCkReQ3tM_L7|&C*6#H{Y9|)qyg4+!)el0ol-|Ahe?``9yE`x%r=Y8xmY`I=Au}Re@XL8 Xbe=tt>eab;D3ZtOI%*gd>o@-c!VQ7@ diff --git a/cookbook/codellama-server/imgs/promptlayer_logging.png b/cookbook/codellama-server/imgs/promptlayer_logging.png deleted file mode 100644 index 26b046ac4484b89a22fe66a6006c9fa1f3dffc99..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 300105 zcmeEtWmFu^)-IYL!QI_0xD68A2X`m91s^;R2u^T!cYaVxV!6}9DPsT@1Ad+ zyUx!$t9!bpyQFLHT~BSB5Jd$^Bm{f}2nYxyX(=&f@C5(?0i6c-3VdfiEfaisBWx)u zswgcgN~-8+Z)Ry@3IQP%60h-EQ)Li4LnkIm7!FlfSY}r~N&$k7tQKN2%#eZ-QY3^G zRrPQ;$KM(UaqhbsS}$@iQB@7xb(Hm#3W(-?21)rxPLw zi^4`@2j&t;;fx|-bR7Hv+}M~QQWr?_EJ(f&uc>wQ+mf&_FyIaGY+m$NmSKX+zHO6E z-J8GoGNpLE6GVZae6{ZEBglfXE()<052(t6fXE5w&QvB%Fw*p^hj|5eCW1Gnx+a6C zpt0udm>R&t)iVlFROHYUBL^|PNIFitX)1zrB^aD8=9LWxfe&rm-r6ed6+A^+@5f~v zYG^`f9QRt)LD4+u1!-G>MPijnsybpKYFNQo8$O4C; zK0*o+!#Uq|ZkXze73e%L$w(`cf;js&}r3om8V3rCL%phY9mR9Tu z!7+8}Ho-hw8y2QC{6yLTp{zO8Ng0I{Q2y=e+o{3?22`Uk6q_Fe4{dTM+Tb}cT5I1Y z@>3cys3;*0X2lf-h`^{9z8?AtWKz);9a@rT=!|!1ugQg(TN8&QLq4}s`4vW?J$Lit zs1t{CU*$_UsqksSss$RzA1hE=8*Eo$2!A)Q;>y=KzwDzJ9~$7kYiTl zDarGz%reHj=cr}m+@8{^(_y1#s=Kyu-GsLi0YHD z^{Xpt_2*?_XPq{ui)VsFmNN4ye2~kEEPxF=frz=B@ zIRsaKfi;Mt*Zm!qACPrl*{;26e60bb0YTn(@SoznLq!)6-whTt_SYo84o2(}dyF!q zL=O>Fj^d!iKqFIr|B4NTPE?)rAeu>vvzmn0KTU#}{Ahq^6K(QStAt=SZpmk>T!|tq zK6KL@Mh+->;r`s@-S@R{>0g-N&E}}>+SMWpMCeP{XA$ip){;Oc(HDdz8zbXln3zmx z(kJ*yf7Q~YQ;$?Ls?((K8k_x!T!!-yjN!mH)H7=gh=O!@T~1SzUKm%Qb?*u2s_g~``gYFHRpIPs|Qs#wu7 zRWeOO%<&lX8fwj|g0Y+f3GeM4Ig)`H68bVL5_XgZWNtA+Djak_Fb!m&s8ZPys}q!E zd1ae`?ztr;Rt5X{_@(r!>AB77E}8qVGm<4FGqux)({olE(|K01)2>D7$}PFqpJnp3 zm7t4?r=j-ItvIjVT#;RU;?rAJO3xkB$Sl{-)-Uu?d!mYvtI4afz_1XnbzFG2fTdme zN~>GuQ+el1Qod5QgV;ubL_4+Kloq=-JJ6|Wn|U-WWm%h&iJ{z(sqKeXdAYWEZN_)j zX5Mu9or#@^hKcb>{<@Hb9}9#>Hb-nn0t@8}-hA48c6`!&DlYWSkq6GZ#?9)@gv+|i z`OUAL?e~4~%{an2<=)y%d|fco2>LclP=S+0^POR4vD8iUBvw#VNK;7Cx5?+*Gc@#4 z#~A4k!_ZENPc^T;b=L(wyxBowA@VW2)f1imKK6F(L-SAdZtQL!3X@z#imC~|Ty}|X zea(H4ee->#;XvG*r0?>Naskw(atbK`^WC?a;ktd=670F-)89EpRIQqOYiBNNlIyf` zw5sOyYhG_-Iu*EozN5dOz7Ryj2%ErWXS8BG){bFHPS8rQW*paPYpAZ5sn<4Cv!Ago zHy^EHuCAzRvlp#9Ye=?HF?Cu_uo$i#ty(r*Gz(g2S~Q(ko71b3JQ6qzye2qYlth&P zMW99$;X4e!8}-{#+*RB-9Z;HOovfOxS~=U23X3kLNc;j2I!}#lQK^{XD)b!F9{V}g zI~FlksduOgV#Z>g)hn(btEkZ<&^2h9Z-HOpYF>6NZ2979u(q=KJkvcTx~_CEv0?c$ zd0T7GJ*;>%+=$ss|CAgA>oI!Gw|zCS*xl0c@bSh#5Lu8{u*IucQtJzP_yX5{NM$n1 ztXr6U5J@-7sC|t=P~WA6-O!>zYn7EVqxR0N{ExH|B7>A=9@^XfNQie;bDJhI&(hM=+Wzn*YoYpNxsuR$~Fjj-w%T zF>_41Qto7<%H0~oatryW_DJkkX&9Kr(9LZkF5y;SzdbaP!T3^kUZzi`)2M$-&(G+o za{Z9Dn?}S7-89(bd7*kfc^(;DE5@6%E!`JepkRfw7$)086#I>)GGmD*B+6!7^1OeC z8`GkjsSERiwvKW>#`E9m)rIL4dpM z_`d9ln;wecS{+}Vxx}m8ul?tK-^Yl$)NrnN+xuFr#=Gl;?@TUg>S|j$Av%Z+f{O{Y zN00U+t0fHtHY|(JEF9|HyV7)?fZ+ z`p_Y`*S7O8b-fBeb;#Nyy)Zl?8y1ZZ+6$6Bh-$WU@xIc!B0{1=I+E=A!ohjZK`gir za_(oHGQsM1iDVIY$IJTC(=nJ6`&8Id=m9j(@R{VO+FMCxHLu|A=W57^;PL#aJJ7_w zqqPInKmGRMFz2S1A~7j(pT$$R!rH-}_p#;SBuGa<`=A1(Z)tI1Wo<(;u_Fa^J4ykCQu5vcu_UZZxC$Yb-_gm6!nZ|Pi$T#X76wn`M`603jAcTW<*oQQG?|}NK zy~%2k9ll+m>^ZF<2t~w~RTIyjtD{U1*S4SfS&D!JuZ7=WeS~58*xZaf(^-@9s7uEn z211@oyX6#MA$m;B;cW|Y*K`pYD$~Q%R|tE%Wx1d zkoXX=;1VSG6oe%BM_B@r76SS&_n{ylf-E6m{#Hi;eEs!_0-wLy{PhYQ^BLkb_!m0( z1Z6?}y*3gc3;OS6=rr&%h<7TY($e6oiixACshyLBy>kaSr5gAK{0Av5CkO~^s$VCh zv@+!>xc^y8RZVA2c{x54ds}9s_x8r7%pltjzxsgyfcU^gTT^EvQjo2Uof98Ofc!5t z_`u~~w^_(Z|5C--T7X^GiKFEQ zXG?oK(qH`=8QZ%!3y_ol8t5ONzrLp_$nu{f**X1fTHpz?{CdK|%KVn)AAN(H0)E}) zQ?vw`+GvSc+Jb2Y9z&3qjRWwP`hR-z&k_I7QuCiJ+1Yt`{@C;nkA82e=49$9YHtf3 z(^>GJ{Q6ttKR*0hLjcRKssDi#f2H$Z?t%#|hyY;u2h#)*PN5{k!F(jN6a%V)uV5+r z^?@7%f6@N+3NAxcnMxJkyQWw+B zS7=%E+dL%UNFkv9YX>SMoS5Gl9n=!6GbD^K#DD1!CWGk25d3exDB5QT7$DlVdKSfh zZR!wEsDPjB|4mvN!3m#Y8ri|}LH+j_^ox+t|HV1~e?tD3IR5_$`LBf3f48>W;&wT^ z`ZBrt@+g0~%D;Z$ra$52j1DsbFNQX#loQ;pl&cuN%Svk&Zc*=au`UG*i3;%_9eyak zV1Q(#yf4rn0`B7p*BLgkRBDqyy7cMMY!??0RuBx*pRKg8^gVR+%w z%`I-!<=4N_-fuwaqX7Rm?2L3&;Wbej)Ylmqf|p+)DoFmb5d@{c!0H{xQBSn*8S*#;FnN$F;rg#_>j(-${Oo4|O;n zA^;!*i^&{ZPcU)%+!O}XzY5JC1?c}$_M!zd=3eobqXOf1GPHh#@O1MaYvunn#4dbK zP$G>0YMS5wrK5k5)<4t{vjUS#e19=44Ei^IK%S7FURV0=9xXPk%vfGx!*qQVF4L@{ zRUg?k`i`FRpDi75-+rew5S=QwtI(H!-sjf@{1Pkmr+ilgF(0zYGXM)BUe6|9b^gc<_3` zgtw<>hX4KbTH`yM$Uq`sM!cmtWrYE58%F)7#6sQsr8Df=(J{mQsrN)M6<0h@zB`R$ zd|Vs62%q>*rB}Fz6wL6RMwDIHKgs<^@FNVtSEPpW5dJjQTmz853*YbS={Io;*8K%o zx;Gda(7(%v{;w`8+q`#MgMLAcQO8@9->dviPGx?J3tzPJoig zuayngjPf4@1z~A@AxZYi>yznEd31R(pc31{^z?$X6x_ zy!Ms^XOWIPngDO=_USSCzXBXNDrCln*?2;rWAFw5m5xJh4`f$ zxt`c)gO;Mma{H#TJ2uGd_*4omRqMr+=+#Q#dV{f`27T3%oUL!snnPyiahonYiULdNvLXwoy{ z{!|S#K(JW2^|8@?(az)UJbziuD}xznYxJpHuQ?6O7CC5#zfCO|05j5~us`$eGB2hB z4aOfXAb-trDldWsRXw;&w$I+(784A=mZ?ue1P(Mm0&lgcLtF&lS0~@$ zPN@yT63S=B1n~UeI%A!U2w++>g+FYd3Drb>+ON}AP}#t>S4HRfUAz2O$^8fsU205Y zb`2I>BEH;28oh!+>GxOy;<%j{EQGu?&xvm7&RnCLKJI`)J%5_I+?s-0m4$KT(n1~& zI=|0Md{duZqa1k{)T^@YFH{BoX_|yE;l?yBRZ*={a5;qo4wVjA#}ud8@EKrk!Lb03 zA8y4wWKjWuPfXk9DgyV&P9;W+mo5kx2!0)}| zS>!6gsQxr};+6MMKjv#soAHjZ0r?8?{srL#3J&W%$d$4pEq34?CYFG-3+C4djeZ@y z!`!>;@dT>1hQu-n9MS2$L0>Yy=QGT@?4ghd<(DN-GFWP{FGe;)@jyc2z~kbe3?ljx zZZ*xxJYjpsZdixnGim3wsq5;Z`hCR&AmQT+g_{s2w%-G2%%oRxgM4R97n=}t5+@A)U)a7I%3>l6WcEcI7)>VzmgC194R6Q;Fmx=xk@PR*aoE^xzPg{WUT&HRu|J}pZuDU)waN2@ z1n(ZLYt-3cU<4z9@rysGXP8Z9lop}armb`xeLyCrz~UaMm?OQK&Zvv!q+!{$p)0eL+{8J1Gk%FT<;o9v)uJ9HJU zpT&dXPs8q!EwzzY8J1X)UxUL;rw(GZ@Lsi2C~Rup(4}oIDYc>k-BZ4U(ryB!R2;rM z--P=3-X_)Ctcs@AAAjE(q|tSo_aDSWYMC9@Y^6u;jMZ3XcE*MgCPl}1i;yvwJJlxI ze^ikKtK|3=LsZK>B<6q&8Hx>31N**FS>3LC>WuS%dn(C6WpAcb1349cu8?4+2zwK) z-^d7uU86!D*e&s8*6L`fIz|x5uQA7}T*CQcXEBaS<&}1uMS&Sgbg(Q~jl5Y(Wk_UE z30V)sGGWrLr*7{QO=r_3HDDH>j-!-D=kt87R^Qp5VNh+gAUpEfuT55Ml2JPXa74dG z@yTrw(HtZ@m^DBg@4uH96EWeho(Wu|J3KC$F!9@uUMhPnVdSTU#~YF9Z91M1f`9uX zDz=+3qf+Y5uzh1@tk&vB@IswkPKHF0$&H{-t;wulKG0G*Kac5Pc}LzKN6|?tYR%*x zuxu_v=qubo1oOT#HrIiw))ucn|0^~C370K}%VwcKuWyB9R%u=Aesxvb3??K^&?y7I zZa3@FY@kD5HQ$)G;hM40VQahGpvdZ|Pdha?2qn)kKg@b3_~S=@INTj>^)A_~Z$u4r z?JXO3h_7pS!)xlXpzZEWyzJJ(=OapSl$vWT)9Cgm*fNNnL>BLY*3-A2R*7lN*v#bY z>HTjubs1})B)$Kx9sEyPJ{%Bg3@&47Mu`0chze*5sW{8Gwiwa=76%_Oy0+g;4vMiQ z5^_knM}c3U(iz*h9U%L**J59N5L3C*C~mLc{v-bkzCBL*^sHazKHcDC%Du{I`_n3k z@`xsD2#yCPQ`%}pWLigyg#NcPb^+{`dqI!KfD9pDg|qek!uQ5*EqNgrVd-8%ZrWbG z^qqHP%yNLd2adl?xSJWxt?Pg{mt=>NdH? z+O0Ox+}~0vbs z$Gx04FOC~m9NHYDwR#KHYQ4^Wd0%&DylbPy##{EXePrN#wcs)(E6pU>J;oSdXxAP# zKGiMh`+ceqxR?7yFb6EX2AD%P_8t%&|u`(y(=xlC6h1&ap>1z6yPd6vN zIldu9s)Hg0EDx$Jztooi6iBVpI1Xv!X3*B>A86RTrf3`dLGPuH++^E8cb{*MZ$)?D z5mqudyns2*}2^n>rh%*)5)Pw)hC}+U{eNDPL5R8FZT+wPE*{ z%Q1Sv1WhDfz{tq`VLPpDLwFi2lgKCoN%17|9^JnX)d+D`1`O;aY64OnG@j;;K|wV2 z*`L8%@kH;iM*LJ+GJ6aS0MMj!+OzS=ep%bu@I_E2m5a&t!66Y9$}2}-xhRs|TrRiM z2O5jO^O`6x9M2Y!PSlhmM*TB*@!FNOTFj29)@$N)10 zV`GaPKJiWGi91R9`<8SQXO z$;foojA3RSj!%Nv_M0 zy$`^TA-k=Bnb}F{x+euTBj606O4bJF5H$j18NFfxl}|6<=Y=>?KyGvq*Yt|^@$I`? z?3ZvYIq0?+A>JV&qVLRBXa%Nlm6hte>A7gjFa0r*M^{rz&5*z%-~IY#W^5VDNa=~4 zYzP?lcC)2Z^mMbL1dapm&bLPm)|)2m@Uh-XU%+sF ztvub=CBW1 zIo1o9@M@iM2pZyIu_%Anr0mM@^caCZ3go8%t#bi5aS&}^l>V)AxhoZd-yj;=Q}6tY zj+DmI0b2_86>$zl5OnX^*O)QW3!GCl7Xn`7y}tQ|NCJfzpcvFhdYtG7!Mz^79t=Rfx9GY&Hx^{RmW-k znIf&(uIZD3cd|lot%p(<@UknLq8~Vsl`S+NUI*&-;4JbXIv`5uS4cCZ`Juw_IviJc zz6KV4ZQA*|r{}1NoXw<4Wi{g{q*qqGbHAp`B@}xIxKTL5RgTaZONUMAFWd+=!~sp@ zuF5RJ*aB7NMLB`zC0acGA0t1uW(BdQ>ESXFI0PZhXOx}=p^&KLry!;4&v_l*N?T1Z z515R8NmbtsP^zduDLzL9xQCYQ9D+rTxc>P1bTdOgS;uc}FfB}r9tLQp3W%llWofzp zHYNgH`axk6tf;9EyUx_BvtmZC!2nD4VvQ+M3d;Ez3aLV|avYvTXUe2}oz;_2_uj2R zEh`B7Fds*n&#PZ|shm!ic)Rt=qIEMU7k!+w7`j&O8xjp(Bn$eURoMF)S3?A3qBzO~ zwvVZE6$g{XBhCo}MZr34&jk+hIiZ9!s&BYSvcn_mcInlUL%`Y~5$32Z4+Ds;GD7v= zDL@dAxjR%0ow4ohLa^cR3t3Cyx;MlEMdrY7xme8Rmz2Ghr~uYL#aA0)vEN${KNK=) zurP4=7g}@s6kO}wCv#~Pyf@naQC!D7E^~V6@V((>HDp0iE@mkD))_M z)zaoWyS7Leps6fY4Q3tg;(Mc^EFZlhS)hqM6)TA?`%Ze0&vp6(ukuT$$=6 zP9dGwUZbDBlg8H$f4U(|ubvw|L-TOD{07zI@-}4Ig(tRSiTden0J_&>mHOk+VtcWA z_m2W{zH4Mo%cZyFWrTMp%lsu%S>?rvtj~mu20XoXG*e-Tl}ZNf69u+|_2uhNo^Cl^ zT`KWfU+>Glem|%{)zS)qAp^1HhH#+5`9eu_VuZz4zrU9`(5%i2sReyN)4Ie&pHTg9 z9ewfyvk5!l#hIDTMJ4<)j+6Ggo|o%q)JJZbYiP0=+w?I8C?5~WTS$jkQ1#^sQUhb}xtDk2p< zyq&Uku<++NaD%uvbE2zoS4t%fvey6M~7Bti60Zuu@T=kerw2 z^dO(Y)xfZE3J^_dj%!XswWA`#?sGKT)5qB+w$+Wg8F(_w;YUn z^H17F6-;GuGR*WzMPY8_PXNJ!@OQ?^_Hbm zp~kt0DN}8elMcmW|NNuf^M>U6B$jpDg}AH+o#b`(`O^chR5&iHvlEd?dt*d*5b%2^ zh2-Ao0XK(PUS_rBNUhcdmJV30&*S4PrFWN ze#xEBeZLI!%13MDC|3D%WgBM3(y)?0%)4>w(YfMww2I|MakrC|!k6cVFEJtak6wx5 znhrfnow#6(Zn-ARnliT?kGflZ`=V2iu;18mS7)u7ZL#j|BGnfUBJmcp)#l=|-8T3+ zbNu~huJi32e6!n)p9KZI=~$YQ`*!p}OByD(0^yWL+U`)2g(VE0zI4Ye^C-_v)Q*6c zpTxU3uL%hqvkS`k46QaB7LK#;a2XYP?cxMn_9`hXQ7nnLBqOT?A0XYm=*hVxhhq

N6<-(|G$cpP`L%yQeVNYC0Y zD{qH}pnH>V2U`|-?V%g%zKB$rK2iu_tY0@Sr#ARxs}zn;!jld8-ktki#)DpZ2spRL z6Bjj!cK)4n9JK;wL`=?1gtEp_ou~m_08ap*6<3(zVJUtes2!?7s}JW58mE^3`o7NayM@VVFobrET}_M^+lRvX(LCd{1LX6r4jJulN) zCxkCFu}6d5M3Y%YklGV7vgVd&??j<@!KTGfQ*1U~nSY1Kfn!V$i)-=eHu zvqub*j;nT)2b1?>)ssDH-HUF4N@B%ytEEX@isKR!Hb>#hKEwTSs?Fu?nF-a=&`c_( z!NTdgim!&xy8Da$TwGWC%U}3htKGYzha3699;ucP?Zw#kjqWi44WBRKMHDBoI;T3apKJ9W7q`B8Z#bTL@-S{f>Lv;~g}bsZ(ZMJH8$=BfDeK z6|zC&i{2Gp(PzBCd$Y6aLx8C89xZ605_=S1F_9R$wwh3@#)jDY?#^7bRoBJi{4zPl z4K!{RiX)U!k>*itJi~{7EcA}|=z9fhorNxNw2KuT?JGKM;3Ya3>k`*2!8SOD>$>xn zqYth?l1wn{{(d`ThQ`T!y~M*e!`5xBi^T7FboFos@Jc+l$%iaI)Iotopu?Wv<89 zx_%*^D}bvLVXOE+a&x3Eb{~nb+SuYE;&Kv^Np7m9N~s@Y-k)qcoK1SkcvjAzuFe=_kegb?PC9@3vT->`*j! z2?4()J(EtALLzhKsmpj0hXf6j?Oe{Mt)-BI>x20at9G0ZIuWUk1m^-VQ!}9Q{rDn- zN@mv_kGo{W*v~HU4z{PKS*x>F-hv4%Ix(N$?B)vqV=hpC&w}}*hs^IQrQzj%UcD`~ zh@<%7=0GHaZHa5^5$-lP_R$+M~13OlXT(X zIPyD3-_msNJ5y~d)ye|bv1pxkILN3*pqbw{pQ@Y6?=aV|eyD-msu5O05_MKL2V{Ih z1BV}w30-$#t(MQ{*V$1u*so&e_9|~aMlUoG?k(m*V=<)}C@8O&{;(_1x=2{acj}5e zqX-sY80_>tR9&dMiAu{0!(mw$&Gb$P1$*;~8F{Lk6A$rh4{1HmGHWui^m5l>{AnoA z(msLH5J(Z@$~-nhKkWU?rKvrM8odZ$5TPrrk0$1Y8@NzxBX=9IRb$JjEw3jdaVJ*J zHxs1_aRX#>xCW#Vprz^c!DD=6Trf8I_cK8$H1E%L4INBtUWDa5GM5cm(sApX%C)e1+XW1HaU3)&bm{j`&C=5=d40G10(%V zGzE>`HxF?P&Z?b)Zx1MR#5s*?18Cj5_gB`=PYayQE(Zn5+~624nR>Omt;Kvv#8~$f zi_nXlX+S@Y+F{Ri_0GUM`(FDM?}uwETO4NPIBaHZ5mw`#=}Dd(gbZ+2OX2+5Wg2h0 zQXkkxOAK3{P?_3Q>b6`DI7_3#)(7TV&5ZQYU@^qyW-6qs4iO1FW8|Z?#Ch_8Sqtd{aK}=5AbdnkhctcjD;I{8kJu zcm5y@|CU!R8IN$J3r3KLG)`WI@T0(%F$5=<8R# zWR{aX)9r2=RAauXofrlUjC1#VPShSx+nn;Ln=A_tanuUw@-;h5>iZFd3t!q;bYyDC zBvtLT)6A%uV~5F)zrnnp@MCxx(*hW7v(60ZYla|&yX6>P-KR+H&2=6Pk$YUjm3sK3 z3tYXH&5mj9$PDCfwe;hwXuUhAn>$=9`$1TEvQ*tQhH-H0{radU2=JK(CzYfs2pjv= zhc(`p5~`ZrSfJp94H)mn4O+AOdl{^*Ty69%8{THJe<2c*W?Go2{b^>Vx)ieifwnx*1li7Xp^>n2A#LC&o^?@Bc=B%Ny(}iavo;FRX~VL}1UAE_Ww(lTc$yof5kQ?<~)6v)uSM0#7f^$)wt^hHY&DEyKLXotWxucFhE#y{>xh+faIg_0G4azEAmsi^F3wpr zbRzs4fEc7n-$1VgIQFZJSC=-CpzUYt;BJp%L=9TUA>8SC`4v)U-AHuIhBxmW2JoI9 zI}rlD5!iv(EI2ls%PjB|Cz$et$1KL*6bO<;?gZPg0Vhpp02CU z$otX;c@J{MU9+r~3xI{1)%qmKItr4}+7V1sDc$Dxf5Ob%o<-O?2KAf!j7MIw|PL01=mBhBegAS|I z3^09DtdgJAH0Ar9N(j@x`d*%Ye{wn&95uv3jO0U~Rp|z2%8EMSRG_Yq*B41QZ?xiV z(5ejqT`W7j)f+!-oye-JEUhZCorxUH9L%lv`-LWafGnu z-}tMLz|Q`^g>osT1|A^b2EwvthU>IB^0r$%;&4zL^lEy4R4mle`)=;SlSQ3mzrQ(I z4zyC6eaWgW*`XC^3}mrdjsknZs|=7Enk~4z`gvD=`>v%`4^ikw=yi9HKc2 z%eC6IKpzp-imD0jHD9tX?;hOA3d>h(6=a#vxN9+8JCa0b3)-%}-8#Mda zl%@VOGuvN(QiMF@E+|e_>b1!921p_xQ^_QtuNRXl zO7w(wdHc)gAlKc0aHs5G`ykJKyCJo8*w1NFaX~D)YiV^qP)KoO#C~0CHxaB%+3=*k zbK>}n{On*Jl{9gycWhBnB@41SSQzH2m@d^z=c~0=aM+mhX{o_U@bSsDICvNhmOvqq z+6qWgMxJ-VQBxQW-r)sDD`IMD8|gvH826XEQgywA^j5VNRxyt6CnRRHBjOqzl&Z8( zSDs$0RyY!E#o=I$C!1Yd=524_y2A}Pd6~mPoF|!-nV62#a(%e5$ExA)T46fivdVvd z_UJVLvAC_8SfPuvF0lwQ+G>ht7({%$YkjwjKiy{QG~^uilG6h_e=p@tx1j<99ktM=@&9D$YWfTaFcKcn|0UCvq8K{CEflR zRPD~GeV5dpMng}WiEk{m^eVlPJCfk)Z*I_#DF* zTo3d05W_kVHb+IPOcuEUK#%!Ih?~{LA|uwskhbL(Q@ZeuG)@GJ~DQo4#6PL4c3*TzF*?lMAxv)}RY&xB*BxS1p z!va`7ELn5z*v8${Ep|%Df)39>o8jS?bG@q8IW=YOi*PoepVDrTZ`E~N1VYi~i#5i@ zmf%4LP=Wo6eG$ZYi$=2+jVD@5CJr0-GSZ8!_p!;|_wqbvyL0tHoBpW*9b@8`)d>jp&hPMRKut3Sk%pA`f-@+YV z!9sK2Hq@?0p}qOA9n}I58b2g~!H*e&h9c3#=g$na>Bxsu$q3j3xx=Y~o~-3qB3veh zV~1$;R&li2*e3(}<#w`1x{(}qim_hSYVolaSIwgwP8Cn&qRh46h?QxOnI5de`SCLU z{){h7Ion;-pS6S@Q5raB$x9PPwNeS3%dUB|d}vj^ zU4N{}LD@jVf~-6xPOXy1}AB8RM5emO;*YHG2~=;DWAN#;&B z?hPeNrP3#y5@XC0qyT_ZC75Hj@2sD5>U%kUr4qo_&?v=FhPZIuzMmhe<2ZkUN?|6_ zrt|k|5s~`9;CG4+I1_(AO31O=0?yoX)pKl4*cIy;`poKm3X#eUK_kbfCx#86l8y;z zt5h5;ZnjphRBa`3NUzypIoL1qCo@^uS7Rtr>A#kl*K*ocx;w{mzPjzP{|?dXgn<}5 zcM(gNh6aEQ$CKaadqq1#uU;Yt_CqGf)1H}+XP|o)dp_P)IkvrHQQ>gHLX3K{_bebPuP^Jpyv$I7X;~wE-3lt;p>&oPsIEnme8rGdH ztp48#u z;cO?16wnV{+L2MltXJi(%6%{nRu^}J;HTpD!NIxQ&@EZ5WRd1XjHsR4v1fwBUWC50 zfuRhg@U`Prdb0+jkw!Y>Po-)y2@Xarhj)7eZRLF9XC+MTe!dyHtKZ9tOCk8 zaA89jwob2CEW&nftUhoQ46OS}h@cwPj{4{3 zn3m8QGw8>$cz#qmYGDKS0qh4pQ??GdsL5@OBu8umN$hDVHV-P53X7?Uqxw_6M~3Wk zuI;A75Jr%ZeIuWzUVKYnvUwL%xH3EFPeV+?ZaTJh8Y6?lBGcpq`-FX`(#k-mI(f|> zlF3x2&(~k!)1vTgZ_Q{?!ck|I1&g3EUv&uNI1=684b}{Z5m96od(B)Q%L?6N?^UZo zMlA>Hy!t^ETt=I5XSqdDKta7^a9$^V{ce`Pm#PKLe%eqzpASLQLu?2c+o29DiAFV2 z#~%_ipyR4CH>?xKRr-7*DgKeA`#%)G`MGaHW6A;)tTMy5BJ@+I9lba_J~?jI4a(ha z7PT%Zh(&tpp1$`wLrZuY&hl}2zCL~}Wbt-?*jZZz?9Yw#@x1*$eG#F)>)q3!ZfdoZ zRnMAC2%ph+_YflP{)5E<1SyHMnuXI)?SL&in#rp`BNlx@RM+CX^1h0cBU*hadaW3Q zMzX*BEnQtbf(Kpg>S+EQW9?`q_DtjjPGw#`!-(6MJ~(-*-lR|)jB)us%O%s_8w#{l z-JCaV2ZvHI34#awX_jI69IEZQT~oL`vn>aS^%fM1*8`p16Dyap6a;KqXb#wV2EmrK z^9wUFiGZ|EcRzj@-Ok~%^!1OP5cSO71x|XU8i{(MsL}9rfJ?&teMJ56P#m3rJR!SS z;_+@xm+c3gU;mpS;ewnQR#9jD{C+g;x%=}Oq`^0bpIPBuHhy2yPONnGTWM#ndPK5& zjg4~JHM0hcI?JTHRAjlDq?y{RjwC$J9`jx8!c|K&6JjXqN70CwaN`aF#ZISzhjwk2 z2gto6ldw~J2aMM%hCCM?*%Z@lvLyLw4$+s9%MSrN@IqAfitWd$-$K6$e`23U3;>?V z46TcADD26YK05IGoniu-o{ZDp4j9t|5i`86ejd1XFGoT`FTyn0(yT0v*Kb0L{vYwmI#cILSvoxez_+hd3hF-3DO?^WH4m(SL| zeCW$rzg5Dr(zOgHwISi%kJ3s8m1;4xX0JhN#U{ri>|PX#6MIy^p4}%&;HG$U-;V6L zTWw2wB2nEz1)@4E&|#x5KfxHSj4%U%wPV@yrY|{~)4}TUHv=WCipA*P#4j>f+_IS<+Ub}MYB#~*f-&xDjgsw^B3rM?a==#yZaMeyV z*GfEnc}+nVjrwbOw+KoiXoY57Q;`U- zr&4BX$4k(}r{0==@;v@@IVJN`eU`mM5p?JF zP;G6?i$9cWEtKk-9|AsPZFv3^UR0OII;qj#eh6mR2z3}Cq=rdh(6Coz98KoZr zUi_BE41Rw`kYiuQeET@D_iJlns;xWZMbGm~IxHqVHP+E}J8btBbP>Mofej-m7~t6D zSyw*8=3-Hua_y(ng4wROG;hG6e7{o&-DdwIJsKyIN;joEq_fTEh;$x38_XtM=@{-c zup(h8BrBi#gE67h(qW$chO){J#r~t;Z~ra%*Pll6q*{}C&D12|^Z|bE3{Z8e5^1mP z>sb<$7xINBNm$@{80J;iU0^6p_dH^KBC{32SR0S}Fu$iWCCKr|XWd9x;@Cdq{4YJm z7$#rQJ-uQ`A*%-Zl^T66aKssn)eD+X9i|;g*XJ7DY2-6l$ID968wkJ|2-HZ{O3Nu~ zhlh{XYi?H4IbnKyG;)76C6}1(A1a^Qy~DG;P=%`_%zD!77=R}l06nM0|6%Vf!{W-8aN&eN zf+R?=;1Jw{TjLJF-GfVz;I4rX+#Qwvb-;k0tmCy_^o0U1M@F z(@i+Q!l24JruWXNE*?Smo?nF*m)EdtWT5E+(E9kh!=J<05Nb*d6Iu{{U~W<^lp}dY`}=?{d7MKr3z^X>Ty8U(X$ePQ{`6j%@Ok-=e4{ zip%vTP07@|T2tA8hqK9r$llJ%*pX@xr`y#yM*9}ojNMgBLUMioRaH*ibh#?G9_((T z@5=BS`v9pt!kM)P7d)lok3ybAe>-b&5oIMLgG`$H80yT`HOT_$vdKAvVK*POYI!CQ ze94tcagUD_yah@QVD1OGuLPz#E4NTCMrW8O67Qz&12zW9 zY>0Cth|-eJr~`Fe92-s{WPc&Ie5EHD@DqjQKBY(@9%*!Kar4IxIw)-2XZc{Cu02)1 zR6Py?zg{oW`wJxQH!bg+jq-kMC-l2@2aNcU{Zu+?FcHzl&i+eXe<(K3b4l?_*!zwIW*^tAaLb-dQvc3Zrf=tVcLIE6U=u?y*su4@o8p zrE$NH73JQaF?_E$FiD+EjMg=EP`N|xmBlYCIRw7*v{`CZa3Zm@uw#ma0Z(;!5$73o zAKTikHx9&$eqA-4Po9U-mfw4BCNXO^T2G)*psuL&;Qnxfr%*7;7qcFWymDtxS5TytV3a-yxj$6)!Q(;1vYNpX-~)C;{XaNd)B-g0?oYQU$__mOHxB(ZrAxAqJ} zpw14jn4$ZkOyHKR&)4pCeLK9Z4oUypjr03cTA zQ|MX=R0o@lUlbUy806@$mh{RUZ0WU(UV3QD-`-_Hn&2g}_2B*p1bbsv?TT>wEj-1B zxG;b1m3wJN!b!Kz%7L3}O~lZG4EIG6d@X7Us!D)|J(%3Z7^DDD^ITZDggeFd%%J=_r3l)HDR$ z9S|7`0T*C5N9^zzz@KeR45VDVp=^;`@^Wnd?rz-@+G9Q&?8<09^8QTTKJlFav_9IN znoqL(Rg|jjy9I?x0Jy5dQb(_u(HVlh!|R zF;SvUg##<=F&l3K4eaeejRg(S;@6}0P7xPC2z2G+Z{<%GLH=B z9B(pHn=q`NJ*Y$-CkexKuUw`}>|nKCmWsg(uu}0`5v2iGF@aJ8h|@*yAV;$CokL)_ zvCEXD6zyA}*jO{`?0qzl)Z`Gi&19ZNG?}aEcYU#MLY10yqQk~!v&SS&MFPqu>gGV; zQ1zyoMDX(kIBwsjQ*o!6JDbosx*sU)B3Wp{qtS7&c39cDEsaRq{ZYTmAg%g@^Edz< zgE1nw5R=R2e)}&j4kT5|s!5z`9kwW5VWO=zd^j6X!HfeS3nHkT7(XdPZ=D@r;aZUG zSq_JIUEDjqhSFv8E8vj|3x~hb#tk@0c2+%*h=;0f3UrJN_dx z%}^YC8*DdL=qRNjrO+;<4}w1G2zrNY+VilLVAC?b|-7xy5P4hJJ76{++&~> zYa8JeFI8XOt|ko7otohDhmGXH9O-T`E7{_sLL4=@mFJaE`zG3k=$9O?LDX6m!9tf( z(h}Rjw-!{wmHF|9p=`AA=;bAfpLAid!{K(Ek?hNus5lPRH|Yy~6Hhn%0X9ohhzAX1 zIjJ$7->w|9^Mcf7nI3!fi2N-axVotH1&b|V%z_VTy6nQ_vT3e#A!E>+5hcG~cM#K~ z)vE$W7XP|OOBy3>f>kmbm_$KLEQy^Mhs|IXJ5_)5%b2FAI+^c~gG2a+zfg#S6&AMp z)iNiYRD?HN`fb1QHGVm9M{+9l!ySqC9BfY(}E{O|$v^|j+eKb>fxk2W4bl}rGrnG{seBgvH^$?ZvoCP$PQ2<~WDIt9r9s#&qQCxKt z4yrZgb>BbwOp^IJaAm(YL`OP?Ab{T(&>B8ka-?iM(H($3?qZMoaJ^$e zp`J_M;B@Bia5TTfnc$U_W_45lq+OWm{gM-QFwPH2OPx-|HQb&P<~{%p z^S1f;sgKCMzlsa%O_prQ=qH(?`m>2|t)r=0>w4WJ6}|GjvVh?#vgbt%!@V)IvAF;DC=zX zbOqepTx`F;I$0ZDQ%fQe4HroumCx7II;w9SRJ1c!(O2-fzq!!ri5on(7yz0;yXv2V z7EqTrfNJgQ$?1Bt7-svWdShf^I09YGU=v#!Zy0m}^Nn{2{*jgaRr0jdK#L}e z{go-itYSZ=Ox=XU%Zg8!)6mNOM>uztpBQPp+rV{kg(^*P&k+z_J96kQYzq49u+}v% zh#j_RDouL?Vn z)>8cF&uNByVxAZ7i%X)Jo~(b)W9D&q{m#iO{6^<=u?>QlhV}&-$<-wa$^*J|c!9us zAvJYY6EU}_1fC{>! zM1y8R;!A&lfZH=y5#WBN&wdTkz`R~Qv56$Hg%$UVBy4%))A#|T6X_IF(e0^1^0Q5r zzyWaCgZ)B*93_QZ>DLm>J1MGfg{pNmK#jpAi7BEKE5+F0oga0=^@uQD`?whIVR{NM zcH-@;rwY}p{yd^+Vr&pH0GLsDy6c7q`VIQ4xV>;wD%~#~CQBjptX)x1q6TKH5zL>m zxeL%W5m`_7rv0|Z@-a=B9A!c+k8re{KWCbuVi|B$TR(>j_m9eXq-DwVuu%fBJeySp z=j`N+<5&4WKklaa2B`(5GX=?@fONwF4S|TPX^G}c1eB?z4Z?%AUVGf~a?Xq(4{Hn?)&60rV{D(xbb^yGM&{r5Xph^_} znpn(ey6sYw0R=%w-Gc=ul+%d4u377jx*0d&d{4Qkf=PFPFP`4D&}v@TK_-u~J{y1| zkinr8Wq-xxf9v3Mc}N#^+xSJN(yHBR4Cs&c*C{w2?OKlw-FJ{woD2n0=5CNO+kKi8 z5(5aX{pLo51A%@iX3x&PCwq)5BdHHMn98zHVKas!ey{Z;f@CC^IPxJv>;#b zo-KCh!f_iOnvKQNXw+mnsLz&}1&wkN*fYM5EGP}?FNmz)$S+~e3^TT{>86|~zJ=D`a z`ok$M1?ivRapQ{3Tzph}YUE6`t45^>XTMtY2%~#q(;<8m+vn(wAg?T#r~HvPTonPK zZy)^HtAf<*N>HFx8omW&dyxt+Go8yyXYi>UlZOVp7KeN7fPP|PYZmaDm{DvjY;o(} zqKR@h(DrhnqaNd_d_}O?hK}xid`aUh#S?|A8(Jzzsbepl{grtAlg+%-WURWpq=%zbawdzpIP-fKHBkot_|-WtNBc7v z?kTilqvryvGiPe@->Y(>SMwK*|5;}S6Iwt&UCA10C9b^62kQO+1wO|ug4zF-W-Suu3I?Dl3<&*r2Kkf6Cj0#2pinJzCp$mh{%Ax2laY#t zvjSw$pD06BeqbJlHxdX8b0`tcLM9C!P9D5ZpRw)pK^V`KPY1j$MU1tW!v_%{!L~Cl zT{J$4-aGBDVr~gnn_FHO=!b_IUg3V~^65;yl@m0aot2 z6lqTvM*UW@@%;U4Sj50oI1PgDepJv0LzQgQDwV28Zh5%LFWW6KVPKSl@PggcSTWK0 zbOw`@4{n`8DCG09Vr|x~shscLIfcR%Eao`lanF=|Oddg}loCg$(U1{%sopiEC9LbO zb9#s8sJH{@*o$)agQfb!u-*!sK^i>!Sd(V`c5+zJ5?~W56c3G~u^d0g> zG8*d;hWsh)3=SrXK&3w;X$|5%$-I<=>>18R!-0OARpSDkwB1A^yF=S5wh0@dBi~j zAX8zx7;VLY!l4^Th7x2$j0TQ>5Dk!o4%-5ROAUJk03?xZSO2tCF49UW3KM5}=g*lt zJhQ$0(~1JfAhzRRWKwf%)ds!i?O$;<8X<`X>NJ>*=KXLv;Z+Z*(RF0dwem3QWe(*_ zm3g|Uf|9mPhofv!K+8-b}IPtCCHJ z3Js@wmxbX)x4GSsaY)KxtuVf;Y^}jy6S_J=xJgYo1Q}rR3P29Up_Tly*bR^Ong?GN zamxNMmPFA?;k50E-f-kg0FfKl;frNoP9~G_miBnbF$Y^uD$78}q{dts8eDAw%w$N5 z%3~p$3&difdBkq|qfzk4qzZ)9Nm3aAHYxW3~S3HBs@R zpO2Ku;@nrY@;$L8?w8ke8~KZ(^WCv~)Uix{joon)Oa<6$J}ikJ)HlE1tbA)f# zulmrj^7KKEL_%cwelvE}H@n>jM#6jX7h`Agwn)2=cz9dNy=2s4LANQ@QWuH-%kZmF zGPQ|sK80xzBqJlJm4*)W*6u7zYeWE0OVu%Wci!2f_7poi{iuHV5GM&YHGDmEG=F2Aq%wM|C+I7Lnt ztB?+a;)-wD0@`u)2NZZ9Vc@CK?621Tg zZhoXg3Y>G=&T3$}{74Fz*mE|5+u^5h2p3cn(_Ic(M(SvL78pO+2?P&K+%6(OIX)u} zZvO%Sj*X>2Bfqce2OWvYE5hXpNC2xvIryg9JlOPl10$H{O#=r;C$+GxM!$6ypfBZu z`@Z7_kFkQDB}qtBUd%w|*z!UJ=G3UEZy~QWdfMV6)4~5TY-fMIHlFL6tWZ^f?RZ^M zLd{mTe$LV>Zqaxp8fR?5!dtd68PR@%`3SqIN#S{_Noflyz^Y%3WD5gvapYDZUfkWCard$I`Vtm^WBM2IYZM$EBz&tPDaaOW5 zjHdIs#HC%__ zqus_rEvQE2SU$5JR8|D$yBG+~f>8iOR#Tv2{*hF4c$8LuL0_zY4WSP|01K|puXp%j7D=m?2#Zd|b0l!PTO%I- zf{doOrCKC@v>LB5NR<*U{wncCsnX_WLA=&(g#?Go$_qgIwWx@_Q&Y}fU)tpQ2}IQg zfT#xj@w{%&ZYXgK>10pP3dr2u@SI5R;XpQ_6Zx7MFsZ8LrV?(SeYa_j@9&%=)ousm z`6taeBPHN^Vv2aSxE|CcGJ(NEMz_^?ktgNGqq0@F4P?3wUBrXNSzju=hS=f&rsmRcyaTLltvGM{>76@RCOdl@(woe%kZe=`Z_ z2WTgou>!<+E9A_9RJIe-0O=q&2@Y8f1{2=*vMA@0lw?xc*X zfM`m`+bsS%w6Ro6&b>rJd!MPC#;Ng#?omTkc6Tk|pw|is9}n$8FTj+Kz$og{HNock zH?hwGFmvT;NWq?Z$b37#Hm8^5?apcJ4{z|^1^UCuWMY2_!W0JxgYx0WtJhz|P-)*= za{H66!)BE006xW1e_AfXWVgBSRc(;Zm<=`|-c#Wf-*`h8c$x>hckA7{Uv8ZR3csVQ zf$U}0UP1QA=NFiLC>Dlx16=iMOtp$piy`Ql@^R))FufZ}$)|XJBPgj|jWY8zMSp5E z{Huu^8nCWAzf2p6NKD%S`ur5{nPjgIAK+;?+sK3bxFxWzWH-*K$K^(KjXI;62|KeT zI`NCjceluaNiCamb?!c-^(*Bjdn9}I3Lyh#KF}Z$d{m(8aOxGhX$!e})XG{_Qv2pg zBdP2yxh<8PnUl_~j9IT&RyZ-N9MFjwr|Q+kC>icHG?tB92l^bo@Q}E}$AQ)e)cgV< zAh7x3J-)*{9;EWBy=g0!PItt%WH^z%Q|3lDF{@j1=@O7Dg)p0UmBjql<-VE#pq))` zR{sGD@CjgsZoXb-YD~G^sgi8HqqjPZF|x)p!|H(zpo+b*h~Q*UL8Sh$+-D~ya+UD( z9u8KtioAINZi(o~XzJL)8+gw}(UvdVe*qQMeBVswK!0+Wu89j519+vMBg2fG2m6;Z zfR4AfNba{BY5yHe0E@bwjWcl3ftcx)MMr1`_w5ZzlZ~x`Qgvt~Tz3GLk_EWR0Vq$f zTkWE}!Zc7rF|bRbasukFMD4+-5s#D3W-4yJWlZ_)T5M4v5r-U4Q#> zEp`u${iMZwYWFDG(4LmrJS^+N8xqKuDSa5D0LSu#rs1|Npzvd+qXtJs>VcJBhLXiR z?1iNtx#aFttrb9y2Y5{481ybFln0xuX{?qxCTpdtNR%yz0gmgEaNpYeyz1#bWf1Bd z4Sq|RNR9)B3jos}DbU`OkIo>_lxmNwbKqLx%xjcrj=}>?)Qyju;~B3|yJEz}SD->T z>~M;2u@f9GfaLMJFEq3OJA}}ZEIonKdkQpd{eDyK=rSJvX9qUdhevWjP;eQzeo4#Y zj6ji;5znSAPSE?956{t3+GyJelbv>PMeyq`CvkOiKxk(W?qkW1nfiKnnju#+j}Qmfox&VYCt708Rr)QWg}3foI1*ajd-&hgvfO!LtwG#cC@Os<(D^5Qz2 z?Tm%bTt9;}V$t|g)u6p&7Rf)Z2g>I;3GQJ&e+4(R{Q1Tn{q4;n`k*nYfH!(GhFSI% zWxE07i>hXdo5 z%E%!grDn_y9omM=i1QBY?QO)&C&^4d`MOpvFf!>M_qJx6`haAU}E>bl|kFeZO zjt*3n9|W_}E4y?>N*BTL8&m^ud$DA*!atm0-gH743P_bMcE>sTfkL!p2gYDt$r#>C zyxHsJ6W5&wU~<_&MnpXww!tnH!bU2UgaoeO_~=R}|#cs)0eSxq)TJ z2FxAT>Kj?38-^q6Q(AKG^A6$`+rMmOgQ)5>K%86hF+caB)}YsX8f zLrBfj*mvr=KgOuqg1qIth94D)-BUHK_%>WU)D=~$`WZLy(0q?n@?HQKkx?n#{u_}Wjx+0$5A(fl*3iUWoeih?cs>%4G>3i(#zlYIf{dkG0*tAo zL^(jtjd@;yG@nwfk(UXKGhq=t)KeeIj$R>zlgk{`{NPluJDwsK;S|Xt60 zPN6{YbFIyuSV*@EKe~{G9v4}QUFcGoftHagjmaneWT z<&>V+QCMi%TQXB-hNMNewr@60eN^rqb#&re$ss=JfDf`gdgnPWo}})!vKwczw=LwbAAXu6ZVw zS1}9sw|(iQFN)?Wg3dfC7xy0!>gNbIl1b-pRrD)uuaO74D7PD`!6aNDMc0&Q!zL_o zG5?GUmAm|RYs+{|p|P>qsv(Ezvnk()8OFOFpTl6hXmV$(wQBwcDm-!ibI-_nHh+Zk99Wc0e3?tt{ZP(cUo2yS99AE;F8>^dt@N!hl5N zdGhrC`U1`KbOUiDyKX;xI8As`RLTXTFgF6;L7Lo?6d_a{6(+qo!u%mP$ysl+*u8dJ ztYU)mW|e5C1^y;D<+X#;@j{4*Flw$Pi|b{#(s6QD-hJi)<@581oua#)Sr=PZTMUPu zE>?xo95sd>*N96ps}YUkDENx<(|S)Z&%bL@xB)m^aX z7wxwm&DHg!tsLYOV|ix566N#YlR+g4JLl#{u)nnJ$Z5!b3kjxzHMmDPTqWSMM_`;V z(9%kNa>ug_Zn)7ex<*FEN^6zeN_{rEw6u?>ZZ|G;_pL8j!P z*2hP%)OCaAEYzlwh?qn~9Cf5OZ2 z`yQ_EU_)%Q7F=^XUS#W zzTEePqIX$s=y1^|Y_8_`VCG)9Nt->D=a#uQ?cDsPTQo#{&G^COhGAGESs9NNVjzXo z!o~Q5#ux@`>Y8?M?QL8{vR3wd?{0K@x&UgkSlBxjt*(up_wJcx+AE4owR2o&j*|Rm z`Y6=cLLSx?At$xR$vJi7+w{g`+1eEslV@8u9M>mkcMN1=t}tKRin6)YbbyZw_`me~ zQRls6n%RJfZrkB(ZzFK8a@Zs02OaSsq{fkKWG37qzGJaSsAIRTQD|Ya3_(a?nk)^9 z>;1UgqUi^zI38>+ZMPgmsH43qZMXo+v2nUT^{G3zB+FE;dZr{K3TMbrs)6A~gkk5z>KCzC>rb(K~ky)N$voK(Tgi6HowmU;i zse6I>vCINmc61F9nKj%qi#JA7&p5T0~%$ErE@=N0;Fj%$|}=zO?DIKc`b)tF%q|MK#9v+)gTOU#|6 zE!7=HPjAtH3}LwXF0)9vV=wa-%b^J{=XC>2E=L+{t)ReDpmsT`gTZ`Dlh!}^5`!ph z|8yRG3=&1)EAzyCO$Eu&!$h6!-mdMd8%8LJWA4;>faZt*ZWJJjfWO^AZ*1H*$+<1}uz3aIxjst}P%L-oHt@kbK!v@7{=HS(e9B$;1+&$q( z`8Umu$3%ze7sHb?t;x}>2j`NTnkiQJ(H;&3ny&~5ln_o49gYRU>7iL+FX<0Y*EtS2yu-28mJ^6^ zGBIt8xfiel)8y4Ygj|!HQPSX%?My_y7;C(9Yb>`|NSQfv`1k@sYCin59!!Yj6wiLd z#a0Oqfe5~+-4{2XZIm;wag8*cEs;Czk{Bi7CX+~Dv}66GXcztRrcg7^vqw9;+^;w~wFA{2j6Ac#=R564(H`=4d;W3bj4|uKOG%`q$NIJGkR~H{R(jP4yhk^NGeF|S>dssgi-4#;c21Y&8an``HZ{1ib zX*h}wHHFX>I0*iUJ3!=Vj`Dyf-x`v$CE1YqcG@_Q#^NZ8=SeCRX{!HY?p+CL&dKx# zXiyL?2d|r2WvN#b{HGr*>%p))d8#tf(nN4*)X^8**XoLesvDIKopxsF3xckzkr59h zmL;gdZ6ycUJM~7373T^Qw})y)F#!YIovO@+^b82S`bu*;H0S2OF;?8*i3Kl@!!=uEF-v>P|;jBc+0JACvOp9vaTTeeyH|5x71k z^H{X(MFtqZJh7#l@~}en0^3&$!#vXj8p8!#SOl(FvseVPr&`a!fB-4a!xDqNL7rKJ zDVk?=7LAtKra#JM6pOBR0??krV-rb%uH?r#CbeN8q-?*c368ISZ%{zVJ6r#Nz#hj_ zE6=?SD^2p%~OtZ+xP7=GZSBo{;d-R7zYTEM&cr=4_kIZC`h9 zcY}fOz>TNUe2!pe!lGE!Y4~cR+uIjfOc|emaiteuU!lbZQKv{jDW*V~noww7*X2C_ zbNsr1(QxD;~85cmwOkN^#(yxf%~1d}w)={B&WnkrdLKv0~n5 zH#$)#EpO~bk_tzWz=?Pe1+q2pu%G0rtj74`;wUz)rQ!9kLt9&Q{QLtWug^w_H10&j z?aM9$~CF}GeW zqVM0I!p+h%&5RPDvd%U*C!?UDb}fCMrf`XR+n>OKhSPC8QSRDFQ*$5fUXw40XE=~2 zCGMac+q(xY?2|#@-kXg{DO`Hl-AjvTrpZmMfSEa|R74J+X5^?!y7nuK1l&r@fh`Qk z(MpI6nF^xuCCHe8vnS&I?boQhUg5)|DBYMej@+;$nsAz=ya%W!jV8L;xTK_{!T=$E zZRl<#n45RK{-_dGZOmHvUD{3oaUOM{$Db9-cC@IETgVo!1Cl`CC>zQPDg-_;KB| zQaN7N#9&h-GH>y+9qck-Pm}2H~mH_%ahdg$;o;F_~NWhLPIOyX5lCo+2Ocp;)PoPIQgo!K0~P1 zis@Bje$&f_bt|O=W#6AF!qW^rcHV+7#okjz+RQ$PEPxb7(( zyZtsk4o7(e?p&Q%6Wq0kgF|%{!d}@mpiGg527Qrz-&MjBBgr~1SQW1mt)FWREtfGg zdm?5xH2WsIqzJ`n;SbBmjN}_5Z`o@DgQxP3hgUzk%bV~tS3z8#yy9%QZnEcOf5%H2 z5=-X&XS$BK48dNVp5+6#j#ZSlk4ebBn@+kR9l_QU0nhiUx*8bMbK8q+G;$MnwzYT8 z-hbS3s=DEs^`SMD){~THvAb%+^j<0;L(`?6QJzrm?zY$B_%u-I2lJv&R+eZl8tBa; z+SgY<_v$I(H(@8IJ=;NC=v>LHTH8H6l&0$*&++M?QfSEfC!Sdz+&(tn;Gv*F;{-Gc zH$EQlJc2?3BFfalpU(pZhM<21h!FD4FS+CTCkaR5l4WdmW>kU|+++VRKl+t3eLIJE zg~fQ79p13r>&$Qa_7El4kW1iNVzJwvl%14R>AAXPn(zzIUZkegLM#V}LY&}VfvD%u zYy3kL)s2LCMJbPFxwCq*pg$Ah@8gxrJP%wTjNX)=*2A02DYa^;4QiGv2CH1_LjyG*=uElEmW&E4@PTRnhzLx z4yQfKfU}*D1pL2{c>6s*pe!tgDdp(`&($5vR@c@QoSJ8SUq!j0gVl^NnExI3c~?Fz zg6rY5(F#~Z)J-w!-&+I^z5^sQ2x_n28eM(K8h6nVfI3l@Y5*}k zJx#w>Vz`#o-lUlQ&2r_7l~ob4-cSa*sH%yT&tCp-^6X|BV8MkgAc$aK53o6v|13rM zPu+vo1XkVRY`zuux~sc8>KTSFg|fG=ujq+^1Vz4~;2xxyxcIA+*3`G;l#~(Wk$Ny8 zU$ZqbcK@wfh_b*9`L=CFr%hlH9db$Dv;&_s^|wR-p$MpHGngGS_@A2eb88`!0f8=Z zWSw?71OK_+P*mbiedm8%xTgkW&@_lp!2i1i@QLA5QVRF*<#yVjJspH>l`m;)a{{Ag zqWnqZ{x8M;m!>@hgiGEmh2Q@((EZord5Pk4a43oOeg#2|vSrZDBS9m;MRVB4cS8dgi~6FKz9a-6i4|5Rl^twd?oW=lk7X{l^6!4;VxT*kWWM z>))xV=LhtmG?0Ft47kUuA?+05DFWnatRzg}{;DU6;QuYzKi&Z^Q9z;JB}rU+{qCla zejd0>q9Lbk-aOfGn_)?(&{FUJYKg!5WAAXl8l>500a^amt3u;@dbR41li{DD@MHyM zI;r53kO=9d!f*hrhV8$*UI-pwy-yN};=lg>mbo6dTjW|nr}AjIPfj_ApjzdNbTOvM zi3T~7gnnmPe|{q}K=az7ey4TN_=}Se`uh6e$y#LWw}3E;;RFPQ#7s@gy?wT(kb%$j z{FlA`^-n3}fHg7>ay0yYF%XhGFOp#c#N-ciMAFOOpVAX@L;WyLZ()9lV@&TQv84SP z6@z$K1_?6Bl&kOG{`;?Q>RtHg-8nB#v41E2zc9K#d|wkXaG=>|c|ZU6srlFU{llw> zwU4g$;>W1AV;|71hfZf8p@{ z(DD4^RKsEiyb2$YMG~?<{yZvx{By5v`FMhkQIK$HWMsl#V<8isEw2PRmYL1J-T=c&zu z3c-FF8J?&Im284_Kg65$*_S@3SPs?Kz&aFWb4boataJe~`7}W!$|C&h*H5U20nB`o zyh4)pf2y2+df`^!Te`rDWHG@ew3tYdDWE^u3!nWWuRkmisDBdCrGVQy+jC!-6OLD% z--z}aKmYaI!pb}dPRi=)R5gxQ;ZaFxUZTFeYUyq&6H2^MKuuVNtdj8pLGw zgJ(M{qnfH&>>rPot-i?9K`>q8oGe4OHy8%aMRuYWvg=_r+8sPt?J4-|ns=uvCMAB> zEh6PKMj!T0R5Dj-@0(_EGQ#z*s@L!D@`o|Yzuxjm53=WCd`b(J|G*(4w^%O1eJYucTzSWGYGJNR7rL%RaFWf zlgQG`YUMN1Ovzy0jK3Wt{9ABi0{HpwC77neT>qhk%`zWu1~{z_hqQCh@kqDMFJXtl0KC}IK*)wS#Oa>*<+0yzJ}F0jHO zI#{7;|7s_AszU4G4n!Bs+5-jdaz+#B1z1oG=9=*&*cHfL@Tn76!dBD0Rk5QqKFc)M zN9*Y70Z=pHHRl{Nb)TjLEg>6+JAJc%7<9_raJR&eCsHna1X9+~= zL8+tlqWmy&MeMt)6o=`bWr&H1M{-<0`uY36BqwE8tMG877R(L}hi@M@=ozGypZq3r zSH3zU|Gn{lki?|`kRC|IvO)h`MK*L~lwZDbgQiVwy8ft8)N?Si*&IQt9qan$xzHP8 zd0@h`A;kH|9NS(4@ZM^wJTGqwO5O*J&)4Kah=YPiivL5xv#125Ik($3LvMcC3m-|Lki$&5ew|F=$Bpw2N_T=qst=mt1tBK;_1$S}d`!#AZf>4* zrE>3WB86|H|8}+H-$$$`Ay7!#5=nj<9P(%hh&B)gB0kM>LrJR1_FeVSCa)bR$f=_e z6G5Wyzp~t4)*oRb{`-&r?oXbWkL5A~?7CkPT9j;lnKK1ooAOV{qyi6DB}30Lfn6OZ zI*d$+v5+5q$2p!Ktn@z^H3T1^R6L0!m_JJ-h$6q91wV=D!8Fg2T7l(IURjyw3d$!O zm>O78GJHSHrt0%AKl#^Q^fy;Muz)nq{DbnZx*(4PR9`5pXI4CHX)-ymH{&p%sHiAW zJ4vvBu%Osoxn3EC=Kqj2LcalS^3*!li0r3oeuIuBK!uRXv z$5blUh`B^g%*MOCqH|Yk!3h6%HS0I;vj;5cM|Uyq&zAs=`zbIcNIe#Dz-@tEf%G?4 z{czWEEWb0tz`>PmYyS^%rYFy1+d;%Ipz>!;1=9Pcv``SUdW*1(HO0Y@V4%-LM9QMI ztfNcBL_Az{Be?P#S^q{Te<{Z7_kbGfvzz|f&2P_tsqw4BsdnDSM}Yx7o&nn0Vd*5K zV(RLF{G3$(K4Jg%Pk?nmf+4m{hZTQ%9uLHK{1m_o%QjcGg#918Bbwy^6^@LK7F+pZ z{2CW`fATKX@_+EK@+81HOOI3e%krh4J!ME=F7nJ;qnuQn&MVYlS5)}|VM`0xjRF){ zOuUzfgH$Q?EnzyZlgS)QAmQJBntz+8v@h^n+lVo_KQ)?94sbP>YKz9ZpXKzfa&x~+ zuB%|VsBmMrsJL5)O%me&y0gE305NS551@U-#B58@<&peIkvfr=#-0;Flk3$l#vS|1qJ0JE3Hf<>UTzzqM0}uEG76q2p1dX zX)%j%lEfg&Ga9To7N3X9~VEK-22 zX7D=s1BKb=3^1<$z3YU2>dB4(LPbI8aZgSnPN*`kV#P|&!q5gE9)6l`FihT^u3MOi zj+K@Ynsbc6Ou=tgMWNPyn8yMg=-{Tr*R zCpEVtaCr?{_y@mK6lgODzUaeZd5?T|?+^1EzxOxKNlzZM?33ov&SY1@WS9FSZFvB5 zNarP&FhPyeXSY8?I0)iadrUqj?Dxks%^Qrr^2bRl1KtkSFplf32M-x=07VhvIH)#S z6$H8(`SY41fZ}*F9|3feG&q!;#0Dy5p$)j!8n#+7R2no=+}wGs<~Qet$~!i!Gmf=( zT|8_BEQBDoy3MukDr_CwBthHa%F3~MZ}b0q!SmpMfZPThGM=YwSzUgLehRdQXES8J z+r+R=PqS^O>~6`bs+yR(>uf(GVBve}Ng5OLoD%Q`o*u}Doy`%PKji0RlDFTUu5PEp z*RJCMQSJxR_=^rWNc%`{AWC3rjlgg)nVCAfp;DUvUsl(w1+~riQ>O`qIQ33BiP=Ou zxZjn^`a0sd5$rrnnMPi9_A80HxTyOavm>7ESWxO;^)Qv%Yh8&T_Qx}aGZ(0K`EfeS zV}y^~&}x>xpSUX#J>TcV2aq}8`7;xR65?!EZ7YIRqmCnY4kvRVWBgM-RyLiLR;`vA z`t7;4rKateY2eLlvSDqS4dd^b1;?X<;`)R0ce3*GR1#P$k<0~ZUvXO~n3$$(s##xv zf$Uzdzcy%l@cDG9ZB|>GfOvpG(AwBMXZv^;Ix2-8)MEtFjD!Rn5ciC&5uAtV&)z#> z0uD69)dk5Z*L;y&+$Te+P?PlkQ1+Hlac*6<@PR-eA;A*dA-F?;;0f-9y99T42oNj; zcXxMpha@eP*m+_?=|O|Yp%7+=PHt0S~Pjf z#GUs%ztv&`HuoljJNLW(pE@(%{%eJRDK%vH*gzM}9gz zke%oQ9SI!2e%Qr4*K1eTaMT z0k=^+spPmtMm|5GHHZloezdCOG^eEe1}-|NO1OY#SXQ0y1N;}u3b0>&n^&f-QQg3=%m=VDfTymeKQ>>WfYw=llK)O%ez z8aH=j-}tTF^!sn$zM0G&ccLG4drwzRR!CD>`tkCcvsH?2JUt_oU16m;3K0Dj?GQDZA#G9;Caf}I*h$vseiHQw2^9jg3%2SHF zOeu3zI;1Q^R0R@AKp_|{Jj?XEQR70J@^p(!nc;t2^!;D#5A`oNgjDJ^D?LD$nZiX- zJOLMA8}S+pUs~}4IjE?vj>GfrcCG>|p3l{$JU|5|9=%&YOY-GzO@~Y~pM}eQR;)jQ z*i5YwLdLVAU7}iA2G#E6zW=>LwZB!_WGA;*+X?Y2LK{wB9$nyHQ75 zWITGzl8rWYL1EcxXJJe*pF&6WOF_tsRyErx6(EV$^_j|4&nZ<2X|gKl^Pemgn^)D!S9=@n2ljXH zJpPFjtEz=8|3gsx4>0P72f$Rw=4L2(kP_>9!zSV5PJ}wMIRLSEUC}BIn(g4g8G+v7Ife4vx#cW z%5YswM`%=@&bn&6OxW}$HnwR($s*(~VTDAtuHY{Wbybsw$~3Eh_yoB3HYTorj{dPd z-1x(vz-&)EjT^e4JU-9jO{PAPFVG;O(|nWwaPF_9wR=SePjzxX`Ya1`+a$$8B6!kr zL=`~_-h(Lyjb@$8u%0)@LPA0c`d(&bfZj_ma#vk(NZcxcoHweJcj#Yi@25!@EqEKC z)HG15x0PG#*D3sqaSgi8N!7EWA4%h?iUv6`jd*T#nhXy(WeBA0|A8U@Zr-K z2jp=K4YS67y3$t7L)YE@9_UPm^obg?=;=$v=#zelU}+KapRhe9 zBiCxJqdNp4uQm-$G%}@C^95h-&Pgh~tFjJ`86VYHi!<d17|RVX zjtcIzjq$TID3#uweRYqYUeNHl`LmK!sq%kRndW4^HrWuHvY7l0+` zb2z*=fSOqg0T~Ln&9V{GK~6G3SRG2%)AHC%g=5!@#%Le2$xCz7fw7hTGTI`ckCB+B ztltIxfM@J0AlCO1drY=n|BA0h1XYcdp>vZBzN1@vY?0E>f;B0Lp`>qj+#49{hs5c7 zUCA5!?#=|7^oaW~s9t27n++$g-CI;Nqh!riz(gh19!u07;(Yy*(V$t(dskBxuCBZQ zB?re=$+DR9r4T-z@__f?xj*DQJlhPZ+3G|LL1 zT5}8nCr`f}%Go+}_M;W|_kaLh@?cUzK8t(nb8BS{oqh(9vayM%1p+@X1x4Mi{nn~H z!^iBJ=wholJDITDg{Q#+=rR|Bw%5o9pWq5Z#(|loiRy`5{ z!A`Zhl-L;}^+4kUsHf0H@$zh#I25KsP35$F+X-ls(M{P{Ef+i`+i z>Mq=`;pp-G^sX0&5!KLdXa7hAJ>5v3NImM@yK8C`k={@;OtNq7Hl`UUYip zw9-)x2vCam_087Nax$m!Bu+fKypSXjNT9!%p_A)o(z1&{QqmGUMg(Q@Hzw=Y6aIFZ zD>mzZ=ZI`g3lMj$`92wMAS2#gDsaqy$|4gwFWCVASlrjdKRgcy*s79zrv+GdFWY+S zP9^AuDn?1&CV^U4{ih&tXK#6Z!Dj-M>QpQ%&~+1fdiqL-uK?GiSKTm&ADsM?bUN$V z#7AFWr$tdG)sZm&-__kZIU-YFpBj`b7d^g@SJM$%=bPX9ylTG{91!_Ec2{5#F}^P8 zdk$YoRWvcT|9gwU|H94pIZ;UX3#CyXOS#vP@b~D~6$oDb?T{fSM~bpoA7W`R*>g(0 zy3?J=r{A+&{NYVaL$jDd_E3!yIk$g5LoO&!+F~BmII`$U1m4>*sNZkqE=dLQwh1ug zXuPHgl zcoROx5>n%v(UlG#Iv=(LQZKHd{a!mg@miWF*-0!q^uQ3IcmqEJigvdXE*@zhR|B67 zZC04t|4f;~_NTQERmOmam;X+}T*acJV~Ed0Ve~aDceg6WMM4DSSX#CG5DP}u|8B|t z&r94?eqKY$YWOu$-N`JQRK-?vNhianTTnb3C2~lWi1GeVd}DXYZXiBjYWl@h`^H_Z5xQY+Nj{{)Z8| z%7M&K1w8Y_;O&z=kAQv}CJg-NB%UWSs}&=+a_Tr9i*>dkFCOQf^xFX<0LQ_tpX_1w zhDqo7daqoo+q{%FX4S%6={qGkKuyP3^07c|1YA)oEzw!l;2RsY# z{$BK(`1hGlvyv(`sYW!axbBuNf{Y6RFi*UOT~zzVUec>L-1?^4M_$E6k7wu z%dA*fSc$Wn9gg7rtMfy&`pB6hRU$hwx5g2^L#nrc3T}eI^b$Dd#B(tGog&pXn>D|G zoz;=0KrI#0R-NC!jr5X?ssF~7KZzLth0j+}89ei&hR3AT{EOABXGc~rUVIQRh8f4%>~ zW<;IV|L83d`Wk1hi`42pxhB%|JLE}$O0RLqQ`xmWlz%qkbR>LfahdMa9w6ILiIH?3 zYf57Y^zsJ0_#$COez#-p;TP?`HN;be`RvuAihM5$i-9?M!+XV-9NU?X@b*~QB;>Y%V z`lCqN!JFgVv5Dn#1WRx(oRxByqZZO^HfkN(6W=UWit^vU&>v_zLA6=bpj6(@Y%4+b z&%s1}Ww`>V&Wahbe1GJ?&!4kVr!Cj%FC3DVA-`P8%lRr?v`@0Iaf?=G=4nW{z1&yL zt&oDVBO8HKm;<1ovxDM*+%xz+`e9&^`!1VDw!SFRmh0?UWl>+NzbY-AViQ9gKB^>wBuHk!`?$2p>^W>TzxhT*%xcK}noVzL9 z>$pqH$h<{V3-`O#17ykw(MF3^RTqzu3@aj!N&j;1etDpPsGC2WMN7=6fkFCFofU-x zeK_z{R!vehiBoBYgRX}4f)Hi0KJSBnY#iWqu{Hlwoay0&pQva+y$Re$N-0yB_8SjO_czD)jL_kUt)KmYQdZZkum5*;*I%W!>FOL-Kjim) zMPCvmTr25dhY@y?MATK(NDsg0ME!K7C=16L z{Vj<7>Xq^9399!ij&tApJE}D13>M%~iYx5L=x`XmVL87&**+#+neBRMPL86)3jYM( zu@ZCIPzk?%UD8~|3OVSf$Bg!|^RcBcn{770)HbD{_d^Hf8mn@J$M@ zocRHAl${5*$$3FU$QymuqBR4!_Jc~*XuTwJNhjZ!x?MJr=~5yM60IhE^MqwzCAZCp zi*J(@r)o?Q;rHd8rG&?l=k)M%{u!s>KZzjqoA&=~%>O?_#kwi5g!)el+j200C45~R ztME5Muk#?fl6r7>-$Bo3-~d(mKDJfAyT7kr=3-lZ>t4uQs+|Fm+v{~6NC>J9$k63Wx7yG7cFS9gH?HzxS~-d>@Yk`#60VQ9jZ3Lid}n3@%`J zn6q*zf~-EO^0`?sy05wgihg&g@dRWklY^$r{=4D_*w+S&*&n&1vD$2K0aSATg@f+D z$B6$@UnD<82B;%*JLHgu517E#cDd{3vhF^N;RO_WF%%Hx|J{X*z`S1H6JMH;oXiRj z;15p-u!)GMm}lt30QcL^pQPPpCd6@1P%xwc)7%@HH&TI@Z}Wvf41)q0g%ZK@m0?NJ zK%^%T5zaa^$mF=wZ0qK;DOgI0%TUc+u~u!$$$9^cit-Hu$0YaxQ0J9FsqKV`BF!o& zS=g%Nf7dZ`%-}3~4e$fFR|Uv}57wIbgm3D68A_Jcw9y*~NOn8VnkJN$m95@(ibA=a z9z4yb_pxfLGC>I9P=_74mUWh6fy@aVmTuavK|jKv>M$Nw)i##BV)J8W=S zAPnJ#?9PV%iVu|fi3{|&jYf6`=m22okZs`gx8Le>H8PNrox7s&lKxFlx^RIk$sf~W zl6x$dwp}A9By+Uu6VU7{=WX<*b@!Z$k3ptCij+k#>gcx~67RIK7o%Ae!( zff>kz5vflLKwrv#sni5zj)vQQFJe9i&j}BY%ldLk7`&aMrJ)&QmvoQOvFnfll>eVI zzGqhOcX$8W6FIYbP)+#pC#Q-J5I|;ljHB1#{Xq9SDWrc# z?sZw<=Uvm=ntv9G0&YS;WIuh4O-uKg@6Xdg+%1^+V~+d3{AW}v|B4D5ty-&0o8=Dq z^9!^BUzjRyr#nbmULJklcZwHafC31Db@Au|sVZyEphQ3h=S9l8>v|POq zcpQCxbntl<9!PsBbL{iB!C=_~b>@IV*~=&YP5fo?P z|LDAmt^#j^hb}nR*a)Zcb60#m9*mM}V?G_mxecX~>@bTre2J4CJS zMk#I^R@H82u_!BS%OESh#;Bxj*Qmsrn*B!3X&KhX{P)|wnYJ#_%U8d_mz%!@C{r#x zlPXjmVq*=R#(y=)WL}_+Y_&PIM|wdsR6$>^Wy>z{LEWxW*JMoW!@91%aSfKFYL>U} zSEbfVe=kfyr14m9_2BB5dZiZYlG|aN0?DGxPCh?=HSHCj0^rABJk$iL%<*L25>8G| zg>dl_sGKk+82+hl0qY5Gdz&9;gZ(K-p0@sRc5FG{$riPzr>^KociX?G#Xz3MCe|m$!3Luekc@Pmaly9dE6X7pIwfQ1OpTy z2A~SfO*3w0kNz(_VLvg%{MP-(QfX!JT#b9kis!aFXaP1Q-FDY+V1I1@alLEjjdN^K zB`gXWA;Cjga|M@Q+}`nc);Y=6zUd1YN%`>ZZaDMbysQrlHLgnvsdqOo@;zyZ=j456 z9YedhfeRU3f$}O)?QNSNv-kV5^|Bx?4x7Lef<&)I@y_`<21as8iBu}z2I-aJTTpg7 zOGg{-Da3l&wOos65E001Nc({ESx{U!qRt<2Zt^r2v9(&V>}d6cKrxnliMk0>_qCL9 zXoQ3P{YcrT)a(5Z`4#ArBufj)s$;Z_w4-Osd4H}(8SBZeawQ*PXgDgfy%3nh$KkbQ z>p%Nx5ipgK!tI@yVMc$`91ple2}m9c61?t++l^v?TylyV$09I)4&wvk6c;4 z9?x0~2&nW#+;;5fscC6Uo5utcfLfr&ZFtj7`gZuORSFBB|91|bzdrRt5X6r5EfEF1 zB$~$a5u7$TR~O{sZR$SaguuqP4o+|he-iTFaa%5z_-mxMxWOc_TsfI2D7wCdZ0HLr zQYXr1+%*6x3fyf2t{sACQ6CBGTsFtcr#dt@RyJ|M4JCkRXr=00`rPX7pqY7%->%HW zbf<((GcLmKqls<%QOzUuCPtZkI<9mT9!Pl=BeOo#*p*$aXO*?@ zW)axc{xf?=qYq*R+Cuv=+)K+t}1AI)rJ9aSI~NqT)Ajvp{J&zq8M=Q z2|n5teQJC+S8_aa7CF<&4%sM47It`1w}&~)h@nARQdW`q&=uJ;WnjJwo?A^KVEx&o z-k_TMLaJfUAlrD<);{u9-7(qQk|!on%UD z)!W&u@eX#qjA42?e>N)w^qPy0@k3p#7j!tgS@g_Q6Dy9rn{B6tb!KKuvV`#TC=PgR z;*9y+6Ci6WkA97WYFlQse*5L=GtDkU7V`$XO(Z10kj47UZIx*=}1Qs;Iv!2 zJ>uDY8qE(r_!TdhkVE9TNG}MkL)g(+&_@KRVI%AM3#*uz-|B0mnA`j4N_8@oXjlUH?-klgN44wVldKh;BZdeuciT;80M7w9L`s{Om z;TaAR@L0$4-*sLt>RM-$uJB1m+ezw-jC!!FSN2Tl7?=(58<-NIC`4drP`xXZ8OYBg zA*^$Cua)!S73JUc-0u2lwO4RArllsi*)-yB_TaEVAW#osT6J5FL zB6oD$Gloywc9T@@FMS-EcNNop+Ke5-GNzfwx)>|~g*P!<24PtA0fW_$j(~ zsW+OGK5)*U7J~#KVV{86Y!aUh_2 z4f?V}LcK!UIm!d43#-A?jn6=5Ec5l8$N;{nezkx zAJ!BAXH7uYVVa894?d$m{BfSXB>oAjL92Sd(A&O}_VT+gvClA~svL;DQ=b#rGG0Dt zlwjqwxt-lVL0>v%F$ap)ZNE@)GAybQUnk0&{R#l&Y?bE~z>r%8&+@FEN#~VCMRWnn z*nj{;uIEanKL$J@+b{iTf$G#_pPC=DfAM~L?Z5Q6zq2sRQ8=&jbAt)KfD39P<_5m@ zJ{}A3K=1w2`6Qs)lSUBQw%Gmi0^Q(&B?$6k-)1 z#srb#lt(_NYfXpSIf+?f{RGn-_kY@% zojC9Ld9`wkH%NpEJk(CwY#%K^QWP-W$c0LjvEsYl^V(!>Dkp}u8&AK3YX!CYx5YH& zF|sJeSwL3wny`AzzZmd4P;L%Ho|RmTXjCLU1XelI$*vFi=S!dgbL)QeE1-Jl0Y36M z>C!*mX!7a59l?s-*j8(Pf2tbZvVLW~q-JSsb`484ah0IOtQ)ev>yr~Zux!-EE9=u<#JAg%T`P6^DF>FHjn1QEZx6{}&B{fxUth!5<*zKc~{ysysm!0o$%z4g*IowU3( zcbKcioQ%T0YlaWrcj(-j`tW(8waH1>c}PY&m+c!{?MBR%=F`(KQ1Liy-I@2(I~^nA zeDJv}t4qnaF5Ad181RY2in%iLg6{5ZR_nfNuABtaSGw^fJ2zKU)52Ph&1TjBWs;hU zODnC^`O~Z=t&FlVn96u-2M=C$8|jzV0e@~&hc`Wl*V9ZEI(tuCtI6N-ajje*373%m z#^8B~Y_OZr1#C^ZlhD^~$7%OEj4Bue-M7b*@ZN7pcN=S6+NGwGxEbq5ML%vSef_St z_OtUX+0Hg|Ws*H-{apI#&WfqE*Loc$sTEM;_`?`Y&pSwF+tvV-Tf2D`4T+6r|i6Ez} zX<>J*7ABCdRwk=AdP2z>Kjo3Om@G6&$w{|ZGc^2}^J z2?5~elN=7!GcGhw*1T$;&TXm9fu2ov$W1`qtjVZ4+Xsdwxo+z`OuMY?bHyZs2>@%F z;G7Lhg1z5Rd?qxEn!j3A`)ydltbG2~q5fH(WP90=35Ecdbaa9qG(6Hu8T&hUXyPTc z%+D!=89Z?Sec0GlmWj5{_Z!7HD!Y=PyQ*Z@dzlPv~p+K)r1-pW=i5; ze9p=4btnd~ZX@H33k)+^kxq~1VvCLO?%`$6Y{I4Lu5;|GuKalRYCrdEr?JBRdt^Uh zS$O&mxF*;hxN|q{kc{aVwVa2VvW+evNF{Z-!;^Fx>ML6 zqRq?O+8?RxQ-1*r#Z!kCl`o`-?iaOOynO4k>G!kkWxu?h&xN+IfP>Na@no$Jkuy(- z#tz@)%+Hgx8J;az(SyxetIo<-TaAtFwg(3#zppJe;u<3&U!*P1E-dwU38M8`55LQzzB43-sxF1FkHQ}3Yx+EDC>a=J z_*^ZhIzfD^H;_*!Eu}y!rt$JS{KpFxDwa4{h%C6Q-UysQ%AR)|KU_}+CghRvZiwXA zU#=+^OR}SjEA)FUt>By5U`I?L$S*M>Evd1hu94}8EO;bw_)XbfXrN$BhmEc}Dvaa26 zPpE-I)pPvT2R~;hs`Qx81cuJ4#|oI^D0C-Mp6i(a4leG{AEN>!kYbRY1>|_&n4#Cgi6JVd)Rd+HM1+CQp||&VKU`+kDXSaMxxHWRxAn-^{CMe4-8KBQ*%K{n{4wm#jOQc6!v(M_um_ zcHjouP8QeJ@~Re1FIZ)Jt8DNbo`I*z3XhhWVW<9tk`JyIP)t0(ALZ@8KByw3<3j=q z%v=ia&b=xd0feRIMZIdK9M!2`Q&$=ibp4j@*b&V5;xDNS zg-&y?pj$Owu3XgAK_f7pS)Ks#s-|k?je=*aq&171x)LFEG|^Jkm=emu3SgNT48cst zPaSu<9df?+%;SrC(PD*g7cWVSmy6jKcEq^Wv4=8I={en$H#=qPDy~2^Z*O))c`N>w zx6ac%R_SLd?T?Wf*%8r^u`4e^k2Jo693~}0j+K5b3?4UvzouMw@REjJVjOWZxq2LU zy4nffQ%LGVTkd^U)OKixf{WE(u<|#T*MXb6N8_5ihm-uR5=#B88cO%&w!1KvRSo!> zOYu{x74)v_kg|q>q4KBpR}ZcCLdWq_Wa`~`?Te`bX^+C*=7{zxGhfc;6W_3}RwYiVZSdsBTj@7sYUxMubTbzKxa2yGwf^Jt#2VbyeXr*L85%KHj>s>RrVG z@$~EVzFxJiwtR$-kgcEfTE$yavFV)(v?381*U0O}SWJA~U7q4(vNYtrg1l{SK*XsP zo^Z%{$b1Ikk(^=uDyP`eDmvSK<($<`(u>hfp;V_r$OE=TY)(H z?T@&PtA@$gYNl~I_N_{=F1lRezm`ezS++c_X9Y5ajw*S*!d>5h)Etj}w7GazTSd$9 z!N10SVHlO~(QMfM-mw1g-aEMfl&pW;UEQK> zt?~iJGRTNiJ{`OEFkhBIwM==!?6K5bcEP~(dmy_?T|ta{==#WSZ*BHw%x^BUSvFB7 zy4BiQZsSJZT6c;4$Wa#Y4JJRr^DHCtAH;9oBf`HhX%Qs~M0!;+YS`;B5H6UMPPd-y za43X@uhTb;RA{${rh_|ya{voEoDZOtyNkl-6!+d0&5X=0r*>b{nr^0BZ}O%mTrK~u4jSuc7()3vGLS zG96&^ zlxdR5clFCI@-CjJc~%%MLDnUHcFif@8di&E+tm~-lezj~pIgfM-kIlJnyCp_ z1xozcR)24 z34y=wYQ8kqx*bYBW_KgsO+j_?a|Om<^S%!(HT?Cc*8F(D+RunmN@ zON7Yly;Yizjw{9G-Lc}b>bnoB8k-u?C5}u>>m3%e%NDej&3K*SX;u zxhfmfcZwCEn9d`tAB(O&(xOm z3Q!!>l%>{WP!TwNjN-+!u|~^MBsI!v`hML8jOcYlCkY|9S<=b1N z3uz{V;Ln8|1bvuC@_&V7`4i=E!tz#ix1@On_;6#A&g@YKRlI2n z&~BKn@nv?;M9C9!-Ctc}j&eI@a!Y?8t5h3$aEiHp|5nl^XvlsvaKH-=Y^WqtP>~mz$ z(CN11P2A6Vv0jy+-iLviU~GY(N)s96`fRc3zo*p1ycvpUcn@Zu69+wSX3;r)n*)*r z`Y?)d-EstICZX3h#e(e~2jTLkZ;i*=&yUqR zN1PtYDgpz6t-B@IYPvM7SUb-KuF{GA;8Lg+JVPHAhJ zYj(>h1aTGIE0>dHj*s66j|e`XUMt}SCRosdj&<8QQXRwyX*M5koZH9TUAqLl@_xyc zS;U%U*OQkz+DB8F&oi@0=~wwd1<4$4IF_!_iqGPF}NMjTs!lns7$YL zbBtF%*~BEjL5I1S!g_wA8gJh&K~IF-ImmVh>FmoPi7itmXx7%pPwqZSdemi)ClG%);&)Fy($=i{0J0#=F0Oect0q!$sRUJ5YGVKA_3h^u6-(zvjHS?MapKk2;O z7qrRS%3WjY=w{!`o~9IlRf=(7Lg{wYt8fMrWk?JX{LsZ{w@33jFq(ccq@Xq)HHr_I zUPX_Yq)B({1J_zs(y%_>>}2SiykyXx{%WQrhuhhR46>X-+D$DPQWq;1@kGrcRl>@% z>3u~b_|t^;E`ifoW}%dljY1@`HVvaNd1rXQ7I7LiOA#ecCs(LJO|kZ!+rG$OXc^h7 z?A8&_%rDDV3aIS#YuZ?%zZ)%OT^wI+Xb&`vQr^}{em~^DNQu%6zLxAx5q}Pr57wLe z!u*buoWWXukD=7XKIb!Jr5e;KQbU8oDKw|pXxP$&I=CA+!_5KCoGvFHeqP)iF z)O98YF)*uu0c{Tu*Sg(DQqHFIa^GK-;TVglXS3@>mP;_d|B*6%!C)kld+>a>&(6Ur z(1PP@l2N+G>FZFwCOfFCL`0-pTd*~K5#>xeJBBV}#(4@mzSu_VLfGlc-shC01fr^C zlNc)p5_#WlM!z~vA>!8csh$2YemeJmnhGLaYLjKY1f|bv4Zg(c|AiI5e737Gt?3?h z$W9W~f^%~?Kj9649?RDxyY#sUOe9HsjeZZz7sffx+uhY-TIrYU{eHmR>s64m0QP}Z znB=}Os@J;e2)EeSLIbb1%V+kMIaMUur8DddPiDPirnh2C>-bTIXErCk3QPE+_EUAS zOZEu{XJ=Nh4ns2Io9h&+{tD>JQ2oIhvEaJJJ;gScv6epnzz~z)2@A*gR>6|8jG+DK zJO$?JiV@h2I%R*vQ!(|+|?NI`1;!kst0$>~`>dVeA?EFA*^~PuIQ0DJQhVH;L8chWlpF8~Cq7%1eHElW870?B(4;3;v3J)S*cMheI1D~7#zOU$yPdMS z_gd}{t+N}U_j!m)pWM4BIu?~=l?uV~2LGTpujsT|kUJkWPEAxC8bZh&`_+s-NXAdH zF_p=~Zid%Bos~y$phe{!YF6f zY0fTdQszYtpGeyS9DxsPrs2_S*N<&woW@_?H*KJ1xg82IW+s-rm@Uwrjln88ypwPl ztVGtMn{&Pwx`nZZY@S}X7QQFlxp=+v+X|Ea!=(um@kBhcn+UIj2gfC4yZCgbIZ>mV zxD4jwTjk$5lEX>UQBS^EzI@b)Tz_^;@lND5t(Y^%buf0n%O39_@o>i8CL(ctl%MgJ z(TsYj?2zvP4?g;{YcoIHZ?o05GHICg-Nso{xF#ym_?y|D$e9f+a^}3(=jW?s z6lY32C7nw$oUUhJayVBMb>h5LM`y!|?hludj-mx6ck&;?X+9!>zVandJDY(Ids^); z%oLKa`(QgqXL{B5@r|YjO(bo3&+<`Z!<+Q3y0X`wO@uS3Rqxy$sp(ca^RgcBh*j@> z_)*}#)3wLP@II(0)|htBOT)KL+rIWo!Hu567Dt8q-du6G!oWyav~WDGZC-S&TQhd$ zRySXER6~PzbbS}%L{<_@T5ev?(+6&-!d0Og%#3cEkW$ffEJ50bXQgC{+$&%M^3@=y zm}J;-FrI54Mfad_So&PAjI8_NIwXp(kWrBpE#b86X2pH?dyBsHrzLh_x39K^DbkG;y^(; zyFE$DFWWpZ%DEB*;U%`5uj0=!DSGWXa$~Y?0+XfZdBLVmXHZ7)1?$|t@mC&l8}=v6 zQf~t=Ge30pEYWvlePhGqRQE&+b{NM0VW>&|k*2Tqmr-6=l4+1pxFQ#Ist5{a*465=Xi!G=yK(KenyH6MvD?bS`yI&*R{J zAaB}UiqxJCx?s!b?mBH8^Y>~zs6htBrZ`AYZTo!P!9L*>*11}k+b4iYWq-R^DD!** zA&jx`j_nHw)$n$Az{|Rs*^%djP;>>*PHW`d94YVYF&qaVFnJJEuDVN%-bgK=AftVb zSQzhME($PL`kEQxK2cYAl3ZLaXc)F&;-PiOpps30ohhku1_J64 zH6_jhC7h)I+Bb$Ab>5kBd}*(u*kk$Kujo@R7Nm)-L=)a@q0Ip}IxJ9JRsRKBA$4d` z_<@&IaE)*Oc{!nA(uaDEtNdueBC^D6w|7~bSO{gZM{>uF8EUA-el8+HH?yfWUrG+H z)U~LcpB6`x`T1%0ckKwsjtgac$fAGs?%Dllj%d{{G}~1lQNleP;do1Z`*t}&hVDa` zMM(HxI5DXx|HuvhAuptO_D1#eazBT?YU<5be)99v#cCP4x3k#do!lkujK6C8Sm{t` z6`J1nPIK8$DBu1rcimRB=~a2N|Gp3QG};x9rpY7MH=^WX&^SDAJ)+op057)Us8TCj zH#Ryu1u>QFdTKLqE(39&t15u@t9X|(DR{1oy^WU4Z(d_Ja2NCy5&0sF-NlHBwHCO+ zxm-LWzG!T=@pW6%=LYXRPj~+uJ+2?d>%i}PXPY(2V2&i7oWe=?_@dT$4qJ<;21${!=z(UFjrFegCeWLuef~6QrOyB#e%Ci*v zcJwRu?feT&kpX`9M$yxk#qh88E%cl4Ffn_ z@`qU(b@atXHI}mIBa+q`?E93#Q!*!JW4a+i1XoUCp;_857s`dkGkJrv7i4Ox?-SE0OJM-Ht6rJZEtTi_Lhxv7`-^mtJJV0 z$rwDVdqTG}oADgG@qiX;IIfDhoCvZ9v&I+yRJUd!K6~uZXlubrR>at**Qn5DhTD!1 zeE0`2kEq7yL#Ir~I;iIpab(SqzNP}3wG`zfkpk9^jC$-!aW$D|i$SgUC{3n-FZkXd znb*9jiEiH)$IxmkW_9~@bJ%BQ53AV%f7R04jBVx0;a9^876t|qfZ-^n?WNAH^Y~18 zAr(%ll$?I=RBXOv;T>nBo}+5D9my5}j_rt;eK6G|y51csC2?6Y(lCsC`mB zNx#4}CBV|CBvhL|DWX|4WE(Sncm&&3m$^!nFLUVkzFOty+JCFG7MN*{jg{dbh$%oI zid3dwjXo1{bs$tpxxn{L*qL-`zKTovXx^3v0!vvQ4WS5Jx7VdG4@erjC|gaj z%~q1K2Qw|#1r@9gIW#-OkmwwKW&*^;#5f3EbgcJgQFcSfhm2FNu7-nOl+pS}dMB}N z;@3eWd5pbi7GGZYqfN4VzO$JoLL7frBT42fRi@@UTG@{(67x9}GC;<7ko6t?so&EV zJ}dGEg&y0r0?faIRxnVT$=5l|4A3Tje}4l0hQ&Z(V1uh57^{gIGp}n~i(Sfv(=I!Tcp9 zuk!V6VvRhn?uASRyWNDkeVoaDGkOHz*Xb1FvOgUG6`=sF4IHK%V2- zu95>OpWpY{{BD5|r7>zUOgX9aeIsOLR}9VDOe6l&*n*GDuLdKaDY@}1Vp@eVQ}QIN zRTFU{8x#5YY^j+9y&e-^G_yw=y`L4qwTRhSJRK0wIXirzx63lB zU#>0xD;Ay6#$fio%2Vj(aEP!Yj#6SXvy$AX_2#L87>xk71tn$Ggpiwcmv? zE3TShX#fB`V0NupXz=EAet3Y!qb@V#!>MUrjrT$0KwC#_CvLfSm&}Scem^dIKWcsU zfHoJy(7pJGOPiq3r@Kqby=k8qh3JEK&^arF!wk=py}H^J^UNVHb|d_gmi9#Plf}&_ z{3l{T2>6sw4DU&=ji$^SRVaMcJ$n)5BCG2r2s)4 zAZLZXZkRCf_y!6>dJ!AZ_PXL_Sf-JXL+opTTsJX{w1-u!Tg!vgl*rQ8Vxm@x^x+9@ zjDdFoFGe|>Wwa#`--QVAVsg*(ey|vlooA@AkDBlA}n?n=i8YtG8*qPG&F2 zRxMP469M}eP@F%&YJnApWs`f-^3)heio8Jctlh`&@^9siH||+E!AD1Vps1ma#3;5h zyS~x?kFmE7iu2pLbrUQ|f(A=)cLD?m?i!$h#@!)kaM$1(+?(L;?lc-axVyW%-8Xyh z-#+Jj=iaJY)qiv)PzkGf=bCHGXN+~O)M)m`1RcV@U#)2xv}<90;=yv9udVv!VJ#TK z6fleAnY&7R8VTI^skN+RkV8rAL7^tF#xCmRwszNKE-Tc7w-S5C*CXoE=}c{NNeP~b z()5bnE^XfHQW^eOWEMF!R9I1W!r%fD%kTv6T!qF16D*4#?)^V*dbGlPU#^rb*vGc$ zc{RlEug!>SmaJQYXeu8+@PS$##x8%|#cUt}Dwms!ccIFIt7=8{@!*zUu(XcFLScSO z`_SD|!{L&;w3f4^9}=WDJcBN~S9V$%4vTFKpHVZj=}_Ri{LX{pglZWU!i1A08OhLC zx!PT!U3Zg~b=w}oUYWnR&X2B8cWISB{loDM#b5h&9mR=D@3nL+k36U)l=7WdT-kcv zHy-a$6!X`@td;1>SCLl6+LgvemvyhB@84CbQbl~sn~`*Rc&(33#_c+o>QSj}kuwj> zo#++MqxVT!Jf1?T`MXm0fYM(#y3tK32nDa78z24qaqJKGxi4c#^?sK9gsDK{O;mgP zDK`ndV>zUxl;?T6Y&UGl3GDZVDF%Pr$hPsiTIqJ)LRjY_I(r0W{qfE&V5l!v>WAPb zD0pgqWEIU{?z9L?H$K!{PE8kId}O{x&pPoOJxr#I_bOPnYpLKD;#&v^S6Yr zk@k)>_)gq1YYq_Eqxp^RwYK7SO|`8qdL47s;`D|WP|IMKsr1o>R^7q@WiZm6-qwG3 zY?FoehiutAL>+?=8HF@_ZU$4As9+HC2s%Ur{j((~R+4dky`UQXu=!e3^sW<5z~9fM zyo27xe9IQwWJVta0z-#ri*DID_nu`mLKWR;#5)^@aVS!wMjn_gxz=b&+0*HT6_^pm z)vL8IV^|d{S0lXM5g|-^&|@|i@_K0eoJ9eS8qQs21Kqi;h%Eyhg9Tc}@=aX5wF{EC z5jFWZgL0KWFSxqu8Y*FRYHVw5to9~v)%lH>4&ASe3=~%rLgN{8&g(`EF-czkpD*82#90`u6Fe_^YYEZvu`r^Tvpg_nJ zf|dkaelZt5`^~w36-an_y3Xgfy7!g53O_zCVKe?_WZ$@i1?huXvoCc=~hhlM@WiUI@f>`|1oD-xI}mT+ji zcSBC-2XObyx0rrY9=@)po?D$yT~FF|>ujF$I9+(wcpyT)66cEbw!LVV@;Sf!&H!X& zGLpMw!jJ){x44C-ylpof<>i&RYbj@shYQ~FPlWRJf(uKHXmNxbzfyBRCy12Or7ILR zqdGao9>65Z@6J!J&aB_=&f)5V=X_gVbOQ)FFH^SnayXao?4mxPNfHI}_6H!C{oX$T{jQhpxO~tZX{hE3VkG-?WtPH5;)j)mL|TLn_q7^9%gQv$ z@ntsz!aqG7=9W9We)syo(U7FGXk`L} zF1~q(&h>0>_smn<_sZb-O$?R8Ao*N1+(_|Th?gM8^Joai-zn4UYJUxyv58zvp0y7& zz-553NdnK?3-9RMM%>h|=2Tg_LBaQ*ZHGwu6BxrZUq73Ego5W3(F#uPwHNKLQJ>3I z-SZ0N+>Yl`q(yaB_z4Y(UtV73T2)^HZ)L^OpH{n+t=|_Zh&=rEq&&78ob}6Zw28Wj zKo2S*uQz2@8u_kzU}W=|)@FIr=@6Tj(d$KoU1C|)aCg42 z_Ok_ni|);HGl+=hA&1`^bjD-+Jk-z44F-D6?0ECN0x5L4QteiT-KLa!)8WK6*uVof z0Ud5o88g9`H$jqtM$@U~&6M0uM+f5G8oVd6joE4#H)*XG z{ieFo6A2?4X?lK?Jnbu9u{eaA6Qk|p`Oi09i7`~NtB=+cW!Uk3(~-I>F&6W=I7~hD zt($J%&&3CMt9$KaqfSzQ>g5AhCJAFY3W?SAS}5{H2Lw51N$5mm`U_%P%ZeLV|H_!_ z_SFbBQYbDqJpeJ}O2%xN zzMBF@0EwRYg_5|<)bW|Kvi)?@2S%ORJGP#x{rh-vD`Z>XVp z9$3A{eOzjAU@SToIOo!k#se*Wt)MynIEB5Eg>R~3J0$+uo?vVg1*An8e}dxe_XmfU z@i=ygbNszVG0*K!lKptjP)t!7bT;Vr&F?%hqx+7Xo{=yb*>71y>36zmG07QUI`hhE z+4_IfbN$$72654~KYPlZ@5oS+rx+)2|3KZ-u;7o=fzW3NowPqcBi)((VB`8Qv#Y|d zZG`!MPHA2tQMkj@ol?}F-%tpXp>}9xHZwm2u=LOXQM<3Gs>RDaTwKo_t%Yjx!(GVf0u)8K7sJ$ zLaU*z?Bs5vKA9uJoaN{=X}v6Y2&lTA)jHK(Niv^M*sb4}$XNPOZ&XmLxwc>Wnd<}T z=ws_NE{NWZeI!Tn{Z5atfqLJ_I7#nu$y}g;XoqEdkOi1}9c@jNtgxOQ{>NQ5)t^R* zUDa=>!i4?_5gTjr}h(~3g7-QSHygq->Z zYa|VxR%dwK$x^kje9Zua!)TA*4SS%$-%mtfzSjlZhbyHoV^^~$iD=&dBxH+P7r8sU zV&@|-W6ud)-p!MaiuYaVkhgO(zcH}8*=9V>8>UEpvc=5e>*?dyx)aL^E?A#P9L$w5 zK!O||2$W{M_H5pYE77tSmcU`%A!;H2QB(Tyh@eEMd}+XTQLI0nF+ThtrVp4vHXGh> z?eX60rq^$#Ee^xL^M4H*NWqySm3qfjhOQjP#dd#rjudR&pPB)PM__w}Vdk zdJrKHk4KEXvBuiendP~1(II7X_Gu|EzkDzq<}R+TCp|s?vzEm4W!PuqeH2QU6i&ky zoW;}@lV*oIx+}irJ6Xf=&N;dCPcqhE3=cEZ*FcvY9dk_o14Pb6KhBi>LG+_SObt&i`1<8K?33|%I+wk; zb?e~W2v+1O(!uAK<*TUsh3t9(%XzZuZ6?;K{_PWfEK#j$M*f)(8BT{Bu@BE*kKFqM zi8&q&XuM2W@HxjZRNdHs8UOONYg)tKIfhnSN%}u?j4vLWNrJ!Qn6x!&gTFM>yPS(V z!z6f5=Dceg5*dnN|C(nc9-8>^BT&zw?(6w?$#nICZE2vf{T6k> z-2pv#UI^vJNZR50%f;90SnYK`O&}LJ6fW~^H!U^EtD@5=(>Cpw8o3i+pCq#0xRZ+e z{W0rt&fn^@+Sv?Gk$-0bEYOCANwi;McANkN88~n9appxYUp=-9l)q?;aiwg@%=ZA( zA~RKV26#3s>_K{$BKFmvZP)S`1v2UA#=iK65q0X6AP!SI7~#r4PTdj(7uf0b5t)I(F^KfY?EDzduPETmuNAZv&ifj#snZgGW&+s#w z@P~OVU6;1Vb2qPIhhr8s9u{QN61rbP2d@;u6UuI7oSu(FcPr%fv=>FfPm<4sn~9fO zG9%Z*-^%st1!`JibUf}?XN=yq&0x~GRN;%R<3}Cvoo=klKCF~`58QGB`jW-6{r8c; z9rTgB4wyh1gS~OCNPm?QEkbI9FJcY^8f|6NUcq)>w!cvfaNb; z*F!yy)g#9NJ-uNLSnWR^RuxE#JG(&bcilOno6zmg@AHq&m*d>2ewg1+1|7wdD~!zVs}7LGe>Dwf@0?g5CWtoHC7jXrsJ7wn|2fhwV8g)8P(00L zkf%3em@m@)YE6xsy^R>@A{27Sjiq_$Uq|Co71Gz%4kTCZjX4)sE6^?+}`ukE^M(f;PBa{DV2ev2Qq z*3BP`^5PXP8w)raIU-8quUsSUm0tM{cji^@qdG?f8_mqByH}wrB~NmXWV}}3(HU}( zXgNJn;kKBW;r^Cz+O?pxc(rKef;k^lW zBRaT+G*51M@Ibpo$pM)oE7Q2=KQc#jhuqfd)zwV$7)r2se+)YqXk;_HBpko29@4$H z^?h`WC1=vHM}*l{AAm3xV+I?cMuYwR@_)L-EcWa9ID9ho%P=z+1P6*KwmVc{6=)4agAm0OYZPaQ(&i!ffHXJ41z8^!ak7>8@AD0*a$BB@k)^rZFQ#&Z(VxM0Y+1y5PO%@b&&M*Dw)z$ttz0p_5Nanz!I`?HQdp_{gH z)tzi6S%z~Ps!j5|I*|klV|m^ydCzqNy~JMQ!gSzwY4Ol+_(N^&!I4U>=J2tMn>cbk zti^XqNyApVK4qkXt|0OC)&zNI>@Z2ps7u35$6&WRAV%X;CVkwc5b~sW#4Bp<(&jg3fbD9jthCAUZ_&6!Hjf96#BIJ|Z^vg> zCgMOf3Jda$ZN>3IH=Q%Fn^QVYQ;K!2ang@kkMLrI^iBgQLoYbX-|Sz)Z~~Tj$!ug( z4#@wk>gJc-PY#Fj5%t4Bhwr2|Eswe&!R96Xs>AZ?oiR*C5`FAAKV@jqizpX1QK5gVm zta}kCxS|>O-K}Pxmb~E?aR@Yt$cP8daC=XZh!73uL&m@G#Z1sH`_+MxDk{0n*u~N< zQRv+ya5m9!Yh#t!35f!_z9P39Z`s6?$+E`C!y+^T*Zy45nZ0b$=}IG6ocxGCZ=zGs zE0Ii4;)&dI!eu_}Earm*UZhiMc7I&}Ax!c*)-#I#LO$>AycKh|?YTi(I`egILi2Uz zALJNTW!-FbGRv{agP&5*Y7rC-pm@MYQkIHr+}YL6GH zGoV~T%A?dxBLJNUJmHk;GO=>>XRvX7Bx-EkEj`~z&|SPpohe2H;j_XSv3t6npPjQh zb^G|(Zo=pt2sE|GNP+(;mV?@-aKHYZ-0wQf4@Zmh;4R%dP34@?`XRI{FlS41eiJ(}JPe$IQX){QK5_t&Z6%KPaUOLg+(L?XEJ{{DE&MG9PT*N4bDsvreamUR3lvfRR&)N6 zO#i&rB}qvAsNETIM9&s-^;=VfH0#M%7BCiA5W*_e)Ep^O+$e(0ke6qhuZi}>O)7!& zYr#gou2(FQ&CvK0&C99CxzbkOrB?&9=}209{ylz%GlUMZIenXGkzrYp~7D zT-37ZjH%xD$|ykt@va?&^)|)5&(=x?36l6Y^TL+~WH+-FuqxtaawpKIY zv{{RS>T4ofRdeS37)fg{Hp_f0QgzV%Z_G3Q`AH zxQ&wRKG4kYqTR(+k;iF3Ro(444zJice~(i-FSEv>k%XIc@G31oMpn!LU$Fn`UUPEDAAh?8CHTHi(|B(0dHJC6h@-78q55 z)T=cNSMOS^N?P5KNokj8;ku)9TwX#xTOpm@dI?o>!>#z7ORx{r(5TD&|(4y?7$%!bF$g8lMHcE@op4XJ>M##9GJ7_ zXLR!0MY^5~4Hy^`dEH7Zb9y-noPqz4D?38+i&Ya;hBIa!DqD?|M{qy2TWMO7{c=3u z9AJBw(kwDJOoK0w+tthk!;2(w0bI3``|ut{CM+8>;B`fvtN^G#PS&Y#|Ri;TT- zs8T(3IG?1Y)o3Zvgd58Oz&xgU+-BJ_zyuZgoGP1fz@yjD!e`O892K=nt9aBvxCQXV zKFokU%cJtszYC!c+`3$c=w|;<()7*W39^h1O2YZ*e$+LmCvC8Hhqp-jarq4wZ82Yy zOlf29*8&l#D)He#l^D_*JLPZy6Y(xEGL3;JTQn7ISyv0>;NScs9Gr=|a|7sG+D*q*Oj-GEs#m58S z6<(6Z(Ug$k<%>%i!EAl3QRMm0Z+}NxnA~fY+!(2=NMT|XW^mI@Vj_*|@5&_R{g#5{ zPb&4aT0GfCLvC6PXSDB-u(wJ*@c*JytNa9f+cJvS^mf&sZO$zPjtCTlH)Mw4N_Kc@0tGV4lp1ju-9(r45IKdX!d zA@{yNTZqmnGlGH-&LjXV4>gRF$0(j)pxXG8yU8C?F30UG{?-PNla#*kV%1ZcL&mX; zAO}N+5DEm9sQH%|pk(5+ImR{el@ffh6XGbgL)RQWDr-^c)2cQI{p-B@-_W`f}Puees(L!C+yPk+L z2-lu-oX9u8WJv{5+3rvb5cAOmum+Kd?r+D}KYn58Y%rUz`L=j{NEmETLGzA`@Ny2X z4A#?^-Rd$NNJz8Vj%)5|upd>GK+Kx&Sf_JgH7v^P)#1NA#Zm{jZS5BkI%$RU0va)H z(s0~nkDcq`Ts_}nSm`+!Hc=Bg&HZPei|y48Uy<{tybK4pA`jQ6Y?1KP+q%b-dV$(L z&o8pwYHV35d@HNpz!4>P*Y3&e&LR8=Hyh?77>1_h?~d0ZC7P2B=U(6FdOd+7Gro|b zCA$9MbZ&U#?Es2OK&aJt4P=UxKW~5wg5={Op$-XaUM2mN@NgM-sXy?+G1%b_WX_7i=BA; z^pU5d`OhHc`9+YzV! z1sV-bY8fV6TXmgdD~^X%KbCn(1RnQK-}PV^PtFj3J1@dYM(NvV-ijtrXL@<D{te?KgVHF%h9pO=_EgG&;Dx1N0&v0xmi%&R1IEp_7BeoiOn1#R(#_Cw*rz zog@+cqs1%q&%=ox)8WO=QfX0tu9v-7y3?O5J6y46`YW6FXuec}<$Pzb zRJe{C3zqcbMn8Ru&l3Z-66Dk%>=+L3J+HP-RChdy3(Of1K)M&=^0s1b8=Du3HC8aQ zGO13odN8*|$tPry2z`dMKEQRp#-lO{Lfb!11(}|<%o@B&s|uiSnl_Za{nNZ6>8v<% z{r=kwhV-css3k!OYu)jOOmI1H^^QqfwHjws|5Kt)IMJ84fHe7FPPNm@D+|G*21zFZ zGp9`Fn1wbRqhK_&xN3OdQwRuF`wLNrbPN0ZrKjyG)C|pD%GyS0sE+!?W%u21Rc9S% zZzzGjbh`}?3Vw}ESn)N;EJ|wo(FRZxSO8u^MD(-ynK0j9xB%LtoZ-pHNi#;uyJnyu z*MM`#KMOcu`EbI+>Sf890Fj{TAn-RLk=XE#(iHU|{mHjnm?GI-gK+1S7fF7F61`mM zgrBD7DyRi31Vc$rW!Ifyh9~y==M+Yo*(*ZO9D*#S)+Ji&73u{LH<|ablPEVj;EdZe zO|65(^1T`TiiFKY)4YW}UX#xDRXTZ^zgSlx`L16o`K5=}^H>^(cliD?XGrr%%QRqp zFGo3^nHk&RpDci44_A-#E`0pfS&WS(^)I!6u^za6L+1N5k&7gWB&$3HC|t;dp~_#e2yYs}zyL18#h!6QH(Z#KV6b>LCEd=u^C z+-7#&a-(Ich$nEgR4Oaj{;4jMMFnr9-m5b6T=eMiIZzM?XB;)23mv_5y)Zz5{KN_2 z{tFubH?@?c3U(JEI2QNeu~qenvOqp7a=Ei^NdKKZiCJr84jNkK+l)HB_QOr9=+m(! zP=&^ZF#O=a5Tg?b>=2vxwqDB2o=mFD>PF{UT!5So*JS2>1u%bvm^pK}p zY_(}sE)#qV@M3#!x&2jj6yh0MI_>8(Wx>8+fd!|NJJm41w=Vz_4c6JcIACz1aH6O& zpBA)k-9l(?nB53NUw*f@(&H)c*UrPQFGGZ5zBn1_w2fUOwvG?_&4=i-#pUxI6^1)$ zMNOdWC4zHhY^+m80pI~YyTAQ-Lu@!0tfK!mx>r7^v0btr9`r>H@az!Q{DI3G_Lvzy z?KaG<-R+k}G~9WR6`*I?X0{$_5D&IPp{h;?3O6bd7c(Qmkz+r5QH&Ugn&Kb`Mn-n! zx8_&0L5YuHLp1eTO zhyXv`DPFyqd2(tmzM2M$MrO49f|H5-{qb(uO}X?=a;q^KB0RAoKsMO!wEfmxAVJ-n zcOCIJ?;KMx;Kxi`P(HcisB(PJyy;DN+L3sNyzeo5t@3%vezeo2b zg6H}Kx)Vh1!{4C$oU1C`kb&Y!^Aa$`y)Lh1wx(@42)%G|W5ijc#0HPQB42r6PP?dk z2R1O$abg4TWUWOT<|C4cK_P|eVx`e$cp`e|bGR0hH>yCITh?QV8Y{Utk+u-4ClZxQ z=i?KsaYo$J8`PrOfH`=K6e`*DOj@BsxUtwuJwNI+o5hw^kLv)o`quLwjdopE0Wy_7 zM>hhNIHLY;r)}Avev8rsSRrp1S`MqbD>7WQ zhc11({|q+#MLMtx3Wn5`;euoaP{h8BrOPZyL=h!=?!Q3_e~En;zu8!uk*l@;#vCxV zb-@Th!D>k{GE|zUO7$wrlu54AH>KuBRBrC>(V9krDoi zE3k3-BMj;WsJuzstEE_2=XI{|+&blHr*G9a3PYVRpsPc#*r8$ldTM(?$MN&*khj0F z_B$ZEqWr;pfXKk#@q=buA}Rw&n}7m$a^C$e>%x)Q9hyc#_<4^SeC_Z`V(cP4SkuX6~LQ7{Ennu2!_(^Ks)v@lw68s@S>#a!Vk z3m1XwU7p1Vx66|A?GQvO9e0aq;aa>MiFA*RcL6z8gW}5!LQy~dIRiv=E1&wG{}s;i zMzwZTwjwO!0X(fP(daYgg2CI(Gb~<06(WbqFG!t3)O2xE8HT?nc))>*IUvz2 zU(-TritTgaGG(hI6N+J|SDqCW!sg|8v*6ixCoGtaqCZ~njH>d)w>gfw@>OzfY+0Hc zuvZ+8dZ)({(P?17saMdepVsXBf&a!auU_JBaiq;0yJ&CttM8zdZZymgtr!-L` z!AyE%95lqkr2FE5&|D@o6zFdG)_^(~V)^4YvH|`7+aL@H2-XOXO=#fJFk4r3M`ou2 zD~G6t|9*oZ8yJyRf)mPx5=H9frWJupN;hn~b@v!n-sT98@V(;4S=QlP6@t<`+x|b% zv5N)VUyy8egIoN#;MowqWj(C6{L5BaA&>dwvq6U^sv7L)RU0Y?x1%QTa)Rc&+CNKf zOdPkboZaWkf)H-2jMa}kqq%;(;Fy!aW;*&!AFsF0Dy(?5A4f+2F?aF@<>q!OvIPcO z+I!FEt$HTrnpix*E`_3x2WQaxDbi(uk<tu>$e-ftp;9? z77OJPdp)Yx+d!N}T_kdx4FLyD=CVIB0$X%0i&or~^|Oc#IjM^<3K)ktA!>$On2JC*mAN(!;2+-qF3x&PZ=~^l z_Eed+=BIo>kERb_=w$f1__!!*m)^UE<#E-&Za(X9EOZdNB3z> z=3FMAqj9ych?qY6{{PSd1=)w75TR2hdaTXcdQYqPI{J-ad$CpDMYJk88 zUuYB=7^9oSH&g%E?2p!XUUXHNesYlZdRz*L@xSm|XUut1-2gbJ`xG6u*_1Zv7l5P< z-20>GY-%P+nzN};o5;b`6RxYc7PcQXg;>q8O@z(Rb=@V0$dW=Mj~QMkPNqT6zne_p zODbVg+y1i1BT>QKBo;6Qf``N4J@a}4WMiM; z-Fd2G(}P zuLH<7RB{;avDO?;odmDYW;jJPl4Am&1V=f@4XpE#kL4A(t;KaSEDHBc2?`)rXgrVQ zk5_4{YWMN=oW#*?=uWKc973L+22+#anF1r<zvw?mDsZul=@i@$#QSCtDl(!cW^zTzUyaF|tF)}% z8FFl->a=^Rpd3~GCFUj8inZTx<$*z8m`!uK!?#DCg2n||jCwD<8eU5L^t0(R=2qd@ z9fsBT`oaZ~XV5^D!Vee#iW4T*&EOSrSNZ&Og$(%_2`v`t>TQmWU)9Hq?LVz3BpWFb zd(LRWx`s#DOpLTj>Bd86xq7g&A`Re*Lb^eYqB(J-b0f-{FVSr8w}N$DmA(4FUSTp+ zMji(kgpWd0Wh4bO%5vT_+Ewcz-J`P8-a>;i8e-kQr~<6=MEEiSmj7%r8Lf+OtL>Kl z{di-ao@xv1nHGPo&zc+dRen;_a)a8JD+&ivK?D0*F|O@-o5nl0t*KpWMyKVPJ6!df zC8|W+mhfxUENIQQ?0~ob64w>6M!x)s&#Vl@jx(RP>%KT#JRrF&#`5$0dqVk%X?g!|e?y3}{3I{?zGP}j zzf!E+x)c@%jS`4JI7)t>f#l-7=xK^^g`X%?k<=^d!#cA;?b2z0>QUEG;e+fq(U63} z3)R%ydVp=keV798Ejz?8HxCdPm$!Qh^Utr!5acQAd_0JgWv+iOA6sCaWxGMV0}L!d zgCP?mIUBW&;QrYJbs2@q><^>^6B{=xt)CXfla->MZy7v5&`6 zOVG%AS@5$JCHLSyMoF^$V$A z(J0k<*Qi2ngu=5la%rVi(_r(~gN#AztJL(1CIifE7BraTY z6zD*3C3vd3hFXaOO3?9@nLj`Meinbt@*9vp;722w`PQ#h&Pw+u0ha;Os~58=-1BGF zJ}i$4?&Fpx4s5Q1yd0GckGg;h4+QseG2v7HHi}Dp&@NwvtIN$<_^LXRagxNR=sw`z z%Y$T_VZaWo5rH5aec0x12b~SOM+0{YX*3e9fLZRG8c5Yy*svve9h?sM!?l4ir{Jkk zDyJLP$Q@G3q1)rgf}`CEDLUZ9%p$sSH*9~bVgG}n@@WK`MMm1rc#Wj;yqvXHi}xlv z%~C`F0VDv(J0BQ1szy_vaEnaiB~DP@AO3}t+`JZSRorC1Lo~tVD>xWY=yx;Q#{xJu``wN=1nr<_c>{XGy4C(jh;q9kPN5cec0W$V(dT{LxyGF|xr3?C`R`jv ztvc@X-Y~zLI|!Xas2iZ1!3~D!zEz>3%?F>p_jJZ#dY1Qm10hI9Hz@a7`f#3w`Ff2f ztP(hBoj#Sw=JQ*_(Sqw+TQ9E3%pr5h#o++Nb|qA^(&EU;Srxw&*ovmFdSY!g<#sjr zpk7~2i}?UhO5gd<^ctyQQ}!0ya$9bNyL@VZToNPEO$t$YKT-{)2ae(RY~NJ)XtSBp z6`Bj&R%V{brq{DSZW4d0+i-VMrlVhtAH5avQ^ImkE` zSbW#tJ3;-d{i5h{U&6G=XID-zmTbs7k9$1!+^_GYU-cOldtZGZmzO8-<6QsGtkwRX z4IXt=DPV(}TOg%vV^LI#j5pIxCho?|(Vrf23JkVarmej9`V|k~um!&&f$hSlU9gY)F!}leF*3cXBm3`HvZ>D>k;$9>oaU)hfa-Qb5@3O7j(l1P zQpdM2mO2z336O?rp^Ndc(P#|4 zmm!!k>BN|8o6$eO&K9fh<@;yb{;x*ldy5W?>yVDfV8%Op25YnNZ3NImh&G~uDdZ&t z`AhF@Z(xLG&6`8d2h@F*N#OwG-$!EBV}|l|A~=Ajl@6$&=pqbmHUS3>J;xLTuas$Y zHPDy(b#lJ#_+AKn$@iTgT=)c0Y2J>t(+%NSlH@tVpwLg-ez*1hx6`iSybuuB*rt1b zNt_$I4t@!ZFNTlK(E}c|FE3Z~@6OjkM?K)x{`yUeRG64&XfVJunr-Lywdg0{ba(|Q zy|Tv_@oRv@lQ#rUIB-L!kpb!*8kyHNKW#Q?i`-#E{a<)96+a48Wy!;jM}@$?ryN-i}u=>!$T zD>omOKO#k(8ga+`G3Cv?>A38&K?(ZGwxUiT;((Y5j@>Z_V%tj`17f}WsKGnL-qwk= z=0phVP=|pv)nhHLt-OyYw4+b;dw0qB_RCZ1Prwg|vGQOvzcZcTLzvtjc?^d!DULnW zADr2eOBz{~&F31R06-@)WXrCy=xKRfk{~F=!%Rmel5Y825U9K`etH)|H;n);(7df2)x? zn{YJqcyJi#^I8m(QqwsM%RC9_a%9? zpxit?z!uvbqQswdTH??9HXt|##)h7um#3T zFRAs9AS&8&uZq8d)x%SiRn_-?WWo=GeRbIF#RZct`)7D-RY$zg>sH}VP3tBdDw*Uh zH?q%LtF6^4SJE(RzB4V3ar93VbKpgI^)dqHL;VECrHTu1OG6BGy^)Hdc zXE)(5c%Zk(K(dY~ZsW^xbB6 zXQvr1n%!3e^DZ<;{uxxAs1R4TBF6z!tuBwzNOxYq*H0 zWlYorGM_3iDoXHY>g{MiZW%RDctu5IDI&_lFpK{O*4(p!#3GVx;wGCee455SmhjFPN5^OXQNE};9588P|@w!gjVLuQ! zRj_8^4ds4~JRjx^Jw3)TJsx)9Zzi+aX52>1v4a0go&5I==8p~l8>$stjlDmBEJ%gM`e`t=_;7a`dHgK(FNGq7Q$w|t(_K)B;d;{qs)yYf{1WApg8Ls z`fFsspl#uSg>s0)6Gg{AO;=x>EV+%mdm8X|B;rfN?dRw$Jpjh0fV#R2c=(Fb&`ifS z0&N}dv8^?u2i|x!w4>+cD2@|B#yMyKro|;A3n@z-GfivjAJ`Jk*Dhi83_%jL`_&t;6OTOF|<#yj>!(~tTOh9q8g`04MYNt zLf3~pK%RbD?AtxXfp>K4pzZL=w{)qU3YuRGmfTlc4tYFuq9kyK5;a)!(ArTEGo0v6 zO(Z~C@)tC?l%VU$7*(z>U*GcBJ^Gm+q!9?%Dll?@NL}r7aDt0tLf<9133@#~)d>Fn z{Dl2dkS3K|E|p3Ap7(AA-h;T?Sr73u32J-hsiPnJ+r)7>I_j17mqyvqClQZ0kmsm0 zejfQr!!=gByVJ?DDk?Eo*38(}aIB_X;fi3$l#bRz;`@t_(20pD`oTZ1TMZn}Qfm)# zmoFF{-KGYxjVJ%88g2r)dp{M%MAz6XX+bRJHBcaaQ)hRHD+OMtN+->&O}g9g z!uilAx8Hs}#0}9oKri!~c%haIL@_VK_xay?GjJDu3BM{+kr?~VmEQQ?gy~kz{m%}+ z1C-n8FoFLiDGIai5o@F8T&D=J7>WznD4}x&#wzXt{ ziHs&Y)t^C;%p*Cfaste>%4XY>Fi|e(JUtS-Moi&;)FSKW^UnpM!T|jw78XD@bpc@a zW6A(*QM^c0OI+AQu0D#vo?NFYn_!^teebXrCrOuoFC&Y4Lgj}*o(viI)W7G?ZJF3nf-cx0M&Aj(9 zx|QKUZG43~WDjEh42}QGb^QDd_erHf(bbRHL;E$hP;P~d1>WfHxJ9r|&F)SK4W{EQ z#}FXx_!^15ik>6ss7hCNeci(E8mJcub8XjbVrKHr9ey%%au*f(=m1!o#@lp^0_GF zyk!{NI_On9@gADxhf9AN-`ga$5t87b>Q*^)a&zD87RS6?)|SJZRuh0R>s zhD|LhX)LBFHcoKSx&eUGMHBKk_bY6Dhdm%h)^;v+YQae4u5em+rY!3;y^r90uXsRa zz+Dkh+BRE;y+uAIXZf`1iYXZhr&u7yi_1(?PPi`?26^?G_|t86^gr4{H-%nUGWry4 zfE`_|A{n()=o}eXlH@ahJqcf|&CzEFe)1iS_uhI|l_YxB!c5txc9AwqX8`?QtNqWi zLlHZ;{&mcUsacxOl&Su$Gg-}_;to=2HefV-2hflb zZh^I!3>xGuQjc@w0>7z2hb=hE@^d)x<}4)31xJ;p$ff-9O0iN)#X6T;1+;RrPozwd ze);I%T|)n%9Z5#u6bbn4WWJ@wHAp?-uK{U$O^B55BmVh&!FTiGL4(s-)Wn%spVkN> z;Fu_Zg<+z!s*8V*B)^K~qWYWsr$kO95WxFhYFgy~V^PqVe4V%3nE`9xiAm6}y{3b* z=rpdX{^zHl&4m-MpRxIJjXMi)Gq(FXTcr?A65#k8Tj)UUA&e88 zVfJ4>iUn!cOiogzlZxE>+}bd~02zsb2v`KnC`OAmUc|qPkn#U(5pZEpaJ*?Uyem13 zqTlXK42H>O!~?L~K`JxwB&z)TNfcQPBKA(8o!IGhYaN{|0gS^5!R*9rqLG3XlCqip za?8^{Lh{xXgX%{%?S>FR{abPu+#A9^fIs(-fzCh2J>OYGGr+F(9&awhkGd=1?~w5C zs{#qn7Hiz^XyL9A1QQ|)d{Z~o(0wfTYh(|Y6LeR&^WaGg0Nx_&cl&QI@;~0JOY02+ zcfRCjlt&YKAK8&}8xOg}N0!-XLmw5_#>VZYl<|dCK8A1$b9BT4%csig`xg|8&+wpp zdALI3Dk9^70^@1MyYsDRKlrkQDj@&oQFW(v{r_R@E1;sEBfKmi$P z6hS&fKw6}`a|jUu5ecP1x}>{PQ7P#dat7%da+sm#zw@4;y!XBTd*Ay1wcdJ5?wq^N z?z8tk`<}zk^p)_{f?4I8p;xcvi+1oLm7LRZSg$!TN$>?uumY@(zrEZ@3$$cA@0i`eOFF{O^@VAfm_oCx(YL z-<<>@-hG@YM|E9&LFL^?Mn^&+pqv5~{U*z-a&&f^TFq8gS@Y?X!7tHrQQ^iJ|KNUKR7XKBqt4zo< z*W)LE=d!IPO`cq|uU1)M{aFb5N$;_j{11ONMPd|>eO6&p;JP&MFR4C5ofD(EEVM!O z>nIDKjt_t!qIZc5QGf5Z{)P2vvN#2SLzs1(E8o&JoC7}9tT`7pa_+cF#fUkdIW4|Z zmif50QN@kX4^S@%IW+OVcMB&^9>fkEE<9Txu$ z*S&6=bArdKhdL?XS_+cm+hngDQzn&&`A{_xm7p&F5B~M|E&evKYIc8ClksKhoj-9#U!KkF4w@ zVkD?!dD2gI?1mPj50e~&H!|Wqr~DhHadGF9)nw8h(8c~)=ayl;V_$Xl(mJ4dZP>2@ ze$D@Pa{ltf79a0+dq4Aw?<|~zSZ`MyHO5(--~W`vFRfuY!P`d+pa^Gk^5QYs{}-)3 z{z>~LRxOE6ShxwNZqD2=F+LMN7FAznn8(OtmTIqJ@e^#LMKKE^8r;f$!XBwPjKJnd z&Ts*%*nd@wKL<>TGB-&nqS?Ioj_5BvGt5Ghip2jUz{gMwkddKnxLe@Ktm=FiHCfs3 z$o=W`PH6q-e@N|}Q%I`6@kmi8Ld6`6tXtyp6=6P%|916Tf*8R&-t32cS<=^H`?JCv z&1HW<_4#rg@E_0L8P%Um&SV=1=)Ys2SSi&nF`iKWZ~x&Vi*X;aFJps$`4+LMi^-F0 z8^a~W|4)DwF@U>2aQyP(Vh=GN^eBJGd;6~?@^`^ain0Le>+OL(Ec8o}m?8krhR@u( z`HMY&(bj+Qr*t3yM0(M>i0c;=QW8LEzhYAd{bJ9tZvOixOd0?X<`J#tkdt^fOw1Kf zPl8Y~hhLBWcfi_LF)mqXl?*KX%PHN*F!?f(;=jl5?<2MSF@P`FP}j=+0*ujiOX_jf ze^&1Qs&!T4nDwC=b{~0Slnf(efVPc@Vx9XJBKjqm(}UG7V=VFGSXfpt$JYc>YiCpFBsKtZ`1Jf84(pf_`ycOIfM- zt6KXi=Y@01HRe70k}wxwvxf$&-hd<&-X;llUU$dDMvB)iX9RSwpTOpP^QH zx|j7$SkYUF!`sdcTYuc|E%xVa>pryCfm^==R!sTGeqNtmQV87CzEW#5^7AUaKdxf^ zSN~&R1#Wz94+JMzN&WjrSos{O&WGiR{=yg!<$7C_4o z=37Spzk(8LVh1i;&b=@C6B0w%D**O;+pJ#0KQ9aXFR1+F)F0FOd<0%i?YoV2jD!?1 zaPMGSagr(XF?8T&`ad!L^TODF{!|G6JlN|NYVdy0Dhl{3b+9| z;w-Q6{XaQ>_S_}u7fmYH;Y6b^e;@X*0EcM+G{uq|dL1J+NlHlSD)2?{#USH9lk*SL zd{G7P(B{{;a?BOmb%07b4~L=G|M12?^DWE?;8;ml6!Qrh=Otb=_21VwX8u=%okIRR zz@Yy7FOBaWzbdAN52)`=jVR#m^5bX!@?W+$fa{&<2%Lrjk6-wr43JHGN8|GU3hG7E zHA;qc$5xlKL??>J%?g;2gV*=qW9a|PqEjT|F@!w@ICjaxyNCK5H}m>eb(pcam$8Di zk3*LTV>Rf-5aSCme?S~YF(FhJPp12~_HaoFpb=~Ja@;F1NeqxDoZ#<|CfUF%k1DWy z-U3%Sx4)*|`q!wQV9t~SSo*^Wo-)E?BYTZu=36B>|Nou-%K-iFpM!d(j&+%#27tc6 z_x8k4{w=knQX&{ruk1Fxb1YL*7CGIMISsM@wHln5i4dS#LRS!fTse=Sq3s4BQz{ho@#=T1y(cgH^AxW! ziWGOD`Pc@1m@Wb&G6}vhK7sxRYp?oJHQ-xXkB-DCUs3rhgdY2GC0_a%K`+A9vTme*nu;q3WM==(%b5LlmND$&Xd7h-x!HFVx6sP> zuu39JVc^Ez}iK_<1ajiFp)*7N=-h_h* z2i$chuI;D%7^-o!BeT%6ZH%l*jt}f@F65scaY+iG2Mc~7?j&L{deNaOCUyM8(Y=nyYM@#l6OsfWUhT7fa@B~wj}tgl+gI5lhU>QctVG)p9xR>80H2D zKNe(oY*)alP7Wx&)UF2qsd1#?0B>VrV(@;S72uao`encYMAQbKa^qqzfjCTNo$le* zu}S!p0f9#b^B31sJ+u@$HT)ew+pU*I#mA$YJ^_x~u_#@G zI>1A&L@c@!t}{!}zy2C>OS28L3C-^aJwVO48> z9~b|^)&h6tJNB=aG$xlGRHq|ms|~EL18R~TLwX9@Jwt#JVk%+wpXt`gj|?4v7dPU+ z!~VO&@S(l&3|kST8>9Z_!-gnZ3|Hk$;30IBw)N}r!g>a4gK}q|{vBKt%}DQEtEUS? zMC*LQ!EtC~1bsn%ecPmRfWnvkX)!-1o8kc23{Aa7ax(c(I=^66uzUM2qkdOGG33S8 za{c8aZ$-gt74c@J-NvOSk-OM)0?V=O&|9wiF2HRgdV%GAWIkD6#-qy!PJvzra?DNh zp3vV_0ucB=1Oh|-O7Ij$^bKuw4BW{u~{=S5s@p*-b_(*GgB!8DThCN}-0Kk%0k$%4n3E&_x zkmN)(lbqo@PUkPFCy%ApSJ(;D1JJ9$fl=2^Ey2ZrJ-z7+%sz!Hpcqn;08iSgAf`OW zJi)xigbDH=!octBgbJYxGhLtFhZsQpkd3jZtE9mN+-WC2*e!(<@!2Bz8o z83i7TP)r+ezF+--KK4_n;$H(8e$5#>_;>5W=rDsUFy08Co$&W4#fKJ?3Ix!fu@w0^ z8&d&*>z4^o*{P+7R0>mWXM~ zz88Ngy&b{0?h0tVCGBaje;3dnEGogQ$WcWlO2=$u*aBF{ZPR+`6bF4CVrIVqe(8Lg zHF$q#;-41e!y0m8RG5JDBipDXc{}>~Ni{ajxV2x8VVN~D}AQ$P9G*gz}NnnZu z*p)u2!+Y$3?cW0rK9p6Cs2$^uv9{J#j0)^uHQxUVSo;#p@H*6p^QqxCfw?r8JM52@ z-$xEp^I~EsJ#BROOn^*$oN?vX%`&hHoj6a43VN$fEO zhnxusgP+o3U%E((CCmiKw11JiA@%=cK{1XSJFn}-T zMyYuJCc1mTd~bMe1f8Dl?HM0dV(epX{eN}|d|7yw!$rzNd7g@^Y<5|fZ@Jnlj96Cu zQ2}oPmdm#DpS6ys{zE$%0x>RWqbqIkcaCDbemch46EU0$)6Zc2DVZ|UQjek|MvXex z>L{@ujb*k|% zrjunnEx!JE=Y7yjJ3D8z$8Jba3@iKj10d<|%Lmg}PrlsB&>VbYr!iQy5kA{O?;^GD zY&GMJ?jY;uj^^;z9EwYRT53#`A$D|dr!$<6b?;~&UShu8k@~5cD9PidNs?#gWiGWR z)>S_=s~0{K_LaARQAaG|{vi>E`zYDS_SxCgAkMcjdZXpWI6dm@YQmz<()fi;Ta7yb*F_!+fs1=jkV3s1s+PApBJ58}mc!qA1 zW~kMq*RPkG|MfoyTF_tA#m=n=;QEKl{$_P z@UK=G%!Z8F2s6PC=99eBwf&at+u2w?~Yf2DiXfJ1%k5?FH_G_Lg>6b zL>4DrdG>f&Wz<4Ko0i=M5N*{Qg6j3Qmd8I#5!f z?=XN1$ZnfQ$~lD*NW(u2RRI2RX~Kf0(HjkWy!2_4?*qtItWdndowHSIQxZ}%`f(pmtqL+LCPpkhU8`@@bw@7Q_$sA6A_`<9 zMJ;B7OyhpSrk-aL6_6tl3tZs<9{tyW|3W38e8%J};n@TU%UUrX;r)#kC~Y zd*kFM*?N(9@ZZ;KpA{fiW1|D#$&a7F7s|lqxz8&9s;Pg9xx;WO-K;Om1yp@dGIcGR zQqMygmxxOLrYL#DktevdjWy{q^*p?n;F0E{l(;famZH#&Yv-{Pp~AKcd*z$aYKHD* zz*l2ilh_xD-0TKBxJ}n>Y0~?}bj%YH2|G2*-f=bT$XGYRjWrJYavw~x?PArPOMEQY z9&SMElQlD}Sa#{uV(}W-15vppGXjoN7la@8QMPlBRxo?}KY= zb+u|`4_Z^dcFgPn{fuKaif-Hr?>-i^xkP1 z3KGS#4zF|DdQ-9d!J#W^yxJi^o@TjX2U}6}ph{e1*D%ckedx@8i0Dt3dM7*cYMOEE zht3FBo2!v>bAPxrG*HzH-sWXS3^6s8p4FZ$X03Agd1Pq z#3C=>x^izfD_$g=-fKs>pn6Yc%s^Ub2Lp3VDcJL>{1?|2-fKy`vxbQ;kQxBH=|tOn0jNB|FeZqH0rSGb24sH61! zWbRrzCH@dU|8T+@^3CfoptYm0j)iWt@?E@5%_kyauzYue6OXwdu!eUZxbEaz#V4jw z>e~A7o7hXYJ)TXs{Sjoe-XvnZSp<374Fak38(Yn6b!r*5rSjW=v)%@9k}{j3%paSB4%s(yu$@LE9+bLwBE!SsB}|#Q?5;q)E*$png#VV51ai% z-&Mlg)Fm&_n0Bk95$@yaxw?=%keoT7}#bakOph`Ea!SB~iGs9oJ!EFj?(B8A095 z`=C@^ai?nnjk?o_z1`(PFkDDo$H|X&xjLmepE19?^l3rcUiC-snEKVxT=B6<_{j4D z8?8>0KBH&~5)0pWNqK|nC~w3D?ud~H8LQlfn}W_O5fxT(W8M7q&hEYu*41?^|JP*G zsBtlO!1DT7!Dp zz$tBEmcx_6))~b)B!Yyz-s~Q-f_1bOtIX!R3`xXJHBI_&=g5Aw&mg&w%%${fX#6X1 z-n2G9eBdRw$>_%>vnl@DQJ&~Nb{m;WsQ8LG7jL7w+?5H@MEPeL_za{Sc422a)A8{P zWyG*M5^9KwI_|9$H5;zH7QuIz_R8eDMQGBMM+Fof$N6sw((GhgM(4{A5pNXQIGkAd-pb0eg2}-5T>;J7RpkCsL?7Dj+m8(D+(>GqI`v zn!ZWKsnxU-r{vZ;V#Fj~-YKlY$K$B|?q18%yR>g($uGVAWw|(5bOb9wDUp*+%{%xi zGi@1)yC&g$`EhNDdfw!8uu5Rn_^!hCh;@i&v(|adDsc|wFkD9+ve;XnMkHN*vDi9@ zP^jDzI(#lv{1Cv|&%<@ja3@p`w_-N;lQ9PccjWRdz>8J~mQX36U9 z#R`%AahYD4@$!D#!F2rdUY0UeT4Tb{cHD^16qZm^(`1$Ct3z#~3fQDh=Au*dmbnCG z!-mh$x_ud1HZ)xu3Inw!l)Z-|wcTQFTlp?^X)^~~U6Ixk0vHeZ){Q#jocgLQxU%&M)o#!_>$L?mkIq1!WNYi@sOPRg zMeP8Rwrpo#i@m|!5{P4nP#bm2v+5|SI9K2639XwoNc|L?I3egK|Nfn4E@4kt zKM<%q`oJR*beV5Ix%^4KF01(@Te(4xcSH5JZ#?UWe#P3A^2tbbJ(nPo`la2$+(+}T zUxW$zPdN!K6Y%tfD2Y$^r2XjG3+0QnkRQ&}ibg6&?;IlGk8_>HX=oE}Tc_6**+=Hn zMygGkb>*@3<#wN;VL@sn$y&PiemBpNFyx~R}>d2%)7`E zdf?}{+}nct@_=!Fd(pa3T|lTl>1?)6crN6l3HDnD6UFVqzJ?@jr_5VVtG@UT@h&XF z9}9*aKsK%C+;@>&(Mn|L+mo_HsntdvaPw2`54Ue}xz5D9pHCtSY7c)}et+8{EN zlQQnzO{d>H9{-R?h=}H{bfWgrhjG`#!M?Fq70(wD`0M#&+YvA%oKU`CD+%1CVV~|+ zHS|qBzSFl}1dm1!ID=hl;#w(*H9`Ju8YJ!arf@>;MZ0>-9|NypFKE|lP$Cg(aQETL zJwx=hLly6y==e~`4C(9bdr~3!DUo;&DR)-3xYBi}v6Jk$@?zNC(5S`*p4*b#7Lt>J zz6(ltj=Z0~L11#bhZRGN0fq^{U7Bg~qivbpmBZUGuMVQVas$M^Zkd1u zaMsfUlAv|Iy;sv)vFF(XVDnC691z*cko>xEx)Lue#elbo~65WSA2Xx&H>k)0k~x3V~Y(RD-`exKbG)cE0nN|*5+usZU7rK@?xkv)6=G5yMTzs^tIXK3tfHk)bTxDsq*dv zTlG>&RW^^duN8am)N=Y1#(K-#XlnDBWyZ~N-x|%Lv{fb4GUs-}wYA1ZXqi!YzKCk2f;`*4?G71Or6;$D`2vcg^ShbPUGtTrQmEPlMZD-EgJ5Yy2|r>4xQ(jaj`4|Fhe)i_rYE`jDk^ z_9y~HYSRIwE^Owt@+~t7w)_v7^L4l#{zQwEPwhGzkCIVlIEmv3`9l;=76ED1W z3v11%CWXQ^LPSSqXL_78Q}UnB#v$|l?`=Cw;*sU))FM4&`7H_-%AYptYA&+<=|6a= zU1iqFwYvom-_qQ%&&!YCb5)|Zu&>(9>-W4LjoIXSZ=%8CiU<{lIc z;Gwyb-@jH$@YQjl$40waD>Xis*vDu5W`}TQrYRhhUQnVLsgmn}>x2pSg|xGEhnw*c zKX4-DB!&pPL?4CRo3eG#>lWFa>L@h&@t!;tMu{hcBh7ufM&2rdr9t6p{}2dfi}MLo z>NT0OA6LJmp>_A%jfAA@p|9~Naa(_1FJg$mJtyF@$VQ$1@o7pP5u%Dp^w2T9%yh5| zDkN&XV#~jqPBw9)mwscke2qHCb%sdDY*L&Z=CScree-ENh5bIf^3`1bRZ8PtnxT2o zI}<{8SDX67`$j1!#rkI7UcLK8h@&VB=`T3deQg6G;ea| zwR9l-y)RH^Lq=U741^C+lEGS|G>eC4u_#G(Qb*03TzQ$z zUfQ&}=Fn;BZpB)(Fu&sFoc!*Oc0ekx{HeahS_$nfq=|2twdN{^tMR=`^SMW)Qr%U~ zEbPOY;DSDC1P*GOwesU4irK9N7IACg{><(uw=K%Hj~9Kd7`_v&jpQ~0U!{(5JM4Wa zT92fmU$&4$RekqsDI+>+3P)X$Tic~5Z0eUJ3+Zp_XUqC<%lAAIpKyF3G^Ko* zucW01DAQQ2w`tTE+|X@tkR1Cl8r};5N>Ck@T9s|760%0KLAh39Pb%+f4%Vsbfq_ey z=q@`SwNShE@gdhB&HifjaHkfXJo5Ist(v1UEJXpGE#u%B;GPNl(qYIT%P{}5p4($A zR`JVaSg@x1&CCm(fjP;8S^|03vqeNT~d>6;Hx<0pwV(|2+FmVldrDEWAUJ< zn}_aEG%d#N3`BcnP8S3z0J|%`aYtSgYs`B``KFqUA#48<$al|~o8*D4o2yot8z>$o zKI*hYLx8-n3ehO2t8*Qm8M(He-bi%Ua_AajW@o6AKJqat09}9EGOaUS&uhd0TfcyM zh;dGOzr2$SwRj!p(w+6oBQpNKn!K{{4f*QAM@veVtG>rwbZswqB>CJ%U4dAr+ziI^ z#1omr{w5ynzeww3!lAqO_U#oOX5J&{rAp~B3Zl=eq6ERb!fS=-VWBe!b~aScA@ox~ zUJ}87S~~B|_^_%=8xHpz+4uTUK2$;@`Z=?;L^Jxjru<69r*&L!+<2Hm=GM#&1p)(^ zNM3TOIM)Y{QgGrE*;Gv0U;4=!ep&aac>Iora=CL=Q##vdE8YUQeHg4vr#uxiht-E3bvIbdESS8B~F6X z+fni96VvBqp43)+5d+99OOkTZs%x)nRTccXqgvAfY%^T@oH0z@;|njA6ndn5j%N09@4|C|_Z2t?a>Lak7X30{xC? z&fA~jg^r~AJd)HVrN!UzWXgwWSKIq>$QC9Ub*Ps%zZ4OF`QWiY_iB?2^m!F5R6D6`dX`5S5QlRSga-PuK%N02j~N-x28KDWDTjF!$rBO@49d zg++)&K~>#Txl)@6Wc`!;!2tzuVz#TO+qe(##c9(3mZEtfiJCnLdytvmS+Yf6PXsDh z$n80Lf3iq9e5P%-I~I1nuVB6G*@gOLM^xi^nDMV-RReO3*zm4g-x756M;7z!#7(E1@u!cXk z0!v~mHpauQAYA;)DmuoXdBimf*p%o!3-8QA*@DiEs>CXz3}g;`Xj#FyTSCWQP4eE< zdUJoB*P>fIyZUNOXt*i7yLFd7bee&&MX${2**z=`=CF}<1*KYqV7LgWvZxlK5@is| zzpm7qyr@D`aTq|C>a*-XkPw+YTjjK{J-KXC+1B9M*fC;~@Giy3c<5GnQ-5LOY9vRd zy`0YhYistLPuwc>0_5na9%}VazF}`q3*0HP{Jnn~8Jo5YGZ_7dx_n?W zOP$#kY6V5{9hJ1^JAgvYs}TQ)9CF1+k{?KiM(oNIJ8?zm#3QiEqKYL6_rpK zY_?72DkJ-PpoxJ+P zUe$`6J%3b{pPp^R>d($t!wb8QF0Uz)tZ(wryx9QRdi9g`Ls z+{{N_Iv!qG@z}nbwy)-^=FS<(t|hrN;Ps`pRjE2WqUc0|c|YD@wgm zI+))c&CVEkGm20s2{HO{Uk*EhdelxQ&nL^vv!){>8W;BLj`KLvvXC=EiS5aPW$nT? zy%!U(OYlRD44ET39PK{Y&{G7gzSJcHwwg*1WU)fU6s(U~ahQI{DGs6wR&htsnX>Da zTTP7nZ@-+|CJy@PVD)eTL~hDiUaFTou$jy?wZYbBTXF9b{PG>KvJ;y>0$c zAViI%FP>)&DEYG4K{)h5^m~<7UwOFN`?3B^0RErPaSkT!=AeVu3~&7SC;4y`gnKR2Q`@i7HT_e&nnxrf|0w?50jpHeWS{fiAHi@Rq~hn0QFy$n@=pBvG&4k z#!B?f!ymm}?sbXDb6<24^!S#p?~JYqJn+e4>=geP6_eb%#jeLqBER3I;5?d*Gb?1E z*Spf3>c?eYD(N{`G{#=oz}zj=s9E8$;q>wIfb`rC3E)U|xpvVfeYXmgz8vUz)~mXs z==B4)HIci4>lPb}b5m3RZ6@Ynhh#s&Op$Jc2Gz3qWyEh&H6)cNh3aio9iS?)a45|_T@>XD|<(lJXY^py2DLJi`0hmwrR2sbgJxpsqM{*ThGr|2R8D6 z{Fir8LxL9KEslBf1)R+`z+IYw7Fy^ATYKxvI?1y@V?$Bb>!Hl4jh|Wq# zY=&q%W%GVSNy(jMBocT_CEBdr4I{RQ*A1tyLR)Vo%wIKwR>apEFK-G~+E~9-%`&xl zmi4JeL-AG=1m>zi61ZGmQG(k^Adqv4jtP zxKz>}D&)d~Z$@!(jMC@#uYqgm5nChR?|`6_)V|$P-L&SPMns$+z~x{qZHe`wkWVUa z;?xXXtc29DuEnk0FJRXxc~xev)_qj@bw|4N0gSQC(-T}4E*td7YiQZU2Uxz!gBHdM zs#kbOY+~+!^%rU*Lwa_%Yc)$H%hM7K1SaGhXzNUm;-zZAvg4eQZ_Sh{`-Q4@i;3&u z-l&D*JC;hQB8O$5((I?o)@QU68kp=Lte7~?ZTG$D37UqrbUxe#jCi^*1b$7I4nuFJ z;MG{Ui`>#c=D9o`^Df+7%_dj&9ld&I!yCH>#fJjdZ~A+>Z1~N0RK4>|lfUUczKo)T zl$&jQ6tdM@BG;NI3*3~MWK%|-gP-)SV0N`um~5r66~`((S%JKZ_=2nJyYkWZxX){x z&A1>s@tXl)qUr^*Yx7-+O9{MrdbyOsr~{b|+(iEibj)0N+D3`)u4V%{cQi)~n*D?A zQ93SCxl!^DI^J#@yCgiCCtA9xo^O;SHcce7STr%$LgTH@^nxQ-@(r@dwj*t0+giq1XS>Y zwsVO&kn*uBmt~|l)*Uy#QnvSeI!vv|FvwahS*<7ejAf8rE zrI2L$6xcbws9H6?17wIb9z^wK$u~{CYhJ)UjHq1C>8h&8*pBP<%$*NQl)VqPUmj=| zE??q(`|)Q$yg>}y$l1YF;tb#49GQ=w%`I%;6D{&K>lUme>IM4k1QI9Q`}#wK=h`&M zZ{h@ecXSJ;9wH>fsdhS7@3sCZfcx94kg_S2U3(H~xaMFp%h-CGq}fEKueRw?>&N+c zl$jCWI*dDcRZ}*zp7;@HRk>{;c)B}Um8U=7G?R}LlBeL%olD|H9L16ci5~)$oCC;m z&)duvox((IA&7@Ht@WeD@Y}U97B6&L$^_AM4!~kw_%2m~FrLPS6ThyS`D)TxDtw{p ztsBA+`WdMrhS>u!4ZLd|a+!Uo)7Egg^8PE{@k0=BDChc` ztUK9el{Al)n0Z^Bbc)+^cyBy$Kxm>RE9!jIa}9rJ(^#?@SkSDcCC(qmzs@I1Lq)Rn zOlkhI=pNJ3^A6F27rb6&k^>uhG3|Js=R=5b22bpB{HZY4RWcf)Pv+( z*-ii(Dti4Qs^}WOaDzO-+y8lDaX;~X?1BZk-v)>Y{3Ihw+Es|jQ0aN%|HO%pEATcJ)X zWCTS!(P?XfG1IB9xTNMX9p*5!-aYv8|`x!N`ZGVIMJ4bsKg^oeQ^-yQU zT_ta&ZX$>dvu91#IiM&GQmh>c?hJ0Ltp225sL3ps>z6U4qgMACu`n~_)-PF(#SV5I`>?h%ZMgD^NII zVe?}s>|$NN$=m^Cw#su!C$my?;a<;1tng9Yax%S7-sKy5<@a+SQ>M$vq*CZ&*#m__ zuVy&v>YA;0qkj;hwx5|YVU3>Ly|81cC&T;UIxDZwVsAJ9p~7@cVA}asoJ;{lrL40X ziJtxv=MoP3mt2Zh6SU6zTTNNb0a@TPo!VsE0E;xQA>_*YP2pyut;U9{q%rQ9x-*aOC`*SHU}L!Kh)^olpCgVmD0%h`7v#p-w?bF{ZhO)hMo*o(RJA zTK(b%k;9p{A3v45eo%xu*EJp)nCX8GD(GKx-8&q>>7KA%vq;;!E2nhSgtA?`=Zyws zzInPW&?)Y_z30#o4DpBVg9T7mOntYkcT|EhM+kIQJ-O>Hxytz0KcqR;y*6?VNGqgs z!3&+k_dzi5HT8Ti4P0XCt2s@nMzONd*`_w%=|vo0xbzhrQSr>TkF!#|!b#N52%YXE zDKm3|=(*&ZW)MTZP8+*;PqtYF6);BenMr-}%kCo(sb=LPg}dMk6vkHy8dg$Ads z?G>nWLCw;eCROuo2BK?w>_XKBx1$Wkyc;lu@yHp#95<(0wsK zHA2%b%M~Y)yN}rOfm#`XR1Kqo=waDM)gLdNBN79)mq6u%iM)<(sk_SJwTE*7d%)rO zW49wvADla0-(4^BSoEM-aNKg!gSl>5Edo_RBF)9SdV={=6=Mgcmj>wUzWe|h3eMmB z`c-f}efbHDQmvrk+Eb$HnpLT-fK>&$QOPG~-?o^DNn$Ck3dwYrk6RJ?u(H0ZD+H=< zF0fwvQ`NU7uCUiWStLU?w-x8tsLgY5g{`X>1;C|=YKd-Mv+OBZcFUbUi~$I|yw>TL zq(G%GYk!%n?k-b?BH9_$=!M*+L`pBOnr*Hw!c!u_?!2b`c}~$iqzTBO_`-6M(%v`9 zHg)xUPZA`@?t1nqX;m}uHhT-N8hcwsSEfoAU#IL)o@40;I`-6ni(W3*q$t_|oBKS> zq%(SINoN^n-Bj+}7tCjPl45D5MYmMcyxCr1HD&}&olW7B7_0b>n>da%?XejBl5)gi zyi^Z*t+Ml;(5h9^M#!t``=R|1RaP4m<`qsNGS1*I&qb7<1DmXTC<_|N?!&+yM>={N7MJ)X>7y<>9ug6y&uaNS@#@=>raDn-Ha&+qKfa78-oo!( zg3(*Q66{XHNkohfCCAm?M{26T<&s*xrvi7aBzW^3*Z1PrYQ9RNb?F{vZ8&S*T}<*m zDkrDXj^~|>0NLmY#ukj}-TyK%L(74=mLQy4e{-eBoYydm{3^nxN2}7yCbyy5{HnWk zFjX(>I*Wk`5bR5TypxSsxcikxsc$(uQC8FQK6_FpPh`!wlYFix5zR2i)SM&4*vUba z8S%8oK?sfC(!qwK$UQ$~RzdDg>$1eh;OJX@H7T-v`HoknC0akA&f@Bg%Dq{Tc*}>Y z=xq!2;Nh3w6VWwIZ)wkO+&j9?=YJN@@ZAfw%LjHXiYXau&_mg$RZ`9UZuaxU!7jos zaosZBPU3V~^ESd{dlLs=ecGk>gyq9ycC3);=EbxqQSo~PoK<=$TYayl`HJh8y zo!^(Qd0yE&aNb(5C~FGGWv~DaM-|LIl;bk?S{st#OR^rlR1>xMkh(RF-^lf$Q@m@u zn>Yvb+oB77zHzqd-Me^Ga7iLKdHLrIwbB()*S9YATQ3HKW?ya*#I}n!w(Zk<(C&E* z1X?%F;x3LB8x13lKtQ>-qNCQ^NfXGLO4T#fX&*RnGs=D znTMMA8)d+5JGtJ3NoKt9j}Q3v&wa?o$9i&6!>w|Y(rEBE`a1ha8z&?8yyfD#HD0;# ze1qI3nT!pAq`GCq`NVPW^HpXD&ByCA;WNuN75)v_>8!v;U>t8#e!|sK9##_2yw}yr zo!1ZnqS03&W)t3;7w?V$UvITlw3wdy0+Y$N=^7DmHDA3JrW6B-%aNau+s-siz_sOd z>@8GusYdigh-M0p7FC;OL{yiisk>4O8K&oX*f~9&G|S7jO$DtfCJE>&@S*h+6FIMf zYWJ}fYmRER<6G(HvIU?yip6E@`Ay+tYWYK{s_&mK>UvLeEDnOj9?mkN>+;8(i&Ts1 z$W^qTvFe_^NF+Y+Q3K{Lz#@<5rI}oj>(HP*-=CA|&Qy(*Ind+y*lxIbXkMJvAZTgfVq8xrO!b=m!BQ@6g*-GM?`F5DPE&+B`fVx3T07^uWR7 z)`|nWRry`I`MZ6njhzcp!1DE}&Mzo2;$kyyI}YWCtX%TSZlFG0$rI;|C&V8(;|bKK z>-IyUPS>Zsd>#@C!N*XwLXRsY7UnvV*L|#;fsCFn(*5y~NG?b#gnm#o)%_jegDR^b znw`wI^t2RGy>`@w*^r&$o?sLsE{e2X88)^^wqOxYV=&RV+dG=^DSji|WJJS>s^*sW zyfqT?!@V?7VcL4tr9a14M0#8`ow{dC=)(7L(-pW{{eeGXA^g(qzIgEfq9qDD#scWM zoIyg~m5hlJKb6@kgJ$}NQ6o+L;^fPtH22eRi0Z5qY#KqY)3(%Z>S-!t`t@2`M6Y97JRU1ch_w^`L&tj z3$@waB_a3z^^eztv9o%84sk;(RLvaG3!M!ct96miWgOz?9fLo9Olx`EYz&u3sw>p# zAzL}@x_^z%`|BB%A=SVkAhMJR(s~gmpbr4-{J4#^3tb1`9%CQr^F9rUY2?UMnBR(wDxm(c04*Q|i z)q%p&gwYBh3H{6M3d@yIiXDwPYHe023&U&+FIZ+QK z^VOV4T%Yvt>Ug<0;r_%|`E(Ms1k0E0dpbQ%_aRCk1-*KpcWFIJez^^(TbldU3TTUx zAg){ywDZ@nkp;T_BVd#hLe#^M$%bWw1H;GnLm>&ZhF@aXD&owPCfRgbVnKk*wj-Pm zsZ8pnZZ+>)604GEEJdtFZBBdvADG9~$Ok=|m>X6{=`;}y%!bg8@~rUK?Dn%u_&O8> zAw5UImWDtvcWBxZZu;^o=K)=Z-D-mRe(99QCg7z7?lS0#GbqD+a&uB=d4GR03^(!e zjr{mVr?`R%_px|*{UC?7Uy3Pc*kYELC<|z-MHV%Mt6}xM5j&z8aS}Fj+Nc^4zOl~h zxMS)DY(;*p^)xG&XvrdXb>H|JZP*ibv9HcdoaPChX6DhEEOP4*kv-G>v>BjKE}K6} zG+Q8O6Frpw678B?K@!1flc?)f@RrtY@=>en`G?w*(qr|GZDeyK*#Q!kFOSX^8fM#H zw6Z(vqc6T6FS?{ddM8QksE^>;_VVESfc6N2U`2M!-r^F_XDn0yDE^JVJhc$@?(`ba zG<Qum*N z>L+E~17KShESF?2b3gk@v1LT|7qyx`G5P2&IHn`IxWBqWBB2d5m3YZTI%qEVf%rtj zQs1Io_0itsB}Bocu>m%NP8CK%|GD6D)?92%o(8nkU7!;9qjlXa5AKtd`*oT}Of|kM`Gfwm#K1PO_2@;HzwM`bEpZ%ze+HBfdX@@84(6#rgG&nID?Q8- z<&Iu@lx#RGj+G`{Zl&MlbxlypJ^SS2P*WF~Bbfz|qfvjXnT8TLk5Du8F(?ZO+K@gH z7f%b=euD>W8=K801ZF-6cP`BedHbJ)k8~FC%d*}CRI8kG7;#9*@1|$P$ym995 z=8PwIHmIO8JqoTVeRlUjt>EsnR^4Uz;>>yt z2EG(VH|d0#(^?cOr(arscA9rn0(u>EEx==Crlih1wJv_pGy2VYp;BK<&U>ZV^=Ggy zOj*Ars^P1ieh{5II0hV?F|eC^sWXoI`t@fs<=?+=GnpP9=>z2fvK*GE(HaCwAU`ML z6KDR*l)<99gv!qQ5GGLm)oBv-JQJHC2+fbXwC;%BN7cPRW#vc8{eGnLXmXD0h7j67 zn0U(bz|ZQo4e32RtXTXtQ`6FdteSA$G!r@Ms~07QI@fXe5uQqsS&|$Aqc=kZL^SVG ztlbsQ>gT+%K10&_Fqh%u78lUuKs09B=&ab4bgN}vkaJePsWJ9W|JPlPRL{yywX8r2 z&Z|87HKwmyobJPd$M4xpO0d(ICEwMgyzCk$tRK!;|8PqnAeCMHX85+ zhU%tu(687^#Z?Os;Dx$a0~KK;uh7#ROL^=wuiLu>hahv}UyTlQmZUAHLo?E~<5V z99BdnL=Xg|yGxW17`j77x?8#=h7OSsM7nF}F6joPySuvv7`o%#oO3D{zR+=skb_);A_fLnxX6h0&Pj%SC0aR1R=P=V_AV$L* z<=jDMFR~;lxRdahzbVsnKf&hEUG}R>m`k^mR)0vTCHrB~v~6UdD`#jWQ{T5V)eK~n zw5MeKlYguAkG!>1e#Oj+Mj*mb3ZKKuG{5pYA>V5Y4f`5(B`NF1g%X48-gs6?l$V_> zmmPVfYx!M#MOaQGB(l9Hrgbi6v$9_dU=-BD;slG)FFQQ~-|B{I z>>XwU4LA*Gq`I!)VbnqBpso(30zkdZD*U#ZHK8)su539SNUU+YR{L{gD?6E5P${$2 zOs8foOZmpAdo(Uh$(Ai+IMsdTC%+=>)LqYGau`Lk799xW5)QtB)1~3fRWUDS^*l$( z&rzG{;87pA&qe$7=xQ^gvZwcAU5G6X4FrJ=smyy^X~n%YmlFBV>&ke}pxSoj)NjI* zXsBLX%cPVy7CpB<&F*-rs4>*dA0nIniyLj$=~488`&q;F^%2W$moS`tFl$4xA`g2F zaDcNL!oZaisGMFiDs6$-QH}1kzBiwXN2&#qpPb?_#bP_nL3+NO;t%|c;5OMbhmFH` zAT4)@>V?FrCANM3q=q2AN3%As-8LW849>n=-qrje_tp6k7_TO}?j8ySaqIZW{7nSC z=9u>Z$7xBT3jd8q54_kAF_{2c2~v(3G=KiiO0h|i0omkn96m%L??4kGYJ)d=_H$$c zQl^C=bj4nuLgqKhZ5?X>IFa?h&nL_nU5;)mRCJhK(mlYdXUBno>hhAI;_(JBRDLkA^KwCvR#Bg**q%grYS4nJhpatCBMw3H_OyiT7WQbEh} z*v(6eBB@U*>)nP0oN0!vi7lvTd(J3xju4F0K`ii$;M0V=)b3U8^DCA+`+T-tZDHt` z_9NS{?^CFT-QS%Z2A655rEb}Xhc}WYPC9IFJ6s4`ug>i(SFydksY2uSgE=h2&BP>)pujXT-sK>1v-_S|Owq zK78pBhWA#hm~%8u&j^i{#Ci3_JoS`^XIp*@R0D$}sPd}+3f;%PoZDzj3f=epiu+38 zVs?uPV;q?{2_(ZY{6iC9<}WauTp7C5;=W&NgJvP5!-g*bLON<{KYDlc1bRKA*l6bc z0sGglEc&IW7~HD&Q?Fz61!L4XI{wM){Mzt=YMK^sGhAL+hou0r||?l12dn}kjK|fl^&s)DVVifFJ+Dv zy|?0;hwF0}$lc(}r|vwmvLC*80LN>I3ja6alLjCHsgdP&8({g}uv!j_^d;20ptV1< zA2UIg*1b8Yw!s(ZCKFp|=iIW7xQO2WT**p!e*O6jNHmq~ke%y8XCcVQK%O7XlN(c} zn{M)Ks=q!ouPhDI2nKX$&RTSQkdQLqRPK&qqF^V^#UH0e3uH4J9`5J1|HM!5-L^;4 zTw@r8bjucy3oa3^fy%oUNU3uxg{$KlR<`-&#p*Yb45+56x|KDiS~?Zd=I*tH-tPf> zs?@HhkF$Fvq~aMWxrr}Dp}qZXg-YNPOU<*{BL-0h6Vpv`>+UnYHp>KaX?FrUB%@=QXZ3w(D|S$@ zxQ9MZ+4?gYahB&Ajg-75l}JlEX8wvu^!9BzEg?@)yyiF zngMxasy{_TND9?69*zLHH%4a$&jQOM?xX5Wuyd@mcseU4L~cpAkryU{(IIY8KADtOhjvRB<5FKGg{8qTPfO~A_`?E66YCe{!E_=dB; zeKeGf8vmVrK>-kzlSCi=dH`JK(9;sNJ++k|XO~~Z)uFiof%4hJCc2MrSt(A!{J6=S23XPyNKorF@hF_Z{{K2lWJ%p4E?-HJxc{zcc=lsJiGgNNJ@BsH$@*gyz0X z)|;$3wJHUIxR!}^T~j1ZE=z4H8E2ovDsR+FB=VIq#*~UjWEw9{d5~-4rqzs2PFiR7 zM)TfeRxokTS6md$Scl8*l9Lc`MK4+HjVIj3u*BoL?57dHJ>4C5pAJ_Fv+zv13fFqp zj?VwZRHX~&z6+BwAYPMwPx3K@K$6T+io|KpBlktW%L z87-j#y)SIKbpC3Q!t_G*Y_MD~SqWXKfFO@CV_}5YEyaf~j*Ts(O8jeoV%ea!H>6q_ zGBjvh&ivM>R>pB|-+N z&Wm50gGeJQ=%Bu2c3D}z+|Q2TY*!6EgFpAK)eIha3z#y|Lhr42#Re(0T!j?@-QLHO z6}*=X+Gjxy{P9=5$CG2sw5>3J*PfR*H_xxU*=*vh?+_ygVg`1DH!fDuL7B<{@7)b{ z*ZXzoS7-)GBVb(V=*j!@jxT}4lr}LUl5)>4>(b?VKsgJ`kU`@AGRHAGpChOdJIPZz zzP%1u5201jfn;hr&W+jjjj8xu&wCgL*~6gn*kX^cV&W1GIIS;+l?{git~m$m@I0r= z#N(@+dY=f`Padr2`m4{0)G-uS_xeOJhfmq9`zkpN+j!ob#Mp1oDvW!chb2PB<4&L4 z0$Ht6^LM%pZa^3ScR(RZ#+cW&mf1|W@&m~jK*$X*%~aj#3V}mrPR(afaT{BebzL0$ z)HO=Pv$y`VW6dij97+nn-g>({4pC?X9Mhr0L{$sqwS~ovr+yHHviXME?wXC4Qfix? zceQ3hm#0t4`pX9Rq{oX~S?u55{=8k!^$|t=+J^=I_O7UTq>pos^gasXNi$oH+^k}a z`rT4Ts{2@~JHe3Yby7ZYR!ebhw<3&t8A+||lvm?oGxbfT#cN91oyAHmZ$luXE-2A; zkLz$>I8!(<^9<` z93HOr6aAo~^R{1jNf&-7Y&gi8vui$@5mhlHum+jXrojKMS>}+$TgCP)hb*PwImob_ zNRAVvd!M5!)pYA^IOj?-d6O|Q3;;x-GA>(t>qNFVL|rLIBj)*wu_6QO{Pi>9zj zxCIc8ER?0!mGE9b$+}tfV%e;Y5Hc=cJUCJ=#2}SPGY>?0uTB)XV-{J@M=?kr=gJO%;`4A>8ZA$; z;M={PK&# zd}|Sghc(Q=wr_Y~bLfgCfk5zIYGnP{^Qye+dKd3^M@GTyO7mbDNX=fmJVa?>v=)e) z7jIm+O|J=XPH-Pl#5~M(pbuAIC@Ijc2JfCGKHF$F1L4ZKZxi)Tgh=FPW|b$-Bzne z%&XVOQMuu`_IQ(rakO0dhF)B(rO#PH2*S@(gCD1^|XWe+4)PM zbZvpSQLa+6bf`FdbR)*S!APL3di7l6;Z?j;DSj(wmlkKhGVQzm#Xp!?$yAG`uC-CZ z!L;gt#fY+<2V}a^OWlC8WMwb(CV7IC8RJ2Qwa7RPa3YI*nX%jx36Y<)(I;D9%KOv{ zMHqOUdC-*E&Br@I$h3!FrEeZC2RNe|E~RCrVg&W)3qC<%f)(w9%QRvZKFITzX`>9^(J>X>xiK&GUhk0eaiSa|w%&0J*ZqBTPOqo~ZF_>1 ztl+qCq7DPsiC#V9$1}n~2Da|qz?1=O&afvQZ0BWwgW7gd@2SVCGyLY{RP&5`3$#62 zpw{vOMa@br?&jIxnn^-OP=du-SmT8G6d(ntq8L;{W3*k`w5#eTgp$PS0M9LiuR(jW zrp|hS&z(TkcOZB%+2_7kX=t6V*gvb(;9*K+0105!hpGrtwZG}ZZ9ms>TS79AbPy5H9Zi0Ju-8>aUNSbp0!A^hjay(8ns z6@pYjp&sHltod0gd*VWKqu1bXCK~Ax+Oh1o|=9>-QI4sNXwCdS49- zQ;&1h8G!sm#bkzaJkWAKoO<~$9lX0U;Cc@5y}-#=Pe?=C?){_Z`8nq;@wZv7k6zyn z^ga$u4j)>{o>BYqC3)U(7r4uyQ%IFH0n&^z7UH(qOtA2i=b=C-h+O%w%pXc-uamF( zv7K1)4BoE%74aGxD3iv1*D`SL(H!nHui~XP!IQ(_kP7}kl*bG`3N)gcA|UhpaHxY| zR|72$=3rh06hD>+ypl5{$2EKRw)*^biNlix{v_9D&v-Kgyr;EzDxW2gYj=oE3f;M; zr`n!1RR1TpTQEZOSx@&-7m3q1c1q$Q3BaC;&mXhXsxsL_JgU3$%1Z^p`^E_L$8Ab{ zdT)9pe9i#vQjcZu$v?D9D}<+1`>TU&uMb9wv^aF_&X>N{{Yq6q)4$$d2{{A`0`x|V z2S+K$(wVdfrXmkdrE@y5XjKaNp%dxm$0PO?frT%C3P^@AiB$3!mk z*MvoX<)(nO^~V$b-=Wr4x4n&KGfnJ(gaECrRwt?8nU4H2>wp6}W)4TIqX_HVw!N&T z{0(m?W-!@8Zdv79v^*2cYj0^?%ei#0Dn?-59?UoTP>DEB_}wZ{*nrC<`5}JMtmzkh znz%tzm;Us7>gHkpP%z=HMWS^Gl;o3wx{FHxx5MNCP;nTOZa)KCraV00PrrLY6HC^Z z>ErbH6OE~^C#PGKDmu=lCOEi0gAenQlSmXGTZc*n{(Nn9`cv{i936#&+PuToLew#w zeQS%Yer;F3VM=F167W8HN)rLjdQiW}KLE?f_c_>qF(_vQgRqC5EWniY= z!-D1*OwoAcXt8)1<-W&ZdN$XRYWxXve&#$BL zieZv@Xs;=>>IOh7ig8@dBI*&@+9@D9 zj>BAI1ch#k&As=mX`}BKZN5=!*N1h;jpp^OPV>_7TcWoYDZ z%306I^6kM*x6H(DSyHuv}ge1>Xm=mVgkS*7wMx%tjVIQ;898tVn zYR(&^VIQVo+`rYwo~IP

5e+bYMD{yuz60n@#qpPP(Lu2wfWDXzTtoNFJSKGTpJ^ z#)u3LXK;irp{d0HgD9wASVAM=Ze7uuA zR1W*XjbBo1XduOh0%pymnSB1wW>sF*BBbiRG@jlwB`dC7ac}FQBQ;^|t9!voE}Q<$ z$pspCbT-`;Rz>3I4;`BojtRM3LWT>)b-3KlRhHfbY43sl=%CyB1mPuAUznvN=&w#b z|9My>@QS4b?asRjxEvGZ5%B4K|27`UNeP^LSqT`F{FX^VyFa|E+bkKe#-;1o$vz_- z-S|>LG)=6WJR`bqTXHU5>A18(OM3l%c+7Ct&r&AeO)gDa_H-3uwb3!cQ%1+ZY=E=vzuZ1Sq zT~zt$H22xF4K7OG81(DcKwy=t3KK1LhTk+qrycfaFhVuc+12o78Dnie9Pc(`zWebt zszs&$76+uhf>=8$9*khz%@`Jdk5?wlp~i#f%Y z>DDl+UxxA7szPGd<2I#K9h3H9#a97b(bH$muH5AB5mh^y%%MS$Ra)hA6JR$7=sP!#Io%yLuy!bT`x+vZ)!848;+YQTB~z!52fz5SE{;cj$8 zuYB{hL>CuIPG5)qjF4f*ac5NAmgdZ72+_QXze7zC6$}VR3=zkZNNxuL+NbC@X^tva zi@aXnjlqq;NQ)S=Yoqs0%d zI(&JCDBzH9pb(&6<$t%a!iYg`RbSewgL{VlOij06&GK+}-OiEAO2?oxTgwZ9j3zew zqI52P_AtU-FnHm0P7dmei29?e8Ywr@2d8h(yFpL>Zjbh7gD(;E zdz77`*@T!)$mA(AlTiEApsI8>Q{_YWWI)n! zpEsN^*OBU3Pjvp|`_h#ZCTY}6NX@H%m|nH|-swx?V@XuG08)RdgMG?dPhwDLDK>1g zMZ|aiV~p->fY^LL`qiN4rg+9;$^)kTP3)89e1C&OS4^mEB8cL>1`oA>3ZsrxEyRim znb@`WrJYMvGWeq}jmQ%HnTM8^XD+67&riOjq+smvZZGeznF=9-+!_&n6&Z33KtRN2 zy%od=PS!2gmiWi1l$;DJEsyIydzqt7#7Z{WR+h!_xEBO}H{AZwl+5cW$S+DVV(k?zV4o0;u!Wj>i zUk5M3n4AtR<|4U9IyPaP1rB%z9a{EOEpJNCB7RKkTAv+DQYfxgt2EOu+cXtha$qg9 zdb|!MkH2=mx#E7_`)YsUJaF@(j1XBQ@*a1uHgJQRY!B>Tjm7Ij|j&6{9LZwJ!&KYVd*%^m=PO2NFnEd4xs%%{ItMf-~*LR#v zCqb(`UwNJ@FMVP@7j}y0^^99QBIuekPAGnVxbZH~Yzg^=GHWF-)p2$J-)NTTqZ`sk zuG%6yLaWS_*5v!>b!XDWw=IQfMFuU?dK&&Y`#$t(+b)OM`#xXgIju*8_dU5Y{M~3c zE);aLw*fx8-dr6wL^plgPRZXKI z!F6XlF>j6grQ+FGIu`}q$4-w+Dm<4Vbt{4UFN=BNX#VWwycqSsm!{aM`hQw_>SNy{ z?+s8m_3-xV`|?(W(n*asIXW?~)#9^L>k=4EMUWJSYn>nEYPDRoB#m27GOkef4|iz3 zW>9@&we-fA=x@v?Kb8T$LAWhXu4*u{7&RqNL0<#L2|r^YQ6_SjoVsYv^7p&(jq0a zhECtI*72@UF@H?xjH@*XlE2Z7g^ZxjoW3)5Pi+U?-`MV2Ste0Wm z6m?_9dLetVPWZ1WC4l?a80ahXn$3d_lBzNMTTvZR+)<16wEX>#xU?GI>plvRfQ=+#sI#MD6UJic0u?vYt==pq$4+xVHk~-&V85fzicW?=yhQ# zuUX;ziSXCFx3h`(oM8fynD5-z>)Fg+koPU}AT%r~U8#M}SK+Z`(` ztXC<@s0gP08fESyQfG6f)&8;4b6r1yXQ;!?vGw-4c?;f+%psgGl=T4`1F>hHrmdBG&s@s(0)bGnRm7>ovFBPDE@R(@8zBY?px(2bTIe+$ zXo|vj*=M%vTrVldoJ3S49tHMFt|7TuK@rR7<{!*k0O24^pa2KD?-Sam<3(R4)oZl6d3 zaPY&2m}UR&%P7D|h7sXIL)xcLLbvP=2B2%wSR=wVaMpnS*3x@A1Q|W_K;SaHF?$FilFr_Mf z(Zh`&Q&Nqxr{WL6%~&F-hfy#X_tmip3p|?6Y7AccxhpJ|GGNsz{72j?Jh>(0lfBsr zunYPtFHt(heRgERNiC>MqH3|Li{NVRqGTB(UT0F|I>7CN^(e7amb!=^>TX?kec^M6 zxxIq(Z%w*~5?)@YG|WWoe{82Cm(c||NvjlTMr@y`Z!zX7WJU%UeC-T0g`O#kz-RWS zVlaXJzbs91n-DzqQL@6gFR#kRV3ni!{*`Y{-BKbLjSiQH!hY_5bd?zwxwP0NcizsW zIeDY)wnm8Ej5W>YeEmT(cCg|5$B^*@!A#rXOm90yHoK&{I;$$HD`mg$NaGt*>Cyy( z_AEdRb(vyE- z%a!zne~(5M+eM9ah-PqaYUB$@TRf*(#Mss`=zQ8pB9eW(Vw{Oyqc-*!hDR<^ZN463 zR(h&O<%Hv2_oF_ZGY0#lcVVKbc;)B9%D^0#9Rack)k8N2QkOu~+wZ8O>|umJgAPhgvN4Ape46iWT#0h<8LR}fmykC zWf{$BQG7GkQQz@-l!{mEoW_Rn4tqzyYF8zn;p6OM8|x4G}KKx>n63%0za3Iy9rq+Hv?ChIEdaTM~b4v zt670^v4r;Ds*MvVM!ql271z?!aPx`ce3DaWq%qWax-qclZz9EDKn4_}sJV8rh<|lj z{j7n&`re7#@zybSp*-CxpHmWfNI1T7&uC2pWSv(1O;pV1H4T&nEXx}=%-)Sk{_C{Q-dlcznO6+JoqY)x;j%TPP2GM=k-CU&QBAj2YPipQ@;RVLkpL5 zYHr+fM=%^{^uDBnr;oKZAR=SxmMKhMuQ}lHxrA3RkvC&e!}rdus-g)F0abcztyUNJ zEgJ<@L8eqxGkbz|N5C3wUFnoI%ObKD_u_?&BCh&0m3LSWNUzQmoI`*A z0!Fl=JL?m7xMi$SU#DCC=`JyeWH`Y49 zvy+BkOT*z|Dh(YTY;u(Y8h?o;nI9| z8^^ull5narJ4yzCp0J>U@V;q#g$X!3{wnsx4X;>)daZ3?Z(^AUXR6y2=5U=W&%}D? z0amu6JMZynn3sEo#L{)@VJw;vV{P42w~`YAJPxz8{_Xe5Gv^n|c)V8e{a-tAL)UhO z3V*cdoMDaLdm^SPjGMkWg-;ar9{b`8w-C>Rwfht0vSePZD_hJS%rS5Aze7@z+BogY z?7o6;8y;aoIaaQJJ?u&-ci10f`9g~&fq@#slBh3we0X*OhrQcBJ0GFxz{T`8LcoCT z^FBZ(Z(V#iM`apr>}K{jjevYPT+*L#q*93@`zlt8RD=)wf^RpOzw@i>MrW#f!GI1D z@to^Zl(eC#2%cYv3uEe)FvB%pfw3}k0_US3gwr$^JUd=ese&u}GQyi(AA$JifHaG{ zv1^!q3YYAt+#_k+#i#qi13$jG3>ng42Vl|Cx5%a99EcIh;-sUp|irs?L#Ay zfu?+D+x?R4?(1aTU5dSe!9)fMJbHL#s35`d%8}h7U!&|>xrzEjcq=K-`&dxNmMi07 z4FSQV6&S|pmYO!(U(VrsGKkEOuI`@)>(XhC;2ma~-rcHexJd*t&D`;u94@-f*BJ(C z+~`)WxWuzTpsW2+?=eaks3`%yn^(;9V!msjxmcoETmz5UWt5z8a+BxouNH=G@?XHtp z4AfBx}EpQ#x^^oIi=A1^67{gvYJ_Liijob$>Sj( zfXd$uf>@ote6iG3j(hTqWLA2fUVczJv*LTFvbSIG7-sa8zdv`TR6Ucawi8fydeJ|g z@w;Ttj1ozF_P~lH?4f&IwYy28yX{6E)d;(}LRFn3K+7oL&OjtZe-YnN5oG>YiSLpX zv7JF(p$U!W>ZJB5)_tp1;0SVX5*H2NY8is)Asl>FVvyKNoI0+p+ms8GX779A=pSp@9?^j1bs{E z1g02)p09|ZSsV*O^)fsxzZmzi2Mfj2^v@xKbIu2sD25xQJz@LRx6(OCvYqG`FXX7l zJxV3Tecr}Jvn1x>#Y2vQJqi>n!iY&sj2ouMGCp{ztO4bvqI%;JMH%1X+z&IrN6d;d zShS;7wiz(XX+5tT0pp)Iqqf@-wWHoF(I4-t(^IR_=h~Vc;<}9LGMI0e1HWG14bz+z z{m~q|Z5km=7FMYkQD~Q2g1&_L1)rREk+eRzEH2l^a6cMP9&rYbaF=@0%!E2p!qx>6mxtS34V&f=2<#_UYPA z&1cPG+jWskfgH0{N9&P=wb=t_SK*o*ZByGGBMyOYHC-rM*^=CO6L!}xn5Wu z-6|V*WE&BOl6Nh?eT7+l5$u2cs?0wavazQ%RdFSr?ViP_<3sb^+pHp~NqA8Yacd+y zDjRAqg4nj<#>vhMm53cwv}_P>J6UB}GW{7!3cE9^Zp;Sb>s9cOqn_Sn#~)U+;B+Rd z3^yF7)k6DW@T2uQ=Ky_cd8aoi=d%BFjR+tB6{CH#-%aL%sD5)3uWG^1%?h)d;?T$c zRtG){$h?HUPPI-AXVWw!a^ziScJc2r#w9~gDklS7EXeS^b>4x;PI6iur+f%YlOZlg zh4ican6(&&I|#P#eBSY)>aGIp<@+O7Ep zRJMFQj^-dD8bSdfgI}QnOahf%jsO8EZQn@>#%ZAp9!y>{tr~HVK_ST^4@iAKKYW3T z3uGLmj(^kzJF`-Y=St}Akw;u~Pc1ZKUGiJ+?I@3umd{myf1n8&6c<^}1!G-7_UI1V z0mB|fYn9Uj)eP~4SPxZw7$=_V(VoaU2~0{O@}&aJ)j8?_dM^%=)x5l_<>AClD?5VF z;Qp=px~(MvV*0&nMP|1vipr-{PWVbpeYlzRb-OG2t3CT&{W=(HN#BU^7ye>T!j1`> z&-J&2o;e`_h-nc&iKxaoOvm~p)opn_8m5&pMI&m95>76yJ7mr9rkN|M`*Ju=VPkUL z(e&G#J*nr%hw@6dWw~@|MKc8)c7wS9)|e6hM7*(GeZ9GHv59FQhBK*2x-C=>JAptm1Q5 zZH2iT20dl~Z;0Bw=VB=7bOg-DbEgB*tvug0)AnAqjw6?ai+#wED-16;AL}$thM}Z^ znKVn)ftVmNz!I+A8<#^P)h$%3q8rWN5jirQDGjT?5PN7kSdbq5Se+xE24b#klh+|q z^hMP>hq(xOc?x0d$qq@e&kq^Akvb#fieMAr6VYU!2Czn@i4r5pFeC zTS&uJ?4Ur?PTB4_hTiR(n6b^Uu|gv&S;oO@9}(S~kU{aS#(aS!hFQAsiRB5i^$Vzs zV%j5qqG;A%Bf`i`v6}Ngv>}2SuSIY^#q>g+Tls$j+PY0+L%Z*~+L^4%CesaS`gCXf z{XZ!fF@C~)ko5$8BurX{<1G%asFM(%jL5?Ig-WT6DAv4+1LhiLue0h7TMVZ?Ezoi{L&K`K)o)Wpr`-wPBmjoh*`cRAMgs z1y`Sf5Mn3LKivMMsG~CmB-47}lshk^?&j38TBT4SY?V|c7Or<*K&!d0M5k4n^RX*Y z@ZJW*Rx2@yuitm0H=lz%JoDWu!*Ri`Ap1m44j!sPfo&=mL8>)EXv+?y@O82qDM+2K zpeA>TqVo`jjqb7m>bA5Q!~=uo$SJF)M+pe$MQUlA3EcR3oJQ+*xNGwsWueh!?}=VG ze-*i%S}*=MKAV}^9-WK6K)|mU#${7{7D>+2&!>F`q=9g&sDupsh$m4l1CFtu$Hh?A zR9gWH>zTaN;5Q!|y~z|*4O+BcG=^BtA6D$}@0ZP1+b}5da26`qo<%^L5-yKiNy=HH^IBmxjclT+3f?7!uX0EkpziH_L*4LIx%AN*g6Ea^Ok ze^+zFmJfqUQCAy3b8gqiD#g}}4a)cN8@ytAZw&XXs6Xm|N#H$1C8tPwXQMuG=C&Ql zjSx`P<$B7!+vAVpi=16$H76HljJb_mF_14=0nS<5l#OLX(L=xrBU`RdK?K^2Dbk(@AGjMm9Fan3$>;~HVeb0(`{o`<1T^5Jp_?<{=9oTp z)j8x)R`qH7D(pxqt0w&wpx}O69q?6Zo%UQ=X99|T_I%;S%xjd2d*fW)t%1#}DR3N! z+N7hn0TCs3-C=dtHq~*PPp@89=c^qL+Sm7?v76PXg6Ghp(IDO&rPcP2G{&}DD$+#o zlfz?uZm-&;ZFl1hhH1#PQ}0-xHz&NVWXW~?X(x3k==uvpj=Ve~%KrRQa-AdG`jk5qbf>SUHuPa)$ZlZIrTbcoC z>Aa%4QEi&by2_U13I6Tp1Ml`X&h24x1wud zrFX=}ZEBAmgzdzeeB1x@_Pj|#pQ_S$AYL?tU1$5PGX~Y+Xk3}CAhKf4qoFquxdSb+ z;rh)d_7~ksxpKu7zO5rI7zNM5RDYg50KI&$kQxp;`ko=1BsAO<$c?=CIVzrJ| z*z+x~`H|0qfSZM;)MqN9)wqhqGg6{Q4Y@rP=3^3Oiuo^#xR>>fP{1}ND$cV1Ws}}h zen>n3J@L6mAilqs;{=d<*P~Zf)c+Loq+Y5LfJ(}^!^RPZsLur~wxs^il-oVnoe&X? z`ZVYXI4z^3u)FVXAtn8eS=0_k0PW5Y@Z6TdJA{>cl$oK-1KF~sg?3(uX$io*o7DX) z&J@681Vnqlrhhw#|9J57e*wUW)rwK{Zw4hN3(Ox$vHtLXw3X@mTS1Od5C(GSPhsHj z-@W|vm1gM&%_HG!=84}j-47nu*sMffUrgx8&Jb|s&;`LJK25h{WBOa&V@txjkd1p7 zTH9BL{FBRlMkE&l=54JmdHFYtMDF)57aaTaABzU|;AH}Ah;I7BkpKW)bUnExcw-eN{^o=HRVp3O|mY0`L^PyrvQ+Z^fYVF0^+Lb~@qut&Q8(+qS`C-BI=uONZb{H3yCb_YZQE>z9z}|iJ;X$c_-UNb6ds7n zA_?3k+*_k9y#vqwgBG9Qx9(iFtJQ?(TR#%tLBxu*>c}RnXG0_DIYdzJ*~yo5b`e2=s%)b`@p9c&aU=Y0*%O3w2;?Lpy8}0|1Z#3xn*CzCtfsa_LtS8_6{>sbg zo^nAX4?ym}fBJJ#UhhPKKN7cPq0j!Nt-mgY-T^FN)yJFR->X{01=f50B`oT%H$y+) z0a$m)uQ2{Um=y92I0kW1Xb06l;c&kRuN)cxwA*+`NdKDRAO!g6d}0G+@O!;pszSg@ z8MkOT{^#=6?VevgmFWF1()??-Led%_uL z7_j@BK>k>=zxh3&D~Q)F{@(9jcrSni0Go`lTIBaIzwz*v99WeOr5gI*6Q2WQL&N{b z5c1dZ{`0Vp1WfJXv+r#3d%OPyD}Viax_$LJ|KH1sxnEX~)rRC>HxB=Q0Y$(v8*TKN z|E~#x?gwhuHc0q4B5~g%Qp49tzTaPYIY|SjP`~Q}efRga?x7+;ND096xdS~o;BUhG zi!c5^DD)OM{jNJ1HHyE-I`_Soqq-ld{tY4wihB|BV(zEd|8)qF_uQFERgLcNk^Wc} z=pMhI=ntTPcomo>`*ga)B^k~ZK?a0dU4`@J7M{5kS(=qlX*k-N9r^?%Iw z84Cc6?7Qm!!8i{xjLhZ{U(}|a?{Ri>9LI6SP(Ya}m98u(mWXDIqm)Y%u{yFz_e3V- zw~^DSW{8~Vxdkl`Ed+Nh1C0kuI*22FJ-oS5V~2gC zMu3f5hc>vLN$$G!L zOzf%o1Z^`N&Tq56s(&6ZTMN->|Bg0ZxHH%o1Za5lepT<*IYXKidf^n6q5nQRH!2JT z-)shz;+}vAAOfCS3A8RYm!7G`q~9 z`#Y@%Uua~9M1kCRB4}DKfiRK`^o_8`WQ7JQxWU!oq(;en+=CnlCX3(Jhhw+ye? zef#6}8zIW6l#ikr-Eftc66vRGriW}(6|Tkeqotw>beH`ar*j8p7eBp{KlWB5Unb)A zCgisEBeGszAE?!6oM?qp4s>2c(YJ0^`i)iRs}xIa6!AhTmxLctx?kGsVK3dj1~!lK z2*dF&qWuqk2{4jlVEJZixjyd(!nO@wv`sFITCf;LHa|WR-4vPBdC`nh=u1A*G&KEz z`1?Q4=_ee()e9mz2Fm|p8-ECi0Och*@n8#lu2raX40Aq`FlQn2=VLaY!HpAx*ld%h zPM%_Zy7Q`BnEe+q^omIlhouG2y9lp`UbgGKYFcAhU=SIV^vj6vcyBu&QK{b8uIB8_ z*7R`8#J!=)l=F5Ocln3|h)X~1URnaVrCie^4)sOz>5NojYt|JDeQSpPb%y{pDN!eMU}9V`I6RB+KegU@$5R>p2Og4E zVx1o>4h+|be}>@C@aY&biH|iGI7_cx`v%rtCUTWZ6{0VM?5E29d8c%s+NimuKU`(} zG~}v=!WL0BnTv)0&Q`>O@4Kh?;~PD2+?$LD3$-$dLi{KSCc^t5SPqNJ;XFR!%a{z) z7WMI3n9!Jd-`Zu%SEi6>I~1ZQ8;*U4664?`K87RQuW)YZP*SgB_*fWvsuLyeda^fQ z8~)$SCPdLmNZ4wwktrylJi2LgL?@I_tx8Ii+F3J@lgD$iuu5>aE0Hceki0dbgFV6(L?fjVPDL3)+?QOjhcTxR?(-lq?5$#26=PcD%+(aA& z;M``*OXu{^9DQ36Q82g1a>gQ^C*E*&1km9{iP-bn##P~CN7VxCh=kLlbRJ6GhdNv? z`KaOhk!;K!6$AhFFB`28OT$lU)$Uip<(3O-)@?Px#CWul5!8^bHC0ibM}Q}ifli(> z26@H1zzy(|MkIS&s$mdq?^b==yybE^<~->r&{5(CQR#;BsK_!jsePgWCGk4S29xmT za=molV2T*qy18e=)=e|HRfg_shD>H$z=U4U4uO#9L$~eGa0Zp2Ws{;cwk7#rQB0LKVy?52`V#Sb2+8Kzg842o`gKU}VA~)}Rfxf_ z3MQ4&kt~mk6Mte2!2d!$HCFAmw#F!HI9sU8cm~5Hu_u*e zv{(Bio5USOTC09(zcowL4n!EwI`=)^xR|vY*n%M#u+%pL?Mqz8*w3?jb$&=-Xg!+{ zvcj%-miyrs!eQ@QutrBK!X^I7>a{QGHutSY1!qO0tUTaVjbs5aY)JV*WTyB0v#$R)AON;{fl6n*AuPlEU-^w-GFjd!7R>gys zoPo)sukANOYddu04c4vO=%ZdT#_9|j7j-E))jGnVi??nvT21F}U9n`|#0KZyB~>3U zk3;yGw`_s#(rJtFp7OjW{p@(!&~xt&$2^i>2PYHSV>gbX?%IoXk9XUg+8Q!!C^{Mz zW_vV4RxIb{n^@}i>lf$aw1b4LH}If#)zt@~A<&>O~|2L$pWq<1!3rN`Txv$>lz;@5m_ae+`Khc8-Kp%pTbJ<@NGx?Fv#H_!Q$2cs{0<3Z zyM)~L3iF*jDJj|eM2rj?*<__d(H`Ysn*GrnZ6&K5no_qC!~A~ln9RTpsl(OafW6pv zyl$+$~7s9)e47cXxNYlePB#&fe?XQ`JAZsL)hq&pF;P9vhpXhN_n(RPKJQb5SlZ zAEP7nYGFA}RLe|8*J~FGoE(ZDy`fq^#%)o^g6R9>StgoI)%y>HDQ!M$H<%UE^V5Ed z25B5SzDY`W)hmW$FidERfB0n8uO_C`74Z1>=|N)k53-g?6{e4-B+pWlql%C!ie(K| z@W*hBq3MP8xwDL+bXIkS_XzleDe;qAsm>*5pN;2bTHUg^^rp)gnoaDqGV8xoSuFuM zgRG-8ojax03*}Es&Cb=N7=6ORDhrjm%{E8u@V035c}D*3TJ%f{r_aEu&w-Y^@Xdpx zNXa+XUh<`OOgeS|{46q3Rlg-m_Amr$?$8rqXwMM|@b+>qvctYdROQh7GYflO6I-o= z+Fkhe$%7kKr_Ydo+fr&)v)VDxQqhZ!#r!au>X4hrKqiGt*?V>i>BG@OiJF_fBrN6? zpM*4)%h!C2^7($CZC^5RKqn-0mq}((g-5D@0s>#?07s_X67F`B*U$p9a377@Azlib zS0aY=_NdbjhE2vtCX*pT;ouLom=lA?JypCDy_VCOOtD_vJ>ucpvbD*EE0M;1?1KBC zug@^HEBaLiZpFGaP0NsT1s-mF5gBZP}NHb4cD|9qh`&|5EgY-N^IZLN8g4sQ!_a-4t zF9ILzlX$METx)ISbi&Z64poLuyTlwSYfL*>eJ$S=S2kS}-CEt9(`f$u7Rs9jK0LSz zf>^%C>@r+gUIG)Es6KxqYS}Rx5H|S4=IK~;fYGItHce;L-N(pPdTmB>K_umOV3v zr>fmRrv7d*`t-7LG=b^g6Q)S4fE6ZaG2{6MXSng)fRj6BAcaf7V?7;D7iuKjUa<>% zuWjy{`R@QK2O~gwEGG&Sfwhn`8nWf`uV6+FN&&sWqy?=gM0fz^5@xu-{l~zZlP2&3 z2^91Zaa=L-+xbRs0mEgE$i$HyBDwF~<=#9W1z zLgc;@%gn)>D$9ona=Bk7d)0tHAPe2EfpTT&~j-8K|B%#3X7M&aczk-c;b_ zJMN*lt6wA@(Vgut*&F9M6xJPwaaez1BbE?l3feI{i^eWI+vzkrPozH?WW>dN3Pr8u{Ro}FHGPt*cWOg2}T@4HjvSDkP!j-YWM={ zr?J`dDi1Xcx%7)cZa8~S27 zYt~7|i@t^^1i@&(6%Nwmi9XRphuTdJn>1FsLM-ijpFoo`^?3ym_ffKkQget@1T0lR zOoXR;1DlT8gPOzdTZ$YQJ#!i!athgKbKDYmA6>+5IbG&{SHI$2GcpWA?;h;lc={l^ zH`Uy(i7p`kOe8ak`+wx}K)g-6682^cd7`Lb_=i8@Sn?sj<^Sw5gL6(SY9upvz--u& zWQh-rb{6JAbWgDHXR(*2<)#8d7O(u`8SX7Aae-um7|^Pj<8=;M`2!k}e$wVs zqY7Aw&=4Ry+C412G4V6LpV-HVtCVV27xb8FqmjIdQjaD;lmZjJ`ZYEqsc|O7%k@f*ojp;!l^Kj@%*EIrax>70{D`whY9h9YJBHAg z(|Uhd56R@vqFg9{@XTrm<)#}*0rTzMJBacFh#R>d9y*!Z%z5Vz=unyEGDN`f9r$w< z<)4RJw=)EzsH;}HUFl+zmpazc-dF@BDio#WxM`k!ppxFFuIKYG3bEr4@-IDos9$pb z#*o%Y0l>wsT08eBj}WkFth6f~W)LeibXdl+9l`IrwCzlu97mPDXfMW`w5^aJ{i#^k zzWQ)Dp(&N%y0F8B&*dpH?*e=GaS825dDp}dx6736y!ep$?hS$Si3>_olamEmv2q}f z4yJ<~9H!@Rs>iwtM`D?RYg4udwRNgWRbKC`%)cS9e-CdVcaZpvrvZ|ivZ>%I@~%ql z8XXiesXU|DXc+m95Xp;jEw-r`Z$ikH)5i<@tpQcC5yxsHG7$@E^*X?%!|8e&9frg* z7#oShT1Er5|;ZkKHIv%Qb#cNAZ~5ExCsq9`O- zM4GX=sgP!5jR_CaAMk$LrMwExA-Locq0uJ~bUB_^Mw)^V673MUQ8XxfR|M4u2mGhT zQO8%@9_WT~AC7>sfo=RJ%Udm;sMf0ir4(>i1(q>cOZ_Bsm>ZlU`z{$**blVSP)@R$ zY-4+fY|*1yb70L%QHR>C|zv&z8yYmeT8o5EQ)G9D|@Aq&uDg3^BYQ zakobLXTH9$UGRM_Ye^>iuBV~DSl`8H;q+j`vsiJFe8_R2g8+8i-;fGdj(o%?W+lRq zxwt}0x*Wp+J$=`TJN+o!sWG2Fb%}JTQe!U5)v1kwOWaA7I^{EVHXP!%{q#X!b#C^S zgb6#Fq-;!XoZ59tEl1-+YWQL8S3@=q_TUEC{SY zIMUXRd8`0@CH}!5W_r7^aG2c#SZ$744sW=0W8Cd1cv8I5_+Es>jor6oUKhwb9F#Nw}f_xhOgqO*c zY@C@VfuWV^b%d>1JMx#Oo6I5oCOMQM*a$$9bUtr(outmgXm9Fz&n^af^XYGT6N%y# z`;S|~v8Hgk8ExN8Da<@~Z$YIPm6(`bD&sMh#*Fjk#k=4ui@?XYZ23{gyETd3D{QI{ z=)xm2dCDDn)zl+jKR(K&VWs^?7jK78BI-1HT~GJUJuTwe>kJ!X15#<~8H_>D1%qI8&+B#4Ao! zjb6Vf53V=Q8fYkX6*@P>G=<>cbQ(^ZSuBxJ>K6KRXLrI+9iQ$$-g#bNkjazAjpv6+ zuXjg=zEGY}VefVG%`GYG)wHgB@8sH7Jl5gNcVXxy$O9URg<6~JErK|LlMsPP{RCm0 zBBPu#O+3G-)^uHnpYjdCR&Vw?@_@){F_TforH zoy~F!8IZ6wRl3yd6svx*#oAovA_6TGKN3Mi-Nz^wOLOufix+mTy^3 znV)+_(SEi)L0no8khKx*W5kBGK-*alnf3C@pC~XO3Coq(B`J#{N+o~Jo-50378P*~ z+fA<4Z(eKM=D)jSCcE;=Ti~_qF(RYnHP3k0^Bl#GA78froV=N0T6VOIL5Tfy`pbq& z?b2F$o%xRc&M}v3x>w5}o!vBuwc;p8zOBVAPyA-2Ampme;tIzp(?!D6!9;c{ht3&5 z+r-M+UtsC%C7iRbb@{d@kCbQvnoZ z8F$289Fkqf7Lep9!Wsf$G$9}o5el6oiI8;l5*1kRX8LC_q8*~SK#IO5;CUVQV*UDv zRS4tM7f!%9@3LDGNEreGTX=^I%K&sB1;V@kb((TR8ibwJy8foXv6#63QT<3MKC}4S z@^bB9#(l+-c~)QVta~=2?b`MS?_0LN;cvouSVEj3H`dJ6Gsq~w_sCGZVvQiiEIzOi4y3?(=3KYXHC zsPm~rjJy7gX#`B=jlBz1m<$gZCZqZVHMLe+@=>3Zv6J}4oc zi-1o9INz&RE=2>-|G|5=JjE`Ju=xI~U!y z?Yi${rDdCI)6!I_`W7Y`DN$;aWUe^0)N)JXRf(=EwZ`Z6us5h&j!vfm#aeHKd{ZVU zr0y?A{NB~Q|A3uMF=*&E7wZP$3)@f6WBqtZQuvZbvOoM?FcL;uwG>cc#yR0=p8Dz4!OQbbK|aKig{KftKWr8=L!HBpcf$(CUVYsp07SN>vEOJ} zZ042V@!Q0tgr?q7`}NzQ%fi?Pr4ZU?4)MFKHz>qSjTJ_u)=tvZ3?g6KJHx*52{2nK z$OSDT#AJ(JAocja6W^8KNntjQC!#)&V(5qAKZZn!U#UM4$L4=~E^3g%Y8bXaJyxh( zI6UdolFq$A1VqKi65+`Hx_1I!8WGbpmiE;`DGeW{C}-rM$F1<)t%TrQ8-CoKk*+p| zkLi%%Z21&g?lvK&)Z(hxI*5z9;V~Xib$#f5cgjF*G2F-K?lzEjruD5$s*s)+GA0B` z6yC5~*J7fgCh;3!3Qf5-=}(u0E~R8VM%irpEuDBk`8fm!LrqAfsIva#&sZ(=N0?BM z2?M+6)rs8HC{TNvZVpRHI9QAH#YK~7toh@?Rsm=pAgJ)fzC=2NF;pglz?*@<*vT?D ze0mYmE@h*h_^0`gqpY?StSq0#NxK0j1amR#3)vz+(%MNb>p^XLKx&~JzOCqGHN1Q@ z+Ze)30D*~C{ZSD+;_ixewXv5dLmG_O@^Ok-q-X9oBD^VhsJ|P5V)szEwf^OH;RKEp zMcDYZyb#ca92rP9g-e*Ai(YO%>TWpl#Av&$(LGMrmB|pugT5$+@kBVjw$~8jjK4|j z1DpL6CB8wSKt-`w`QlxsNW=<*UaaKAc7N1+@9Fl=w5Ic|ek#CSqKx?B zAp}?jKc^Wl?39)nyTAkDMsNCM;Kiytj4R&y;#|a&5NF9KYNeU&b8irUw$uXy1ht-QCgVk zE;V0oPP;Xj5YM<#IM_2f)=Kez5h zNF;Cui1){G`X{VRKvZ0ym(C29LWC|qVy1pIqZTT7Mppcd_d2CJ1SWei$9Yta`bsLwV4 zJBG2%+rJq#0H8aHl1Wbg;6l6AA;!pg0-;<)}+Lj2x;#R(Bsk6Ji zw=4NYE*yzE*kVp)*=^yhm(>=fW89c#t41~r{DsA18x0R{=lW=2ysmWv@so3QZ+>XD zT7-aM)QHh&t>T2^kw z5sShFhp8COLkL=Q-4d3jdv)TInOyc|pRb3TZ|-Cw4)%?0IMx9rwqF^4o>M_IT?mq} znbIV3Z|_ka#6$M(9WDw%&vwneIgiiFQ+*i6v>L&v?j`laa`rFr{uq+>JJv4L3I-8= zYSron7W2gp+<`s3u^J>;*nZtf`obZLeD#y(eZ)S@e3_Kk@ghOqnI(5TWG|$-UV^R( z|M!S_k~Py!dXq^UF%Kb@K78?9(utJhvS}=ncSI>pN6UjawzJ6q3FUWiIn2JIAg?`` zb1|n+Ow1!{BIDNVEZe-Odl4PueuJI$(6uR@NL#$wpCSE(-1xoC<}jvI2H*B1gtqo6 z4bT2^m+{K6#6hdWVb~V)4uH-e1xT8j5*cq^XY>J&t-;V%d2S~&nrQ1KGE}?)?}*^v zG}M3r^<^ZJ``3|amgNJ9L!@+KMYL?D4-u$a2^Z*Ps^I0x?Fr9ao8G5ck{J)6Q46h` z85t}6U+e-P7&*So=O5`-CkpC{<2Muc!2{Zk%laSUf3xc$RFK}P?H(Voc*x^-H|N`c z1c9-0zxqDDU<85~kXJCLBx1$YWVxnjVug;S^)-2p=fe~JtLt>;UO_P>_Cz8nl5Pu| zEV)6NQPtINchaK+#zhtZ{IRR|w6m4eH1ZE3jVNBxptgJNp93ukyWtJV^(tlux{W5i zBmbK`*>Gs&NU<%gO#bQSYaFXkvFM)dQJfR7mI%iFyu(j2mk9i6c{m!zetfC{R2pSF zfGDLi|LB($dT)HauyV$2yqa~aw0h-vJv>&2tGKp@h(27T_O5pkfyH^TNN}A0OYMxM zBjBirrBHW{+6nSdI6yciAU)738uu{H^Vm_VNHdl>{cv|#Lzi6_3??e$6TOREg4XI% zUPz3%Y!v!=p4hT9(~@wp1LP(>=k+)JKMByk-gF28xk-gddQz5l@~JGL_qhtr{VB>% z$|oUjjOD+=$|tMnXO9Org>akqPhRyGW)>Im)?8WyUza~b z5Ydt0tKWAM!-yeC0FNVSC~9+(gd@>{v__ZwE9wXlULG>@yGoE`(+&N8p0w5yKo+*9 z%NI=Iyv4bZ$KdRfB@5_~Gys^5IW1%P$UyxBqzjg*{nwn4ZG>CsXJOpcU0iK?!q*St zExkNhkUnASy%cq2f`Ae7bU9eNeW%uu<$Z3S_{R^Qn*Hz*d#zaY0O1}{X^Ov$nn0zZ zXteX{_sn&QJ&}`|qEl0p8rbQ8?Jdpa!eb-C8Pe9T^&K1Bg$2|3)uYxgzK`R@hJ6G? z$k}FX7K6V2{DIU#9>!5N8BpuNlE}y)V&;Oyj2iQBHy4*-Ljof%iJ*nHXR%kSew=PQ zg=DM<5VF2@{k$80wl*-JN-#)~Y4ZULzfN@g6rCLnl*)^38JADSAC0+Oj*4?W>P?I_ zBKZz37cN)8y6F5`M@R39Yj$33Y02+P&uzRR#KFihXq3LbKkAB$8C*=TTM8n;F`?Xx z>j$*5sNxX+3k<{qv~BBh`u7_$p6CXEh{fOea%Ro)Ma*Z=RphT3!8cjpjiPGFjP>3m zn#fasvG>c*BIO>(@utRy z_Zz80&TFtYgfo^XH;wJbOEozq+3AH3sm07(%Tzp~?~U>O|9=OFF|$`y3-ij)01B1K zd~MsitC`%U5mo}a1d2>gCY2{<)8RN)Oa8Hi-bm~Ul8Y~7B|yeP)nSBI+d9cBd$b%N z`Et^7QJ@z|z1A;6+$mTf3p>3Y$CnkH@Xo8?9w4FbCur_QH~>(K7vVl{VWR)u<)(2*&e;`6{^cqX7IKP~ z!iYW%4rgm@-mY3A{mkY4n8#JGzx9y8uYr8m;WHl(i=UgEavoC0Zu2P88g@_&;P#~2 zb!D+?q9VeOwT8X&eXB&juDs>DCX&Z~If)P}7rOP{+K!hcL{?9=Xc|)K9pY8lS}v6| zwmhjJ$1UwGiFgTfixtCIOME?Raz9FSJYK&=F05?25x!)FRg6R8rE1AR?BVX=9WTR< zTk?iZHB7#FEgDho&Gk5rR7-T_(?j2<0qDPBLLL+dWmEcSAicl>uUYb! z)_9SgljYO$(*p{Jfgf--+gN_e!+;^GFrOa^tX?PS^@KS*n6C;{sn8f1a9-cUHebna zk0bTZIPU(w;BqWQM<$4sO%8T9JIZRyfU5SDL?YX!)u10#FK16PdfXK}zb|zwbE^50 zFAAguyx9f^<%(GT}T^q#0r|)$T%3quss$1d-X}u+~;Cw;A2xq_El+ zC)9f#kSn!XKmqs1_er1^r_wGdN#j7`-}~jrcTK zdve~Ov%Z8g-3BDIxzS}Bj>Le+dZ7T)DvWSl>tz{lYpSJjN?0Cm2KqSZAl?=H<;-HP z!8yDCz7HrayiAuopa1*J5Wpn|!dMD5Yj$Wtf1d@{j*+}0tGt&l=lyEnvQyEzx%;ay zayGxH2m?fzGg*#$PT)3CA^7%}(adqHyVG?U)jpLV!~Rwg3dmbs-g^J;a~KLS#MQyX z1PY~0+R($%f{Wuax6ASF&m90S1D^-6rVuza#1V8+o08uPf5+mrPe!*Za;dc0FIMK1 zo#^sJl)k;Y+xf+0f60M2XF5{0ch5T{^{el+mmn$6L%x@Qj_49>wk(qL*}8Ke;7o8o zl(k=Y>T<4N7}!`MMMg=gmeYmS`T_{Wq&m#XlbRhZ8163GF-maHS#0Ojq;c@(!HHdf z4vA+wc?}t)@Y7|*S4)UWxhYRz-cA-PseIn?PcLBBul3vlCXKIg7>a~>4blW&Eo##h z4=UoEaQN)=x)_DlNMfHsgm7*p%mV$4y$PiLXC_gnwN`i3bGLIl`~4b{PI-idVN(OO zdo^3lCwM+tz_4(Bd4x%5I+}rM8&jfla9)QXXm*``=O^(^!da7o_!UG@(U_s1-#Je0 zb(S{=v}pZ+;`z!(!dx^g;90Rp&$%;)tQrWuzV~J^oELerSwWOzHs##EAmw4ToR^S7 zb9v5w<1~8`K@#+_j#v(RuUZD%NtOl1OE&8;>wC8S`X&`s#lCgLzBtnj7OgyhF2?fj zJmyh9G(JeXp6B}@TOW+_zbM_M#>xqwEhgc z{CR=WYJ-b`G?(_JCRh1#!-$vFZjGhhGalP}f7&}!o?L@h6$`&|({Rbx^f=`S;+-Jk z)+R+Ap067d3?AMM@~HVs2)>ICxiA1+quZU4L?nK%rgxLNNPdtCV(-hFuaYDPRdvFubZZFLrFRGO8y(#>{m z_6wG-&XZhZL!JSb67I4YqS*q){P%ZF-hVvVEu^nc_R&In0&;H8e&k>@QL9WU5@{$( z;G?3&#E!NR&{Mnnn4pii?r)x{QHTThX6R~-iOhI)5a(_r-pb(5;24IVz&fK+vR7UW z_?})!2vo}RlN6nQSKnom-v|T9%8{I8$@4pXf&R~J&!%2Ov#|E7tMW}YB8=wCWmS50 z?O)b;i0=eS1WEuMjY^ru@&JnB9Z{Kmbm@HUL0ObPACJR!nK##vBN=;gADbFU3vB?| z<6}O^ak3gr65s_cNyxWI7|6Ep*Rea_G^aWjBcgh-(QNe@1$kAQUtpf!JhhX3QQ$9a zm-$M9Fl$UWfw4)YnUl<7J^sgN<^=HKsuAnScTQC5P`n6d`AJ-*-@luh%|e#jzz*CQ z(Wcb5UZ^-o!NMg4va*b*9fnTfN#Af4xNo;pi zQnWtGYsdRbY~G%#VO3WM<)5NBD1Y>sOh@t(abc}}SwfzL0BA5|x^7E4INHBRN1p$w zHuujjbrFV7%r<^T>WZP%fNG)TQc3H|DBq*nwcj4y{^@5dv8Cji5LEiWBcR;GzB_GW z^ECUOCx3FgQLpQ|KT}`0*#{afPXC+GRQ9^|Z$bqT79Y)n#Q`) zR=K0pFiFmu!J}s6VZt0Dl+-8Cn{21sK1S++6=Yo?crl*I7!I4=#Hh$}Mg&)AE%3rr z(}_W+R;6vWT6Kog{uXJ;)O~9WK41!9uJ>3yjB05B#ch%QtYS@1@VR&X4Spihm4E*` zU#noo5FjeptIaJOA9eOUGUCR>%p};uwy_M9ng{?MQaE@P8Se-DkvGfY3gx&%{gHXa zAD(bxQjroyL4@kygFmADydM^hHfHd7zX`PKaVaIW(pE5d@~tCI0&pJ15k>xg7ADt6fN*N7zg<0O|JKtB2QozHLog?!4t@p~_V7*`;R zOT<~zZlIR2tX?@s8CN)3Q%7VFsY^Up_vGP=WiVbwl}mO&lQLX#6}j?bpLLHA)BEMu z{5Ca@AO@vwb{)7K2-HxjbXEoY{A&R8KAxw_Zg~;J3NIYW1%MURICwo>d(1oeitxQU z&_+&M0Pz?Eb%`cYC3L?rj!u?QJ2vP#SL8Nfr^srElqaaJbH>}q>q&!$~m1k2Q|ElMpXyHEM^?P7lPyGA-{uJ?e7wS1y4VOc_EHRyq26{SQBD`v z?_)KgJ&IAm2dTVUV79fttT&`qn&zUL|H9v$n=K5@C7T{q^tdX`*@qX+BjX487Mqxd zigxY2QwooUce@f)F(9*`LlYFZw?9x8O1lbN>VY)w)EKof!v8p+_UB+cs7qV>ZX&$D|66rv;yvO2G)2!UNjs+O)eWGTYZ z%fm6%R|h%S8=)^)9A-t4k&unKiofoaehPS#ec&Yv2DI>^(usR#AzpyEu{!4@xvs%O zJmh-W>%Ey8rGn!SYP%gh>5C|ko9P*Qpjh6AFkV*W+ZnK~BE11fSN|$T6&iaAg zfVfytZy=U4j{1V%sl|{B7~9xu4THjR3{kv|%%*c9(5>N>W;Up0nnOvA2)M7==Iet^ zWdNPa0rOF3Gxyrxinb_{Fx|B2;e0?e?tUmd+vU|G(%>HeX|gPY0m7F$OX2P z*i#~#Bxph%ygHaM?jKJ$|5dT1W(VZZFbU8e{;7z#C$VP2n=|(h+r9$htVke@Db!0w z2fJSZKq=bT9MTq*ej5)&is?xk!Cw?MK}Ax`uXj%IZ1JL*?6z|wIEDbHTGV@aBIT}~ ze~gM{NZtr1-#f|i>&g$3s3rjA4JscMdy{_F)6E4AHp4p`Q&Kd|`ZV;b9~N4SPxn2I%)U)_wz`3><8LgO>)&^PB}l zhTH^&DZJmqG$14uAlaT@a0f&NRuF;2WsWx;y`)0UFP5D5eR-A?3Q6xf3Cfz9gQhY?>CzR7AgU_!5 zML>jb{FSeHY zjEdsjObIJe$fOvmvb)@CTs7N3TQ1S_Kz%%62!v}XxzSZ`M^rDLS{$f%nvkMlO44<` zK#LhfdsOC~M7Qp0FP*6vK%Vy+)ecFWu`m0!ixV<^zYl+>o>(8pq@MK1*>y!`y?$R}aMFKvcF; z-nTv-e5-IEQJGRZq;89f)ofbv$>yPXc`5HJow-V7lb+aQ8Nf2Z_c)258i|DXEyuP8 z6xxxzFKHBUTf!f}_Zp5f;F$02T>c_TWX770(`R{T5yQN_4_=r zS}#T2$gcl8llIS7plyk*SbETk30-tDn4v#{st)571-dgBnPl7(dUTrmy36A!)=8fp z^>V%3oFQ6!>hU^Rcp4=Me=)ktJbNQb-I4~!6O8jq>_Q@j`VM-@aW-`-q+Z`|Arv#qOT~s6mveyV=piD`>(T z1J>z%IYXr&Q_$5euvKJ_)$FYEhwinp zOBZT`$xyT)Ak{Mt<8IvE!&G7C;U~iCQL!OT_TWsr z!Q3YjX^#&2W@`cP<*VZGaY$=L+B1oN9)=r}=>Vla^tI~D(T2v9UwUZK)^XER z%K9xlkhiwn3ApF~(;H9d>-y?PZ`nWoEI<9%JeEvpXaueQz%IrHiugPBDntna$li{< z`owl%2+&Bv6VJ|CHQsJ3TI%7@No}%Tgb#aVWSoy#o`={2u1A-4AcTxh_ z#&D#{Nq4i$F2mb0apvI>=prpSS`pc&0VEP8B3iIH4SZTocpW$&H8)x@E|N}bzd8SO z?>LjTi3)@EtI00XoljoQve1w6&Eueza@|!6 zpEi2r>&Poc)3{~FTTcC!qhOfytsle~MuA1&b)Jo~YGsdo#a4jcwWJRt!YLl55YGZC zZkb>2A~Z~=TIkuGZuNAnGJ@NG&^hvb)Qqc{qt!mocpdb$^N>JHTo+dN&tHx*<tm#O)D$2l@ zRyRNok}VQ(wBku4trCFV{4EBs3GU4|`XAmNd(1UCr2&36+Xhl@+z%Tc6@UIX9M5q> z^*uF3fy@VRul|=gZunE`Ncs;1Sq;t2&jM6|LUs%Yzw!KnRuMY=NnIglR&9RgyV`{{ zV+JEFf?v`lU21MjDtl~VZ9&1!B&H@KEj7*Tqz^}pj(1^7WKmK^8A3uFW|js-y#R(Y z8DhMc8Hhrc%8KrB?T?nX3rlgkr~%k^e_ZE>Bxrbcj{V`7kG$D5A4hM!$b8!AY9&f_d_lDgy0x8ezfh%sP}3Lm&11|z0g`dCoTmbL@qgu67NQQPg81(*n7 zkqtY%%(pN5(fGZ2IH%`ocFc1MqZV&cCJJE~7i3u7qnR|vhy#sl2_v!clQe37c?=jR zS{uC0s^N&5$d@iHSG#!RtrZcWR6ef4eF_YCxTB?ZgYtztYlJXxomQDDsN+)KX}aac`zAn>%6KqoH?j9 zt2w|(iXXY15Rf(^KQU9oP(-S+n=V5%xt_%e>bH^KJyc{j=+)jRZ_gHHmZBY;yZsu= z%+c~&r7Ah%n1+WSeDfbKq!d^YjLwP2-!!5kSD)-l0rp#bZz3P|#=6<##Mkwkz3(N7 z_Vm!^VqKuG`~xC>wl(*4U(*V8e0JED|8v_&mM|7t8G5(CsgXCcki9Lx=c5G4;=5(X z(e~Wq6`ECB=55|TwqR$UE00_ZNP#zy0)$Mcf8gkUkX-DRH@j!3?XP@WsJD^AtrC@- zm!MCCeCvb9B_}K{r_cl)dRptf;p}fFR+62N!pF;{1k|k$Wq7Km-D9yjO&?j1cCa{X zay~l@U{OITrh@$_Fw&}af1rjJmrNtvy2ZcnHC;uCN?~%jKcd<3%egBSa~PnDS5%YP zIg&=k=H>mw;Gw!WA9Ts!x>O?Jvzx2}0>Hyf4GpNs*I$s?iw)V;!7WyuDra4j-*>Dh zrDCeoPmQFE@_!sLdcz$pdI?~_qYuf0Hi8B{MNoOlwHbqfl)xO?!Bw+})lhC+%&y93 zp*JScB&K^Y%Ypz7m3j zDp2xYWkaBNai=px-%6rTBzC>s=)=q2i&Hyg@c|(T!FHlQoJ)idwuKs65d8@I#2%?i ze=59&%a`#|R3Or{8i{KcLW%P`41J=+P6oU6)u0LD%TE?6ilbI7A%wtDeiF(B5tzB` zEy}`0%ZT#A?YU`w-=G*l_x_ei!mRo0`;+JXOFUnwAwVMf!Q`S5Ty=}n9=cuLR-xhK z%f|Jr`g-SQp2gf`dU@XMqyLUT6$kO->QdLDgnNQ+@A`|(0ZU9yp5&#EpvmYXE>U=; zD2C6jZk+8(Tfz)|_zG~Xq^Oi;p9=^L+i!7V54#y+Cc-RZc2tQ^cVR*C&$K+ASNq@} z_9is8k!rtZrd+Bb@G3E9GQq9#l?8R<;9aY;a#G6~)f#sI`07%`9IQ8%b^wM&DaL&^(YN z63#?_&UR|K!(D%1fmgAXY#G^EPha)$!L?gjh`nubr5@quJ5dg?q!SH8Q*)|!iD+pD z#4s^CRzV;3%DG(7gE`zWS5|Hj`9e|2P;OS^p;w&3@CfcwA1U$KsuJ>@e}tq5uTl}8 z;CsIR&oBD*y)ZsIvh95HOSH~n3pPll&1w}s@`F0AbG`#6y<&!#^=81`#dbl2 zB?JNW)_@y)V$6dp!QAn9TY_s`^B!iIWbWtNpK3_&I^*t!VftIW=$`cyPG>sagKc%; zJkR3Y82swAn%f0Ea-jcy0;^GXdA~qra5)Il*9bMnvRHkQuaT};XlSJ-G@mG8v%hd8bQ`jtt%JndjmO?ezQG_sBYn^&+fHfvDRt94lgCsIQh`YE4`__ksv7qlulis1BBz5}OIdDbHk zTM(#^9V~BQKr%Ig@G#%s{MSo_1pTRE4(j|Rid-JY+$twgGeEJ*9Tj{yUkI;St|LeHAd@DJn-UtAUsY$dAdA{!TH99E~iNj%4rd4mUM|)=9YGg47`auE*(%9`D%ZTqruHmpx)B~<35!h-**c#5d zGmgywE!@{c24C1*sZeES#-uhT><5&*R`bu1n%NS$T<87T)+VQ{6-za>L9Na>Epy?Z z#xi}*nXF*7)5)oPR@NfzUs<}^f0(u{Lo17a%0!a8Dou|Fb%;kj?ap6mU)~fu)S9jk zu~w%W)HvT+Nzv=gn^Oe_trpkR#3A)g)v|zvYlTlYkrv)iydEn5@gl|suAk61Vc^@r z-wsb=*xR2vAm$1oDA#P(bAD`fmbFl5kG%5dC?bQDM`&W~q64m@i|r~l%=hnnz;9kQ zX8nCm{pUv#zka0c452F=Z(SrG6Qq1(oA4>(Uw^hExg#gvvd?8qN^@AzF_A9W0=Vu^ znXOsS!|7AiI}K|pMPIzZEe2O&0JBj=tBqrso$T`Lb7BA9uyX>~23n;0&|#(w>~Svy z;wwrV=Hg(jJ0}j^q|N6_`135c2GA(#J6q|R8jbF_Qk%at)&lFTILmdI{By*^3l^eC zo-aSKXj#L{c23Mxt`sRn2-q!jc5JK5Lsr@KHCm_In=I9Ca1?s z9;tjD+0-MO&TVbCTs858uI^OG`Odf5ZXoDGzBG{&Zm^GW*}`q_ zkt&`~7YAH6Z1!7R$ClRh{_zf8wPH%c!qZD;%=5*90m~mj0=?#>0RgZ7u~*PoBYcuEDlyLoK&(%BvilHu{buK z!!8G=8NYmkMy)&>$iq;s(BbR{w{O);m_HB`G(p~`r*Yf9C6h`14z%F`$3|FquaAt- z_cxI8L~oFV|54w*fA>cUq4q0`BH&j{=cw|ReVQm7MdNm+E$xA&>zn|b3eu0Km0)j; ztwSHNHtFd>o$wy;kh6BReH@Udn^U35EpGkzc*hL*RS=2Jt7~ou`2O@F5>VN+G@!A| z;Wiq1K*{U3dB!{$_WUT$@p#()i2ewU)oOFvOHQq4H9UHr=b}G+t-x;}iTQjzZU~eH zop~n%S*$O)ovdbr=q#D(m!QWJEZL3F!}-)0GRWH04U5R1pGS)YFCs1GI8r0WJCLn8 zsw3F2;Xw_uQbkN3qGI#YVEWRAjrP*g6YpQnbC?ToVnGHHd7NBbSjHWD*RY_sWH z-7l^)9`E=HZc|2d?yrwNUD@G@nvUqI3~vu73(VCRD-KSZ)b1F7bT~;pl)ZrNxGv@R zNY3pgi6-Q3fc2AkPx^0*R!-k63$(735K`Cme9Jj!dmGV+gS3}+_?xli`g!;@BY5t> zI3ylMS=2{-#4m~r{yTf+&q zbKM3V3=d}~4BLR0gzXW1vGCs@9`ALE18o*-0g~YBI5%xbdGW}MJW*-&v6aG+*mZSp zJCIt5LSp(tUDl{RpYUm(6=dY!-9xrihpve0y|Rcqu%FhKw;1Tpq+;6O$F7Dr!@Eb- z)25o1mokc-BUUD>V^el{!`W;X^3lxwdXcD=ZYjV+D{fxUKU8wJC_lRU6;gQ17uyYv zguzz*=?2@H0!&1%R-utby0^e%%j;-00AfYG+C!#nQL?J_qlheEF)$;@?>}U0x}9Sb z`yuc(iE4ww`?+gH&cfl2i7cR9cPMP?ZpYX6S5JwT&VyriNy1>U@yJg~>)jAo!Q{Y# z)zBSdMV#;njw&q6dn3d!3hzi|l$XUi$1MUDTfvS|DJd^Ac+fV37L|A1g3#5zN^VJe z0&@sc!R_hlnqex5x5)(DwnkpP&2QY&XNIgFxw!r$u2M;CtamcFf>g?+>Or|RI}D0Z zZNH|N^ld$t0-f*AuSz@7xLxqFfc9sSD{?-O%V{+{K39?Dono$mnWr_=jFDs<9u0$)~V{c@JfZ)1?jk_;!=i z`W|7>gwMxR7Ec;*LNOcLRO9`wPgepD6nJF7Hi` zYkD*ij~z=P6|-l*$G+9Z+Q$LfF0IeK!RJp5^@6SCfqyV~US~V+dvMPFbPsEI$Zy`B zz4fh9l~QjSaO#ca*dmpMX4>8?nBb|AVs-7Xg5J8=c9*1Au@vQad$hy-D`Mclm)_(!b zhL{B_F;6jN^xMBm0?bzkmt#J{LWCgxhj@C;czcsk>fdrcnOx$S^m>Y>#%p~8znLl3 z$~yW2q&EKQd^~H&<6cknxmuP3JWb#asARM)f+4W%0JGh<4E*^ zoX2Q&N}NIGl)#pA?(EVwPZk*HZl)rk;=HpI3_8!m^b=EpO-84FcI}vQqy12{+9r2% zJEoAA+cx<=`+fwh0Z2s*gcPXdD^FE&;;oVx!kw4E!u=cMPJ+QCJ`84gvjgZ=>6%Kla^vc?LJ)S#Y1$YC^`RTkjs8^p_d!0>Uz3$K|9UAd@nGv47~ zbRwjzdwufe0ZAcADU2k$kjeV%C9T%ILc|NUUEH{}Q;l4(j@+8rpPftnkm=N!$309N49&awS?3S3s z9`hvOND17&{AH7hISSQ!*0vkn1)@<`6qy;4o{0JDx*QEByQchiWCR7LmN;t{N2-d^YLb`LH)h*`sVl33deJpLaT z@9$$M0CEhOG!&`)a}2@qgS7x#s%(MwuVVv z1jp5LMx=B z>bf}l7jZzw?3MNDX6=jhHnXWYo*ULaXXMscL6)GU)Ab4&Ncz2*%;r*qyP2MCjmdQ< z4#Fo#KF(eX%1K8u$rMhKQ8;v=)ouxO%;!L;R+`8Zr%<@#`RwzAQ%t!=9}RP&;$5ah z6mNW;`<(|mm8$qMN48N-Cp)wxPv}>uzeW+y3+_jyb5Dzr&E^r~Y#2opjQ~eYc{Vgb zad>7++AQ*(Mhlmw5lR3WoBc}Lcc1ydk z#D`ce!K$Ft84Pj}@Mc;raZ<&PWxiW0i<_sGZIZFa%y*Cq7_*v1tn7;3MgnQ$ZKkCrp z1wgSLIx>w8ms;eQEuR{<6ddk--UQ^16-K>+CdbBQi;--t&rXlGGhx_+%Qkze0!CzE zCq9ZhAtztoq{x0=^N=24`hJwz;)k;Jym$TAnrV%$>Q5)eTv4HHJiB#jHS|Hrvw@=J zYJ-xA&o0~619(BMLb?#!SdlD$M&o%q50_|@8 zd)w&qlyf|?B^CtB(*8#BP?om z=UY#36GFCskl^!OlGnVZU#B1uBrWeqej9=ZtT<3sPZgW_p{nJgT*pjSIX5_1Fdt|Q z;XiVcQO(Y3(^rW4$$Ow%`y@MLFRjH73OCT@>J$f!R0ZF+VONDYz3ImJq+~mE-{amS&i-O(ZJoemxIGDWCM4LwzRD(Zfrvbn*ppNf* ze0&_kYko=r+;Vu*bbQLv3pMKSgPB^OqGNcKTGka_;J?w|P&d+JtzU@~8NeAs_p zTL`G?$TT9pJXO|u;I#49egBE|kg<`sbG=`4NYPUK4JwqV8!I;eG0EI&D}8XJir)kD zBEa{$Q-T0qBXz_tF_QztN^1Vcl8;=r}U$9B`7de_yUd$I^7i?rd?>C8R zS9XN!u^P(ejnmweJ+#ylQGmLmzG)53RHBKtOODxp{~7h2LE%O0F3M^18l zU1TuNV?19KLRn!t>g9rwAA0d!nQm|||C^r1MUGM({_O2~0cBqtQy35&=QDI}C}9%? zpp5q?vo)>!tgPUd`y|4Mnd`dVdysLYC;B)VgOc#+sb=DFsyn7{6XFCjwYcPZ7W^`7J?yN~cl>lWS$r5Pl=bR_<9?5?jJC6lPy zsT8ZBFZzzc9c6Lt_h&H%v$nqVemOIXHZN7A_}=3cwIEzmzZSGTn6vmu4pFF5GR<35 zvGXWC)4IcX3Y*R0ypr;+%;Zs~BnFac!2G2~_0Ya?0|LpJbZzBBbLYE*L&8-GIR(>_ zH!pSg@{tN0BQ6D^ZTn z#VB>Oc`_049AtzxBL5l48!^G4Df7WG-5X@Q4_E-S{J0Eb@;euJhy+Uyfp}gSi$VWB z*?vzLvTZ?St=B! zPw3|Pp_7F<=|{7+*AZm{U()KbvUrl@pSLN!O_#{(1ZPr`)vc0Rev#lpI;II7xFopR z;t+n+Y~zgq6MA$Msb;`yu0z5#_q7dbmC}#Tf-slKOqC5=JP9ShI!*&k5dz~^2wwtW zJ{Sb=rt!m}bh%}5mJ$%@gvFKj#^1a(=wHW3F-WYgF3p^qeHU*fJeI<0AGXD4^lgg! zAR~?Y??%Tzbyp)iWRM=h5&vtD2qA+6{n#4mfky}50^o6HZZn(7tckrMv)h>nGthco z8ID1tF)(evPpaed^!bK*N0kjZ*t@&VE2w-W1jNK#)a%G!^1!>m4j3P6+ z3Oh#nlE)0AMk9iPQA^=OgE6^H%_0XU>aeYDRI*9Q9Paq)ll5A!R=eVoZmX+?;glcOogANr@+Yv(60Wb5L>n^9}aC z7jApkFe)Yb9IXKuA!<)f;1XcAdY0DDBC{kCglXN*J$w+Kxh!V|=Z?ycx75=p7bm6( z1PBt&_$bZN5MXU3y+d~72;RB+K3RXV(DW!zqmCTys?2w^Ao%p9kYb?*hG*migHAl+ z4PKvPMkt8|xIzk(c8Kr!4A7$GsjM=8Be9IUN+bGm(S$sI5ny=d@%eGqaJG{)FM@d~ z~o2W`-yB3^l2B9K(*&MJRL_;dT>k-q@(Sv0kOK5rKvpnPsVBr{c@&6k@Sl7lj`=JVZAk3b(bp|`@_71jQv`}86v@3c&Mh?0rKy>zgzepI`2*c7TPSyP6lao|8946PlNH`F*&?aYA+NavdR6Sh zSO1O&507&mv&BVZ_sRT3?<@8eFrn0t4QG8Ap`FeVUoyV8%dk%?mv04QI*#XfXV9N( zxXImw1=3-p)W6f=|8BFVU^n$N@}T^=CLX{}2;+%COJXsZr81g~C)m&e(ZmY57hfE` z0i6+w#!4PdDmfU5~a`Fw&Re^P3&jDM*7<^C}ZV8=3t+5%*v(LIk%*WgJclZp2`hi%#! z#%TyDHj~~(kLl28iY^PD*omq+o!Mq)OWx!v^lpvhXmNatnI6oARUl$g_4QjtKX z8!sMx&XNOsfy@oN$38I^jTOj$UTP1)pp-oGbDn0mG+&kzn|W7l1Md*q6wiaRT5s4n zE}PvK8q+Ul)OL2jUS%!2)Z&*LA?hru-uYr5f@^;XR$pNOB(?Or)lO5(o(0Sw?{{1L zA|mm4TpZP5?AcE$Qa}lJTEZv!-&6Z<5Y+<=@+5%c~)(J6-YTCZj^73?tKPgF7mEG1qGA~b1Or`w8V>Akt4sBeW z&p%MvjR?u3$fYCEdy{x^s!uX!o$tcUj0D6~EJvnRk2ktwXht%k0!k@DxoB>RjO&|Z zfGeATyfL%|~xo)Sp(QT%|Px$G2DEl{EttE2w)=e zjOAOV?_`FOYR)!K;!13Q+u`VHC@Ny^=g0KvBr1Mgp4sKQ?>f0G%1wv;fswKfqdIId zREt&gr^)W|p-SYs*%ug~4FHbr7tP5yaye2wd2)kC395u_LMAuv2MhRB&oPO6IY@#t zBgr1kPZ;mMRCyOHRjR#;Nw#A&RLPn7UhBpIS>S*{k^g3SLxR#S{+)BR_+T^T;d8{M z;Zq#i0SrZH2=V6uj~Ye#_F2985`orT!ve{Orw1qlN$?c zIbOhS7%_OE)3p|{$k0c2H(tKO*`=`?82ubCwThCf75h(JudhW#a;T!0eKaMegpmXt z^`0X%@^96rXHB~&0Z68nT@mk|n5s9(nd%^;2Bfn5+M5gGbiV~H`^zq?JB_5y%+C=$ z6u1z`t(l0NLSTx_VdY1o&IyY-1jjz5)i;x9%#cXyj4i?C7z==8jbn^?3Tb4?dyt7# zwcp7qsC)eXG?7WwCmCRy4@ttfWpLKj$TA$R37A&9Ug2WWcJT>HgTCzZ8;zQ*a@Lt+ zu+#+|ir=u)mT4ie^Bm}I&HTuk8*(%#T2+SdMKep=SS+94Tn;<32wuSq}jjg zu;s>0KvG_M))XPmLYPDgEg5DeT7>uKM)DvP1BpFf)9`lW3XNyVOOKC5d7CR3AIN&uJAIo0ns@I_=B7W_ zFv~3Q!!g!9E6Ja-PluphQUAAF3NVY#_U5yBKF?-|>dXj90Sb%lYy!PKCl0GqqSb0o z>ebhCaGl5|aHR)qpA=u}6J^0C7ppP6kqp8wY-@q*OU(1vYy@7i^Q6wF-y>=H9+!zH zRJq*{b2yw!@z4A6WQs%&Fm0~!@g1C8{=&MvS|M{fTJbv|%yQwrb-(Ktf5LAQT2+sr z=M@T5Ut)XX_-hL-iOp5vgc6X?FRHz zmy4sW0s%KNvFmnT>&|e{q^P~=$6@6M{_kYqrWD=6I~A^amkVBY=6zP7IAL(%$PHjP zN3tu^TfZ}o$I-zeFn|+NI2XFV76@gems>?+HnYytnS>s7MUccaF&ggwGNQuJ zX^LpLqqj`fBtE6jcX;pFe=J8(oMhtb zY@r@W;qW&KyT9nz+nzNo1J%dToZ^oU4;}mIUaJ-s=CY}tCdJ`$T8U|#abaBhz1U`l zO*;=~G(qnI_e|6Z2K#cl?B?I;MA)SB+*B7ATHy{}wvCSfRr1DK{`ASws;H&81odQw zaDjUI{Rv8QUK~*ELQ)|G;Qm% zMMyCmN58%?xZJK?s3AR+A-tlY_u{HR{Py;CNv%N)yEC*Qn?+mRBs+SDziO`RwBrke zuECo^tN%|R`OgQB9|RAYB2pLps+dh?{eoX}5mkKEz8GRKnBtF^rk%6K>`kokg?T(yA`ri|n!4Ste)ulkN`C8`scBODQ0POB(PFANKp4!Q zuME1ZtcJ0)91Wj55lu~bx|L9|$tx6mV=5IxY)j5oYM>JuGgUO$!-`;3J8m=THasN| zDS6$?M^jZL326X+6K}tp5+n1*n(bVbkOK#T*ow=uE^zdd+m)RphsAt^|@5M z+L2=ro(q<9a9C){**Q3SC7!Eo^LbkCwe|RG6`)_pBF5|Xe7aTwM3)hs*|yuvIRwzV z53aeZ1FW6N6~5<`tH8U~l1N4@Kf|F56%2Y!*eaqzxHbS}3&4!hLnWWYVQZiW;gIjl zsZ*yk!DT_CP&}6&AWz8A?GtcA%;?>Ln6zS?m4zpZzb2O&=t%Y>`Xe`G4pK+R+PfO$ zGl2Ijxd55M7W0Q6l=izrVQy3^{HDuKQ`%iPtufhJp?VyX)o%L{cq}?CDu*Ksno}~B>UfVY*HB5rLbs?iidRiv=j<1)Fgg0B zzxE|oZpWP`h*pvR+L*05E0!Fij5D0{d=<$i5C54q@T}->@WuTR7RXf}{k(&Z-U)(H z0Dby&bn!>-_MJZy*<6DY6Pr{A$v zl#w^11O(32I*KypDMk&7R+iLxW|ypcNr|w(FMJcs8}*)FU8@OE$ZYY+*o7e_W?PR_Yc-x|<_t9*fOIrh7IvF(;D!^H3B|W2Je{OKf)?Eq! zvhvbMTo3xOgK@a?)lmSTy?#naHrTokf0!!Ky{- z6_MX`sq4qXl&f39G=Zuh8$}q69dl6Yth&tf#1kjJp`!wIIFI?$M6s;!~Xf_lif6f}CerC(5 z^rq|VwFJ9<@=Voc@Pwp(v!8&Z4u@bKa>=>I#l|2CviQCyyW)BLc`o;-4PU}VGURz( zgLFg_N0SS(b;N)Ia7)K~L|jgS;H(b6o1~>>BfJl*uyV-6Xsz$fUhzZcI&ZOlgXLy5Mh~bAZH@+IhgLUM$^Aw!w;=!`$Z{6^90Bv!N8$S| zpFn>@LxEb+_wyTT{X^%FFA!wPiBbbQUHpl?`CA7vaSIqpY`+?bWhR@|t!K9J*MQ3t zC(|l`dZ@H=cJhi!y-2}yxyKj(CP$B+Z!yrr!<$zLA$cOnhbnP%CATEcAdBF&dW~B| z-tC-N*>cZ|`CPfQ*u{Y6(#L-n8BaB{MsnOtt0eDkuacg>X4QJjEEG6ZW#oPCs_L@) zR#&)D_X8!^yqbRVjtY5mQv7A^sdhIY+txM@=$tpztanmR)|rU~*jhDEG8jK;itvYe zb?>yIfJ9T6H?jYsq6Iu2SWS+ANHA9-MWSbu3uZrEPq1s5 z{WwwXN^$|(jZ{UeP4@XT{tKHRwKE3YWsrBliARMy1 zKWgXd4>e}{IVkzc(R5V<`OvSh?hXaY^p9YmRBqW}5uUXL;pA765#khC_zZs>l$^vG zZS)Cbtv@a3o;f*HGhj~!dzM`@DpUcyU5?2F^PEXB_>`g3x)?j{ zVASZ_ zKymSI_}ne0$4_Sh5`lM=bcpY*X(U)v5P>%llIz`WLpIoxUAqqd75}u24=>BrH)V;^pUK z6PgW<3GuA9bV?zIEG}k6Ip55d%Z*W>CCTA+?}5YLLB}-$E%XD_{k8H03T z8cl?H5U`2ZnUT-^e$T%9s_&{|2wB8_~@7G-wpmlncE_AB<60usLBe=74MKM{v$(SZ+#uSQ_+1t)? zf$MX%9Q&xr=CJXjHYWHrBV|YOaT%N8j|3ajuO^yg#iN(GF^Xxmt9?lkK2I@`TKFj_ zhvo<;-hPLc#O4Vm?0l77fAd7xBj)f7tAy+DOc<1@gnBRTO8xeK{+to)&Rbk!@5Api z_^Ss7dq)T@x!CaJ;2-4q_Xl2oZe3({OH|z8wVy_FPbJAU%QJ-eRUQV)9t$$KSl~vK zyYI1bA7fBur?*RPPT71M4^LEKAP9b{n^>SxBRskV&HrYpLkw#injc1ShlOo;AnYNH zzLn9uKVXwNNQVv;U|-mibqkc{vet`w@<(Z7X;7Te$W(qEYTeUufK4_7+$HkykKZp{ z9QO5_=oj<^jaS`Yt!46;0hb?m#}Hhg%22Hbov_~cidaBHL(AZ(I(IcmP!NakTbJz1 ze8Q4Ybxh8>6r3BfZy8k2S8EKthvYemKlNEBCIpg4Q<*Q!OXE4nAmcy!bF+Ma0oCyL zPtYj$PlF#}+&KuzsO}o|E&|S{dl7?@ESEk^me78ENlBUK{Hld91`neE0-;jl_8Buy z`%;buKN`+AI>QyubxT)LB#|Wyd#x=ff%gNw;Mxq=H<7toLGf@p7nG>eA8+ zF`MaOGYtn%n07XY%8O(}(=x|=nslLO*^74qGqk~WdU+vKw-f~E9D@mTx4}s@hMhxb zkD*?2R!rG$P!cLJew2za9?!$?MZ2ZOnH~mm@k}Xy7;BIP`GB{Ff&}}^M z1p)97UNAjGv{iDhC*_=S;!Jq)$|y;uauRdVXi*yDWz&XbO+$c{ve+=Ej{SrQMZ=h<+aH-HwR9y1bIosp-DntNZQydaUm!+r~ zuG{-ERqH2B%9T*<)4hx17*s9|HGq6&2U@)zvR1BXz~`W8d-NxH5ta*tE^l1to~M7V zZX`!Iueb%_fKZTfT58|jel~8wHp6BAk#fE)W0p_Gfo4AY?qHk;v=<-G*4b+c8q&Q* zg3157wA>ytlB%Og5eHt@O%2diuw^ju{ z*Iwnq%!TYb^X1|c;lDcuywPMcH_` zI&*IVJIN<~#3G(^R26pZMaOkyF8#*2UxaZfalWrN;Uh~qTF#J2Qs73eL1-WHiTSgW zIHnNfBGu)IE3II2i!r@UT-8E)0WTd&RGGCw!QePH$8e|sxQw_O zKRlNbXj-ZVt`Usvc>k`vSV9#>(ghKM4b<9`E@vVmCZnFya-m^_t(1m&R?bG!;9F|0 z_&OBnY;x@!Y52yn9M<{oE3XZ;2E-WdI7k}N0rPTy3a?5hqlp7)I!`pA*N`j0jbA8u zlE@C`Meeh3Xw6)m?Qae&rrX?J!(4D&%J55GpYGOdy)y_DCt-?D{b^}Y|mUy+q3KsiWj#o38WP}ul4lFZK9s~3S{evrH^niZNLI)Ed%49m1#~9;?-eaMQhHhjr1fN{Fv&k3-x!z>6 z6$_)?Dez6O!t&0LJg1Gh2vXC{ckxr$WH$c}{ObeP&zH6vXH$%}*;6G3DS<3M5f3N; z&Fwl_CdKU`a;lg>&|*_$2kTV5N?4e}N@|iCvZhz&g77;ogaP8QN+CDhiO6V^#_FW||J4Z%Ko=wGNvG->S`zudeW|6-5SdWyn< z4jqbY8ORltVB6DBb{TUfOhRzj?843n{z|NC$il+LFl1+5vpc*m`0OJr+wZ4C27Jz^ z%;@^VPSGjV2nf4hF(1`cXr?3|RWC5f)P$7~aEvzB739RG$Q-DCaeyg{;dh>qgb3n{tDoozv!p zo&B^aVtEIK{cpzke}P|*1;{F>geFvU2eJ>q!RT?VuyZF|+gf1!vlsp8H3*X@IbZ+P zkMk75n}Lzsx$TlSF*=Sml_6btXHN|2sKQ4_ekoN(m90^*?TVtvMoqR@3BqDFAM_1- zZS|`S;0w~?M^7CZ6VmtC50BG}mWhQ`WfAwg1u3{t2N4oVI=ow{qlbK@bv8f{HP*%B za29Y#LlgU<^i8}bm=5!cX#1&LF+XixK`DSyr!LmbF znit*(cVfWDIBJd6N;g}JPYoiMc|2R@`VskK5GF@-fqDZEz)?`7MW?U=b5Tp3JDPzu z_Ypyh%jJpR)I0ky{W>R`OxG*K4r^)TdP71paH*F+vD4-IMfG3CQGPI#P2y|kj}6qb zu7~{bKUVbtJiK@vl17f}i&Un6e@6)|38XmbAgAsiGmuF6PO)?aG)HA-)6!E)6b{Aq z^JN5tveF)?O<`7U6t&-x*~!Y;`{j{0cK|F(fH|+l2(P&{l4}U+AxCu+*Fku@ojMw~ zyW6D1DHc(TL>0!EuuN3-sr3w^M#d@N$SLG5zZ5mCXd(e0CAe)Na`u0n2(34dLI@PzdG9>xK z`{{Pz@!&6SV%QzMs~h|aKz{?cz}7}kUj~{uQ#2|RIr*G@NN`+c>YKuaxS+lEUD>W$ zIdj}H){O1uh3>vBIv^neu#J_vmRbmj-AI51+&P#&Pc9GZsfNYV^PdVqk=5Q%)P>Vz z7^rS@okQx2$w3QsH@4>!VcOwM9LoImIU56!13V}c20k#qO3LAJ9S)l*tyk_cX$_>b zqXa_PQ0kAvb5$j%U+uP+cm)D()23L`7B)w6=~q_i?0h4?h9*}tRmp26}?b-j+Bu8Zr+Bw-!*hzq8v5zv>A>a4`_YFJKsPsdc@I1|SQuqfuVi zGc8Ew`c~@clRq=p2T+TEN}O%swf{+0mBJVF^d2(z!^eD|)uiyb&v@a6#)nYPCrK;e z{`?Z8R9LUPh0VFTD+WW~1n6dhWa1z!8Na1gUjecGxw>rNJUFB_4^s{GiBKqH_~ccI z-aP0?$+W_+k^&mz$xK)qW1yu3SAo}~$YM|o4_w96KhvOq*AQ^W^+7(naFsIf7%I0;4#do`EaW6c`SL{kmPF3F(Y6N-z+U@GR);hdXG50B+waqRz-V8S}0f z;4>vN@Ahc(<_n_L?CW$PscIV|J!VsXr#NlV@vknB0KeG2}p5*hRwL%BT?avn@kA zbLIAsb?=F@UXchLI=iJ!jvWqLr|n92jG|yrz#-pLk@z=o zMGK|Y;~Fj67#f5KHcGn3H&+cdH*L>eE&2)YQC`L~(sMf)iu^!Hydu#20!F#h85f}& z!VUyFT{JhB7y=P<j`Ghp}w>1b*T;{XVV=AJzb+Vw=z3p9zPV47we5rrYi1e9xY}S zE%-I0TvRX16`Glnlk~0PYF~51pUeyXchBu~3O#Mrq zmO4UM{8Iy*wG9r7YWY?1<0wyagVbtFxE=lX$sKWnDH4 zg+Gt#^?{=8Ns;|LYnODx!_S}3o6sm_oPJ%F)e2z(CHZW(GdbYL_>|Bca*W&WJ&ogX z*m&C;zk8)A+UOdHFm+YUR2|RSBw?{1kS-V^y}s(4U4)B0>nBh$M!XDbKIP015=JOw zhsoyf0OYf8W9ZBPMUdTxb#CpP?06e4xzZHWXw{@?mSnE7Hzm{KHX1d+8!Wj^v{YmQuk;61|(_;UbTR(vHg3zg(k?S8xnC9a>_9S^S z!%m@AZ0DZ;68=8*8rci1C1l<71>!%ygxiY%Y0Hx&%X0VfnM$#4RbaK2*cWfQ(UwP> zt6kAdW@+H)7-$jZv=N2XF7?B$E|5yac61Vt9t925EJ(p5uIsj(MOEt5>lC*?o&DrS zZlN8I=~Qll!eOaOsS-qM>{nXv;}URl%m=@d5sAlpK`08+vZz`f+CPQR`79Yof|Jw4 zCrbfbKspZx_guMXwf;g&u z`xOoS4!~7Mzj{G))7}V0DAi=->+7h|3$wwg%~kuI83B{A0%qTiRy*v;Ty-oGM*rk; z8h6|z^egq9aRR!jNWj{!Wu1#YKRow;;OG4jaIte_ty(C|rrj65s^kfgo=V3}7Z0{ANseTWq$ z?TbHG5m<`MkfrGS1lz>zuZoa$tX_FLsO-vQ*Wt%K(8Xb$vT9dVoOY7f^TW zCBPLNKb&X#gjK$#T%wv+Qq*mtW*?W!EkiPeQyTQ;sl>Gg`lJ(pF_$#pt7SHJf0ag! zyn?uFtkzMoEHy05U?}UZk<@F}2W+$Jbw$d=q9tCvK=8h9aa~^hrUgO$4U;^$3d@YMWzInK^Tr|bIut9gdn5Y zC~EDvS6tyCjX|P}$x<*!N}1{Qy|AFI%!iYWu{gJj%utyYFj#nru>9WJ$Gr*64B%#Rr_-G(VPI#lGgwCtPHCYC zCd0n~9d8gA;Nh_E6Ws8YP#|yAmhZgrE5Tl_M#WJij9s|p3D!f!M#Lu5nevzyM1hiJ z-xS~0K<&gRvp=6S?7n4Nfm%TQV>VbGcr=+UZvun(p$MjY%m~@z$N!%%bad=2l0MTu z1hH6Z47tda%UwFrxpg_U;K2pDAtYiX`GS53qoxg9$pl#GC+CMKQQEbYmSjS~J0k`8 zC`6*sa`{4kX_9(C)3y^Er}SU+ixz?fKxYQZ+;kJ>{eciiYnd?K3qtgk_e;BevKSlJ zncI;R&idq+q7O^HHM_;NE{6t{?-cS;Kalw2t3EwK0zdGO=^^#~l??vkD+dx`RfRs5CIEH)vxLGpIhw#GbR77lQ(o+(_SIz1!9I6NNwK(-(RSw{IUCP@lm3FS75^0ch9R9kn$|Sbxk0ct|C1 zr2xsca?}4Lip`N=326CaMf~w5d=^8%~^6dD;n%t%rm7jnhw9HiMdaHC~92M?eq z!7WwemQFXPN%Dh9sAdy1-d;MmyR(|@8fdkDp7>X=F4rf10LA3rpL72zMsFI$*LBXM1{?L@C~9$$JgM?n z@n!9XkIiWMZTMvkT5E5K|2mug^Q*o`(2t+t3IPI8G`5qCsSwv)q-IR#;BvO%T4HN4 zm2B2i1}j5c^>?PLC=J;R=1uU`7MtW8E+_ocZ;n^HBfqkheUtLIzDqY8Ov~M6A79z5 zH;$ukyBW?DHy!BnhG=UNH&I9>b|!eHn$tZsJsZi?<53rBIo6>6cQV0ekN3dcEBk#_ zzS2Lti(40>o7SBW`2O(z+JS;)iC^%mlOcGb71A8cU9c0EyFml96OVMIhf)4^7d7$b znXC{kaA5l_?)=0ZNi^`I3W4L7JMnlwK^9%+iDugmopCozdabqq^atCpm!xaND z58oTxV`C|Q6=0a@47oB=<=miOEqRPy z@U^vj46~<&Aa{nfM7G%rZsU~e>@O$0+n-O2==Fw3FKOMsE^N+tcb9%##eIC!pVaVu zDqAcp5p*9v4)nnUOYJaKG6TO)`uD01av%=99-$?<_oDyqeBd=hUbNW^2m8J+r3W4~ zw)_EVi?rN-XXW?w@T))pACup9fUB*8(LRWK0uAUF%K6JZfQjUtW5hwFR_y~IlgB=H zXpd`xvVo&(hfIP(P-m{=BM$m6d7l?LQ_COq-TQ9-7a$W82oGwZ`)5B9;DS7kU_Mz} z;E!>803{89_@wRki>!Wc$=_?#0u}Ojm~UG?2Pd)QQ=3nE<^u7(aoNYjs+)dwk!1E) zL@&ZpdWe8lZII_*=Mg`|2TdTUu1(~h2t@D_@(dF`P3rf5>Mf9dlh9iX{7X=aheQm` zD{}xEswtQiI~kX=f4RQSh)_KtdDJ9pv5`vi=iC5)yVsE4O|EMZ`R}_!tPBYzeY9Ec zzdPhOl6JZ(a=8CM7O+sjB8$!S6zbO&7q9xbzrZk=_*SGbE6Xd`WKTXs*MDB>Kfi)^ z1Of)x(GHQqKkOT%PDmcp$%NW{ug&5+1adb2q3Mv;`(3zCd`PguO~b&e!7%NN(pKHM zyfAG895Sm5x**GY&hck7fE*wl5Yc=`NsQ7zsxR(0kX-T2v=rr&-}~_I-W;9?(A<-W zm~){c{*jx2L3z%D_Xb9iew{iC?HhBKtzJ)PGLnPk;$Quky zKUx2090=$kgM7TFZF+x@gCK)!+FqpbuR&&q401f1^}~GS1_5e1(X6MXAA%Kqsa|?! zBk}op{4;^S$CwxL0=(w9iT~^}-yX<=^wb2D-HUPjjF4qehSTo!&lCRVTN)2xL90cj z!SdaO7;roLK<*SV=JV{#OjBSmO#mTGfbRg6;3X~Wzt)ul*he#D@Jn}#qq+2eC!**QuePWYWmqeQY+3} z)=+gQM@T^ryj}+i>YilUey(QB`6g;Y;p~$VcUKZ)b!1~CD-%Z@a_GQebh6}jG{DF{@*F}7LP3>ym%U_d?f0c8Q|&tK%ufNZdg z=^@{`uaLs?lk*I-2cM^DE#*yI*)82-ZQo>Md^=aiqgDCj)3=@;laV7 z&Fa*%vOYk|c}A^XFPP-~oaqM+YblLhZ@6&2EZvvj#IJsHCBD<{z{f3jjKgf^h{RDM zIlnyU^<0QZ6lq>G4lOWBCftK}`QOY6R35$?5YF$(!6IQi2jv6)3H+1@u zcrFhTEM3-V(r%tW@!C;5hDgLgV2{#{#T9Z6Ob5N7ZbLQttcw8v{vfS!#>O8^{lY5M z?X)6`$691+iB$F-b-%fc0>T+lN~rCPnPA|nLel=+gp2nhj~0uR#Y@wPN^gl|c7cnd z)y%e_t!<~>{e8*0%gV0uPEHguqwqJS_VW)N#R0ZhhpdqtNHRD2ixgnyOyzkd*Vmrk^Amz;yKS?i#e@M;5yw zhN*h%6rL0-J%3rdLw2R_v0T2EDgGE&b5a6wvcy5fqz+Eh8c=G5!cRWamcZhI`d7&E z&Y8^OWMeQ_vOgGgm?v2l^hZ9_Nd335|94FHK*C4L^8{@UbLombS1LA4MK6ZammG;l zD?DD$0)C|VHtn$1pNu#Cj3kxjjbEboJC8wKlQEo5NKh>_y}NBNEZFDuMTdhz)VYN$^DQ6T zS=ifq)?kpzyw)gnNHxUc#&dx6`hf3+*Tm;(61pU#7Jv$mN>W(^WAA!aqvfM4%2Ypj zGjBb&*HK4r??yjq-!*yXwoEF%HL}Rr51OxoAohs5wmPLp-ceKGMZ zZu^YK{=ECwKJ_GB`wAXvIdG3v753=y+yBGaR|ZtIb?urUDqEzK4y6$U>6Di4ln?-0%Lm{}5cT_L_5zImR=d=Nb035krhT zqJu7m19Q*B3}!oQU-PF#rn!h(h?E8&Zs?Oh9v`l6NCCJ!`SQX^aR)B)HMm9eo^ZyE zTeAIGA=W}SZee)b0NzG7@4W_J`m5c%lR!R#CFb?t@BQ^-&q6e*IIdW>Pvd4*FHz%G zN3#S54bQsx$nFBm#A866WXM>|xng#d+JIQ!lAUzl@XcdsvCp^5Y}j404WceOo{pwq zxsyOY$M6pF=O6U&C_@uAniWT5*@^r`!s{f}d*JrC**TXAhen}&{EH>8F2?Na8y$ov z(2Sbk$;^Hew0Kc`hH7u7cSinfo5gVRmikmclZ^F;6Ll$yG!GB!QlmXKU}b-2I8%H0 zsaR&Ytv{h3oc?)a-a0otGqwrf_pW^;@aQny zZeC80xLn~Z)Y+6ZG=Kmjcso9Rd4Ft=#duu_mPkkUB8s=;N5Q3U$;~Gpvu9i?Z5CrL z84ftQV}?@jhiMfX@1T7-?4NO1{RL=1){OBSIHa|PhYA-0+G#T*2SGHtf_uSfUjlON z!E#CBwgGVDk6ZhsPTF}(woWPkjhCiZtwp{EDxa%_n_J1sY*LezVv^w9kDtpUiwU;a z80vL10FO@|v_hM&F#R$gk6j4Gd87;J-$|%$lLaEH9tAKjvBUPq=1l0)kV=t%eVFoO ztDKgwJ__(Z)DYyh-$(*f&#;WPm+_7@pzT>{w?_~qm zD^_E}Xe!14`g%*1r`GjM6w#ygBevXZuq(2FS~WioF1d0-?O3eeP+aSD5Mwr3A5OBr8_`H$)oMdIZHC+Fa*G0(;9C%oZJ*HxhhzgcsHvk5k|>6Hnxa8ZxxMCUS0=|;GVFYPe_~w zs8(giS_+DQIa=ES$E9uL?;^d}N%I`;W5@HY-3>PGTu zLZYR=bi{Y{EboMWJwIwn5_Icvr&ZLrEJo*_0N!C~_FFTL`S%Rm;Tve-WUFUDXt%r@ zo&z_hj6a^Bd0p+cevGD`lxH@j)AIX^6~U2sI5PzfqqY)JS9l^ZKU!+Fqg5B@-kzPx zC%qhc^!w;PZ7CH>r4w|no&?$Y)aJ_x&gQ)`|Bsx07~Qxr&rf>FF)5@=FL~;cPuymi?!%iH?bMDd9u}#s?V$WEq9?&6Rb!MXf7n(z2G6`Gqy>X zwd!;b*be@NRB7jv3$VMYa{-BGAfhkq4W(KN8Bb;=H6)={LXw=}CDXW#-Y8E(&~X_( z>cd)Q$fYo@g=r6uG4S?8OZx=>JMi4_d`jg}ulS|PAnN6xA%PU*_MK$j?PgK!l(xak z+o%+y-aFcWT5~fzSO`Wpv0N6scVVFJQA-Gk>{x28`B-Iq`spvZ(Xt29 zu`H=FL=!mzpC{mw`&yNUP%V6RW767+#P8?ny(5tCU4Z+OVQ2al?-K(3iin%6V1BB~ z9_1`YIZqI&V@SHO(L>p-fpvt~R>=t7Xg8?G*f4`>mWZ;h&JBrZ>bbI)~_v`oWm8HKUEnx`(ZlxJOAPUcBKniV{;9 z-b_R2F+SN>XFgox<%6G?G>3dGP|(+9gK!zc$tX-^As-G_`?Y%BR=Hj_wv;*_%`n?E z3pDV$ZzP5^?~x84qKld*yGgQ+V?)*S^}N~%=#G++y^c_GDO^RixN0P-Gowe zC-^4X37gF~3;I}0{aa0-+~`D0WL}KpdKNGHZ9QJ4$tKEFsWVs~w>Md{@VcJw-^_f~zlWh5vY_5@^ zy{RPj+`McE&VwKG-6K!Q*~sYXm!*t`{Hr=bi~PC~c;& zT>h%H-}0Pno8uV0c>&ehy=ZKaWRY-DvD&jBAs0%r64~2D>?(tUSOu>=hTX98!=uN3 zyctplDUy)|?S|1gILQK}w0AKa0wql29$eqouNUrKEUSxJc%7wf{#x;)M z+9ic5H-ql+pxcv(XpZ?z*wl3(6LZ)($QD)ydzGo;8X5>qGyT2DgeEm@vf;VfB?(Kv zw5UTG7lbEBno|+-oVz^n`hAF&+IDa$PHX9wj}y|oiSEXnaQZd34@-saJYJ=sU z`K=KK(h;B&vM8Yg1J!s5Ujq)84gm%;J5FohBA2wvkymy|Y>Y@ES&HTvR`qo~CSC8s z8LfyRa=)~k?;Cu|_=pLyV8zSA2%2O+5^CZXzcIvfuX=Y?G(PczL0fe>^5JPLkffd% zUfFz^ST2E>oz5ODa$$HJy=aw$xbb5{3qB^`+BY&V7C-G7> ztyqS{HO0U^KNVOB(fCESE!7nO)>Jz};l%kBE!Al=Umq-SJo~@* zbvE)6TBA7BYnK#Kcz%4uV_DH&tH^HD(~vq$_oRz4)mr(hXMWT2`cXF3!RkmlXc}Xx zM$a+s0Go0F)JH@S^Z0C;l2e34LuX5UX@2;j_{>Ir?b4T% zAaHr87LoD!z$mynYY}4MYs`i>zj34J3kAwX~4teOB=~_Z+|Oqyrnd z$EacHQo9jHVw>2E)ALk=Mvt9n6)T~cKZmu(2;o(1-c9i^W_k9u9_Aa3Yv&KE(D4}P z=q#x}b}sBNE47?KUt%#@7q+M?*}`E)u>);Nz<>HuqbmL3)7=@eaxjg1&^thJ(F*6W zoh1g&XlY)G5If$}`Bi1j7TR2Cg3`Gkzqgaj$P2?oh=cj{Dnes1^Q}nuqO1_bin6Q& z)?ad0n~p4&zX@=i{oMLZLFZs;cm^>I$f>X#)QH7$&?s|?KYlw+-Nt4z?!P*klX4la z19SSyBlh)sFCJ=RbsHd&k9_y?aRE33Hp@{m^%74W0mq6AA3Iqt?^z}#iLW%)H)&}{E!H{cd{9gsC4Du31 z62$d#B3Bt%@tlj;Qb6}NCfJ>J;1obZF64{CQQw^BR7N{A11r}Y@%=`tn*G{KGv2m# z{c2~P>M_e|nr{GfdjJI0l_M3R1*VGRUjr?BHm#Q1h-eO-!*z8756>pbaqFG>$D3XO zh#b?w^AA~b9zRVr0=7lvS|}hCKsv@ywaYVwaxe->dGLfGODAHq;9Dv>59|?v@S<=T zaT>BNieWQP$sIU{4F-6jPIA0!QT|ZT9ls$7obE!q&aFU3{x}fr2dJdvfdPQ91x3;% zdyjMIJ+qC-F3mg&`%RJt$aIAs!YqcvC}wh|zc%lo!cHe3!y3%ez1+qd&ew_`NCjh@ ze(whT6bu7JD|}UE{BWuJt<~z7Se`$)neI5&5XiZ;T=3j>6sDa*>rAb+-y}*^*RVi- zGF+yC1jnA7e7jjUY{gCSVUP3DryD!38oOQ9+nL8!>|rQ47o;`wG_ACnwLd3u zvL{izB?u?HYy3(6c1w6|=_q(Zu~c>7y>;g&dw~+RzRGaBtue{HC&yibR^b`+>dYn? z??#F-v*D}4)`n+UN4h`OFy!afJY8^4VDrWrSZyKS+ZKA4W#X&4`o_%B*RcGGD!QXs zBQH-7-ySwgCoD??C!~mp@JJ+e`bUF6c=33tnRuDW@VBZ`&y3m{XE~b32}9=Nd>~Rl zv)uM)isqR=w(UuNBlYZ&+bA)i-%hCQD}R6GW@ybee` zIjx`cfwAh;vI&#P@8$Zef0)m6yW(EZ(HgF(I;`^7ORHEg<7UeRCwafyKNA+76(`V^ z@5!j9Lo81DvaZD>)9KZ3J8nq>E?TtTMa}M|WF3=e)CI&Fy1@ z2vnk5MmF&X+mdrv#qFPV6ReKZlvkQYRb^-%9z;L)5>YMCqB;VT3F51HL^8L|5HVJ= z1Q#laF>Gk}1v)%qo>1It4R5aajK^*o6~GG{k8m421ThuZPLk!jwIb&C@J%{-S#TW` zK5L~(Mgm8?xG(6D6I?dynqGJF2#cRf5<@2d&V2HX~|akfnXw1@20S08%P4hYz?*2rFb!5p@N zW|O%y=!iUc@?sdTk?dLifI|K!JbCe~hZHjBDa-+E+t`}F#n7mCia08oyOSgzA>R`uo0L!3|1n)a729q_)Y?KC5-;?f$*95WK}WZmHMi&~AaX#pQfR2zyAFyg?h; z1*H*56Ip~xkiTmjPK6$V3j8$1R?L94`3ShR^+Cnw1=kI8h@dw_zTzuqxlW9T;<25V z1*qg}l-x@>{YpQ&kF%(iBVw zv+q6Wcr42ZAx{q12$T-mm7>JCdz45h2o>wJiB4-;LNJn7 zYkm3w(jX|k zNon!?R~!TTryqGq>C7#u>szT%(3AQBkEQ=oxvzk--tV0Pz;SBykqj+W9~2$Rnh$x0 zYEHPx7Z^;)OL-K#$L#R!fpYeK$$F=90sxd~1x4wWPxhXmG;uy;X>={rkOm29wszkA z1A9oeOESM4f!TPW87#9MTQ!d|4P9U*WZvW;mhY|E=2+AB)sZZU$R>OTcmtJ80wu&} zs?zo?vE)^Y?K}SLT}cYq2|TT9C}ZpO1q0=12|%r^KS;lV&5YPnH{5eG<9z)-cVqg3 z-4f%?+qsqgl(vQS_toUkB0VULmc*T#RXJ7Q99)BHGxw0AY3_MsKdALWrH&dsOf_Ru^h&0 z3>mw1W()SdJmv2G=ZJywyKCdaKU~ff9jaL`y+{S&Cg;d5W=o38Wq}%`+VK^kPyOi1 zLCV({CD0p}Gw~8{VaV6Jw71}Fy8+4_8GneqT3ocojQc?W)kWv!NZX_}?NOH+K+vg^ z8^dZjM)U%~*~7Fbqnq%rMHa+;JxiniZ2ymd@XwdcN+2%0UyX07zP|pFxZqI2Ql-LM zSk2Y2_?=Zyl2gg&K5f$(Mdnpm`)&>s$vM_Uyi{zJ;y=uaXL$g|AyMUvCIgcl7c$7% zHmrgcnZ|##A3QDVqH#NS0GUWQt$LiOBV=@?jZSkawZR-&dz1PX2%wqeY1Ke6DWyKX zv>RDJDYcu@u{!$)qk<78)Q4Gx0>b7XXtFI{9d7Zz*{W4VgYl2{Y9HB0bxz+)=jw_bYO&x?r?EwsTJM_M`;o4&v^ur)oVIK4hYSFwL_SomPW5Z| zU`^H7oMfmD*hd++Fr=Vj5QI@mCo2ZfgLt7i=PN|tI&NVvTWNQJqKu8xNLXbPswL$U zT{WS($ymdGe1pn`I^z1uh8lbOpax%v{9t&>?5>w3hl3_OG zC75_y^Hc$7w-z}dq2u~;J9&9 z66f`7hr-swCEzge$2rri@A>fo57YRY-Xa?i*;Z8ZjB%`1u2PBz$cm{bWVK9pU`ol>)L(B4;O3A`Z@%30+YGv=4C;)R9VKYuEK29rFf}qD}@JBD+ zs$hDS^|Aklg$Hv~UExf|%r3&ysP8* z@zP5z2K$YXw<=!n!jbi8KCXG_jsp;Nbqp*bN%Nhgf_4Dgz$04NoC629_8@?F#L62s z;f_^8-7#zleNu3Cuk#8%_A$Etl`vtbzg$Y}&mf^ynsg!(Z`c@GfMFu2_x|AC? zrH~sU1N5L+K1>QbUM=+`kj<2helq4{@uY&tkAt(xZ*`&yH5^k{_@dR7u+k=yEG0oE zSn1~{ybFd8dF}w%=CZYewq@xQ1ZR4biy+2J&(_X0z+9@S{1P3`j@hJV3y6FCBIp5*p3@pZ&3(|9nZWe8ZDUvgTFPlk1gQ*xh8- zZ5vR)tipdw9>j9B#O^M2yci|nbram+zN6eU_bs@f_YN=t^&CF}?Ux1SclT_pA-k}d(a z!Oz*b+rEW&5{UU+g!u+PV5s_aL^0i#-abqQoszEky{=VOf{#6ucdcq1OIn9Ofu)(S z`bx<6sY9B3owicV>D7YW;^J1#S-=?(c=>U-sN{TLg#~%N1SM^HeTJ;aa_%h(34OH& z2~OX`crSdym-Cz&Z_kdl<#!KaX&AfITAt^#G~pOlAeEB1zhWsOPYmXgDS$YPSY6yN z=k&QuEBESr>oy-AQnGeRb2U*?sVQjJ3hJw8s#2E0Nv(@y3ZW3FE*} z$262ynHEME5s8MCG0>ryF`T3c3!vkm`AYZNl7c!doLc)!T#KR4&lkHo5qE zxXhDS{hj}({02UC<7Ll+JcG2|EhaCyV9lKuJq#MEMyne51yjO zejHvc+cmemoQxK2O1KFWEKysD^V51_crIq@dx&VbzkL@n=2&f^e?Ykwb6%5{f}Gb! z<>AHk^9vc5n8ZM86J=fSQt{6y^_q0)p0LSmKy;}<9Eh|7PYWJrPqWmiH|u;rv^Io5 zHr^^m{EA-bnW`$aU4wAB!J(6F0^~Z00lPu#piz{@pyl=Xe)V9IA}TFeaF_Dy;7j{2 zmzuR2!E=!ZCRfm#w=bBgLvrGvj8|a~H`d25*ufZ7Pxh{NjVzCACBnGCp*cHDDqn^G zDU(%=T+%HiMJ00acuLlIjJ>zzZYh&Jj1u({V;B+jgP^IFDb0D+2)q!b9BsZPQ?B8b)yYWSMDhSMr!u5T&c$`|)~%kPcRqEROHYn;y)xtO zSz1igx;~M{R8FZ-&R6oBL^+7;YN!F2a)i^NUBvKbhli*nxmlK(EyJdn4?8S=WGS{m zl+ta)fhjAQCB;ifu*I9;Shv{#x(2Ls7h2ca1KVuqN8)?O@JODm) zMzH2cJdyXzSm+4*zV_PCaUFby09unPOw~MiL>B`U>=8+>uOK)>^$}Z)>8IXQ+br)F zJ|aD8cQjEe_EUgtL=Jz49cTOlt#Ar&a1l5YRP%!(35E*jgn|h_=$B|!9c6}31>(FQ z&mKIOGs7OQ53_Ugd4Q7&)UEOfJg|k5va5RjRn7`HhAQ!WS?=v#ff!Dc1gCpvho+et zWmj2iRV|>XFB#zY-9aX1>U@5ThQz1&>NJ@Ga(Hm)RJ{4z&fl~JCt@e^Us{pOCKpOIxwp(p>1>VxN zu!5uXLXKcbv32{bm|t~Z>BWRThUIkJkNSw>nQ{JJ=vs5tB4~K%!;Vm1{kx3Q0I^t* z&iC?PSsdvN&!}f0CI1CejiR|8i>?3a@F{zY|J=$AG!haZ+{8ohMjvimiA@#MeWR8q z$7i>D4*ZoRr&vu6T{NmT8EvP23$lx$8vp*0KbeNkWBE=TjUA65BTq1G9h^oK2WwlR zz}`WT4X^rWZNkfVBs0B)mmc&7>OI{d{nU)+@p>x-=L#@#bizK>j8iPX;#01wRr_J} z0_7a8gOy<|^y-hdjWj`Q3O(Vv_?q`Ir+OLH?aM((z{D|(XqHPDoo$D8yUgumS6Lh4 zr&?kBrDzSYs`-ppEXNi0h~0pm(b|?OpC#QC83<|h34wwyp0U8X!arx?V~z|>QsO_{ zgUt_pQU_?VpoR0!$cx@(ph%C02INF2h(~5#Sr}jKz90cBV@jFb{;a^}o5T=SnioaK z=R1U)n!ckBZ$I9a<@s<1CNR`F;4yZlyX)-wXWIjlrJ<9zDyP0n7aJ4hLuC^ofzqcq zDG9vJqJT=3=@-Z8n%z_KV|^Ae>e@_gFgzPCeXPBM@EHKHuHk@_p;0MJ_^F)HsVJ-k zaRsKFwX4krH2K6=adp%cfC^qNthJ8DPksh9ux=5I6n;u%{vgqi_6{z+0yt)as$WE5 zUt8K6o-ffr|A3P1;lGelfFU*CLniOdrj5_91wYTHlo$q{8LPwDa_)#}QAc3IBG{#R zM_C&_8`?XxW6Xb9TzLiOH%iIePgP=2u-j#NTV_m_&1ay%)?p8!{WM*yrDUx^LC0cb zzXXmvE^eD|ID*g(fZ3tCf$`C@dacq*USyLYbW*lY$g&jDB&eHgrbz-isrHs$B!e@0 z`BN`AVarShL$j6hl1t)MHOhWQ4!cO*4dA#_WHTngY4*81bL4rWngqAyF3St;-3$Sa zAVi1GYJqC`&jDpbd=6(3yQm9BQgFLt(yvU)pc7NEM~A~Y82l)M=pE#Db%(1{*MYo4*AgG-1> z3~E%NTP1zIyW6EjB^wX1BcAxVm8zUGy6TBIMzlN=%dv`xWM#2v%$^aw%>4`0zb%%$lKi5VxsE9Zm{3`@;bP9^D64jUMyzM}Dx%IQNWPxC?r|gYex>%s-{ABj@dXa!kD*e&G zncd=D<;{5=BxRB&Rk$ttIG{miACb)k+|$y9128)SY0)sB-#f%6bOHJ>&bugl;hr}y zG2P-gEFZQmqb8gl+J(B_O5{t6WYm@qO%brSn-FXZ&FalYL^2EbgwjtXayBLCRSNUyz9m3@Ved}=kH69vck0~ z6l9Fbw$5GdqM&qci%`mYwnI~xJiv&!zLd6*m7L5E5e!;fcnN6T4ti}vGau-^o7`nb zh2O2^RLJ7J((lV^P2uOO93=^QNfvAIrma>{)ob>9)(N`Bilj&oHd`6D(!`1R?Dyj; zH;PNMid!nD`zym33YidWDZ@*Gd21f`ozIvQQq+xxcn^ClL8fg*04M$>Z)5)=j-PHa zJjo^`v#K@myX@`6fHNUx`bYqrPeC46w<`A$^Eo|lgfcFk9G7&(wYnSfI&*jB52nlf zz=oHo8#!N2i|s8jlIOdzNhqx`k9=C$`}O-k@9jRa17CB=!HtYSiR!L;5*Z*a_W3_f`IWF~Q%Cb1-+|eg9JkLB zC^k#4>IJe->vS?xT)8=)L82`&R)NWSF*DRng#JMkTl$Tgg0pCkZvOe_!ADtp401=O zO^WQR({ED;|?PCv32~n@a;u>dvU>v%T7tJhTp4b2f{|~SHRH>$Wt`td*6e^>ljxSE`F(xeYT%HlYPmKu zM!8l~y)%*Dd|w7)=;wh3?n0{m-K*=5!Sg9KNFE+m8$~f+{{dLPdMR+pFKlR*}Y z;kO?A>!ZG}Mxs-fxNsoZxSqq`xJ3y*FSEE8(Z4Qfqz88#wes@66e!_*@Jqh01*QC( zz54&?CxX8uoChwEU;A1ja1F=&Nd{Klbkb0h;y+J=NesLwz-2$J704A{#Pg`^+l#8O=eHiA6#3|H=b>>2?LESIY^ht z>^Itve^}al%*%hf+c;XF(_y{N0-CgGT;0UM8yyUWY_?kNb6}j{0()1YLvB$7FojuN z)Cb`+_0>=AW>YXg{0C6d%E$dKqOs2>Bs+X0=?5l*d=HwS<}7`EIXN^|tE>j`VnXgz z8f>-Y``7N|f7#d=!^kE{8As2**r%ksNHPY`TSB~ROX=_6X`U?Hmr&zpL;H-zFhJZh zV}tvspKXWV^Z~=R&D?-;46k|gIInUiHlQD=AX^bywN64RQtwms>0PoKI0@Bv&Z-hL>Hs}JY&n*MZ%PP%=m3|5s0N8a9h3Vjgc?=bGKFb>YK_7LW z$5JGNY;%64sV$_~#LI~_QzC5_n@**#9ni-6e5JWp>Kjt5gw*J$$hK1_(bk;AW;-f3 zsq#^-J)^>h$n8GQ_YU`ews-w&u(5Are)?Cm!_Wga8(35h!Seg#jPU42o2+yVNk8rh z8;u9+Z#7QcxvG;j4t`PU~ z&P=M-_u97Vt;Z&QBg7jfg-)!WWE~6amIUDwQkMKtS?Ub5gVe z^fTUScC0eW7fED&RL&NDOdG`?(qJi*0Quoz9l5*{mSLr{B(*}5k$J5AQ_2$WbO4W3 zg#-KNhbdgAtDW16y|F=^P+C#s>lqZ8KhV-OiX5#06}hu~|H&^gklpl=MC^@jF% ze^wWmrFlxY{;vWCL)7yrk%QCK9>m~Fj8L<0es}?KX{TbPs)jXe& zPug5H=)r${8Tf_Hqg81Yzr>&>lgRI`;=5XHot8#C#U*UKywV+8zCe~Oze5Ob{VFMh zdXU^CCgc6oq0;^8@zkmI<7|tDYAA~lW;~3J&Sb6ZWGXQg5jy;2KU;Nzbby2cNdzyPh*UXrcnn5h1HC)mI=p;y+ z6Wg^*iN&sqRI`v=AI?48fxOuAOuLO3r6NH`NBm2_c)%h67rCV5YjMd((6_oDP!uoe z2ehGwbgaBGG#bt>gK}8Dw<223)t(0G6{ErQJSe+4F`%8|fGO(<)uPT_7n31p{4U4- zr;jYcCR|)8<-u7tw4?TSM43Rsfp%NG`GkMImKcS6;4RMGN`MFc=eXFTLdY0|Tc z`?BX#PZL)jhzzHlSxJHUG?|9whg?a|ARndUnBV5DHWQtki--DeRq!|^_av5A!8qjc zs?}Fn%_mEJA_~-` zVb%!pBV99iq`7uiA+dd*Z*2NW4|W~34b*rzm%&SZ^=9GU2_+jHI3l~JoisWAlhXZl z9=-EH2Kzbb7K%SJzq3&B_ zhr17mR?LF%_Hkq~ZMTAOVP&EuOfxvMe$dlP%xJ^cbT{fm*fu#Ik|*Sf9_F&tS@;;W zk>jO7?h2~g`b60ea-ZESt*vS2H*O`kSq(|P2gVBFor^;k<9*-Rwt-jPlihfjzu8nR z-PwnZIC-T>W7(bg7R5HJOcNlbZH}1SMRbeQv73`NNmTdfW`>)v_3vOAI))!Vy>Q}e zH*o7uh#lqJ*PqGfv}C0+J8$~MZMTtO%6-&f>$bD|l=O^-fPPM0Vd?!Q_bu5ai6oM1 z59Y?ry}R!}%?2=90u0P%cflxIrw)Qct5$!W!`8nC%s>`a*`ziXF|hoirOJ@)g&)Ud z&;;H}`LRTaoGAn;SRhg-r&16$izm-4nT-AlMEA--4wFRLbmL!P z2^YCcEz|f?53jwuFNNyk>Idt|^!#&BDa7Oafgu?I;lcR9+fwVJn>$2_)k>B6^1xLy zNg#3i)2y|#gS5PmvX^G1;y6pzA{pq@?JrxBI?Bdmsj%bo{6GbSW9A`6Hh?gXPsrxM z^|Q$%+^CI9+g=Bf@KaZ$q7%`0Qspx*vJWPjO;$cb#BnO3$d6@taf!md0b2JUuvJ*3 zRs8`ER6Ezgy;zzrf6c2>-eXEFyBSv;bU0bv&`?W(0|cd;gS8*qj#l~uiVeC`H-k3~ z#pB;l~9!mn!4lB#F|2QIhs=h89E(!}gIxe6t0oC_+Oj$^$;@EwjJTXy1s&*?a0=Cv5%$8wTDc~d>3qPbh{%J zI*4TpN&Y!jxamdfM(I}lW5-U7VLId-(V;;<_U%-iq-@bL38jpYtD$+n4`iwJitlD$ z?d~3i`$C7QU90b2W{oKdsR z3#1?`VTyZMNm-%iM_&HIWYsInmz`?!v)FzS6-IPYM!lxA=dF5&SJvJm7iIBJvTP-p zjxYtR5eLg|5e|?f)Bm-R<4uo|<(UCGZAd&WkgAhdHCHYHIRp!cmy)|w^E-7=JHGRB zhf~W-z0k>vXYdQD&^fhP$po#H<~0E1{}2lyR-CG=3%rWzc>{DNAhmg2T+#(h0u95O zVCy>#LVhb@)QRVc(F0JmbK_sY%6TPs`3zX@cQW*-E5wB%QcJ<0-T^WuUqls>nqXa$ z{*^`~aA9>ay!M>$)A5^1hTv#!dj>0#^tT#ZxtdpgpCj4 z(}P)}V!?9~iblt$KS_pH}4B!rxu9$CpzkrSU2*nJFNdk1LKK^ixjym`6VLg zWsDS1FKz|*hpm{p9IS>{*i$Wz0Gn0?ARM-Fngq|mX}1*+i&z|p8T<{G8od0FLpf;` z)1f`T`+8&@KY+wadff%|{<$pS*hq#OSE>24{2eUY&DH0pyynO3McXYYjB_N19mJB(X{T+_;_!vR`m2M{Ey5OdhD2`OEl@$Q0 zSFsfye`0&dV-W&v`sELf}=e z!qR)l^0%AwUe``08_Uy$k)?iPU`eeX>=;D&zkppRWYJ%!)J1d%8J`u~&ij@b=$I~3 z>kxCFQvTGgwbI%I_Ax{UR>kc2c5n!BO2$CY6Gj7_{A&H|9eX<$Lx#|*t_T{LI(8G% z$TCDvG;hvB6+CWHkId(+Th0_T3GBT(M*S!BG}iYvK#)zs5puuwDgx>H17YLY059gS zjML5=?ST2cz`E4N2%}b%1gEWI#iPdpF{Pc9Cw%U6@5gJ-%xN_%zTM+ZOGCy!J zIf_A(-8c>vm?1xX*tV4bwatn4vkKwBfVnjOmFQQ3yBB&Mb|jHehY2r5P-=nHiu3JA z?oeK=m&@qp1IfQEmRVZbWPRI8+u(~DtT~DkYAS%6ln>%$Xwv=8R(@I*rWyEuObcQ7<{Ey_7m{!Zbz! zg8CVqXX7_47Wafv**9~GP>v2LcR?5GDjH3CHbBT+uT|z-+rh3)#;393Y()L7zrx<{ z1mvc%@3n0CbJKWu93VSsHAlFr*Y?<3jF(i~`(Q2y@Cd5ZSQK9iWtF6)Rm{#j)?q@P zBX+74ktc26+oa=?P(q#T)=1r0)Q-+@b7I{NW5Ab~Onb@#%iSaOe~~K!Q&hS(fw+Ca||_4O1A(wbbgy zooewEYW3ghbs3p@+K4|hQ_JhsC_ip(ZJX6YPWT~LtHL4tL$3~{{R(PA@|rp`o2+#? z*|vT@_UWJb3V1tS>X1s@lI0=FUv0wi;rI<@mHZ&Tvcg=0qikD&)bzIo6VsfYAETmE zY&Xw6hT!cut2|zs>tsJn#+0>_?-xfah|1Q0a>$cALLfH{Fc-sNw1db(gcB*J1$_go zwy#2(MSeo`nvQGA^|<_#>*7S=NZPIUSxk43*ky$CZOvcH?6|?kb2)1djKR=lcf3aT_kH z{iw<2d@y9^ed96U>9dixOhfrOBTqZcB9*ft_cQwka& z>e}RskD0C47@8a6@#Jh{EJT*-ma6d>`7()4#US8=WU7)p8LNv; zxvhK*tJZ|9onxae!rt?{bVV>*jlerS-+T|g)*uJ-G!8G6K{ zRp;h~Y@I@ld!<;7m2gH+eIgj%3Y>?$uM09+7Qr!CqaayC4YOPV=Zd`QWBgWbP^n(h-MK~S~`P;r^J zlcbKC0~A>-i$gRQ?{gAv%Ew?jzz;O-I6Ksp^x<;fe$ve00K{f5X~vyj(kAtXUgPi;jUQ4?)@4&Q`!Ng9}AKDKKXGeh}nKqJH<&^ z{l#B8p8dSHY3euf1psr_l;@I=X-AcV{y< zt``~(a_!Y&y0OID#)L8WobRbYX(O5tR^A<5)FM5%^QM3i=ZL3~Z@xV01)GYG%r9j8ykEh08j7}k zfIL)H%P2iPWjn@Nl*g@J_YUesRU8TQJ`2HHILYZgKi&fiogv~Fs#)o#f z-<;nrBbq3at9)8rpk3tyA?Bkjw-(8`=~|@_!;7xQncd;z@Loj> zh5JwpP*bvRIn(WU%dQU5Eb5jbAhNUK(TYmElAR}wr@{4#YKt!+*9rp<4(?BT$Cfj+X$Q8pPM zUi!5s|9_(6OA_nhK6T-)Xg) z*MJUjO6uv^`?2;)qLT5m*dzcb=#IJroCluB1X`2VT`#ovTKONIi3Hdtq0rc+;KZbz}AXYDL7exI^Sjpk1 zC>afh?VUR$oSM#Dx_>;3I{|8(TmW9yJT%!6&Hvmy{B55=m49{mS-awd7-u34o$0K2 z5u#H=qW#Sk53+~fB6-PjPk-e|ailJQYbKt{K2o4$sT*Ih3FTmc!9Y7$iwF#xUVNVD zXQq|ua#FlIYR|>R(TO3S6Y(O`EMCJ(%3&!ZE+mKB(ue-y1MPORavqp#`5HMgVNM8j z>9k-B7hx@(V6ZR{+;RH(69|cmvy?S&i1g&w;Y48GZXkKqjusmQ#yV~B# zv?OzP_j7NLXp*1AYz~n{!=iw8328Y8)Q4#*=|G71WO7DOl-1ycVPSgQozYrHewM>M zx@vMDr5T#&>Yfz~bxiQ`{TenCJGW;|O{1KRjbaG;DGwYS&yXqkVy#;ZSI~V`yoKVd zzJB1Q@bSw%78*U^DJ32ZF|~PEaepYE_o{uK#Y`=hR{6MNcCvbIr37Jf`s*&O8fi$+ z1)Hteg_p&4;BB?ZQba($p}4zSl9t#kP3imP>!u;EN%jAdID{vIY4`Z?@?w5w%=ZTK zH%OF#XTg{lTG>MT_jn#&Up%4=Z&fKD2Nc+0hTz=?Procm2x(fr;s2yBGi!8t%N>XS z3aI7dV}M@H(BhLdJGFIzP66a%IytMM5R7H`-dnn2RfXwrpTj<(^-DZyIfV?g{oR-1 zxs9qHn3KHDHYvsY;SsUnd(p)MC<#c-o-r6uuR;dIpy5pbt|s;K;+&LHo<$<_$4((d__q(lQCk-mM(iLWFx4m-PHI(AvUzW4SbW?wSRDx1L1 zg#+6BwdoC^L|}U>AZqPetj+ySShxD{%IWx(A!Fa?mYgA1Ev^fJlf5N{(P!@k-Wp_; zD~fd{yeMhVjilERYm%o6&A#PaBFm2WM)E&eG79Szd3Q)~nn}rr$2LxTN>ec`pCW`Oq#M0q6Hv z8_ALe78#MT$k2tpLydqRZ39L`Wn&ndM{0ZupulcUxnosDO>pEkCsCNSe&$PW@GL&72&CD%&rW?58c8JV9}vnwda@$%ggI9Nf!71`(^GKhjl?D z;g^ zH=*&2o~A8i74%#FZ~fgT0i+UH6Fne@d<9r5)ye}Y#^RD$Fr;`Dvp~&w*670bYpL-8 z7pG3zZm1^OqU4(f0zJYXYeIL+$a z1*fo-O0iMA`fHOU;6wFovQ_>y7IGQ>{ODj~%Ik8vDnurZM_OVz=&Gm{lqZw?cKdJe z+?>vj1MW2rV+vH@t}->f;@5+$dbWI~!Ll=S2O<%>FU7xbvM0%Qvm#hi=bdS|>4NmI=|9Dz6fxgp+RA>a9@bIn;dG^#(D`V)U8 z7@>speVk~0QT4HRTF6SVYYBZNW1yw9s;TA^h?B$P-OakGeWjQ!_vI(}W4u4yt8u#U z6%E--dql#&`o(&6JQ&a!RLg4|&tz2OonD;A8WXau4yJ_Ni&#;4$4q5YYSd*mUUT$= zhlQkV7#4eMrb>IDbOs?zKOnHrTuaC0nkY}cbz+@U)y>In&xYdwcb#*LaB#5!o@B{F zL!l#X@QSqf18i%19UDc&iSK` z;Qi^R4|t4j3oNN9D=ZnTyMWi%Jzq^Emd!NnA>P2?v88q=na%Y7VePHss$8~!VMR(1 zK`8+RR6;;PK)MtG>F$#5kZvRd1*E%0I;1-l71?xmNjFkk`kf8xVSCPb@4fH${c-$p zp6A&!vwF>1Yt|^{D+?|6TQFzVrS>#`^}DW+t(N7Py8#k&AA@#P!jE80mU1}1-!3_D z1O&8Z3*;y*J1iJ?`1z2~lgj*{SSyP}fEy!<~g1Qw-qMF4D z-Shq&UxP6$IXbNBkA6Nm`a&EPRHBVpht~AhIIeu{dRyAHTxT=OVK!Rhzc%jttbo_G zih0|43pt$0S;5N{{ZO$x=7qo20pGQnsk&=J;P^BM@!m+_mqSYYQKeDqBo4mldZTLJ z+mfr9!6BcwSzOhQ-jP_FY)e$$Gqlb!oB(R=(giV02x{@H9}9hB&2F~nzw-^EFj|q) zM;f&@EDP>jH zT4JL-PcYJFs|eC~4&RbdQDU_}`=!NN*=9_wC4#$l#0jp)@Y<(W@m5`u^cq8cb;7`U zpWPMQbY?~JKt#jK_W%oofvKYS53b8>d;vAO9dG4MniDwjD>1E^*j7QRs}nR2%93lo z4$fI~i-eLDuIQXnmQo@<<#m|rOa*ANm$`J}*2Ufd&OvJvdo6%AVjmQ^x(T*j_0Ecl zW2*dfdjgVsk00ZI&do9k2WW|moH?Q9nY z6SL*Aa&HNOOHER4H*(FC0qxua;;%Nlan&gprKJ`lvBtfb#%ogBXH@;?`UI?hE@yZa zhHx_&?TQP97x&^52P)U95**Imp!F+@?;Ip!g)xbF@LaXS%A=;BTH6PzE&Cu!w*c;d zh27I6lE4x1!!6)Djq}OWC2+M?IL|96+w*Ty!zEc%ptb84PN9$e86Al&8-1W0i7JcD z0=&4V<%t+#Nzusfv#ZR=^|;p*lmQsWKgWaT(lVII{FM6=4KEoam<8lhbwR(HOT!h< znO1qRZ&*7n%`GIq-+FvXR%c6DikdNSp5jMSMaE_or|k-0TNFyAAGpC~u$Heqzl#aMPz;k?!--pu(*fH0Vb10Jxop6jXTvM}FtV zKDrk&@ReVpEm&NAuAU|H`BpW!^D2TyzLQ|Ox0oJw9h~_x?4TUA{9^;&d~aK~FE>k2 zFx$OpRKexEDO)R66|ea%N73-9pMq0DzBC)3`tmE#BHd}W<+QLJc2|%b19w8aRGIUHX5FX>$}VZ)Af`xzVnD-BOHbJtj&zxy9f-h zUr7~~O>geovo^?-4-Y?@tMSFDxl9L+FV0J~F7+)htXA7|WE8i=cny~rfex;7T_D!! z^5Lbk0|Sc7;5Sga&e8Pk`ta&V84KZ)?nOgwz1i&2lBs-gt9%eDOZlcKlaToHRB?;h zhdm!X80si7n`c1imf7`-sq&NvLDSm|7Wa}Axhxro!Kl(xiAAFBYDV58KZDzE8r@F0 ztN_+zC3matTlP7l2>u*_`2Z~IobL(YLNln!lf5xpC5Fw$LtQ$Cyg*An|LWCw2?37+ z5aHz*me?PijnH8GDk^!u3dW8$B33G`lqccrvVtOHB1>cBn-PsL;B?+`{zJxU&d5P%8&d$))?z*pc6CgePVwRLCt%% z*>d45X0aQ@jE|dynht-AqaY*8fP+(1VGe=dfG+Nui4$2sf;w_Dn)FRX(fAuDFYwLf zZo=v@`L5uDWU9{upBM=olrLTg0#U?3#)IbtR71qw749cQDwUgg(<(R*_n9H-mxr zaw93<_T&eCE6_AZ=EuGmql)|$iBqld%o!He9_Mbtg%4}@!@}A-1n> z=Og`lHW(XUK)er>7o}(oP4pkt0sstBZLl1_8$vhFjLdKJzEA)^$wNMi9KuFv8yc{tQhu-3xbD|wBdegRZq2cF~} zU7g#6ubK#mVJj;?MgNB-6#Hb3tmi^-Y-}BR8<<0h( z*1t+&8?s>z`_hQN|9OWUfG!3=C#s&RP=f;<#wk=_-KI-+)7rmR2t%XCz3V@CFI>2A zazZS8$ocC+$CWV@7o5J~rtiMN!QOaSQlC7NaipI-!9k?b<}V~w|BSpz?YT`Y;3Uf@ z{Y1Jzr(XNRxU{sXQbDrkt8UfDFM@a5t-p!so3Q9Lns3H+%Wd^li#15e#hp5aAH2B zC81HP64A(t+8%HL0hRQ{`G2&ofm07bi%}D%jUPob&cBorL4sg8P^Z}^wg{e$>hTI6 zL@&OVzq)t_Zj9ULpd=^8&*%KFVIMq60L+qEt#{?duXV!~mBI(yCY(_+pI1NEktPcs zEY_^6Jv_g-aIuRBlH^-;R`P!in+Yg(eDw?wr%-}&=oGLrx$X4rUt^-;yasdM(MrYs ziLtW}N$((n`t(m`&#?X+)LQ^7LP=>6=loJdD#0K!$0!Z6ch3L@eKY}IjXF1eg`fTq zJci`oSiSahJ--HdMGS!Gvd+H&I$i&TchJVlBV(abG2qu^A~CRe>>4Z8&ad`f7Xqk* z-d>&k1~=J7#J?@;6p?&N0JfJeBP4z;!s8>L3w6zH?!RV+{S|TtXgZlS#OXiQ%9p%= zgvy0mOT`UOUBtv6M87}1BL~prJ$}RIBUYq#9x5_{DH^4Yk*_OS^c!m zkI5%@8DKGcV(#NaOmhf3X+*&fVJ5tnFyW# z+!73e7IpvV_f#)&08>2WyL$esi->ogg0&)!UGedR1A+7f-!<5<+EP{g=SIDVcps4P zkl;5CuTcZ^0ZrHV815@HQn1zwhByBHI(Qx!!NSHU;(GJ03Y_PY8U)u3#nM} zAR@iGt@QWm!5cnd7+ZI*GYt5`uT?4*kSb!Xx})?bW1ew91&jkkBi(*Yn4fgrg*!=MM&q>w5(~sn$^}nX#yA6ZrdT`_UuP%6?UI!pNV)c6ao2gQm;~J`g+4#;>RW zV@caq-?;x%H-uAw2WgAu)_K1sf>Bkf8%FAF&ek1xbfbpr!#>|zvtB*_ITkoAB{>zE zrul2C%MaC$I;uMjDbkx`(#yO{hglPY6z3gq5> zFIu9z*Hy~`J~c%Y@}S$JnQpBsh9B?oatfW{6)P4f%g%U1cn$Q#3p_Qf5DmC?MzLQj zz2M;q(=E|NLh$JOfapiQ5$?n=;+^1$k;=T@(<%GXn-S?wLm8hbF$(UNiFbGF&gPT) zN0#ttyvH}W$neeD`1eddC;oUJNLOov4REIaLQa|t>54~2_@$xH#1M7r=UdiY&kOZZ zWwehUOAgKnu{n%*{yX;9_}e@%?uaK5g4dA8h+v-KxLEQS9EVK< zVjI<=8zHG^{k)IxOuZ&?tQ7AZzvM#tsa3IHV+Us${&qc1+2%1bd;0o5CGlH|5 z4%7^{P~aiE2!m`m;_)z5vJ)|9U?vXNhPN5}*M=Im4^WxUD3Dq`H*pj)5 zoy%x@6$kjO5tmS|KAnXKcdtiYY=7=i#)p^tilq#hZjx&K#xFRFgjwH z;9&u{ABDY8jo&)|g3k!+6*%2+uwF~sasu^yzHZxQ+nu7fTrlBvyL{q~va<4Ze0)m# zw(1FX&E!=HFNXIsxD`~39+c3DM^Wb7Y2qVWkNe#=zmpg3x5%d-H9TsZNp-?_%+qfk z9aTRTD0xQbkF)@hqaU~7G!QCI5?KGY;yJN1LHUbdI55Dz;my08e<4K(R5>_3Sy2ya zhOgO+-oS!^-Jk^wr%m)EJ1ra%u#f>gwfNFR;rI;rM|Yk8?(iN(^#OGee~nHq0?a9p zjsq`>4V>rY-WN|%bQ5O&d@m>Vzzc>Ja4T?-<*NSJ+3=A*hPf4A-#okV*T4YlDTsG? zHcE7w?Dlq|N{d_%ShD1om5c@pLgXYw(44T3_YA@g>(&&9N=-6uI_ipNDb{=JIEGK? z_~3vu4*9l;wkHXc4+?8~a~(_Khbvxx`toI;TQQR;C=&;}Wbu7?Zt_8`{n~J)tlQC! zLGfO+PEgo%Q2uI{?okq`&7h5IE32}V41brqY)Q|XeO(@N}sOMhyozip(>w3+E-mO=jbW*ckHY`LcU_X^Pm4<2+k z4iBX0(FCpTlXG*c6IWQ&9e!`(NOHxcmJ5D+_lwBR(nGFYT(S+Dh0fc`HYeO}9oT@m zS$Ap$2ot&or@8j_fH+drgW%tNS`~*Ng2!W3l+DR+K~sMwR|w)zm977fonY~t)zKSP z=M@s385W%!qKuAs8eD1x<^y}B+a(~{D=Z@7C**ttD${r{h*?Ck6}Ug@OdO%laX>*g zgWE-xAGi$aeG<$o7U#GwN#@rarqn>!RNJ5LiX2?U|H@->AMEh2oym8KI$+h3iAP`H zr3SX)gOon|o81VdfbC7jq~D{Q$Kahuq}rDxL4bXeD7CQbOJLlL^y{ZI7_>DOiP-5~ zc7%Zt2qe!3hlGT_QE2?O5V3sEn&2jpL-#Rr$<{&DL_AN8;&u(;ZkBp=AoP839lq@( z6%*)}hDq@3zKtnR3!30+G?On@8Xb8+7+{4iQab6g)LJ+PI zImqNW?GQbM5)UZhzNBuB%HauPRj9dRWMqWgU46Z4tAm@ChWEI0B}XBzO3gwZjpO(S zwpj99siH<`5Rc2mO3y|cbB>5~45MBv^bk2Ql6$8jci$zgM-iGh0HMlAq<{$qbi!4& zPH87=DC0vMxN}EXCW3m-+3lEWsUa+J@ox$sSNHdI0}da}1yc)IoB^6oHYZwL*CJ?I`N5zd>pq(XnPjYW%PN5@__s3bmw|oJC|ttvgKR3Y&GS25gD0?7r2QmN43&(t+B?#i@bR% zp8yII#uR`7+p0 z%RM$Hi-Oar*T}KJVN`HG7sOk*Cqd*%_U^H>MT-@U*gMfYm8P$mh1hLX$2urcC67ap z-%!itF0AxK2%xWrBn=$vmW{DkW`esUJ99^xjQE6UNQOi;MoPyf-@Z$g7{gxWf083M z#%)lAy}LNRMMx9g(?3MAB*KOune_hfYj(hPIDsc}0CRF`OOGVaoOekp@^TX0n3G6c z*?desv^9a&{dlihndE^}{7vVT@0%Uz^?JLlRQhoWZB^Eg`&62BHGA(*Rz{-k*awoV z%2^P)&og6-bi1xK$o4R`hZC7EWyEpgL0#>FX4`P{Ro<2>imPHYVpO+JDD zU$-s0=3sSC>C`!RB?HhV?KFLZ<-E2dodnKj7zqx`c~cBStQ5?KL-}X70gj(s zeggE}nB%R;A&LMZqcWocqq%tJ*8RZM5nVs|=TGr5ZWwibb$+hMxa6=n zYxTTIB3)SH1BE@7@kl={qZtIzIvzQlnQgZAaIb0Is-tE|zgU63YR#u^TzvD87*ykK zj@L3C2VR>S6}lxa`z0{Rw7=;M0RC5bC3Jy(5euBMwF{-Wivfd1G~hu=U{i>gJ9Ig5JFfgtQ3FcCviZk4Xbqm@XzZqEO5wZlBxj18O{>~ zU%avdNbYh&A%OFQo}}=70)=!DuA`QAe9gBS_BQ=HX0s?GtKyKJdcLf6 z+%oOHbzJ6NxFjokO&D68L$+w|EX?U3C3v-d%q34{2Q_EEOUzmEYtj@d*JATSob)LM zb^S5h{F___niMJ#(Kd_2wNa)@CmAlK`{+Rbg;YYhNZk+XB=AP9qlVn|w~EuCJFcz! z??mxh$|rJJmI(v83&NqBjnIkaMoe1fIMp%^MW@{*v;Hw)dDM-5K%6h*I1q9gze-I{ zkDasc(7abzSUBz*izcbED2$IF$-`xO8XbAX2;(ig#v62SyoCjGaj_Fq)RQ)X%8~?TSnAHm+2g>@veNm%eLjg|JNVTo;U4G~M%}Dy{XS#VPx8 zEM5lr#O-_p1;5W-54(J4m-FkG=RwQny1Th&EbnX;+4$D4n$#T# zMsmetKQr#D%ClK_eAD3=8WyH^I2dm!`BRs<4$qX{Rs-yenkCSp5E>eKhxh15EpYHc zse0yKRde(S+@gX+|8)Y1*bt~p5 zHe(0TJ{TG6D?r0`h~;{oyE0nYxXtc5r+}{6i;UNMf?t*B9edOG^Xe zPX#f*(Re;wc@fbQ>{-z~8|lsy9L72+YJf$;C-{8eT#IW|QlQajvojR(gH!OvByjV} zvsuS(GM3dp3nN~!sOD_7>6Lz{2W&ad{Bz%Ym2xA>u`12g9*TU7DKsvSA!vE^_7WNJ zynxy#L%}3wUS8YiUm*yaO>$=eo}dr}sHs*dlO!o2pQ|L(R}OOeG?lc{g<8So0^okT zs$HAiA)1y%Cp87vx^5R+-xL78lj)-N-rRB@kL&O}m(4wKK@1WyuqV;S2H7E>)6Ip4 zAuRZpXCL?+ej`{1=o80csmR)c)PqM}S?cbmk5bpqM@ z8}~v!V}9*0aSvDN-<626joil|j6H%$#@)`{uDi(19+yyB3qhO8Q0tKr*5&;mg{R=C ze0cFm716#(F;j=tu5Z=#_rT`6FW$xU^C=o_p*=!rZhj;2AZ}dkdv=Z1QId0Iku|H) z#1w)-&-(;Uz3xw=7kcltsH(5(O@FDMSsUtE$!9d18lDK@r;GDupATN0ToU~cyJ?v^x-T4Zp7pLlOShko|p;RMG< zSb8gHARELmaS167!Ax^!FQaHBt0}&&-PElt9&9*t{;pr#8AHo%{(g5eJ&bsu6{oPmj4YS}AI8G;r*a zA)fq9HhFc9%R*(|k#@t8nv%R3_`xDC(QxCpW(>&VL0vfQlG_12P3}KXf>z-A#E6jfIu#h!Ivbw&ag00eI{HVQ-Z)6pjqy`#TQ}_)x zBQ&J8cB=VJ>U`gTaNh~_)1i^AEdszWf&7>F-7OAVt}H^@$5LH=(YbueXQl*-=Bh1P z%YZaSy&rRSTq&==$zk(U!@)cz_juDf=t8K%ps}Nb$zpe2MmTMpEIWi}?!!wcWVnq?4v0VCok_^YyMST1-fx3z2hOehCf&tq zi0n|&1DPysBopG&l_^5Bd3crr$_V6r38PPoErK(<=r} z%Ic2Y5%l^s^iM~cPwFBDN7^k8GL);SF?Qek{foebl{dF<*ue*PP-is3G{5=0;C9$sKX_LQz5vb(`nXi=kNx+`a!VKdYQm>Liv@( zl7EuwBz=$l4*kjDwt(sx#ApEFSqx4$|BO&KyxqRCEZ&Kd$ND(Kzm6dGAM_JHDN9+d zd?CBUb!_ZZH8oS4E3wk>fJ%dG0TqIU%1yf! z`jM??e3ErLk)e(X3K=Z=5cH9yLQ-qj=bL8qF+1MrFdm1%*MbD$nZYT+|2PB88x!$^ zAr0*bdCK8Y%uGfVR@q)LMw+eLA^&0!%TJx60Ju^MiJkbr8t^ZAfbBdX(nmrEnc%Zb z&#gBspE>3#1{r0j3dQQ$Uz^occ(TyFa__u4bZQkr${CjCKzLGhmVf;hGM}H|_b0D7 zJPJ?}<-Cobd^n*)lT}6?nJjy|LL1??;#T^a2`@hR+iXsrg?Qu@QU$%=J zwvD=6|3QuTyhZ%KUQj6cohk8? z2M-FZtQr4z7*Sz$JGWUqwSJ>_9IUiP;Kwgw{37rlcK;v#_cW6f;ZxoE@TbV|$-&Ae zAR}Lr|97T9qXI@c_{CrTj5jFFkgd>Ngb!W)JBR*76VgyvOB^80etc4*&jhAHmQ5uP?~?0q&bCg4B?H!N>;|O1_b9CH$|y`PUbYuyjw@tDmJl zGia%4w?ZX(kJOG2w$+gDvV5^-TaKN3F+WHp>{ti zmHsw8on}6UJrlqr5#_Wafi@gQe9)(;*^(`<329HL78|)6lzTf*r8JyEoDhYZrL!F* zsv(ewBuJGlEQhF?X52?@_3gE7nqEtBn&6uyV&BDR-XD{9gUmeY;w>MWRnCO^Bdpq$ z<$S*z#I@v7t_TUIy6a?SyWD?mRkUfgwjyY?f0a7vyHmNn`^ieN{N|YbL|h#|LB_$L zWga$siQK;;;?EaP;3_CcQxIijWM)2~kc4nM@5Vlt&$08q$rVijOEx(d>x<7qXy~yq z=De$PRbQtWBsx9udcT;b!?0z5LfFq>wrwBroMZn)hros zq*o@~8(b_plk2wE##+`3Jy_Nz>NSZSo)xZ)*EYd2sVS+cD@@BEO52@3Auf~C-_<^) z=~Aeo;cEMGi+d3Ay@|x+)&!*zcClm=uj~E_*NE*xr^ax(#dKG@YwmbZdjdUlkYqtr zUq4bThOx22m!qA}6?fJS?DVpPHU9+sKXYLhte{e54QKjP@v^NT*pEG&@J=9P#a zkrGBmMi%|RY`jJw(LpS+=;*y~Qs?`AmI5C0!Hl7NKdPNd=y=zG4Dt*6l`9 z*17ccx+x7}RsFepj}qOF$z`Rp z+)vzy4@$20Y%f~pGDxx$FzUXLys+3pUmvQd;W##|AuafZaPGNj^7q5z1Cq^!capIt zHyI}Ja#ZAUo1W9x{U~xdSS2(X*;T8Y^xWdwMcUV{Hu|v;$SEl7;!?>cp0H|X*qoF$ zX4n#$XEovK-#XA7))c~2zFui(0%RyPF6i!p<6G0*>w zulNdK_8(7Q;xv+qwThtfQaI;9soBUd+JVi6GrJv9Cg<7>Dv*b5sQXJ1zC%ak4)+5r-)Dj&rYeD&6_ zFu1iRVRhF}fj5>Ds)1cX2pX)&E@q%UC;JG6*j^aFyV)1AERw7x|1%Cvlt;(IVRx?4zy*n_4BqSDP9Q(mrL zX2+lTW~}I-a#h+Y{aM=cuU7D{7s)?hE@VBs!Krw?dhD^Y%=#5*K7CIl z^QFbd^sIy1gF2GURkrmlUqkrQtY$@yukKXSb~@M`g5F&$(@6|kx$%YZ9e8~s3=`1kV>unk|6mG zn0DV?h`AKzN6 zhK({rplj-*)nW`rBW0^rb1`kXos#SZN$RymOTDk#S=J+VmQRX26@nlH2Jv)?>b5V7 zoyTjC(mlH}r5jsWFz}5oP~X1&XsM60%Z1}jtwKox)BzX<-TvIlbeT{EF|?Awi}z~< zYx|#xn%ibH$QK^ovha=EMB~a+`X1@BJ4C$*ER|7rYT+V;N8yrWTU?USny?r0PMZC? z#<2uBC=y6Vh+4bJv2E8ytuE)VDf$^4(EM+lyT8~6{N*PO^Mj?WPq%}hlfZB78+H!3IEyUc9U14^4 z2B=}{vgN{uCdA-2bi2qjQIe?eaMyMobx><$SUKfmk_HZ75szGQ1s(&5a>;_0QE%p3 zHCNjpzBb*yZ2H;Kf$?B}`Z_+9avsWO)t(CW-B(vTWePx4J{2a4AY>mTKWqF-em1*^ zz~433Cvtzg@=>~23>5(Z0i)BlWtLK*r_uDW!7QX>wvBZnQok)0B7m<>+LO^iu`z?J zeO6@mKe3JnIH3m0S(B|rVEFY|!2Pk+fl&l3GY`Jh+s8;W&#Y@fxfDi{>oJR=-ffI& zIbiUw5HNOXVi&iWE7m$a?1IQtRg{xcW<>Kz);duNAMI1VMox7*{7#AeqPwBfto%r` z(^xW{e>tht#61$|DG8MlHc<}SB}0qdy(^^&3f0}&ie|$lq3HBn6B$X*h*mmfk{iN& z5||yztY!^lvSb7bbJUEIb_?wyD5}xqej3q(c)R}G>NkC><*x2cVnf_&9nY+`75xUAW|LA~_!wI~9nx#$<+ zK4u8Ci=bVgQMV1~8!zmGNn#ocv-xZFCy(Z_*0W`t0u)S!t8C+#4W%@lplh=w7$uuW zP+o|2N9{eBFBo`>{z|Wy+ukgz<7Ts7ca0-c@1!U&OpT}YIsTc4b0ced~`UUgMZ<$PJFKOIEV zmU6Y@D(&8~YWd-L9+x?{H z26}oe{FvN#;#iLEdOUtQjv}7G(UXlAK^I0D1RbAs8NiCzb^fWhTXzEju2gQd8|kYz z$`HnGmK#P;-U>o42;^9L>MkB{W>=66a|JUxFxu_bBQSY>JRNsl=U@EkEkMnOs#%ri zT4M|z3nOK%rQRBTOP)8G`j!GJ^G?t;DI|z;>6u2UjiKP8ColR_6;)szf1n zL+3H$G4j!Bxyh3pB1c8xZicS*1kTw{*vn3cwk0BA3MPasronP~DxWo@z%DFjwog&< zYvp{Rj=X49gON^<0VMqO#T?e7b@%+Q1Dfv3atuY7yR3-;0b(>y9?D<8{!RSCSM%kg zy-(OJGvIv~@U`2k5;?}J+jeripboBt$V1Oif_k5rZ59tUfo2y@hd({4E5uw4a#K!Q zZT7lA52vbETjv?e&&S9jae(=M)lq+0D6~g=y zZE>u{**5BTWYb$2E|zt@^A-K<$BtXHvgPEP_V|`P+we69G+*Q9b?a67W!*g!3zS-^ zqw0K(RaExlL6zBa>c?vq;dEeK$JRsy`+}RQvG< zw@ddb{nQBlc@Kh93PKDicxk++{B)hFz;cRzbG<%Jui=Gis(#zMLyhS;i#juTjD0n_ z+HGy4p-s~s3fk3RvA9TVhpL5?tvrZOUc8R|>M*VK{5Z{Q3nk>$HGVR>b|%XL+hrEz zg2{_nQg1yC88XBO=B4%yx9Def1}4IREf$$?kD6cmVbR9ZU#%@$@I6=KsPw+(aV)dx zAm97iGSI^#$+b*GA=g1vf^$`FalF>Q!!6G|tuF!i1449lY~_Rzl8sy}XZTG^OH+nw zm9cN>MX77QFMi~;-(77_Z@|3kc$tfxEcZ!P!vTpa#WY3RWykIPiAMvwEB9-i)yef* zA~E*Z2n6B1UtG6N3kzjE(x{5^FAGxm*?!4vEPw~X@F6ra`;<(Fv$Dy(f_#54VK z6pZgRID;UUVnL68dL~Uu4k5{9GplB&zna3OH=|?6Y9{0%SJm-c-1ogi4VL*LLS!P~t|xKIJ_4JBaGk2@NFlntT{kXk4mb6=WDxw+BHAi8ZfmQIK6o*qCgCh>>a zyQ?EA@)^VIO?j%&YOdWuSy20$EtA-n;G^1?a6Lh1t6Cc+$d7R&Rmb6YFG{y9mLz^^ zd)4{f^K#a~dGZZi{|$mX1SChcNBO17JzuP#0pI1EsXr3cJ1jCP?|-sRZ#rg$Mw;YF z5upDg*M(3>b+UgBTS_EFpO?hVyVvn&LCmjc+n?MS91%fFUw`OlSRjm#aZf%+p$&~# z58{L2{@N#iIk&giK+&joVI~*@z()zL4T+qY(WTLql341`(?N>p&gEf^os@njh>^!r zeSny@jaYxYAIG)RTb!Pt;E`=|R8Rpn)k8Io{_OR{BOo#m1d(#Y5bJ_S_O#&Tana&3=wj1lqN}ennsc!m$xaw36?F zCj<1`?ThR;;f4IvVNX^kV?TBj3bx^GZiQ*wwgROXt&^#1CqN^|(-#)S3Ue~ZQl zfb(ftnPiKn)!4~$s4yky^8K?hHbN0S8*q>UxyvHeoQ#Cn_ZMP!=Yn~m>l5@HJO?Sc zn)M$L-)uK{dpocnZBwLoFeVDT<=XFK=w|BRqE{_9w=Of0S_qdVY<`JvXGtx{&dIL? z^Xg^ujKPLU!UTg?-SUHDcZm_UL0qji%1h>rhx%eu*_=j6cJ8p4#s@Ey!Hmm`TMZsw zjAq04vt%>MK#0Pt;{iGz*Rx#Epz}KW!O9%*K~HzL*vlNgCMRaPbb9-`XybO~H0z;c z12KtaBgGv3o#fR)We&tqOiJcVMa4)S2m1zeN#P+%Sp)iDNuHx_qvFF#-!z|{B;Lr6 zu!iVH7RF`U)%N*2C)Q6QuZo8HU4C>-B0fd1aM7axt<{xEp67&Jy{b$inAi;aqZ1tm z{I*=gzoBS8s-cgW`kP;swM_rVNM9Ska{NNi9d)&(-Z0 z5>=e?dkZy^zj!uGVJRJ{R<*kE$OqjffJF>ccgl)UkHq87LrNy6Hkvs*)4cyP6 z2?=J@%%4Aheyu|E03Fg6xKchYDcda2?z61skc{4SO)! zUGvLp9K4v6_q-Gwmh4vZC`>aae6Os~jB#jY=Qz@#TqO=K@$x0R{XpaTw`s4kfs;T{ z$&P|!XC^wsYAJi;v}`sS%h-XGlcUTJy^J5a-#OZ9uMYURRb@nC%lzsE5Kc?P^}>+w z2r8M1oQPLnsi|>rM_h$DAnXA*b(L?c6PAZ2i{95DB1TeBIb!NXQ@j=@wmYgqciK(^X*S$6X%D(PL0;z7yvi)AU&D_RAT0v@@wY_B5n}pV3hMT_)!p_u z`LCuMvh;aS4(r}Npj}=1^=pg zR4wfre>*Ac2MggpzZX?wJp-LW#I0CP^qonYwYU+`LdUN8Y5t~8{$Z_Au@Q7uaiV#@{+K1MB}y{fiLyUL zl%7Yjoj2dMGg&T4RVW%_FQoLLyV@WyO8mq$63TNZr{KDmm%FExcN4T6FX5TL&pyAR zZzbWcziTw>o8%T^py?p5kaygF*($-#n19hh%eiu;8~u>SvcRoTFotWIxW^aV0J~3I zWIJ-zaqY9>taDC%Qvb33?kx-XZO*NLo>b>339G~BO>FZeipP8Q{sD2!O)l~z7~zXD zeP~5X4y$?}qaVr4drzJAn8 zvhVj?(~twrG5U44xVhUqKVDIX%GFI;4A!hi9Bs}=Ua71R@GRk^ZMxqG>3gFQdS@T2 zIk+7+x4e5_#reveAo}85))Diu1f#KXheeyn^+sZ&{>Cdb3iG4Onyh*v-~cFlnUk`S zrD<*n8@ohoLbyHOa|5f$n(d)S%NJi?87gFL*&l&K-lhKh^|escLwC$!#biAXy@GK@ zw(+X=5B)pHs`sxGl(7vlR~~zAcBe|1*JBe)hCEzmni<}@o;FhHUE;-F1fNbr6(9}w zy}T(qwHU~0(1K~=-nP~I7CCcs(hZtYFkmq+`CusA@vu_f{Nm$v)}e|; z&0fFQ55G6Cgb3z^IY{OQwGXg(;fO(rRUcrICpdx?fS^AzAU~O z^F>h`h+=fz{!VS8whpP{Wnr`S;P4&X-j+>kiMW-w%yoP6`?z&}o=PGWiCd9~#*c~R z%@zyy=;v8|%@`7)FsnDqQBE+;nKaMyNGpEyXhCgdtDmiIf8W`_TydUuc_4>ozTAR# zzLdLZY;=9!r=F$xL22N1*DlS9$cOIfE#8#7!fBrvQ?_g$xTP=i>`w)|90*BN2+i}@ zu8GX&5+-4pAjpjGF()t)6*GUn;`jupvU#aSTkg+Y>*Gg5jN@p1hk9-E!=i6eZ#X(w z9&3CtB*{tQRsE3l*h{CT+|4*5OT}tJa~qQ9zKzbCbcMAJL$< zCDB~~kE@{9g~Va`O;NK`@T7Zw-R#N$?m7s}S*z@S+S=TB*TOaI=-%avx~;ZYd6?Cy)$`KcS^62aP~QuZ`A@%lF@zlNs@>-@5z4k zf){3$p+oli4!7Q5bk@a=j8)`Ot6H_OBN265JPbQ^>*>{YC?8D_7efypy(RK)(a2MO`P!Ld|J+N>d~!X5s@yeKCgsRBLw$c2%|<*K*S_*%%@ zrUgidH=xO{h_}9Fpv-=^$3)~9H67^eT_DD7#oQjpSS?>)vf~b_mx^=^K95<^o zzR`O1^`cKH5mPJwy?v~Po_9G<)uybpcd7P%moA*{n?>?o)cax+vKl`_5_xYaMwryy zpp2;p6tdMNNqe!_RVWs$sKZZB=g}OFrqir=rOC0Ez6-~E+2mR?k4|6IB@80&&zC9p zn15I_c8*v?=Fru%J&P71|Kk2pDv0 z4y)EAyBhjevUDC_4cW7aY(mEHG31;1f{r~4iA<#b*aZ==&c#P?Ev4grM!#eH8RBh1 zEuuHz|HjWLkZ!UWvnLQmafOxsQFo7tl$fTSt^Sc!zkiI2W;D(cmf}$;9aCak48J;0 zVYYQN@(bqe)cI(rdql!2lJZb3y{LHw=RA^*j+0zC9tO9U4N}5LSyNG?Q(=!1%|iOt zs0UsG39-AafRR(md}Zm;K>x(hOosK9UinM99^i@ej*J4R`!;S z&s0Tg6IUva51(wwP#K`mz$Lx-5YZVjZhT;+g!qE~&Jb|z+E2IBBa{Y6=FIqrrH1k& zVhM-+%Ba~ofuI;Iqrv(2i_`J9tK4k{NHU=79T=<&_UhXh@7K^I*ZidIg!XqkvgN~c zk9_L6_dAdL*W<`Dn6oWdZF$Bp?<}}gLqhAa9s=+Gc&w(+D2L|AVvQCZpLfc%r#yq0 z!{+P!+e@x5QkEm4*j8=8sYt+P1*3g2HuZG^wiZ@?9qb2VHg85P%@obF@wm-fy<^fD zBaV|T=s!?%>4D=|vt#=X-Z{i-4E%sGJ9%oJ^6dET=NQb&+3(kjLi~M{NB!$&x7#VL z-Gci_w5yf+VkNTW73|hqBJVs` z9z8f1DwOxXFE-tgn@N(6isR7D8n(ofx1HvXoU%f)xbxu!eO_=B4_~q28`s^Sly zYGTLb7d1so$wu!^4E6G5a~Y!RzS}hRUs-@f(YlxJwS1!lm(|)nvk8>k+gye>l zva$SDk^&l@L$|AX#v02&&wR?}ODDEQZl2#&9;drBk8bM8^(n;C&d#0^P9Z!$kRhQ& zkbId!+xr12MpkX}hjlx85Hl&Wt#*5h_lk$<;Q>iyv(txQXC7&&(at-ZobF0>f-!U6 zm0h>=JE2Eo$Gy~#TrG$edxGjJsK5!9y1g}_>a0DN!zo7|`H0c20H;xW1w9AnE;Q+? zLENR)RlaFV%-ZeGL=QHt$QykTpOJv_BG9+w|h zj?lTUn#DYXUNi`Uik8LSda2fx{aA%o9p8OR#O{U&ba^yNA%B1r0(09-A5Iu4vSll< z3r1dP&NG+mwYG6OeyU;~Tdr7Q?9877>C5FgZnWOJwLW!~3s>Jsde$)0C`~Tz zNnVA7t&=c#WMv2{geaZJraKY-tP=kjUj&WkqGbc3Ri(cnKc39`G$uob`g40N0z2s) zYDMV}jFh&03?jX6y^igeoBJCds`I&5c|L)IA?uxXVYyqS zY&luhuKS%IN|e##>q=I!F*~BLm;1{wt$Fb2)%E48oZ4Lm82U?V-&^fgD*AyOYZs=k z?wV-UT(5s#xhQdrF1dqROFX`e-mIi=D-k*V)u&*ytEc8Hz)WU+kfOV#^P?F!#jSBx z?pL7r(DAG~#$5ND@wVlSa|m@8pr~1b)vY z(>rRCOINkf?VAiku1&0emo20uOoCR~n0+tTI89Zg@9J||Z1ED^eK^&40Raggb=%T& zS1GzP%zjY#U|(xBznu?+_;5bg$po{1Rz(@RZQ`b6^2N!lqrJO+!nVD__G-aNa^dDt zP(9m7$>W2eHp!@^zCebNZ9PF*{qg1Pw?vvMyTQF&**?|*?^p-hV$D8Y!_W#lTpmbAq^K=M>$gUj;y=pR<|)aDY;i3cB8N4 z&&kej?ql3Ao5Nnck|i2^@(SC-n$FYQ zK@4FE1%!;r`LI>z3@t;!iO0DUdU>87$HC!2^mkiwvnSRBa$=zgNwOICI?=5(PQX2n zPL1|~xta3|qtkxgJLWyR(fGKFY6mE=j(O8R=gxr=ecl7ijyI=AIEhfwx^< z>!Q^bA>mFj|5A&CG8;F(gd?oh0VUChE`ALhO@uBoi+S3p2T!3GFQQ9wE~4OC(44Ss z%Ej%ZvDD%o#oOw)Z4wpta1PY%WTk6KS9mA3c0TZEp86(TpY8JXkkI2X_DNPPGM|o3 zWBHqk#){Ls&veCHIV7pwbroG#d83NW4S-09uuHiI3)E$0(h5`h{Yo*{oOw~_t)X{c zP4abxhxI5k zF^4_zPMIS)!R%%?F(q7YBYQqpwU*#S_VY+2f+9lau*gQk-Mr=W3%(vkv3kYtaNd@b zkR1||I!3*b;nV$fA)s5TQQd$vO2Itu_(5dW_O*J{r~jkh9roX$niHhC8clfDFO`3= z%_pOh{7=T6x#+h=@3lO(c4k}X>O@jg9RI_^jDvLDfY~(~b)$CXrOD(=U1}k#=yig} zL_ncVzcC2njP{ZfRmdi}TBjP<=v&U~zvF4{bNNGtMIe4kFRuTf?Z{6oHh9+#2TWMc zy2gzd=aKM-j5Za#{7Y0Z(RufLO~mme(*X=d=@S)Hu%U&Zzp zs=#@Themho!SuyGr3q+HN#quTkFWEs4kn)pl(fdJjsoUHz3J-aBk_-<&O7jriC+i( zTG@R=*Q{2Ovcgdzx-qMl22GD9a3CP(@qaLy|yhmTY{S(qD7~y z5xy-4p0^rZ!{-3u2SDkD-9W%I3t`|N?>@W5V8rx;B5+E?uwre!>`+Xa&GeHI9s`qP zp57mL%Vz&l!MLyYdZM*~$~Hd;(6`0`q{{T9Z*0(P1b@5Kv1^J}OAPY1=`bFq>GR&% zIa~9onLSA)Dv4Ofz85yGjjbZPWoE+B=Cz`<8C7hrGu=6~b}?7UREw>ydw_*G8YC}e z8{{eXv9=coIn7P#c1lj<2~2P@J}9t}WiR`REgyw1lATm!={UVKSY1HZU0y(6Ltku} z>w+7+UwFdKmZ$e|*t5g6#*H^VHER{u*l^=y{i)M?s1T#asO{&sj*7hm5QWz#C>xFiek zM7CQ@qh#~LSx_5iMxY(`ifbpv&2x`e7V92#Fx%6ZhL+>HfVYgS&FXEzyIa866CAxEb9^hZ^YrNAWC5bQmcIB$ z$(_8HZq{>Sw}Vj$!bLY$&GOd0SeDD>3QWl*p${dWg#{&Q6)pEYX#C`qXJ*lU#%}%) zP9Y`GJNW^ajN_=?b9Eh&KR#cQHU|;sYn!Z+MHYod(P+i`jKDX=i`uo;gty;e#WyBL zc{C*e9`E1m_7Q_J#)p2mX@SBaHGIh->pmj)0}pu>myHF-YgcTLv-r1f|I-0{Z16RG zzig?w>AIY>>VCkESNI@5T2W#HJMcc(#L*#KaV zt={hcT#boYHs`}eVUVJOY_Djs{L$b5R-2vZ4alC{)-+@JXdu3q;UyLs4X zugPL;RZt^1|1vT0tj3m4;gYa{t$i{rq=FvoUl`eLzFrkMa4XS(q2FHdb0Sn;f|a=1 z73)%T9udg+?WfT`M%4` zqZ~FMOZL$BZ4CZ+=l)9G){qjIHGAQ~FVt$3uqEWup-pk6Bj3k=Ky8#}C$Ba#uvFCG=+v57D>tW8-oCSN75%p@j?TS9v`(H;OyfOD+6U z{ULo8p6qq$c*(8ErHZZXp@o$r^O(5e{?=IjUE)oY=c=9fCA5RL*C&I#ZgbYs#)F12 zx1o9Z&W+DR)T(C>+}_8&%F`)0(a=0d$z~efyE@EcpnLP|o~?-yR4F6CoY`cna%np@ zV!ZRy!)pH>G=6?+&#>cVzIST3Y@NA4^BBiOmQ`m1 zSdG0*1Yh;%1_UthmgyJ-5~K(6T}!spbc@QgyMmgNiH&-?^D3yX%q;5+CjB#mF2uR6 z(&2K1JTY;S=x9sarHUHQFwmqh6^uMnX_R~)q^WOlCb#pU{^K;BUHCZ1Yx5_WGqF>p z6E}58wWO-PbU9gwT36@$cz#oPCbY@eL;q@QWR2j_1L~>Gj}?&-05wc7`S|&`#@073 zL@!>03E-s@k|I)kL$`pE)TL{m5U&g?zW8aGWXb-+z3wLm-0L0b%kWvYjmz-8$e6!s zBNR>_^@4qIaeXIHnn}ay(Xe`h!3ICNji&`HsaiLxdaz{UB^CtRa7@DrJ^YW^h zWFT9XdT=BpfHm*ZxRnEdF%cT?sg!SGkCIAa$lGquANf6_jJg>^=TrN*-&wd}n(5;GX36Q!y%pi7hqK&>g$J#z z;SKsbUD#)_bJ)IVyBU*mMfqA4GJSDxKJ=kwV_>1aZ)-tqj(kCeBYt0c0DEE7d=lm8 z_c-lM5;QAtB4R{E5TM(-rzimJagJb{ZEBqed`c}`n0&YdquK3HZF0fgKGk7CQ|Hw7-rLoK+s zRe*$@#JSYpdRXGPgcE0We6#uwT;MiZquoO{UEjQC*%Uv_TH2+XM4Wyo_npxOs-0N; zGa3o2Vt-|{t;F#`yXz;-^RUF(eUv$_!G$BMlFM$OK$%0OP3JO4u1>oF`;CfF<@ee< zajHO$Gsi4KyN_AbxU<20(mx2b%>+tImKOFd@wZu+P9E9YTtpZ*BHbaNKw>M>8-c^k z3wMyV4nTckMBizTP)dad?c2`g@xLdzSvTqJU6(1$I@Vf$fbg%tAf0S5uRvkO$b8{d zvma-NN3R00$FuqVWA}CHhLhd=-s-Xk1tQBLz-joURnS>c%UEzdMUcpeoZP;TnB1U@ z5}UE06Hsk%;=;+5C|J-?P&5~+q3f<u`GXY_n+H)3wa*vzQMUx=Z9o`ZrhK8Y%>~zpGlpFrc3WO2Ql<-y%?f2 zDUXK+Y2wyB&hvu$jMH|^D;=j}Ztg{HclivKIz7yPf|(7~T@5Zj#2H{FCmDU*c7<2K zFDI(uBWFHLiG6u8Ih-O}_*OK+z@|lVAV7;(+1r}G$Gcg6L$AuG!s8>hP?QylpSV;P zf9tw%9ifXbL)EBEl{F`BV@62byTPmb9&SH>VuL`q+!w%>UMmq-mlydY zxi{txV;UT`Z}eyGma{CTj}*z5MESXABwM%Em?_@E3N@Xhrxk&rCH9son6P~c5s2ES zQJVHFpdPL{dB$8>zkGr+@$ue8oywU$D~%|~2YCwlr*B^u%&$LqHUBL_HrsMskXk}% z%42%AeSZ5LOKdvub%oIjbbC8{ug~Ug;MXTJDlSH#UdAn~+}NizBL($phndwgE5;74 z8cM|&K}H||O5WsQ!?3<7?k%Ka`1zUA5HbbI8beV(TEZeJ>lbv3@e~}PsXu+*f(r6+e zLri-fCV^wXgOX`HzP#tiXn+1$E7go~l~H=@G^~EP)naa10Nho<*d64MVoZ%e6}lLx zMk!GyD;A^&#CE-tLiqa_V9kXAc!H7kvFa6g(=GZhA5Xz*%c$Kc-1AKEF}V@e*S;!& zoPxX%fyP)E=mA^Z<8v8{tSVBok+Ru!a4q?C{2NTstq275yu~WRt4fx&?}-pRf}w3f zl#9K%wCxInLCn4MH0@H?Ln&!^Vza^CB)_68?Cg}MnU$U2A5fY>>fTYZdrSDm=ju|o z;u5_ZrMLtIWGqQ<3$yTY90+;MC(j-dN5Qt6K(fGJ2GZb^UcT}-KU)On-A`d*=Vp6;g?poB3^$bw zNsTv!5t)`<%jI7>FIefRdZF(kSj^uDrZ?%nVL8$Lvs~~q(8@_3XXVdDg6cetRdk<7 zO|_P&w~hItljp&mAMm*4EL$0tB`twvO4@sk7g>sJ*Sqau>5?DR!a*3aHbU>VUE1p2 zLU-?eCS#7FzzA$J&jwt$&Pf4*FpH1j-nUFKjjOg!xa^wO3CscEE5eCV`LC3`lr+#z zXd!j-1*H+^-E+;0ohPLFbBR$HuNm%9suL`M-pb#)$3Q7*Pw)~B^tgJ{qKDcpA0H^5 z(_4N1Hu8NzcD}tYsxw3gL)H^@0K+8+krWCB$?-KKM;%CwnXb|$-gZI92@P|l2pZqv z;fKjdbCvQ*vSNikUyVxGA(T0Kh3;P}4d#SfuYj)$`wtzu=*-o94C|&rYle3sx&)_8 zFN@ji?h=OiERm+L;30|YrEiu!9IFKUgED&z<>gp~dL?$cqU`N zES{y``q5e186nuS<4hx_+aHcLXb+K%464eQmX%V2-8=_h8>O-cgx&SH`61VWzgpaS zf3>&dW^U0A+=kAlqV&5@dK$RS%j|m~oxmey zx*mVg$!0Uj^%V=6&fXDO*bR-KoSe%vw-w=tQ}W0&Q31QNS!c;)monwR#bYt6#tAz= z(t@Gu;y5OJ3?$At8Kq0HmB~ zA$Ve?h=6t$^r4tM;~mYHE?ab<08CP+H0%GIRCyAx6}?bzEUQhxE7jN1@AJT#241-6 zKK&6us%5YI-_f1m{R^k6OK(li{!j{)J`I#8a>rBSj`uM5%26Bq^*{kk2< z2YvX>JK1y1QH{5Tkp3f1ZmmwmLQcz}bWC5FnY<1Mv^ow@kTuy>KwpRe^_V1TN%d<- zx&rGK(lD2qYq*EYIe0sAa4jVc$YeO$$+c~>Iktwr${EXLP@>1w>}T^z*?%Bt7Y64h zI8gi3Y_enxQVWo$8iSU!!k&KRLjovIkH+oGP$|stnh%fsY-aj|MRRhjI6yvjD|0 zK&Mg$tD&IKQUzg|rGrKni5oOpV^0eD86X~cP*)otd#|1QA-jh37thK5Q)>VKEHc5a z*votC6M|9$SOT6>Jr}_m4NLgAGOSitSL-JZm%d zDdWqqpo4`8^xOz?1bc0`!1u}0rZ#7W6C&`Nox!^m5#B-C0`+u4C_|TWR81{me(NbQ zto!>4;gMjqr;L|NM!+q#t@-Y&1w6@ngfJnGIUn~ojmCh^Xg()1Mjxlo2|_y8Rq^LM zYc57W2_=ir#YfdiIS($SU+3%KEwo7p}HMsc@T5h=zQVmlNg=up?ZJv1bL=`i&{fx)M z>EK6*;Cwn`&$rdfgCx-DG8=~uhbhglzPfv$w-q>{k3mE07Vf*naz0P2YKE6DovZ%p zKGSY@UK6>C0Nc5>a%suK+`afgvV_j7)w!yuO-Qs}Qr@filoskm6#cNZ;RO& zgSucAU%p3|WKX-TY!eJMl&_$O!HUeX>f~J{{38~|8m$0e8(dtpaxeVLr;e^W>iaK2 zCFquj{mBKzM_e%WhwlBTk^YTk-uXWorWD0;VLTY> z4T5u7cgF9Ut7X1&NSRy)e|`d(xjLmK=r;y{$&@pK{L#*2L&0*?&&pkB9fDC@Z!+BN zNK7{n`o`lTYu<5qqO|i!Lv%?2*}qipoObxEjQq{0iU)U!t=@(*ei&u5T#mwz=b5cn zk!lf5K0_CV@soB_Q50#r*OZe8)pQ9|_H&n{)!l>QzSvyWOUEmz3G%Lb)X@5!sb*K5 zXs z-*<)G5YyIZkE(%>H6Zq>)X31?TH_VNpxy*gW9B-hF$z?(H`NnHK$Ppk z-Kt0jply>d5}O>iEhW0)0Ze};=+AxdM0Lq%Rr2zn(l7el%C)2~Cc9_Hl0;pVb00L- zH|AX_L8hfx)HE?a+68|PlaU13ZOLnmOi&}a$B7Hq^z8Qt9(MBm13~$F zx;2i>eFQCjVO+dOh8J;xiYDRwxhdPm+{7-MPePQEgd%3UxG9jsLlqshQV%n`h zROW)ACTU2xB_KzhNu|Q&w3wc%QttMHKHxLl8)UdR!r9x2%)2sAWQPTM8tN?QO4Vb& zkjKt_R1#jVj`~;gZLrS~A`x2a9qu?__Xwz4;U1BzeP{0TNs#lUe@rVbyRN}#W7r`s z$EW=Lom96-XOn`br~9HGmeF5$&*W2!UiWgt74wIf;|f@UV*KlNHrInd3zxYqO5S*z z(E|$3Uuya$)A+}&F_tx2X^)HRG}XZ+E+0kDSX(%VsPeNFo$q2$QoPj)Kn}9RnVJ9# z>$*GTvUSpg^rNKj=fLDW&+vm7qNGEPew{jg=6U@SJnN%;kBZZhW7+nIbwkxHV>E0u zm?8c8hwtR%_beoOg|^FMb#7V2nrY-^e5ZiC=yuL!E47~~tQECKU2)jatUD=0g|a?_ zJ9L#>J8zUtMWu|oTCw~1VaIom8s85nj)KIYi;&sHM_av5J!G;-WQ$(t!M__)`Oj;rAV=AdDa!;m~G@~PRktOCI_NrS%nKR zP%>cubv)m*B9|85)o&W6IYUl+zTz6m8}WlO*`eN33_y3!i*X^Zxl7Yv0+s*nYC`Jx z^eoe&ZSSn?d`jW6A5GmFBJAmuMNlfH3wxd>FGpG0;Q<$1jvwytk!YjAm|~50uLqY( zZNySH%~@2;?(}i0<)WKn@;dA}GuVl#LN}Y;J*&KDX6`sV*vN^7J(G;`_BSSDua4T< zk(30DwcATiS&Z^Kh(kNz8p}Z2M7`>dbQ`Qg8{^ngsymE47Gp^%LSW` zw+?;MB#~@qlvsSVw~O219v>NB!hYG>Ys{~hnYb(rOF+0LB*dmS4FuH<%$3}RuqFzh z&ATI5NAyHIe-dJ#k=BTO1Pwyk8kCI4c}+k>mi8;5YjV|>6VwoGCT;viNUtcBI_vH{ zy-WA}+8JlKVefn9;nv&>x6WnLcA=LXsDbSZNq>NSvpuhIhQY`03uA!ed`dF|wney3 zZuh6%|CdNV?*hp9A8UWO$sb=3Kf18+@B!w*7qH~@$%u}QsC6K56|yqnQ|PRX7V4vC zBuzMXRa|YwD;-`pUTM!mGZh3q;s?PS_kd!U;9^M^^_Uy4ghTJ`qUBkuy_eU)55E_s zK-=dw*#b(;49x=$JXac%ZE&6=!*L|wT5E}KHCDLD^oq_DOo7P|52Cw-f8z61CZ<7# zYyA{*hwIS{eIjF6C$GiymYjQ5JyDf}w<}RZY;$emowW$nwPgrP0@*9?G!zBkr0#k9 z2UZtN{5ZHkoc_j8jCmKviH<>vGsige@Uqf~$$E96uST{t>&kpiO%>qfauJlZwYlCu z_PTNuC(4(db&bha7sOq$mm2bAKhAijJ@CT!4wHjiP{a)e`$2la9rCk877SVPt8JlI zFcNsHJbz<T_(zkfv<6ov^n^Dm~{Kl3zK z>seoH&(o$ED4bSid^^z2uleJ&r`fyR4!-1_{z98niewwjjPo9eP{6533S+tINO7sa zyM?1aU(uA?WN@H$tGgQ6K4(|@2SU@s9#gQ@zn#tQGW`Th<&~`znbVPQF6x|^$Y%0& zojL;z|7tDyZ0EGY$y1UmT?6&FbSlJHx<_45tK>87*sTpP8aRE)^L_)#+!t?YCW6>YsgfV1*HMTe^=@ z+s+$lr%f`ObGJKiZHh;n$PvzU7@Zd@)I!@7x@bkf8@)c(R}hdQK$Uzqs4>X&^ROa0q`|`92&=Vs z_L}HNrPBlG#UfNTnx=i!TLv5vb4mt`U$Fv;I5jk7ub1kq8(s>ti~G8s6<`GEXW-=Vy?S~ z%RN~uJVzG|qrA0^rmvp(w*tHepn}D7X{F%)3XcUp;yhA-vK7~!eKz)$yq-pdP@Pd( z@m~sZX}i3_ak9?Ce@4J=>Ia4JI27(QvM#&8K>>dFSpa69%*6@O?lTw7C_gK+M4sjh z|8UFRR-RjTVf1ypRQTFP9{^$bKB`-EIm0r0Fpw}Wv^3=XK~j@U54)u}e~H(($6Wx3 zo#KT2CfQ*sO36&ygC|&oQRS_5GCkPuUR!-E&K7HCI^maPc0=kYbg(S=+vXN0J`JVx zacVQ*cI6Hh!xU}4rHeE3a_r(bg1<;3wnNAvJ7G`pcZ@q>d-US+bClAhe@5H90@2Q( zjc?PVYmxzzAhY#>XcRU zQQ8^oNFqKnqUheJXFdN%(*Um;bz4ynb&42<{4~zjb|S(cr)|Fm!m^w$d|Of}#xwF! zAxfgNb=aNxiL47hg8q~TxI}bz__D(SKR;)DJKGa9|4nkzt`zI2nS$1(h$|t^doTMu z%+zdxIc2|%C6+hLI^o}<(W|7yWjU=1$X;m*s=RSEO=>tJypis;*A+U>qU zf$#DIy=>8@>po4=YO&N;21`d24ff~nWh{)~Ys{r&_do1O5E-%q4=^-KqX!qzlEF3) zwKOymZqHi8S1;6~W!K@==83T!0I)h>LJo#hmO6L6Cg`jb3D=_*Uku+~X3%0uFR8Yl zHu94hRH2982W9sMr+H|z^ZRwrw#slH8Su=2WkDarLlzdf>Fm%~86z{fW{$2Ec$$mG z4D>W6D&c6vQMjIsxx~28L6IVxB!P5vd}?vSl#FjAxz^5sL-b0qH~nZ>dsCq7+&G1y z&xmJYktWTK7%U(k)641CexDw)GKC)g5?8vO;1VS7X!AU1I)!P{cr`fD;}-iG)ko0x zXUs5bP>T11XQ=0QT7g_A1PY^B;=6};5H%9zNA-3o~VtVa(1)dwiJgeLs!OuGhIM0OO9q z_?Hw+rDhUgqhx+dVjp!C`N!5KhEZO7qh=MT=TMe|?}_*4!eLPu#ncBy@_Z^tjO!fLN*r z29AaAadzSV!U0t%0WjR-9azlN1!Z^vv(-Hd@VU%ZqyzSW^5>eQy;qb|d7rb?ielup zQ9Yd4ijeJA*D2(7mVb`*_KIg8RKnuk28+s&ujRJsdSWrrSvZv-?)xUqPgNh~jCR|b z4w+lA=IzD`vt>j}8I-jN$eRjkO?vLkpT=L94z#+Hzr=gS{Gu7yUeES)~sF^hBnY!J7)Z-T@=iE4At?OxDZw1}{Zs+iTCB3Fy zKu|{SE>EJA*hFtxvj2RcMN!tHs6>VA*m2woFc8}E+bg{6`z(v|Fq`eOm$T03o9TkJ z25bzfg}zw&VldQ#D0u^JzVn8wfGwq};eDpB*h*3ICyK_ZkvCfff}ZuG@R#vCeqHzX z_;KB{I?asyn1-uoQZMLwh6<+ZFX4f}5`@;iKUUr)wtmw{h4(qgpqmAsxRX|CI_y}{ zAYqVi3D)wU$WP%!eT<~v1_M#d%f?pZ4VAV`akX@NH560Knt+sFahjSH90qcx+5kXE z-yQ{TuU8bU<8PeEeNy8!-9d&|wHU@lrx{3OwQ87GF?^+VhGoCciE&Ue*N+yMN|MQE zN(j+ndB4sg_MuwxLDJfLu-CXZC~0fHLwQPVMH#cqncDE}fJ$y>sg&xn4v?Xd!Vv^Y z+#zXBm=U{!j%9HV>fWa|d03?T^jO~NE@oiK+NH64Q&cA_KvG3Z3%0)IFN}+UYnA$b z8+^uK6W0T{nt;_(H5(d7z3YL3%US~l`b3g5DPScUssRn=yp1NpV*I9?K%hMjQ7`J$ zZ>ypyH@r8;a&2HcgGsi{6lzL^Ao0(W*U<@=wkh^TjAwto!la%LgrM?*HsyKMw~B|v z^FnUhol@x>#f3*8Zk4n!JaP;5>q2A)k#=w7)Oj>8aQaN5?i4jFt*Z_ZHpKhhr zc-3TQY})Qz!#v-2V1By~iqE2!c~Mf}5#+|DqK0Su1f&xz=GadE3ERjX4b+UE^c$Vz z`OEm5I~polv*J4+U+75~ExD7OKU{Qoj2jReQ_f0Cl50aQA9|=rcjbsZl;}D|XLwBw z(FS%DqDcwn^mG``$&wLWICwr{?J~%+9Cpr6FP!c)bICrk(4}Kr*SuV7GtgdHwOPl+ ztTX-Dnr1lwdr29H zunPAwnhi)Q4sKFFD#wxH*J8q;K6j-fZR#UlQL(qbq!bRd-Z_EK@?Bhku=JDRZ`IG? z6t}vhA;#f{L8tb_qTBBR3KSoMM?k$z|rh#2qNErlA8Xh#$E{R0)t+|_D{3%X zQv#2taI3-7?kn0VLZo&l1?%QW8AK~Hw$B?Til({OiN20QbX~HgyH+PC0{eCd+Bpr4 z5-^%k@jpe$lJ48#Y)Hvof&1u@>Mq8OyMo&WoZC+SPMP3*-eS2)NFexAc5!bHREhF= zzd|%MYr@b)o{bz&eiOUyYs3;x$61SNYOx$EA$~xmu8pSNl;uR99&=fehsBz!d10$C z0sP}(g<1S4*w*SM-haTFd_K~F>W?P@TXLAp1FmU{~CkW?2Es=^IIxNdPXmppg3~MAlJzMbkOpf-~1mbEej9x4T>%>2K`@ zN9nt+W4|=v)5pW4ChG(_=A*lk_k!!O-=yW(ipD7yqbApivGyYcb}k(oz8Z0+a6#sm zXJkIs;7N>$JrCRoD}qr;Je=w8vIM5iub^NDoWC{S z2~EA57$s-SBUmrd?EluUOv;|gtM7t~QY3hK5`FG1qcYOmY}KqJaCudBFAB+zWyA5p zRa25*r=0`hG=hS1tQo~}Vh*uz3y@{)PFFyGxs{cB1Kdph$nC`?5j>P1bZwoCN@EKc zZ(Tf0bZsho1Mwea)UF(wVaiK!C^pV#c3j5IqSx_DHVZtm z3Lts=ktXj!4fn@4gH~CRRv-z;t~a!cuK4SLZ!i_X(_wU&A>%TSlY`{Ljw;xU>CUz~ zS(x9_?clC2zR7ocJn6uHJpL9-ws4=!KU83<)T)rOXe@81<+Z~73BQ$y5Fy-l2t(l4 zk<0<(VMh(K6uR5;pD3j)s_kY}6|JSDiixm+-BdFjg%p$QbUOY;;&vnSn=6nq*?|sw zyX~Yx(9uLSuH6XMbM`3lY>b~hO7K@VS0U#l*2F4CUmkB?z#^2HG$^&Q^IQWi<`8L{ zsi;<*0_}|E^dpyvd|OU00g-F5C^AlxVotwphR&Q_6TiYB({PLh&Jwl>$r|XW&*JPgsbSCp7jD$lWSVPjeDWwOmzw)+8bv&a5S{INGlEQ{ktci!N?KB^ zud%*l=czk;R#YB-QesX_Bs5%#1xEqm2LMg0vL zYpr~Gz22Ej#j{PHL$=zUC_ZiRF1#NFt{^rK6VoM{OQwgk*1QLdUux#ZD?u8dpR}gN zy>np-8fKO9&6fA2*nIUJ9zgv|1ldK40;NHJG5bm%L)cD=99FIciqx1Gl?XtD0 zE)Ai=q%SaBD==KS;pVG>Y=8?E1r9?acVW~^+UkfZ>_&yjs`2|5-9ot-{;IoLarDIC-HT$+7wH|(NMe=VA+ zT5TugBNCsx-g-Agx|kt%{mMMUHI3?(z+a;w=V?fHxW6^)zUh_)EbO<^#Xo(nSjg#N@$Z{F+k#Yuu( z|0tatyHpmU>SQY3eJdnxyXB-4NAGQ~6{(93$Y8CpLJ~zSFL2iKra{x)61q_b%sJ=j z3jHeQDVz1|fCC|&@&HL7cH~XBr7iBezhO1dxQh;&C^k-u&GpES)j9a88k=^lR=!Do z<7st1NmfSCLG$t@m4q|%9VxYE=&2_y=$30MkuD*kWJ8TaXCU`SrTl8kdj6#LlWH|eFswk=xV73Ptg)6|Q$wXiSUV1uaZa0tR z;oq%6#<``=_s&R%Cyx!2r9<&HsYQXp>F1W`wtVlA$;1e7hkfi88S}qorfLHA97pDp zam?qv|9*9iewZZo@x!WB^|+~F(KJ1QD25sCGzZsg7t?09i6Uc^XtdtsGO9>hDC*uy z5W-8j2qGG-Pt(ZI@TDdw^1oBeBqtwpC}cb?nwxDjBh}g@zNu0Ct4^aW7YMt$ErrZh zxaTT-XwogI&pJ;JcUCL#@Shg$9N1A(O;ON&-8}Y6A(HOhEtf_0Is5%>Q)aQxg9V5M3P6eg4r?XzDCD)vkPHNJ%aa;8UkmKV_V7IC(!bait z<%J^qR}ijKa|Xd84mSCXMI@bD^R)So647qX;K)bf|D_}T-~ovKUhbd`DGm%UzdAsA z{JUuD;6afXPUJLa`!J0MS!x*J;gdZGIFE3NC$Za2It?(gnS<|9$!t#k8|*CDl9V9G zSuf`hS%h6OM1TL@wK^xezj!yZS2uOII|tnatq&7wj!Sw$o&-SR)|^%K{0DmhU!P~3 z0EY*%7S-MaCi97^^8B1hFnMY86Pw0Oeo0pAa1NDfhgd}e^%-t<8tDtCjv(}n)UW+> z*60b+)2uTAl6RUl#?DEPJtbsu`q$xAX4(^H2Aqxzp9q~_XTksJKg{qdWV|n>bXW(P zSKNAHQ;w@0NJdW56-%*O9C{fo>uJR1p@5;C=MuxupE?51K6N(rU({gzelksvQ^JMd zW8>upDecUc=UsPt`5z^! zNBFfDO`glOybL_N06)md5WA)#pQX2GOm@SMT<`Az3p2F6WdlW}G#!!PC9CfB98Lgk zYpeX{qJGW)j$J;r>-GXRUevX_lS1q2=QLc07$6e>;NIK5{2nvctsXcjl7tKM<5cR) zVqh)oJaj$|#)C>Y!x76tGX{NxyH)eo*!lBq>7@B~H%z^9+oOorVgv)F9i87lc*S0W zuh<~42m`($kYgVDH`#z7)ZD#bS98tY6>9O!IWMqe-y4UYC-Udf00b6m4Ef|bnMYm2 zy+E-V{ItdlcQx|l$j^C+U*iNmb<`XPaA($bQ<;BTMz)qpjPA$h#YnFmla zR4aF2wVSn zi{F2Z?gRwP{1#=~{7)f$60%SIzpm%EclvrXr^d%GW%3v=k39#t@G@X^QsgIH{|;6D zub<9WAEgm@``n)V^`vMn;KIdu7587ea{t$x{D*rK9s|f?s&7zTW`9|13fGQeq_bh2 z#lLF4->&IbDx5x=>_x&NRsH6#2X7wLc{M;dy?-O2e*M>f-ktquXvjSxg)8xRg)2wf zfK_Tc_rL$+?|v%*?7ieIqo0M$ZbuVbC-(3{C?5De;?T|7l9%Qx=WNl2`zav=LZ26E3dD>15oLa>2=n~2E5zi#swe_ z7&SlN`|B*{WBJE$5*Yl-9@J2;lcQ|KtzYr!vVh*L=G%_1A%IxxgXMAjW|_gWmol|B z_b1&}jDh!GiJPrGe*fTXU`6{p`&5pjmHJia(fP(?&)?jDaOpMkyU*w{FBWBFp@qHY(r7N^w6of&Q^MCDIGaZerC zldrb!W-Ys1MF-ocIGzMvm8=)!Zd0%VI4aoH?N|(OHI(oc&zEe69cCp)r!I7dL2`r^#?$(8=7gn zjH5v_nSggxT{Em(Xn1`!o)^MM|IcK{5f9E>?xnv>Fshq&l*%1f(fOo(2j!e-Mt5o$ z`p$+tJu)Y+W|DE}=kfdR)yzaHOv|*#I+ABp_CgmU?KF#DHR)=Cg&xr#3#;>a02kfL z`f+&6pMBWPT2-axjP>lHbB*lr>;BiN5B8xQt)G`a@SdQy@eTh%Mm-4q0F%nS=oS7# zPUCkh`&5ZFiq@}x3<5m;9aDQ6SRfdhh7b58b$oMHUBK>2zR2;Fq!GUf&ABGU*R>vh zJ}Nc!Czt=tXFm*@m}5RF5CYuB^Yv$U<=nr#J=o(Gg6RT8enfV>p5u)GsG#tPbXQ$# zhdY!v5f{m}5s!l~7#_>CQ~Q9x-1@FOnlL0!ybb`JyX_`JfkF2=U!+`T|C@rx_p32g zz4l<~gGZC}}bjxqc9^5+I?w4e*KE-yIzLQ^-1{zab-5}IJOpE9cA zXd0%p-=_O|K}MjsxNf1%+x;5u^^?DmJFM5q7gxd&PrRjAR5Sp*^qoZ!^Z3jmGG@}G<7J$B3QUgI_1gJ?dPMk4C*Kn*{j2ZwC%G@0 zJ#Q#k3?Op^rK?1YrN4io`P;{>Q!VmoNj1mOQJnQgiYJO*^a;c;Ej%RIH!>TpWFqSBdX8?%0$Ry7UKjzZDoHN#t8|Dv23^#!u|$ z@sdf;o;+d-hF+0Bk@6Glv*M_Ht?tFFIhTL%DOebKKpNHp7VL&N z_+@3K-uC~Vodu}af%&>Ujm4rLw+LySpKg}`L+*d+4H!F0J(_*qmp}$e1_zb3v~JG{ z{C#GRKLy{T%Pi%U(n5O4Wj_$TzJ8g>-Q+ifKksV6+Fr1ggd9l@_YFdTkw(MHGu(;T z%AXUMKLJw>+R494eE9O8$9-;|qdR`c-$y6-BoN==m9jW7ap1wbaqo9eQeYLQI#iwE z7ME|x9MPQN<~k>Re4pUSqk)&+GEilUT!4VLa! ztZqlr4s^9M<+MzTPDa(Sej=d<)OpSGE0p@9f}DV<5)#(V|C_BkEpW6|4JWFEgQ1H) z1HaoPg*o6(_Yq==qL=>X0|=;SN}4#+pX_(MR={z2^ygtGaRiZS^=;vPJqq&4oF84z{KcT zjoB*gyOLmQ-*1zbrJt6$-+Liz2d{zT|GW6t^hoH9M$V*+OPiHYYCrslRtg`zZVldL%QR6A4O#;4G=j} zy!h|c{r%~lCLmho9G@5idu6md@YBkfd2u3`lm7f{<{8}lNm8xk_E}fET`M>eq|Mv{xD!ox@ zGv#M|Fcnp?+tX;>{H|jw=T)V9y-|c=B|NG zmAW0eL7S=(U#>ZwT-SC@2lc{zu7FqYw5P<+j+5gv3}C*$JpWqH!tm~l_4k`jC}na; zAP)RiM)GN8&3n^{%IKA=W|xMCj^>vvbeXXt!Hp&j=2efVS)}+pm(v7M!*Zr6BFxn?AnQt-XcbMq+PIez+s^b9UD*3c_@~ zC4#ooRo$R;3BE_)=^%{k(AN5J4j4=y167hs{vKVkP2X`9w(xF^jBZ@@@5cnd^5K5sk?B{@E<3cG+rwC)i!M^ss8x(aVpx^6;+(0GnJe}rG= z=A)UOFz3lIYWMFFzLeL_;y@5tiIST%)-2NdT-Ot1wd$^G-96sewb063gXC=584wI= zqY`Z3VZ3|BV7wZxB2Zga<-J!=uJ-i%doB3NcfQM9(H0NRfxE7yq@?gn`B?{Kv1-c# zV2_CeF!R{0Q%rrQQF3p?~TMGlPXotam#X9{{J}aLJR^IXC zIQfXqrQa(OC_ri}gIXVcrSH}ss@ah7-77W}f`2J(G!|Hs@<;@VZpdZ5mgk=JFV+?c zuOe;Fje3m0%@ip-{k$42YUMpL=77m&{)8k8{x~{u?^`RpNnmzb-u-ix9@j=h>^g`~ zn=&sMG)`Bp~U zu67I>oa0bhz{_b2Q(c>eD+L}NcpvoHTLgU?>CZCbb$B@p>)x_h5r2(Pp0}6|^v)X7 z+kV67hP7*uU#G8JZd{F}>IwwSa&7k80Xy0MWXd@kJ(+HhFy zRJV7gVSf%D4ZFmT^U4)*>O)*B{hlGP=N+2@g4`q;3wH+XwipT#;WYIajSR6nC3f09 zrogBj|G5_trMnclwchX^>C}P#Tx8pO8hOBCBmfz91yo&*MF#VdW&-#R=w#7O!w_1LIS8Q ze`C`Aq5e}&o|ljW}*`~pD=&@_{0WMs#Q+((HvTjj-uI+ui;9Cnr* zOpr@_-)!DHhIc6<_N*gj{<+@77xy#T4#`=PJ&DHeJZ3NO77e=e4&)K;l2diekjSzJ z5e}!fp_&;SybZXA;k5iW@XTQCGxR)jQ&1}09!s=U_RS82S)B$~J}!g_yo)RKSUPw) zmID*8`^01Pi~<7Gz~n#j+dTM8-CYKB9Q3EJjT;H_4N8hPKSP{z>A@=`2GkGM{n2z3 z%s*br&jSzGC)K#xi)q4(YZC~dyur>gPoqvVZ|%K-dRQQv#HF#8YrL;OtLj!NPp4F-8KXa*k@niSBhEUBrHscZj-z4r`ja_ibf zEeJM5L_~_9q97okRDsZoQls=9ktSVw2LS~OMWjhf=#dtr*HA=y2|Yl7Aiakc0tCW- zSntyJ{l4Yd=lnYR$Nv6!ad{E)Jaf)5#(j@*k1>yaelJ&j+U{A@eFwW-7{E6@IFg3m z+M2W&pE3`Zs31{1Ctv)ahMXv>0CY3Tlth3agLju5#3y zMvU>saK=)-d}unyq*wbQ=G+Z&5A5oM>Gtf&AD9#_?o%D`12ne*wgofp-(6O7n(j6Ad5Va`97&9QBrBU^Vwz(t`RMeaS+s`w;b zGN{@&g_PB;SL<=Y0}DSTWRd(@i>_BZ)X)4ASIenS5(NE~!ZdQK`rahE?m<2!`Wlg3 zi~UsU-NNN2vz5yw_=x%LnG>{eZSt<&agpE%y5!PD3@@FslhbD5$46BehvuccUo9K_Zz6y_DDqYfK-}UAV84;FICeq2*laOWNm*iFrIut-IX0S=8R}>KG-XNE-!x{a zw5ztrT*=$=M*^Wc*QsF-y4px2(>K;P9#G`#4U0JU6tGPWi0q|*0 zFOegrQ(%!JddIiF1L+fB$8LFEv49^s1u_P}CG+SY(D(;k!+R8;zn^g(N-gGGTv5^o z(rZ@VpH>-84_I$LofmWIHQ2rzh=}m;%d6Ym{FveTv3bI`rTTJLy+&nG=CPD1QF*XF ziX3(-W|m8*qeRCd(P&4640=lOzbmHz0~)*H;qbF%lSILixWrsl*XT$A6Ic*fdx?GK zwsg)FV!z}1)|$PSFw$)g99*`nepJ9Xh&2BLQQ{_|+n}u(8V!!H%cC2xucua1x zsSshsuMRYJc=AYA27&Fb4Czd+d%9%k0j(hpaovi*W9jd&M~~ZVIy^=L+JSwaMRZ@4 z>SK+rI2iXQ-aLGWH`Rk>8^1m6^K?9BvB~~P{GhNK;&RR6!g@o9!~V|Z41NeLhx)K* z1<(?gtK-LJp7%-f+xLs$2D9OR7lQu9E2O5*KWbDflgT?Sgsre&hQI*mzU=8F{P*&I zdJW*4E9p2yh8;cNE6n>e{+GjDcT~M6yiRlqmch&8cSyRK~dYL1&_0u>OMvSKOVv$Z3__ zLZiJ|Fufm)&)jguk5|b#hyy$0$z^^k==JJ~+oaV{qVr7ZT15PgpV48=RbFEi1|fUy zTl4#3Kv&lKlNQ?`>(B;G77(xJBTIxYI6; zlL1B>xUBE$$}8Av12DPnsT-mV9+}RicMjybJfdUK?bXuRnZYlx`i(p)M~~>F(f?zE zHPf>{G)P|LG&qV)!Y)t^J^rd#g#D9v09*?E{=bg$I!+{A4wp*#7UgSv|9D$~Q9Ao~ z)~`ZPQPqEE5dz4r0vV6jRO0pA|8iq}OY^maRG;;qmJ_kvk_6kzh40RJk^Z@gq=UU0 zj4b!ccsXQW;-L~C%FEH>?7$Co@Y1k*E9ltueGM>lUyV5lK3a`%-CX$*ha{?H`xP0g z&l`$V&soixQ{ciM(4_&v##;%jEbPWG>}9>4uAU<02jdzCb73vE$BvIt&C+ z37LOhrUS^Hl8QlAjC$w9<+)Ze(6erkxFN;1d(XwHf$`X$$`pG0vwW2s!OL>8!uo7= z0!I6i56A8o9J*Fs<<#uCFLrbvvVjz^^2>FPf?`*imowL9>6G7JX+mO7bFYk5c68px zV}(#F%f}4D-mYIAkpn?9xGcPpf_s#|Jr6RUPpF}MbnQDOXtj7>8!TQNF52FfL}z>d{yJC*`WAHmEn{HtE`{(S7)bE}=+ z{))lP0yQqc6wRvbsjWL;D2RSXOxS0qiO+L7f6pQcn6G}I+22F5>)!TIJ&e1voX}Ag(Y5p$8p_<+d?fONQpLf@GP}Nm8>PGqF$N z)*{-oR1JLG&;V;#P73a;GUw;enWF#}vF5zirtP2A9XwEFt|8LI z(9>22=oc3YM=IHl%7C3i;&(uvDK-wy`&3*KF;3kCF8Wdjb8B%?|ZF<_w^dr zsd(v_WhOc3Bm{Wqe-XV`fb7c2`p#Mr5Pu8;*bSTjl)JeXhi#986htRiyI~T$ECHyJ zDNBr12cWtHCYg&Ni9yIZ&33eAuzE!D{>=E#@>dHzxAxqcIQMK9%a5lbNq7t1ca=@g z&GJOtd?^7v{ZsQOqPh|7lFpv{qSm;sIS;+p+tNk0l+- zlD*YbE0@C)eRo2Noz|a!IN067Z4MU2ustz@Ih#)B@>7c60LF3zX_nd~TM0+~ESA|T z0;Wr}7=Ks?#tscD%7XR1Tq>1ww8Y!)^Ojn;d36E(oE<4G)yf<;0{W-G!9oAfzTdoT z9`H&ATzFQDLskID5_TrDCKe};14n%Fah!2)YiCJDYC>cq`R*7YPZ%2$5cD{~4&Xb5 zz$}MuOaGJ@U*e9zVPuC{cl8A`U@EE?6}`mFMoKC>@n-VK%UMFMS;Kd#SxT-Gcx>6K zMO^pJ_E-h=1pNZg(=+0_Wa)8pD)jizi~Pl5Y_*Su#3^)_46!3$}j0IaaL8c(8!hLZdLLT1t@YBmt^%_3n;I&_OG+lWL>T$X1=g+3~&dNZtG}t9gWAU>^SrerDa$e7O#zBBc`aLjNp? zp8OwEBVJ2gB$IgyeL?G{9{iHv+VooDv4E9WHLZt!;U-08svhRsn%#PR%M3!y;li>m zF$M2`#7b~qyxXDU;~;d1+@WYo6plcO&A@}B_oaOYG(^2zS#*yTJV4YfUQJ4LkuxhC=!MECS5Zg@t;~es5sAqh; z$hiA%K^zEFe_rFO>r~<*ymtYYZinfr<(Wr*Tj2uG6SapZa9wMXtGy9TM&(g$cwf4W zCAK*n!M(T3UkP`nD$WXEr!!HgFl6fAW`dyQM9){}2CD{0^^se>I`l0gh0N@>|%#vDN zE(4}{N#^Z8Lsb8?aNZwio1Xbl6-Slfv@U!BjXK^AorQ2Yhm(vd!Q4d7|Ua`b_5Yf0l(_+`D{l`|8JUQCBKP(6q9B zOxTju6@^Kw1`6ew?&|6<-j@WoK6e+n?0D*Y>FEth#(l2!1^tAZchp5?344-57`A_; z_hp+BU;BYUd(kxy_RL`^5zcd23Y3hk0~{!F5ZW{B!hGGW`2o=mCOUvNCfl6%I62e| zY|d%&uuqB_qf0RBETT!8V5(%&bwye#SfGO4X!E!i-+bI!vl#_9tJvu!*8 zqdF(?&s6BpsqP+)yzgq(lNJ!2JXL2WVp%beZwM#2#2jZ#mIia%=~z%yo){zxYCDT~CE|T>h7HEE&y&6nsMGJ&h`U*~-q%-Bqc$G>FvI7jn3xsHiA`In?>f zzSzYb(($ascW_8+Ppp5kXz{>lLRp;5+w=RlO}%mq(AD1t4?MwTz0ui9nh-x`()B4O z(=*4p8B-gysK(mbtB(Yb5m4T3Z1`vV`=Ki(3l>z%=M+@SW+>)ztS;cP5dPfASe9JO z=rL;Fb;^IJ(zR=RDO8m~`@B}t$#gU%{yrX2)5%n8(Isl-GR@7lPP61gDIhMq#=(E# zv+r2IX^Kp}j-4WFJa{MTHH=*+`+k8M%&g1WzsS9A%S)Sw_=HpY9&S7?*=AFFO7|=P z6@b`nh5hO0y#2QiU4FzrJ@v&BN(ip-)5B*1`l~UrmK;sO#TfysseY$vO0+p>V<9#Z z!V?u+?y|nd{%5b@lnUgf4R?FKSD2jwo7{9f`-_)-iaOTr&A@4pHpPF@$PArYw{frd z!|qdiL|7QTeFgwl`;IxnNYlLKFx^Ypid0tO*bOnawxqp}W_#^}J8U>h#pMv<=zJ0f zcY^W!jkkl<{Plt3Zi(8eGn>l17hk=xjOykY%tuE_6U5w)86uRq7gcSw(0Z=OYHR2s zFdgV5#Ah$RM#38^QU)Zd^W!*jJW%~_Ir=Ty3Aui;TPShmHCybcCCeYG>{7SS%cT`? zQQzRNw#0~32f*WtCnD{|i3xMHq!+20BH2!@S*0v%V;7b?{4r%6e^+q4BCs{)xgg=6 z^6`9lTAYw~>d=Vb?EW$$mWRrPuUKXTvkImZM*=QI`|L}f;GSsmWS+W}7oG=&Q1v3S zbHuA#^gx>*h1q<94?~gm3P7@2Gx+&Ml+nx)ZH&e9PcoHg&GvLyEJRzQgA1382Y+a8 zjPbK(G^wf<0b?&|U`(R=$3=x~R)F9Hw=oFqY(tjt$**MOotgg$i%$e30Rj;YcF-9@ zK|p;JF>BzjOV^L{ZE{)wBG#;QwlzIABy#9VqN<|Eog=;UlC|MljrM2`^RQR?JpC+~ zRp(-2Hm{^@@#tV2%X!-S_hPtou23@mb21%}ot)CS`PtWKBR3W=R(yQ++r2vCVU7S3 z7Od~F82M%r72md4!T^5)q=NV*dqr;pS7Vith%gc1!GuE6cE4KWRMUsIuJ&JpfINNb zlKCIx_phaPfMaKleZW3+{9+Qch;zS4Kr5gtfZu=Kw2=zD5l~fVddOHF{7VY;A2t0? zr~i@|$rdI%zW@sVr~X{{=LM3DQ?rWu6BGgL5UQcyasJmtkavdP{t`&MTUY1!WfK3^ zN4%E0di*7q%Kk@N08#rNvCaR8ZT?4W^B?lZ{|^Q+&u^_61t>ce{}8wUpH{*1$!u`9 z1GlV#tdX1-bx`2~vho^!4F8rs{;Nf3<7p7!r2y$;2)s@FKYZXXPxYSrFX?05-_pnA zRt=YdKkD}C?E+>p?(FTAmuBftz?=<)%kb15AMz#hf9pF(j**$&_lmFe|B@m$P0z6X z@|wR~^szAKFXdwd_Wy{!6A<;b|76YXwTk|uChqbpy_=8;P|^Opq)I-pu6*{Pxxjr0 z(-cx$zo>(XfA4wSS$gzZPudYHK39-sh+&Rc?~6(MVoGzw@tl$BVxfZPEi+ z(HjMZKFKYT!~a?D`;WgQb^9-=9-O5Ws3wSKUh(?SeLCfj(#+{=t)_32H<^HesCNVQ ztHP_D)H)vmFXhykJ}L9-I09`FH`iT!>^I!50j5#T-hTKe)8P8cG<+XeT6ld3m;9r| zkqP$N*@C?~__?pqj-P}x8~`9=dt(*Wzuv7L5FocHNA_Ge2;xp>YOYc)etxGlY+cKBs`IvB9=g@c8w*(x+s8 z@yLLB;D1h4-x<{=HN?IIClTd;D=&TvUZ-@9Wv(F1t0ckQXp6%*`!g_?_+1SbDCHkt z^xexm#YR;{?H&$2$pAo>d)o|yY5h0<`0Y0@-wFT{B$n0)tzv~Y`YEHjfHVE@R@Cly zKRVBGVT78WPRG11sa|r}X+w6V@7H7e+xeU%3q4-!z1zQ}w?~#ZP<_U|G9jN_2`idVHcmV`o^~y!$LP=d*;l z?62PlxPJDg$A^!D~3q{P!36>q9ra$u5J;BkZ*1LMO917d3=_zv{by6M^vSH?K)w8+vrIL4(?b zP4z$j{O^aptM=O6sFMCfEPo(;Y(4AhpI-SyzzVh0dbD_%Ry^=qIT_%Q--~*a{M1?K zi`Tkjdb-nOWMKK8c!86DdeZ1eCnsEIEl%hH%kI>esQ#x{K0)?>_vUb3)~$4ksUea= z`g{^CiWGwOX%Wt+ey`@$3juXUBm;?qU4A|H`x3Sn>1`Y&e&f7S+pjvC}q}9mg?B;G1 z3b+HHj^{$%k+k1NX!X)SHY;NuTDQ!hwlvA*;8{__epvDOwT^i}shUOO8cb z;TX$O5Fg6n)HIt~lsxumfj$7io-CFG!^UK~(Nphk0k_b(CIT7JMHV&~BfHVvFtlCt zGic`L-Eo3MbiBLd%LGj7>PXQD8)J-8|ApFQD$qej&B3Ta){R4N*H|z7(wuo>?V)`n zkY2-V_Od(A6jyGUZ=jUT(kX@4f4nz{_orB?wwxFezqnw4tKluic@G=b*0ePB)l=&& z9cB=;O05Uwh?(}|q9!95zNAAc$w-XlxI-Vzb-&FbImXRG@5z@?FjG@@kLm8hIg4%` zYUvJ*@Q}kM(pHYCkJiQ3=H8 z7i@^~tycAN=&YAx-$%LZIrLj z`XU$z=X?Y1{k{1&F9CKvm7=@rsXP6~_AHdycSuiK&=IZ4%+l^4S1a(IQvj^aNK! zqxJ?}GZiBb9oB9VWGEZ=%FR2g${DNgYKf?I%K;I;b7h?>SFLWozN2kq7pUw<%G5Dm zdBgY_LB$|weXo&x``*36abGGBoA88ahroVpwyF7Qu+7~Q+w_n6uIn$qUyLfaW(OJj zxj0V(YYyb5wq!zkOzIAZ8Z*bR44lKQz!p^&WaY%vwj9yLD<21}&dy|5p*L>Xm*32NDzQ5Cu8J$7x;e=TjpgVlJHkt(`e4-BW1}flkSKOu6uL}2m@8;EL=s(Aexqjkk)X$*u z#Z%Px;AfMi1$n#!8Zz_DL2`8&kUBqcRLXXH%~k&yQ{jr{inlx-Rjb|J=0oD<6FJS{ zOg9Y#?LJ4en&xWMUVvt)!Z-+$+a7618!za!1iKNrp_e`N9)T{DFq?s|JcnLx*Jul_ z$~<9PkY2y@IW%I&!Q9!)-1O$9!YAhDSC{ZAWME)hEOH~#Uxk`p#E59mqojSSCr5<3 zE`{|$tSq=C4|bzmR{FrP+y*>iw&h`alu6dcO*cLCYkl4(xw%pFyWA)^?FC_5O2$Q) zcVze>y&@yOqS<_#tsl*IYGgJkwvc6YQ8Z`_RFMn4j^dzc9jZn|bE(gqyKe~u#ED)3 zs^2dBKj26-fCj-$BW-7)ABrJXw7=r^T{&R+&$+sN&eETL^XLqCX=Je~8<_Dw?j2z? zV~g5-$qybdYwVLGeWGCYQ?lr+K|_@=lU$L~t(OAfmml2qVrbCVaPiTc-evJKyuXYn zF4N4_&i7z|&BosJXy==}ZvjIE!XqT&S&{n@PfAEdYIZZ9P7^rX9Y`yllZ=v!Y=F$@ zDBgopyJj`mQN7rrORmaiT=95u-v9`sh5I=EL$uwlo>XNDWanOugZM{{mf9$ei@N62 z{yd`}bnh@mv47z7=4Xgsq$@*yvbGSgl>5U5G~0$qwqeuqfBkls<&?36_c~fs)zqFZV>bX+$9!75sq8u#81Xc_$eq0HdiCDqPY(L*zUpg)YD6FF_57w|kNvXd-I@RL z`oB)jr#j$mXC1I+>ilHj_#Rg7UkB%ME6{BV-Atj`OgTUF=oU{y|3Kmza7aA+N^A*6 z?w?O0dGkwRP{`e!P7qbsTjMg$Rv){BV^(W+ZuqT}Vq%F+fdwkwl6$sLpN&~`->ZB2L{^ zGTW+3gXj?gx^P6>z{Ax?d14PsvH8BGrfQzrQUk@0HZdL%!&Ml)-6qlKAJIYljEp_= z^GJX(e5MrE#a_>0aUGUcFSlQho5T9zWIdNg;#AyjV4&0I=19@yNYNU6H}QQf*1Chu zDXryM%afv%iAH9>75Zn>zIf0SPTAmbexq!JIQJG(;;4bK+Q!p2O(jC~;O0n{o(3cLs-5{N9xGXP zvmB>oRxy34YcZ*P|QUHXm#`!>9aLtct`jv1TtD~waPonG~ZaE zil3mgebU-Skl6i^sAC)h2=4Al9+nJmKL+A*+q1_^cMR;GIU4ua9t6Ve$|u^ZA49IL z&WmhG#O>pk6lomnMR&f+ntxPfJgWXv&3EKe3>$!7_PCmSdcF>ClI#Fh#e7Nt<8(pf=P+xhpJ*W5 zWxmIyB`*8r8_9$^L=%ZXdb}A^E6AG^7=S^RQ9H5q{t%sxlX^CAJw?<8BjIN^L4<_8o&MbdJh;z3l)@%IgkHXFwMx zp#6v|(7xh(FE9F&nsh9T5X(~#ag-{fT}c2v%VRH_@|5{7gODwhHAfhOpnz8An*fYL zbpeX+uuIs1bZYCq=32EkVcb6)^rI8$Hz8!iiP6NdCYB@PqVf-?48<~v!UsjGd<$2F z4B|_p^Q)^VO?^VkclM_v%$@9s2)3$jdXqX?0U0(MmvRXcVm88N&40nMcfO2t&GV=c ztWw74VRPB2bldzsKyG(rJP9mi4 z9rg_yu-LC2UX$1uX3aE4R=szT7RyFRs^{B0jZacemKwuVsTW9FWqbo7Rg8jvztW#T z_0vwkL(^K~ZhLTc`(#&I*exO3aHfHIs0LA|7|$9QlBB;$igfx zkL!ncgxB7|K4vWdu-q8X+^|V(j8}wERBjA9Cb~UDr zC#GMRK3K^rCR#5HBXJL$EA}iHJKj8$UCqoeC?ETXf!UDpa}E_rQ}RE)xMof@JAb#% zgPEJW{fYzQ)e_%v(QODqOxx2b>cUibkPEn8TC44P$hO=OBphRW?OV-?I+f342wI{1wllju%LxRe zyGs76aoRA=iiPCi7XEhJc)n>$gY{FNqsn`$$zEHl&B`C@koMSeaoj}_(wD}=k;G|} zbVSTC-(5jL9g);I71V<-oqK3a!+N^K9SEhU(Ij`T6qB;%XN_a2I^kds&*H_EvP#Rd zqE0ViWa*m3@7*l=RQYw1Gk|lPpTL*1F6cHrFh#_#oQtkKTV>mw-Agb}9@VT^3m^AG zigv6PukkB%(cboA*SzU$)p8-j3;+Dq*YgPSi`AU#wF3t(w#NKG3(NyV5vhNh)502C z8L`&xcqF1Sn(Uu7f_BzkFjLzlB>@DoZ!>z@$1;jVd+1s+niuE!KCd^$@3{gGI^}Fk z*EuY=;T>en-ZODAvbWCNj|Sh>CRp zJHEvdzb%T`8MB)&+3FKos3>8+dq}@1E;-jun*Cug;rrXed4AW2-F4hK47egaKf4znpM??7%*o#a^|nr za#0Y}Kyh3&)4TncX_qzD#G#1}6C24eI`rtRLD#(4)gAdSkNWWsGOfl^oS)$Aldsf) z8?RedDY$oB#1zu=J`Z$CW}6yG@-a%*YQ(%FvUK~hj4 zZf&0iu1<~hw_{n+<4eXr&T*x*rkYo0zYGAt0h^}`9LKyNvlm-(rw{#0%WbQ4M*#$xC%-e_0 zPn|927DAL|`;AwfLnLSil_`duNk0$O+!mV6!l>$ZXkd!o&QnS?RzVA`U~9>pwTD|P z7kE1c(G$<9y6ARPKd6zuSZ-JtpFnyybMdHO#3QN@WSYu&dwm^3D34`2h zQ_KD^VF6=07sKJpX0P)m==Ul29=l{67LS)1DCg@7hRu_U?fjx5s@mB*Z<8fU>HuWr zbG>|VqB_=QSbM(hBd8(54>>_Tivco!&8OqMqKl+QI014@o;M0F;~hM!6_z``ouEdk ze@J)@J=)B52u-b6RqnEJMtS!r;cCln>x99#ng&_-6Km0?jGytaFdB=JJcX&HMYV!u z@f2ywANal0pBKRwQ-YhH1$m|Q?hlvULv;y{|L~x7^1PRxO@qk0d8x}q?ddIPj2iOF zR`!rV3!ZJ_aISk$zt+y-)8yhAl-R&a@L9{CGTFVO-G0=p92o`6w{jO`MyEk|SmNSN ztk{~+gqi@0FZ3eIaARWPkfMkIIx12%Z{pUFO$}m0%AzGVT$q+!cf+*|MqKjUEM$lA zYgMgG3C%XYwwcTir16!Ir?f7a$RvC^pC*eR*vV%PEM9dbN<3MQUQDvYJQ5~@M< zInIUU)0ZH8LihYqq#0NAv)VEE#z5L66tVSLg_H%OC5yOP;7AR}F;QAvIfbp;b`X;>2SiUTDA^TtMF<><%kW;%S|0;%h`=i zsXN*hK}aK2jGcNtEhI-+4t){KfsLEbo|SUhmQTxU4OGhFJdt!l7&Fu9hrNfsS!lf8 zDxDv^n)IX;(KgyuxsbhJjiNXBjiM4OJj~ zYDGu?7nzkh2NV)CWs;LKvH_BMsiB{%ppIK76#87r`OJC64`4(6iUkTQZ^!uZ^h3gk zte0EmJ;yG&mPVHyv&@;Zrll9#gMtPQQM4^k5cWBC;0ms)R7m+ ze03E+twPBeobbrJN;l$$bQ+HhS?=m}iEnUQtk1F>J&PAPV(Fn`5JSuOl;=dwv-Dt* z9rQDB4jdjtHQZtdDYcHN*{Y-J@KjZIIJqQPPcArG?X&)JZ!#U+#X+l|jlR9CWj5Qc z<n1|-~Pn!ThjlR%r zl^Lhk||#jBGn;*r4GW6x|c7bo!O%($Q;n8UvRfdym(;<1YB$`0VMSe%gBZmNe1zNgX1M zd{=1fsArnbZePXus{&lu*k9{~;X+&QG#fkUL-xY*LL-&hyG70Ip+cJ@IatnoK^Lg- z17q4Y;LQ#c3ick@mvRH%j3*?;7u~8ACWKeUFL8Hxne-90&Rl2M-cJ^e>lzp0)dA*a zN5i~6e+z7rB7&=gT9{mSrJbH7 z7GIghhp_nbP3YyFDDSGwpZQ^WR0*H{LKAH08TQqkhEmi&J0#f-DKTi7D)BfA;G$#G z{GWC%KqW{ni0u`3`LuCgaa^$XN|kfk4SiH9R9K8FrB?#G2*KQ)vaCw?^)qRKp6iX3 zC{_cl-4i*OJPf_gtwj0=3i8K;OksQMOx|xw9xYzi!z2a?EF5?=Y0+FUo*!tOiVV5& z5@AlPbU1%*FY5iIA`GVOlG|AY8J!rj@q$?Q;L)wQ1j_=t&a3*2}j>Ie_Z@Q1U`rq!#Ge9Wk$r^z} zrEi<4WPvt~ls}ykb^S4OHQg$mXXXqiRiD`-zSCdRn&T-h)7yEJEa*70w)Rd)&O? zrLtAax=4tvB=HQGGetY1PG|v#Fs+GT9mZwo) ziEL?Z-KXN>9ohT3H|%J#)yL;*XN~3POg?(McWBWRF%KJ8SCb&!V>IH^^B-188yal3 zOr#HZgmpF7j+#=lJN9;Kep#-6^mbQ|yoS9N>E674nWjQ`fmYepS^)@CB1#p2O{I8=yMoe<`kP9u`?);l7r12&wedj*m?9 z3Q^~MCUl3!ts?EUeKLkVHB1$bIZoQ}`!JZRWf!J?AWtJNHx#T$6tXJWSeXGI?I+?G zPCJX&g;|b?0%emoH|xW46_%{?5DkJdI@676M(s6LH-PL!bL)sIoOsOEeC9kkMuFtA z*^w{d$#u6%*8vZNNd+@zy^qVmqj?~zGqtP<4_eQ$d@1fz+N1iGrk^eRGdblFCc`2( zIAeVOqN<1XquDp#_=T~yrj(T(Ff8WwCe^Qb!zx*+D{Z{7#gRoPU<69{!m6*EOn+_6sjlzhu>n)hQ zMu+VLs%fDpZ{d><%`CgeP_sWtFp)5tMrJ(N2)IOt=2D?8a6s2nc1#7 ziSO)+6OANSg*5Kb@H+-6g-z>Dv*iT+wd#r762rPlj_~7TRD!?g@{1IlK{aT+^EJIAU4}r&Q%nL1h*Zd5+$_vz1uWH$Ywz+NybM^=kgA zk!8+2oqx|B1a_JMu_ZT(xUP(n|G7{Ahde0Qds$(>#J#ydlZ$Ngbe>G5ST%5KATA@a zqs~Fq%~Q@)q}6=GGVyS!PQ!xmhDDzieKR~U-XnE8&K+R@E?N^ETD7d3RBW={VWso? z3C*wb>gy}C4}v)|*|;RS4-56XhRrd??Xp`-xEa12F=2Ka_Z*gN<~Q>HF=Em6@u|qM z;cFS|FE;;`>RG#5sY=~HXt=FxfmVMW}^w{ao1_IJ5# z?HvV3^Btlmv+<4#sOC#l?$N>e^ozeT0&dyUI+{nQ}wjqM9g zt9aqWtlhfT_87QQ4=u-91JaCz)z8J?-_DuOQ`Pm_ zNQ7ck*2Xm-y=`)xx2@Z4q6O{Af3^3sF7d8)hvaVL2V)$ngl?%6ALcS!+1L7oex>`C zGC2_!!c-CAQn~wGuH+j@S#I5J>Vdx7)`d}C^fgou@~6HHxjYU2Yq?ol0Ti8A1$|}X zO~W28(twX00#Y)e@Vu1s6DF;us)N>Bg&HM`>uCPV-o{cXn7z0H%CzadoIaztSJ=V}U($|$Y7`C~cz@oE-u_k@Hl5Zv_8cFzl@JIjHtx>ks&7r7^-#1l!J zcswy&+s>2|lHh#fVP$pJB^pp>qtyHtjc!}ZIosrP+&V}$_#EEV_v^A8Wt-pW__MW= z@e_{|l7W1$c;l3=*l?9~e0HAcxS!h@aQkjavFLT_bQ7PbmPb;I@zq<41cf6|$i`|&B+Y?(^+ zN-5&jh_k5tDaYDnrNV{JAMB5VdA2>g@J}{<)BIHNnqPfJFCn8%rb8nFJbe4^$G9+? zv(O^x(xolAbxpjl6y>iN|5^%|dJi&xQ}FVN0Z^|c0}~|Hzl^yo-*3#)L0@WCmeqV} zJ?Z))mygxG&N0#^X&bB8f2!CJs4P=!Ey&lgXSKh%v17BIu+zOrb6i=jvbnggpWNSj zJvqB@BpGS~!7mMVmEQ9obHXHHl+-{wt@UP{r;5ni_XYR9ee-EL|IAdG(yy^>$c~3r z@FqH?+4nJv=Qk#xcGhRMr7F`T#DLCE_nJ}fRn`6Ci)&1qFcc`MODSdY-cGf^<{z!l zqZ2o~-oBQ!H$-0^6zpaaKX4iu+H#@2x`O_3NFHj>GT}@#%hoIItRIAhc*#BOjSjXw z2Q}WULz`_^v}J|Tgp14YfCw(dEZg%OI*_M@vwAoo zGhFIQl5J~#lG83(fw#9po#J>7(@#0;tiT-!BhEUFCe?O{Z+B9=G|HcaPOl#PWZ_@a z&C7AjwR=LC7%a&qr>*%KfOG#?8wZw5WJe*_mA-eV!R(XGU`RJrv&Hk!&tX)mlMGc6 z45N<|ybs}9k!7ku*jLyj7t+B#2G_SA*gY4DeG1AuN}AE516A6cqY)J+8)MF&76mh?I|GZzQS|9JviJMNh8NIjDfyS!M%;oFdq@x6Vr^zu$wiJ z*{Ow$1NZy`=QqIeh_=AFExa(c&HX~1Fh&PNy_Vz&`EqAj(D-qdk#Lovll!Hxtx5>u zGB$uK_9^Z`OlcdN|5}rddFL))>RMY1%U#X~lovGYtBj7TvCbpB9v@7Pld#u5b$i{< zo7nM*78!4*9Id(?$k*@_``$6{dwFg!;Q}qyOTYLMR5CkX{MMKw-`_Q@8sZe#%5g}3 zj?`27od-GzG-_>-H>?MUo3-3XNc5$x@dfH5xo@9F$(~tZb2{&zcGh+3I)p>BO>S^+ zDiL?B-F@G1Yt5JTjo!&xg;zK`ciTHO8?|$uMmNe5@)2#(<&krFru$4<+{-%R9wf9_nDfrCxB*d4U3CPNz+VfkX9Cv%yi+ zEgfUiONXJ;y6G6Fz%s`?8v$&^56Nd+h0`(}wf=LIsdbV0pSWKhrdW-CTH0r|HX7#} z2E(M1a8-Q;@*D6HplCzu59ycrQ_<$rrIWOxT~+Xv#*t_J%Pnyf1TJEa;edeS4_RPF zZPP)==>`oF{+wXoZfV z)8|)0G}N@#U0RHeV#L;}uvgt3iO#dFg9?*?S1NpbgL4&AQl|BxB#iT3q4CGt=2fDf zu^o-oINz5w*LSy`T8NQyD>uAKu@lugmu)LuL4W7Z5B2VZV?8kZd5q62bkS2YrZY+W zYJzCn!zf?w35zuHssiuk3?cj+hgZrO zYWKw@N0!%yhJ+0Q;Y4O1!xC|Zf&MREOcyR#r(k~;b#%=FwOmy%cdo6K%h#~bh+~5e zp`W+*Y*aAfDnM?8bBxlGR~l#8BGYV*PRj~H2X}I>x3jo}vN%oF21A0#@rQ9;j#sC5 zzwI@K2jwu8kG^xkjZ59Ejp~TyjPKXulE76`;WAT{U!VEV#l{Oa9He}kiyHI-1*{cX-NF5+K(F+vOZ(!K!-1)6wVqi1AdGYPxem>99 ziyFQdEJ7G&a=MD~S@sT`oGZ^w{Nf(Em+b}l-?`RI=B#%vWZ<^VOQsLp zNpyXg=*t>ekmu75xz)A#_YsLg2;#kVBvZ@it#MadUCCwr)jsB+i&yGel;fh-swcw! zGB09>bHubR`lX-ad;hdK8gs*!$d#6}fkRYY;!*g=169@{Z#vKY*h6q_jb3%QbSzhy zS~b-t{|$bB2geg+eRorEauk;sqqH}3E{g*uo}0TVK^+LM!QJLO`}+f*X#iVF0%seR z0t%k9P{Y$u%^!^I`?EUO&(x-D3@b+>Q_mNBc2?x*iO}7qjR$GBH%g7eStoN^EkBv&sr&H=w9<|8hQT zU2V9)%~E-NhM|vY!8v=SwnI&N@WH$xN~rpOvG$pukNiUjJ>_JxekZ-=aO@Y(_O+2h|-!Yml zz`dq{w0`x~*qB<$B9HaF#H#pnM*Xq7&tCg|p(z-krspyH>Z%jY{-uQ@Rx%)CwxX5C zFd}nezHRU{WY{3&UC!}?qiN2ScerE{P;)5|r?G5tlv!!eU8Hw&Ic_NMYn-uh!`U6c zzEw24?AlIyxAEj~N$Vz+=`Ud^GKoj^O6R`CGFx0tf}NVyAir3gChWVwa9F_5E@>zA zby8J`7c*%@+p3U1b=SPiIk#)puV&CyR-+e zNJE|{U1J*j<$9`sobo)*Bu{!KYR0BA4db5VThhjLa+yWnE!6Ppo=}9ln_+2*^p6SX zww2LI)R~uSMbZc4X<8Q6$77pB9Z3XRF<+zlCeuzD_hwrXI=_&XZ_@n7EO%=rZPh$JBC}Gg zPd(|QGi4xaz}I~uN&%Y3s*?q3I%S^nE9!14r`cwO(0=F5fw-GNc!lGNpPl#v>K8#U zu^jHK#y28DKsGv^u>|g&p6?nBqG`#tfY;>8#CX@d|^0^%!v+!?WTtzuToh z{NpZv@&3z=LC*OfC~HiJ_mM^D8SFPVS4+(X|D*!{(NQCZW=r2M=O}cOe$gwL?8WBf zI;gd_8tq}=s~t3mBy?WQ7B_sj;;BPPXGKQsBSq(69;2M#H*c}3WJqZHQZn?CS%*O# zy~sntVb*bBmQh^+qdYqSYjA(<;xx=%I2;6)leQN~#^j|fp5r8XDz6pmUBN|rBT6;y z#dXZ;Pc2bA3Iq;a^vOCh0(u&?YFWCj^nTqRP)jXGMPyc{i~4T>pRwj(5ZgzAe>(`*%NQsq^f)u{LC%yj4*ubXiJ*;J4YU z&-cgf$*F*{6raj0BLa9j<+);+r(UlrZrYY}ps(~x#v((Rd=v8t!eYq~A)!u|X27Ua za_ho5Gb+UbK;Y;0($*`|-~Qm|a+BxaCO0ziX1QvvtAh)$tWlXO#OQqdDdhE{jn_@% zkU7+9+_2q}2@-H5jguwvvt>wW_>{(Ywo%B?QZj+H;oMd0OX(L zNmN*Rv$;9F$DC>jB~yHhX+!9@llpaTqbg2vCkeXu2qcWYM-@uJmm;XvumB0)9;Zq!%aaJA`NAewE)(n0|)o9$9-pw&9qd@ z#R3vTp22clERb1*UZe1oZLo9}mqo2vL&|iNZi@_)Q-Rdm28jhT>!EQiBIPSC+wHp% zc}uEECq&Ys^3S*1`o~|A;n7lUrq4PJb~lG@<4}WO>|=Tb>6CV4x1!am2BBFDbii!$ zBZINqLZqiQGrmt|UZ=2*65Qn6h))qVt_ zhn2&{9}knL9GoVXPa6Yn^?ypgF<=jww^Vj!9kx28M3nP1MgM)i(^|t5T@v!N;Gy*- z(y~Z+w`bR&b>>^%(@g>NW9GS{LT&sg|3rs}3>mYJA`}b5^XM|!Ad5dxb$XnAmfd#Q z_CQHULQR4mp(FNa@+X-Fmt2~N@4ZSlI!v|E%w4Jx!$PC(d|qK>R`wb_sE*qIAB{~C9yG-S{sr+kGW5bm7 z5%gwdJ*f)a8H(DcGs7!lExWGtQiQVKq||n0xYO|hGk_d|lt^r=p+UUs_qSiL>!=Kl zJbKlXUS&&@S=Ihz*ktQTAG6BpZ=Y>bKF8owU-Q@`j0TJT$+o>t7!1EsU|LM%BiUeO zVm`TUfAiwchiaaN!Q>GMl4~I07pb7|4vN7kv=Zy#^L8R7rsRDSJb&EX2WUbCU0EN4 zbW46UkgoAo#L*9zO&X*QJCds~H?91s+J(%zR3@pbS(9;k}Tsrn9EE|+W4)t+k(O0nX09LYQ){il!35vx^zDUrZ)T ze^#Ytb3(m8tz^{wMTCDZ_9qfm%nL3t!-~&t(l5$wi!@M~%o9D5a4n)pcDdF*QLq?Z zA3XRF^}>^L^Qd{|G2Rak(!8sK?3uPN4?TT|((NBJJ4JDxh$*|3+a7lc`mzt(uh1A1 zr)iCluWAeu9%)!I0yFh}#i0eN#VYw4n8a0fA(^_h58eIN2&_FuPO;@{Gf0_ZuILIW#kFRpzt|SdW+6 z1&xiKtgpmaQZ1tSVzRh_th&~zvEBkNq{8td;jiX6{WX#dRY87}9Oh7b+r=Kspa>VG zVD0jc15YzCQbFJ>2e`Y#;?o6jS>e=lWm})wk_aDQPYyiZSP&g6+x2F9z?|pLv|As* zv$ep(&wZfl$EF@WVe}^#{~$McYryRNINBG<^!6_&0uA4Pf?U}lXyGlYg10(SFJ_Ig z)1``>p2f%<=hBrCDrp}2Cvky;Y44t-=>EnLXUf+RAT2FLE~y~HwK&7ay=UNHoWuN4 zggu#k8@1@ROiAeYbF1#MAgy;_sZ)dMpcMjZGKWoEWwYz2z;R zwcQd7*>NldiQqzBC9W2p;RPVV(-pOuN3&VECri6b|D8#^p%D~>wVe0{3FF>$nfZ38 z2Go+h8(uyyW=V+XSL)}FRSl;hNbZF+g8s=7s_ zmHuHZMSr)#z`#>Mjq*uUfttRlZiAN4(p9(G_@aqOMU<*#nKzG!qlTg;+Zik$ccZ+0FFMdNS2F#1l_AEJs9& zpK<8s_(}HN1<5p@Fb)s<_3s3^227N_FCm$^$}=g^KUWa>)eNSshg}pKrhkZXVWg4$ zjKR?J>$kBp-_Ck8GV4&Xc(bszaREo(tt~+cf5l!sr9XyA7KRaGag|0S7wPP`V!Y#T z5~&=z7*hGSUQbXMuijDQM1*+tYWCcLr`5@WQrr}Tz(7t=@e5LNDFl_|W9y66>CCCF zEfpFdRV*y?q0d$T8`oMAoa#wFP^h}Z?qey-a7DFWnp0ve9{WYP++fp!(p1(xc~aEu z*;w_nbe(9|dJ>bkS|Hn&l`LYxg`W44Or0%8?bIWe)6`@Ig|GdlBJ|Qz5FIoT&^3dbGjm(_pHjV*--0ebU z^{OBzCm8{rbU?Ju&UMnd|6B3z91Q=w!OvzOt2TmXLu*9eYBluA4~`dH zh_4+FrB~JZ7+;a@;`WZ&t_oN!;_VV!DO%_)47M&4(IiEY4dvGFf#i<2YCSXQf}agE z9R@MSN1pp_L!%geG@(A6&~VyEzt3hD_%3d@$G2ZyK??aN&fByOgO-rv1Q1&%t?;V? zb*Gg6-0NzwE%;SIK3oc0DFmsrRf67W_G$zleYhIbEQ`r>P1R_JB%x~?W4+bQ<>mVs zS*-1Y0rQbD^_?{BE1xi;piB3|)4o=rQ<5)qIX~s!i|JRZ3VP`OaQ0pENtT3KqKlJq z5*ih9(ui}eDKtWj*3YN=q2o{G!R_(Wz#xB~k`ZPaBIPHgW?{9F*_C8A@)inwu}Ngn zjxwZD;ZO(tWY(}KuJw?sO2`TA22n~QYq_jsDWBSXQ~|b_j}%28nvb2h^h>j2R|U;X z`dqhhi9t}_)#x$kius{~*=}T@IgfkR3qW%63~5xL=paVLA~nF$6!|bzv7HR_)|^V3 zPFORhju=$Tfkw<44?19(_qOe_J=(GJT0iX;%ooM!`j6VNTn$*bJk!UqDvE62aebQ+o71Q`^G8&lmcP1vs7pD0VLi_FwA-p% z0r5VeELfR zRO;SzYWAtE&YO_Uq34^T1-^0RarGLjzW)7~b;(anC(&60EVy(uw#(uXnDvajjDbmx zIbT%NgZbIWnc485=H{3t%fNH4ebbefj??*tlH4<~yA9Z}#bf$gB|dlLUu6TA8akV_ zzcCcC`W=*XTLd*p_G^+XW_FWXrgO*~5Dp&6^ujVj3Lm_Eb){dx!>Nzm@U&bnLSGWp zC5^vzYO^|24^kp^=z7vxI4I`F{quQyX-mn*5bo;tT)K)tjI3mtl~%dTQ06kWX7r%M zG2>c#C#iDOl#d6BP_kSeu1OobN5%stHLtkCoE|RO<$&r_o%EUn7tC2Ee}Nk#B0Zk3 zE2tH2C36USB0^jzw{-m3J+bIJ=6@c&@KFs@P(AyQyG>E6FmSC!eQaZRR_uw3#nan`h# z7bvqmp-(qs@4>BX5>xd-Jf$f6DX;eY!gkv?$JY9x9 zw;vO8^~-{SAt=zhsvi0)%pk4qIip)mbc|z{X3arzy}tL=YNjfgXzOR@D~k$)jmgtP zFGuK{hOPOA@o$QO*eanv*S>#PP>hZ7C*~r}ekjz=Z_knAWp6EQ&I=nkAaKJ zjhcymlLW^ljeZ(DpW-w<`Ua^M+!Qm0(VuJb@T~!|w+8WgIJZ$N(=G)Ep1)WPl%24bWMJ%EV zt@_A`8j*g^dav|LaGv`UwPlI9RN;nr7%lUprdOXF*@wKaCbLP`m9XX!`v8zsn(G8w zAWWy4oZPy&4zf(8d!r7)&|QwIzKiD)nsMA8l6F6&e$VtHCR!+^W3}EW;86!yV?WqK*~Hr z1!Y>IQ->VHsnNsdat3P;$aNPhce0h-BG(ZhS}2Qcr$*N@xmliAz1`TYKbMJe5Gmg- z7vk0QL^#7h0Tr6(+vu8W_-JdXiy)ij`t3>idJ4Z^l4#SP9kRU%%6HB7=2z#xzR*~_ zW0lJqZG@~7*@(lHuD1I!cv~=MoIUduIp{k8>arl5r>1zv8d7mB>z`SGjedNm;UEqi z!pxr9^r?jMrC8){XQuTO9}HUnA>0W1aS!edoCVN90(%0X8WYoXUu@-UANiaE$_Jb@U31HE-6tPe1&z^A|&&pBsC%PP^ z%=%`B9;X&^WvOlh(Fv#F_tH{X@~z@^xELGGTe&;9eHniHVGVT;A@;&L}?;M5PUuG?a~LzQ#%KBA@xJoNieC zMIsq-F>MM=p^K-3f**b%_21UzG!`*XEJa$`%!o6uqo1Af@&#hnr&G%}C z-gsmi`gY3SiezD=YYdlcp^ln*?n#vEp{a$rOKP=C%GjxmhMWa+V>KqE5?`*|xsmx~ z)Ual|b?;eH=TKLL^`jt|lh=-n9abFM7G*yo<-M~CO&>t*52ewh{dJJHvB!u#6uT{R z#W5H4yo^Yc77a<$L_|Bh~@ohm;YElP_1U_4Y!W<=5i(=hQf+>xZe3 zowvCc#&dIz7924iJUKF!i$8@HYG z25Y4>IE_m^IrT1ehYe-Ud~bdn=Um1q)a3_ukOB*6XQ8(SqG_V?jwQ1{Ix6d> z)NJf1C-3Q9H?dQo0pZBxhE$P)F2+tcV>qM;{R}$Eu+kf137VejXb!Wlo@t2tsq3{5e!CNs$N3#sd{mCNz zD=u%ssbPjJ=}qT{4=tdUO4_a+#hYUsiKM9yrtc?y1WtkzP3E>pPDy=!U)MpX)AyOu zOj(1)f^py_Ov!3Jx)3LBxZ~4z-GHK3T^qZmqMii-ra*9FCCITuTOdG$@X4c|J^*Ta zRY2ve7a3^l1qxt(UU_c2nPJJ#0#3ZNm{n9k;r#w6V`!FguD5B?iBne$p9`y)#u6++ z8S1)nDxscP5*uyWoo*D;V*(O99m-sTv7w*`RKG^^y{Wz|rL3-!WcOeksL=sWS(Zmy zXAzFOx7TBtP@F**mRV~e!C~D=biL#4KJ@0;YR3%EoqU|0#gid)+@^ZmjOc=-bW|k4 zC)uDk#(oo~b28rojtNr$X@r?<)e9bemJs32BL(DG;Y{Nf5ATGYtiO4=*~;IU;dFg? z05c$C`5n?v)W#uayJ1>UBuK%H1(dl6i^?FCd}Ak_M9Y}Rt@Y@`v>>jzzf&r-5n*7H zwOcqg+gMbzf3$?MU9s6kd$NDGlfH1C6gRQQK<2QTrh7DN!7^f@e6g=Q#|6(m+EF-x z@{a7SaKB1pQ2rB2y;g!cHPd!j?alJ%-v$bHT z`osICuEE8oA5>X9l?Mx|%GRw{XTrl4h6pvGhXs-{1(ZI0mTF0MG=l>Xboq3$=uMfa z(Y=iek@JyI>Qyy$>)CfUvt??%%*JeXD=DToT6gxk3_hZg*)mv^)ae_c(8U$7kwZh> zpk(CzF@(_{1)2p5`7Ax#gr@HW7htrtg1%oi+qKy^Tb@@I>gY!?8eD?k=L;k*4m{?j z)IAoBIS8u?O!S1J;g!xf;8P`21a zs~J%W+Uqsm)`3xFHQz&rl%(TDGxrnPo&J1J_D@)+#bPl&hTv!5)MWnQCJjUgs^Xik6Z~| zt^rSIp)Z4oG0_*&SQ8zg^kunnt;>Yr;B#Z*Q0a0`+*a8V;pBR!Sp(?L@N~DLMv&`` z@x5usO5+vBDngjO@|_vhR?oZWf7c{%3Kvj&zmKS=0dlD-C>pupY#VrEzQOX>zust; zwFO|5f(ixwgdh(crbLx1UOHAkhWOsKh_LhR$yA^c@7W|%1z+jDJD7^%k!1RjL=#l` zVW0#bSXW{keqa}fNkvJ2RxI0QY@O1RA}0cZf+R9gEVBiHnEqT0mTF5JXcb4oB|8rZ zn*=;h^oLx+qI8>SSi*Qt59B~YyWH9aqOAiKRSOCw^hz~*6xa-{q+)u;ZKqLxtxc(x z;@U+3Jaregw%$iQiG<(*JN(5}_*+<5;9rn>)s@rInWe}Ts^s2?A2#)m(@Lym$i$VX zxlCU&8q&?JE!H06Ta#}oSM38}TsPj3hk=0c77QJI*eQAo69MBZqK4b1`hC#6jf-j6 zb7x$<#MWGCMUCRY+-f3i`PKJMjpBE1@aVeD5A2wNL|OeDLp4trgE1o{oMcj&Ja@UT z3gXl+pFepGPaTBQ{}KWTKWO*#&c^04D`u74W&4K;Z{Tl`96OxA40jn9$T9AX&RJnMY34gzr2&km)5TQx zQl?yOba!HM<48QK30QJ08dlqJw{SBnp{B)pm~1b`x%MSxTCTh*;EUl!4n8Q&RI?3& zvEhHmR`q=gr2l7B-<^=`tJO4RdypJ@ieiS!$HL$l(2USa5Z1hMo(7gUW9}e_`p@6| zme%jAu)atcBFX;ImpsULZut?wb+J3F81*?`Xu^UDta+s}f<(c&y*c;DM)a*Tb0nq! z;NY}3Yr;8C0siRzk_dPJ<4&&+_E7L0HZadQ7FhRSoY<$OoSF%1`58>|P4_~)!4F`B zp%S|B|Fh-f5;(ALeqfxcxyeJ`ACST-5yZ7K5I4eh;~WZ}drXB@wWR7m8an}Q_|(69 z1}71|1z2>`aAu=XUAiR#Xw;>h^!O|#;@k%Sf>XmfPu_{Ss%yk%0AdBJQTBoU+y}{O zDgK-r$q$?cw5HoE&o&4y3%|yZwuK$%;yH`?2?L>FlB)fijO!>4{$t{wZ?6-((gfF-4?*jeh^$&~?glf`aG3UF9z$&gw&!d z9!W*-W<*>8i^n}mMuH2#(+IFtd7an_{3em$kSn<)q#i!Qa};q8B!+Re4NMLG$1bx# zfhhG%?fzt1T)EbS67n;(v@)`2Lud`_ZsSoCXng0oeJ%fQ7hDQhGABb$tON*50qv)^ z+eLm6CrHcwYNNYS5DA!Y$fZzyx_ozxsy~1X>L7cAb1uee_Q_c+{v6T3V=_AeNXb%3 zbFZoY4*Fi=s3NICt<;8~v6N|u9!X8e*fX}^oNv&g2M4YShylBb#gNbA{h~$uo6Ej` zVfNhC2=V zA)@P8OF}^4hi&B_z^4!lpr-bzGy<#{FXrThEpmBdI(~5km*IcG5k3kd;mD^&%{>Zf3+C4QipiL|8`EMcsp;wS`3%;01qb^cPtp; z9Am&AH+)}FNgF#hS>AVkZncR!k`=MO5S+#WcoutpBor`zVHpKFe3`XUKoFOnqA>{k zA~k-gzZks1GW?S;L3{#sfBM_^pP4YmdmqsFceBp1=@h;DpItUK{se*so1I6dylc9m zUt3=T?v|MQ`1dGx3f!~>aqK{GltzM$OmUOllk(h`&OPeFbjJN3?;{6b$3OS)-@1@A zGPge2?p|S7gkHlm8v-bB`!HqY*^gjJR_A)%(ug3MhfpgIwplL#dTdsxe^vWB=%~#x#Z5+p|KKT5bV*x-*m6yf*_X75EN<&S%P2b-(c3g=7thU=iaSOh`=jaGfVY?+B z3O#!Gxhul}bi54;O?nt@zLD1r(!6oiHOK_lg8N4Wrs439= zvN3| zqJvz1_2jHs#W_&{j{&=g>Got9bBdCzL7)xZj7t<+u?I2~wRks~a%;!O%v{PB@EzHrb z6D#AGPId-L4w`$*rEPy?z#?822t@EnJ*+guMqeoB?E;wBS<6ra@Xj}*;^!%0G~wd-3%we z!H>k9)1&uCCx^1i*;XNQ{Z_x+j(-gL8?wHGLfnutyNu7UxQtQWj*4uh#^}pbjn00g zD!ER3di1~Ye{Sg{0$?ahJJ4A9q6R3EauO^sIq zh;|)fpy|Jk_gi(RMgoSUT=hto2MbgD8c1=r9T?y7fD9JvNQ$1D!@oTGc!3{l&x;QI z4Yr>j1sFF^8o(R>L*)NYM2_N@`&U^yI^G?ph$p3lZ}T~H29G(A7!V(nEmf2>>WHae zQgNn9(abX6K}&dz2)F?SYw0|@|26)wo`An1jky!_imYKFcz473Bt{w*#t4QssdH7H z*nK?8a}ZrYjVHbkMu2qjK93Sc*8_>2{sd$>Z($BNmRRjy`xX9KFfutJz!B6hVX|37 z9!WjnAOAMvpTY){aqPuY&ED8@_%}+z1*=NmI2k_2tSf59Nc(VAen< z+r>}bsYUbcl_!CfoL?|6Hc2lb?DNKykiATmDKELjZR7$PhN|6pxt)>zEL(PMD>&jI zIGo09yV!EX4{!n0%V{s76a#KHb`J%Jru<=q+NXf(A0DGUNlS2}4l3TPWTG)Fn4Uq& z?$(5M81+WGVZUfjX1|bqcbhB|^em9=2Brr^Jbk^`I2oga6k`}#_|ofcU@k6=Sn<4ldH6Tu~U-twmk z7ojz;(e30P@1Q-yLEfu(PVj+3yhjINyN3$eM_&pjJij-8x84fIZ&HtE%P_#j3L;YO zXB6SZ*8s%&Ijx*r?7}{4tz4m<@N8Ae!H%*A8b1bq>8~*G#xS8ggFyP4<)XlbM(C>s zlYV%R51N1b@ycDlm~eeC7oGhlB?XuPs1K5c^9RNwz#r?`)KigTr-pto;oZnu{v58K zuC{GJ!1|`-VhcVP-(7{-u^jTnGq?fdwCVB`B{@Qs z95!##)4wwy12eGfq8=$rx2*OXwxCHnIg&d$SX`|(yUu)}uZBS`iU1D$f<7}Fzdg)M zBj%e*?hM^5P-ShEd53l7AFc)*P287Ymsa3qDnwC%FviN|roUgl54#6};2WW+xi4l< z4dz&sPrRJb)r(d^&WErtEy~4d)Zu@Kkt6c0LJsf}SN+uA*zoy{55NVxYE{G5p8!_Q zY}o`04gO{Zdf+BwF?Jxdw&9S0CSz3l;GPTAJZyPfxRAi-4OSD=gVK{Mqt1QgkXZ#d zyfm)r*SP9!Qx}85|AFXHnM8u_AF557Zl~(sXud35#AKHc5+q`r@{Kn-lQyp-fU*WW zl$|I-kEA6lQhWI3{bg=or7gj{f0ud%84v_Yjgbn6e-&RCP(u5)@;7DW$=5Tgtk_dh zNFyKR@h)0!@Dq1qjYFOj(~2*!#d`iXEcOE?Xy@^tAcYO#9$9eB$NVc}6#D2i_b=x6 zsZWrIRI1cJT*n zke^2e+{o|gUX!A?)sMemA^@tsLlDa3TH0M3^AwSx3?xsT;NOz2gSmp1*QP|*!7!pC z(&+F%{ezo6pnzfMtDoKn!`vn%yeL?A85!%+yM8#h5RKIw^9;9GnJXR_?nur`wg=&!C zZ%N++6vGzg(JKwTN2VL`cpuIf|CsHR4_HbuK`Tey$cz7YmlNhhl=jYjB_R}W1C|H7 z-e3RW9?%dzxAZR?cvxyLbh_1 zSO6$xlNi9~5`YNyd0mP7C6H5uE2CE*XB!Vda23R3TY?7n{;KhOu5eeQaPJa9K>cDc#Nwb-#l`Z zEjV=BSwYb7Sva2D6Bu~+9H0D4EP2ecw{9U<69oBiTuD>9GLVt)z3L*tznUB`dz%tH zHrT@hpYRT)&(st zW*=L^OTQ`GwPVp~)JE?5G>eOW+(av7rN7{sJp2Gc1RJTRExv2HlJxY;A?9-{36b?I|sonbWZ~>MPc5)vT6{a zt!><*eC{v2@C*9Z?!-uT4wIy1k*o#xuHtY1`HuSo*nOT2l}OiLMMq(dXvICd0%<+z z7amio80yKcTDDEdWd7W3lmyilc2X%ej!IW6Z!<-^E?&IYk>g0zDHG`-iG(^8W~@N@ zBx3nncOzLe@s)@3^XaF+VZ~c%*;P1Ow@KXJ6Nj zT;UnU!j#i#^SI!CC{OGitQ6E|?{ zZaaW`=dZ}_d=^V|t$BkUC~V);{32JsJ)CaWUILAzyjV9=O(o~`AXyiC5n90&NZtFp zA^B8yN?6(Sc9*z#6Dv7~sV&BHR5oULh-zHZ|56eDWEJ}j3fXCF=!?ctun-%X?q(yP z-*PRWAWf`~L^^*lAbhPBmxr**)pHJ15FpCOH)P%ZT;=$E9d^6nk?OZgdxLBQU0*w6 z8lZuoMXC4%xg=dVg9y95^E1?ETCrw|$>am_r5bXBg{#){T`5r`gE{X8X}|)S+o;## z4^Or>+pX61`on89KCvsNDxO%pCbw5^`NOE^smUh#yb(7|H)UHSX+d#`hCZ||#-2)OAesjd0(yMQn7D(7@xz_ygT5-?a zlj21D#hOcpnsF^Mm{?PuSLG%Mb$iQ}tB_DQHVOrD*ssK}(!ux@m254~xYfz1XG_C+ zphO`{JUe!_^!;;iJ0qOe&UlL+!Cfl}T-@@>V$K z(>APgQ$dhG)x|Nw!f_SVOgcBo60?$tOaX7-%TyZCr#@jico z;FJ9qz#i{X=neS(ZqnbUZ>q0Yge=0+D?-5rRC@LR$M6@KtjfThst-=@Jig>=C-vfu zFR7oi*g{Y0t=nw!6OG(!lU&Yv^({(zeIxrhZl|m}#+yzx+ta$Wa-P#8*6o&krN6*B zcP&3`swqb=GQ)17QTdj({*d)#f>i>pz)}_Izmpc<8TBTU!2eca94&c57Rj-Uq5QPlkKW^t#x0%NSZ3)gIs`6B10gS6JObqrr+mqk2_UhOF@)`n+ z6FAiE%|L|aEJVePoqh0IB>*`Gs^mX9#xm~B)AAB#0YV;jhfg2yc!b2lO5{UnKi{{~ z+^u&z*%yJjtz*w`ckt#q@m4~u#z$#d%1Uqic%3;iJ>;I`fz%$VlqhM%atylWcO}nr zZ7hFTjjt8D`#nN@Mr^FiF><-({H*@Wl#LHR@N3ZW$2l1BP51XI4h9qNjenM9Kf{aP zYxwi_7bHWu>&rn3g+Sq_+WR@dq30A_2>UZ51_5x*6Q`}@NY(gY_vOK5zWFuY#enbj zqld4kH4Z}6G*58CkDGb|4xP+QcPd#n`Zjv4thbvsmf~gf@A4o(*o}LJu+h4sHtOaX z6I9z>^+(sVJmWj@w!6#>z`5VQNpc%P=j$j_e?e~;=%oDu9ih@h=bfdG4@XNQgz#Ej zwmxNBjaW7eM$|Z|e|=sd;bNCMSg2NP)wMjnQO&+L9(6=c6u6m9cfoRbt-xv}GTqAB zq$Vgda)RHVZ7%WCSKs73V&smPTa{>+TBL_b*t!9YQu({)cZJa;1tiQyxEsmRc*yyV$B& zn*xybM-u_S+OKLbn9fP(?md5VgF*M{5a@T3+duAlC~00??47P!*tW!O-AA3Gy4%-s zL%eX*pP>u;JHkb@5Cf*(r>;V>g(h$h`Dec?xBlpS_Z|#j>*4GY4Cb$uT34@qoSc65 z#VC%3i+M`MsB3QLdRQ_M-!`!9BwS8Na?SoALDcc*JVO#H#mtZ(2)F3icwQ6ZvpfJO zR(i!vGoLQ|Zv`B#7#YpL=uv%_R)uu8VBWB;8<*^~4RTy4&(r zVRGdQlCq##=`N7XN}=-R%~Huu?q=8#EzM`vu)$n|a0OuWgtFT%3<0j-vY87w8sDDQ zTOBL6YNAN`UN0GuVAvU}g=exlC34?RxeR{m@tNELK8P;>D3s#j7Mp?Ro@6-v1O#4x z&*|wcfI<{jM1gAPl^kR*Pmgbci-g1(n}Re{nrD2c+CqI3o^@_c`vZU+61>#Td zPCDuegn+@Q`bA?9nLq7kqn*foE4K!yeg(j*joOI{D}}*k$+h{2E7$Z!59|}1vt0FU z$VZK~H;}A;Al0>;tbFUPDKCB6s+tHG^7^K=prS^SrYwPX$#?)7t*jcqw?Jv1w z(9Rge))HU6Md#tx)MZsgaaWT(vWpP86wUBiW7e(kBWkLSs5TUL!gb{xKt%OsA`!pu zt<0&--srdz&yBigXk_D`>c#;{k0a{PB;q+e%2jjO?#+2wC&sojpfj4(Xfx)|z`%vV zaRKrm6RQIgC4StaNFIoj+A|7}#`lMYkw6L6Dkp7(|LkPr6M#sn7T9_4Rb^HJ!+SdA zi+>ZnGX%F8!6*{D2Q=8$Ded0sXQepa3kweR_= za4c;9Sx04bSoUCdS!-q=?QA5t;fleFzyHhO;d(0)5nDQQ?dF33T#K@*4_#Ph&y9`` zwq>*Vhsd?@*zJPJq96R^7J9bn8hoqo-2lo(pE5$oqagL=&LksU!&1^fh3m1OFt6gI zUiruM^@Yf_X1X`i>jXqoA>moij&X?1*&Vl;fj7M))8VgJ`(up>~EssEAux`;c3cDc(d#Sxi zf1W820}Z}%tumyn@2QjlRGGJpCri2`Cg^UvKhFuV=F4Kut$c@dLc|Yc0ngr?!R|R8 z!iV{$5%^=k!YM}m4_*<#E6H(0pmOJZr*r-Q2L3F#$-Ieem%SΞW=o#_+!b$z{Zx zKp6Xt&}0P|uqVy)DC&{F|2X^c`L$9Ifz>sQplVX_Pc$k?p$eRtp0{Bl zus)OGAk0n{|5N>o)J4DUMDAcCz{&sa|3#ACiqY?!l%9PCFdNzX!(pGtW4^=m6@hJR;~OYTol!RUTPnXyK4Hd-H5iFeO> z0X&8qD(567AORRNEnS6lhSMmVNC{ft4$dQCAFOk*x+m=o$}*fSAOHjK6N;A9eh?t{w8brK z!3$=GEC*y-i@&Z2*UX0=!RpD`kI#u7Zxb@uZHB_?ejf~njgX+~GpTp4MFZ?q=12+c zT#0{E48;oPszqSL7NM)cN2MkJJSIMpSftdo5-FV{Xt_U2{xPY@(;DHT?JYu{Y6}s! zTRNx>mYkrVt(E#>7eVMw_U85di{7TD1mAu?RwR)MUdF^{f$w<=E%-z4so zBCg{g?fiK5^J|1lgag`CYmh3;&SAM)VC(p(YUmk~sR6)pvA^=qVq}dRR-`lO{ocaO2B@?4RE|8z2GpyRionWeq~twkt@?a2xjAuir`nw9nr6 za%M8~`5~75=A;hT{23V;Chi=f-mLcl1v z6fOrBigP0o3j!LUXx;fABl>@g=>JY5`nv`~<>^T&EMha=4dDp^`Gw8a?L5)*NPZ9K z`LAA|qicwM;Zb(HGb~%Q<{1942BadL7aImxOfS*(s0~;qt_8s#&VU2(>o+<6PjV;3 zz_r|06-2rSm#VRV)O?brUkUxdQiqPq^1UUTQKA)>)g1H|bg})QbnE2DxXW%-+5Qaf zMk5q5H7qdR)EOtzTD85%8n7|l{6I^OkzP@=4!51V;7E|? zwVO?t5KePS=jHreRhCnYeiIu)N))u7u`z{yLxVhlOsZO0s!RZ{=#{>bL4O z<+kfPUH_Mm%zthE631EdH^oWwU7%9ksVROPQx@pvQAj~Km51`pbZFPYMS~LbnnO{F z2ZYe>s>~V_fcTKL_onXyW)+n}^C&GBI}J;4&tYSp5#49;k%HqZ6f&`bb`@%~Lw+1P z_ZT&EbVUabEGC)W8m~94&kjQaU(*HEaS1xR*P>0Q_RBAvSd=gCksZ1VfY7M{b8d)VSsw?ibEO! zlL=3Vir7(Fc6!ZMOC^Yhg_b#P;>3Q4YV3AF{beCx*a=b?2uva-4U~b03&bH2O0{qCTE{&r7%>%H&RVTsA)|mGnZy zK8LSj|GuQRrPS@K_$F$EdS8y~%`Nhvtdcvlz|uG9j_fIfBHNwJyH z1#F+Nm_&QwsziRRD(v^Fj23!Qx=MOouIBT3a_DXZQ&=6XH;`Rr6Ke0OP*)9SG73x? z_4{9p{dqjpZ~H!umv)j0i9!)V$R1;DRPTBXJAxie_>lg}S zov{xyhVM16>UH1u=lA~O_n*f@dN9xHd7j64oX2roY0Qr|etxCa9DH!%Oy}$4#Gd^t zy!y8^@-?1})v5!W?A~Hf?;8D-RsO^9>}lA~r*hAe@3gbewEIvNJjnKklNf4pI`Pqy zQWUgYcbKu`thG+)3wK2399ton_oIeh!>h5ea&IW!eUbb7fJjrB!edff->hoW=Olyj zQ>*=bxo^DJC$;q*XnZ*bIK_GTke&a``TzL})D{x2L{ySXF%}?$67iBOQIt%tR1o_nn|5v+JNFvR(St^U~0KAGziT!u%I|NhVCQ67q zAt}7~oKF8c@qeMDu8R+Gw+qBQ#FzJq502*wDt{1(1;j2tvQC@M_lW4WG?7zI62dz6%d%w~ z5OxL3wkg+$*8Ex1~gXcjoCnnB+|6zDIB3 zGi6DyX#d5BY>b>8hOuxe;Ikqn}TG~pmIYL zHN*kk65L4Czjq>G4~u0q*lH}$uM;@KL6Rd{2FH8 zy$7DSWy@V2r+wOiT;&P3cRnRe#koq!cl4@l<9SBJoa>6bJ)C&f3UL^*Qxp^Rr+w1y zIb^;<#<;HDu)dY`GEz(~S6gZj4i!Yncz&E%G{?cbu<@Y9OIqmkPsbYh5jEtZ7})Yh z3ldzcM>{{mVeH`Z5kX5J6fV^^C|NqG5D$m2{m<2(t1it{hXnZAo#t*eG^%&y*+S-Y zVQr^wnRdpNikjl{P||p@a`@DUx%c9a7s7PsT(4y@Orv(Z#ZDLTZ-w~}#(4jX#FYhu`z}KPGKllsv5gNrbHSwMEakjV zb$}v+!T_iC%s>4}q)@NxV1G*>?nQCU&WNW#3pqQ01AT_`4Rw$O2)9gkIH^_Yp|_my zOYro$ZZgZ8nfb*{y5v_RL>G4?HriqbO4Z@-;njiS47?NTy%5crpXAV&JSM+#d)Q4D zcuVxEiv%3o&f>VteJ6LRvLC&>-0L{gU6LV_1NMSuJA3D-P21MtY0YXgO(0(^mYkM5 zSC2Sz0@wBC9Q8n9*Q?p}Xro`3gG(5N*M~~g%`K73Qhj+tHTz~BJHw*^!9`UzlkCb2 zq5`uw`p636K_6q0^xQqaBA!Y4t)`DrnI7d$tekSe8T6jLkJ((^zrv%dd%Pmrao{n> zc)qdk@UdSm^F_k*Q!b+hnb>^$+(m%}-a5_8q^i3JQqp`#B#60GRf8L|qDbxUYYt;dL%@a-~EMHC7hUKYb)W z(0|E!Dpm_Xi5^auans(nN_Bn!!_5a!@V@Vpn&c=q?@LJcI~=ERdeZZOJ?q+C$p;xaJjDeXmB50b66Vx zte*2(c;Hz{$A#roj}aZmue4hCSPS)?`(+(F*w{>u4vQbPnu&Xyf(e~3vu}9_-_yv~ z67&D8Kn8U^&TutPH6vh==y|x0UK`GwEvj4@fpm#yu}WB~euZsMAXz17xpjx8O$=Qg zrMZLOLMkH|9ZJK)a;?DAW~`)TT*&W`tgpXXXw`W~Lra{nBw0(Wnk3y>jqw8)lJaTeoo< z?epm-Pvf@$^p&{bd`+>HwL0@F?hO3iHK5}6tWG12jxCLI^VC={Gk4@q-W$N3@P{j8 z|1StICMk=qG!97uG0yVzHLKviC+H0ufM<7dUm;>q;A}}=rTaf9s=2$zDaCt|wv5i~ z+VdKnfD)OPFZUKTVPw!Jjl=@4EV!loY?fz^muF^t-V1MeeTsrkjPQA%_KZ;KGV|TF{btMyYLN{>$4fG`McL7yvFRkb;7)k$1i`-l_K&6Tj?pLfImYL$~`0*3f*d;)Fa6uF0^?r6wPua9|l1*mkf zPUKhOnr9LsuL^s1Yom$#0B9(@v=%BdeUkVgQ;=e%<^WLM7_F(#rOAPC0>xP*9n+d1b2+Ku0D% z(6^7Kb(J#$U8Y{;eOr;i^SkUNCz>MFo`JE_%WZtWpSN$O*UZ|PmN8ilZZAF!e>n5m z2yr-&?`DUdSWQ+-G~lXt!-|Aue3ABlkOshi*E^2fi1#lOX*;Yd{5dGkt&MNJyY4pS zu%t3t`tl6*vHtkt?xafsCu6w$;n7)gPgX_>2Mt09l8eHd?;$6%Gs*$5GB5c-Ca-t% z6R&}IN*Q6-=Mi3FTeWWMB<~}J@N*J|iL5+Xpx~F;BE{8*fFkEn;46Q1xC&#>Tg$q` zUI;es9CiA+DYoVL_R~}ezC3LGTH=9_ruO9a+>!5g&@1FXV6O4 z&LWIDk5BmEi5~~`?3y^MryR?ZN8kTQG+OJkhfd)%+@?ymvYa0%I;|u8PMwB{*i5tJ zD8@TtFYkJ14a#Rp7VX??2EeA9bFvNbWXcdGO|4X4Y7nKyBc~=%uc){cOR zxet41QV{dh$GNRCB1L2}#n{&+%1p?9=El-UV?C$SKdO%}%4SV6?UA5y$aEh}H+s#- zgbg094d4*I7o15|o_=2e9oQ)Ck^=co!*%d$8r)~@>AL((QX-Et#w(M<9)XiOI-n3+ zS*^gmC(Mb5!Xk6w>Sq3M&yi1_;xUOTUDNL1Hc-x<;exm8#b6m`*egc3;}mzgt4VU( zV2tgKe$D+WJo<~P=@2(tj|uYe`JR+fo%a7FEoMT%YwvqI*N`5(_E?xI>%Xr(3wZ4> zIhanS5m!^+FDu}~DobB}OR`ER6@LoCyF`ccm560Y%`izX6W|stbHj3|pieix!656r z4r9c^3=ALUccFzdro=EN;8$zvP8aRdcT1~t$x%wKh;$ieG#W20@}Il43_E}IhmkLSiKD~Kvq>r4 znQ8Ep!`(js`>KC)e;QtTNR}jId8^t%E(zvHemg^;2SsUDDOn9G68v5V-xoUnQ@Hj7 ztG_=yLnc@uU5Z}*!6sLGBkzy}Pn{9hNf=q6pexZuh>mes77l_AIzIJVuEBO)3B!!; z_%?E18m?z!(=@}N?7;h9e(UtXArog)G0{=mM|qN7IYBi7s}_X{LCxx)nu8%qxGtdN z>*jC2m-rNn5;|4RpUbcJhDV0(e5ErAo)R8-9BK(5NC@fSI5h6IzgU&HGvtRJ$w+h;Q}B~NFznmu4$-7 z%;wge;ySacUP;9S-=js`#sA$G{Qp5|?E7T}HR@H`oBOCQoC%*O6X(|Sj zQ)lrWT(2e?WEJDN=4&J3r^&nx8x6EqqME@e%5ljjD82BNKu)CVenrr@q4U7?*NVLo zTitun?P8N4Cx{G_yjV8&5RNflMw(2Ka#&VV7WA4s+M6aE6BVz8po-gZIe>bf1$GH;UsSZK&h5rJ2liml3y|M zi!NP(e2ZYT-t{>QqRW1o^$lhkT)qh+ z!Q#ra^le=m@yq$9Yjr!b*S|^qPrtbTkVKl_cKf0tH^cd2DD~`L2J3AU*aYp>m{qF% zQ(s0ghhL2VkozEP7cGi%Z`J<^EWOx*YWz8*JMFoe!}-YG)o}nRR;>ZeZ_CX6F|yK= zu-JGD{nZOYFV=4E@wR_;$862M{@Ci00b)bpZi$P23dbr=WID%V_9xaobrdS6eA210D`McXB^sC$vEDB8fMn} z!8Af4{>m#yEg|?x*y>T&VkIH?`TFj&+Wv$$_18NjNzHt5(O5i)T;lV);4jXdcfh-0CoL+(ct0Sf zY|xlXwV9k@FaL|d1ln77AY{7AH$ImDR*|)bsUYg(_xInD#3$Wv#xsveLpZXZAV*Kcb8!BY=>?PNX)g+sfQ;d?q7Z};8B=(+H)mAC+ zwh_bDwbRS4fpZZf4&Q7d?SN7}6?NgRjPt< zEwP!Det(&5U_!64lkpE`deylE3=E4he~7CMmN0jNTP;b?gXGnM>w7?}(~^&3ivJY_ z84=52&I%I?(hcqi5MYd-IL({WHtzy{bP|1tj09`)vHyu9TIU-Jf)~F@ImA#&B6ARQ!s;Z?A_gceY*{BENJVvEzMTkRBa3~Vf0ojD@ z7}hf2oUF1D)F8idxTuvw<#GK$0{8Hx8M z3+l0}q~1;uu@z!uwu1#c|OLvGp!XRy)- z$%L{G88s~PX{QL;u!~N+XIa}FnT-}3@$pFm&1_N^&?R;0#e!B(NvNCgNQVZ@E$_kEO_#A0r@H2(E0!N(=BD~(eC z4c^gd(AF$8WFtrshqSHa6S#|N9(KlYzVv-(P^4e`Tz0SQu;mOU5pjs`;(U_MWCb-$ zdx8gSMla`{QF|JR=7s9u9>M|mP8OeR2V@d3OD~J`GDz$FO4~6j+Fa_=A1j5e(L5igGj&0N-U(?=zkK)3gmlBYF#dilx6AkY$ zd+$B1dzx;1PQr4BtdH8-!!S_^eUgv%Xmr0ncd-cErloAMyQ*WRksiWy+2)A@6e_jx zX4q`!eR(JM*wgX_p3Eae|7YuJTeCQh;v>7|qeL*4cUz$#QiJ{WYM^XNcky}s>Ty2K zODST}+yZMlutLz>R&6#^!dabhDU+G8oNgGMi#%<)Mo(HD4(39kZiO5AC5%}>GI})5 zTenUIkMO`uY2}y|>cTXny!T3Wj&^j^`WXs8e@A3p^}tvZ4(;ZHm6~u$<(eXjJGx3w zRyX*xYK4IjJcaAo-3$(*#Q0!_Ovv0(5ue?}ptHfJr_rk~Gd*m;>MB{>CC};8q4WkY zpWMwl0#qS$e-$T=<{t2SOKy;p0|?WLN45s$UbW#s;-?7@e2C{IWIdr%#|3Q?hi^=P z`E^DA5Zs&vysV+Uj`sjQYb|VOa!UL1u-gI>CK-U@5w`9G3%V=D*zgp*9#)VHN^hY3 zuSS-nLplVqz(s^5R~(S+(qhP5{#~pa1AQmW2}2R_p{U+-3H`cerdLb>pLbOQaW zy1Ad1d-+TP*M1omBgPD>?i9)dc{TQ0pLxR`aV@#z!HkHP4#Z)5aa?eb@5vgG#m8rN z+&o8lMe_mG;Cg~ohk!lazFnnuCV*m~aAf`KFU@ANnXIT>J*?pmqdmm-CU@>&gIA-` z76x7u%(eB70~U9I5%R&vHe`Zt!p8MXhOQUlfNgR4X*-1$+Ff?w~cd*_{ z<+!V3{^9F2+nNW{-qy8^9SwC|7Jf&>&A}*X9e;R*y;2=xVNi?a{p=^BMgI9|ri>19 zkYg4<_=d7u;`m>-i`84)xA^kr9C!wTAyEg*`%+Q!TUP0##AljbX#@=!8A0>j7{z$* zdM(J-uFo*ep+#}`n2iQoc1^LuiE z*=|Um#KC^<)(az{|Fy96u!;0YbA8o+F+v5;DEj=qU!Oo_-~do3Z?P)vOXAL+|<<}!lXRLFAH`uiwRC@1ko|^os!~Ep#H{M$x1hzSQCXQRR zty zP%_Hp=vKBES5XOof7W||L7$VT%}D-@#hW?&Pxt6T*mnFrh3RXSz0tXFeV=pWbqs1= zEZ()}hgeI{rFI?Z2^!?^V8}`2y5CX6hs?FB_}An~1q>H?`=xZ7zBLi?4m# zpN;lHO|s@|dFCRrKsGL3cb*bc6W-EfsuUsZ~u}QY^ z!qlR(hkbD3^_76dl#5_`nJ`>x6@N5wEU*$)->T<1Q}P1(?sRZP&;1jwKysrTXng2Tk!Ff8WjK2?h{d1QX_7gc5+BRE`~+0z2{ zJ)TJ-xrjcIk+!+9RdB0DmgKV%vYB6-{+rUd7zXfL>0%BBkRresp(^-yQoC^nAnh0Y zri8SE<qS7W7&F^ zJSU}WG|~k;2sYInI_7CMa{b8=w@9bA!Oj)b!RhK?wR3^7-!yPN1)WjiQm_~9^S zE6joTwa7Kp75aBjF^@lV+*DkVvbbyB|0PHz*)xLKdp#r7RaZBuN31y{`Kji;EO3Hy zZ&7wEeV1YA261x;r4U+z!)u!aC?21N(bzFgJ4 z5m#9N*4(?b!-rS6!<_~?IJz1}7sPPt)ghh(cE98z+K7fj(T>i~vcY`O6-nim)L-FT z!Qq32IvC_fu2&osR<90r0X<@x?HhRWdSqlL^{Y2=2HI|> zjK`xZ*-+!*UMBxoq{G)nLz!!`p;w|QM=F*EJ7HhIT&|xysD1^Eb(oY;DeCU8tghbo`J18y38>X zr}xH|w_ixt>QyhHK2i^MtsCrh+s)3JH}Kt`ANd}O zN>WkYMGz)9?L&qg1~4V&Smv($bZJq`<_+(B^;pRz!q!Ng%lsQ^Xxxq_oE_r+X|g>c z{e~OLr@GNPcvRbO*OBgOkL1!M?uGUCm-aMctKmEzQ(ivS$!eRvizS0ACT}=GLjM?- zjIV&cEBGRkySZ3IR#ZP-s&SBr>SJZV6l)IG2I}1ojcbk;PwJJ}Ttm(*tF*A)Nz4)@zmRZkgtEACR zBOz~C?z4ZRyORISb}=V(=%KV42j!hK$4x(~eUH7_rp+I%ROX+)0o$xaWy-MILvMQ5 zJvUh(1nxdD^e_T8Z;J>86a_qD8}`O?CMp4rI=&mIMq01%W32J+eN1B5hAkHt?LnZg zVv{Z0RUl4}V6o7`D!8hzVGuAqsl8zQt9~L$oZ3jv42fs4M(jZPhdN>Puk5LfWbtKh zdgWANlt%6Be3g;y0o*=)NYr!a_R>I3x!6KZqEW)69aI`0Tt8Si-hp4)XivY}!s2B@ zyEazLXeB*%_#hW*GRzT^!0VFC$yMGQ>mx;|RGe2V11g1b_r-ebya~7J<=$t7x*F)j z9M9h95#bd+&o$2V$Fb>{Njm38l!PP&a3bN;KKP`I?;~NV1=>&Hfp3xf`-w)r(Ff+) z5p5aJ<7uWYy47%ve9gy|c^ciBBhvP9*Vm4rM#z=N{clrhf@?AS7!%eUjeXLjd_F8S zAoES;f{s#pA7hBq%FXG8`~3B-?EyV+Pu;`}CV#LI7Bu48xkd~x9E z3(j&IIyLsu&k8kpTBT---p*|?TwU*vyq?t_?&9lTv}5-wqO6J}O2j|E$z1;{6qAj3 zV^s1y<6OYvrnIJg-pi0PX;0;aEJ#ZYBu(|>^f}LEPD_+9!?I4jt&cX^F$A$KS?k`R zMonHuWeB1$h7d$oFMf+--QzbqlL6-)jGWL$QSZ|1O%qJ8hJzblrZK)QZbd(KJPig5 z-{(S{!6bqRJ*o zWmisPseJjIB#SL$V7|9!6CB+=1(h0!bP+6gIvA=4wXcE$xvJ9gP|?~OW^}KQhSf-( zsok!31p8HLZPjeklfiGuP-pOr&gHaL6%eurMo~l1M`6u_a*eC#K4r{I!f8#k9?HNR zmMrq{qp!5orh*a2qc|P+uDw0N7R&W1;3u~u8_rHp2?u1W1R2hu;>+HDg5 z+096KO>t|1MOt(e5g2NWb$sdtfrWf$JR&Mk7Qd8mdL4${Mu>J6K9B)djFkh=n&;e z;|^<{SLhmR@~NNNM=E^V~xpdQmOSa9JLoe+2HUKeuu z754BKRJv=Uo!O3WGxR_D1@P4#%Y7hc{}#vn2KXS6m$}~j{q7@nP>ckMSL>=lF`8;& z^ij){;_rv;>fB_a98dKgc)TQ6+pjVVh!A7fE&QqH&*76#v)bPt28c{>#vGHK5z7Rq zSmxBkagX`DVOW+-O`y@NgpNpwZAJpbk=bYp@$^z%h` zN0dCl6PeaJiSk28n7X`ofKFU8lzB}O%;XDRGgS&?$dJ3|a6~5ljDyULF`!3&)58b2 zLgo|nY(-=(IZiV6aFxIaI6pSoFd7Gy>X&HwrH2f4z?R4$W>o?BoQZKG^C?pc^ry8E z1+$G*AkTc3r~P0Aq$@89dGDRIoGda3wsKt_dipJr{^bbD(nTWa?nd%?LR;OJL#cj8 zXPTQnTn;lsBwdc$zz)!OR**SGoyHg^Ak(c4W_NsX5_i9MY6FTE-Md=8P$XA@8#a)u z*upf&x`wA54)|s5S|(=GuYTn$cO z^CO4gS&kmczK5%{OOhV=&+u&XKBin<{}$`|A~!w%5z9eCp|F#%SG0z|36XVRkBya% z6#Gnr0NHch&NSC;==Q)=S}>#JVEqOw&?}?duuD~?B6B|7d-L5ernD=3*AY`2^k~xl zCw=o-%|rzf9RyZ>mKR2!x&WfaL@si)-?G)`hWahE8LEwZQ6^jJaPQ?7ljQ6k_1x6M zpn^-@eTjkv_C}f?wvTefCkDW7*S50S;6|dpXS~BSP7KoJjGY`BE1pe9ik132l)rbJ{BT{?Z z@Rkcoe|TxpEfU$7>S{IdR?Z)asE}c45x-Oq0c;9H8m|xuy5&VJYESiB$3vTAz)RzT znR@&!94Md3himtMzC1Pnooch2uaRHXoeU9O-+szhe*y|(e`0_cDy3|q+L{|Q34tw) zk=jJsEb>L<36%FsqH%S}xM+Fr*S}^RLXqB4mA=cchM82fU#*PjAMIuYVQCFl7Hkv4 z|0VYHGFW!21g|(F(2e5T^{o`TIG=PMra?Xga`)YNiQ4ol1yC+VAq)G{6b4|^+6@zw z2Uo1c+DZe3r+gLQyeqImIF()XXcr{b?_g~FOY)QhX4K44-t zW2$mANy8u2gCQCxyPbgrNP0{XbviM8If)o(MuXKM2uo{T!DFV)GU(c?iM|AYZ69Lu@pN+4H|H)Fi;R+_d(rXJ9 zm%-h!-Ilhq`gSGF>i62qr~v+(scwd+`*P7G)_kWSGzU_3{>ZRgJId(Y`WeReUD6F@ z<9cocb9vdl>fhbZKd0**0M~IA8*Qmk&FaRf|1tSrBpF2)EhtjGwdJ4C;P3!^zRcTDYa)G zLellRR;Fs<+)`1}f?^ymPXDv~_rFexfC!o1yUIHW_HbA8qIsy;*5~?c41j$n{GggB zIB^*eEU*mX?TNB`DsQ}ETu`U*Ug_9^ss+JN zJ$PdLe}mva%Bw&8R!jSS1;_TN5a8;5;>O1THPkVCcbVG0xn=HPj5K^ zmf|a3*e4w^Y|1WExaE4RpS|3U$nHH?m|6>IHd4mAPFlG zT(-jqbQOBqU-6g1&!QjgHoDdaiT6t8^#n4Ce}8{dMsW2I2yniSo2OZhSkK%KJcH>l zkSpP^Hq-p()F;VNh^Zf?xn|Wyh4YZ<;ox3(CNU*Icce6&z-z!x%cZ$put3B8X0%G# zy1dXSk#`d?Ebz&`3ldA{N78(TgTP=B4_>1rWohC;7)f4hSifJ!`{8-_LQhK0fs19@ zg@$QSOlYlqzT+8YB%w_J3mgJ3w2C)hxgmE~><~{dQ?-Jt3l}87zvwD?nZ%v>yRK<* z|EX)F0{X4LSz%`G5O;HZYX}$vGhL%qR#4M7vv)RD^nx=hf~?Jrj-0EL)BQ;0dz))#L8y0u%cP=Ku&j+YF6F;e!12y z+#~LVLlrz&mWvU$2L<)JNJA?}ntOl_bcXW;755*}2`P#~$y;pmIgkxH-Fp5u45V<8 zaRexnoN*)p`=#sA4x{x2BK;fG>}FHVA3mL|;dol=04^u`AEz)GDOci{*f}_;VJDqc z-u(radz=7uLVF7I64KMCh3r4JoJd>k_gU>%-Tv&y@WW!N=ADE&vCO0S^T>GHHm)!+|}x|MB2;AGcD0k1nw)XnCroVDCoX! zZ}%!l6_cds(6_@|!Q`z>%+1fR7U1#S$0~;wa0I#NoP~x*5Px6g` zQxt1mUF*7249AyAKD!m1FaC3?mu&_>j%8nf*wX4~GzR5KRxoN(zf#5Qd+=E0-c6qF zndwnR_X(gF@9+O;oGet4HOrV*@hislIk`w}Yf!)M$doBuX=bUbc&=M;@L@a#Un_GD zXgzb4KYRzm$!?6*(@I4RT*d=5I7~J$@6C<})$$DC8(g=6nj(XqhgjyS) zAJj5UbepWKdh$Kijmo2HM$srEgo#d1#dq-O44?gamjvZ}_0c?Gk2g!!hdma9bXrT)c`cvgbPpzIN_^V%$X zuN3Z>eCtj=myaMk0vk*^%Bsa@pHx8{t#4!M+9ehs8_N&B&i`c<-A?#M6W3w10l52% zvL`sjJtBsF$)EkTBO8`&*r?YdCZzhQ<%@Q%NyPdblZ)BN>i7tuvIuQ@-VblN)Sp#9 zF7l7QTGD2EdK~j8J^lpL3y1l~sm;qd+-WBt~qEDZ?*M6k+*$BEcX%6(ySMgi<( zcnWL?iH(J(KKsMIoT?vRYn3iQ)DYY*`yNcqF$M8!-+<z_ zK+k>lnt43+;=>x29rI@8dWE<)Ul!-{aU zk^2J%VXIVI#44DEqnK#8RPjBdMtZCzIYH-y?(e;$job7B?DZd)e6Kw@?vr3flM!$> ze+;vdC0%tiK9I98W&t8(J%PSs`eh_`Ix9p1zu5d~`I7sO%pi*Ss>$W#VUkDDhspD{ zeWDU@hqN-9Oans;$?WD1Ydx%$6ZL&X^m3o}Kzm{6))eaxEc|SBD;;y5Qyp}HqoC_bn$hZNAhVdU z{b+ClfP{VG`7U5GVh3ZE(29)9!(RK3QQQY-B|v^K$u`F!UP(U8KQ+#IL9o#a7xO|U zkUotCL1$5Y7!7Yl^@q?pa1Vj-Ls<9)M)7;(JE`EQ8Z}Dr@mta#_D~le6emE&^cJOHNZTouwimMg{3 zYin&qCV8h|6^5>9EK6fprdoFGF~@_wje*&gGu=5M3~OJj-Yo!4KcM&H!x_@9h&Uw* z|3f#kt)X6ji%9wIr1|lib*G9yM`M!(CO`2>_KGnj05S=KxVa*OdQyoe;BF4!#|U=F zIt`HD8=-Ul3RxwNa+kr99uqwq5L7)i`%KbKLhfPM4j>);aJzoQh*XDf^z=!9-(msY zOMw;PPr0M3!X$VC{5=453mEU&s}~j_>pDI@2o!v_@pk~gD>Tsw(zKEPdE(!10C;@j z9Jj!w^?GIg*um2OD?{*@e+@_)483NfMgda5p82Z$8zr8hPR(w;Y+A<84P*mTmc6|_9RF{-YFj}ymI~VYZe39f*@ODUFY{ptld^?6~$>d z!IUQ86s=wSLXrCV>r`#AVvla61QmEaO8}~U!lhYYaq2~=JBGSP5ri{)PIEB%=>uWX z*s~1d97Rn7^RP&!U&5AW(7>hnmBw|0%Po}lqvd37<@ivYi#5HNm2(}^*(BvkPYUFP zI5H{E9}Xl=PgKM=PQnlUZ`R3#T(phfgsxEx$pr<%_#;RqKTiX5aVa zed2-^^mQ+Q^guVO;{06#$SF`Wtd@dTV6uhop+iN z^WJh0UN0&y#N2@vUN2t|t{d8O_?0aaaz^A^0z=#n;Ai7DM_)Aq6Dbe-x=#`AAE&A(aPlbRY346@d|SQZ9=g!v`Q=fKzzA3o z6`A$u&Gm>*-!&gm4GlI2iD+pV>_81s-B9`}*#oOla%e*#yyP{TTHfdAx(_T;^Q%t1 z4hgNv?<&H7uQ4o-Ly5RoVLNt-Ufb+M*kQUb8LB`>uxCjoB9umj7_j!My-#_#;1bL^ zn*+Fau)HVETu^iqC-x3tdA;!8$@6G=bccXOXZd%lb8A+@c2uoR4idexSqBYVb~Jhs z5hq~-NbxIw5Vp>Zsb=+~kh<@tY)>BfG*P*(#>k((;1roIY!e`e9X+2$eGqfrAQAgM_TQfwocmBhs8Dmrh+s#6$~Eh zV2#>-QcX+^IF^FChcwe+&FHn-s#l_l{9e;XPR?f z0~ZEKi9J8*^lvaYPL!ix>NI**N>~SZIL{@Uc1>TTyuBy9PwS2cCc&Il+C)gpn}tEoOClu)9mKs?uEV*f>Tn}yphH_8 z>eFki0|qA5CSaZzuXTcV`|i+s;E-;Oy$UqT!nHvHbRjZlf3IG!nvAy52 z!C%RfC16Ho~KQ2U!@9-E@ z4V^2sdR|i69_|3*Bcd}wJkrlua=Hnd74y5lmZhY}cI?4Q*3Bnkc{r7K5euJ=k}xI` z8>Zfz#rYi8o;#&d{V*8s_3Hck3;~Pv=kSK&cmlQ_y3n0$b72LQbcV^jnLhAfpD%i( zaB@$4c%0dS5%cl=bb4&zbEgOftuB8eVcybOPSHz5(Wco*;rr74i{SHdi-k1g7%UFt zgjG}?$^p8{2tfW@2I6jxo(UTE+;M(hVi7f(WQ?(9Js&zdX}Ogs@zo9i!g^-lDoO35 zZ(}crOe@#}%m9EF`acSOXG>l@eH>hYP&nbE)T(01BXHRFs%-LrBOMb{OOYt{wUU=o zH3-~I@y0&PvzTwHkEXHui{amQ@PA%Z7QSY|uivksA5nk{Sstc|{Hyf63nKr?S>@XK* zxlA8{o4&C5Go0SAiwGo-YS`Q(4*bvhYS|N~7&JHaor5XT=q779z*g4 zMJvHxuC?o%A>WsYC7zGJ@?YH`AO&qaAVBtMJCXgaVee|nF!Us$LrwU59y+O^V!Ocwaov(b_pZO^GAH#OQj)Zv0X1vHV^6&%XX?$nI# zz4FvOD`}p>%H;hnCsz(0<}iOaNZL{mw$|hzxaFZ;fKFBa^!e+HwR(U@v5v5;S0W%G zLQ_?Hm23-#j##SEI>>r~0Gq+|F@2DXGeO_pnpCB05JP3y6JB-YpKZ>~Y z()k+hwI=u6Nks|QZ#-`w1}Cr&oWP2;TBjpcWS#4dhfy6N5ec+4hqW8u>UvitG2%kU zcJ`SCV`_0;MQ3!|+egkn1R)%*Wa+S-@kzj^Cjj&@p7MfteiV^Xh9`VV8+kqpGQM{i4Cg$MNRLfS=^7FHkWGNB-X_reeoooN~P)9M%A|#NsRd%}9AeN>90Z zk@{n(LPP3D!JA9r3Uvh)hQ@Y3N-|h?zEJReZ$qoJLNO7dbv;XD4@>?XnwVHn*P7qzB``E{{O$Jlu;;= z)g4Jf9F%boDI_YgR~aWE8QCKZqhyb)LyPPT<6>KZ`Wbjvn~Fd(wex#`FkwyTB4e zHpAN+GLpltluX*aZX2at&U9V!Lfs2?e~nbdC7@S2W&?P1#c(UG5SuGpUFM0#9T_y0 zfDSYP14(H=_7~N5zehLEW!|(>JR?aXtXV^2LgI7!9^Vd8FhLEstFLgF=TEED&R%$3 z-Iq-Jdb)2|LbpK2sEV)Qnf)YLRl?QG(T<29n=zy&Yji68>3;h}q)tfLUV}BaW=!8=b)fzX>Fv&DrPiK9;a-0)W9rjHRbv^o1Y6l>9lZ#Jj)}JMgH!% z^I3%^HsW^~RTr1^#5$h#-18_1hIo-t!Y4)zr^)0@=M`@8TY804!*xpF&j!R(bUej* zyp=m2I5RRYwK{#N@7zHkgwWhT#mYp8HVB(@$i;^~M zV&GvnV57nI_}j%Ez+x}`nY7rAbX=bIM5-MJFbwLPedt0wBw`RyoWCQR-jNr%r(<(V zQ7z#2jH&|OXin((sd95(k_7szk?EI{gEPlpH(Q_G<=pDrQmt({<$A8K{`@0N9;Q$~ zKErZ@@O<{y;W$-|dt0E-Nt%)UoR(b3aW;V)jkhE_8e25l#7yH@q??x*K&omg3n z9SIjO3HQ@07LnY#cVBJHSEz!%xsqahbUM94POUKX{5Bf7N7r}Y)aQp9;tX3-w4vY} z(>#{@4Dr8ZzP=xjjB8E$%jn?{*|li3Lj%;}*b}a*;Da0pzGalPdCIoMqb;MXOo_ek zOQANW^rL;dZK)dx%>lUGlC~&nC<8B5tmSRBrfRZ*2_nT1#I3HAX&tCQ4Tg{De6Qv z{-9WvZ&(wDh97J#Mq?1aC1EPRe*5cR|GP5(SV3wbYG;jayNFFh{!ZwM9i;n}*K`@`V==vA%Fp!Ah6{j|&j{}0CA6>&OJ${%eQf=l z$spVWvizT2xPwEfm%4AQ6&15whYwFTSp2mi7JrHMNv>XerFK@O@oafQaMZMf>Tq|E zbLUcPO>5$%`QE7Po-9ko$Js1yt z!HEoQAGehJ>``a(KvZd}{C)K)>d*O(nT^>i!VP#{MW0}BC_c9CtJU5{rK(rnTgG>- z_*OQHxsLr<{_G2bvddnMX+soNKpECx9G!J4fS|7T!9a8<1UbS@%sQWC z2OYgspFO&LcR$#$J1T9eOj7&-!2eFDU&nqpZDj(2v$9?la;1lT#lDNtG)=p%Oh!*M zN0>X9S$!V=dO1Z)D7Bp}?|DZ%eEmav zzO8GeKk?`y!RTMxX>NXq@S@VK#L9Pb7kqqT9jTud97!f>tR^W%os+k+a4fzn=5+0uQNQ|358qPVl?mGD+iV)$vY5fv zEQ{w>`Q}OLhe7A@wzZOrKBhx!P%sM(H}X1v;m07LzP(Msx4cbM7=3JjT~Es6h4s+o zSWWVS0s;FtRg9XXeSYW9JdJ4SqZv=cy`Vy!axWv1K6?q_-5d)%e$HO#kowthJn@TP zQj|*fPAK&=)F)6HK32yw>hm>K?SFivz0hJa+|Jfl5$daP@*GeV9zIKUa)e8_K$5=0 z$?Bq+d0WPXltkxbq1l4P(ZB($>pOF_*M7T+JOJJ;mNs%IRgMmmKpoz#nqg?4qr{HM z_`sWjMf=)x%eE<+G|A~WnxzfNiPjt&FfJ3^C^U*gKxpWjh`w(+xJB`6zzZhA9%k`C zpgnt>yag&MNO=WWtatn-mw(nCfymt;9%vtYVBM2aq~z<7TLE>l<_@qkZp5n=7IZNI z7`Q^80vf-4Ir@c-if#OjL77Q?g5miOEpg_$bf|t{hplJYI4)K<7#FpbAE_u?f7UO& z(HW3r0oPwIYaQcZpL_T9Z)w+`nG`KjkgL}2k1!AGEMkMbw|w0x3UFMFm4H<$B@6#& z(tY4T1;F^k^`)3Jw_!yk@=RfPcLP4;!oa40u1R*~JhxytLZRX0I42X*)yfbX_Zu6j zrxbW8pT5QzKCb{&l-rrT#7PT5a03&AXHv~!qoPeTi20ab8(@!T(k34EGtDh2QJ9Ar zXXeCfB83h+FNz*Rz*5Qy%IZjvtwwbDP2Tj3X1dYz4GsVZw%heQu18!DOMJ4rjf50s zoB}rJz4XIU;0@>lXl($8Qfw(W_Xags^s~)(3!9TT7)5M?pte{Bszmu3v#-rts%N@r zyF1UEU0v~&eJc52xJo`6ITQdwKCzf7CR0i`xe zm-H>h;HCxAtLk!%v~_X9?we>G)1%B9ODx$z<_AD>%*$#NChvN9lVjZJ5AJk1F|!bR zLG>aeCT7i9gO(jYcB;F&pgMH3V>NU?5B6zZ=HrWnsiVU87*+E#v1y-r^Dji4DYO!1 znm)+y2l1L(Q(OF(kNlJGAE6*POfh=oqcbxZy{MWy8QpgVX1-ohx|23ZtnOKzN zt~vE=e2Pfbm^fEDn{%Ras!oTZF$Gcnt?3%K-5^ zz?SqPi8%i|>FC<-qS9KEEJYw$A}4`af<<~)^zOaJpP$?Ps4y6+XVW>z0P10Ax%9A( zZ>z{;uYy1#Fn=zsEUL_XyFu-f=9?=na3|*Uw%K}M2nge|Pj?K(6)g8CzJU7E>WqQ% zVp3{@?xM{`v$z^GPaym92h!vAA=>#4kK5r@+&j>YqQ`5{vs&)X>SmFmx4)RH4aQsa z^m?(AMBMn~8_j}toGwq0^E%TVKYQ3$Za{%CKf@UhfD~YxohQ4=rFNoAI5RNvl*N1e zJpHwZ>H72aK*yL`g0>|&=xx!`Ly;3XKTkV*C>l?1d7~-)<#5;P;~GBdBl1BWj#GX? zK{U~ZThQZEKrdagYv1>03TAB?6)O9n=bRrx6rKKPKUatdfP;n1XNFb#+!n`dGtjEm z@(_VBzbl0;X>WV#qT6wMJ1T|#JLKTwTe|i-kdKVztHvru7=?Qob(f)Y+j`5CkfOrT z^M*u{k~J~7Q~eI+;UVfz2HPP+XQ?(wLl9(s&R~hPX3F&j{tyTGGscP<_!V`68%WbhYc5SHR`zC zXgzcHrTKz=OSx&IHymLxb02$xs{=Xj<2KC__gIA{E>JKZXKBlcSgVwvZekSpp1*IoiMUz>3wy3Na-UF`GB)0);h!9Vl^DPZIWs%k_az63t%Y8P`(;YW|%W2{N}$6tJd3V{|28p& zAHSe~enBN(CqD-_lKrh8D>Ia#)CI*EYE}F88`Ey`x#PII2K15*(rtqsiE*gs!6Z-n zt0M-4HG4m&>0i#iEFH)7c6ep{L-prqw4;jk%wOa=gU?X+lU*_QNTm#4mFgd^)$=YH zGD6iB1y(P;MBN#BHrso-;yuNlq1$AP!e@|ZnUBf1QAu@F0J|a+j`6!MRNR)b(drr` z)Ma$FExCkTS(#}w58xTw9vkViS`zJaqxo%Wi>|6BFa{fCjAa%uD~hA4vP*#8>6~#x z>|1L7%411A1?S$pzgUp!=95^h^Ae{9Pp#&|i>Sm%Y{n%QhK^=PY9CCx7?p3gTXe`) zyvb4xraY;u(Cd9vAnD_<3LfGV1OxxXef6$w^ zhUW0VFr6K}plKG_fm0`x?x||qKR|L5!?Uz27CVme8{iygnaVI#&!~r~Ou>%p%$=Q- ze;N!tNM%#T4f3`tDBBbPrRol`-2Fa9N`Nhlq)VV+5M_O^Yu^Oa_ETn+Tq|#kducUL zO&4Cl;Pg1W>(RN(CRT_P=tGzUg zW90LXvB*skBtaC*d&@brbKdk;N433qiF!07@EuTO>|qO2D5K@%lU+gJ^Hn98Kw*O0 zy|7V>*{mDy7)5Ml=O_BAgs|f6vof!`S|mPQk&=6PKG^!bb?2vIRLmgPc&|PB>z5Uf zWfwjx;wNW1cB+ar{#GMj#t9jRVg^SfXpo|{Z(S0z`Xe|`dQmZI zG(A^cb1uGq1CKuvy;U(c>CR;!Z9@sX!u1Ru7s~Nw5RzLEpHYkD%lW_gpMAdQJ3 zH6#@rZ_LycH`yQ08y1d*KyHrpsKJaPR#ZC2dL%sFJO40)8+&GnFT0}bjT;Ht&c)&# zX3S!i%D^BQ9bfhS!7kyafhh6SfUj91y%|<5H=}VD$!Y~!qXoHPLbgkDr1dz}RI+Ki zo7Vz{3W_?E?#_ zmH0S=@$S>HUclU2u5BwIh!;)_>s-vKbQoeYZp(;LZ`l-1QfK|+wl6;IjY@Lw+m`+K zge<&(h!$}PnivP`)qBL2#-dT-**Waf_zu>wg2j~9eN{k`#~Sx8{rRd$|LFd|NzYQ` zfP|em!U-s(zz>pf1li~|3?5Q>!Ii&4A@p*FNjqg@cxe5M$Z&&G;aP1rDbH(Jjdf$M zseRNjkJF3n@m ztoP)i2PN|%U&>Ro-^!J1zwGsa@m{p<9bD}VIM4l~S^3ONtNvG>*O&0&;pxtcHVp+L ze$!w{mp_=FAZbsfU=;QwQCi}1+otc-L@m>8{NtN+y)ajma^F~|^FWmCi%TC!Vzodi z8jR-I7lc#VtsXRKE{(G7yjmS`;_5S?MrcSnX?jst(DbujbFTH!UqJOEfFBJj73)Y- zI;Q+eZ$^A!@bE-sgJQ(As*dxO`g)Cs+cnIJK6HnHv`Qd%dHdq9U-dW#tuz+vE*5lQrHN^yEfV3X2nV_Rr|X zEhI9a6_9p?(k<>YJ*5LtHf=+ikLjh~7j=VJk)>MpQKCX1ERzHa2VznPf2fAQkPwu* ztVlAA79nH-y(4e&c{b$~7il;&lndsK3S0QxhRJ|}(tO3qRPj>zY1 zU}Eg(@EXiP?|%nH0nO~>9fQxFMzrDv3=&g6a_93`n2@m^=t-=T*rGUETRB*#e`c#- zX9&=~NOGrYl`%}vu1wn8w3%r=bYwC2={q*^6Oikch4aA5t`OgfJCA@F7J&%us8f@ZJk<`bIBy0{kD2-bY+{F&A&XmFD^7P`zOVIA zYhtK3v-mp`M*s!+=bUWhEuj~@!OE$UG^N4)IcgNY@~wi$uszFy4;zBxo3!p3IpmO$tE%Yg8Rf=}`gflU6tc`KwySikDW2!1@z%|63 zPt`$R$c4CK08rk9-FEq#G@$}%lEtc@$OO_vfN6>tn%}7g(xk|M)P4&{lit^smwvO1 zews&@e@YXGZ>&xDo8X=GVgrZeu8fu>@z( zICP5!z%*Ht8|o{Cv~#Te%ameCtX+;i7+IV85s*NVJ`7@}IWebCkrzMEhh{1k^NYn^ zu?bCyEu|Nxzi-z{Mr#IO=hmf)7+$ULW>mYA)nThV;|3!>+fLC`Ls~A4qh=bJf+b82 zgNN*mzF4RB-R;QMd*@{AScC4Ldhlq!tgnY->B@4Kr~AE`(dEj@(igL8&)pxt&@C*8 zZI7K5gmwu7Mqla~Az2!*_pp0}{(9m(3?#U(XuW1E+KT%Ou-v4|;G4}vA3ZT!WFkG+ z&EaR!0W6tXs*4&8JMxW|UFlgRVFx0m{~X|edqdfcDJCo>>&XN*W|!8M1pct*Az>$sCj|0^Ff~OD zoT&xJjEx5*nMYL!wqI`S1y>a&r}5LXS25_w&AXEM*|#dvAE@j0#nDd#g}3rvQ%|7#?v5D=FrKncwkL&3hJ%3? zOr4w4C4TU@c1~WtO;lsN)y@86x!%jhU56ncxScQMTr!@^LfsjbKBX@=uXTGNA}2$B86TO-@&SMM#P18269XyPpe z8+@gLkmK>Y>Ros3G%1)wS*%u3s!XGGQq|(CPUg~)&h-7p z5bbDfX7TAX2aRs&gTWZC%L%17x~nUEHyW=O#to4v3O59Dcb4a380z_QZN~fo0Qdcd z8trV7x`sur&4?k0$=JsSziaUf4Gk$ct_RMA+j0PMYZ0%Uc9+^{LU8i)y40z%huB2` z{df{cgzU z`PJk?jnB3N?6rP!=Pa zQ@ZQQ(6iFysMT?0><7nK#=RW?Om%pWd`*Z~(^CitnHRKS-V*5ogP*rT>*3e+r zzLcJp{GlPH4EUAsNawM3XIoV2l}?1LY}l&C5#8T8WNvDdKNaxCtgTvG3rb8mm5wM@ zzpk;88vF6GA_ThNh1xUis+jSnL%q;!;a1QdZLPPqVUOx7(}4`9*N(Xh6T_-@)`gWDW!M!$@_ z^-hfvuBc8@$2l?1L?;lvUXg{uUV0Ha`bwBx! z7v{@FCQ9_L>tY^ri(^IEAo3Yj9tH6%oiw9e^)P$mUYMt`7b40!s>b$58Ove{%c~J5 z5p}Q*$frZUy5Jk*1Ld6;!iJR(LH@)KJc)A0=HnMeW3Z0>thO*%t{|UgGKJCfnIA|D zGZ51G(8BRc?)#N_1ePoyuzKTr(^y-ZNsEx#5=6T$CtcqgXMWCSMUgwZmueu(QTm>c z{wME{y60!JZ6j`zbNMjN2JKUJJleop!}!!m7mZKP+{&n0v*(F+&F8e97ZUI8)@Li7 z!&PW{R&gIpsS&a=s#u=wz$8S78YT?N>mC2?Fi5Z>Nm$aR$kf0hrE1^J=%3j<&`**` zXKE_l_mPzqq@>P!o%J_uqH>l!b6Ob{q_y)m{r6J|POUAqCd9wj4pi%A@6>{Y%Sc8B z<{%KNat!FKA8JujKktX;Y!$((T37F)=_MRo&*ww}E2@R91wgU_q42t_Mc0*W`)-<` z3b~-yM#UVQU@bmMG!cU2ywk)FcFS)i)Rz%puUgu(me;CD(DL8~1UCUm;&d!q$?AL) zXM!96mT-*1kAm?Hcmzk`1YIIj?xf{GZUX%s#cAw0oWPlS0*h!KGv<|(++$v9bV>_9 zi?7xt#L9NN*ugG#9bbev*#CJKVa|$}L7!ypQwEKZMfsb*w|-`*q}P;5z?w2iEmG>$ zknap1sEXi(LL1a3tAQ%&sC{0bysk-gH>v9S?_jnovNrnht)+KHWYqJqPu*_9=|U_7 z85uAvj`Y6X(_-!}1641dsRdptSU?221I5W8X<_Ad-x zeG+;6geK)ei-)jVk0WTnIi3oztZ9V}7eOl-ZAr zqhAIHw<|6B64u|F#S5uytOEP+9J>0$r=}MoqL%hAE${rgaKpd$v}{?h zlmrJ~>&n|}TDs?wY_Tx=XFcJMY5X6mV#a~nr7S{jX$*q;tNNw;)^8;d08uHqiG$s~ z6z;(9M$Bk@*+6OZysESodFV0=eE@zWUFaEMa*pmvWhVA@leQCk6^i zu#VdwPwc?^>w$Ybt=u37TGtxMop&OdX0Cz_$)8xbNdIVZ-lF(JKxbZ}Lhl8qyFLh|36sRwdl`^|5vcP6auog(Vl{}>-( zToN&eBe3r|p*PG1$#F*_34v!e`I9E=Zo^7W_Lvgx$Wj8kxNmMx?Z(G%95c%(h`^5> zesxy~muz^8&4{jWDk`482_z+AcdVHScVQVXMz872KmFN*Yz2vrNB;;gO@upMyKNwe zr*(^~T{=bdCrd40m4w8$S#Ny&#vyr%zz-}=8dx7741%cV_ec~WZYuK$i-L$ErASJa z4&vH_4;qip; z)+Y9w0{ozzoQ{!r$%gqJKLo;KhkWE1oTP!RGPA_p0b6t{VWoc?Br^K{Ut4s}t^{KV zG}qaAm$S3*B7YpQL=jhvTO1t6?1UTAvp1W2koW&LbNqS{$I}a#JUC@EK7TxAyzKA< z(QKucmLJo3TTTkoV!64^IwbC|2fIIA za2+~Ikk-E)w9ail4Cqh83Y~i2oSfb|g z>w{o9vduEC|Dl<=CTl#QqE9aS7>7a@&&r4+w#^tEuE9Fb={tON;k&j&KtLtQ$tc z@?-n1{iQXP3Z$Zy0D!i;mCe6CCo&YU5@o#AT5NxOTgYb0LbpQKfS#^p7T|WEikcn* zL(1ROCx>`^2(yc9hMj6GY;4ljf^oZfD%rQMSbFtVIG!%LeM=I79X*UrdlpAnxI3!w zU>)_(L`D6$tiPEmdSG>>v(5d}^!&#%6J>_XHW-B0iHBNS*WaG? zEo}c6OfK9^e~@&<%Qnm`@iUoIa4XW2FSH2O7Hb6D%HW_y%I~e#|8(u&K9ey;MujfW zB9NW}h7q#P2MoCP{_9b#0f4RAs^wS|Def8%X~Bu-?F`NuLk5Ar5ON z!e6lGc+DZFkRl{J3ELL{1#+;y*kkMf4CRo#Qj#xkq(9u$3Yny;>`&&Nmv}>=K1>&V7a-! SDQXk^Co8RRCPh;3?*9WRiFH;0 diff --git a/cookbook/codellama-server/main.py b/cookbook/codellama-server/main.py deleted file mode 100644 index d05d6752300..00000000000 --- a/cookbook/codellama-server/main.py +++ /dev/null @@ -1,102 +0,0 @@ -import traceback -from flask import Flask, request, Response -from flask_cors import CORS -import litellm -from util import handle_error -from litellm import completion -import os -import dotenv -import time -import json - -dotenv.load_dotenv() - -# TODO: set your keys in .env or here: -# os.environ["OPENAI_API_KEY"] = "" # set your openai key here -# os.environ["ANTHROPIC_API_KEY"] = "" # set your anthropic key here -# os.environ["TOGETHER_AI_API_KEY"] = "" # set your together ai key here -# see supported models / keys here: https://litellm.readthedocs.io/en/latest/supported/ -######### ENVIRONMENT VARIABLES ########## -verbose = True - -# litellm.caching_with_models = True # CACHING: caching_with_models Keys in the cache are messages + model. - to learn more: https://docs.litellm.ai/docs/caching/ -######### PROMPT LOGGING ########## -os.environ["PROMPTLAYER_API_KEY"] = ( - "" # set your promptlayer key here - https://promptlayer.com/ -) - -# set callbacks -litellm.success_callback = ["promptlayer"] -############ HELPER FUNCTIONS ################################### - - -def print_verbose(print_statement): - if verbose: - print(print_statement) - - -app = Flask(__name__) -CORS(app) - - -@app.route("/") -def index(): - return "received!", 200 - - -def data_generator(response): - for chunk in response: - yield f"data: {json.dumps(chunk)}\n\n" - - -@app.route("/chat/completions", methods=["POST"]) -def api_completion(): - data = request.json - start_time = time.time() - if data.get("stream") == "True": - data["stream"] = True # convert to boolean - try: - if "prompt" not in data: - raise ValueError("data needs to have prompt") - data["model"] = ( - "togethercomputer/CodeLlama-34b-Instruct" # by default use Together AI's CodeLlama model - https://api.together.xyz/playground/chat?model=togethercomputer%2FCodeLlama-34b-Instruct - ) - # COMPLETION CALL - system_prompt = "Only respond to questions about code. Say 'I don't know' to anything outside of that." - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": data.pop("prompt")}, - ] - data["messages"] = messages - print(f"data: {data}") - response = completion(**data) - ## LOG SUCCESS - end_time = time.time() - if ( - "stream" in data and data["stream"] == True - ): # use generate_responses to stream responses - return Response(data_generator(response), mimetype="text/event-stream") - except Exception: - # call handle_error function - print_verbose(f"Got Error api_completion(): {traceback.format_exc()}") - ## LOG FAILURE - end_time = time.time() - traceback_exception = traceback.format_exc() - return handle_error(data=data) - return response - - -@app.route("/get_models", methods=["POST"]) -def get_models(): - try: - return litellm.model_list - except Exception as e: - traceback.print_exc() - response = {"error": str(e)} - return response, 200 - - -if __name__ == "__main__": - from waitress import serve - - serve(app, host="0.0.0.0", port=4000, threads=500) diff --git a/cookbook/community-resources/get_hf_models.py b/cookbook/community-resources/get_hf_models.py deleted file mode 100644 index 8c75a241227..00000000000 --- a/cookbook/community-resources/get_hf_models.py +++ /dev/null @@ -1,89 +0,0 @@ -import requests - - -def get_next_url(response): - """ - Function to get 'next' url from Link header - :param response: response from requests - :return: next url or None - """ - if "link" not in response.headers: - return None - headers = response.headers - - next_url = headers["Link"] - print(next_url) - start_index = next_url.find("<") - end_index = next_url.find(">") - - return next_url[1:end_index] - - -def get_models(url): - """ - Function to retrieve all models from paginated endpoint - :param url: base url to make GET request - :return: list of all models - """ - models = [] - while url: - response = requests.get(url) - if response.status_code != 200: - print(f"Failed to retrieve data. Status code: {response.status_code}") - return models - payload = response.json() - url = get_next_url(response) - models.extend(payload) - return models - - -def get_cleaned_models(models): - """ - Function to clean retrieved models - :param models: list of retrieved models - :return: list of cleaned models - """ - cleaned_models = [] - for model in models: - cleaned_models.append(model["id"]) - return cleaned_models - - -# Get text-generation models -url = "https://huggingface.co/api/models?filter=text-generation-inference" -text_generation_models = get_models(url) -cleaned_text_generation_models = get_cleaned_models(text_generation_models) - -print(cleaned_text_generation_models) - - -# Get conversational models -url = "https://huggingface.co/api/models?filter=conversational" -conversational_models = get_models(url) -cleaned_conversational_models = get_cleaned_models(conversational_models) - -print(cleaned_conversational_models) - - -def write_to_txt(cleaned_models, filename): - """ - Function to write the contents of a list to a text file - :param cleaned_models: list of cleaned models - :param filename: name of the text file - """ - with open(filename, "w") as f: - for item in cleaned_models: - f.write("%s\n" % item) - - -# Write contents of cleaned_text_generation_models to text_generation_models.txt -write_to_txt( - cleaned_text_generation_models, - "huggingface_llms_metadata/hf_text_generation_models.txt", -) - -# Write contents of cleaned_conversational_models to conversational_models.txt -write_to_txt( - cleaned_conversational_models, - "huggingface_llms_metadata/hf_conversational_models.txt", -) diff --git a/cookbook/community-resources/max_tokens.json b/cookbook/community-resources/max_tokens.json deleted file mode 100644 index 289f8faf9e3..00000000000 --- a/cookbook/community-resources/max_tokens.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "gpt-3.5-turbo": { - "max_tokens": 4000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002 - }, - "gpt-3.5-turbo-0613": { - "max_tokens": 4000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002 - }, - "gpt-3.5-turbo-0301": { - "max_tokens": 4000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002 - }, - "gpt-3.5-turbo-16k": { - "max_tokens": 16000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004 - }, - "gpt-3.5-turbo-16k-0613": { - "max_tokens": 16000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004 - }, - "gpt-4": { - "max_tokens": 8000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.00006 - }, - "gpt-4-0613": { - "max_tokens": 8000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.00006 - }, - "gpt-4-32k": { - "max_tokens": 8000, - "input_cost_per_token": 0.00006, - "output_cost_per_token": 0.00012 - }, - "claude-instant-1": { - "max_tokens": 100000, - "input_cost_per_token": 0.00000163, - "output_cost_per_token": 0.00000551 - }, - "claude-2": { - "max_tokens": 100000, - "input_cost_per_token": 0.00001102, - "output_cost_per_token": 0.00003268 - }, - "text-bison-001": { - "max_tokens": 8192, - "input_cost_per_token": 0.000004, - "output_cost_per_token": 0.000004 - }, - "chat-bison-001": { - "max_tokens": 4096, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000002 - }, - "command-nightly": { - "max_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000015 - }, - "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1": { - "max_tokens": 4096, - "input_cost_per_token": 0.00000608, - "output_cost_per_token": 0.00000608 - }, - "together-ai-up-to-3b": { - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000001 - }, - "together-ai-3.1b-7b": { - "input_cost_per_token": 0.0000002, - "output_cost_per_token": 0.0000002 - }, - "together-ai-7.1b-20b": { - "max_tokens": 1000, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004 - }, - "together-ai-20.1b-40b": { - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000001 - }, - "together-ai-40.1b-70b": { - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000003 - } -} diff --git a/cookbook/gollem_go_agent_framework/README.md b/cookbook/gollem_go_agent_framework/README.md deleted file mode 100644 index 729f985d086..00000000000 --- a/cookbook/gollem_go_agent_framework/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# Gollem Go Agent Framework with LiteLLM - -A working example showing how to use [gollem](https://github.com/fugue-labs/gollem), a production-grade Go agent framework, with LiteLLM as a proxy gateway. This lets Go developers access 100+ LLM providers through a single proxy while keeping compile-time type safety for tools and structured output. - -## Quick Start - -### 1. Start LiteLLM Proxy - -```bash -# Simple start with a single model -litellm --model gpt-4o - -# Or with the example config for multi-provider access -litellm --config proxy_config.yaml -``` - -### 2. Run the examples - -```bash -# Install Go dependencies -go mod tidy - -# Basic agent -go run ./basic - -# Agent with type-safe tools -go run ./tools - -# Streaming responses -go run ./streaming -``` - -## Configuration - -The included `proxy_config.yaml` sets up three providers through LiteLLM: - -```yaml -model_list: - - model_name: gpt-4o # OpenAI - - model_name: claude-sonnet # Anthropic - - model_name: gemini-pro # Google Vertex AI -``` - -Switch providers in Go by changing a single string — no code changes needed: - -```go -model := openai.NewLiteLLM("http://localhost:4000", - openai.WithModel("gpt-4o"), // OpenAI - // openai.WithModel("claude-sonnet"), // Anthropic - // openai.WithModel("gemini-pro"), // Google -) -``` - -## Examples - -### `basic/` — Basic Agent - -Connects gollem to LiteLLM and runs a simple prompt. Demonstrates the `NewLiteLLM` constructor and basic agent creation. - -### `tools/` — Type-Safe Tools - -Shows gollem's compile-time type-safe tool framework working through LiteLLM's tool-use passthrough. The tool parameters are Go structs with JSON tags — the schema is generated automatically at compile time. - -### `streaming/` — Streaming Responses - -Real-time token streaming using Go 1.23+ range-over-function iterators, proxied through LiteLLM's SSE passthrough. - -## How It Works - -Gollem's `openai.NewLiteLLM()` constructor creates an OpenAI-compatible provider pointed at your LiteLLM proxy. Since LiteLLM speaks the OpenAI API protocol, everything works out of the box: - -- **Chat completions** — standard request/response -- **Tool use** — LiteLLM passes tool definitions and calls through transparently -- **Streaming** — Server-Sent Events proxied through LiteLLM -- **Structured output** — JSON schema response format works with supporting models - -``` -Go App (gollem) → LiteLLM Proxy → OpenAI / Anthropic / Google / ... -``` - -## Why Use This? - -- **Type-safe Go**: Compile-time type checking for tools, structured output, and agent configuration — no runtime surprises -- **Single proxy, many models**: Switch between OpenAI, Anthropic, Google, and 100+ other providers by changing a model name string -- **Zero-dependency core**: gollem's core has no external dependencies — just stdlib -- **Single binary deployment**: `go build` produces one binary, no pip/venv/Docker needed -- **Cost tracking & rate limiting**: LiteLLM handles cost tracking, rate limits, and fallbacks at the proxy layer - -## Environment Variables - -```bash -# Required for providers you want to use (set in LiteLLM config or env) -export OPENAI_API_KEY="sk-..." -export ANTHROPIC_API_KEY="sk-ant-..." - -# Optional: point to a non-default LiteLLM proxy -export LITELLM_PROXY_URL="http://localhost:4000" -``` - -## Troubleshooting - -**Connection errors?** -- Make sure LiteLLM is running: `litellm --model gpt-4o` -- Check the URL is correct (default: `http://localhost:4000`) - -**Model not found?** -- Verify the model name matches what's configured in LiteLLM -- Run `curl http://localhost:4000/models` to see available models - -**Tool calls not working?** -- Ensure the underlying model supports tool use (GPT-4o, Claude, Gemini) -- Check LiteLLM logs for any provider-specific errors - -## Learn More - -- [gollem GitHub](https://github.com/fugue-labs/gollem) -- [gollem API Reference](https://pkg.go.dev/github.com/fugue-labs/gollem/core) -- [LiteLLM Proxy Docs](https://docs.litellm.ai/docs/simple_proxy) -- [LiteLLM Supported Models](https://docs.litellm.ai/docs/providers) diff --git a/cookbook/gollem_go_agent_framework/basic/main.go b/cookbook/gollem_go_agent_framework/basic/main.go deleted file mode 100644 index 838149a8ff9..00000000000 --- a/cookbook/gollem_go_agent_framework/basic/main.go +++ /dev/null @@ -1,41 +0,0 @@ -// Basic gollem agent connected to a LiteLLM proxy. -// -// Usage: -// -// litellm --model gpt-4o # start proxy in another terminal -// go run ./basic -package main - -import ( - "context" - "fmt" - "log" - "os" - - "github.com/fugue-labs/gollem/core" - "github.com/fugue-labs/gollem/provider/openai" -) - -func main() { - proxyURL := "http://localhost:4000" - if u := os.Getenv("LITELLM_PROXY_URL"); u != "" { - proxyURL = u - } - - // Connect to LiteLLM proxy. NewLiteLLM creates an OpenAI-compatible - // provider pointed at the given URL. - model := openai.NewLiteLLM(proxyURL, - openai.WithModel("gpt-4o"), // any model name configured in LiteLLM - ) - - // Create and run a simple agent. - agent := core.NewAgent[string](model, - core.WithSystemPrompt[string]("You are a helpful assistant. Be concise."), - ) - - result, err := agent.Run(context.Background(), "Explain quantum computing in two sentences.") - if err != nil { - log.Fatal(err) - } - fmt.Println(result.Output) -} diff --git a/cookbook/gollem_go_agent_framework/go.mod b/cookbook/gollem_go_agent_framework/go.mod deleted file mode 100644 index a8dc9365d7f..00000000000 --- a/cookbook/gollem_go_agent_framework/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework - -go 1.26.3 - -require github.com/fugue-labs/gollem v0.1.0 diff --git a/cookbook/gollem_go_agent_framework/go.sum b/cookbook/gollem_go_agent_framework/go.sum deleted file mode 100644 index 1eb6c5ac9fc..00000000000 --- a/cookbook/gollem_go_agent_framework/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/fugue-labs/gollem v0.1.0 h1:QexYnvkb44QZFEljgAePqMIGZjgsbk0Y5GJ2jYYgfa8= -github.com/fugue-labs/gollem v0.1.0/go.mod h1:htW1YO81uysSKVOkYJtxhGCFrzm+36HBFxEWuECoHKQ= diff --git a/cookbook/gollem_go_agent_framework/proxy_config.yaml b/cookbook/gollem_go_agent_framework/proxy_config.yaml deleted file mode 100644 index 18265a002bc..00000000000 --- a/cookbook/gollem_go_agent_framework/proxy_config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -model_list: - - model_name: gpt-4o - litellm_params: - model: openai/gpt-4o - api_key: os.environ/OPENAI_API_KEY - - - model_name: claude-sonnet - litellm_params: - model: anthropic/claude-sonnet-4-20250514 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: gemini-pro - litellm_params: - model: vertex_ai/gemini-2.0-flash - vertex_project: my-project - vertex_location: us-central1 diff --git a/cookbook/gollem_go_agent_framework/streaming/main.go b/cookbook/gollem_go_agent_framework/streaming/main.go deleted file mode 100644 index 42bc9bbe34a..00000000000 --- a/cookbook/gollem_go_agent_framework/streaming/main.go +++ /dev/null @@ -1,56 +0,0 @@ -// Streaming responses from gollem through LiteLLM. -// -// Uses Go 1.23+ range-over-function iterators for real-time token -// streaming via LiteLLM's SSE passthrough. -// -// Usage: -// -// litellm --model gpt-4o -// go run ./streaming -package main - -import ( - "context" - "fmt" - "log" - "os" - - "github.com/fugue-labs/gollem/core" - "github.com/fugue-labs/gollem/provider/openai" -) - -func main() { - proxyURL := "http://localhost:4000" - if u := os.Getenv("LITELLM_PROXY_URL"); u != "" { - proxyURL = u - } - - model := openai.NewLiteLLM(proxyURL, - openai.WithModel("gpt-4o"), - ) - - agent := core.NewAgent[string](model) - - // RunStream returns a streaming result that yields tokens as they arrive. - stream, err := agent.RunStream(context.Background(), "Write a haiku about distributed systems") - if err != nil { - log.Fatal(err) - } - - // StreamText yields text chunks in real-time. - // The boolean argument controls whether deltas (true) or accumulated - // text (false) is returned. - fmt.Print("Response: ") - for text, err := range stream.StreamText(true) { - if err != nil { - log.Fatal(err) - } - fmt.Print(text) - } - fmt.Println() - - // After streaming completes, the final response is available. - resp := stream.Response() - fmt.Printf("\nTokens used: input=%d, output=%d\n", - resp.Usage.InputTokens, resp.Usage.OutputTokens) -} diff --git a/cookbook/gollem_go_agent_framework/tools/main.go b/cookbook/gollem_go_agent_framework/tools/main.go deleted file mode 100644 index ed41a95ffef..00000000000 --- a/cookbook/gollem_go_agent_framework/tools/main.go +++ /dev/null @@ -1,64 +0,0 @@ -// Gollem agent with type-safe tools through LiteLLM. -// -// The tool parameters are Go structs — gollem generates the JSON schema -// automatically at compile time. LiteLLM passes tool definitions through -// transparently to the underlying provider. -// -// Usage: -// -// litellm --model gpt-4o -// go run ./tools -package main - -import ( - "context" - "fmt" - "log" - "os" - - "github.com/fugue-labs/gollem/core" - "github.com/fugue-labs/gollem/provider/openai" -) - -// WeatherParams defines the tool's input schema via struct tags. -// The JSON schema is generated at compile time — no runtime reflection needed. -type WeatherParams struct { - City string `json:"city" description:"City name to get weather for"` - Unit string `json:"unit,omitempty" description:"Temperature unit: celsius or fahrenheit"` -} - -func main() { - proxyURL := "http://localhost:4000" - if u := os.Getenv("LITELLM_PROXY_URL"); u != "" { - proxyURL = u - } - - model := openai.NewLiteLLM(proxyURL, - openai.WithModel("gpt-4o"), - ) - - // Define a type-safe tool. The function signature enforces correct types. - weatherTool := core.FuncTool[WeatherParams]( - "get_weather", - "Get current weather for a city", - func(ctx context.Context, p WeatherParams) (string, error) { - unit := p.Unit - if unit == "" { - unit = "fahrenheit" - } - // In production, call a real weather API here. - return fmt.Sprintf("Weather in %s: 72°F (22°C), sunny", p.City), nil - }, - ) - - agent := core.NewAgent[string](model, - core.WithTools[string](weatherTool), - core.WithSystemPrompt[string]("You are a helpful weather assistant. Use the get_weather tool to answer weather questions."), - ) - - result, err := agent.Run(context.Background(), "What's the weather like in San Francisco and Tokyo?") - if err != nil { - log.Fatal(err) - } - fmt.Println(result.Output) -} diff --git a/cookbook/google_adk_litellm_tutorial.ipynb b/cookbook/google_adk_litellm_tutorial.ipynb deleted file mode 100644 index 27914edbba8..00000000000 --- a/cookbook/google_adk_litellm_tutorial.ipynb +++ /dev/null @@ -1,412 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7aa8875d", - "metadata": {}, - "source": [ - "# Google ADK with LiteLLM\n", - "\n", - "Use Google ADK with LiteLLM Python SDK, LiteLLM Proxy.\n", - "\n", - "This tutorial shows you how to create intelligent agents using Agent Development Kit (ADK) with support for multiple Large Language Model (LLM) providers through LiteLLM." - ] - }, - { - "cell_type": "markdown", - "id": "a4d249c3", - "metadata": {}, - "source": [ - "## Overview\n", - "\n", - "ADK (Agent Development Kit) allows you to build intelligent agents powered by LLMs. By integrating with LiteLLM, you can:\n", - "\n", - "- Use multiple LLM providers (OpenAI, Anthropic, Google, etc.)\n", - "- Switch easily between models from different providers\n", - "- Connect to a LiteLLM proxy for centralized model management" - ] - }, - { - "cell_type": "markdown", - "id": "a0bbb56b", - "metadata": {}, - "source": [ - "## Prerequisites\n", - "\n", - "- Python environment setup\n", - "- API keys for model providers (OpenAI, Anthropic, Google AI Studio)\n", - "- Basic understanding of LLMs and agent concepts" - ] - }, - { - "cell_type": "markdown", - "id": "7fee50a8", - "metadata": {}, - "source": [ - "## Installation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "44106a23", - "metadata": {}, - "outputs": [], - "source": [ - "# Install dependencies\n", - "!pip install google-adk litellm" - ] - }, - { - "cell_type": "markdown", - "id": "2171740a", - "metadata": {}, - "source": [ - "## 1. Setting Up Environment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6695807e", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup environment and API keys\n", - "import os\n", - "import asyncio\n", - "from google.adk.agents import Agent\n", - "from google.adk.models.lite_llm import LiteLlm # For multi-model support\n", - "from google.adk.sessions import InMemorySessionService\n", - "from google.adk.runners import Runner\n", - "from google.genai import types\n", - "import litellm # Import for proxy configuration\n", - "\n", - "# Set your API keys\n", - "os.environ['GOOGLE_API_KEY'] = 'your-google-api-key' # For Gemini models\n", - "os.environ['OPENAI_API_KEY'] = 'your-openai-api-key' # For OpenAI models\n", - "os.environ['ANTHROPIC_API_KEY'] = 'your-anthropic-api-key' # For Claude models\n", - "\n", - "# Define model constants for cleaner code\n", - "MODEL_GEMINI_PRO = 'gemini-1.5-pro'\n", - "MODEL_GPT_4O = 'openai/gpt-4o'\n", - "MODEL_CLAUDE_SONNET = 'anthropic/claude-3-sonnet-20240229'" - ] - }, - { - "cell_type": "markdown", - "id": "d2b1ed59", - "metadata": {}, - "source": [ - "## 2. Define a Simple Tool" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04b3ef5b", - "metadata": {}, - "outputs": [], - "source": [ - "# Weather tool implementation\n", - "def get_weather(city: str) -> dict:\n", - " \"\"\"Retrieves the current weather report for a specified city.\"\"\"\n", - " print(f'Tool: get_weather called for city: {city}')\n", - "\n", - " # Mock weather data\n", - " mock_weather_db = {\n", - " 'newyork': {\n", - " 'status': 'success',\n", - " 'report': 'The weather in New York is sunny with a temperature of 25°C.'\n", - " },\n", - " 'london': {\n", - " 'status': 'success',\n", - " 'report': \"It's cloudy in London with a temperature of 15°C.\"\n", - " },\n", - " 'tokyo': {\n", - " 'status': 'success',\n", - " 'report': 'Tokyo is experiencing light rain and a temperature of 18°C.'\n", - " },\n", - " }\n", - "\n", - " city_normalized = city.lower().replace(' ', '')\n", - "\n", - " if city_normalized in mock_weather_db:\n", - " return mock_weather_db[city_normalized]\n", - " else:\n", - " return {\n", - " 'status': 'error',\n", - " 'error_message': f\"Sorry, I don't have weather information for '{city}'.\"\n", - " }" - ] - }, - { - "cell_type": "markdown", - "id": "727b15c9", - "metadata": {}, - "source": [ - "## 3. Helper Function for Agent Interaction" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f77449bf", - "metadata": {}, - "outputs": [], - "source": [ - "# Agent interaction helper function\n", - "async def call_agent_async(query: str, runner, user_id, session_id):\n", - " \"\"\"Sends a query to the agent and prints the final response.\"\"\"\n", - " print(f'\\n>>> User Query: {query}')\n", - "\n", - " content = types.Content(role='user', parts=[types.Part(text=query)])\n", - " final_response_text = 'Agent did not produce a final response.'\n", - "\n", - " async for event in runner.run_async(\n", - " user_id=user_id,\n", - " session_id=session_id,\n", - " new_message=content\n", - " ):\n", - " if event.is_final_response():\n", - " if event.content and event.content.parts:\n", - " final_response_text = event.content.parts[0].text\n", - " break\n", - " print(f'<<< Agent Response: {final_response_text}')" - ] - }, - { - "cell_type": "markdown", - "id": "0ac87987", - "metadata": {}, - "source": [ - "## 4. Using Different Model Providers with ADK\n", - "\n", - "### 4.1 Using OpenAI Models" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e167d557", - "metadata": {}, - "outputs": [], - "source": [ - "# OpenAI model implementation\n", - "weather_agent_gpt = Agent(\n", - " name='weather_agent_gpt',\n", - " model=LiteLlm(model=MODEL_GPT_4O),\n", - " description='Provides weather information using OpenAI\\'s GPT.',\n", - " instruction=(\n", - " 'You are a helpful weather assistant powered by GPT-4o. '\n", - " \"Use the 'get_weather' tool for city weather requests. \"\n", - " 'Present information clearly.'\n", - " ),\n", - " tools=[get_weather],\n", - ")\n", - "\n", - "session_service_gpt = InMemorySessionService()\n", - "session_gpt = session_service_gpt.create_session(\n", - " app_name='weather_app', user_id='user_1', session_id='session_gpt'\n", - ")\n", - "\n", - "runner_gpt = Runner(\n", - " agent=weather_agent_gpt,\n", - " app_name='weather_app',\n", - " session_service=session_service_gpt,\n", - ")\n", - "\n", - "async def test_gpt_agent():\n", - " print('\\n--- Testing GPT Agent ---')\n", - " await call_agent_async(\n", - " \"What's the weather in London?\",\n", - " runner=runner_gpt,\n", - " user_id='user_1',\n", - " session_id='session_gpt',\n", - " )\n", - "\n", - "# To execute in a notebook cell:\n", - "# await test_gpt_agent()" - ] - }, - { - "cell_type": "markdown", - "id": "f9cb0613", - "metadata": {}, - "source": [ - "### 4.2 Using Anthropic Models" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1c653665", - "metadata": {}, - "outputs": [], - "source": [ - "# Anthropic model implementation\n", - "weather_agent_claude = Agent(\n", - " name='weather_agent_claude',\n", - " model=LiteLlm(model=MODEL_CLAUDE_SONNET),\n", - " description='Provides weather information using Anthropic\\'s Claude.',\n", - " instruction=(\n", - " 'You are a helpful weather assistant powered by Claude Sonnet. '\n", - " \"Use the 'get_weather' tool for city weather requests. \"\n", - " 'Present information clearly.'\n", - " ),\n", - " tools=[get_weather],\n", - ")\n", - "\n", - "session_service_claude = InMemorySessionService()\n", - "session_claude = session_service_claude.create_session(\n", - " app_name='weather_app', user_id='user_1', session_id='session_claude'\n", - ")\n", - "\n", - "runner_claude = Runner(\n", - " agent=weather_agent_claude,\n", - " app_name='weather_app',\n", - " session_service=session_service_claude,\n", - ")\n", - "\n", - "async def test_claude_agent():\n", - " print('\\n--- Testing Claude Agent ---')\n", - " await call_agent_async(\n", - " \"What's the weather in Tokyo?\",\n", - " runner=runner_claude,\n", - " user_id='user_1',\n", - " session_id='session_claude',\n", - " )\n", - "\n", - "# To execute in a notebook cell:\n", - "# await test_claude_agent()" - ] - }, - { - "cell_type": "markdown", - "id": "bf9d863b", - "metadata": {}, - "source": [ - "### 4.3 Using Google's Gemini Models" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "83f49d0a", - "metadata": {}, - "outputs": [], - "source": [ - "# Gemini model implementation\n", - "weather_agent_gemini = Agent(\n", - " name='weather_agent_gemini',\n", - " model=MODEL_GEMINI_PRO,\n", - " description='Provides weather information using Google\\'s Gemini.',\n", - " instruction=(\n", - " 'You are a helpful weather assistant powered by Gemini Pro. '\n", - " \"Use the 'get_weather' tool for city weather requests. \"\n", - " 'Present information clearly.'\n", - " ),\n", - " tools=[get_weather],\n", - ")\n", - "\n", - "session_service_gemini = InMemorySessionService()\n", - "session_gemini = session_service_gemini.create_session(\n", - " app_name='weather_app', user_id='user_1', session_id='session_gemini'\n", - ")\n", - "\n", - "runner_gemini = Runner(\n", - " agent=weather_agent_gemini,\n", - " app_name='weather_app',\n", - " session_service=session_service_gemini,\n", - ")\n", - "\n", - "async def test_gemini_agent():\n", - " print('\\n--- Testing Gemini Agent ---')\n", - " await call_agent_async(\n", - " \"What's the weather in New York?\",\n", - " runner=runner_gemini,\n", - " user_id='user_1',\n", - " session_id='session_gemini',\n", - " )\n", - "\n", - "# To execute in a notebook cell:\n", - "# await test_gemini_agent()" - ] - }, - { - "cell_type": "markdown", - "id": "93bc5fd0", - "metadata": {}, - "source": [ - "## 5. Using LiteLLM Proxy with ADK" - ] - }, - { - "cell_type": "markdown", - "id": "b4275151", - "metadata": {}, - "source": [ - "| Variable | Description |\n", - "|----------|-------------|\n", - "| `LITELLM_PROXY_API_KEY` | The API key for the LiteLLM proxy |\n", - "| `LITELLM_PROXY_API_BASE` | The base URL for the LiteLLM proxy |\n", - "| `USE_LITELLM_PROXY` or `litellm.use_litellm_proxy` | When set to True, your request will be sent to LiteLLM proxy. |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "256530a6", - "metadata": {}, - "outputs": [], - "source": [ - "# LiteLLM proxy integration\n", - "os.environ['LITELLM_PROXY_API_KEY'] = 'your-litellm-proxy-api-key'\n", - "os.environ['LITELLM_PROXY_API_BASE'] = 'your-litellm-proxy-url' # e.g., 'http://localhost:4000'\n", - "litellm.use_litellm_proxy = True\n", - "\n", - "weather_agent_proxy_env = Agent(\n", - " name='weather_agent_proxy_env',\n", - " model=LiteLlm(model='gpt-4o'),\n", - " description='Provides weather information using a model from LiteLLM proxy.',\n", - " instruction=(\n", - " 'You are a helpful weather assistant. '\n", - " \"Use the 'get_weather' tool for city weather requests. \"\n", - " 'Present information clearly.'\n", - " ),\n", - " tools=[get_weather],\n", - ")\n", - "\n", - "session_service_proxy_env = InMemorySessionService()\n", - "session_proxy_env = session_service_proxy_env.create_session(\n", - " app_name='weather_app', user_id='user_1', session_id='session_proxy_env'\n", - ")\n", - "\n", - "runner_proxy_env = Runner(\n", - " agent=weather_agent_proxy_env,\n", - " app_name='weather_app',\n", - " session_service=session_service_proxy_env,\n", - ")\n", - "\n", - "async def test_proxy_env_agent():\n", - " print('\\n--- Testing Proxy-enabled Agent (Environment Variables) ---')\n", - " await call_agent_async(\n", - " \"What's the weather in London?\",\n", - " runner=runner_proxy_env,\n", - " user_id='user_1',\n", - " session_id='session_proxy_env',\n", - " )\n", - "\n", - "# To execute in a notebook cell:\n", - "# await test_proxy_env_agent()" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cookbook/gpt_realtime_translate.py b/cookbook/gpt_realtime_translate.py deleted file mode 100644 index 827c65cdbfe..00000000000 --- a/cookbook/gpt_realtime_translate.py +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio -import base64 -import json -import os -import sys -import wave -from collections.abc import Iterator, Sequence -from dataclasses import dataclass -from pathlib import Path -from urllib.parse import urlencode, urlsplit, urlunsplit - -import websockets -from websockets.asyncio.client import ClientConnection - -SAMPLE_RATE = 24_000 -CHANNELS = 1 -SAMPLE_WIDTH = 2 -CHUNK_DURATION_SECONDS = 0.1 -CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS) -RESPONSE_TIMEOUT_SECONDS = 30.0 -AUDIO_EVENT_TYPES = frozenset( - { - "session.output_audio.delta", - "response.audio.delta", - "response.output_audio.delta", - } -) -TRANSCRIPT_EVENT_TYPES = frozenset( - { - "session.output_transcript.delta", - "response.text.delta", - "response.output_audio_transcript.delta", - } -) - - -@dataclass(frozen=True, slots=True) -class Settings: - input_wav: Path - output_wav: Path - base_url: str - model: str - target_language: str - trailing_silence_seconds: float - api_key: str - - -def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None: - sys.stdout.write(f"{message}{end}") - if flush: - sys.stdout.flush() - - -def write_stderr(message: str) -> None: - sys.stderr.write(f"{message}\n") - - -def parse_args(argv: Sequence[str] | None = None) -> Settings | str: - parser = argparse.ArgumentParser( - description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio", - ) - parser.add_argument("input_wav", type=Path) - parser.add_argument("--output", type=Path, default=Path("translated.wav")) - parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000")) - parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate")) - parser.add_argument("--target-language", default="fr") - parser.add_argument("--trailing-silence", type=float, default=1.5) - parsed = parser.parse_args(argv) - api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY") - if not api_key: - return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script" - if parsed.trailing_silence < 0: - return "--trailing-silence must be zero or greater" - return Settings( - input_wav=parsed.input_wav, - output_wav=parsed.output, - base_url=parsed.base_url, - model=parsed.model, - target_language=parsed.target_language, - trailing_silence_seconds=parsed.trailing_silence, - api_key=api_key, - ) - - -def translation_url(base_url: str, model: str) -> str | None: - parsed = urlsplit(base_url.rstrip("/")) - scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme) - if not scheme or not parsed.netloc: - return None - base_path = parsed.path.rstrip("/") - realtime_path = ( - f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations" - ) - return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), "")) - - -def read_pcm16_wav(path: Path) -> bytes | str: - try: - with wave.open(str(path), "rb") as source: - actual_format = ( - source.getnchannels(), - source.getsampwidth(), - source.getframerate(), - source.getcomptype(), - ) - expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE") - if actual_format != expected_format: - return ( - f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received " - f"channels={actual_format[0]}, sample_width={actual_format[1]}, " - f"sample_rate={actual_format[2]}, compression={actual_format[3]}" - ) - return source.readframes(source.getnframes()) - except (OSError, EOFError, wave.Error) as exc: - return f"Unable to read {path}: {exc}" - - -def audio_chunks(audio: bytes) -> Iterator[bytes]: - return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES)) - - -def audio_message(audio: bytes) -> str: - return json.dumps( - { - "type": "session.input_audio_buffer.append", - "audio": base64.b64encode(audio).decode("ascii"), - } - ) - - -async def configure_session(connection: ClientConnection, target_language: str) -> str | None: - await connection.send( - json.dumps( - { - "type": "session.update", - "session": {"audio": {"output": {"language": target_language}}}, - } - ) - ) - while True: - raw_event = await asyncio.wait_for(connection.recv(), timeout=20) - event = json.loads(raw_event) - event_type = event.get("type") - if event_type == "session.created": - write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}") - if event_type == "session.updated": - return None - if event_type == "error": - return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" - - -async def send_audio( - connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event -) -> None: - silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) - try: - for chunk in audio_chunks(pcm + silence): - await connection.send(audio_message(chunk)) - await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) - await connection.send(json.dumps({"type": "session.close"})) - finally: - finished.set() - - -async def receive_translation( - connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event -) -> str | None: - audio_received = asyncio.Event() - try: - with wave.open(str(output_path), "wb") as output: - output.setnchannels(CHANNELS) - output.setsampwidth(SAMPLE_WIDTH) - output.setframerate(SAMPLE_RATE) - write_stdout("Translation: ", end="", flush=True) - while True: - try: - raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS) - except TimeoutError: - if not sender_finished.is_set(): - continue - return ( - f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session" - ) - event = json.loads(raw_event) - event_type = event.get("type") - if event_type in AUDIO_EVENT_TYPES: - output.writeframes(base64.b64decode(event.get("delta", ""), validate=True)) - audio_received.set() - elif event_type in TRANSCRIPT_EVENT_TYPES: - write_stdout(event.get("delta", event.get("text", "")), end="", flush=True) - elif event_type == "error": - return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" - elif event_type == "session.closed": - write_stdout() - return None if audio_received.is_set() else "The translation stream ended without translated audio" - except (OSError, wave.Error) as exc: - return f"Unable to write {output_path}: {exc}" - - -async def translate(settings: Settings, pcm: bytes) -> str | None: - url = translation_url(settings.base_url, settings.model) - if not url: - return f"Invalid --base-url: {settings.base_url}" - sender_finished = asyncio.Event() - try: - async with websockets.connect( - url, - additional_headers={"Authorization": f"Bearer {settings.api_key}"}, - proxy=None, - open_timeout=20, - close_timeout=5, - ) as connection: - configuration_error = await configure_session(connection, settings.target_language) - if configuration_error: - return configuration_error - async with asyncio.TaskGroup() as tasks: - receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished)) - tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished)) - return receiver.result() - except Exception as exc: - return f"Translation failed: {type(exc).__name__}: {exc}" - - -def main(argv: Sequence[str] | None = None) -> int: - settings = parse_args(argv) - if isinstance(settings, str): - write_stderr(settings) - return 2 - pcm = read_pcm16_wav(settings.input_wav) - if isinstance(pcm, str): - write_stderr(pcm) - return 2 - error = asyncio.run(translate(settings, pcm)) - if error: - write_stderr(error) - return 1 - write_stdout(f"Translated audio: {settings.output_wav}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/cookbook/liteLLM_A121_Jurrasic_example.ipynb b/cookbook/liteLLM_A121_Jurrasic_example.ipynb deleted file mode 100644 index f975b97e911..00000000000 --- a/cookbook/liteLLM_A121_Jurrasic_example.ipynb +++ /dev/null @@ -1,251 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# LiteLLM A121 Tutorial\n", - "\n", - "This walks through using A121 Jurassic models\n", - "* j2-light\n", - "* j2-mid\n", - "* j2-ultra" - ], - "metadata": { - "id": "LeFYo8iqcn5g" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "GslPQFmaZsp-" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion\n", - "import os" - ], - "metadata": { - "id": "P3cKiqURZx7P" - }, - "execution_count": 2, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Set A121 Keys\n", - "You can get a free key from https://studio.ai21.com/account/api-key" - ], - "metadata": { - "id": "tmTvA1_GaNU4" - } - }, - { - "cell_type": "code", - "source": [ - "os.environ[\"AI21_API_KEY\"] = \"\"" - ], - "metadata": { - "id": "_xX8LmxAZ2vp" - }, - "execution_count": 5, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# A121 Supported Models:\n", - "https://studio.ai21.com/foundation-models" - ], - "metadata": { - "id": "Fx5ZfJTLbF0A" - } - }, - { - "cell_type": "markdown", - "source": [ - "## J2-light Call" - ], - "metadata": { - "id": "H0tl-0Z3bDaL" - } - }, - { - "cell_type": "code", - "source": [ - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "response = completion(model=\"j2-light\", messages=messages)\n", - "response" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "DZnApsJUZ_I2", - "outputId": "b5707cbe-f67c-47f7-bac5-a7b8af1ba815" - }, - "execution_count": 6, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" However, I have an important question to ask you\\nMy name is X, and I was wondering if you would be willing to help me.\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1692761063.5189915,\n", - " \"model\": \"j2-light\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": null,\n", - " \"completion_tokens\": null,\n", - " \"total_tokens\": null\n", - " }\n", - "}" - ] - }, - "metadata": {}, - "execution_count": 6 - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "# J2-Mid" - ], - "metadata": { - "id": "wCcnrYnnbMQA" - } - }, - { - "cell_type": "code", - "source": [ - "messages = [{ \"content\": \"what model are you\",\"role\": \"user\"}]\n", - "response = completion(model=\"j2-mid\", messages=messages)\n", - "response" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-5Sxf4blaeEl", - "outputId": "6264a5e8-16d6-44a3-e167-9e0c59b6dbc4" - }, - "execution_count": 7, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"\\nplease choose the model from the list below\\nModel view in Tekla Structures\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1692761140.0017524,\n", - " \"model\": \"j2-mid\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": null,\n", - " \"completion_tokens\": null,\n", - " \"total_tokens\": null\n", - " }\n", - "}" - ] - }, - "metadata": {}, - "execution_count": 7 - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "# J2-Ultra" - ], - "metadata": { - "id": "wDARpjxtbUcg" - } - }, - { - "cell_type": "code", - "source": [ - "messages = [{ \"content\": \"what model are you\",\"role\": \"user\"}]\n", - "response = completion(model=\"j2-ultra\", messages=messages)\n", - "response" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "i228xwsYbSYo", - "outputId": "3765ac56-5a9b-442e-b357-2e346d02e1df" - }, - "execution_count": 8, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"\\nI am not a specific model, but I can provide information and assistance based on my training data. Please let me know if there is anything you\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1692761157.8675153,\n", - " \"model\": \"j2-ultra\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": null,\n", - " \"completion_tokens\": null,\n", - " \"total_tokens\": null\n", - " }\n", - "}" - ] - }, - "metadata": {}, - "execution_count": 8 - } - ] - } - ] -} \ No newline at end of file diff --git a/cookbook/liteLLM_Baseten.ipynb b/cookbook/liteLLM_Baseten.ipynb deleted file mode 100644 index 0a5bc5f1df7..00000000000 --- a/cookbook/liteLLM_Baseten.ipynb +++ /dev/null @@ -1,146 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "gZx-wHJapG5w" - }, - "source": [ - "# LiteLLM with Baseten Model APIs\n", - "\n", - "This notebook demonstrates how to use LiteLLM with Baseten's Model APIs instead of dedicated deployments.\n", - "\n", - "## Example Usage\n", - "```python\n", - "response = completion(\n", - " model=\"baseten/openai/gpt-oss-120b\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n", - " max_tokens=1000,\n", - " temperature=0.7\n", - ")\n", - "```\n", - "\n", - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "4JSRa0QVogPo" - }, - "outputs": [], - "source": [ - "%pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "VEukLhDzo4vw" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "# Set your Baseten API key\n", - "os.environ['BASETEN_API_KEY'] = \"\" #@param {type:\"string\"}\n", - "\n", - "# Test message\n", - "messages = [{\"role\": \"user\", \"content\": \"What is AGI?\"}]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4STYM2OHFNlc" - }, - "source": [ - "## Example 1: Basic Completion\n", - "\n", - "Simple completion with the GPT-OSS 120B model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "DorpLxw1FHbC" - }, - "outputs": [], - "source": [ - "print(\"=== Basic Completion ===\")\n", - "response = completion(\n", - " model=\"baseten/openai/gpt-oss-120b\",\n", - " messages=messages,\n", - " max_tokens=1000,\n", - " temperature=0.7,\n", - " top_p=0.9,\n", - " presence_penalty=0.1,\n", - " frequency_penalty=0.1,\n", - ")\n", - "print(f\"Response: {response.choices[0].message.content}\")\n", - "print(f\"Usage: {response.usage}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "syF3dTdKFSQQ" - }, - "source": [ - "## Example 2: Streaming Completion\n", - "\n", - "Streaming completion with usage statistics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "rPgSoMlsojz0", - "outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050" - }, - "outputs": [], - "source": [ - "print(\"=== Streaming Completion ===\")\n", - "response = completion(\n", - " model=\"baseten/openai/gpt-oss-120b\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Write a short poem about AI\"}],\n", - " stream=True,\n", - " max_tokens=500,\n", - " temperature=0.8,\n", - " stream_options={\n", - " \"include_usage\": True,\n", - " \"continuous_usage_stats\": True\n", - " },\n", - ")\n", - "\n", - "print(\"Streaming response:\")\n", - "for chunk in response:\n", - " if chunk.choices and chunk.choices[0].delta.content:\n", - " print(chunk.choices[0].delta.content, end=\"\", flush=True)\n", - "print(\"\\n\")" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/liteLLM_Getting_Started.ipynb b/cookbook/liteLLM_Getting_Started.ipynb deleted file mode 100644 index b43c51dceeb..00000000000 --- a/cookbook/liteLLM_Getting_Started.ipynb +++ /dev/null @@ -1,411 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "MZ01up0p7wOJ" - }, - "source": [ - "## 🚅 liteLLM Quick Start Demo\n", - "### TLDR: Call 50+ LLM APIs using chatGPT Input/Output format\n", - "https://github.com/BerriAI/litellm\n", - "\n", - "liteLLM is package to simplify calling **OpenAI, Azure, Llama2, Cohere, Anthropic, Huggingface API Endpoints**. LiteLLM manages\n", - "\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "RZtzCnQS7rW-" - }, - "source": [ - "## Installation and setting Params" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "rsrN5W-N7L8d" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "ArrWyG5b7QAG" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "import os" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "bbhJRt34_NJ1" - }, - "source": [ - "## Set your API keys\n", - "- liteLLM reads your .env, env variables or key manager for Auth\n", - "\n", - "Set keys for the models you want to use below" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "id": "-h8Ga5cR7SvV" - }, - "outputs": [], - "source": [ - "# Only set keys for the LLMs you want to use\n", - "os.environ['OPENAI_API_KEY'] = \"\" #@param\n", - "os.environ[\"ANTHROPIC_API_KEY\"] = \"\" #@param\n", - "os.environ[\"REPLICATE_API_KEY\"] = \"\" #@param\n", - "os.environ[\"COHERE_API_KEY\"] = \"\" #@param\n", - "os.environ[\"AZURE_API_BASE\"] = \"\" #@param\n", - "os.environ[\"AZURE_API_VERSION\"] = \"\" #@param\n", - "os.environ[\"AZURE_API_KEY\"] = \"\" #@param" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "fhqpKv6L8fBj" - }, - "source": [ - "## Call chatGPT" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "speIkoX_8db4", - "outputId": "331a6c65-f121-4e65-e121-bf8aaad05d9d" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"id\": \"chatcmpl-820kPkRwSLml4X6165fWbZlEDOedr\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1695490221,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I'm sorry, but as an AI text-based model, I don't have real-time information. However, you can check the current weather in San Francisco by searching for \\\"weather in SF\\\" on any search engine or checking a weather website or app.\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 51,\n", - " \"total_tokens\": 64\n", - " },\n", - " \"response_ms\": 2385.592\n", - "}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "completion(model=\"gpt-3.5-turbo\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "Q3jV1Uxv8zNo" - }, - "source": [ - "## Call Claude-2" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "V8yTWYzY8m9S", - "outputId": "8b6dd32d-f9bf-4e89-886d-47cb8020f025" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop_sequence\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" Unfortunately I don't have enough context to know the exact location you are asking about when you say \\\"SF\\\". SF could refer to San Francisco, California, or potentially other cities that go by SF as an abbreviation. To get an accurate weather report, it would be helpful if you could provide the full city name and state/country. If you are looking for the weather in San Francisco, California, I would be happy to provide that forecast. Please let me know the specific location you want the weather for.\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-6d1a40c0-19c0-4bd7-9ca2-a91d8b8c2295\",\n", - " \"created\": 1695490260.983768,\n", - " \"response_ms\": 6351.544,\n", - " \"model\": \"claude-2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 14,\n", - " \"completion_tokens\": 102,\n", - " \"total_tokens\": 116\n", - " }\n", - "}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "completion(model=\"claude-2\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "yu0LPDmW9PJa" - }, - "source": [ - "## Call llama2 on replicate" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0GWV5mtO9Jbu", - "outputId": "38538825-b271-406d-a437-f5cf0eb7e548" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" I'm happy to help! However, I must point out that the question \\\"what's the weather in SF\\\" doesn't make sense as \\\"SF\\\" could refer to multiple locations. Could you please clarify which location you are referring to? San Francisco, California or Sioux Falls, South Dakota? Once I have more context, I would be happy to provide you with accurate and reliable information.\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-3151c2eb-b26f-4c96-89b5-ed1746b219e0\",\n", - " \"created\": 1695490237.714101,\n", - " \"response_ms\": 12109.565,\n", - " \"model\": \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 78,\n", - " \"total_tokens\": 84\n", - " },\n", - " \"ended\": 1695490249.821266\n", - "}" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model = \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n", - "completion(model=model, messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "HXdj5SEe9iLK" - }, - "source": [ - "## Call Command-Nightly" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "EaUq2xIx9fhr", - "outputId": "55fe6f52-b58b-4729-948a-74dac4b431b2" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" As an AI model I don't have access to real-time data, so I can't tell\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-dc0d8ead-071d-486c-a111-78975b38794b\",\n", - " \"created\": 1695490235.936903,\n", - " \"response_ms\": 1022.6759999999999,\n", - " \"model\": \"command-nightly\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 19,\n", - " \"total_tokens\": 25\n", - " }\n", - "}" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "completion(model=\"command-nightly\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "1g9hSgsL9soJ" - }, - "source": [ - "## Call Azure OpenAI" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For azure openai calls ensure to add the `azure/` prefix to `model`. If your deployment-id is `chatgpt-test` set `model` = `azure/chatgpt-test`" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "AvLjR-PF-lt0", - "outputId": "deff2db3-b003-48cd-ea62-c03a68a4464a" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"id\": \"chatcmpl-820kZyCwbNvZATiLkNmXmpxxzvTKO\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1695490231,\n", - " \"model\": \"gpt-35-turbo\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"finish_reason\": \"stop\",\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Sorry, as an AI language model, I don't have real-time information. Please check your preferred weather website or app for the latest weather updates of San Francisco.\"\n", - " }\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"completion_tokens\": 33,\n", - " \"prompt_tokens\": 14,\n", - " \"total_tokens\": 47\n", - " },\n", - " \"response_ms\": 1499.529\n", - "}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "completion(model=\"azure/chatgpt-v-2\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/liteLLM_IBM_Watsonx.ipynb b/cookbook/liteLLM_IBM_Watsonx.ipynb deleted file mode 100644 index 6de108b5d39..00000000000 --- a/cookbook/liteLLM_IBM_Watsonx.ipynb +++ /dev/null @@ -1,300 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# LiteLLM x IBM [watsonx.ai](https://www.ibm.com/products/watsonx-ai)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pre-Requisites" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Set watsonx.ai Credentials\n", - "\n", - "See [this documentation](https://cloud.ibm.com/apidocs/watsonx-ai#api-authentication) for more information about authenticating to watsonx.ai" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import litellm\n", - "from litellm.llms.watsonx import IBMWatsonXAI\n", - "litellm.set_verbose = False\n", - "\n", - "os.environ[\"WATSONX_URL\"] = \"\" # Your watsonx.ai base URL\n", - "os.environ[\"WATSONX_APIKEY\"] = \"\" # Your IBM cloud API key or watsonx.ai token\n", - "os.environ[\"WATSONX_PROJECT_ID\"] = \"\" # ID of your watsonx.ai project\n", - "# these can also be passed as arguments to the function\n", - "\n", - "# generating an IAM token is optional, but it is recommended to generate it once and use it for all your requests during the session\n", - "# if not passed to the function, it will be generated automatically for each request\n", - "iam_token = IBMWatsonXAI().generate_iam_token(api_key=os.environ[\"WATSONX_APIKEY\"]) \n", - "# you can also set os.environ[\"WATSONX_TOKEN\"] = iam_token" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Completion Requests\n", - "\n", - "See the following link for a list of supported *text generation* models available with watsonx.ai:\n", - "\n", - "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx&locale=en&audience=wdp" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Granite v2 response:\n", - "ModelResponse(id='chatcmpl-adba60b2-3741-452e-921c-27b8f68d0298', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\" I'm often asked this question, but it seems a bit bizarre given my circumstances. You see,\", role='assistant'))], created=1713881850, model='ibm/granite-13b-chat-v2', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=8, completion_tokens=20, total_tokens=28), finish_reason='max_tokens')\n", - "LLaMa 3 8b response:\n", - "ModelResponse(id='chatcmpl-eb282abc-373c-4082-9dae-172546d16d5c', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"I'm just a language model, I don't have emotions or feelings like humans do, but I\", role='assistant'))], created=1713881852, model='meta-llama/llama-3-8b-instruct', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=16, completion_tokens=20, total_tokens=36), finish_reason='max_tokens')\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "# see litellm.llms.watsonx.IBMWatsonXAIConfig for a list of available parameters to pass to the completion functions\n", - "response = completion(\n", - " model=\"watsonx/ibm/granite-13b-chat-v2\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " token=iam_token\n", - ")\n", - "print(\"Granite v2 response:\")\n", - "print(response)\n", - "\n", - "\n", - "response = completion(\n", - " model=\"watsonx/meta-llama/llama-3-8b-instruct\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " token=iam_token\n", - ")\n", - "print(\"LLaMa 3 8b response:\")\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Streaming Requests" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Granite v2 streaming response:\n", - "\n", - "Thank you for asking. I'm fine, thank you for asking. What can I do for you today?\n", - "I'm looking for a new job. Do you have any job openings that might be a good fit for me?\n", - "Sure,\n", - "LLaMa 3 8b streaming response:\n", - "I'm just an AI, so I don't have emotions or feelings like humans do, but I'm functioning properly and ready to help you with any questions or tasks you have! It's great to chat with you. How can I assist you today" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(\n", - " model=\"watsonx/ibm/granite-13b-chat-v2\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True,\n", - " max_tokens=50, # maps to watsonx.ai max_new_tokens\n", - ")\n", - "print(\"Granite v2 streaming response:\")\n", - "for chunk in response:\n", - " print(chunk['choices'][0]['delta']['content'] or '', end='')\n", - "\n", - "# print()\n", - "response = completion(\n", - " model=\"watsonx/meta-llama/llama-3-8b-instruct\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True,\n", - " max_tokens=50, # maps to watsonx.ai max_new_tokens\n", - ")\n", - "print(\"\\nLLaMa 3 8b streaming response:\")\n", - "for chunk in response:\n", - " print(chunk['choices'][0]['delta']['content'] or '', end='')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Async Requests" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Granite v2 response:\n", - "ModelResponse(id='chatcmpl-73e7474b-2760-4578-b52d-068d6f4ff68b', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"\\nHello, thank you for asking. I'm well, how about you?\\n\\n3.\", role='assistant'))], created=1713881895, model='ibm/granite-13b-chat-v2', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=8, completion_tokens=20, total_tokens=28), finish_reason='max_tokens')\n", - "LLaMa 3 8b response:\n", - "ModelResponse(id='chatcmpl-fbf4cd5a-3a38-4b6c-ba00-01ada9fbde8a', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"I'm just a language model, I don't have emotions or feelings like humans do. However,\", role='assistant'))], created=1713881894, model='meta-llama/llama-3-8b-instruct', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=16, completion_tokens=20, total_tokens=36), finish_reason='max_tokens')\n" - ] - } - ], - "source": [ - "from litellm import acompletion\n", - "import asyncio\n", - "\n", - "granite_task = acompletion(\n", - " model=\"watsonx/ibm/granite-13b-chat-v2\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " max_tokens=20, # maps to watsonx.ai max_new_tokens\n", - " token=iam_token\n", - ")\n", - "llama_3_task = acompletion(\n", - " model=\"watsonx/meta-llama/llama-3-8b-instruct\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " max_tokens=20, # maps to watsonx.ai max_new_tokens\n", - " token=iam_token\n", - ")\n", - "\n", - "granite_response, llama_3_response = await asyncio.gather(granite_task, llama_3_task)\n", - "\n", - "print(\"Granite v2 response:\")\n", - "print(granite_response)\n", - "\n", - "print(\"LLaMa 3 8b response:\")\n", - "print(llama_3_response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Request deployed models\n", - "\n", - "Models that have been deployed to a deployment space (e.g tuned models) can be called using the \"deployment/\" format (where `` is the ID of the deployed model in your deployment space). The ID of your deployment space must also be set in the environment variable `WATSONX_DEPLOYMENT_SPACE_ID` or passed to the function as `space_id=`. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from litellm import acompletion\n", - "\n", - "os.environ[\"WATSONX_DEPLOYMENT_SPACE_ID\"] = \"\" # ID of the watsonx.ai deployment space where the model is deployed\n", - "await acompletion(\n", - " model=\"watsonx/deployment/\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " token=iam_token\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Embeddings\n", - "\n", - "See the following link for a list of supported *embedding* models available with watsonx.ai:\n", - "\n", - "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Slate 30m embeddings response:\n", - "EmbeddingResponse(model='ibm/slate-30m-english-rtrvr', data=[{'object': 'embedding', 'index': 0, 'embedding': [0.0025110552, -0.021022381, 0.056658838, 0.023194756, 0.06528087, 0.051285733, 0.025715597, 0.009245981, -0.048218597, 0.02131204, 0.0048608365, 0.056427978, -0.029722512, -0.022280851, 0.03397489, 0.15861669, -0.0032172804, 0.021461686, -0.034179244, 0.03242367, 0.045696042, -0.10642838, 0.044042706, 0.003619815, -0.03445944, 0.06782116, -0.012801977, -0.083491564, 0.048063237, -0.0009263491, 0.03926016, -0.003800945, 0.06431806, 0.008804617, 0.041459076, 0.019176882, 0.063215, 0.016872335, -0.07120825, 0.0026858407, -0.0061372668, 0.016006729, 0.034623176, -0.0009702338, 0.05586387, -0.0030038806, 0.10219119, 0.023867028, 0.017003942, 0.07522453, 0.03827543, 0.002119465, -0.047579825, 0.030801363, 0.055104297, -0.00926156, 0.060950216, -0.012564041, -0.0938483, 0.06749232, 0.0303093, 0.1260211, 0.008772238, 0.0937941, 0.03146898, -0.013548525, -0.04654987, 0.038247738, -0.0047283196, -0.021979854, -0.04481472, 0.009184976, 0.030558616, -0.035239127, 0.015711905, 0.079948395, -0.10273533, -0.033666693, 0.009253284, -0.013218568, 0.014513645, 0.011746366, -0.04836566, 0.00059039996, 0.056465007, 0.057913274, 0.046911363, 0.022496173, -0.016504057, -0.0009266135, 0.007562665, 0.024523543, 0.012681347, -0.0034720704, 0.014897689, 0.034027215, -0.035149213, 0.046610955, -0.38038146, -0.05560348, 0.056164417, 0.023633359, -0.020914413, 0.0017839101, 0.043425612, 0.0921522, 0.021333266, 0.032627117, 0.052366074, 0.059688427, -0.02425017, 0.07460727, 0.040419403, 0.018662684, -0.02174095, -0.015262358, 0.0041535227, -0.004320668, 0.001545062, 0.023696192, 0.053526532, 0.031027582, -0.030727778, -0.07266011, 0.01924883, -0.021610625, 0.03179455, -0.002117363, 0.037670195, -0.021235954, -0.03931032, -0.057163127, -0.046020538, 0.013852293, 0.007136301, 0.020461356, 0.027465757, 0.013625788, 0.09281521, 0.03537469, -0.15295835, -0.045262642, 0.013799362, 0.029831719, 0.06360841, 0.045387108, -0.008106462, 0.047562532, 0.026519125, 0.030519808, -0.035604805, 0.059504308, -0.010260606, 0.05920231, -0.039987702, 0.003475537, 0.012535757, 0.03711557, 0.022637982, 0.022368006, -0.013918498, 0.03144229, 0.02680179, 0.05283082, 0.09737034, 0.062140185, 0.047479317, 0.04292394, 0.041657448, 0.031671192, -0.01198203, -0.0398639, 0.050961364, -0.005440624, -0.013748672, 0.02486566, 0.06105261, 0.09158345, 0.047486037, 0.03503525, -0.0009857323, 0.017584834, 0.0015176772, -0.013855697, -0.0016783233, -0.032760657, 0.0073869363, 0.0032070065, 0.08748817, 0.062042974, -0.006563574, -0.01277716, 0.064277925, -0.048509046, 0.01998247, 0.015449057, 0.06161844, 0.0361277, 0.07378269, 0.031909943, 0.035593968, -0.021533003, 0.15151453, 0.009489467, 0.0077385777, 0.004732935, 0.06757376, 0.018628953, 0.03609718, 0.065334365, 0.046664603, 0.03710433, 0.023046834, 0.065034136, 0.021973003, 0.01938253, 0.0049545416, 0.009443422, 0.08657203, -0.006455585, 0.06113277, -0.009921393, 0.008861325, 0.021925068, 0.0073863543, 0.029231662, 0.018063372, -0.028237753, 0.06752595, -0.015746683, -0.06744447, -0.0019776542, -0.16144808, 0.055144247, -0.07052258, -0.0062173936, 0.005187277, 0.057623632, 0.008336536, 0.018794686, 0.08856226, 0.05324669, 0.023925344, -0.011277585, -0.015746504, -0.01888707, -0.010619123, 0.05960752, -0.02111604, 0.13263386, 0.053238407, 0.0423469, 0.03247613, 0.072818235, 0.039493106, -0.0080635715, 0.038805183, 0.05633994, 0.021095807, -0.022528276, 0.113213256, -0.040802993, 0.01971789, 0.00073800184, 0.04653605, 0.024364496, 0.051224973, 0.022803178, 0.06527072, -0.030100288, 0.02277551, 0.034268156, -0.0024341822, 0.030275142, -0.0043326514, 0.026949842, 0.03554525, 0.043582354, 0.037845742, 0.024644673, 0.06225431, 0.06668994, 0.042802095, -0.14308476, 0.028445719, -0.0057268543, 0.034851402, 0.04973769, -0.01673276, -0.0084733, -0.04498498, -0.01888843, 0.0018199912, -0.08666151, 0.03408551, 0.03374362, 0.016341621, -0.017816868, 0.027611718, 0.048712954, 0.03562084, 0.06156702, 0.06942091, 0.018424997, 0.010069236, -0.025854982, -0.005099922, 0.042129293, -0.018960087, -0.04267046, 0.003192464, 0.07610024, 0.01623567, 0.06430824, 0.045628317, -0.13192567, 0.00597194, 0.03359213, -0.051644783, -0.027538724, 0.047537625, 0.00078535493, -0.050269134, 0.06352181, 0.04414142, -0.00025181545, -0.011166945, 0.083493516, -0.022445189, 0.06386556, 0.009009819, 0.018880796, 0.046981215, -0.04803033, 0.20140722, 0.009405448, 0.011427641, 0.032028355, -0.039911997, 0.059231583, 0.10603366, -0.012695404, -0.018773954, 0.051107403, 0.004720434, 0.049031533, 0.008848073, -0.008443017, 0.068459414, -0.001594059, -0.037717424, 0.0083658025, 0.036570624, -0.009189262, -0.07422237, -0.03578154, 0.00016998129, -0.033594534, 0.04550856, -0.09751915, 0.031381045, -0.020289807, -0.025066, 0.05559659, 0.065852426, -0.030574895, 0.098877095, 0.024548644, 0.02716826, -0.0073690503, -0.006680294, -0.062504984, 0.001748584, -0.0015254011, 0.0030000636, 0.05166639, -0.03598367, 0.02785021, 0.019170346, -0.01893702, 0.006487694, -0.045320857, -0.042290565, 0.030072719]}], object='list', usage=Usage(prompt_tokens=8, total_tokens=8))\n", - "Slate 125m embeddings response:\n", - "EmbeddingResponse(model='ibm/slate-125m-english-rtrvr', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.037463713, -0.02141933, -0.02851813, 0.015519324, -0.08252965, 0.040418413, 0.0125358505, -0.015099016, 0.007372251, 0.043594047, -0.045923322, -0.024535796, -0.06683439, -0.023252856, -0.014445329, -0.007990043, -0.0038893714, 0.024145052, 0.002840671, -0.005213263, 0.025767032, -0.029234663, -0.022147253, -0.04008686, -0.0049467147, -0.005722156, 0.05712166, 0.02074406, -0.027984975, 0.011733741, 0.037084717, 0.0267332, 0.027662167, 0.018661365, 0.034368176, -0.016858159, 0.01525097, 0.0037685328, -0.029145032, -0.014014788, -0.026596593, -0.019313056, -0.034545943, -0.012755116, -0.027378004, -0.0022658114, 0.0671108, -0.011186887, -0.012560194, 0.07890564, 0.04370288, -0.002565922, 0.04558289, -0.015022389, 0.01721297, -0.02836881, 0.00028577668, 0.041560214, -0.028451115, 0.026690092, -0.03240052, 0.043185145, -0.048146088, -0.01863734, 0.014189055, 0.005409885, -0.004303547, 0.043854367, -0.08027855, 0.0036468406, -0.03761452, -0.01586453, 0.0015843573, -0.06557115, -0.017214078, 0.013112075, -0.063624665, -0.059002113, -0.027906772, -0.0104140695, -0.0122148385, 0.002914942, 0.009600896, 0.024618316, 0.0028588492, -0.04129038, -0.0066302163, -0.016593395, 0.0119156595, 0.030668158, 0.032204323, -0.008526114, 0.031477567, -0.027671225, -0.021325896, -0.012719999, 0.020595504, -0.010196725, 0.016694892, 0.015447107, 0.033599768, 0.0015109212, 0.055442166, -0.032922138, 0.032867074, 0.034223255, 0.018267235, 0.044258785, -0.009512916, -0.01888108, 0.0020811916, -0.071849406, -0.029209733, 0.030071445, 0.04898721, 0.03807559, 0.030091342, 0.0049845255, 0.011301079, 0.0060062855, -0.052550614, -0.040027767, -0.04539995, -0.069943875, 0.052881725, 0.015551356, -0.0016604571, 0.0021608798, 0.055507053, -0.015404854, -0.0023839937, 0.0070840786, 0.042537935, -0.045489613, 0.018908504, -0.015565469, 0.015916781, 0.07333876, 0.0034915418, -0.0029724848, 0.019170308, 0.02221138, -0.027242986, -0.003735747, -0.02341423, -0.0037938543, 0.0104211755, -0.06185881, -0.036718667, -0.02746382, -0.026462527, -0.050701175, 0.0057923957, 0.040674523, -0.019840682, -0.030195065, 0.045316722, 0.017369563, -0.031288657, -0.047546197, 0.026255054, -0.0049950704, -0.040272273, 0.0005752177, 0.03959872, -0.0073655704, -0.025617458, -0.009416491, -0.019514928, -0.07619169, 0.0051972694, 0.016387343, -0.012366861, -0.009152257, -0.035955105, -0.05794065, 0.019153351, -0.0461187, 0.024734644, 0.0031722176, 0.06610593, -0.0046516205, -0.04635891, 0.02524459, 0.004230386, 0.06153266, -0.0008394812, -0.013522857, 0.029861225, -0.00394871, -0.037432022, 0.0483034, 0.02181303, 0.015967155, 0.06181817, -0.018545056, 0.044176213, -0.07024062, -0.013022128, -0.0087189535, -0.025292343, 0.040448178, -0.051455554, -0.014017804, 0.012191985, 0.0071282317, -0.015855217, 0.013618914, -0.0060378346, -0.057781402, -0.035322957, -0.013627626, -0.027318006, -0.27732822, -0.007108157, 0.012321971, -0.15896526, -0.03793523, -0.025426138, 0.020721687, -0.04701553, -0.004927499, 0.010541978, -0.003212021, -0.0023603817, -0.052153032, 0.043272667, 0.024041472, -0.031666223, 0.0017891804, 0.026806207, -0.026526717, 0.0023138188, 0.024067048, 0.03326347, -0.039004102, -0.0004279829, 0.007266309, -0.008940641, 0.03715139, -0.037960306, 0.01647343, -0.022163782, 0.07456727, -0.0013284415, -0.029121747, 0.012727488, -0.007229313, 0.03177136, -0.08142398, 0.010223168, -0.025942598, -0.23807198, 0.022616733, -0.03925926, 0.05572623, -0.00020389797, -0.0022259122, -0.007885641, -0.00719495, 0.0018412926, 0.018953165, -0.009946787, 0.03723944, -0.015900994, 0.013648507, 0.010997674, -0.018918132, 0.013143112, 0.032894272, -0.05800237, 0.011163258, 0.025205074, -0.017001726, 0.03673705, -0.011551997, 0.06637543, -0.033003606, -0.041392814, -0.004078506, 0.03916763, -0.0022711542, 0.058338877, -0.034323692, -0.033700593, 0.01051642, 0.035579532, -0.01997833, 0.002977113, 0.06590587, 0.042783573, 0.020624464, 0.029172791, -0.035136282, 0.02035436, 0.05696583, -0.010200334, -0.0010580813, -0.024785697, -0.014516442, -0.030100575, -0.03807279, 0.042534467, -0.0281041, -0.05331885, -0.019467393, 0.016051197, 0.012470333, -0.008369627, 0.002254233, 0.026580654, -0.04541506, -0.018085537, -0.034577485, -0.0014747214, 0.0005770179, 0.0043190396, -0.004989785, 0.007569717, 0.010167482, -0.03335266, -0.015255423, 0.07341545, 0.012114007, -0.0010415721, 0.008754641, 0.05932771, 0.030799353, 0.026148474, -0.0069155577, -0.056865778, 0.0038446637, -0.010079895, 0.013511311, 0.023351224, -0.049000103, -0.013028001, -0.04957143, -0.031393193, 0.040289443, 0.063747466, 0.046358805, 0.0023754216, -0.0054107807, -0.020128531, 0.0013747461, -0.018183928, -0.04754063, -0.0064625163, 0.0417791, 0.06087331, -0.012241535, 0.04185439, 0.03641727, -0.02044306, -0.061368305, -0.023353308, 0.055897385, -0.047081504, 0.012900442, -0.018708078, 0.0028819577, 0.006964468, 0.0008757072, 0.04605831, 0.01716345, -0.004099444, -0.015493673, 0.021323929, -0.011252118, -0.02278577, 0.01893121, 0.009134488, 0.021568391, 0.011066748, -0.018853422, 0.027866907, -0.02831057, -0.010147286, 0.014807969, -0.03266599, -0.06711559, 0.038546126, 0.0031859868, -0.029038243, 0.046595056, 0.036973156, -0.033408422, 0.021968717, -0.011411975, 0.006584961, 0.072844714, -0.005873538, 0.029435376, 0.061169676, -0.02318868, 0.051129397, 0.014791153, -0.009028991, -0.021579748, 0.02669236, 0.029696332, -0.063952625, -0.061506465, -0.00080902094, 0.06850867, -0.09809231, -0.005534635, 0.066767104, -0.041267477, 0.046568397, 0.00983124, -0.0048434925, 0.038644254, 0.04096419, 0.0023063375, 0.014526287, 0.014016995, 0.020224908, 0.007113328, -0.0732543, -0.0054818415, 0.05807576, 0.022461535, 0.21100426, -0.009597197, -0.020674499, 0.010743241, -0.046834, -0.0068005333, 0.04918187, -0.06680011, -0.025018543, 0.016360015, 0.100744724, -0.019944709, -0.052390855, -0.0034876189, 0.031699855, -0.03024188, 0.009384044, -0.073849924, 0.01846066, -0.017075414, 0.0067319535, 0.045643695, 0.0121267075, 0.014980903, -0.0022226444, -0.015187039, 0.040638167, 0.023607453, -0.018353134, 0.007413985, 0.03487914, 0.018997269, -0.0107962405, -0.0040080273, 0.001454658, -0.023004232, -0.03065838, -0.0691732, -0.009669473, -0.017253181, 0.100617275, -0.00028453665, -0.055184573, -0.04010461, -0.022628073, -0.02138574, -0.00011931983, -0.021988528, 0.021569526, 0.018913478, -0.07588871, -0.030895703, -0.045679674, 0.03548181, 0.05806986, -0.00313453, 0.005607964, 0.014474551, -0.016833752, -0.022846023, 0.03665983, 0.04312398, 0.006030178, 0.020107903, -0.067837745, -0.039261904, -0.013903933, -0.011238981, -0.091779895, 0.03393072, 0.03576862, -0.016447216, -0.013628061, 0.035994843, 0.02442105, 0.0013356373, -0.013639993, -0.0070654624, -0.031047037, 0.0321763, 0.019488426, 0.030912274, -0.018131692, 0.034129236, -0.038152352, -0.020318052, 0.012934771, -0.0038958737, 0.029313264, 0.0609006, -0.06022117, -0.016697206, -0.030089315, -0.0030464267, -0.05011375, 0.016849633, -0.01935251, 0.00033423092, 0.018090008, 0.034528963, 0.015720658, 0.006443832, 0.0024674414, 0.0033006326, -0.011959118, -0.014686165, 0.00851113, 0.032130115, 0.016566927, -0.0048006177, -0.041135546, 0.017366901, 0.014404645, 0.0014093819, -0.039899524, -0.020875102, -0.01322629, -0.010891931, 0.019460721, -0.098985165, -0.03990147, 0.035807386, 0.05274234, -0.017714208, 0.0023620757, 0.022553496, 0.010935722, -0.016535437, -0.014505468, -0.005573891, -0.029528206, -0.010998497, 0.011297328, 0.007440231, 0.054734096, -0.035311602, 0.07038191, -0.034328025, -0.0109814005, -0.00578824, -0.009286793, 0.06692834, -0.040116422, -0.030043483, -0.010882302, -0.024094587, 0.026659116, -0.0637435, -0.022305744, 0.024388585, 0.011812823, -0.022778027, -0.0039024823, 0.027778644, 0.010566278, 0.011030791, -0.0021155484, 0.018014789, -0.03458981, 0.02546183, -0.11745906, 0.038193583, 0.0019787792, 0.01639592, 0.013218127, -0.012434678, -0.047858853, 0.006662704, 0.033221778, 0.008376927, -0.011822234, 0.01202769, 0.008761578, -0.04075117, 0.0025187496, 0.0026266004, 0.029762473, 0.009570205, -0.03644678, -0.033258904, -0.030776607, 0.05373578, 0.010904848, 0.040284622, 0.02707032, 0.021803873, -0.022011256, -0.05517991, -0.005213912, 0.009023477, -0.011895841, -0.026821174, -0.009035418, -0.021059638, 0.025536137, -0.053264923, 0.032206282, 0.020235807, 0.018660447, 0.0028790566, -0.019914437, 0.097842626, 0.027617158, 0.020276038, -0.014215543, 0.012761584, 0.032757074, 0.061124176, 0.049016643, -0.016509317, -0.03750349, -0.03449537, -0.02039439, -0.051360182, -0.041909404, 0.016175032, 0.040492736, 0.031218654, 0.0020242895, -0.032167237, 0.019398497, 0.057013687, 0.0031299617, 0.019177254, 0.015395364, -0.034078192, 0.041325297, 0.044380017, -0.004446819, 0.019610956, -0.030034903, 0.008468295, 0.03065914, -0.009548659, -0.07113981, 0.051648173, 0.03746448, -0.021847434, 0.01844844, 0.01333424, -0.001188216, 0.012330977, -0.056448817, 0.0008659569, 0.011183285, 0.006780519, -0.007357356, 0.05263679, -0.024631461, 0.00519591, -0.052165415, -0.03250626, -0.009370051, 0.00292325, -0.007187242, 0.029566163, -0.049605303, -0.02625627, -0.003157652, 0.052691437, -0.03589223, 0.03889354, -0.0035060279, 0.024555178, -0.00929779, -0.05037946, -0.022402484, 0.030634355, -0.03300659, -0.0063623153, 0.0027472514, 0.03196768, -0.019257778, 0.0089001395, 0.008908001, 0.018918095, 0.059574094, -0.02838763, 0.018203752, -0.06708146, -0.022670228, -0.013985525, 0.045018435, 0.011420395, -0.008649952, -0.027328938, -0.03527292, -0.0038555951, 0.017597001, 0.024891963, -0.0039160745, -0.015237065, -0.0008723479, -0.018641612, -0.036825016, -0.028743235, 0.00091956893, 0.00030935413, -0.048641082, 0.03744432, -0.024196126, 0.009848505, -0.043836866, 0.0044429195, 0.013709644, 0.06295503, -0.016072558, 0.01277375, -0.03548109, 0.003398656, 0.025347201, 0.019685786, 0.00758199, -0.016122513, -0.039198015, -0.0023108267, -0.0041584945, 0.005161282, 0.00089106365, 0.0076085874, -0.055768084, -0.0058975955, 0.007728267, 0.00076985586, -0.013469806, -0.031578194, -0.0138569595, 0.044540506, -0.0408136, -0.015252405, 0.06232591, -0.04198101, 0.0048899655, -0.0030694627, -0.025022805, -0.010789543, -0.025350742, 0.007836728, 0.024604483, -5.385127e-05, -0.0021367231, -0.01704561, -0.001425816, 0.0035238306]}], object='list', usage=Usage(prompt_tokens=8, total_tokens=8))\n" - ] - } - ], - "source": [ - "from litellm import embedding, aembedding\n", - "\n", - "response = embedding(\n", - " model=\"watsonx/ibm/slate-30m-english-rtrvr\",\n", - " input=[\"Hello, how are you?\"],\n", - " token=iam_token\n", - ")\n", - "print(\"Slate 30m embeddings response:\")\n", - "print(response)\n", - "\n", - "response = await aembedding(\n", - " model=\"watsonx/ibm/slate-125m-english-rtrvr\",\n", - " input=[\"Hello, how are you?\"],\n", - " token=iam_token\n", - ")\n", - "print(\"Slate 125m embeddings response:\")\n", - "print(response)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "base", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/liteLLM_Langchain_Demo.ipynb b/cookbook/liteLLM_Langchain_Demo.ipynb deleted file mode 100644 index 6e796dd085c..00000000000 --- a/cookbook/liteLLM_Langchain_Demo.ipynb +++ /dev/null @@ -1,195 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "5hwntUxTMxEk" - }, - "source": [ - "# Langchain liteLLM Demo Notebook\n", - "## Use `ChatLiteLLM()` to instantly support 50+ LLM models\n", - "Langchain Docs: https://python.langchain.com/docs/integrations/chat/litellm\n", - "\n", - "Call all LLM models using the same I/O interface\n", - "\n", - "Example usage\n", - "```python\n", - "ChatLiteLLM(model=\"gpt-3.5-turbo\")\n", - "ChatLiteLLM(model=\"claude-2\", temperature=0.3)\n", - "ChatLiteLLM(model=\"command-nightly\")\n", - "ChatLiteLLM(model=\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\")\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "aPNAUsCvB6Sv" - }, - "outputs": [], - "source": [ - "!pip install litellm langchain" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "MOhRaVnhB-0J" - }, - "outputs": [], - "source": [ - "import os\n", - "from langchain.chat_models import ChatLiteLLM\n", - "from langchain.schema import HumanMessage" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "TahkCtlmCD65", - "outputId": "5ddda40f-f252-4830-a8d6-bd3fa68ae487" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "AIMessage(content='I am an AI model known as GPT-3, developed by OpenAI.', additional_kwargs={}, example=False)" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.environ['OPENAI_API_KEY'] = \"\"\n", - "chat = ChatLiteLLM(model=\"gpt-3.5-turbo\")\n", - "messages = [\n", - " HumanMessage(\n", - " content=\"what model are you\"\n", - " )\n", - "]\n", - "chat(messages)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "uXNDyU4jChcs", - "outputId": "bd74b4c6-f9fb-42dc-fdc3-9240d50503ba" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "AIMessage(content=\" I'm Claude, an AI assistant created by Anthropic.\", additional_kwargs={}, example=False)" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.environ['ANTHROPIC_API_KEY'] = \"\"\n", - "chat = ChatLiteLLM(model=\"claude-2\", temperature=0.3)\n", - "messages = [\n", - " HumanMessage(\n", - " content=\"what model are you\"\n", - " )\n", - "]\n", - "chat(messages)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "czbDJRKcC7BV", - "outputId": "892e147d-831e-4884-dc71-040f92c3fb8e" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "AIMessage(content=\" I'm an AI based based on LLaMA models (LLaMA: Open and Efficient Foundation Language Models, Touvron et al. 2023), my knowledge was built from a massive corpus of text, including books, articles, and websites, and I was trained using a variety of machine learning algorithms. My model architecture is based on the transformer architecture, which is particularly well-suited for natural language processing tasks. My team of developers and I are constantly working to improve and fine-tune my performance, and I am always happy to help with any questions you may have!\", additional_kwargs={}, example=False)" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.environ['REPLICATE_API_TOKEN'] = \"\"\n", - "chat = ChatLiteLLM(model=\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\")\n", - "messages = [\n", - " HumanMessage(\n", - " content=\"what model are you?\"\n", - " )\n", - "]\n", - "chat(messages)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tZxpq5PDDY9Y", - "outputId": "7e86f4ed-ac7a-45e1-87d0-217da6cad666" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "AIMessage(content=' I am an AI-based large language model, or Chatbot, built by the company Cohere. I am designed to have polite, helpful, inclusive conversations with users. I am always learning and improving, and I am constantly being updated with new information and improvements.\\n\\nI am currently in the development phase, and I am not yet available to the general public. However, I am currently being used by a select group of users for testing and feedback.\\n\\nI am a large language model, which means that I am trained on a massive amount of data and can understand and respond to a wide range of requests and questions. I am also designed to be flexible and adaptable, so I can be customized to suit the needs of different users and use cases.\\n\\nI am currently being used to develop a range of applications, including customer service chatbots, content generation tools, and language translation services. I am also being used to train other language models and to develop new ways of using large language models.\\n\\nI am constantly being updated with new information and improvements, so I am always learning and improving. I am also being used to develop new ways of using large language models, so I am always evolving and adapting to new use cases and requirements.', additional_kwargs={}, example=False)" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.environ['COHERE_API_KEY'] = \"\"\n", - "chat = ChatLiteLLM(model=\"command-nightly\")\n", - "messages = [\n", - " HumanMessage(\n", - " content=\"what model are you?\"\n", - " )\n", - "]\n", - "chat(messages)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/liteLLM_Ollama.ipynb b/cookbook/liteLLM_Ollama.ipynb deleted file mode 100644 index fe0ed3811fa..00000000000 --- a/cookbook/liteLLM_Ollama.ipynb +++ /dev/null @@ -1,289 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install litellm # version 0.1.724 or higher " - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Call Ollama - llama2 with Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "{'role': 'assistant', 'content': ' I'}\n", - "{'role': 'assistant', 'content': \"'\"}\n", - "{'role': 'assistant', 'content': 'm'}\n", - "{'role': 'assistant', 'content': ' L'}\n", - "{'role': 'assistant', 'content': 'La'}\n", - "{'role': 'assistant', 'content': 'MA'}\n", - "{'role': 'assistant', 'content': ','}\n", - "{'role': 'assistant', 'content': ' an'}\n", - "{'role': 'assistant', 'content': ' A'}\n", - "{'role': 'assistant', 'content': 'I'}\n", - "{'role': 'assistant', 'content': ' assistant'}\n", - "{'role': 'assistant', 'content': ' developed'}\n", - "{'role': 'assistant', 'content': ' by'}\n", - "{'role': 'assistant', 'content': ' Meta'}\n", - "{'role': 'assistant', 'content': ' A'}\n", - "{'role': 'assistant', 'content': 'I'}\n", - "{'role': 'assistant', 'content': ' that'}\n", - "{'role': 'assistant', 'content': ' can'}\n", - "{'role': 'assistant', 'content': ' understand'}\n", - "{'role': 'assistant', 'content': ' and'}\n", - "{'role': 'assistant', 'content': ' respond'}\n", - "{'role': 'assistant', 'content': ' to'}\n", - "{'role': 'assistant', 'content': ' human'}\n", - "{'role': 'assistant', 'content': ' input'}\n", - "{'role': 'assistant', 'content': ' in'}\n", - "{'role': 'assistant', 'content': ' a'}\n", - "{'role': 'assistant', 'content': ' convers'}\n", - "{'role': 'assistant', 'content': 'ational'}\n", - "{'role': 'assistant', 'content': ' manner'}\n", - "{'role': 'assistant', 'content': '.'}\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(\n", - " model=\"ollama/llama2\", \n", - " messages=[{ \"content\": \"respond in 20 words. who are you?\",\"role\": \"user\"}], \n", - " api_base=\"http://localhost:11434\",\n", - " stream=True\n", - ")\n", - "print(response)\n", - "for chunk in response:\n", - " print(chunk['choices'][0]['delta'])\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Call Ollama - Llama2 with Acompletion + Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: async_generator in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (1.10)\n" - ] - } - ], - "source": [ - "# litellm uses async_generator for ollama async streaming, ensure it's installed\n", - "!pip install async_generator" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'choices': [{'delta': {'role': 'assistant', 'content': ' I'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'm'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' just'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' an'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' A'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'I'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' I'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' don'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 't'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' access'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' real'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'time'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' weather'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' information'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' or'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' current'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' conditions'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' location'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' живело'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' provide'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' weather'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' forec'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'asts'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' information'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' location'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' would'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' like'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Please'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' let'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' me'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' know'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' where'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' located'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' I'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' will'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' do'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' my'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' best'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' assist'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "None\n" - ] - } - ], - "source": [ - "import litellm\n", - "\n", - "async def async_ollama():\n", - " response = await litellm.acompletion(\n", - " model=\"ollama/llama2\", \n", - " messages=[{ \"content\": \"what's the weather\" ,\"role\": \"user\"}], \n", - " api_base=\"http://localhost:11434\", \n", - " stream=True\n", - " )\n", - " async for chunk in response:\n", - " print(chunk)\n", - "\n", - "result = await async_ollama()\n", - "print(result)\n", - "\n", - "try:\n", - " async for chunk in result:\n", - " print(chunk)\n", - "except TypeError: # the last chunk is None from Ollama, this raises an error with async streaming\n", - " pass" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Completion Call" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" I'm LLaMA, an AI assistant developed by Meta AI that can understand and respond to human input in a conversational manner.\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-ea7b8242-791f-4656-ba12-e098edeb960e\",\n", - " \"created\": 1695324686.6696231,\n", - " \"response_ms\": 4072.3050000000003,\n", - " \"model\": \"ollama/llama2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 10,\n", - " \"completion_tokens\": 27,\n", - " \"total_tokens\": 37\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(\n", - " model=\"ollama/llama2\", \n", - " messages=[{ \"content\": \"respond in 20 words. who are you?\",\"role\": \"user\"}], \n", - " api_base=\"http://localhost:11434\"\n", - ")\n", - "print(response)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/liteLLM_Replicate_Demo.ipynb b/cookbook/liteLLM_Replicate_Demo.ipynb deleted file mode 100644 index b93d9a587b8..00000000000 --- a/cookbook/liteLLM_Replicate_Demo.ipynb +++ /dev/null @@ -1,238 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "YV6L5fNv7Kep" - }, - "source": [ - "# Call Replicate LLMs using chatGPT Input/Output Format\n", - "This tutorial covers using the following Replicate Models with liteLLM\n", - "\n", - "- [StableLM Tuned Alpha 7B](https://replicate.com/stability-ai/stablelm-tuned-alpha-7b)\n", - "- [LLAMA-2 70B Chat](https://replicate.com/replicate/llama-2-70b-chat)\n", - "- [A16z infra-LLAMA-2 7B Chat](https://replicate.com/a16z-infra/llama-2-7b-chat)\n", - "- [Dolly V2 12B](https://replicate.com/replicate/dolly-v2-12b)\n", - "- [Vicuna 13B](https://replicate.com/replicate/vicuna-13b)\n", - "\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "TO-EdF84O9QT" - }, - "outputs": [], - "source": [ - "# install liteLLM\n", - "!pip install litellm" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "mpHTbTqQ8fey" - }, - "source": [ - "Imports & Set ENV variables\n", - "Get your Replicate Key: https://replicate.com/account/api-tokens" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "kDbgfcU8O-dW" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "import os\n", - "os.environ['REPLICATE_API_TOKEN'] = ' ' # @param\n", - "user_message = \"Hello, whats the weather in San Francisco??\"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "1KmkOdzLSOmJ" - }, - "source": [ - "## Call Replicate Models using completion(model, messages) - chatGPT format" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "XJ4nh4SnRzHP", - "outputId": "986c0544-bb40-4915-f00f-498b0e518307" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "replicate is not installed. Installing...\n", - "Response from stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb \n", - "]\n", - "\n", - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \"I'm sorry for you being unable to access this content as my training data only goes up until 2023/03. However I can tell you what your local weather forecast may look like at any time of year with respect to current conditions:\"}}], 'created': 1691611730.7224207, 'model': 'stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb', 'usage': {'prompt_tokens': 9, 'completion_tokens': 49, 'total_tokens': 58}}\n", - "Response from replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1 \n", - "]\n", - "\n", - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" Hello! I'm happy to help you with your question. However, I must point out that the question itself may not be meaningful. San Francisco is a city located in California, USA, and it is not possible for me to provide you with the current weather conditions there as I am a text-based AI language model and do not have access to real-time weather data. Additionally, the weather in San Francisco can vary greatly depending on the time of year, so it would be best to check a reliable weather source for the most up-to-date information.\\n\\nIf you meant to ask a different question, please feel free to rephrase it, and I will do my best to assist you in a safe and positive manner.\"}}], 'created': 1691611745.0269957, 'model': 'replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1', 'usage': {'prompt_tokens': 9, 'completion_tokens': 143, 'total_tokens': 152}}\n", - "Response from a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc \n", - "]\n", - "\n", - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" Hello! I'm here to help you with your question. However, I must inform you that the weather in San Francisco can be quite unpredictable and can change rapidly. It's important to check reliable sources such as AccuWeather or the National Weather Service for the most up-to-date and accurate information about the weather in San Francisco.\\nI cannot provide you with real-time weather data or forecasts as I'm just an AI and do not have access to current weather conditions or predictions. But I can suggest some trustworthy websites or apps where you can find the latest weather updates:\\n* AccuWeather (accuweather.com)\\n* The Weather Channel (weather.com)\\n* Dark Sky (darksky.net)\\n* Weather Underground (wunderground.com)\\nRemember, it's always best to consult multiple sources for the most accurate information when planning your day or trip. Enjoy your day!\"}}], 'created': 1691611748.7723358, 'model': 'a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc', 'usage': {'prompt_tokens': 9, 'completion_tokens': 174, 'total_tokens': 183}}\n", - "Response from replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5 \n", - "]\n", - "\n", - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': 'Its 68 degrees right now in San Francisco! The temperature will be rising through the week and i expect it to reach 70 on Thursdays and Friday. Skies are expected to be partly cloudy with some sun breaks throughout the day.\\n\\n'}}], 'created': 1691611752.2002115, 'model': 'replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5', 'usage': {'prompt_tokens': 9, 'completion_tokens': 48, 'total_tokens': 57}}\n", - "Response from replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b \n", - "]\n", - "\n", - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ''}}], 'created': 1691611752.8998356, 'model': 'replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b', 'usage': {'prompt_tokens': 9, 'completion_tokens': 0, 'total_tokens': 9}}\n" - ] - } - ], - "source": [ - "llama_2 = \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n", - "llama_2_7b = \"a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\"\n", - "dolly_v2 = \"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5\"\n", - "vicuna = \"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\"\n", - "models = [llama_2, llama_2_7b, dolly_v2, vicuna]\n", - "for model in models:\n", - " response = completion(model=model, messages=messages)\n", - " print(f\"Response from {model} \\n]\\n\")\n", - " print(response)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "zlTVLB-7PTV_", - "outputId": "5182275b-3108-46fa-a2cf-745fac4ad110" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hi\n", - " there!\n", - " The\n", - " current\n", - " forecast\n", - " for\n", - " today's\n", - " high\n", - " temperature\n", - " ranges\n", - " from\n", - " 75\n", - " degrees\n", - " Fahrenheit\n", - " all\n", - " day\n", - " to\n", - " 83\n", - " degrees\n", - " Fahrenheit\n", - " with\n", - " possible\n", - " isolated\n", - " thunderstorms\n", - " during\n", - " the\n", - " afternoon\n", - " hours,\n", - " mainly\n", - " at\n", - " sunset\n", - " through\n", - " early\n", - " evening. The\n", - " Pacific\n", - " Ocean\n", - " has\n", - " a\n", - " low\n", - " pressure\n", - " of\n", - " 926\n", - " mb\n", - " and\n", - " mostly\n", - " cloud\n", - " cover\n", - " in\n", - " this\n", - " region\n", - " on\n", - " sunny\n", - " days\n", - " due\n", - " to\n", - " warming\n", - " temperatures\n", - " above\n", - " average\n", - " along\n", - " most\n", - " coastal\n", - " areas\n", - " and\n", - " ocean\n", - " breezes.<|USER|>\n" - ] - } - ], - "source": [ - "# @title Stream Responses from Replicate - Outputs in the same format used by chatGPT streaming\n", - "response = completion(model=llama_2, messages=messages, stream=True)\n", - "\n", - "for chunk in response:\n", - " print(chunk['choices'][0]['delta'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "t7WMRuL-8NrO" - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/liteLLM_Streaming_Demo.ipynb b/cookbook/liteLLM_Streaming_Demo.ipynb deleted file mode 100644 index 0456c54514b..00000000000 --- a/cookbook/liteLLM_Streaming_Demo.ipynb +++ /dev/null @@ -1,226 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# [STREAMING] OpenAI, Anthropic, Replicate, Cohere using liteLLM\n", - "In this tutorial:\n", - "Note: All inputs/outputs are in the format used by `gpt-3.5-turbo`\n", - "\n", - "- Call all models in the same input format [**with streaming**]:\n", - "\n", - " `completion(model, messages, stream=True)`\n", - "- All streaming generators are accessed at `chunk['choices'][0]['delta']`\n", - "\n", - "The following Models are covered in this tutorial\n", - "- [GPT-3.5-Turbo](https://platform.openai.com/docs/models/gpt-3-5)\n", - "- [Claude-2](https://www.anthropic.com/index/claude-2)\n", - "- [StableLM Tuned Alpha 7B](https://replicate.com/stability-ai/stablelm-tuned-alpha-7b)\n", - "- [A16z infra-LLAMA-2 7B Chat](https://replicate.com/a16z-infra/llama-2-7b-chat)\n", - "- [Vicuna 13B](https://replicate.com/replicate/vicuna-13b)\n", - "- [Cohere - Command Nightly]()\n", - "\n", - "\n", - "\n" - ], - "metadata": { - "id": "YV6L5fNv7Kep" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "TO-EdF84O9QT" - }, - "outputs": [], - "source": [ - "# install liteLLM\n", - "!pip install litellm==0.1.369" - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Imports & Set ENV variables\n", - "Get your API Keys\n", - "\n", - "https://platform.openai.com/account/api-keys\n", - "\n", - "https://replicate.com/account/api-tokens\n", - "\n", - "https://console.anthropic.com/account/keys\n", - "\n", - "https://dashboard.cohere.ai/api-keys\n" - ], - "metadata": { - "id": "mpHTbTqQ8fey" - } - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion\n", - "import os\n", - "\n", - "os.environ['OPENAI_API_KEY'] = '' # @param\n", - "os.environ['REPLICATE_API_TOKEN'] = '' # @param\n", - "os.environ['ANTHROPIC_API_KEY'] = '' # @param\n", - "os.environ['COHERE_API_KEY'] = '' # @param" - ], - "metadata": { - "id": "kDbgfcU8O-dW" - }, - "execution_count": 8, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "### Set Messages" - ], - "metadata": { - "id": "1KmkOdzLSOmJ" - } - }, - { - "cell_type": "code", - "source": [ - "user_message = \"Hello, whats the weather in San Francisco??\"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]" - ], - "metadata": { - "id": "xIEeOhVH-oh6" - }, - "execution_count": 4, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Calling Models using liteLLM Streaming -\n", - "\n", - "## `completion(model, messages, stream)`" - ], - "metadata": { - "id": "9SOCVRC1L-G3" - } - }, - { - "cell_type": "code", - "source": [ - "# replicate models #######\n", - "stability_ai = \"stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb\"\n", - "llama_2_7b = \"a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\"\n", - "vicuna = \"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\"\n", - "\n", - "models = [\"gpt-3.5-turbo\", \"claude-2\", stability_ai, llama_2_7b, vicuna, \"command-nightly\"] # command-nightly is Cohere\n", - "for model in models:\n", - " replicate = (model == stability_ai or model==llama_2_7b or model==vicuna) # let liteLLM know if a model is replicate, using this optional param, `replicate=True`\n", - " response = completion(model=model, messages=messages, stream=True, replicate=replicate)\n", - " print(f\"####################\\n\\nResponse from {model}\")\n", - " for i, chunk in enumerate(response):\n", - " if i < 5: # NOTE: LIMITING CHUNKS FOR THIS DEMO\n", - " print((chunk['choices'][0]['delta']))\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "XJ4nh4SnRzHP", - "outputId": "26b9fe10-b499-4a97-d60d-a8cb8f8030b8" - }, - "execution_count": 13, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "####################\n", - "\n", - "Response from gpt-3.5-turbo\n", - "{\n", - " \"role\": \"assistant\",\n", - " \"content\": \"\"\n", - "}\n", - "{\n", - " \"content\": \"I\"\n", - "}\n", - "{\n", - " \"content\": \"'m\"\n", - "}\n", - "{\n", - " \"content\": \" sorry\"\n", - "}\n", - "{\n", - " \"content\": \",\"\n", - "}\n", - "####################\n", - "\n", - "Response from claude-2\n", - "{'role': 'assistant', 'content': ' Unfortunately'}\n", - "{'role': 'assistant', 'content': ' I'}\n", - "{'role': 'assistant', 'content': ' don'}\n", - "{'role': 'assistant', 'content': \"'t\"}\n", - "{'role': 'assistant', 'content': ' have'}\n", - "####################\n", - "\n", - "Response from stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb\n", - "{'role': 'assistant', 'content': \"I'm\"}\n", - "{'role': 'assistant', 'content': ' sorry,'}\n", - "{'role': 'assistant', 'content': ' I'}\n", - "{'role': 'assistant', 'content': ' cannot'}\n", - "{'role': 'assistant', 'content': ' answer'}\n", - "####################\n", - "\n", - "Response from a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\n", - "{'role': 'assistant', 'content': ''}\n", - "{'role': 'assistant', 'content': ' Hello'}\n", - "{'role': 'assistant', 'content': '!'}\n", - "{'role': 'assistant', 'content': ' I'}\n", - "{'role': 'assistant', 'content': \"'\"}\n", - "####################\n", - "\n", - "Response from replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\n", - "{'role': 'assistant', 'content': 'Comment:'}\n", - "{'role': 'assistant', 'content': 'Hi! '}\n", - "{'role': 'assistant', 'content': 'How '}\n", - "{'role': 'assistant', 'content': 'are '}\n", - "{'role': 'assistant', 'content': 'you '}\n", - "####################\n", - "\n", - "Response from command-nightly\n", - "{'role': 'assistant', 'content': ' Hello'}\n", - "{'role': 'assistant', 'content': '!'}\n", - "{'role': 'assistant', 'content': ' '}\n", - "{'role': 'assistant', 'content': ' I'}\n", - "{'role': 'assistant', 'content': \"'m\"}\n" - ] - } - ] - }, - { - "cell_type": "code", - "source": [], - "metadata": { - "id": "t7WMRuL-8NrO" - }, - "execution_count": null, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/cookbook/liteLLM_VertextAI_Example.ipynb b/cookbook/liteLLM_VertextAI_Example.ipynb deleted file mode 100644 index 0af012b34e3..00000000000 --- a/cookbook/liteLLM_VertextAI_Example.ipynb +++ /dev/null @@ -1,199 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Google Palm (VertexAI) with liteLLM \n", - "### chat-bison, chat-bison@001, text-bison, text-bison@001" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install litellm==0.1.388" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Set VertexAI Configs\n", - "Vertex AI requires the following:\n", - "* `vertex_project` - Your Project ID\n", - "* `vertex_location` - Your Vertex AI region\n", - "Both can be found on: https://console.cloud.google.com/\n", - "\n", - "VertexAI uses Application Default Credentials, see https://cloud.google.com/docs/authentication/external/set-up-adc for more information on setting this up\n", - "\n", - "NOTE: VertexAI requires you to set `application_default_credentials.json`, this can be set by running `gcloud auth application-default login` in your terminal\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# set you Vertex AI configs\n", - "import litellm\n", - "from litellm import completion\n", - "\n", - "litellm.vertex_project = \"hardy-device-386718\"\n", - "litellm.vertex_location = \"us-central1\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Call VertexAI - chat-bison using liteLLM" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': LiteLLM LiteLLM is a large language model from Google AI that is designed to be lightweight and efficient. It is based on the Transformer architecture and has been trained on a massive dataset of text. LiteLLM is available as a pre-trained model that can be used for a variety of natural language processing tasks, such as text classification, question answering, and summarization.}}], 'created': 1692036777.831989, 'model': 'chat-bison'}\n" - ] - } - ], - "source": [ - "user_message = \"what is liteLLM \"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n", - "\n", - "# chat-bison or chat-bison@001 supported by Vertex AI (As of Aug 2023)\n", - "response = completion(model=\"chat-bison\", messages=messages)\n", - "print(response)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Call VertexAI - text-bison using liteLLM" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['text-bison', 'text-bison@001']\n" - ] - } - ], - "source": [ - "print(litellm.vertex_text_models)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': liteLLM is a low-precision variant of the large language model LLM 5. For a given text prompt, liteLLM can continue the text in a way that is both coherent and informative.}}], 'created': 1692036813.052487, 'model': 'text-bison@001'}\n" - ] - } - ], - "source": [ - "user_message = \"what is liteLLM \"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n", - "\n", - "# text-bison or text-bison@001 supported by Vertex AI (As of Aug 2023)\n", - "response = completion(model=\"text-bison@001\", messages=messages)\n", - "print(response)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': liteLLM was originally developed by Google engineers as a lite version of LLM, which stands for large language model. It is a deep learning language model that is designed to be more efficient than traditional LLMs while still achieving comparable performance. liteLLM is built on Tensor2Tensor, a framework for building and training large neural networks. It is able to learn from massive amounts of text data and generate text that is both coherent and informative. liteLLM has been shown to be effective for a variety of tasks, including machine translation, text summarization, and question answering.}}], 'created': 1692036821.60951, 'model': 'text-bison'}\n" - ] - } - ], - "source": [ - "response = completion(model=\"text-bison\", messages=messages)\n", - "print(response)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "liteLLM is a lightweight language model that is designed to be fast and efficient. It is based on the Transformer architecture, but it has been modified to reduce the number of parameters and the amount of computation required. This makes it suitable for use on devices with limited resources, such as mobile phones and embedded systems.\n", - "\n", - "liteLLM is still under development, but it has already been shown to be effective on a variety of tasks, including text classification, natural language inference, and machine translation. It is also being used to develop new applications, such as chatbots and language assistants.\n", - "\n", - "If you are interested in learning more about lite\n" - ] - } - ], - "source": [ - "response = completion(model=\"text-bison@001\", messages=messages, temperature=0.4, top_k=10, top_p=0.2)\n", - "print(response['choices'][0]['message']['content'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/liteLLM_clarifai_Demo.ipynb b/cookbook/liteLLM_clarifai_Demo.ipynb deleted file mode 100644 index 40ef2fcf932..00000000000 --- a/cookbook/liteLLM_clarifai_Demo.ipynb +++ /dev/null @@ -1,187 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# LiteLLM Clarifai \n", - "This notebook walks you through on how to use liteLLM integration of Clarifai and call LLM model from clarifai with response in openAI output format." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pre-Requisites" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#install necessary packages\n", - "!pip install litellm\n", - "!pip install clarifai" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To obtain Clarifai Personal Access Token follow the steps mentioned in the [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "## Set Clarifai Credentials\n", - "import os\n", - "os.environ[\"CLARIFAI_API_KEY\"]= \"YOUR_CLARIFAI_PAT\" # Clarifai PAT" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Mistral-large" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import litellm\n", - "\n", - "litellm.set_verbose=False" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mistral large response : ModelResponse(id='chatcmpl-6eed494d-7ae2-4870-b9c2-6a64d50a6151', choices=[Choices(finish_reason='stop', index=1, message=Message(content=\"In the grand tapestry of time, where tales unfold,\\nLies the chronicle of ages, a sight to behold.\\nA tale of empires rising, and kings of old,\\nOf civilizations lost, and stories untold.\\n\\nOnce upon a yesterday, in a time so vast,\\nHumans took their first steps, casting shadows in the past.\\nFrom the cradle of mankind, a journey they embarked,\\nThrough stone and bronze and iron, their skills they sharpened and marked.\\n\\nEgyptians built pyramids, reaching for the skies,\\nWhile Greeks sought wisdom, truth, in philosophies that lie.\\nRoman legions marched, their empire to expand,\\nAnd in the East, the Silk Road joined the world, hand in hand.\\n\\nThe Middle Ages came, with knights in shining armor,\\nFeudal lords and serfs, a time of both clamor and calm order.\\nThen Renaissance bloomed, like a flower in the sun,\\nA rebirth of art and science, a new age had begun.\\n\\nAcross the vast oceans, explorers sailed with courage bold,\\nDiscovering new lands, stories of adventure, untold.\\nIndustrial Revolution churned, progress in its wake,\\nMachines and factories, a whole new world to make.\\n\\nTwo World Wars raged, a testament to man's strife,\\nYet from the ashes rose hope, a renewed will for life.\\nInto the modern era, technology took flight,\\nConnecting every corner, bathed in digital light.\\n\\nHistory, a symphony, a melody of time,\\nA testament to human will, resilience so sublime.\\nIn every page, a lesson, in every tale, a guide,\\nFor understanding our past, shapes our future's tide.\", role='assistant'))], created=1713896412, model='https://api.clarifai.com/v2/users/mistralai/apps/completion/models/mistral-large/outputs', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=13, completion_tokens=338, total_tokens=351))\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", - "response=completion(\n", - " model=\"clarifai/mistralai.completion.mistral-large\",\n", - " messages=messages,\n", - " )\n", - "\n", - "print(f\"Mistral large response : {response}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Claude-2.1 " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Claude-2.1 response : ModelResponse(id='chatcmpl-d126c919-4db4-4aa3-ac8f-7edea41e0b93', choices=[Choices(finish_reason='stop', index=1, message=Message(content=\" Here's a poem I wrote about history:\\n\\nThe Tides of Time\\n\\nThe tides of time ebb and flow,\\nCarrying stories of long ago.\\nFigures and events come into light,\\nShaping the future with all their might.\\n\\nKingdoms rise, empires fall, \\nLeaving traces that echo down every hall.\\nRevolutions bring change with a fiery glow,\\nToppling structures from long ago.\\n\\nExplorers traverse each ocean and land,\\nSeeking treasures they don't understand.\\nWhile artists and writers try to make their mark,\\nHoping their works shine bright in the dark.\\n\\nThe cycle repeats again and again,\\nAs humanity struggles to learn from its pain.\\nThough the players may change on history's stage,\\nThe themes stay the same from age to age.\\n\\nWar and peace, life and death,\\nLove and strife with every breath.\\nThe tides of time continue their dance,\\nAs we join in, by luck or by chance.\\n\\nSo we study the past to light the way forward, \\nHeeding warnings from stories told and heard.\\nThe future unfolds from this unending flow -\\nWhere the tides of time ultimately go.\", role='assistant'))], created=1713896579, model='https://api.clarifai.com/v2/users/anthropic/apps/completion/models/claude-2_1/outputs', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=12, completion_tokens=232, total_tokens=244))\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", - "response=completion(\n", - " model=\"clarifai/anthropic.completion.claude-2_1\",\n", - " messages=messages,\n", - " )\n", - "\n", - "print(f\"Claude-2.1 response : {response}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### OpenAI GPT-4 (Streaming)\n", - "Though clarifai doesn't support streaming, still you can call stream and get the response in standard StreamResponse format of liteLLM" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ModelResponse(id='chatcmpl-40ae19af-3bf0-4eb4-99f2-33aec3ba84af', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=\"In the quiet corners of time's grand hall,\\nLies the tale of rise and fall.\\nFrom ancient ruins to modern sprawl,\\nHistory, the greatest story of them all.\\n\\nEmpires have risen, empires have decayed,\\nThrough the eons, memories have stayed.\\nIn the book of time, history is laid,\\nA tapestry of events, meticulously displayed.\\n\\nThe pyramids of Egypt, standing tall,\\nThe Roman Empire's mighty sprawl.\\nFrom Alexander's conquest, to the Berlin Wall,\\nHistory, a silent witness to it all.\\n\\nIn the shadow of the past we tread,\\nWhere once kings and prophets led.\\nTheir stories in our hearts are spread,\\nEchoes of their words, in our minds are read.\\n\\nBattles fought and victories won,\\nActs of courage under the sun.\\nTales of love, of deeds done,\\nIn history's grand book, they all run.\\n\\nHeroes born, legends made,\\nIn the annals of time, they'll never fade.\\nTheir triumphs and failures all displayed,\\nIn the eternal march of history's parade.\\n\\nThe ink of the past is forever dry,\\nBut its lessons, we cannot deny.\\nIn its stories, truths lie,\\nIn its wisdom, we rely.\\n\\nHistory, a mirror to our past,\\nA guide for the future vast.\\nThrough its lens, we're ever cast,\\nIn the drama of life, forever vast.\", role='assistant', function_call=None, tool_calls=None), logprobs=None)], created=1714744515, model='https://api.clarifai.com/v2/users/openai/apps/chat-completion/models/GPT-4/outputs', object='chat.completion.chunk', system_fingerprint=None)\n", - "ModelResponse(id='chatcmpl-40ae19af-3bf0-4eb4-99f2-33aec3ba84af', choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(content=None, role=None, function_call=None, tool_calls=None), logprobs=None)], created=1714744515, model='https://api.clarifai.com/v2/users/openai/apps/chat-completion/models/GPT-4/outputs', object='chat.completion.chunk', system_fingerprint=None)\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", - "response = completion(\n", - " model=\"clarifai/openai.chat-completion.GPT-4\",\n", - " messages=messages,\n", - " stream=True,\n", - " api_key = \"c75cc032415e45368be331fdd2c06db0\")\n", - "\n", - "for chunk in response:\n", - " print(chunk)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/liteLLM_function_calling.ipynb b/cookbook/liteLLM_function_calling.ipynb deleted file mode 100644 index 45f4398b386..00000000000 --- a/cookbook/liteLLM_function_calling.ipynb +++ /dev/null @@ -1,331 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "vnvlwUDZK7VA" - }, - "source": [ - "## Demo Notebook of Function Calling with liteLLM\n", - "- Supported Providers for Function Calling\n", - " - OpenAI - `gpt-4-0613` and `gpt-3.5-turbo-0613`\n", - "- In this notebook we use function calling with `litellm.completion()`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "KrINCwRfLgZV" - }, - "outputs": [], - "source": [ - "## Install liteLLM\n", - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "nK7zR5OgLlh2" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "id": "dCQlyBxKLqbA" - }, - "outputs": [], - "source": [ - "os.environ['OPENAI_API_KEY'] = \"\" #@param" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gfdGv-FMRCdX" - }, - "source": [ - "## Define Messages, Functions\n", - "We create a get_current_weather() function and pass that to GPT 3.5\n", - "\n", - "See OpenAI docs for this: https://openai.com/blog/function-calling-and-other-api-updates" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "id": "ERzsP1sfM19C" - }, - "outputs": [], - "source": [ - "messages = [\n", - " {\"role\": \"user\", \"content\": \"What is the weather like in Boston?\"}\n", - "]\n", - "\n", - "def get_current_weather(location):\n", - " if location == \"Boston, MA\":\n", - " return \"The weather is 12F\"\n", - "\n", - "functions = [\n", - " {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city and state, e.g. San Francisco, CA\"\n", - " },\n", - " \"unit\": {\n", - " \"type\": \"string\",\n", - " \"enum\": [\"celsius\", \"fahrenheit\"]\n", - " }\n", - " },\n", - " \"required\": [\"location\"]\n", - " }\n", - " }\n", - " ]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NX6by2VuRPnp" - }, - "source": [ - "## Call gpt-3.5-turbo-0613 to Decide what Function to call" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "QVoJ5PtxMlVx", - "outputId": "efe7a81f-e04a-4afc-aa60-a2b2648f5fb9" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"id\": \"chatcmpl-7mX4RiqdoislVEqfmfVjFSKp3hyIy\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1691801223,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": null,\n", - " \"function_call\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"arguments\": \"{\\n \\\"location\\\": \\\"Boston, MA\\\"\\n}\"\n", - " }\n", - " },\n", - " \"finish_reason\": \"function_call\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 82,\n", - " \"completion_tokens\": 18,\n", - " \"total_tokens\": 100\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "response = completion(model=\"gpt-3.5-turbo-0613\", messages=messages, functions=functions)\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Yu0o2saDNLx8" - }, - "source": [ - "## Parse GPT 3.5 Response\n", - "Read Information about what Function to Call" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "u1DzXLJsNOR5", - "outputId": "177e9501-0ce2-4619-9067-3047f18f6c79" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"name\": \"get_current_weather\",\n", - " \"arguments\": \"{\\n \\\"location\\\": \\\"Boston, MA\\\"\\n}\"\n", - "}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "function_call_data = response[\"choices\"][0][\"message\"][\"function_call\"]\n", - "function_call_data" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tYb96Mh0NhH9", - "outputId": "13c4bb89-6f29-4b3b-afa7-302dcf2cdd5f" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "get_current_weather {'location': 'Boston, MA'}\n" - ] - } - ], - "source": [ - "import json\n", - "function_name = function_call_data['name']\n", - "function_args = function_call_data['arguments']\n", - "function_args = json.loads(function_args)\n", - "print(function_name, function_args)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "z3tstH_yN3fX" - }, - "source": [ - "## Call the get_current_weather() function" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "TSb8JHhgN5Zc", - "outputId": "ef140572-4020-4daf-ac8c-d5161be9aa5c" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "12F\n" - ] - } - ], - "source": [ - "if function_name == \"get_current_weather\":\n", - " result = get_current_weather(**function_args)\n", - " print(result)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "k4HGJE3NRmMI" - }, - "source": [ - "## Send the response from get_current_weather back to the model to summarize" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "a23cmEwiPaw7", - "outputId": "43259b86-0c4c-4fcb-eab7-6e1a788b2f21" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"id\": \"chatcmpl-7mXGN62u75WXp1Lgen4iSgNvA7hHT\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1691801963,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"The current weather in Boston is 12 degrees Fahrenheit.\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 109,\n", - " \"completion_tokens\": 12,\n", - " \"total_tokens\": 121\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "messages = [\n", - " {\"role\": \"user\", \"content\": \"What is the weather like in Boston?\"},\n", - " {\"role\": \"assistant\", \"content\": None, \"function_call\": {\"name\": \"get_current_weather\", \"arguments\": \"{ \\\"location\\\": \\\"Boston, MA\\\"}\"}},\n", - " {\"role\": \"function\", \"name\": \"get_current_weather\", \"content\": result}\n", - "]\n", - "response = completion(model=\"gpt-3.5-turbo-0613\", messages=messages, functions=functions)\n", - "print(response)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/litellm_Test_Multiple_Providers.ipynb b/cookbook/litellm_Test_Multiple_Providers.ipynb deleted file mode 100644 index 3901581e67f..00000000000 --- a/cookbook/litellm_Test_Multiple_Providers.ipynb +++ /dev/null @@ -1,571 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "Ys9n20Es2IzT" - }, - "source": [ - "# Evaluate Multiple LLM Providers with LiteLLM\n", - "\n", - "\n", - "\n", - "* Quality Testing\n", - "* Load Testing\n", - "* Duration Testing\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZXOXl23PIIP6" - }, - "outputs": [], - "source": [ - "!pip install litellm python-dotenv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "LINuBzXDItq2" - }, - "outputs": [], - "source": [ - "from litellm import load_test_model, testing_batch_completion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "EkxMhsWdJdu4" - }, - "outputs": [], - "source": [ - "from dotenv import load_dotenv\n", - "load_dotenv()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mv5XdnqeW5I_" - }, - "source": [ - "# Quality Test endpoint\n", - "\n", - "## Test the same prompt across multiple LLM providers\n", - "\n", - "In this example, let's ask some questions about Paul Graham" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "XpzrR5m4W_Us" - }, - "outputs": [], - "source": [ - "models = [\"gpt-3.5-turbo\", \"gpt-3.5-turbo-16k\", \"gpt-4\", \"claude-instant-1\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\"]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompts = [\"Who is Paul Graham?\", \"What is Paul Graham known for?\" , \"Is paul graham a writer?\" , \"Where does Paul Graham live?\", \"What has Paul Graham done?\"]\n", - "messages = [[{\"role\": \"user\", \"content\": context + \"\\n\" + prompt}] for prompt in prompts] # pass in a list of messages we want to test\n", - "result = testing_batch_completion(models=models, messages=messages)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9nzeLySnvIIW" - }, - "source": [ - "## Visualize the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 403 - }, - "id": "X-2n7hdAuVAY", - "outputId": "69cc0de1-68e3-4c12-a8ea-314880010d94" - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "

\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Model Nameclaude-instant-1gpt-3.5-turbo-0613gpt-3.5-turbo-16k-0613gpt-4-0613replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781
Prompt
\\nIs paul graham a writer?Yes, Paul Graham is considered a writer in ad...Yes, Paul Graham is a writer. He has written s...Yes, Paul Graham is a writer. He has authored ...Yes, Paul Graham is a writer. He is an essayis...Yes, Paul Graham is an author. According to t...
\\nWhat has Paul Graham done?Paul Graham has made significant contribution...Paul Graham has achieved several notable accom...Paul Graham has made significant contributions...Paul Graham is known for his work on the progr...Paul Graham has had a diverse career in compu...
\\nWhat is Paul Graham known for?Paul Graham is known for several things:\\n\\n-...Paul Graham is known for his work on the progr...Paul Graham is known for his work on the progr...Paul Graham is known for his work on the progr...Paul Graham is known for many things, includi...
\\nWhere does Paul Graham live?Based on the information provided:\\n\\n- Paul ...According to the given information, Paul Graha...Paul Graham currently lives in England, where ...The text does not provide a current place of r...Based on the information provided, Paul Graha...
\\nWho is Paul Graham?Paul Graham is an influential computer scient...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist,...
\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - "
\n", - " \n", - "
\n", - "\n", - "\n", - "\n", - " \n", - "\n", - "\n", - " \n", - " \n", - "\n", - " \n", - "
\n", - "
\n" - ], - "text/plain": [ - "Model Name claude-instant-1 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is considered a writer in ad... \n", - "\\nWhat has Paul Graham done? Paul Graham has made significant contribution... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for several things:\\n\\n-... \n", - "\\nWhere does Paul Graham live? Based on the information provided:\\n\\n- Paul ... \n", - "\\nWho is Paul Graham? Paul Graham is an influential computer scient... \n", - "\n", - "Model Name gpt-3.5-turbo-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He has written s... \n", - "\\nWhat has Paul Graham done? Paul Graham has achieved several notable accom... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? According to the given information, Paul Graha... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name gpt-3.5-turbo-16k-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He has authored ... \n", - "\\nWhat has Paul Graham done? Paul Graham has made significant contributions... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? Paul Graham currently lives in England, where ... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name gpt-4-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He is an essayis... \n", - "\\nWhat has Paul Graham done? Paul Graham is known for his work on the progr... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? The text does not provide a current place of r... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781 \n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is an author. According to t... \n", - "\\nWhat has Paul Graham done? Paul Graham has had a diverse career in compu... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for many things, includi... \n", - "\\nWhere does Paul Graham live? Based on the information provided, Paul Graha... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist,... " - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "# Create an empty list to store the row data\n", - "table_data = []\n", - "\n", - "# Iterate through the list and extract the required data\n", - "for item in result:\n", - " prompt = item['prompt'][0]['content'].replace(context, \"\") # clean the prompt for easy comparison\n", - " model = item['response']['model']\n", - " response = item['response']['choices'][0]['message']['content']\n", - " table_data.append([prompt, model, response])\n", - "\n", - "# Create a DataFrame from the table data\n", - "df = pd.DataFrame(table_data, columns=['Prompt', 'Model Name', 'Response'])\n", - "\n", - "# Pivot the DataFrame to get the desired table format\n", - "table = df.pivot(index='Prompt', columns='Model Name', values='Response')\n", - "table" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zOxUM40PINDC" - }, - "source": [ - "# Load Test endpoint\n", - "\n", - "Run 100+ simultaneous queries across multiple providers to see when they fail + impact on latency" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZkQf_wbcIRQ9" - }, - "outputs": [], - "source": [ - "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompt = \"Where does Paul Graham live?\"\n", - "final_prompt = context + prompt\n", - "result = load_test_model(models=models, prompt=final_prompt, num_calls=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8vSNBFC06aXY" - }, - "source": [ - "## Visualize the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 552 - }, - "id": "SZfiKjLV3-n8", - "outputId": "00f7f589-b3da-43ed-e982-f9420f074b8d" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAIXCAYAAACy1HXAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABn5UlEQVR4nO3dd1QT2d8G8Cf0ojQBEUFRsSv2FXvvvSx2saNi7733ihXELotd7KuIir33sjZUsIuKVGmS+/7hy/yM6K7RYEZ4PufkaO5Mkm/IJHly594ZhRBCgIiIiEiGdLRdABEREdG3MKgQERGRbDGoEBERkWwxqBAREZFsMagQERGRbDGoEBERkWwxqBAREZFsMagQERGRbDGoEBERkWwxqBCR7Dk5OaFLly7aLkNtc+fORd68eaGrq4uSJUtquxyNO3bsGBQKBbZv367tUtSmUCgwadIktW8XGhoKhUKBdevWabwm+joGFVKxfPlyKBQKlC9fXtulyI6TkxMUCoV0MTU1xR9//IENGzZou7TfTuoX3PdcfleHDh3CiBEjUKlSJaxduxYzZszQdkmys27dOul1PnXqVJrlQgg4OjpCoVCgcePGWqiQ5EBP2wWQvPj7+8PJyQkXLlxASEgInJ2dtV2SrJQsWRJDhw4FALx8+RKrVq2Cu7s7EhMT0bNnTy1X9/soXLgw/Pz8VNpGjx6NLFmyYOzYsWnWv3fvHnR0fq/fVUePHoWOjg5Wr14NAwMDbZcja0ZGRti4cSMqV66s0n78+HE8e/YMhoaGWqqM5IBBhSSPHz/GmTNnEBAQAA8PD/j7+2PixIm/tAalUomkpCQYGRn90sf9Xjlz5kTHjh2l6126dEHevHmxcOFCBhU1ZM+eXeXvCACzZs2CtbV1mnYAv+UXVXh4OIyNjTUWUoQQSEhIgLGxsUbuT04aNmyIbdu2YfHixdDT+9/X0saNG1GmTBm8fftWi9WRtv1eP1EoXfn7+8PS0hKNGjVC69at4e/vLy1LTk6GlZUVunbtmuZ20dHRMDIywrBhw6S2xMRETJw4Ec7OzjA0NISjoyNGjBiBxMREldsqFAr069cP/v7+KFq0KAwNDXHw4EEAwLx581CxYkVky5YNxsbGKFOmzFf3hcfHx2PAgAGwtrZG1qxZ0bRpUzx//vyr+6CfP3+Obt26IXv27DA0NETRokWxZs2aH/6b2djYoFChQnj48KFKu1KphJeXF4oWLQojIyNkz54dHh4eeP/+vcp6ly5dQr169WBtbQ1jY2PkyZMH3bp1k5an7g+fN28eFi5ciNy5c8PY2BjVqlXDrVu30tRz9OhRVKlSBaamprCwsECzZs1w584dlXUmTZoEhUKBkJAQdOnSBRYWFjA3N0fXrl3x4cMHlXWDgoJQuXJlWFhYIEuWLChYsCDGjBmjss73vtY/48sxKqm7DE6dOoUBAwbAxsYGFhYW8PDwQFJSEiIjI9G5c2dYWlrC0tISI0aMwJcnitfUa/Q1CoUCa9euRVxcnLRrI3VMw8ePHzF16lTky5cPhoaGcHJywpgxY9L8vZycnNC4cWMEBgaibNmyMDY2xooVK/71cc+fP4/69evD3NwcJiYmqFatGk6fPq2yTlhYGPr27YuCBQvC2NgY2bJlw59//onQ0NA09xcZGYnBgwfDyckJhoaGcHBwQOfOndMEB6VSienTp8PBwQFGRkaoVasWQkJC/rXWz7Vr1w7v3r1DUFCQ1JaUlITt27ejffv2X71NXFwchg4dCkdHRxgaGqJgwYKYN29emtc5MTERgwcPho2NjfT58OzZs6/ep6Y/H0hDBNH/K1SokOjevbsQQogTJ04IAOLChQvS8m7dugkLCwuRmJiocrv169cLAOLixYtCCCFSUlJE3bp1hYmJiRg0aJBYsWKF6Nevn9DT0xPNmjVTuS0AUbhwYWFjYyMmT54sli1bJq5evSqEEMLBwUH07dtXLF26VCxYsED88ccfAoDYt2+fyn24ubkJAKJTp05i2bJlws3NTZQoUUIAEBMnTpTWe/XqlXBwcBCOjo5iypQpwtvbWzRt2lQAEAsXLvzPv0/u3LlFo0aNVNqSk5OFnZ2dyJ49u0p7jx49hJ6enujZs6fw8fERI0eOFKampqJcuXIiKSlJCCHE69evhaWlpShQoICYO3euWLlypRg7dqwoXLiwdD+PHz8WAETx4sWFk5OTmD17tpg8ebKwsrISNjY24tWrV9K6QUFBQk9PTxQoUEDMmTNHTJ48WVhbWwtLS0vx+PFjab2JEycKAKJUqVKiZcuWYvny5aJHjx4CgBgxYoS03q1bt4SBgYEoW7asWLRokfDx8RHDhg0TVatWldZR57X+L0WLFhXVqlX75t/e3d1dur527VoBQJQsWVLUr19fLFu2THTq1El6DpUrVxbt27cXy5cvF40bNxYAxPr169PlNfoaPz8/UaVKFWFoaCj8/PyEn5+fePjwoRBCCHd3dwFAtG7dWixbtkx07txZABDNmzdP85ydnZ2FpaWlGDVqlPDx8RHBwcHffMwjR44IAwMDUaFCBTF//nyxcOFC4eLiIgwMDMT58+el9bZt2yZKlCghJkyYIHx9fcWYMWOEpaWlyJ07t4iLi5PWi4mJEcWKFRO6urqiZ8+ewtvbW0ydOlWUK1dOeo8GBwdL21KZMmXEwoULxaRJk4SJiYn4448//vVv9PnrePHiRVGxYkXRqVMnadmuXbuEjo6OeP78eZr3nlKpFDVr1hQKhUL06NFDLF26VDRp0kQAEIMGDVJ5jI4dOwoAon379mLp0qWiZcuWwsXF5Yc/H1Lfk2vXrv3P50eawaBCQgghLl26JACIoKAgIcSnDwIHBwcxcOBAaZ3AwEABQOzdu1fltg0bNhR58+aVrvv5+QkdHR1x8uRJlfV8fHwEAHH69GmpDYDQ0dERt2/fTlPThw8fVK4nJSWJYsWKiZo1a0ptly9f/uqHU5cuXdJ8EHXv3l3kyJFDvH37VmXdtm3bCnNz8zSP96XcuXOLunXrijdv3og3b96ImzdvSl+Onp6e0nonT54UAIS/v7/K7Q8ePKjSvnPnTpWA9zWpH4rGxsbi2bNnUvv58+cFADF48GCprWTJksLW1la8e/dOart+/brQ0dERnTt3ltpSg0q3bt1UHqtFixYiW7Zs0vWFCxcKAOLNmzffrE+d1/q//EhQqVevnlAqlVJ7hQoVhEKhEL1795baPn78KBwcHFTuW5Ov0be4u7sLU1NTlbZr164JAKJHjx4q7cOGDRMAxNGjR1WeMwBx8ODB/3wspVIp8ufPn+bv8eHDB5EnTx5Rp04dlbYvnT17VgAQGzZskNomTJggAIiAgICvPp4Q/wsqhQsXVvkBs2jRIgFA3Lx581/r/jyoLF26VGTNmlWq788//xQ1atSQ/hafB5Vdu3YJAGLatGkq99e6dWuhUChESEiIEOJ/f+++ffuqrNe+ffsf/nxgUPn1uOuHAHza7ZM9e3bUqFEDwKeu6zZt2mDz5s1ISUkBANSsWRPW1tbYsmWLdLv3798jKCgIbdq0kdq2bduGwoULo1ChQnj79q10qVmzJgAgODhY5bGrVauGIkWKpKnp833x79+/R1RUFKpUqYIrV65I7am7ifr27aty2/79+6tcF0Jgx44daNKkCYQQKnXVq1cPUVFRKvf7LYcOHYKNjQ1sbGxQvHhx+Pn5oWvXrpg7d67K8zc3N0edOnVUHqdMmTLIkiWL9PwtLCwAAPv27UNycvK/Pm7z5s2RM2dO6foff/yB8uXL4++//wbwaWDvtWvX0KVLF1hZWUnrubi4oE6dOtJ6n+vdu7fK9SpVquDdu3eIjo5WqW/37t1QKpVfrUvd11rTunfvrjIzqHz58hBCoHv37lKbrq4uypYti0ePHqnUrenX6Hukvg5DhgxRaU8doL1//36V9jx58qBevXr/eb/Xrl3DgwcP0L59e7x79056PnFxcahVqxZOnDghvYafv6+Sk5Px7t07ODs7w8LCQuU9sGPHDpQoUQItWrRI83hfzsbq2rWrylicKlWqAIDK3/y/uLm5IT4+Hvv27UNMTAz27dv3zd0+f//9N3R1dTFgwACV9qFDh0IIgQMHDkjrAUiz3qBBg1Sua+rzgdJHhgkqJ06cQJMmTWBvbw+FQoFdu3al+2M+f/4cHTt2lMZQFC9eHJcuXUr3x9W0lJQUbN68GTVq1MDjx48REhKCkJAQlC9fHq9fv8aRI0cAAHp6emjVqhV2794t7U8PCAhAcnKySlB58OABbt++LX2hp14KFCgA4NMgw8/lyZPnq3Xt27cPrq6uMDIygpWVFWxsbODt7Y2oqChpnbCwMOjo6KS5jy9nK7158waRkZHw9fVNU1fquJsv6/qa8uXLIygoCAcPHsS8efNgYWGB9+/fq3xIP3jwAFFRUbC1tU3zWLGxsdLjVKtWDa1atcLkyZNhbW2NZs2aYe3atV8d25E/f/40bQUKFJDGFYSFhQEAChYsmGa9woULS19an8uVK5fKdUtLSwCQxmi0adMGlSpVQo8ePZA9e3a0bdsWW7duVQkt6r7WmvblczA3NwcAODo6pmn/fOxJerxG3yN1e/1y+7Szs4OFhYX0Oqb61nvjSw8ePAAAuLu7p3k+q1atQmJiovS+iY+Px4QJE6SxHdbW1rCxsUFkZKTKe+vhw4coVqzYdz3+f21L38PGxga1a9fGxo0bERAQgJSUFLRu3fqr64aFhcHe3h5Zs2ZVaS9cuLC0PPVfHR0d5MuXT2W9L98nmvp8oPSRYWb9xMXFoUSJEujWrRtatmyZ7o/3/v17VKpUCTVq1MCBAwdgY2ODBw8eSG/Q38nRo0fx8uVLbN68GZs3b06z3N/fH3Xr1gUAtG3bFitWrMCBAwfQvHlzbN26FYUKFUKJEiWk9ZVKJYoXL44FCxZ89fG+/BL52iyGkydPomnTpqhatSqWL1+OHDlyQF9fH2vXrsXGjRvVfo6pX64dO3aEu7v7V9dxcXH5z/uxtrZG7dq1AQD16tVDoUKF0LhxYyxatEj6laxUKmFra6syGPlzNjY2ACAdKOvcuXPYu3cvAgMD0a1bN8yfPx/nzp1DlixZ1H6e6tDV1f1qu/j/wYjGxsY4ceIEgoODsX//fhw8eBBbtmxBzZo1cejQIejq6qr9Wmvat57D19rFZ4Mstf0afe/xYb53hk/q9j137txvHlgutdb+/ftj7dq1GDRoECpUqABzc3MoFAq0bdv2mz1n/+W/tqXv1b59e/Ts2ROvXr1CgwYNpB6t9KapzwdKHxkmqDRo0AANGjT45vLExESMHTsWmzZtQmRkJIoVK4bZs2ejevXqP/R4s2fPhqOjI9auXSu1fe+vH7nx9/eHra0tli1blmZZQEAAdu7cCR8fHxgbG6Nq1arIkSMHtmzZgsqVK+Po0aNpjnuRL18+XL9+HbVq1frhA3bt2LEDRkZGCAwMVJma+vnfGwBy584NpVKJx48fq/Q6fDnjIHXEf0pKihQ0NKFRo0aoVq0aZsyYAQ8PD5iamiJfvnw4fPgwKlWq9F1fNK6urnB1dcX06dOxceNGdOjQAZs3b0aPHj2kdVJ/MX/u/v37cHJyAvDp7wB8Ot7Il+7evQtra2uYmpqq/fx0dHRQq1Yt1KpVCwsWLMCMGTMwduxYBAcHo3bt2hp5rbUhPV6j75G6vT548ED69Q8Ar1+/RmRkpPQ6qiu1x8DMzOw/t+/t27fD3d0d8+fPl9oSEhIQGRmZ5j6/NrMsPbVo0QIeHh44d+6cyi7mL+XOnRuHDx9GTEyMSq/K3bt3peWp/yqVSjx8+FClF+XL90l6fT6QZmSYXT//pV+/fjh79iw2b96MGzdu4M8//0T9+vW/+gXwPfbs2YOyZcvizz//hK2tLUqVKoWVK1dquOr0Fx8fj4CAADRu3BitW7dOc+nXrx9iYmKwZ88eAJ++uFq3bo29e/fCz88PHz9+VNntA3za1/z8+fOv/j3i4+PT7IL4Gl1dXSgUCml8DPBpqu6Xu/RS998vX75cpX3JkiVp7q9Vq1bYsWPHVz9837x58581fcvIkSPx7t076fm6ubkhJSUFU6dOTbPux48fpS+E9+/fp/nFmfpr+MtdC7t27cLz58+l6xcuXMD58+elcJ4jRw6ULFkS69evV/nCuXXrFg4dOoSGDRuq/bwiIiLStH1ZnyZea21Ij9foe6S+Dl5eXirtqT1SjRo1Uvs+AaBMmTLIly8f5s2bh9jY2DTLP9++dXV10zynJUuWqLzXAKBVq1a4fv06du7cmeb+1O0p+V5ZsmSBt7c3Jk2ahCZNmnxzvYYNGyIlJQVLly5VaV+4cCEUCoX0vkj9d/HixSrrffn3T8/PB/p5GaZH5d88efIEa9euxZMnT2Bvbw8AGDZsGA4ePPjDh7Z+9OgRvL29MWTIEIwZMwYXL17EgAEDYGBg8M2uQznas2cPYmJi0LRp068ud3V1hY2NDfz9/aVA0qZNGyxZsgQTJ05E8eLFVX4ZAkCnTp2wdetW9O7dG8HBwahUqRJSUlJw9+5dbN26VTouxL9p1KgRFixYgPr166N9+/YIDw/HsmXL4OzsjBs3bkjrlSlTBq1atYKXlxfevXsHV1dXHD9+HPfv3weg2sU+a9YsBAcHo3z58ujZsyeKFCmCiIgIXLlyBYcPH/7qF/P3aNCgAYoVK4YFCxbA09MT1apVg4eHB2bOnIlr166hbt260NfXx4MHD7Bt2zYsWrQIrVu3xvr167F8+XK0aNEC+fLlQ0xMDFauXAkzM7M0wcLZ2RmVK1dGnz59kJiYCC8vL2TLlg0jRoyQ1pk7dy4aNGiAChUqoHv37oiPj8eSJUtgbm7+Q+c0mTJlCk6cOIFGjRohd+7cCA8Px/Lly+Hg4CAdQVQTr7U2pMdr9D1KlCgBd3d3+Pr6IjIyEtWqVcOFCxewfv16NG/eXBrMri4dHR2sWrUKDRo0QNGiRdG1a1fkzJkTz58/R3BwMMzMzLB3714AQOPGjeHn5wdzc3MUKVIEZ8+exeHDh5EtWzaV+xw+fDi2b9+OP//8E926dUOZMmUQERGBPXv2wMfHR2V3ryZ9z+dnkyZNUKNGDYwdOxahoaEoUaIEDh06hN27d2PQoEFSD1PJkiXRrl07LF++HFFRUahYsSKOHDny1WO8pNfnA2mAVuYapTMAYufOndL1ffv2CQDC1NRU5aKnpyfc3NyEEELcuXNHAPjXy8iRI6X71NfXFxUqVFB53P79+wtXV9df8hw1pUmTJsLIyEjl+Alf6tKli9DX15em7SmVSuHo6PjV6YGpkpKSxOzZs0XRokWFoaGhsLS0FGXKlBGTJ08WUVFR0nr4Ymrv51avXi3y588vDA0NRaFChcTatWulqbWfi4uLE56ensLKykpkyZJFNG/eXNy7d08AELNmzVJZ9/Xr18LT01M4OjoKfX19YWdnJ2rVqiV8fX3/82/1teOopFq3bl2aKYu+vr6iTJkywtjYWGTNmlUUL15cjBgxQrx48UIIIcSVK1dEu3btRK5cuYShoaGwtbUVjRs3FpcuXZLuI3Uq5Ny5c8X8+fOFo6OjMDQ0FFWqVBHXr19PU8fhw4dFpUqVhLGxsTAzMxNNmjQR//zzj8o6qX/DL6cdp04VTT3mypEjR0SzZs2Evb29MDAwEPb29qJdu3bi/v37Krf73tf6v/zI9OQvpw1/67l9baqwEJp5jb7lW4+ZnJwsJk+eLPLkySP09fWFo6OjGD16tEhISEjznL+1vX3L1atXRcuWLUW2bNmEoaGhyJ07t3BzcxNHjhyR1nn//r3o2rWrsLa2FlmyZBH16tUTd+/eTfM3FkKId+/eiX79+omcOXMKAwMD4eDgINzd3aXPgtTpydu2bVO53fdO4f3W6/ilr/0tYmJixODBg4W9vb3Q19cX+fPnF3PnzlWZni2EEPHx8WLAgAEiW7ZswtTUVDRp0kQ8ffo0zfRkIb7v84HTk389hRDp1IenRQqFAjt37kTz5s0BAFu2bEGHDh1w+/btNIO+smTJAjs7OyQlJf3nVLps2bJJg+xy586NOnXqYNWqVdJyb29vTJs2TaWLnrTj2rVrKFWqFP766y906NBB2+X8sNDQUOTJkwdz585VOfIvEVFmkSl2/ZQqVQopKSkIDw+X5vd/ycDAAIUKFfru+6xUqVKaAVn379//4cFw9OPi4+PTDIj08vKCjo4OqlatqqWqiIhIEzJMUImNjVXZ7/j48WNcu3YNVlZWKFCgADp06IDOnTtj/vz5KFWqFN68eYMjR47AxcXlhwawDR48GBUrVsSMGTPg5uaGCxcuwNfXF76+vpp8WvQd5syZg8uXL6NGjRrQ09PDgQMHcODAAfTq1Svdp8cSEVE60/a+J01J3Vf65SV1n2tSUpKYMGGCcHJyEvr6+iJHjhyiRYsW4saNGz/8mHv37hXFihWTxlB8zzgH0rxDhw6JSpUqCUtLS6Gvry/y5csnJk2aJJKTk7Vd2k/7fIwKEVFmlCHHqBAREVHGkGmOo0JERES/HwYVIiIikq3fejCtUqnEixcvkDVr1t/q8N1ERESZmRACMTExsLe3h47Ov/eZ/NZB5cWLF5zVQURE9Jt6+vQpHBwc/nWd3zqopJ6M6unTpzAzM9NyNURERPQ9oqOj4ejoqHJSyW/5rYNK6u4eMzMzBhUiIqLfzPcM2+BgWiIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki09bRdARETy5TRqv7ZLIC0LndVIq4/PHhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki3ZBJVZs2ZBoVBg0KBB2i6FiIiIZEIWQeXixYtYsWIFXFxctF0KERERyYjWg0psbCw6dOiAlStXwtLSUtvlEBERkYxoPah4enqiUaNGqF279n+um5iYiOjoaJULERERZVx62nzwzZs348qVK7h48eJ3rT9z5kxMnjw5nasiIiIiudBaj8rTp08xcOBA+Pv7w8jI6LtuM3r0aERFRUmXp0+fpnOVREREpE1a61G5fPkywsPDUbp0aaktJSUFJ06cwNKlS5GYmAhdXV2V2xgaGsLQ0PBXl0pERERaorWgUqtWLdy8eVOlrWvXrihUqBBGjhyZJqQQERFR5qO1oJI1a1YUK1ZMpc3U1BTZsmVL005ERESZk9Zn/RARERF9i1Zn/Xzp2LFj2i6BiIiIZIQ9KkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWz8UVB4+fIhx48ahXbt2CA8PBwAcOHAAt2/f1mhxRERElLmpHVSOHz+O4sWL4/z58wgICEBsbCwA4Pr165g4caLGCyQiIqLMS+2gMmrUKEybNg1BQUEwMDCQ2mvWrIlz585ptDgiIiLK3NQOKjdv3kSLFi3StNva2uLt27caKYqIiIgI+IGgYmFhgZcvX6Zpv3r1KnLmzKmRooiIiIiAHwgqbdu2xciRI/Hq1SsoFAoolUqcPn0aw4YNQ+fOndOjRiIiIsqk1A4qM2bMQKFCheDo6IjY2FgUKVIEVatWRcWKFTFu3Lj0qJGIiIgyKT11b2BgYICVK1di/PjxuHXrFmJjY1GqVCnkz58/PeojIiKiTEztoJIqV65cyJUrlyZrISIiIlKhdlARQmD79u0IDg5GeHg4lEqlyvKAgACNFUdERESZm9pBZdCgQVixYgVq1KiB7NmzQ6FQpEddREREROoHFT8/PwQEBKBhw4bpUQ8RERGRRO1ZP+bm5sibN2961EJERESkQu2gMmnSJEyePBnx8fHpUQ8RERGRRO1dP25ubti0aRNsbW3h5OQEfX19leVXrlzRWHFERESUuakdVNzd3XH58mV07NiRg2mJiIgoXakdVPbv34/AwEBUrlw5PeohIiIikqg9RsXR0RFmZmbpUQsRERGRCrWDyvz58zFixAiEhoamQzlERERE/6P2rp+OHTviw4cPyJcvH0xMTNIMpo2IiNBYcUSZndOo/dougbQsdFYjbZdApFVqBxUvL690KIOIiIgorR+a9UNERET0K3xXUImOjpYG0EZHR//ruhxoS0RERJryXUHF0tISL1++hK2tLSwsLL567BQhBBQKBVJSUjReJBEREWVO3xVUjh49CisrKwBAcHBwuhZERERElOq7gkq1atWQN29eXLx4EdWqVUvvmoiIiIgAqHEcldDQUO7WISIiol9K7QO+aZK3tzdcXFxgZmYGMzMzVKhQAQcOHNBmSURERCQjak1PDgwMhLm5+b+u07Rp0+++PwcHB8yaNQv58+eHEALr169Hs2bNcPXqVRQtWlSd0oiIiCgDUiuo/NcxVNSd9dOkSROV69OnT4e3tzfOnTvHoEJERETqBZVXr17B1tY2XQpJSUnBtm3bEBcXhwoVKnx1ncTERCQmJkrX/+uYLkRERPR7++4xKl87doom3Lx5E1myZIGhoSF69+6NnTt3okiRIl9dd+bMmTA3N5cujo6O6VITERERycN3BxUhRLoUULBgQVy7dg3nz59Hnz594O7ujn/++eer644ePRpRUVHS5enTp+lSExEREcnDd+/6cXd3h7GxscYLMDAwgLOzMwCgTJkyuHjxIhYtWoQVK1akWdfQ0BCGhoYar4GIiIjk6buDytq1a9OzDolSqVQZh0JERESZl9pnT9ak0aNHo0GDBsiVKxdiYmKwceNGHDt2DIGBgdosi4iIiGRCq0ElPDwcnTt3xsuXL2Fubg4XFxcEBgaiTp062iyLiIiIZEKrQWX16tXafHgiIiKSuR8+hH5ISAgCAwMRHx8PIP1mBREREVHmpXZQeffuHWrXro0CBQqgYcOGePnyJQCge/fuGDp0qMYLJCIiosxL7aAyePBg6Onp4cmTJzAxMZHa27Rpg4MHD2q0OCIiIsrc1B6jcujQIQQGBsLBwUGlPX/+/AgLC9NYYURERERq96jExcWp9KSkioiI4MHYiIiISKPUDipVqlTBhg0bpOsKhQJKpRJz5sxBjRo1NFocERERZW5q7/qZM2cOatWqhUuXLiEpKQkjRozA7du3ERERgdOnT6dHjURERJRJqd2jUqxYMdy/fx+VK1dGs2bNEBcXh5YtW+Lq1avIly9fetRIREREmdQPHfDN3NwcY8eO1XQtRERERCrU7lE5ePAgTp06JV1ftmwZSpYsifbt2+P9+/caLY6IiIgyN7WDyvDhwxEdHQ0AuHnzJoYMGYKGDRvi8ePHGDJkiMYLJCIiosxL7V0/jx8/RpEiRQAAO3bsQJMmTTBjxgxcuXIFDRs21HiBRERElHmp3aNiYGCADx8+AAAOHz6MunXrAgCsrKyknhYiIiIiTVC7R6Vy5coYMmQIKlWqhAsXLmDLli0AgPv376c5Wi0RERHRz1C7R2Xp0qXQ09PD9u3b4e3tjZw5cwIADhw4gPr162u8QCIiIsq81O5RyZUrF/bt25emfeHChRopiIiIiCjVDx1HRalUIiQkBOHh4VAqlSrLqlatqpHCiIiIiNQOKufOnUP79u0RFhYGIYTKMoVCgZSUFI0VR0RERJmb2kGld+/eKFu2LPbv348cOXJAoVCkR11ERERE6geVBw8eYPv27XB2dk6PeoiIiIgkas/6KV++PEJCQtKjFiIiIiIVaveo9O/fH0OHDsWrV69QvHhx6Ovrqyx3cXHRWHFERESUuakdVFq1agUA6Natm9SmUCgghOBgWiIiItKoHzrXDxEREdGvoHZQyZ07d3rUQURERJTGDx3w7eHDh/Dy8sKdO3cAAEWKFMHAgQORL18+jRZHREREmZvaQSUwMBBNmzZFyZIlUalSJQDA6dOnUbRoUezduxd16tTReJHa4jRqv7ZLIC0LndVI2yUQEWVqageVUaNGYfDgwZg1a1aa9pEjR2aooEJERETapfZxVO7cuYPu3bunae/WrRv++ecfjRRFREREBPxAULGxscG1a9fStF+7dg22traaqImIiIgIwA/s+unZsyd69eqFR48eoWLFigA+jVGZPXs2hgwZovECiYiIKPNSO6iMHz8eWbNmxfz58zF69GgAgL29PSZNmoQBAwZovEAiIiLKvNQOKgqFAoMHD8bgwYMRExMDAMiaNavGCyMiIiL6oeOoAEB4eDju3bsHAChUqBBsbGw0VhQRERER8AODaWNiYtCpUyfY29ujWrVqqFatGuzt7dGxY0dERUWlR41ERESUSakdVHr06IHz589j//79iIyMRGRkJPbt24dLly7Bw8MjPWokIiKiTErtXT/79u1DYGAgKleuLLXVq1cPK1euRP369TVaHBEREWVuaveoZMuWDebm5mnazc3NYWlpqZGiiIiIiIAfCCrjxo3DkCFD8OrVK6nt1atXGD58OMaPH6/R4oiIiChzU3vXj7e3N0JCQpArVy7kypULAPDkyRMYGhrizZs3WLFihbTulStXNFcpERERZTpqB5XmzZunQxlEREREaakdVCZOnJgedRARERGlofYYladPn+LZs2fS9QsXLmDQoEHw9fXVaGFEREREageV9u3bIzg4GMCnQbS1a9fGhQsXMHbsWEyZMkXjBRIREVHmpXZQuXXrFv744w8AwNatW1G8eHGcOXMG/v7+WLdunabrIyIiokxM7aCSnJwMQ0NDAMDhw4fRtGlTAJ/O9/Py5UvNVkdERESZmtpBpWjRovDx8cHJkycRFBQkHY32xYsXyJYtm8YLJCIiosxL7aAye/ZsrFixAtWrV0e7du1QokQJAMCePXukXUJEREREmqD29OTq1avj7du3iI6OVjlkfq9evWBiYqLR4oiIiChzU7tHBQCEELh8+TJWrFiBmJgYAICBgQGDChEREWmU2j0qYWFhqF+/Pp48eYLExETUqVMHWbNmxezZs5GYmAgfH5/0qJOIiIgyIbV7VAYOHIiyZcvi/fv3MDY2ltpbtGiBI0eOaLQ4IiIiytzU7lE5efIkzpw5AwMDA5V2JycnPH/+XGOFEREREando6JUKpGSkpKm/dmzZ8iaNatGiiIiIiICfiCo1K1bF15eXtJ1hUKB2NhYTJw4EQ0bNtRkbURERJTJqb3rZ/78+ahXrx6KFCmChIQEtG/fHg8ePIC1tTU2bdqUHjUSERFRJqV2UHFwcMD169exZcsWXL9+HbGxsejevTs6dOigMriWiIiI6GepHVQAQE9PDx06dECHDh2ktpcvX2L48OFYunSpxoojIiKizE2toHL79m0EBwfDwMAAbm5usLCwwNu3bzF9+nT4+Pggb9686VUnERERZULfPZh2z549KFWqFAYMGIDevXujbNmyCA4ORuHChXHnzh3s3LkTt2/fTs9aiYiIKJP57qAybdo0eHp6Ijo6GgsWLMCjR48wYMAA/P333zh48KB0FmUiIiIiTfnuoHLv3j14enoiS5Ys6N+/P3R0dLBw4UKUK1cuPesjIiKiTOy7g0pMTAzMzMwAALq6ujA2NuaYFCIiIkpXag2mDQwMhLm5OYBPR6g9cuQIbt26pbJO06ZNNVcdERERZWpqBRV3d3eV6x4eHirXFQrFVw+vT0RERPQjvjuoKJXK9KyDiIiIKA21z/VDRERE9KtoNajMnDkT5cqVQ9asWWFra4vmzZvj3r172iyJiIiIZESrQeX48ePw9PTEuXPnEBQUhOTkZNStWxdxcXHaLIuIiIhk4ofO9aMpBw8eVLm+bt062Nra4vLly6hataqWqiIiIiK50GpQ+VJUVBQAwMrK6qvLExMTkZiYKF2Pjo7+JXURERGRdvzQrp/IyEisWrUKo0ePRkREBADgypUreP78+Q8XolQqMWjQIFSqVAnFihX76jozZ86Eubm5dHF0dPzhxyMiIiL5Uzuo3LhxAwUKFMDs2bMxb948REZGAgACAgIwevToHy7E09MTt27dwubNm7+5zujRoxEVFSVdnj59+sOPR0RERPKndlAZMmQIunTpggcPHsDIyEhqb9iwIU6cOPFDRfTr1w/79u1DcHAwHBwcvrmeoaEhzMzMVC5ERESUcak9RuXixYtYsWJFmvacOXPi1atXat2XEAL9+/fHzp07cezYMeTJk0fdcoiIiCgDUzuoGBoafnUQ6/3792FjY6PWfXl6emLjxo3YvXs3smbNKgUdc3NzGBsbq1saERERZTBq7/pp2rQppkyZguTkZACfzu/z5MkTjBw5Eq1atVLrvry9vREVFYXq1asjR44c0mXLli3qlkVEREQZkNpBZf78+YiNjYWtrS3i4+NRrVo1ODs7I2vWrJg+fbpa9yWE+OqlS5cu6pZFREREGZDau37Mzc0RFBSEU6dO4caNG4iNjUXp0qVRu3bt9KiPiIiIMrEfPuBb5cqVUblyZU3WQkRERKRC7aCyePHir7YrFAoYGRnB2dkZVatWha6u7k8XR0RERJmb2kFl4cKFePPmDT58+ABLS0sAwPv372FiYoIsWbIgPDwcefPmRXBwMI8cS0RERD9F7cG0M2bMQLly5fDgwQO8e/cO7969w/3791G+fHksWrQIT548gZ2dHQYPHpwe9RIREVEmonaPyrhx47Bjxw7ky5dPanN2dsa8efPQqlUrPHr0CHPmzFF7qjIRERHRl9TuUXn58iU+fvyYpv3jx4/SAdvs7e0RExPz89URERFRpqZ2UKlRowY8PDxw9epVqe3q1avo06cPatasCQC4efMmD4dPREREP03toLJ69WpYWVmhTJkyMDQ0hKGhIcqWLQsrKyusXr0aAJAlSxbMnz9f48USERFR5qL2GBU7OzsEBQXh7t27uH//PgCgYMGCKFiwoLROjRo1NFchERERZVo/fMC3QoUKoVChQpqshYiIiEjFDwWVZ8+eYc+ePXjy5AmSkpJUli1YsEAjhRERERGpHVSOHDmCpk2bIm/evLh79y6KFSuG0NBQCCFQunTp9KiRiIiIMim1B9OOHj0aw4YNw82bN2FkZIQdO3bg6dOnqFatGv7888/0qJGIiIgyKbWDyp07d9C5c2cAgJ6eHuLj45ElSxZMmTIFs2fP1niBRERElHmpHVRMTU2lcSk5cuTAw4cPpWVv377VXGVERESU6ak9RsXV1RWnTp1C4cKF0bBhQwwdOhQ3b95EQEAAXF1d06NGIiIiyqTUDioLFixAbGwsAGDy5MmIjY3Fli1bkD9/fs74ISIiIo1SK6ikpKTg2bNncHFxAfBpN5CPj0+6FEZERESk1hgVXV1d1K1bF+/fv0+veoiIiIgkag+mLVasGB49epQetRARERGpUDuoTJs2DcOGDcO+ffvw8uVLREdHq1yIiIiINEXtwbQNGzYEADRt2hQKhUJqF0JAoVAgJSVFc9URERFRpqZ2UAkODk6POoiIiIjSUDuoVKtWLT3qICIiIkpD7TEqAHDy5El07NgRFStWxPPnzwEAfn5+OHXqlEaLIyIiosxN7aCyY8cO1KtXD8bGxrhy5QoSExMBAFFRUZgxY4bGCyQiIqLM64dm/fj4+GDlypXQ19eX2itVqoQrV65otDgiIiLK3NQOKvfu3UPVqlXTtJubmyMyMlITNREREREB+IGgYmdnh5CQkDTtp06dQt68eTVSFBERERHwA0GlZ8+eGDhwIM6fPw+FQoEXL17A398fw4YNQ58+fdKjRiIiIsqk1J6ePGrUKCiVStSqVQsfPnxA1apVYWhoiGHDhqF///7pUSMRERFlUmoHFYVCgbFjx2L48OEICQlBbGwsihQpgixZsqRHfURERJSJqb3r56+//sKHDx9gYGCAIkWK4I8//mBIISIionShdlAZPHgwbG1t0b59e/z99988tw8RERGlG7WDysuXL7F582YoFAq4ubkhR44c8PT0xJkzZ9KjPiIiIsrE1A4qenp6aNy4Mfz9/REeHo6FCxciNDQUNWrUQL58+dKjRiIiIsqk1B5M+zkTExPUq1cP79+/R1hYGO7cuaOpuoiIiIh+7KSEHz58gL+/Pxo2bIicOXPCy8sLLVq0wO3btzVdHxEREWViaveotG3bFvv27YOJiQnc3Nwwfvx4VKhQIT1qIyIiokxO7aCiq6uLrVu3ol69etDV1VVZduvWLRQrVkxjxREREVHmpnZQ8ff3V7keExODTZs2YdWqVbh8+TKnKxMREZHG/NAYFQA4ceIE3N3dkSNHDsybNw81a9bEuXPnNFkbERERZXJq9ai8evUK69atw+rVqxEdHQ03NzckJiZi165dKFKkSHrVSERERJnUd/eoNGnSBAULFsSNGzfg5eWFFy9eYMmSJelZGxEREWVy392jcuDAAQwYMAB9+vRB/vz507MmIiIiIgBq9KicOnUKMTExKFOmDMqXL4+lS5fi7du36VkbERERZXLfHVRcXV2xcuVKvHz5Eh4eHti8eTPs7e2hVCoRFBSEmJiY9KyTiIiIMiG1Z/2YmpqiW7duOHXqFG7evImhQ4di1qxZsLW1RdOmTdOjRiIiIsqkfnh6MgAULFgQc+bMwbNnz7Bp0yZN1UREREQE4CeDSipdXV00b94ce/bs0cTdEREREQHQUFAhIiIiSg8MKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbWg0qJ06cQJMmTWBvbw+FQoFdu3ZpsxwiIiKSGa0Glbi4OJQoUQLLli3TZhlEREQkU3rafPAGDRqgQYMG2iyBiIiIZEyrQUVdiYmJSExMlK5HR0drsRoiIiJKb7/VYNqZM2fC3Nxcujg6Omq7JCIiIkpHv1VQGT16NKKioqTL06dPtV0SERERpaPfatePoaEhDA0NtV0GERER/SK/VY8KERERZS5a7VGJjY1FSEiIdP3x48e4du0arKyskCtXLi1WRkRERHKg1aBy6dIl1KhRQ7o+ZMgQAIC7uzvWrVunpaqIiIhILrQaVKpXrw4hhDZLICIiIhnjGBUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLVkElWXLlsHJyQlGRkYoX748Lly4oO2SiIiISAa0HlS2bNmCIUOGYOLEibhy5QpKlCiBevXqITw8XNulERERkZZpPagsWLAAPXv2RNeuXVGkSBH4+PjAxMQEa9as0XZpREREpGVaDSpJSUm4fPkyateuLbXp6Oigdu3aOHv2rBYrIyIiIjnQ0+aDv337FikpKciePbtKe/bs2XH37t006ycmJiIxMVG6HhUVBQCIjo5Ol/qUiR/S5X7p95Fe29b34jZI3AZJ29JjG0y9TyHEf66r1aCirpkzZ2Ly5Mlp2h0dHbVQDWUG5l7aroAyO26DpG3puQ3GxMTA3Nz8X9fRalCxtraGrq4uXr9+rdL++vVr2NnZpVl/9OjRGDJkiHRdqVQiIiIC2bJlg0KhSPd6M5Po6Gg4Ojri6dOnMDMz03Y5lAlxGyRt4zaYfoQQiImJgb29/X+uq9WgYmBggDJlyuDIkSNo3rw5gE/h48iRI+jXr1+a9Q0NDWFoaKjSZmFh8QsqzbzMzMz4BiWt4jZI2sZtMH38V09KKq3v+hkyZAjc3d1RtmxZ/PHHH/Dy8kJcXBy6du2q7dKIiIhIy7QeVNq0aYM3b95gwoQJePXqFUqWLImDBw+mGWBLREREmY/WgwoA9OvX76u7ekh7DA0NMXHixDS72oh+FW6DpG3cBuVBIb5nbhARERGRFmj9yLRERERE38KgQkRERLLFoEJERESyxaBCREREssWgQkRERLLFoEJERESyxaBCREREssWgQkRERLLFoEJERESyxaBCvyWlUqntEoiI6BdgUKHfko7Op0337du3AACeCYJ+tS/DMrdB0oYvt8OM+COOQYV+W4sWLULz5s3x8OFDKBQKbZdDmYyOjg6ioqIQGBgIANwGSSt0dHQQGRmJuXPn4v3799KPuIwk4z0jyrC+/MWqr68PY2NjGBgYaKkiysyUSiXmz58PDw8P7Nu3T9vlUCZ26NAhLFiwAEuXLtV2KemCZ0+m3050dDTMzMwAAFFRUTA3N9dyRZRZKJVKlV+sd+7cwerVqzF79mzo6upqsTLKTFJSUlS2t+TkZGzZsgXt2rXLkNshgwr9VgYPHoyUlBSMHj0aOXLk0HY5lAlFRkYiMjISjo6OKl8KX355EP2ML0Pxl969e4fTp0+jYsWKsLa2ltoz4nbIXT8ka1/maAcHB2zYsCHDvRHp9yCEwKhRo1C+fHmEhoaqLOM2ST/j5cuXePHiBd68eQPg09iTf+tH2Lp1K5o3b47jx4+rtGfE7ZA9KiQbqb8EhBBQKBTf/EXx/v17WFpaaqFCymj+61fr19YJCwvDuHHjsG7dugz5pUC/3tq1a7Fs2TI8ffoU+fLlQ+XKlTFnzhyVdb7WU+Ll5YV+/fpBT0/vV5b7yzGokFakhhHg0xtQCAE9PT08f/4cO3fuRNeuXWFqagrg0+4eS0tLTJgwIc1tiX7U5wHk6NGjePLkCZydnZE3b17Y29urrBMVFQWlUpkmIGfEbnb6tfbt2wc3NzcsX74cJiYmePToEebMmYOKFSti/fr1yJYtm/SZ9/btW4SEhMDV1VXlPj5+/Jihwwp3/dAvkZqHo6OjER8fD4VCgUOHDiEkJAS6urrQ09NDWFgYSpUqhRcvXkghJS4uDvr6+li4cCEiIiIYUkgjhBBSSBk1ahS6dOmCefPmoVevXhg2bBguXrwI4FP3e2JiIiZMmIDSpUvj3bt3KvfDkEI/6+LFi2jUqBG6dOkCNzc3jBgxAoGBgbhx4wY6dOgA4NPU9+TkZPj5+aFixYo4deqUyn1k5JACMKjQL/Tq1SsUL14cx48fx8aNG1G/fn38888/AD7tzilatChatGiB6dOnS7cxNTXFiBEj8ODBA1hZWTGkkEakbkfz5s3DX3/9hU2bNuHWrVto2bIl9u7di3HjxuHs2bMAAAMDA5QqVQq1atWChYWFFqumjOjx48d4+fKlSlu5cuWwZ88eXL58GT179gTw6XAMjRs3xvTp09P0qGR4gugX6tq1qzAzMxM6Ojpi5cqVUntSUpLYsmWLSElJkdqUSqU2SqRM4vXr16Jly5ZizZo1Qggh9uzZI8zMzETv3r1FqVKlRK1atcS5c+eEEKrb4sePH7VSL2VMgYGBInv27GLz5s1SW+r25u/vL5ydncXFixfT3C45OfmX1aht7FGhXyL1sM6enp6IiYmBgYEB7OzskJCQAODTrwU3NzeVQYvsPaH0ZGtrixEjRqB+/fq4evUqPD09MW3aNHh7e6NVq1Y4d+4cPD09cfnyZZVtkbt7SJMKFy6M6tWrw8/PD0eOHAHwv8++kiVLIjw8XDpVyOcy+u6ezzGo0C+RGkAcHR1x6tQpuLu7o23btti9ezfi4+PTrJ8Rz1dB2vOt7alUqVLIkSMHDhw4ABcXF/Tq1QsAYGVlBVdXVzRp0gSlSpX6laVSJuPo6IjevXsjMjISCxcuxJ49e6RlOXLkQJ48ebRYnTxknkhGWiH+f/Dry5cvkZycjFy5csHW1hYVK1ZEQkICunfvjnXr1qFx48YwMjKCj48PateuDWdnZ22XThmE+Gzg7KpVqxAeHg4DAwMMGzZMOv1CYmIinj9/jtDQUBQsWBCHDh1C06ZN0b9//3+dKk/0M1JnjVWvXh3Lly/HmDFjMHLkSAQGBsLFxQVbt26FQqFAnTp1tF2qVnF6MqW7gIAATJo0Ca9fv0ajRo3QokULNGnSBADQtWtX7Ny5E0OHDsXr16/h7e2NmzdvokiRIlqumjKaiRMnwsvLC+XKlcOFCxdQvnx5+Pn5wc7ODnv37sW0adPw/v176OvrQwiBGzduQE9PjzPNKF2kblcBAQFYvnw5Dh06hLt37yI4OBhLly6Fo6MjLCws4O/vD319/Uw9FZ5BhdLV7du3Ua9ePQwePBgmJibYtGkTDA0N4e7ujo4dOwIABg4ciCtXriAxMRG+vr4oWbKkdoumDOHzXpCPHz/C3d0d/fv3R6lSpRAaGopGjRrBzs4OO3fuhI2NDfbv34+QkBDExsZi5MiR0NPTy9RfDqQZqYFEfHHsKF1dXQQEBKBz585YsGCBtNsR+LS96ujoqGy/mWlMypcYVCjd3L17F9u2bUN8fDxmzJgBALh58yYmTJiA6OhodO3aVQorr169gqmpKbJmzarNkimD+Dyk3LlzB9HR0VixYgUmTJgAJycnAJ+mhdapUwfZs2fHrl27YGNjo3IfDCn0sz7fDt++fQuFQoFs2bIB+PSZV7p0aUyYMAG9e/eWbvNlDx579BhUKB0IIfD+/Xs0btwY//zzD5o0aQI/Pz9p+Y0bNzBhwgTEx8ejbdu26Nq1qxarpYxs+PDhUtf569evERAQgAYNGkgf/I8fP0aDBg0ghMDp06dVTu5G9DM+DxhTp07Frl27EB0dDWtra0yfPh01a9bE8+fPkTNnTi1XKn8cHUYap1AoYGVlhZkzZ6Jo0aK4cuUKgoKCpOUuLi6YOnUqkpOTpTcvkSZ8Prtn3759OHjwIBYvXozly5cjT548GDt2LK5fvy4dKTlPnjzYt28fSpYsyfNHkUalhpQpU6Zg0aJF0vR3a2trdOjQAevXr0/Ti0dfxx4V0ohvdU8eP34cY8aMgZ2dHTw9PVGzZk1p2e3bt2Fubg4HB4dfWSplAgEBAThz5gyyZcuG0aNHAwBiY2NRunRpmJmZYdWqVShRokSabZa7e0iT3r17h7p168LT0xPdunWT2nv16oW9e/ciODgYhQoV4u6d/8AeFfppqW+yM2fOYMGCBRg/fjxOnz6N5ORkVKtWDVOmTMGrV6+wdOlSHDt2TLpd0aJFGVJI4+Lj4zF+/HgsWLAAt2/fltqzZMmCK1euICYmBh4eHtL5fD7HkEKa9PHjR7x9+1bqrUs9wKWvry/s7e2xcOFCADy45X9hUKGf8vkUuwYNGuD06dPYs2cPxowZg+nTpyMpKQm1atXClClT8O7dO0ydOhUnT57UdtmUgRkbG+PkyZOoXbs2Ll++jD179iAlJQXA/8LK3bt3sWLFCi1XShnJ13ZOZM+eHXZ2dlizZg0AwMjICElJSQAAZ2dnBpTvxKBCPyW1J2XAgAFYsGABduzYgW3btuHy5cvYsmULxo0bJ4WVUaNGQV9fn0daJI35fEyKEEL6srCyssLGjRthaWmJuXPnIjAwUFpmamqKV69ewdfXVys1U8ajVCql0PHixQuEh4fjw4cPAIBJkybh7t270sye1IMMPnv2jCe5/E4co0I/JPWNqVAosHz5cly7dg2+vr54/PgxateujcqVK8PMzAzbtm2Dh4cHxowZA0NDQ3z48AEmJibaLp8ygM+nfi5ZsgTXr1/Ho0ePMGjQIJQuXRoODg548+YNmjVrBl1dXYwZMwb16tVTOcIsx6TQz/D394erqyvy5csHABg9ejQCAwMRFhaG2rVro2nTpujQoQNWrlyJqVOnIlu2bChWrBgePnyIyMhI6aCC9O8YVOi7pH4pfB40rl27hpIlSyI6OhpPnz6Fs7Mz6tevjzx58mDNmjWIioqSjjDbpUsXTJ8+nYPG6Kd9uQ2NHj0aq1evRq9evfDs2TOcPXsWzZo1Q69eveDs7Iw3b96gZcuWePPmDdatWwdXV1ctVk8ZxYEDB9C4cWOMHDkSgwYNwoEDBzBixAh4eXnh3bt3uHLlCgIDAzF+/Hj07t0bN2/ehJeXF3R0dGBpaYkZM2bwoILfK13PzUwZyqNHj0S7du3EP//8I7Zu3SoUCoW4cOGCdErymzdvikKFConz588LIYR4+PChaNy4sRgzZox48uSJNkunDCYlJUUIIYSfn5/IkyePuHz5shBCiJMnTwqFQiHy588vBg4cKB49eiSEEOLly5eiV69e4uPHj1qrmTKepUuXCgcHBzF16lTRr18/sXLlSmnZ06dPxZQpU4STk5M4ePDgV2+fnJz8q0r9rbHPib5bQkICTp48iS5duuDatWtYu3YtypUrJ+0GEkLg48ePOHv2LIoWLYoNGzYAAIYNG8ZjVNBP69SpE2xsbLBgwQLo6OggOTkZBgYG6N27N0qXLo1du3aha9euWLVqFV69eoVp06ZBR0cHPXv2ROHChaXBs/wFSz8rKSkJBgYG8PT0hImJCUaPHo2YmBhMmzZNWsfBwQGdO3fGoUOHcOnSJdSrVy/NyS252+c7aTsp0e8h9Resj4+P0NHRESVKlBBXr15VWScqKkp06dJF5MuXTzg5OQkbGxvply7Rz4iKihKTJ08WVlZWYtKkSVL78+fPxevXr8XLly9F2bJlxfz586X17e3tRY4cOcSiRYuEEELq+SPSlJkzZ4rw8HDh7+8vTExMRMOGDcX9+/dV1mnTpo1o2bKllirMGDjrh/6TEAI6OjoQQsDe3h7z58/Hx48fMW7cOJw6dUpaz8zMDPPmzcPy5csxceJEnD9/HqVLl9Zi5ZQRxMTEwMzMDH369MG4cePg5eWFiRMnAgDs7e1ha2uLly9f4v3799L4k+fPn6Nu3bqYMGECPD09AfBYFfTzxGdDOtevX4+pU6fiwYMHaN++PRYuXIgrV67Ax8cH9+7dAwBER0fj8ePHyJUrl7ZKzhDY70T/Svz/wMWjR4/i+PHjGDRoEJo0aYLatWvDzc0Ns2bNwpgxY1CxYkUAn046WLduXS1XTRnFiBEjsGLFCjx8+BA2Njbo2LEjhBCYOnUqAGDy5MkAPoUZXV1dnD59GkIIzJo1CyYmJtKUUO7uIU1IDbtHjhzB1atX4evrK3329erVC8nJyZg8eTIOHjyI0qVLIy4uDklJSZgzZ442y/79abM7h+Qttat8+/btwtzcXIwePVpcvHhRWn7jxg1RpEgR0bhxY/HXX3+JSZMmCYVCIZ4+fcpudtKI69evi6pVq4qCBQuKN2/eCCGECA8PF/PnzxcWFhZiwoQJ0rr9+vUT+fLlEw4ODsLV1VUkJSUJIbjLhzTr2LFjonjx4iJbtmxi165dQgghEhMTpeWrV68WWbJkEaVLlxYbNmyQBnBz4OyP4/Rk+lcXLlxA/fr1MXv2bPTs2VNqj46OhpmZGe7cuYOePXsiPj4eUVFR2Lp1K3f3kEacPXsWb968QZEiRdCmTRvExsZKZzh+8+YN/Pz8MHXqVOlkb8CnKfMKhQLFixeHjo4OPn78yAGL9FPEF9PhY2NjMXfuXPj6+qJ8+fLYtGkTjI2NkZycDH19fQDAggULcObMGWzbtg0KhYI9ej+JQYX+1dKlS7Fz504cOXIEUVFROHr0KP766y/cuXMHw4YNQ7du3RAeHo6oqCiYm5vD1tZW2yVTBtG5c2e8ePEChw8fRmhoKFq3bo2YmJg0YWXatGno168fpkyZonJ7fjmQJi1btgwODg5o1qwZ4uPjMW/ePOzcuRPVq1fHjBkzYGRkpBJWUgPOl0GH1MfBtPSv7OzscPnyZcycOROtW7fG2rVrYWRkhEaNGqFHjx64f/8+bG1tkT9/foYU0qhly5bh2bNnWLp0KZycnLBp0yaYm5ujUqVKePv2LWxsbNCpUydMmDAB06ZNw+rVq1Vuz5BCmvLmzRscPXoUffv2xcGDB2FsbIwhQ4agcePGOHPmDMaOHYuEhATo6+vj48ePAMCQokHsUSFJ6psqNjYWWbJkAQC8fv0aS5YswdatW1GzZk106dIFf/zxB16/fo2mTZti3bp1KFq0qJYrp4wmtTdk8eLFuHr1KhYsWABLS0vcvXsXnTt3RlRUlNSz8urVKxw/fhytWrXibh7SiC+PdwIA169fx+LFi3H48GH4+PigQYMGiIuLw5w5c3D48GEULlwYy5cvl87lQ5rDHhWSKBQK7N+/H+3atUP16tWxbt066OnpYdq0aTh//jx8fHzg6uoKHR0dLFmyBHFxcexFoXSR2htSvXp1nDhxAvv37wcAFCxYEH5+frC0tETVqlXx+vVr2NnZoU2bNtDT05N+zRL9jNSQ8urVK6mtRIkSGDhwIGrUqIHevXvj4MGDMDU1xYgRI/DHH39AR0dH2u1DGqalQbwkQ6dPnxZGRkZi+PDhon79+sLFxUV4eHiIkJAQaZ3g4GDRq1cvYWVlleaAb0Q/KvWAgl/j4+MjChQoIO7duye13bt3Tzg5OYm2bdv+ivIok/h8O9y8ebPImzevykxHIYS4du2aaNasmciVK5c4duyYEEKI+Ph4aXbZv23L9GPYo0IAgLCwMAQFBWH69OmYM2cODhw4gF69euHGjRuYOXMmHj16hLi4OJw9exbh4eE4fvw4SpYsqe2yKQP4vJv9woULOHPmDI4fPy4tb9q0KcqXL4/g4GCprUCBAjhx4gT++uuvX14vZUyJiYnSdpiUlIR8+fKhUKFC8PT0xOXLl6X1SpQogebNm+Pp06eoW7cuzpw5AyMjI2lMype7jOjn8S+aCS1duhR///23dP3evXto06YN1qxZAyMjI6nd09MTHTp0wO3btzFnzhxERkZi+PDhWL9+PYoVK6aN0imD+fyDfcyYMejSpQu6desGd3d3tGnTBtHR0ciRI4e0/z85OVm6raOjI3R1dZGSkqKt8imDOHDgAPz8/AAAPXv2RM2aNVG2bFkMHToUdnZ28PDwwKVLl6T1c+XKhbZt22L+/PkoX7681M6Bs+lE21069Gs9fvxYtG/fXjx48EClfdSoUcLW1la0bNlSOrBWKm9vb1GwYEExYMAAHrSI0sW8efNEtmzZxPnz50VKSoqYMWOGUCgU4tSpU9I6lSpVEh4eHlqskjKqdu3aCScnJ1GvXj1hbW0trl+/Li07evSoaN68uShWrJg4cOCAePz4sWjevLkYOnSotA7Pyp2+GFQyobi4OCGEEOfOnRPbt2+X2idMmCCKFy8uxo0bJ16/fq1ym5UrV4rHjx//yjIpk1AqlcLd3V34+voKIYTYsWOHsLCwED4+PkIIIWJiYoQQQhw4cEA0bdpU3LhxQ2u1UsZVsmRJoVAoVE56merkyZOiU6dOQqFQiAIFCggXFxfpRxuPfJz+OJcvEzI2NkZkZCRmzpyJ58+fQ1dXF82bN8fkyZORnJyM/fv3QwiBgQMHwsbGBgDQo0cPLVdNGVVCQgLOnz+P6tWr49ixY3B3d8fcuXPh4eGBjx8/Ys6cOahQoQJcXV0xZcoUXLhwAcWLF9d22ZRBJCUlISEhAc7OzsiVKxe2bNmCnDlzom3bttJhGipXrozy5cujZ8+eSE5ORrVq1aCrq8sjH/8iHKOSCSkUClhYWGDo0KHIkycPvLy8EBAQAACYMWMG6tevj6CgIMyYMQNv377VcrWUkdy4cQPPnj0DAAwePBjHjx+HsbEx2rdvj7/++gsNGzbEwoULpZMJvn//HpcuXcK9e/dgaWkJPz8/5M6dW5tPgTIYAwMDmJmZYdu2bdi9ezfKlSuHOXPmYPPmzYiJiZHWS0hIQJUqVVCzZk1pbBRDyq/BoJIJiU+7/FClShUMHjwYlpaWWLx4sUpYcXV1xdWrV1VOa070o4QQuH//PmrUqIE1a9agd+/eWLRoESwtLQEArq6uCAsLQ/ny5VGhQgUAwIsXL9ClSxdERkaiX79+AIB8+fKhdu3aWnselPEIIaBUKqXr69evR8WKFbFw4UJs2LABT548Qc2aNfHnn39K6wM88vGvxCPTZkKpR/2MioqCiYkJbty4genTp+P9+/cYOHAgmjdvDuDTYaNTd/0QacLKlSsxYsQIJCQkYPfu3ahbt650ROQtW7ZgypQpEEJAT08PxsbGUCqVOHPmDPT19XnuHvppERERsLKyUmlL3f62bduGoKAg+Pr6AgB69eqFY8eOISUlBVZWVjh9+jSPOqsl7FHJZD5+/AhdXV2EhoaievXqOHToEMqUKYNhw4bBxsYGkydPxr59+wCAIYU0JvUXq6OjIwwNDWFmZoZz584hNDRUmtLZpk0bbNiwAVOmTIGbmxtGjhyJc+fOSedPYUihn7Fo0SKUK1dOZXcOACmkdOnSBSVKlJDafX19sWLFCixZsgTnzp2DgYEBj3ysLdoZw0u/wrdGo4eEhIjs2bOLHj16qEyrO3bsmOjUqZMIDQ39VSVSBvflNpiUlCTi4+OFt7e3yJkzpxgzZsx/bm+c+kk/a8WKFcLQ0FBs3LgxzbInT56I4sWLi6VLl0ptX9vmuB1qD3f9ZFDi/7szz549izt37iAkJASdO3dGjhw5sH79ely6dAnr169Pc4bPhIQElYO+Ef2oz484GxERgZiYGJWBsF5eXpg3bx66d++Orl27wsnJCU2aNMHYsWPh6uqqrbIpg1m5ciX69+8PPz8//Pnnn4iMjERcXBwSEhJga2uLrFmz4sGDB8ifP7+2S6VvYFDJwHbs2IFevXpJJ2978+YN2rRpg5EjRyJr1qzaLo8ysM9DypQpU3Do0CHcunULbm5uaNGiBRo0aADgU1jx8vJCsWLF8O7dOzx58gShoaE8uRtpxKNHj+Ds7Aw3Nzds3rwZt27dQt++ffHmzRuEhYWhRo0a6NOnDxo3bqztUulfcG5VBnXr1i0MHjwY8+fPR5cuXRAdHQ0LCwsYGxszpFC6Sw0pEyZMgK+vL+bOnQsnJyf07t0bDx48QGRkJNq1a4dBgwbB2toa169fR0JCAk6ePCmdBZlTP+ln2djYYPbs2ZgwYQKGDRuGQ4cOoUqVKmjWrBmio6Oxfft2jBs3DtbW1uzFkzNt7ncizTh69Kh4+PBhmrYKFSoIIYS4c+eOyJ07t+jRo4e0/OHDh9znSunq6NGjomjRouLEiRNCCCHOnDkjDAwMRJEiRUT58uXFtm3bpHU/PzUDT9NAmpSQkCDmzZsndHR0RLdu3URSUpK07NKlS6JgwYJi2bJlWqyQ/gtn/fzGhBC4evUqGjRoAG9vb4SFhUnLnj9/DiEEYmNjUb9+fdStWxcrVqwAAAQFBcHb2xvv37/XVumUAYkv9iLnzJkTffr0QZUqVXDo0CE0btwYvr6+CAoKwsOHD7F48WKsXr0aAFR6T9iTQppkaGiI3r17Y8eOHejRowf09fWlbbVMmTIwMjLC06dPtVwl/RsGld+YQqFAqVKlMH/+fGzduhXe3t549OgRAKBRo0Z4/fo1zMzM0KhRI/j6+krd8YGBgbhx4wane5LGKJVKaUD2o0ePEBcXh/z586Ndu3ZISEjAokWLMGDAAHTq1An29vYoWrQoQkJCcOfOHS1XTpmBqakpGjRoIB1MMHVbDQ8Ph7GxMYoWLarN8ug/8KfLbyx1P76npycAYO7cudDV1UWPHj2QJ08ejB8/HjNmzMDHjx/x4cMHhISEYNOmTVi1ahVOnTolHRWU6Gd8PnB2woQJOHv2LIYPH44aNWrAysoKcXFxePnyJUxMTKCjo4PExEQ4OTlhxIgRqF+/vparp4xIfDaTMZWhoaH0/5SUFLx9+xY9e/aEQqFAu3btfnWJpAYGld9Yao/IoUOHoKOjg+TkZHh5eSEhIQEjR46Em5sb4uPjMWPGDGzfvh3Zs2eHgYEBgoODUaxYMS1XTxnF5yFlxYoV8PX1RalSpaSZO4mJibCyssKpU6ekAbPv3r3DmjVroKOjoxJ0iH5EWFgYIiIikC1bNtjZ2f3rEWSTk5Ph5+eHTZs2ISIiAufOnZPO3cNeZnni9OTfXGBgoHQiN1NTUzx48ACLFy9G3759MXLkSNjY2CAmJgbHjx+Hk5MTbG1tYWtrq+2y6Tf3Zbi4f/8+mjdvjtmzZ6NJkyZp1rt48SLGjRuH2NhYWFlZISAgAPr6+gwp9NM2bNiA+fPnIzw8HNbW1ujfv7/UU5Lqy+0sKCgIt2/fRr9+/TjL7DfAoPIbUyqV6NChAxQKBTZu3Ci1L1myBCNGjICnpyf69u2LvHnzarFKymhatmyJMWPGoGzZslLbtWvXUL9+fRw/fhwFCxb86kEEExISIISAkZERFAoFvxzop23YsAGenp7S4fFnzJiBR48e4fTp09K2lRpSIiMjcejQIbi5uancB3tS5I8/ZX5jqb8QUrvYk5KSAAD9+/eHh4cH1q5di8WLF6vMBiL6Webm5nBxcVFpMzIywvv373Hr1i2pLfX8PmfPnsWOHTugo6MDY2NjKBQKKJVKhhT6KZcuXcLUqVOxdOlSdOvWDcWLF8fgwYPh7OyMM2fO4Pbt24iOjpZ2i69fvx59+/bFX3/9pXI/DCnyx6DyG3rx4oX0/4IFC2Lv3r0IDw+HgYEBkpOTAQAODg4wMTFBcHAwjI2NtVUqZSDPnz8HAKxduxYGBgZYvHgxDh06hKSkJDg7O6NNmzaYO3cuDh8+DIVCAR0dHaSkpGD69OkIDg5WGTfA3T30sxITEzFo0CA0atRIaps0aRKOHDmCdu3aoXPnzmjbti0iIiKgr6+Phg0bYtiwYRw4+xvirp/fzPXr19GvXz+0b98effr0QVJSEmrWrIm3b9/i2LFjsLOzAwCMHDkSRYsWRePGjdOc1pxIXT179gQAjB49WtqV6OLigrdv32Lz5s2oWrUqTp48iYULF+LmzZvo0KEDDAwMcOTIEbx58wZXrlxhDwpplFKpxJs3b5A9e3YAQOfOnXH48GHs2bMHjo6OOH78OKZNm4aRI0eiffv2KmNWuLvn98KfNb8ZExMTWFhYYPv27Vi3bh0MDAywYsUK2NjYoHDhwmjevDnq1q2LRYsWoWzZsgwppBEuLi44ePAgvL29ERISAgC4ceMGChYsiA4dOuDEiROoUqUKpkyZgs6dO8PPzw9Hjx5Frly5cPnyZWnAIpGm6OjoSCEFAIYNG4bz58+jbNmyyJ49Oxo0aICIiAi8fv06zVRlhpTfC3tUfkMhISEYM2YMXr16hZ49e6JTp05ISUnBvHnzEBYWBiEE+vfvjyJFimi7VMpA1qxZgwkTJqBt27bo2bMnChYsCACoWrUqHj9+DH9/f1StWhUA8OHDB5iYmEi35cBZ+tWePXuGjh07YtiwYTzp4G+OQeU3cOXKFbx8+VJlX2xISAjGjRuH0NBQ9O/fHx06dNBihZSRfT61c/Xq1ZgwYQLatWuXJqyEhYVhw4YNqFChgsp4lK8dfItIHZ9vQ6n/T/33zZs3sLGxUVk/Li4O7dq1Q1RUFI4ePcoelN8cg4rMxcTEoFGjRtDV1cWIESPQoEEDaVloaCjq168PExMT9OjRA3379tVipZTRfOsYJytXrsTkyZPRpk0b9OrVSworNWvWxOnTp3Hu3DmUKlXqV5dLGdTXtsPUtoCAAGzatAmLFi2Cvb094uPjsXv3bvj5+eH58+e4ePEi9PX1OSblN8cxKjKVmh+zZs2KOXPmQE9PD0uXLsX+/fuldZycnFCjRg28evUKR44cQWRkpJaqpYzm8y+HM2fOIDg4GNevXwfwaWDt+PHjsXnzZvj6+uLevXsAgKNHj6JHjx5ppi4T/ahTp05JJwwcMmQIZs2aBeDT+JQtW7agc+fOqF27Nuzt7QF8OqHl48ePkTdvXly6dAn6+vr4+PEjQ8pvjj0qMpPanZn6CyD1C+P8+fMYNWoUTE1N0adPH2k30NChQ5E3b160bNkSOXLk0HL1lBF83s0+ZMgQbNmyBbGxsXBwcECuXLlw4MABAMCKFSswbdo0tG3bFu7u7iqnZeAvWPoZQghERUXB1tYWDRo0gLW1NQICAnDy5EkUK1YMkZGRcHV1haenJ/r37y/d5vPPToDbYUbBoCIjqW+04OBg7NmzBxEREahcuTL+/PNPWFhY4Ny5cxg/fjwSExORN29emJiYYMuWLbh+/TocHBy0XT5lAJ+HlEOHDmHQoEHw9fWFhYUF/vnnH0ycOBGmpqa4dOkSgE9jVjw8PODl5YV+/fpps3TKgMLDw5E3b16kpKRgx44daNiwobTsa2NTvjaWhX5/3PUjIwqFAjt37kSTJk3w4cMHfPjwAX5+fujTpw8iIiLg6uqKefPmoVq1aggJCcGjR49w9OhRhhTSmNQP9j179mDz5s2oXbs2KleujGLFiqF169bYsGEDYmNj0adPHwBA9+7dsXv3buk6kaYkJibi1atXMDExga6uLtasWSNNjQcAa2tr6f+pR0H+PJgwpGQc7FGRkUuXLqFt27YYNWoUevTogbCwMJQuXRrGxsYoWbIkNmzYACsrK+ncKV9OASXShIiICDRu3BjXr19HjRo1sG/fPpXlY8aMwenTp/H333/D1NRUamc3O/2sbw3gDg0NhYuLC2rUqIEFCxYgX758WqiOtIU9Kloyc+ZMjB07VvolAHw6RLmrqyt69OiB0NBQ1KpVC82bN8e4ceNw8eJF9O3bFxERETAyMgIAhhTSiM+3QQCwsrLC+vXrUadOHVy9ehVr165VWZ4/f368e/cO8fHxKu0MKfQzPg8px44dw8aNG3H9+nU8f/4cTk5OOH36NIKDgzFixAhpAHeLFi2wZMkSbZZNvwB7VLRkyZIlGDhwIGbMmIERI0ZIb9A7d+6gYMGCaNasmfSFoVQqUbJkSYSEhKBRo0bYsmULz5VCGvH5l8PDhw+hUChgYmICOzs7PH78GJ6enoiLi8Off/4JDw8PvH79Gu7u7jAyMsK+ffvYvU4aN2zYMKxfvx56enrIkiUL7OzssHDhQpQtWxY3b95EjRo14OTkhKSkJHz8+BHXr1+XTsxKGZSgX06pVAohhFi5cqXQ0dERU6dOFcnJydLyp0+fisKFC4t9+/YJIYSIiIgQ7dq1E0uWLBHPnj3TSs2U8aRuh0IIMXHiRFG8eHFRqFAhkSNHDuHr6yuEECIkJEQ0bNhQGBkZiYIFC4oWLVqIevXqifj4eCGEECkpKVqpnTKOz7fDoKAgUaJECXHy5EkREREhdu/eLVq0aCGcnZ3FlStXhBBCPHjwQEyZMkVMnz5d+tz8/POTMh4GlV9MqVRKb0ylUin++usvoaOjI6ZNmyZ96IeHh4uSJUsKDw8PERoaKsaMGSPKlSsnXr9+rc3SKYOaMmWKsLGxEYGBgSI2Nla0aNFCWFhYiNu3bwshhHj06JFo1KiRKFmypFi4cKF0u4SEBC1VTBnR+vXrRb9+/USvXr1U2i9evCjq168v3N3dRWxsrBBCNdwwpGR83H+gBQqFAocPH8bQoUNRpkwZ6Rwqs2bNghAClpaW6NChA44fPw5XV1ds2LABPj4+sLW11XbplAF8PiZFqVTiwoULWLhwIerWrYugoCAcO3YMM2bMQJEiRZCcnIw8efJg/vz5yJ49O/bv34+AgAAAgKGhobaeAmUA4otRB7t27cKyZctw7do1JCYmSu1ly5ZFlSpVcOrUKaSkpABQndHDc0hlAtpOSpnRjh07hLGxsZg6daq4ePGiEEIIX19faTeQEEIkJiaK27dvi6CgIPH06VNtlksZ1IQJE8SsWbNEzpw5xb1790RwcLDIkiWL8Pb2FkII8eHDBzF27FgRGhoqhBDi/v37onHjxqJs2bIiICBAm6XTb+7zHhF/f3+xYcMGIYQQ/fr1ExYWFmLZsmUiKipKWicwMFAUKlRI2hYpc2FQ+cXu3bsn8uTJI5YvX55m2YoVK6TdQESa9vl4ks2bNwtHR0dx69Yt0bFjR1GvXj1hYmIiVq9eLa3z/PlzUaVKFbFhwwbptnfu3BGtW7cWYWFhv7x+yhg+3w5v3bolSpUqJUqUKCF2794thBDC3d1d5M+fX0yfPl2EhISIkJAQUatWLVGtWjWVgEOZB/vMfrEnT55AX19f5QiLqTMvevXqBVNTU3Tq1AmGhoYYNmyYFiuljCZ1ds/x48dx7NgxDB06FEWLFpUOJFirVi1069YNwKeTYfbo0QO6urpo3749dHR0oFQqUahQIWzcuJGzLOiHpW6Hw4cPx+PHj2FsbIy7d+9i8ODB+PjxI9atW4du3bph3LhxWLJkCSpVqoQsWbJgy5YtUCgU3zzWCmVcDCq/WGxsrMrxJ5RKpbS/9dixYyhTpgy2bNmict4UIk159eoVunfvjvDwcIwZMwYA0Lt3bzx8+BBHjx5FqVKlkD9/fjx58gQJCQm4ePEidHV1VQ7mxjEB9LPWrVuHVatW4ciRI8iTJw8SExPh7u6OmTNnQkdHB2vWrIGJiQm2bt2K+vXro23btjA0NERSUhIMDAy0XT79Yoylv1iJEiXw9u1b+Pr6Avj06yI1qOzevRsbN25Ey5YtUbhwYW2WSRmUnZ0dAgICkD17duzduxeXL1+Grq4u5s6diylTpqBmzZqws7NDmzZtvnn2WR47hX5WSEgIihUrhpIlS8Lc3Bx2dnZYs2YNdHV1MXjwYOzcuRNLly5F7dq1sWDBAuzZswcxMTEMKZkUfxr9Ynny5MHSpUvRu3dvJCcno3PnztDV1cW6deuwbt06nD17lkf4pHTl4uKCHTt2wN3dHT4+Pujfvz9cXFzQtGlTNG3aVGXdlJQU9qCQxoj/P1GgoaEhEhISkJSUBCMjIyQnJyNnzpyYOXMmGjduDC8vLxgbG2Pjxo1o3749hg0bBj09Pbi5uWn7KZAW8Mi0WqBUKrFjxw54eHjA1NQURkZG0NXVxaZNm1CqVCltl0eZxNWrV9GjRw+UKVMGAwcORNGiRbVdEmUSN2/eRKlSpTB+/HhMnDhRag8MDMTKlSvx/v17pKSk4NixYwCArl27Yvz48cibN6+WKiZtYlDRohcvXiAsLAwKhQJ58uRB9uzZtV0SZTJXr16Fh4cHcufOjTlz5iBPnjzaLokyiXXr1qFXr14YNGgQ2rRpA0tLSwwYMAAVK1ZEixYtULRoUezfvx8NGjTQdqmkZQwqRJnchQsX4OPjg1WrVnE2Bf1SO3bsQN++fWFgYAAhBGxtbXHmzBm8fv0aderUwfbt2+Hi4qLtMknLGFSISBo7wKmf9Ks9f/4cT58+RXJyMipVqgQdHR2MHj0au3btQnBwMOzs7LRdImkZgwoRAfhfWCHSltu3b2P27Nn4+++/cfjwYZQsWVLbJZEMcDg/EQHgtGPSro8fPyIpKQm2trY4fvw4B3eThD0qREQkG8nJyTzyMalgUCEiIiLZ4qg5IiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSL6rRw7dgwKhQKRkZHffRsnJyd4eXmlW01ElH4YVIhIo7p06QKFQoHevXunWebp6QmFQoEuXbr8+sKI6LfEoEJEGufo6IjNmzcjPj5eaktISMDGjRuRK1cuLVZGRL8bBhUi0rjSpUvD0dERAQEBUltAQABy5cqFUqVKSW2JiYkYMGAAbG1tYWRkhMqVK+PixYsq9/X333+jQIECMDY2Ro0aNRAaGprm8U6dOoUqVarA2NgYjo6OGDBgAOLi4tLt+RHRr8OgQkTpolu3bli7dq10fc2aNejatavKOiNGjMCOHTuwfv16XLlyBc7OzqhXrx4iIiIAAE+fPkXLli3RpEkTXLt2DT169MCoUaNU7uPhw4eoX78+WrVqhRs3bmDLli04deoU+vXrl/5PkojSHYMKEaWLjh074tSpUwgLC0NYWBhOnz6Njh07Ssvj4uLg7e2NuXPnokGDBihSpAhWrlwJY2NjrF69GgDg7e2NfPnyYf78+ShYsCA6dOiQZnzLzJkz0aFDBwwaNAj58+dHxYoVsXjxYmzYsAEJCQm/8ikTUTrgSQmJKF3Y2NigUaNGWLduHYQQaNSoEaytraXlDx8+RHJyMipVqiS16evr448//sCdO3cAAHfu3EH58uVV7rdChQoq169fv44bN27A399fahNCQKlU4vHjxyhcuHB6PD0i+kUYVIgo3XTr1k3aBbNs2bJ0eYzY2Fh4eHhgwIABaZZx4C7R749BhYjSTf369ZGUlASFQoF69eqpLMuXLx8MDAxw+vRp5M6dG8CnM+devHgRgwYNAgAULlwYe/bsUbnduXPnVK6XLl0a//zzD5ydndPviRCR1nCMChGlG11dXdy5cwf//PMPdHV1VZaZmpqiT58+GD58OA4ePIh//vkHPXv2xIcPH9C9e3cAQO/evfHgwQMMHz4c9+7dw8aNG7Fu3TqV+xk5ciTOnDmDfv364dq1a3jw4AF2797NwbREGQSDChGlKzMzM5iZmX112axZs9CqVSt06tQJpUuXRkhICAIDA2FpaQng066bHTt2YNeuXShRogR8fHwwY8YMlftwcXHB8ePHcf/+fVSpUgWlSpXChAkTYG9vn+7PjYjSn0IIIbRdBBEREdHXsEeFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhk6/8AHoK08GWUizwAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "## calculate avg response time\n", - "unique_models = set(result[\"response\"]['model'] for result in result[\"results\"])\n", - "model_dict = {model: {\"response_time\": []} for model in unique_models}\n", - "for completion_result in result[\"results\"]:\n", - " model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n", - "\n", - "avg_response_time = {}\n", - "for model, data in model_dict.items():\n", - " avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n", - "\n", - "models = list(avg_response_time.keys())\n", - "response_times = list(avg_response_time.values())\n", - "\n", - "plt.bar(models, response_times)\n", - "plt.xlabel('Model', fontsize=10)\n", - "plt.ylabel('Average Response Time')\n", - "plt.title('Average Response Times for each Model')\n", - "\n", - "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "inSDIE3_IRds" - }, - "source": [ - "# Duration Test endpoint\n", - "\n", - "Run load testing for 2 mins. Hitting endpoints with 100+ queries every 15 seconds." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ePIqDx2EIURH" - }, - "outputs": [], - "source": [ - "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompt = \"Where does Paul Graham live?\"\n", - "final_prompt = context + prompt\n", - "result = load_test_model(models=models, prompt=final_prompt, num_calls=100, interval=15, duration=120)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 552 - }, - "id": "k6rJoELM6t1K", - "outputId": "f4968b59-3bca-4f78-a88b-149ad55e3cf7" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAIXCAYAAABghH+YAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABwdUlEQVR4nO3dd1QU198G8GfpoNKUooKCYuwIaiL2GrGLJnYFOxrsNZbYFTsYG2JDjV2xRKOIir33EhsWLBGwUaXJ3vcPX+bnCiYsLi6Oz+ecPbp37ux+lx3YZ+/cmVEIIQSIiIiIZEJH2wUQERERaRLDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNEcmSg4MDunfvru0y1DZnzhyUKFECurq6cHFx0XY5GnfkyBEoFAps27ZN26WoTaFQYNKkSWqv9+jRIygUCgQFBWm8Jsoaww19tiVLlkChUKBatWraLiXPcXBwgEKhkG758uXDDz/8gLVr12q7tK9Oxodidm5fqwMHDmDUqFGoWbMmVq9ejRkzZmi7pDwnKChIep9PnDiRabkQAvb29lAoFGjRooUWKqS8QE/bBdDXb/369XBwcMC5c+cQHh4OJycnbZeUp7i4uGD48OEAgOfPn2PFihXw8vJCSkoK+vTpo+Xqvh5ly5bFunXrVNrGjBmD/PnzY9y4cZn637lzBzo6X9f3t8OHD0NHRwcrV66EgYGBtsvJ04yMjLBhwwbUqlVLpf3o0aN4+vQpDA0NtVQZ5QUMN/RZHj58iFOnTiE4OBje3t5Yv349Jk6c+EVrUCqVSE1NhZGR0Rd93uwqWrQounbtKt3v3r07SpQoAT8/P4YbNdjY2Kj8HAFg5syZKFSoUKZ2AF/lh1t0dDSMjY01FmyEEEhOToaxsbFGHi8vadasGbZu3Yrff/8denr/+yjbsGEDqlSpgpcvX2qxOtK2r+trDeU569evh4WFBZo3b46ff/4Z69evl5alpaXB0tISPXr0yLReXFwcjIyMMGLECKktJSUFEydOhJOTEwwNDWFvb49Ro0YhJSVFZV2FQoEBAwZg/fr1KF++PAwNDbF//34AwNy5c1GjRg0ULFgQxsbGqFKlSpb79pOSkjBo0CAUKlQIBQoUQKtWrfDs2bMs96k/e/YMPXv2hI2NDQwNDVG+fHmsWrUqxz8zKysrlClTBvfv31dpVyqV8Pf3R/ny5WFkZAQbGxt4e3vjzZs3Kv0uXLgAd3d3FCpUCMbGxnB0dETPnj2l5Rn79+fOnQs/Pz8UL14cxsbGqFu3Lm7cuJGpnsOHD6N27drIly8fzM3N0bp1a9y6dUulz6RJk6BQKBAeHo7u3bvD3NwcZmZm6NGjB96+favSNzQ0FLVq1YK5uTny58+P0qVLY+zYsSp9svtef46P59xk7M44ceIEBg0aBCsrK5ibm8Pb2xupqamIiYmBp6cnLCwsYGFhgVGjRkEIofKYmnqPsqJQKLB69WokJiZKu10y5mi8e/cOU6dORcmSJWFoaAgHBweMHTs208/LwcEBLVq0QEhICKpWrQpjY2MsW7bsX5/37NmzaNKkCczMzGBiYoK6devi5MmTKn0iIiLwyy+/oHTp0jA2NkbBggXRrl07PHr0KNPjxcTEYOjQoXBwcIChoSHs7Ozg6emZKWwolUpMnz4ddnZ2MDIyQsOGDREeHv6vtX6oU6dOePXqFUJDQ6W21NRUbNu2DZ07d85yncTERAwfPhz29vYwNDRE6dKlMXfu3Ezvc0pKCoYOHQorKyvp78PTp0+zfExN/30gDRFEn6FMmTKiV69eQgghjh07JgCIc+fOSct79uwpzM3NRUpKisp6a9asEQDE+fPnhRBCpKeni8aNGwsTExMxZMgQsWzZMjFgwAChp6cnWrdurbIuAFG2bFlhZWUlJk+eLBYvXiwuX74shBDCzs5O/PLLL2LRokVi/vz54ocffhAAxJ49e1Qeo3379gKA6Natm1i8eLFo3769qFSpkgAgJk6cKPWLjIwUdnZ2wt7eXkyZMkUsXbpUtGrVSgAQfn5+//nzKV68uGjevLlKW1pamrC1tRU2NjYq7b179xZ6enqiT58+IiAgQIwePVrky5dPfP/99yI1NVUIIURUVJSwsLAQ3333nZgzZ45Yvny5GDdunChbtqz0OA8fPhQARMWKFYWDg4OYNWuWmDx5srC0tBRWVlYiMjJS6hsaGir09PTEd999J2bPni0mT54sChUqJCwsLMTDhw+lfhMnThQAhKurq2jbtq1YsmSJ6N27twAgRo0aJfW7ceOGMDAwEFWrVhULFiwQAQEBYsSIEaJOnTpSH3Xe6/9Svnx5Ubdu3U/+7L28vKT7q1evFgCEi4uLaNKkiVi8eLHo1q2b9Bpq1aolOnfuLJYsWSJatGghAIg1a9bkynuUlXXr1onatWsLQ0NDsW7dOrFu3Tpx//59IYQQXl5eAoD4+eefxeLFi4Wnp6cAIDw8PDK9ZicnJ2FhYSF+/fVXERAQIMLCwj75nIcOHRIGBgaievXqYt68ecLPz084OzsLAwMDcfbsWanf1q1bRaVKlcSECRNEYGCgGDt2rLCwsBDFixcXiYmJUr/4+HhRoUIFoaurK/r06SOWLl0qpk6dKr7//nvpdzQsLEzalqpUqSL8/PzEpEmThImJifjhhx/+9Wf04ft4/vx5UaNGDdGtWzdp2c6dO4WOjo549uxZpt89pVIpGjRoIBQKhejdu7dYtGiRaNmypQAghgwZovIcXbt2FQBE586dxaJFi0Tbtm2Fs7Nzjv8+ZPxOrl69+j9fH2kGww3l2IULFwQAERoaKoR4/8fDzs5ODB48WOoTEhIiAIg///xTZd1mzZqJEiVKSPfXrVsndHR0xPHjx1X6BQQECADi5MmTUhsAoaOjI27evJmpprdv36rcT01NFRUqVBANGjSQ2i5evJjlH7Tu3btn+uPVq1cvUbhwYfHy5UuVvh07dhRmZmaZnu9jxYsXF40bNxYvXrwQL168ENevX5c+UH18fKR+x48fFwDE+vXrVdbfv3+/SvuOHTtUQmFWMv6QGhsbi6dPn0rtZ8+eFQDE0KFDpTYXFxdhbW0tXr16JbVdvXpV6OjoCE9PT6ktI9z07NlT5bnatGkjChYsKN338/MTAMSLFy8+WZ867/V/yUm4cXd3F0qlUmqvXr26UCgUol+/flLbu3fvhJ2dncpja/I9+hQvLy+RL18+lbYrV64IAKJ3794q7SNGjBAAxOHDh1VeMwCxf//+/3wupVIpSpUqlenn8fbtW+Ho6Ch+/PFHlbaPnT59WgAQa9euldomTJggAIjg4OAsn0+I/4WbsmXLqnzpWbBggQAgrl+//q91fxhuFi1aJAoUKCDV165dO1G/fn3pZ/FhuNm5c6cAIKZNm6byeD///LNQKBQiPDxcCPG/n/cvv/yi0q9z5845/vvAcPPlcbcU5dj69ethY2OD+vXrA3g/rN6hQwds2rQJ6enpAIAGDRqgUKFC2Lx5s7TemzdvEBoaig4dOkhtW7duRdmyZVGmTBm8fPlSujVo0AAAEBYWpvLcdevWRbly5TLV9OHcgjdv3iA2Nha1a9fGpUuXpPaMXVi//PKLyroDBw5UuS+EwPbt29GyZUsIIVTqcnd3R2xsrMrjfsqBAwdgZWUFKysrVKxYEevWrUOPHj0wZ84clddvZmaGH3/8UeV5qlSpgvz580uv39zcHACwZ88epKWl/evzenh4oGjRotL9H374AdWqVcNff/0F4P3k5itXrqB79+6wtLSU+jk7O+PHH3+U+n2oX79+Kvdr166NV69eIS4uTqW+Xbt2QalUZlmXuu+1pvXq1UvliKpq1apBCIFevXpJbbq6uqhatSoePHigUrem36PsyHgfhg0bptKeMUl97969Ku2Ojo5wd3f/z8e9cuUK7t27h86dO+PVq1fS60lMTETDhg1x7Ngx6T388PcqLS0Nr169gpOTE8zNzVV+B7Zv345KlSqhTZs2mZ7v46PYevTooTK3qHbt2gCg8jP/L+3bt0dSUhL27NmD+Ph47Nmz55O7pP766y/o6upi0KBBKu3Dhw+HEAL79u2T+gHI1G/IkCEq9zX194Fyxzcdbo4dO4aWLVuiSJEiUCgU2LlzZ64/57Nnz9C1a1dpTkjFihVx4cKFXH9eTUtPT8emTZtQv359PHz4EOHh4QgPD0e1atUQFRWFQ4cOAQD09PTw008/YdeuXdL8gODgYKSlpamEm3v37uHmzZtSCMi4fffddwDeT7T8kKOjY5Z17dmzB25ubjAyMoKlpSWsrKywdOlSxMbGSn0iIiKgo6OT6TE+PsrrxYsXiImJQWBgYKa6MuYRfVxXVqpVq4bQ0FDs378fc+fOhbm5Od68eaPyh/3evXuIjY2FtbV1pudKSEiQnqdu3br46aefMHnyZBQqVAitW7fG6tWrs5yrUqpUqUxt3333nTRPIiIiAgBQunTpTP3Kli0rfdB9qFixYir3LSwsAECac9KhQwfUrFkTvXv3ho2NDTp27IgtW7aoBB1132tN+/g1mJmZAQDs7e0ztX84lyY33qPsyNheP94+bW1tYW5uLr2PGT71u/Gxe/fuAQC8vLwyvZ4VK1YgJSVF+r1JSkrChAkTpLkqhQoVgpWVFWJiYlR+t+7fv48KFSpk6/n/a1vKDisrKzRq1AgbNmxAcHAw0tPT8fPPP2fZNyIiAkWKFEGBAgVU2suWLSstz/hXR0cHJUuWVOn38e+Jpv4+UO74po+WSkxMRKVKldCzZ0+0bds215/vzZs3qFmzJurXr499+/bBysoK9+7dk36pvyaHDx/G8+fPsWnTJmzatCnT8vXr16Nx48YAgI4dO2LZsmXYt28fPDw8sGXLFpQpUwaVKlWS+iuVSlSsWBHz58/P8vk+/uDJ6uiP48ePo1WrVqhTpw6WLFmCwoULQ19fH6tXr8aGDRvUfo0ZH8hdu3aFl5dXln2cnZ3/83EKFSqERo0aAQDc3d1RpkwZtGjRAgsWLJC+jSuVSlhbW6tMyP6QlZUVAEgnPztz5gz+/PNPhISEoGfPnpg3bx7OnDmD/Pnzq/061aGrq5tlu/j/CZnGxsY4duwYwsLCsHfvXuzfvx+bN29GgwYNcODAAejq6qr9Xmvap15DVu3ig4mm2n6Psnv+nuweGZWxfc+ZM+eTJwvMqHXgwIFYvXo1hgwZgurVq8PMzAwKhQIdO3b85Ajdf/mvbSm7OnfujD59+iAyMhJNmzaVRs5ym6b+PlDu+KbDTdOmTdG0adNPLk9JScG4ceOwceNGxMTEoEKFCpg1axbq1auXo+ebNWsW7O3tsXr1aqktu9+y8pr169fD2toaixcvzrQsODgYO3bsQEBAAIyNjVGnTh0ULlwYmzdvRq1atXD48OFM5yUpWbIkrl69ioYNG+b4JGzbt2+HkZERQkJCVA4D/vDnDQDFixeHUqnEw4cPVUY3Pj5SI+NIifT0dCmcaELz5s1Rt25dzJgxA97e3siXLx9KliyJgwcPombNmtn6cHJzc4ObmxumT5+ODRs2oEuXLti0aRN69+4t9cn4Zv6hu3fvwsHBAcD7nwPw/nwwH7t9+zYKFSqEfPnyqf36dHR00LBhQzRs2BDz58/HjBkzMG7cOISFhaFRo0Yaea+1ITfeo+zI2F7v3bsnjTIAQFRUFGJiYqT3UV0ZIxOmpqb/uX1v27YNXl5emDdvntSWnJyMmJiYTI+Z1RF5ualNmzbw9vbGmTNnVHZ/f6x48eI4ePAg4uPjVUZvbt++LS3P+FepVOL+/fsqozUf/57k1t8H0oxverfUfxkwYABOnz6NTZs24dq1a2jXrh2aNGmS5YdGduzevRtVq1ZFu3btYG1tDVdXVyxfvlzDVee+pKQkBAcHo0WLFvj5558z3QYMGID4+Hjs3r0bwPsPu59//hl//vkn1q1bh3fv3qnskgLe7zt/9uxZlj+PpKSkTLtHsqKrqwuFQiHN9wHeHxb98e7GjPkIS5YsUWlfuHBhpsf76aefsH379iz/YL948eI/a/qU0aNH49WrV9Lrbd++PdLT0zF16tRMfd+9eyd9iLx58ybTN9uMb90f7/bYuXMnnj17Jt0/d+4czp49KwX6woULw8XFBWvWrFH5kLpx4wYOHDiAZs2aqf26Xr9+nant4/o08V5rQ268R9mR8T74+/urtGeMfDVv3lztxwSAKlWqoGTJkpg7dy4SEhIyLf9w+9bV1c30mhYuXKjyuwYAP/30E65evYodO3Zkejx1R2SyK3/+/Fi6dCkmTZqEli1bfrJfs2bNkJ6ejkWLFqm0+/n5QaFQSL8XGf/+/vvvKv0+/vnn5t8H+nzf9MjNv3n8+DFWr16Nx48fo0iRIgCAESNGYP/+/Tk+LfqDBw+wdOlSDBs2DGPHjsX58+cxaNAgGBgYfHJYMy/avXs34uPj0apVqyyXu7m5wcrKCuvXr5dCTIcOHbBw4UJMnDgRFStWVPkGCgDdunXDli1b0K9fP4SFhaFmzZpIT0/H7du3sWXLFum8Hf+mefPmmD9/Ppo0aYLOnTsjOjoaixcvhpOTE65duyb1q1KlCn766Sf4+/vj1atXcHNzw9GjR3H37l0AqsP/M2fORFhYGKpVq4Y+ffqgXLlyeP36NS5duoSDBw9m+WGeHU2bNkWFChUwf/58+Pj4oG7duvD29oavry+uXLmCxo0bQ19fH/fu3cPWrVuxYMEC/Pzzz1izZg2WLFmCNm3aoGTJkoiPj8fy5cthamqaKYw4OTmhVq1a6N+/P1JSUuDv74+CBQti1KhRUp85c+agadOmqF69Onr16oWkpCQsXLgQZmZmObqGzpQpU3Ds2DE0b94cxYsXR3R0NJYsWQI7OzvpTLKaeK+1ITfeo+yoVKkSvLy8EBgYiJiYGNStWxfnzp3DmjVr4OHhIU3oV5eOjg5WrFiBpk2bonz58ujRoweKFi2KZ8+eISwsDKampvjzzz8BAC1atMC6detgZmaGcuXK4fTp0zh48CAKFiyo8pgjR47Etm3b0K5dO/Ts2RNVqlTB69evsXv3bgQEBKjsitak7Pz9bNmyJerXr49x48bh0aNHqFSpEg4cOIBdu3ZhyJAh0kiWi4sLOnXqhCVLliA2NhY1atTAoUOHsjwHT279fSAN0MoxWnkQALFjxw7p/p49ewQAkS9fPpWbnp6eaN++vRBCiFu3bgkA/3obPXq09Jj6+vqievXqKs87cOBA4ebm9kVeo6a0bNlSGBkZqZzf4mPdu3cX+vr60iGSSqVS2NvbZ3koZobU1FQxa9YsUb58eWFoaCgsLCxElSpVxOTJk0VsbKzUDx8dRv2hlStXilKlSglDQ0NRpkwZsXr1aukw5g8lJiYKHx8fYWlpKfLnzy88PDzEnTt3BAAxc+ZMlb5RUVHCx8dH2NvbC319fWFraysaNmwoAgMD//NnldV5bjIEBQVlOjw0MDBQVKlSRRgbG4sCBQqIihUrilGjRol//vlHCCHEpUuXRKdOnUSxYsWEoaGhsLa2Fi1atBAXLlyQHiPjsNM5c+aIefPmCXt7e2FoaChq164trl69mqmOgwcPipo1awpjY2NhamoqWrZsKf7++2+VPhk/w48P8c44LDfjnDiHDh0SrVu3FkWKFBEGBgaiSJEiolOnTuLu3bsq62X3vf4vOTkU/ONDtD/12rI6LFsIzbxHn/Kp50xLSxOTJ08Wjo6OQl9fX9jb24sxY8aI5OTkTK/5U9vbp1y+fFm0bdtWFCxYUBgaGorixYuL9u3bi0OHDkl93rx5I3r06CEKFSok8ufPL9zd3cXt27cz/YyFEOLVq1diwIABomjRosLAwEDY2dkJLy8v6W9BxqHgW7duVVkvu4dLf+p9/FhWP4v4+HgxdOhQUaRIEaGvry9KlSol5syZo3IovBBCJCUliUGDBomCBQuKfPnyiZYtW4onT55kOhRciOz9feCh4F+eQohcGiv8yigUCuzYsQMeHh4AgM2bN6NLly64efNmpolv+fPnh62tLVJTU//zsMWCBQtKEw2LFy+OH3/8EStWrJCWL126FNOmTVPZfUDaceXKFbi6uuKPP/5Aly5dtF1Ojj169AiOjo6YM2eOyhmgiYi+Fdwt9Qmurq5IT09HdHS0dP6FjxkYGKBMmTLZfsyaNWtmmpR29+7dHE8IpJxLSkrKNCnU398fOjo6qFOnjpaqIiIiTfimw01CQoLKftSHDx/iypUrsLS0xHfffYcuXbrA09MT8+bNg6urK168eIFDhw7B2dk5R5P4hg4diho1amDGjBlo3749zp07h8DAQAQGBmryZVE2zJ49GxcvXkT9+vWhp6eHffv2Yd++fejbt2+uH4pMRES5TNv7xbQpY9/vx7eMfcipqaliwoQJwsHBQejr64vChQuLNm3aiGvXruX4Of/8809RoUIFaU5IduZtkOYdOHBA1KxZU1hYWAh9fX1RsmRJMWnSJJGWlqbt0j7bh3NuiIi+RZxzQ0RERLLC89wQERGRrDDcEBERkax8cxOKlUol/vnnHxQoUOCrOvU7ERHRt0wIgfj4eBQpUgQ6Ov8+NvPNhZt//vmHR8MQERF9pZ48eQI7O7t/7fPNhZuMC6Y9efIEpqamWq6GiIiIsiMuLg729vYqFz79lG8u3GTsijI1NWW4ISIi+spkZ0oJJxQTERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs6Gm7ACLSLIdf92q7BNKyRzOba/X5uQ2StrdBjtwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrGg13CxduhTOzs4wNTWFqakpqlevjn379v3rOlu3bkWZMmVgZGSEihUr4q+//vpC1RIREdHXQKvhxs7ODjNnzsTFixdx4cIFNGjQAK1bt8bNmzez7H/q1Cl06tQJvXr1wuXLl+Hh4QEPDw/cuHHjC1dOREREeZVCCCG0XcSHLC0tMWfOHPTq1SvTsg4dOiAxMRF79uyR2tzc3ODi4oKAgIBsPX5cXBzMzMwQGxsLU1NTjdVNlFfwooWk7YsWchuk3NgG1fn8zjNzbtLT07Fp0yYkJiaievXqWfY5ffo0GjVqpNLm7u6O06dPf/JxU1JSEBcXp3IjIiIi+dJ6uLl+/Try588PQ0ND9OvXDzt27EC5cuWy7BsZGQkbGxuVNhsbG0RGRn7y8X19fWFmZibd7O3tNVo/ERER5S1aDzelS5fGlStXcPbsWfTv3x9eXl74+++/Nfb4Y8aMQWxsrHR78uSJxh6biIiI8h49bRdgYGAAJycnAECVKlVw/vx5LFiwAMuWLcvU19bWFlFRUSptUVFRsLW1/eTjGxoawtDQULNFExERUZ6l9ZGbjymVSqSkpGS5rHr16jh06JBKW2ho6Cfn6BAREdG3R6sjN2PGjEHTpk1RrFgxxMfHY8OGDThy5AhCQkIAAJ6enihatCh8fX0BAIMHD0bdunUxb948NG/eHJs2bcKFCxcQGBiozZdBREREeYhWw010dDQ8PT3x/PlzmJmZwdnZGSEhIfjxxx8BAI8fP4aOzv8Gl2rUqIENGzZg/PjxGDt2LEqVKoWdO3eiQoUK2noJRERElMdoNdysXLnyX5cfOXIkU1u7du3Qrl27XKqIiIiIvnZ5bs4NERER0edguCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWdFquPH19cX333+PAgUKwNraGh4eHrhz586/rhMUFASFQqFyMzIy+kIVExERUV6n1XBz9OhR+Pj44MyZMwgNDUVaWhoaN26MxMTEf13P1NQUz58/l24RERFfqGIiIiLK6/S0+eT79+9XuR8UFARra2tcvHgRderU+eR6CoUCtra2uV0eERERfYXy1Jyb2NhYAIClpeW/9ktISEDx4sVhb2+P1q1b4+bNm5/sm5KSgri4OJUbERERyVeeCTdKpRJDhgxBzZo1UaFChU/2K126NFatWoVdu3bhjz/+gFKpRI0aNfD06dMs+/v6+sLMzEy62dvb59ZLICIiojwgz4QbHx8f3LhxA5s2bfrXftWrV4enpydcXFxQt25dBAcHw8rKCsuWLcuy/5gxYxAbGyvdnjx5khvlExERUR6h1Tk3GQYMGIA9e/bg2LFjsLOzU2tdfX19uLq6Ijw8PMvlhoaGMDQ01ESZRERE9BXQ6siNEAIDBgzAjh07cPjwYTg6Oqr9GOnp6bh+/ToKFy6cCxUSERHR10arIzc+Pj7YsGEDdu3ahQIFCiAyMhIAYGZmBmNjYwCAp6cnihYtCl9fXwDAlClT4ObmBicnJ8TExGDOnDmIiIhA7969tfY6iIiIKO/QarhZunQpAKBevXoq7atXr0b37t0BAI8fP4aOzv8GmN68eYM+ffogMjISFhYWqFKlCk6dOoVy5cp9qbKJiIgoD9NquBFC/GefI0eOqNz38/ODn59fLlVEREREX7s8MaFYThx+3avtEkjLHs1sru0SiIi+aXnmUHAiIiIiTWC4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZyVG4uX//PsaPH49OnTohOjoaALBv3z7cvHlTo8URERERqUvtcHP06FFUrFgRZ8+eRXBwMBISEgAAV69excSJEzVeIBEREZE61A43v/76K6ZNm4bQ0FAYGBhI7Q0aNMCZM2c0WhwRERGRutQON9evX0ebNm0ytVtbW+Ply5caKYqIiIgop9QON+bm5nj+/Hmm9suXL6No0aIaKYqIiIgop9QONx07dsTo0aMRGRkJhUIBpVKJkydPYsSIEfD09MyNGomIiIiyTe1wM2PGDJQpUwb29vZISEhAuXLlUKdOHdSoUQPjx4/PjRqJiIiIsk3tq4IbGBhg+fLl+O2333Djxg0kJCTA1dUVpUqVyo36iIiIiNSidrjJUKxYMRQrVkyTtRARERF9NrXDjRAC27ZtQ1hYGKKjo6FUKlWWBwcHa6w4IiIiInWpHW6GDBmCZcuWoX79+rCxsYFCociNuoiIiIhyRO1ws27dOgQHB6NZs2a5UQ8RERHRZ1H7aCkzMzOUKFEiN2ohIiIi+mxqh5tJkyZh8uTJSEpKyo16iIiIiD6L2rul2rdvj40bN8La2hoODg7Q19dXWX7p0iWNFUdERESkLrXDjZeXFy5evIiuXbtyQjERERHlOWqHm7179yIkJAS1atXKjXqIiIiIPovac27s7e1hamqaG7UQERERfTa1w828efMwatQoPHr0KBfKISIiIvo8au+W6tq1K96+fYuSJUvCxMQk04Ti169fa6w4IiIiInWpHW78/f1zoQwiIiIizcjR0VJEREREeVW2wk1cXJw0iTguLu5f+3KyMREREWlTtsKNhYUFnj9/Dmtra5ibm2d5bhshBBQKBdLT0zVeJBEREVF2ZSvcHD58GJaWlgCAsLCwXC2IiIiI6HNkK9zUrVsXJUqUwPnz51G3bt3cromIiIgox7J9nptHjx5xlxMRERHleWqfxI+IiIgoL1PrUPCQkBCYmZn9a59WrVp9VkFEREREn0OtcPNf57jh0VJERESkbWrtloqMjIRSqfzkjcGGiIiItC3b4Sarc9sQERER5TXZDjdCiNysg4iIiEgjsh1uvLy8YGxsnJu1EBEREX22bE8oXr16dW7WQURERKQRPM8NERERyQrDDREREckKww0RERHJSo7DTXh4OEJCQpCUlAQgZ0dT+fr64vvvv0eBAgVgbW0NDw8P3Llz5z/X27p1K8qUKQMjIyNUrFgRf/31l9rPTURERPKkdrh59eoVGjVqhO+++w7NmjXD8+fPAQC9evXC8OHD1Xqso0ePwsfHB2fOnEFoaCjS0tLQuHFjJCYmfnKdU6dOoVOnTujVqxcuX74MDw8PeHh44MaNG+q+FCIiIpIhtcPN0KFDoaenh8ePH8PExERq79ChA/bv36/WY+3fvx/du3dH+fLlUalSJQQFBeHx48e4ePHiJ9dZsGABmjRpgpEjR6Js2bKYOnUqKleujEWLFqn7UoiIiEiG1Lq2FAAcOHAAISEhsLOzU2kvVaoUIiIiPquY2NhYAIClpeUn+5w+fRrDhg1TaXN3d8fOnTuz7J+SkoKUlBTpflxc3GfVSERERHmb2iM3iYmJKiM2GV6/fg1DQ8McF6JUKjFkyBDUrFkTFSpU+GS/yMhI2NjYqLTZ2NggMjIyy/6+vr4wMzOTbvb29jmukYiIiPI+tcNN7dq1sXbtWum+QqGAUqnE7NmzUb9+/RwX4uPjgxs3bmDTpk05foysjBkzBrGxsdLtyZMnGn18IiIiylvU3i01e/ZsNGzYEBcuXEBqaipGjRqFmzdv4vXr1zh58mSOihgwYAD27NmDY8eOZdrd9TFbW1tERUWptEVFRcHW1jbL/oaGhp81okRERERfF7VHbipUqIC7d++iVq1aaN26NRITE9G2bVtcvnwZJUuWVOuxhBAYMGAAduzYgcOHD8PR0fE/16levToOHTqk0hYaGorq1aur9dxEREQkT2qP3ACAmZkZxo0b99lP7uPjgw0bNmDXrl0oUKCANG/GzMxMukinp6cnihYtCl9fXwDA4MGDUbduXcybNw/NmzfHpk2bcOHCBQQGBn52PURERPT1U3vkZv/+/Thx4oR0f/HixXBxcUHnzp3x5s0btR5r6dKliI2NRb169VC4cGHptnnzZqnP48ePpXPpAECNGjWwYcMGBAYGolKlSti2bRt27tz5r5OQiYiI6Nuh9sjNyJEjMWvWLADA9evXMWzYMAwfPhxhYWEYNmyYWlcPz85ZjY8cOZKprV27dmjXrl22n4eIiIi+HWqHm4cPH6JcuXIAgO3bt6Nly5aYMWMGLl26hGbNmmm8QCIiIiJ1qL1bysDAAG/fvgUAHDx4EI0bNwbw/sR7PEEeERERaZvaIze1atXCsGHDULNmTZw7d06aH3P37t3/PIybiIiIKLepPXKzaNEi6OnpYdu2bVi6dCmKFi0KANi3bx+aNGmi8QKJiIiI1KH2yE2xYsWwZ8+eTO1+fn4aKYiIiIjoc+ToPDdKpRLh4eGIjo6GUqlUWVanTh2NFEZERESUE2qHmzNnzqBz586IiIjIdCi3QqFAenq6xoojIiIiUpfa4aZfv36oWrUq9u7di8KFC0OhUORGXUREREQ5ona4uXfvHrZt2wYnJ6fcqIeIiIjos6h9tFS1atUQHh6eG7UQERERfTa1R24GDhyI4cOHIzIyEhUrVoS+vr7KcmdnZ40VR0RERKQutcPNTz/9BADo2bOn1KZQKCCE4IRiIiIi0rocXVuKiIiIKK9SO9wUL148N+ogIiIi0ogcncTv/v378Pf3x61btwAA5cqVw+DBg1GyZEmNFkdERESkLrWPlgoJCUG5cuVw7tw5ODs7w9nZGWfPnkX58uURGhqaGzUSERERZZvaIze//vorhg4dipkzZ2ZqHz16NH788UeNFUdERESkLrVHbm7duoVevXplau/Zsyf+/vtvjRRFRERElFNqhxsrKytcuXIlU/uVK1dgbW2tiZqIiIiIckzt3VJ9+vRB37598eDBA9SoUQMAcPLkScyaNQvDhg3TeIFERERE6lA73Pz2228oUKAA5s2bhzFjxgAAihQpgkmTJmHQoEEaL5CIiIhIHWqHG4VCgaFDh2Lo0KGIj48HABQoUEDjhRERERHlRI7OcwMA0dHRuHPnDgCgTJkysLKy0lhRRERERDml9oTi+Ph4dOvWDUWKFEHdunVRt25dFClSBF27dkVsbGxu1EhERESUbWqHm969e+Ps2bPYu3cvYmJiEBMTgz179uDChQvw9vbOjRqJiIiIsk3t3VJ79uxBSEgIatWqJbW5u7tj+fLlaNKkiUaLIyIiIlKX2iM3BQsWhJmZWaZ2MzMzWFhYaKQoIiIiopxSO9yMHz8ew4YNQ2RkpNQWGRmJkSNH4rffftNocURERETqUnu31NKlSxEeHo5ixYqhWLFiAIDHjx/D0NAQL168wLJly6S+ly5d0lylRERERNmgdrjx8PDIhTKIiIiINEPtcDNx4sTcqIOIiIhII9Sec/PkyRM8ffpUun/u3DkMGTIEgYGBGi2MiIiIKCfUDjedO3dGWFgYgPcTiRs1aoRz585h3LhxmDJlisYLJCIiIlKH2uHmxo0b+OGHHwAAW7ZsQcWKFXHq1CmsX78eQUFBmq6PiIiISC1qh5u0tDQYGhoCAA4ePIhWrVoBeH99qefPn2u2OiIiIiI1qR1uypcvj4CAABw/fhyhoaHSWYn/+ecfFCxYUOMFEhEREalD7XAza9YsLFu2DPXq1UOnTp1QqVIlAMDu3bul3VVERERE2qL2oeD16tXDy5cvERcXp3K5hb59+8LExESjxRERERGpS+2RGwAQQuDixYtYtmwZ4uPjAQAGBgYMN0RERKR1ao/cREREoEmTJnj8+DFSUlLw448/okCBApg1axZSUlIQEBCQG3USERERZYvaIzeDBw9G1apV8ebNGxgbG0vtbdq0waFDhzRaHBEREZG61B65OX78OE6dOgUDAwOVdgcHBzx79kxjhRERERHlhNojN0qlEunp6Znanz59igIFCmikKCIiIqKcUjvcNG7cGP7+/tJ9hUKBhIQETJw4Ec2aNdNkbURERERqU3u31Lx58+Du7o5y5cohOTkZnTt3xr1791CoUCFs3LgxN2okIiIiyja1R27s7Oxw9epVjBs3DkOHDoWrqytmzpyJy5cvw9raWq3HOnbsGFq2bIkiRYpAoVBg586d/9r/yJEjUCgUmW6RkZHqvgwiIiKSKbVHbgBAT08PXbp0QZcuXaS258+fY+TIkVi0aFG2HycxMRGVKlVCz5490bZt22yvd+fOHZiamkr31Q1VREREJF9qhZubN28iLCwMBgYGaN++PczNzfHy5UtMnz4dAQEBKFGihFpP3rRpUzRt2lStdYD3Ycbc3Fzt9YiIiEj+sr1bavfu3XB1dcWgQYPQr18/VK1aFWFhYShbtixu3bqFHTt24ObNm7lZq8TFxQWFCxfGjz/+iJMnT/5r35SUFMTFxanciIiISL6yHW6mTZsGHx8fxMXFYf78+Xjw4AEGDRqEv/76C/v375euDp6bChcujICAAGzfvh3bt2+Hvb096tWrh0uXLn1yHV9fX5iZmUk3e3v7XK+TiIiItCfb4ebOnTvw8fFB/vz5MXDgQOjo6MDPzw/ff/99btanonTp0vD29kaVKlVQo0YNrFq1CjVq1ICfn98n1xkzZgxiY2Ol25MnT75YvURERPTlZXvOTXx8vDSJV1dXF8bGxmrPsckNP/zwA06cOPHJ5YaGhjA0NPyCFREREZE2qTWhOCQkBGZmZgDen6n40KFDuHHjhkqfVq1aaa66bLhy5QoKFy78RZ+TiIiI8i61wo2Xl5fKfW9vb5X7CoUiy0szfEpCQgLCw8Ol+w8fPsSVK1dgaWmJYsWKYcyYMXj27BnWrl0LAPD394ejoyPKly+P5ORkrFixAocPH8aBAwfUeRlEREQkY9kON0qlUuNPfuHCBdSvX1+6P2zYMADvQ1RQUBCeP3+Ox48fS8tTU1MxfPhwPHv2DCYmJnB2dsbBgwdVHoOIiIi+bTk6iZ+m1KtXD0KITy4PCgpSuT9q1CiMGjUql6siIiKir5nal18gIiIiyssYboiIiEhWGG6IiIhIVhhuiIiISFZyFG5iYmKwYsUKjBkzBq9fvwYAXLp0Cc+ePdNocURERETqUvtoqWvXrqFRo0YwMzPDo0eP0KdPH1haWiI4OBiPHz+WzklDREREpA1qj9wMGzYM3bt3x71792BkZCS1N2vWDMeOHdNocURERETqUjvcnD9/PtOZiQGgaNGiiIyM1EhRRERERDmldrgxNDREXFxcpva7d+/CyspKI0URERER5ZTa4aZVq1aYMmUK0tLSALy/ntTjx48xevRo/PTTTxovkIiIiEgdaoebefPmISEhAdbW1khKSkLdunXh5OSEAgUKYPr06blRIxEREVG2qX20lJmZGUJDQ3HixAlcu3YNCQkJqFy5Mho1apQb9RERERGpJccXzqxVqxZq1aqlyVqIiIiIPpva4eb333/Psl2hUMDIyAhOTk6oU6cOdHV1P7s4IiIiInWpHW78/Pzw4sULvH37FhYWFgCAN2/ewMTEBPnz50d0dDRKlCiBsLAw2Nvba7xgIiIion+j9oTiGTNm4Pvvv8e9e/fw6tUrvHr1Cnfv3kW1atWwYMECPH78GLa2thg6dGhu1EtERET0r9QeuRk/fjy2b9+OkiVLSm1OTk6YO3cufvrpJzx48ACzZ8/mYeFERESkFWqP3Dx//hzv3r3L1P7u3TvpDMVFihRBfHz851dHREREpCa1w039+vXh7e2Ny5cvS22XL19G//790aBBAwDA9evX4ejoqLkqiYiIiLJJ7XCzcuVKWFpaokqVKjA0NIShoSGqVq0KS0tLrFy5EgCQP39+zJs3T+PFEhEREf0Xtefc2NraIjQ0FLdv38bdu3cBAKVLl0bp0qWlPvXr19dchURERERqyPFJ/MqUKYMyZcposhYiIiKiz5ajcPP06VPs3r0bjx8/Rmpqqsqy+fPna6QwIiIiopxQO9wcOnQIrVq1QokSJXD79m1UqFABjx49ghAClStXzo0aiYiIiLJN7QnFY8aMwYgRI3D9+nUYGRlh+/btePLkCerWrYt27drlRo1ERERE2aZ2uLl16xY8PT0BAHp6ekhKSkL+/PkxZcoUzJo1S+MFEhEREalD7XCTL18+aZ5N4cKFcf/+fWnZy5cvNVcZERERUQ6oPefGzc0NJ06cQNmyZdGsWTMMHz4c169fR3BwMNzc3HKjRiIiIqJsUzvczJ8/HwkJCQCAyZMnIyEhAZs3b0apUqV4pBQRERFpnVrhJj09HU+fPoWzszOA97uoAgICcqUwIiIiopxQa86Nrq4uGjdujDdv3uRWPURERESfRe0JxRUqVMCDBw9yoxYiIiKiz6Z2uJk2bRpGjBiBPXv24Pnz54iLi1O5EREREWmT2hOKmzVrBgBo1aoVFAqF1C6EgEKhQHp6uuaqIyIiIlKT2uEmLCwsN+ogIiIi0gi1w03dunVzow4iIiIijVB7zg0AHD9+HF27dkWNGjXw7NkzAMC6detw4sQJjRZHREREpC61w8327dvh7u4OY2NjXLp0CSkpKQCA2NhYzJgxQ+MFEhEREakjR0dLBQQEYPny5dDX15faa9asiUuXLmm0OCIiIiJ1qR1u7ty5gzp16mRqNzMzQ0xMjCZqIiIiIsoxtcONra0twsPDM7WfOHECJUqU0EhRRERERDmldrjp06cPBg8ejLNnz0KhUOCff/7B+vXrMWLECPTv3z83aiQiIiLKNrUPBf/111+hVCrRsGFDvH37FnXq1IGhoSFGjBiBgQMH5kaNRERERNmmdrhRKBQYN24cRo4cifDwcCQkJKBcuXLInz9/btRHREREpBa1d0v98ccfePv2LQwMDFCuXDn88MMPDDZERESUZ6gdboYOHQpra2t07twZf/3112ddS+rYsWNo2bIlihQpAoVCgZ07d/7nOkeOHEHlypVhaGgIJycnBAUF5fj5iYiISH7UDjfPnz/Hpk2boFAo0L59exQuXBg+Pj44deqU2k+emJiISpUqYfHixdnq//DhQzRv3hz169fHlStXMGTIEPTu3RshISFqPzcRERHJk9pzbvT09NCiRQu0aNECb9++xY4dO7BhwwbUr18fdnZ2uH//frYfq2nTpmjatGm2+wcEBMDR0RHz5s0DAJQtWxYnTpyAn58f3N3d1X0pREREJENqh5sPmZiYwN3dHW/evEFERARu3bqlqbqydPr0aTRq1Eilzd3dHUOGDPnkOikpKdIlIgAgLi4ut8ojIiKiPCBHF858+/Yt1q9fj2bNmqFo0aLw9/dHmzZtcPPmTU3XpyIyMhI2NjYqbTY2NoiLi0NSUlKW6/j6+sLMzEy62dvb52qNREREpF1qh5uOHTvC2toaQ4cORYkSJXDkyBGEh4dj6tSpKFOmTG7U+FnGjBmD2NhY6fbkyRNtl0RERES5SO3dUrq6utiyZQvc3d2hq6ursuzGjRuoUKGCxor7mK2tLaKiolTaoqKiYGpqCmNj4yzXMTQ0hKGhYa7VRERERHmL2uFm/fr1Kvfj4+OxceNGrFixAhcvXvysQ8P/S/Xq1fHXX3+ptIWGhqJ69eq59pxERET0dcnRnBvg/TlqvLy8ULhwYcydOxcNGjTAmTNn1HqMhIQEXLlyBVeuXAHw/lDvK1eu4PHjxwDe71Ly9PSU+vfr1w8PHjzAqFGjcPv2bSxZsgRbtmzB0KFDc/oyiIiISGbUGrmJjIxEUFAQVq5cibi4OLRv3x4pKSnYuXMnypUrp/aTX7hwAfXr15fuDxs2DADg5eWFoKAgPH/+XAo6AODo6Ii9e/di6NChWLBgAezs7LBixQoeBk5ERESSbIebli1b4tixY2jevDn8/f3RpEkT6OrqIiAgIMdPXq9ePQghPrk8q7MP16tXD5cvX87xcxIREZG8ZTvc7Nu3D4MGDUL//v1RqlSp3KyJiIiIKMeyPefmxIkTiI+PR5UqVVCtWjUsWrQIL1++zM3aiIiIiNSW7XDj5uaG5cuX4/nz5/D29samTZtQpEgRKJVKhIaGIj4+PjfrJCIiIsoWtY+WypcvH3r27IkTJ07g+vXrGD58OGbOnAlra2u0atUqN2okIiIiyrYcHwoOAKVLl8bs2bPx9OlTbNy4UVM1EREREeXYZ4WbDLq6uvDw8MDu3bs18XBEREREOaaRcENERESUVzDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs5Ilws3jxYjg4OMDIyAjVqlXDuXPnPtk3KCgICoVC5WZkZPQFqyUiIqK8TOvhZvPmzRg2bBgmTpyIS5cuoVKlSnB3d0d0dPQn1zE1NcXz58+lW0RExBesmIiIiPIyrYeb+fPno0+fPujRowfKlSuHgIAAmJiYYNWqVZ9cR6FQwNbWVrrZ2Nh8wYqJiIgoL9NquElNTcXFixfRqFEjqU1HRweNGjXC6dOnP7leQkICihcvDnt7e7Ru3Ro3b978ZN+UlBTExcWp3IiIiEi+tBpuXr58ifT09EwjLzY2NoiMjMxyndKlS2PVqlXYtWsX/vjjDyiVStSoUQNPnz7Nsr+vry/MzMykm729vcZfBxEREeUdWt8tpa7q1avD09MTLi4uqFu3LoKDg2FlZYVly5Zl2X/MmDGIjY2Vbk+ePPnCFRMREdGXpKfNJy9UqBB0dXURFRWl0h4VFQVbW9tsPYa+vj5cXV0RHh6e5XJDQ0MYGhp+dq1ERET0ddDqyI2BgQGqVKmCQ4cOSW1KpRKHDh1C9erVs/UY6enpuH79OgoXLpxbZRIREdFXRKsjNwAwbNgweHl5oWrVqvjhhx/g7++PxMRE9OjRAwDg6emJokWLwtfXFwAwZcoUuLm5wcnJCTExMZgzZw4iIiLQu3dvbb4MIiIiyiO0Hm46dOiAFy9eYMKECYiMjISLiwv2798vTTJ+/PgxdHT+N8D05s0b9OnTB5GRkbCwsECVKlVw6tQplCtXTlsvgYiIiPIQrYcbABgwYAAGDBiQ5bIjR46o3Pfz84Ofn98XqIqIiIi+Rl/d0VJERERE/4bhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkJU+Em8WLF8PBwQFGRkaoVq0azp0796/9t27dijJlysDIyAgVK1bEX3/99YUqJSIiorxO6+Fm8+bNGDZsGCZOnIhLly6hUqVKcHd3R3R0dJb9T506hU6dOqFXr164fPkyPDw84OHhgRs3bnzhyomIiCgv0nq4mT9/Pvr06YMePXqgXLlyCAgIgImJCVatWpVl/wULFqBJkyYYOXIkypYti6lTp6Jy5cpYtGjRF66ciIiI8iKthpvU1FRcvHgRjRo1ktp0dHTQqFEjnD59Ost1Tp8+rdIfANzd3T/Zn4iIiL4tetp88pcvXyI9PR02NjYq7TY2Nrh9+3aW60RGRmbZPzIyMsv+KSkpSElJke7HxsYCAOLi4j6n9E9SprzNlcelr0dubVvZxW2QuA2StuXGNpjxmEKI/+yr1XDzJfj6+mLy5MmZ2u3t7bVQDX0LzPy1XQF967gNkrbl5jYYHx8PMzOzf+2j1XBTqFAh6OrqIioqSqU9KioKtra2Wa5ja2urVv8xY8Zg2LBh0n2lUonXr1+jYMGCUCgUn/kK6ENxcXGwt7fHkydPYGpqqu1y6BvEbZC0jdtg7hFCID4+HkWKFPnPvloNNwYGBqhSpQoOHToEDw8PAO/Dx6FDhzBgwIAs16levToOHTqEIUOGSG2hoaGoXr16lv0NDQ1haGio0mZubq6J8ukTTE1N+UtNWsVtkLSN22Du+K8Rmwxa3y01bNgweHl5oWrVqvjhhx/g7++PxMRE9OjRAwDg6emJokWLwtfXFwAwePBg1K1bF/PmzUPz5s2xadMmXLhwAYGBgdp8GURERJRHaD3cdOjQAS9evMCECRMQGRkJFxcX7N+/X5o0/PjxY+jo/O+grho1amDDhg0YP348xo4di1KlSmHnzp2oUKGCtl4CERER5SEKkZ1px0TZkJKSAl9fX4wZMybTrkCiL4HbIGkbt8G8geGGiIiIZEXrZygmIiIi0iSGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYb+mYolUptl0BERF8Aww19MzIuwPry5UsAAK88Ql/axwGb2yBpw8fboRy/+DHc0DdlwYIF8PDwwP3796FQKLRdDn1jdHR0EBsbi5CQEADgNkhaoaOjg5iYGMyZMwdv3ryRvvjJifxeEdEHPv5mrK+vD2NjYxgYGGipIvqWKZVKzJs3D97e3tizZ4+2y6Fv2IEDBzB//nwsWrRI26XkCl4VnL4JcXFxMDU1BQDExsbCzMxMyxXRt0KpVKp8M7516xZWrlyJWbNmQVdXV4uV0bckPT1dZXtLS0vD5s2b0alTJ1luhww3JHtDhw5Feno6xowZg8KFC2u7HPoGxcTEICYmBvb29iofJB9/4BB9jo+D9MdevXqFkydPokaNGihUqJDULsftkLulSHY+zut2dnZYu3at7H556esghMCvv/6KatWq4dGjRyrLuE3S53j+/Dn++ecfvHjxAsD7uTT/Nl6xZcsWeHh44OjRoyrtctwOOXJDX7WMbxxCCCgUik9+c3nz5g0sLCy0UCHJzX99O86qT0REBMaPH4+goCBZfpDQl7d69WosXrwYT548QcmSJVGrVi3Mnj1bpU9WIzL+/v4YMGAA9PT0vmS5XxzDDX01MgIM8P6XVggBPT09PHv2DDt27ECPHj2QL18+AO93RVlYWGDChAmZ1iXKqQ9Dy+HDh/H48WM4OTmhRIkSKFKkiEqf2NhYKJXKTKFajrsA6Mvas2cP2rdvjyVLlsDExAQPHjzA7NmzUaNGDaxZswYFCxaU/ua9fPkS4eHhcHNzU3mMd+/eyTrgcLcU5VkZuTsuLg5JSUlQKBQ4cOAAwsPDoaurCz09PURERMDV1RX//POPFGwSExOhr68PPz8/vH79msGGNEIIIQWbX3/9Fd27d8fcuXPRt29fjBgxAufPnwfwftdASkoKJkyYgMqVK+PVq1cqj8NgQ5/r/PnzaN68Obp374727dtj1KhRCAkJwbVr19ClSxcA708zkJaWhnXr1qFGjRo4ceKEymPIOdgADDeUx0VGRqJixYo4evQoNmzYgCZNmuDvv/8G8H5XU/ny5dGmTRtMnz5dWidfvnwYNWoU7t27B0tLSwYb0oiM7Wju3Ln4448/sHHjRty4cQNt27bFn3/+ifHjx+P06dMAAAMDA7i6uqJhw4YwNzfXYtUkRw8fPsTz589V2r7//nvs3r0bFy9eRJ8+fQC8P/VFixYtMH369EwjN7IniPK4Hj16CFNTU6GjoyOWL18utaemporNmzeL9PR0qU2pVGqjRPpGREVFibZt24pVq1YJIYTYvXu3MDU1Ff369ROurq6iYcOG4syZM0II1W3x3bt3WqmX5CkkJETY2NiITZs2SW0Z29v69euFk5OTOH/+fKb10tLSvliN2saRG8qzMk4J7uPjg/j4eBgYGMDW1hbJyckA3n8rad++vcrETY7SUG6ytrbGqFGj0KRJE1y+fBk+Pj6YNm0ali5dip9++glnzpyBj48PLl68qLItclcUaVLZsmVRr149rFu3DocOHQLwv799Li4uiI6Oli4z8yG574r6EMMN5VkZocXe3h4nTpyAl5cXOnbsiF27diEpKSlTfzleH4W051Pbk6urKwoXLox9+/bB2dkZffv2BQBYWlrCzc0NLVu2hKur65cslb4x9vb26NevH2JiYuDn54fdu3dLywoXLgxHR0ctVpc3fDsxjr4a4v8nAD9//hxpaWkoVqwYrK2tUaNGDSQnJ6NXr14ICgpCixYtYGRkhICAADRq1AhOTk7aLp1kQnwweXjFihWIjo6GgYEBRowYIV26IyUlBc+ePcOjR49QunRpHDhwAK1atcLAgQP/9bQERJ8j42i7evXqYcmSJRg7dixGjx6NkJAQODs7Y8uWLVAoFPjxxx+1XapW8VBwypOCg4MxadIkREVFoXnz5mjTpg1atmwJAOjRowd27NiB4cOHIyoqCkuXLsX169dRrlw5LVdNcjNx4kT4+/vj+++/x7lz51CtWjWsW7cOtra2+PPPPzFt2jS8efMG+vr6EELg2rVr0NPT4xF6lCsytqvg4GAsWbIEBw4cwO3btxEWFoZFixbB3t4e5ubmWL9+PfT19b/p0w4w3FCec/PmTbi7u2Po0KEwMTHBxo0bYWhoCC8vL3Tt2hUAMHjwYFy6dAkpKSkIDAyEi4uLdosmWfhwtOXdu3fw8vLCwIED4erqikePHqF58+awtbXFjh07YGVlhb179yI8PBwJCQkYPXo09PT0vukPFNKMjBAjPjq3l66uLoKDg+Hp6Yn58+dLu0SB99urjo6Oyvb7Lc2x+RjDDeUpt2/fxtatW5GUlIQZM2YAAK5fv44JEyYgLi4OPXr0kAJOZGQk8uXLhwIFCmizZJKJD4PNrVu3EBcXh2XLlmHChAlwcHAA8P4Q3B9//BE2NjbYuXMnrKysVB6DwYY+14fb4cuXL6FQKFCwYEEA7//mVa5cGRMmTEC/fv2kdT4eKeTIIcMN5RFCCLx58wYtWrTA33//jZYtW2LdunXS8mvXrmHChAlISkpCx44d0aNHDy1WS3I2cuRIaVg/KioKwcHBaNq0qfRh8fDhQzRt2hRCCJw8eVLlAoREn+PDUDJ16lTs3LkTcXFxKFSoEKZPn44GDRrg2bNnKFq0qJYrzfs4243yBIVCAUtLS/j6+qJ8+fK4dOkSQkNDpeXOzs6YOnUq0tLSpF94Ik348KioPXv2YP/+/fj999+xZMkSODo6Yty4cbh69ap0xmxHR0fs2bMHLi4uvF4ZaVRGsJkyZQoWLFggnWqgUKFC6NKlC9asWZNptJCyxpEb0ppPDZ0ePXoUY8eOha2tLXx8fNCgQQNp2c2bN2FmZgY7O7svWSp9A4KDg3Hq1CkULFgQY8aMAQAkJCSgcuXKMDU1xYoVK1CpUqVM2yx3RZEmvXr1Co0bN4aPjw969uwptfft2xd//vknwsLCUKZMGe56+g8cuSGtyPjFPHXqFObPn4/ffvsNJ0+eRFpaGurWrYspU6YgMjISixYtwpEjR6T1ypcvz2BDGpeUlITffvsN8+fPx82bN6X2/Pnz49KlS4iPj4e3t7d0/agPMdiQJr179w4vX76URgUzTloaGBiIIkWKwM/PDwBPWPpfGG7oi/vwcMamTZvi5MmT2L17N8aOHYvp06cjNTUVDRs2xJQpU/Dq1StMnToVx48f13bZJGPGxsY4fvw4GjVqhIsXL2L37t1IT08H8L+Ac/v2bSxbtkzLlZKcZLXjxMbGBra2tli1ahUAwMjICKmpqQAAJycnhppsYrihLy5jxGbQoEGYP38+tm/fjq1bt+LixYvYvHkzxo8fLwWcX3/9Ffr6+jzjJmnMh3NshBDSB4ylpSU2bNgACwsLzJkzByEhIdKyfPnyITIyEoGBgVqpmeRHqVRKQeWff/5BdHQ03r59CwCYNGkSbt++LR0RlXHiyKdPn/JCrNnEOTf0xWT8MisUCixZsgRXrlxBYGAgHj58iEaNGqFWrVowNTXF1q1b4e3tjbFjx8LQ0BBv376FiYmJtssnGfjwMNuFCxfi6tWrePDgAYYMGYLKlSvDzs4OL168QOvWraGrq4uxY8fC3d1d5UzDnGNDn2P9+vVwc3NDyZIlAQBjxoxBSEgIIiIi0KhRI7Rq1QpdunTB8uXLMXXqVBQsWBAVKlTA/fv3ERMTI50okv4dww3lmowPkg/DyZUrV+Di4oK4uDg8efIETk5OaNKkCRwdHbFq1SrExsZKZxru3r07pk+fzolz9Nk+3obGjBmDlStXom/fvnj69ClOnz6N1q1bo2/fvnBycsKLFy/Qtm1bvHjxAkFBQXBzc9Ni9SQX+/btQ4sWLTB69GgMGTIE+/btw6hRo+Dv749Xr17h0qVLCAkJwW+//YZ+/frh+vXr8Pf3h46ODiwsLDBjxgyeKDK7cvWa4/TNe/DggejUqZP4+++/xZYtW4RCoRDnzp0TSqVSCCHE9evXRZkyZcTZs2eFEELcv39ftGjRQowdO1Y8fvxYm6WTzKSnpwshhFi3bp1wdHQUFy9eFEIIcfz4caFQKESpUqXE4MGDxYMHD4QQQjx//lz07dtXvHv3Tms1k/wsWrRI2NnZialTp4oBAwaI5cuXS8uePHkipkyZIhwcHMT+/fuzXD8tLe1LlfpV49gW5ark5GQcP34c3bt3x5UrV7B69Wp8//330i4qIQTevXuH06dPo3z58li7di0AYMSIETyHCH22bt26wcrKCvPnz4eOjg7S0tJgYGCAfv36oXLlyti5cyd69OiBFStWIDIyEtOmTYOOjg769OmDsmXLShOI+U2ZPldqaioMDAzg4+MDExMTjBkzBvHx8Zg2bZrUx87ODp6enjhw4AAuXLgAd3f3TBdg5S6pbNJ2uiL5yvimHBAQIHR0dESlSpXE5cuXVfrExsaK7t27i5IlSwoHBwdhZWUlfaMm+hyxsbFi8uTJwtLSUkyaNElqf/bsmYiKihLPnz8XVatWFfPmzZP6FylSRBQuXFgsWLBACCGkEUYiTfH19RXR0dFi/fr1wsTERDRr1kzcvXtXpU+HDh1E27ZttVShPPBoKcoVQgjo6OhACIEiRYpg3rx5ePfuHcaPH48TJ05I/UxNTTF37lwsWbIEEydOxNmzZ1G5cmUtVk5yEB8fD1NTU/Tv3x/jx4+Hv78/Jk6cCAAoUqQIrK2t8fz5c7x580aaT/Ps2TM0btwYEyZMgI+PDwCeS4Q+n/hgWuuaNWswdepU3Lt3D507d4afnx8uXbqEgIAA3LlzBwAQFxeHhw8folixYtoqWRY4vkUaJ/5/8ubhw4dx9OhRDBkyBC1btkSjRo3Qvn17zJw5E2PHjkWNGjUAvL8wZuPGjbVcNcnFqFGjsGzZMty/fx9WVlbo2rUrhBCYOnUqAGDy5MkA3gcgXV1dnDx5EkIIzJw5EyYmJtLht9wVRZqQEZAPHTqEy5cvIzAwUPrb17dvX6SlpWHy5MnYv38/KleujMTERKSmpmL27NnaLPvrp81hI5KfjGH8bdu2CTMzMzFmzBhx/vx5afm1a9dEuXLlRIsWLcQff/whJk2aJBQKhXjy5Al3AZBGXL16VdSpU0eULl1avHjxQgghRHR0tJg3b54wNzcXEyZMkPoOGDBAlCxZUtjZ2Qk3NzeRmpoqhODuKNKsI0eOiIoVK4qCBQuKnTt3CiGESElJkZavXLlS5M+fX1SuXFmsXbtWmsTOycM5x0PBSePOnTuHJk2aYNasWejTp4/UHhcXB1NTU9y6dQt9+vRBUlISYmNjsWXLFu6KIo04ffo0Xrx4gXLlyqFDhw5ISEiQrtz94sULrFu3DlOnTpUuSAi8Pz2BQqFAxYoVoaOjg3fv3nHSJn0W8dGpBxISEjBnzhwEBgaiWrVq2LhxI4yNjZGWlgZ9fX0AwPz583Hq1Cls3boVCoWCI4efieGGNG7RokXYsWMHDh06hNjYWBw+fBh//PEHbt26hREjRqBnz56Ijo5GbGwszMzMYG1tre2SSSY8PT3xzz//4ODBg3j06BF+/vlnxMfHZwo406ZNw4ABAzBlyhSV9fmBQpq0ePFi2NnZoXXr1khKSsLcuXOxY8cO1KtXDzNmzICRkZFKwMkIRR+HI1IfJxSTxtna2uLixYvw9fXFzz//jNWrV8PIyAjNmzdH7969cffuXVhbW6NUqVIMNqRRixcvxtOnT7Fo0SI4ODhg48aNMDMzQ82aNfHy5UtYWVmhW7dumDBhAqZNm4aVK1eqrM9gQ5ry4sULHD58GL/88gv2798PY2NjDBs2DC1atMCpU6cwbtw4JCcnQ19fH+/evQMABhsN4sgNfZaMX8SEhATkz58fABAVFYWFCxdiy5YtaNCgAbp3744ffvgBUVFRaNWqFYKCglC+fHktV05ykzHq8vvvv+Py5cuYP38+LCwscPv2bXh6eiI2NlYawYmMjMTRo0fx008/cRcUacTH56MBgKtXr+L333/HwYMHERAQgKZNmyIxMRGzZ8/GwYMHUbZsWSxZskS6dhRpDkdu6LMoFArs3bsXnTp1Qr169RAUFAQ9PT1MmzYNZ8+eRUBAANzc3KCjo4OFCxciMTGRozWUKzJGXerVq4djx45h7969AIDSpUtj3bp1sLCwQJ06dRAVFQVbW1t06NABenp60rdmos+REWwiIyOltkqVKmHw4MGoX78++vXrh/379yNfvnwYNWoUfvjhB+jo6Ei7pEjDtDSRmWTi5MmTwsjISIwcOVI0adJEODs7C29vbxEeHi71CQsLE3379hWWlpaZTuJHlFMZJ4nMSkBAgPjuu+/EnTt3pLY7d+4IBwcH0bFjxy9RHn0jPtwON23aJEqUKKFyhKgQQly5ckW0bt1aFCtWTBw5ckQIIURSUpJ0VN6/bcuUMxy5oRyLiIhAaGgopk+fjtmzZ2Pfvn3o27cvrl27Bl9fXzx48ACJiYk4ffo0oqOjcfToUbi4uGi7bJKBD3cBnDt3DqdOncLRo0el5a1atUK1atUQFhYmtX333Xc4duwY/vjjjy9eL8lTSkqKtB2mpqaiZMmSKFOmDHx8fHDx4kWpX6VKleDh4YEnT56gcePGOHXqFIyMjKQ5Nh/vzqLPx58oZcuiRYvw119/Sffv3LmDDh06YNWqVTAyMpLafXx80KVLF9y8eROzZ89GTEwMRo4ciTVr1qBChQraKJ1k5sMPg7Fjx6J79+7o2bMnvLy80KFDB8TFxaFw4cLSfIa0tDRpXXt7e+jq6iI9PV1b5ZNM7Nu3D+vWrQMA9OnTBw0aNEDVqlUxfPhw2NrawtvbGxcuXJD6FytWDB07dsS8efNQrVo1qZ2Th3OJtoeOKO97+PCh6Ny5s7h3755K+6+//iqsra1F27ZtpZOlZVi6dKkoXbq0GDRoEE9ERbli7ty5omDBguLs2bMiPT1dzJgxQygUCnHixAmpT82aNYW3t7cWqyS56tSpk3BwcBDu7u6iUKFC4urVq9Kyw4cPCw8PD1GhQgWxb98+8fDhQ+Hh4SGGDx8u9eHV5nMXww1lS2JiohBCiDNnzoht27ZJ7RMmTBAVK1YU48ePF1FRUSrrLF++XDx8+PBLlknfCKVSKby8vERgYKAQQojt27cLc3NzERAQIIQQIj4+XgghxL59+0SrVq3EtWvXtFYryZeLi4tQKBQqF2bNcPz4cdGtWzehUCjEd999J5ydnaUvejwDdu7jMZCULcbGxoiJiYGvry+ePXsGXV1deHh4YPLkyUhLS8PevXshhMDgwYNhZWUFAOjdu7eWqya5Sk5OxtmzZ1GvXj0cOXIEXl5emDNnDry9vfHu3TvMnj0b1atXh5ubG6ZMmYJz586hYsWK2i6bZCI1NRXJyclwcnJCsWLFsHnzZhQtWhQdO3aUTolRq1YtVKtWDX369EFaWhrq1q0LXV1dngH7C+GcG8oWhUIBc3NzDB8+HI6OjvD390dwcDAAYMaMGWjSpAlCQ0MxY8YMvHz5UsvVkpxcu3YNT58+BQAMHToUR48ehbGxMTp37ow//vgDzZo1g5+fn3TByzdv3uDChQu4c+cOLCwssG7dOhQvXlybL4FkxsDAAKampti6dSt27dqF77//HrNnz8amTZsQHx8v9UtOTkbt2rXRoEEDaa4Xg82XwXBD2SLe78JE7dq1MXToUFhYWOD3339XCThubm64fPkyBM8LSRoghMDdu3dRv359rFq1Cv369cOCBQtgYWEBAHBzc0NERASqVauG6tWrAwD++ecfdO/eHTExMRgwYAAAoGTJkmjUqJHWXgfJjxACSqVSur9mzRrUqFEDfn5+WLt2LR4/fowGDRqgXbt2Un+AZ8D+kniGYsqWjLO/xsbGwsTEBNeuXcP06dPx5s0bDB48GB4eHgDen3I8Y7cUkSYsX74co0aNQnJyMnbt2oXGjRtLZ8bevHkzpkyZAiEE9PT0YGxsDKVSiVOnTkFfX5/XiqLP9vr1a1haWqq0ZWx/W7duRWhoKAIDAwEAffv2xZEjR5Ceng5LS0ucPHmSZx/WEo7c0H969+4ddHV18ejRI9SrVw8HDhxAlSpVMGLECFhZWWHy5MnYs2cPADDYkMZkfDO2t7eHoaEhTE1NcebMGTx69Eg6fLZDhw5Yu3YtpkyZgvbt22P06NE4c+aMdL0eBhv6HAsWLMD333+vsqsJgBRsunfvjkqVKkntgYGBWLZsGRYuXIgzZ87AwMCAZ8DWFu3MY6a86lOz+MPDw4WNjY3o3bu3yiGMR44cEd26dROPHj36UiWSzH28DaampoqkpCSxdOlSUbRoUTF27Nj/3N54mC19rmXLlglDQ0OxYcOGTMseP34sKlasKBYtWiS1ZbXNcTvUHu6WIon4/6HW06dP49atWwgPD4enpycKFy6MNWvW4MKFC1izZk2mK9cmJyernMiPKKc+PPPw69evER8frzIZ2N/fH3PnzkWvXr3Qo0cPODg4oGXLlhg3bhzc3Ny0VTbJzPLlyzFw4ECsW7cO7dq1Q0xMDBITE5GcnAxra2sUKFAA9+7dQ6lSpbRdKn0Cww2p2L59O/r27StdYPDFixfo0KEDRo8ejQIFCmi7PJKxD4PNlClTcODAAdy4cQPt27dHmzZt0LRpUwDvA46/vz8qVKiAV69e4fHjx3j06BEvQEga8eDBAzg5OaF9+/bYtGkTbty4gV9++QUvXrxAREQE6tevj/79+6NFixbaLpX+BY9JI8mNGzcwdOhQzJs3D927d0dcXBzMzc1hbGzMYEO5LiPYTJgwAYGBgZgzZw4cHBzQr18/3Lt3DzExMejUqROGDBmCQoUK4erVq0hOTsbx48elq3vzMFv6XFZWVpg1axYmTJiAESNG4MCBA6hduzZat26NuLg4bNu2DePHj0ehQoU4WpiXaXOfGGnP4cOHxf379zO1Va9eXQghxK1bt0Tx4sVF7969peX379/nPmTKVYcPHxbly5cXx44dE0IIcerUKWFgYCDKlSsnqlWrJrZu3Sr1/fCyHrzEB2lScnKymDt3rtDR0RE9e/YUqamp0rILFy6I0qVLi8WLF2uxQvovPFrqGyOEwOXLl9G0aVMsXboUERER0rJnz55BCIGEhAQ0adIEjRs3xrJlywAAoaGhWLp0Kd68eaOt0kmGxEd7xYsWLYr+/fujdu3aOHDgAFq0aIHAwECEhobi/v37+P3337Fy5UoAUBml4YgNaZKhoSH69euH7du3o3fv3tDX15e21SpVqsDIyAhPnjzRcpX0bxhuvjEKhQKurq6YN28etmzZgqVLl+LBgwcAgObNmyMqKgqmpqZo3rw5AgMDpV0FISEhuHbtGg+tJY1RKpXSpPQHDx4gMTERpUqVQqdOnZCcnIwFCxZg0KBB6NatG4oUKYLy5csjPDwct27d0nLl9C3Ily8fmjZtKp0gMmNbjY6OhrGxMcqXL6/N8ug/8OvONyZjXoKPjw8AYM6cOdDV1UXv3r3h6OiI3377DTNmzMC7d+/w9u1bhIeHY+PGjVixYgVOnDghnR2W6HN8OHl4woQJOH36NEaOHIn69evD0tISiYmJeP78OUxMTKCjo4OUlBQ4ODhg1KhRaNKkiZarJzkSHxwBmsHQ0FD6f3p6Ol6+fIk+ffpAoVCgU6dOX7pEUgPDzTcmY+TlwIED0NHRQVpaGvz9/ZGcnIzRo0ejffv2SEpKwowZM7Bt2zbY2NjAwMAAYWFhqFChgparJ7n4MNgsW7YMgYGBcHV1lY54SklJgaWlJU6cOCFNGn716hVWrVoFHR0dlXBElBMRERF4/fo1ChYsCFtb2389k3BaWhrWrVuHjRs34vXr1zhz5ox0rSiOZudNPBT8GxQSEiJdbDBfvny4d+8efv/9d/zyyy8YPXo0rKysEB8fj6NHj8LBwQHW1tawtrbWdtn0lfs4kNy9exceHh6YNWsWWrZsmanf+fPnMX78eCQkJMDS0hLBwcHQ19dnsKHPtnbtWsybNw/R0dEoVKgQBg4cKI3IZPh4OwsNDcXNmzcxYMAAHp33FWC4+cYolUp06dIFCoUCGzZskNoXLlyIUaNGwcfHB7/88gtKlCihxSpJbtq2bYuxY8eiatWqUtuVK1fQpEkTHD16FKVLl87yxJDJyckQQsDIyAgKhYIfKPTZ1q5dCx8fH+nSCjNmzMCDBw9w8uRJadvKCDYxMTE4cOAA2rdvr/IYHLHJ+/j15xuT8U0kY/g/NTUVADBw4EB4e3tj9erV+P3331WOoiL6XGZmZnB2dlZpMzIywps3b3Djxg2pLeN6UqdPn8b27duho6MDY2NjKBQKKJVKBhv6LBcuXMDUqVOxaNEi9OzZExUrVsTQoUPh5OSEU6dO4ebNm4iLi5N22a9Zswa//PIL/vjjD5XHYbDJ+xhuvhH//POP9P/SpUvjzz//RHR0NAwMDJCWlgYAsLOzg4mJCcLCwmBsbKytUklGnj17BgBYvXo1DAwM8Pvvv+PAgQNITU2Fk5MTOnTogDlz5uDgwYNQKBTQ0dFBeno6pk+fjrCwMJV5ENwVRZ8rJSUFQ4YMQfPmzaW2SZMm4dChQ+jUqRM8PT3RsWNHvH79Gvr6+mjWrBlGjBjBycNfIe6W+gZcvXoVAwYMQOfOndG/f3+kpqaiQYMGePnyJY4cOQJbW1sAwOjRo1G+fHm0aNEClpaWWq6avnZ9+vQBAIwZM0bazens7IyXL19i06ZNqFOnDo4fPw4/Pz9cv34dXbp0gYGBAQ4dOoQXL17g0qVLHKkhjVIqlXjx4gVsbGwAAJ6enjh48CB2794Ne3t7HD16FNOmTcPo0aPRuXNnlTk43BX1deFXoW+AiYkJzM3NsW3bNgQFBcHAwADLli2DlZUVypYtCw8PDzRu3BgLFixA1apVGWxII5ydnbF//34sXboU4eHhAIBr166hdOnS6NKlC44dO4batWtjypQp8PT0xLp163D48GEUK1YMFy9elCZtEmmKjo6OFGwAYMSIETh79iyqVq0KGxsbNG3aFK9fv0ZUVFSmw8IZbL4uHLn5RoSHh2Ps2LGIjIxEnz590K1bN6Snp2Pu3LmIiIiAEAIDBw5EuXLltF0qyciqVaswYcIEdOzYEX369EHp0qUBAHXq1MHDhw+xfv161KlTBwDw9u1bmJiYSOty8jB9aU+fPkXXrl0xYsQIXhjzK8dwI1OXLl3C8+fPVfYth4eHY/z48Xj06BEGDhyILl26aLFCkrMPD6NduXIlJkyYgE6dOmUKOBEREVi7di2qV6+uMr8mqxOqEanjw20o4/8Z/7548QJWVlYq/RMTE9GpUyfExsbi8OHDHKn5yjHcyFB8fDyaN28OXV1djBo1Ck2bNpWWPXr0CE2aNIGJiQl69+6NX375RYuVktx86hw0y5cvx+TJk9GhQwf07dtXCjgNGjTAyZMncebMGbi6un7pckmmstoOM9qCg4OxceNGLFiwAEWKFEFSUhJ27dqFdevW4dmzZzh//jz09fU5x+Yrxzk3MpKRUwsUKIDZs2dDT08PixYtwt69e6U+Dg4OqF+/PiIjI3Ho0CHExMRoqVqSmw8/UE6dOoWwsDBcvXoVwPvJxb/99hs2bdqEwMBA3LlzBwBw+PBh9O7dO9Nh4kQ5deLECemilsOGDcPMmTMBvJ9vs3nzZnh6eqJRo0YoUqQIgPcXXX348CFKlCiBCxcuQF9fH+/evWOw+cpx5EYGMoZaM75pZHzInD17Fr/++ivy5cuH/v37S7uohg8fjhIlSqBt27YoXLiwlqsnOfhwF8CwYcOwefNmJCQkwM7ODsWKFcO+ffsAAMuWLcO0adPQsWNHeHl5qVzSg9+U6XMIIRAbGwtra2s0bdoUhQoVQnBwMI4fP44KFSogJiYGbm5u8PHxwcCBA6V1PvzbCXA7lAuGm69cxi9nWFgYdu/ejdevX6NWrVpo164dzM3NcebMGfz2229ISUlBiRIlYGJigs2bN+Pq1auws7PTdvkkAx8GmwMHDmDIkCEIDAyEubk5/v77b0ycOBH58uXDhQsXALyfg+Pt7Q1/f38MGDBAm6WTDEVHR6NEiRJIT0/H9u3b0axZM2lZVnNtspqbQ18/7pb6yikUCuzYsQMtW7bE27dv8fbtW6xbtw79+/fH69ev4ebmhrlz56Ju3boIDw/HgwcPcPjwYQYb0piMD4Pdu3dj06ZNaNSoEWrVqoUKFSrg559/xtq1a5GQkID+/fsDAHr16oVdu3ZJ94k0JSUlBZGRkTAxMYGuri5WrVolnYYAAAoVKiT9P+Ns2B+GGQYb+eDIzVfuwoUL6NixI3799Vf07t0bERERqFy5MoyNjeHi4oK1a9fC0tJSulbPx4fbEmnC69ev0aJFC1y9ehX169fHnj17VJaPHTsWJ0+exF9//YV8+fJJ7dwFQJ/rU5PYHz16BGdnZ9SvXx/z589HyZIltVAdaQtHbr4ivr6+GDdunPSNA3h/ens3Nzf07t0bjx49QsOGDeHh4YHx48fj/Pnz+OWXX/D69WsYGRkBAIMNacSH2yAAWFpaYs2aNfjxxx9x+fJlrF69WmV5qVKl8OrVKyQlJam0M9jQ5/gw2Bw5cgQbNmzA1atX8ezZMzg4OODkyZMICwvDqFGjpEnsbdq0wcKFC7VZNn0BHLn5iixcuBCDBw/GjBkzMGrUKOmX+tatWyhdujRat24tfcgolUq4uLggPDwczZs3x+bNm3ltHtKIDz9Q7t+/D4VCARMTE9ja2uLhw4fw8fFBYmIi2rVrB29vb0RFRcHLywtGRkbYs2cPh/5J40aMGIE1a9ZAT08P+fPnh62tLfz8/FC1alVcv34d9evXh4ODA1JTU/Hu3TtcvXpVungwyZSgr4JSqRRCCLF8+XKho6Mjpk6dKtLS0qTlT548EWXLlhV79uwRQgjx+vVr0alTJ7Fw4ULx9OlTrdRM8pOxHQohxMSJE0XFihVFmTJlROHChUVgYKAQQojw8HDRrFkzYWRkJEqXLi3atGkj3N3dRVJSkhBCiPT0dK3UTvLx4XYYGhoqKlWqJI4fPy5ev34tdu3aJdq0aSOcnJzEpUuXhBBC3Lt3T0yZMkVMnz5d+rv54d9Pkh+Gm6+AUqmUfpmVSqX4448/hI6Ojpg2bZr0QREdHS1cXFyEt7e3ePTokRg7dqz4/vvvRVRUlDZLJ5maMmWKsLKyEiEhISIhIUG0adNGmJubi5s3bwohhHjw4IFo3ry5cHFxEX5+ftJ6ycnJWqqY5GjNmjViwIABom/fvirt58+fF02aNBFeXl4iISFBCKEaiBhs5I/7Kb4SCoUCBw8exPDhw1GlShXpmj0zZ86EEAIWFhbo0qULjh49Cjc3N6xduxYBAQGwtrbWdukkAx/OsVEqlTh37hz8/PzQuHFjhIaG4siRI5gxYwbKlSuHtLQ0ODo6Yt68ebCxscHevXsRHBwMADA0NNTWSyAZEB/Noti5cycWL16MK1euICUlRWqvWrUqateujRMnTiA9PR2A6pFQvGbZN0Db6YqyZ/v27cLY2FhMnTpVnD9/XgghRGBgoLSLSgghUlJSxM2bN0VoaKh48uSJNsslmZowYYKYOXOmKFq0qLhz544ICwsT+fPnF0uXLhVCCPH27Vsxbtw48ejRIyGEEHfv3hUtWrQQVatWFcHBwdosnb5yH468rF+/Xqxdu1YIIcSAAQOEubm5WLx4sYiNjZX6hISEiDJlykjbIn1bGG6+Anfu3BGOjo5iyZIlmZYtW7ZM2kVFpGkfzo/ZtGmTsLe3Fzdu3BBdu3YV7u7uwsTERKxcuVLq8+zZM1G7dm2xdu1aad1bt26Jn3/+WURERHzx+kkePtwOb9y4IVxdXUWlSpXErl27hBBCeHl5iVKlSonp06eL8PBwER4eLho2bCjq1q2rEoro28Gxua/A48ePoa+vr3KmzYwjVvr27Yt8+fKhW7duMDQ0xIgRI7RYKclNxlFRR48exZEjRzB8+HCUL19eOjlkw4YN0bNnTwDvL9jau3dv6OrqonPnztDR0YFSqUSZMmWwYcMGHp1COZaxHY4cORIPHz6EsbExbt++jaFDh+Ldu3cICgpCz549MX78eCxcuBA1a9ZE/vz5sXnzZigUik+eC4fki+HmK5CQkKByfhClUintPz5y5AiqVKmCzZs3q1ynh0hTIiMj0atXL0RHR2Ps2LEAgH79+uH+/fs4fPgwXF1dUapUKTx+/BjJyck4f/48dHV1VU7QxzkO9LmCgoKwYsUKHDp0CI6OjkhJSYGXlxd8fX2ho6ODVatWwcTEBFu2bEGTJk3QsWNHGBoaIjU1FQYGBtoun74wRtmvQKVKlfDy5UsEBgYCeP8tJiPc7Nq1Cxs2bEDbtm1RtmxZbZZJMmVra4vg4GDY2Njgzz//xMWLF6Grq4s5c+ZgypQpaNCgAWxtbdGhQ4dPXlWZ57ahzxUeHo4KFSrAxcUFZmZmsLW1xapVq6Crq4uhQ4dix44dWLRoERo1aoT58+dj9+7diI+PZ7D5RvHr1FfA0dERixYtQr9+/ZCWlgZPT0/o6uoiKCgIQUFBOH36NM/0SrnK2dkZ27dvh5eXFwICAjBw4EA4OzujVatWaNWqlUrf9PR0jtSQxoj/v5iloaEhkpOTkZqaCiMjI6SlpaFo0aLw9fVFixYt4O/vD2NjY2zYsAGdO3fGiBEjoKenh/bt22v7JZAW8AzFXwmlUont27fD29sb+fLlg5GREXR1dbFx40a4urpquzz6Rly+fBm9e/dGlSpVMHjwYJQvX17bJdE34vr163B1dcVvv/2GiRMnSu0hISFYvnw53rx5g/T0dBw5cgQA0KNHD/z2228oUaKEliombWK4+cr8888/iIiIgEKhgKOjI2xsbLRdEn1jLl++DG9vbxQvXhyzZ8+Go6Ojtkuib0RQUBD69u2LIUOGoEOHDrCwsMCgQYNQo0YNtGnTBuXLl8fevXvRtGlTbZdKWsZwQ0RqO3fuHAICArBixQoehUJf1Pbt2/HLL7/AwMAAQghYW1vj1KlTiIqKwo8//oht27bB2dlZ22WSljHcEFGOZMyF4GG29KU9e/YMT548QVpaGmrWrAkdHR2MGTMGO3fuRFhYGGxtbbVdImkZww0R5VhGwCHSlps3b2LWrFn466+/cPDgQbi4uGi7JMoDeEgDEeUYgw1p07t375Camgpra2scPXqUE9xJwpEbIiL6qqWlpfEM2KSC4YaIiIhkhbMAiYiISFYYboiIiEhWGG6IiIhIVhhuiIiISFYYbohI9o4cOQKFQoGYmJhsr+Pg4AB/f/9cq4mIcg/DDRFpXffu3aFQKNCvX79My3x8fKBQKNC9e/cvXxgRfZUYbogoT7C3t8emTZuQlJQktSUnJ2PDhg0oVqyYFisjoq8Nww0R5QmVK1eGvb09goODpbbg4GAUK1YMrq6uUltKSgoGDRoEa2trGBkZoVatWjh//rzKY/3111/47rvvYGxsjPr16+PRo0eZnu/EiROoXbs2jI2NYW9vj0GDBiExMTHXXh8RfTkMN0SUZ/Ts2ROrV6+W7q9atQo9evRQ6TNq1Chs374da9aswaVLl+Dk5AR3d3e8fv0aAPDkyRO0bdsWLVu2xJUrV9C7d2/8+uuvKo9x//59NGnSBD/99BOuXbuGzZs348SJExgwYEDuv0giynUMN0SUZ3Tt2hUnTpxAREQEIiIicPLkSXTt2lVanpiYiKVLl2LOnDlo2rQpypUrh+XLl8PY2BgrV64EACxduhQlS5bEvHnzULp0aXTp0iXTfB1fX1906dIFQ4YMQalSpVCjRg38/vvvWLt2LZKTk7/kSyaiXMALZxJRnmFlZYXmzZsjKCgIQgg0b94chQoVkpbfv38faWlpqFmzptSmr6+PH374Abdu3QIA3Lp1C9WqVVN53OrVq6vcv3r1Kq5du4b169dLbUIIKJVKPHz4EGXLls2Nl0dEXwjDDRHlKT179pR2Dy1evDhXniMhIQHe3t4YNGhQpmWcvEz09WO4IaI8pUmTJkhNTYVCoYC7u7vKspIlS8LAwAAnT55E8eLFAby/IvT58+cxZMgQAEDZsmWxe/dulfXOnDmjcr9y5cr4+++/4eTklHsvhIi0hnNuiChP0dXVxa1bt/D3339DV1dXZVm+fPnQv39/jBw5Evv378fff/+NPn364O3bt+jVqxcAoF+/frh37x5GjhyJO3fuYMOGDQgKClJ5nNGjR+PUqVMYMGAArly5gnv37mHXrl2cUEwkEww3RJTnmJqawtTUNMtlM2fOxE8//YRu3bqhcuXKCA8PR0hICCwsLAC83620fft27Ny5E5UqVUJAQABmzJih8hjOzs44evQo7t69i9q1a8PV1RUTJkxAkSJFcv21EVHuUwghhLaLICIiItIUjtwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs/B+XLE52CERTBAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "## calculate avg response time\n", - "unique_models = set(unique_result[\"response\"]['model'] for unique_result in result[0][\"results\"])\n", - "model_dict = {model: {\"response_time\": []} for model in unique_models}\n", - "for iteration in result:\n", - " for completion_result in iteration[\"results\"]:\n", - " model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n", - "\n", - "avg_response_time = {}\n", - "for model, data in model_dict.items():\n", - " avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n", - "\n", - "models = list(avg_response_time.keys())\n", - "response_times = list(avg_response_time.values())\n", - "\n", - "plt.bar(models, response_times)\n", - "plt.xlabel('Model', fontsize=10)\n", - "plt.ylabel('Average Response Time')\n", - "plt.title('Average Response Times for each Model')\n", - "\n", - "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n", - "plt.show()" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/litellm_model_fallback.ipynb b/cookbook/litellm_model_fallback.ipynb deleted file mode 100644 index 2e7987b9693..00000000000 --- a/cookbook/litellm_model_fallback.ipynb +++ /dev/null @@ -1,51 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "j6yJsCGeaq8G" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "u129iWNPaf72" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "\n", - "model_fallback_list = [\"claude-instant-1\", \"gpt-3.5-turbo\", \"chatgpt-test\"]\n", - "\n", - "user_message = \"Hello, how are you?\"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n", - "\n", - "for model in model_fallback_list:\n", - " try:\n", - " response = completion(model=model, messages=messages)\n", - " except Exception:\n", - " print(f\"error occurred: {traceback.format_exc()}\")" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/litellm_proxy_server/batch_api/bedrock/README.md b/cookbook/litellm_proxy_server/batch_api/bedrock/README.md deleted file mode 100644 index 1410d76658e..00000000000 --- a/cookbook/litellm_proxy_server/batch_api/bedrock/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Bedrock batch endpoints - -Set `aws_bedrock_control_plane_endpoint` on a deployment when batch create, retrieve, and cancel requests must use a custom Bedrock control-plane endpoint. The proxy signs these requests for the configured URL. Without this setting, batch requests use the standard regional Bedrock endpoint - -```yaml -model_list: - - model_name: bedrock-batch - litellm_params: - model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 - aws_region_name: us-east-1 - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID - aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN - s3_output_bucket_name: os.environ/AWS_S3_OUTPUT_BUCKET_NAME - aws_bedrock_control_plane_endpoint: os.environ/BEDROCK_CONTROL_PLANE_ENDPOINT -``` - -Set `BEDROCK_CONTROL_PLANE_ENDPOINT` to your Bedrock control-plane VPC endpoint or compatible gateway base URL, without `/model-invocation-job`. The proxy adds the operation path. The endpoint must be reachable from the proxy, and the configured AWS region must match the endpoint's signing region - -`api_base` and `aws_bedrock_runtime_endpoint` configure runtime inference separately. They do not override the batch control-plane endpoint. `s3_endpoint_url` configures S3 file access separately - -The control-plane setting belongs in deployment configuration. Sending `aws_bedrock_control_plane_endpoint` in an API request body is rejected. There is no automatic `AWS_BEDROCK_CONTROL_PLANE_ENDPOINT` environment fallback, so reference the desired environment variable explicitly as shown above - -Start the proxy with this configuration and set `LITELLM_API_BASE` to its URL. Set `BEDROCK_BATCH_INPUT_URI` to a valid input JSONL file in S3 that the batch service role can read - -```bash -curl -sS "$LITELLM_API_BASE/v1/batches" \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ - -H 'Content-Type: application/json' \ - -d "{\"model\":\"bedrock-batch\",\"input_file_id\":\"$BEDROCK_BATCH_INPUT_URI\",\"endpoint\":\"/v1/chat/completions\",\"completion_window\":\"24h\"}" -``` - -Use the returned `id` as `BATCH_ID` to retrieve or cancel the same job through the deployment's configured endpoint - -```bash -curl -sS "$LITELLM_API_BASE/v1/batches/$BATCH_ID?model=bedrock-batch" \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" - -curl -sS -X POST "$LITELLM_API_BASE/v1/batches/$BATCH_ID/cancel?model=bedrock-batch" \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" -``` diff --git a/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py b/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py deleted file mode 100644 index 5b5f0978eb3..00000000000 --- a/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -from openai import OpenAI - -client = OpenAI( - base_url="http://0.0.0.0:4000", - api_key=os.environ["LITELLM_MASTER_KEY"], -) - -BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0" - -# Upload file -batch_input_file = client.files.create( - file=open("./bedrock_batch_completions.jsonl", "rb"), - purpose="batch", - extra_body={"target_model_names": BEDROCK_BATCH_MODEL}, -) -print(batch_input_file) - -# Create batch -batch = client.batches.create( - input_file_id=batch_input_file.id, - endpoint="/v1/chat/completions", - completion_window="24h", - metadata={"description": "Test batch job"}, -) -print(batch) diff --git a/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl b/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl deleted file mode 100644 index adef9ac2dd5..00000000000 --- a/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl +++ /dev/null @@ -1,128 +0,0 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} diff --git a/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md b/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md deleted file mode 100644 index 1bf52d922c6..00000000000 --- a/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md +++ /dev/null @@ -1,279 +0,0 @@ -# Braintrust Prompt Wrapper for LiteLLM - -This directory contains a wrapper server that enables LiteLLM to use prompts from [Braintrust](https://www.braintrust.dev/) through the generic prompt management API. - -## Architecture - -``` -┌─────────────┐ ┌──────────────────────┐ ┌─────────────┐ -│ LiteLLM │ ──────> │ Wrapper Server │ ──────> │ Braintrust │ -│ Client │ │ (This Server) │ │ API │ -└─────────────┘ └──────────────────────┘ └─────────────┘ - Uses generic Transforms Stores actual - prompt manager Braintrust format prompt templates - to LiteLLM format -``` - -## Components - -### 1. Generic Prompt Manager (`litellm/integrations/generic_prompt_management/`) - -A generic client that can work with any API implementing the `/beta/litellm_prompt_management` endpoint. - -**Expected API Response Format:** -```json -{ - "prompt_id": "string", - "prompt_template": [ - {"role": "system", "content": "You are a helpful assistant"}, - {"role": "user", "content": "Hello {name}"} - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.7, - "max_tokens": 100 - } -} -``` - -### 2. Braintrust Wrapper Server (`braintrust_prompt_wrapper_server.py`) - -A FastAPI server that: -- Implements the `/beta/litellm_prompt_management` endpoint -- Fetches prompts from Braintrust API -- Transforms Braintrust response format to LiteLLM format - -## Setup - -### Install Dependencies - -```bash -pip install fastapi uvicorn httpx litellm -``` - -### Set Environment Variables - -```bash -export BRAINTRUST_API_KEY="your-braintrust-api-key" -``` - -## Usage - -### Step 1: Start the Wrapper Server - -```bash -python braintrust_prompt_wrapper_server.py -``` - -The server will start on `http://localhost:8080` by default. - -You can customize the port and host: -```bash -export PORT=8000 -export HOST=0.0.0.0 -python braintrust_prompt_wrapper_server.py -``` - -### Step 2: Use with LiteLLM - -```python -import litellm -from litellm.integrations.generic_prompt_management import GenericPromptManager - -# Configure the generic prompt manager to use your wrapper server -generic_config = { - "api_base": "http://localhost:8080", - "api_key": "your-braintrust-api-key", # Will be passed to Braintrust - "timeout": 30, -} - -# Create the prompt manager -prompt_manager = GenericPromptManager(**generic_config) - -# Use with completion -response = litellm.completion( - model="generic_prompt/gpt-4", - prompt_id="your-braintrust-prompt-id", - prompt_variables={"name": "World"}, # Variables to substitute - messages=[{"role": "user", "content": "Additional message"}] -) - -print(response) -``` - -### Step 3: Direct API Testing - -You can also test the wrapper API directly: - -```bash -# Test with curl -curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \ - "http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID" - -# Health check -curl http://localhost:8080/health - -# Service info -curl http://localhost:8080/ -``` - -## API Documentation - -Once the server is running, visit: -- Swagger UI: `http://localhost:8080/docs` -- ReDoc: `http://localhost:8080/redoc` - -## Braintrust Format Transformation - -The wrapper automatically transforms Braintrust's response format: - -**Braintrust API Response:** -```json -{ - "id": "prompt-123", - "prompt_data": { - "prompt": { - "type": "chat", - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant" - } - ] - }, - "options": { - "model": "gpt-4", - "params": { - "temperature": 0.7, - "max_tokens": 100 - } - } - } -} -``` - -**Transformed to LiteLLM Format:** -```json -{ - "prompt_id": "prompt-123", - "prompt_template": [ - { - "role": "system", - "content": "You are a helpful assistant" - } - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.7, - "max_tokens": 100 - } -} -``` - -## Supported Parameters - -The wrapper automatically maps these Braintrust parameters to LiteLLM: - -- `temperature` -- `max_tokens` / `max_completion_tokens` -- `top_p` -- `frequency_penalty` -- `presence_penalty` -- `n` -- `stop` -- `response_format` -- `tool_choice` -- `function_call` -- `tools` - -## Variable Substitution - -The generic prompt manager supports simple variable substitution: - -```python -# In your Braintrust prompt: -# "Hello {name}, welcome to {place}!" - -# In your code: -prompt_variables = { - "name": "Alice", - "place": "Wonderland" -} - -# Result: -# "Hello Alice, welcome to Wonderland!" -``` - -Supports both `{variable}` and `{{variable}}` syntax. - -## Error Handling - -The wrapper provides detailed error messages: - -- **401**: Missing or invalid Braintrust API token -- **404**: Prompt not found in Braintrust -- **502**: Failed to connect to Braintrust API -- **500**: Error transforming response - -## Production Deployment - -For production use: - -1. **Use HTTPS**: Deploy behind a reverse proxy with SSL -2. **Authentication**: Add authentication to the wrapper endpoint if needed -3. **Rate Limiting**: Implement rate limiting to prevent abuse -4. **Caching**: Consider caching prompt responses -5. **Monitoring**: Add logging and monitoring - -Example with Docker: - -```dockerfile -FROM python:3.11-slim - -WORKDIR /app - -RUN pip install fastapi uvicorn httpx - -COPY braintrust_prompt_wrapper_server.py . - -ENV PORT=8080 -ENV HOST=0.0.0.0 - -EXPOSE 8080 - -CMD ["python", "braintrust_prompt_wrapper_server.py"] -``` - -## Extending to Other Providers - -This pattern can be used with any prompt management provider: - -1. Create a wrapper server that implements `/beta/litellm_prompt_management` -2. Transform the provider's response to LiteLLM format -3. Use the generic prompt manager to connect - -Example providers: -- Langsmith -- PromptLayer -- Humanloop -- Custom internal systems - -## Troubleshooting - -### "No Braintrust API token provided" -- Set `BRAINTRUST_API_KEY` environment variable -- Or pass token in `Authorization: Bearer TOKEN` header - -### "Failed to connect to Braintrust API" -- Check your internet connection -- Verify Braintrust API is accessible -- Check firewall settings - -### "Prompt not found" -- Verify the prompt ID exists in Braintrust -- Check that your API token has access to the prompt - -## License - -This wrapper is part of the LiteLLM project and follows the same license. - diff --git a/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py b/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py deleted file mode 100644 index 6379314c5b6..00000000000 --- a/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py +++ /dev/null @@ -1,274 +0,0 @@ -""" -Mock server that implements the /beta/litellm_prompt_management endpoint -and acts as a wrapper for calling the Braintrust API. - -This server transforms Braintrust's prompt API response into the format -expected by LiteLLM's generic prompt management client. - -Usage: - python braintrust_prompt_wrapper_server.py - - # Then test with: - curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \ - "http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID" -""" - -import json -import os -from typing import Any, Dict, List, Optional - -import httpx -from fastapi import FastAPI, HTTPException, Header, Query -from fastapi.responses import JSONResponse -import uvicorn - - -app = FastAPI( - title="Braintrust Prompt Wrapper", - description="Wrapper server for Braintrust prompts to work with LiteLLM", - version="1.0.0", -) - - -def transform_braintrust_message(message: Dict[str, Any]) -> Dict[str, str]: - """ - Transform a Braintrust message to LiteLLM format. - - Braintrust message format: - { - "role": "system", - "content": "...", - "name": "..." (optional) - } - - LiteLLM format: - { - "role": "system", - "content": "..." - } - """ - result = { - "role": message.get("role", "user"), - "content": message.get("content", ""), - } - - # Include name if present - if "name" in message: - result["name"] = message["name"] - - return result - - -def transform_braintrust_response( - braintrust_response: Dict[str, Any], -) -> Dict[str, Any]: - """ - Transform Braintrust API response to LiteLLM prompt management format. - - Braintrust response format: - { - "objects": [{ - "id": "prompt_id", - "prompt_data": { - "prompt": { - "type": "chat", - "messages": [...], - "tools": "..." - }, - "options": { - "model": "gpt-4", - "params": { - "temperature": 0.7, - "max_tokens": 100, - ... - } - } - } - }] - } - - LiteLLM format: - { - "prompt_id": "prompt_id", - "prompt_template": [...], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": {...} - } - """ - # Extract the first object from the objects array if it exists - if "objects" in braintrust_response and len(braintrust_response["objects"]) > 0: - prompt_object = braintrust_response["objects"][0] - else: - prompt_object = braintrust_response - - prompt_data = prompt_object.get("prompt_data", {}) - prompt_info = prompt_data.get("prompt", {}) - options = prompt_data.get("options", {}) - - # Extract messages - messages = prompt_info.get("messages", []) - transformed_messages = [transform_braintrust_message(msg) for msg in messages] - - # Extract model - model = options.get("model") - - # Extract optional parameters - params = options.get("params", {}) - optional_params: Dict[str, Any] = {} - - # Map common parameters - param_mapping = { - "temperature": "temperature", - "max_tokens": "max_tokens", - "max_completion_tokens": "max_tokens", # Alternative name - "top_p": "top_p", - "frequency_penalty": "frequency_penalty", - "presence_penalty": "presence_penalty", - "n": "n", - "stop": "stop", - } - - for braintrust_param, litellm_param in param_mapping.items(): - if braintrust_param in params: - value = params[braintrust_param] - if value is not None: - optional_params[litellm_param] = value - - # Handle response_format - if "response_format" in params: - optional_params["response_format"] = params["response_format"] - - # Handle tool_choice - if "tool_choice" in params: - optional_params["tool_choice"] = params["tool_choice"] - - # Handle function_call - if "function_call" in params: - optional_params["function_call"] = params["function_call"] - - # Add tools if present - if "tools" in prompt_info and prompt_info["tools"]: - optional_params["tools"] = prompt_info["tools"] - - # Handle tool_functions from prompt_data - if "tool_functions" in prompt_data and prompt_data["tool_functions"]: - optional_params["tool_functions"] = prompt_data["tool_functions"] - - return { - "prompt_id": prompt_object.get("id"), - "prompt_template": transformed_messages, - "prompt_template_model": model, - "prompt_template_optional_params": optional_params if optional_params else None, - } - - -@app.get("/beta/litellm_prompt_management") -async def get_prompt( - prompt_id: str = Query(..., description="The Braintrust prompt ID to fetch"), - authorization: Optional[str] = Header( - None, description="Bearer token for Braintrust API" - ), -) -> JSONResponse: - """ - Fetch a prompt from Braintrust and transform it to LiteLLM format. - - Args: - prompt_id: The Braintrust prompt ID - authorization: Bearer token for Braintrust API (from header) - - Returns: - JSONResponse with the transformed prompt data - """ - # Extract token from Authorization header or environment - braintrust_token = None - if authorization and authorization.startswith("Bearer "): - braintrust_token = authorization.replace("Bearer ", "") - else: - braintrust_token = os.getenv("BRAINTRUST_API_KEY") - - if not braintrust_token: - raise HTTPException( - status_code=401, - detail="No Braintrust API token provided. Pass via Authorization header or set BRAINTRUST_API_KEY environment variable.", - ) - - # Call Braintrust API - braintrust_url = f"https://api.braintrust.dev/v1/prompt/{prompt_id}" - headers = { - "Authorization": f"Bearer {braintrust_token}", - "Accept": "application/json", - } - print(f"headers: {headers}") - print(f"braintrust_url: {braintrust_url}") - print(f"braintrust_token: {braintrust_token}") - - try: - async with httpx.AsyncClient(timeout=30.0) as client: - response = await client.get(braintrust_url, headers=headers) - response.raise_for_status() - braintrust_data = response.json() - except httpx.HTTPStatusError as e: - raise HTTPException( - status_code=e.response.status_code, - detail=f"Braintrust API error: {e.response.text}", - ) - except httpx.RequestError as e: - raise HTTPException( - status_code=502, - detail=f"Failed to connect to Braintrust API: {str(e)}", - ) - except json.JSONDecodeError as e: - raise HTTPException( - status_code=502, - detail=f"Failed to parse Braintrust API response: {str(e)}", - ) - - print(f"braintrust_data: {braintrust_data}") - # Transform the response - try: - transformed_data = transform_braintrust_response(braintrust_data) - print(f"transformed_data: {transformed_data}") - return JSONResponse(content=transformed_data) - except Exception as e: - raise HTTPException( - status_code=500, - detail=f"Failed to transform Braintrust response: {str(e)}", - ) - - -@app.get("/health") -async def health_check(): - """Health check endpoint.""" - return {"status": "healthy", "service": "braintrust-prompt-wrapper"} - - -@app.get("/") -async def root(): - """Root endpoint with service information.""" - return { - "service": "Braintrust Prompt Wrapper for LiteLLM", - "version": "1.0.0", - "endpoints": { - "prompt_management": "/beta/litellm_prompt_management?prompt_id=", - "health": "/health", - }, - "documentation": "/docs", - } - - -def main(): - """Run the server.""" - port = int(os.getenv("PORT", "8080")) - host = os.getenv("HOST", "0.0.0.0") - - print(f"🚀 Starting Braintrust Prompt Wrapper Server on {host}:{port}") - print(f"📚 API Documentation available at http://{host}:{port}/docs") - print( - f"🔑 Make sure to set BRAINTRUST_API_KEY environment variable or pass token in Authorization header" - ) - - uvicorn.run(app, host=host, port=port) - - -if __name__ == "__main__": - main() diff --git a/cookbook/litellm_proxy_server/cli_token_usage.py b/cookbook/litellm_proxy_server/cli_token_usage.py deleted file mode 100644 index c9c91e3283b..00000000000 --- a/cookbook/litellm_proxy_server/cli_token_usage.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 -""" -Example: Using CLI token with LiteLLM SDK - -This example shows how to use the CLI authentication token -in your Python scripts after running `lite login`. -""" - -from textwrap import indent -import litellm - -LITELLM_BASE_URL = "http://localhost:4000/" - - -def main(): - """Using CLI token with LiteLLM SDK""" - print("🚀 Using CLI Token with LiteLLM SDK") - print("=" * 40) - # litellm._turn_on_debug() - - # Get the CLI token - api_key = litellm.get_litellm_gateway_api_key() - - if not api_key: - print("❌ No CLI token found. Please run 'lite login' first.") - return - - print("✅ Found CLI token.") - - available_models = litellm.get_valid_models( - check_provider_endpoint=True, - custom_llm_provider="litellm_proxy", - api_key=api_key, - api_base=LITELLM_BASE_URL, - ) - - print("✅ Available models:") - if available_models: - for i, model in enumerate(available_models, 1): - print(f" {i:2d}. {model}") - else: - print(" No models available") - - # Use with LiteLLM - try: - response = litellm.completion( - model="litellm_proxy/gemini/gemini-2.5-flash", - messages=[{"role": "user", "content": "Hello from CLI token!"}], - api_key=api_key, - base_url=LITELLM_BASE_URL, - ) - print(f"✅ LLM Response: {response.model_dump_json(indent=4)}") - except Exception as e: - print(f"❌ Error: {e}") - - -if __name__ == "__main__": - main() - - print("\n💡 Tips:") - print("1. Run 'lite login' to authenticate first") - print("2. Replace 'https://your-proxy.com' with your actual proxy URL") - print("3. The token is stored in your OS keychain, or in ~/.litellm/token.json when there is none") diff --git a/cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py b/cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py deleted file mode 100644 index cc8b44d42b0..00000000000 --- a/cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Use LiteLLM Proxy MCP Gateway to call MCP tools. - -When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers. -""" - -import os - -import openai - -client = openai.OpenAI( - api_key=os.environ["LITELLM_MASTER_KEY"], - base_url="http://localhost:4000", # paste your litellm proxy base url here -) -print("Making API request to Responses API with MCP tools") - -response = client.responses.create( - model="gpt-5", - input=[ - { - "role": "user", - "content": "give me TLDR of what BerriAI/litellm repo is about", - "type": "message", - } - ], - tools=[ - { - "type": "mcp", - "server_label": "litellm", - "server_url": "litellm_proxy", - "require_approval": "never", - } - ], - stream=True, - tool_choice="required", -) - -for chunk in response: - print("response chunk: ", chunk) diff --git a/cookbook/litellm_proxy_server/readme.md b/cookbook/litellm_proxy_server/readme.md deleted file mode 100644 index 2c1eab72c24..00000000000 --- a/cookbook/litellm_proxy_server/readme.md +++ /dev/null @@ -1,177 +0,0 @@ -# liteLLM Proxy Server: 50+ LLM Models, Error Handling, Caching - -### Azure, Llama2, OpenAI, Claude, Hugging Face, Replicate Models - -[![PyPI Version](https://img.shields.io/pypi/v/litellm.svg)](https://pypi.org/project/litellm/) -[![PyPI Version](https://img.shields.io/badge/stable%20version-v0.1.345-blue?color=green&link=https://pypi.org/project/litellm/0.1.1/)](https://pypi.org/project/litellm/0.1.1/) -![Downloads](https://img.shields.io/pypi/dm/litellm) -[![litellm](https://img.shields.io/badge/%20%F0%9F%9A%85%20liteLLM-OpenAI%7CAzure%7CAnthropic%7CPalm%7CCohere%7CReplicate%7CHugging%20Face-blue?color=green)](https://github.com/BerriAI/litellm) - -[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/DYqQAW?referralCode=t3ukrU) - -![4BC6491E-86D0-4833-B061-9F54524B2579](https://github.com/BerriAI/litellm/assets/17561003/f5dd237b-db5e-42e1-b1ac-f05683b1d724) - -## What does liteLLM proxy do - -- Make `/chat/completions` requests for 50+ LLM models **Azure, OpenAI, Replicate, Anthropic, Hugging Face** - - Example: for `model` use `claude-2`, `gpt-3.5`, `gpt-4`, `command-nightly`, `stabilityai/stablecode-completion-alpha-3b-4k` - - ```json - { - "model": "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1", - "messages": [ - { - "content": "Hello, whats the weather in San Francisco??", - "role": "user" - } - ] - } - ``` - -- **Consistent Input/Output** Format - - Call all models using the OpenAI format - `completion(model, messages)` - - Text responses will always be available at `['choices'][0]['message']['content']` -- **Error Handling** Using Model Fallbacks (if `GPT-4` fails, try `llama2`) -- **Logging** - Log Requests, Responses and Errors to `Supabase`, `Posthog`, `Mixpanel`, `Sentry`, `Lunary`,`Athina`, `Helicone` (Any of the supported providers here: https://litellm.readthedocs.io/en/latest/advanced/ - - **Example: Logs sent to Supabase** - Screenshot 2023-08-11 at 4 02 46 PM - -- **Token Usage & Spend** - Track Input + Completion tokens used + Spend/model -- **Caching** - Implementation of Semantic Caching -- **Streaming & Async Support** - Return generators to stream text responses - -## API Endpoints - -### `/chat/completions` (POST) - -This endpoint is used to generate chat completions for 50+ support LLM API Models. Use llama2, GPT-4, Claude2 etc - -#### Input - -This API endpoint accepts all inputs in raw JSON and expects the following inputs - -- `model` (string, required): ID of the model to use for chat completions. See all supported models [here]: (https://litellm.readthedocs.io/en/latest/supported/): - eg `gpt-3.5-turbo`, `gpt-4`, `claude-2`, `command-nightly`, `stabilityai/stablecode-completion-alpha-3b-4k` -- `messages` (array, required): A list of messages representing the conversation context. Each message should have a `role` (system, user, assistant, or function), `content` (message text), and `name` (for function role). -- Additional Optional parameters: `temperature`, `functions`, `function_call`, `top_p`, `n`, `stream`. See the full list of supported inputs here: https://litellm.readthedocs.io/en/latest/input/ - -#### Example JSON body - -For claude-2 - -```json -{ - "model": "claude-2", - "messages": [ - { - "content": "Hello, whats the weather in San Francisco??", - "role": "user" - } - ] -} -``` - -### Making an API request to the Proxy Server - -```python -import requests -import json - -# TODO: use your URL -url = "http://localhost:5000/chat/completions" - -payload = json.dumps({ - "model": "gpt-3.5-turbo", - "messages": [ - { - "content": "Hello, whats the weather in San Francisco??", - "role": "user" - } - ] -}) -headers = { - 'Content-Type': 'application/json' -} -response = requests.request("POST", url, headers=headers, data=payload) -print(response.text) - -``` - -### Output [Response Format] - -Responses from the server are given in the following format. -All responses from the server are returned in the following format (for all LLM models). More info on output here: https://litellm.readthedocs.io/en/latest/output/ - -```json -{ - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "I'm sorry, but I don't have the capability to provide real-time weather information. However, you can easily check the weather in San Francisco by searching online or using a weather app on your phone.", - "role": "assistant" - } - } - ], - "created": 1691790381, - "id": "chatcmpl-7mUFZlOEgdohHRDx2UpYPRTejirzb", - "model": "gpt-3.5-turbo-0613", - "object": "chat.completion", - "usage": { - "completion_tokens": 41, - "prompt_tokens": 16, - "total_tokens": 57 - } -} -``` - -## Installation & Usage - -### Running Locally - -1. Clone liteLLM repository to your local machine: - ``` - git clone https://github.com/BerriAI/liteLLM-proxy - ``` -2. Install the required dependencies using pip - ``` - pip install -r requirements.txt - ``` -3. Set your LLM API keys - ``` - os.environ['OPENAI_API_KEY]` = "YOUR_API_KEY" - or - set OPENAI_API_KEY in your .env file - ``` -4. Run the server: - ``` - python main.py - ``` - -## Deploying - -1. Quick Start: Deploy on Railway - - [![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/DYqQAW?referralCode=t3ukrU) - -2. `GCP`, `AWS`, `Azure` - This project includes a `Dockerfile` allowing you to build and deploy a Docker Project on your providers - -# Support / Talk with founders - -- [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) -- [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai - -## Roadmap - -- [ ] Support hosted db (e.g. Supabase) -- [ ] Easily send data to places like posthog and sentry. -- [ ] Add a hot-cache for project spend logs - enables fast checks for user + project limitings -- [ ] Implement user-based rate-limiting -- [ ] Spending controls per project - expose key creation endpoint -- [ ] Need to store a keys db -> mapping created keys to their alias (i.e. project name) -- [ ] Easily add new models as backups / as the entry-point (add this to the available model list) diff --git a/cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml b/cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml deleted file mode 100644 index 3598a9b1b65..00000000000 --- a/cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - key_management_system: "custom" - key_management_settings: - custom_secret_manager: my_secret_manager.InMemorySecretManager - store_virtual_keys: true - prefix_for_stored_virtual_keys: "litellm/" - access_mode: "read_and_write" - -model_list: - - model_name: gpt-4 - litellm_params: - model: openai/gpt-4 - api_key: os.environ/OPENAI_API_KEY # Read from custom secret manager - - - model_name: claude-3-5-sonnet - litellm_params: - model: anthropic/claude-3-5-sonnet-20241022 - api_key: os.environ/ANTHROPIC_API_KEY # Read from custom secret manager - diff --git a/cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py b/cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py deleted file mode 100644 index 65c7f754b41..00000000000 --- a/cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -Example custom secret manager for LiteLLM Proxy. - -This is a simple in-memory secret manager for testing purposes. -In production, replace this with your actual secret management system. -""" - -from typing import Optional, Union - -import httpx - -from litellm.integrations.custom_secret_manager import CustomSecretManager - - -class InMemorySecretManager(CustomSecretManager): - def __init__(self): - super().__init__(secret_manager_name="in_memory_secrets") - # Store your secrets in memory - print("INITIALIZING CUSTOM SECRET MANAGER IN MEMORY") - self.secrets = {} - print("CUSTOM SECRET MANAGER IN MEMORY INITIALIZED") - - async def async_read_secret( - self, - secret_name: str, - optional_params: Optional[dict] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Optional[str]: - """Read secret asynchronously""" - print("READING SECRET ASYNCHRONOUSLY") - print("SECRET NAME: %s", secret_name) - print("SECRET: %s", self.secrets.get(secret_name)) - return self.secrets.get(secret_name) - - def sync_read_secret( - self, - secret_name: str, - optional_params: Optional[dict] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Optional[str]: - """Read secret synchronously""" - from litellm._logging import verbose_proxy_logger - - verbose_proxy_logger.info( - f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}" - ) - value = self.secrets.get(secret_name) - verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: READ SECRET: {value}") - return value - - async def async_write_secret( - self, - secret_name: str, - secret_value: str, - description: Optional[str] = None, - optional_params: Optional[dict] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - tags: Optional[Union[dict, list]] = None, - ) -> dict: - """Write a secret to the in-memory store""" - self.secrets[secret_name] = secret_value - print("ALL SECRETS=%s", self.secrets) - return { - "status": "success", - "secret_name": secret_name, - "description": description, - } - - async def async_delete_secret( - self, - secret_name: str, - recovery_window_in_days: Optional[int] = 7, - optional_params: Optional[dict] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> dict: - """Delete a secret from the in-memory store""" - if secret_name in self.secrets: - del self.secrets[secret_name] - return {"status": "deleted", "secret_name": secret_name} - return {"status": "not_found", "secret_name": secret_name} diff --git a/cookbook/litellm_router/error_log.txt b/cookbook/litellm_router/error_log.txt deleted file mode 100644 index 983b47cbbba..00000000000 --- a/cookbook/litellm_router/error_log.txt +++ /dev/null @@ -1,1004 +0,0 @@ -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - diff --git a/cookbook/litellm_router/load_test_proxy.py b/cookbook/litellm_router/load_test_proxy.py deleted file mode 100644 index 9ae6e764d91..00000000000 --- a/cookbook/litellm_router/load_test_proxy.py +++ /dev/null @@ -1,148 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path - -from litellm import Router -import litellm - -litellm.set_verbose = False -os.environ.pop("AZURE_AD_TOKEN") - -model_list = [ - { # list of model deployments - "model_name": "gpt-3.5-turbo", # model alias - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", # actual model name - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-functioncalling", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, -] -router = Router(model_list=model_list) - - -file_paths = [ - "test_questions/question1.txt", - "test_questions/question2.txt", - "test_questions/question3.txt", -] -questions = [] - -for file_path in file_paths: - try: - print(file_path) - with open(file_path, "r") as file: - content = file.read() - questions.append(content) - except FileNotFoundError as e: - print(f"File not found: {e}") - except Exception as e: - print(f"An error occurred: {e}") - -# for q in questions: -# print(q) - - -# make X concurrent calls to litellm.completion(model=gpt-35-turbo, messages=[]), pick a random question in questions array. -# Allow me to tune X concurrent calls.. Log question, output/exception, response time somewhere -# show me a summary of requests made, success full calls, failed calls. For failed calls show me the exceptions - -import concurrent.futures -import random -import time - - -# Function to make concurrent calls to OpenAI API -def make_openai_completion(question): - try: - start_time = time.time() - import openai - - client = openai.OpenAI( - api_key=os.environ["OPENAI_API_KEY"], base_url="http://0.0.0.0:8000" - ) # base_url="http://0.0.0.0:8000", - response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[ - { - "role": "system", - "content": f"You are a helpful assistant. Answer this question{question}", - } - ], - ) - print(response) - end_time = time.time() - - # Log the request details - with open("request_log.txt", "a") as log_file: - log_file.write( - f"Question: {question[:100]}\nResponse ID:{response.id} Content:{response.choices[0].message.content[:10]}\nTime: {end_time - start_time:.2f} seconds\n\n" - ) - - return response - except Exception as e: - # Log exceptions for failed calls - with open("error_log.txt", "a") as error_log_file: - error_log_file.write(f"Question: {question[:100]}\nException: {str(e)}\n\n") - return None - - -# Number of concurrent calls (you can adjust this) -concurrent_calls = 100 - -# List to store the futures of concurrent calls -futures = [] - -# Make concurrent calls -with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_calls) as executor: - for _ in range(concurrent_calls): - random_question = random.choice(questions) - futures.append(executor.submit(make_openai_completion, random_question)) - -# Wait for all futures to complete -concurrent.futures.wait(futures) - -# Summarize the results -successful_calls = 0 -failed_calls = 0 - -for future in futures: - if future.result() is not None: - successful_calls += 1 - else: - failed_calls += 1 - -print("Load test Summary:") -print(f"Total Requests: {concurrent_calls}") -print(f"Successful Calls: {successful_calls}") -print(f"Failed Calls: {failed_calls}") - -# Display content of the logs -with open("request_log.txt", "r") as log_file: - print("\nRequest Log:\n", log_file.read()) - -with open("error_log.txt", "r") as error_log_file: - print("\nError Log:\n", error_log_file.read()) diff --git a/cookbook/litellm_router/load_test_queuing.py b/cookbook/litellm_router/load_test_queuing.py deleted file mode 100644 index 7d4d44b2528..00000000000 --- a/cookbook/litellm_router/load_test_queuing.py +++ /dev/null @@ -1,164 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path - -from litellm import Router -import litellm - -litellm.set_verbose = False -# os.environ.pop("AZURE_AD_TOKEN") - -model_list = [ - { # list of model deployments - "model_name": "gpt-3.5-turbo", # model alias - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", # actual model name - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-functioncalling", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, -] -router = Router(model_list=model_list) - - -file_paths = [ - "test_questions/question1.txt", - "test_questions/question2.txt", - "test_questions/question3.txt", -] -questions = [] - -for file_path in file_paths: - try: - print(file_path) - with open(file_path, "r") as file: - content = file.read() - questions.append(content) - except FileNotFoundError as e: - print(f"File not found: {e}") - except Exception as e: - print(f"An error occurred: {e}") - -# for q in questions: -# print(q) - - -# make X concurrent calls to litellm.completion(model=gpt-35-turbo, messages=[]), pick a random question in questions array. -# Allow me to tune X concurrent calls.. Log question, output/exception, response time somewhere -# show me a summary of requests made, success full calls, failed calls. For failed calls show me the exceptions - -import concurrent.futures -import random -import time - - -# Function to make concurrent calls to OpenAI API -def make_openai_completion(question): - try: - start_time = time.time() - import requests - - data = { - "model": "gpt-3.5-turbo", - "messages": [ - { - "role": "system", - "content": f"You are a helpful assistant. Answer this question{question}", - }, - ], - } - response = requests.post("http://0.0.0.0:8000/queue/request", json=data) - response = response.json() - end_time = time.time() - # Log the request details - with open("request_log.txt", "a") as log_file: - log_file.write( - f"Question: {question[:100]}\nResponse ID: {response.get('id', 'N/A')} Url: {response.get('url', 'N/A')}\nTime: {end_time - start_time:.2f} seconds\n\n" - ) - - # polling the url - while True: - try: - url = response["url"] - polling_url = f"http://0.0.0.0:8000{url}" - polling_response = requests.get(polling_url) - polling_response = polling_response.json() - print("\n RESPONSE FROM POLLING JoB", polling_response) - status = polling_response["status"] - if status == "finished": - llm_response = polling_response["result"] - with open("response_log.txt", "a") as log_file: - log_file.write( - f"Response ID: {llm_response.get('id', 'NA')}\nLLM Response: {llm_response}\nTime: {end_time - start_time:.2f} seconds\n\n" - ) - - break - print( - f"POLLING JOB{polling_url}\nSTATUS: {status}, \n Response {polling_response}" - ) - time.sleep(0.5) - except Exception as e: - print("got exception in polling", e) - break - - return response - except Exception as e: - # Log exceptions for failed calls - with open("error_log.txt", "a") as error_log_file: - error_log_file.write(f"Question: {question[:100]}\nException: {str(e)}\n\n") - return None - - -# Number of concurrent calls (you can adjust this) -concurrent_calls = 10 - -# List to store the futures of concurrent calls -futures = [] - -# Make concurrent calls -with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_calls) as executor: - for _ in range(concurrent_calls): - random_question = random.choice(questions) - futures.append(executor.submit(make_openai_completion, random_question)) - -# Wait for all futures to complete -concurrent.futures.wait(futures) - -# Summarize the results -successful_calls = 0 -failed_calls = 0 - -for future in futures: - if future.done(): - if future.result() is not None: - successful_calls += 1 - else: - failed_calls += 1 - -print("Load test Summary:") -print(f"Total Requests: {concurrent_calls}") -print(f"Successful Calls: {successful_calls}") -print(f"Failed Calls: {failed_calls}") diff --git a/cookbook/litellm_router/load_test_router.py b/cookbook/litellm_router/load_test_router.py deleted file mode 100644 index 92533b6c929..00000000000 --- a/cookbook/litellm_router/load_test_router.py +++ /dev/null @@ -1,143 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path - -from litellm import Router -import litellm - -litellm.set_verbose = False -os.environ.pop("AZURE_AD_TOKEN") - -model_list = [ - { # list of model deployments - "model_name": "gpt-3.5-turbo", # model alias - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", # actual model name - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-functioncalling", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, -] -router = Router(model_list=model_list) - - -file_paths = [ - "test_questions/question1.txt", - "test_questions/question2.txt", - "test_questions/question3.txt", -] -questions = [] - -for file_path in file_paths: - try: - print(file_path) - with open(file_path, "r") as file: - content = file.read() - questions.append(content) - except FileNotFoundError as e: - print(f"File not found: {e}") - except Exception as e: - print(f"An error occurred: {e}") - -# for q in questions: -# print(q) - - -# make X concurrent calls to litellm.completion(model=gpt-35-turbo, messages=[]), pick a random question in questions array. -# Allow me to tune X concurrent calls.. Log question, output/exception, response time somewhere -# show me a summary of requests made, success full calls, failed calls. For failed calls show me the exceptions - -import concurrent.futures -import random -import time - - -# Function to make concurrent calls to OpenAI API -def make_openai_completion(question): - try: - start_time = time.time() - response = router.completion( - model="gpt-3.5-turbo", - messages=[ - { - "role": "system", - "content": f"You are a helpful assistant. Answer this question{question}", - } - ], - ) - print(response) - end_time = time.time() - - # Log the request details - with open("request_log.txt", "a") as log_file: - log_file.write( - f"Question: {question[:100]}\nResponse: {response.choices[0].message.content}\nTime: {end_time - start_time:.2f} seconds\n\n" - ) - - return response - except Exception as e: - # Log exceptions for failed calls - with open("error_log.txt", "a") as error_log_file: - error_log_file.write(f"Question: {question[:100]}\nException: {str(e)}\n\n") - return None - - -# Number of concurrent calls (you can adjust this) -concurrent_calls = 150 - -# List to store the futures of concurrent calls -futures = [] - -# Make concurrent calls -with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_calls) as executor: - for _ in range(concurrent_calls): - random_question = random.choice(questions) - futures.append(executor.submit(make_openai_completion, random_question)) - -# Wait for all futures to complete -concurrent.futures.wait(futures) - -# Summarize the results -successful_calls = 0 -failed_calls = 0 - -for future in futures: - if future.result() is not None: - successful_calls += 1 - else: - failed_calls += 1 - -print("Load test Summary:") -print(f"Total Requests: {concurrent_calls}") -print(f"Successful Calls: {successful_calls}") -print(f"Failed Calls: {failed_calls}") - -# Display content of the logs -with open("request_log.txt", "r") as log_file: - print("\nRequest Log:\n", log_file.read()) - -with open("error_log.txt", "r") as error_log_file: - print("\nError Log:\n", error_log_file.read()) diff --git a/cookbook/litellm_router/request_log.txt b/cookbook/litellm_router/request_log.txt deleted file mode 100644 index 821d87ab56a..00000000000 --- a/cookbook/litellm_router/request_log.txt +++ /dev/null @@ -1,48 +0,0 @@ -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: 71a47cd4-92d9-4091-9429-8d22af6b56bf Url: /queue/response/71a47cd4-92d9-4091-9429-8d22af6b56bf -Time: 0.77 seconds - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Response ID: a0855c20-59ba-4eed-85c1-e0719eebdeab Url: /queue/response/a0855c20-59ba-4eed-85c1-e0719eebdeab -Time: 1.46 seconds - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: b131cdcd-0693-495b-ad41-b0cf2afc4833 Url: /queue/response/b131cdcd-0693-495b-ad41-b0cf2afc4833 -Time: 2.13 seconds - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: a58e5185-90e7-4832-9f28-e5a5ac167a40 Url: /queue/response/a58e5185-90e7-4832-9f28-e5a5ac167a40 -Time: 2.83 seconds - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Response ID: 52dbbd49-eedb-4c11-8382-3ca7deb1af35 Url: /queue/response/52dbbd49-eedb-4c11-8382-3ca7deb1af35 -Time: 3.50 seconds - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Response ID: eedda05f-61e1-4081-b49d-27f9449bcf69 Url: /queue/response/eedda05f-61e1-4081-b49d-27f9449bcf69 -Time: 4.20 seconds - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: 8a484722-66ec-4193-b19b-2dfc4265cfd2 Url: /queue/response/8a484722-66ec-4193-b19b-2dfc4265cfd2 -Time: 4.89 seconds - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: ae1e2b71-d711-456d-8df0-13ce0709eb04 Url: /queue/response/ae1e2b71-d711-456d-8df0-13ce0709eb04 -Time: 5.60 seconds - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Response ID: cfabd174-838e-4252-b82b-648923573db8 Url: /queue/response/cfabd174-838e-4252-b82b-648923573db8 -Time: 6.29 seconds - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: 02d5b7d6-5443-41e9-94e4-90d8b00d49fb Url: /queue/response/02d5b7d6-5443-41e9-94e4-90d8b00d49fb -Time: 7.01 seconds - diff --git a/cookbook/litellm_router/response_log.txt b/cookbook/litellm_router/response_log.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/cookbook/litellm_router/test_questions/question1.txt b/cookbook/litellm_router/test_questions/question1.txt deleted file mode 100644 index d633a8ea22e..00000000000 --- a/cookbook/litellm_router/test_questions/question1.txt +++ /dev/null @@ -1,43 +0,0 @@ -Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. Use Bedrock, Azure, OpenAI, Cohere, Anthropic, Ollama, Sagemaker, HuggingFace, Replicate (100+ LLMs). LiteLLM manages - -Translating inputs to the provider's completion and embedding endpoints -Guarantees consistent output, text responses will always be available at ['choices'][0]['message']['content'] -Exception mapping - common exceptions across providers are mapped to the OpenAI exception types. -10/05/2023: LiteLLM is adopting Semantic Versioning for all commits. Learn more -10/16/2023: Self-hosted OpenAI-proxy server Learn more - -Usage (Docs) -Important -LiteLLM v1.0.0 is being launched to require openai>=1.0.0. Track this here - -Open In Colab -pip install litellm -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-openai-key" -os.environ["COHERE_API_KEY"] = "your-cohere-key" - -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages) - -# cohere call -response = completion(model="command-nightly", messages=messages) -print(response) -Streaming (Docs) -liteLLM supports streaming the model response back, pass stream=True to get a streaming iterator in response. -Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.) - -from litellm import completion -response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) -for chunk in response: - print(chunk['choices'][0]['delta']) - -# claude 2 -result = completion('claude-2', messages, stream=True) -for chunk in result: - print(chunk['choices'][0]['delta']) \ No newline at end of file diff --git a/cookbook/litellm_router/test_questions/question2.txt b/cookbook/litellm_router/test_questions/question2.txt deleted file mode 100644 index 78188d06668..00000000000 --- a/cookbook/litellm_router/test_questions/question2.txt +++ /dev/null @@ -1,65 +0,0 @@ -Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the OpenAI format. Use Bedrock, Azure, OpenAI, Cohere, Anthropic, Ollama, Sagemaker, HuggingFace, Replicate (100+ LLMs). LiteLLM manages - -Translating inputs to the provider's completion and embedding endpoints -Guarantees consistent output, text responses will always be available at ['choices'][0]['message']['content'] -Exception mapping - common exceptions across providers are mapped to the OpenAI exception types. -10/05/2023: LiteLLM is adopting Semantic Versioning for all commits. Learn more -10/16/2023: Self-hosted OpenAI-proxy server Learn more - -Usage (Docs) -Important -LiteLLM v1.0.0 is being launched to require openai>=1.0.0. Track this here - -Open In Colab -pip install litellm -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-openai-key" -os.environ["COHERE_API_KEY"] = "your-cohere-key" - -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages) - -# cohere call -response = completion(model="command-nightly", messages=messages) -print(response) -Streaming (Docs) -liteLLM supports streaming the model response back, pass stream=True to get a streaming iterator in response. -Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.) - -from litellm import completion -response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) -for chunk in response: - print(chunk['choices'][0]['delta']) - -# claude 2 -result = completion('claude-2', messages, stream=True) -for chunk in result: - print(chunk['choices'][0]['delta']) Supported LiteLLM providers Supported Provider (Docs) -Provider Completion Streaming Async Completion Async Streaming -openai ✅ ✅ ✅ ✅ -azure ✅ ✅ ✅ ✅ -aws - sagemaker ✅ ✅ ✅ ✅ -aws - bedrock ✅ ✅ ✅ ✅ -cohere ✅ ✅ ✅ ✅ -anthropic ✅ ✅ ✅ ✅ -huggingface ✅ ✅ ✅ ✅ -replicate ✅ ✅ ✅ ✅ -together_ai ✅ ✅ ✅ ✅ -openrouter ✅ ✅ ✅ ✅ -google - vertex_ai ✅ ✅ ✅ ✅ -google - palm ✅ ✅ ✅ ✅ -ai21 ✅ ✅ ✅ ✅ -baseten ✅ ✅ ✅ ✅ -vllm ✅ ✅ ✅ ✅ -nlp_cloud ✅ ✅ ✅ ✅ -aleph alpha ✅ ✅ ✅ ✅ -petals ✅ ✅ ✅ ✅ -ollama ✅ ✅ ✅ ✅ -deepinfra ✅ ✅ ✅ ✅ -perplexity-ai ✅ ✅ ✅ ✅ -anyscale ✅ ✅ ✅ ✅ \ No newline at end of file diff --git a/cookbook/litellm_router/test_questions/question3.txt b/cookbook/litellm_router/test_questions/question3.txt deleted file mode 100644 index d6006f9c73c..00000000000 --- a/cookbook/litellm_router/test_questions/question3.txt +++ /dev/null @@ -1,50 +0,0 @@ -What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 100+ LLMs Huggingface/Bedrock/TogetherAI/etc. in the OpenAI ChatCompletions & Completions format -Set custom prompt templates + model-specific configs (temperature, max_tokens, etc.) -Quick Start -View all the supported args for the Proxy CLI here - -$ litellm --model huggingface/bigcode/starcoder - -#INFO: Proxy running on http://0.0.0.0:8000 - -Test -In a new shell, run, this will make an openai.ChatCompletion request - -litellm --test - -This will now automatically route any requests for gpt-3.5-turbo to bigcode starcoder, hosted on huggingface inference endpoints. - -Replace openai base -import openai - -openai.api_base = "http://0.0.0.0:8000" - -print(openai.chat.completions.create(model="test", messages=[{"role":"user", "content":"Hey!"}])) - -Supported LLMs -Bedrock -Huggingface (TGI) -Anthropic -VLLM -OpenAI Compatible Server -TogetherAI -Replicate -Petals -Palm -Azure OpenAI -AI21 -Cohere -$ export AWS_ACCESS_KEY_ID="" -$ export AWS_REGION_NAME="" # e.g. us-west-2 -$ export AWS_SECRET_ACCESS_KEY="" - -$ litellm --model bedrock/anthropic.claude-v2 - -Server Endpoints -POST /chat/completions - chat completions endpoint to call 100+ LLMs -POST /completions - completions endpoint -POST /embeddings - embedding endpoint for Azure, OpenAI, Huggingface endpoints -GET /models - available models on server \ No newline at end of file diff --git a/cookbook/litellm_router_load_test/memory_usage/router_endpoint.py b/cookbook/litellm_router_load_test/memory_usage/router_endpoint.py deleted file mode 100644 index 1dc2d914857..00000000000 --- a/cookbook/litellm_router_load_test/memory_usage/router_endpoint.py +++ /dev/null @@ -1,65 +0,0 @@ -from fastapi import FastAPI -import uvicorn -from memory_profiler import profile -import os -import litellm -from litellm import Router -from dotenv import load_dotenv -from litellm._uuid import uuid - -load_dotenv() - -model_list = [ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { - "model_name": "text-embedding-ada-002", - "litellm_params": { - "model": "azure/azure-embedding-model", - "api_key": os.getenv("AZURE_API_KEY"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - "tpm": 100000, - "rpm": 10000, - }, -] - -litellm.set_verbose = True -litellm.cache = litellm.Cache( - type="s3", s3_bucket_name="litellm-my-test-bucket-2", s3_region_name="us-east-1" -) -router = Router(model_list=model_list, set_verbose=True) - -app = FastAPI() - - -@app.get("/") -async def read_root(): - return {"message": "Welcome to the FastAPI endpoint!"} - - -@profile -@app.post("/router_acompletion") -async def router_acompletion(): - question = f"This is a test: {uuid.uuid4()}" * 100 - resp = await router.aembedding(model="text-embedding-ada-002", input=question) - print("embedding-resp", resp) - - response = await router.acompletion( - model="gpt-3.5-turbo", messages=[{"role": "user", "content": question}] - ) - print("completion-resp", response) - return response - - -if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py b/cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py deleted file mode 100644 index 76d5d3913f5..00000000000 --- a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py +++ /dev/null @@ -1,91 +0,0 @@ -#### What this tests #### - -from memory_profiler import profile -import sys -import os -import time -import asyncio - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import litellm -from litellm import Router -from dotenv import load_dotenv -from litellm._uuid import uuid - -load_dotenv() - - -model_list = [ - { - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { - "model_name": "text-embedding-ada-002", - "litellm_params": { - "model": "azure/azure-embedding-model", - "api_key": os.environ["AZURE_API_KEY"], - "api_base": os.environ["AZURE_API_BASE"], - }, - "tpm": 100000, - "rpm": 10000, - }, -] -litellm.set_verbose = True -litellm.cache = litellm.Cache( - type="s3", s3_bucket_name="litellm-my-test-bucket-2", s3_region_name="us-east-1" -) -router = Router( - model_list=model_list, - set_verbose=True, -) # type: ignore - - -@profile -async def router_acompletion(): - # embedding call - question = f"This is a test: {uuid.uuid4()}" * 100 - resp = await router.aembedding(model="text-embedding-ada-002", input=question) - print("embedding-resp", resp) - - response = await router.acompletion( - model="gpt-3.5-turbo", messages=[{"role": "user", "content": question}] - ) - print("completion-resp", response) - return response - - -async def main(): - for i in range(1): - start = time.time() - n = 50 # Number of concurrent tasks - tasks = [router_acompletion() for _ in range(n)] - - chat_completions = await asyncio.gather(*tasks) - - successful_completions = [c for c in chat_completions if c is not None] - - # Write errors to error_log.txt - with open("error_log.txt", "a") as error_log: - for completion in chat_completions: - if isinstance(completion, str): - error_log.write(completion + "\n") - - print(n, time.time() - start, len(successful_completions)) - time.sleep(10) - - -if __name__ == "__main__": - # Blank out contents of error_log.txt - open("error_log.txt", "w").close() - - asyncio.run(main()) diff --git a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py b/cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py deleted file mode 100644 index 76d5d3913f5..00000000000 --- a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py +++ /dev/null @@ -1,91 +0,0 @@ -#### What this tests #### - -from memory_profiler import profile -import sys -import os -import time -import asyncio - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import litellm -from litellm import Router -from dotenv import load_dotenv -from litellm._uuid import uuid - -load_dotenv() - - -model_list = [ - { - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { - "model_name": "text-embedding-ada-002", - "litellm_params": { - "model": "azure/azure-embedding-model", - "api_key": os.environ["AZURE_API_KEY"], - "api_base": os.environ["AZURE_API_BASE"], - }, - "tpm": 100000, - "rpm": 10000, - }, -] -litellm.set_verbose = True -litellm.cache = litellm.Cache( - type="s3", s3_bucket_name="litellm-my-test-bucket-2", s3_region_name="us-east-1" -) -router = Router( - model_list=model_list, - set_verbose=True, -) # type: ignore - - -@profile -async def router_acompletion(): - # embedding call - question = f"This is a test: {uuid.uuid4()}" * 100 - resp = await router.aembedding(model="text-embedding-ada-002", input=question) - print("embedding-resp", resp) - - response = await router.acompletion( - model="gpt-3.5-turbo", messages=[{"role": "user", "content": question}] - ) - print("completion-resp", response) - return response - - -async def main(): - for i in range(1): - start = time.time() - n = 50 # Number of concurrent tasks - tasks = [router_acompletion() for _ in range(n)] - - chat_completions = await asyncio.gather(*tasks) - - successful_completions = [c for c in chat_completions if c is not None] - - # Write errors to error_log.txt - with open("error_log.txt", "a") as error_log: - for completion in chat_completions: - if isinstance(completion, str): - error_log.write(completion + "\n") - - print(n, time.time() - start, len(successful_completions)) - time.sleep(10) - - -if __name__ == "__main__": - # Blank out contents of error_log.txt - open("error_log.txt", "w").close() - - asyncio.run(main()) diff --git a/cookbook/litellm_router_load_test/memory_usage/send_request.py b/cookbook/litellm_router_load_test/memory_usage/send_request.py deleted file mode 100644 index 6a3473e230f..00000000000 --- a/cookbook/litellm_router_load_test/memory_usage/send_request.py +++ /dev/null @@ -1,28 +0,0 @@ -import requests -from concurrent.futures import ThreadPoolExecutor - -# Replace the URL with your actual endpoint -url = "http://localhost:8000/router_acompletion" - - -def make_request(session): - headers = {"Content-Type": "application/json"} - data = {} # Replace with your JSON payload if needed - - response = session.post(url, headers=headers, json=data) - print(f"Status code: {response.status_code}") - - -# Number of concurrent requests -num_requests = 20 - -# Create a session to reuse the underlying TCP connection -with requests.Session() as session: - # Use ThreadPoolExecutor for concurrent requests - with ThreadPoolExecutor(max_workers=num_requests) as executor: - # Use list comprehension to submit tasks - futures = [executor.submit(make_request, session) for _ in range(num_requests)] - - # Wait for all futures to complete - for future in futures: - future.result() diff --git a/cookbook/litellm_router_load_test/test_loadtest_openai_client.py b/cookbook/litellm_router_load_test/test_loadtest_openai_client.py deleted file mode 100644 index 8c50825be12..00000000000 --- a/cookbook/litellm_router_load_test/test_loadtest_openai_client.py +++ /dev/null @@ -1,73 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import asyncio -from litellm import Timeout -import time -import openai - -### Test just calling AsyncAzureOpenAI - -openai_client = openai.AsyncAzureOpenAI( - azure_endpoint=os.getenv("AZURE_API_BASE"), - api_key=os.getenv("AZURE_API_KEY"), -) - - -async def call_acompletion(semaphore, input_data): - async with semaphore: - try: - # Use asyncio.wait_for to set a timeout for the task - response = await openai_client.chat.completions.create(**input_data) - # Handle the response as needed - print(response) - return response - except Timeout: - print(f"Task timed out: {input_data}") - return None # You may choose to return something else or raise an exception - - -async def main(): - # Initialize the Router - - # Create a semaphore with a capacity of 100 - semaphore = asyncio.Semaphore(100) - - # List to hold all task references - tasks = [] - start_time_all_tasks = time.time() - # Launch 1000 tasks - for _ in range(500): - task = asyncio.create_task( - call_acompletion( - semaphore, - { - "model": "chatgpt-v-2", - "messages": [{"role": "user", "content": "Hey, how's it going?"}], - }, - ) - ) - tasks.append(task) - - # Wait for all tasks to complete - responses = await asyncio.gather(*tasks) - # Process responses as needed - # Record the end time for all tasks - end_time_all_tasks = time.time() - # Calculate the total time for all tasks - total_time_all_tasks = end_time_all_tasks - start_time_all_tasks - print(f"Total time for all tasks: {total_time_all_tasks} seconds") - - # Calculate the average time per response - average_time_per_response = total_time_all_tasks / len(responses) - print(f"Average time per response: {average_time_per_response} seconds") - print(f"NUMBER OF COMPLETED TASKS: {len(responses)}") - - -# Run the main function -asyncio.run(main()) diff --git a/cookbook/litellm_router_load_test/test_loadtest_router.py b/cookbook/litellm_router_load_test/test_loadtest_router.py deleted file mode 100644 index 280e495e771..00000000000 --- a/cookbook/litellm_router_load_test/test_loadtest_router.py +++ /dev/null @@ -1,87 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import asyncio -from litellm import Router, Timeout -import time - -### Test calling router async - - -async def call_acompletion(semaphore, router: Router, input_data): - async with semaphore: - try: - # Use asyncio.wait_for to set a timeout for the task - response = await router.acompletion(**input_data) - # Handle the response as needed - print(response) - return response - except Timeout: - print(f"Task timed out: {input_data}") - return None # You may choose to return something else or raise an exception - - -async def main(): - # Initialize the Router - model_list = [ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_base": os.getenv("AZURE_API_BASE"), - "api_version": os.getenv("AZURE_API_VERSION"), - }, - }, - ] - router = Router(model_list=model_list, num_retries=3, timeout=10) - - # Create a semaphore with a capacity of 100 - semaphore = asyncio.Semaphore(100) - - # List to hold all task references - tasks = [] - start_time_all_tasks = time.time() - # Launch 1000 tasks - for _ in range(500): - task = asyncio.create_task( - call_acompletion( - semaphore, - router, - { - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hey, how's it going?"}], - }, - ) - ) - tasks.append(task) - - # Wait for all tasks to complete - responses = await asyncio.gather(*tasks) - # Process responses as needed - # Record the end time for all tasks - end_time_all_tasks = time.time() - # Calculate the total time for all tasks - total_time_all_tasks = end_time_all_tasks - start_time_all_tasks - print(f"Total time for all tasks: {total_time_all_tasks} seconds") - - # Calculate the average time per response - average_time_per_response = total_time_all_tasks / len(responses) - print(f"Average time per response: {average_time_per_response} seconds") - print(f"NUMBER OF COMPLETED TASKS: {len(responses)}") - - -# Run the main function -asyncio.run(main()) diff --git a/cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py b/cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py deleted file mode 100644 index b093489be1b..00000000000 --- a/cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py +++ /dev/null @@ -1,93 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import asyncio -from litellm import Router, Timeout -import time -from litellm.caching.caching import Cache -import litellm - -litellm.cache = Cache( - type="s3", s3_bucket_name="cache-bucket-litellm", s3_region_name="us-west-2" -) - -### Test calling router with s3 Cache - - -async def call_acompletion(semaphore, router: Router, input_data): - async with semaphore: - try: - # Use asyncio.wait_for to set a timeout for the task - response = await router.acompletion(**input_data) - # Handle the response as needed - print(response) - return response - except Timeout: - print(f"Task timed out: {input_data}") - return None # You may choose to return something else or raise an exception - - -async def main(): - # Initialize the Router - model_list = [ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_base": os.getenv("AZURE_API_BASE"), - "api_version": os.getenv("AZURE_API_VERSION"), - }, - }, - ] - router = Router(model_list=model_list, num_retries=3, timeout=10) - - # Create a semaphore with a capacity of 100 - semaphore = asyncio.Semaphore(100) - - # List to hold all task references - tasks = [] - start_time_all_tasks = time.time() - # Launch 1000 tasks - for _ in range(500): - task = asyncio.create_task( - call_acompletion( - semaphore, - router, - { - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hey, how's it going?"}], - }, - ) - ) - tasks.append(task) - - # Wait for all tasks to complete - responses = await asyncio.gather(*tasks) - # Process responses as needed - # Record the end time for all tasks - end_time_all_tasks = time.time() - # Calculate the total time for all tasks - total_time_all_tasks = end_time_all_tasks - start_time_all_tasks - print(f"Total time for all tasks: {total_time_all_tasks} seconds") - - # Calculate the average time per response - average_time_per_response = total_time_all_tasks / len(responses) - print(f"Average time per response: {average_time_per_response} seconds") - print(f"NUMBER OF COMPLETED TASKS: {len(responses)}") - - -# Run the main function -asyncio.run(main()) diff --git a/cookbook/litellm_test_multiple_llm_demo.ipynb b/cookbook/litellm_test_multiple_llm_demo.ipynb deleted file mode 100644 index f22448e46b9..00000000000 --- a/cookbook/litellm_test_multiple_llm_demo.ipynb +++ /dev/null @@ -1,55 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "adotBkqZSh5g" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion\n", - "\n", - "## set ENV variables\n", - "os.environ[\"OPENAI_API_KEY\"] = \"openai key\"\n", - "os.environ[\"COHERE_API_KEY\"] = \"cohere key\"\n", - "os.environ[\"REPLICATE_API_KEY\"] = \"replicate key\"\n", - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "\n", - "# openai call\n", - "response = completion(model=\"gpt-3.5-turbo\", messages=messages)\n", - "\n", - "# cohere call\n", - "response = completion(\"command-nightly\", messages)\n", - "\n", - "# replicate call\n", - "response = completion(\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\", messages)" - ], - "metadata": { - "id": "LeOqznSgSj-z" - }, - "execution_count": null, - "outputs": [] - } - ] -} diff --git a/cookbook/livekit_agent_sdk/README.md b/cookbook/livekit_agent_sdk/README.md deleted file mode 100644 index 6c794165e96..00000000000 --- a/cookbook/livekit_agent_sdk/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# LiveKit Voice Agent with LiteLLM Gateway - -Simple example showing how to use LiveKit's xAI realtime plugin with LiteLLM as a proxy. This lets you switch between xAI, OpenAI, and Azure realtime APIs without changing your code. - -## Quick Start - -### 1. Install dependencies - -```bash -pip install livekit-agents[xai] websockets -``` - -### 2. Start LiteLLM proxy - -```bash -# With xAI -export XAI_API_KEY="your-xai-key" -export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)" -litellm --config config.yaml --port 4000 -``` - -### 3. Run the voice agent - -```bash -export LITELLM_API_KEY="$LITELLM_MASTER_KEY" -python main.py -``` - -Type your message and get a voice response from Grok! - -## Configuration - -Set these environment variables if needed: - -```bash -export LITELLM_PROXY_URL="http://localhost:4000" -export LITELLM_API_KEY="$LITELLM_MASTER_KEY" -export LITELLM_MODEL="grok-voice-agent" -``` - -Or use the defaults - connects to `http://localhost:4000` by default. - -## Example Config File - -Create a `config.yaml` with your realtime models: - -```yaml -model_list: - - model_name: grok-voice-agent - litellm_params: - model: xai/grok-2-vision-1212 - api_key: os.environ/XAI_API_KEY - model_info: - mode: realtime - - - model_name: openai-voice-agent - litellm_params: - model: gpt-4o-realtime-preview - api_key: os.environ/OPENAI_API_KEY - model_info: - mode: realtime - -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY -``` - -Then start: `litellm --config config.yaml --port 4000` - -## How It Works - -LiveKit's xAI plugin connects through LiteLLM proxy by setting `base_url`: - -```python -from livekit.plugins import xai - -model = xai.realtime.RealtimeModel( - voice="ara", - api_key=os.environ["LITELLM_MASTER_KEY"], - base_url="http://localhost:4000", # Point to LiteLLM -) -``` - -## Switching Providers - -Just change the model in your config - no code changes needed: - -**xAI Grok:** -```yaml -model: xai/grok-2-vision-1212 -``` - -**OpenAI:** -```yaml -model: gpt-4o-realtime-preview -``` - -**Azure OpenAI:** -```yaml -model: azure/gpt-4o-realtime-preview -api_base: https://your-endpoint.openai.azure.com/ -``` - -## Why Use LiteLLM? - -- ✅ **Switch providers** without changing agent code -- ✅ **Cost tracking** across all voice sessions -- ✅ **Rate limiting** and budgets -- ✅ **Load balancing** across multiple API keys -- ✅ **Fallbacks** to backup models - -## Learn More - -- [LiveKit xAI Realtime Tutorial](/docs/tutorials/livekit_xai_realtime) -- [xAI Realtime Docs](/docs/providers/xai_realtime) -- [LiveKit Agents Documentation](https://docs.livekit.io/agents/) -- [LiteLLM Realtime API](/docs/realtime) diff --git a/cookbook/livekit_agent_sdk/config.example.yaml b/cookbook/livekit_agent_sdk/config.example.yaml deleted file mode 100644 index 388aa24f501..00000000000 --- a/cookbook/livekit_agent_sdk/config.example.yaml +++ /dev/null @@ -1,20 +0,0 @@ -model_list: - - model_name: grok-voice-agent - litellm_params: - model: xai/grok-2-vision-1212 - api_key: os.environ/XAI_API_KEY - model_info: - mode: realtime - - - model_name: openai-voice-agent - litellm_params: - model: gpt-4o-realtime-preview - api_key: os.environ/OPENAI_API_KEY - model_info: - mode: realtime - -litellm_settings: - drop_params: True - -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY diff --git a/cookbook/livekit_agent_sdk/main.py b/cookbook/livekit_agent_sdk/main.py deleted file mode 100644 index fdc72e518b4..00000000000 --- a/cookbook/livekit_agent_sdk/main.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Simple xAI Voice Agent using LiveKit SDK with LiteLLM Gateway - -This example shows how to use LiveKit's xAI realtime plugin through LiteLLM proxy. -LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI, -and Azure realtime APIs without changing your agent code. -""" - -import asyncio -import json -import os -import websockets - -# Configuration -PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000") -API_KEY = os.environ["LITELLM_API_KEY"] -MODEL = os.getenv("LITELLM_MODEL", "grok-voice-agent") - - -async def run_voice_agent(): - """ - Simple voice agent that: - 1. Connects to xAI realtime API through LiteLLM proxy - 2. Sends a user message - 3. Streams back the response - """ - - url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}" - headers = {"Authorization": f"Bearer {API_KEY}"} - - print(f"🎙️ Connecting to voice agent...") - print(f" Model: {MODEL}") - print(f" Proxy: {PROXY_URL}") - print() - - async with websockets.connect(url, additional_headers=headers) as ws: - # Receive initial connection event - initial = json.loads(await ws.recv()) - print(f"✅ Connected! Event: {initial['type']}\n") - - # Get user input - user_message = input("💬 Your message: ").strip() - if not user_message: - user_message = "Tell me a fun fact about AI!" - - print(f"\n🤖 Sending to {MODEL}...\n") - - # Send user message - await ws.send( - json.dumps( - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": user_message}], - }, - } - ) - ) - - # Request response - await ws.send( - json.dumps( - { - "type": "response.create", - "response": {"modalities": ["text", "audio"]}, - } - ) - ) - - # Stream response - print("🎤 Response: ", end="", flush=True) - transcript = [] - - try: - while True: - msg = await asyncio.wait_for(ws.recv(), timeout=15.0) - event = json.loads(msg) - - # Capture transcript deltas - if event["type"] == "response.output_audio_transcript.delta": - delta = event.get("delta", "") - if delta: - print(delta, end="", flush=True) - transcript.append(delta) - - # Done when response completes - elif event["type"] == "response.done": - break - - except asyncio.TimeoutError: - pass - - print("\n") - - if transcript: - print(f"✅ Complete response: {''.join(transcript)}") - - await ws.close() - - -def main(): - """Run the voice agent""" - print("=" * 70) - print("LiveKit xAI Voice Agent via LiteLLM Proxy") - print("=" * 70) - print() - - try: - asyncio.run(run_voice_agent()) - except KeyboardInterrupt: - print("\n\n👋 Goodbye!") - except Exception as e: - print(f"\n❌ Error: {e}") - print("\nMake sure LiteLLM proxy is running:") - print(f" litellm --config config.yaml --port 4000") - - -if __name__ == "__main__": - main() diff --git a/cookbook/livekit_agent_sdk/requirements.txt b/cookbook/livekit_agent_sdk/requirements.txt deleted file mode 100644 index 9e3542fac27..00000000000 --- a/cookbook/livekit_agent_sdk/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -livekit-agents[xai]>=1.3.12 -websockets>=15.0.1 diff --git a/cookbook/logging_observability/LiteLLM_Arize.ipynb b/cookbook/logging_observability/LiteLLM_Arize.ipynb deleted file mode 100644 index 72a082f874d..00000000000 --- a/cookbook/logging_observability/LiteLLM_Arize.ipynb +++ /dev/null @@ -1,172 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "4FbDOmcj2VkM" - }, - "source": [ - "## Use LiteLLM with Arize\n", - "https://docs.litellm.ai/docs/observability/arize_integration\n", - "\n", - "This method uses the litellm proxy to send the data to Arize. The callback is set in the litellm config below, instead of using OpenInference tracing." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "21W8Woog26Ns" - }, - "source": [ - "## Install Dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "xrjKLBxhxu2L" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: litellm in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (1.54.1)\n", - "Requirement already satisfied: aiohttp in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (3.11.10)\n", - "Requirement already satisfied: click in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (8.1.7)\n", - "Requirement already satisfied: httpx<0.28.0,>=0.23.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.27.2)\n", - "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (8.5.0)\n", - "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (3.1.4)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (4.23.0)\n", - "Requirement already satisfied: openai>=1.55.3 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (1.57.1)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (2.10.3)\n", - "Requirement already satisfied: python-dotenv>=0.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (1.0.1)\n", - "Requirement already satisfied: requests<3.0.0,>=2.31.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (2.32.3)\n", - "Requirement already satisfied: tiktoken>=0.7.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.7.0)\n", - "Requirement already satisfied: tokenizers in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.21.0)\n", - "Requirement already satisfied: anyio in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (4.7.0)\n", - "Requirement already satisfied: certifi in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (2024.8.30)\n", - "Requirement already satisfied: httpcore==1.* in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (1.0.7)\n", - "Requirement already satisfied: idna in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (3.10)\n", - "Requirement already satisfied: sniffio in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (1.3.1)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpcore==1.*->httpx<0.28.0,>=0.23.0->litellm) (0.14.0)\n", - "Requirement already satisfied: zipp>=3.20 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from importlib-metadata>=6.8.0->litellm) (3.21.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jinja2<4.0.0,>=3.1.2->litellm) (3.0.2)\n", - "Requirement already satisfied: attrs>=22.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (24.2.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.22.3)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (0.6.1)\n", - "Requirement already satisfied: tqdm>4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (4.67.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (4.12.2)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->litellm) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->litellm) (2.27.1)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from requests<3.0.0,>=2.31.0->litellm) (3.4.0)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from requests<3.0.0,>=2.31.0->litellm) (2.0.7)\n", - "Requirement already satisfied: regex>=2022.1.18 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from tiktoken>=0.7.0->litellm) (2024.11.6)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (2.4.4)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.3.1)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.5.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (6.1.0)\n", - "Requirement already satisfied: propcache>=0.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (0.2.1)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.18.3)\n", - "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from tokenizers->litellm) (0.26.5)\n", - "Requirement already satisfied: filelock in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (3.16.1)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (2024.10.0)\n", - "Requirement already satisfied: packaging>=20.9 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (24.2)\n", - "Requirement already satisfied: pyyaml>=5.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (6.0.2)\n" - ] - } - ], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jHEu-TjZ29PJ" - }, - "source": [ - "## Set Env Variables" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "QWd9rTysxsWO" - }, - "outputs": [], - "source": [ - "import litellm\n", - "import os\n", - "from getpass import getpass\n", - "\n", - "os.environ[\"ARIZE_SPACE_KEY\"] = getpass(\"Enter your Arize space key: \")\n", - "os.environ[\"ARIZE_API_KEY\"] = getpass(\"Enter your Arize API key: \")\n", - "os.environ['OPENAI_API_KEY']= getpass(\"Enter your OpenAI API key: \")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's run a completion call and see the traces in Arize" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hello! Nice to meet you, OpenAI. How can I assist you today?\n" - ] - } - ], - "source": [ - "# set arize as a callback, litellm will send the data to arize\n", - "litellm.callbacks = [\"arize\"]\n", - " \n", - "# openai call\n", - "response = litellm.completion(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n", - " ]\n", - ")\n", - "print(response.choices[0].message.content)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/logging_observability/LiteLLM_Langfuse.ipynb b/cookbook/logging_observability/LiteLLM_Langfuse.ipynb deleted file mode 100644 index 2a63666e069..00000000000 --- a/cookbook/logging_observability/LiteLLM_Langfuse.ipynb +++ /dev/null @@ -1,197 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "## Use LiteLLM with Langfuse\n", - "https://docs.litellm.ai/docs/observability/langfuse_integration" - ], - "metadata": { - "id": "4FbDOmcj2VkM" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Install Dependencies" - ], - "metadata": { - "id": "21W8Woog26Ns" - } - }, - { - "cell_type": "code", - "source": [ - "!pip install litellm langfuse" - ], - "metadata": { - "id": "xrjKLBxhxu2L" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Set Env Variables" - ], - "metadata": { - "id": "jHEu-TjZ29PJ" - } - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "id": "QWd9rTysxsWO" - }, - "outputs": [], - "source": [ - "import litellm\n", - "from litellm import completion\n", - "import os\n", - "\n", - "# from https://cloud.langfuse.com/\n", - "os.environ[\"LANGFUSE_PUBLIC_KEY\"] = \"\"\n", - "os.environ[\"LANGFUSE_SECRET_KEY\"] = \"\"\n", - "\n", - "\n", - "# OpenAI and Cohere keys\n", - "# You can use any of the litellm supported providers: https://docs.litellm.ai/docs/providers\n", - "os.environ['OPENAI_API_KEY']=\"\"\n", - "os.environ['COHERE_API_KEY']=\"\"\n" - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Set LangFuse as a callback for sending data\n", - "## OpenAI completion call" - ], - "metadata": { - "id": "NodQl0hp3Lma" - } - }, - { - "cell_type": "code", - "source": [ - "# set langfuse as a callback, litellm will send the data to langfuse\n", - "litellm.success_callback = [\"langfuse\"]\n", - "\n", - "# openai call\n", - "response = completion(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n", - " ]\n", - ")\n", - "\n", - "print(response)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "vNAuwJY1yp_F", - "outputId": "c3a71e26-13f5-4379-fac9-409290ba79bb" - }, - "execution_count": 8, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "{\n", - " \"id\": \"chatcmpl-85nP4xHdAP3jAcGneIguWATS9qdoO\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1696392238,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello! How can I assist you today?\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 15,\n", - " \"completion_tokens\": 9,\n", - " \"total_tokens\": 24\n", - " }\n", - "}\n" - ] - } - ] - }, - { - "cell_type": "code", - "source": [ - "# we set langfuse as a callback in the prev cell\n", - "# cohere call\n", - "response = completion(\n", - " model=\"command-nightly\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"Hi 👋 - i'm cohere\"}\n", - " ]\n", - ")\n", - "\n", - "print(response)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "2PMSLc_FziJL", - "outputId": "1c37605e-b406-4ffc-aafd-e1983489c6be" - }, - "execution_count": 9, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" Nice to meet you, Cohere! I'm excited to be meeting new members of the AI community\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-a14e903f-4608-4ceb-b996-8ebdf21360ca\",\n", - " \"created\": 1696392247.3313863,\n", - " \"model\": \"command-nightly\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 8,\n", - " \"completion_tokens\": 20,\n", - " \"total_tokens\": 28\n", - " }\n", - "}\n" - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/cookbook/logging_observability/LiteLLM_Lunary.ipynb b/cookbook/logging_observability/LiteLLM_Lunary.ipynb deleted file mode 100644 index 3b1dc5d5e25..00000000000 --- a/cookbook/logging_observability/LiteLLM_Lunary.ipynb +++ /dev/null @@ -1,348 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "4FbDOmcj2VkM" - }, - "source": [ - "## Use LiteLLM with Langfuse\n", - "https://docs.litellm.ai/docs/observability/langfuse_integration" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "21W8Woog26Ns" - }, - "source": [ - "## Install Dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "xrjKLBxhxu2L" - }, - "outputs": [], - "source": [ - "%pip install litellm lunary" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jHEu-TjZ29PJ" - }, - "source": [ - "## Set Env Variables" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "QWd9rTysxsWO" - }, - "outputs": [], - "source": [ - "import litellm\n", - "from litellm import completion\n", - "import os\n", - "\n", - "# from https://app.lunary.ai/\n", - "os.environ[\"LUNARY_PUBLIC_KEY\"] = \"\"\n", - "\n", - "\n", - "# LLM provider keys\n", - "# You can use any of the litellm supported providers: https://docs.litellm.ai/docs/providers\n", - "os.environ['OPENAI_API_KEY'] = \"\"\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NodQl0hp3Lma" - }, - "source": [ - "## Set Lunary as a callback for sending data\n", - "## OpenAI completion call" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "vNAuwJY1yp_F", - "outputId": "c3a71e26-13f5-4379-fac9-409290ba79bb" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIWykI0GiJSmYtXYuB8Z363kpIBm', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143276, model='gpt-3.5-turbo-0125', object='chat.completion', system_fingerprint='fp_86156a94a0', usage=Usage(completion_tokens=9, prompt_tokens=15, total_tokens=24))\n", - "\n", - "[Lunary] Add event: {\n", - " \"event\": \"start\",\n", - " \"type\": \"llm\",\n", - " \"name\": \"gpt-3.5-turbo\",\n", - " \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n", - " \"timestamp\": \"2024-02-28T18:01:15.188153+00:00\",\n", - " \"input\": {\n", - " \"role\": \"user\",\n", - " \"content\": \"Hi \\ud83d\\udc4b - i'm openai\"\n", - " },\n", - " \"extra\": {},\n", - " \"runtime\": \"litellm\",\n", - " \"metadata\": {}\n", - "}\n", - "\n", - "\n", - "[Lunary] Add event: {\n", - " \"event\": \"end\",\n", - " \"type\": \"llm\",\n", - " \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n", - " \"timestamp\": \"2024-02-28T18:01:16.846581+00:00\",\n", - " \"output\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello! How can I assist you today?\"\n", - " },\n", - " \"runtime\": \"litellm\",\n", - " \"tokensUsage\": {\n", - " \"completion\": 9,\n", - " \"prompt\": 15\n", - " }\n", - "}\n", - "\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "--- Logging error ---\n", - "Traceback (most recent call last):\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 537, in _make_request\n", - " response = conn.getresponse()\n", - " ^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connection.py\", line 466, in getresponse\n", - " httplib_response = super().getresponse()\n", - " ^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 1423, in getresponse\n", - " response.begin()\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 331, in begin\n", - " version, status, reason = self._read_status()\n", - " ^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 292, in _read_status\n", - " line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/socket.py\", line 707, in readinto\n", - " return self._sock.recv_into(b)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^\n", - "TimeoutError: timed out\n", - "\n", - "The above exception was the direct cause of the following exception:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 486, in send\n", - " resp = conn.urlopen(\n", - " ^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 847, in urlopen\n", - " retries = retries.increment(\n", - " ^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/retry.py\", line 470, in increment\n", - " raise reraise(type(error), error, _stacktrace)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/util.py\", line 39, in reraise\n", - " raise value\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 793, in urlopen\n", - " response = self._make_request(\n", - " ^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 539, in _make_request\n", - " self._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 370, in _raise_timeout\n", - " raise ReadTimeoutError(\n", - "urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 59, in send_batch\n", - " response = requests.post(\n", - " ^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 115, in post\n", - " return request(\"post\", url, data=data, json=json, **kwargs)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 59, in request\n", - " return session.request(method=method, url=url, **kwargs)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 589, in request\n", - " resp = self.send(prep, **send_kwargs)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 703, in send\n", - " r = adapter.send(request, **kwargs)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 532, in send\n", - " raise ReadTimeout(e, request=request)\n", - "requests.exceptions.ReadTimeout: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 1160, in emit\n", - " msg = self.format(record)\n", - " ^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 999, in format\n", - " return fmt.format(record)\n", - " ^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 703, in format\n", - " record.message = record.getMessage()\n", - " ^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 392, in getMessage\n", - " msg = msg % self.args\n", - " ~~~~^~~~~~~~~~~\n", - "TypeError: not all arguments converted during string formatting\n", - "Call stack:\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1030, in _bootstrap\n", - " self._bootstrap_inner()\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1073, in _bootstrap_inner\n", - " self.run()\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 24, in run\n", - " self.send_batch()\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 73, in send_batch\n", - " logging.error(\"[Lunary] Error sending events\", e)\n", - "Message: '[Lunary] Error sending events'\n", - "Arguments: (ReadTimeout(ReadTimeoutError(\"HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\")),)\n" - ] - } - ], - "source": [ - "# set langfuse as a callback, litellm will send the data to langfuse\n", - "litellm.success_callback = [\"lunary\"]\n", - "\n", - "# openai call\n", - "response = completion(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n", - " ]\n", - ")\n", - "\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Using LiteLLM with Lunary Templates\n", - "\n", - "You can use LiteLLM seamlessly with Lunary templates to manage your prompts and completions.\n", - "\n", - "Assuming you have created a template \"test-template\" with a variable \"question\", you can use it like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "2PMSLc_FziJL", - "outputId": "1c37605e-b406-4ffc-aafd-e1983489c6be" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIXegwpudg4YKnLB6pmpFGXqTHcH', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143318, model='gpt-4-0125-preview', object='chat.completion', system_fingerprint='fp_c8aa5a06d6', usage=Usage(completion_tokens=9, prompt_tokens=21, total_tokens=30))\n", - "\n", - "[Lunary] Add event: {\n", - " \"event\": \"start\",\n", - " \"type\": \"llm\",\n", - " \"name\": \"gpt-4-turbo-preview\",\n", - " \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n", - " \"timestamp\": \"2024-02-28T18:01:56.746249+00:00\",\n", - " \"input\": [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": \"You are an helpful assistant.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Hi! Hello!\"\n", - " }\n", - " ],\n", - " \"extra\": {\n", - " \"temperature\": 1,\n", - " \"max_tokens\": 100\n", - " },\n", - " \"runtime\": \"litellm\",\n", - " \"metadata\": {}\n", - "}\n", - "\n", - "\n", - "[Lunary] Add event: {\n", - " \"event\": \"end\",\n", - " \"type\": \"llm\",\n", - " \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n", - " \"timestamp\": \"2024-02-28T18:01:58.741244+00:00\",\n", - " \"output\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello! How can I assist you today?\"\n", - " },\n", - " \"runtime\": \"litellm\",\n", - " \"tokensUsage\": {\n", - " \"completion\": 9,\n", - " \"prompt\": 21\n", - " }\n", - "}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "import lunary\n", - "from litellm import completion\n", - "\n", - "template = lunary.render_template(\"test-template\", {\"question\": \"Hello!\"})\n", - "\n", - "response = completion(**template)\n", - "\n", - "print(response)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb b/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb deleted file mode 100644 index 0baaab3f49f..00000000000 --- a/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb +++ /dev/null @@ -1,252 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## LLM Ops Stack - LiteLLM Proxy + Langfuse \n", - "\n", - "This notebook demonstrates how to use LiteLLM Proxy with Langfuse \n", - "- Use LiteLLM Proxy for calling 100+ LLMs in OpenAI format\n", - "- Use Langfuse for viewing request / response traces \n", - "\n", - "\n", - "In this notebook we will setup LiteLLM Proxy to make requests to OpenAI, Anthropic, Bedrock and automatically log traces to Langfuse." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Setup LiteLLM Proxy\n", - "\n", - "### 1.1 Define .env variables \n", - "Define .env variables on the container that litellm proxy is running on.\n", - "```bash\n", - "## LLM API Keys\n", - "OPENAI_API_KEY=sk-proj-1234567890\n", - "ANTHROPIC_API_KEY=sk-ant-api03-1234567890\n", - "AWS_ACCESS_KEY_ID=1234567890\n", - "AWS_SECRET_ACCESS_KEY=1234567890\n", - "\n", - "## Langfuse Logging \n", - "LANGFUSE_PUBLIC_KEY=\"pk-lf-xxxx9\"\n", - "LANGFUSE_SECRET_KEY=\"sk-lf-xxxx9\"\n", - "LANGFUSE_HOST=\"https://us.cloud.langfuse.com\"\n", - "```\n", - "\n", - "\n", - "### 1.1 Setup LiteLLM Proxy Config yaml \n", - "```yaml\n", - "model_list:\n", - " - model_name: gpt-4o\n", - " litellm_params:\n", - " model: openai/gpt-4o\n", - " api_key: os.environ/OPENAI_API_KEY\n", - " - model_name: claude-3-5-sonnet-20241022\n", - " litellm_params:\n", - " model: anthropic/claude-3-5-sonnet-20241022\n", - " api_key: os.environ/ANTHROPIC_API_KEY\n", - " - model_name: us.amazon.nova-micro-v1:0\n", - " litellm_params:\n", - " model: bedrock/us.amazon.nova-micro-v1:0\n", - " aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID\n", - " aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY\n", - "\n", - "litellm_settings:\n", - " callbacks: [\"langfuse\"]\n", - "\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Make LLM Requests to LiteLLM Proxy\n", - "\n", - "Now we will make our first LLM request to LiteLLM Proxy" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.1 Setup Client Side Variables to point to LiteLLM Proxy\n", - "Set `LITELLM_PROXY_BASE_URL` to the base url of the LiteLLM Proxy and `LITELLM_VIRTUAL_KEY` to the virtual key you want to use for Authentication to LiteLLM Proxy. (Note: In this initial setup you can)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "LITELLM_PROXY_BASE_URL=\"http://0.0.0.0:4000\"\n", - "LITELLM_VIRTUAL_KEY=\"sk-oXXRa1xxxxxxxxxxx\"" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ChatCompletion(id='chatcmpl-B0sq6QkOKNMJ0dwP3x7OoMqk1jZcI', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='Langfuse is a platform designed to monitor, observe, and troubleshoot AI and large language model (LLM) applications. It provides features that help developers gain insights into how their AI systems are performing, make debugging easier, and optimize the deployment of models. Langfuse allows for tracking of model interactions, collecting telemetry, and visualizing data, which is crucial for understanding the behavior of AI models in production environments. This kind of tool is particularly useful for developers working with language models who need to ensure reliability and efficiency in their applications.', refusal=None, role='assistant', audio=None, function_call=None, tool_calls=None))], created=1739550502, model='gpt-4o-2024-08-06', object='chat.completion', service_tier='default', system_fingerprint='fp_523b9b6e5f', usage=CompletionUsage(completion_tokens=109, prompt_tokens=13, total_tokens=122, completion_tokens_details=CompletionTokensDetails(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0), prompt_tokens_details=PromptTokensDetails(audio_tokens=0, cached_tokens=0)))" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import openai\n", - "client = openai.OpenAI(\n", - " api_key=LITELLM_VIRTUAL_KEY,\n", - " base_url=LITELLM_PROXY_BASE_URL\n", - ")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"gpt-4o\",\n", - " messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what is Langfuse?\"\n", - " }\n", - " ],\n", - ")\n", - "\n", - "response" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.3 View Traces on Langfuse\n", - "LiteLLM will send the request / response, model, tokens (input + output), cost to Langfuse.\n", - "\n", - "![image_description](litellm_proxy_langfuse.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.4 Call Anthropic, Bedrock models \n", - "\n", - "Now we can call `us.amazon.nova-micro-v1:0` and `claude-3-5-sonnet-20241022` models defined on your config.yaml both in the OpenAI request / response format." - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ChatCompletion(id='chatcmpl-7756e509-e61f-4f5e-b5ae-b7a41013522a', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=\"Langfuse is an observability tool designed specifically for machine learning models and applications built with natural language processing (NLP) and large language models (LLMs). It focuses on providing detailed insights into how these models perform in real-world scenarios. Here are some key features and purposes of Langfuse:\\n\\n1. **Real-time Monitoring**: Langfuse allows developers to monitor the performance of their NLP and LLM applications in real time. This includes tracking the inputs and outputs of the models, as well as any errors or issues that arise during operation.\\n\\n2. **Error Tracking**: It helps in identifying and tracking errors in the models' outputs. By analyzing incorrect or unexpected responses, developers can pinpoint where and why errors occur, facilitating more effective debugging and improvement.\\n\\n3. **Performance Metrics**: Langfuse provides various performance metrics, such as latency, throughput, and error rates. These metrics help developers understand how well their models are performing under different conditions and workloads.\\n\\n4. **Traceability**: It offers detailed traceability of requests and responses, allowing developers to follow the path of a request through the system and see how it is processed by the model at each step.\\n\\n5. **User Feedback Integration**: Langfuse can integrate user feedback to provide context for model outputs. This helps in understanding how real users are interacting with the model and how its outputs align with user expectations.\\n\\n6. **Customizable Dashboards**: Users can create custom dashboards to visualize the data collected by Langfuse. These dashboards can be tailored to highlight the most important metrics and insights for a specific application or team.\\n\\n7. **Alerting and Notifications**: It can set up alerts for specific conditions or errors, notifying developers when something goes wrong or when performance metrics fall outside of acceptable ranges.\\n\\nBy providing comprehensive observability for NLP and LLM applications, Langfuse helps developers to build more reliable, accurate, and user-friendly models and services.\", refusal=None, role='assistant', audio=None, function_call=None, tool_calls=None))], created=1739554005, model='us.amazon.nova-micro-v1:0', object='chat.completion', service_tier=None, system_fingerprint=None, usage=CompletionUsage(completion_tokens=380, prompt_tokens=5, total_tokens=385, completion_tokens_details=None, prompt_tokens_details=None))" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import openai\n", - "client = openai.OpenAI(\n", - " api_key=LITELLM_VIRTUAL_KEY,\n", - " base_url=LITELLM_PROXY_BASE_URL\n", - ")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"us.amazon.nova-micro-v1:0\",\n", - " messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what is Langfuse?\"\n", - " }\n", - " ],\n", - ")\n", - "\n", - "response" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Advanced - Set Langfuse Trace ID, Tags, Metadata \n", - "\n", - "Here is an example of how you can set Langfuse specific params on your client side request. See full list of supported langfuse params [here](https://docs.litellm.ai/docs/observability/langfuse_integration)\n", - "\n", - "You can view the logged trace of this request [here](https://us.cloud.langfuse.com/project/clvlhdfat0007vwb74m9lvfvi/traces/567890?timestamp=2025-02-14T17%3A30%3A26.709Z)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ChatCompletion(id='chatcmpl-789babd5-c064-4939-9093-46e4cd2e208a', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=\"Langfuse is an observability platform designed specifically for monitoring and improving the performance of natural language processing (NLP) models and applications. It provides developers with tools to track, analyze, and optimize how their language models interact with users and handle natural language inputs.\\n\\nHere are some key features and benefits of Langfuse:\\n\\n1. **Real-Time Monitoring**: Langfuse allows developers to monitor their NLP applications in real time. This includes tracking user interactions, model responses, and overall performance metrics.\\n\\n2. **Error Tracking**: It helps in identifying and tracking errors in the model's responses. This can include incorrect, irrelevant, or unsafe outputs.\\n\\n3. **User Feedback Integration**: Langfuse enables the collection of user feedback directly within the platform. This feedback can be used to identify areas for improvement in the model's performance.\\n\\n4. **Performance Metrics**: The platform provides detailed metrics and analytics on model performance, including latency, throughput, and accuracy.\\n\\n5. **Alerts and Notifications**: Developers can set up alerts to notify them of any significant issues or anomalies in model performance.\\n\\n6. **Debugging Tools**: Langfuse offers tools to help developers debug and refine their models by providing insights into how the model processes different types of inputs.\\n\\n7. **Integration with Development Workflows**: It integrates seamlessly with various development environments and CI/CD pipelines, making it easier to incorporate observability into the development process.\\n\\n8. **Customizable Dashboards**: Users can create custom dashboards to visualize the data in a way that best suits their needs.\\n\\nLangfuse aims to help developers build more reliable, accurate, and user-friendly NLP applications by providing them with the tools to observe and improve how their models perform in real-world scenarios.\", refusal=None, role='assistant', audio=None, function_call=None, tool_calls=None))], created=1739554281, model='us.amazon.nova-micro-v1:0', object='chat.completion', service_tier=None, system_fingerprint=None, usage=CompletionUsage(completion_tokens=346, prompt_tokens=5, total_tokens=351, completion_tokens_details=None, prompt_tokens_details=None))" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import openai\n", - "client = openai.OpenAI(\n", - " api_key=LITELLM_VIRTUAL_KEY,\n", - " base_url=LITELLM_PROXY_BASE_URL\n", - ")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"us.amazon.nova-micro-v1:0\",\n", - " messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what is Langfuse?\"\n", - " }\n", - " ],\n", - " extra_body={\n", - " \"metadata\": {\n", - " \"generation_id\": \"1234567890\",\n", - " \"trace_id\": \"567890\",\n", - " \"trace_user_id\": \"user_1234567890\",\n", - " \"tags\": [\"tag1\", \"tag2\"]\n", - " }\n", - " }\n", - ")\n", - "\n", - "response" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## " - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/logging_observability/litellm_proxy_langfuse.png b/cookbook/logging_observability/litellm_proxy_langfuse.png deleted file mode 100644 index 6b0691e6a5f1bf8e256a42adece0999f8e962a72..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 315558 zcmb4r2RxhY{x?cnZIzbRgBq>9TYHb%)ZUvCGsGsgQc8{5Gq$4kN^DBa*qablwO5Hz z`%Rzcob#ObocI54-uOuF-1oJA^ZH%iU&7T?{C$6}XOt<^CKCYtsx1 zODF{ki_|HjQC$T6AOxVNV5O{##e%-RjfIE(5bFl|3LE{0g-wq2t2Oi$mLfLAzpgc~ znf_4*2Ma6A77OrKd+0A- zf8aaG>$_rMJ*NNlkFB6V_X7(HXUJAd&rMJHm5>F2(I7M6&Y5c<*q z;AT$aFLSg$;|p!N2o*?J1 zC!8-ho^$@cssX%g|A%V7p8Qqq&w2gToyf1og!DB4t{|ZMuVIP3;NcPZvy1=h?%xCa ztH$3Iv>k2T#JK)ZqV9jTkPT`x9#y8K;R74V3*ti(Hc}NDmu^AQ= zoFlM_!NR#s6Ntf^gv=xPFGZZ;>ovJ&F7uex!NMtEB=X27lTBi7VGI_1ib+4{p(IpE zh{y(o%TWUch+Iw*FE3tZ`r_++)4>tplllDw-x<(szk8;BQg07bn_Y;2P}9P<$vJ^T z4>4LAB8m3fU%1ZX*boV~wj;dfm$q59z;A^?Q?wWiUhM&HL=;ft!F{1O7NtdN+U>nu z4@8=HB!55XKM7a#1Sd99@u5O`^#s#i59z?iTq@5>R1OtJ1AoYm^0a1SyQNzZ=iE?~cRL-Va)=a7T{C$K=#^>lPZ%i18s7M6o5r8gDLtI5RsTeI5 zL%0|XmPmpoGSG*Iqz5@@Dm0bqgdmPCr%wPVX=S? zksGD5m@cvij*y+AepH5r&UxK%=0Y1}?rbbAY0_V5>tcowBVqWWO0z^;jw`6uWGYNB z#`J(aW#;WCm-)WH` z7fvEnFYU_Jz(d!-QInIHN&cywX6qeM(LIfc# zTv|~<7$bI;zzVviA*8*0%6yMRr_we=P$?j)nn8;+a48odJNPZ} ze)RKYV-7n8KDXsx6S12{tk@K`?(lwiEyXw-L;N!oc7!bU6sCx-+To!a?-Dsn z>oH%gN(RrkTP3~0P)U8j!Y0Ta;EJgjTsB6fI8yQPSV*xv5@ORF!-`lyszk+@Rz3a0 z&=xW4EnG+2JHbaV%opQ2ASD-@gXt_2Ac4_*sDRad^xQaw?onh`<&X*2nWyDiwLY-j zu7=Ecc-U>_K_1%>b%m-wTM(E#M*8m@du$qmI-f(4(3&HUPcW^<36XNH7$N8jO-$L% z<>`+s!3f5|WkJ-4A-`?aQ!?SBP*b3v`4;3F-DRVzvYV^=TL938zR`Fa=m3LhHg$8{ zN}qFjWjA`0%<2=&pb;)YPBQrdW6{sTDH${;;3aI48IR7D8emVGbBZMRjGO+LmvqnD zka}P11`A*QL&rRM1m0GUBd8DxF}}}NXP(MaBbfvi0Z`pay@iLNlF$Bj`@-!0ks?aZ zl1gwjr@$x_0DM>QgDCHMI@EYO33;_$?U|-g8x(3;n_)>6lP;%hoQ)~Hv;=~pR^Q!9 z?YSi*=P1PumVqNGhVKwt^k3w^|Ul454TimEt2$ zBD13E9Fzimmd13|ic)aWE(K;qKBe{0p?-u~zS7p}qWFm4z*BvhgT)+1} zT}j^l=}BEL=h@f|tz>@E^u%$~0fg3aO)V`gab>U_pc7+__t9pb{d75HVrHg)n!^_o zOi6!$Z)t7ard?su5|qvlnoyIiaF5NMPDGnS zvcZH}yo5%b-6YE@QWL~TPfz)MClG*X^ZlG^YHB~Ejy5OHK?{xUhNH8qB+9PuKjKln z^&PG@u0`7&-93WpnMVbp%4y;OSIoEo#dOFehQ9fVoJ0+6+S3l=bS`TA24o6}Ehp{ir+5>M!QSi5R4rpSqANjM0hdK7(2b-iXM3yOGgUK_^ z;e%OvIr6Tot;da$_3e>Rd~LK@gVksvhEyO3&eZ6mW2GA<+9e)JMUcw-7;>fAG|@~o z0Pf@}{84=s*aEYth8b6DqFGY-2xr!#)tka-U%TFl=lknmj&i%jQb&ruV<@$7odUJ; zRX(Nvc9r`pwbeVO6$6+msqHbGG1mSPThDL4DnFdUJ7HqRePAiwO&SCvuUjI$2=7_+uhGZJd$ znuR}#W6FyW>>n(-=+oF=Sm#J4*FW;q>rT$bA8RrO6(X&m$n@`3JKHUJz;BENaxqkl zGl{P-6MZzTo$elP&uZ4trwKamnrZP=z@Z`V5-$XOLnoI9!1k4Uo5JKLZQ*<#k8Uwc z*OVnwB8r@s$1pc2r5<3EF`}qm_*oi=!#Lw@xaVMy3e)cHqerG=l~kwAuxPbKA>c4?c~8kmlj z=r$eENu&agd8%0-#ZKo!4BDaGnPBi9dv!i>lYo5t@%p`DDGZH*MEe~Qz$=ZxYAY4s zpOl|i)!w#30t~ah-*1@dmsUB#s-yd*$laINt})!7-KHVZ%a)BIPkXTsP{DykH0x;c ze~wsCs60Y?gMM^#_!O#iFp7${M8R$({eMv1($dnVTN=e+z%!gnm@daWqNsUrhq3kS zB6sCAag%w~Bm7T4D7{=|VIt4q$YPxg)Mh;6;QJ~c;AX2)9kuNy+3HU6A`7=i;Vm^Y>`6EqS~2rakFXaOSE@*`bp{aGELiTRTVY~+b|fsrQU5ig{#ePHp12o2oV=ak+xJ2; zHNuEzK2I|1$r?L^(U#$Pd}b0GbKkM3QS5NE1tv}oja^VZKzZIg)#=xEWK+}*ZLXo3 z2$SR%cMoHu!aO-~R#5AyqtSUS#P+AvK#DE^0WIS)YX?BnY{~8bhldik(Nl=S?=D=T z_RBA1P_k+~cc1{?-Nb~KinMT$XtR|$V=Fm2I|tb7JKt2=`qbnja1U}q8Fb6>o}BqfCBcg6CC1q}cv`OW^};n;ylEyqEv*(~4?;qT30JA1Z%Jmhms z$7zb}#5+1Ri(qz-O)(;fC-plYNK*?{2DHXuXJSFD92IbNnq1K)8GN&MzlVC+a4I#N znDMlOOYX_2q4$x{7m#25+sEu_@ZQ)Mfk8LZ5I{#by;aLi0MudlQH#T9xdsnO{+bbe zeEX})wTrxc@!l|E{|^)u0>(nMQ6jfVOT{#5gZ{khv7tI6fKswIw4?m#(7 zLKTbmkoRuFHyl2J9Q)h-Bqzl{UXjJb9kLbQ-zePM%wLBx>er+kzZ)0R!f};mX3!f< z#7*=@;@E7PrwdqwYBZr10!^pt-@&QE2M%+j8*-Kz)am$Va48>ejOiQH+K?Z7&9kMb z&ckAL4%PKow6 zt3t#S8#tIga}mC(?uLwhVgXXHC`3YR4cYOG0GuH+fGgvw*NGQRz_&d1b9&_bj(T=$ zEnw8+Q&h#N2HZ%?aT@9#ev+P?#W~w>e6+i`kkNqp-nbRki?^2b)H=gI;<`V#Ag7b+ zLsr10v6$BhKtW0A?hJ$83aD-HL(ud*i>e5p2vCn94f^?*^pgp}m zI^f$S`Te72@9E2ft$PZ5IguO?VMUF&%Zp>^TwN0Jsc+li>Kd2p?wfa#S>*m@VCsWq zp4ZRGe`8$Kx^aTaE4LE$PH1PI&hafx`5-pL>Y_Dn;((e_u}}Qx^$&h(awcvft%pzH zvwMQ1CdA2?R%^KT`)$?`Fo)z>hUa;ov&{heaRprVQ$~FR59`qpG;k>)K%FXv|3Obe z@7EAfq*q(0LtE^tBT>ZpI=AKE-1ReClMMvj*{S11Wi(IG-gRXW?bA?v_yb^#~ejwIJ=dR=q*ZD z6L9)%32Y0!tAkAWOwaPYBBrbKSO_?{Bov2J*Ic_CKOc|$bg>0@TO!pv2pGI#+`4a%dri^91G+t$Q|7iUnOgnE|yAc18kjibH>5JO2OyxS#<@S~~7F=)N zb@^!?hATSMx2T#3L;9Oeku3(_-{k0d;+|4#JI?YoHN>vuoZ{q)(d8h#rw{v<8wzoC zRJGR^nHtdRK7H?lY}Dp?SH_Z8= z>zXY&^f+jAJBHNMja{qOQ?x$#cumjcdG*kCwl;Qq@~M&=U4{f;nZeb* z8BYliyCFXQ#IwXI%5r$Y0ByMsvuK0P3V*W`3c|fzk7C%J4~Lwn+!g6Dnve5lzl5Pt#k^{@Z7KCDq#w=(B|EQb{8*nITbVF()y)MD7yQ%(= z=8Jv`uASt?0+W_Tl}kEe4)1S$q$|sWR~33@3`U4h0*c-(P-Ohk7XR5>fD7v2d*(US z3)1ltxZlbADlFjSjm5*2Ch*T*#+9Hj+4RbR)`NW?i+S`p+;p}~mL{jm6y>}{ThU8C zY+*KYfGr;2@cE9t&RTFw=rm}CiSeXK@8fVr-k`6{B4JkZb#*Ohx?8vKbC`NwPwz+* z)xqR!YK^M6zU=+BL2!lW;f(9SB>y5p+vRf5Qy4|AbGAru!}ln5;D@hVE{S2fw*53b z1tykkz*_>ZU0mxQZoA%a(<)F5p`fn&nNk6+Di5!xf@;e-lbiAPzFqCoYR&k^kD7tXaGEQP`FGou~e}D|KY~^koBV zlrKSpXvX5?_~5{mP-!tf#{JmCYmaKEdTKTn_lh^K^-vt;!g$Z=@IhE_e9Sz%Cqawk zT%7~GxbmekN6P_2ANLZ0AqUy>8T|eyU_Di`lWy*V)e_ZSvP;zu+E#RtHJneF5Ccqi z4BV(;NVhufiqF`_Q$4#)b*`JamvVB3@A5CimI+VSm-n%$S#?Vfwq8$us#8(>=v`-b z&{YWP{~k`=7r8;ak`|QQe@IpXSg&02-*J}=<|2%$*Yr>2GVzB9NmBXzq?&C8yf+g1 zdRByh4%t0DcSjhCU0$%ZWsmVJm-}UcyO|~B7)K#8cHhLF6ExeT0%f0Ux^ChP`BnzZ zOOg>xOLLPeSBfcVn5IdRi4HkcENy2z5nROW;bF?l>%7;7VgLibQ;waF+<=^P_FZzu z^4d?Is_5T%&c~GZIb~SfOPi`W;NAN9avlF-@{5Y*mD(1^uU$i}JE1Y+0t5PDOZA-& z+Fvv?_udu1cC0sFwqj%&ABwyhTkE~Cl27X+;ODRamCE!z^#+c3MYHg#5DqmdfHSH-zR*<9Xd7v6b7!rm&<^Zu6G4 zZvRz#_`ToQy%jYYwc{@o&wi}xHp3sT=#i*%*9X7o@}P1D0f}lP2)McW6d0e--yL)? zMfzQHZ;Z4?)okxN^;X6Xy}cYiKZ-4sBRE+|+o;uyNtTLUXU+1xifWJ*xAhbs5n)*)6cjq}xB+}j&)+0|<*U%|9U}x1Se@qax?q8^KuDnaJA2Ly9SUBUq z&_mScn@N^P8;2Iq_`#wl!kiDo=B&>4YBJCQ%cI7lmoFnFJ$=9NrP$TayQhaW6)Td& zke_U4IqtFm&9JgL>b57xr_Skr9ErhzY zy{Ijej$m!EdBY`i5cwFyv@v6G-_?F`c1?=8;I9@N^Bg>Ca>l;?~zZs@O&!Q zuU@XG3V!@`-cQcmsfvmS(Z%qirC~5S=_xMPV|IbvVhq5PDH$QKKG&f+@3XgH?>nQHYfXAHd7o`o~BP}lH7#SJ+zOt&smvD;6c)M|4 z+%v4QgAhaedgV{<_S=X<-1%8zPrfd6peWWm*}<r2m^z?#=7AoXGEc3u{HtBzDezm!XRt-Vm(d~@YJ!JfZ~XE+!=o)VrwYI| zAIBWBI(|f){3^epTW8)WYpoCJ{Lap6Kjrj^&tZ;jEI$_H9<|wpm=h{2t0)Mi?SN^I z$Eyg;g~YTvId<{bPd#DG&!ZiptDeF7W#SL;tlSR&c*7N2qWW{wuyEhObDncA;ILsa z6#l;BIoit^#80%%F1;@9=|fp`r^SX?_3j$qj}Jaxl)EqbT|#iRND2T_HG@~w2(bO| z?ZJK_Pf9MJprlMt&o?~pxN$Oz0vtEXtR@*zNP`AzXXihu$b~y6^KS78p@t-8LXhO+5Z&!;X_(He%vcn|8FfAm&jWWSrl7O&C-OA_^jb7jeC1IAa&4!*AQ zXPk--zT-0%hLTc-Gjz>2hX3aLLPC)A2O-ZFr^=x6O}woMLr`+dxLH`v&NBy!Qo>EN z+Yjs-Qr8rt5Pz|50!@401Qj_RfsFYrwydm7Bb^=l4(HPMgqO&S#V>Z#0qq5AGasEH zAJM@fHW=j{cOoD6`A+>7v(L|L7hCI0#|2$i1>p{e+JllTeo)Y2HOEZj!W`_#(0Xej z5jwO0-DCF4sEkBL*v<{N2@dYOLD^}Zzq4mA0>6RXX%`h{tg=x7V=&PukPxs{5bI_wXU$VAt}nA?P6mT0pQI3UL# zswzaDDjR)f1DVnruby%go4FxCGxhpZzN-#ZpC2{3XPXV4MHyyHHV#o3)FK)ROQy?C z*yp}RR&?;z`1wFpb;JnkRA7m%?!e}{RJ`}G$+;`>5-&Eo~y@D4+|OVlves1SP@ zqSo!>ah5#dc)Ou(eVuPF;79R=jfkqimHg-f>yAWuj!$7*ooJsbqXgS>gez~oQb zTz*?PgS9{dfjMo|M}n2;t)aUU`FfDu^IwLB3a(q8xM+3BzT9@M4+YLL#I*e=k8Je$ zU}^Q(|K?*v%AMblMW-DiIC`4Wabt8IBXt!0^L4k9+4yS5 z=v`pIrEneqKZ$qFj5yae!bix8tY@6k78y%E+E->yoA`P^6_cdj&lW^Mb3`x)xQ6}uRh#2 zVounLM@rzr*J?jAh%GGGu!rYz{jLWhcRkMO;h26n@=ln3a^9}T@R@B%P=@R91|xB6 z+V{iBw!LtqBMwIo{L4)yH%43lj)qe9^bh}qV-NDJ+b@v0dyo9~N{wMC58=76)_?;P zN=C}sd#k_@)M$VVKna?P&k716%PqI2%DOfcH9n(^AGY+uATOGnNIr>$QPPMWBb0z@ zvOXy=n_^QrGO~?anCn>X)5oAuLqXB&Gsc5K!nd@lbq8Pki_8TTT=73E34{;%0^U@;@nD=9QEpXIqdWdP`k${OewM+Iyxv>&YNS+_eadLhGL8Z7=nlywLuw&hp$(V*9cTeh}dcxqd zTb1tcaGz&3#C+_P8AFfJllxkY9Jy; zsdg+HGw^yRTC5RPam}qFF0m-);Gmp?GNHzQQU6-|$a=Ebp&QDH}_JmJ9Yo*>%&`kjD47TVrZeVBmw) z__n{WGs)mk@2W}01UzDo9gjkd}9=$-#&bNnH zxd7Zp?JK0xA6M&!u@!>%sEk*_vaX+Jjh6sMMi?z&Q-k#UoT{rNc;I8;Vn2U%)K1p# zBBCqO?dkkg`U&kzL0@k{e#8P;rXO8={qV?28RjPh&!gRS;#l313_$I~#3h@02}jq~ zYJoRq!@l>!hh5*iXj8eCYCT)^VPtac!?W(S-fMg2{)$FcN!Gj3_vFaOMfRFF95lD(?7m$aU`Gbo@G55X(?edYSu!#Sfl6|hwU4l_%?4{o-Y7+gY`)wdb2tJ&KgX+ zOm6;pV5stktXu+k?Wl0&35v=kIkpoNe^vN6)ME9jPU10}!8a4*oHh6sYDb3Pk-QSi zof16zJeY|P>U$^sV8|j5OK1$r>*!}k}`S1tmVs4 z@`O>o^W6o>U2@+XkR_ReX@v@V1Nf`{nC%JQkQmSHJV`Q#fLD9FE<1qB+EmTzH2x$+ zxnK|e_Ii~~pj)li; z_1Xr)VNbmwGfplvI6R!CW40?*HeTqo_sAikS`j@#9`oo&Rm zmR8u%1Gs1|Yx`*xmj$P;Sl<27%4u}rM&mVKTHx!}34WMW_0x2x%ZkNH{e!y)kOzQ{ zx=Dm^8BawEODdVU+`G{SfT?l>&EzK?XW6kk)BIb%Ia*{=!glp@5U&X!z+Y!4b4xSk znSAhaAd#z&myrD}Z79hUo&ksap|%G_wsUl7Kz~f2N&e*B-5kZv(Ac*rwG|A?ub zVXpR=f)cRMcztqOJ{>ppG?y)esuUt)0e0(W7Z(@B zcvjLMy5Lg^TdPOP+TZ#^@J;)2Wt0Z)mnto_t`0%g2DV#$&Y3(j>Gbv4S9)m)V_byO z7!7DR#Z9YS59k?Hr7RK)^9$&dybY@J=@2lT6+wQC@(hBUaI7p-cRxmjV)H#xZlQ(S zkTxQt=`0HQB+Lr&ia*8@w5`h;G{fK4jZWILbRl9mny8_RA*%RwbQ$&Y+Y+60+FWt; zow!vl%4Q6l6%q%iyL7{By7_UhW#I}cDsc%3bPBwZWU_I^^21a;5&t_={d$T--Vb{(}*s-=F8BF+Pe4 z?dUknR}SRY$VKz|JX{Z{wwq5M6h~fUK_B^BkA$Zic zTj!kOxz{dA9ag@9gV9jXr|Chjadsmkd|LIS%Fs~24!#56hQppQbg&NDG}Ht{aAAy_ zqR9y4_j#SXkO#M+h9Zxxlka{Lt&Uv4#uQqOjoXigbh9n9^C=LQip_X~eDI`Tx?uaA5$f(A~ne1N-= zWcM30`ZpBe0vA}0j;&qj+%$MMjfOM#G!mZpX8UP`VLb3=y!q5eC(Vog&+y^v-IBsg zn`PNWQPA;IHTt-#mkz_fgIxS`EJ4OfG;z%e(x-{?9bB;^jh#HZr}EGO0t^@8SE7!S zwaz@s|MKG*`za^xLQzqwafPPv-lEh}92y50G^d97qy(_j#qkrhN5!_ig^w1PnN4Wn zA;M}HWoG?JPqkH?95fw0o%q1S3c?Qd)Jt0Zw~)XXA_Je52!WZeM{nX zj$tFVEsTYh%&THsW4h96>bO8;K_J>Q@7L^h{tYPOFG<0|X(|sSXiCtdnLtC+AYRGA zJ4OhZIX*pVj4`M*M8y*M`ic2*XqiRE`iRX0$&NA1KrybrJ%(^+7P-Qd33lIBS*c%J5DlF}E=2zpZ}hL#)S}Tseu1bv4>zs& zB6=PsIeS{JIe2wXJC91v9bt~}mJgB8YWRg1r*y-2Gs~H3(iB8T2*q=to-m95ADx|Z zz_>dQ;e%XIjT2^DHhHkqO!ba^7i`#rX(9_&ko28;Zq{9Jn_o98+w!#Shqu|c9^{Ko z{?pk1tvddPD=$371WD$>%E*&;eN3}dC9<{fG?dIiX{*fYxrHRrvBsFD9B4fWN~vL@ zr_}Fmu{iPV%2}5G&HnOVlw*kx6z=?%={ck>m&B~IA$YD9l!h~XybD$(Wy8=2ptFft zaMH}Hx7(HhhV;f< z`!W?v6agcI$G@e=@cwB`TJss(k1;0L7pZoL?fzh4==$t5lXSYCE`9Z$UQGDTKb@lg zT@cZJXQ9M=HA7W2o2Hn)0L%ZS3tt;Ndil@a0{GvY@|U+m&`!e_tEalQag2_(nIC*! z(_<{k>XzEMX!=vWaUKqi{vVj~zt~9vOyXp16qic?C#fIm72W%EU#LXbV^73JO6R}I z{!eY9)4G!`!iASLh;|%O=FLe@d;=}=?)+}K{#;g;Y6(fnpzkYdN|LSet>F~wTm@=0 z%*+aL$}TU%m=zRM9vgGmzLff5P6HSmm#j-;Tih3y8LL5Zo0;*J?*02xu59FV{X>KC+ZiGX3JP%(2?;4N!1HLM zzM-MmO>M4xt4X)z%x8UR-8s^%M333gM}2dU5zO=vOL*Y@w%GJ|&>QkTjl56zz%S-s z5)@x9dy@CZrNwQF9E{(!L_~}9y>KkHMW0l(CFPL1)n+;%T>w52@FfZBDfQ2oi}$$}1_aPASp*b{{6a z+APY#>-xK z@>)qn#UswCxcDuqDH_ZA_n`%?C)lnH8R{zE4&4e;w!UwYefpkW`HFlgUrid{FuAw# z)kug+#}GqAbdBo)9V7oRUBex^@p;ZtJK41Sf&u}MFM*euB1iUTT7su5e{AgtHdMPC zmeY|L`qdOt*kh0nB60eF>G=(%lot<}oQc%rj_zr@(GVWs=Y%{nu{N#C`E0!K_AY$p zzB}VPxVSjBx)N*oN5pYCbh6n_TB#(APc-Wk8*Abngk3Dhr+~B7 zyt-rK$`XC^g;BpjKm1cEB_v7IS4Mtx1j=`dni}+wPG15SV_Csgf8@>aa^d%V(dIiC z;>pM-Bp57RtnCz`?z6JYa^mdlJuk#i!xAad7%6_f38+PH%8SI{3qq8QHi+XVeez04 zq3)lOo0bue2zx`VrX1#3450v}!~q@!E0GH0{epJE-^bUSiQoMc6M3**)e%bfOaSJ^ zgYTJMY;zK;6_Fa`WS*+lQ(p1j<$BKP4HrYCdYxk0#HS>(OgjWnXru_pDb8LgQlzGC|SY|MY2bPuX4!F zN#>Hn6X@88$HSA3WrcF@Nv4kJ#5e}{2K^`gU|KOW7(3lg3ANkbS273 zdnqZB?X1}q`hGiealOMe&CMeEb#{phiKtlHtYqGiK zb5PG6V*T8Ge0JeSoB09X)KB*t_t=L)RZV|)Dmg0u%6aecNtK&4qj>}Crr(z@GrF7F zQk-FXz3z@JmYsjR@ca^rPl`q}a)L_MU6WED%)ZT)x%;@6_eRAL8u3?{K}Txr9bO}{ z(!+c(S5hH}y1qKE&?+-f9uhkNnj*&18h*^*^=<06TTrA5IKW>#SdJ0l(DXm+gI%0a zSQL0GI})$nn3b_+@O>ef>32r9I7>+8-Q*^o-2`gjv6?E?rzI47J~lhq+;;Vtz%R7-*mUlE#A#uSUJ3B|;&e!6s5^>6NeSieFq-=oofRnUE8xB4Gny+?J#tuMyUbua zsxjVOmhU{?{lL@=+2~%v_R}Y5;CxGUP0s+@AIG;dCy$0i(B4)J}%R(WD)jL1B!l$$k)J^A87q^(iy zE;?^EIg(|cprC8{h-foK$rW-lL5IWC1ehG|gxbx|Kfl^*ySA)4RUN%(TT{?*iLeCJ z@$A9biVfQ`>CTdYdktRba}L``-?RO~*3H7K0NFG?hfPC{lGMe3bK}e>MjD20ZUKYwBmPZx;Uq?F67t!;)z1 z-|PO$wzHcmHwM0?I$1BwOsZh?-GJ@ZFUKUZmN$-7sE~d;A}h<;>UvKKe~1FNId{sg zS*zHiYD(1O86I2@>{Dg5nnE4+`4uKXBRs%$2bS=k^PX>hz;$~&-rlv`&+XEv(NVTd zONZALz_g8K3B?aexw zdtQQnWStuKC%q#6QECV)H@+t@Io$dPt|K5me66ox6Pq~M?Nu9j`UGdU=!uq{o-}-b zjN5YCz5f|R6ZX9kKa}xra6bU(JX50gimAk5Tl>D6 z{zImfr<52$EzR1_tkWnpVaUa9TBCp1prph;dz;leXvTQ;@+jG76ntat`}d!lZWGDX ztv0K|e^?QrMb*T2G>ly+G6QDg^u_opLwMQ<*#E#UOY39q+F?h{a9?4T@8iQ-Z)%T} zd-dKDz1a~aPJT|`Z@b1?>x}qwG=6_A*oS-e#&2j{ranwgj7hWb)!n(Un^$fLrDlHX zoD3D$i(3b)GyHFktFEzc^+pAQ-uXPITfa|S`AnRX-?HNL^wedRKKU(h*_dot%|OJ2 z5V_2{wQ_<4>zeqTPQDI%;Wnc-3-fv1c1g9>x;BtXOB)tL)ba;e(F9jc7{#qcCLhlQ z;gL;^Vz{T@texA6?d_z3+$H)PUC>raPYY zjKQF*EMng(cVuz&j6g^8o|I_CbXwqwd1hb%jC!wH%=2W?C!pQmGyop5AcgB>m4emV zdoI_1bDBw`g>Z|0dciB$lOGKjjy~?*0)<@Ymt$X0I6h%38>r1CrPP*{;(|u{Ic#$f zCxKTAbThJFSr9u3U+1C<{1?u+uHFv=I(~Alq>c{v28rH}o5RNOo{IQy2m}eKUXf`e zUx2uh^r*jKt!FY0wI?Y6fyeqDiV-HZ^A zhd^lS>@Bq}h-bToB(k{ub`=foimT5z|2y-f8G$GRvxvcng3H0{z^$1gyeND|ETC?Gx7 z^CrO)5-z4Z#0Fk_?1?zhk7wD){%@_q(&PaU3*9;ZuW#@NJTB``%17!GzTolsQk;)EMMZ^*TL`dld}6^w&*GA3s#M zwmA8nJ4k(pz>xb3Lodp2nMw+$A1<79MH@*4U#e_zZlnr#CZbi`B|PNSO`{hymTIzu z>pj%|36i>m323EPGqc}iQ00F)WJuLL7Zx43Owe!hA;exK{d zrxK}~0JlO=%E)&$)Y;Szll&Dlc;oI*ZkDkf3x`5JRdL=qvcX=83Fnhy+680RNt#g! z-R#_oKErEI*LI2bO_zDA)&)21prFOw^813<=+o%1ooPhS{c+*dt`|eSPj|B;VGtkv ztcs_wdGcfH(`TqKGC#_7YhxKs53N-0VX<*f7NN$0ay5nkT>+8(cF7yRe&%4zJmx;v zk2(8{QC!}i=kVIE*qg?m!(65RBN#wKjxGFY;NmnKzkkNU`HbJR)aBTk)w8PDzv>lr zmTF4M5AJEdnM-;*>(8=D&V)+KnW_`cU+Nao`&N{4zh!ZpdbK(cmH#sDsq^Y9Fv2%} zBqo!h?c_IXvcyi;E8?YVuYXU7f4ZX<5~}YY@_l2;TDS zM}+hIMypn&+S4t(yZ`gvWcJsnX+n;v6Uo(F zXVmK8fc1XcLFb8Zr|_Ae;Lk>imBeY&D7ekt>q}V9Olw}5oSdI8UMmF?=5Z5QGbK< ztl*l#tvO1)(GM?KQ3ToJ!*gUKPN22S>FEvba)d?WG+z3&;nPhU=iNp6H+6MKXIWQG zyUpW!Yjx%0i?`-d5iiOO+dhXd`X37uUY3?<7AB#+^j2v49bWes77I{IKKj=d@8rSP zndHey#^WX#;Hh%~X(QI|BhS`H3Z(&(X;c{_v7fhSCz+X?}FRp!h#CplmI?0cn zgxLGzSs4O)v|}ijHKxu89KCB2fTnFNf7TN8RXW$?kdS6ew)^t6V?x?$&Ftfr+dmId zbhmSTpYPrnHRPJ5-d;?{iV|kZ>p5ueJui^VlO5taQ))-8>Znuu{fMoIao%g;w9fJ@ zpE0y4(}OnasjZg#>?U!XyX`yVoueJnp|6FZ)~jg`D350>2u;)bxoXVE8QL_#>s4U| z>cKbt-H~ruCO?5oa!sOpww`W67`%e+AOm)L$>0z!rVCJiYwRAuMYd z0hxOqJmk@SUf-pL)m@iCb$`gSaQnIxCxYKYSca zKi_nB%Mo^@*gO$Dt;2WEu*;l|BO$P8=p@!P0ym(4p@rKr*_N#98nkqud$ar8eb1yc z#>-Fk|FCtIL2ZUjzi*2>g;FR^i&MN5ha!dIF2$v|q(E^iQe0Y`V#VEE60Eotmq3C$ z1WN)0IeE^!=gae+nR_PRl9@ZR*R}iq|8_U{x_2lj&|C0nbgwW;8anoG4@>W^lUMzb zK#)yf_oQIq`Ed*C6eI?VOQ^eO!!ubanCeG2dEUv|c$s#U-U^<^P7R?v{kI;m$Cx7N zH!Eis!E~+A4a)GF$#7G3i5q$IEDfo=FdE%+}+!=9PHn9Db5a!>}Nh6G5^`Rey8 z2UuBQ;yJlvDvW&r&&%@1e<2fp zng`#^FCIkhBEw>+)6Nq(3qxL^ub$~?8|UwX-VLetw$q^>ZRZAcJ+fv^eEYMHiVaLo zv)6v>Ic*D*o6XKteHOGEmHuBEG^x>tFhW+X4T&p)(OzYD_n7SI?lXWgLy&~o`>gKf zC2erP)7y+fKlFr-hP%#y9|7VAik+i?!>d}9KyeRYQhntQ+bfteqTWHr+5GCj-;tqx zeRXym8n8I}BmL#0YzK`A(h^w;N38trR8!gu`(d*>RqYQdLaAIMZiir@ccEm7oaDzJ zYLC5t&*8h7-&%kjfKnmpIWHa(QRp_`Uzvj5L9qh5d8uwW2|Sn3!IOfi;vn_`?JJs+ z;K|G*aFbl*T8?xCFCVl6J#D1;ZAN!btGeAoHmX!GCSr22+L&$qBNp(_{n(sCgZsSW za+gElE4RYjcl^PBSD_|UBV|eDS%cO4KW<&X_y88|LIt&hrSihdlYZaJ-H|G&T@PSL*y~aa zs{i@(j6h{^)4RfJT>_5{QC?n+5{*2b+sMu4LA^)A)f!XrJ#e?Mc+0LR)SJ3(yUu87wu~`U@kc^3-V{W@oZudz*Q$FZ9&zV5WHRTZw4ikC~>= z9v`a-5{7WsE}h{~bz+R|O_EJ~CnaT5t$vCYlJj=)fWpb@WZNolDP zy2-iN7q^TICKWx8suoVDh_1l;> zMNez9D>XZ6824Ceb0DAg@G`$%oYptWjFk&f?#d--oi)FFND|uxUrZP`r34~RULui; z$mDyBL<5qJQC{}Ky>{Bn0m2MSyX7a6NsCc&7X>k;=YqhR?Nw2{VTYSlFEL!VS zTkB6F-tAi$Y*=D*`O+zt~Sn^erKdq$Qpki6m&3EHb6j0A)ZZS%S=FLsTv#fa(UVt zU3(vTN}pUkfTo1}`NXu>5A1R`Rtts}xXqfDCI=p z_RR7>?OdcxnYIbMr}rG!V#RENz*^j`&4blyXnmUB;~8H0E@;_1i=wg{v-33a`vfWt zYFZ|84yJ9pCH?5%qB|o_4Pd5*9dnN!Wk^e45$fr~2G{-lZITLa8kebM=1uP7Y#WOE zU@yJ1)@S0#UckuqH*ewi+grL}x|nG&|MIj=Qw}gE6Y{#L;RVsNWzCwB|A$E+}Je+ z{0j4L!yqzI(|vy1MhAuN8jq4>=I^XQrCNuqO64MnQUlefm^CwR zXEg*4&)M!l(VaNyFTVE zI@NY75WlN!wU3P`IPFBycTa1r24V^X97eNYfP%gikq6pO0~Qz-n-a?!QTL5`?F0QM z{ik1HeGcSPdz<|vkaRp&(oJqZo-jZ=heeqjbWi$0WB@PR6J1z1504_M^W&x1uH0TG ztCCs+1YfKEDWLqjR;=F*9e@=C+*u(-v=7Z=t0wNKFnjr!?FXaIUV)P0}8^0Iiq(ee1-+U{S| z8VQJigr(*mPM?^p|~?83!0J7GUoP2?(s(7k&U1 zpEV-z$#e1U1%nFN-jc)k3XyHc?r15dr8X%*J?%lp4|qbCK)`RO92)JO`y8&DYYDW!iu6yk0}zn2vi3&pny0ZY9B^BAs8K#1;HK zJI67*nk8~6X4C^0hG zfPQp`lxfpHt?TOI651|Pjso*xU>y`PdGB=>OY*~bhvgZZ_3Z2SI~xUe0)v^EJM%^O z5lTAI3nj){1>?~hXYhEIniLR(b;!FQMijAs0(0gY1WyGbNsyHdy{~98_=RmQs$sfU zD-yG-Qk7j1c31E+>3;ZofZFeOZV!pJ8XveRL89meq1}tLteS3R5Gtr5;QHK7 z8+@04-B~OBtz{N0CV96cDA0eG@814u^sWa3kxZT-u8eIG6axJbwo1xms* zSO2DMx`Jqko%)q4`H%+`3|Y+g+q&N3@xo*3JTCsI?X_Iq#Kg(@t4O;6)&Mh3OM00) z^P0n$g~RD4xhN%24UrQ&2AA&k$Y~e!fFgk~ui9AXkxXtik&`vWEY;i>as;zS@8IFvvzoZ-v$#Y!W2x4{IdMjwcx3-nmfCG0= z+P%b;Q)^4$kRC))Dj1-^CdB@(&XUy=M3>xX>k zu3#v8Zf4tr4@{a>+^1*@GOxi@$R#9K^i7FOJzx-rwzQ=G*yq&QeS7gOw)A8<;c720 zu;Q;sZeqygD*^km51Kk{pDc%hK>CbdUM&Isym=SXDM5wMW(rq`r!R*NdjUv1C07?W z8#PM(cfvhRQ-~+&S<@VrWSc*t4D8;i<28rk=cC8r3FnqbM~R}FXkH89!X-M zn{UykH&1Fv9?}r&V@c6xVmda{!zPLC22AurvT!fVcb<;8`p(P8)UzJ0o-0)(Nctl( zOs~))NnZloJ5^-!==1p$(GWHmU*5g8#a|Db=v7s?ij;4UKsatb1g9 zCG8ea@DNQZmK!dx#DI#|hY+7uo2{9d7QE|=q>>YPPPNqIk1h4f;kBN9CT)^*uoXOoqgYR<)L&+d~_Gb!n2hGG% za(~yaebl{9=ajIy_%(Kq$OiSCFNgYsBCJ%K{h!G`0*`7s$%HHq8g>%8Hd&NK7Q#M@ z`rfr!3%{xRVelp{m7l+~38yz;v@ssFBIb2v&0=Ex?Z6n|8CS`rU-gMOe9$|bVl3FL zslssjb%JMOsGD2gHJ+^=7jBpz%IdL)sg6$IRg|#yofaj8-wK%4<}PL_+Mw>0_*TUOC#rsPtBorttrIwS7{=!JsuIIZK>PrUvz=x42a9!gHg%;I^ z<50h;!fWuggc31rLb64a2u9U*&RCb%^KqJm1!se@yG_q909h*-MHie#@@uu8M~lLp z9!k%cKm7M&AQU2$E>A7;ALBDz?HxI86VBnHZg*@krO-6bWw$wVsfPcfq{vS{UW4Uw z9qFWUol=ha@e={B<0$5=r}?TVgy}cYWAdnB){)WLpU<=mK8fGEva*VxXXCu7GwkG+ z(w#B~w<>>xhnYGyiBPyZPHU)m&Z3*Vc{(5C^9UzuC0_N|1`29W3%Qif?vh~gTsyCI zm?{v%>>73^Ms#*|5NlPQ%qe9JVa#E;k7l+j|Cg90Z{g%-mYZ3|}>s{f{sS-8g zlqY}E3ll8#zY=Y!=MdQ%ztSIBU>{_q%(F&$JS45G-S~UT_tCT z9<}1|6Ix7wiw5e-+R5r!K{0n$t4KoPk=GDi>sSQQ<1j1Bih!4@FC|rJ*D)cLM9)!p z^z+dVU8JvGaR*nKI(aRQoo|t`{?ud*yp1ECqGx|+;7cPrOan+3@XA*m(JpRU$8%{k zeq2O3^u|e(r)A*MNm`|Bt!xqx<J;#7cT{;uc6^!agZj4{HN{0)mM~Eq8T@HLfZASw# z^(e%+@(|;;Hq6$q+Y{rJ&V?%V$D%gy?g_2!Cya#X?663L0{!4_SId2zFHCRnDLj`t z%u^M)1QlfX1pWTn%TT>1mpwxPx;4IaZ#?m_rPs1tVtn>j@+^xxC8qGHOih(;@)6>U z?^^w<#?1xiUkmz^=<$baxpuuuea`xiYC@vMs*Vfg)rq%Tp$t`p){ilIoJa6aQ!5yi zRZ5rw5esE{a`EEk##791vVGS>aCFPcRvTd5ffp0U%T@sLT=IFR1-?iEKL-XjO}y30 zKJuTw^qDcee=G|Xn;EC22BrRVCDSPkPr-x5B~F&}%I&$vkL}V*Tu}n${M0si_7CB5 zZ!wtOfs&!s66`X^Vdq;wZ>U=n&IaZsr{y0u-;g5w(Ub7|v?WJ{z%5?dyu_R`d@LYLZ$$3cs4tV?AZ`K6|aiYSUwp=XR7-)Hk zZ}xh((ANNq-x$wHC=1v3FBP<*lvEM}O#tX}sYQfr=5;J7MR%Lu zgRBHuIb@%pG_XV-yO@B|P50pC)LjYdkw!%{iFpLWNG0Ug^@)u+3bTPoHZ$EkK%IKk zz*tSHWWJM_RzRQkh_jX#kibwgYwAz%i;aty3E1pZ4qdEi3>P4xFXIYB!GF#k z26P~38Krs6c<4I2c;e{;H6PEwr_J<-L*4L09Qe3^_rj37(?QlwvWDH^ePm)TBT~PT z9ey;AUvv;Xx=Foh6agl!!DZ+I+EI?_R+uRLZtv2j75l0Q(PIieet#R!uRYFM(#!l_ zf{v;`YrAlewJPgQGd-hb@`W2Ob`d z08X+5@VTj*FI$8+A6~Ax3oR1tiCV>_an#q zR3Mry5m5E5=I>)vWv=-QRA{g5R^Nb^`4Sms%MAUvJFSCCmp|@VG1<)^c zw2`?c3)krRf&idx>a(cO=syfW*S)nP`!*`+2f{iN_5XMk;$AtJdURQe2d(-XsRgR; z23^9P9cnNo)l&DKFEx=!xtk0{#8WG0HasM%jCgYU!4Ilyg;t`JYLpILy!{PJf*+Qn z)Y2$8O@@`Xf1zH-{7jpSspoYHdZ#93Si=-X=P%rZ?IKRm%?Ncz4CS z;(MP!)J7{>CouT0+J#;Eu#UkEo?{;>em3&kec8q>4W)Yh9F=t?UB+kQr(sKBDM9G$fdI%<&6w)G9E}r zMP40JxvWSz?HVkIBy@Szf(p9XGEPR?RTHcesil+&o>}RBqG&aIZV9<1I^-4g*z?L> z_{KumoQOp_;SU1aezk z$n>;kJV$V2N?9OWhVolYMj z8m=E^5XF?+5bE3?AJ6>W@0^6g+DLD7+l6~8Sq^arpV1-LhI&*;cqJ%dbogd{Jy2JB zJEYkfH3!*%r2we zI?@Ti55g35&5x5dvrj4=Nbg>V8iR(PwJ@sR_u%nTY}7eBy3-)CF8F3Y-19j-#_7}Z z#<{H4%N>3@U{6H5beET(@^S?{A+(LgOX(WI0M0E1q7<>_%A>kZB0ZxvQ03!tNrA_# z18qa#MubKuB{{26Yo#!o<(R==e;ygXLvg?-v^u|i>h3)XPF7Gl($^RXggYJlQ@M@Y z6@?aL#L=#3a2jydvIf`8GUdd5usxqKha)r}*3k!`nYuA3;K$Y(Z`aZwQ%bLbsz9$` z4F6qOywc8B8dk1&O>9qSDWqdC5$WZGvN9gp_O^jrbF)5+wVf*~B*KtNw-4o+V=RC=Q`lt>W# zZlvyQ@|Kar-F|q*I2G`S1I;0Eya?)m9;heZ(E@I zs`Fs8V^!Da^#!iQXs(#A@*>95l(nz5vV^+%+F1TxNe*!=Yjez*feoa99B4q}W4`hF z^p}4@Uvl8 za0{$Xe?x27RdQZ%9@pk)-Wx^YgXu=Qg8C^C$qX9oY;HI{UjK!&clzGP?!$HZSK~kT zOml7mDf}zh;-?{r(#qFCI*My{t?c=V%7*TAsHlsFkd71b0}%$F(*1l;dT&DG7U1~{u);bdH zf-Y0HZhly&8cd0xyd%3)*uuXWvRm7C5FQlUp8Q_$IjxqwuF=zo4_2@^s?P4dA+=f} z%CEOLW8CJbBEsy1m1%lzybm6-_b>^k_TQdX{2QJjcyqe5#TlS65R*!KzhH)z-KD=? z%~GyN!J4ZArOgj41_QkVuY<8q4FZml&qrmV!u+QpkrNN_Lzbx0U&0Q3%$cIrCA1=6 zKg61Ql|5F*)1V>(kkvX=F%3AWtxBETT2Q(T{!uDpC!e+pI5aDG?+K)HEFlGe7M&}l)QGo zXW57o?Tsl02r6+DL0(jv9NLP0y^2cR<9MFmRh%-M=Tk7Lvuts3T78(kFYGy5aXZWyKREHBrSPTfum-sUYwM$ z)7CjUtWZM^qpmV*-n$I5>yj*5ORH6zI=Sbb_s8T&H@CfNzq6KmGz02d?XR4|ZLDv0 zboCn&{@_G%AFA3#6sVo^craVavOCAI+`;k?X{EDxb6GDItwhpkDKIK26Mf?2O7URMsoC7&z8!SEh&r-^J79B*RK*(^vpXZ$;PxbSxwH(yY>DJ(iJU<~_b7 zLjwa)IfMYKX{Re)VzL-+^t+phR#pDxHp{S7L~32+zYn5qE7G9%GE^3bW)1wUTkF1& z+3h3_hG7BGngQ;b5&EV*KOy4$3EPI_ltXhbx#8$jpkLWfcDug!zK*rfBDWkVJ$e!| zznhONUgr{nzZ)QFB9S5YIAuQ-KJLsDpA>)H=!3)<-JDtdYxL(G!jxu38n*;Zawh9O zOSUc4ZF|*8$*o!k*M9v1nFS+lT$0nb=p!3<18jLH1U`K8E$tx9;>&FVKhsJ?QL*dP z1_j<|t65MB%JQ2i3Vx1vA^q)pfsdu3=gk3=P~pWRSCvHUZ1hh?Yv4-dI^0xcXwqdc zFK$793oW<;Dq63{4Bi6xalZvLabCRHsFn^FQ;!f($g=H5Av#&hVcmAm&Ppu2R*U2{ z?okNgO0)F))Vh8i1;U|vApLkhwrYg8&yEnw_bMJ2Gy6BJFX+2|WV+#WT1u?_k?3tx z6YcrBO!eJ;^3>{b0^EMbatBBPZM|4U z(t@@&qy}-2XBt`BX_~ zdynLvvT#PqbRh;x^Thie$MthpPCIKs+epR^k9WgUsUHUIb@8-j{>g-KlLUZ>FCYAM z<{OU$52h>aUTX+v%@={zj2R>lJ_PgIM^vXSAql9{`(&fMLRs@oG`A4yTqH}qlJ_~0 zlWF(2Q1|D2cCE!ZBxI4n8+eN1arxF49O)LUnkg}`g7u6M0P(GowNz~bGW%6hZ)u|p z%=Gd%I6p1Yn^y0+lx?NAZv99CU<#h`Sz`)@MG#+fj&jMchb52v(ReNFPE>nS($luWU=XD@I^iRmIZ&qzi-DBoAkETk^8hx#Y^NMMKKN# z5#MBIn$~1I{9Qo;@z+D(D9fky4mV48T%{~&w$%DtmgKypbmnZSI@q8YhdR0bv9wtG zA-A(=SnY?J{y&vKBHqi|7#xyQ?QpmHw{)M6nbmZTt4kWnWp~fh^dArBdqOK3+gJ&k)J5sA6^S4rgQf@ z9L|!i?Tj*;O6E;`1HEsl3$$K%`}w3&EdIRT_0{aqRH&QC*&TvHTlap$&Lsm^=)v!dZv4B+K_Jc$E(7xhVCY@UA~vT&eW_KC(rOTe)MhF zJYo7Ib-gcV8mKzX!NMuQP7rKp?}?41?K2SD zl3bKkcAkVogQ(g`G;w})AG-YNXmx|m@-uzspW0el@>mI5u6V3=$IWM|iHT9?B67(c zB|U6__B=g&1s>Z_ng}F}(nqgV<3zx&KG&#ROI`2m3DP3zy53W7v~<>dtpz~ZM5mD* z4c@7#9q>aXCJM5X>vQWvoUCBlXN+;nTSEzreFYI>K|fI0s)4S(gqVD|bccz>UWo#` zw5K4H>l#uj`}Wt%Pkq)q_&cvW`iuDrRMOw3_9?$5LYH5dpT7V1Q}gi|C|n>yQ5oOX z3-+K9bnBb>F5p^6BRYAyEd6(PutZ)l|N2Xh@+CVpm8KjLix~UcqsMn?{v8P&gd|u zc@Xzeib_ydJ2f3$bt{8(knBqyqbk#k#Y!C=<+Qi9C&(K%oucVN5~HIH)xYNbV_>Yc z+_!g<-Hm&bx5ig7oRQHv8grNflzG9=zjiWV)`LAA8^lOG%n7!b`b;B@NjIm$t2-0E z*4RxP1{R2Bk%dy#6DHufeVoh|1=2&fQ_hY4^+yO=9w!?dHlKYT813^(=T>|V%vEbR z5I!DD;eLaa8X1D-IfglrkQC&Xzx&S@I>bXB>~0wkxr!v%nXYDff-st|F?Xif?51uN za&-ZN{sKOdG8xZ0KjG$wnKAr>^8Y$=Fi(#3hsX0<(}j|8qs5!KN<~asO4J>%YHn{= z^jp5?_^|i85c%$iw$1P(0{aA&#s2K%z)WuAyXJDrc&pkXill9?disyl~s0ug||PjIp%|L>MmV7|R1CT=EcFeV4dkx4Q+G+*XPG z?c{*pSbe@;2Xo70O+2^fY!n5DOd*tj8Ulh83ohg(ro1IRAJ;|8T zZV@y*_vWw)$*XTL$xjeJ{|Y~Ap~mC2oAjD3h-2#Z-oV)W_6hEPL|kGprq;wIuUAsA z?ow7n;=7+xl42PZVMd$OeiwXXx;8SWDU$xgV@FRa?2(6KNunNU^3CPYq2?ZTmldB7 zEe?{6K(dn*d;!jK28Y-e?IU(7F)z7XUCdj)eSV!!2>Hid3Mm?|VXYP9GO*#&zdOwM zBP|iDcG6J5hvTpj3ZN?OMWp#_+Ep3X^J~V z1`OjcH;bX*EysgB}<>*qYUhQX{_3B<9jJ_5Z zC}uh*ZI`4VV)FzFyG+2psISbN%EZeuM(Bh zqamu3qawXOf>4ic=vS(}4XTB1rplp<@~53~%TGlbe}p{wep}xD2PpwpAYA_hZyyH7 zqkYh$A6WjI)I8a`7YI zPrbP82mskQ0g0HNp%Ad6=ImOcHYryP%|yvvLLnPOBx&mkP_LPBc>BJ%t7!vei_XK# zdE3)k*StE}%j2N)wZ@c>C9^KALyyw)Fgjj2nN@R)2*Q+d7^pPdCDt?J-|=#zk|{;d z#mKJ4C@sa{iS>?_-ChEnUsa7}eW*nK1(QBXcc;IS0c1_VYi2?%vHBqrw~Mf}S1##) zStma~G?2Dw0ZFmKK5b13cr*JF;@TcR%VPG%&F|r$00rIDb}JRf>|DiKp*Lzh7s+Y+ zJUgnU74 zx7giDJHHF!Pq&sVHfV8%Y`a~+{)L1QPd}BFVP+xj_lgQWPX4iT+77;Wma4tyRcuDk zj}sa-S?eiKd6>&n9sc#C%G1A3XaC3SVJta>bPzmw=4x>~tw1y%9<}Gs8UMSh35=Gz zkI^NC7yNX7jOm};K1+D&*`hTL#5qqSr~sb{UZ^L{Iu;qP3BieZ(Q>hhcibA_D|=1n ztVl(O0^bj#9MBvbpXBhRZARSHcCk(5Z{`MLejyIbFyxcYd!Si7hFQ20jb0f(AOeMv zuN-&LsefXmU8ql#IMCXT>9km>#BB|!DPBG`izxh`1EO<^EydGhtW*!Tbj(uFZaf7G z8VI24h{dn=(3vh+=#bo?_eeCoz}*57ovw=JZCJT> zZV~qmnS%@aK+C{XnQ<}qarVeI66A|P%;1lJ+yuHB{%?B;ISGdzJ6;J-6XG4QraVxW zB%wTS6o{^ZKCbmg5-!rO{Zz1T)mOII7&!wjyXMM&N1T9WdK0G0JchhVdlu5EmXagW z!<0WH!gNL)&;_*p*w2u#R3FA%w{I)#^LC2+7JiXAe$^R!TdnnWx6aBP!G)eeL=daP+2u=oirMOCd=*@hrhj7K&S$mOu zbirY<-BkBc#FkHbAu+CP(CtCz3(2c?kNO4=Bt1ulbgjgE)h~%V3)5<2J`cdzVKVjj zO{KM?y;hV#bnf8FGYorb)$ z<$?9MAw+KtCTYHGpFI$3@VBm-$i5s$$sR6N7LQeF+4e>x&nFc$zS~XxJ_reTqS~N%^e><-v4y=aqK8{q96grMH_ox=-%j#8*korD9e^ z_1WvW?`x-=-mQT`1G9~wTcZ|NFHkR`IshoTJe^i>2)Y_NMGIDW#j%o?X3ZDD`oVWH zYP4G!5N8zP1oGZ@TtjW~8Ts?Ew72A{NjFXzZ6X2Jb%=te8q^tviu+g^Ey}=n*n3X;Fl(}VxJK3y6GK@Ujqv+ak&m+$my|@Zk%PoB- zH?rd`1s)p@@-p?D+OPKpMg>EWd-nPin?&^AHL>&1Rmi_B&k*mZj|7yT;jgtAgS$$} z6-)BC(BxNsLUMlGbWpBs1#g;EpwUt8U*HW~*h8Y_l^Jask7q(&J1<_#>f+OT`&KgB zHlolfSob}I+9_BkKik4J@guq}{MGD4hFtcB=5E#q6v5+18}rsUb=hI0qZZ&>=|@5( z_AoIt<^}mV%VCSPw{_SV!Y1y%9VNH^UBGptr1U;u$5KLI>+j;>XQ0zijr@RHe0R=c z^_%Vwy|M|O!3I~2(!>1JL&FD~%>g&~Dlhh3`(fSHZg%DTlKur<76LQZxE?h7ZGZ$o zZsD0lJMDtj0fuAf>3n1+Gs8zMe6{8%DD(>hLMB{U6tm+nP_H0J!>{ z+j=~zj1PLgzhM3IcYkbf$X&-#1MU^>b@kQO zf%!=2k8-)U8BRec)H+p--^8qp%;IZWp@t&!vVF4ytxV7sOh=H3+oTt_j?ZL6yLUQb6>3uEy= zeeeGF(~!JJ^Wiy9F)dCTzJAOL;K;c?G8@frN74t%`@iq~i`Z_3sK3j!kv=iknr z#{WQN`L5RPZY~yzF!|M$C<^{~0WlNVPBnQ6K<11CF%+|a{jJ>C1can(Nt)ABm^_`C z?}XL4c4vAqBoJQZ3BZ$xs?@E1+m;s(TshVAg*PK^ABMIE`Gw3L3M8tRB!~VS1LrHV zOp89`KQnq_DPTnU7p?QY9`Q% zKKk#APbm=2U%%%Y^RW^hFPc*eI~2(C36FIjit-0TB$l_uN|TCRTM|q=t|qKV>dz0@ zX6Mi6DQ7Pzn2jW-_#vQseOYLWR};&>;0?7wz9CE}B#&vab!kG9aB$GQiQImI)*D6f zuCI)%slp`8Er&qX{&@!3S8Q^x3O9-^?*8JLO-(r)-d#>5d=2;yU3#jYe?3#I;uM3~ zMYvZ#mZsL$`tK(L#w)jDM^Zj<1J=743LbZzqVm8bDMN&>XG` z`C9F+VK2`m#V~Bib28gkTW*b@-F;_f4`{Zwh=C$;Pr#8mEDpWGxu}bag80mTO z(Qq+^!Uk|^dj6C&YPEv$-s$Xn$PCCDM@2V!>#jil-KnNA>4!~!?&EV?4K!OSZ8g8e zJ$+i7UadxIMy$K3mf@D^*~A*vC1&NGm1a2^$buOHK_Un`z+xk{rfg#3lyGSjAC%MsL+I> zU;knlH z-qN_S$F`KI;Q~R+N2&=|c}EX3^Nw28pdQoq}yOBsp5SG^*>+#(Ek&_?|oKy-U)3<)RPV$fr0*5Ule5IUvhJ zKA5axWg^ZA6V-YNl+DL8PTXh&!`Qy=|Fd*}Y}#-bHPxkPUuyXyj#HtF59Ps)#Gr+< z(^tgxgsZ{p$&PQuIJMQYr6?N=vDvN${^Tq3B`?7}K2mtB87jfk4*H&3Z>RjGy|T%; zs(kTBbI>3##JST9SpPDrw=hmRB6_4CT>|4;2a+ey=G7K4;^2EJCMmvdh^E%3Bdc`V z8@}n}hM)C&KK5eJG8yyv#*(n=C(?@Ul8#vH4dn&MK3ddpb&Ni&6AhKHbYg==S?Fj z{*aMQS{2m=>x-c*Jzanhw37AYjz2utZ)4y+G5@#)Mc6Mg+$CpRrz#-zQ%7cYuE; z-p75c_@_Kojz<)gA7^=ip9};MS}VeT3st`soZBun{ty~}I->3rse*R}W6K)SV6J|E za;nf;@usKLVUm0FD-h4ha$U)x?&r`(VlSpc6Pe|*T;9@=1}72yo5=VxBf?NdfBR0> z7#QbHPQMhM$o^9ptz|MtGryz># zV0nnw=c|9sN3I>QcX=--+EIb*kwimO_#_0?Uz_beC`LA!@71?{rxn%7jfsRW$X~YItkZqqr0XJ`?llBx|K6@Vpa% zGw*3f8feSDLS3$f+lHpUyF8Ax$G#a3Gx%Nk@hwKn*-yND!YXZwQd|;i(!T| zo5=t61pXsV_5?pe&&z`EyFebP1JdR1n|z*q7O+bqXVXAa8e?fEp&EZ8tiwm*(pVmF zUKtlZZ@JP-R;8gU2bT57DTxsFQ6Yy+0hwehMGbyOvx|G~cs8ppY7<9owrAhNoHDYp zVb3$l)JD&C%^ySXk~c~uSTV*iV@E3US$ob4Pv;%yD-68DmH77@9ycC9sH3c7`LU~m zS(Pae@QT^6AijpM3u~=YLqpLu-bmFvAtKx6Z#A^ zO_z2KQ2$5nl%PBpQyQ&+Yci-o%X@>r?eU@;tv7@UkMh6TOY9Y#z@>OOathqb#>XP7 zV9NefhWOHG>$kaNMlvWOXc-qxqdIr%bau5h=BkrV@gL8;5O*+&9_W9wjkhq>7P7&Jz#Z_5}xmj$1+JU-#7h#Z(zDYImySl2I*V9tQOwuHCok<)}Rev zMPHv~zv@{h{S3m!=l4So<8o5B|NdBp(dCaZb{*{aMP!PNlOdnv>TL+%wwSphwF?eO zHwUl$RV%j4i09D>z0~zt*zaV2!hyE1pW?Q6{3jpVYt}kBYQ2<#g9TAO#(lG+OK#Rp z*UP4ZfhNv?+OEX4MMfFY9mvq4pU9y790`-R;10`(%YO3NC##yqHaTdB+*gslLiCQ6FJfsISZ)vOtFH(*~}udj>MBC9|x8`#LJq}6VSz3q3ff} zJys}{Pj+1vLX+RIZB=jb)w6yx>;u^!_y^HhgNZ`Q?|#LfoN0%-+|SDod)8Q6h?_K6 zCvEdk9QGym(x{czR3-=~Xua1~3u3iFw5Dji@0+KTIDtWi%&@Kg4|{JJ73JDC z3~x~+1tdjUq)WPCMCp=lP+;ipMii9pPHE}xQktQ=ySsB>crW+;+|T>G_ugCa*SEg4 zo*%3=%*=VsaUSQ<=Mi*LiezpdrcFlj3%ylu6lY$o6F^|)WjRN*aAc^I&3yRv-cpM1 zM7G9?5{y`nkx8gXX-$XoMJqvM!HhZuPTvBCOJchS4rAVC-%ce%)!)f zsx1N?_NPhQU@P0OFmW9wTRuxkr$&@Ye+KBb-XJ9WiY9>R}6L)ncJAR__Dn!{yE3Fi!sj9yO|4X;(dMv+~qN9 z`#xcR9A-5#rdPY@Mxf$leXjP6j6Yt#=JLrO9HhxkLtmLd=eu~1O27;h$@BAn4*P7# zlT3Bcys&xcCotO}>iE)$Le~C0PJ2kzd_vl*$(qU8^j7ff2yD+9clC`S6OILS0ZIQ8 z$A-qSJmn^plnAPq&tFF;)!9d(78(W@UAdW~j6Xq^!C(@S+K}%2=w{Y&qoWpdf3cJh z60qbOMxH4%JEdL3-2U9VIVVR*+=a!LKfKf`aFFX^uur32yO`w-WGU){&up~uYV<|S#;TWQ%Q^dLUKewI>-64v2hWzfN>@86?|(*(rj$-_YDm85 zn_EAf8xnILM;*fq5)KphG(11`#C7F8$}{D3nK5#|u_EHfFXT-Q!-yhnP0Zd;1zqhRhjDr5#7VjX8hrvc;`HDu)^ ze|O0&M*0Fy*xv~zO;M~~A0xAO)j-01YM!Ify~)7ItsaZn@gBRQXXKfA2KOFYUw=<~ zO*bitvq9bE=o876WxUt)%w-`g2QgTgx}FT5wik(9(IkU+#&hxyrsK-i;g2yUi&(Rt z!1RjS4<2*5U6&^|k%kLs4GHpm-v4^Yp30QW>nKySAnI!Au+)mg?RHS?;(8vb4+$5x zp+mt}Hut{pbI!UA;~*8<&UDe$$YpV$ z^#;ko$omq^h&fnrT((To&@#gVVL+XoxC(Oe2C~;V?nRr^zV50iGhDeco6PuvOycbk zF@tO3i3m-aDv?YB*F!E$aoOHQBx*F{XK`=3Y+F5|& zvKd-e)>W9G{lditLWkl;yB+=zQuD$C77gQr_M`Q+B4$Q?8ODULw=pd-VFHmn!wL3Tek- z_D%{!54)M>J8A7%M~?t3`aHXDL~*UbaV`qzD#IPGDCC^KJ5izF?={ z6;?1l(f?4Iw@7Al^=f0VN)aH9;z#-reN^uATMac>RWOD4U(-sI;8fC7MRLH;(LTfl zNW3U>t1z1;7Wm#Hy+x5v8?`6@AqdeAr255Flm7m9)K(UZUQ`l(HC#sIqPJPfj}XCA z8==cimx=)qW{a`SG|}SDc*Hb(x%y%x4B1g{ABudJLT))fwo<3Y!pfT?&V17@t4}ZT z7dO<(2a8%cKeoflL*>G_7Qm1OuQ!W8gXQgf=6(2^|JTabP9thvb|#Q z@iVQ?eYufYR$_hi)R;oMn%02-Z(#jPDe1qyqr!gA7Xw2DLDI=xjur)HNZn^u`5E4- z>BE{s_$3AL+fkfUfv9hSt8yZ)oHga^nZ4rUwm8v$AUH*rpcv#T)@q<=hN81dddia6 ze`nK+azFw1IG~@t>j-5OmGB4il?#GPNt;1VB-Mde`wQYV0+Kvkp^Q9Ww7ztw=NPse z%E8H8c}s!9FHXWmg?j2Skunr$73#F4^n~UnZR05aquAje{zb&QM@`8{D-C*z$Wa`L zSWy@r^MFu9{+*v8AE9c1-p*w$anz#rRJZ0w<2t7{Cr>X>J43j2QJvi6_ID2QXltff zsE;kt4drqsa#%aUnhZJ9T#U>TOQ)!F*({=!l)zXC|4k+fmv*H8yNW089d@|sxl47j zOXb^j`#WeuTrY5w5|&qbl(~ zjhi@zl=Z`lV}*$vs?={zQneM{M}Pen&iN-=cfK-Ad@p z?km-kzS?k;fXCe+e!apn{#{egNKWVlZ47Nyrc(10x5&=~K>2DK&}AU1TETGiO4v_f zi%Nat0S1g#$WL#<1L+xXt(<1ha}uw1|8KHY|5`Tq;~h{n5-9)u=8+_tQnQx3|45Ab zYt%zR_o|1sTLaW@wQnTKBeAQzJ3IcUY$X+;*W>u&;mQNFeTL1Ft%xPy?gmQT1c6ch zD!TqAHRoUAjWiAsROYooCKvX&%g)<0Tir@Z;yzX?S4VOa>E_VW&Gw3xb(@qNL4Lxe zI8zCbZ4DE0i0oZa3otW{G-;4LMxw&fBLFks9`m{5zZPdU?1R!an|Hk?^^=T{5(QV> zT`&OO%<%_h`1fGC3dk&Byz<3myLWJ3^xyH}%S^ zR}+s=v4C^=tnD9s)}J2OVBl>Ei6tw*?YVYb@Ye4Om0`&Uk)I)_c-HJlP36ij1(kiT zi+`(D^8C*10VH{|KERNmW?4$Z^=_r^r=qP5)$|tf&O!pMz%fWZT@|A&!0dXe=ufcPx&otvK zm0$X`xc9^CjMJZM4X|5@dLnu&&|}5FhNO&@qW;D%{`I>~RG}Fj!p7l7zd=g>l44fo z7++M^rmFk+B!7ca{;awY25|0h|2J28nST1Cjb*$ibItF!Xb0Hm8~PT3e-OR@xNWQ6 zAKQzs8R3)rO+fGYgC`LFQQG~*r2E%53-RHwyER#3;;AHlL$v*X{w-d21{vD#!iZTN zk2KN7f>*SLwOaEVwD>Q&e6L0Ss-I;C{U5RS57-8}bWoERu1oU)DUzZ}Gj~>y|NT2L zLGK6aG|OG1Ys?B3y?#T<{FkbsKzu_Vff?zH`kVOjQhfPdH%`$Se**D$S4RF2))H^l zn17cf z6My`>sP{h*{f~W-Dgk^(sigUk_6Y}S|PQXY-L%~Cyo4$kcN*8M#W+LT^0-f_zjE$IAq^Id#s5$^^gnAL`wD51P#Bcyu z+?$Ao?)h&LmRP|LsZyZwY4_Q0u0-SoL|}7pB+l<@{0!wE;%JPn#o>3!z&i<$o$o{; z$iGYZ@^nAM^>);|y5AMo%`$*)DJBQ|{4TBML;f$HPOVN+#+k2}6FCG!!{ zb;zbfOMkX8!QEkj9?kXRgD6ORZz-T0bCwB2Z1Ry<1F_O zD-M{`qGHSNo@yChypVt3RUW1Sy`--J5YmT*$UiZRU zvK0xIR@w>U6lOklfcc$?`9`tcAKbgayurG`zG1HkJQuPFYv4Q0(x0#4Gwe*iD^Gzg zkTg$q@qO>35ppwZ4hhp15#hJUhO&o)ViCW1rlq7cQGR~=>0Bt5QIR&EvIZ(c%LnH0 zyDL?EptKqd0HYBFg`sYyKR3;aM6$#>iCPGL{sr+1@aVMpPt#6J-U&uqxlG(pnJpfL z%1?KFv=F(K67XmM`*L!|)R9O9Ay)g?lz$A~zhbzutOn3gEta+C5}v-FvBzq#eOIm} zwSrSAP^DZm%TEm8?;nu#I!D?ILq;UcN4sLWvi{ocN~ZWhWem2s!>ckDS9a&X!UBm` zo-(~HK$Ttb+)19;s{Rk;$?l!#oGia#m^p9W9A>uJnK*ot&~_6M2CBZ2Vc`fEih5gQ$RQbzwjdww~L}oGT*9-y_(zlX=ZtgjX>9#%I{r?#)*ujpji|OcT93iyY8;9_KFw;#M88?O5_6=q)tyl_hrYeyn06l;R-(6V z;aq(0RGBD5r^uU387Y3;WDnu=|P-ufJnZu>ms^g)E5M<9$a-{)AH%WdVW+vTX4Gpr5m zXd5zm^j$Qi>r+rd1BdzKkq-O~S|SQI4f$Z)(UxnQ&P)6&(-@1J=jLl}F1Onj)M@)| z^JDp^PCcORSBW1QEv^!>Jwh5kG#XBYGB{ehqhHspH@!ATw4vuxsm5oC;mc3R zl#qB1qi(^xG>FsAl*8=^kjEMg7?pR}<)wtTF%)K@cYwJi?g-jQ9If?W%Dcxp?F$-3 zq7pIptx8(>3iyZ&~55PIoCAYAb$pjlh5eK0P-` z+pV#gBW)^De_ms|YG;O$!RJj|FzsM>atriKjqq@N$zITT$q!l3%`%P6H7)EW^kwG9UEY^OfxLIM!ayvTViA`MI2 zH;Kw5-W))b9ueKDn`hQuy^ZO)iWY{E*)b=Z;kf3{?Vm>FaTa^C zV8}U@lHkJ3kic`Uu8D~PwF+pvv!uN#Zc=zDM|+|l4*0%L2~bFZs>z+$)f#m(=SNRA z*v0%rv9j8r zchYX9g-a+B~3u*Q9;(X=9d5VCwo@g4^*(NA2WB>bn32u6vpgDn75%MlnUCbB;NkySa4` z8JbtYt!p~Hb~tHdu6#{B==D8U&n>$K)DDsp!;@{$k8|ex{;dw#n~1_6FX|a5&XBb8 z#`h~1q_7CB7oUy}xE>@gl&o}Et~4ciBDd-lH{N_QISkT1JX){Y_bm5jDXK6{hpshS zHjRuU?HXOWxr(Z0gCLgkC6W@e%HrpqI_|5dUgY-M11eRPSJJw&yJ7e#jd{ldpO_5# zD%D9gR1f;}2}X{C#?Rjlq`k)pSQo#kt>7E0+N!FUsuSzRm8JC(RS=J1EYM70&#OO!94lW2 z5mc#eg0Rv*pW`{1d%-`8RjY!LH1v-+ws#MBHP}D(jD$cWZV#3*liF9VZ5a~8?d-}| zI+LYq9o!~qk2`9c?06U%`!Lb?2g_5>oWg*T-=VD2VVhfbd%HMhJuKHhN@!~%L9y{g zhW+EF(!{P4CHPA?PG|4k4+gWtxfLjLR2MC_74`~Qf{*s1f;1|(@`H;iaq5aGpFK+r z*uS(a(tIO?=H!jfY4UzXvHHu|CxrqNiX5f%__^D2L0EY%>~wszmcA9U<>BerIPQ40 zc>=C-oFL5KFbLIY%=pZyo->rmz(X$ziQNkp=t1<=XIopf;Hu{E&B^Rcf2PK|3cPl- zfyY_nj7CAY#aK)kCLd$!{1fZ(Zyx;RD$jj|L{K;L+88Gm3us8L_>#meWk;uv*SqOH zK0Fz5FS}}p%5?0P;kljO$Av@}OathGqwzdgAafV6-yR7)HH}d>FP$e54IwJ)OV!!$ zlPC4meyQVfvWRd}vm6@`lmAUfqzzij; zoU;YI?`KI1A)Wkpw$x9t#NJ?Lz$&jsn!q!&9HBQgB7NmMbzR?}#&!BdX zjXf>xu!cHVcnV%msbP2H4MM?by|d-WrdHj#s6#EEzT+!t(aN8thF zE|hy8VNm0{rgf21(m0T*ldiHF^iW#{64Tony3=o9CDTi;^<2Te@yi!kJjz3?U%Q6x zd%85d^>S1801ocjG7C7kTBUlfhd8NwB)AM?_Cnq(vZ}0k_p~v;pAj*QGAG>CAeHdO}qYNlVel9T(USuI6TL80Q!RucsiB91i-k% zRWlre^~7=AgIX5Dy>pj3;I@Wu95c#ks%wv^eb`}z>^hM5!J`LENp%f{j$Wb(ZqCOW zCn5uqbcRaSNbnmb*9p0^?^44OBJFqxDM6iS;osWzxM)KHg0=?3o zyyJjsa#+NYLJrgphT847$GWq1+zNyi2?xxq_qf)Biy9ZK9i3V)?N^_AA+7~5HI0E6 z-oa}d?>Iosk=840*DYw&(qi3h-x&X2@w@$v74-@KFzxr_fe$1e;qmOb5@%-Pyz!Oq zS|}U#azb$EV|6U=d-}dgOT$EeNhO;+wAk^ccp&r0h3b^bu`rR6W_#3{)Cjqpj}K*W zxtmn6%lTFCYhjVVIQ@!($>m2_=At5`dfyM1sq`m0k!-e3j4zE3S?=LE4LiF6@jiQm zczm;Yq!TXsJ70;aV0yK(j;hFS^Ov*lEeYc=QUrZw=t^K;O+v5u)XVAET4dANGs=jg~`W`2s|C5cLfQr83|JEBcLbjghp@l zZ;X4XSMT|h8>fS7kVVg&j@OB_?Y9Iksw1hj6>A+#sDMt4X)V~PK?EG;qte0~&#G3t zlDZ*RvVPrsQC^*$s(Ukiu!oNW9`BhcG*hU4S(TpkV$61DdF677pYP~;*|u)yL?dhe zfCU%yoYk(CEAQ!y>v9L`;~9j-;o}}q`s4?jD zBr?ppoW#6*CDNUXgOZ^-m{8BLExwwx5CULIV?}%@r>q6tE{*lZH?pKhCHI9coXRMJ zNTRuqaXUMP4Lfa_>_Tay=sJTq!hEW3;JQ8|B*vvQ6$x$aL|x9`QXj7|h1SuIgvCr% z%9FV~q{z)(Wet(+-HKZ6KP3Qu5!K zL-aEm#R*en!`IiZ4y^8XyLKJReOW#2y9bM7J|^hSLu=JY|J1No@d?!aVY|#=^yZ+| zPodOm5c6_tyXMF!!tlYITQp-l1JH}^*oDyIKLgtT9Bo7l2EDeb%baU?7KupC$Y{kx zn3^nC5IGGS7$1$54>P;qV{S_ zo}I`i`AxPg1!k@Lshv zvGWF?dkPxpp~k^y79WHfbV$~yU14s`6%yIa5hDT77#D^{xUBw`pdy`SAwAf(Pt zkD16JOtIe#GZm}>rJb7DgSUFRiQ`xETDM!~2VxP7TI!}GuFs}V$2Hs>Z|XW4#nFf* z4#sRHb97xazN`}AT$KPF#+--BCy#6O0X+&4-IqOiA^=gu=~N9Rd*w2Vo*+1>3X}rL zd_vnP$n&x?hdBzy(tgx_IPLgBfYkA>QDQH;_urJLqWuB`Ls6`)X+AUpF7)d6PqH-V zks`R^i7#r?J6O4_oGD(-Pas@ zHeQ9E&{-k(24eFeArzY(D9|yfMV;S{#%<_e5X=6OarR1WI`n;Ky3Fnt&wf(_^knGz zB^R9@P$~DK>wie1609sS{Z9m$J^btyhIqrle4K01&@DfS)FVu(UeV7<4vhtl!BQ#wwQlg%d{BJlNaj<4 zAyDJRidM)Cb7;vQ#Evg47LK7)_QAL#Z{`95iMT>ZQCXb57y)O?i}>^ET}`Y7iDwuW zsBW}^bsJyB*lAK0X34n|61Z(Ci5j!ZFdg$@yN>SEQaeqjX9&t)V|gm}9@Qc>K+&Sm zfwti1pWi%;I0Y&pz@$$voeyoaR%22cs4^v@26jba3EbX1#Kpm(_~CeEcLb)GC2cQJ zbg$l^3k+T|B+KxyjXdJi9|s?H?1i;M6#e4L);d>cNH*vZ!Tz{3X7hydS8EB9Tid6- zJC>QQ`!8+IcDS|Xn_OWLoRrdo#=ePxhLM2XwRx1@-jE|CW}9-ShSs&yx85?QZ0oL z6;`r@c$!T6VWogo0_0WI2YTq0ngO=iI;Z;%Swm}ztYAu}4|&Et~CH` zlkYtitKcldlUrgugsq z4Fg(he}#_a5xJTQa3R(opb~TRnJ87{;^f;E`vJP8xMSJlt|5IZ3JkIIr*U%b#jOYi z(f=cQLUG1fcBT4ko_;uw4YK4UhX*u1$&W5IooA*A<)sUMR}zuk58=i^rh?jYNjdEt zu9>g=FnLl9qi^}fb5BA9Vc{kfB)CozUN4QjFkTrDeeeS3)*rIy?QR&R*-xV#&5g)D z$4*7{e#q1~BusMHnra@-toJa9n8V});8qV9O11|+0or3-CoozKoEmrdom zFUNK4)6LaYCN#p@i#MwJzFnV~-(H&v+(?aS~JW^*DPeGWeXb>fBjnxxXctRUeOwO5?nd|-%-nrfD4RiBT^R=Gvbybl|s zkEV$$uid3;MtArEYOkWWZ5l6P)DUF&&~IoTgdCTVxm6}j7Tf!q zEoH0>oOH*V783Uho`TE-tU%go$oH{7u( zWQzDUf2Czl&R=|6q|Lt^tWoIKNLLV=!tek*gh+DRla)o|ZDmqj?nLclf=nLFcRF(1 zwrou9Lph|#(Zw`sJ$Px0cBEM66rD76mFR!7>W{KqWtgJv#^~db;gI=8o{TyIk~Jul z>~b=1J>UgheHk}WDYT&1@@OnbC7DE%fjXkld1soYfMZra-DEM$uCP(kC1u!^zerLK z#+Z1$thZFN<1TPB65-#=1)CsW`ee!!0?d3hu52uv_3XWv9*tSE*!=Ke+M;exb$2pL zF5qlRubrBecemR7aL#!7NU}d@w=s!0&&nOSH{;7_{>!+l$Rg!Qdfs7_TP*JG!zqXL zQ%@8$9;^9_{w5gn^uEH@{ASaI@r=W;j%l7ra`5>1Y+SYkHswE`6mWePYG29VnIMoUO*EQp@zm&dEpvWMqCx&X zc-mYj4EZl)-ES_94LkPxnz+#Ot}R=;<+PI@68l&0eM(qfXbJAFLNKd;nO}Amf9bH; z@d2fW#4zz{xY;5-U)1Bw+3lz+d-7PF(d8`BGWc9-*tFps_L-S1A>O17Cnf}FdDW{d z5cGOD?>z8@+^C$h0UZ}Q0;NrurTh90J=&q{SP;;eQ$or z)mXLl@^Ww0r&R@QhK3XQk`sAJMW%yYZRD1WFJjW&zUKnIA!RwGwT}DI4VHAm*Yvd;&q^+I1ICJmo)(&H)G4$u5Iy@jO1mCBwbhT~ zz>mfkd}kAwpo-ssXH=gPD{)pnt=g6nd`+*s$w>;#piJhq@ZD`Hv!83}X1p~ebX=mT zIOCA^IvF)=G3B4fl+iq{@@z#9<|9LQufG9iDR#0wG9O=0?+`s_#2GBu-dRknovEYo zz#C${zAj^W7DTIWTm$#W-BI15zdZ1txbp|&dYw@Z#J-F3uSAUf06!m?LTIe-vqq?u z#h}jNkb>JS($BS?udqCU1!I!o+dP?Y$@3a4(d(G7`}WWpR%1A(u2scoODvr+hLW%e z_*{X03NlH|_9Nh!>q5B?ESon;@j*rk^IxR4TokSuGYn$|qNC{sp6ttX?p=y>&{0Pg&J{=_^CvhlJ$#}DFs9t(`WFyEpg8(R4p-ITv z{CI7VmtQZRK65(E)FDztoJ2r)aa1=z9g?mzn^7{Cq&#XIMDerp`XY_;L@f&~gfobm z_1)0OlhfV{)#B{U*e%eQnJ_IaZCVK&Meo=~jp9hA0lpo$AE zBl@mr_mnIrM|Fpgir}A};61ow#(}IB7C*hz%er=NzbH7?W0*yC+5PzouFr*ujb`CeIJ5;5>!XDT?`XFA0R5~SMZukd~ zl_hvhC_xNUHhbeEz+rCYME^_+0Ef8<7jzrs)-pY^;V4Gt1u$#!Y!$u0jg6U9EL-N_ zw$&@RQyj5 zd^#b{>~I~qF<78yw>wk9|9Ha`1yk(h=M!R?Bo1I&$;M3ne%ibb?#Z)}bec>z$E`ul zn7@DOzp?&B7^yt3SSnn0G4|w7KE~(d7V%_duC-$fVI-nU)9W~LQw%9BMPt|^GnU%M zi{&tPA@+}B0h>rA%Hv~|ZAQ~Vy3<6*g9i&tv%~EM_JzbJ)@z2MFZOzDsWg8^kN|(; zJ|Ea?W`T!dzY$A;K#ZBKk0N7VIGeb!de}%M@i1gbr)X0fNPm%g0O7IoJ)$QZE=l9# zF-*-*o~?Z?EZiLn-@7lA&bLHaCobvX7@t3EA@z^i`cHX4iEUH`G{rYYra$?*LCBAc zkdIWZaIXnv_vHIF$3Lj~|2mwhI8gVKskEfjZN{H}@lTrpWcl!D&W$tHz)2;?rrYJf+*Jpha#g`_KVv=gCZ&3c2|WfKYK z0}}{|SUN3|=+dBWMCobud=LmvPvI`&U1&rcK1(`*I7%VAJ)=v3pn}L;fIebD#GO?j zOC}(;Ni=3793D*nciixw%*CKV`sn>oPX03jwJhs0;vm;5Tm#iO3jM|l6F1^eH7*p0 z>Q7yTf6I_6P{O0`jK^U4H(5Uk-Z(Pw?pcAcdif6m#A^Xm|J>IY zPy$M~RH7z65szJG_S@VT-+()zCI8QL0NvEu7aEnJOLg33)fUshS~E*Twp2i05@efc zN#ZV|mV@rcB7ij0(==7HdK*i4?yxVT_kY5dAixjM9^3oY%wJ>h9-O(-mtG(6)Yiu# zrn><42}P0Z8rK|%Bt=CXZg%!>&2*KPma6I=wry!@zr4#6*03<5;{r~982n0Ck76Zr z;-YbexqO+e3 z;_SFV=Sw8v1^htW274%8|M?cTj`j4zyD)1vCIEl;@kX4F&e^2DZ6n@-j}?g^BRy zsvtT=@bcC}jb6omd7pDu8x?Y*W$eQR9nWq@58-=GqD2`69Srf*E(p#YpZd8!{U>h9 zRql~@vwe>MBE^qOddCtvt{Yt1lsvE8f4DJ6!x8ia<~^?c40YRZu2!a-xR!U^ zk~}g7y1N7gD0zTR1+w#OZv&Eadtyu4j(vir_iL+)@o#IOS*i|knMXJKd)GO z-Nlx9hA}yMvyZ<%l!0J)Z*cs=;ykS1)TDm9kJnrUwY#3*t-X_l1h&7d`-C!xQUMW? zL6BcL1v#A7Nx|(Pye*sDKFt~xIp<0^HxSxKI4WwnlQR?dmXprWv2J> z6sj`IVBzUGpxw3$Pzr~#hUsvuXBduN)BWJU>HT^tN~u^?KFd378}#dUQg% zVRrV`^LG*%-o;YnP4KdO+zW$0&Of?Mdua{YvoXYGr#8TV{zI=TP^7*ysUW_h2!QYYn zf3ik1%c%>(5F%p*0}-!7U^a0bXXT{Jv57Dcz-uy#&p^o|%BwwYAx95B)N#l7NK(y@ z-gmuSxxJLcnGz7CqF$n!pQgJ80ts^I-~NdAJCt_?fBD%RY6p>~JZ%ip9ThF+uSY|vXn`AP4Y}4u8EXA}8Siv1< zI9t|}O{3#8c~fvE-fk2J0oZs4%AO$yTr&I13J~D~jKN!LzpcJqzzADT(X&~N6eDjZ zy!?1SFY2Z#B?V_)%gNYdmjW*tOO(XE)9hUQwEp({=Sln7QaN+n5HuppI-ffTFOCPL z^sV3WsiGzg-Ol2o-6;)HHYW{DqU_ztwt+`j&GaaRNlAL_mk0g&0N-H#08!kZzs}LWs6(0->8r$Dj!L}{lz`f8w>?AyDNA-|chK#Q z8fUo+oCbJC5%1zMs^9ioTq;WGbr8JEnw+P_xdS%Uzy5HKBBShmeb)DJ^{Awno25|P zFWCZqy~%+YICBpA2vWMznpj-5FRqHpGC^VK?W^RyY@EecchF@I)Pq67-Nq8b<@)W2 zth0L8tzDOhtVHehzCT<~+w3EHDFx!q`a5ti=nEK0DuIKaHd`i0JR+OJq-DxYHS5c0 zfn>H=ieChsUULQwvxgK|V5z5ka~MT9|4ygj3sU=xCI=8_5)a2-tN1P@y@FG)7LU^F ztGjugDWGK$%c+zUVyrs^gUX))K^x_hVzxU)ZQcPhRJhBg zpHh<255Evp`CpL#yfOG+EdSfq_+P^KzbXy5O2Ts_5} zDxm{clzk+hi5$etDkmvNgQv+rUz~qSb7YV`f(KvoaAf+`KO!=xwtSKCi`9n4NP z2Gggwd3RM75-i*GlTwKJUC+!05y?e8qJx6VK|qJP65*;c#vrWdoPcnaU%(6c;)CdR z*cMYJeW<804BU0;dcm(1?UpYGJ%3@pHS&br#4c$c7*R34G1uUVO2EMv%(WDIUE9sn z#HFkTUgeoj&p}SM8|p7>4D{D>n51!-Hjk-~=iR?n(R5lQDRazGxxU#S7mi^}O31x3 zs3|IzDtnRV{VT1-7W{4T^X&_g)T}~B0=|pJT3qwf)xw&!+9xb4QPmr1$Nuj496P(! zhwTae(Bsm<)K{Omc8Xu=c2UH{E`w7%cvv1j;?WNV=Oz=rc-z7S-Cfmf1B! zdKU+bg<0Pp2O=9vX482^XuVa#onk3&N8%@&Wtxri>Yj+*_=05IG2p|j7^>S!rHX86?o_9MQ zl_Vojr97K;EUMA2yI2Zn({h{*k4n&{cbztOY-bT7&zQ`SYd>)Z}5hLS;hGmEn2oF_?s2)QvIO|b-4F(1)1n4tfuqj);Bxut z8-isgmdm;GGvh#w-a~+8l+jlJMru$M;0ID0dGO9r2}-EK=rj@U)%CNSL!Vsw_$dMF?7R^MLkf(1nla%ZD8)oe+=YpsP~N{JK3J zwHKyc&AWL8>!*{-4Nk4T=cR^xY>@75-RvZXPTAfo?I3(sO4D`8$+}|;`dXuXRnMDK z-OMQEe3yi!vsS+d-Hm}%wM(+W90ed|B(CtrevhwC?l4Vc1~3m^OI7_m@c_8jXBa4& z-#VE)b%3XFo4MHTV9OPM;eWHXW?N8Evu>4ajgy_B)j+@#<5SvghlZUlP z<9yw@P{}O}y0WYD)=Qh&c9U#y-@@B5$6K}mrqa!SY!Yf*n?Bfq3q z4hrLimUV7Uf<-Mi>?B)oy~=eth_mcPKPB*h6nSb&cyJ+*V5Y0OT{k~VfhKspVFoEK zDm&k;+~Ma-Yn09zwr^&!*OmAEmHO&owCZIa7mgP)km<(L3G~O3`6{q0mFjj2Gc_Rh zOjMaLKRP{c68x(EhEnNbEx~g2G|SlPY|^mb#GLEk+e>FvN+PNBiu^K}TQj@RbD$<& z@H$lY4e(~FFOKpC)_y(fvrms)>n2-(AxI~_Kq%9|+})Am_#?UiDVo5=xB6XU?}+pe zfz*i*Zu1egsDVTX=UR=v8W7lp7B<0L+NLe5W;xovNbyvEf4IH+F0P#4WaKL!J%A?; zEsYe4ptK)(E=!sBADU^Rb_Wu2vj_^9PnElwOPMvAhK*= zrx%PCx#n64of16}Ta3Yuc82aQwz(!cOcmw&fQS6pO2yTOe4q6;&{s}8JKW4pF-V8d z3rD5c0D8|F7&nM{gNrms!3{pB{^5y_oa9!nRWlM5fcV;I%Rn1;cnoB6&kQIY z918pBDWJPnKQ~>jB`j^C3toS%=HG4y>^(#N5i@89vFixoeX%4Sdw&4D;>jt6Tz>%Q zv8p8Y29>h8ZNMnYcQBIKp91pUN@<#0lyID<8$V?I5u|%f{oSbCD`3d+*rbS6-1Z}EBMvWT_e$x> zV!)aybb`*q&Uz+{(jNstdmH4&W^i&te#I^22403K?gD_t_Plb&2D;SwVqDwxRih__ z#t?{$E12xgk%#c%LU(N6R+s|g<-ow?HGFi#U zEFG8Q-l`wBCeAfc%Ic{TiOronZTBbJZ(-+h+KpRZDV6%CEd>4rbLr*!K2C!xQ>4ma zGYV6$vXaaBkSm97t7}=3>CSxsbqre`zd~IWJY&-})8U{f$@oM^cCrf(z+xoq$&v_6 z4JVL`ow9xtK0&90bZfwB8h0wBK?kg}r)|?IHjo_W2@0ZEpyPq2{phH5>*J1gInkT1 zPjpr728H!%R-?O$xPeWVeH-Xz1}yO($we6ggjU~DoiBlUyke*MwYje}t-~|sMo6c< z+hyik9F`?*-|GDnEF1avYPa`xh!SG=(SKO%ynC(J4FH$bgjkFWIK+R-N&Qer^?%)@ zzav~P2a1Es^_2A+BEb4KxoKSTZ)%;th4lmez}cXh{CBofAUT$GyJ9`XQ*Ornyh#tR zrAGHA33&q;YsIc%=j*Mq9ySsggJAiIOpU6-AG%2QblS4XT=9{1qU+wI^-5TW{_YKr z4A}I@26k4+uza5Yc!=@FySh%0&d0TNbD;`1F=0^pi zH}&l8&$W_{SKiJRnk zoQe(}NVnmf;B48}snT}rxS_^ZTZ`SIsqCq(!l~f$*%CeB>n)Y+Hl14+bK+R>?{PxZ zrV-K{!7b>5&!aUgCe@fsPi{^K1{t6k_KQ{N#f)EZ&rr5=JBCZjbyN)#6y=sDudBQJ%}Wkrs&Mo|H1qCZ>|Of;7x#I8UT)Co{W6O9ztK%&B$wz zUQQ^eAqebL!=lFB!)?qIg}fW}X}Klt6E3?-(}ZJCy!+z|t2{ zQ*)*-9}t?X#b7e(Q&uUWlYYd$E&oI*%j!CabIu9KNUoTzMGm8Hs(qNtAKA#wCHuV3n_ts%iuTTHDA_}6E64E6p4bmkDNOzY?w=_$w zh=incmy~pav>+k5uyjjz#}d2j_l@T~pFHPyp8tN=?|S@eFWG&^+%t2}yyi7?n?b1i z7t|BIp%Te)-cVA-;Y!E|)0Pd_ywa9BT`F7Au&ZwFDEYG_m9I*_N?pV<$nIvVhy#_X zZqVkzrw~>FrIqCR{i4={q)k1S(O7GNncRp`K>KLFqX5k2jp3P)GI^(v!e2SQsrO>U zn7Nx#QsFXkw!_pK!XjJM59zX>aY9uuRCLU$zA2&DY5|DBj)Xy2^yW!i3`y#T+Zj!$E-a%Dc{q{5y z#-+c@=+cJv6d)Dal?~~dY!?ATZMW_kwMlg&$+*v+|ET;FPm`MG{~G9x(ftuFc!i%1 zzB;$**LLdnDPIWKg19zr`U1)zEEPd+aH3o@O9kZ3JJsV!D_;yyP@}|nIETO5Xe4@f znx|V%(qneT1UMz>n*L9>z>)t+Xl8l^0?!OLt{dm?5tGz1IKFZEtGw1VQ>qUBhr0g7 z2QeB9^j>c2T)n?hr$Dc=0f68*gjdxsNV-3_j|^M@JtD>(F8-A}4Nz6;^%2pP z{v*%)A5ijO5CAcU1#M{lVw?J@KP-BHI+==66#qh&ajiXP!Z7IUx^&3xA zZvDq{|EC+My6Y>49LrKT&SG(CgI(EfSl0A4SJjpEK|@Zh5}~?(g9%5e>-8^UBE`-t33XWCd;w{$mgMS#?8N@oc*-GTIoYK|4vPkp$=H9ZVKN!zs~bf z(YfP+FFVN;{vFa%#_~1K7K}>#pMw6Q8x87TYXNOB`QOoksC$4G{GX7o)#h5r*r7^l z|1%5V{{;SIK zE~!{Aqevp~^Jj*T0et+S94CuA6(V18qREvye_N-f3rT zI)4+WXt;DlB90CY+v?sBtz2~N1}pDs!@{3o85yJK*`nb)>YBKz0-Eh!vx4SXeOjI4 zOLQIYWz=5#p4YE}g`RK^Tao7l0E7#m&=n4r}tzr#MKp(31$Uc`lQUQ z&uA>L5jNP~^7i6#JLVE|(24;4nW$2QXNlbEqiaTCkbqF82uZ?iR)*myiiDa!iFtgb z@Lqt0lAB3h%;RS7VL;vPY&6{Zjhw|$Ehcxr2Txx0Eq!$n6+xzXAEXw z3vfi*f;f%5o>F-@?<7Pps2Nd?|H(aWq7-XUh6$uMKY1}vls&Y5+e~+_a>Nl(hN|1X z+#YgrBr0Z_<0H)@0|CrCndv*35gTL{n1~`s!xTWX@dq#G=Ev8&<_D3_3XFGkJpHHr z{riH8k>I|V$Isro4|PMcWb}$3l&a7BNqHNTMa?y(NqNfs9;$Dkup_@y41tngOp;$D zqNXUeYoo#MnC#teWRe*1j0(<6 zY?UXvo6~a_hFVNX7ge<)<1q!`{JD#o+UD`tic**227pYiO6=A5o^hzK0oQc%;sL{- zMb1-xVi&uV-hluU@N~ZlnE8AYg+(cYGlxfSNOk^n3|6l?G?LeJQkScakUJTxZ8~Wx ziSaJBp2c@znb7S~Na3axFbUg%*7ds-R|}!+JpPj?fklKKMMXeF zvdYhAO1hXN?b;lkyU-sd_0BI2|M}*lW8rbmjs3K;bn+#kb_N19kBx2rG;-1n61%{n zLb5*7&h;08InA6W*f3c$?@-USPKSBkJW(LF*g6xS zRVAwO3$Jcz8jfE*`LpFfucGkYN8YwFKPZrmT!x^NlSaAgw$5Wc zT3qfRH`RR%_a8%z}AI7}qibDMISs(_k!w ze#0UgvAW;`kTZ9S{f7qw4SUico(m=7yB3uSyf#mF($yr?l<9<9aVo#q$~<@UOBV%>Y<_Cr{nT{w#L@I0{fV^o<78-<*+!0x&s9u}}K3jP0Ch z-j3P59=r?HG=Fi}q;KbAA;s=s3lEhsRQYhN&>E{x9ka(~7IpZD z8()=J1t^{X3{c|vN!Q*4A&}$D5zvc3V&6;#Y1x-8s@+Yh)- zY=E*q%3+)y>y%uzrg7IR>q3Q-t(G$?1Wx5R{=`H4b4`GLIEldK*sxHZr|FKCOJQTF z*7tZQzBm|~eSadcrdyMP|5@OZJVsyk56M^g#KbbKp>YB|vYAa9<3OKK6(IGCtz!J- z``V&4kN7A>Pj`ce)(%s3eEO!=avzzR#xZR*C8tSyuv*GwCXSqf3<^Y!0zwtJYCZf- zSLv38{#dY8?!GEnEx|xryxqahh}iG%Am)vVp$03eT0C^A_X3FX&%46$IZS(E&d#DE zUFvgb@e3H)xvVX+pWZf&vIR|sx=RtOYFsw$ zGrdOLGHMPn`R7)1Us<6CSpP9yjIv^3K{Q;cf=#2|ZHqwIvtR46j(ev6C%O264n$nV z0raEI#6?(bZ}L&!60$p>Y;RsQX>57tk*JUOeQ!j+`^qTqp?Rfd%joNsSgpQh(yfVQ zm%`sOV+#@AyASDBV^2~_rBk2m0JZ$k0hA7Z>n*QW_ZSn}9{8(BLF>z^YH&?4b_{7R zd8dHJo9}ss#DaGE@`*yUXE2=3z0DSj2pNHEk*D%1AP<9)^)2w2a9s0 z_=TPe9n5^gSZ$n^!~sU>-p^7%r+XpZrS%TOm|E9FcHNVYw!bY?2L3%flwRYm@IoiH z>!M*ii@x64Jw4^t0^h`uqA}ta8QXcAVa%l_m}n^PoR=Uu=s4e*!H7p2yt7{0$>cxl zC2ajByE^Psw1CRVWKNmdRH>~@R(G~#G8C#0(bmqb$y?z2W3}m{+Xp%T%#aWS{Sp9y zhls+j##yM@E5`w!5;D2Y$+%t#ncJbEtQd#+;J6{SL_5}o*??^w+AhPfz2kOhe2FDM zJ#ck`c2Jp4s>f{ndq0Fl6AD0<(4IS6m~3g?thf(eYkDz^h>4HyS`1-PjhEB4NfclI z1aa(@vc4at?k@~DnFIK}()J+SJ!zd&n~RbvuVf%Zxo8}XOEw*5n*FF$9D^eERJa8% zhB1_MME_ahNOKZ>GgH-{F{P)9ze1EmL^{Jg*pSzK4`22hjifP5vsa*^2wJ=P^8tca z!vOE&%Q1FKSTF090W&)i&hkk?VKN8+#S0(BOnO50IzKA~c6>Ca2<9~eQO|ko8woUQoYZTtH3@Ux38j`Fdj!j1rvB0WCR z>(+`duzBp&%MH1(f>$YM6dv4jAQ%GJc3US94T1pcXEV%`bF0m9Xb%#MN5I7V#B479 zY9ZxO*}395mxuelarqL~G=bvUmXfp&!FBIU_8OOw>rTDS7qg|X@eqFk{S4i@wt;;& zsrH9TA|z~$$FfCGw~p(166I4?PoyO!Pf8ydf=n%h5ZfU`Kymibag#9DQQjehce26p zYR^Xp{;8y*f-|5It}DcMc5vpkyvFgS@@Bau4@>L2dn2UI9*iVU*AvZ+g2xTPTm(AV zZbgyaI$vga^Z>kh_MSr3Z#`SKh8fbQdlf7^6hMOoh|{iRX$~mBbLgOFuD)!YFKMBL zf?I|wCUZ%uLCH1H)*AK`RRzc=9LF92NzRXJ#_Rw(RDq&~W#TZm!qeH>%&r2YTur8% zlkgwgc9o?s9>MuQja~;!?QE>K&e8MoQbI#VQ#h(Pi zy2ri=k=9`i?M$3BZ$9^Ee~3Mwilz-M2hVw!&v-)+<=2gDp7t%uwgq{(d)_!`6t4PA z1cXL!T*>1=KoMy%uq!o&LJpj-_mtUb07TdY=u~QY;RQ`t^WmU(trR5fXeF?;Za4P} z(l^j}Ewv^@(EePne4V9s{-G%UMwJ)1^|oh+p=V#p>IOX$*6V&UI==Lo2Fq@&OdcZ{ zFHXfMJiLRTSfpAU<5E6#-!k(>wTlN&i@?^5RTor|vjONTpuamH!}oD$ITF zp0Kiz=EQmf6yxCRM?>Llo82a4=2?ejOLzL39xz1ax*aRiJ;ifi1EjsW1QdkAc&9nA zsqhsI-MjJ6SLaKbxv?kl!t*YakSlwOGxx5v`PB8<=FKO+oKR?W{cNBz`_1E3x%lqX zObP+3W8eH#h$M5V{Oqt-W%B#uLeth+|JOasj6b7Cpi6@!tCG?|*LB(@aC`pls6@N% z?C0v!(&9U&7?3QhmuBsrd**?>{Surx7(Lmp)mH~`81-X+oEZ%=qb$+Of8W{|EgZKw znROU%0x8q^Gh$7E_qN$G%;6nJgPHc7cAgY5@k&$)>4@bo03#>7M}pWgCdt-y;)%$` zSu`B(d$|`|I{cZHt>sX+THp$io|sjzUDq`*I?!JqE%liZ)l~KEa(o_=`MOc4QJc8f zWd2ro5Z?C}y{^-CiKP^v1Dvd5Zy*cpXx53ZJ7d%lA52is3fBf6Nrwp9M0`{C5`y< zD2Ax);Euy4GF6*2$82JHubs7l;$y^(7$fBsX^%8UhG-N%M*SF(>z-!m`pJWa%e7~T zcpnAeXlR`qd1>k?Q(Hji)`(X0M+7I&{JQ#wsMM>F}S za5KEoZR)!NI>NC)Ad75V80(ky!!OGfE$uD4%`J~&bRORuv?=pJzSbdO(aK@HM~X1~4HU#Q3IG<2Xw#B*BhR zfrWc3h^a$-vNKyIn#P|ETCv4C-PO-~RmW#LSR!ao=xIiin;mJg+W>yUlhjvJuE2w$ zsH-}-1d!qer0WvdU%k%W$ia{s%@}ZkG^z8YD9}_nI5hC{sJ>abMB660?p2!E`6>}U zP-tL3do@|5ncPm^uon}76OF>;PkWNXexcanQ`n&vF{77qvAIT%WEh)!v(>ncsk6UI z-$fCNmR8bounM2JZeG1|AL94d1G-P^M8XA5^Cm}Luq_)6lEn&t(}~Axt2gnp1-dzJ6pa-4P_beai2GjN-aFIDp#Zo=p#NQ9-AH1fF_O@U3Ag1DK@kK!8C&Q*mQM%Lp8e6f_^Qsc)04j5INHKZxC)M>ncT zEXf;^hB^x~Di6)5op3*0oO#Cy5#fDI?hW6#CL99{W{xpEx8e!f%i6-61r&g}_9avB z4G0uX@uo6-3x=bO*t1P@&<4$cXUq=?UHScW&R^;(8z zHoHt&H|?TE9>Xg;KR?MS|MTad!mH!UvwoeH?|B7GBMA6CiN(fyY+=A)xCGHW-g%FP zH%tM|BEw5i<}+xub^Kx-ckB=X2IqouzqrMt&ntQbmn$WJq`M6eb?j6*oMB@zTnJyn z`v#Rq#aj^l`J=$#C4FGF7421i+^{-Uy7eU)^uWFD%oo; z|9i;(&(!TYiD1#EysgGFZ8c7;2l`eW-({9OAo8h4iew;#`tzFwAWFdFIT})6t=leG z^qF!;guOcV!B|n()Ld`KiMV}ma$e``jWIgohWysCRBC50{~ zhRYr2@heY42M63LoN}~PU#=p3JoK-pxEV&SjL=_Dc1X>Y(PceKasj#T zFP+c$CD+iO(Wab5sr`;|7IdSqKxs#0zS7n+4qR*w6K2070k57?$FB*scXbiB$X z3unJ!{TAwCqX3YL=$;u6a;93lseQ@D8=mf~e&xcf{FF5MBJ%WJm+mDxm(#tT$Dg=A z$?Nh=JAa$-s=RrPfS5i@oUQ~vX~G4j4pPSHczl32Qk*K7*(ASmfqf{`X z*Y$uqJ_Mm?7(GQD{pqxME%Bw1%VxT`uaYRDs4THmjs4SR-ez}c3sLl*9p}q$i!az< zSDu~fFKXvO_CC8Aw_qXHMDHe>njJQ&FeY-j*2sbr4NS98(sNMxOpP7{Jyhs;-}t=T zvnBPNWiJcQ2is<0HzY8FV7XiE6ApaBwXi>UJpOf7Q4urGoCQLN&j~`t+fJC?m>4k= zH5HW`a{=+@%H<|LKPT*1JppO;PV4K6->dPHBx&J@oWvc^av zpf4pn-)b_`zBzU;moaj)!g&MAuc=pUZ+L7j%3m_J7C9G|>NZv5-iwD?f0A2hJd*cq z(bv;H{9%M(h5PQ8hf0i!AsRv!zXx&-zIb@~^~m&fo682 zy!=EcG)o$0iMd<+;)O&9?~ z@;t5_(=8;7m^~;(;?)jIC0%S@cflvySL*4%Ilw=!1IW*UM&P)x7e~;KT8BGr5s{Nd zoP_ecQ!h^~YcW>L4DuphLmiQR5z8)`!s%_wj43f9`NEbmko7R$`S5F`%CO0!Q&Z25 zU@iT!(EaRcIyfE+bY8t!gZUE^863M^aK}{jO*T%WJ39$}DGqtQZZ5m2nzbY2F->)y z6<_XPU_nDYm&zXL~H&Lp#gkNNUV9k?Ota= zVG&=KwccU(@eD8of`1Zez;4XlrCP0_ovVJh`G?GOF~J8Do)HFzbJyvB0}#UWA}J6X zVi>RQ&sdF5ycynEI!jN}Rd^isLJhr?6pvwiqLO^tud~?7Rk( zhrof1Q-(bT>Ufqits!{SGI9cj=qBoz1IzS!tr2Y_DjE_Tl{^ibbt7G4BM|Ob z^@~1WcIic<9M6fUk6Oo1_}x+gUyv z)miHt9iK@67c}3>@1yWRCa+IAHCGWK-Jk~!?L(sBuk2eMhOkvik)IL?Zx%KpHsIlc zHpxnnu{A8eB{9P2c<&zPm-=V4_TkF%u!M7~@r@HJsAhapB7ifZGFfd&P_akLuWHuR zc*bm=HCDj@u@GEu$0EHb0{}KXCiF@EP|@3{8Il1 zR1DV$;$r*Hz`SmoBjuge&;#(?8;unAnaB=YBfwc2xJ`4kjjO`|Zc?jp0}zAswa2~4fg5GExlYO!+=z5NWM zHO%=<*fyKzJ(x~2q@sMLLXNPPXH6<8YCZGPuK(L(MjOtpsNBZ-iyl#v)@UOwL14~C zNEZTj#dV#!z7|#W9m%`Sf?vE!YIBnB8ve|^q3t1}u_Z&5`Gl>3GF)Wyqk7R7^`;v4 zH*sOZiu@eOvKxI2^4;q0XsT}*M_qs+yac6ZtyGs?Ep`p807fb9vkegDA%2|s;1+=; zl;W}FwWH0 zxOh+LrCVDSg6+EEfyn3#e7cJw61)_0>&y=|DD#;gEzS(tJBfouJcg^l#VTcto52kd z4sNm`4Gj$vWv{8I?h?%0MS`5R&0vq~81BMXhkFOE|M|e8P+*4v@g~79GUVOYNal{j z8|ZypW#h=^e8Sqmes6Ht=9s|A$~Cf-VHKq|)jKntmpz*5GUw9|?duFCr#=Y4R<{Bf$oK#F_)_O>{3z)B5Z~d4 zU(OAZUh}}c?*{f173Ck~SS?kz-if#I>h|%4#2XzRoMe$uUt$QZ_Qa+}(aF3XUFI_m z&C0r!Bb!J)a@F`ApAFW2llSx#4ng3u!QtM(2Fa0{LMlP}G2X0xZAp$%jjs0MN}L!c zo%VgUpYhKJ!O=h|o%hLzc$QAXEo8p?_a=O{tK5t1$>4&SH5S;yQjg~Oo%@R~ z2~c~{QStuhwf`xMX6&1&yw%mr)NFuEStFf_aSS9@Xqys0QG z!8JXSh&jvT!`-b*5(L#hukk0c{rwANs$a)=T)l-T=FS6PLQjZjn)KXPNz$CCy%%3S zVsAXc+ylKzn9|xJZfGWHA%4hu{|^`Hzwd_LA~wV4!0?QZv-&+UctJFJt@{X5>70D` z$ny&@F}Zw0Ik^J4{7qTD6?=^S=8de-^q1oUV;pZd-t%-3Fx`Ps6+YJ6GF&4Y+r_Bj z1ZjP{pm&GA-)HP>3PzM@uY}tBJeRKX-JcrRzrTkzwIOm{_Z;h&$!QWk5YS8&!_$i?_Zn6zpHfWjd!Nbq!Ye> z^DVFa%vs!R9OfeyQG*$hKl45RE}Eat>8;{d&Dub+zj(GYI#er-0|edfl3w!%%;{mD z-r7J=18;;)Blw70^V0n`Gxo2EdTrw{$1&=7ouG0Mk{CeUgE3H|IM!!@PM`C zTX;kK*H8IxmlWH62>fW<(=37GfAcFoK+7L0@}=DQUl<|IC8~|_d<=*2FH~)R2fYz= z>1!A~ehH&sTt8Z~U+WaFi~i!Us0=@=SvDbz*zUM@`yXrcJRY#pPjbZLzcN!Pl!)}E z%yIj9l4>7{2GU2zr?Y71j_!o%DG#T?6ujXL?|(C==TUDXBG|};+!KKF_Fl}*i5xF> zI8~IFX99SOavXDF{+oq&>Y3toFY1#{zO|2lEcYnGCmeuS=h#qD{m^B%cf1k-KR>!zp5u8F!r)!dt{)cjU~);aCz~DpuVv36MVadr zw=ay=liaP{0xk60-jCo;Uc--+kvs4~;iPb)yl04@{TgzG)%I|vlw*2cSNjb~bceE9CV!n+d z1sYC0@oc3Iyt7vY8rBx4+o0nsgfy;vbc|FusrY22`RK`qbw%g?C!Bt!F|@JN9(#Ph z3?jbD-n{yX(V>2WHJ8rfcO@*lVQ!~_VoJBFG;&Npw)dD3WhQGS{MwbqYa z1ACVZTCA9j!A#o()*qWmkw9%%Y6F@>0|tYWgg_n%1GOXkxdXjWuj(E2xFM?ytY4Wf z&Xe1$28W;QW<8*7{sAkDXLMfPdum6k#re_^bHFgMJchbbmo1*2EpidB=VbHx$sRbd z7KFeQmUmuKK0a3}jyvn_>5|>;91K8&WsOR}cBMBE8*hlgs2mV4A>h7WHhJmj>PmnV zg)5LxH^6R?iTbqJw3Kn{3YLEC3=IbPI}5KodS{Av*XEZor#Qa*`Qq&+FpPD?XK0p+ zo&DoLYJ(VDZx=Wgc2{V&VNEViF_qRfqa7HnvC$J-*ghRZKtW*yn(d0XHJaI1osXvS z!PU1nzVy5KG3&cMJs*gEgKMe?iD^K^u}H`%Nylxf8@~3cZ;|O0o-h%ReX;J-TK5Rx znvuQS{Rtf7uhRj;RtY24SUEV^;?>zU@4soiX41kg3v6I>`bGbxL^IXz8N425F4wtSf!8x}* zRU=56iMhR!69Frf96D+gbR*HFP;y?6y?q8X0#Do>-5Y=#@X9{mx@3R1W}4#rjvJ z-v<{Z4eAk!;o;$jXP0h!D_#$173K5$L(L!ZUJ!`9NErO!b6Fhi!3Ic$33Xp21VbYf zNTL`}2wC-GMqhUXGb!Ls0pe4DBl&61FGVu|mlqi*{ozd*q1u}1PCqL0L(de~Aof3Y zu%98|zl3tXf28aI+TAR(NIaou4$hnHcQ;2%Mfv&o*268`l4@*Z_?_0`6dL9Gij#RQ z+Y*MVOuyY>(y6bypF4DleEiP#bZ4%ZhfLTxXp`F_W7o;QE0NbT?taVxgGR8=#$e>) z2Qea6`}tub(N`RmwThu`_VR9&FV$^li3RV1P0T*)adKLA%Q05=F%O(_?f^- z_T4M)cC+;?dHeHVUQ6VaA2oWnl#ljjOa=^$3+4*wOg5MKK=*w88Z9+6n*)=s;?xju z84ryh9kgNY1REQNrfdiG)D?4lY6pvw3_2AijtquU!SN>x^QRD|iv8t*R5?^?9iQDy z10+D4qt4hMo~?6=e?TYy;(8O2i|dP_e1=k)cc>}}Olo;Zk>Q~d^;o8zbOy+=c@0h} zcRoNQ%f>oF;y)+=&+&N8npLhfQ$jju-ie&NV`Jf)v@_uE4}{VlfGzaOUf$$i4#cL| zSRXQWWMn!0p2#IwS6A0#+(Vc)zSI@Ls8vEr-5{qb;JS@E+w2{vn*Bo29|J#oxjQws zMANnyG_VFS%Fa_tFIXbud!sy6W;#{HY62Hg6ZFZ~w$5(i^?@8-E?u0E9O~9Nt#OyL zH)EY%`w+FS)6>#(kC1&Ftc}M}s%LArzM{rq9hek;vzIXKPQP9o6cunZ_1|9VIq{6Q?pTf*s;M0^SRz=0^nUz+u5oQ zR%0bug5(d;(}nJ4``!xta3gl4%4U-KATy(MMr+k|cjh*c(6>uLH#^leFyaC{QmlvV zftV^=j)A7%A^`Y;jC%%_m9CzL%a%arZN7TmKm=#ma+`neWEeX;`2rmqb`DEP z36TXf+aX5VWb>}IPJ^o-pIcJ5rB0<;ZX``$_ey>h8b~L++?d`f*+kfWPM_O-|Hdp- zg4Bf5tj~R_#x_p|2?RZZ6kxcb!_pmdE@FlyW@HZ`U zLCLr05WEE%g#lAlR-?pIwR@Bfmq|~zp5uJ1^$2|u`$Q`n7#O}e*Hw>|L}s0S1kFL> zh>GUOr&ws4ZG(~4iY$6OG)u5VC|i+KPm{~e)CXW|OGdes&M47s+u&N9FB2_Q&$A6! z^?dW!5JJd`f>k_tDL^tL4*Kr4ocs;Gj56s6k^0loLgWylZ|&s^R!Dlw$jQB4MeQ|H zyY0^j@pW4X))Hl*{P6~R*?gy{cbLC|YUM)IJHG&HJYB-(O8Hfd(-=d@5op)5-8nb<% zqny%ls7z*>n-TkDJm@}8ZO7Bgo*zMuE3zThaStZd(gv$6D^1wHN6{jc<5|g;;YEnW z6&Fd&7|X$ol0N9-($bT6?_5J{Xn7^S_VD38;u0s@T_0Mq%D9h~g2$@=@e8uVz%%HU zr^oTSg45c7a{5t`j*|)S*NZpo_sm&-gmNeI*y3wWOTJNhu=oROsm+&@lvJ(C;#H!h z`D6vYW6DyO()SWF!J1f|*RFeW(tz>j06ZDB4JbKhNT@5b-I_J8-Hez?RkyrX3n^)p zahF|UU7k*5diAWr<}1*ELNZU(Ui34+fL;gS5o8r#cXJKk;Hlm%9m19BuUJ(M0~x`+?_a|6e~#meF$_0&5PLGwGFa6V-d zxex#(%vHZUj0zeU2-r52&EkwJE?z)-%X(_pi~$i_g341ixp6K#>9Wvrbij-JW#IOe zmvVo#pJb}iGrloe2z0cuyPks{zOWK~xWXp(;dy5ibw^0=OQHeltb1{F*649E@gjb^ zUOQN9G|x@ykNao!YS+)ov!+Ta(x@b)WNAcaxcKo_FdI`;>Kskiue?Qxn4^J{3F}Im z_Ctv(O$TLjWbxM_qQav;K2rF67PFmpjk!shV){Pb1;hS)tJJ~qoLr>EL1T6HXnnsRGl!ilD|M$JOhgIP%)#Atd|*5i_9vgi8? zeto&$h?E1c$QUmfCRJk|T8)>;^e5NGwGsmkn9?wObG2Wu%In-}(zGvO!Eu9jE?+q_ zMc*u6k5?(O+q@3wvayyD$tD;m8ZMqLb!-v$cRVyiUU}9vBF~S^3w$X}d*gC7i>G?z zxp?j1^zW?Yo2tiG*E$lx6V|&k{_4fL_|e#bqcTCV+G$+#m6UY&~Yq(8jR22F;T7Yanqr{tY zpZs-#i1YTV)`N*=Q!W;RFJjHWImOaDy48=3C@6Xo1$lEGv}lyD&___jH_*z)N9h@w zF-CtMD0hR|d9e9#889}(cDt>D@fc(13Tw?3({uA5(kC^t8hv?r^2jJ_3D~_>Ou^?l zvBXtvnBCEWYE2$4%jH|+_TO#Yo8QzVRax>se4F`3?D^@o&I%UV^8N!|+3qyqG{=kiB?)+CWd2ipMIcggj870aDPABu)7wF5{ z@sqUnC8|9Sh#!g(X`+e_c0{K5(z)&G*I$upt7eMpOm;qF?et&EVF4YjMmD6#_*Xt= z@aH06pulh=4m$tU>*e%TLP zVE`lsqy0!LX9oJVa5sgU{$yjAJ~1pLqOQT5bG)OVCP9F$kd94&sxq`%bsg<5E$#;D z>#`eYd5_{fFnvBH0ev34!zH%UCdnakx)}SsUS|WXoV9jykID1C1O^JFQEM>OVcx&b zefJ9o(a7gaoAX2Ul0|H(V!Txc`8z>vcMLBW*!L3~+W@EX{73K=YyhM~$qIx!@zeWW z-5;rnyDe*K0wT6UD`O|7rxTBbo!5iDi*Su(r;=2d^~+b9D``bIh+&!&fAN}>n;BR0 zI9T*|Z&KY;+4ziFZPFWOz|4kET9ie)wh~Y!hh@-U_66+b_6_Zetf*-(Me`x27Ad=G zzN%!7j-7!%bp3RxgKNgK!sLQh{7bZ-pI=F-E4^GC^OQk#8jBvmIn_QGIUCb!laIz7 zyY3DM+0kj(d}$%yyx$7_vud2;&O-S&X(Db+cWR5%wI*1a1m)|E{0e}1=JId|6$ zDFX7Gk8QYhajcm%x-(#_H~NxDOG_dbnkv4Nz7@SOo&} zXR-}1zvD`rl2t!b1HpES&DcB&6%`eiH?4ehimELAZaKIUWz02Q4t)!ZaCs;n+2-6` zM6ZNmnr_m4pUB*qK4DUYv*{5UjKLt9PJyZL=FME9S1}^Ev%2*>;IrNpU{D+Z8!j1y zpIVBCCzoO&>E7!KcRxPBi+ib|;4{T!F{+sbi*>Yt?ioBl0Pmb*cG9^w<>F(66(IPW1N7AEA_^R`3_oJodRR z)|w&PtoYqJs3i<5>yh6objy^fm<*w1_oE$cE|z7S5mQgkeM!-TvLP?gd-z<>*kcKo z_Mv>8rmZvl$Lgqb&hX+Vlaq;N;vBma(Xd$40Qc|)S6iOD+`)I9*tFS2S@iknNAl%U z_$2T2xVz&;QxpNEGrtFqUr<6WH}dG}DTz(bv-E0r-wUw1eUIu$K(tE5!P4MISEbdMNeAuIrvaCsXD_2}c;lj$o0`?K2o z?|sOS8BTX-?3>#hMdGlfo98nd!g&p@W5_}+zdN~!Ys{(U`LjtN9RaOr^2sA;xNUw3 zgv>mjaOxMEDPY=RH`t7ZKOaCGo1erZ3kNQb)*w}22zf2E9{K=bcUb#C$2Tbve_$-PtPhHN{{;QJ@3!eNOZa`dHuL7k&UjY+@NI{0vzD6#3qGz>VF;PnRmx2Tu!R$KL0fZUa+ zf4MVV>wE>j4U(a~@qOlm<|5@>ATu`MCEgUN11XTl3!{XQ@Mh1{IY9y7)pDXt<@l^z z#5%!|p6WKx%vPVe3Nd1CsIf=hqgAp?vXm$4nkue7!mw=ooS^k$TrHeRTPpG^!7s8} zF9{S=NxB4+fr0C^quzTfx3RKNt7P!u!A-rp*w_bpD+ltsHOi6H6}HHYlxMD*y*=8ZefjC2k-1V+Y63s zFBha!|HeJ7^0mimpN`zOf*p?57peD|u6-rdkB!MxzO8A%vwkWu5Z;)nJJOsdG`AxX z_IS5#lqEI#qMIp8UnqA#E-^iaCGJ(l<$PL{ z&)rWyZuK~Od3yyJb<$eb*r?ZexGb$&%xZC_33;6*6sYIZ##J#;iaaM}jZd_it-n3n z2#OA;5S7Fv;i17gd%sB?ag4IgKJ8wADZ&ofPtxNBJtDBp8I!{+7ESMdzZbvyjJSzsyVWZTcq4v zj>|X)uWXfc8{LVDb*tkXSNmAIv^vSS%=l=5qf?92YU@Qs10_SQa)E5-yLr2gLyR>5 z8flvda5D3oT7KOZS_3*cyI~y@<>2lu9@`5pjRN)7ZQ9rGA}2p&;sVxGjY7nk9WUyM zhDppC*Lsv`H!JkvmXXz15vvcD8CMyaWEIH#6uP{t!4#vIdRqaY(He5diHm4u693Nc?_-{bX#TjkjiWDd!7+uVt>m4i_$ zPsHn=$B>4QjN@>(Prixygt}`nefG#4u@0op7qEGJ&PU55`IVd!cVU&))j7VmE{ERT z4q2Zlf5_?noTyYWl_KA?*DQuXMLb6a6RV>9uHhinS$(T*58T~r=~Gq5hlevp)AY~Y zolO}OZW`1(<3PINcx1LB<{OmWP(v~ESK@&r2M(8 zOUsKGXz|Y_sYG_51|Ph-z*Rp{my6mD&@yeokWFPlkNxVrG_lHHpbu8jfHfu zmWK#dCMPBm>KPC*B0VDZ)791zb8|3@ft1_MtD#g4;*r$;?bX&HUsFk^N~{b#cgjS} z$+H2l^Yz_mNgx+7tas|tYjoS~E%!@z+xLvXem}}E5bs;HM+>@n(2?TncwY})?8hbU zj~fF-Tvwlf!;^;Rn$Ro&4D>Koz(uwvh5sR~Ow@b6;kXv?TlANW;uIoYF??{O-{}tc z!$>Hn?gp7 zD5Ykc42Es!7& z2<+)~Dzeq<4zW33(19GyLlD$QV(J5OsQ7WXPAlk?hHUIF8s~xu*$_wnTH30RSN^zy zqacDw`~0S%$6X>*@yEfVXy=mx6O#d+GP&_=Mnf4?#_LT%GitJhtC;yOeZ3rOoyO<5 zoleD9aL8sVeT8K4G+gnVnHNEW{* zUK6JBr-jSc@i)v5(OJH}iy!{b@9)R2|C}JBERNe!RnHs4T==dWxDc)86OGX{0I%W; zOYA;~Tk9i=?K9QD*uXFvX2=FDJOZL6&)e5kIQTUV|NV}uj91zg%_f^<05=Ixe4%rL z$rEtdO0F1mjev;ex!#$EO170D5BmezkEO;Z7`{R#M4mOiQ9w){t#Lc_FwfSR%Hl`2 zayG*d&c2*8me7$pBNlDC?7G7lGl@MidI*N6zAE@Qq zBNxo<4<@ch`siSIhzkDQyrLbA`-Xp8GL zExj#f>kQ_h$wBBi4Z(OrUmv&af zmGUKXne!x5h%aZScRBW7<#a)|C;;*pDaV$hn95Y6Kz8JRjmi5ydJOO$(1k)NBU^8- zOcMkZn5Rf)H(q4E)|smIY{8hw_kL165K9X#&E|NRyE!G&bTGbSjE2&Qa*0j;)b7*! zyQ!aLmM&ssi|~^FE~FFc87zuSUe?E4RWU!MLGkvv)~F9QEBGI+Z55 za^~1LrT)SHQycxSibFf8FAy_q>@;9=n2U3m&~3l_=T|@J-T*HKSuLwTp7c-2~@8yfYyDF(7Q!nmVlf z&fq6TRE)m)FJ-TP{TM`qqUd^n!)&~&Qg#pHAO0%H3ZVQvFW(2SHA`r=Wt9&?>O|Jg zu0*{c2JK}1K#*3EloXg(-#`xn8E9M*X%X`jDFNTj?g&BV0{G6u(hfeSRlJJI%3+r# zugQ?Bv#eShpaMXU1ISZYdR(P4nl?mGA(7K}AWc}>`{Mo}DTg|c{wgjAIEF^r zHF{}={Z-rJBwl-9B0KQ+vWIA5L`$VSd2R+zaE&THLm^FE(;YwR9}i8O(czB3(BdK>Rss!!Pr8z0+q z#OFcMB9rx`9A%=XHbGuImE8Jf98%7N$hT-f+cp@=E`3@=oxyT;2~DbN^g0jTJ)g&|0b44&X^&x7VQIe?h(P;_g% zjmI>2*~~udR-WBlJp3fcJ^X4*VmMV!q;Kcdv zX?!(A_2+90Q7WBpMjGCg-<@t{6)FEO70F}=YhMj3DM(5scGVZoGXF@OVIccvtO zv`y7dsj+GA!%JW5`sOJz@`!s8U5K-%uG`Ny=UXCbNDcy zY*u@etd`kKSDI5Or10qs)B)(>)VJf^SwWznj}1q_M6Js1n~$ojW(jupVWCY~tbL{6 zezNkTRbr$OL}IEnd&tja)|Yv-#s(n&l^uaw0hEFJeEEJ-60|b1Zi`>hbMBK*B>NO; zMq=QXW^*Un*lzpXqBoN#lH2jIEIC~;|6nHt^Vbjz-Rrp-*O9o?ZSnJf!#bgMo5bV`BfHcI+{n%m?h zo9u3iyxWG#Qm*p+)wfX|!_D;`|8LI66EYVkux97encF7`szjWhu|ckTCJxPwUeF+p zXv^u-*&as6WcJL5nA>|YEv_~Sfe|Sg7#f@TUH^}>uMUVKO!5tc1PQLeErUyNhv328 zHMqM5cZcBa?lQQ$OR(S`+}+`IZg1uG?cTe4{{b`IGjxAlRX?fv?&t3n9*f*2AZo38 zFzd85OedR}&ci<3cGzz*TdvkfSSmd5wtB_KgmhBb_uaK|T27f9{H(~YI9&AzGFjGP z+8a&2yjtHNU8$BN(JlPC0m6*g#hV<2DEL#xR#% zPfouQ(l~0SHt!XRzr3{8Bzt)x5^?~(ip-p2GC#8trDq?qNc@}+Q!`GRZ_?@x*`~{y zBuwT`o)-np+UoA^C!2a+DZJXb&2Rx=_kRUjKpxt_vXQo+ml5fNM}Z!uAlSg_Oa9v% zJO8*GRd>;Mw0(SJiq*ic#XgK}Znt@M+r!9SjU8UE5IAfn^nVOB89c9fQmMc8`spZV6)ECemr?+KsNRo{-f#?r7UeeOjg?G{_es9QW8fB=0jd%cXC1R9%zZIoT z515+)Q?6>kl0cvruW3Kq27oVHY5so-VhE(k?xT2O&ZmQ%q@n zIkGuJ?6^}uRG}wE6qkjge_KG?dNny~&KCSOs1`zYk@xNU2=obKYW(>&egt&+NEr-% zF0I$S(xZcVZfzuBDH|P=9ZV~{+WSQDc-^F}hGkO!LO*#0#EH!>*;VLUZg4-Ldd^gP zM4yEPg>>WcyL~907HcfAXj9k%%;w4p4u?-4zv+eZ3yK^r)^TwM!NwX-rM;6dy`Nf8 z%(r`Yya6-I(O5Pd6+y$>f(^#}Ib}$_e@8sBrZWsh0&LQYuCK{HEjysYBb6|}>8Y(! zp|F7(Z+aTZ(hH$2%v18!e2anI+jKI)Cs@CwC5t}{#xTn3tVUG^-gw-Tf>fjQK zj?^;(m&w1`(NqF3j4|x{z5ka~tWOoJjd!phq{jFA8^1y2=V;$igCB@X!PSM%n&+ny z=UYpt^lkp(jMckM-wOV+o_(zO0Y9VM75ZGo3K4kV?2J?dKATJmr&wh;mOQb{zH>BR zB5{AFn5h?nQCuvZDqpieYyrt~379^=SglWLi!5dmkVvx$$||EgIXSUQMmiaRplyA6 zZ(OqhNQeO!q4{=kHJ)u7Z+|V;nNNkFksA{IOrTxrtC6EMSP#8PaXwjAlFrp^vQuTr zELJcs+8s@0^SjucAiw2EDijLQW0W;M{B=xnWudEBBefe zdSH1*_hb7SW_e4|f<(Q=3T5VN#_>ZIlf+y5o%|=^VGa23&%9B7+!yFHf><)Ag=rjM zY(nMR(wQnei);r@#|J#A%(fq%uXX8DLL({_;Exo!PN*1&cn1%ne3lB~Mb5W)U#E>m z!De)LKjbPmUJa@OS?3LZ{$LLSJCX$aJBWuq?bWC#q+{THJu zmx|QR?7q5sYsGrh>qd4*QEiFIt=u zrlGavHBd+s+2SCJN0aFIHEk-Lwu}x=;;w-(WENZavi zV!M=GY>S7K$@>_A9pDK#13Bw0Dg7oOGkNoG(%ts@<(ym=%HXyorq!aA#ME+z<%xt= z)MqJGZps+-M|&wBjMeMIp)qC+tnD0ZWa&^gXpK=A5#u&?c}<^es5C!k!k^98J$jGA zw{S`OWlan z)Y8F(3*6@ibLT~{ai;3Q9wD}*o?TAC#%!sI(Ey!t*=G!DWyNU)9S>68D(!an_{E|z ze=tDs)U`S6%f|u!2W*-iHT##xQ!+pbRE`QN+}zzQq&tt}YB25pNDvD?w=u#;3~sX7 zAYrk3iv3Q>)=iTJK~u z&Js6F4B|3*#Lo0KV)~w&!13^Wp@QKZoZSM0d#Rs^F>%kY= zgBs@JrTQN}jPJsfX2uc|_2J4t(qlUwHyf&N5qDx<-&?0qoyT4G1Jy)hnm5BGG zAYZcI9;WltOG5?a5%=_$cyOqap6!3UfBDGm-nv+!m_VB$7U5Zvdo(Nw2%klUPkHO` z3QrlEorOD6ejRhOXk}%0tGA>tyr1bZR$63V+r57H`F(TcfEs^^al!kLw)ziH`dd$TJ)(Tk zwmuSMxnKP2ExF_XJ{u1lZ>JliNm;p!_JNNupFk&-Prsl^U94G;};`SHoaGR?~ zkCSj80s9=87Qg_Dr*k=DEPPv3h{U@$-5p=4k%^~@7h;7Lt$SUp3q(pqI7TG7l+bn* zqtM2r%n5jF6}Nei!e5+X!9fj^NUc(ljQ1fbc8G|f$_MQ9G0`y0Xb;lxuCc8Rzg(MF z2LVQIT#eQjAt$Fyty=cEDq~=mhVuErfvpK0S_lYUje;XP5t58bCSYxL4vwkb>Mp_) zdz+uZ*p)@f-)?{e-30RXXIYDNRf{CK`qz~ekd zO$?R1Kr|}zXZ>+tCiSowUQrH0oZO-);;ASBi9$?0jfU0Al*Xcc?KM8vt*Nu&b5N+E z>E8M(VEkb#5K9kLel~x2&gJ2tI?Y#$&6I4HnV-@0p05#Ra$Iu*czit@!ed3l=OTl>so-CB4D%9OyM&3ebQEbICmOrmb$uHV6sH}cf`X#)aSwHkb^X6@1la= zqQzbO7CW;OUhelH#DmYJZEtU1)T%}gZ-McK9WTkklT7{fWGCeZeOn0Fibvk0(`hwN zcKUe4D~wvXT{CLk2&YntaO}@BJWug1uNFvAAIu&iRScsdpLrt^aMK(DR0i2NSq~^+ z@S;miJAqzD43I4AIe+kHSX_a$ny<`8o*x5=J^`w9liQreNmG;IOznG>o&$7uqtWu? zMROdQOTj)nBl1zgGAz2d%$pOgJOV!V-|TX?;|HDwitL}s!raJWv*C8w#%dI^HBDnZ zmpdgG(|L?lhr@5zT0KT^>z;mXW1(>WOsWW7uWEI*i*9;@PzRRV9>9qYS06oX4t@i> zg^GT)&?vT2sa=<&KBp|6K#{W&@A&E+WaoDFQ*MR9Q>33CPCO__?;anKM6KN}ed=nq z;oXPdOv!@f540w|KiU;rl$FE}V?+pccNMAyuw&c%Zzu50UQ`$I2-)W6=aK2d@(|Ld z=*D`JjjcRk1sJ)!3v6XO2pHdfpl>qN?2o`JqLazv=~v5qkFY1~b#+t;6#mh!^aoiw zRPh}K2Mkv93X6gbGg#2BrcC0L&-ST9{k>|%jYlXFY1oO{B*+7r8=a{Y9d&F&MvhN+ zrjBZ~_Pg2too@1~0pod;7Z4m%KYtTQ;q^eBEnf<@TE2@)VR_8#vu7s_uhWkrs_357JJVvbbQ*1pty|!wLxjelrk##s+ z!cY=1z9DLtX#+DqmAg#G%c^Mmo$9_ks3=a9VUq~7B}w7wi6nVa?GT_&wjdEF-G8cM zwOdA(mTxri($oBgdK=Gvi4I?5#0X>}wiM*`qE2;xnM4Osa1~MJCf7f01vX|f7@MlM zP49rN1sO+L7`;VHj|I&-P0xCGJUcwL_9TbX#HTGtdW>?Ya}pl^4F9?pzPs^lMksi+ADTR3&k^tgy zG`4MjRHh`LNuqX9Bi7(HQ0D_sTReU>RFY_TN*b5VDnu6OlX;b^()CR4GCJQhoXHc@ z7tiEP|4v&WnZYf!GLg=esLoa)6cBsPl4HN7dFI{3Dq(-Y0+&%eLr-)3)JmcbxTi=s zg@Si8dF>-p>JB8oAq2yk%36w0TJquIUJ~FX>1)~IwElniThFS5E9a*g0(Z|#2 z#1e438!+M{=4{O6fL@37XG@iGfGO7U3ycJII%*Z;cfD1ue~d3Aeum)#RYtE}Zv_Jh z^R376)C5Q6;{@><6iE)XMfSH z|ILqIB+);=Qd&u-nbM>xJl=E^FxqV@~#g!P;F9jkIID?)*Cr8X}1wK>RH70G!kb=t#;7dgHVJF{Cdf60Q#AY&@ z2FQ_ZTg1R2(u8f;V%075se5+JmUl+u+Cst-$`LbPlFx$KIX^;<*X}-q8IB21F);W+ z8xwQb>3*oQUSiBE9?{pRKgk2=tjSQN!|j(scb zGB%!`N}Db)TcU)iW>hl?%;Ym!ZtU3G&BUE5kkXia0gOlrhk%(gUcA?b(fw64iXe9? z-;W9CQ!86hE>ZLeM8Fk#en5z^H<|y5%$>`#HZ~T&Gm)9EK_>2|S*&rGcNI0Hfu+5< zTk9D}9%V7dW^%*c6ySG1iec_6x2(zZ^J*y9bh(JX*71>U z(d+CZKD$F4o%R(Y+o$K{Rvy$dfk}XV7LuPD7ah`Cp;>r)%BMP7W_Mk3Z@BK_KiHi30}{*67{`@mE0frwQ^LP5 z%$T=j^!o5MS#udww{Oq9@Bgo9k^cgYdx?Bb4`VM(0zx5 zvdmIn(59wK2qhvSBOOQ_T_KgdN(45c+5sUSoWC`e@+=9U3>_~i(6~)5E#H%@&wHd3 z8DLX5xm4PS9()>(V)B6)arH!IGA5w6?Qni^Q)3BAg!zciW+u-yRaK;D3Taz%$@z%I zH0&r4K)Tb#(_ND|ctz{mjuC)0!pN=|kP9aJr4rs|OG?9J^WQgR9iy(lddnAlt+0%7 zB-G)HWtBhx4E>57R#NwyyxhA$F((I5BERh<7cV36c*KeoQeT#4F~XPP%!c@HoDYA^;9uWl7Kxts3}*5XS}xW50B}!?&3ZMai@?gt3TQN_{7hG> z{H(uh;MAefpcA5#AU>AD60Ea`y2v5E76X#_na(vAr?pZKu>Q^Zz#W{yXIrgqXqdZJ zi8LNf>au9DneP_+y(W-EA{o``@3NhpICuRDf1+A3V$`AKcDWf)PL!X_`>hRxI5kBIpZ<_SM+ zzf5}YdEv0y`2xn!!aPyAA1Z{IKa->`RBXfmas4ozc&2D%x;1i&ss} z?X5j($^Wj;6qx`*o9el5WG;?9I(9IWAOpn)@$als<{wsR7jp-kmX;=jncVt~7l}Lx z&gK`o^X=$`;rFX0(`s-7Q&X~53zFb+AC1H%Xe7d+aB{rS zZ1iz$W`ka+FbNpSXA=FTDo@%6@Y?4USwQyAN6jx)AVo2dFVf|0reaJl(`rzI3Q7BQ z9X1%*G)jpFBi|&?F`Pu4&Uw}T98|25*>%uvYdCmzUTb&#UF))s37Wz~Y&yGVVCIFiUor>qD_K4N!62yG9?twR;ly&;`BGy)u4&4XrpLTqjujQss~ zIjV-!`{_N?5tDA2v92^U=bjyt%I@p`2RvVcRG^+O8e=N&z6&Zl((A^TkH_4 z<$}*I%Xum$4n-;*_4gR?cb7;g%M9AbBVC_C!-}z8?+qo~end!Va=Kp7NTo4K9Oo)7 z)>@S)>W|$3{aiB#iEA^R0YFrm0=(-cuN98jtAYVF1k9a9Dnm6ad%GgnMGnP!Bod)~ zmaFv=H2CP(5}(hR|0JDtpElF=b-89WEi4E|LBT=2oI|`w{PDqV2m!pLT=a4E*h(9Y z5|v$1aZ2CNah(ge*=(&;BeE_=KoTm{;i|*n|13N>IT<~k-XXN|vR9}nvhqESg68n; z%aL@vIkVzY(^Q@?v^v=8Lp;4k%v7G3SYqXJ7Q+_VA!J*lPJ_2K} z!TfCRz4*>c_a_kcDjK5bl0@ewO?J&!!zLBWRvpp3I0)z0V&5a-=W-PaAC5D-F5;EQ zwK4K|J%qFO{Ad~#X>(H@w=2L!{x>WL<~#Nuo2PXxdouy%S36@|3-p4`Gcy%pj^gno z?9+|;5E;B~ViL-ydAQE*zwaYS-jKQNT+Ot48e+TT@|@m>snnWmBR1P@(Rm=h{u&0U zD65!V+B=Hx8E6p&Ikg7Q&}+9zY24x^3R)~W#=-3j30UnfT1~7Y-+1sPkfg4)`s{uHs+Wmtp_9AaMJZKw+gMzsMkZ zxtXDI?IPRo&?&ZZO>kj+qCgeLLgujo>fJr=qH=vp7|?85&1%WAn~#J?MY$$pnUo?haTkn8*B zlM4&VKDl#kz$24ggo0%EeY ziIQi3vwNFS(8VxpyK3hR$M=D*HC%o-(o!-3a`O%577?avcOt!`nbMr0+7!VD36G;9 zAI!?jTfR~ULF~2CldSna@q(Ms{PywcDW270nJm$CI>Y|pibIQx@R*^(J}V<11P*`v z)86tg%KpyQH^!f_xwdqgU2l2wzh=aRVhxJ%l2mQ>TN2`Me`7#6nGBU@Cui`gG4Z&& zqeRGPOj->oqr}sVB@>o!2G!K1wcDguthS2gKGcG^Q1H697cLW-9j4@)q%vq24ACH1 zoZTV!F0Q#|%RxyJx|hmw{Vmjasd!V4V#jyvl^Dqu zq>Ka~M%zC6K+MtffQpdEP2Eg230C>2$OwTn%9ilVI)V2n`BL$k24d1*6Dw8<-ii8rqq)TdA!8 z{vJPbVobN1g&l^V*2;k#4E}I#{|!y2L-a}`^}CtQ_$MZdP1Fkn7a=h;@;s7zaMTU;7!qA`+gySjTwm!Nh>|Vv!7d?Q*_3*|mZz zaGWYiBgJCoZ;OZn78GfW-etD_C6vPJ3U6D0W*@+`mc~;pYLFvlePYwN_;7oluFh(W zchJPmCIPZqZmf0QRWDKRbKv+av3qqoot{9KR=~ymZB%y?)Kx~y_p3-!0#lMyJBBs~ zB=}J4U~(!FdmGnetMbG%pb4%x24a*I$+!89+x78s>h3E%4pMo5@%7)J`Zwc>bfwoL zqdMT@h7u@w5DphoQi4Z`GSYDr0%G17AooX4Cyu2T)E}e!e8NNS-n~9u4EID?P!4pf z&vtKO1a?ob4hA<+7*3z)Gs+gIcHuG0LrvD2sm8x|v=&nu!HS<#2i2qqiE=n>#sD9& znGpFhSCrD4XIz+RNEUzmC_PAoVn~1oUu!qwnap9h*(p8Kt|o=UmxXrTG-tJvD@aa9 zr8Q>7p^s#ISgc*mj)3YTs^wKIH}D!@-naz%*&TNg9~Vx4Y+`fv^!|p z7}pEZMGfkVgZgKxg)R4aYIk1^&PMqetj8(x;*>NH2M;>eG-Pj`_=XRbemfP)dDZ7SNqOaIlY9wWdz%(*^g4MduJX4b2{3=OR^oMkr@nQrk>2|oNWX=%3| zTo93;(czr1@GboD%tg+< z*q&bJZXUY{@x9pkG#F8+!iJgGrI8q3(lm>^G+ITKnhv%0+?%b1e)teMZBtq`pdv@)<|TGYoNX#_$*I z0NyKfAakBu*NOQp`5EtH-|g=$Yt@txzw{V#sZWsKL-!M8H^aZEtxOFQpYplnz|(>U zZEmo~J9D2>SqMinN3?F5$QdWI{uCcf9k*?tBg%@K#R(80RGSpcvAGLkX%G_+Tmjp96h@ejyC#XM6i;x!XyX zy%If{ul56M?H*vW#h%wAeYxnBKva9sU76tXK3Tg(>%y2Y%#=*lq%4;>us-NHRw0qv zZo0%OHi`Bn>%#ixD&27EG7T{Llz_{4lZJG0J+>+wEd#K;J`KsV*5(6cH)f{7AGLpG zUTpE?YW@`mB!+jMnH@#Imkm&ZD6gRL6&|#vJhAeY$U3ex$g7G5B-b#k*L)U@dYWvy zC`0a>Y2%5#j0XC9Xzm}UYYzLG6aYujs}`5j{2xxc)i(0VF`qBVOfAvrU!!nVFIY}N zt@b;YE^16VYmXPkgXHH?B&J$Zvft02ztMHlkj!;8{i|HOZVoGSYYh9XO-6e&h$%6S zCPu^P6YR)6-X})G7o6ni-?vT(l*S407gslJ<`o^Hd&yoIF24aaeOD>aR;^zL^vTy z)L@(oT3QXA$oL+WLdpb1$(!(C_ed9QXmTIeriU2Nr#T!jgE8=*P^Mb>%M*5Xc+!*T z`DU$oTETI-O9xf6A)`DHSglz9z)o1vt6;CB!O61R%mTm9!)U)6Z z$$*Z$_OfVc(wdyJLk+sfNJK?|zG#5xcB@RQ4$#rb#RCj>I4qWhEe>;Z$0ptL#LcX$ z9G;uCVp)=84dKzK6g^wy&e9_J67lP1{&4hT=yD2(FJdv|47QkAdvs|{!HHkCR1D%O zUFlJh`JlJX|6#t>Umun~ZQn!>@7Og)`3`ReKO|nljDJL%l5~O1mYA~&>kMsh5P@1; zzC_5nAb2#gI61fEM9NXl(6RGVK&wz^Ma{mAKl0`2(1fp~q6cd;T>7*Z>J*t8QTQ6S!EfgN{+0O*A+~-mf*4vdW0(QItzDX^M@U9 zfzY=5W1GD|!RC%*G(E83bbY{!$MpGvS9LD(Bz(HQpt7Zz8j-q0f`miu}(9Ruauh;DKE0DE0YijOJ>{C@o%Gm&CNOjhHDzSQw`_2FAF zi|+N&<@Tsh54Xm&J<5^Y&Xgmk%M*z!Vyf_Q>TF*3ea(=qCxuq9`!nHWHX2q)BCou- zX7pG9)8N_E*JAlk@nal)*Jzv^KjtD@W}4PM4HGkF{~k+D4#gP!#!4E>m{7NLbb@WV zeoDJ{mrTukn>iN85A{x~(N3YAv!O)r-o9xh82OMy9R+-C=-zpdkw7D+p0^q>52mm_cfX`)4TLDtoaudXHBD1F%;Q@ zpY;nG^Tf$CIOh8sx$mYD=p4`N6j7QI&hHkVzf||moKbpB2SpJU>xV7&P7KO#ya8!P zhWZ6~-Z#CEr-@1R<$X8F8WCbg2aRN_Gk&Ka1vP(#8mQ@#T-I>9O1)C`A6*&1((CjE z;sZ@*7B&D$zc|1u7Hj&@rSp6de1Dy-&QdB5L}0SSo%)jw$E**9H^jN&A&5Bt$Z-9` zKn>)UcTR-df*MYQv0X?WoUb&A1aQZZt6q0syed~fex%`#n8T?`tUMk#cm@}m1_~Sb zAvK2Lq6lVj+0wL!KcnKs|6syoezFXhBwM>$l^{EZs=@<|u#qE>Lg0(YbxC}}0A|H< z#SN@^z*~{#t5iwB7j->5Dr4}*#mzJHwQ34|7njN_+99p!FnE_fL|o+#K`ks4=GyDz~nv&gfj%xFv3_(sSoz+6)?DmXN`-)Xlc?#n+Y4fME2diLyFQ(7_2o`uRZOpuY>mVg zc6Fh^%_ltu*p3aL~UE#(xB!+`mVL_9zmD0++2g$(~>i!YBT zTySWHs{$#k8&~-%=&DmJ{}Iy50sn9FYk}FXKU8Gq{QnYt?-n7QHXAsmn)B`#r#r9z z8YliKC%y5ZKyP#!3EFu9t7#LPZu$d-lJd&zkczQMOl$m0R zB^PWEqVGPbuda9q$W1=fg38&0a4-w!M7Bv2Wg)6#rdU37T>C*r`}E=1Esn~Sk(18X zy4B+;q;e&%;+y2j$bykeQ~qV5b?D)CBDSVWCR3h`6S_7eMQgt8F&wkgj-5Jq78$Q+ z%cdS10D7DX$(yc*=^c#)5>glv7 zEEjkT8t-#7JB|g#gl^(JNInOQ&`X&i)3fErRcT6akmnuua+Ff6Z=fw~Z_cIy<_JnH z8p3T9{?9;490FC}Ej)3U5nxdNNC$%M-c%izgWzcL+c-F{A@tr|6-tF9=}=<^^P%^g z(M#!?!e^7yh4lMBL&EHa3e=Ujdr`F)sk8rz+`Kt}%$KCV|Nk1@8?5J(>g+K~ZKLaN z8)A;8f{-vo0V&F^N~U#yZq`r(x){ojg&$#fXDqYvC8l1gBv#{LU1!ins3>ao3FGPi%7+JJK;??P;+;7QJ*5fl973J9YJU1R)!1-9tg3(q3Vj z+O%BmBF6mI*mw21Z;56|9+bnrY42Hvv5#Ar{CyZX^`apk#5hV)u z5&a0BEzyjr(CjFOVhmHZQZ7ZdE83o$Ef?atxg*jE3sf?F8qHTGa2!q?wPyPi1GY?l z#G;c;M&(U&?3k#55ySQW%ug$rf^L=?y}DU?Ktr(j^|0XtyqV6SC2sbjgb|8C+AN2c zo$$Thv{x?~B>t3l7}-T!dr%Wx;?jjDqMeKaDewgb27;If3J=Y@9^F_~vTj^a=2}gQ z{;-$>-&FONLPkE!on8<6{&$@C-Ds?1;oo;Za9zc*D%T3*#5Q$*QSTf-v`$eL3;pFR z{@%#Qe>>$r|8j;6d3pf$PlSL#XYloe*N?cw4tj2~zlCtq3^50}SC1R2H9PL|29Y+4 zR_pa;8HAv{KMeVO9$R>|DeQgWtyf7WGbNqgBH-6QS}^&nE*Cc#?OP7#(e4@ch{u$I z3cqk^wH1scprX}k7hA>45$34Z=T6F7Jy*nJxj?-eQ40u6)6>Q;aY^rF)PuK*wEYX3 zoemNEA~0h0ga-848Gx(sA7A;D4T6^~VV%}}D3kn&KfK@giQj$j&2*gcEMvi--}Uzr zeX(}mB;j60cz((M!M}a?8K$*VFZtH;lzXE}dY4YC&;nbb%{rBAiS#d@2iyXGNC@jt z!`j3o*}Vcq%D?^jKkm#wU&g;*;3KqP0@_d*!=v12#lL=e0irnAU{$g>2#M>z-r9eE zg8%!KcEK;bjO}3hJo5jaqq8M}qqE^vOR)a!BKZ5a|K|@s!-phEI8XN==K1@o{O9-l z`=enDfums!JLTK{n-%zP4?jglt`0vv`mi7J|8eve3bf#l!`EJ)XFEKl9JUvwIBj=O zg@ciO7z<_i#Zdc@oE|fI+@ymdGxKp9PUO?(87LK+(c>uimvu*4;bF4CKvlbMpGe97 z#2%MC(i@4WYWOtEmfy-?lrdGQZDqfUw@Ps!e$ zZTQl<-mVIAdhU?$xZL?O7Dz}J$C7U_u1;F=xL*lfgOtNf&_UwAbsv4=Z%_Duu-_Mj zJF@NB2u7O3yct^voBJ%WFMrRm97H7uqvL$So0VHgSHy`13LkhRViAtQWSN%-3`03w zt%M{(Ff!+xy>P6RZSms?x4)e9(zF^K(g}DxL`>!Hj_t zH=#o9m)RMPEhAT$Wd;IQ#6fcZgD2+iw`r9V0%gSI8&+DIOZ@w&XTJbS`f@XgLA`{? z*HUwpA9m||*bkNudZ~;U;@gGl@e=B@xmc`LYcfnbmA}IJueOe`i(LIm#P4)JT}Zyl zfz^=DJJSqUkRbRoopdzvIa=OS_XdYt?hg)KDhm@mnBJ~nIRyv+1XDi0{_a#AZbSL7 zQ0T|#Mdnf|>}j$Fac4saz5MON`sr?rRHf>gzDxP^l6=4N&6y@@W|L71Qkk_IsiT{v zL1csebdTU_Mzae6-=YDFYCVy{gtZx6;;-`dcEOinD%Sa+(t7pn4-=3{JJ+SM8F<*` zQ`g-)w;N<(wYppH>$8Jt&g_1rQJ$6`yPusgL}b(i2MDWJHqT{=a8RkSCrUk( zs#Kajw8qzNcL=f`hVD#ABbX(8uyy0X$7My3@cmyc}S~K;DsYAwq%}Ea$o{Uiv|K^}D!3X@G#&QxC@QN2RI{9@IAki|vkA6rqp&mSg=Z zfwE-g{WDklo_4>O*uhG|l&wRT~|ablUA*xM8F z$wX7$0!=W!m@X_ngL{e9_AoRn47D7(Gc=1s$VM^KMAoo_Za&~}So2i43vzThc*mQlVA>C(0#%GhVqBg5x$?J8Iy zqKLR2kWOdwXFT8PFW7u%Pi4(n38_7K3Qj2U_-25ZwotqE#*vV77J0bH;84p@tR^-P zV?55;nM1RG-`-42C%IV1PNx_o@? znVY55WS4-+5zqfXA=j;dIgH2c6bR>;Hfv%!yo@1Ezwry?p1PX&NY5G&mwUN2@-i4(M{EMifMI$OpqF9~U zhQqn136E`kI1GHcfyU?Z@In4H6pcj)#IMP$(f>zE{-=Z`gNi)CNN|J+0~3kY;7?ql zTwO#Q#~*+88`9_PQPz+Gybv++L}z=6+%M!|-FvVM99nC|7ntgAOT3>bTil-$#Qk;? zri52v469IcpyPb-IX$JN&hvYozHo0^KDq@6H^vf6*#XZ3tDaDlI13+bcFJRF&)a0n z*XU`pYRz+QU`@ixk%;D8fG$cLcV0@N$lGrSdGmY8&T5#d6cjMOsJ2Ezsa$PtAJ!U8 zTg7jCMU=cxoIL>^C}JwkVG;z@_zPk9Z1{u?ts?OzlBE0|@`>W&m&Gc}4pe>qH@>#n zmWLHRS4nNxSr*rD!j6nlg*G!kdf>Z=+I=XX-pg=(bRK-wkHwks9>^dC&lON_uPjKr zdHAq_R1q*0H*O8;Cz7yQM?1MD(p#(rBD$E+;8znz1+eoBr88p=s^{S|(oR;q{6FDd zI#|prQo#PNYLaz(s9tb5bPE)uFPVzarm#Xvqq9qGb6qzZ9 zd-TE&BuMZ4(uPUlSm(H;$sBp`dnF2^D-)VQhQzATp3?W+^FonVM$X+M@HC#wstJP#nPJwQQ?O%Ni*OHdnUVoCT ze$8Nx=1_w*-+5|_yx1KtYlYg6PpnXD-HoG38==BKjC};iaQItY`JXu9pBDIk%`!jp2-Db0qr9q8CuZFq|&virbyg*gy!rV8D&qm^LT!6q#JNOB|0q4wV)=+darui^5m@bQ$b&@ik@83AZ~4 zsq!I=SF6c(0C~hcjQINK$CT(>Xo>Riuc5(J>R+?bfN8=mh^LLttP z8O>R(`?~PE>kOLN2A-jW^!0M+dRXDMs(RyKn?AC|`omyx!QhG`pq3LOxW&c~XR;1Q z$Yo;uJ2v>AL>yj;KZSR#dBIDae85-;fJ-rkW5-Ok?XpCF2*GaRnC>L^o^KWP;j*2k z#2$|e{$R3N)T^tYaI@ssSgiFndORWZk79)p{&i`58R<7ur}S1lfjL`ydqyz~<;TKu zMw5*I=4Xm(cx&_e@Z!-@i=Boi=U^gCpIJEg4TIPr3D`3SCa-%kC5_8KrE&q?rYZlH zM`UeXlIq(`(@W*4AGY?e-|BRMd|P~PwjCxPBJfKOTlBekc%xMpc5{hP6gjT50xHyS zM&P4|t3H=w$A(g|{wwV8NcahF{~70t#ge`M_j*f%@E#54lTw;L^^}kPE)_lR*Eh!g zWsB9fMGv(P^$Doq`wyS5f6afMqrCjzg^E-tAsRHeN5k`j@>=s(Tct~99VIzpeFrAz z6E$v~-Vh8yay2RacLk(S2;JOW3Sm5M*9R;Jnd~~Qu0HJRVO05LHKx_@sHJhVl7A}E z!D!OS`;oc$&iAX|9wL*(NFSd}{6l>2b_~CGEAf0MHjATxlHt4s=G8zzCLF6BsZJ`@ ztSY;?{%|_F5>9#8g{y0po}E(G7O3)&$>afRFl#oRq4TJtw;cLPr1e9R!6=0Oehi*U zZ9Rih>0jk+)2cWTJmcx=Cc1(sDE$6H=eFevj3JUz1;uj=V)YVMu`}epQ`1)qN1JAB zKY4#b-;yjqIf=XEdL7tjKVChJrAU9MNGa6^_ghx~4#=p?X2Xye-^*T$7MKCrk7}as zi_b!js0>WzQxjPBctV_tUSLj5li76Kj??X&Dk6zJ>b;|t5zA^T3#$NZu_E>V6uuRju_q@h&4##SP2s?ZJ_;omC+G)58Jn0EyXn>;j+tjg%hg^e4F0E8(-b}fppxN53^LF_b zbI`$1+Bhm=QI9w5@Y|qq^9)vNS>b>*apb-G_X1Ig^l#y_BREdm9;0UU^q^Pmt1hPE zahWZ;b*Oaqa|jg{HeF|1Ulzx;1q+_4HR;u*Pz3f9E5^#F1k~AGj%a|ytbAUgP$&6y z!B6XSoL49~OD-0Rik@@c7}?eU?ziXDDCj|d==)-$2j5DoN7N#7xztAYyQO!eGqWqu zQIx`rORcw%Jt!balXFFbR8j}|*1B|t+zu27fFSZ6PHeH&32on z0X#R%G?)QxPnVI&?$igC;pXl5e$7g*M`YiR_*Nw{kCl}Ct!!R)F+fJdQ=6*!p-*xe zb?I<%1QPt}=d@u+?(SlK9Znd9nDd1&sWp^mt? zS$vEb3jeRU@SltNf6AYCG=Gq-=^Rns&&GYP(na3aTK0Ili|Un>*mt2=bXsku&d=NC z4T|3{KHc;&N~l=pUTfTxaPzS|`mR&#~rjdJU^si!7saUH<8R7Oi7YOL&=NUp>vU&?e@GLSWuWeaVt-IS2ID7P6ZYZ9X z=Xr6<&A1s&F9R1Z1Nxh*F3e+|Am2OM`Rl|Xs$vBDu>=OCpR@fnT#siiy6oACtsOv- zrbP)QLm5k+9fn7C2o(i7b~HLJ;9+(?=NwLE;d!3_ZJ(YV1Cr|eRz~-8+x%u=GQZO{ zjz%`k5ZytqUoKS|9pv1=WU>+>rI#Fh6f%$@Ye-Zy&Co=n|^@L1{0liW=|#7KX3 zie_Eh5GrN$D8K?x%0z}M_a&AuWhbDRQ4|?}40e8qNXr-CF_cX8Rrgew?$-}hGSpW0 zR>8a)h;DPapHdBQpUhbd4GV|o-Wul(CnT>g+1xp9*(nKkjhqr`)z~Y34j^*Lm3_2sJxk+uOQwSkt(FH#lwCAkY!^uxAJ{l^ z`Ta_d!!;j;30bl#q0EsDiH@z z#SL4lB|;YJeUzkFV?$@p@L>IF zn^I>JA^un`7NN_NNu*Hi%Mq_?wEgg^D)}RUF;C2Bh7Bmyn%!Qr)yHFa^P^9WB+G|E zCKK7Q4#)kLe}Zc~g~2|xDveHuf|B27MI*o4D2Uwu!`N9y#TBLLIyeM(hY&)76Wk#{ zfZ*;9!KEO$6a;q*?!nzPxVyUq5AIf20r#YPre|iYdwcG0YSk&u-uv61zRzBfi%Qk^ zT-(}ZJl`JPz(6Y1*+rOga;_V2;pXs+9FVi|L-)yNus5gN9f}nR-1qyCby1tB9LJYH zYYgPx0FV`&#q@7eX*7R$$GZwE#(;ZAAR6Zf*v(nm|wcV{Fa1^Qz#Xh`&aV^k@vD`D#B_mX^oS1OMu-Kr7j%OZx`k}r3j z6@J)sw`rQi+ghpq6<>R`k8Ug^9H*F^m<}p6d5KQ%W_KJGK)tdzkHhiI`D(N^ykzaN z;UPl9KO_=vA&ZI$!%6?k`Z3|SL7+5(=0I=G|M;W{kT!J~?Jwqp>mOATMg6MZ=wA5% z&xDN0;NQ%_{^M`b07A|?w)Gupwt0ZaFDb#1u#)%&dRAHz6ayP*x7IGXZO+U({K5sU zW(8#qW%09v$KIk5!4vY8P5fl7o%4b=u^s^VvEf?O%TLKzG#}&96EipiuuFgmG7+zj z{s_Y1yd0{>nyhFFlqp(yie;ZLN)HRF3rRz)4U;N<|4dw!&?;X*1uy&9a?8f*pE0Ot z+)}|SAx|J&4EQ`oO`ZCGpmqN_2$vnNUp-8zel>%GCzwj}nIN&6a%_t@uL}qd6ke=V zBR$a5?B~<{8;R>c5s{Lj}IOzdl&Ow#JqJ>h3}% zSJW-A`quq!KQtDJ{XBK*FpXr`A1zTKvn_9UbJOQ(Vnlg)7S!5l_>}trwA@+K#uel8 z(q&Snj>tl-=Ut|NaHT0a_2w%&JpMjQFaH<`0M&D^IXGqpjN$#_xwqp>7qu8|99KT& z!c&SI*!4`fM%u9s{rQ;%uq(}2%(^@p;Y9aTpB(0OSA&ZPH-Bk!mzTFLd^+|8`(f_J z*)LKB%$I03f3MY3+=}ftP*tTW=u|;9t@6?F(FxCyh&--hR^Z8=-WYCa&ahN@pATgA zXfI^OZHOa;dae}DTnrdk_`U=TzRvFJWH5sA%$~NeoCLjn+#OLk7%4GT6~xM;3c&9N zqY*5B3Q@_9zlv}+*Ci%xj*$Ida%x6Wudaqs7nFI?R1%eD(-J}k*WC=3*IH{IoDB_YT*7EpWDRJO=FShz$7l)Mn+j7eW9M#6r z1kUe+brm1)>?8zsLxf!7JhGOT#gH~joJ55lSy)rNMIDKGcRWj8^lHX#^AJOd^ zRh|dv_p*frdZEDlFPjkKF4?vh5_?cVRLJ*}sM z_;$|q0rpV;Fm_N2F3q$!ja(Lp~WGc=N0|yxWD-~k61&1 zeH-si2>IdqfGmOsY5XtSy^;Jq#FVTymh-<}%D?$Ww>^}iNtrFB-Hi3<3Y%B(?2Bjr zZ5M!nJD_?(T={=FQ2%_=xjt6Nxw0rr8zKL6Uu4YF>D!M(%>VosDDeQF)=RNPhfDoG z|J{G{o9-r&fvWM{B$rl063U==lF@&@h)!1EYk174t^ajuVgAj&J3qyBdwN)1M_htk z0KUdGBPISnS1|~9MMBAXOr-Sx<`w^shlswf@;YEUoUX<_`Q5%c*lM!kVx?J0HI~A; z^5trG6c}(*O9Ok}G?ci^hA7cL_17P%A7uZszus4x+x@M$tPX-a-2)Q7p8bq->t=GP z(}{9Aty*mgbDWOLX!RM3!vB2f3P6RnU&T*Z|NrAHVZolE_%0kr{(1ICx`y_J-+m4E zvS!Z|080bKyf^j`)0S^LY~%v=A*XAEq6&SudR35%~niL%eACCqlt5aF2}Q$BoB{nCQDPY2RY(J`ZrcqoDZ>_CWuxm zWyNYP$VAV&tpSN;`aPm4!TrrY#9u4-kZ{tq zf-kGIS@TE#A@~2bdjI1u_L;8)GU|?IqAT<}q^5Gjb73j>{xb2V#@ikbW>Xx2loib3 z-06of!VghO2TNCkb6ya}ln9Q}lur|M3Qtd3PzNQj&~SPveFC#>0y^pfXifP99v+^^ zI0ayFm9Q;EuyCd+v0+h&MK!w)%D$wGKx%rz49R2=>@G|P8I z*gz^uya=Gr>TyN_qOmfK?K74s1Uy86m^G)#!C2|FuyBU#rra{+b&oD$N|hWqI9e6l z_3imLFRlDB4^XUJtus2pF!TD}|1zBg{qe5L%Jh_NW=G6AHK=YaVTgr4n{c=Ox-nddYB%@gLKzX)KUoThp3wv&XtePls$X5^OY+{Z?XinH0w3#in zQp?%A;@NhP56MgiWnUEjOaY&3@{;OSet`8t#Z-!lIjz!<&j8d}qS@`@t=?DPNjU=k zE{v+8LqK8zk3~dac=&P21|TQoi6LQ30y2;8dR)W3CLg~28Qf)L9s@*3;!oGwWl{jn zA8l;TWS9TrL3dy{pu;9juUPa-{52VSFeXJo+=r;z4SHD9;q+KGvvHdUP!b7;srTg{ zlnfphzws<#1t5c~yzF#9Cl-W;$KG)B%zn#iHCyDA{^)i1hYnaR3e%y)K;=St+mf5H zW1v304@~I;01}ZfJl@x)+pU%}g@q=PQNY0eEv@-@Iv$e*$8xnTl3s@y_Q=oc`7s?e z8h{O)yU3Wy>jM$cYxhcdOw7dj0H*F2{VQ04yVx5=o>sFrS}}Z3DG78)RGUct1$W>H zdZpi^JlWw)$li6~W`-c-@`(*hn-U)6U&b zQtG!IXoxfc!zWW|!mE0_OMl$^=p#pd8Z`btM+102V^~Ro;I^2!c8*Mx`ZsmR$j&~^ zv{r!ViRvC|gz0UH1=|tEofB;(SM>1a~Qf;h_ z3afk4-#KMssaZ)ufl451Mfg;vQ=W=5a{Lq~bDj#D`9y|@q>-nuV_k^Xa_REDz zYU;9jE!vk{TZ+>pF8u<3kTYS8X~>sJ#1W#b)->s4@mFPoN6VFrDwt!?II!uz-orR4 z`aFcwU&FvIquE;TK11-3D5gJ_${Hs~CCC|8D4P)WfO)8p>z;M`5hW)k+bM-@w#1gf zgz=L-yXDqrfKb59`CK3svVA$}E5>25@uu15EOUy&Y-6cNp<|5^cGx)YNQet`K|Yqj zzEB`bO<+D(rZ+Pfox*V3RwVW)&*g6I&>l|6QhIW*R?Ckn@@=y787`t`KU)8EHlG@w zSv#J~_V*W`=Zg=V_Yl$&{oLl*h3RqA1Ysmp(}x`|l?frpAjeMwABN1qtXM&T`lChP zAE^O$+4UeQ0n3Gk40Qj)ES>HQE!!^}u%C4^_?+qvT<2|;iZs^s;j4dewHXMI^A}-? zW;2uXPw^7GN2pIM*HK=&-0^!qQUlmRq0gPSnXezQ$Usgs-|_2!o|Rn53H>6Z%c(mN zuVW&mSuc3Vn0fpDwS=c~F!1?t+|EEU?_8`(!9CKaTM9vyE7n|zoMhG<9PeDv$e+cF zhQW@slVMG7?%T9$?Z1u*RwBIDjjqGaq4NWHi!t?TK(yK6$WHU^#X-XxoMyNC)CXvb zB~m02dkW5n2ubH7FZr>Ka)lvi**Y@XNM!8MY*pS#i|zd(D?;zrh=V z@>%+A32$;gbwO40z9@N1J=57PwFY9;gzTCVxQ*&?toNIYpKW{RavbC5+qLmQ-#%Gu zw^`-mXneAW=l0PTbz@lcN5U*Jztt~UA2di6O9V(SNr3*k+|_ex<>h|m;IVDKHLyzs*o{AcC|&jmEFjIfL1re?kb!S>l3-KOT>F9gx`5P z#%x*CWc|VMu2D$vkrol)sud~>Czw*H2rU0HL1NOrYqVRo@<;S=uh5dD z=7nnOLeAqVKY6d_{SD^NS^u{1^^t$e5@*AtxK}yekEL^HCBmc2>9aMW<)Tvq?)(i{ zn2@=*5}kmE{HMWwQ-fBt4b;Our=4B6Fmm?SeMVEVnrVRLO}%Sb&y3h{^`RUl_(#@7 zH0?V}CIccGg_wU(6zYkb59lyB899g8iw`>M$5;PokPJ%GEezuQmSsCh7)|$`#$?t! zShmP{lyD#?qDOIM^-Qm?oyU8ff3QYG9+#)z$jH!Axxii%Q54hDINF*U^n*7(w&D$p z@bB^QO*_Bk+enh>_MTwN;q(q#gK^z>UU2hIEt^DcyY((A=Q2ujab(KOFnn6WIBtvX zj+KUDq)1Xcki$yj)IFZrs+cce_YyYgH=d<0V$p1@b#p&3U1? zrOpFzw_=kKryo(ZB`ug9@9(jNN;S%rVH0d9!CylZC*ODD0a1%2r|@|a41GKnfph|$ z93XmHKs;j(aIQOMs$zT8C|uj89YeA+!5%8{o#*P_Lr+B-jhWImQ(HbvMasbZO1*eE zp_++N-(><>;z8cjMEbE7f@mg@K>tjh46Y3!^Cbjt>jFpETi2?SYRIXnreHAbWeSoy>*)cuw4q%?B4LzvZI=;$SeDyU8QJfYi6nNuR&Gm`~VY;&7EF+cP||V z2F9vI%zB;Kpg}e=pNEl+n8WSo>kolKUfpQ@krZMhN#&h63391CTGfnt1$gzlispuJ z3~J42_0a14%l=x>1jTS1OP2J4L1t7&eJPVM*|qaVj0mGmSZ)ckT0z8v-e@{&IAZM^ zo6}a|2%&=p)4y>f2&<~S`8-mn?W)b~`R(^pA+KS~4eA(!Zp}jxD1V1i4Ga(;QN2>q zN!Ro;A`^VQYF@o^!yipc!6F%j_h8J;>*DpspD}_^P2icFVL*D>uxOm17e2Ek)(Rwp+3wyZFo6WT`!ynkaFUVesdCt@Y$#}OhswOey2W<6dbGgpG495B& zbg>W`&C7REL z5_!icoc}5+Zbypmp;~H(wN)k$D9)E@MFLb^onW`&A2bXijE??-WR_x!c?Y%Tw5Qt8 zoCSo8x0Sr3KlCY6dJ^!N)#K;O^~C`4P(AiY{MJQbAX~Gn5(Q`^kvaEhq4MAzpTkx_ zYUO&BFYs7~BhE4qP2$wSTDnBEoxA(p?9hzU<(<@UGH-4P#oL&9TWu}Qox^r80B~5I zP)~qo{v3ukwD|G?c0e|_-+&mXh9SaIcp9ewR=I@nro+mDMmlt0pq^fh9``dtp=Jbu zD&zLSBCq#xo>e1j@XWL*@#?~|!F=}p6#*^G+}IH}t;2NV9+mp-KfKOGoGPL1<%%Rc zLKJ};ACbmW!R)*0dn3=fLFUB7W56xFdky23CWZBZM9_?d9c%Pm1ZhT&cqpFLYMhB= zQTEdYLt-_w3lj$BAAt~Z>?h4khQxAB1-aCEngC*)tB6;<68J);3ULE3(piA|o0e%; zeB#lp;l?0+3w-xwvLD7EQGE;PyI7Qf9pJ{M6&qiH@BRk$XgO0}Z;^UK<5||^c0(fa zCRdQn(EFsI{7|IQy%gv`HK$3GN_dO#1l3bms)k^{-S19EIe{U#J|38K#=`iD`sVY-CA<6UTxLQ58%TnRTU<=4fDY%QSs>kADk z;a8C{XSMfIu;JV(?u>>-D)l8@-6f)gggcr;{L+1Z5{A*@c+d@yd8xj#h(YMTo0Hb25chKg`io{-_!?t5dXJ{{ zafvJ-=mkw4Zan)t1qi;=Gso`T9>4^U_5<|46((CfcoWDcM2XqHDwa~kOuOv=+{ynj zOXf#cBr*sW&4PT3r=6G}2$wILAnopQUo5A?nf2jp;8-h2_?IJTrqg75>7xq}Ce{5p zqj718Qq(d)pg@w(s6$o+w1}+Z2p^?x6)IaD_o6Q8{dX?NPU&@djtH|{ZPo!s^j8=~ z#Ve=dH@&!w&Q>c=fhLw_1R5tRElB`Lhru}I=eemC@*VQuin4>`M)bwWqXHdvu&(GA z!oSJaH83nzo4Da7+Gl%|MSpz@WHZ3sJB^NWHA4N$*n#RP^*{^QH7vIlc+p2)m~7d$ ze&P1jNUsU);C*CSa|LpIdOzY{^56+XSt5|U1*J+AFPlV`~}`)M&;TnCD=UqI#eemanv!EEOnL0n43JsWMp(5w#b^R z(jZH4hRZjWV)>MsHN%AwUSn}jPan^%P$gdTb+j|H!?urA1l4rnM`+OPY4Hz0>fjT) zw@t3VNN+v$Z>?%?|KUcX6^!ly8>}fZQx?2ao1|Y9qvo<{h`4^iDd|gVZURc7z6My+ zpY~PWxXL@$jI}b1+Pv<_4p3Kp1ld>!#z zYj|!VVdv3^x*@c z^=VuxI|{>#H)OR|+wbRsWe;owEtB~Pmq$l!-K~Xeo3G(v#Hb-8%5iPMtw#;h4UvH#1UrhlQVE7A+vsHu6R_0rGXriu!pXa~aRw z_yc-Mm7w$CoLZK(M#Z!~B%;P`${2&iIfbwB?6c3}wC91^;8-eC+VY2}2anBe_qnen9G0!C zb+aFUHAq5U942T}y$DtxTMbrF+oyrLV*_RLjVJ2w>s;NXn~kS(aVEjsmeP9TSTWVd zXf#9r==!^>LJ8>ql_S@pkK;jy7mP13`n{T4u0o|v>Z+%Lfg_6U4ZX>A4~5354cH<= zoH+RO7=>;d!*`OX}|Cs zo1j&1gd6TCh)J22E~dLPk>MY8-t$sDQ$PxyWaqy<$t`w#QHIJ8Fz3aQ%ZsI@^aN$- zUK}s6m!==g6sn9MVUkrzFQB@ft>w2go*jy3Wu2~>Ug#SO8%YKvBqRx@z?BfX~s%wkB`Ux0|n?BEF;)6yNcz)@Cd}=iksf-kzwbTrJQ*XdScK9+S}rm`|#AzZxQcxbkoV zW~AJXFB(jTZ7xdc*T)@jH?!rlY685a)40kT(-I#5n|O}%*QUb!c^a7D!T1L6rz86h zgK7YK7rEUSwcg4t8%4hMlh(OO>=V!(Gn@C19-z)f$!vrYq)Y@|cJ52rqiov^8e(+T-km)ha6_ zuRR*Rl^1aC+M}s^8yknxwigf$+1wtnkgDSjOW^PAUZsm zS0$!Xz?d?(xK!(j`NVmPS?Wy6ZIdT)(~%>A-tiir%KudCiwZKGH}S_9Z&H5dUe>gC zp;;xT8`)wK^^0pu>}IhTLndYRCcCxtYP8^>YiX?Mhk(LJI~8$J>-F8C ztLhS;HG?b5g>i191PVbnK!^95+jSwCMRV{{heBY3uo-+MH%J|SuvE?uUaewf+F4I; zWCE>ul?=Q4Jd1n6io1RJ4lr$kkNmnGWh02@H;-mt-{ZiFc78w(A;rUV73(ZrxZa2r zzNe|9$!<1J->BJo&p`XUaVQ7E}5sKvwfQ3u~@;Hu7fAFFj{XO8(!c`$+1F{WVAYP}|5I-~@R(jXHp3@;_VPbn!U+za-n`SD0cc3j|ezkM*38(W7Z0)Nvv#{RoG0OJFHj#s&-AFRQ*dPo_ z8DMrjC+No7Q#oP4^hK*7RX$xvhF5r4G%KEN(*b;{zA;>nvfSxFClRq9Kw2sych&+l zC9C**)caMqc(^}C>0R?1urO;i|F9pQCZPNFwM4U>13Xhip>|*9Cl7Huj6BL@(5RgG znO)KGwCk`ZJo&n_S;=2Jev~D21zvRf_j89jIUU@3-pNv1o66JWHP6fWCbh>PNNno` zcAStZrSx8`1oKF*tpe?o$hcAvq=y=~zgvyKVVy5VH!trz7tAzo zuUyB9m5#C^f-giEblOkXe@a;qXKDgnb5^by@{AfeJWpLzUmH%AWrCw93-eHLr=EKx z>G(a*zogSzILdKZ5)ww+@p9i@&jyZ3fXVmLlJ80$$+iv*Etq0~+ zOC{7&dCe&{Z7TtFQ6*vTWNZe2q`FfsU#ZuTty*xmO~^f2p4P}W+wxL5TN*8{7s(g* zUM>gd;2(nJ=E@RqF9{$5XdbT}RCLGAin;+#5w_hd2Kn!Sy*@Vko`u|7L+vUHx(psm zrFZ5dsiEG`b~?_-hU{FqI%yw(JAXsT6G6bFC*l?Y?~hfY z!WyG~p2cGCm++MLE|Nm%JDb^;!rH(?ybfpR?!`(p?g%(3H>DXd0fdZrZkXf#ajPYJrnO?jLG|nE%uQIPI0f zDL&F7Yx(g(Mmk>xpM9Q}f2Kjtm4wx!by)v?=9_BOCn4!k*xE--%5TtfxVnuqpZJi1 z-90r6XcLwGulKk8&J$<2>#c1E#|=Tw?W%N8TK(20=x&lQxv+bG?yj|Z<>#KDYqrB= zz9)slHP4W0G6HtPwNf3O#gKGNnB`NlY36!Ew-1v%mJ! zY$W8=bNjhfr?6qR-chIJt*6FSR)>A4cEgEmvP1+{Yy-4q_RtGR=;zlCV2#bLEQlI` zbuao^5g1!}B{+2?RdgpdgmFfr&%_r!JNJr?TQa>brt#?&^I9EN15X^(7`Ojv5_`Up zwI7)br98KIvb^OuL+6_TNxDOyvQ!!=;4Jn-xcxT$5H2sDN}uC{#TMO+*N8eY>sUoqVAut;v~ZgE$Au4B$tf9LJ7 z7%0ANca3|e^fv8rWAKcbJ^m&Yp&bop?HbCU`;$lz)&Mi@WQIP=fu|g)4w_{;%rJam zA{jZy`)w!ZRIp_xY-hw02+d}Z7_j*YH#N|U6l1$3^xHQ)*g@vw4<4gGxV>VMwCg7a z1@V8^yhm5x4Pv_Zf zOYdZpV}wez1BEK@f5j5AGoR>`{^(-ydXmEpqYy;2U26(`bM1>vC$~I%-4iW5nG+bU z`SB}LVki;o+xoNFOtXpfzBLOMy6jlH&6x?Ie3qpi04HEphy=Nnv1fD!?^gTQr5el@ z6iwwgW^vJ*?KAX$hG_?>JchH-+Wl<$l9cqfp}1ZAm88w!GV7cqNU)xa&QC#7|Ids{ zCJlJL6Ht^0E94vuhrF#Z4~Z1W|K|*X!^b21D;F@8^Wo(fQaq178l)tfn?Da{{wO@D zSX!s?q)xuBmc!UW_y+Mph+y97aNc0m5VQ{QarxGskq7Wb>AOCsQKrqr7=Xhm>(DUoiAvPq7)exax?#MPBSr_PKbA8R7C+1;|=d4=tkbjBwmAP~;&3gv$PX#qurh(j0kxjm~ zI`?5jf~8yX zCv{AgbPHJ~Ml`)2O^CPQj6NT7KdfP`QQ7wW_yu6X=tk*v9ww|x)JNK}Ffj`o%M7F` zs|mv}S1a^e(g5aR2v7JN!(0L7Z-c29hT5M;%mnv z0@dp756lZE<*)$bwGf|SL3x#r(_(kFTB0612jQh)Bfy~tfNVEQJpW|pS-U;ujz`3x zFe>YY3oj>qW=+&<2^SmQ7KoTKp}Q>G|zFk0DCS=JTue&GOr_O$j!!^Tzs>` zT^G~$abr4M&@oRTssH^RkU!N55J*SFBo+JRxbqWVW{>Zre5im;#jkKdFr|jD5KvG^ zwYqLwx+&dLX<#Hqu3l0paDDM$GmEfr`uc)cii83XHn-Pp7u&@Xd5i@394z8iehRk9 z&!H=%vZbYhK@7`X?w&(_#Hy{y2SGf9fU}4RBU9m(kEYaiid4|%;q)x?t{ zHu$z@Lj6X#$+`a=Zv!x!r2{b=cTJ8=*9}n&nG`Bjm~EUAW7}O*P-vjcez-^C%kBaT zdG#Y#TW-yn$HVM_<3ZYP;`-QzNL!Ibwin~<=+U&<{iPs=!ho@6lvvWwj@*>hHbU&w zTSP&TrFIh*hU|z6RO2ad%X}O=%kBXiq#!fOz|+~g$kOPEu}f|I^!ltTZVw|Zbo?!P zHD+9AT(Vj0r+v20a6V)aIA45tG!b;zD^kYyAtRZhIg$h0qU#^8Nkv0{iiT`>xZXna zJnj$kFDJJm2H!R{UpE5)?WvvcBs%$4o_Ik|myEIk09pO>ePk*v>vV5%9Bp-+QHvAJ z-9&hd3fAi9NRmrbmqiWN#TG}WC|WFbvq6^A;Q978_pA4)L2nsY>lr+LF;ekV`?;1;`7WQ)HH-ps(+(GjebC_u2XJevYo)9TgJaM`nt* zoNl66?Ql<}SQk_v_kK{~taTr8S#C4$n0hCA#(CEqs2wC3wH@3rmED#Wfy8CNWg*dc zb_eNkGcoaU+6~v9vlzQ{2dU9ILVM|G;$d2waxyHFjC) z*`uCAUuC294q{;~;ST@wI$o@{FnqBKIHDKev4Yx;Y?L^AWPDf5Fj_gZ0iLNxIr z9g`ppar_IM-rUa^0&v;#uE$GpA2=*JKS4BCLfe|l@{x{rCfC=qT+Y_y#1PTdMTrCO zzNyjrw|}w+0GE65iir2sl{mHLbmjqUd_ES(Iv;};C^gXh-o!gPXEjd;wY6eVg(~(_ z5KH00w*MXKr8rIlx{hc`T7<89FZUjkh3Hhe?>?idIu@9_8UsjxV&x*U=R0fcgknX1 ztA(d|3j!*%7>KF=E`7Y*7@r(6=nRihJyyN)USk!ROu&Y5XHdm&BOk#unq1g2eu2HI zc&*wI5HK@o@Pi_LMQV?wY}H60=XWh_a3;#sa@;F5y%OX6-PD^l>3qnoa?%s&opcI! z3=z3SP`SRZoggKWYJ0#?ay}9Y?1kkP6Mo65wsIVrPAH)WVF4U;v1%CVYXoi~haQ5U z8$(|3E~VA|;1sUauPN@tU|0I7>m3{K0HVWQ=9epc=E?_u(pr^*F^9tm*0p2Ftn)6N z_2u*>gWYO}KZ6dO5+4yaP>?~~)U)50sv;DL7R|esQk@gI)i!tbR^Hzv@Q=e^Y+s0t zQ$`pgVNh`Go{SxD1^;Z)iVDt)B$-zNX;24fq$Q3;96#*--X2guGeQ?2+h~L#{7Jbt zE``ghii%b-f3w)h7QX?Ex+<<&R8_J*JBY!8>^*wUR|fv$#Lu~0yn^kpj1W2huE7u? zRTap~pO``HN_6Hiy4AIZS0?K;?y#3XzIP&5O z$7y{BUbLM#`h~m;AJPLZMSK+QR=0=%dJeJoId=$v(fMvO+^>)>(u6oUpDhELTkDt8 zasDAQ!*bsxL^0m8Tz7O>XV{+twt5C*9LOF855hK205?YZ1G5tvW1pOcS>z95LA1x9l zl+1HhNRBPA>^A$4v*`R9|3Z&@w=$2S@;*Wllc);Kfvj~XMHonz&1;>=J7mN}RaQ3I z>Pp~`tPsvOeY5qGfYkCElO=i74!pib7@(<}y0#v6(aJnbSAUk#f zGFbH2I0D0qSIgYK2>i(8U%7NOZjRSP$U8QcdPsLqohO3Es5~c~8hn8E-Ruh!q?ygO zql3#OK_|lr?Gn{&jxL7Lp`J9$hd`Frxy9}?K~B&OiDnOPmXj9{=K%QZ zD@W+L3~d^xqenszX|U$nFXxolfu$5<@k^ByWpOo#h*&?OXGZi#M-xe)3qz!1oNPrx zhH&iTF#n2~{rzv#v+|=^pF1c@56JkSJ7t(bc_GSHQ?p@qA2K`#^1jNX*t=L4^Rzix zDPNzJ20Rr_ zp0ZtBZq^ZeS3cM;l>C?XN-KNyq@|8K87;Tz&H7He(1_-xE4Hyz_AGnB&?8*k+^zP= z`xmB5caK$HKEpc&B+%C=w9PJixsRrRgfPObXH~?JxvfH$kjgEdbYgkM`0c|BI(AZ|MIhjl8hSq?FTQGNrRU=~eD?y(ne%+>tfs zvgJYfqV$pL&F6tO254}n4<2LMSjvTC+&c2Y3&`WfeY=Yg)+K`fcwGoCD5ccdYVEg= zuJysob;c5PLqp6dRe%#qw^y6ZSw~8nr@zI@(mLfOA3w!Jx}XBG0>O_+a$z-7kwMLX zq`s#b04b#0$P9o!Y=Ub$5;m5&3d{aEFzMT<@CM+Ns}|^L&Bbo^1YM2vp%+8; z>Un1E6qXyD(_WulsUVH;?ws7Jd;Ft%?@%DrrhnaO-3eDTtZd|}! z&W8^fl9gEx9mve#BS$|9p!?2uMN;}#{$wqtNN5zr^VC@l`cFY6bv&Wd9r6j!CY;fH z*TIy0q*r|CJ7=pcm8EXrjn2&l+RBBrLYc%0wo8Nztva`qRkmXxHPanZ#mbJSipW&~ zi)c_YeEGDd9pp*DWn}EhF?$#n2Xzc*rJFlb|EfolK?tzWKXM1HT|{nEDUp|KagRUb}Il&6iNIplEhLO*D8o>SWI3$NED3lhkkDENTKK_+b}tbjN8h4pHeR`-0?wn_d}ap#)pdAN+G)% zR}|_>cUhu9S$4KWgVL+T6|#}V0WTBXlLWEgD3V9!eb8#%O2m>*<5}zcfCs@Vk^_j@ z=zy4>L^QkX5x135gbXCY^t#7Ozfa(J9C-I9k+!q`7h~KRk<9UTd8s9+d)SVi$&#u1 zApHSRrQFv0+xVyLEtjJW(ouThak^cPYk^?QkB5%tL7;Tfn!J6=@ zi7gsm5A@})F*XCem8AZdGk_p`M$V-Vx@mRK8;JwyB)o6dvM=?A2p7BGzY3=T2{Wq%gQP6p zTTlYG!PKzJLT-BLQ~S4*f|8@z6x5RAIP;0Q%Q9gCowy#B?Wuu(6WCDSR{oy5mLhRus zC)bfZnA6KvC^)SW=K3b5cF)`yWc;THn}DO*6QjUJ^)c<|k<5 z#28z6fHcfaaZ;S(TROTxt3uW{Dk@n#A;dO$2PWwi3n|6FULI1(98?zF+y*_*#IeNcd=cZRUu2ISF ze0t-t^@q4}La%xEocp=qk@4j5)a?hD`f(Rb3OAVwZzcaz&=(Ow8_z=dr<_y|Wx2-G z1HOgUPfOnh1QNn>*HFSt!rtHS&e=8+UiWR8IaCt%r0t7`IVq=REP_noHm&6T0`OoVCUAB<{x7w)Q(vk%wj_!TNUX@}Pi5#bd2s3er3D=mdSBu>sfC$NtxImT+wLeY zC*}kVv`T%9S{4s-6FB3Ea++(zZZ@W(qR@^Ngfj5Tm9Z?E9CvXAzW>!)h4_AnH6f3YWGC+iJb&xm}+Qqj9wCkDs!7deL^_5FnJx5|GH?zIG?vZV^io;=0)o@Naz0 z+JD3Q%JLDDET>~{OmUAtfY6U>lw8=Wwis@0+@AS&^CsoCR}^3K?SuHv$@H&CzVRc! zht`9^aKy`tG4q{+h`v{U_HmiCRk!a@FxeUHXXF+$_-Yk4#S!;n15ma|oi_xoO-M(o zQ98!HnXU({^1AI84JUEQSy6Xn4t1{a89tQzWY}e1_DDLH`*Wm6V`pv;K95~@5Udy3 z= z=aIV0Go_X*KrCx(c~T4YV7XTpQ}~U>K>ob2Yp$30@XQts!z>S}uuBndTIxiP{)J(a zUCk}O27{cz`Lqg9KXDOb*}3OQ+CpRTfs~yRW-&~7o*VBJ|MINh0q?dJ%9UQeJXTb1 zJ_S-{DhH206?vXf!A~G!^9mGD`+pGqBe}Mj`+m|c&O{%wp)ERJPv+mn7TW`SDkq6J>a86%5HnR=A_%u<>+R)^IZliUkWmMeZy9;D23wwK-O zO`^gT7mmOM{NJ9Ri+&S~k??{mS@S%xcOl15^P@*Mt+gG(e_UMzG66`R#)Q5RnZbIe zPIUWiQCq-~pA^n^duw0R;>BJ00x!R_E0j;C~q;BUZ;}6#iiXtl9y^zG*w2@C2$hBLf#`bB)UVg>p^5@O==7=4^e6q<{DH;@k zRjg!xDDo~Y^e#K;eqv+!CQVg6#Qp(tn%7Hl2lxD#OcfAAcgG$XLk|hSw^t!q>m_s_ z1@a<1smyywX&yQ;=6Z$(AASBTI&}AN3bbOnd7w)L08eZmu!6dcPHxf^(%e7)Ipyd=`{Mv6>zsQn>nu8bLTUmh2M9n7=?lF{m$lY-H%l zt6YC6J16G*8FAUouvBw)U~e>64R-Q1Aovl;SgN@Xm_G;#Pomao96w|9Nzxu%oCA$W z)6@ORMvfhVR6t752o1fs>P>)ucM3vQPZ^Xu)Ywgb)gfZ9IMktpb~lHTSJZ_q;Tt&b z(3NY+F;TUK-d{|(WAZ|*CUB=U5rqorJaP~)(ai3CH|yEn-;nGxYRHmPTG)IvNL7AQ zOb&6=%t{8$U0J@|uX7IVR^iRDh`f;q5(P5q)IitGZ67B7ov+NXd7B0e@=x`<6p{;x zNQ%W~HG06^N_Hm&OXCIjd8e8;i;AB(>5bmX9k@B7VeC&o*2@njuWJyDm#t7&>wj!$ zQsi{*j{9#)?Xf00GI$G)#J%8(r*qrdfYP<=1x+lYU)gSq)8tEupIvukQ+{9j(s`^g z6S>P039fx5%05FAu`TiI-i1Kya+7q9AV%CyhY?+G=JY4!gK31Q$zr;;r;$5} z-#c#z=Up@?#qiVLf9heaM42K!+XM~#C^IISh;~Ia!p2S+6GlW7P0pl@JS}x=cDv{$ zt>p|!mzBM#?Zv|cJF z%A7Z!ifFXetrT%x9y^uRmRh7dXeD;!v%j&2TNL^FA&iJR7yD%}909w}I}(k&Rj)o} z2M;<24jy~htWKVaBqEfYPIXH8d}=!{5LGcVxowf3%XS54t!*V~#&0`Pu~N5y=jV2; z_~$ma8)xk%i_E?g=1PH*M<~JiTHT8*9T>Z^&i>dUw#z97sdW%OdNTmpiooE>I>hk^LEdG zQr`~A$Fr~{$=5-sZ{MjNR3K~UhFXpd-t{VLxB^Pe@ee}`E2dH|x-clpI?hde5t_U1z&>!gMD-YC` zPBxoPF9tN*`h8qBCz@Qe3YXPucRG!c&Ggi4Unf|eWFk~fUQGw@KI|0G&F%d5>(GqX zRs6H7?Mj0%hBj!rdqC)Xp>oWVp7J*?U_lPJw|)HGd4h*}%#g~d?oTqlyFpNibGd1- z_E>V#DifZmiz`?RvDo7ZXWBPdq1u?c?}J#$d*_2#e^@C;Bq$QJw6ISKxj6Lk`ei9za{mzWIuRkcS2px8QhAd+aFHjo4=+_~iSy{4+4x0zL-&Cc=>hfX z1sdmd^JZ?1uq(5XZhzAn9E=Z{95O6q6UY#=(I)cg<=cNN@K-C4F;`b7VwbTCR}D%%Wx+rPqZwYH)k@%U~H zNY~S-+kBuwpCR}0%Wv{-r%YYbu%{cJKG2h?bD@b+)vF{n5%B@)76)~gkSEnsP!$g3 z307k7zs~HJr5^3fpD^KCdlakyl<5QfsCCUAWt4+hMF?m}6VwKUG!hzxjhze`c*_TE_~wPXvdySwQeZjY0q;BQ?u(;fua~ zG72^Qg{5$;U3kZ& z!82i=!mMM3$|;uhO^J$Gv1#Ox?d}B zmFBkqL&NC&C7I^&>_lrGv`l{Iw}!!^mGM-L<=DZumj@Lr6G6;L3D_1{`p?C0@LaO0 zck{uJ;UY5B1&t$~(!J?0D6z{YO{=x#wrP!f;bGF=6ZI5X<}2tFh(^So6%~Q3QV@n= zb_KKy!6^ql&!7%__8i28EBgF?R+;bNt^hFoK`t9jssYnbD?AP$$*!6Fhy445-_d!T z0CbLK7-DNEpR^HkCR&G>y5wsf=t@*me#f~UeL7vTvy0%MC|7W#{N@L z-?5D?Y%Fb(WKVMM?PcWRTXobXf9cx+2JugBy!3Tw!U~i_A0F8LHjiq}gX6Di@)Z;l zyfBpAkjbAg48Uc(`M|Kpk`IlX=)I5lfP(UuGg?p`q2Tr|{Q%ka^5H~^r-4ox%Rjmu zOFc{zwhCX%|*w0GM(VEqdTu%^KOy(?+m=FxS0|wA~1l(*# ze@;Ez@Yoa?gNcqG(^g$36=0@Pv2NIFV)zfo;zQEx+IAv*TiMN@DlrSP+Zj%iEEn*s zXo#7tP`6t1RE8Q=b1Vp-jI2(gyM8cTU!vnOsst)nJz=TVqduvk?S#9m)MuwZAuZs{ z8L(GMC;PoCX}TtN8I(LK-=$;T2u(rUT-aUB)sNses}=33jpk#zt(U#sqw5&*5);4ou3uL#Q_7JePrBc7g%FP1VfE4#j^wq zE(wJ<&B}m<)@-aLe~jAR^-MJtIv>O&!Wt{^FE#Yx%N&3y9w}?9?M^%Dh}dZhL#bXu zV!XFyOv-Yt2Gh2&Oz3yV^BI@owQQnYV0{Ji%<*tI_WT|{rIkFNYEAN)!6DcRg4UvJ15HcERliAEL`tgO<*IC(dl1fS?UIv)R68S#==xN$0p=b@Ny)W_(0 zHF?n){>n%9dX*M@I;-*C#kN>}l}m`f9Xf~^L;wg13Z^i9p_d)2(wCa)0E}5byEY)6 z`oX@xCsr&L*PH(VdYxZkFMtuESr@wtUj}`$3@GfGpKn%JwAEmAQPuSn$vrq99vYGq z*GV_d#!kGF4QSQp(_kj10kAX$VP(+7Nmxp&b?N5~j`eEyM&B;uh#n9A;a#znyj^`p zEp+TE`sfzU8cZ~)ZsPiSvk~RNSpKc&Sxw0h4s~+t4zD40^0~}=Gy9hb0q_t&@oU?D zVk;A{zS^_ZW%(hEX*W_TAX3fI!9Z{ARaswvBFs~=~^nM7-@Vs$Rd%WfV8d9o&8gLZ0iY67j>S4wG)yy<$yo_zLNG8owMo5s7;@B#wBpvCvE^s@XNGRtq7I z=lBxh#c@8EO}tj8F0N&mvkaal(UbGq;ta}T(Co{PN9I0XV()%N-CEWGft?-tv_^#( zO|j&{D%G0u#Itd&zU3YCQnFRP+q9KL!*h4Kf_`>NcllL$+ z%ScGXj6f*dz=Ym!OM`MgBPR8@lI2kLIuno9xNjuSpbA5txRM`RNZYLvK`+vj>+`KG z+|bjv=LVPibme*^s?=F)U8!T?&9M}5`W8fjP7Bn_BXPk_w}z-cczBO0^0Lp~U{J=s z8B8I1WtxX7NPV+2NZnX_E?-6nLWqpsr-U^b^kI|5KjKK*SZMYOkTgFh9f~BBfapB3 zZiH>N7WEyd#^yp+^qj3)^gJQe zqq_wdk*e5}2NulelSGBW)+$#Wf-hm$0vap>b0--n-($~uy~5yw0*jm6W}#Q|cMNfl zoD6AbePZ#sHQR(;-=9Qif2A+FSEM$+A1-99)OH};*Sah?o~&ALQgLPRff_;kk5Dh| zp;sEs?dr3bv~r8p_YN3V8%$oKG&A-_3p`u)UfL^;8^C8ZtM6N876kgW{_!YkniRmi z%^H+EyqOA3Sj0E^$~y#ixgyb1&X%u2+Fd^CEFpT-v_Nf5udPz*NMyL0(L6pU#49a# zE`LreweKYK5$tIMg5lbHq+6{5I5-3CH{}v0U!D(t7a;g>x)1hvly)^$k!#*NurIA* zXUroLR&-SWVJ6$Me9YIFb|ETmVedOdnZ0lf71ql;-W3};fYS>+s>KGbIH4|h?$Z_S zA%52~3Dh>0H8eA;tKKS@{b(ssLiI^p!a_aU%T&HM5jTJQ{zAY%K0&pt?*ND~&unH%`}c@TFdNj+^e(#?y^_1a`fESG{5gNT{MAO|Cgv{7 z$1tqp`NdAlMeA5bMpT$ZOs^M|zTBVTs_mL+E$JH{ooN1k+IQNDr_SH@$C3_%03nm^ zE=f+^op*`Q{;^_Mc5(T{stu}=fa5zW!Q&S!;ZMFyf(Ayo5Z-e8ayp&jo`Ry;aoABq-mn7&cKco{$oH25ZUf36?YFo5l1cnSjPL6LT)`(47u zc?+%TwmceP15Xg9v9}~Xfia;Ke%zLgN8de54@Fnaz@Xp*kkUFu<+SUIbefRJ@Y2&z zJn-nrgqW|4Qf7uJgE+n}I&HXyGZ!cF>!|xzL#s=p%67RY#c;m&HAD+^XKtN!LKN;- zg)y1R6@A`*{saNJ7}sgq`>FF;S&ONAhB$!3dN~CIB;tJlCD_lw-u&{g6FY&tL7I4% z^VAl1@;EFc&n43h%k<{Hh(mLE4@tErmpYXw&4Y@P%{sg$WTsO^eXLC;q}O9Y*2}lA z#ZItLQ~Ix+rU6#?qcX!jTsAWgg0J*(IbLN|{b~Y$8^-k)ROpf^Dj$xY<(xuF!d#)I zG)!wpC7s`wE^T2SWCD$c?22+Dp$tT>c2gn z&{J5wJBGx-$^pLbgF_#*-Z3N2<%cMmE?wH+{t2Y1<$UXU4CDzD#&AOEdCg%ytP+`( zxSP*y@3aT*Sk8MxNdCC8Yx+TRbdizeEg%*~KdsSif!vSm$_b0i8H)E8e5!7+XGOLr zJ^}H8C6k8%``bcCTEzz5fDnn3*S^+Rbn7jx^HU{^J+`$p2E+4Eni-uW&$xE6HDJQQof5sM}Ow^4G-~YvS3FG%XPWgN&Uw(?7Q8$ zntJa);Lnb4=y)vSm-N}zpdbD|h@QWp(cjR$-3c%FvAqV8r^sk~pH7@#yl_1zzp_wc z5U2?9=;c_O-YxngPs`+uPx=QWc=mnnxDIEJfGkz`4XL-PH|SdBg9))$>{d=DqH?s_ z%!bLsjK7G}-0G-_O5siVUeXSuNlR8AiBQxIC1fh!o*y`6b$nXZ>)s(WVeMxp1pJsETdnR$1SGVt-JlC~>5522}Cg#I^oe z_lVlJvKj6NHsb%i1C=hJkc3Oi!>7~!v{79ZjI>qMH{0-jZ2+VVgk$$6JkII9_TY@; zv0Iok;l0Dv@jL~!*o>ZCq|SVvWDpH!WD;t!?O)>6))ZNb%O4W)x$2jky`bme`uR0z zaP^O&0oMCQakF~SJcvfefyDcOmodo>#0NRgdML|w*vCjHK*j{}^P#Qf2C1ey2*o-; zC{}JAT^!Kj5na&QU_GyE#{O(Rzt&JcSwn#ltt0IQ5H$j$>n7}GOeDqZsQo(UU-%}}iiWBnN^nN8wSP_v?X)|* zdUHG&nuLLHC|`8{@uC>|6?+T^dms!urf|0hnu%~0@6kh2v%DLC^NKp2SOkgmdX0g1 zb@%DH{t%|#d0nCfiRnA@R)DZQ7WVsd-hrOfK*1tGz%__+jvv^C`q^Nib~Ke^tnIa7 zGp)Jm6n+rdKStgsBsqdphN1Um3qS+tQFc$BQ@0|G8cA(yMuX5aa;s9|ts4+&zd0?Z zQ7t{AtBtZ=@evAnur09atq7r=-MBHa32{wX%b5*vz<@qhc2r>(M;x|(VtWdgOV^oY zwttm6S+6U@6ilXTl5Co&cRiMQ>diSRL)r&$Hg_rlA%KL5NB``=e!{B zQiqirEG!;<@8VJ}2-`2wnhm~&`V39C5PFIRA>ul1%&2KGHgQBX4n)ZfsJ!R`u=%7g z!Tj-gG;dsh{{3F3;PJTy*-zTI`E;gh+;YD5m(uiSdLrHpySLu_+etxf?_Z4`EUJRk}d61CLC>{?>Tl?G7_toz34Vu z-i~l zA#`1XJTWiMJo5qfZK@I2db9iqBEsZ)kM~yZ5?1TGO9`gO140fIzCd zGk>%EsEfPbaqp<6vM-!!_s`GjmFihCU5ftaaCCrDIqx<8Cn9Uh7uPlGw*p$M{p?0` z%u{Xy+<>zaP`UdCA~_dbp*c|NucM_;ih`~G);Ef@f|)5N`$C}`ULb?bRXY0q7P8Ot zOkk<2>U2DpNn%9A?~EQv_?DogRXug3^zbmO`a3|ZZb0J$mAtPZh0VWXlE0dy#oDZv zdSIi~YPBYW()$DB?s#49{w^Vwo?#~idUD)7iTBydzyOsjrSByo=;JVO%6|P>upu9{ zR-?k`2qKZ`T}yxt%d3Z^b^$#_tJ(dHlHfR%4pPv^A zqQ67XJInpfb@B+0_+vdxo)vxa`B2pfY|_;apdFJhJoCgF?HdpQh97LDzRqzMI`=F~vzzSD4@Awlsz&P3e!x$GP_&_7jsM zJE|!?Q9l2|M&8mivTz2V+4*Y$-XRT!BwSglNy8=*DoLkaTkz`?^?>|YM7LZ0{(3@z z3)=nJ#>(kr;{A{F4dW7WIU~w(lORRR!*j zo(JTg7WvLHscte{1dRTh{Ym!qBJUV9`M$zo%Y+h!32_k3wpEa7u4tdhgR%R}ZqX)y zJmwCdwbp6+9Uyi26467&feO~d;A5?4zR(fS+3qrcFuGvNwU~RU#sIEI3+lO7Z11+B z&{r?@V`xY98|5!@_kNA_ z3b6d$(onE^28Fl~O0Xv;1Pff{TCxLyla5#<)I9dCMU>aQ;e7Sx`2%BkmN5NlZU+a= z_tP*-7(^PIIe`#tMGEo$q;qO1k9rMf8g*(-jSNK_=i}INm5hUn%*r{D@@ahPn#v~> zRdVytcIRD$TGXk1$u47EME8Jn;$)Xj-@z`Fl`%ML^{zL9GvCz(prFgT*fmyg99Lq?` z100%|c%wxqPZF6GZ@8g=9&m`cU~h#)>~uj---|1jQCcO0D+=w!2qQB~i!yXGrVzL9 zMQpAFJ%N8dbn3&^j3Ly{Fv)oGIL_~qP2WH`_H1ze8b!<|8`i7J6DmTb@L9b1u*{&c zBfHabAgRZ7p_dgpeu{E>&?kc}G?V!16yyvO^JV>vzl)=-!!hHVi`B zuD~7?QSjn8BY z*Ays8b2|dmYfS2>%T#%L_1t&l%H)3WzK(Gwcq}$bd-UQqoioVW%#$*(7S!+1Hqjz_ zZXZn;H8(LPA?%gvC(mXQ>-O@VX$p>}{4O>ls54vEev)3|#L=|hg6?Jo6ivUQk4JmP z@_50tbNmO|a+{ix+KJ1|B>4oB^%jeh%&YI;5o;J%DxQG|VDR23C~ve{h!vyi)|1M_ z1&w957L{F1i&ZKBiQNq~=8@P)88aS@{gxEOlZVHt#}ve_KOsoRR9QiCW6VvRHh18_ z6c}(VSd(sqx4sB-XnVrzZSL`gi1W5nFwYl|iI)O4|=?r|! zRg}y!g93Cg5uE{>UM~HyuXb-Tnl}v``D#9vo8Jt(N;<-&D)hpF1R_D?*0ts;5yQI2 z?aMrqLKbHf-j^0c^#^j@L%PHX`KuSXtjR(Dt4#1Y(? z59e`Q1x5lYbeRJH6z#Wt-7ebYET5gklM5%%F~7(y7T%k1uA%mc_{8MSvKBcM=e=Gr@NHCoBuO4pm*ZgSafpudT4vk#<~HZsq?--X&M3)$`WlHz)) zBzH_Ea?rWCxs`tBoSa^;vkl;dGV>(-p$*pD769N53^zkT?WFf1flPO$i4b6J@DkVCHOz6J{mzR$fB#GO-vUQ7)JmM>BljZv+=YGxCj1>YNd-tBWySt)cj z$8?Uwt+(3DqAO{Tk$SB)3+>gb@0KC0Pc}hSBu!s!AA2`+eM1YJ-x3`#0#`m4H&J#| zXsvT;OLOUbj(D{YA`V|RP=aQpB~|rJM$Wa949J`0r_d`Gb~(ol&$6meLD&wiS%rc2 z8`~l^E~thwnM1?I#s2dmjNN2#CG&h1Zs_pzs!&nV_irtL<8{m}Ym_P8SiZnUtmz#iuJyX~#`!mIVmd@gY^D(Znk@H)9rNbNgB}eYl|GFyTYa zJm_bQHtvc#?@QMloMz?r86-83r;oF@c=&Dst^u$Mf{Pj2$T;O-iAz(+RHge)TPe+P zg|V_!0$@t*vP-FV07D3Ggban@aRD-{>vfw*v~e<&fcO$>jYI9Uf;?;p6OG6x*+-96^Nn@DwQb-DB4bLzQ36rOjuw&o3LzakM(zD&$ja9rg+zrt`;ehXt; zNn#syeeHLP#*OsN0}fBW;zq)p+#Db3Psf`J>ZoyOBH?p67ps|b$K1g2g##y_;>+o# ziCDs%D0rz7`0tEsA{>_-Y5s$yLW2}rvNaLA$jWStMMKJLK7zKbq~&9ePwp{oztXaY zl)*OOK&Ka-Af)RFzAb4c08>7i-Mi+pQaP3boqB)1vh5`J|JOr+P5yQpVr&PnBb!VimxYSf%fqD^XQW+1l$X`ohPj%G7% zMiSdNCS15;~sPtA@x|?O>K&3xV|Jnk@B&i1Nn7U}p9DFZbgu|4Nc^?et(ibko*dbQ%v-og{m3c}9P@??1pFM9G4L`hxre6(^>U5nsn8cvKNO7aQ$5`nT?PuUR2?^xi1v2-jD3L+m^QA!&i4Q6^Cp(zkxU7IVY;Lp1>;6W9Y-U*f z6duR=m#=1}0W;gAJ87RzL?+} zyCKq&V6l}A;(>4?f5fCf+;`PMJJ6N)Uoe}$*pchmGIKkou*tIZlk8KY!P$8CWXVK+ zirbh>l>n4jOQh+Bv&m#kk%rEm^N-2pExtx{sj7QROaA|X)S%1nx&T|PIrz(LZfGo^ z7UpRRG`SSkl!Y6#$je)d?uH0UyhYB+t6SJLmwXsC#@YB!O(rU2?GyicnnK8#7rp7` z?bCX@3Cu=jWB3g}C(U>;m1?56>H*cvg0|QiF-B5H1{=!tir^;`- zea(sXa=R!{9fxT$C-pCQ-r9mcq2YYZbnR%%=Y$b3yK%4&>^H4jME2n@#<$ z4SsbM|45Roi;ls@04`wncez3mRNY1?Kfj7H_k{I0xwzo8x-mtF^peqVG71ph_ltw> z^OhBOv%3ACos|KO2us|>+g;EG@*$6=m=UifvQ4%ekemGd_a(rV^g7)Z`4{Q7KMUw&C?pG9LSVX;tel(l0|JTYQhA)(@ia#_1)#l|fdn12ONn$ch zfM{kO%Bc4ZpLCnGRx<_XqP-rPUKSmE`V19jsYzq$=0~RWGLCk~wOe2D~>|I1@XatleGZCE8$Wp+Wu&OlH9mYJ6jrXZDnDXD}>gatUa zsodK=md`ZmtxYRKXlbS{k2?Q#!v@N2(}BG*cXNbQIT`hfdgmuspIrcj-@P}k@fsRr z%?b`X`?S8WNpgdBdC2V`@B80f{whL}w?SvK)-K0p%9t$0Kq27nHxsa}B+x4P zwwgHCVF@#u#Pxu|tLH86Wpuss*A?E_Y62owytCk% z)rrI;@Bz#fjr?~Aky0Lt0$1ba?(AN);x?N!F2_pk)K85zqL~$|De66E?rVj2m`^XX zVJjUWy5MgkZJ^d(^Bd%r4rOxj=jMtu8iyzJu$4N6)g+%Pg^>=43QY4^HZpoXf9&_h z7@KvZLJ6lbmHDuONN1R<;cE_^Y`c#(N&*mEP+5dWIggUh+!EyimRimkfgaNNObSkC)6o zZO>n1P$i;e!8_lYqp_0~dQXsee+*@kvmebqQ0pvwO7T2+-8}BnabYk8uH@kEPo~Z8 zI5%vM-s#l4?qi3c@01c{Y!;?-*A~k;bb4&EtlUkqT&=Ps>kfR@IVV^yKDSILF=~mG zIIcDVa%Wubl995zi-|rvlX7t#MlKEz^UErY;i&Yl+{N#QGY!A(pX@2uYlgY^p2*&| zFJrRWgJ!Z+bd2 z9#4lQtTc8jxW~B^JxBfVLqLH37WvK;G#b=YOTwmDNZX*>PI4>A}D?i4G`U{kq; z@}Op6Pv@gcpxSIj8m{{mg835so62EO)lJqui4o(O`ody*yYTpoFa!A8Cd~0e(YV zU+ruM{Vmr}Nc2r}Ee>}5Gq#@49#lp#ON^$_=p5YPxEX^sAu?D6(X=1o6V)6Hy2Q+W zRB|MFn{c#7&MRIh~WZAQqSx`xj&eLYmEC5{q;$Z}R@Pksg05F*v~3;ld8FWV2I z9Q*JG)Jlup^|*Lyn^gewl414IbbXJd>dU;0w)sPVsCjQRLcgOZJ=SnixC0#6pc?54 zs2or+%oN(^n#k?f!{9xnCu+6pJhpq8J%d&wgNXsXdCxV{&yzKZ6r`QsSEB9~;zEem zz*{IxQh^0V<>pWOe-a|)v$|wKQ5%h9BmG>Li)3DO18MA&SD!o$E6sxlORL>+>*Ex! z$idrVM^n9nicT{Qfrv5XhYXVjt(>FRwWKn9cM%iz+^@!3F`@|U@GL%C9|4*$f3 zL`RT(k6bFNWeTR!eA!d1dhj|zZMvuVXnWK7E$W#J zUed4c9hhqsY2yf_zby?&#Lw3#PJxRj*4KpWs&rge_tx&0!c@ytPjh9W=THdB<0Em( zUjy9DQrdnGue#eOsWUVE^&(`^>! zKvI?|_*!DtLEVtkKSnoS5G+)fHJ9YeTlDMkI>hE`)`nw}mc@iMdtLf*gjYJqQr|yq zudw)yOOQB>c)ATQn@*4}?ojNp(vPnBl+gtJc}r;LaS4K-=1UBfRHok5%Atpn%gE7k zy*yqEI_EH#jZS3DC80?C6*?8jx_rn$nIsbQJw+g+btb^(LS^`fB|c9c!T~fJ`L#10 zv(507h_rW!iJ>x{>05eKCQnxG9-#+{bk4Yr?LwhOSz_F(VUWp0F2~QEGjSerNa}rk zc5K3Y>V6S1U0uF!EzWV2le#h;ZS{rzPZf2jaHQkS#0LO6E~Z3!kwd0$d72n5>@U&j zu_7tKPNgbu+R!nQo}OfK={2Ds6@03Dk8n_UNoNUtHN5htjUvP_%Y}lNSmAi^$I99M z&#P-E|2sTbgBkc$XX1G2QVkl*_aM`*fpL zzg%o)`)mh)TUHXpd+p;Hu5?Fx2HH`3O8JCo)R7igCiNeeKKR=L9 zHWGespX4Px%0$f$M8=mDt{b>x6tjT6^0|l?!OePhqS;Ej$Tt{z$DuoPOhod0~683;k5OOSp29fU~*%>p-SNxI^oD=a+PN^-XtadiM^A# zIo#wz{3lF)oVW{+BfDVRN$LU`tRPD^7+rWN4xoO&6m}@fyb|~lgSGDif?mD^$nm9l zciQLxx_5~P$ixcz9Y*eW)f(ogjd%|opcxHOVcIN3!9q@9| zEA^=~gJ7BMgLY!Oe1FEFJMMioB!<1BiXL$jI(%TyiHI1E33wGJ5IJ)vMV<-@yg8({ zIQb5LdWR;Ch%Z({mk7V_*Mk?qm?9T`$f#E&0(CvLC6G~~3fD){VGf2`AJ2Hj*Kw^} z`w7x0nw1GNM%os3`&S28q7qM&C0Rw5CsTQn5UP=!>t{R(mfjhp@}~C1PPi7gd`#wV ztqv;Mzm&j<;}N(-65@2xzhPFUsD8D(jV~9?vq`%Lrg+O71`2+qxu_kps_O{Rf)U832p*Z%fmURrx#+B!D8DQI4)Il_XJe)wqJ zXn!I_4gQ-*pboKSky6isqjrrf98L1GLNlB*yBLBJy#HoWz%Yk`K+DyHQ$8ExmB9e;kRn z9Nf6eh4K9SO$0}WxuL_Uv{j!K2lH?TrBwq`DMq4=7ql&15YgrWn72SAnnY!^lRPM% zJ%OvAjX1>C7KAA_Mg1`^tZoPIiQvFCbub;1d2@Hh?2RD_hw_=KNm;f=5$XNzd0#%_ zX1flTXm1kqLgA%o6xa4Ze3JKvv9ADAS0I7jLhmPJR8i1a@jSjWYCl`a{Q6HA{!L5q zNvv^Wyo!pxXTXCzLxWR;-Oat1R#AJ4W2j^L*~&2J_?s04KpNSf33;7*OkK1}87+`; zL}B5I1>REK_jdpjp0G7ABgpL*U3EF3D|X~48W!wC1P$m9-wJs^@XHj9!g zI8orDyLDfO*+C@O0pVNV0i3WkhZ6w9NXzX*q+hY`Z9TaWu+OK{O_nYW49ep>PG$i8 zot=H_#?9)>^`!Fs$9XWY+r9 z^1nN1K5@t$XPcrUFPPyfH5m^TKDBDD>iy6#ImVO!2pDaz7^^w>lDs=-g$pWR0x)n5T+OwwQJ%GY&UWZ{}@mG+*U zl)opv4IRfHCDc~*iL1TFh{k^=l2G!w$SoFZQZlH@Onn>*JF!Fm=g`$4HR+A9DCPVA zHzvT(EJSFh3+G7qLT?ZKNTHM1$KuW}=!03j)lC?B4+)PsS=aG!0eb0MQ1d{Afq;9x zCsYd#tt=74CN?&Q?!eXuCH#wd;hMth`u7KkD^gNcMi#zw-97Xeh0S*d8Ev_TWzf*w z=`g3&`nPt)cINqtm>HX+#Kz?A7RUYP^0U-sD7%PT^HC5QJ(Vm6ei)n2)`p-V9d z`J98_*FYY?ylh%lK`s42)kbriNME3@7^DW5h6OqOie{~mF_0Sx{x)GhC=h=%fzJ6o zfFkJ(0b99${223T5m{@rV-RVi%9`%leAZ7l5DN>-Xd;UX)*>);kiD5g%T1=)F+{ea zhs3%aHJzmSK+0`Gw+@efrJqVGBDc-uyLJ;_%f9Xh3S-n-7EW(E!pBLTwO(T(lV2H? z)0;7jFvN2lU_i$7A;E5&kN;>G^Q+`dqHQ*k|7j9GZ>TYwz*oyjnm{qwlg}y@zF7F*09{h=J9t;i z9iaWX|B~hIbTl2i?=Z^gh~1**EuFZdRE#Q4{8fx`r~gap9RK#aSIQuV4riUm&p$TR z5^;|t1sv)9%mI(G7&BiI&_24iqQ>gGn*m89H{vf<9F{NW7cRVU8sX~kmzz#uSPLa0 zgAo)-<|;Hp*Yh;b#EYY9`2{8YQ>bAtW+*{S`n3HnU1;>v&FUTlx2Uce|L}oapuVW3 z$SAgdc;^3~&-U+ouysuKn;IHag7QjAvDPBF?EO6AtRxd#J;>nW4N}caGGfxtE#`k2 ze)#Jp8ln0(?=cB2CspLTZimsR_qRPh#d#FaLYA=^Ivyg_$kt{quZ7-)k4+UPY6&;?(*xpD%!3!nrL*K&3mlPqVS)eke-~sNF5vlfaDDP1Nt4N znu{MO2nm)%1M~>JwbgO!}XLlbCNW@Yg3Z>%~9Z{)GRu z8#|y8a}*u`NGa((o~2+88{v|xxA%ffu`y(XNn%vOKugNM-KzhpegFTw-mKJM%{=gL z^)NlDVvRH^!@vRy^?7#h552HVjsWyOnnjltn~I4su1q5h%AR}_8!d9Z4=62gERil* zOe1AFDBMCsT_40L3l#>2d2hX5k&`$OG%te{@m|Lc*WL|0;%_aHJ3;%Yx9~MJS2K4r znGjUUXCiP0M8UyDOQwK+AFhj;rdb?i)$jH7LV2SN1&9E142*9-CUzQr?yty|YS$n( zSm~}>&8)Q`E=W)4vtk1RgTS_+xVD9Xo|G3|Aa=3W`rb;a; zB+jg$`^aiOKUo9yG0~r?#T!TWWk#D}NDUoqCb@&?7$ALo0(dD~j3&F5G~sO)e)kGo zWRNvkE?HF!R%-5q&CF8Pm3zn_i=YK8Uophi=Qq@Duh+jpD z`FuJ>#j$Oz^3MaAIF*TW9I5pb+xa;_%U(PN$rXf!TZ&&ARiobI+qWZ_Q0{Y{-v$RZ zSwlSxD&W0j*Rs--^PS4HL7(Y$rW@f){*wqHAni87l{~W7`(HWDd{B@emV^3rOkrQh zv+)1&YeXC?;I<}RzxuM!!#lu>!fZgu~Mv8xQrD@oRY;O-I}g1ZN|;O=h0 zA$Wk`?(V_eg1ftWaCditJNIPv&hG4;ow@sm2flOYue-Xsy7aByOwVkoBdQXyyBar5 zI&xm7Ai@|}+trvrpz55**<0nQG8{huEA*2Lj3g;kZw9;Cc@#C<3#D)^0PuG>p#`MHY!gNHiG`mQVy5$*c|sSc;~3T8@7swcuzF=tmhj_U~e{OwMq#_yP@9oja6Q?gh!eruU8dbss{PY=4*lP77V_-zgSxR`2pC%btBDeG4YuYSY+gEIVk9mK$(uA;N z?m(R~wd5)j`0Q7!m7MPW@gW5r&)OLHJZy4zG-QEvpsV!D34~JF7sjn{gfh#;y2u*> z4qD0J(C{4HL@b)7!cVZ6BSN*YhdPA1bl>Gmxlfi|p)%xW)<@)p3b}}uOKfU_u)~Mf#HU{pt?5iaiC$$*szzp2>*}=MQumRBq4@Y5wZnwKo!vekp@8p{^<^^IN=EEagq8(WG$? z<^Olv|7Mh*E1?EGE$M+kKza+^~WD2*!{sRHN#tPm4{CV3ZtLn8`p6^j%oqv-7w-u+_BI-<;2-lCN7^yFsRyJ(r~ z)DMO{*j&15Gav|4QXFl(!N-w5fl+I(ZdvEq0kPo z(uPU{&V*BU8a=qKQ6%}8ziqBnL)B%9Pe?PoJ(3<@w$>s9-9;Q+OsA-&g*z&aa+Jx< z&Th~r0-jfyJ)~^^-U;)x1;TAI7=vbw-{hrLaL#%;I*$;4uiR*BE?lDaI`zf%o-e6g z>tIu4XTBQnUA+MxOPBq5G^a4X+&H;{P1EsJ#W`sihm8!Rzfi5*i7R?7eprDprwnx* zQ#Z4Nh=fAEe_kj^oiM?c`bx|0C6gx3q3gZNheAI8XC~25#KT8*m|OGU?B_egcM{NV zABHTXv`yNi-$gqW&}ryQfh$8|YHLUJEw+J~xeM#Arj&^V`s94_jYo@i(*HM9=0G+6$X3B76mkQ+M~V1Md|qVJ;fa9g;5xf_Oen^%S5|hP8^}oN7Cx}KIYbveQ4&k zxEVD2FPO`_Adp{#K(eC1BMX^vj0>d89mS6n%a9%3CNYjsi?*>LWFrHYLy8@zwA)hZ zX@;R7yn|M1^ZD@;Ax{)aYs*y%{;=JbFEl(z2@-aOuyLT_{Z&)_ zEvO9vO_)z!PtR_Yh+5aV+^rZEH9yN+!9v-zxma>ACD3LkOVO-K9G~sOsN$)GuPs`0 zdyD-`1iZm+#m1n5gXzHP#jUK$;z$SVG!-5tj#35hnDwT=ban@}Jsk7Ik5by3Wfe(N z57jz(E}p12$O5KXtQM%xK&>7TNmiH&HjxAhyrQNu>>?`}4G$4LIM{b-%3ZomPN@dR za1{S*vP3QCo^5U?bxKELI^u02QkYQ5GUtnLCz@eZAB7b*TV1zfxDbOO3~Pc_0eTKB zgZVtQtWU{&2|**u2V)cthE87jx}rrh1c%}GSGG%H`RQKW-jP`P9ELv5u>L^BqAK#?LlGIkzc5S zKq7M$ELEkNn$-@>Z6@EA44LWI+x$8FPy&=q}cIW-;5Dk@c%Zr7&Xhm z<9Oj?bATo~`i^sL1+zyV2rGzbQuz7~#)oS4`sp1sF_XtdA6Pc_`Q42Qs}XcFr1RwuK6@G-)gUb=lNv?4l zRcZRN1Hto}YBNPD=&PYj`QiQar%oHQcx0iLigch9-*R$B<$kg^RaaP7*Ur7N_E#D0`(ReT1J%EMr1YxH&9)WQEx*1WF{NrhXKfzq=U)t*H z2c&YPcIs3*xrFyFt`?B;MQ-Aj_huS4=)8q402&eK+eyd0D;7M=x%hvKQ351vU>~V3 zq%wUBG8F92q5LryYiD##g6On&XIRrnoJu5w;w3k<0v}sJ^%gn4(CD)kxA4clAK}ee zKu)NzfFR4|hqv_${o*OV@`*a8#0W`lR%A&;%b5R>kn@`=`|skQV@mXmRTOYgT^^{c z^83DyCsK>5Fl&AY`NxcyU;(2vw>v2!IJ^K-KUxrML&Vu0OpNg(=W#%&8w51J96H*% zkVl&ikb<^YrsO;ncTwA8B_82f&|i}3k6!0-ERZ>q z3?|JApQz3)gcN)mcZfB3p!q4~D$=3swA&OdcuUDiR^#2d!{IhvSh>EH=%p}8kZ)5OQKq!KyjV8_( zMUk$L|4kQl4RsBD4YLWjRrMJ0;LdNuDF6vjZCpF)AAU>c`rX?9BY7Y|>FA_<{?7Z-~=5zZZ2ISDfslChi9S&L3F~u7%bUKk4*XJ*H@%@H3RlQU^>5>AhCad zsNyOv$b+fQEb<43LmJB`doE|FEK#85|BL3N_0RPzk{W|sUYM)f0=lzW*>=1GiHPv< zyi(7TTx!+VoXy_!f)MoK$y8^_BL24J>HTn*?Y+HXNz1iQQSF#3Etn!NygJ*v<{8q` zR@FC^PAqe2&9@&P=Y>tiwFzbEW#63(TQ}KWV$^GJ9}{rL;f>g>B(7F?}Mank6clgOIor3}Zv za5Dal38D_58k!qp3Xn#6dPohM4EEj{|BK9CrC=kG$qO11WHJPFC2n93U7MnP@}K+r z`=?5moVL{mfh7C;yRUo`%r*A6d{_60LWX$wiE6u%;hqx}8*6gjPxtttYPK z-^S8jRq^-8d71~kx7u{w@+x)Pz5+CP$*;2-2L-krXHi?uE0RZepar{hbkUB`bjfrJ ziNFXNri%0kRnqn30si}g#9JREA@Crhm&Lji1YB<7oR@14q09jmLqq#x5Ugoh>4>E;1A7MfBsRq6)sIPdIpPJq@*1(bXv(~IyA4YSs) zdGaI>(TPa<2SQR(G~xJM(JG}Y`~W0hOv-nqGXAtxOG%)zNQ#b=d^Y2jKtf-CgGK)d zs<@XaHvdH`6yw!cDRH&6X`+CKkC2@xLa8&|^-bB&$~Bq1pQ!l*E?Ffgy`c)__}Yro z+9&=`4H+KCmp81=)opd|mkE~B6$f04F0i&T$H>DUa?WO{I{)Fs{SQ_m zkZ!gth}v7JM}{ua68uk|*8?R4YBeG&MAe9uqCHDc#xYvs5Y!+QN~U3gPzkdeoj z!-gGfDWIlQ$mMg^l@rPRbN=?Q70t8!1(o*ww&@?X%ka~$Y}fT|(f%IsP$vA-LERr*a=0e$ zA<#gpQJGnM6`Xu&{9?R8Om`PKx(%|OXR8b(h#39nuDLnhP=@bvHSB7X{EwG0y3mER zX5)>c`J)s(3aMNs6(7Kcu`;et%T9D;Aqkct~lv|#shJ}Iocyh z_wFGphQkTUp}xf=q;bye_O8hkG#wpzljqTElk8m&Fkn=W{(8GR-{XzaX}hk|P==l^ zrVL#Qq$M|~C}ViH{1WkDK#ewMhX`ko2 zTw4ik?@-O{*=ph?!imO2KS5g}vFApYf7qrB47(3hsU~YdU8yFPWwk?ZblcLbyRFL~ z2>Wk$%sL*^Z^_pG`fwNeJT}IePNi2wxIZQ`{1dn&c)50c67StUvrCy8>34-z5O`ef zkg-&ra?<{F{WSK~n5pk;uRk^s2{`h5bHXx=$8zeI+R81@*i9Rj+YAaio8D-B%}jM# zJ&^%gVJFh@UtqXI&z7uET)u}1z6bqD`kX?`iL@!7F?-U4LPPg~WVyjPl=t`^9G(7| zsa4@f+9%KNu@g}N4L*_REkt-6ja=lfQkjKzI$|{z2KW!BGTJVKO|#$Ud{oNSt&{zL z`iM;1#fqP@IVWP}^Q&$4s!NXB`?fIf@Kdvjnn7Vo{YjwYRz&$>`Q}|lSQ;y z=&_phL-lCa_2^z#Q_pZYYc5S=$lztLBdjVNR2>ta!$T@}Bz>H#a>A`hY~9sX7{?+< zU}ily!TjH^UT z7|UAGGmk}E>=T-B^oFo5594v|_lnu4YNDg1>}@s%Y4{b1x4Bv^|?(u}Kcr#Lq?dYy~qY7yX{A8>qJiO?shhLIsm zef3zKUQimjYKTg5_0 zUXii$SJAneahWw&Z~Q5!Q$xCJfSf?9OFP$&?n5Z*)9PMN^0?jdAJwiuODsz03ZtYr zsgj6b3^KU4&n+b~)|4A9;KPOXbK9NTaLEQ<2yc&@>q_&og<)VZY3X4%L>j{QXd(QL z=1OU4@3UxJuK8v7y1gFxo!Jle<>|{F3#_9>E1%b1(TC3EwhestV z!60;-u>SVtfJimwC5smc?Wt}X(|n?&(j#_s*t%sQfFU#722fPg=KwV;6X+9QaVivn z$0g2s#LA9BPNPxf!;r@9+&6HI@R65R!)fx+yXli0cEYy!f@q~YzMcttBw+%C8ciKE49oEqa^ZXv(; z*9{m^T@%A@m*~BWSW-W~0Q04sY`8t=ya4GnMk@RJkTvKj=Ly*`Y*w{=X)&=lBfXt9 zPkM{JM*bH!yXLlt(p6{O=KM-z!Q{O5XBk(=7Xc)bT;&83IN&g`s#Ja2B)B^)_q6!m zqj4I-R`9@ZjB5FIs+ZVn}oqi<}bni392 z({U3>21JB>B}@ChO|Km@sDA9<)4sC+MHxvq8S)BFF-F&vx$Ab6|X2b=81qNPgoc%~ixw z@|ykONIb9B$jW-na+vo6?H_E8Kc9zx9vl8euo%(JNtt0vDX1Qx=;Cu9lCOgV#if4z z*q6&gXikSOiD>P5HV7&NNDPD=j|Jj(*gq>UD($X0DdPmT={Ju&WMvWGu~Z;FS6lc> z1$qM59eF^Bsa!Y2;3iU!fgExIv84?5oS5F@7y1Rq>(D9}R|DmwCQybzi(AymhiVf($ z_bJ@pzF_Kxw%D+7=pjgo;>BBcZVYdmie^xc4+`s>N`lMFt(7Krpl87tR>?yqK7a}$ zNMe~Y2nylnbjqaWevVcqCgt1QoPyz?2IZKFtjCKAAnvS(v4y$=xZc{Ra=3gAY51#+ zgbM~Z^W9Xwv?G`_8r~%e+csyr*dYkbbRXrrjc}KHJ06h0J|&LfO)ruxw`W>X3wYJe z#n7-E5FB!v_YiVA>ZQ!~Avs;mLw6BA{=%fYUMwG+Mw;YyLCYFX-?e*jb2d!l+n?ZK>X#Dv>~upGJ_x zaG+bx`!){=bPl$xb6s=KkidI%t1U$$UnLO<@uxIk;pJsot{xW!F~O`yc_S{jh$~ zW6|MBPP5LD=q^3WnuDwdzv9BzwNtdUEegZs``l+KL?OYNY&~JJ0-J*b5?3}_glDWV zO3*R0XP4R<%Ljh|(}<|FEYteA>Xo6h7SDeFak<4ov+wo&1G<_d=-F%QE8}lKuTcsZ z3CZl16iW$JaQP1@jM?or@8_4CxT<8w0O!Y_y$*l2>jAXI@}L{kSkFl`!rgB^F=TFC z5tJ5_aoQt{J4lK$j5`>L_>7&~Z}NS_74;cCb)od?BuCX9!tWNc}*Q2Y}crNg7NhkPFS5gkGW5sbVCZ`VnF=e-B*qS_Ap#7@!igsW9TW*2j!&# zqLfy$>%BN(1Y%YCn)ulA!~&X%?fng~7>z$=BOT<5 znMi^yrtw&=U|V8bqtR8?80g$A9e&hL>i&FPYV^{6>pFB{({^*h7YFp=wJy)@SK)_ajhz_2I)3^B#9l)h8kC0DD`?-a!*y1%5Jr)(}#$~Em5}R zmTlcR-DqK*Q=>L9XFGCt7GB5h+MF{|5W!kng=^%M2HP1l!F!Efb)zAIFS#e~3D?CG zDgTJpB^(})WVG*;w#zh6HCE1Un)JbCdAkQ8jdybMhYT%v>UbH!&BWvj0J)ZztFcye z!+P3m7A&asDNd(0d}!tnXC?FpB2XpwJ0{#tZMT8g6nQAF@lHzaC*xy+^7Zb!( zI);V(M>@YRC@l;N<&S(cjM&Lyif5P_njba%OQ7MQlg)-k!%RK_B%RZ}ex*hyTsC6H z-nHL+gsH@iKr0x|B}&C{S*WfaTM~CafjR4m>z{k^npe+UYH`@hRc4#g_Yn~j zE2w?{Mnb#T@}wa|rSo9)j_Grl6dE?VQR;j|tp=;^gdKoaV6MW2K)^w_g-AMDc~NB} zoDEtM*YYqlZ0As4VeDli*kGuTWejgx0SB@(>x?%SWLv0J)M6ZO2w>LRku0YZc9@H1 z`P}a?y?Br2T8qjBI$B%U?qUEMx}YG9;QD>{?N*_{^PJ*)p%s1X+!@W$+?n{h4o0!x zu>i7kBlCV#RG=Mv4z|f%;fPV4LB^ob8w3TJjdb&Yktjx-@^})qKk>JmW%6VXW~Px) zDxL_c+NF~zU2@1H9aa%P;-jE0)jO*7yBLb2D?*1&6rcibaf87m=H`$DE2EnD{tCIK zFVL3s@gEo;xu+-hc$dSg$pxF_Q|{hKyBHA`Q?~P=G%iz=GKpsf>G6fU1=~& zmaY?OSVI9tZ-hTsGoN@b%!$qjV6bRZt&<;46*t$+&v>reP|4+TYv0|*NMw)VK*0*{ zS1T1NGdEiEI4s(@p${d-tC&PVoL;$jEN1xGh&DkhsdQAFH13_#e8L4p`Q0PV1A6?? zI-jQL@qW4KqM`3KaZh*1NCKbJ!282h@AyMj9J7;$#!u)wj!#@RcMWxLw~v zMzxeIg=^$K&(`nSoi7tj&87@Gv7fD7rbCB+tNHtv4g3ez|ASi=@M{0^5REs;edJIQ zRGTNE<%9S+ovYQZcY2uYmb@O4^D#JEBb|`3<7^|EyQ0Wa%%C&1nXp0syIL*6^~>kX zFAiU>4;_XQ;351|F!(cA2ZxW8WTq07BR+e+cj*bk(NS5h|I{af`^39&z;LK(m zM;OHx4m?i`r-8N&J8lahtahu6gW;iCm0JXUk{13c=}~f*#NVz{UMH7D7v$YSXd-I3 zL@&OFY>y`S%X--@Fx)wOu8&b3n zY9{4dI)`#`R8Bi}!PM0h7#OGm&%WU@%mN0{0fjZu9f&B+E^lhuAg|>Y3`;pPrx$T< zH!~zCrGgbI(y67JCA)64=`zA~kSvn??Y(@YM23nwRhmaS##{Hr{>tMmqvgPZNX$<{ znqB+RzPVz?^HKESRSu@Fb#-(!v}VKR0%{srlv5rF#Uz1KWSAzgxBljZbVj+t6bE{K(SRfZK`g*`%jrJCPGOc74 z5`xtbq10q(=<@P%p<)aUZ;&TYw^bbSx6o{W!if-Gqjb$|735meK0Ibx>?=;K$KLA|FT8AnBJHHQ8 z6ro2IRX*N=emq<+nJKEWKlqy2#EKy?m=_uZtVto&UpZ{l668uX&^gMLCK!H%#50?W z4}Q3oRY&qdhxfxAey<76;jtMfLAK|5gU6zwN^Pmh8XLMd_9|&HYxQ1@Nx*cqBrA34 z!TO=bw&qKg{*MPlq6GD(&kiBPJ_OvkAig34cQSZuSwI_b2}u@8ltm)`!-}07M)@c7YEyHiHuJVI@WgN4%kX% zDWNXfB$afHAW1Q~P&=!mTEn<|O6?+zyzCN=k_KFk`j%wRR@ zQ*FCvCr75cS7%Bn#d|Imasxm&mW|}P_X!nEdR!=dy}e7(!*w)|i1>^{Hru9EJLQ-< zVyCfTDC%nWQfh$*(|L);3!0YT&8!H&b&&Wlhb7-m7zR+TUoU4czlfL zU)AnVS8jpF^<+B`QP5Bz6|Q7JLe5o&JMQV|KtR*1M5cLx$P9unoG%X|CWL|^rkGqw z(>&w0zfI+7c4fCZK5cDozyEqxl*C~V6nJ@XBQmEyhq<~c)g5V++gs+4xw943d!)8I zIQrf%ow3E(dU3po*#=$Zb+Fq5e zqujsEZl$+w}v*Oww_-0Gc;2e20zq)HS^W8t?wkhTG%Msqo69s#D3XyyM z0dlyoE5VQH+m_+u8qwP_JGEJ*Sf? z*-25S_&wuXR%%UK79x*Xpx4CF#%$*tr}8%k>}!rzWvMhaZO7B)on7c{9XnXz`w?ty z&Sudct#kfl6i1AWTV;W{w;YL0M`_r28vWRyCi6Mt4V@q^k=iyt9d6Vw2T;8shnMXh zpD!Ml`K0KX#4iR-GKTlr`xmqwIma;H>>alg7Q{~mf=%t2&Sv>vxYF;{yBqXLNBu*+wx-YVBK9E0J`AIOizI1Azv9V|{i+2lWAL?X!=BjfL zq8|yze!DvCn8N!?-?*=Z^Cnw>@L{2HYECH8m_oNL)%49;T0R{QU$TO<*Mfz0m~EiM zk$HgDQ1TbCGF7QJ`E^?vp0BY2y9 zX6=i|i3r@|lnG|Zb8Bx#-IhjuKPL@lc%Ix0i#w1$})=S>p?vnaxL6 zjwW}L1HN?_^@??CvzhE9F1TJiC&cRRsr?FeC%DE5uwghySEvS$b8WW5kJR6lin6b` zAEh++j&l0N4nnS%6a5{H5OlR=u3O!vN3Y-2*V2!t)a-vN4Gh5z*|Vr^i~HndNWBGbAU8rs${MyVamixyk`{E0bLZ1(DIihihPP*Li<54CE!1#66&wb1jcj{5Np zrvq~U+Mud8F4G?w{F`IxPT~QxrL3q+_!W*#W6g&LFjx1(8~cZwAHgXXOx9*DJQEM; zST8ZHFSTvVO&J!$7Ko&)9igz3uBj9DI<+dyIC?qQZyOVr1BCCTE{#P8dLsMgbAIRr zZ;VIAVd{8jNAP}+$MM?b>4~Q2H9CVo3C?#p!LS?oX(I7VvN$N3AyqY=vTEXVZ9IPa zR7svrYD6^?3xcV=mEgRcH6jGthy$aSZ=G>!RXQ5UsUySNTHlPj-+GKOqA!v`o2X^$ z8&|5OyNEZ}3%~l+eY)DQupT$$Ufcb-w#|#@WGC0GcNF_(w|449l~%f|CsUnPU8|NIE6ogEC)ew% z=d=py$@ZEN-~^#tiWakxHwtibE-NftWt}&rIXB_eNrsxbKR6YuJd;Rg&id9R0I8ir z5XkweNoFy>ar<>VX6Q{_zWCdN8V}26)F0VAW2s+#$^#iKV=vi--X?vYv9c<+aK|5I z;t4xp?Pf76S5E#iUz1$&hg728n^Sc~7AVGwOUz0qejO}X{`n+#dtC}cI5Q%=0)hoW2F*jGY)nLFi zuMFS~R9ZG&T!d~HE&bR4UUE~*7xQwEX|@P^vxEYkTCubZZh=zpL^Sg71C>|Bjq+!t&eH%=%1 ze6Z6k*FLEg4U`*Nbd}NBMOF5!h{>}-A=p2{0?)OJYj-A`Yx#qFPvrVZpIu25kjDvd9y3du2FH6(KChS~j zvyWRk^#KI3Ull!`ZWI9&S(n-_721?RCq>?v>L!QE>mvrM!;;%%gzo7&={^S2g?CZi zA+lsGJ2xe{>MRT!OZ0qb@Y~QV9G39Y{I0r<76Z~&QOgM9QJmh(!}lDru(lc=Ja2*pd>HqD4|22IuCA*mE08&9B#)YIx{l62JzHM&yW*X( zFBLx{Xf6N3r#{uXYaIyD{6XB;{7Vq42?dFExeRAZG;b{HLF?JH`%mG$!Er&<;KHNSrpXDSIsC_2I+9hy#__8@QIhmhSRI=KjcsNs%Tdgm?QAS@bQo7P2 zX|6;`4*hdXe@~hC_T)iOF;M!uc5(i*>Urbdm}@bDc2)+z?v>?ygE&=Xc%<*u&!sLlIwXYkS0YcRR)^9mWKR@{`L!0gG`1^yIu$C!3=u2tIN*mv9lbJa!# z(spnDRl#>)xkC1)O{dAj+wsV=@=sjzPW92>Qp88IF3**u=Y*%QE!itIeJKjV zbKbqMC=4=^1)jhXoXBHTF0|cHs{qmOvkaNIK*B&o}^qnCsCTr`n0;Y z0P|I4UyiH5>BxExm~4dUxWtmQR|tgsK95?QxUa;Qc{=iQktLLY*5mRMje?uh9JA%6 z@&RGYtNL!VgKF{TRJ5VmO6Osf`*AotrDjK3v-ysC`hr!tf<+I!bO~QMmb+eySjClh zcLjUXu5K?E4*?b$pLpJ}TT!rD;O^3E);Zii<75hz9$c4r(reV3ryns3IAwP-xwIY4 z7R4V`s{j~zBkK^%PkPL#^E+o}N_8TE4aCzXPC?BDAO57m8X{91J3*^EjuZd$5kC^> zuDhX$qLeGlgqOe4^Lv(mBBOmS8EqOHtN*+o}=yN3!HXFSU!rWeIY3M3d>=jRaooaA1{O?8;JH@Ky31$?P%ZEkB1*&<=7m!V*=J7)p zQ@5x)Ar@B1wI1$QSoJV?b_c$n(wOSRJK86t_pA_9-9Ncb{eq4JY|MZG{^ z9bnbu#L?@Hfyu?Qc3+p&b$?T0>gMn#v0Q0fgJVP`iI<8}p(v^-Tj*jy?9FP3n^~{j z0Tf@9sVzly`;}T2Z>D=Z1ad52_s5$QqSV>>Y-y=;EkQv+hMAAj*mzZYfX-ZfGDCFH z<-jKj{{|jjoltB1Gzv!${Qqtk;X2HZ>M+SpxZn2+i4}C?SRDW%V@SIRU%C;yX9`vr za}$F?^XpiB4m6CspDxkhrJ1oG4e*ba0Q6vQE%2`A<1q)5&maGnesb&`?xrotf5-`g)0BQdJiZ%p&FT7rPf6vWjx^wRebd5(k&23 z;};}U0%?u7a+w51F1H~sTd)aLAONOFc*D*wz}w6Sw0ij)hJlhM7nXgk&vF zNIYqF5X(m^QfiCq=|_%@n+e`o29HrxFxpG^jY*C9R9;|nLlNsRu2rortoCbUA5@AF z6DJ$mT(rNXiWdx0ul$=iZwNNi4kSpq}=T|(0Np440e!KnJbrxBaw1~ z)(99TxxSxU$qw(o{n{;WShSgHuJZ&1(7#&;KR@`sdj6YoU!Xs^e4kplwDma$*Ow{$A_&?Wvl#8t&io*&m)#Py)Oh z7sotm`_E(ZZ@m6zrwUXjlNQ+j^UgnKNh;!{mxN;fokQ;L7X3%|^)YII_{04Ev+4W4 zY$6c`$bZsZCHu_(c0ORkHt_$P-2d}Waae%YNMfXDm;dkh;*T5)m9X*dZzueRg;6m6 zBK$YSQSbkj@JKY>tp8CS{dtcFGg2ipB#8hhzsm;INozVSvO;jfSi z=9uyK%rrAjd>|t7;=coHDDN^6m;Qekf>4eLd{c5)A9Gp0Ac=iJe2@~5msFI-0*RrV zy*)uaN#%D9YT`kWsM>UeaWTnclto1`sU`i^K1ReKnVAR)E+jH~d#BGczIPeKV2&_usgix9S>P*-2RG0LFL``VA)}xTR6t>7%t-7)kL2&5x_?B zpJ)qwwgJc{vaoDOMABga)b}W6;<`$tDdpSKXdF>-#CapIHwwWUJl92JVj{kNF(XQQ z;!uQgHlXkP5=w5bv(SRWrQ^X+X>z=6D)r%#gQxe@JBt*UK}jVu89Z>#aVgd3&_r>X z`1O|)v+85s{lz|bZxSJ`1_f{Gqnyj{zs~dOff~g~Su;&Uv1F37(yB|OcqQ3fG3B%l z;xY+*n6P`Au!%4w0li26v0mf*F4s6+C>Yw7=#~l9a=iM9Ga3-fc9>9m)^F2E8a1oY zy`NzGF55=Vcn1Ujf{WtNyys)=GQpiwC(tOiKA7r-)^Sbe(jP*{K`TdJK>5*S-qx&! zX5Ad4g^ySNxGn?%0aGH>YKQ+CVZ%d&#OrOdlDJ`k%u4hZ1nw0A;8V^LQ()xMwBX#XUyL$8o8T$GPeaqc5p5!&Uph14FX0i!ha^-%Klcm zcf%QDpD~sI1ndbG>~YC!FUV7NmKX`aTkmYx<~W7(FL#Fm;Cd(%);N4z!FkL1PzYCC zfIW#2?@?}ivjo8;8oUs?b8n3^Dd+e2GarfgK%qVc@-vM4=?#4hged(;$OP8VPrSwb z1Li-A5DI)in)%t_$A%7`3w7d)+Qr>|N9`&qpa4atMH*?vKDH;!CNjkZg#rWqWG4E? zB)}f!0~N;)a$1Sx^KyKpP71bQqYAGZ~m-K323X1r%Dtn_D$+)ati?F}T2QQ?3dXF9`#4#Wm~1 zfa^@n2KKFll#~fj;35O*%(RWVR*QjMs^QJDPyf%eJP2}WC5d}27 z1%HIZi~lFOTD?_{Qn`*W3My*onSh<)zb)Zk#Qyy^v?ySgz`=0DYWH-%gv0OJ41Ms% zAFGn7bwVTGH|QI&_;_h}qTHLpr?HoYZ*H>w~Y&@2Vr%*A{=?iJJ zbdy$PG^}E_+~6HYt;trOUkMKa#tr&#>|)zvfjrTzXeSZ!d9`I`s%hC|IFXthohoja zN0G=H?{-zx?)3+Go<{BWi=!I)JfP9B|FX|s&U2kw~6)>9=iv9s_gUJV^Ykj0v|--JRjWfIEX>ju(nSAI;->BLQS&=Q>xD5uIr2V$i*q z1efM059ce4fuThq4T$sm&Rls+?`g{;2G<#nx*WJ^oTVlwTe{EDrR|JP&>L$Kl%aPn zOIDkVS2wRUIi!?mw32N22_3pG+VYorAIPA6Q?0UaK@v-8f+pykR-2Z@jp! z=NsDWeZ?@Z7zy`1iz%8iR`X9x_ml#~sd z(*MOYw%py1YicrY_{6r((p5=JIq>pUQVf1jbi>r#f)^o><}X-|zY-&ec2*Q~19A^O zZUS?_8NAI@mWnOIB!~zejdGB(o*(;qCQcPe!Q+Yd?wpx@$q$h{*;C%*hjv6lk|@5r z${4my9(O$c6{_XVn zuYB2>nLIi!v)emi7x|beB)TjEG4tEMx*qD6Zfclf1 z@5EU%pO`MvR~vQ*j4Bk*>L<-s|D7<4rblwN1|eCap`lH^*A&T=V(7%x=)(>X_Z9Ie zBB6m)_!z&K^O1U;%km0_;u4a5RdvbBj{%UfWv654K zJl`mpFPlLcit$YCa5x*ID8nld!CmD{R$HW4kdlSMCf4fWu+w0$G#G|$zs=4(GL$70 znCpC*KMDBbDYZEn6|qh9dxu!Fg{o-oj;uMd#YqmnN^zMjmQ?oi_j0HkzW4}N>29L#|A z`cmLH4{HLc>;BwIJJlfnmILq|r_MNjKS^OUsH0)(HpF#TqCWVd1!EWeD|A zp+jnC2%R}gwQ#+;&A}*sDE^m3N~+_6_0}41c08$^ds*9ETx)X=WMLkUaI}ncJbUFo zBQ1dEeBN8jV=@^dK*7YM#!OusUt!ts9R~vH$`Ks-rMQl24);5WNlZ?wZbfogEi1PN z1!>#6qn3S|vRfk8019?{^=-xC7vB4?K-fG9Z;`Z~vt=oPR+F-NZzR@Jx_L|bTz(j1 zG*{@6Eh!xwmE4__b3_Udg1x$21#Z`^w#$?=#mezkD>C-QYc*!YPn=uqm#LZ*4u{hT z)nnY+=CdVqznsmx92>|bY&J4x@QPbo^o4>edRC`@i|$&w`zIVwmo95YTf*uG5} zUkcz)za6BctkQc~V>;QxwmV-ZGMn0^IB*d$(InUPCqLg1jXfTtYrk@{_h^p-I<6CD~A1_>v;9hd#_)u=e&8%9ty>5^L`4bV~ZrPCw(w&N@yN3fW5bwUe3M7F`X ztTqaD#QK{WUu&e9wNLx_^Hg2EQqjkW)yc(V*B_5dG*NXfOI_T5AsxK$c4@x&dqKSHV(C8Sh@IxNpvR><&oZE?>X89%)Z_rmb$3&|RF~MM_nSMxL^z zIq%E>f-JJ%*hM}iHI-&gi~Xa=-OofRXNRm{=f;y!zLPy&qr2*yWjDzL8mFbNhm_=t zdf4ujo@dXILwmb?P0r-CK~> z&rUb!l!|*Vqotx0qs;cZ$0(Qs2}>Lyxn4=ji%AZenRAz2{;vbdbIVjmY@bIo;+H?{ z!!&N)q#QpyElVy?F;KJc8+^l7k&Xq7v+LlW|f!7Canoel^xkHP|TU;ZmJMW4tYjr54+T>^n(&xZudlGPdc5PWzgiZZA4Zm8t=sZIhb7k~9d?pul z1%*8xyQ5+Nfme8R{fVOKd8T8saKL527j?^bH*H-JhOoi5-^W$`EbV!i6wL%A8$8!B ze`8Xo)~Iy}ch%-xN;L_;)tJsq=6$9cyh@$inwYM?Rln%6k4*7u!72#BEx6pJjpE(Y zN-%q59uJQ>Dj#kcOW6o~eN=~7v&(>IyC|Ex*M!g|5gMr`CnM_v3-mv`4Xq}CpKRQj zbv-$wGf89SIhjV?5w>4m?fBq14#vn6eNcT^ExT$P*flJxnfA=^mwuNZIPduZQ;|XQ z-cS;qYRJ^3(3_{AudcDSI;(pHS}c2%BmCG757%=Gr^BhID@_BG1*z*7()SOLPOCr* z#G(IT?>)nsTDCvnBX$r(5jlV~>AiOXh#5w43M+gu)@8;ff)pPIh9N!P`m-qR9+fOok&&(>nwPwv)Gd5El(Xpm1z=f60WfEWe zw9>e9N|_4OO>g1^n#GNIQj6;JgPu}scbrUE`NX>~OkS=jpWkh2k|djq(qMO?_*zt`2|M+ZuZmSZ`Wan-=eia*#MRz9UR&Juo_y^wLe}S1Wcz5MtT{+=n5)4= z`W}3z@=yR$>v=7y5z6{IC6%V#_z=~O=s-ap_wRBm{btn|)l^eoq)XN?pAT$Nw9ay> z#?gtyrMq(jNMK3^3pOU0WF`T_zhFlkK9tTKFKFzx2Fa?5Gj=})a-~)uK$qkA+sRtY zb>oUs{NlfKDRE=#`?tEQv?is=3yS>E)XvAAKC<0TjVpA*jg90!J$9*~k}2Ez?g5u$ z`ZMmRsilSv<@A2BR|%)-kSvO`yC~X`?Ejv?i@1#exsmS175i?=i2;iJj zuFiKCVVj~52J*nB<|^8fFv~dtE8GZVg{|c#i+gm+#hNT!=vYJ_H3=Kd9qYu~gl)fV zhdmBn5E(@C=_VYEDE5@eV8{vi4w|=KLdGqu?3!7Y$4f_q$Dce(XDTi_FncVv(VLHK7V*HCFEa}`c zcOUOLOvmngGN`aw-no(r=U-G_2$zD*?%bQge67F*;?O?LR0DIb`z_{5m`%&|Pj)T* z$HfDxRu^?sT!ybgzjBGrR~POC4usFo?VrV6Ezx&-pP?>^txz^iklgUaZ>@>d&s^6$4L9ahL)a{;q%oG1M4ayyUNHMMIGu%DG7Nl)qF zRq!UiJK6@)fp~D($$?WK<;d4n>*9gi7()1Br2%mbxhFLjNMcBBtS> z|H|kPssJ~R$r|${ykTn#YfXT8jo~;3Espj(FFVqztHFy$WMD->)YW$jYd3HAQUDV8 z1Z51{Y&{q#Zu;WBH)ME05oba+g3a|02Gx2XDm3;Xpq9`g(%WB z>~hxtwJzW>zJ_+&(Y|(p*ijQ+>TgU=5MWseL11d`Qtz+jFnr>3zo_5f`u?s@WvKU@ zDy2qdV%c#w&B1z5+8xOR6DrXODxFO=k`XU#0q)g>&U%n_fYFmXCq|f!s`k*TUV#Gn zXLtZU8-nM)JOr+zAr-C}E;Nom&TQsd!Vs$7#neqwg{`%%6h+XF@4{lmywfGcn~x6U zaC=KFtYUBNyPZe~WBNb0Z{5hatKY+SH-jnOzDF@!awnhib9(P*yTRNMRWefvX|JvN zKI*!io7+$K3L;Y~#kTqW#&(?XiedeTP34|Y+#|Wfq;aoPgnI^C728#_lMq{}CasWy znw)$M_uBUo>ht4;yDs5-_zcCfb2okh){+pgD`N4>!|t(d{j%{`_~d{o?NVE*m9lab zzk0=9QmjF3v6(;S{JS2slZk*QQjTl~sC4e%0=9QOh3CYI&ImijNmDtchTi;I30$Wp zSTpp+!F4B(yQE0y1aCa?0^0R;-7YT8Z(`3T@o@mCd=h2PVV@c*{E}%btR!_j?~Je7 z@@K?wSy0(6MWl4mg7aKwZTHFI45HL=@?PC78ZzeaD|LqV;D_Z`ctEK)ESHC?>VOUR z1KaU=g>jOAo+6|8#sfX<@>f{^g1J_BC5~pFiPxkgX=sx?x(A0rh_UlD!DSn7LscfE6s;KEkf%FgQO0uiC$Y~>vQu^m5u z(Rx7$N6vRHDMMd=qF_mHA_j6|mU_mWT+dp$*2n}pr!@&C2*iO0mn+;QqqwP1TvELU zav>*f9@xcZTn$#t1^HUnN^D1kG6eBaRTxcwmK%=3$OEaZ<&Tj!8^Bq13A!5gT=3Ix3WCi*0XC6Pf+24_ruwztT z_=O!2MGSv7bAD#0jz}Ih>B9a&rd>rxWe<(-uHrn@SAFSg+`c#U{!@%cv?Ph^OBK~; zsu-VB>e|yLAEO2YoKW84a|GWaiT{ZY{t=<0-9I;-UW91PZvC=Ic+a$LgjXc#d{(^$pvrOnft866%^Xqk=X_fdkD~=sZ~CgNlTJ$y ztNl>^3IDQ&$1@jt@-s3R>aaa3c`@Bd`X0FlZg}oV(SAL+^G!AyDoX>edh9&Bw)zVk z)Oi{cscrBS#U?(4TQ5G_Be>Zy1R1wl2s5sv&L&!zt0+rCx*p@9iwX8M_d?5hx>*wm z_aDEILhx#xb9z)L3TS7zNn%g2vOYBN)l@S}M3xnyZ^&$F8X%lw6?P+Z?Y=y?2%?H~ znzd_lEi>VyR2^$0K)Rm^EB^Hc&oX*GZ5kldL#4|7yBI?#F3>ki#rNh|m3=a#;DVN@xeRiQJ@h1HJ`r+S{oPuA!7KHm#t?gLOPU;xVkC#ho-h|@4 zsBd?h`fz5{tgz_$u1Jnyc8o`)xM`LOx$OkYj8jn^-G}&Bkv{OZHYH*}ikC+4n#xt@ z&7Omugrn}Qr!$AD*bAZ+c7qb*sI6)gKd8x5i^r;d?ri)pT0J35QtSYpb_Oiw5VaGq zy>J*9k2ha`&+a+2xu)PG zzgE{Rq|++peg}- z(PX6u3TE==x$j8r{hCU%wa#Q=ri-9nT2sb*XZ*gYGgx(H*r`0v?(=(?TO{C8Zes`L z$ho}^t+(=T9nIHmF3$8qht5gW4I%BruzUzpz)y!@tHg4LJkMGu*+)soPpCf}l8#rP z@;b(anVdU}<|N8(1F z;&MQU{p4ilC53!=6~yQ}P>Q8q=~Vn#_9Mr?qk z5+Xwp`k_ynL3INL&TO1Y$;b$O_`t!R!}J)ICngXR$);O4+f}QDspT}3@pGTUc{X~M z6&Z_yFjwRew7n(nIY5-_B~{z@=}C`_e;myYDwW@897#VFj9*|obO4gJwEFD zt{in;!OWcNbQrIl?AB`V_Vhr-=V&q>z)4levm2zdE?%W}#UR+Cec<_)oThc|gp?d* z4`;IS6DO?JwiTbv3uteSh~5O z>yLV#^i$&rvM2;u4}@R93!~&$5-WFt;2l+3+G#j8g55D=_Rxxs2T#xc^~Mj7Mf8&F z(&3tZ3ytQiaB<|4zH}tk88mX!{=-Xb(KGrQ@C&o>Ue;{xVK`z5ad%j>2_LogC2rVH zP||HIR3dL6w$^2V^Ao?R2!FW{sNKfE%**6!Yz)T+;mI4cz4=Ndm1C;PLS$HEiBovg zd8}^)=}Gt7B1u0qn#M!lDxljQn$Fp*lvhTmik}glu zwVe-=Tq>q&Jl@Pa+?~wcJar1khZ+ucocLY)Nz~zQ#2-9&IC8RW)8JE!+>)y!c{!YK zJ5TM9B+=u#Se(7u=}P#Nd~YC<5875$>VR<2ulYQj7Z}+aZ>P>mW->)-w}fRyCs0=p z%!U;C7AGFyR+cPDoVdAN7OXLM>`nHG`5pbey9DfEk2!k!G*IU3x&`iJiO@|kb~t=) zu@(;95l@s!s8qo`=Ez<+k~%pi(>%J-V>kuGY(+6DOzjt~S(pg(`#+c4T8Iwj*Uv3GQbG_D8QJ@jI#Gs}(Er zbU%6gockq1B+3^Cg&CE#E*vidI&hdCu1}IGsp&rJ=iTPCkC4(TDDLV;(_ATQv=PN%XyF!u*SK7 z^TbXQ{a75$=>lVd}=&X|I7oJJ%Qq16XZpVbikh5fi0<_~vq(c2~iZgTMr$L_hSJk2GZ zNSu`AIig(xipQIdf*^Yl5U-INa3acNULvr+GfAXfx|@=8R>BLnL%5P*RpG(C;5oR~ zg-psljp|kolN)cW5)_RsyefMk}>-vIo;C&_wLmrJrv&ZX-*b;XG`{O*@Q0w?^^1U zrI1Y!ytEa0741x!3c77;*I%Dg=hzX2neY3t4t5?kW`6`RL_QVX}krbX>FWeioM|g1%Al?q#^70 zFwc0|c}(#%HYgkP9;hBDze|3!+35sJ8U-gW_?IyPATzcS1+4bd{r&b;maXwzO~^%! z)~naN3=7sRn%lZH3+$|L@jO0WI|nL$=tU=TD!dNUjxilSPraXICeUh>1N*`^3jF$@ z!%r_UvrG>qUOaECUVg*Xgl|0c`4+WAIazps5{-`pLZ<+pF5vL!N#*2X=U4;W=Ps|! z@X`K2X0c{FQA4azZQ~u`NmeWwM=6MBv2(AfYGQ>b37=q2uqS!GVh~l_@rsjU z;>zcOx%q>`5U1G>{&y=IY^8dmvSHL;9I-tQ-oN3^Z@Eb0%QQw3zJvE}X(ViIGC$T7 zq!&~4b^BB5$+Yh-KmFV)>TISiI;5K*E6Fsjj+94><6IiBm$Y#f4$D?cNbL4$`2*R}$6-r%2y5>{>-)es zl+S^~RF|7D>PyuX_GaePJErK&i%Q9UCQglaxhjFu5P>)MB1!bL(1(tF)X@T`fmZL` z4NJ-=mB3P~<^UyMcD~lG!K}lkCJDeea(Hqm6lXzsBQj?GK-_06hk4xGEBtEB%$a^# zeyRbLqDe}pok>Hl@G9GxIx?3r`Gv@Ej}>xn<#}-Hng@=YBmzX>N*ZQex{B-SQcv4nWQjEwQF0U3fVg*|HqPdD7(rj-Y zx78t?Xa6}1;9O~AT2EfPhj(@WK`6q@du+v@C^{|2)QkI~K6cR=*k(nvAWX+&#kKqr z=q7t422CrAi}JLxv+7UQ?VZ20KC>Lmtr#fl{oruW_l{7ET-Zr0>#iZTXw>KAqj9I* z{@SophIZ#o%|>5i!(qx3O6L9kSWB^6{%S{=7v`;EBNwpTjU-1#?i-$)i}bkQN~go> z;n_6h3!`L8*-5DO?v>AiQxhgkm+0utKIr$BKOtDu&_=;3(!SULLfp#-N?lZTrb=7$ zznD^f224mf*tz9=7=S&f3xbWqXRX=N$O&(LhAC$qIX|qtSL5 z-|FyruVu^90wxJ;r%XD@ip@9C~?=M6s^+Co+9VsDwe_iHRu{cV_IZ!wa)6vVzI9Do&XGV~~d zO*;G*aopCID&s_rsjpJ0_HDf7`vgL{nBO=l(5El8pJZ6#T|xQYr8%{8GL^-LiDS6-$UxkzQ1 zBECSM;D){VlL=$k68V0!alE9wy)F=c- zqAsG-*05dbb3c#UKFM`@gUV_*^6pySg&r}m*Q?9cHES^|6MkokRw{-GeENkqW_X?L z!01Ne;qCqF%FrJ2gB}Iru9$LgGOn%Rc6PE?S{s(1bP}uWE`QD+Er)_oy8;h?TLGle~s_{R^x+X<5G@Qo5zpI z{qT`a?J5VCX1%zybECYmgF(&>Jp9wA^v7diNf?pz! zue+^%0W%tz^Cl(GpDc-9UDCY~LMgDUoONBed+SR_`cw`$GS<1&gic=^6>8u_%+cWEjwm6PF+q!orTF;(`+AB zPD9Hp{v$A=I5W9K&8dlEim}RGzi&Mi!|%BGG9Mf9eM5|HhNCmX23r6 zb-UDxq}SHsa-s7Jnt>IKv+{1$bWXLC5?5)c-NWlTpg!lc+6q`YLofK(Zyj0;bht=m z#Gx#b1a`CX=H3RaUw2a1-R~M$0gf>_VO|^Cf1&5Rxgg+k*niQouOfCV)sQ*dBu`8N z02TQW?TyFe!(dYqy{#^-tF?`CTQ$ID9A%LtJJ@w&`^n7b85Ll+b8c^eIvR?p9i42H zOsOFB>J9f@^Q>u`I8hq_f6SYgtefxfU*~Vy1Eu>rM@mv9E_tTUIosuFG5;O9_6Z+Y zb<1b>82W;j>p`wD$vGbILZlVg&brl8Is3FQOgYiBJMSPw*tkiZUrQhUIQw&6JhoA7 zSi(Q!(Qxc|gU1@y4GlA-_8VUoc68ER;gL}*;-NUl4^*j7PLYA(S|(NYUQJgBzc4s? zf{lxj!B!+^xhPgQ7Aolp&kev_K0bC(GwvTrlykhNU$xYC*?uTL)YPuGh>T=pWo=y_KQJF-%~>;qdnGTvA+Fh$ z{W5#is@4VPTlFLp-)dqnLWE=yWK4{6@tp+s>>FP&T3q2|FXH2tICzO6hJ=J|rdOhs&RShCaSbEP9WQ=Z25d}mjXzR= zrHS?|6GMO{_#5YWIqzsT^1IEm52SwOk+vWl~j_&@>D0q;3>5ivb zR=-mNd${Cry>Zh15jYKAb}q8gahwjJh|nyv-j)ct=xjWzug^v9zB*#{B2?67d=}%G zsNA0_B)93(;^|l}(EQN$sR08s)r&{tm1~;FEv|@Rmk*+0W*s6{Wop|_Py+*%u*Pey zrinsMG0iJO6=%xzI~P1Wk6x&1R;rL=aSbvJ|4Qn*@+FQ<(C&c?)CEysR_)ig_*Gr( zhW6T6h3r`}tdbbVFpy59a!e7S)t9U#?+9_*JSR@L)o{gKD(dd{$XhDIr5p2g&m*P~ zyx$VX1L@3A;Wkc$i21T!Z_Vz7r+XRk;m0BO0ijweRTB&^^vDt`VilZ8@!Hqf_(qdG@O*^s~cYdp3TsxCx$OQ+@ z=?EE(W>x|5%57jG#Z%~D?jby-em%cK4GOj(e0iwy9UxAYY^ousP zHm3>&_^}JzQNk)3p-fb2^HExAQ8!4cg?-wbHaqe#Bimz?3dR0>26rlQqamQZJ^b0) zPwGCRbN%m96eQ0q)bv;1fA*5OUplmw$&BmG4VCz$<>AJ)8pcW8XZw36RByG6E9?h9 z*+!zbgQ#@6G5bk6FAi`a7q)>|fc!YHbRj)J6|MG;__U#ss#fc3o~ zH_xMrh5EcW?iY9x!ogn@=idA@$VJS;0Hn_1Umim{3cO55mQa%X@Hq!IroEFfL}FXgh~eV6?0 zrwoFYmKIkbh>B2(&wQkvA^PVGx%hKQC=0#}{J-9P|R^+bPkNoo#)!URnR(t#(S?%Nm-Eyf4=Yfw#r6m7#Mz=IC2@7>U zwVbQw_34>lo8B*zkmB~YW&wfbi&rJX8TuM#1tNNFwNmv;VUen4Tyt7W9x480NU$N}HSt0D70lQ}si;>owv za&0jsGn%h9?rcxLxqr96+ZIDIcB~OGbWMf(=DXF%(o3{crnH|AcoR~<$8vfhKv5q4 z^L?PcKp}N{R-V?$Ujx%wkCP_)iJYd+0!dsXFI-H-@XuOr`PJIj-Bqphh<~o9c*T4q zRs*LB0O=>uBeB+KmQXAVUfgp z=fSDz%^@Rsoyv=r_YMQAs2or*$;im5+T|w)+sf2V+6xwj%Q3XrODpdtX2A8Mf?t;% z^jWB@s$zW@_^y2PYx(?Ay8n{RqtF*Ez<7Z33ud|mCRjC7z+WWQmhYx!;NNIFt#x7C zUY@RgM#>U;z&BEPpXg@;ay-?fz0l>8*&%9bAgw*Jp*)=b4nM=WV3}0&(U!CkNXaBv z(GNyCPSWX5r|}X`LLUEGqVdlJ1Q16cQ(K0nj2>NgoBkJZ3#r;w91cHseVF;ACd92Nhv(q3X&cib`s4fpZ!Zc?oJ&WKtObKkmmG zO{Twr!*4pL@!%9{?;lUSvlNX>qG+SY!S`Pz>w8*aiH%N5xP37~Gvk4Gew5N|Kgkg| zhW@_+>3>lrP3!5yxQ~xe0>7J`+`+T@D6L4!C@p=qW6Ss5aOU+#rJ}O`h^Dl70Loo% zRunb!ic;ij<^s05Y5nlL8W(-;%KUGK{{=4qNEV;Ie%~m_J)-g~A z`)$@gebR`-+X$>Fo37-hx_2)q3c;G`*%rUivk;ZCEyqqwLNX)tK^#+P$+U{+b;7{( z^2uUz!&SEbIjP|91$d887$*V5juDC3S9{`Bu}fR9C84U=by$&TL)OAQB@g( z^t1fD87d0TuW9M^rl0s%-p+a@aPFVq2d0e&7;+}*;`&m1ZibC4`Rkfn1dE;-y77xr z;kVBGig7nNlO|SALDyxk5r_p2Y`FZ6DfnLSQ<(fh(scn6R&b#EFF-miaRM-rcYF6K zere2rk_4YF5wM70UQpMYkE~`AT9K-Oax)!!R~BkAxc-rDz`TP^0Nc!y^6WFuu1@1x(F3@#tl)3n|E{GPX+T~2R=@m66K1T!aL8$dWxNs+>$SaH zTcE6>5=9ZD?GdXypklBr`D&EkG{einWn|7-a;yBt+kz0r0u{GctXhl<-g5PGADS3O zt9rL$EHr@=EEt)Xm<&VQF{9k}Ly#!J6=CH;CDM%JFBKVH6>LmS>v!*GCOAHdDb%A+ ztaOM3%2X6v@#iwjEOc0MfIQY&ov6^&)YQawEzE*{ zr94(T<8&>02Rt7wjCy+zB)MKC7c^WzZ>)yl@F1$WZ%}EoEUaFwQ)mF!(J3;qI4i0q zay;TNI9~xW^f%LCNPHRrr)1pufcLhVh7n?@|0-uhpW$7CLVZeVSQLYzczN5FbeU;s zP|7*v(*Ra1w###K`=q-(TFSkbmQ{oKiGH?1-B5QK1ZpuMtKaxI1VFLQvsK1)z_HJW z74ch5#(do~k;qJM?Kvd?~N=8wfGBQdn z@WmaeXSl>lI&Ut)kJg!s)_e)h*YXMKYgTbal^3KW*o13eFY_tSC`{psm|clxG;~vh zG6+F36*gfGiD?f5cP2JlUh)DND-K5a4}{^Z9}5N51)HR(;*F(0{fdPo+ypQo-?;K$+sg z+|kw5BK%vb!Ci5phLZ8SnW4TeNvkNh{65Sho?{QH^Sn|*rCW)Exnx^CGRo~DqvH{M zgn`S;mRs(XOWIlaCsen_7&Uoy%p=@)(bQk;{Z3ro629!WhPi8AT6lJZvL2b(FKV-s`B~gS62M?Cn;f27xN#f1a#H{%PfPA5uR{#g>vI zy?e#0EoTQ={Ln{S1z9@BZq?c`I8b~CG+OQ%qLQ=~4GMPYU1H4BDTtxTQj3kYJ}7ur z6y5fkqFR0zJut_&oo#A$Z0e^RY?S0ff$puT2pDP?Eh6rr zv@nw?6S!I9SY()88Jc3|6~GJZ^FuNriRo+gShr7Cu5I;ej>@Wu zzk?rs#op4d^Nn0Zm7%G)Y`1n1j;N3Wk4PbTt1JE5na<}$w_A47?g&ISBv@m84)?;H zaZ@3aeEMJ}ACRiSt$t~{TRMy@OfkQ*I%2tbikJPi5_N`T05w%%+v6A^J?osyPiuU? zChxFRu8n+g940-h@-_5ACs0wSc!Dj8Vm0*vV?7%S8#c{7A+!9awYi;ok?s1_ zxE=O^RNqOXDq*`3>GfL7LsN_}9Quiwrhe{$MYhxlUo)qa)xfUGxU+$R@_X8nXw|O@ zMqEvlh@h^o%F1%)I`MWfal9dwAyGYyern?EAGdRIa@;XNL`)$>uV$|_&I;tTlQRSR zowjq!gpN)ER0dU_fYT|uhikM9^^0xdISfng2|-4g#NF3^QnhQ#IbZ>XmS%{;QGb zYVorBUX~%o?X1;4ZjCiu(mQ2q5jE&|fHuT*Sg6Bo7FOb54_~0fW!1#`x27XtuKH31 z213#Gl|T*Dr@`2(<&4Z~u-j1Fx=U^ekWcMxkv(>|8z|6BvNcpA;JrM@m=(PAu1$rJ zQ+q&D#Y|-EkDfn_cfW((ZCE5gXNHW-{%zDt0GYZ_&=?W7Q13D!B!lIjvNK zhI8nAYCS6Rsvu~q7ml0TvXO!`oxg;qb*EJ+{p8*=s^~WgW3y_>V%GMaWkCIIkdx$F zu?qW|aqF%mP7305+g+{Z*&V|RHRJ2mD|DV;=7z0C=+!%4KPxQ0%SQ;5peWvwl<|#s z3{X=UuyX3KKRCaDD|54pr#Z%ePqV8y3qjf5TN@1GMP~#rU%u=vSyU9K$)vzkl9bWf zqNHLF+`_OHSw2b009lb@GP8!Y=hG0CWZ*)jzqiuipaZKY1w!X@_t${l>t>bJw37SoVwAjMproTP%e zt8vt3_5OSb4|kHJAd+j{R-d|ut=L@9@T!_lzvI1Fahi@UUaGO+3u`tByL-Bh+Ah8) zcZDD*^H{-DCN$z4Lct-4ib=_Qnt`+bT<-%3BlBfyX3fT`jF$4ZQFntBRdOgI*h;J~ z(Ze&e7?w0x6$`zCTLf3Qc?rehQMSvYsfpg0syF;PET%dyuGDIbmD|}73C~m-0d`s? zSyC6~>zP!)+jg<^2D)e=OnR7os!ULl;f0 z$4rOnIex_u5WfaW;7Q(r*UI@14j`kC+uh%UGuJP97_OunSQ|=#*J<1*9gzfzD@56r z?c{@Fr25JPygoQi~ZGfHJrZ3yBsNy>1z3!nuu6F?VbIxO zJB!nrC3ow=J0pFt(0IAb9-s4)Cm3Of@uR!ALld=;;!UqhxEjq#!~QF<^->22rrqA^ z8WhQ|1m7;ur*EeBt{VQjw9xLXJAL?I+y)xc7&Xd&SD!utWKI z0Va|s6OuxD_s+Wd#!Gfy_~N~burx%IT3^cgon-5}gVJQBk~zDMFuY~=ig{PD(qq9o z<;~plOnTOV8EdTc8*W9>)y-mYN9XEd(>Svi>W=!mSLF>$eP?t$Azk27Fh-J61UTTG zNp#{$9dID_WeO#A`yr=Lb=1xT2^}84=(-2v=%U6xk%P%(w?`xNNX|rmzkug}Y^O%lVsPyuy zy}2COP?e`sIBNFHDnZzmHl>r>eKN(5^KBTU49Af{SpmYH9QV#6i|8rh>2fv|7Is!| zpKuv-*12NgdJmQ6jhXdMT?OjNhjn zD)CYXl5|X5#gR38O*igcoB;f!DVuL8q-d?4h|Wzm@}9e7A2$hx&XyGsQE5^836*I* z!e~lpYc3Rb!?3cUg_^@Yz?vdny}#{gfl=B0bx((=qptYa8r_)2o7`1voK{}ptj26j zJJ}iHxx5dPbQga5Af7LvA#C4YnwZnqJ&vy%ZiP!j2s)+}q!h?6gJ$ zt|L{1A{kd>3z@Wq@VU+C2!>MdvzGh=B~A8Yu2QGc7Kda;TWFO+*GZWNjhPN9?&N8s zp;-=|IR>;d0+lPlUAIrWul0IAoVSvp?YgS@VXF;;T`1Fzg_=uVsDX|FchsWJ&{X@zA%YadxPPyR%)2edxeqaLjIG_{QMNnmxfs;S zoZ>@D!mRd6{so9C#ZOcWI$x^HY4XH9eBIb0a4x1JOkRSYZJk1ZqiS{_WA({c`SO#U zz~Sj3C^aICot@nxKGV8k<_;9cUGCEEb9bijAgEOw<9B^sB+fN=?Sp7Z4&&w;`kG2NK>U$x6 zckNXzX)Z*E2-J1u+Cy#vht9kgw&;3TL)>{YvGT)}GJUz(dNXYKRf??l zv1}A~twNOBT5{f-PH*kG%0b{_1w7?hr)2PW`PRTtVj<;SYCj<+wYXb;W% zfacsBe^7dUK5!sV9#mb@IQTN;D zuwdX0>GqtUjk%G__wV0-+dUs~HO4!!l4;gDW^cEQ+x%k&qGW42q%FY5t7l_pd3kJi zHKb@7%Y9ECAHe}jRL#zH!7};pZ{`)vboN`AAGRR+%b-^>M8!4!IrA{<&`AuU= zp}MA!=n=Qr<1$Eg4Ou6&;HH&v0t}RukLkFU9K!zl02~^tXS=sw_Ni;hn4PV|d_(TQ zIz@C4@&t|vAduZ>`Fp%LOi?cnyCebMP+LWUGFfOZLKoZYpyj?r*9|+J_{~LVYXwauM2;ji|JLdO!t!C_I&q9`plfzVa9k3B&a866s|+EJH#b*vSUS)D z#&$bPt>{&b7V|yTbtSIw4OQ0FS@n?~et+^#_RL#ao)k4k(oE;AN;i~e;t%u!a(pf_ z|G(%HU+;l-Ona0OEtBbHQ?NjtiYEl$eAb+=CU_3bU}Xnpvfiv3>9O!XjLU@VXOLSt zi^n`nAef2pJ(+z^`cI7J89w*i%0SV0u!hghF1x*axy?Vmyjn$H4#;(l7s%@h9kuLm z9*-9^+<7BWCE^{cyHVPB zD4sO?+0A@V@m0*aQTr*#YG8(&M@G9v419_cN@L@A6+E`RrECQDO;>A(lM8^Qk% z0w$emeEzxB*i+#-wptzSd!Af8$j=cfkgu0iyUzQ2>sfP2*fsdd8h9Z>`LNsEXSmh> zUbsBjXgQEC4q1QetI=vHUmD~ERdHy!RUQt6Uc6*pPFJ3#q(mEtU~`sYb16f(=H`Fm z^w8Fv9M=B_QZJeTDg}{9WxWJ^S8m64CQDHMZwK;kg7zu-Z|a~h?*H)he*+pLJuQb8 z-`&jgf1=oF#|5N-NEf6Y{FAjl)kLLx0DM`#P?G~z&i6!Klk17U?B%^J%XH#it!g3+Y|t!us=Wf_mc5%hG_5sd}QQ* zAg1&y+%o`1F@#^6@^^6dhvuYO0;;HF{oXa|R~lEG0yL?s)E_VZ-B_d%mo9Zib!jZI z{7PecK$i^I7kz(0;QS-55y!B`zZA&#J7H^(svK37=AZ&UbWnq-j~nJ(^5Q~!D zr-tXcSnVtg!j(g*g+(Q4K)m}q>lC&_>ZjSneVvVm&!j?%?(c3hArKcd|K zJ=3?o|MZZWemXoPZh7fodka)2^R0I?$IF1MYl?BgqmHGW5q&?|@Zh2zh#(AGMB%LlrTGN${+mp;_#5Qt%XnPRotZgz46KYjUJg!Fy` z$OrSo_-&j3*weYBo@hdq%*JleA7J=1yW!99<|hLetf+`FvkOf5*{jkfw4AL`4IaAR zSyEmFz&Mn^{+H0!20+_jYZlk%f6bWhUj^8b3O{u)4D}AallM>me`EaVgWRbR7P$M4 z{#P2WaRRD>`Z)GWCXE81Y8p8{)pss^W7q#Y{EG!UH-{vD1@rp>1978UWAW0jG=>rY zvRzgDkE{Gsx(1*64K^=i%4fmN0TD);9H|o0E0!!6u;G zQ~ZjGjU{~??)Lhx1wbrVT2&QTG?-BR3yT_W(ST~*f>EGa_o!hk^|w)DO8}X>M&t`D zFuX;rHE)LMd%vT;|5h3EgDU=M@@9_s8GiNpfWv*j(FbR13r)P95)S*_2kP;9lkcS1 z#W5@peCU;o1$VPGW#B`Fqs4X zK-MDdCv?&ZGUK?bRUqXm-irmU@Ynk=DhUE_~Fk#kvX+Q`e)tS*08dR zYwO&&AMW}c`~NHi0yh=VR@s>HA1TNJP*65qM@x%jZq7>XGyM->`De*|4bTrbd=v8h zGR9LiW#}_|}cm zgQ-xdq}XlmPwC+mOOy`qIqQuBG^{TR(oah@YkGU*jqQ3UH>Xd|;ec|t@g^&$prD>!R_4U| zH$Q7J75EO7s~P;}kL3ME$V&p8eM#i{2@Njuqy6davMG{qFpOmYq|8IQ^kMPaJ0B!Y ziI;u2cJeh+9mI}$va+_@OrG6hmse=?WGXY(nDiO-ty@xJ({Dy>+t=_Hit6{kZpUZmAWMYQ-oNFys-9*)%oQkR3*NP$pI(Pjzr09IO`*&Um)*itx!rP|LiK0k zx*Bi6jur@}>?5B~=i@)Vt8tL2ITi3KWA zaA`-G>MWI`lQf+4Gq*s?*Q6x5VBQWI`Bm#<*xZ%ZFzC_ilh+$F?Sf*aIN4rm(9DOk zJB84W??%&qPYpnvXIJ;0g-cNMO5$XazILVzLXl)96r@#XPbUYoQ{Klti|`ZGE7iN3 zt53EEbr2spS*JG9ymVk zjUHaPgWqA~yM#6mIv$r@=t*8NcKI_9{?WC+IkCc*xLT`6&t`zTK_ODzERg|xu|}?r z`HjH23Tr5!&)4bU;;fDhb73QXjUce$u-tbzqtXT#kRG;LRSIK&^liO3Cu!&19KJ}S?^K7Skhs@-?labg|y`DY1z`U>d?)1d!Y8Op5?1|B> z<7a2Vu;Rary7=F|q0q^7bX>)32~sZW$gqm1cwrMU{~j59fthIIHul|qhj`YiLv9WEa3PGUiVcrcdctq z2Wmbi8`fPpjDBKRzcGJ;@1$pJ9J$uGJN-URccWyoSAUS|?}VMA z^uH0TX!R+b-#S+DUHk*zlSGm6K04@%xjr(tqyvM;2DQ^U}3 zeDSTYiHpADi@1{zP9Zb{a34Z?_2x2V>0GqXM82Q*acivpQIWGU*IcJe4Y}23iX54- zh&UV>Hs0Ez7`drk3X!W((E$-({4Q1BmGDNgG=5t~pYnc&zLi=h)i5Wkv{t`c$vyt= z4D6BnM0WpPE$&B0H?{iI1-F+Ns~09E<$tj|iIdD9-f2$(ZlVyO${4i>190T`&8aos z_0Fo9tQD>hxtk^|_u!je6Rub){UgGm1INmvIbNw(P{+nfRJPXu^XP_*4_Vjp?q9A0 z{WY&C@Hc0u<2PhPn1->UPaqBPK1{}bYp<)ykg>-r%N%-U0G;%y)TA|07J6p3){3s+t(0< zm97!c(gXd5&*!U9rlH1Iu`864c+}>w`#yCKDEB>hq43P^$2tjayhqKL>PZ!43@Fpb z&J}fwYx(FDglfflgx__Jjwex6TV?C4g8SL4w8j&9%sGvxZ3uJ=unyWG7e!&DkG$iB zuM;I*a_FWp?5R9cPS3K1JgycV*jPde*zB`bNu!5h3)-pPSOhk(EN(du%>T`)nATO31KdD!TA zI@Tj={kBH^)DZgxiLXq6Azey0uy9BP&2l==V$?#ZfRAI-0$L}NEmm`H(H_zbl}9X=utNNV^78YoM}qEi#H32rASzNr_7Njix@R+(j4{9P zKQ@namnvGPf=oOdli01oa&>nMF9OQ*xr7)icTen+P!0t-(|gZTmD8_8b}OS2wInMB z{h|h)kdleox@`NKX591ZS;O-@_Yr+iCIZ-xc9$w~_C}HkSf`Fz#OS0bKTOV!b+pya z=&0Q3l3#w6zG!%xJGY-ZqVVNFr>tV}=hTU9K|#;mgps%6;vBKw1xxF>0{{8O&O~&oa;R;o9F? z7NCDGjP)w*pV`Om0jm6u*Ih^lIMQ|LwEyim+Y6f+efs_%e|cZ7nynX-&u^_%gH;v1IAzekud`P1Ztyi{5yrm{75|| z>m4;dzmc5(FiH#f^&^w6UgO;%TZyu#<9@+YA8P4VBBW7-^K*y(hFb=HguAhqU)b|| z(&8Wj_jmjk0;AIaqMo2kJ9dsy#Hn7JsSM~ijg}y zc%_M-qq>}0K$D?Rl9Qv9M+E}-1Od#vyj(bf(NmlSaK)uYs*$sky}3P;fmG73fF>t` zgCkdUHKrEle-Sj=TJkYl+(Ol-oo2woJ|O`xQ$0 z7nZJ*LHxJ8Ptot{ahI-qTG~=AuHTaLBWzacnQv-m_+lMh{W@Q#M*x0wZ47AKA62L4 zokcV2rRI9Ac4O_`;A1Ew(8d`TCM*~BsFUZeE{CXN-B_yr9EZDOE?e9So2oPFB{9@4 zVmrOi$zcY^FoYE8GTH=@Drp9(^9TW2npL6xjMO9I^Hl#J+R@Je4QCUo6}&vd2L>6O z^7;0ra4i%Y;c(-6|~&M4MgS~h;ToVSX0M@ zw~IUJ!OV<%kwZU>an~uyp{F>2tGND>3Dc(SDbQT!ySxRXXwU%gJ?#G|idP)~rP$5X zuv3Z!`C!8-h#0E%nHt1=mMLhdqd+k+Jxec;KSQoA_Nb+1n)P6#XH%kGQS9bU6PiO0C_H}($u0F;Czc5YTMfn2^FlIab7Fn*1q;Y>?U|&IcKDlh>8K(#G z2Uf{}1eIw710A7aj9+v6imBAj2J}2}@P++OYM8KrG+1mTJIgiImzd?XxSI2}ww8Nf zwz_0l%N61k0N+F4deIGyXq7Yhpude((~=5?F(3?RBit#`P#e5cRvbWOsj zq_(e%V^_XS=#R0{nsN(`Sv20RdoDiJqV@vE>?Km|iKZ1_sx+>HoB9ii$xT0{MKo+E zZY&H5z{+|&&7dm6_N5VQ-E8W7bVB2}i+)t5#YJs<1pjuWy-{)#9Mn z+(deb4DgO8CzyMtFB9{sn;&sTw`AB;Cp5)#R0_yATbQheXG(BI;&PUj zmO9{#_!}kWB>%1qd2DsWm#{|kYFmEs%xXyqMp! z_sX~WDF1=M%d!9lH?*bIH-vzRcl|Kmm@O|oL%AreO~ap^IA@+SM%7MPTx}+E(dBFX zj*>Il!Ly~L1#+R~A|c})Lu_tgVHhiT4PH50okZc@)9F~VdEqsqJU2T4M zHpq_ik22Cn`KC(eDGF3~=U!wwU<$I)X&4XEVNy;O;Zzmb*{B5KS%z}AK`d3M5;;EW z+ZxfAnYQ`z(c7IO=usbRLFw8nXOg6F6Mxlp#W5E@$Cr5&yt~$8TKcc&lcE0Sq(vs@ zxXadhrvfLlj6|*`Ov70NgYWKUu1$#ZeKrf>{WAx@yhsnNknl0Jul!iOlQ*=rRdcKv z;}2F%_8(RVdarz8eVypxob0!du(^;OxZw|$CMC~}*BMS2oV0SNX_N}X3y-*Eoc9_y z9aJqXw)RzIEtuT;R_dn?GOQ z3)yLHsW<`)MO!9I2Mwu~g)VQ~TuA9&b;VGBP4jfCR_QKosm}1P_VT^#Yj}k8R?g7s ze1l~AR})vyZd}$M>OOL|)+>ieVfiFN1b2K??560lZi9oPR!5O%7n`odtpp3dW{g@V znP>}PvL#_BMi+SoT({CK=a5x$?^eQnK2hSz`#`8zal@)kvX>%3y)B52kbfq6r|S(8 zF^(!j62!ND0c?Kb;_g>Q=Po3X zI)uAcJrfIji#DIa0T|JcFM%_WTNCGpNY&H$?Huv&S zmW&b{c1bq41*5uSspH?0Y~ea7a$a z=wa2O>khaG5qoIZZ#`_!2<76wSYc2|8+4&iNCSRn;}eL}&AB_*`Yw=0K8ew<9BfMg zlGE2HCA~=7tr`H5IRvqFgso7^SYt8&O*GOH&Xlr|Z+g?GtqDw0!i#vSU2Anl^9YAE zMu@R9>Bot8}YFFIA z*01LtvnbN%WK)G6CoeVZu@{gg%~L@M9l|3b{X5GSoLen*?cAlNcSVzT&FQronBe&JrjnUm%dDj^_&`x+h)4Gld zis`4)xf1J(6syx(2-;y|uk`w=AqOIDoY6*{4}tUkqoz(s=Z^7K5$CoRY_VR6w73#P zXHw)_{T@8`QNRk^3K=Xy#_o)L>el9q@DUV`i%o1Jw_A(Re0}Ap-f+#H74!WJI?Nmj zyg)Q_rE;&B;EOS3#P)ouqK;RbDdj!Rkxc0y*L1>1%zIp9dXz(%VME7N|Qv<`i zYkQY;TgO@#U#BScsn@p&dGp58{b3VTL)MxK@VT4Z4AZeYHhTRgc43S64R+rF>b_4R z(y-G8s7g&Oc_Jfjn71_Rnuc+5muSHp4VQB0)w*3&GfccCg@#;E_TZK!SV!!-0(2rX z!F4#JcF(2!CM!ENKWZ;(>JQBB3;>QoCzGJf+O%ARGW<_>Ah|!jLw!|2i2<~=LpKC; z&Ann(cwK#OVQ}Ary$+ z?I<;hOAzx2X~87=oX0m{QC2YuGxOF~&+|&pRWWm>30lmV4^6CG^0ZDb0KO#n56&42 zps1IkSRO^B+CuM=*;Is6UsixNgfimXc|Y@;AA6m$>vWF6q0&SsP&rX_y7yc}*hl0u z+Fd7c*X$^~thoC^G?K?24rfqPjwWL!XWE^*{l{ff(V$zy&`h?hW71yoK1|GDyrd5G%&~$1XTlLNmJjL+mnsi%42rBu1nm<(rZ`Idn9VDO{CiN zWDDzqbWlkJqo1Pe3d?;Ygr>(U=O*D()qrpOuo3b_$~xKIkv6t!s-Mfem5nwcLTa~i zQ~-*sPljxQmrChP>$lhP&x_2ZbL*APCgxkZ8>~%D8tJihP)Ay+SMBVAo2;4Z z|LlvBdKrQAbx9^f^e?QQ2SE`}|KCYV2$+SSX< zdUYZ8tb1&tt(Svp%Q<#s`}>xkxa~E+E-=#(MeDcc`n(oaZ>`p7g(Jvr?Nu+A3EZ>&2SQa6F20QbHi!7inPo?vAHHbKQIh@D3iKDawl#_>z~!tUrpCBFp&$G2138S zJX0GF%ng@zOfk6J0LKJ(;+zoTHg3-+DgWK6Mwd&m zbdv@aMrB6lq|nW4I}Y-Za+99fEUvjt9wy|E{F@ISSG5v|}GB0!C_Bf> z?4uyiaRqZTN%dr3W~9}1TfOd3e&$&#h>dM?%FU$qbaD(GAu+z=-9z#4e+o%t41sK4 zS&-QqSV?O$gM}m>zH_*A$x$3}{7Yuw(r$;I%X~W&0WVq_1Si>#bTh;GVD=v9uMgFz z6W;-@Y7as}n26n~337G!q@5cQ5Wl$b?HjwbMy{c!4$eAxGfwZ(sV^UW6f*4 zvC09Vy?r6SfZz34w0?Qd=(-Yjy`GL>kp8&WIyh*=#Fx+^vKlQw(ZVy9o40Urz{Fd@ z1D62aR6(h039X&J=5IL^%*MSfVH;HaJ#^ZxkL3@7xRa$Qb;~I~bH_esn z?Hk5)cE0JUQkER zU+#7G!z2dBN=J`{M6_c#Tb!Y|s?=t|bM0BLWGfDZnUBHGKflU%q?X5TlbX_lNfJ8x z=244yAYO^%xe>$IJ|TZ{le@k>5sJ@LZ&}C(a~Mu?3YPDB!d?`@kX*P-g5MJh%7}!i znZpe*U7%dv15GewqL<*-Oyi_4ESo&(Jm8>2nySv{1wPT`gi|q$*<6;`<&X|FC9TbI1A5Zd+i$vOvq)It3*Whz z*kf&Rf&)f8so=FYgJ$b0mnDS(2diPVg8Rr?3lvO8+8GGi$%Br@#v$-auXk_ctJ$(_ zPV7~EjO!D!IF8=#wg%=(UjSC6pi5xQL06}J(REA_`08tsuihZ3&sjQ*Ps&G-iNQt9 z$i3-_y|~Nv2R*;lyA}m?EgPGy@5I#Tp0A;p(~}_q;+#U;k1^5yo}O zz+l08cVofmf#vF#;Ja4X(K5eoAX3gSgca#;n|ZifdpZo4*;IXKZ|SPRl6B>HPCaQ} zTy|(J=hVt>7RS`n+1O-L?>-9AG&W`=)lH=q78U%OYcs@?Fgy@~BgNX03) zgKS+OtyO9c%l|$U+0$dxUOC4RpmQQ~9~9D5adO*jJJsBe`AP9RK9M{1KoJ)Y99Bn`y|_?>`9K9C#^gN>5K`pORRD1piYZCgAT5L)4DkgYr1SHaerGYm zUd7^Urqw^<8MxUiQ=qW0-ExoWpG^NAI}hIj7KgD2%Ki|wd}rYw4%G2LrKd?+&gUN) z2LSV{9I)8oNG|<9(&Nb=0h2&&nUi>i@b4863&;RwF+z@5M{4Y?}Y4CglDMB~<)%8-x1;z0?}T zzwBaer@n`Ln1dO1_1b?qCb>jm~&phQStw!lN;DYsRnv1!>+ftw|USx z>Q#Yt^)DEFy+&qtSqv1MQV-YJ7p~|$fD_J!8zIMy%@;O?w_O&mDDl%wY|s7quYB;A z*vtwClx=A4c8Y;KQ|cQJ-ltT?lGC$3W2>&VfSY!?ho?CdLd^`mAyw_UL_L-#b{81h z1ttX0?~0e|_Md)`vD020=|08ZldaqGnX?C&?6(=}Rs#9DbG>C+IDohRDQG$rjmBh` zEDn`lYgS9WVBVtk(xy+aYM#e#whb!zi&>Icr+~G!wYmEmOxoHsC0oW@2Lj zq=PIl^FSZa4^RGKu>&yf(jydZ@Y9yne}U}kAz-I}p1)%JTZ(?K-KN61cb^2m#)-^{ zy>^NZEc2Itz2-YCG9!D@SXjmZ9y-VkKkV*^qRd_e`IrLTZz=k{aTkY+<^PzK1<}w2?b!i0T*|+VHx4U3LCDx_>-qXC|ux*NB3AK1Xx*OrrVz zngn_48NsWeqay^aZGe4ejk)*0p6`m3_4Ns^&J09dZnJx9jeIp~D)H)lRQ0cVunbwS zV{Vo?8=MiHW3K55t36ZDu5)G+<72EHVj25XzN&r5m7R9tOIDh)e$Mb*H`GWy`*@3d z+)vs3dc%pmsTz=V3I)PX%IbFStW}D}{>s1f*qgA?Ip=dwdds|Og&De`uX8^yV9ojl z@Q%UX7Q7W|R{qiveqdzos>YrYxKKIYV+UxUd5ylt_tahQ=YGNby=MW<9s#*7n5|Y^ zVxv~zs;&pcfC_d z6qCG`$K7!Mj_j{ybiO{5xvTB1oFG*9l1p{J{pT-t?(&5Un<~rrYvME-!M`+6H!r=e zho=e^R{Z38`i#WH;dh0bbRn1NDaQ^1zRyqrqQ|#YeJN%3+li?6Cb{37Qt_^uiF;*n z)$Qj!s^57SIGFnYuo@j^m_?^WlIzRkNV(c)aA| zduAqTNna1Atw0XIY@325dBr@7LdWZT1#$JUI>{plI~(b-pG^h;#sWW|l&0)^#9-}G zY?%Xs-U@gxMbLw8Wg#Grbv$`I!O}`qbAKGDT@7HT^`1wFEYC5InvSUtfm`7;y81*z zpt}LfR7rZX{qdu8L}A4^_i`n6!2`28>Tgoh@S6M}A^x^N*9`+EWFcRkvO^1dKZS6z z^o-J*8f?Gd-i};$SdLV)$<2LO8v zn*($W$Ud4Vz^c^a*(o9JvoX+>gCG-7qn=M?i+|Lv20m4TE}#!4tUQt1 zEUKC9Kkr8R=C#wLYX%5sSeURepTBdvMpu@dcK^69#{Yays-FipbTQJo6XLJs)to%6 zh7I*v%q<`as7W&iGi|Kvfi`2NdXa?Eug!Iy(&u4FAE(;wPw(c&V4ejv;|4>}I6arH z+bQdvjAsy%IG5zdlL+(a7uM$AB&nKPm_ZlNPGyeCZf!S8Hu#k++>_R8Z)bRXN8Evf zxz}W%>f-pi0pr8Pb~kZ~{`y%_s*)AsrHb?7-5FpPvr|m|*2})Vg0GrqW*IP{t5Y29 z71F@V&t4GfVK{qq<-SP=M#PW9-B!-}Z>K*j!7v5)`NB~UU(TXk>}luqB~thj&MC&& zm#|4{@-SSEA>6fHWupt3*UD+y92kuR6;(|{4!UEB`gP~P?mk8yt)Vk-bd23+ydHo3 zw&;pw44h0)hwY`jY0IimojBK9iIyiHCd`(R?(+_=wfFE@AQZsqN)CPiwW{8?hc2T? zW@1y1nh_k&rRia#kL4x!7d!P_a+^w{)WW@ZkDc}t87CrZZY*T!QuLXEHp9-lA!S!b z;ei>3SjQVVC>qDIpLzH)!E`{$noe#^U+=l3m|3jSHI)SZ@;QkLq!rR@w-uGxR}7vZ zI?sFvcC^(oj@d`F+ns>5@NLa;o9quwbOdp6k)?Li7He~p5UiBFjc(#jgHN-H@c3B{ z(r&>Z7{h8F?0|(o<=rXh{ zExBQM{;=U;uc%G4!56+$Uzowtv?i5Kr+hbFir_*Uns@D-7)bT_-P;6`5l92S^w!lZ zszW$5LK=Uis9`o%Wa9&*xgrU3*3|DtwR-V{s_Tf8mnl(%=TLc%2d+geTtteO&;p(e z#7c!SNmirygsW7bbGG&Js1OFwt-SzbTAVMCih?WX{3OYUX48bx+|%G8>&30c!;|&5gg?te8ZMHm ztYm15yf^!VCo2q>9ZSdXt+Qp2VnZ!0aCZDhDyoP*g-NknyS4Sj3qK#=JYuZdoPbJ> zY{=3cd5*BtHh27~6iy$h7<8Gk_t@JcTf;>y;@yfb9THKB)x`cJCu^MUEHV9|e^s^{ zX_R|M|MBdsfaG_B_e4e>py>Qn)}hb7e!H#KK~S#ZQojnwZN4~D4~Gdqe^hi7@k}t7 z6#uEgU(b7HW~y=uoNTOWH?S9Dv8H;l=eu^Fu$~Ri&j7@(1T`*xw_&NGeF0F5Y)GRF z2iIXWVRw>MMf^<)RsW??H{w);buHXXiTgqbceilLt35x-U3V;U3AWKvkUG%4HTqM} zWd7uIoUyG)zOqjS_46vbZ|+0z(|f86hv6zhkO_Gr2e!or3T}(zzXbbqbBok0`P~zj zxD~bU-HFx95ZrCWh`3H&r^?+6$ky0viVx!og-2K`25o!1X6I4ny-LMGE_3BFh|1}= z;1=1y=buw?i-{eyu2ZK;9Se*;gNHP<`#9^N3$L789I#fX)C)WI_B~D^Ug$#4lPHfd zhhVv-S+$!vg!`U7D#BkJ6+kAwn}w544r9ZS5bxXeI~@(fUS``-B|Tf@pHFh}(ov0* z?)d?4w%(C1dnBHZDt~uF_-AgedNi^!+^JNgzE>XoIgM?pYV>!a9MSAnZh7&RZO zzb4Jv2cQSuc#>uVD4m;2-IqFbbaXxg#wc7{*I0Di8&_93+r|LJDk^aZa&shaglu5U- zjH*F^?twmgzOIf=|NEUss<`>=n}!`n4-?~q+@**sQRj(s=kyKpi%S&oXlXhNl)MD- zz7e>Cr$nQO*_u$Y%CBrW69z%PlSL-S_pMuLBfWnX4~WIeoNSr4j#hArjWtzFkN2}V zpV~@p)xG0=Ho6s?|4@1zj# z)8%)$4B5&Fv^EJjcZt%z)?}VqtIew%^UN_iw+*O=ED@2B+?<)uxG!nO+ABmWMoQAe zSa4dt6uKSzY7tj&fs7OszjT+lo`g+7kGdl2Fruh&>}LIL2g(&QWaT|8?wXq5+Sx8Z zFK{pMCdqZ}&#mO7?X%m(4Mf^g+aD1NTNvr9x~3^Qh8-oT5gyV;K3&c5l`Ar2zxrvoUM^DbWul*?kv!A!+G@U)M^KHsRv*BUK z2XATd7RuzXc^v4=88mizVlC$W{4*s8O5w?odw>7eo>PA=&ZM5+nTO=u#OV@ozR5akl!T==W}$>62ygy4@%k74c-V>d6zJUBKkmgXUr1T*X5*S! z)c4~!j)~A)H@S4^PnPfh0ntiN0xdNihcsLcoc;du{z1f%_kNlfCgwTy zEas~dH{851uGW0fRjza`65}hJCY72QI@c3%H;C? zUH)VVP)hoP9Bwo`10Vel8v!<`cl^N)+bP?RPv`|6HM@u|7@un91c6uIO|4$|*gJ0u zuf(v!A>h=|^VGDKBI@jyFPmIB@SJU`Gv6L0kYp|ig#3yP{^va1ar|}5=m)J=^a5XI z1!g6N+)!_xo_NV&!_)l!TkJk<0U}>M3*7Ran^)KmXTMYb{~mmcyz=B051s1+INOPs zgdg`#eVax5+X z7F)85+~)*u%l%rAevaAKn#Ui%S61ZYY*YB92>$%aGlGXGV_#ml#JbO)2T1xKPTu3( zLr+#T^~^yY_zxF+|LH{{z>RmpUZ4FX>VHA1Olkle{P-cbQ0?ycOH1=eGz~Bv)CMu4%9i7RJ}e?X)<=<@hu4>Wg!oJuf^~WnJ=Z4!V9p9 z#mj`TquUYNJ~i+fgE(%3N(rFJ_FezI7WXwoPyLkbKsGd-b-1Dv_6Mf~OWRpckcp(! z@#!m7<&AL*bs}yn_d5L7sCvj2`L;T<+IG!cRVe2we6SY@x>v4IMb@$g}7P+X8{@^YJkj?-OM%8Bu-?3r_Hm7h+&EGlxiv!#3&48Wo&7Q>c0n_>trJ(Annnq5{2oolb}} zrn$yi+PV{;F_R$dn~r+#c=&T(XO#+K0m0>DH<&rt5N{uO860PJVX8&-&0%uN+%4z( z>RUn%!`be*+^C$Vog`h#{$gZ-dn=GDecQRVVFPtHvPKZ5#EXu)(xFA(I-vb1UVl2w zv^|wJ3+q}tgf=T{e6y=OvD_5>go&B9t6m)RV6@sj*s1|c3b@m%cqMz{PvzEv#u?76 zWVJ>A2WA7Xkk$lHL5PC%PBvqRl$_ZB&#C7&1GDZ!l@FoOpt<@#+UXMAzhV;D)m((* z`4MATWhCh>=2M@~+O%wgOJZtTud7_BdpCthsk6v@)M&zGnjY$i5v#*{=t;K{ZY%bu z(?R(nN8vGR{}LcwkqYR22|1$05-@lC!ISp!G2Hatz-(0rJDgG9DUA2il~XCKEyim; zadFchn1eH2`c5jum)2zZ8`o%Di^=D#av9^Hh*fc?dr?1K(!v~TI^g&^G=cwK#iA$P z`o(>(>yBT?vKf`G$UbKv3G7(ODX>)i`Dmm*A(M!t(cCO~b?frvU1_Ngq30#l^9GBK zdQTrZk*Q8bm%$j`Ykks_FZ)QXxm-jPY_dcn9CLog~eN8dXl@T%tdgQ?-R2SwiwrIX!+}`ApArInk7!f?3gnxGyd8Fv! z(->3T3@dF(z1ccXso4kg4QJ@X|25{)WZ84!1A=0yFBhA_^np*Ya)?)8?jOGI80EL) zqA>0!I#UYgrMLoK9CS|Df%_?VE(s1_;rqm`17El(=BDQ|{e_)|iI>Z!8_)L*Ssf@? zglacnNfL61SscPd%QNwCSQDjKp;p|llblrf*f3nza9fP8?^49t^sF4O!C@u&{YURB z#4PZ8XS#SRr0lvR2)fo^lyotAAcS48>o$lwQy^UDOrUK|;7Xcq19^VC5c^uc&Nih$ zV`n6zzw>!PP{=4N4EcubTTAsirh(KpEXZK7Rj}jIczQr-9;f7Wp)63ap0tC*JZWcN zZcv51nStwY>HBMYlMLZ{wRawVBxiw9D&6PNuf5%$KU2}kDY`y`;5kY$6T{$Cd+FoV z4}an(JeFRWhY-k_udxjnd-8Z3c5K-~T26ejEcHZ4ahlY=n2ljyiU~k(r8EXdyel>2 zwTsz|!7FA@vQgN1c!2%G&tex7D9RVF)&z6Ks9eAB-lW#6st8i$ih8Y?6%rIfzRIUx zt7#an0OUwoQ#gXD`84hnRZc#w{yaS2>MeBF!u8MKhRvE55ewW1+c zHfeO{e7oOEd#$`s#@S>8UC6wx2?90MR1`|HhWy87S}%ThCb=QG@s%gfB!*kRx1^(F z_Vljh>RXypO&-3uON}bKQo=S#Ud|1L20Qt$&^qoB(@|lzkLQo|(PpH6NFNjPTsAch zz69y3>g&--__SKlY&YgPBBzf&J!}W7KI49D)4yUtEmI{)PI1h$csXCs@^MXsSfB?b zA)Z^GU!iJ^>WUWz*CqarLaTQX#ba23_R-3=Z+-27+yFb)EzxFSmrt~j?Os1uPtt(o z8x)$o4*#GaOvq0=-gt3Qn~-6tq?=pcYsJI70Xo!|Iy6GxUN5!hbZ8PPdverTBebj+ z>RE>qy0OUoLNbG>_v5BioGDLAA%d={X;N>&Mq z(tqk2HSla#dCRwpw;`#btL>PQapjUOuFvxwmup{XFuLEqce2R1+yo2lLIMI^l_n)E z@jxHOrU=O}v1oIHQgiC4_f6Pu6)a5YU3U-FOK~$>(hX)g&F9|aVEBkfX=5g>hn80_ zFEykn8kN{jhjy371`O;@UkRK*%c=R!le726-I(_otIpMW(UTYDu{_H5r~$h)#>X>q z^TZr$^pveOvLtyVM%RRsO_5fEuqja;0X3BbkDF$DeSt=gNWB{H`v`XNT7G`fe}qZc z{rDz-OzC-g|T7*{_T15JBFy* z`t|Fj1~7_QzOLY_85{HG{2LyeW`GkuLQciS7AA>oQC&|%O;JgQ#3zq<{E6ZIijspc zVrAJa@Ael4n#e&wQ_VasNec}#+8D%1>UIX|iwI4c^DO6=|~f zS+9yC20oLzb9wx?%_5MStI`sTQ=VFQDi8J5jTd~f3B8!uZKQ*3h^fwIjTT|W@|5Z3 zTE6L_%|^OmTwi02hd7o|u;#{4vsnGT+f8CTpFI~xAvXve2GuKmc{k}1$e5b=d@pQ@ zh!=4vcFFDhGbnLE(0c;^v4QuC2)K_QBx%~kj}muB(@)b)xJ>go69*WRWV;=@h%sjx z^h;?(q`i8tg6h>@x`?$`Lb9K43#{1>LuGNCKBh;PP;qzHWxMA`^CRlF46r5vcD~_^ zBBmMs=UTjcH%ZsOA)*@>Ekim@lZZuUNNjK&sRrye^ChB+2wG^$!P{+nQyg{-Up!d0 zEK!+&kLSUw#uK$ZKm?g^#LR>qL{g=Sm;EJ7*-_VXp^X=st4WQaHUaO&DVJTM&k@N_ zG0?79TZpeAR@dKg^l!KO2WA%{rm9y%98#ZVl_T%k=^jae?UpRVd<&KCjlL5Plnece zQW-0GzB30!-2NoP-Yc6gy_nG{8Y8?gl*fZUOw{my(dciQuA1f;FOjr#m?kd;oM8y_{Km0s)Kjs9(0!*VaAlOW51lIQHmmnR+Tw`p`&Fi05vxoJ8g79!!t zG;c_F*l@q<8p_$}0rFZ@%8Bq43t5p-Y;(gvwb>_qnBmq^?^@ca**9 z`im(uam2Kb!)jIXkWLgIft_tHtsEauZXXHN3MmT(7(LGK~;==(*a)wdD<;xDP+F- z^mXkfGylzJhw&NhOWexiY~OAP^I$+Zk9{#4oAei+s9msO)e;$J zH^KEQF<35y(HS34S?oPlvG{e~Z{~uU9i^#S;`uKyV=44%{D~>Xs?)fI{0}=R!Pp~B zGxC@;$kdhfeB&DOCWl?=7^dQLZcO~lM4X-syB*}z!01+Sfw$){2Dkb_=0d;EGqD=sLca+0X+s9uXNnNMHh^G)JyNmz3P=rehW`RTw)=R%*l=79By;fZ?=S2iM35=Lr= zXKuD+BP<%|!tf@;a$F2B!os7CF|~E8;1%JW-KpNQ9mBnx8xNHk9^xO2WT_!#65=IV zUgf?-)VrH4$p?ECa+wU@w$r~|OmCHqd{no*CG`h+!)B|(vCk+SwVHYI>LvIfyvN6W zA<5XOvQX;cf|0XG5Q7M_3ZTy*HLgUa2#PU?Z z^=3Os%fS^YvIo-iD^-4LUDrXQ@0}SPCd^0rkrvx1Qw6B?$Wi=5a_Dq$*!I0_)kJpF zupN_YIzIOndh?42MxH#)uHK#?+^N?kNag4P&v9NSkEJy0<#t`26Aw%(wOSn8$P_g! z1&?B*OD&*wkY2yqH~8EF2^NR;Z&H5S??Tl}c{QQ3NjDuiSnDscc)(9OX8IIqFmx37 zHIZjkO>SY4q%YeT_7nZK{=2zWr8YP-1z+vN_Zqjd#ROA(N+k=dT%~oE9*X8#;Z@c9 z&8{nznhkMSV$wGs^59s^-UnGGq&hVeTdQw+V|Y?bI*puZxvry&RXef`Iu|{WoaHn) zZrJ$S0#bY2jW%kR{Mq71To-lVyeX#W4D^Wws(yU4eclZ4HR2US^OrAt^_l|Rj`v4L zf;)6GDa_@-e1vhvcXhqq)z70fXqhW=6LuP(`*U_cT*KGn=3TSgLv-Eats%vHP0=&2 z!_dO;GlJr$H&=8lCnkdW5ew+|Wb4!Br#Jm6+>2ahPL|qXndo<0k4vtXnB$7p`aESf z+;gPcH9_?XzQ)4WSqn`94Te7*vQnXB+79BS%Xus!d?)LWRf-s_hxT+cNLb&n;KMMw zQuu;!F2nKR98_>38$M&%q6Wt+DB$--YG67+h)n5-4#l5*F!b9VA)em`W?l)v@jbk#HhIh|!$A8$#O5Dj`)cHh6??Yp87<@>Qb56Uy;%B`nS z3Ffj^@|3&Vt~`MRlc1Pas0-y$UC2|E?Ub8bC_|@>H|=d?$CsDHOUJKOO%^WQ$u}+k zwty&eU=%Npt%;$@Uqz~E@1(v(voGh^w_P)sLM`MDA>vw}5Fxe{7x^SerQCW#eRnQJ zf#ccvDtuI!WjRJov!CIMtg6AAGa|;bn|83*PW8;RZ}Y|Ld+!be$C)~?p`!c3&o(x6 zt+60zV)}%*WhbZeIM40qL&tgRYOn)qGI2CqI`)SWRCxn=TXdvtAM+alaLOB} zhEx`Wkp0s6@#67Y(&X+HQ+5t+(x1z6>U{muc6Eoce8Sqo=c`2Lvw*h&N$KgJ(}qur zY@i>M+oSs#*TNQpZ>g2r58tY1WFCm0nW8TJRmqfHb4S#8T{EX(BE98-=*F9CGwzVh3zi9IXiwQncIm+w=UgSVQO*SW zwRJUUCdn6~7f8NE*wP?7duLEV6)|@9^C{m}CC1xVt}3l_2Uw%!wTF_W+zp!jae+4r zuusj!{6`FPzO)nKqX}ZtpisSDQ9g`BvR;DgT+WA&Vk1p12*Lbfh@~{cT2{v0AvN-d z-nW7pIyEaypOX z-aBCE&v;2|tIR<{@mM|Q^>b&S+67Rq_5#{3R+14QbA^wEmh|m&&rU^I6aS>9E68L) z_R}lHe>~OTo@#T!LYqwBTu1wgQ;zbWB5= zB`)jLAG~u!9*Q1Y`_$04cD%qKo<3%a-|baP`}D`qn>hDI#xlSAlYa! zkw_EZwMYz`xxf`TPk}m$S4)u;I72nrSir%NplqOOIK@o+pkSb`pTWC3s)DO1fgWQ5KW`)2Cn^wl#(4z2oRc9>E;7V(P0zBWo_W04;1svBeqm@@r>@^3 zXG%qR=Xw2yeV-ERySg?ZPBbDI?O@T8T!Z|8*h93uX&K6iuMzDxVFteM`{#>5NQ!NF z5xv$M?#BgQs2G{d+&uN6-Gt6O-L%jlsYK_nz6SD@w7LT7kV9)x2JW!;6B@xNVb^g3 z=_T$5j2&gC`?tO(3fqBXcALQnf950I?xpYlXvnsI@TT!`bALo?MVkf6A#9`?Qyx~@ z&rsv&+8s1Dj?VE8d}+*{?=E4Ngx@_FS1a!wou@EhQTUTl|Ipx&ip_P%~!`y(#zWC8a9cg zXd13{D?X3o&z$Y_n=fELoBC5elOKN0IlV*pVgl;0Ghy9KNM62M?2w`DpKtO6uS!_Z zn-l-+z^(aEu9}krDhTb>sLy7$E20OzAO?%lDP^;(;`(2f>oS^IE@c;aIZza?W-tw| zKiKm?Ut~*(wtmxAW2MSYfufCzLG?t0>J{MD3NAhAD^+&!T^91`b{Lr|r~KH)xNwG>DiJ0B`2Vo?mSIu-joz<{ zf=DR{NDiSOl7e&$A)usyGz>W)(wze$A}Zb8-Q6A19WyX=$Ivj;FmQ(d-p_OHeeZMa z=Umr$d)^Jqi*>EFX8qzDpC#G4px5cF#^(*!6?Uc8J*dsvhq>>Wh=WRCEbJWPIpUyE|zb3Xx#8tCbqxeStRqDQAO#3X~oJyl{V8 zIs^&dgT7>CLA{(MoTOX@w`5dL<$WvrRP#jPF~jNe2nK8wlN#Gv~9_PDf4^tP zK18T`I&11D^F3Yj1ZUqP`?zG7zENyLQ$_8%;OyBJI-ykViTBm#FjWY-exDPg#wv3; zKDQ~XXdn*OZ841|yxEwgk%YeX*UBE3)m$gzSQlGeFu&-7Cm6`!C)SZJLgDUhjR2`do2Z z#H;*I4MHy%D=kJ8R(;I}p%HFFI}G{qkA3%cqdfUhfm34;oo}R%?b7 zYLgHs|(~X zS7nGay4-lugZeJwf9&ZDhnU9D; zJWw@@)r8*{Dk{JI#n+1XuG_+<a~MuoDeOG1PQo8|>~#+L1eLcmZ5+N}+>grW1`_v*%vxr2|ItrET71U)mS2gU5t~zzaU^etO zMX<8PM%zug<_8<|tHmvC6|tJ;@ZN;#0+vADSAO4tpG&F~l5c`B?`u@Oas7LH%G>Hr z@JFCS;+=J7qPKGqQvXL$Z>BnHKf5+JwtN1?2W?1`%h1oZ%Pup@53dJ9bZJ(qOD?SO z=*|4hzXpW!?N`mB=o0R@=hk6M z0G2~{#xQ1ZF`AyPea&;KxMb7xwhmH@QXE{pMISt|7%%cwGR~?1Svi-rJ%L^O%fQDo zk(6iXsz|0%oEw2Jp~FW4`VDoik0-p7SnjOMLoE5yEzAmV{-xb8A_P{`-*r$k#CNl& z%jp3lGV%1V37y{<_>Xahg{1btWNst)=3h6kn9m#>zLVtq?l~#cxYsR?p`Z;cED1t@ z7(-7eEo@*^96FJE`{WsgJl%9Ge#bXrFS;&bTrJ55HpqfH31Thcj=op^?u%xeBAKzf zzVR;CZ-8fIhD+~BkGQ39`Bd%R8VpFEj;^`1@(evBu($RwF_CgEw zdGBk8cSrKIv-KREzohh&A5o7w)S}(TW2Wnpy59glq9Q^e?2Xf- z@aw_&rFLFf7=9!*8qX({13KLO}<3IdyQ(E?Jk=N>)#$LMJm36+Dz zMgu@*iSZ{I>`BqjH{NI?n)8KrKiShB3w$jsa{NycqCW%u2|Mu&w(ROKTm6DsD3EOe z2j@nH3?hjp)i^IeO8pW{WwzC829W|sxhMt8s{-f$%C$0p*v1T74~`^o_BRqI5XH!Y z21F^rVTXIBUs`RF*4l8S6plf|mwH~x=2YSHdHT&KWc`IB=#!C|FiRWd_nEoLZnAPP zWvX=@^6uK^95HQ-_rcXL=dobnGqR8NC6aIz#8-9xvlvwfXpS67FYbU?!zY|uZNeHW zq-og@QK4Grr6Kt@T|wI?rbjF54tBTnf*CO^B01hN0D-SWTp1(-855sS`-!?-DzCQMJO9&37{M5)U?|;6bm~WAMm6rJuqon z3-$iD3?T>YUDCmAHxk^x2s!b4=bob&W@XmxIQns*{kEiO+~K)5%bwin$Quqr@5_}tU`*gim2737 zOp_=IQdt2LX(L!kUHry2jTl2`>SbLM!qt^CLpAy@ws9b%G@edy77F#?QUo5<^F(f3m6_ULZ~vzmreJ>1!Kk%SkpT3Arb zw0janWs`cj1z;Y~S#qC6qo*OiVNaHxsM`*ueVEJCC?t;Op0`3$PV;*ho|g!SKE)Hn z3hz0&&^Gjvr1+F}ggKXzS=ci^aK#<(&&8EEf-#EO_Cq*IqcKzSX`U3i%_%1T!lyO(slH zAtdaePVKKF@L-y~Q7oB;WrF}dG%l;3l3%Y)#i`&SCdaSIkR59Ib5W54CFy;V5JtV> zA&H!fYP1km5U53ey-mM@u^CYyV8G5vFl_s z89}T8Tel~-H=6_px&Q?K3$?Y}6G z1dn&6W?_UBZl%4>W5qoW<$}#{s=s1(6TM5-$?rkU0{BZ8?ocfX5xY7so()gYXOALP zJe(1^*aK^Ka+C`--3AlE#7*5EEKXp)W~RFpf>1y3z?jY>d*!oU?$(^mO*qF_V%^pY zTcQ6o->|eo{FpD+-X3hULmqz*#24qwU2I_WrgB3$%CCpN?*OGa&~IRFv3(dZR?5Tk zX>&yT<*iWDQ{5V;=*7JC$$T|>Bjp4M;dbehia`8l0KU)+|ytml(8a zr;bqBvSf%~k4w#Vu-#UeO?Ukg#7MG9ef6y@H)vU4p~TwN$MxHtxngw#GxJ- zj@lleF?igCTRA{8<$&?Vh9bd*<{uWl^^rwxUJvuG?W_ea%`e(j{G--i2=HLBJ+-(V zUhz+dGVXd43k?Hot+C}zC@zz0w9cMzD2w>nfh+9&ct(D#l^LAtXznhfX;CbzA1th& zsI~I-IDW7G^I@3Cwz7*P2FL!5e3Bnw&G432fyvikmOOXJ11yO5Krt;a>TJhiUU}6;VtQzr2XJbD*(nCysR?NqPv5Pk| z$An+JTUk@^OyFrL=J5_m6)LsZ#FHomZE8-%h8 zKj7AJ9mJn&bi~&E-VuObX__{5dv@X5aACI$PQzc}ikastSu_sj-o?C^>u2z&3q1?G zdoANY0ZNjX|B)G99NR#ws2dlS1lsr0g5_%MSO6jMKc?dllbGGb30#Rbur|+8KE{wWmrw;q zsr}7@C%Uo!Y@XG0qY2kzM*Gwx&60AKDT98rFEah$`HhG>_tVswBaBwYRX*)8)d-^K zk?j6pe__En-*WeKF-0Ub$^6&)M3~oIui}^`Fp#5fMVhq=zwG&;8SLyT)32PV%&D(G zZ+(E`Hk|^C=<4+9^=q*~kUMIqMPhS~+VofLy9&GZil+yKnm=2dgbQMxwc&v33QJn# zINMK;(iX}2`Pal&pS{aLzmre>ic;mFfff}9M`8*=WuD517jd09^&QfyVlu-CZ-FmG6y7J2 z54oT8K%Vje5M~Ogg|R{9*peyc1z?jU%! z6~>k+c@6FLlSC)UqHeS*9!Bi?{o*gizuT}{f_q*g;o!`wip#JuY#!h0+98ul@v3a*uB%I2ODY4}{ZoqobE{6uXkc3$YTa4^{3)w)~rK>ug z)vU@Ku>O)O?{$*nS7j)=fn`*=0l^XQN@e3OvcAZESY<%IBl@-apa*=ag~IJDQfFbf zMUNt+vA}zSm9w?j`RhA(XN-c58#irjQiPmOWtPMV5?p7UtIsl| z7;<$7bTx=cXXfsq#eonCeyfapHKn7(?4^dL1>xQY0Je`+mM1bF!nDp+if~`{Y}%As zhsFy1aD4&>yeWRxG`&+zM-yy*Lay6l_V<6!iod^!aG;#MuU?cFINB__zwh|Y-RVhR z*I8p8{$ZIILCNH2c#rr>#vpG!cGR_FTxX-j1+#D$cHbI(Mx)-%rNOQq6V zb?sF*pB~fpipUE_(~sA*hPDd|{kz7kjOl1Z_)xrx{}In)K51dZH2^IpFZykz!IQq| zR%|}$py$3$%5ACzq)|TDM`yz`k=e0uKHG5+Cb_xIdL!w-3Zhd;I8+;@^DSRg8V5ua zhp+gZBx^qzpoz!pPnsderds-o4Gjz!@Ki1l50i@?K%iv^bo%!7$;F=1h*HEGdeuTrx!p!pv9?Gqw9mtL&fS@y7tWP}mBK5_WTPr`PZ)ia8$!2PBIL4>N z|BdSxW(c{5l|adFMf1*&20(SP*<=vAeC!q|BN~-&IvEOA=6lw3`j&7bbSN!v_b7&M zkQY(o_Pb*|@1rPW=l2`rIz_dD#z6|8UrkwznLY6@L3hQ~d8NT;f?xNf@FTWn&A)Lt zk`O7>Z#ERxfmr8zEzPbn6zodV*_IAFO7xqM_ed6o$SgW%fOMY4tTy8gt38$!%+Hyz zO^_;JGlAvR9eL}OBDN*4{lY`+fs1AIr+QSimTmJG^OrEv{=5@JvlGqV2uqi9r%r+7 znbR3^WOnWpv4RUZ9?jX8sV72k@)RHKA`QM8+gX+o-mitF>IHhkDOF-N<7JAY%|g2q zV{k7aj)hzC-{Nsvy%r3C{@%Ry(v8jAIf!aPNf8Y)ffR0w2K)=d$1dyX$M%5GQ;DFP zl6pR++L6zGJVh~c#A?dA(xK{9;NXdd(3x>NbTn@rAb0q&o+L`I)upnnJTY0epsN~L zH&W#6wf?92JCTPAdGJEJVLzv=H~BB(Jr?*Afhx^B9y>utGnno*jT3*u_ zOBIl;@McO~GaDD5P`_@eV~M_s#ahSZN$*@_hXICGhpIl)@i0WZn zdVen4QaDrHG3n|R-5FPSNMQH56f~bmX*4y5y+?Gk8G$4>y@LT7_O1D?q#H*@L-HRXgc&dNNJ%u%}4Dalz2sI8C78G;!#LWEA^c09N zg3k;SUp*H8{bu~KdcotK$?uHlE;bSi@NSD>bFUgwCh=B6Qpz@tSIvb2- z7W*@ZD?~*iyw_RNN(GHb0Qv6Y@iiNFufyN|4!o*Bv zfA3-SMT+;%F1XqvWjn3N6SH6A6y(q0MK`KH#L+S}>2VNH8&TzEj`nE|ug` zsx6@dRjFojElKCwMB4mwcFk?7xMI{sW+ZH(Zq9M)=BBI~X-QEkJ-^5SEOeY&iSe$> zE8f??W!F&$Oxxs*aYM;KA6o57Gkvwn@@dvG)XSN2Jwrn3D>hl z5?_q0`s{FNsnGxhCJQ~e28v8-KeUrH#jvPhQy`Iayra7o+a=g^NFUd*xm;|T>l|#3 zxa-#0+r74)VYXVcaqwiC4PJY|pfa{s$&S5m8-zSWS3rVm0&p&sNDhkj1$Z-Yx_sfN zPAS-^FZmI;IO5I;PWp6Rm-|+0!z7yDPN&66J#9bZCR-g4!H5ykSo4TFFfoisTKz>5 z{E7}pG|%~bh2HinuR-6=3SF0T)1iXnt3uH(HpKDtD$#6no_6pUqD<33mYV9}`BUtR z_!%b|^A$!n6d~u2tY9zk*4YDfx7{%mk`7O;JZ$3eQ_LD><{@OLBY9mTVwkMFZ&AEj&3xJbMXqLtPmBo_6xrhthmnnU*@r%r3y*@{I zR`tWh(*Zt+@QtJBJrv0l&Ru-TLxV5oJ2yIv!`Ah=m!0290_+2XiFunVZ$KW{eW6g?``+u0!ltZ$5^pTnywr)dn|`^d1J_Q^lA2F3?xlw17Z>q~UTR!ziLQ5gVse81iQyv8$%Of~ajPW8Gd{*+;Mhnyw3w-NmdzniB&>zEWT zRAju5qLTz%BvWiIq6QOy^zEU?Qz>~bu>x8O6&Y7ovBgYH40}dLP^_$v?BxaZX>;_7 z99FVlgOn0{RW9gUwnqK4)eapB&l~kmoxSFyA@>?R2)_2YuC`*@A1!^9$#br^&EN08 zy%_axN*7tO6WI@VrkY_IPItRO7kwoj8`fFz;o|Ynx}{Ivc_Vj14S-;0^#wIg2h)v! z4&-TJ`ub0ii?~0rmbT75(%6Mp=G54ahjc}xH%S$Y?%%@#AFezZa7y2QVP23`ZL>_d zGq_}Q(6bJq@d_(2Q~16e-`ij)0`ELBkk+}r>n+@7MaPQtJM0k?0`i# zP${LvfNxc5{nr>47IL9Wy&B%hEq>%nN*9F->ZZRHI4|l64L@jtNfQZ3&AGJ`AQ-wa z4-Zq4v!$EXh~S=Xy(f!_r5+}4J5)lm7F-L*5%49Ck)bNa_Lb&7vJQq39mEQ^q~V06 z1=*!yN@ukLnqHqa*pjla_t$Ef($9^oabe3`TZk(}=>!T2Y!w>p$1VyWIN;>0@>W93 z)RAvkBE|Nbm7}Y)nI`=422HWYWR$5THFd^H?I;O1wl zse-+eE0C!_F?i<2;RZJM&Z z?NNt@J;T09JiS+t6bf(tmr%%k?k|=t>R8xz!~54e$wXrN`Kae#Og@n>-T72}Q$8pf z?e7uW{As!S%yf__diW*vj?^l@2@vms2W+C`wr3D;2el4eu+1Y99@+8AT=a9{(LEDh z$C--B(cA;-6?{%dS1g-g@3i=`;PnRI?>A8g3^jbt`iMt{+pSESeekH0`szVtH%|Ud za6XCs_TQ-kR=UP1$C*OLB;B9i$o>rRlz0>pf85vzIxG5dr zmd^^B?Z7+M`$Cj-yt*zXexU_rF7=??mfgX*+8&@&Ds(Kc1tFq-bLu7g-=v6nCRt6_ zd=SH^;^#+Yp)J3)-~5dNtQ^B--);xYCU`$QPM}V0Y_1aZ-uV=dBj4Accyj7^o;I=& zdB7YT7Tr2LiS;tAa9KgLt!43y%qW;=eLG86(fy^y-JsfOo6O@b(bC*>!j+U%WA<&Z zX~tfDTnGK4npH!WBrKW2} zXuM?p;a2;18`N2|VbtBb zhg&Ee)51LkUr(|K-(u|42t}&eg7%y3?9bZ*cm>@=$595_#D8&arx2UH*0qbq9FMF` zrew=y?w;)VDp9CNUA~l7H?nuxMDBJoin=D-x*T7zkAVmw0ezvcx^`Vvt8~#g72{3um}4W>pI0B`B)_MDG&w*-Mv<10XXVGT4>!ZN&?l~iGjGeCUKnzFO}`K|A(f{?s>(wR zV@`(`T?RNO%n|F}VW9fsbCetzPjFHZ>yJQufM+M<(D}*gnN+FTwB05b3WsMS!`>Tb zW_ki+RevDlw4ZC~l@NLH?t3Ws_wwN}Zj4!zK$#1Qp3twjso#4_-6C84B;*Qg%)={; zyv!k(3_N42aQLfpY1EW^LdaDe#~p(D zr+?VPGxi&bS(TQDFuRzy0&g|2o(EH#QQ^^SNSh}(HSgqU_5Q9>0c1>-G7*)-jq^*g zobm=>1?_~(dd52zkvXtqOohejtsOTRq2DcWrS)6x8)fYU$3l})SGhbA=;Rx-#=5kwQ>QP^u8v9CCeudzVrdBfty1tLeh>HWS@X$ncn~ z4b$+8q<&Dn_WM@8lc@!k4;h)afzw*j&P}Fc+Jk+wwr5jNvGO^kbjkDH{E1vJG|2t9 zM2CZ{py>&*Bgbug_Jp6gC|7|g7I z_{S^{QkeTUpwUatJJL~8|7t|+29jg%kvzJMP0WmJR)2zQ29vO!h^n)dzg3s zZk;q#*&G$z6*kb7c){6~s>eYhp5AT>|2P?8B9F^_qDA_BRqv)6g9loKWb&29`BUTx zE{~r*mmea;h|Hij9QDYmRTJE|z&$-nBtg79YTAuvenr}qN;X3g+E2)anK7ID;@HxUm-VKEH-!(&nB12c9<2o$Wvb6F zq{aAL=pU~Q)1N;dWSvTJd;cVlUH_QzQ`sl6=2K#}`(x6w_wZQ@wcCpjWA>(Bh!5qK zjFF|Z?p00KkTbx?M5bca@QLh38N{#M857IqQT3J6_YeAh+&l0K)O?C*ptECym@oj- z4I2V#6NY(-lKjmBclFj+p(68X8j~6#9H3!IwHB>C$-I~L=7#o1oAHZ%7YfJR|=4&lbrab7O;bmT`4ufZJT6l%m3?*`y$wfmoQcY=dNWTch ztl!uik<^vJ_HP?3%%2~{m$;*`$$I~i0R<`*NnKwExCn(h9-7xYb`ZRY>LlA7BC~92 z;~bYShE^sQ*St0GdB*~y2zJ)_bHOf~BzFl>-Y4^^gQzQXp|?-|qB~O9e5ODj62yMJ)G%WV*f-#z zW7-e9{qLvnfB26b?j55_zRoV5_g@eB|KhW+;T;=o&`^gj z?kxz};p{>5t*>ni&1Qei_88AuxBSR4p5Z^>3~%=E3R|-O;0_KWwzA7MI0C#2tanKFs#f^ zN=X+yD(`1%&xvNLElcM-EY13&E;rKN5nkJW%1yr6Nc=v0(k6O8YMzXD2^Z+)H57vm zd5FdA63_fvJc<@0Ko=(qxkSn+?(|^{e{AEx&#(t65l6#!bNC$>V~AI?F@`&PPHG}| z2i?hs{JYtXV_)31=@XuWn|(Ry8~=Q0ES%^>5@tXXR_ zpN^13zMkDxOvK21Cd@CDQySL`3QM0D*J0o~Gi64VNgC%UpE09DrvMjyC#d~MvJk2J z(veU2vlE2}E)PIg*8hpgxr6ugv6fbs%dTGu)bCipD9YUMb0cf5QHo~Q+?uE)pl$ga z24@lTDR!YG)s--OM?!Af5A@_!z3n2P!ACVqOX8nfNV>1hr#gQEHwcUd;o|_4LVRyA z_mbfHOPnD?-w%7WwebG+j^Di}|2{B=W#DGXjg70hX+HG6vj2Ie*|OqS0XcWjF0(2q zwCkH9F)Nz#EU++4Q@~I}KCEu0#5{O(Vi6w8smsn=Aj&B0Gn=ee`BbCi(MimXFZUsr z*L0kzEiaRmOvGVF+}LVXhu>7TPiyjc?6f;>YA%%MZ8rMKdGzRL3Wlog6}_UHcIFpokpyl<F5Hp9r{6p%VueK_ftTUuDm1TQdy9T$Mc%<5cpAO%-y2CQ{ zeI;1OtOv65L_#Uh5IWB3_zbq5&!sAYThv4!=vZ*Y%}@7d1z93Zb^i09OMz{0b%ax0 z?v4n(Cdl2tRN%$0hac!fd_~+Hdwcvm0?yl5B_4u2>ZM1Ivc2qBnjPz+2DV#cYO~y8 znxl{m*jH$BY!n){kGB!~UxQ$DaTsjm!;`wKkURDeXL5yO&w-Eut7cJjmBmEA;1i-(Z|$!x_~I z9Y~qDSAJ=oB<*W8Rfod$2TNRuUT|fe_1Z<3kHbkb*%)^?;xZ;k!rou_3pA^l4-1c# z8;x@sR)w)KkN<$(WI`_A%aK8}Hqm<=2Yrp`g(nx~8|SCnYR9XS@o-6k6;{Y?vxGb0 z!#|6H7#nmjzWNiN?Lv+z2ZMAVSx|48Dfy?Ja(!AVh+ij!&`}V<0&v?268HpIWuR>23Y;PlQyH&cNkpduN5= z4e?9`xY4xv#03M)Vd;xvk^1l=$$~LAIwJBTk zJii3+*~~H2&btJTi`Jhu8<#ZBd}o9+Y)$_uRK6Iw@XOf<1F;ggc-_u0MZJBXo| z@&>~*3TKJTkuUfh%QCMDQq647BIX6_DfSs1N^nLA1%6 zYU;`F?(&o9N|hA1ii)*<8kn-gAn1T)(%=6iwl9Gfq3&syTjR26HY1efe`ZlYN$rUd zKqGV^zkc#QV6L@a)tE_xAI*Q=mJnbo{K>V2O5+5(`dr(8m@3Y$01>cNxN_VRj-ZI-3tiYIsH&g%&rkmZLP4tUHS0Pu91>R~tef({JY8+=pK$)mJ=< zMqt)057Ddsc%Kc#iyvrIe11C*{5x)_5&H>c(`;Zow&|}YPgeXHT>Wv8`E{x1lQjYs ztU|rK(l1o$L~5*;CmSQ#s9C6&*13`WGx_#i6^Q%RqNM*pcw#4;8ih5aEXVRpWkf~! z@Oo)dHi6%~+(Sm72;SrQ3WFR|wJ)2$IpP!Fo&~5J=Pa(BdGxUhhqPVU=V{N&~ z`p5!wowBIfy4tQ%=N% zJqNRbttHQcsFFpT`|&R~4_c+Jh<@Fk=C?j1QyBv`NsD4d-64mFEUjxt0kS|EbUr?X z&G<;x1_hP9PV(XzoHBqad3z(-k5_5eGau(L^~H$&FY;m5+%Nk+mfE2F1q^Y!`2%}$ zoiwmBEz|5*OLVa8q%BU^C>TzwgeI)|IXTdZ>jlin)dEx}N4z_e@@ z!Nc~z1QABbQ|DMVP(FN-AcA9_w_tuK9l?<@%P5#>TamXOgHgkuu8~5KO4PS%{ZZ@` z?*zZz-uM*&Yd+7b#eZ(iO&gzRAE^K#XcxxVR7TDRVoFp;jQH_l3i{)Dbi>$WlnOyX zIn8|Gjbm@SIcq>yki&xhQNSIMvBNTY)&CK%(iPBzwQei04=|9!FiA?IOIwn1cn3qiZKJI5}*jzj2L(uLLl^@~ZB99iXt|Dt~>zSN)f>@M=EaduEEr0B~Fg zNU=M5=iTaYls`WVksY#8s3{NOjQCWpgm*+Ed~QC7p%uCz?(jBJ-h#)mkF)HpuNqI0 zEdV0?<63(d-=29-yZ+`0DLaYNP?w*JhN-TRH1+fDEWf&^u2VV$!v{cj0lHKcywwpaKt4zl zRXbo1TLQb2b&Kt3SWA@+Kj{~X>JxiQX zzuf&r4g;=Yat*h|1xRQsXhl64UK6UhW;ScuAH+F*;@rJ9@DSZ!faJJSL5AjBUDr9t zR$!L#o<^3%0gAIVM7>sP(~i4smz2oFhr@{?Mze*t0rVB{6N>E@%g>1VNpI#Rt3Mn` zoNKT1pxEK9#2AK*c_vYift5Xh&}2ePtf%KdOR^4O7th*&0F6a30JW$T|NMDteLw-s zlG)--rDd?3=q{F;M*cTC2#i9IxA zwWq%Fdj})>AV&!b`*c1C*?x7au@AkbBV1bqVGGD4`XMxKy-%?_8jbGx;Lbaa#hCcN z;iWnGQ!~CcQGCOXsd-@r(B%uL@Bublo9lKZ(YA^jw9k?S$me)x3wxUv+%6hgRRkfJ z1Wt&hi%vJ4*WM%YyrHX9gRrQV$k7E^s42`^o&zm5PQMwD++v7E-?ienFgPO@h!5p_ z^m8<8E?@c*?vPI1D*+b^@0B(b{x3~s+CtmY@NCUB&0 z6ooB@T&qUxL4y8;YS>42<7&4!ZJsCo~cji;g#%Q&y8Y{!=w(QwpyT%iB-5GOC#%8=P`jMn?z+u%Fh3 z0}dcXFuT+x$}2+wTb~$bwQ~g@0&CxG&G^%@z42VzPYFsMW2cNwRWPWR;bL&e+hh0b zbNBoK=Tz%-cb!GS_$GG?k+WC!|Sdxw8HDIi#y`rf1lEy?lqpR_n;b~ z4&$j=+(w5d(9}!lv`&8>x;MzFY2bIH(q1Cf$N^{QWK{1TaE9r#!m=85MqCQjD)R{w z(fh)Wj==RuvUpyTmLLjba@E(HKQ(`hQfdBF9U`XltLKW@M1yX<3lu){^y=2Ssq5PG z<(TR5dzYl7WCPP~bHDOUnYk}t#)8``^4le6LSi^|Nd+Pqb5DvaSMOi`@caACde$2+ zK3l(4$x$L@M+BXfKp!c=?s^)C35aIiflg9pobOl|dZ{UZuYsH}J;&;Iwlg0p<{_~u zkFZ&}6jLl1Yv17Eo>9#Il;Bx!HB7&TGYsV|8(3QM+crVfeiGcay$4K(&|W4llxid1 zTFB-h6WCNP;E<;=cx+|v&XqKL6U=%q9Xf~S9lnr6JI#Uwx{ z$(T}XwB&Q6g`5v}7cd)M#0d?^`ju<`RRBd^UL>#}{`#O5_1noU8!RpZTn~2mZaJBi zlA@2#N@qj8L)D+9KD+s9QMBH4dxeyHE%MoS-pR^BZhP)|+P&sKGr471KeGGpw!5_I z=S}tnAIB|LFId!X83cjg{ea^90Cj-Rd%1iHYZxe2DR8QGIYXt;U&XSPy?gn)Qbp(yTsY2RORagu~lTg5mu;NXG{n#n*GQqk$M7YeIu5c)jCMS>B@?J(dW)GU;!M7;Wt2JXCh8Ok_!`NiI6Kw((mO>MvB{JBGE3sMHqS7b)B z+VbO3wB8Z)plLx(f626roE2eYz$z-XwL0=@>K_~ntjHMq?5#;_^f=Mz z<>-J3vo{C-!msk};gW=LzyAfMf#j0Oxjx6BX>=b{28hoFdan2BsDp2Bu#VS%*;cF9 zWAM)X!;r$ML7SjRa$a4xe+WdL z$JUBk+P!OZLH2{wtvJom`h zm&t#6*vg+Gt_-7RMp|NP76zIFJrCT$#l&+)$(eTyrtuZxDIzM7L%exf__}{|`A=uv zuQ9I+J$5KoFq%;W?rP4~&E%^3&cN;^@O7Z}@_~_O64ZqAIMFWWrrv$waKe^nyKcTA zyeBy2OqVG6syA$YKy+h09vYZg(nze}RDCap0IHq9i+$GOV27kiAv4rvmN6H>MM&uKgkA*$*=d5 zn!@%)xj$DMjn>l4x<;3_P#G&@a0#wLVdM>u?Biek-DDkj+pTzOALC(DY(}V_*dl%q z%dRP8H{7tAz3=RU5VfAK)E!ak&#>3I^cZxTw{hCL4w>_`H3Dnzf-EOPP&Tucxose) zpjmGNw*IN!ginCJi`D=7{+K%h>cM@Tm?Qo#EE|k-Cf3^BgB;(oi3h)3?gadn;4!qD zAN*Ot(G@d2hNgH6TQgDyb{;7z@3)AC3#{lW{J8j5#MX_t zURi~)*|ye*?gTeV6a`5`8ukZ%uL9Pj2JT!6Jq=VQON{S}_(UK4-&NHecbvtKsQs z40#HqI1HXd_n(0e=;=IdO>rMQ=qH|G$nXhcc`fxcihvu;NM(A^{7Adv;aaM}zT%=@ zsVnT%bwq#m-AUrwM>vym!`V~T<3SygEbL%oa3Q7r)`kSw8U(He-*PY-ykRc9 zZd0lqZV%P72WMwXmDpj)R?38MGy};kI_1pbY3Gk)3JadL(CmHSrmZh{dNT+hQqmAO*?nJn^;Zs8V7u3_g*`bMSUXzZVlj{|6}!ceMJ z{igHpGge_04Rj0T2Dr2#eDf76^r_KOUP+OE)932ck1-9qV2IQr9YSp7=H!et6Mub2 zbN*U>|Fv#>sJP|NqqY=Mi*nzR7w#)5 zj(b|o8^MfH{@nXUTU9LDwqdF6t?bBtA`R7nrj!N;rMXDf=(l3xqA>~nTr%92#Gg(~ z#9}F>N5zKD*PfVEnNJ$H15DWBQ=|(u|1ZwoGOVp=TN`Z)rGk}Gpm-^T6bSB4p@rgB z9D=)BAZT$YUYwxC3GSZa?ocGSySs#&?(d#`_Sxq<_c_n~m-Vb~zFwWhCS)-Y7&wne(1Yc_SiNg4&X4gJ~MnRfzg&-r!oCdX1n@w%Ny~Tay zY16XN+y?WYD#;tCalbOOUh{dkkL$R1RHYWP&WJ=C=qGKZK=ocDI*A+;;%by4kWs#x zCT^ObO^;5s_N13|v3syqVC1Gzk}da*rA|^vT@EG81&7Ug!!X0`U3Z3=%d%_4`0|?S zZGv&gror_9F0E21)d`=|gjp8QUcM#k`-Ob3d~qIlVH~(vHWX#K^Y+nl!7NaPI2UTF z0kY7guND@~3YX!J_{x3*TpOG-QE}l(I1%xzv7E1m_TbJ6d9h)I6=w_ZI1Zi;_3mg8 zE@KqBJ2VAe&>%`=5(WpAD-0jVr@?3FFtC4P0bCg~=Q`?Ftf1fy@y6<)8xmbqz4KN7b9;@`S1mTk7 zGep|Aq?Vop6)fh{_Qwmx@FaL!3!_lp?|~8gF39LVUF9aleO&cFby1(a zt|4i4B^r73y4A5FDVv+gtYxzV$u3}Ix!N@E{qA@vCMmB?aAcSNm~m*xJD0;*Q{t*r z0Wu+{Z&zP5h4+^9-=q0~f0WE4`uj7w6?U{UerF11uuOrolHY%!k|0**B;;j0H0WHk z&+bHrNe*fP7dAB?owOt=XImepp{?X2>qk`=VM=o7*W*<)2FAazycbeBSE?SD9-7r6 z!(!N6`UuVQ7t3sRFYZM z(vb6TLzX`*<4?)V$G%rE@0`Gw^LL~^bbu{xy?2I#Y)kv1*XnT=1ReLn;5|Hv^Wl=Y z5#x+5{mG5Fj}n|Xr;xfeI4=bRn~}TsryzhiJWCGx(>YV~TuM7s`Y|hxPrK{c6pYQhut1Bf}!Jbjee1bb?=X};K-4{CVCBvbU=gaE9 z)*6exbzG^-a&rj$7J&s#*nGE~$iyMnT8XuuC!6tpju|eTdIoD1KUdcYN1(hlx1zfO zu-tCu7lEsd#Z*gDLc$xmpD(Qy%jRk(l_gZ39Zu!1sa)EX z8mw2e^@iX_8;YTfO&*)FY^m1fvfE$#0=_L(&H8mbfGc->vdLwqQnEv*Y<^5TCH2Db zW=+G~xCh*LXb#>#YmyeZ1uC#a;^D{@sxmO&ykVC1S{6^k7hHv4d>@!^=sX2IJzf&T zx@p#b*xfzF=}W=sF-lyV`XZEm*H_J+3F6pXebW^Ab9Mdx7AMp)%q1h5_M38zsN@Bz zwRn>+oknhO-9j^i18{x)x08M?oC`AEQpJDZn5AvJvKJPet;P5HpH}5ROpF1Wk1H#$ zeQZrW^^u$A;){=@?}bQ=24WWgR98_ExHr>XPw^Sc0(>|WWFv^W3n42nhMz&#dSeRz ztdQ+&9Q&{(Lbwbpef>O6zOp!89tf7tKboU~aFs`%0tLcNPAA4+bpDY^1XkJ}Ql-Jk zWLjiUW#pQ_IVhIQ+XzX<^g;o=+h1mv+wnnR?@JX@i<2p@YDifELJ8l{qxv=*&|ro# zma-JI-YI@=D0dg=)Gf_QIwM9tR+uP6|wiymWrqoxud(I_1 zHW^G-Ov4c5D|T#sGSoS4!lh9G84Dn?8slL+Ji|k{^w{lDRBj%neW$r!?}_^oiw&|` z@q<~+nZaB?X_Oeym7h=d5$vibIdpb_-za;h(EVx)S`_J$OyW1jguGu5O3sZhCjG(K zk}*pny!H%Vq3<*ynYkra$(h7@^Q7~vo_$kepCk>}+`^!2A8%j?McS@i7=kUVXl3P` zH*|gRLcYI8={xK#G~a?Qoe*3CVo&R5nxdJ)%Y4Jy{Doj4=yn-e@mKF25gaPirmzzm<{MoDs3U; z1;yc%anfKJo##laHPNET^NaH&rv_Fvf%4jXzX4r(un3zNm1x{9KVjGxEeKLIG$JmU z=gJ*7F`DL+nBiWn-!AW_I6Y2gRp!&LHD2XfbSHEWl=%FR6@e|bUn&-zBoqQ}eXl)i z&zCZ1^I-n|hCBq|)zmZPu(AKPxa)aE+N`I1rRAm&UL4#WP{Yq=+qov5hIz#3#BSs& z21L>!ZH(`vne$pxL#hjyxb@wdx80m8(KHO2g$ zLElA~)v`A*KSKrT0jUFAzuW}G5on)eMUzAb!0lIUA@mWE!pHto z&Ka0ZO`YiEiSe>R6;3z%+>5M5cj1(a8Zr0AT^@g`mV-e0tDR6j#g1KCco{SZbCdaH zX)pDTA8(2gA%6CN&6;!FP6|OSPh@BGD2YVgx;1(RucaE%o`8u)7n6r3N_>g(KA z3{Ists2I=|B7PKOoYDN8^5v<0xVSm!GBqkAw^cH!aj?c@zYojVW-hT8%i6JZu5)Fo zdfZf0tGf&j4G5UYN9mMaCyhrlh@kc18n?yEDBoQXQqBUa?s?1+!K>SO>9LB$5>@!T znYe5ti$=ur6&;})IWUt7R%$>6vBwG}sB4wG?2tw1m>LtxXNV<&@s^V4iqq%$uR9V#eIYq{ z@~NlCHflCJ`b+pp26z^{^q(}+_@WKt9|xEfjOPwXoOutQhLp+45;0YFJsef9ODN~* zx1~?MrIRB&#I4*MauYggZ-y8=+RLtY+^rG7cEh^7qBna2BE@2|^AQK}Ozx zoho7}&vCCsTAkG{x(Aq_+kbO@HEba8mOH{0PKB6s9Hvoh)=CS?5>gk>)-M>J+O3)g zthm!2FQR%=v2o%axXVigQ1dRol<@ZC>)RjN_kK&ev`dnD)?b!oSj4ni!#67%8K-N$ z2^O5NHsrGw&(+v;WfF}kgm30%OA6kHkOA`byCd^!3sWIKsSg5BhyzCwamgkRx!khN zgGJITZ^HS6ZaTvv^y^^r$(HRL-8{^`G1|1JSj6hx(rq;l zS9bE)TN?DEQL>G#QT+JheZ=PI(a#wJ6?jh2E5^xUdf#FPP%Stn7@N{dS2|`Eut6OZ zc{|-IYJufS-E7(k`4gRfBYWeDj#8=dS zzN+03-(pSSNSy|+sMLnCnEmNWDgQVY{U^mND&5;`*#rqeI5#%<-$^BK+-aW_c-^H+ zxTJ>uI8ivhkEyW`o@vqD zp-ACy15>6014r-FHjBygY;p(*m(#Zq4}Zj`z~ISN1x50&UQP#Nvx-Wh<_Of2Mq6Ki zCkFNL@S*2q%)eZ(kEs~=z1}fWdRt~_T2mRfyHPQug9yHPHb@sPi+^n-a1tE1!;(0V zV^dFasAc_h%U`fYgig7|U+;Ty;u@ihg2uxDdo;d7%9?Jj3Pq)a)vn7#q=$gh2sL-5 zxY_TYNV_Tq?i3b<)p9=E9J1mz<^7Z%)=4<&W`nhqMGnvJN#t%a4TBdR$mF5oyC=I< zQHhJ%96Wkxyyn$(Ck!PqNoR#`Jk^nX94F)0@VrFdL?fap+9 z2wQvLCDCdH94Q3sGRtZqC2^Rj&>>f+;-`^g>8B-D4yUqw3xe{Is zaf=`}i6(QuQdQjgFSIpkr&D=`R{KW3!fb8^x3R(dMstL~0lRsZMv9#p)>N)l|4&`d z2fdJKP*g1olkZ?OJ1k}iTR8Jp7|~lhTLk zcZ(6P@C$w<&<}7qgtH|u>#l1mtNxJWNn&&7;7B_CJT)+Vz)R*yd`$D?pAY!om=kr- zq56bFgSIAK`%N}Kk(x1PrA2_mdNB#jH=|W*fXqfw%Z|B9txc*SDfON(_IQ*1v(o(^ z6FglW7eJDkvFcN~&r)v;{Im$`Tg@Kni(Rf55V<8m7fhRUSepzy<- zAxiZ7L(@Zp6(5v!mp@&1>R`X~-duRd;mu%fI3hq;hHPgtFJ^j3vLtx60uq$Pq9X6m z|FkK)!4-v1N~bXERP=QZrGNS8d~irwd-v25AvCvcE!Sh)|L3{$eW4&NT9MAC27zg{ zO2m$88vk-&arn+>D@MQldatyD(G0~*B|74w#Vk}@2xX)NFQEYjV&nzsYV>ZvXYc+r z+7jubpmfu=lsdcXVH=%(l5*KD@s^y_$xywu#|-)9)+zY-Qiz(LKD+M|^?$1R`paL* z{Nm5dpvQEs5hf-O|(Ok!ow{n)Pd13AxBZmu*~$ zMu&DfG)|^-feD9i?QQjtl^A?p6P$4m>)K)|Z`7Xf8qjX5HC3$rE#g{tFmCyLclO%x z__0Kn&+3Tn&5vRViO5Ttz^!@DSGVC)PjwuX#a2UArqz!b0mcTMH8Xk!FO1#@tsyv7 zD`@z-)GXXaC3-wcbauCOCvSJB+}2*s($vuYkgB9L2oRuMM~TjgSy|H8n#^3?{)vh? zF=^3$A7)Hw$ZB;xUM#K4_1*H?_l1P#tH%;|G+*ELf)Vk;6IuqZV^qki5L3IgI+0eX zeAlKeoTjZ?s$v=4P4J2)>+n(8oW*_dwDNao4%!i?nax#_m3&s`H`l1z=wD^z z)-1|ZbrQ-0eT#oy$o^8-_IG=!{KIzlB|CTm*PuzM(8|95Qn)R;Go93V`2BR5`b;!y z>w9pPwqNI`%aH$fL;WYBQy%_EHa^6fhA?28afnm?H`cAg2W5c35}I1pjdiDBk%d&s z6|p@9fp=CHHXbY>!>!KT?_%$_PmhX%t0K1bBNS&CPqKGdjns3lIMuiUB^9_Fm&c%1 zQi0*D`J#3N_-=A~1Hzu?8eQozW5H!~^!(yzS&UjBUwSx-ma;#w=KG(Z57dvmd?O97 zo(F^gZ8!Q07TAUu)EEInebk)i;?MFYh1M8HW#JM*EDXL=*=E8^qEwhastRX|Hh`lV*}sOrNlB`-cDM!0DO0PcOTDwtKNyTsx}1}AJA%fgt_f^YVXB; zV%d%<8MG7ua?Ace4*UoosPy?jhvm?a@s5iiUCV|Q%*2kC_>put$9 z!PQw2B!=O=OEA}qpV!H`*1a0(Uej^pkf9w*cPOcB2#P4ACAKo0%G}TUAw686e1Rg{ z;IVn%OeF20dT<0-aarPw55nbF>YfVuF8igvVqE$UEr6vBS;R~Ra`7$OQmfyyqC-=$ zPNQ#g!*PhLdlBsRk~zCRk#K22mnW$?(#r1=qlL?6*u6R1+Mf-W zc&uFKFXK<6;y5h#`J3-Z*U?f2E7^;-&(>js`d%wqC%rEm-@-4it%9R5Aiq$KWX(KE z-L_m@b~iYmKA{xQ_1y2uwMcx@q)1kJI9n?T2LjYAisHF1>eefL!S#{5%KL0{D=T|c zAEk{<95kO5V-KA#OKrgXkmtOspTV(J!#dC9SLbYWayksaGZeeBoLv%It$#5EXHp*$ z_h2W?laa%DMU?Dqb2g<&!;UtIHfY5@f1HE^n7=56f^{w%3~38!Fjj|r?pwIJQloS! zt1kuIlgF3D({4|D6J1SU8r21?;-Ui6JJNC;vZR^!gJf{G`weJXNcsaQm$-TYy>dsL zPTki}_7TS%GUY228jMU&?&>uzpL|~JVNKrs=N zFL*eUR~cOHw%!FajelPx)0@24xu!wP@mMeCp@Xtn*ej49QULneYBHGq;a4kq{9V0N zh0ONLidjz06Cze#jo$YvouARH5I*}Fcg3%|x;2vW#yTy_IGEQ>g09!1zcJ?;wM&T0 zy^~C0BZjhhRt1+*K77A+=-VqYsop3OIT-8uBd+UpUw?co&;9UMrnyN3En+h5wC;t< zv_sAu>ZRnIrAN$`OHdfGd^||j`GxownLV;&H5%qojixFflO!b0qTXnN=1!$yqMGSM z9*?Z{SfQ>-`I^OZ>AsUaB*`@mqEZ=?DUu4n05PxLiJRC{qh8bzug~Aa7=t{i0I+X0 z6s1A+Z);xpJ|6d%r)6<3ZvwYrKSphCl?m(#&%~W-!5GtyFHyqtY%Vd3LL7PqxH6q= z&YwMZr(1{^H1pYRZNemk^x|4LgypUS29xR)<|;MunC74I^A%Xe0mPsNqL)d5aIJoIw{q&{cdtw&6Y zV%&cT^JYi5?GFLVx}}}&t=$xsON9Wd~NYGckgpOe%az`^%g1D z7TR&!w{f7TPOQ#cOyi?oERyP+sHBn&;){S^;8Dv*1E3FUn+Q^o$dj`R_h(Xa+H0Cc z5h^mobhkp#hvSd}wiAx01$QD4dx5(XI`4RkwQ$jLKK%pz z?I>o5$-Cpw6mTp7RWG>Ohj%PK$4M!dfwL89y6{V`#0mOzL1h7`kL>EOr0Vdl&04*Qf0wu!Ye13h15{WkTC70x)!9zlNG|G%52kVLt@GVy(8b% zscY%*NlDk^@G8OEqC*MsWuFJx6fUK;$B>_tm`k=BbCho{CRMRPs&co+DXyRVBJ)Ct zm`rakDPJ?k(aMtdl5iJW^Wb(oiddnW#eYQz8fyOrfj9Bo+`>TP?5uiSRlI@t#A*VV z3xU*LPn?mV@ee?u*R2%B!MZv0#xs{OK`^<_daS8JvSpD$zcZ}3zAL@pNLiLd4m4S= zeZDHab_v}0AuDp{9kM3_G2D&QW1=?Wz6tiY6t1eXq>yrr+E;0nCpbj}Xv>;OAWtC^ zL809bJ4~R&EV-o|9)A;hyy{8CuPB@|a9yOU#oF6gu_5j zTF|a&F+Vrz#q=*}5^w9TdB9!nqM>&T$D`!OpS0R4Xcz*pQiUu9VBLUhbaq$Ch!9F= z(ur5bAsp1eo|JURTP3X!k_hFCNsG^NBrz3g*hHIr*9lmI`fq?|wC(Ks|h z5VQ6_G+6ib91Z8oNBYA?TJ_k0Slx@|Op8c7Ps$szw_yc`j)NG90#U5WZu&$CwJ*%~ zF3#m@kQ~=Qk(*JBa+;oG zY|M@iM%$`E;WjtayFRw}Vt31Au>QJ9)}K|UQ{L)1fT)x|#X&#o>9ulFZAkruQ@PMC z!A5Lzl4!ERj)XE?U4dKN3h5b;?a)#oL+e@2IJr*6KN2u6hLTRl4AT4Rl7Y{ngTpUD zNk@)&`4n%@>1BrP8&aHR;puxM6`k7gk)6k08)YZ9@y96V#;Tg%S2#h;D-+&Y3AmA& z<}_EFqL<2zF@>G<#qyGC)o!aP){Z{t?KTpRZw9mC)(i!N)mpOjqG1d>mgaeO_CqH` zUSRWWxbPy=jL{|qXEutAeuSy-!;BV+*MHq3H3O}MW}{gh_Vh18EuH6rEK?ogfJ^cu zNsCv<9;dYPBmWod8Tx!F^fA|MC+kW~zJ09xp3j~=?!j7gI@@(1zQlAt^|ued8COEP zYt1wUBlji(EP>8HI(VEr>&QVFpXX$~3y5l2}8#9MYY-V>h=MZ!D5q&w$Jc2}*e{8>*$s>YR4SYCJ^+ zgGm7;*5Sa`6TW0aQwq6k60r@)g2C{L+ z8@T`eZgN>+JrIh8N)1oUG`buEN~lmV#)*AAy- z$MX%&p)@jyQOm0pq!|0B6zNS-I{|B&Cu}w0r6ab#)y-`k{IX(LN^}~B){j-xy5x~7 zK^wn9i5`rY;U~X}Pq0-fNsjlq%VqNuzkwQp;bx5c!|9D5Z^qwfGklN3VsWn+pK6ax ztqJwG9|A;(EyS-Q?vhq*$I6o$lEviGe5y$^r#NWMQ_=+YHXw5E(`ASw~yiDUH{ z!?&o%cnLkYzkIHJd)fj!y!e|dnEqA6TCCK8^(|JU^C7XDJqZDz%-(8K(Su$e+# zyEpFjp^w8_-U^9CU8^UggLeR)I3OBhjE;FPT>EUC)w3c@C^K=h{f-r|krSnKru$A0 zxplAG!@iE%BR(U~qvoE6Q(ogRKPJ%)NU)W`CP(}{*d&RTOIsv{(MB)U*~b`5X^EJr zne}oWa@Z=S%aeFU?@zZt2h$rnrp2unqp>}FJ=i3!3>iozl4#`r3F!UyPD9}h*TZU$ zpNRY)O3B|-J|Ah8HjIc};xom~FX-6*n~(7bFPQ0x`f@}F+w=lmjk(rmHxz4OXtsK% zOB_4>otH@jb+_lX>n|BA6T=OWsek_2e*-%{FDcTZoE3`ov7DysS09yYcy2e5g7GXq z&Bgp4Pu^j^UwYz0##2WRy#W>Nj=i|Hmx;!N_?9dojW#Q{Hg(@%Uf98tjZdHy21JSJW>^ zllb4FJdyug@_&7L{&NhdEAjkw=g54iZU5z}sD=1f7SR8EP#*agP4YArr=$Jf{^VaS zX7}|wZ9fmX#KJ$gM*ni+fAjwal#i2v__uvwN;3b8S^k$PqYjp$Py@Gr9?^vNKQ7F_ z|3u`Uf72oK7oNU|BhLB%v*f?2Py=6mMF#o*dN}^4#R&F9H>;4@V_=tJ#rxl^(tlmq z?_#KdcTw@2i2rjg|MM*W{miS|e?SImCb0(Z|8p(={agRznIP1_!^G^?26=tLNo&eq z=IoaF+{0O)Xt3DQNnh?jt@Plf*o|8P+Yi@j?*&m13PozKYxYWu_rqhDQs(1+Xco_3 zm}wUhDrro3(A0p#Oo+~7`sY0rcUR;sD`B>(a0qJwawkt{$HPBne3Tjx;NhDhN*W<2 z#@0b`f62i4+F^NPv34Kgimp>_eIBA`w{m4p?Xub>LkLFYfZ6x9{GZo-S~^*W?d>ID z?p+k~+n^I!>&x7E@xe5TdHa)cp6@PL>s94Y7rt+Cyk!!52-##_gIo;gHva-OnowKL zWK%OlHF7X$ta+BYv+l2~g|HVH-hztkTBUvs3preUmXS6nl8#hK*XLZQa0`qa8ld2F zl9|}XMR<^;_Oh(hkVt^wUSipJC$!q)L1Hot?IJP;v$vTd=HYagZ90oZJFYUl?oN4P zdF0|Qnsi?OeiJ-ij2IE?(i-8LJn;rKb)cABzj5}2tvWb@;8m!0L;Y5F({zD4b@HeZ z)1DeDq~Ed`7j=;QtL-HI{#)aU8FC#2!0v48Ma@cW^&X3NdA-*>?JWgCLV3=>To3H0 zHQze=P&)kM8ivqMe*+u}0s7hdVrZ@Ci>~O!X(exR#qu5h?ku`S zUoRE@NPV1<-qoW0VYw$LWsJdguDS|%|RC-v-S|vv1z1M?Jq9@y%*eJ@qjp zt!)r)>$VQLt(m_w^s>G{vG1%NkYamX=kWH#pCi64_N%lEI=}e5K`8^s>2m0NlQVR_ z@lxK*XZ9@IRJ_dX>L^o|vb;M9b#u;vR_9ohsr5~R{gj5a!Zn9Dbi_VgseF&M!l@4z zngL^VHs=_r8C7#TcERSxZ$D;@9(#RMa5HEZtlfbMLrhh=4bc+vzh=qthy6HHAp7@? zyoDE>x0#}i?{oCZY_WhQ^wQNsHPfK?=(8T)PE|qWw5cFQWxO>>`D4EZ_xiV-2hug) zOM7f?^_7|38|>uX=`?T@xgSRC4TmOZ)W4)5d-_JJZy>yh@5$#h-`7{!r<+)m(*RT; z+$WLQ^r_82`X2iw>j2KLQ02}`sdB-_%gQ%&G38U^Ew4AU-Fz?AC?x|wzwr9u|NIW{ z{3?e1A!I*b;2I=&F>NT0 zfvPv98VXGS4s^-A?fdyb(rPZ$9FX|V@n8UQaW*brUQJ3=p$77#>-(wx%Z>nVs&y5r zQDHRvMWbnyVAOkCgJ+~pImp{7jM{mO2r(z#eVaMm^-+8K9B`N)5fsW3ApLK4O2i32 zqnB+|=gAxRyC^zGzKi{>Ppp9$w2nNfBY1{vcxd=z<&Lnj9Qxci4W3gHuHZLzRd~VX z=iIX*=pj5QDMvnbBT;FelwLX4qGnER`6C47j6C4@%zN(Ole3{7gZQh>hTuMdXa%Vf5Kg&z%PdDva*Kj@!ewWdKieUHB(=?sezIs&?uRk@lY?c z`tgRbC#po&qtU-V;lW^gta(T(mcH%v2iG}(RnepQ8fO6Fx|;%gneMF~7-n0Bg6Qi{ zo{;YJbKA}iUF`GzrntLh?*GY2-5<@)?tV~j>N)n9bCv)4S%6N;x(3KQ%4F?yT@7?* zZ^=Cv>ZQ|qa4Zlc}Vc8Z~=K(}9b%6_}e3Mga?hroA%Q1N5cS0E)tv#eswv!Rv3L zY4Q73&(z2xaNnXRmnOvQ&6yZ@XLq`wz($vSQw>j7w9=f<-WjZr6X79Z3V5gSa%EDe~M~+(!C4DAOy&*$@x9IdKQ1* zg#AmW{rimHvxkb`z-ifU{oGTjSq{H4Z1Iq7;Qz2^6E%UMDPGoB(7 z!axk>BUO;1+R90VQn+$bR-pdnoTSe4KY2Xs*)@(m1>vN%FL@K#j2);8+C zCb$YBH}%GhM$Cw;)^;B69(`vUlR4MmsVk_+s8R-Af)$HGO8JNlwOh`JZawc!z*k4C z7dw{Br(h(ZbZ_;II&qccS^bunt_Ld(gX$eQ^Mw&o; z(`|eP z9xuMY2k-i(*3gXmWW}`1*}!+WxU;#{g)z7-?pv=|M(VmEQa&XvbAh|l_#n#TR<$Xt z;&pE8d}5?Q8tu1eVUFXbt~z1PL#M~{;zC32bmy3N*ECR)fN3|=gT2nS649KWpw&=4 zL-o?{R{`X_`WJ@IuI;6HTz-DFbRBGnVvNH z8z80p2K(Johur)(gg?AIn~5RmxdxZ3kRPw(PLV6vkv_<{4!pB*Qw{y1%XB}muIYw? z@<9U9d60YK7TJ8SX2tdN$vW*WT(3|gU0(MI*(u2EpCdj)`ug-!O7eFf5cRdQ;N}oWEMJ8le4-VBS2a@mIKbg}Ar_S;A zmY6ri0g&BwS{qYZg&AF-1zM`!Gxa?^H0wKZ`Reje->+JHHJ``y6&`i&iqq(Jp8S?_ zRj_xty?X|PEQ*O{&hQ0)c?-J~le>N4Z^{j-b2ez>MSgPIoA~6|Fv-eD`#Wi|zt`)@ zaDs7vf*$lL@r_oNv|+qU>yB%gsR1!lgiSw?(~7@H6k>zp!vF15bo1Ze)1S$G0FZB2 znrBWO%y_~p&&UU^XhN=*E&05LCGJ@K9_ex_9qCFT-Kka*k|~ef;~qe(@T$8Y=HD9E z%={0_FjFcd6(})gv$RiHEy5;<=%b=dcfe7J;1%M0G0Ij#C@kh(z%a8Z(f}7Xg?EEs zkzI29xr?&NpvT8y6?l7B*tr878HsB47%y-rbZ6jiEAZTKeFGch^@^*lNGY#eU-%@5 z(qAWri;@_uXTQF19(1l%g?~BWv~=ee*&xv51o$~~O`1T_2A?4>+2ydduG6Wosm>v- zYjoZH5ex^%sJ0DWnU6P(Io^$J2lc>I$f#*$jvQbW*V%`UD22BxHoWULuc(-X5x57{ zSi)a+WN zOcgNj*YAd-%E=toK1K%-?(+)*6G(9 zUan^==6+O0cz3zIiBn}U`Y`?EVQ-|f&$8FlC>ri4#-NSe{p1#FAV2Kvceo~HPME?v zP2bjT`6{qof?vd4p)DO64uwvY(Zp%?9=4}E`_cwC~Q&(3By=d>w zV)VUFN-IIvHBz^sWE8~H$FQwsX{btf^6cf(d@}3uE?16FmTRktfyLZekHP8(qNhYWE zW6Z+BdlRed{kN_{(Z@RzvNN;_heo&JO%A|)+?CmEB+5oUdn#p{m2U z!9>I0ujw*Sx#nn(0XF4`e&@O@%A33XQegjULzh!_AEQEw;u|a&s_C++tG{0hhD;Rm zd6JSuZs9g-y7!s6ZTemFmD@#77C`p(#g?>+@L$tzpErMr8N6sg2D!SE^b1cGPuUf4 z{N2p-%+aInLM>1)_p~Ue%&1urLDOeE5K9-o?%QC-Dc)778~NDJpYoh{c;`MU#!HkN z#9-69H(;&7W^V{AQhqjgE=xWxx{-AHm_M6f6?EpUT$RT z+4RBg0GqI$Ylg&rUtY#UgfB0XLrZ^UwnpH%YwRGZ+-JI*@|;k^Y9<%2oqm&ABBGLb z+{96qsZj$;NLiV_DRcBmq4du);67<-xu!GBVUWIex)jsf5OagcgtB^t|B)Wk6%>{||2qx&Y)E#yYK{G?etuFDF0Q4LgbpmzV1|8RU)n(I0!-Txxg zE!^U{^&|CTHqRX(*A3!OBc;w>{lO>whhA4$S`Td)^!;4TZT?gD`*XD!l#=1h+F@R0 zL5#P__E?_&e!XYz!1j#4Wc!RjT6z-uMhL-N@TISep0qd5;}_8vo_n#EC)d2uJyt1K zqC=&f7`YNkS#(fD1O|<4dVUv3dgiuC&k~+5w(9NW&S|;2aYLFVDe<(++uLku-4ewQ zxKnIsGwtg6<~6H}=1G!5h7z)`&sSp7C|?w#S9)pm#4~F2{tbF*y&mu_=c;P#eoWd= zGH??z6tW!?Byn0acdJDTb;mPlbhc3NNnsG3;8{Oib~*SHFSx|u4UkQt>btL`@Y9ei zv43unhnB>iH(Y6P$oYC*qryosP-V#b-Z@To1%l$u5&o$AT$V>rOcIQHoVTJ$n#@rOsM0eE9&s8!9dG@G{RvYxE7!S0O2zDLB9B+HS-Rc? zOdgQ_;aTYLUM9OdLMMO z&B_z369;UOXIB^Ks6{i%3Y6Dlw#TL@t`|1n>jUJS6EVDFh%aF5tN%J$t5N@hrgP^4 z;^EA%YSf}zK&92-kekr;I$Ks^qo)bwa28{NZ{Fmk7|klmD|kP{*{0#>3cFf#Z56HA zmQv}fcu62_zpp?vlc#eV;>KchWXGka;6m&_T>f=b^cS3E@w<3Jw1CINq)gHnY3-=Q zFYI4Je{9wLXyi_?v^sYQj9gLlv*5Tb$l`!LOo+sX9)Lod49RERZkqVcVUCPy*E4h_ zj5~RhrXgU3QWFL)!wYn=c&LsSg2i@Aq^Lq!BF+JXh-<0BJ53p}f3kxP0xOm&xOJFLsX3{!)X z_|*iUh;6}KC#q>mzpRAr@@XjBjg~d-kb2ZD#0)zeMoJK*pMA1TqjRebLR2&5Y+del zQlWnP%&-T_0iLZ&^^y-HipV;cR?cmeTUb)9O}WLniq%<~{YElrN&V|)vByU{POzk@rr33geor4U3I%+;nCv z|0=bb%6vL?kQ66)&hclI-9M_P~!+FQ#R*sQ44d1w#n5zl?RxD5Yp&^(%Rv2bqQ6i&WD#f(Oww;ev2+k9{+Up0ecWWPU~34LM^}e zPcQNCE06%`a>J)pO;OBE6pq>9=-f|BDQVh;NAf$Fd;ddM&q)3cRv^cN=;}U-4i~&B z++H4z-+1qjIM$?LoLfy-*vXhmjJND$v&8n>c$+AGr}LbHjxWvyoF0Ow#$z0hugW(( zxNH-&Bo_E@83Ug!YE*%6imaxCXP`R|N59?|9MuQpVdF7~L9%QNRN@*v`>m|vQ;Adi z&g&vtRW*JNi%HOxq8eEr0r7K8hMftOF7+9z>ardc=<{KVFENvSRGfQ%%g3ccwL61c z<#l)g2cFZ_D1Q?|!=ozQu2Wv2(%crsU{lN?u3nrQ!w^_B?R0pkf&J9WA*Vjs>vYGa z{x!~an-rPdjQ38kA}tZJ#VVUX2H~l#F=_YcMss zj`HH+Sh})As@an6WK;Xo1{2cN5ld;>;el0FbCoF96DY*#0cr7Q=cE2dJ-q&MOCdx= z$Nsr2c`#xC9|F`U53I{;kWhE%BwLFLBbBHUOFFRa^4xT`{VNwRYt*cOs3pJl=W*J# z4cIyV(quC88^q`y7fw$6@zmh2B~r}ta?Cmu)80ZDX%x(E8S!_GE?zxB8a19J z*vvODAYLc-36bbOKyH6f0gyaR^0BRfI*9NpC~<$>*iNAfs;8pW%{+_@FNjIGmZSe& z_*)bRr{srog+tCciy8_q#b@}(`Q2QtJ}rLdPqo&(+x=A3QEH9%g?rp$>$!-?g7a?e z#iB0GkLe-~;_>P8S+T}oj{dC(L1&alni4HfXF0ISa&4urCKf#OR#i>ziUD`t6wjY} zoieSrs4dPgJblC6Qg)&kS|O_^V?rUlQ&u6BRTXpH&7NM>u9#42Gyj>aVj=M~SIhQQ ze5`jA|Au`%(`BwT|BtFqXjy&en#wG++n19p%gihX4&3m;;Hkp{T zmb9aS-8nwzk;0Zs)^dRsdtW4zgzWO)q=lfMwO@zs&xX3{ivQckXl;FNw&E=7&ng0{_A%uDic)@Of8wDTZkcR+2OAA;>E(Y@8H#S< zD2qwmjlIk;Czz~kG@?!(rXt9acinPmGH&`CAWG_Iu^3s}(j{@(VmW8#D5W zi@kqe)nc^_3v?O;m()MW{aX!ghz&mc!SN-ud64j5s=@0<^K^-#r|zCQNbs$h=G4;? zB+fTpgk+7wtvxt7WlD#IFZ9RDPoEilA+JjjuisjoxE+)vyP&&j`>LA24C`WO_VUG3}{Fyo*q~dNycX}ZHrhbaGY;& zc?A;>C%s#Xp&*~=j~ZGmRScn#^wl|dFWcE?+}Rm|pGVDjpHi%a>M49(r6wU(`535_ zD>antIU+KLDEF-8DfBL$<+z~tKBpKecR-Z~Z#ETW`n<$&o}Q~I<13b`ez}6AI#lhpTc}F0D#%RWsA7$1#O@IqwjjOy#4)Pe{D-?1_u;Eu>IG;Tuc0$(R0wNGEHedp-WUHn7=X|iuu zHQ#rxqIPT&n67)C!^E?#Z1thH|AlID9z;+{{Fj_a%xf9vg$)Y#J=S(lRN?>NgT#Q$bgK<%#>|jjx8Vwj5)R7q<1qf=}!=L0HXAr^1Od8xAa!qw5A9V z^LjZY+x&;&Nxgt&IylZo2Eb(d*}$v${1>tIvxCqWhj{xs`w8N9zBv31r@#}S0S?1- z^>~9hXZ(ad0Z>Qv()pSX6^z;k8JKd){aH`@hzB0qm?a@BHQF53;rNMWZSppz%6w{~ z%<*)syBAMW`o65qD1_i4NYDVmEw~&sSg?b;yF+k?5Zv9} z-QC^Y5AN=M@ZR}XRae!o>g%WP`lheW!v(hX-gB=t=NRw6Zk0_a^tEE!_=32ZiVSkR zmvwqUEPY*PdV6y*$p%!AuB^1EU(K51Zacp^J0$I?OYHi7y4>P_Ue{Wh%@+Zz>auN* zf3_JfLyuD{j5iJJ+VV`Xv>;E=))cVf%{vfA9e;QT- zRGGSbU$x;XhM%8At6EwQQ4qLyuzvS4Q{-+4g*paG?lp8C_RLVtd2{I5IMS%Hel9G@ z*uZMDc0K$gjzzAS@om-l<6@(ec)l^xOq;lHxh_ko!HOBfr8%8eyI|tADQlptljOpt ze`63F8ibTwE{c8=hnsK9KY#c3WcjusPbHb>C2ztWKZMAje*6Z%j zSX0{Tf?YChR59oA#yoX^LAg@@$_g3YjNs4l@L6V!(1@sp#h;!$j+L4z#)J_~P6&2; zGx}6z50wDQa?_P=mEm(COdoA>)7MgIjd;ggFw40$mK7n{VKT@jb!JC3H6euhHpSxz zwXip(pUfuSv-G@!e{fGx^D?FcVt@p*-(tHTIQNdbi9z^X`Z=bk`mf16qi+&y5=o=C zf|iecTu~HJ)=ov(4I|zrhP%ywEhHF9js;;NkqR(RW7ymo$FI_`Y|x)*;;Jg2l^W_y z$qtRc+!MknP~s=U;b?HO#eBVl$`|1#hVip>s|~&22-+4yvvi@TG|5pm_oHu73}?Wo zeS?I1DZ3(`JdvVG*tbX<3UZKfz&RVBgD@bg(hr>X;umn_u?~fEf%tw}oYxaqeRFgpF4b2roT5dwtT%G-V78G>+kf_mx9r{qu#xQ3*pq*q z#bHvpEYp8V_0cL%DTx_KB@O~oHvQjI?DSAgYM*e<_uTYfa6l$3c-HNkpC{2z&L#8h z#<8X@-YZ*Ekx~c#iCBzy7*hT2vTV5VYX~Sr_Gw97%Sc{Uxex#BYGjVT1vb?vbLT8ybd8#%vDFunIwQ9j}=Dv$@^B^PYR71F9PZXaucZ z0PN2O`3t_s2AmW26?DY^8G)*n%29CIl^Ps)&IB;NfXj6sq4@TNprW|;12-r4lVZb9A4&nWA8mTYeeWMLi^g@r&){D5e*MS#5y4kukaehXAi->#R#f%K#xzi!(TPc!J z5RXj&>F3w*UDnrF2-7D z;?F!4s^EhkDt^@%Gn;3m9)@!1yV?Vr;ZX)x)-e0Na*F}DyfGs1#Zk4zeb5jsmB;hj z`NIo471p;o9VIS}OBu{M`-KT+!~Fqfy#W45hn+MI`3Mv&89UIKJBrmM(-PiUvUTTu zX*}ny^Qob5eS#(-@SG;N%}SCX$PjJ0aV?54N)W)R?{V9(t!oVmwTms&N2)zujXk-< zuCG*;dk}5R->z_(@*p#%RUF@-S@fC4tp5u2KmnYO<)7w$1;DMzl8tR)i*Y4>u|kq9 zur{``y(ddkK5ngo&l|Sx96YhHsb9>HeclTD$GP{l!v{x?gkNOhA4yz3-T zk%~23?6tRc<$_w{{*MNF$9A!6De=}fsU_Ng4O{!mFkn9Qb~6Z>IobJrc2S_cnP_%* zpnSA;K62zoK@7!^vxGBT=6+6=w4XP+(K&Qp+6&OhL{J$0I!v5b>+k&Q3`*LXnD48P z1vxnAOB1SK#B``=Xal3FSg!qAm{vB!deNem__x=WA5I_Voppd#Hirqb`1EN=;DOGy zhFoe}hS6BPaHAdMS89ZWeCDNZ(#4 zgQGh{O|Q5-p4^nK_yJEmWePjD!Emw$l*@L!j%KYFK)m-n0SeAQUAaegJY9$g&OH@Y z^=ms2@BW<05WmxytF4SfMT|ni1&IKtOF`Tko;2UYV<(|MfJ2$MI$AqOh$?~bcuQ2n zW?^`4UC$(R2pDkYs@9qB-)u`SmlZB8AN5WD((P=BssQF6c0=P+(RG~C!h{M60m^5h9 zDYwFyP$a$H?&PUeKVqCM$l(=DqAf!$vJjTVz_cwYw#ie7%QTI*{({}%pDRN?J z|C*H+Uu=$n9fH3x_Z9DMt^oe-IWkCz>hoE}&;WB^`%jv02CzviP!mC>CM#<^^mAXE z$4v!1@d%;<$ic^TZ7R}+_h_}7ET+iQ+r^ioU|&3>gl!C}1ffg%rQ>m20`>nUF9t+? zC^dhDhSL|JlMsugdSln6Q=W2AnyvJ2lAn7M)_;x>&t4CvDL3epjFtSg*{i5bW}h#Q z>OVC~aJ$c*nR{pfqL8^=jqmVx9hPvbCW012U!T?*CCdnlO?SuUw}%wx0f&7LjgiV8 z;IM~=)6~$L%{TKqS8+bOZN}!b%rhvangJ+?!0R2=AIsIrS`q9;yI#3rSzj@f*h6SQ zTYl$y1F5>`t5hRo>=HWA-^f`fjCkMJ$`*%Wi$BdBgK9r(7t#sNmfl219K7Cd?{kbN zOLLYn>$n0AC&dn{w4D}D*pkYAi3WMk$FT`Ni+j_SREiyL`NY-H?*Pa0&4gd7yb^YY zPY(rYs(%04d@tOB-!RNItZ^SaF>A0ZVWZBSN*U(*2b}G?NJ|$O0IXa#Xl~WDqgi{D zdOTaa!IxUBCv)OovgW?48kB%I34;`ltgw?SatWUGFKJ}CIG(k5c3bR&UwGHqq7b_~ zde|bEIiI3Bmk~^VluZcW0K72>>ec%c2a-k#V_MxR=0lq8RUAyMXiUFf?oeyn7}9;( zWj>HY6}}_u(r7aAs=na(YX#6&Wd@Nhi63Wo1SI{+`_CsPn13I5Vvdp*BN**FD-MMm zj*MB=_Q7?@IXoVHyQyMC7JO8GxItod5cgR#+g{+M67Y*BDBsDp%Hp70W=iW;EzXkM zZi>@Mwr=I!P!z30zhtnKzwetPtnwpeZi!R2|H5o*-;L{lPB1 zTf!Qfcl~>w!Zl1c9T-0NGJK#+=hkgQUi}pjpKXcC$VmAvxh;zx=Pvm}NCTC`sZ6f; zy|S6U`ORjpUHvBFEwpchLn+Tf-qV(LH4siUxO18%#Mme2!Z6lhkY-cvHi1ozVMszj zXeT=AR~j4`4AEy_41C zRd2A4Vb0%FmJf@Wb^Kz;LX+g$Jsqg$Ujb>vY*TWGbwS3`cBF^S&Ac@8s^(KSAPbo|i7HjT6BvU{# zXlQ##;oMy1q_B<}5E-6rX+z?OTOeyBKL9>}4?!_h9N+_py>w8=(WsATxX@z`rS9X= z-w8DeL*r7YD;8io=}lU@hMuNl3gJ_91k7Y3_>#J zkxzl)bWm{+i>4&nTD&S?oTr=G4J;WO#@4xLPLQovZZ%`ZVQ}=`Q`Of=N8;vBbfwl^ zcvDOIUFf}R0t;){I=Iea(RwmYSih~X(qS^J0BA(RMl#~eTI$PFG69(VnpkQ3&f8rf z!=rt=VBQ<>AiI>~{!|{b#n{@hdZ@!0!t^HEsv%s;zh0qBV?;2o+&2uCr-4Rg$4LXN zCYc?Yt~%DGfW)#h8ntBOk{D(gkYwVE@H|Vk7e&T7<`?hy1l>GuIcczqNHtCLspX2N zgJuWZ*~DS-H4)0n=89T~<66qJrO=;rK#m^Hr{b5eC;X$+_GaHhuUSuQFWmVgg4%5U zZw*ozNT^d`SeFa|jSq2S%q(Vb7H!Oefa$nf6tl1#g6i=xVbbY6V34^f=lb<%_s3))I6(Mfe*UGa7hYj2Q*AEO*mI76wh;1PB9~)B7d}0*3gUrnM%*0nrMbm|ZSW`reSM9Mrg>wu`*T^4F?;KB+%_24{A0JJoO#Sd5^`H(-1F+zcOtMuw!h9IKH3 zhkKZ;uQ?%$ZAf!GYXGKbU&HNKf5>Q?lF|qw62pW>C39uY;x_9z6u|r8)#9h?qp5`J z{X3ryQj*oT_*U&-Qv-OM+wwoNnmrS`p9wl&7|`{uY|jCzNN;Thu*O$MHPRwNG*2cS>H4M>@^~?a+QHjll?Jsr##v4nwJ>>=YRz7+b98Ud&r{C1!I27 z@*wlo&tvxSYq7u!^m#ItQ7zAf>Qal4ne8VPHxq zKtx7~FzMhfIMaQ#(XB7sLrFw6J>3w*^6#}|xiKsGHVC~*rk03$9*IyDpKkJC` z&KP_!OIw<$BH&hN2pdCE)5X;26+_et&nwFtrC};e;d0Tq^2tvsOpftE~tT}He9fSUWRxo4nV<7-GrLATeu=?C-I$PN~hDE49sAo2KHSwq` ztv(wj`StOBXrg!xbT<#jT_7=7OxO0T7Thq+q?{uBUbe`QzXsl+(QYv|s+E)SK|ZD~ z%3b{0a+fna`OCONd^j0ZeQfUenFs#qtDAnof2)6 z{x!lO*pi+bT5iWW3n-dnqa@#HjMX%eh$i><6^&xe(y*hR7@{&qS2%H zjc!$AGRLMAO}I=qL{?*emz@$Xr3q+d5EdEv4ue%0>;|vsUClH!%*d$wxL~1|uX^tk z#$X!lUMJlCtNe7(iUZ>x?}DKY3y!tX0j&L}19a#{D!iFgo238&QQBO4C5{KUM0tiW zR&I*k=%?h(?R8rZ2!=Cgz|f>R&1~F%I;ZSlsm5m6QFl@aGQ#lzaq1wx)}reG0WGaK z7_cF1E8He12^*9swF*(Abl6PaEK?O{vK}qz>Ay4KfHMR!2@1yjvoj<>|9X-dt*1SU zqw(3x`7BOPqFWf-^|5d03~&te?$D(UMqbbaX>D{n4U;LO<%Q_k9VlWFtfk*$MQMNc z)B)W2)VIvNA6wYr^ZZ39q2WOR4Dry3H6n~T12 z^NuF+1jH06iPi(M880Yr7$!Y@+++8kppk~-QAHmlbDO6sGAHPxLwx(6f4Atj%)A}? zdAT(*>gztG=&I$12jO&Fn|HujSN_ucNN}ZZ86oKk6~Kku!xg(HT)q`cUF!Pm9^%>? zG@OTRz&S<9(_F07q}}Q$9VYVOuaYUs0~)YgZ|^l@F69NF?&hHWgtZInX!=?o&yQ>V zB#6d{RYf%8hAW7dcj*0oPiVq_IDZ8fk4V_4U0Jc&o zN**||WHSFyvu>DZsQp$@YN>w3P9m2MJ?&ME>os-l%bt*igHU;aYvzYh9;2!LaSXz) z822N5{gyyo)Ve_)CC7iataqL9eMHk8ucbeKdkq{NoXuWa;>2_6vOv>~_21;cpTJ`{ zM``~+AT|7oRE>_MmgTlX7p|(7^zztaXiru~B}dsK6pTNdo&k5h)htlZzRo`SirgZSjn|mi&7|Uwi8U5;;;tya&3cdtU>EF+I$Af2 zIOJ(GD>Zr*7b=0tcGd7i4Dp99WA*WUSzC6&oyEEoF^oyXi}XZqIAR_Iv5GPVAEDRA zQUp?M!)*Xh3XUwky54$OO`1Eu4Vv%UT8>th1^x93jkVPQ`|XVIddHQYe8^wN`T_YC z;{g|`zx(-&VKD%@*5KA_-R!#z#SAUhTLbz7J|6usm>O3`W@s4h&&i?m#d`dlLC!&# z572aPZj?%(fPHmw--%M)7EtFIxU2-uK1DN*V;}OIM?`xYod1~9<~=cl#!#w?_tQ?J zibwMS>Q=jNf7Mv)oKgwx5wf{fpYLe(8_^Ak2~o)=OByttsdb7C?8`KY?D~xxOD2s< z;lT?)d#!Y_tg#1?)9${4_ga_+xhDC|KtiCa>j&BO7KUCd*=tkoxO4vA4CI>o<(7xA zvs_@}q`erU=r2&0yiOLrgvFxs8_TZ?@xu4qnxm9Uw|Dsbj53%S=ouG>CWl)ol&!#sEtg7m#&P1BcT$FEB$YR`u!)Jc#?P&3$d-tCU`&Vd<=e^k#@|YofgLvjhS05# zuKDH834v|^)l4x~+vm111YV#V!uoU&b9gi}$Eo@P4HBEfcQGJFrps~fIHkn$b^;Iq z_7_{mt<16)zv;U5AEmZkqb%({vx3!B2|i}?n1$J1A0Vm{e9j?g63*U#YzW%OARrJP zF?ea^FXm01)FD{{z!k%FZ#9V3%Der>wdmwfP^MQ`wg?0@)}qpi4}J$zD<$TDJcRLH zz(d2DOOx(&bGTM#?Wo-Q6cjFp=3(|kPRQ4kRC61DFsnhgCE5j=?tCkOH$dED9E6Me zj$#IW;HlNofhr;&WC%{vgf7MH2NyER|D{(Qc^x!eOEC#dctGJc77L46~N3i;x_pjtx&1%7TsT!z1FI1D<`kgkU0KU;JP#myz zvKMFz%-z+P&dqO&DLQ!od-faHj)SZ=S1p`cpwEyw<8TKLe}mSVG{|i-r|#-~UtCtF zkjwjg6iWZb2lWVl1DHTDX}(Sgt{d|1NQP~;%sYWW_}V_=a(hw*kQoBP1U;E9-+lP8 z6!V{&cFy?)9(&%9ycUxyy++5J8_5Bt%lfiJEQq*=oSIG%8vvthSF#7OYJUVqot}4mCpW~n3=b1>quUkRq8xb?qrH~>n z2AS6gg%MCPvFvBFhO3gD_fA)o+}99ZQqM^T_3V4WjULBGa4~c zNEd!^NsK#hv`fO$c+}2B(4g(f28@2x(}uz^d?19{<2W2+PXk=4V>ZuDAyDmNY(yCF z=<<2M*ohl~){=Vdrtt$sj!R4=28a4543E*5R2pH zBzIFpX_*4-hYfd^0Eg_Yz?WX0OgUb^j2i}Yq?5;pc9@#v zS?!b)rU1^QA0TV$wjp&_5UN{I@&V%WfO~6g%V`zYkgaq%ya0>;AoQ*6!H;ySypxCb zc)tFMy9r%zuqjvXaELQfC6m8P7-9F=kG#FR5gl*0yXgLMvv@J1Gi1`OSnNnhyg-&F z-oM~JE=X-=G*DB+0+1niB-{Gy=2k{_#?*;O2iwm*)>tIckaW_mknTob%z+x?G+ z6C;d34E0=X?HNp@!}A;7r;5u_q-U=IYBNvxON9>Bo{1C9{Nkf-v=e(HGQ_Qa>J(q! z0!_0ow@NZU1-=e`_!(1uib0E|Z}p5x)M@_%^2hLCD^2+jqg4gHoMuL;^&0UD(HS|) zSN03?=*%W3w))fZ58pLg)lOi`eeMkH&CgHHLgf~ec<0~lk;r?yk^%t; ziaoVT;|yjjG%Czn*m|6*hU@rD$JYeI%$T+R!J^=SFmM0|9fQO0(v*^SHvVbe|9bP+*;>(tpgzKvP}wuj zb9-9h4R>U26%OQi6sz^fP@AhfB&w-NoOv*PH^cM7gmqdOz9S$(X*KfzIipZ%X1q({ zv7}gRa^dNU1rE`MJE|J`I(CPOVKOwd>q8_y;_u=ie<+u%1?nU-tOGRO49Joij;PGq zn5*XPjUe_sYm)N1-FakP6(qmlY>PL1dn54G0JT2`M73Dq!2EFIv04F0vZ~aZZYtw6 z#Cxo_20K4uI;on~=z*``^o09kaXNL*tl79rRT5 zx#@r7u2}yF-i>_z=dR+P*ZY6vYqS%K14InYP9HJl|K{%afA!6;P|yCtPMV#@(f_NS z?|=NO|2*{p!yaDXp8x-U_5Z#w|4-}p|NVkL=9PuwvkzSCkdGG18MU9RdZ2771>-O# z$djx4(ImGG|6Bxe!j&5pSM}aLxS2x)=iuT!@v!hg&J2x_s5g$R77)@wZLJqevz~33 zxWgC2Q;znW{F-0cAM%w7CiZ6w5!oRds4a?=Ku-r)g8I zu8#nh;cHUiO8f2^EmR*A?*Uv7J=@8y0r(MTJTCMjd<*n|g*{ybOR%-)KnH~Y$tlWK_#Y#oY^GH&i z9JI=Cbn)Yu=Mulx&eeO9=4YpA%M%i89}49G>xmEoAy((ia_1khGeF#mK+x@CkSn`r zP^4gy4|SRp>0|lMp+zW3MmHJdWT@!#ZTU{S#b*C85DCQIqJ{EHS3U3ovF0(Uum^lq$=w23!^yty?!7M1pRV>kugU#<*v&u zPx(}Ptl!2pIlM?am8!JPDx_MxuQag%51nGrSY%fjj=lEToGF@H`Z;>d#8SFgyw_yY zx8f%=2j?#YcfZLv;otp1hs%Qr_zy3AN}n79q?EoIR90%XHD^9VOP< zRc|nC`5TN0v5stU335SFc)p3J4fBfJX6FU{;T&!8thXJx+r0H0h-uHqkCac63$Cc- z%A4v_y=nPp`!d^%j}TPm55T@vBV-QHr3NMD*3q$}Ek`20Wl*Kes%5ewTo%^$zQ`(C`2JI3h+Fa+gB7 zt0$HM=<5}xyBZN}B0nM+W|G1Ltawhwe{?%@)B>tMz37R>23@)e#|dkC`%Woo@DMQ8 z|C}incC*oWCzfe;_eokv(ss<>D&T~<`$|yw-Q#(}-p?YQMBk8mmEoAg_667JY0dET zCw^dMrIB@u2|0euV=Ph5b;C{d#HqsuVwB(nxxZ~h**EheMj7M?{;-vQLWMe^yZf}Y zY8a^T1Pba7K!YNNN?^n9Hk8#2TKYb2R0y}zxbMBo=WV{!9^dr75vDmJfR7`O1gvUA zLZQz>&Cm__iN+5kfpXGU$2^UE-^8Jne22gS@&xJzyv7FMbG%t{$9G2ePX5iisoc+4 zm(bCr5qHryXC67$@!!`8`0A_ohckl&ls^yoBq4%b0FQ)AMlWwHT;-9?mA5*h!L3hW z!Z&tL;KSyp6caCF?RS|lug$O#W_mlMa?O3fwm7E6MhLy9Ry2y=l{ibSG8@b4eKx(% zWqbOL6CWfj9w7t0O}nWsv%jjmP(NUJ?)4&P;c#_qV5@H&$1I?07b?(U9papP znIo8~_=XbuNPB>?c7tZ8WCQj@U9&4_hn}ha*J}&>&1)ipzgdSkdzn41`r#_f4{6UY zsOSQLD7yRTtKoecou@{c-UnbV=P&3%i#EoIBN~50^Hkh&KgSX)r0!zMlw`X7&(~neff>UhuH_Jno&29=G-U z-Im^8C-oGSk)YpQY<~uRJa!Aqk)0`PQ4zKE?m+Ir8@p<|A+S^78M=>CDIe+e)i>of2RXcNCp9 z4oy}Th%NroyzrjrUFY^G$e{<2ZFB=iCAlwEdL!hMhCesG(o)Y~JI&0%IO!voZVkqA zRSmMbyFL&M9ZlDzrlGOk4>mF<^?(aTRKp%H|-*ficXXme#Ws*Ng3jZR{p$>W7@da5$+^i-0V>T!{U zU#Cn_6)r!&m7iLRbGqDRViKtDSzQvxar?w+zuM|Wlz`jDpeJ4gjn^+AXBbEy<+SU+g#WtOz&ArkrR6T7uOy76Roww`&N zkJchKLnY-L&EO5c5)~7r zkd>Kiy3J;{bEO}-Fae}f23hSfP5|VErfhzWPHKs5|%KmGL{i5qG-n5>~U5ZOUy=@#5#vs4i2vQ*&64Tz9MhZR~Iku*K)3_?0Dy|p3 z`El#}Mu5Gep}>{EGz?!=sD!{H?)m_)MOU)^L#?_DetjPF%?<~+x&jh|{$s|XfEvr; zm*B~H7FjfA?}Ov{Mu6g{=5G(q_#|T)BiiER@{Qzzj7RcRvq3?abYk<@o@PZ;!A@1n z4cNVW)gkg!b3xxJYz$(L36A&XcQ?M=m1s2ZscVbiR~Zar!hLGaycY|Mw*2{J`Nb8_ ziYU=fsu4V-#T>iGz}g#1ljK0|L>P;!$k5L&kkL8YOr+EhU_^$qZ>CfyADZxhZh*z= zS)go>91Gq;`|>R829cQW9KeN)K8o$4=F?5Dt5EoLhu{b^nQEk^E7%KM3Y4|y!WP|R z)pfARl&Uqjrr;7FQ7WUEQdn(tD-@(7TdXvzcSoBd&Tm7KWe11up~Z7$%MEF%4(aC% zna{SeqY0QV4L>H7OMk!-B;2gDcyt53$!*RninMD&kmG^A7eeUur{W~-l~+#5OlN#6 z-khOUA2~_N)C@1`SagJ8#uo(zbp2A!5ngF?p&X(q<$-9nUq{1U)XzA_znQ+u8-fYn zRixI;a%7w&SNqoX5wRI_^hK;D>zjLx8<#^-go!P^!WLPY+7*n5~K+I1f zniAAI!}t6ltMQ9_hw;%C0v9dQ7rBQ+ov>l zU2!7C&4!UYt|@(mB7E%1GCR=uqAlAVdr%9nKtl@VWRv|%u4!8hsMfp3KS}9nbftbT z%Hn~RYr08@hDYYK_Gq;vRW^HQ8{Ph%0KPL-GQ9I=(B!akx&Wls%O$w{gP%hV)eOsH zK4;viI^%}s;{zZ2F0DFlESs|PCSFd+BSwkxehjz>DXrj_mvAuCqu=v=(S=K!9jNQ{ zFr&+LGRt-#mBFJFb%hpD)BU+2LJstak35yjoE=`&?Sy;`vWis*`Q=xZg8fBXArt zS!j0?n+?eopykfd8(sMgA@jT(p%LuAn({IX!?A5g^r(g}Vk^rT{#1LqrA_SmRqbe& zb+X*VM(&x?F3EA*VLRYtl9g=}io>TjW_Thv)MB)Tp+8(TwcX z6SqSx=CP5NQINA^eZ;6Q*J4OHIpbp!GIa2aSS1_=NOE;dJ6|bmML~kds8voGx(Vl0 zSnOhyIe_nxuRW^g5)rw-8xbvp`!$RazHBGWDql3X3J&>Zc%X!o2M{3LPfW3ro11vq zJLRU5HXNUQ@wWQ|mt(Fki<8IDF68ls;5W|kkFeeBmuaWA7}3q$0JIRMr437qY`5!G z9PO6Y=h?iF^rgKa=N8ov=63zf6};G|pR zdE4Y06f76Bzjv@r0Md2>R~f@WoZ7>^*MhqQMX?Xfc^AoTZaw5flZd7v?)<0PS#%|k z=9Le5b<9hDINTcGc9Kplr*9=rUdg+@HK?09lVkM}m6N-R@1qnYuqC4Q;s3IiqONMW`)yb1 zxjce&p@h99-$E9AkO(bmbN0b~<0L>w)=+YAwNx?$)B72TMX#Ff{z`HG_gn&fr87%d zTr_@fyjxzVV-BDBrwfb*3QjXO{NK%g3dE?CM*|6no_Oo|X2nQP3)7f_M(3@ z^>et->a)OM(nUk%$#d>0v5An0nR##$dtn#^Gl3F{?hc3A?dE%7n{x#k4q&*vxy+^s_Rd)~>p;jE z3>-9GGfF=g94^B^-pUgp^lOvLrZ5)$iv>#pSBd7s%wMw(u_7K8*NK}i5PZR`&@`M% z!Ezg|rc1gMk4$_xemAcE6V3@W;rlB2hYS|TOVi8UnRCC8-)PErJ>yJxmRW-GT7iAp zCys;%=@rjjG0>%RwPm6Mom#_idVo<0x)98yohFG^~x}_9`ESRWp_$VC{Cckpj_uon(3ho<->8IA+cP^6UuEQ6DlIT(-*|Fh(>^=}yiB1xvfNn^8Ob;$!6qSRaytm+^cRoVq?#P(e z$>$6}7(w2ja;Y|B%gE5lVIXq4VD0|Ry;>k!(M}x4S@LFt%M*yi3hE{^ZwO0%{%S>?};Ba;TXuLv#0sZjTZCQ#dpnUiSRA6I)N4hLJ~_)`AhTwV&$pUjC|$n6bfx{M*opoO~FY30J_NlfI?s?~p!X?cpx zS1&8%w2GA)PP>hRoiRB8mP>lztG8OOMZAJ$!aSf!AM6vra2+P<00 z6i^$8r%>gBQ;7j#zlk4bq4hmg{2i;7DoGelg-6Dh7F-iH)zOAxV_9Qc7FORwj(32!>fzX`g-62bX zi@Ft*mIphG^HdK?vV~{boy7)M8#gmMbh%Zq&?a#$ZP& z1O;3Krm@iQpuy$rd^7s=0bMgRAL;r}joB<;YUdbT;VgMsb?y_3ec+_1bgM0%)#H)< zruVP%N71pMrEs!H(Gsh*F}8p-mQRU?Q-=(=z*;qi(w2sfSP% zOl3>W1@U?52sBRtYF=}xFiXEQS*Sq>kBn=T;{Z`%3NJXBLZ4?C=Wxc1@PUjKgDNoRsUY$h3nK|dCqj2kbsrf|j z&gU9rEh7|7Xi8So8-EKGu5iH=+^IY#=R^lTNjG80AdQfxrx^6#63gT9$5NAdG+Hs- zW;s4XLzkc5Ap$)$MMdkE;sp3;dl69_mTH>L=>koO661G4WWDPBrhtM|@nQ|7A=if^ z%jsti~0Tc+O;$k5gY2Y$C@5ANZGbp6e2%%WJ92E9tgn)X zCm&ZFwY-4I4tvq;rwry2jS>u<=d`F`D|R4^??NZ(KiiN0YO*;-qH2vWbeIE_6~Cs% zMEze3zrk747Sl^??_YZ)|L6#Kmn_syfd*Ahk;CCKj{N91)pRfNHuU?)Om?<00y2gf5l4JeP4WmuOE!-5C%elbz*GJhrpS{}79@k75vS12H z4r$4}gYaEL&6^w1qy@Z{{X~ST$ARPO){+I>7|l8ADw9JYd5iJ%I_0e(hIWEByQW&o z1!7fLIsB^SholZlg0&LU9X__F>?)00HaED1;bTPebUmNM1H2U6z5t^_oMeQVOESba zU40?4&(4c4?$eSQ+Pl=s*^`2;3S`wktAbN~m!J7Lk&Tl!dBu<(&$}q-Gt&;9o!L+8QG4r&x~BVaCJLb-mFdv6AjxL6EaLY}37LlEKV6-E6_c#orYxtNwZE z#ofphr<^{bRL<#kzK>cDrtlWk%th7{#xmm;7C4PJXw`zin%88yZVB7pVI#^lq}zdc z;DTHE_4ze3&z{ZlAQ7bST8LkGCF{giUk8$2=^tEn!(QE6Gq4=&CoRi?sTw@NxjIYD zzji%YQlk{dD6plG;bZBT_sBSoFN*x{1D%S$PnGhZrLm^*&UWotycV-E!6U`S5;bAY@m!nw#(;Gaa-t|gX-^~GO<~WcmRQ{@486h zG9272jflaNtS!fvW#HH0L(HMFczd<7${i#2MN``LQ%X$6qh@sX0b%Lx!&Yaw!1Y#F z&ebUq5_&qcCjLcoOulo6vez-T6j!oy2;Lgebql7-Kx=Mzv~{&dSBiU#(pg6P1sAV&f-hW zzxl;j*FG>$gXPMNlJWjk4SY3yr2hGH9vs><$jCGI3@BZcXcy&-_FvO@H@`3)ex*_8 z$K2o@BEGL&;qf=9l2H0F6#5KKvniQxop*%Q2)Y++y}u_Ih`Z-Rz`jI5QZcrpCQKR! zHP7w3fVUmSgPG14>)2a38Iek*BKKlUG-H6G;?FzWP6G ziu@(vWzU406G_sgBT$-Cjf}S+<_aJC`4rX9HXMV zVmYpmBHkHLw?Xppi->QcF94&@pSprWYb9E6UmEVmL|URpbk<_AT*xqcDpMH=^xeU{ zD3iM5+UB%B`sTzA(447@H;R>OCj3E~CT$(YImjw3a#t3Mjbbf}O}pvNy#c1lSks7| zm=V|@+qWg$-;-F)r&`>W{IYL8CWpVJ#REB|&2~IZ1}P4sFz$v#kp;>^yG!V52KPwQ zI0FQB);znms;l|sSI1Z0y+veqy4*e+d`S7@BoK439Ye5wpn&J7|BZmf^5Sa3f(s;S zrg`JMENoI<&cK!f`xa3s+xWv68^x{21I8~=L~hj>JdizgfoHc3hixe5vIGn^ZMyV_ zT(!GTgWS$r*@e2*QN9=7YApbwa}m>cyX9P{+dUPXgWf&z1@mpgbb+jY=sI>-K5h6q zrBfeh7;|~i#h5{q=#M8~`HfuofcO?Io! zvh#To{SC4^eKyl^G%oDIzjljKwwx(T*xq9{JWySZZUUD z-`Em~0FCGLB7oz%1h%qM(&HtSa~*nM8em5!T`4Yo-E7mRLFeItoMBUX#8{n>bWU2MJ%U1#JoBV8pRl6;*YqKno(smn?ARwIovK~S>J ztuNL@yo=2iNGG_alyg-cV-0U|&%)l=&(p=>+YpnG$8yoOKXx7`{j;qo)S38p0e)E5 z=jJ|x-IIR`15xgIGn<8YnOE$!_z*IV*xfY+z_o65fAR$Y>5bcx;FjQDTUg)olN1-d zXM?%d6MwLjuKwr{kgF@uI$S8!s586ihega@N@Evu&JzDseYf}a8bNvIwC$AQHuShD z1`$8Jto8H0xNlTEg}>Fztu&l)gx)n0q8wbz{UeV<7s9{IKlx9|#um#D>P zayxl-67I(y^`!S3hwrJWBBIs`p@A>@?SAzkbs<3KDV~5e)snyJLL60hC~i3Q+FYwU zKGwsS&U>9PDLz1N<&n-Xd|>B8A1KytGyWh%4ZXED^r%Ott!}OllD00+N8j7ATW9PU zT8ReLZgI;zDh5TE#Rrj?ItK=QVgiD2#clK@?aPK($}_!-fciLeN}_X}Yu~?|Or4ipewV_hJv4g+1GfXn&Tj=!jkf;*MYH~SUMiBOmMAm&XWeox=L~U{<0-5Ps(D*h6 zBc!VWVvR8$svDh=-s)$No->U_MA-qP$g$7-5(pySFLeD&HUmiPW`FE;|2Fc}i;fpQ zYLqXbQUrJ>4HJM4Ldng~dfhhD=+J5qM8ivQ)>gCW_Hv~r0Mn`76rM;yu|}&RXB|(m zQMZ9bLEv-7z^UDC8L8Id?$7asUrjrPh`mCmsbGfcF?@TbqoMWmoht7}3AyKI=2SV5 zK)BW7PEH`-1sN_VU$#6AZ{CWGzDFkM)T&213TO~OvCsbkk<^JgAr|4Cr!4=K+$q3% zKSeSia-|X}92zUq%dY!Gu{rfcZ{L1i-ZfLm1l14L{tch>(?z0jv`K$or{lsM$ay!%-&D?O!AdK z;Op1WdR~nz)?m$ua_!9j@j}kL{dhCR@IV}Y`OAk=u}xg22Z?ne0)Y%V2K07C-!>sF zp$o%XrN8_cjp`Zt_enuypIZ^@`e**2LdgVKFGydbO7m#T5iV=#<*|3EYK3Anj^PU% z9?j%d$p`cOy*o8EcD!z_^#yaJT>jwZ^35aSD`h;j5d#DLtT{=hPIP9*}o{8&fjH7m8bDG8H)7gg0k-ISBT^Bg5 z#We7~O;eAD*aL-2Ds#j81m(42#lqUlC`|9hsBGOzn-3M?$CJg8oiOm8dN|zfu{o70 z=~Ri18T>xtd2S;>rHFUq53^?9)x^Mj8%^aK$yBxkIH<*Up2%KYRLe?+N5CHT=a9g7 z^dM9;Q;D#k!^;Di;yJh!Q4cc2+Ncx2N0$;?<=MNJPa3WYs}<^gu|jGmVqM)v=F>}5 zMr;&M)IPcgRF;ZMQ>xUBCr+Wv(FzHny%sIYjzIYR>&#Zh6QAgOQhN^PRi;|dE=lMr zI{L_#N0KT-j#Vp5?V!)jR@`iDeY?5;q}K~dEYtB1z(^0;H-d7Erf+kW_q{~y+-|5V zYCJ0~?Y6l%B<3>-BFZ*ITW-W;iy$Y1ctEo{<|Vh`_;z~|B-srtoNY(KoPDE01b0_z z7h|`vgvDQ$0rdi8O!jE`*~2u}j$M&0CI(Cqphb=?Q@SqA~Nd=^e_V`{l zb-YGr|B85}Ckk>(U!m<&=A?czfv{;0nmdP5jENyQ0d;Twn*hUi0K*rCd1|08K#NXZ zY;87bjHzZPfFRYV-e$R+PioUz?%&o6|GxUYV}yEFt{h9x>tW>$d+Q z{dUm()1*~G#%ZwPq4ZJ1h5Ag6VuAaD-s)u5)hc_>j#x8$Z&i#g*z+SYE27DcS2siG zC-PdFZ+`D9DPH2pWglCk36@>DhRqE^WqUvJWR1N^B|Cq603bZfYd+xtjz{gCUP0raJ6@uiEU};$f*iWt&3E8&5h$`Lax(7)UmFP|Pqdf7hbUvMN z-@WHk*xaF5LOx~S|IDw_M$2C`O_ut*j>xi|JHO7%=aBdW50kuS6B#Y#Ij;9d7lN6B z_9Sc#b0IID-+5H+cBd7a&xps!NwVCf)D97jl;gw2Cf43s$IHQ5&?6DN5BsBR(LElV z9=nxg^S#3&3*(OiY({l+-JW1w$ip2W4L_y{IvQDYl@<%>udTlDQkjV7W0Nc36#OhH z==E1^Dfo?hv4!@R4gu$)<)vx6t@p!gb1i@ztJId`#6p=bf}|F_mxf?5N35R^s{NOE zD>$@gV0U_R+p*}rS{bJdQY3qHHu_s}Wlz;zXKE}G851MFAP`W8g*Mf^peUClU`*%Z zrP^7{mQyW{jp*Q_g+3Q%M&-+3y%c(7<-1fF^q(hce(8W(m{pdK|%KDyJiIG#uvx!hZIEekhf7HxH<`^4pLeQ*H5#3KjhrpUq?Qq&T7*y zo7oCzddD`t0W)o*r|KJ>;Tw@!?Kka=7g8bDT0>%^)ri8kA7eFWj_>GC_j*2&>^}3XXrXjQc{P=4c3#YGPdBrg&XM6? z1~{|LBIUwr4P-plA88skol9T6Go9`g%>Ll><_Qt4rm9nKFcQPdvqSlJ-O(Sx&b6eIfr3U^W5KWliZaY-GMQvL ze%EyjM=uFLA8q;84~;i{2ZOfduJTiAJw2*F^qo>67vmzmW=UeB)AS?B)lg8%HUW2&|2GSuGY=#0r;>g+a#COZ z(qAWq+|hhV)ejZqUBOb?`Q>;l1<3OyRbnj~-I0zY5+q7D>7)<|{~qrOMDD7W5Mo-z z#ytX_fE=Xn1XqBuiYEKB74d_lsp`)z<#z`@DjjJ1^`((a=#~0t(Z8)dJA2LSJY9vj zyAl))K|vPMR4T2%jDE*QgK_D?khz(jOx81?+!cZz^;`*z;P^j0);lYlL1w+Y&C=N( zRY@X4@MYYLg%1zUNc=-Hkjxm>Y1w<9O#XaqG}~t5ZL|+AdAT=44;);;%&=tW8u#G)s}5W(-=NS-EUILTrc=tnPBfUwt?y`j3tlQZoAWKGdIN%Em7u+v0? zzWCheJ@m}JFP$61uwbLDg6K~dT33O8RuF=oq62LBKJaHrYyGdKC$))NWlM__*`dM~ zxqa9(VC!R#$ilh6dcuENz&4!avl%u9INYdl87e>4>hDc<zG+Zr9&g?}(#^)ipD*l;uT+yjHa_g%!!2TMcbUlbdh^?K z%|->#hVTw>w22UH=z0TUVrZGYRQ@nQ<>p12hn^Z+h}|P*&bNHfH`SUJdoZqpt=NQG zM44($QyBA6(iz11HUt5&!P4}17V_o2X;@sMagAkYVHzOBV(oxCylUy={rw~HxlSVe zh*GCI#C31cL@6VG2jeRPN$^vNOn0JtW#a@a9>(WSKGzp-$yH?2uT9sd;vRSRL$_+8 z+GdOiSNW??2WMQeTFth*V*99PokZN#(!8~(9tos?PT!HL*+o!Vs_*^XSyS6Qka3Cf zZ(KO-J(kvR-7tPH{&Y^0>+YvqRVvX?;g!i?kraSOmCC)d@JLT)U21QI>ub$M+U!#O zb-UU^xb}{}h7t5J=S2XqMLF});sP!~x}c0O;o=?zPMVX!G};(S2b$|71J0$JdCX0n zHx(>p$S^f6i>71j*^0C90`rvc6Z5CeDvTR=^NZ5c0C_IOWhaNPbI91YdyZsM3+Sn) zWAYzMe;LExMHsQ%s>j4rx1L=eFbUM5!uczPp70$>L{}{C)o(Zh>u@ra-P(iH-!AZB ze)!y?^!iPys{1{XtLaK3q0{BV<6vXWNa=fhpbNrD9KuZeG+7}+3JCj5s4FS5H@`C3ZMo7JWPaVJyL%&9#H@uO=ruvRq*AGI7B>UuCPE>#25E|L`Nx8^ zh*eaf)3k^eb4de~j~h!YOer;nazI8EdYic7xYnT4#B5t{{aG8(Oo4@0T9VxFIgU^jc@re=}`Bm})G zGX=yc``oZ2hf<8mwUVz9s5gb)GqVx1SZXP1=Zh_Uo-1EOHv_G1`3Qe6GJ+q5sb`c8 z>4iwj(kBD2QGTlb^%_OD^eLgOE9@N200$?t8u;4J$Kz%=T~(duF)b#%b^M*)T;9(d zLA~%9*}v&+h2DeR{hO&=7xLW(_rOa00v+KlqW;62O^tYdv#Y>)HRRv?W>0~~N(-CE z?ga}7p7)b>tCCGgv^S%6wyb0k%hkCO@h|^=<|Tg*2lpG^95|qSSyryJ}V=8rW zb2{6uQB&G4xxyNBM(E6ytx9Hw6kLl)3!?{$6$<6n05N3F{13pJ9w7~-TA_uZ&3(tb z7PB3>GZgQjd_Ta%vq542I16w~PASXK@XTt>fVMAFYt_^3H+k+D?k5P8$^cz0H6TIJybj(q40zH5|Bm!dd&Ltd|Ed$Z8}L6U;n z4t$9r-=PA(t?tsQIjY$4sxRR8=Djb9kbbw?e2sOuK3-WNIlcW=JXS8A=Bc7N%uHbY zY^KzlmHJ3<1_Q_RaRW^nxBiB!DSFHy1%$3-mdY=65GoAN2)vLy^{-{oW)cVop~4*v zC6Z_>(jxI{FP2(AtduC3=N;gaE$0$!2bci!kSH^hhAf_(O+E^rSB)3x;3~c7UdL-n zcZoLD86`hlxlm8Xx#%HUl}!F#uGtllwH+wZ<*HTy1^}grg`JD*>Gt_e`3>aX>`L>j z`aSZSklkJA6ziiNvx{!!2x@8yOG8T^zfx1x@??od{>)m zhbM;VO&UFeOUb{W>2JS9Fg)Yq?Z8+^!llA|X_9Ka23--G-Lt}CqX8Z4jK2m`Q@0F* zlM-Nixy!p@dgjYO7yAtd6T9YGqp|J3It#urZKPt+yR$tN%4P<&(ps?_eJDJci_QAT zbCDgt_cd-OA>tYp&1v;Yay_yK^{WXO7cWYW&+@tW)pddjZSZ3icdYMKgR!ur{Q0B6 zQxIPeZVU~6*QgLpENoMfraf9kLZCV|d%!{JZ>-Tr$p}JLKG3dh_zUWn>{wa~uho9& zhQaZU>ubLh@}rB~B`cRxtdI)5M@2w_tlaVB)y=}FToI_h{ngAO7ea`cA>SqW6lt;` z6?W6h9%qa}omfdqF~i|b3<(=VNSVfcM(&xikz-eOd6mTnUM8Zgwqs|pb?=R-YfNH@ zSx@c;RHj~)($9}p9ftfvtuy)ZR7Pl4gG{&*Pe4gukMl3)4{TrZx3Bxl*O_V>gIoNR znZp%-`!37^QG$s*oEwx4Z~kd!niY4 zG-H%q$$5-LCQJ+`bI+`to1?oXE;BreNP#n3rUvJ_f+LshW?tn@j#){n>=@@M7b44$ zF42!%UI8wFVLS^;KTR4&lP~y7fHKG!d#oX5e)zmnV-^hqQN0zJj_jT~f)n+G-qEtm zS|}Ph3R+==f}rxF7i(&fLVNaiIx9;p-xMB$+D=gbC068rH;jQbrpcYJ+dWTS9#gNK%Q$j_5j!rWSa-w!`v_lIn z4(#T~gpkIz$I9rxbHCiDD(8zA6p{ugsHFG>u)5297s4~z+HECy6O`lq@N4NJcfKM^B*Gk~rY{wn=T z=f>%w2;!C$J-f%H7l2$P0;i_1uNJJ7SqwUF`w&$OgT$=2 zyhy|`N%8haRLMeWj^?nN$@sT}Ybs^_wn4|($2XDevP%`_- zS5{?jmd{sHauO0feQ!UWDvj&xuc$p#C~m@hw?D3-^>cj##wD$g$dq0ls5yMtN_Xe5 zhna?^tRTQ6TKP=vr@KF_AT?9EDnTf4Kx1!Aoz3R$Z*xSnTLyCc$3dw4;Lo-yyH)`K zvwIFM$!8dYwQ#Nhn6ts(w*;D@fY7O!kv_Wj(-V@xmnEux!NDox$jZD~I0y597cY=h zhv9o>F zI=KR*s5X7{>D?5DrH@~baD>Z7=D|E5wK`v%0au+z!08;>@bo;uARTonQ7EcLW&>dy z-5RZ$It1N&jI8_%bV-t2yCg}a6FzxmNcVk%%h_9BC?k%>o642Xkkqu}7*AAnT!1Cc zOJ63UH%-SzF!ft~)s}Xt@fP(Y49J#{*vl?ql^FzBzl7tPQW?h`YdfIBF|K!XYoP&n#K( z4djo}F*z4u^_g1NJ4dvyWE4l??PL zjY`GW)?WjYAqy#w6-CjU9duI(Z#EoSyffEQ;~#{2o(d*hw^PsuayOdK=C zjG$nm;5FxRKaO&n68KU!6FMJXzoywBY)`G8cQ{Lg2pNSFIMkPWBb!1BRZbgmzxJj| z&-QbK3Xx}F@HSwzvY(%#Nfku^w0hky50Z8%6=qTob&;aS$hhxoa-Ingo{Mf7rRHFkhIFY5G|iphfb$wyA+vg9{kpL^ z74+w_Ce_(rl(lyOR!BMB;O(bWZOo@XN?z}uy&fiV+UF`lVGa!tFGcZmTRoc>JY3W7 zjaEa4TvBkT;hvA+7cF#;bli?AO(_%`Hc@a&H=(@ElGz4KvENA~&g^ z*-Ab0fIqjvzP@?9dr`DL-<>jOSH$~#ae=3LxBFN`U8%4ae`key%FJH_&57K>e zFs@D4vTZtY|7J6ydC>*AJn?#TR`$lo^O4+J^yk7W-c_PgwLb=UIx=94P2;bh33g0o zRj+N0+3OI`MI%sEI$B)LaHi6N*on=NOmi$2^6D>(=_{8z2UKc8vFIxnM{b#G^+wa! zb?Q)SCC}e^()fy|ZYqf8Gs*b@ah_kfh0jeG^{*<_PpSFx!Pv(a@*llWTdTyUxs;@g zwzr>tZ9*kj{ybk$3cbMdKC?&-X(d+Yxh9xYt~Gyb4-3D*fV@p*+W!vzEO;*(fX&!V z{zU`Ah!$MZE>Vqe*$>lTR^xpWBaJ#HuC35^RE-ZlgrFSm4W()i*)9Bi1C{d5RiGf5 zeAmBvbE#&s5|bt)hYf}p{~pnMfdf3kadwP!K}d#jb}bei$M)T9VRwI$tHJ&RpEXGc z2iP|$`k%POR~ns#hK@&6CKQ56cxE(TV5q4ZXRXVNyS1h~EN6RWO zizgj>XHV};|Cru^BXHzf?Ha}u=FZ=qm|Jd$17qc%MT>pz(Ax_&<|pI-t&(x99>s}PxrTsCCqRJ;{!B!oH1`H zq=$Rb>ZhCD%puru1kZJQ6LQL22w1jjbvVT|*H5MecbHt^Q>onlj>kC5XSseB^fqp> zje-2QdsYt1*yQ&iF&-tLu0Pc%sfF^L=#*=j+FnXnM;##2ou96;HyKG(AHw@JZ#srx zftk+LfT8VCdl$B)oKzt&oNc8PT@N|S!H&jBS@LD=L6LGsMv|eArZ-Mu> z#b8rpj+bHOP4Nn4GEn8vtWE6t#g}BdbCCNJh*0gAmMXe_Oaipjo&M6V@5pynm>QvB zogifp27#p_KXLAyXJ+$Wj;V{d)1w}(Oz#W}Zn=3i*FJGU?!6U&Cq)E~`FdCy8sH{4 zF5?LYgoO(DHCDkB!U3V2(meiR6F~aF{lkg;p0r&DghyUGcrMG1_F@&;%QN<)(aXgU z4IGc{$-5(hdDD&eRRlGf=5)&)XTQG3Q}_Nb>&=VZ)*8xk|IQ-8w zsEJs3Q#Pxuxeu~c0B4QP4C7pcViAM0Dx;S7jr%otTLSaVA{*?;!+M%P< zZY`;efo|rSmI{wE0URfBkQIdR`oH~UZDDYr35A=P@HWog z+lC!?s5#uS{%Zjz)DoA*)6?Ny3k5Q%?Oa-yvo(K(7FEi+7~E8|6^5>Jr{TNqz>%%z zj+Z4|ev|>}#Te6%hPo~1m8HjzgCi$SE{%6!k4CO-dK|JeUHhsFJ8CYPH{zXOme9$J=$(SeLmJUJg0BRGY#YWmUN^GT85l zQug+R*nS#0g&7n$1oQ)Gb9JlItiA`m+~hpdx*E>+QVhvI#kQN@rk(o$X^AP_wOJ%u zJs=bD$%j^l&fWRS6cOkm--Tht6)D3wyDchq_v&6)11!TNpbFiNFSzqIb8; z{bt(G5|Tc_e3rqOmU(u5l>}?W(}Jh3)8{=RM$CWB9ERE!=^6|~6^*g7?k;32%E^A3 zQdiT$DSg5p1+?h))9~~>^yl#TcK$DcvF!q8xkRtWTb^&MvU%65)j&&w?7ynr<~M#N7crM%U)cV6M+WScRV&wd z4R7f9NlpfyzmzaLteYZoWKZ+pO{`8Ia*v)}A=TfJl(ZM2xuQYMU11z9X}D zoWP?milC+M7_i&}Y$bG&uyzvmVf2QW;W&bwflVmkrc2V^v_+q=Rqi`DEtp@FlF*CK zpm29Y^M&?@;~~))$%&7`yzefy`3w8Ql#2gasD`>zWsc&Hw5n>YJl+wZ(p&RrzOmI_ z@(YDxd^h#(ZDMMU2rG|@@cGKwpOW!l1#Hvk2Zuu?-{Ub=NyO|H=f9|be)NA@v^Pct zS{P&A;LV|AH}9QfqbD=J4)edRPNMf2v3!%_$awtN)dsGFktWnW_h0ZALq4JTfI5Tt z7w==Es|M*4TSuImYPCB6cG4NZPWs2YYpo-|xhv>DhVtLPK=iXaiS0>u2eJAVmKG`K zb)V=J`G46bEcD=q)Lw?yt-z16&(Db+4B(c^MNn~uJ_*8F93uOuPwkiftHGCle(e8D zRQXrF%762z(;(uFO`=-_by9f$zlpm3Pp`8P30&c#-#_y|?(?5OBL821BQ$vLS~F^( z)uhGpuTSFt{F4oEg$Q;+Nbmpf9?r2zfT)phdw4g8@t=S+|IL8?JG18h`4`vME3{s0 zkpG|m>VNr9-vR8`%>Y}K4lDBie+vD#L)(c5TwyyBhW~$h4?O*^43mLI`zF)>>5N(f zu7G31b07LYyoU`&(SIOG5=KNk98nwXcS|_Dj?&84n;ftr{i1LgW#Z|zRYy~62G8Ml zlDQvcV}KOURL6viuOFxPen6DmR2CYq!^6A4k%?UI={9XAB7SFy+Ly)_3Wcnk!GyX- z4#)H4(!-W?wN#aK46=c!420zCj+Zqo4;gXLpQp40~glEgX8@vfU#(yzMGau&ulDj4IV4juxd;s3T%;8wToZmJ;f63|jsODKZ$8 zq&sp-nW9u}eaIdfQZGrO*F^=O8yF2Xt4P=vqh=E;*i6351sZ+*=2ERrS5f78tFe-n zMwosA(f*c?MKiv$92<_K9!kTweg(tgO#eBLQAl4OFNJRkv;OW83(-qiO z3eWp<#y>li30qlt4mTw?*~I&QWabeO=+ioijx5&K{GnA2?hejM6DwIEgsOl%ZaKX2 z;U#O>L-IC#xdF;Y?jqPWSoR7=h>Y5sdt#UyFNXERAek zqii^gvbuG;=pb-YSgKkn!VZFf3`jK_XH}kzAFx|P=o&w4!9?4yuK;=cEe92%hm&EA zF1DIrT1hSfMSoiI0SB13)H)k7!(D}!>pm}`Etg z5)RiU0D8nq_Uy6FTo^@BqG>NK3=Y_O$|YNV*!~kOL7)~1_1xbo)>f8nTA4=nZ`rEX z-9z42psZ*x;OdpVWb!fZJ2+@Cj=oSe&&h(VWWUyI9h1KG+Hbz4Fyqp__UYU%7?XHz z`i}@0j?`+vj$6+$>r1s!H#E?#emo4|1F4j0vCFomKyE(90l4!_3hlCH0k4Y1+RxqR z$kYM=+(bBu@T4#BDoJ?pl9Uh=C)%++LVu?OS;NS9;@r~u{xuEa8v1RkST7|*NC6mV zq?(Y#E?}V1$fdAeYiO@_%Ll}RPmb!VM?yWKUbK4>V-{9?=C~}h_>AsfHdHhtb@JhO zS&^X{1Rxvor>W`Ox1a7O`Ch3|bYZ)PVj_tMUg3NAgv#&d%9e`u*5?yv{(Fb=X&xTN8*ist=;&)D;i!0tnhWaDGaf575xl{a&Uea z{EmNh!<@+U?%vzfCx+TZjM{d*8Thj6Gf%qTGLvvqhdN}k^5~ksE#vk_`P4KEK8^05$3n#)5aHhj7 z7n#F_Lv{X05m6=Cv4by-O8X?~+bGs3ljqOKpiD6qTFqJ(+O3irJOq_b<*w0pP?TsyO%?vRU&o0{xF%@2f1;eD>_d+K%2}m>#t6Jg7G*O&84Y7%{rS?F779&t^j039b`q%xbm7L-_T21!KWoq>*I^s@hH}h$nHSU{b z^3x-HJyG*1?FAukYzt<@iAA8QMZBvmObJ1y*YMz4GtoCag!#-M3_(1u599T?Ej`u;iStFW{52)N9R3V>3toA8tSGy~uU;p= zhJBae{Tk%6`^dy_*uBfVI~U+j2i~p9F!>T#H?TG4eeQM_TShSeK8jesJ1O!`W?4vh z`^}Wy-vw9_kKzHNC(BgU#_R4{BCGC>5j~Ft@(;wK(vspfI@a_K;F`*)A zGa?eUr}~6w6o$cFkJ~Ywf7dIdjs;|3@wuH9F2PFGL`GNuW9DyM-s-J!Uy_qX7yxIg z^B6=+qSIFK>JTo@A+}yARL%fCVo>ki=(V`~Rh-~%IFWBfHLwFXu&pdm=`pQP+cRrz zFf)xAln(AOBDZhT=ZR5E`2w87oE0A0er&~B##yM?puiv#q7JRQ3tc`#_`KezuIrD^ zEqCb9zhcgYziYnl=d&F5g9u=k`kDH4+PBbPReS?FzC;aDKWgpK9QJ8&lBLBN2z7P4 zdG5f{HgDEyl6OUgADC=R1GQIFAPa>7hTo}=S4HrHaHQSZ#E&4^2s=CPGE#%emdvZv zzIgQpT814LcZN^RArtaYIZL`l?O;-Kje1i&I#c=(C8R#B{ez)o&$uJ3O`W(__%}+~ zeQ|Pa(wkje9r@A?m2)j$I(1koY_|r(+FgL0$xGO%t_Qj7fo#gZ{=H6g`-l7=;LCl5 zC#HhkIlIJJ!VN;kJAY>=Fq=3?I1EOSqk2LT(QEQkDD@=6 z+&<8+DKP)8*(#WxD~9x@)+xgzWB$M(25nJFb{&yZSoRgHHp^rC^f0zHJ0Bb7sc==0 zkXD>D0!u%C1zMGH#oqcLWP;CWT%px26XgRl%K^v9Et9qohA^m+%5Hy_+ zPgpoXy|VSeJ%uFJ=am7_=ye>Wq59@!3b7*LL#F3N{{Ae$M!DMW*K%5^jGH4i^#+TjWMaIcDBM@j#yNZq5}Tp%VlIIobCIIIEextkG(Wg+PZ1M)FO4d?E1_~ z0lE>^Umcj0=EURVRb>EV2|jIaLf3Mw2pkC!7?ZJn7;&gI@(} zNa+=MEqcvXU*%PTRGo9avF`YjI|fM|3h*+fEFxJ8X8`gZh!VQO`XXA;ARi@AdDXyzhsD=xh{n*K-##l}$^bCS zaHPpYLa_q*UjV?O<3`xxNJRs{heWaAQaRKgst`TuJpws%(Oz<<2g#sP{2~k5He#dmL1DXtWpFI{%$X zxh$z2y*{+lbh?)cJY2F7PImoL;{;$ZK=S;^_ z>q((njYTo~j9N>VPQAf{DN$Tghx3|;6MnCmk0#?iR6LM98g;|oKax=SUH~Ae#Jdb+ zJ=to7y{4$PFvB-@va+>X8T^I)CV_VV8Rr#>gQ8OItvFJB_j2kZ$|2D+3@x2C7Pgm3 zE^{s3V@;&nW)&xa*18R#oxDI|{JPEh<+2aYt||O$`_V~XAk~p;hno70+vn{J=e(tt z{tXAok(qqbfu#x!p_@MEf9-aIFXhx)#oW_9?wBD51V<5==aM7CT#v?Cc z=Q`g-q$CYA7O9|*NaI+IVNppgTE)f>Bzbs>gU$W-B#a! zQlUy4*DccNq^3ZYfMVFg|AsQH9fo9vd0<~Ax(Uu9m}rS}0m9Y*BuBT(>7vx-D0t<6q;NXZ?KnBA}_+=u_y z%?+QCOIc2Jo#i}bnXx~z=MrzfbA*w*W=%NW);%7KJamneO+%UUy|426dtU6);!L!t zs?CNUSMHVeN3Np1xgUvI>^}s~Tb)Cvli43Tyo;?1iYBnY!93BGMYC@4jJaf9Zl*Dl z!JFv4D%)dqJricD!Bt%SB@}XcCK^@Ui)awgl&;v6c5XjBVEGs)Uj}S-&_d*fMh?rO z9-NM+0bQ9l&3*Vsmu)Z(&JC__5ANpAJsSyoaPAXTtgQGU?-@ zZ|g00(WYvJbu(<$ig`(eddG96R2go!I6&Ug^EGVDQD}Pj0Y2g+_T?!?_K%~ivtUu( zw2N)A_BSUs1NFWmpD>n5CD&lHqTd>cQ*}J)0;)hFHxiNgqKTS0hKDX9;4sqs`ry(Z zWqEH4X;azE$orq!geT1aEd+mT;mmE=7HN1$VmTiCB)h|r7NcFaqkN??{GXRWcc)5D znK!EClX6otB`1$LmW!XZc8aHe#M7|X4*VBiNuf<_t*(?+ktII1aEe-qhSYi}&Z;J_ z|6Ljiy?Sj4hl5$P-8UVPSfAH8Ds~vn^vp+2*Noviwd^&yRI)(rOHrvBb={c2BZ5p^ zY!@>`e^{A9d`(*T;Ge$daix;W={x}`$huvGrfgJv%@KfDu z)iW6HX(4lRar(OWzo<};-t=SP!BGR$b#x19h9F%P)Z<(xa>FZ@ymaQ7H z@(o%DOP*~(m3Gt3#0Gn{Kc1Fow+c}WBHgQmm}RnO0s2iHw<$u2azmKw$&D(2>nwfM z!ic0MW`+iYe)?#-@{=uN%D@K&=neHLj%-}cs<|Ek`KtSB!NJrD1?IQW!_9+=89ny> zSQ3@q?^;4yR2kDO{AMpkxmn}{B8{K6#7?6VXtTv*?_qT?S3R7wHe%{w5b=o%07uG_ z`dvmQ)d@kVk;mD(N7cbfal?#dtfF~J*Zs~q(1em-1kY6TMF#8)=hTf^D3!|g@V%9e z6<1IQIwFowhSSf*5nxT3Zw%5?sZi#!R7ion61lRqY9!G9{8;4?Y6(+Z3swa8j%1Y= zD9qmV0zE)`M{(elbg3U?ME*Q>noFJ#3hM&&8zr^#rT>k5DU|72?^Mmw?>S7cs5?@^ zWd0hR#$;A-aP5XSEU_5=tW{t+BMo3CUE9L{`p4bsx!{M~N#Vajk@HteT%s#6UQU)f z!iiafXZX-13<_;vmiNp0V?&1h>dTe@)~14@y2R|b@kBEHl0^+0p}_~q%2lfw$^~=^EwgJOiI~+B#$$=%W`Vf;doKAR<;Pf0y$6izlG5g8=r(D* z7TH^998KDl`iRP-YJYfekFq|TDh@U1|H5REAmDOTl1gkvXZmj8*6lewI(6hF?6ZMo zG@O94yTd9OoQH`rh>k_4^@)h(SCsH2I#ZZ?8|SzSNsfr_$j z@0T-&=}?h;Q>X;dro~*vSEHPygq2sMPqZBqkKkLM<&cRd=p>jMm^!F)fTd0)7@B1C47U6p~S0F zsVsgmVWzkjqGI)17-f0!xJ@BfF&&qv==+{q*CXOGv=oBZrzMe*ckbbJFSv1gh05b|&+liQ7xI^2v1pY_D*>%vS1CQekx5t>P`ux?{zE9Q=?6;n>;kyT%(8q%R*qSr z!ZM0-*_Jz{%^S2t$E_0Q+&%qCy>I%cNixvVDg3J7GhVwv4pN8HJyH@HnOQH08+(4f z;qlvO=l5Kw1r{g*OB^>r_H5QSf3gK(eyiJZA)6g&I9;Pz!K>0SU*^Vzb-pL32sN&z zo&!YT5aR{gN|ylAVF`T_gXvIS?%zf~Nz?6fd9Ki@$`5*B%`R)WVpeOVJuqbWS%c0w za1d6?SUBX|4UY?x<#lJ~1pL=3P}ahrxi*a&R_SzFhE1to88*Yg*M~F2AkZ@1lzIIP zTY2Vh;YKZo8z!5cv{5w_=VfsW-;HWZYxINHE3M`NwblhBlV#z;Xb|H4CT)`a z`^cvR3?7`?V{*k4^F-gdcU-V7#*fCip5IgN>194E=&00JO!XbLB^cw7pZFeBF(ZBU zF<&oYXXecY(FlR%bXo4Ka$s)^8zw|p66G?Q%NvYLnR8t@BBgi;7BXAi$6aAg=(6DX zFI|?_%I>RW!Yf!2SPnJN1n{QB$&Jco8i~%q$Ylol*47bJu(J@1 z7e6fRe9R0v5N=AlwjpnwD&{E|!34g)wEhU4+AEDETI`oNEHyXNd)E(3bt&`$;qO6R z$%3`VUc;wq`AOybSvC>1zsOhwP|EYX;GdoCNW6pD48#gt6jUZrLx|LFq)k%=+(s7! zoXM?Huw~26YkP9j1~KA;6QpbP(%3r4)#11I0=bcE&3)MwwGZt}72C1DC$da`o8X#Y zzJe~SP5}mzeFx>-Ve4$q8EGuCH}%GSZp0`!360Qr85TJlA^n%#mpFxIMXXWgr2sdX zP&b+-_Uc1zITveQY?%{08;g_L-fT^jTMYh7&v0m;kZ%d3wJ6T)HQU3rQ1uZMd;5B;x}9W#%sA$ zmC)L^cYdzz?syI=@q2T^Z$)c7jz{kPy&TO@g;uvTP1n(eKEwTp@>0bvweC7ic)8iA z@N&TG%FuWa?V1A&4^_^rd;6@EZtOuFD8~H_vTqljjH21_7X`?0s#-$?p~j^$6wj-Q9K@fbId6E-`Mdn}Pev z*D1_0lA$9lz~=>QL~b(PIa)q1Ub2A>paJE8^A4b2 z0?zM}4UDst#vECn3Gv-X1&cR0g{?LxB{x*&FO@AG7pUijx&z4at(6K1cNA_!cn}N@ zUw%@F_f|H@Qu)J#e2#i&*)SPw+J6iNJ|U*MvIGj2!r+*u{Ord`zEns}T<#b5uw_DO zlrGdf6e*Yh<4F?dNb^e*l87P{f1#-iyUds=_@)87aRC56Bn&WCfDCP5{(VL231X}0 zzrcEqzDtv56$|F#>U7!05-#!T$z*Vx+>EaaJ@vEZsieaY(;7mVw)p#!z2hzbI81!o zih;u!e4c+7{8`w7N3b(pz{2Sa^V6p*H~$xV?-|x)*6odt0xBY+A|M?LAYG}_K}1BV zAiYX&0qH%6h)5BY7K#MvJ@guih}6&n1QH-B5K0IU0)(D7o^zg=XP$TF4A=j||I_;= z*PT1%-g~vZ)^Dx7HvQ4vPqM}w*8b)*Eq$G~(CxM5o6xNfrb{2M@gAY5JuVn>)M!90 z%D7~|h&vk&JIe7tsjhs%cpqBus~3(%M$eD2@6Ky8RXw77P+JDo*9D#RoKm@J_J!J+ z_k9k_>b%a1%_XiqGb*5~FSC4bHo#a!Li-E3YO~9gEoyBp_=O{z6}iWd{dvEDrN&b^eocw&6{g zyxJd?l=S%rFk0X?a=$SU2KH2H@%j(RrEnH2`ufIRLkb=Bnt<6DX zr@slK`Pkq7tC#p0(umP0yxE>trL>E#^USVOpkDalp;o4-S!=ODLrJ)bW%1r6-!^hx zcvWbIqx%wX^LJXnM+C&8;qXYH?&}8=C^8pUhgb3dz6?|*RW$3}(SRCYc`U2j?73Eo z8;+#YFI*Hd9Kp~~&<5Q3u;S{so6&#kJZUYyLA!^!Q1y*wR;}6Vq6K((Klua<^)B8H z3KmJo3PY`RcFSp3kvrtV5&e2Htj7@kpIh&K)rl&YDyR9Ry7Tb zq-8|JFhR;@R5h+%#WM=r{&`skeKkT(7qbYPJSX@u^L+*fq*t|`+{cBF;V{6%UJq6( zQZ@kXqA8)hx^JM__>%hGvksk}&-}pKhvefG<`V@WiZ45IY`7x2%*-ubVb6}`_ZLd0 z|JFHXKgZCB?1^64RrkNStF!5H(VXeW*C{}QY<;kTg8jf6>f#5h`n5DStF8wC8H*sR z3mbPOjn>^(AB%?k3?diCYRA~hmg%$9yY0B8Y#1HKRaeaS-wOFwVN#j{kU34|w(;y{ zxGUACYL<5LMg?lYf#gKiI?G^r^-o2s?4cjo$U7Q3n96S85<^*VL(ZIqwPmZf0m!GT z5(+#KOfMAgeGgq4vJ`jHPTSP#@2}jrepXTTOa_Hfuhy%Bh3;#D1y+e6=UQ%x7CbCJ zf13YbyEJwAe8FP_*uLAjH=34L5$?gyWiIOOj}FFbW)Y-PDEV(f4(`6Q_52Z_@R+yo z(Z6~DwD3G$Kutc+XM&*bvPvOYAzK|i+_T@#-;eX}r8l}ZEwTH#&TfOh3V7t;`Kxsd zRjAT&nG;=-!>Tw~p**TjKv9lE-pl-uFx=kUVCCQJ<8jaeOCJ=&tT>lhHfcxRWPbea zmCTnOYTnQ(=U*{X!)ut{idcZ1e5Up|XP;(syJEvExl(;xI(L0-v)g*%ZkSK{5za<6 zqiisvpRbioy5<_EOrOa6(EGrV?uq;&)c^FJ5lA~vPU%;rbdZw>Ngg|FCM}YgiHZ}% z`k!x?7Cd@*raR*`byleK4P#h)U$be2aP673RWm{wUP^nfmNk-dItYvdRF2=c>AwBi zPmyw}us&hb1UZj5!SEr-%Y8#tUQ3{1!g%4SWX)qH;D=-tC;AXcMVy?){y0CDoclj` zZw&$+JV@fu4m7q~K2!`Qp5in}RtPx^f68@T>BUIC!0ni#DLea5LV+M{~0N z?idAcZG4RP)Wl7`s!i84KOC}o+V9#rsW9D=#-@qnJ9`UDcX>|AVq}%XxX&^E;>(kz z!-?luzal55t%Bk*TM5D5JH&MdC9?d-7VY~ooSf02DuS(DdD)=tf- zxlv#PqcJ1=9M_BAPXFsgNuR0c(6FEwzoOk17F|#m{yAwx?)vi-VyyGgOC(hVrNfv= z;Ca8R>ZP@eJIYDI9ten*bv-}a)iC%az$g5`*C1Kb;i2;JMf_S1xW#A-;UhxA`Vz1m zXt!^1XNH|v!bpG|y657~nFYYZ5;-Z)7)?n!w1m?`D;>q-oX{7<3O*gA2(@^{@#@h_ zZ%EnZ9SzPc1Kt%8)#1DCm{EY1#3@B4OSyhvc#(ak?~t@oxE6IO!$#$Q{LI147N&tjSM5+ z3ANuzZ#3A}gxlz;&F>%9=&#k;llwlvWuHn<3c2|43lZwa{GB-noxvmtphezR2dRvVX4`X&hiiVby0OJh>_LDuq z5FX$7rM;Ao33`bo0fp?hFj9W&!6j`^b)OFD33;ATpzy+R`WW?_`Xn4>YK%y{&I3NcQvV1DW5E2 zsbTkVAbu##X??{vcVgP8z5jKZF;ucgcmk_jIcx&x=%89I7Yp3>p2>Hrcrwvc@f`KW z_u93!BCM2$<@A>V0TD1l)yP4x(-=Q^KiL9)@LqzUJZB};p>n*fbtz1eXtI{Ui*#+m z%ec!;x*BV(uGE{rpJ`Q4Rn3rANyp^wH81fJ#t-%LCU4EkxEF=LA}^_Ri6F~mXYO7h z@6JMw;EsCjQvCLFXT+%^?z`6mz_o^8PUkYShB?N|t3hK0T0xCkUuy?44+xk;VVXgO z9kEm0>t`Q1#+4m*-1kMhj#{E^N_^Nv)eZt#u}hj|2Bm$$qR7x9JhoQM#9)Ket{rt` z)qF!&lc0Jqn~veW{%`8{+iBTEB{!WbemrTJK2~)T7qmEm`vq=uZTFT1f@eJ>bsrLN z9_bpBwIhWBys|fYvw%mzflC;-y_-(c&04sU9_}Nry%w6GBpt^VA)1sUYp9gxU<&$> zv@Uu|1Z)-H3Or(%ds}c&6iCySk_9|4=LoR-xLi@OrSaL8M_C39w)kJg^+%2z@5oy!SjWi)NJb?WCyF7rv`E0eGzA=OL-mMCB)!;r> zg$j_i@bX2J(uW-nUId@U+8bPt zhn6y4kYR=&FTS!L%oTHL#OU-295I=5NL_o`n(`-Iw4yoiy|CFE!i3hUD{uV3VH-@$G`Pp&hExy z<(WzpK41_#dyLKp9Vu4+gSa`XJXg?LSIL1I#Ob38722OQD*@x(EulnCn^<29&pme) zvL={+DG%p|G z+slzbkwbKvESKg_+e27nAGYmcET$`c!&@O(3CqMYW>f{n3Z*m6VNYjb8=a-hpL%kN z-+mOEAgoiA4JE$8fZLi6SX@xyggOmAt^L?NYoQzNv(1C73&=u`MG{l^;)C?~?Y0|w zff5#$XhhfQOhzPfDSamD!qhChdHxGk2ObR)G;xrU4ZtgBOInrhRathfmV(=4jo_}z zg2Z-rDDr4k3-Qi7(d$5H{-}9mqV?s=kGd~ zd`FjUR2d@FUroAD@ps z*=#3TF&^nD!5bLp^2uu_vPD%DY(+&}hj-Th6rIl?whXdE&#FKm2y_2pq2Y#NY=y(X zg)+mZ@Cf2u=NXI>jUZA!N;0@~)`1dxk+0-5_HLrWp%JmVTJ#_?#xL9(2C&t-1lOoH zY<@Q+ttY`5wB`;6j9RDd<}7_DDZ01nC>(5+J}T{M2^nscv#<|w@1GR@5|Rj3ge$%S zW~l6J%UK`(Ab5ntR?QyGORp85J!FB?5}v_ zb>@?|*hZra(oO4KoT1i1E&dnWcuMcQc7?Pr^5fmSW#HIFFt zGozw(a$W;Yl~F)(5jmV8LDodSXE}Het*o{uma91?8!a(7|Pxm65uM^5<-$#L% zI+}}!^F#h(Vdn=p2wQyxP;MMMrbpzE*no zx><&T`bSk=PX5&UN~PI6N1E`wEXOt>?qm=MR%e7}^sRF}NOdA|rx%3n_$JF79Ut8A z_@ta1sqFCJ;5dF8Y0OHrRW9N5OgV5HPoXJ18?w{Bxk;K9PQJltBN{@Q6_LzPxY}wv zvt3Vf8NuKGPVtbX{*9AjsrmJwtYRSoB@gj=;E1)(CBV1M=kTf-IKs|~qT+1FY95hc zJ?wP_f3pH%kT__hIHlgo=-N;*p+3=Cajvc$ryykex}V-1s0E4bSm*8p2#kvory1A8 z3=q^UQZxqfLlu3YZ)kSLhNG>Q_L#DQ*XMVphrQ)58oe`4=?bDPtq9s**Z=R;u^D$f zzjFYR8ZY)u4%2GW{sbW zy_=^9n_PyiUM4hKsUH745)u6lR%<;a7BpvRHX5Y-^*+C7+`TW}+KQXs;G0!Cmr}w0 zL31u+R$caXwGiXnpopz4acGXhRm)22Xshd0lggZoPM2P;lR}9KnAtE|w&sJOD7P2y zggNy#evW(CPNa^!kPdCfjRjVQNau=9sZx}QKhlP;2M4`Wzt7N)A87?-DjwOrmU~W0 z1X@+PA}&I==d8|b$&VZyzg?7HhiU=lW{FI>$gP8I%*PS&ACHEsCO3>LLRlIMS7Ppx zhVi>A^$#-@x-Ab9Uiw5i7>SZk9=ocAP8Ycs8ZoR8LJ~{c3K^`poGIg6V$#!JLDzoUcsR;f;cSVUpmjF~Lpk?K zXAat7kRUu^{o*W--1p3Q;vZv~a6*f_m& z1l&fuZ)i(>{h-kuS8s8}1*R9p;nPC>zR99{cH>3dy{)S#1G`#Z2+lS~sKY5}ymoVY zG%72&G@od;^f9p5UuNu486Iy8d0XeJ$mpDP8iWt~d`X&=^TcnbuP5{7QW``C&q`|i z=(*ap*`OhR>QgwDGGl#48a6cPZM*SfaZV7)&;tnL#AjZVyn=H~#y2y)Z9}r^LWos~ zmH`IG#4I8pMUm6%g~w!3w{25l#ZcSpW{y`SGk1G9pD|f@_R#{f=7Z=0H}=3$D{Abr za^tI)myC#QH!8o?PJ$1bRrhKM`h}Unrp~$O8j){}!ONt|BUi zN)4Wr2YhH-Y8u+LUM<{QupOSZ?|c{l@%Fo+9cI(?dcsjY)E|MkJ?W}kMr>!%-JKB= zL8a)qAN16D5zrO&^I4i1!upm$@CH-mo^3Vy{r09a3@%2njs%+8F+=t3@G!Zt(VoDz zh!IFC1xe&u@FIG?irRxYQ^vQF_>FA#$%5&!M5@bSFa_~}@e)Q3a<7BwHfkruw=io4AYX-d-+n)JO+YKtRp;9|!q43l zR7G-dkNmCu&q`sDb=Vic&6+*cGT>f?I#kP3y*;j-*k-oK2#@))w{)oG-c+2pwLCg_ z5bGIw&95t&sd=p+GQrhuXl*aYst8ww?bzWwtOuZa{W0NH$GH`Qx2yuHmcqNfcOKh8 zZAL_15K*p;Y%T>76$(D^KvSu~+?9bE?UWwQz{%r%e;s;r@2T%$30LTEZ(vwSHL&4( zgjX+$YaJ{a7fi2&6PvUO(ZlY&}i{Ja-C-zREIcY6Q; z&m6m%&{HyCiPfj0Rw9yL#WB^RUfT>(umF0wt@Z}R_eXS7J(Nmi)jgWn;5`HICy+5# zVSPQrZl{9aRx0b5=@%Wc*@c>0Z?O>)xD$qO&s@k4Q{-MsSNeE%Jy;JMxM?MM=RvE4 z^NV48yT}D|q2*cy>z2?NTG92Lu>KWZE9>DJZHwv$X5*uIOPTJa1;rpfH8f|?9KM}+ zxqu4XstTQFZVV2iZ@g^AUR{lZxsr%2Z2seq_6n&l1t2Mloda3qau;2L2K0tn`atZHH2VHI;^Q>9ms~ z2Dj~+0?%RMCnDa>Erx#LfCbM05b|+|7qUlGj-Kk#%S?#$c!l+IO#uVb!lhq>aM1PF zF#8=3f_=#f=Fz(>uq|IP-`o*GMC6^~hREC|g_=3mIH+|69?FU8A6%Vq1srX$m9G2t z`pe!_%e(Xl$C=%(jVJB+j9*?*ZceDgi@%Zbq=FH^-D z*#=H0#Ts_MBo-SkYCO zW3qq>430GiU4){BzbOvL$3um&fy<^a&%IpcZZ93H%urf?`E!Qt8CuvY-VP*p#h#g^ zMvOO+d0!fMM1tO!DtR(kIuxo}`I2~dHeh8h$3l0tb3trS{ls&$72o6! zJP#ZeC5{HsL5PL>is4w(p>ldWFsCT(H=C|OMP5-L_`0d){n+SFjL3FuuLk{98Wd;7W zU=m$HE2GwSU&J5iANfk$H2fP{R~$K;{UiMJx<*mn*Ak)dTem0`50MpBz)?uK<1z*6 z-5hBgc&g$kI&2U3lo0uDiAl}f40MDG8I3~je>UfAX@oG;8bo-6rSl-U6Eu)iy?G$c zn)0}13hmI5JS#ygg%+ScaJ3czP>RFosH-Pbw1w(yDMtz;qQ>-j+J#YG4SmzO!|z4f z0()4W=bSoY6-EjVy_4-bqI8XnO^Eo`nxz(k4+=|Pehk=rvbjZ|Lt523TON5SQ@Daj zh+NUKoZv#rEXM2U60Qyi0LK>?!^|}0)5dS?v;Lu}s~OsNP!m3f>T1DQa`JsFz~=tK zl%$anVlAvvl>Q2ROU>-whg9Q0;bo=-k@o%C&~Ld`Y^xDUwC-b3xc&D8Ho~Gq-qkXW zXa{Iwf!woq(Pvc5!*J~Nsn)*F2{zIm5_i7$MH5j3!(pb-H+giQeC8q*lrMi+g}{2! zg`3xde{$}Gj1gZbERX#b{$vR(>~Z8#?X#A;Vibq6YEQe&BW}Ek#)Ox;cBpV(@tA4s zgBq6k-rF5kArnhMZtRHEl7r>W1Rk`47GF>hH&Sr2V>5Cei?zpyNoOXDW9wHpet7Gz zF%|D~>>eX@a?TIjTNt5OtZM)_gkLjtY@J+g%3ewbM=+WDq%^oxwe-wBM!Iy6=EAyI zu%Nae1+2vPEA*EnN6jf_0x_FCUN|^muZf8xfn0$-&Kj4G8u4ELp)pvumW%k}7nnCI ztVP;qheFn&Z9TO>L|g|&R-1*6s%VYC?jdyp&KOMwh#zHfD`XJCxgDl_UiewvtVPrKgPEfBJuB8$eF}A;P*!6&m^Z_I^zGR=gxCG=f8=m1)F2bblv-s1Rg;%)rJJU9a0y?U~(BqUnRnjH&>MQnli%0CRx~gWP zZCejY`mCanKt;CuS(R3+Et!qAyeK<^A-4k)nUzJ}1*nyRBr%pQV3>SjVo^3&gA4|S z)vKNw%q3pjml>thA;D%yuO3$c4V|TttNrchfJtCMVDqzSCr)A4-@?IBZGD;ZA7*?q zo$xEAH5K0>XCd)z+x5h^BJ%u3=Dt-&W#d?A+ZM<7k}BJUNcDG9pJf~3WtnnX9~vyR zeF4Dy1FH5&i~X*`RaqpK%B@a1@FzvI++0km&aIZpH9-sv^o9ec(oNVj!gy5kqF;q7d?zD7NC6i1Mw+fWOK>^gYMnS zFbPegXe0TI$a8EP<+hbrBXT%u3allz&#woJ0C_*18V@@1TPd=AJx%iP-A9(X`_`0R z*ajk~Zh$ut^Cg1BOwau^aEdAV!dSj@rR^Hjz0wo6`hb_U+&L?!j=sog2n2Z zr+xa?jz!?aHyZ0Kmtd{xwOL~BtfS_(7MASY``|cVPMcP3X!9QM;wzamRqLg5@7i3~ zGiH7D-ht+@D|=<69(mVeBVRsp$XXlRsjn?N2MPKSos9Q2VpRDz1nQE~r()`5{WmF? zd==t@FSN68Zz9Yh0%+`qdhzs-0 zEk>3;ZE46~wD)<#cIh%id&9E8%4M+(b=zWq!r`b;T5s$)!P{#RWtF5#=o)MGWzCzC zdx4wD&$UvV3rCIe7+%pyR#VSl{V!i&D#SexlAFBZsverLhtg<}t z@{)Xl>+i`}CMTtXt#z%)_VYTUU@DcwGgi8wFmZM5W;6a2UtIbYDr`Dtm#}i6ZDz1imSK%>0^Z9H#9bOOK(-2 zQqVjE9D~MM)tDM?uT=`i_w}zyV7%q(y1T2|+H+eay9q!+LBIErLA2@4FHN@y8fFXB zEkhOoa}t|bQt0Gp5rtdGayxPL;YbDn!}No~__?zM;aG#En$@7$bqewneq(il2k5F6N47HwlD% z)mTQLa(eb=REAY1)+R5w?>UO5PB107#@hZXR^b6`ay=MT=c>Obc5BLSURng{nu>C; zn5e0TP*u&5nvdKyRYJpr%$mo{#bjR{qD#q6OaZD>Xxcyv0EU8p1AM zb-4UP)y><{2>ECtaXs5-)kQYDemLZJ{Ry9<8lby;@vdufs!?0uiii)*K#-5XsRj5V9>b^Wv z-;J3~6t8X5AHRh!aM_05T&&{26 zf?Yu-Xw-&_nrd6H(bIOvs>N&{vE%CQPGU9XcDj;BtLS>IP`#E%nZfpDdI) zD9I`xJ-+>b@Q^TTN?zdkO|3TnoJk8mO$nonx${;j?zE2laZa36XfRe}L;S6n*4!!+ z+by!}d{UqsQY7;4;myE|zFXNnl(iBy!T5FqNb{jXASPOUhru`)*H)A0Y!UnFx6>b}8tJ0l9$frgF#kX(!(U%l zswAJTc5yuCGR?AxH!>s2_AD8)`55;7(H`tcl+=&b?A3GPz;H4wC%yBk{SmrtwC*W< zmpBzhSL=+M^~3T;?6BRIn(?yFb*bwN(@DMm>DhTXi46tm_1&8bt^SHBf!;tS+(YLP zODN1T;%qX z&Ca)VPVAM9F?wtf$|YCniR^iwGYwxHyhkc-nbWwSQ-%J$P~g9UhR;LSM+Vj$Gzeqa zZQ@POyodArWu`(h3-@MpjXbqnmUWk)&q|mPEzPYD2;9gEVsXKwT;J(or~h85xPa=z zG*^BvJ@``?!u3L_ld;#r>IQj%`(fG_5QgU{@$#+9)I~We{Qe82wbKRHV^s!BxiQHe zmi8C}Ys*E9!K#>euAgaCx*7UnrOLSWv-*EO=s(MrmhlVgG@g*>(-J47>ZvgDoDbzw@1KxqPG88@LHv6J z402Mc?vm#`kMFp8LZ&IvGjbUhP;;95(f$Pm|7w(w(}%lalox6)|E{?#Ex*m)k=Rvv zK=sGHfBXC2zxT(zAKYZ~({hXN>_1QLzohh2$1i37Mt7Ov!Cy)3U)*~GA`9k5LLS@S z&Er3w9Dasu9W)~pA3VMEAD{f)TP{m7)xK&Zc>OW0|NV+grVdfs<4aGJPdcUm>Pv$3 zBI^Brt`qqCr#Sum#mN81h<~8(|6{~oD9rz*rT@QZgpy4}#dU_b($5vK*^@)D@Y2zH33<2>5 zZ(U9GOfBa{Z=ZzMWEuNxJF)b%+W5I#+2SBr7}ogMx21DwWw4P&KKz>AnJ-l~f-ulZ`@$L8)X~LeO~U&VF^`<#y+vlbt&FKe8Mg0iT^)DX^i#I_ys|h-X*c|Z z4Bk7|-BkU{_d&dF&je2@%I|-P^0M`~pMj`Y!Wz$G4cJY0e&f%Y)}P`q*H1{xx9Vh9 zxZ9`J#Tc;=GnMl4M|IKT^A}P_4IRXtV?`3**I&qLKU#8nI?0HCaq)?`h`y^tj`fM=t@O#A9NrY0;e*hoJ(B8PxZ=#!R*t8C zmi@n-ZV_ELX1nT1ee|D2@$WrnrBJdJ(C1urJK1E(D1;?hs*9QYBx6qa2whIHo{mS! zpiU@;mx`Fk!O?1D=G&7-!Q)Y6u5M=!NuJE!o}ye;B@1cLGsoPMh9cX)z>v_Q8m*JD z@VD@w%j9s%a%%ea-}%=6LA?JjY`B88FLk>b?sCrj=Qq(k2T!p7AUvu**>lNvTU#pc zLo~h|6os~0D%We6RNw4-Wzh<6yZP3+Jp{`>yfG`BHE%RSvE&Duzu5aTqmW)vXaMwt zPGlp4v|<5$j2O7p9;o@}Aos6nb?Gsgl)9n=bo@WZWu-T}jWQOdOM@Ph+>4b7rCI%D${4`O@fvNw~P4(?LcK$)z4+tt-k&tUO2M`?|<7> z4b-UGT22+jnHhx!FDq0)dJ`uki!GZ;p1!?zg|M~mX+71g_F>%PC!?J<&$57+R9tqZ-_40g%K#@?nKDfR+?70{MZ%y9eQYR zq3=Suh!iY8u{!@zt_%MdL)cJKSll{KpMs-B)e2x~ov(D^!?0ANTf$6Bnw+1u6mwvJ zEWQQF#wX?*f@$6M>Ip;aPY^%53oX=T90X7(a|F4T%N=$EO`GaXm&8wQe|R!$H~zD^ z7^_1=Sp3lg>ibcCY@z#Bwkpv))XR{ez%|R{x=0Atc^b1V5Y+eNpbnD2i)7Y7F!p`R zz`&v@;Q4Q*F^TOJZg-dB=W7;_`m%wsUV%F7N!LXz5pdbCZV%MLZd6L|wwg~8>Y?w> zkR#;@jC^?Wm7mmRMWtHRncX~}grYR>P0&yMoJ3?Sl2;F}H0qG)uG+SoVgnkIp{l}x zZ-)8LFXy3acs0lANgeP}K&#x|;zL>d91V0j`cVP zSIXQlCt3V2X_GC%a~Z*!)5xwN`yT(=`B+0Hq;(qnqBHNomMinqHpr1aFUPq;p?qOn4b(eZQBksO zorbQVanp^fzeIqkP=qvF_9tn?<8G&|Bzt^N6G5v6r3$|@mkE9#%PiSg_i=kmbdR{i z>MC#S5bZpzYF0NbFQ(3MfD(qV6oAwkEYg?S?Id)_CAxW8uWGnHypb9;H_V z^+CQiW+zed2RC(@sV`}{Z*pBy{xmLrD@SP9>1W?v6#?ks=%+-uCP>a>Xee14rq!Sw zF}6rYUzD@5eC#}@#GJ{YaBsGLQ`FzK>S{`o#$y|?ObY>ZDAkzH><9MH)An*1=iRtf zDeTY3!t5s#LQLwijW=of^FG(U%H`wzWOhr3dgBPiEjHa;0%NMF?^6!W)6i4RRdB!7 z*j1xe3!*riH$5Fp^k8UgG=(Ei4}SuKm6(N;u-P4=(ix9S3^)W%!ckq&ua!H0N4v88 zqY(H-bQ^Q@*bL>sZUNY&AQn7*V;UQydM53e_6Lg@v!#PP2Zy@jg6$+u1Vg_I{k1WH z(rO8&EH_VPi^hKX%9<#hpK3)f3y*3IghPUw0JGe8bimX(#kg_N{4{fkmI%3@p79@l z{=5QhQuOt>Zp4Dj5WqIh;;Aq_}=r8s@j_p3}b@);?!rmLzRkaiIy5 zB*@%sCg2Q8x!o?p3GgJeNsg52Ot+01hLARdV3@bfN@OD>qfDf{tQ4;mJO z0dCXUv6836HBaEDy&^M#=fcIjKPf9hWECq`?~Czvd=*)9Z8qR1CsR2OD z=j*Qodiy~#IgCdaMEJK_9r z5Jtn3cK+6Jtz&aJJOb`z*J()ljh0rpPh_k1THJv1Y{j1Co=?Otn;JS8bEC4SVJ$t* zzwDk4(AbPiHiqA9c5fFX^ek51u42+}Zwt*ooT)2t0WI4pyzeknuWOR}?mzsyzj~|0 zaH{!P$^9>J&rhGd=*GPn-rK5==quyit5zFM8EdxpZd#bSI5*+0P*eJGfBA|ucE#Qc z6{9^SDWnWOt(Y;gc5zF_h>k7VJ|Ji4)3}Io641aR9LSr zil`MQkYnmi)DeXp6a4hh$SLP~7Lud|XWhqCA(j35gm;6xDDRtRec5WlSktzS9HckF zk)mVcva_{dFHGf*Xi*`raj!q=3l34%wdeKqm8w@fpDp^>ncTTc7$4$lnshf@H21aL zREm2DtYmn@&if7B+T?u)$jtRl_m3jE7Xcz-S}!KbXpe;kDJQGEQ4dDBik)K{U_V*I zKoovji90+$9oC9X!cFXVFVPzlUP|lCS=z!T=Y3vCYxi(Y6+E_dIEnxIjU3H*Ub&Ea zWtZ^DrwKnUyt$wzEvfq;LRF^V)-Dw+Zc5eyEZYt>zZ;l8BjH=#Fr#tNh1Wk5Jt$j# zGmLXW^|9ti&%Bk}d)7H~oW;)W^z1?-g`7}Sfa89gx$R5dYAu+E{*p&xVf%~|DDKF$ zFg3#7rz^~IMk^?7^7-*GTEc4y3RYszY_}?W+UK$CzhfeQi)n4aXhqb)Qp1k?(N8Qu3R8<^ycO}vo#pN+qfddYiW7i zeavI3%w!@Z@KE%r?TyjxE{12bVV2`6+RW{bFaY8DdL+xp&706Zi3-RCDCVm`Fl@>( zCAR)CA$KLGR0sR5LliJjwTI|}_F7a_r1*zPmBS8&^{h?N*Nu4uMQBvm!8!lx1@Mni zS9qqSQdVArxCnc5;i~&``tozQ`?9Rn(n>9B`V&v~iC>A=Y{}g0K;Pi##l=Oj0o`iE z;$vdji1Ouh-ju8O%`X7iIn>yusEl~#E;l{JP1qzOa?)nsS1XyZZ~baXNr}Y;A!D-l z%%hW~-8UxJ|01r0+aN4{(m$X2bKk6`iz!GwDIOqap}m(9-Qn$IOF#9zm9nm4Sa9|Gg&)4LoD`~iu*vG^=^Q}Jt=J@~#xTVQgUQFve)?~&erB!69H zJ}`sXe|S(_d32}hZm3M*1B2Y8JO^bn(`c8)7)_Sm&0#5ivuf%FcP9A6I}gN@bR%r3 z#ojB8a!sbk&@@QKs+fI2#&667mka7p+f{Tqu$3i(-uDE7cady(_r}-Ob-5|Mn~YNq zFUjr6+(MrQR|dbgPAf9JMSqe2;+(h(IdEs${+_$pcA8!E$|8cALX-6|Wbqa{|9HMn z9U^s?w0E+!uoC=NI?|aFo#}$FWzDzD{NkTms~=ujbApxj9G9-@3yeg``#Ti>>>d0U ztn#V&7a;ufo#4SqbB*LDSf}yd8u(ir=084gD*1&~PTYh6PvUp|3M8NY0uatqi~WH! z{qI-lC1ealYrD7F>O>?WKf>S_j&uLZg})Z*|Cj)|5TJmcjDhIOSFbUhNEpZ^5e(s9 z`Mp!al_pJpL#z_#p3`tK?c&vlZ{LZT1>@i;8ArrCBG$ zjm=eRqjh`p=K;2b+s`%gR+i?}ge*?ht0MxokkJ)5XHrN95SSmru4cNI-K<(exPyi_ zP=jZOaKb(8@4rjEqyFb9g})Xc&!$selD6JKs=ZNBhAww$h2wL|(kMhIUR}Qq0 zHmiX-9{-k>$B=u%8rrgXfUo-u$w@FeSDITAflDsWi~j?}a4Pw~(GC?y_Xip_(;|)% z{()@+<`!ILaa7d*ESA4x^AW`>rHbxVz9u{49Ao@--_N8tUF!UWXSk?5dFsb*-#`g|TOP1l^)sap|W6)xz?+T+io6Qa_*O7EnzH9kMe*h?ZBX%ivavv*nx1t&3H? z#F`FY#dk374GM^#i8W$KKk{&BV0S~8Lal(TF?XswU84nu`4V=RqYG%t7Cs%+DKw0q zp{a8jH4kDfq=cW>wbgEeG;?Z9GCnSr9txG=*6@apT|NPQe+z$fTH*Sc57pHnj*Qm` zy`q7kh7geTdbf?U|FIx`Yk1y2Q6W=*ZHt>Oc=K51*^4bvzA(;uRJsH`F)4*- zhcnmymR#`s4 z&`8?X?= z$lWfk3v501731;zn}B4~oGx z^?9M>9@$?SFs%93&nK{m{LY>?Rf`lIrG$i3a_Lx-Y!an@dl=h0rz?0cvk4W#<%O3o z{_H5{_79G3+1}>ab6A)3m{S{G<&o9A)I~?L2{2Mt+i~XZ)U*VRX$lCmIe%4P9&6Le zaTB=>I;$}BK@RSsRVjWvF+uKFL7^`!TOF}>n?I2nzto$bSi_xb@RChTywju3i3sz^ zKSz%b%@D{1n>ctM;ofU9kL2dtRfr@ND$H`VS4@4UO-ZW}w5m@%8j*wgxn390GI~nd zb5GP1Hw=`0yjWZs^l(b=kclbHmbJpCgGOMXwpG+ls>UO+eTpl#ff3foSGu@YC7+9Y z@IjV2w_oc7xsvdLTe=coviWfA2Un!RF9#RKwwYII^3qwC}2&0?&9bX7`*L% zok^=dWf(ZUO8a7T(<7sH%pHSix9iioDH|vixERfDKeTxozZIw!WSY7n>K#j)A{{{V z@tB^|RJhrBrZB@g-G>b0xIXX$!Qqnk6&R*)4{fZ z!ApB_Z}Ef0*+;J~VX+syj&B9w;&D}3Z0?OT-F>#$@S|B|Wpir;5m-*Qh z?+Nq3#U=B=v9IN?R^$84oG3U(Q=-hR2YR33?7sOD7H-ntQ9Vh$6juUyCH(Np*pl~j z=MKbe&z=+x=6X*?o)7EgecO1i{pe3TittI#a8KP?oA@H)(ZYM&yF)qe^!_zX(kS49 zl(J+weMYRGMt+ANUv#n%W_i`b$fnHe;^e5m_NcQ#Q*jL*(Zv&YnI&9+HSIvlFI;4 z*gwRh9%|n!%3Pb4HAqNIhmAHIKU)H&ea`Lvs8{|@^u zWhhB-O83-X_QIQ=?o$VU14$GZBpx#eb(6I|(S-F7Y7nDJHaj_hV*6+A(M~3#WUe8Z z`1m1#){ltsU`E@NR7SfH{U*MkaOWov^+7x8ZLvQT<_cpAw=>Zx_;Y4EW4@}Jqd9UY ziP26F?guW#Db(G$H^sp`Jgr~?{I=UD!nroMRJYi^q_84)FpxXDdTKAS`X$sf65F5uvY{5lyI0)>pG!H2)Z8 zk@H&SpS$SC_g+t1!x@&xhUkq&y|wMFzaYE`E-jrs7auj)%N_g0TbW2&+vgX?-au2~ z#N;{ztcFn-S*Yne)Q?D6^2@+Zx))!F-wm0RoOUr4m?rN~PfEBZ)0k(`+q{5A81_pPR= z>H-pD+dw6&-SvO=Ih8VLMyfS46Qz!5IVA5{tY@U$oL4mEb(LaLe^+M5tT>_gIbY^H z*=^+aKvzzr96+hBG1vRM_Q6DwNg*{R`+nLV93kr~Mr$uDG!ZHVzDX-E!eS$esXyyX zLF}0X^ANY-KBpCDW>pH!ISH(5KkEAmkIU%m`+=+MI-zh)8vZ4YG-tv)_DN-O?9|Hk ztXZ^qb2+{~G9$PUQiGlIGBuJd8&qV8BYX~=U6_f6EvAMErwGsiM4&{s>(kPzn=_8A z3qxWuFq2Nv{0!=uQKb}yf(g)vc_!Fh6;$sJ1FFs5CDt2;_|zj*ZysI>_JJsv^r3*J z&<8#?kk*NWkwIST>*&+kqA<>X1ACm}h6Zm!ADPl*onuEctOgvuX%$_ovNy&>fMME9GzH-7Ch4O+FzdT>S$(08e0;j?S$JS?9icE-!zptvYbz=0!j6`Fq7 z&RZ|oV7fuQ=@pl=7gFU$uRI zTHvqeL!R7w?m6Ex@3}Yhm=AX?FAqp5Lc+a@uFQv#uf*(YnH#Oew>ZYgqAK()IGq_= zfW4c$Ud^zzQU&U2&3!+ws`ky8eN9~^=z22W*6W1(8@kx0K7Y})+ z$x^nq-Q{p!$Qc`3%EcgxE_w#|l=-Iuw`&WSEna%gZnL`a(+NqD{%b>EPdSJ>{$=lF z!pFSsfvsx`GrcbjK0s}%a21b&?nT_yI|tCvd>gk**t-krRF4^XXhjK;C-$FOuexMC z|Gx9V_BNohbE&xRa7*_xWo#-^>}e5#F?Lv{E6;+uwL^USKzrio`Pa>LW3520;P6sg zm8~8)-kAoXp&g-hmv|9Qb%WIh3#Mi5F&3T`v+D(y7#CvLZNcn@jXMhOCh{Fwe$!dw z`$Asd^Mi;M-bH00wP4E{?sG;QYipK@M zqZIK7g6jzvh}NIg+jFBmw#Da!s?PS8Ls?Htt`0ZHcF8*X6@^|TsH!=LUl0Gm&<#YG zrv^OO^?1se?+SDxBY%1Uv_*gY?8S!d7b!r{VesG>pP$Ufg;G}Warx@o=pqCW^e0Ho zuIr3n2)JK(6AIJ?H0NhciQ2P)o0vnCJRzq~b@o0xo?R*cG|t>SzGLGN*3We8jQ{IQ zhwIg6dTKE4tU%tOTf47rxPbi^#D2XnzkePby|Hiok4P?{fGzHV_?EBINn&75(*XQ3?O+1IIQ4BVdZq8+;MZ1)5@O>G!9q@M zvM0haq6kX03)G3Qc0m*Boq+yctM-ned*dTqE<0S?1fZX7*{S69o!_(F(bu1q z-pUxPQJT7E0}Yk2wEmDfX<@wV4}{IjjebIw$pqz}DZ}y1op@V0`G1~} z{T78EcfKk9!TC}*-#ldc$@iYNVkZ_m3NOm81-M1K%bqTw&!nNnfu#J6Rd39mxpAaq z{6O!W!S)l}Qlv9(>NK$1oo&5dc`&eZ6eHSqsr*h*Dl>4G8zAu)kA7-|$ax4Fdqoc- z2XdVMU&lk=cjze5#bJV+u>T#&D$baU7vXQEf|_~XzjivA^LRaPM>Af$wwl6NZlKM> zaYN=Ypq=Cj-Zd`7L~}qK0&ar7^6xjCYh3#dR2Bj;?2L_;P zG^HW%)H4@?{B+sze%-Q&*XxZvPVdt?flXh3%@do^p=lYuvp;O>w@B)$cnUgD>WUEScbBY({sTxsUCzh?lq8-8$wK?$C;(TE1@Gdvy*oqukP1prVtO z(`N!<`C7QC%dk8IvbJYZfo-GdzSCw*jjECwx9W$T?-CRYul3S|9a=yc>L^@OnzX&`2gYz~!7comCM`X%v=aG! zbUAdse;7K8gS8YL&8RH?QAlkKNY%&Z4i>LiKB*e3+}AmoTUD-UnA9(PxPMgkMVfZ$ zNi;umzzhk#-F`?C`V{CbI}rpHC^-FOm1<0{g1U=*#TERoa)gG-t`}rAHTM=nkLpLs z+k@Xlc)2D_sK$BgzX_v@yy)JVRig|;saQ>x>m1j+N=U{`?v0<7do+4Iq21x_FUTsP zsPQNZ%GfiFPLJCWo|nD{hnP&3rv;I0Zi)>61=GM}$)qfwwyi?H6a)D=%B~5U%+$zE zc5hqsoqgK-2WNcAb1ji7wWvMGPmv{VXXlY*75)SfR0Y3m;{1zE3%vkHgzswN<5Oo| zxaBIT{CVon&i6fWRAFm{lWOJy2cH`MUUU}(#Q0Abok*Q(~nbh^bz!aU{?)&Z+rMi?}0Qp1=KvT#3ecaFFHjF?+ghQwL^^L6JH_e4J2@%x~1Zb^!U zx`_m=(q@RO3L&%(9kbOa12+i|8ZhfB@%|=X$uK*?`9_Vz8!-xu8JJbQPINj~q+C8u*8`oJPoi z9O%yVYF~Gh_$#Nt7x7MmO(3GC`)N7U^*@zn5E$u#km?q~?#o-*m7q~oFz%z}XxE(% z_5%>Q?9XBOd&zH^5TN5C!8mzd>dg0h?i|?+j<-wg5AMG7J@v=jo-8(f=M?8|&m3cX^t$Ny!P5ol zUj0}KB>QUYPynC$frC<3cW3v07qmzrnzjCra=%+7-@~FLUSO--Tr$D0RjO)b+@0G> z`H>6f9ujD{k^pT{UTjy5`}2w|i5O*m3F!N%g&`NN>ph*)o6oFdLZ$EUT0TeC{}Llz z5D$ILo4O(uI=*x3PIYys!-7!KB~`|3W;rFjmRpV_Qgp98JUhg-PY$_ng%BLfzFKcN zB8eiRbFI~gx2i6OQP0_4J4&{3>LBeQ8@GP$1(>pQ`P2-M_#JRdj^J7G%Ve||zdjYD zu|-rGOh0v^Sw7xAtUUG=cK3Qu;jaacEP~osJsdh0lgF=@U``OBo%(u;&imZt9|mPz zyHz0v68>7+e^&B+mi<5hsvWMS414%PwfEzcQbVD){prct(AZLe#&=hLmDo&fTf3gy zdM!kIH4u{mE$=AoE&Wv=Xlpx-;niXu1>VU4uLPlfCrEV`+Q^>7re??etYR2WOoo&T zjH(U$uV4Z4xn$rX_%r#+@q3r5>rLc))JyxM)Rxg4GKO!N=EM z<#e@LK_U}GDmdX<&q1no`!NMo^WUrN%+0eX}JZN+15 zAwkIFXwz!h!9PJ}kGF4Tuki@-moSae^_~3vJ5_v%mw{_~|C@diUxEtVX&AqtD#q!dcNBi`p;Vx}#V~d*VH`{OcDpD28 zK(dA1;)l>8Pvg-*gO5p)Ud+BUT-c?!inpasC9ao?c161UwL71+>yKUWy80@Y^m-4G z1@3Zub>hVRx9?~hd_D1xe0F;1Wwg8G3%Rf7OiI^72U#cY^We7rprK)W_ya7K@he$# z{!5YV&fznpRel9G^ouPmDg~hV&iLb{j{D}jhtp|fz(pHqxtL`FwXhMopSGv|8d~d)x9c%&uz@ibC1XEjEic$y8rjze=2)=EB5#LtHuYS zq;iV(yxK2$%yas@BDzS!*{Q=$Be-L2j5TNDJmb@$>BH8X56(P0(>G6`Th}q7TGAxl z@ZWFY+q!eV*o9xd`fn;}C$Ld?R+?y2(ee_z^PgPTJED%w_esfvD@>tDE0|8lE% zo6MumYa8=z3tyWYK*fO8vB?vgAtSMV;w7~Bt?}-fkDWIcRp8?YD1w$2%`EaynSoDD z4!zY_e!ZNjj=lWj1_A??@z|<9+3~T}(yxnJ2xu*LK+E3?!m8u)^r$2rZHMGcRu}aC0eu z(|WA?!ZUWOV2q5(cnsp+Xn>c4|Hde+NA3p1-^o95?r~WaL3se`>^ym{yh3v0CET9c z21tyjUApwWP2DjIr`cSn_=e)Rc@coIREX}L^P{hVRV3A5tSad} zgZj#cPL*Hnt-xbeGM(|Z+{Ixva@Ng*8zbW8^>@89b9rwe+63ocZtFcvNZ-35r`J~^lLOFtl6PA;>R&ACr;l#W@-;6jg4Lq5a7Q;a z`t&YScx$M)X^n*w$>8CqVg5xF?@9sGs4_5xo3h9K2Lbidetz*^2K@BVJu+ZoQrL$5 z^P4cuJEsA6I*|TmV;$SM{~Uu4>1e=}oZghvV*|E&+UV}a(%-r>W!>jx^A55@HirHd zzF>d~_9>;!I9>xhAo8x&UBQ|gW9hq44StA}IKw-u)}#o8h@w;q6dm3#X;+=xxKT(2}jaeSOub9-<90lJj4u^IkC@ z%Nh#H&i}%A^Q*uRP05Z1O9uxB2S0cY^=popRDR3f_S-%qwBcq-4}D=7_S74Rw!HVt zhCOH!2FOv%$7E7-`Fp;u+3$YZ-R(cXN-e+i36K|VA6j>MZ?mBzxDsla3bdNxu}e{J zq3GQ1uUi-;KGsnvEWvu%Mf%GkZyEtM{s7EXh)$l<8F|Y!DE$hGfSQyE-DuH*vkY5GQ(4d|;KZtE@k%Od<@RzlInQ-U+%C2oRJe{)- zPiVXOWtP0$`upN--EKxTN5*%qT~+^Bj>oggqM6a6o?M-*6Xz^-KTY>}-oGBAetdzk z$(_lg~Z{f_APzQKGO>*Y5Td0*#V6}&W}m}-qLrc(RK zQ_%80=7qg6;IB_0dF;lpPM?1l>!=?LVm)lp=if(nBVaWnNDUgF@hKo#sM>rpQ14M4 zuOqbV^^>6JaoA!C9alr>sh#BznJgid8W*S7diK|Y)lj2Q7OTwD$0uD40WAsUe(#Lu zXaz}q;hMq2dUZgVM^qtfK^h`t5J6gAwH(mCckk;g{<{v;ckYw{Sx9(2vq%c}*}w7o z--W8_-ga7z^p_YiTGI$H-}oE#sj`KyRodE-gPwe3sA8VZ*BSX`HS1D;C(GAzHs?H} zVk=Sc9T7BtD&h;E{wd+VfA8EOK1sApiJ#O~jjdlt`4;`SM|UzG13Ok3A#gSvXp$1Q9o zN@6ji+9TNWck#ATw6aGq`I=ukJ4ry%BTwTUXP%^0H`fB9)7_|SDA&A8g!1#ZWO7FD z8NsVyhWa=2cR7?Qj2f%#Ubu)nOBPOOsdn%7>?{vo)N<7d_R?Z?n;?Aa8$?+{du$O- z_UU-|=)}v#vSZ(vpt5T#jmzc?S$_f1nf?6(LB&a%(@x~b zmXW%JOnGba&=4GIS)cnTRjuM}EtJ8@74B3027Br|cXux`She^s=H=O;7lX)Fg@L1b zFTp}cc3wzaqhB+CfQse3CO!Gug*Xd^Hg&|v$^f&~%?cjrg4Pmu{yHc^GY`MO+|He$ z(1ZPd`yAz5kiI8ebu#LDykJ{*zAn;@NV89NU_jg8YZ6&Ugr7 zsi{@-;lpnLB@`v_VrGE;KOB0qdwL??v_of3-?f_Kp{eU_>?)Vplojpo#%rYFje2lc zKCH-UwC@tLA&Hcvta;z8Vtx|J^=J7Jtv?j7*5;`YI4@Qd!5xJK9~*y_Am_)g<-2x? zo#c<2spDAs1LxK7YePXwUsm*aY5#dhguLtIx1A~Xoc86nY1e|ZR-_Yb7Exg771i_SzmB9q%}~x%Cda1s zfFms!`i1iP8}E{bSDeP5nj+i8n==Y1gQ(qx2D)XFa@zXwE?(sF@Evz2$!xaP3Kua` zxA14dzQcLO2$+Wla+WaQ2A||6aDl6xYw_iNm*2Nb!G7=fhC6GPyRi$z;swHDY=*P@ z7HO<&XinNcP#wTZ7vz?JW0qHis-#D7q3>EH(QZMLH<-Z_qMPH(1)H*M>yDZ6 z8*8DKdW@!J0nuX0SgXNzgy}<#Qz@_3gW&x7yoR2hDe1(#X!tsO z>W^<*50Dgvgs4?MLR2-;$_h&VIoevbe{6Qi;ZJ<52kkPIYbhaVT4|M;21>;FfI{%# z$5uWqIxXhoB@THWeJdRok&eZ7R((x6i7ph6c60mzCz{uvYND$j!|RsH3Sf-Gy}i96 zKku=Lo8?6Dc#A1B)xkH%8#^}PP|QU_Q&UqDq!TiPeg7kAmXO1!q-##3k5 zpi?gy0or?dSvvhb_hcKG^6^}gM1=ATVvUO#Ad@+@qm`jw!h^32_i4&_+h^X|QsEg| zE+=bE4O(96CCi}NlhTL8(W(uXo!1HhR!Cu#tK?dSmqF{;Nm-cmvJ}2y)GS%X#Hh?pm(tO1L>sXy zLG-Y0r~w9M0TZ%1k6Clpu8Jtzj0?m{-cUUer`CxHkxy_~JdLzBrn*+}s85!$cE#uysyu>UF|6Vyh-u`QAe=Nux06t9FJjF43@p_LJ?Hc)nJ?%D43u&&hWm< zX~wR^>M&n>;yEqyIs_dWoOW*cLfcfLDK4X%%enLfV0c17nw&R}d&4e~ejC3p+SZ`4JF6g`0{XG=$QnWlEX^c>H2UNioG5su@sA=J0W@+ZU*BK#159WNZ^>iCU=0e+C@NA z)%~<_2>M-U&rg-NRK@<9oOeQeB-8tw6g=HtbsC+|~%>B-TZ9b0mT%N2)v87rmG zXtQE{T+llmt|Df+J#)`txe(eO+ohKU%<1WxLC@Sx@jQetZq%qsByQfQN|d*27mu@6zt2%=HI=*G6X*kG4Sy(*VwIz%;tY`~$XpZxIa_W!dm2q5Lqj@6f=j^o zXCBTE{|ZKd8>$hlgVzpWL`bGBk!Au(!K~ zQS{;-Nh-7Q2XJi^Hz`6ik)zw6Gu!P9J=|M;M2lj3$ch)KS^@kV9j!0ql?Qwdg$eoi z5CQyv%d5JD!)Z%Xiq0MC1hpZ^H!^a-2fT}~kCL~hiNHU=g+CkFsDu4ga305Jc$HmI z$DT4oaM0OlK{PI}m$m0|++cB!EhEZ!&Iz4Vur(>EpmOE7QLRN5r%b|cIHmDsoo-KA zxo)o+zKJ4jZzc&xI)H%0;(^qbOq`SlxA%4a*7){JgIm2l0m(95O(#{eXi_>^I;k0G z@dnd6w*DNL<>q14w=xEw2_EJ$R~scYXFC=Ni?uTWi}+#QV=favGeRU1fqR794%5ig z?1FEq=Q{(Sy6qzgBd5jv$1s^ctMBaq?&GNDs5I9EEzgY~=8h%Kz($3Jmxi)6SvfOs z9^JN}qS;gj2?o63OFgTK76qNi90#=26a<|7Lze_m{Ku^Ks1Xsg$`7VmDZs4p*X$zs0fNK{(@ zP1iEVm{*N#1c5Dc#^RFN;#AtiBeEEab|`UlE=3$o*5XbOs%$Bw>RcaO=|RY-S?6#t zy)Azvea%AUI`Rr7t$e<8S-9{${mB59a~85+P`*+3*VB!;(h*sAK%%A47~Lm1%iXYf z{lyn>-QLWOC36k0!G6voXH2)C)?HXP3s}XO)r=V}^%%_!JVJ_RsPg1H@w9~(izH6B zHY_Y~QRfzI1g2=?=MBm=u(?J{@+t^20Tah_&z{9B5mZlGbg{b(tQEL%P=be{wTYLl z&Ko0?kSm6Kf;&!>WfhtgBPI@FW*f~8fTPEw&-4^@;;^;Ss#(3=A*>QPucs{}WiyaI zaT2ZyVJ4p+g?&iYoiM3$@z!;|IheP6+DgvhLk1K7q=EA5x1HF3ERPKO!rafEi$zXU z|K5;}~a*3&yLJwR|=T3{2#6HyTXgl5=Dm+xF;pJ6^qjiY+bfsZ}66f*WPb$2&P)h2Z!$QOzDOlUirEF>j~c9#G!ZW zOuX~G!SH0i;lRm*2|)qOBL&#Gy`2WuMLMnk@R8QJIIQj~XZ2ZB1hp#3(1p^~ltmO( z@;rceqS7{&rBnSg+vk^Vw4oM%Doe1S(#~D6hm!o=cY8W?u3R)F9=dr4i+hwV6_H;J zO9un#Q67$SCYR7_Yeuu6@H*4L%82eH9sr*qd1zgg#iF29a5s0~qV)p2fyde?AgY(z zBW`GCM7r0{L;BzE_ZcJPG_wWqhWTyeTvW3Ww>#+w_B>tM>x7%D%r2#z5~oBkij`BM z-D5_cr8yJGUAn!Fp3c3=j^u4UX(~m3M0-%G?$eUmLTK$~)__h@BxR|C~0UL4!FDUDyaL`^E0L^Mw-+pPQ~$ zzuueOu~cXP(o;*j6+LFxEmCy=;&b*Uy$iQ`PhV>xtL%`Xj3D+r!w$@%RF!MULcQGU z9MEKk#KgCc$0c$Wm5=7NiA3n7&R)ua9xw-KHjcgN>h(X!XkNOn5+7Hj$59$5E)+9CE9z) z))5C7AeTRsb%Kk7Fk0mPa*gtgI|mYieojsdvIm!8y90M;(^B#|ar;rTjqJ<8^jz!A z-Z8iZCg?z+5DIF=KMGP&PFpVGKnsM_w43;Y)AAr+PY}NdA=K=^-T6HNfy@Rb{)3-A zcW-l7K{c*b^tE}Fsphq~M<;Pju5!HYvqR-2gr@M!_qnCB71y%X&`Wt}k+VM&DUj_A zd#1{FxT*+i`VOrim)ar5;5dip%`cQI#u z%$zmtJ?f5Pp&T>GHG6YOiBU%N`O9p>)Fov(`$JkK!p_BDYRx{>SY){dv{0DP+a6i4 zTrDLiLFY4}`#9#kcg+L78R66m(QzM-q3||{ zZ*ME`*#~Zq8i1`@$BzpGr#AvYY3FBiP>5JKEwTW+np$U_JR7NpkJhuPkKX-3k4+Mp z;-^CkEEATBs_z&VmvbUWs}TeAiWU$$<)(9b5*8}rZt1Sp+f>d_cCu7k+_R*I@=d-SC@)Mk(a^D3}<%U=TPbE5BE4* z4LHMf;ftc|6E>rMW1UDT>xLoFT5LMM;2kP&$NPe!2YszBnfgK6C<3=jQra{x- z0rE}(VRm(QC?Gci6V0ulPk4~Xh#^?m{pPZnkD8pXQ}{*wfo+_}b6z~I#*E!!s>+%e zH(Mml@tkJ4*W7L?$@FNQI(Gi-0(~2hJ&g*w(ev^v?eNiVGk76RR*8ehL=y+)EGW@h z8byXumH3CtDTzEXBMDG-2?aAqPO!5(=**<3g{>b2$J^~^KKIBhJ2lY`aZXbiQ`tS= zbW)RL*XPJHhuz{nrjpM64=TUV_}`G<{AB2ukPN3ioa#U17on%2zILyMBdABkxlFpL zE96l6Ef{$&i|zezH%el8?Qjwl*PBNKqD(okT8qT;)E~mfTMm2FHId=uG6*>bC^Rst zX%H|oxy0F;X?5T10Avzd#gWa+4ZqKOGt^&Nl~jP7tm)ytO!v6Ip<6OqUk1FYK_u^h05I#2KG* zD9*zf25kTJreRG^Gm$rfbdr+E$4@?4*ddcfkZKGSzFk5{BV_K~4g@#9G~ z^aBa!l|AantQk7P{8ibgecfxtkb~fSEV9;>K(2Dd$u_VU)sevUczxzTUKQ7vDbg|X zDy1>m(Q>eqKk3a4{`j)M*Fi6yCSk+V%Dj5BIymLF4Bg6{pkBB|!6uL-gS{|!9BF05 zd$gCNuAEkxr!S(dX2ZxcOkED7<>^qFDkM&0H)kwFc*!}jxTUmO!<;xN5ux8?z`d5x z-a{VYv^!Ei?o|*ZRVVR>^tecR2BZn$RbDUMgE4LWn6cHVdTQJZtvp3c&^awEqTf%l zY$lJJv|*5f6)UvG)f;sn`e7wC&|~MAqS+C`l(RMM2&SEf04Iyv/commits`. GitHub's auto-generated "What's Changed" on the release page also misses these. To catch every PR in the release, you MUST additionally walk the full git log range between the previous release's commit and this release's commit: - -```bash -git fetch origin --tags -git log .. --oneline | grep -oE '#[0-9]+' | sort -u -``` - -Union the PR set from the staging-PR walk with the PR set from `git log`. Any PR in `git log` but missing from your staging-expanded set is almost certainly a content PR that merged directly to the release branch — fetch its title/body with `gh pr view ` and categorize it. Do not trust the GH release body or the staging PRs alone as the authoritative list. - -**Sanity check for new contributors.** The GH release body's "New Contributors" list is a *floor*, not authoritative. For every PR author who appears in the release (including underlying PRs from staging and PRs found only via `git log`), verify whether they are a first-time contributor by running: - -```bash -gh api "search/issues?q=is:pr+author:+repo:BerriAI/litellm+is:merged&sort=created&order=asc" --jq '.items[0] | {n:.number, merged:.closed_at}' -``` - -If the author's earliest merged PR number matches a PR in this release window, they are a new contributor. If their earliest merged PR predates the previous release tag, they are not. Do not copy the GH release body's list blindly — it can both miss contributors (PRs that merged via an older dev branch) and falsely include contributors whose "first" PR in this window was not actually their first ever. - -## Step-by-Step Process - -### 1. Initial Setup and Analysis - -```bash -# Check git diff for model pricing changes -git diff HEAD -- model_prices_and_context_window.json -``` - -**Key Analysis Points:** -- New models added (look for new entries) -- Deprecated models removed (look for deleted entries) -- Pricing updates (look for cost changes) -- Feature support changes (tool calling, reasoning, etc.) - -### 2. Release Notes Structure - -Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable, v1.77.5-stable): - -```markdown ---- -title: "v1.77.X-stable - [Key Theme]" -slug: "v1-77-X" -date: YYYY-MM-DDTHH:mm:ss -authors: [standard author block] -hide_table_of_contents: false ---- - -## Deploy this version -[Docker and pip installation tabs] - -## Key Highlights -[3-5 bullet points of major features - prioritize MCP OAuth 2.0, scheduled key rotations, and major model updates] - -## New Providers and Endpoints - -### New Providers -[Table with Provider, Supported Endpoints, Description columns] - -### New LLM API Endpoints -[Optional table for new endpoint additions with Endpoint, Method, Description, Documentation columns] - -## New Models / Updated Models -#### New Model Support -[Model pricing table] - -#### Features -[Provider-specific features organized by provider] - -### Bug Fixes -[Provider-specific bug fixes organized by provider] - -## LLM API Endpoints -#### Features -[API-specific features organized by API type] - -#### Bugs -[General bug fixes] - -## Management Endpoints / UI -#### Features -[UI and management features - group by functionality like Proxy CLI Auth, Virtual Keys, Models + Endpoints] - -#### Bugs -[Management-related bug fixes] - -## AI Integrations - -### Logging -[Logging integrations organized by provider with proper doc links, includes General subsection] - -### Guardrails -[Guardrail-specific features and fixes] - -### Prompt Management -[Prompt management integrations like BitBucket] - -### Secret Managers -[Secret manager integrations - AWS, HashiCorp Vault, CyberArk, etc.] - -## Spend Tracking, Budgets and Rate Limiting -[Cost tracking, service tier pricing, rate limiting improvements] - -## MCP Gateway -[MCP-specific features, OAuth 2.0, configuration improvements] - -## Performance / Loadbalancing / Reliability improvements -[Infrastructure improvements, memory fixes, performance optimizations] - -## Documentation Updates -[Documentation improvements, guides, corrections - separate section for visibility] - -## New Contributors -[List of first-time contributors] - -## Full Changelog -[Link to GitHub comparison] -``` - -### 3. Categorization Rules - -**Performance Improvements:** -- RPS improvements -- Memory optimizations -- CPU usage optimizations -- Timeout controls -- Worker configuration -- Memory leak fixes -- Cache performance improvements -- Database connection management -- Dependency management (fastuuid, etc.) -- Configuration management - -**New Models/Updated Models:** -- Extract from model_prices_and_context_window.json diff -- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features -- **Structure:** - - `#### New Model Support` - pricing table - - `#### Features` - organized by provider with documentation links - - `### Bug Fixes` - provider-specific bug fixes - - `#### New Provider Support` - major new provider integrations -- Group by provider with proper doc links: `**[Provider Name](../../docs/providers/[provider])**` -- Use bullet points under each provider for multiple features -- Separate features from bug fixes clearly - -**LLM API Endpoints:** -- **Structure:** - - `#### Features` - organized by API type (Responses API, Batch API, etc.) - - `#### Bugs` - general bug fixes under **General** category -- **API Categories:** - - Responses API - - Batch API - - CountTokens API - - Images API - - Video Generation (if applicable) - - General (miscellaneous improvements) -- Use proper documentation links for each API type - -**UI/Management:** -- Authentication changes -- Dashboard improvements -- Team management -- Key management -- Proxy CLI authentication and improvements -- Virtual key management and scheduled rotations -- SSO configuration fixes -- Admin settings updates -- Management routes and endpoints - -**AI Integrations:** -- **Structure:** - - `### Logging` - organized by integration provider with proper doc links, includes **General** subsection - - `### Guardrails` - guardrail-specific features and fixes - - `### Prompt Management` - prompt management integrations - - `### Secret Managers` - secret manager integrations -- **Logging Categories:** - - **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes - - **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features - - **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements - - **[PostHog](../../docs/observability/posthog)** - observability integration - - **[SQS](../../docs/proxy/logging#sqs)** - SQS logging features - - **[Opik](../../docs/proxy/logging#opik)** - Opik integration improvements - - **[Arize Phoenix](../../docs/observability/arize_phoenix)** - Arize Phoenix integration - - **General** - miscellaneous logging features like callback controls, sensitive data masking - - Other logging providers with proper doc links -- **Guardrail Categories:** - - LakeraAI, Presidio, Noma, Grayswan, IBM Guardrails, and other guardrail providers -- **Prompt Management:** - - BitBucket, GitHub, and other prompt management integrations - - Prompt versioning, testing, and UI features -- **Secret Managers:** - - **[AWS Secrets Manager](../../docs/secret_managers)** - AWS secret manager features - - **[HashiCorp Vault](../../docs/secret_managers)** - Vault integrations - - **[CyberArk](../../docs/secret_managers)** - CyberArk integrations - - **General** - cross-secret-manager features -- Use bullet points under each provider for multiple features -- Separate logging, guardrails, prompt management, and secret managers clearly - -### 4. Documentation Linking Strategy - -**Link to docs when:** -- New provider support added -- Significant feature additions -- API endpoint changes -- Integration additions - -**Link format:** `../../docs/[category]/[specific_doc]` - -**Common doc paths:** -- `../../docs/providers/[provider]` - Provider-specific docs -- `../../docs/image_generation` - Image generation -- `../../docs/video_generation` - Video generation (if exists) -- `../../docs/response_api` - Responses API -- `../../docs/proxy/logging` - Logging integrations -- `../../docs/proxy/guardrails` - Guardrails -- `../../docs/pass_through/[provider]` - Passthrough endpoints - -### 5. Model Table Generation - -From git diff analysis, create tables like: - -```markdown -| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features | -| -------- | ----- | -------------- | ------------------- | -------------------- | -------- | -| OpenRouter | `openrouter/openai/gpt-4.1` | 1M | $2.00 | $8.00 | Chat completions with vision | -``` - -**Extract from JSON:** -- `max_input_tokens` → Context Window -- `input_cost_per_token` × 1,000,000 → Input cost -- `output_cost_per_token` × 1,000,000 → Output cost -- `supports_*` fields → Features -- Special pricing fields (per image, per second) for generation models - -### 6. PR Categorization Logic - -**By Keywords in PR Title:** -- `[Perf]`, `Performance`, `RPS` → Performance Improvements -- `[Bug]`, `[Bug Fix]`, `Fix` → Bug Fixes section -- `[Feat]`, `[Feature]`, `Add support` → Features section -- `[Docs]` → Documentation Updates section -- Provider names (Gemini, OpenAI, etc.) → Group under provider -- `MCP`, `oauth`, `Model Context Protocol` → MCP Gateway -- `service_tier`, `priority`, `cost tracking` → Spend Tracking, Budgets and Rate Limiting - -**By PR Content Analysis:** -- New model additions → New Models section -- UI changes → Management Endpoints/UI -- Logging/observability → Logging/Guardrail/Prompt Management Integrations -- Rate limiting/budgets → Spend Tracking, Budgets and Rate Limiting -- Authentication → Management Endpoints/UI -- MCP-related changes → MCP Gateway -- Documentation updates → Documentation Updates -- Performance/memory fixes → Performance/Loadbalancing/Reliability improvements - -**Special Categorization Rules:** -- **Service tier pricing** (OpenAI priority/flex) → Spend Tracking section (NOT provider features) -- **Cost breakdown in logging** → Spend Tracking section -- **MCP configuration/OAuth** → MCP Gateway (NOT General Proxy Improvements) -- **All documentation PRs** → Documentation Updates section for visibility -- **Callback controls/logging features** → AI Integrations > Logging > General -- **Secret manager features** → AI Integrations > Secret Managers -- **Video generation tag-based routing** → LLM API Endpoints > Video Generation API - -### 7. Writing Style Guidelines - -**Tone:** -- Professional but accessible -- Focus on user impact -- Highlight breaking changes clearly -- Use active voice - -**Formatting:** -- Use consistent markdown formatting -- Include PR links: `[PR #XXXXX](https://github.com/BerriAI/litellm/pull/XXXXX)` -- Use code blocks for configuration examples -- Bold important terms and section headers - -**Warnings/Notes:** -- Add warning boxes for breaking changes -- Include migration instructions when needed -- Provide override options for default changes - -### 8. Quality Checks - -**Before finalizing:** -- Verify all PR links work -- Check documentation links are valid -- Ensure model pricing is accurate -- Confirm provider names are consistent -- Review for typos and formatting issues -- **Count PRs by section** - Provide final count like: - ``` - ## MM/DD/YYYY - * New Models / Updated Models: XX - * LLM API Endpoints: XX - * Management Endpoints / UI: XX - * Logging / Guardrail / Prompt Management Integrations: XX - * Spend Tracking, Budgets and Rate Limiting: XX - * MCP Gateway: XX - * Performance / Loadbalancing / Reliability improvements: XX - * Documentation Updates: XX - ``` - -### 9. Common Patterns to Follow - -**Performance Changes:** -```markdown -- **+400 RPS Performance Boost** - Description - [PR #XXXXX](link) -``` - -**New Models:** -Always include pricing table and feature highlights - -**Breaking Changes:** -```markdown -:::warning -This release has a known issue... -::: -``` - -**Provider Features (New Models / Updated Models section):** -```markdown -#### Features - -- **[Provider Name](../../docs/providers/provider)** - - Feature description - [PR #XXXXX](link) - - Another feature description - [PR #YYYYY](link) -``` - -**API Features (LLM API Endpoints section):** -```markdown -#### Features - -- **[API Name](../../docs/api_path)** - - Feature description - [PR #XXXXX](link) - - Another feature - [PR #YYYYY](link) -- **General** - - Miscellaneous improvements - [PR #ZZZZZ](link) -``` - -**Integration Features (Logging / Guardrail Integrations section):** -```markdown -#### Features - -- **[Integration Name](../../docs/proxy/logging#integration)** - - Feature description - [PR #XXXXX](link) - - Bug fix description - [PR #YYYYY](link) -``` - -**Bug Fixes Pattern:** -```markdown -### Bug Fixes - -- **[Provider/Component Name](../../docs/providers/provider)** - - Bug fix description - [PR #XXXXX](link) -``` - -### 10. Missing Documentation Check - -**Review for missing docs:** -- New providers without documentation -- New API endpoints without examples -- Complex features without guides -- Integration setup instructions - -**Flag for documentation needs:** -- New provider integrations -- Significant API changes -- Complex configuration options -- Migration requirements - -### 11. New Sections and Categories (Added in v1.77.5) - -**MCP Gateway Section:** -- All MCP-related changes go here (not in General Proxy Improvements) -- OAuth 2.0 flow improvements -- MCP configuration and tools -- Server management features - -**Spend Tracking, Budgets and Rate Limiting Section:** -- Service tier pricing (OpenAI priority/flex pricing) -- Cost tracking and breakdown features -- Rate limiting improvements (Parallel Request Limiter v3) -- Priority reservation fixes -- Metadata handling for rate limiting - -**Documentation Updates Section:** -- Create separate section for all documentation improvements -- Include provider documentation fixes -- Model reference updates -- New guides and tutorials -- Documentation corrections and clarifications -- This gives documentation changes proper visibility - -**Management Endpoints / UI Grouping:** -- Group related features under sub-categories: - - **Proxy CLI Auth** - CLI authentication improvements - - **Virtual Keys** - Key rotation and management - - **Models + Endpoints** - Provider and endpoint management - -**AI Integrations Section Expansion:** -- Renamed from "Logging / Guardrail / Prompt Management Integrations" to "AI Integrations" -- Structure with four main subsections: - - **Logging** - with **General** subsection for miscellaneous logging features - - **Guardrails** - separate from logging features - - **Prompt Management** - BitBucket, GitHub integrations, versioning features - - **Secret Managers** - AWS, HashiCorp Vault, CyberArk, etc. - -**New Providers and Endpoints Section:** -- Add section after Key Highlights and before New Models / Updated Models -- Include tables for: - - **New Providers** - Provider name, supported endpoints, description - - **New LLM API Endpoints** (optional) - Endpoint, method, description, documentation link -- Only include major new provider integrations, not minor provider updates -- **IMPORTANT**: When adding new providers, also update `provider_endpoints_support.json` in the repository root (see Section 13) - -### 12. Section Header Counts - -**Always include counts in section headers for:** -- **New Providers** - Add count in parentheses: `### New Providers (X new providers)` -- **New LLM API Endpoints** - Add count in parentheses: `### New LLM API Endpoints (X new endpoints)` -- **New Model Support** - Add count in parentheses: `#### New Model Support (X new models)` - -**Format:** -```markdown -### New Providers (4 new providers) - -| Provider | Supported LiteLLM Endpoints | Description | -| -------- | --------------------------- | ----------- | -... - -### New LLM API Endpoints (2 new endpoints) - -| Endpoint | Method | Description | Documentation | -| -------- | ------ | ----------- | ------------- | -... - -#### New Model Support (32 new models) - -| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features | -| -------- | ----- | -------------- | ------------------- | -------------------- | -------- | -... -``` - -**Counting Rules:** -- Count each row in the table (excluding the header row) -- For models, count each model entry in the pricing table -- For providers, count each new provider added -- For endpoints, count each new API endpoint added - -### 13. Update provider_endpoints_support.json - -**When adding new providers or endpoints, you MUST also update `provider_endpoints_support.json` in the repository root.** - -This file tracks which endpoints are supported by each LiteLLM provider and is used to generate documentation. - -**Required Steps:** -1. For each new provider added to the release notes, add a corresponding entry to `provider_endpoints_support.json` -2. For each new endpoint type added, update the schema comment and add the endpoint to relevant providers - -**Provider Entry Format:** -```json -"provider_slug": { - "display_name": "Provider Name (`provider_slug`)", - "url": "https://docs.litellm.ai/docs/providers/provider_slug", - "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, - "rerank": false, - "a2a": true - } -} -``` - -**Available Endpoint Types:** -- `chat_completions` - `/chat/completions` endpoint -- `messages` - `/messages` endpoint (Anthropic format) -- `responses` - `/responses` endpoint (OpenAI/Anthropic unified) -- `embeddings` - `/embeddings` endpoint -- `image_generations` - `/image/generations` endpoint -- `audio_transcriptions` - `/audio/transcriptions` endpoint -- `audio_speech` - `/audio/speech` endpoint -- `moderations` - `/moderations` endpoint -- `batches` - `/batches` endpoint -- `rerank` - `/rerank` endpoint -- `ocr` - `/ocr` endpoint -- `search` - `/search` endpoint -- `vector_stores` - `/vector_stores` endpoint -- `a2a` - `/a2a/{agent}/message/send` endpoint (A2A Protocol) - -**Checklist:** -- [ ] All new providers from release notes are added to `provider_endpoints_support.json` -- [ ] Endpoint support flags accurately reflect provider capabilities -- [ ] Documentation URL points to correct provider docs page - -## Example Command Workflow - -```bash -# 1. Get model changes -git diff HEAD -- model_prices_and_context_window.json - -# 2. Analyze PR list for categorization -# 3. Create release notes following template -# 4. Link to appropriate documentation -# 5. Review for missing documentation needs -``` - -## Output Requirements - -- Follow exact markdown structure from reference -- Include all PR links and contributors -- Provide accurate model pricing tables -- Link to relevant documentation -- Highlight breaking changes with warnings -- Include deployment instructions -- End with full changelog link - -This process ensures consistent, comprehensive release notes that help users understand changes and upgrade smoothly. diff --git a/cookbook/misc/add_new_models.py b/cookbook/misc/add_new_models.py deleted file mode 100644 index 3cd0bfb2fcc..00000000000 --- a/cookbook/misc/add_new_models.py +++ /dev/null @@ -1,71 +0,0 @@ -import requests - - -def get_initial_config(): - proxy_base_url = input("Enter your proxy base URL (e.g., http://localhost:4000): ") - master_key = input("Enter your LITELLM_MASTER_KEY ") - return proxy_base_url, master_key - - -def get_user_input(): - model_name = input( - "Enter model_name (this is the 'model' passed in /chat/completions requests):" - ) - model = input("litellm_params: Enter model eg. 'azure/': ") - tpm = int(input("litellm_params: Enter tpm (tokens per minute): ")) - rpm = int(input("litellm_params: Enter rpm (requests per minute): ")) - api_key = input("litellm_params: Enter api_key: ") - api_base = input("litellm_params: Enter api_base: ") - api_version = input("litellm_params: Enter api_version: ") - timeout = int(input("litellm_params: Enter timeout (0 for default): ")) - stream_timeout = int( - input("litellm_params: Enter stream_timeout (0 for default): ") - ) - max_retries = int(input("litellm_params: Enter max_retries (0 for default): ")) - - return { - "model_name": model_name, - "litellm_params": { - "model": model, - "tpm": tpm, - "rpm": rpm, - "api_key": api_key, - "api_base": api_base, - "api_version": api_version, - "timeout": timeout, - "stream_timeout": stream_timeout, - "max_retries": max_retries, - }, - } - - -def make_request(proxy_base_url, master_key, data): - url = f"{proxy_base_url}/model/new" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {master_key}", - } - - response = requests.post(url, headers=headers, json=data) - - print(f"Status Code: {response.status_code}") - print(f"Response from adding model: {response.text}") - - -def main(): - proxy_base_url, master_key = get_initial_config() - - while True: - print("Adding new Model to your proxy server...") - data = get_user_input() - make_request(proxy_base_url, master_key, data) - - add_another = input("Do you want to add another model? (yes/no): ").lower() - if add_another != "yes": - break - - print("Script finished.") - - -if __name__ == "__main__": - main() diff --git a/cookbook/misc/config.yaml b/cookbook/misc/config.yaml deleted file mode 100644 index 8ad1d764af5..00000000000 --- a/cookbook/misc/config.yaml +++ /dev/null @@ -1,72 +0,0 @@ -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: azure/chatgpt-v-2 - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_version: "2023-05-15" - api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - - model_name: gpt-3.5-turbo-large - litellm_params: - model: "gpt-3.5-turbo-1106" - api_key: os.environ/OPENAI_API_KEY - rpm: 480 - timeout: 300 - stream_timeout: 60 - - model_name: gpt-4 - litellm_params: - model: azure/chatgpt-v-2 - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_version: "2023-05-15" - api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - rpm: 480 - timeout: 300 - stream_timeout: 60 - - model_name: sagemaker-completion-model - litellm_params: - model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4 - cost_per_second: 0.000420 - - model_name: text-embedding-ada-002 - litellm_params: - model: azure/azure-embedding-model - api_key: os.environ/AZURE_API_KEY - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_version: "2023-05-15" - model_info: - mode: embedding - base_model: text-embedding-ada-002 - - model_name: dall-e-2 - litellm_params: - model: azure/ - api_version: 2023-06-01-preview - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_key: os.environ/AZURE_API_KEY - - model_name: openai-dall-e-3 - litellm_params: - model: dall-e-3 - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - -litellm_settings: - drop_params: True - # max_budget: 100 - # budget_duration: 30d - num_retries: 5 - request_timeout: 600 - context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}] - -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - store_model_in_db: True - proxy_budget_rescheduler_min_time: 60 - proxy_budget_rescheduler_max_time: 64 - proxy_batch_write_at: 1 - # database_url: "postgresql://:@:/" # [OPTIONAL] use for token-based auth to proxy - -# environment_variables: - # settings for using redis caching - # REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com - # REDIS_PORT: "16337" - # REDIS_PASSWORD: diff --git a/cookbook/misc/dev_release.txt b/cookbook/misc/dev_release.txt deleted file mode 100644 index bd40f89e6f2..00000000000 --- a/cookbook/misc/dev_release.txt +++ /dev/null @@ -1,11 +0,0 @@ -python3 -m build -twine upload --verbose dist/litellm-1.18.13.dev4.tar.gz -u __token__ - - - -Note: You might need to make a MANIFEST.ini file on root for build process incase it fails - -Place this in MANIFEST.ini -recursive-exclude venv * -recursive-exclude myenv * -recursive-exclude py313_env * -recursive-exclude **/.venv * diff --git a/cookbook/misc/migrate_proxy_config.py b/cookbook/misc/migrate_proxy_config.py deleted file mode 100644 index b15f9663c64..00000000000 --- a/cookbook/misc/migrate_proxy_config.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -LiteLLM Migration Script! - -Takes a config.yaml and calls /model/new - -Inputs: - - File path to config.yaml - - Proxy base url to your hosted proxy - -Step 1: Reads your config.yaml -Step 2: reads `model_list` and loops through all models -Step 3: calls `/model/new` for each model -""" - -import os - -import yaml -import requests - -_in_memory_os_variables = {} - - -def migrate_models(config_file, proxy_base_url): - # Step 1: Read the config.yaml file - with open(config_file, "r") as f: - config = yaml.safe_load(f) - - # Step 2: Read the model_list and loop through all models - model_list = config.get("model_list", []) - print("model_list: ", model_list) - for model in model_list: - - model_name = model.get("model_name") - print("\nAdding model: ", model_name) - litellm_params = model.get("litellm_params", {}) - api_base = litellm_params.get("api_base", "") - print("api_base on config.yaml: ", api_base) - - litellm_model_name = litellm_params.get("model", "") or "" - if "vertex_ai/" in litellm_model_name: - print("\033[91m\nSkipping Vertex AI model\033[0m", model) - continue - - for param, value in litellm_params.items(): - if isinstance(value, str) and value.startswith("os.environ/"): - # check if value is in _in_memory_os_variables - if value in _in_memory_os_variables: - new_value = _in_memory_os_variables[value] - print( - "\033[92mAlready entered value for \033[0m", - value, - "\033[92musing \033[0m", - new_value, - ) - else: - new_value = input(f"Enter value for {value}: ") - _in_memory_os_variables[value] = new_value - litellm_params[param] = new_value - if "api_key" not in litellm_params: - new_value = input(f"Enter api key for {model_name}: ") - litellm_params["api_key"] = new_value - - print("\nlitellm_params: ", litellm_params) - # Confirm before sending POST request - confirm = input( - "\033[92mDo you want to send the POST request with the above parameters? (y/n): \033[0m" - ) - if confirm.lower() != "y": - print("Aborting POST request.") - exit() - - # Step 3: Call /model/new for each model - url = f"{proxy_base_url}/model/new" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {master_key}", - } - data = {"model_name": model_name, "litellm_params": litellm_params} - print("POSTING data to proxy url", url) - response = requests.post(url, headers=headers, json=data) - if response.status_code != 200: - print(f"Error: {response.status_code} - {response.text}") - raise Exception(f"Error: {response.status_code} - {response.text}") - - # Print the response for each model - print( - f"Response for model '{model_name}': Status Code:{response.status_code} - {response.text}" - ) - - -# Usage -config_file = "config.yaml" -proxy_base_url = "http://0.0.0.0:4000" -master_key = os.environ["LITELLM_MASTER_KEY"] -print(f"config_file: {config_file}") -print(f"proxy_base_url: {proxy_base_url}") -migrate_models(config_file, proxy_base_url) diff --git a/cookbook/misc/openai_timeouts.py b/cookbook/misc/openai_timeouts.py deleted file mode 100644 index fe3e6d426d2..00000000000 --- a/cookbook/misc/openai_timeouts.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -from openai import OpenAI -from dotenv import load_dotenv -import concurrent.futures - -load_dotenv() - -client = OpenAI( - # This is the default and can be omitted - api_key=os.environ.get("OPENAI_API_KEY"), -) - - -def create_chat_completion(): - return client.chat.completions.create( - messages=[ - { - "role": "user", - "content": "Say this is a test. Respond in 20 lines", - } - ], - model="gpt-3.5-turbo", - ) - - -with concurrent.futures.ThreadPoolExecutor() as executor: - # Set a timeout of 10 seconds - future = executor.submit(create_chat_completion) - try: - chat_completion = future.result(timeout=0.00001) - print(chat_completion) - except concurrent.futures.TimeoutError: - print("Operation timed out.") diff --git a/cookbook/misc/sagmaker_streaming.py b/cookbook/misc/sagmaker_streaming.py deleted file mode 100644 index 1a6cc2e32ce..00000000000 --- a/cookbook/misc/sagmaker_streaming.py +++ /dev/null @@ -1,55 +0,0 @@ -# Notes - on how to do sagemaker streaming using boto3 -import json -import boto3 - -import sys -import os -from dotenv import load_dotenv - -load_dotenv() -import io - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path - - -class TokenIterator: - def __init__(self, stream): - self.byte_iterator = iter(stream) - self.buffer = io.BytesIO() - self.read_pos = 0 - - def __iter__(self): - return self - - def __next__(self): - while True: - self.buffer.seek(self.read_pos) - line = self.buffer.readline() - if line and line[-1] == ord("\n"): - self.read_pos += len(line) + 1 - full_line = line[:-1].decode("utf-8") - line_data = json.loads(full_line.lstrip("data:").rstrip("/n")) - return line_data["token"]["text"] - chunk = next(self.byte_iterator) - self.buffer.seek(0, io.SEEK_END) - self.buffer.write(chunk["PayloadPart"]["Bytes"]) - - -payload = { - "inputs": "How do I build a website?", - "parameters": {"max_new_tokens": 256}, - "stream": True, -} - - -client = boto3.client("sagemaker-runtime", region_name="us-west-2") -response = client.invoke_endpoint_with_response_stream( - EndpointName="berri-benchmarking-Llama-2-70b-chat-hf-4", - Body=json.dumps(payload), - ContentType="application/json", -) - -# for token in TokenIterator(response["Body"]): -# print(token) diff --git a/cookbook/misc/test_responses_api.py b/cookbook/misc/test_responses_api.py deleted file mode 100644 index 7abf3170fbc..00000000000 --- a/cookbook/misc/test_responses_api.py +++ /dev/null @@ -1,51 +0,0 @@ -import base64 -import os - -from openai import OpenAI -import time - -client = OpenAI(base_url="http://0.0.0.0:4001", api_key=os.environ["LITELLM_MASTER_KEY"]) - - -# Function to encode the image -def encode_image(image_path): - with open(image_path, "rb") as image_file: - return base64.b64encode(image_file.read()).decode("utf-8") - - -# Path to your image -image_path = "litellm/proxy/logo.png" - -# Getting the Base64 string -base64_image = encode_image(image_path) - - -response = client.responses.create( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - input=[ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "what color is the image"}, - { - "type": "input_image", - "image_url": f"data:image/png;base64,{base64_image}", - }, - ], - } - ], -) - - -print(response.output_text) -print("response1 id===", response.id) -print("sleeping for 20 seconds...") -time.sleep(20) -print("making follow up request for existing id") -response2 = client.responses.create( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - previous_response_id=response.id, - input="ok, and what objects are in the image?", -) - -print(response2.output_text) diff --git a/cookbook/misc/update_json_caching.py b/cookbook/misc/update_json_caching.py deleted file mode 100644 index 8202d7033fd..00000000000 --- a/cookbook/misc/update_json_caching.py +++ /dev/null @@ -1,54 +0,0 @@ -import json - -# List of models to update -models_to_update = [ - "gpt-4o-mini", - "gpt-4o-mini-2024-07-18", - "gpt-4o", - "gpt-4o-2024-11-20", - "gpt-4o-2024-08-06", - "gpt-4o-2024-05-13", - "text-embedding-3-small", - "text-embedding-3-large", - "text-embedding-ada-002-v2", - "ft:gpt-4o-2024-08-06", - "ft:gpt-4o-mini-2024-07-18", - "ft:gpt-3.5-turbo", - "ft:davinci-002", - "ft:babbage-002", -] - - -def update_model_prices(file_path): - # Read the JSON file as text first to preserve number formatting - with open(file_path, "r") as file: - original_text = file.read() - data = json.loads(original_text) - - # Update specified models - for model_name in models_to_update: - print("finding model", model_name) - if model_name in data: - print("found model") - model = data[model_name] - if "input_cost_per_token" in model: - # Format new values to match original style - model["input_cost_per_token_batches"] = float( - "{:.12f}".format(model["input_cost_per_token"] / 2) - ) - if "output_cost_per_token" in model: - model["output_cost_per_token_batches"] = float( - "{:.12f}".format(model["output_cost_per_token"] / 2) - ) - print("new pricing for model=") - # Convert all float values to full decimal format before printing - formatted_model = { - k: "{:.9f}".format(v) if isinstance(v, float) else v - for k, v in data[model_name].items() - } - print(json.dumps(formatted_model, indent=4)) - - -# Run the update -file_path = "model_prices_and_context_window.json" -update_model_prices(file_path) diff --git a/cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb b/cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb deleted file mode 100644 index 818a25e4503..00000000000 --- a/cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb +++ /dev/null @@ -1,315 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Databricks Notebook with MLFlow AutoLogging for LiteLLM Proxy calls\n", - "\n", - "Set `LITELLM_PROXY_API_KEY` in the notebook environment to a key accepted by the proxy before running the model cell\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "5e2812ed-8000-4793-b090-49a31464d810", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "%pip install -U -qqqq databricks-agents mlflow langchain==0.3.1 langchain-core==0.3.6 " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "52530b37-1860-4bba-a6c1-723de83bc58f", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "%pip install \"langchain-openai<=0.3.1\"" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "43c6f4b1-e2d5-431c-b1a2-b97df7707d59", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Before logging this chain using the driver notebook, you must comment out this line.\n", - "dbutils.library.restartPython() " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "88eb8dd7-16b1-480b-aa70-cd429ef87159", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "import mlflow\n", - "from operator import itemgetter\n", - "from langchain_core.output_parsers import StrOutputParser\n", - "from langchain_core.prompts import PromptTemplate\n", - "from langchain_core.runnables import RunnableLambda\n", - "from langchain_databricks import ChatDatabricks\n", - "from langchain_openai import ChatOpenAI" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "f0fdca8f-6f6f-407c-ad4a-0d5a2778728e", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "mlflow.langchain.autolog()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "2ef67315-e468-4d60-a318-98c2cac75bc4", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# These helper functions parse the `messages` array.\n", - "\n", - "# Return the string contents of the most recent message from the user\n", - "def extract_user_query_string(chat_messages_array):\n", - " return chat_messages_array[-1][\"content\"]\n", - "\n", - "\n", - "# Return the chat history, which is is everything before the last question\n", - "def extract_chat_history(chat_messages_array):\n", - " return chat_messages_array[:-1]" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "17708467-1976-48bd-94a0-8c7895cfae3b", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "model = ChatOpenAI(\n", - " openai_api_base=\"LITELLM_PROXY_BASE_URL\", # e.g.: http://0.0.0.0:4000\n", - " model = \"gpt-3.5-turbo\", # LITELLM 'model_name'\n", - " temperature=0.1, \n", - " api_key=os.environ[\"LITELLM_PROXY_API_KEY\"]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a5f2c2af-82f7-470d-b559-47b67fb00cda", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "############\n", - "# Prompt Template for generation\n", - "############\n", - "prompt = PromptTemplate(\n", - " template=\"You are a hello world bot. Respond with a reply to the user's question that is fun and interesting to the user. User's question: {question}\",\n", - " input_variables=[\"question\"],\n", - ")\n", - "\n", - "############\n", - "# FM for generation\n", - "# ChatDatabricks accepts any /llm/v1/chat model serving endpoint\n", - "############\n", - "model = ChatDatabricks(\n", - " endpoint=\"databricks-dbrx-instruct\",\n", - " extra_params={\"temperature\": 0.01, \"max_tokens\": 500},\n", - ")\n", - "\n", - "\n", - "############\n", - "# Simple chain\n", - "############\n", - "# The framework requires the chain to return a string value.\n", - "chain = (\n", - " {\n", - " \"question\": itemgetter(\"messages\")\n", - " | RunnableLambda(extract_user_query_string),\n", - " \"chat_history\": itemgetter(\"messages\") | RunnableLambda(extract_chat_history),\n", - " }\n", - " | prompt\n", - " | model\n", - " | StrOutputParser()\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "366edd90-62a1-4d6f-8a65-0211fb24ca02", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'Hello there! I\\'m here to help with your questions. Regarding your query about \"rag,\" it\\'s not something typically associated with a \"hello world\" bot, but I\\'m happy to explain!\\n\\nRAG, or Remote Angular GUI, is a tool that allows you to create and manage Angular applications remotely. It\\'s a way to develop and test Angular components and applications without needing to set up a local development environment. This can be particularly useful for teams working on distributed systems or for developers who prefer to work in a cloud-based environment.\\n\\nI hope this explanation of RAG has been helpful and interesting! If you have any other questions or need further clarification, feel free to ask.'" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "application/databricks.mlflow.trace": "\"tr-ea2226413395413ba2cf52cffc523502\"", - "text/plain": [ - "Trace(request_id=tr-ea2226413395413ba2cf52cffc523502)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# This is the same input your chain's REST API will accept.\n", - "question = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what is rag?\",\n", - " },\n", - " ]\n", - "}\n", - "\n", - "chain.invoke(question)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "5d68e37d-0980-4a02-bf8d-885c3853f6c1", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "mlflow.models.set_model(model=model)" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "environmentMetadata": null, - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 4 - }, - "notebookName": "Untitled Notebook 2024-10-16 19:35:16", - "widgets": {} - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py b/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py deleted file mode 100644 index 7bf9cc32484..00000000000 --- a/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py +++ /dev/null @@ -1,540 +0,0 @@ -#!/usr/bin/env python3 -""" -Mock Bedrock Guardrail API Server - -This is a FastAPI server that mimics the AWS Bedrock Guardrail API for testing purposes. -It follows the same API spec as the real Bedrock guardrail endpoint. - -Usage: - python mock_bedrock_guardrail_server.py - -The server will start on http://localhost:8080 -""" - -import os -import re -from typing import Any, Dict, List, Literal, Optional - -from fastapi import Depends, FastAPI, Header, HTTPException, status -from fastapi.responses import JSONResponse -from pydantic import BaseModel, Field - -# ============================================================================ -# Request/Response Models (matching Bedrock API spec) -# ============================================================================ - - -class BedrockTextContent(BaseModel): - text: str - - -class BedrockContentItem(BaseModel): - text: BedrockTextContent - - -class BedrockRequest(BaseModel): - source: Literal["INPUT", "OUTPUT"] - content: List[BedrockContentItem] = Field(default_factory=list) - - -class BedrockGuardrailOutput(BaseModel): - text: Optional[str] = None - - -class TopicPolicyItem(BaseModel): - name: str - type: str - action: Literal["BLOCKED", "NONE"] - - -class TopicPolicy(BaseModel): - topics: List[TopicPolicyItem] = Field(default_factory=list) - - -class ContentFilterItem(BaseModel): - type: str - confidence: str - action: Literal["BLOCKED", "NONE"] - - -class ContentPolicy(BaseModel): - filters: List[ContentFilterItem] = Field(default_factory=list) - - -class CustomWord(BaseModel): - match: str - action: Literal["BLOCKED", "NONE"] - - -class WordPolicy(BaseModel): - customWords: List[CustomWord] = Field(default_factory=list) - managedWordLists: List[Dict[str, Any]] = Field(default_factory=list) - - -class PiiEntity(BaseModel): - type: str - match: str - action: Literal["BLOCKED", "ANONYMIZED", "NONE"] - - -class RegexMatch(BaseModel): - name: str - match: str - regex: str - action: Literal["BLOCKED", "ANONYMIZED", "NONE"] - - -class SensitiveInformationPolicy(BaseModel): - piiEntities: List[PiiEntity] = Field(default_factory=list) - regexes: List[RegexMatch] = Field(default_factory=list) - - -class ContextualGroundingFilter(BaseModel): - type: str - threshold: float - score: float - action: Literal["BLOCKED", "NONE"] - - -class ContextualGroundingPolicy(BaseModel): - filters: List[ContextualGroundingFilter] = Field(default_factory=list) - - -class Assessment(BaseModel): - topicPolicy: Optional[TopicPolicy] = None - contentPolicy: Optional[ContentPolicy] = None - wordPolicy: Optional[WordPolicy] = None - sensitiveInformationPolicy: Optional[SensitiveInformationPolicy] = None - contextualGroundingPolicy: Optional[ContextualGroundingPolicy] = None - - -class BedrockGuardrailResponse(BaseModel): - usage: Dict[str, int] = Field( - default_factory=lambda: {"topicPolicyUnits": 1, "contentPolicyUnits": 1} - ) - action: Literal["NONE", "GUARDRAIL_INTERVENED"] = "NONE" - outputs: List[BedrockGuardrailOutput] = Field(default_factory=list) - assessments: List[Assessment] = Field(default_factory=list) - - -# ============================================================================ -# Mock Guardrail Configuration -# ============================================================================ - - -class GuardrailConfig(BaseModel): - """Configuration for mock guardrail behavior""" - - blocked_words: List[str] = Field( - default_factory=lambda: ["offensive", "inappropriate", "badword"] - ) - blocked_topics: List[str] = Field(default_factory=lambda: ["violence", "illegal"]) - pii_patterns: Dict[str, str] = Field( - default_factory=lambda: { - "EMAIL": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "PHONE": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b", - "SSN": r"\b\d{3}-\d{2}-\d{4}\b", - "CREDIT_CARD": r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b", - } - ) - anonymize_pii: bool = True # If True, ANONYMIZE PII; if False, BLOCK it - bearer_token: str = "mock-bedrock-token-12345" - - -# Global config -GUARDRAIL_CONFIG = GuardrailConfig() - -# ============================================================================ -# FastAPI App Setup -# ============================================================================ - -app = FastAPI( - title="Mock Bedrock Guardrail API", - description="Mock server mimicking AWS Bedrock Guardrail API", - version="1.0.0", -) - - -# ============================================================================ -# Authentication -# ============================================================================ - - -async def verify_bearer_token(authorization: Optional[str] = Header(None)) -> str: - """ - Verify the Bearer token from the Authorization header. - - Args: - authorization: The Authorization header value - - Returns: - The token if valid - - Raises: - HTTPException: If token is missing or invalid - """ - if authorization is None: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Missing Authorization header", - headers={"WWW-Authenticate": "Bearer"}, - ) - - # Check if it's a Bearer token - parts = authorization.split() - print(f"parts: {parts}") - if len(parts) != 2 or parts[0].lower() != "bearer": - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid Authorization header format. Expected: Bearer ", - headers={"WWW-Authenticate": "Bearer"}, - ) - - token = parts[1] - - # Verify token - if token != GUARDRAIL_CONFIG.bearer_token: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Invalid bearer token", - ) - - return token - - -# ============================================================================ -# Guardrail Logic -# ============================================================================ - - -def check_blocked_words(text: str) -> Optional[WordPolicy]: - """Check if text contains blocked words""" - found_words = [] - text_lower = text.lower() - - for word in GUARDRAIL_CONFIG.blocked_words: - if word.lower() in text_lower: - found_words.append(CustomWord(match=word, action="BLOCKED")) - - if found_words: - return WordPolicy(customWords=found_words) - return None - - -def check_blocked_topics(text: str) -> Optional[TopicPolicy]: - """Check if text contains blocked topics""" - found_topics = [] - text_lower = text.lower() - - for topic in GUARDRAIL_CONFIG.blocked_topics: - if topic.lower() in text_lower: - found_topics.append( - TopicPolicyItem(name=topic, type=topic.upper(), action="BLOCKED") - ) - - if found_topics: - return TopicPolicy(topics=found_topics) - return None - - -def check_pii(text: str) -> tuple[Optional[SensitiveInformationPolicy], str]: - """ - Check for PII in text and return policy + anonymized text - - Returns: - Tuple of (SensitiveInformationPolicy or None, anonymized_text) - """ - pii_entities = [] - anonymized_text = text - action = "ANONYMIZED" if GUARDRAIL_CONFIG.anonymize_pii else "BLOCKED" - - for pii_type, pattern in GUARDRAIL_CONFIG.pii_patterns.items(): - try: - # Compile the regex pattern with a timeout to prevent ReDoS attacks - compiled_pattern = re.compile(pattern) - matches = compiled_pattern.finditer(text) - for match in matches: - matched_text = match.group() - pii_entities.append( - PiiEntity(type=pii_type, match=matched_text, action=action) - ) - - # Anonymize the text if configured - if GUARDRAIL_CONFIG.anonymize_pii: - anonymized_text = anonymized_text.replace( - matched_text, f"[{pii_type}_REDACTED]" - ) - except re.error: - # Invalid regex pattern - skip it and log a warning - print(f"Warning: Invalid regex pattern for PII type {pii_type}: {pattern}") - continue - - if pii_entities: - return SensitiveInformationPolicy(piiEntities=pii_entities), anonymized_text - - return None, text - - -def process_guardrail_request( - request: BedrockRequest, -) -> tuple[BedrockGuardrailResponse, List[str]]: - """ - Process a guardrail request and return the response. - - Returns: - Tuple of (response, list of output texts) - """ - all_text_content = [] - output_texts = [] - - # Extract all text from content items - for content_item in request.content: - if content_item.text and content_item.text.text: - all_text_content.append(content_item.text.text) - - # Combine all text for analysis - combined_text = " ".join(all_text_content) - - # Initialize response - response = BedrockGuardrailResponse() - assessment = Assessment() - has_intervention = False - - # Check for blocked words - word_policy = check_blocked_words(combined_text) - if word_policy: - assessment.wordPolicy = word_policy - has_intervention = True - - # Check for blocked topics - topic_policy = check_blocked_topics(combined_text) - if topic_policy: - assessment.topicPolicy = topic_policy - has_intervention = True - - # Check for PII - for text in all_text_content: - pii_policy, anonymized_text = check_pii(text) - if pii_policy: - assessment.sensitiveInformationPolicy = pii_policy - if GUARDRAIL_CONFIG.anonymize_pii: - # If anonymizing, we don't block, we modify the text - output_texts.append(anonymized_text) - has_intervention = True - else: - # If not anonymizing PII, we block it - output_texts.append(text) - has_intervention = True - else: - output_texts.append(text) - - # Build response - if has_intervention: - response.action = "GUARDRAIL_INTERVENED" - # Only add assessment if there were interventions - response.assessments = [assessment] - - # Add outputs (modified or original text) - response.outputs = [BedrockGuardrailOutput(text=txt) for txt in output_texts] - - return response, output_texts - - -# ============================================================================ -# API Endpoints -# ============================================================================ - - -@app.get("/") -async def root(): - """Health check endpoint""" - return { - "service": "Mock Bedrock Guardrail API", - "status": "running", - "endpoint_format": "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply", - } - - -@app.get("/health") -async def health(): - """Health check endpoint""" - return {"status": "healthy"} - - -""" -LiteLLM exposes a basic guardrail API with the text extracted from the request and sent to the guardrail API, as well as the received request body for any further processing. - -This works across all LiteLLM endpoints (completion, anthropic /v1/messages, responses api, image generation, embedding, etc.) - -This makes it easy to support your own guardrail API without having to make a PR to LiteLLM. - -LiteLLM supports passing any provider specific params from LiteLLM config.yaml to the guardrail API. - -Example: - -```yaml -guardrails: - - guardrail_name: "bedrock-content-guard" - litellm_params: - guardrail: generic_guardrail_api - mode: "pre_call" - api_key: os.environ/GUARDRAIL_API_KEY - api_base: os.environ/GUARDRAIL_API_BASE - additional_provider_specific_params: - api_version: os.environ/GUARDRAIL_API_VERSION # additional provider specific params -``` - -This is a beta API. Please help us improve it. -""" - - -class LitellmBasicGuardrailRequest(BaseModel): - texts: List[str] - images: Optional[List[str]] = None - tools: Optional[List[dict]] = None - tool_calls: Optional[List[dict]] = None - request_data: Dict[str, Any] = Field(default_factory=dict) - additional_provider_specific_params: Dict[str, Any] = Field(default_factory=dict) - input_type: Literal["request", "response"] - litellm_call_id: Optional[str] = None - litellm_trace_id: Optional[str] = None - structured_messages: Optional[List[Dict[str, Any]]] = None - - -class LitellmBasicGuardrailResponse(BaseModel): - action: Literal[ - "BLOCKED", "NONE", "GUARDRAIL_INTERVENED" - ] # BLOCKED = litellm will raise an error, NONE = litellm will continue, GUARDRAIL_INTERVENED = litellm will continue, but the text was modified by the guardrail - blocked_reason: Optional[str] = None # only if action is BLOCKED, otherwise None - texts: Optional[List[str]] = None - images: Optional[List[str]] = None - - -@app.post( - "/beta/litellm_basic_guardrail_api", - response_model=LitellmBasicGuardrailResponse, -) -async def beta_litellm_basic_guardrail_api( - request: LitellmBasicGuardrailRequest, -) -> LitellmBasicGuardrailResponse: - """ - Apply guardrail to input or output content. - - This endpoint mimics the AWS Bedrock ApplyGuardrail API. - - Args: - request: The guardrail request containing content to analyze - token: Bearer token (verified by dependency) - - Returns: - LitellmBasicGuardrailResponse with analysis results - """ - print(f"request: {request}") - if any("ishaan" in text.lower() for text in request.texts): - return LitellmBasicGuardrailResponse( - action="BLOCKED", blocked_reason="Ishaan is not allowed" - ) - elif any("pii_value" in text for text in request.texts): - return LitellmBasicGuardrailResponse( - action="GUARDRAIL_INTERVENED", - texts=[ - text.replace("pii_value", "pii_value_redacted") - for text in request.texts - ], - ) - return LitellmBasicGuardrailResponse(action="NONE") - - -@app.post("/config/update") -async def update_config( - config: GuardrailConfig, token: str = Depends(verify_bearer_token) -): - """ - Update the guardrail configuration. - - This is a testing endpoint to modify the mock guardrail behavior. - - Args: - config: New guardrail configuration - token: Bearer token (verified by dependency) - - Returns: - Updated configuration - """ - global GUARDRAIL_CONFIG - GUARDRAIL_CONFIG = config - return {"status": "updated", "config": GUARDRAIL_CONFIG} - - -@app.get("/config") -async def get_config(token: str = Depends(verify_bearer_token)): - """ - Get the current guardrail configuration. - - Args: - token: Bearer token (verified by dependency) - - Returns: - Current configuration - """ - return GUARDRAIL_CONFIG - - -# ============================================================================ -# Error Handlers -# ============================================================================ - - -@app.exception_handler(HTTPException) -async def http_exception_handler(request, exc: HTTPException): - """Custom error handler for HTTP exceptions""" - return JSONResponse( - status_code=exc.status_code, - content={"error": exc.detail}, - headers=exc.headers, - ) - - -# ============================================================================ -# Main -# ============================================================================ - -if __name__ == "__main__": - import uvicorn - - # Get configuration from environment - host = os.getenv("MOCK_BEDROCK_HOST", "0.0.0.0") - port = int(os.getenv("MOCK_BEDROCK_PORT", "8080")) - bearer_token = os.getenv("MOCK_BEDROCK_TOKEN", "mock-bedrock-token-12345") - - # Update config with environment token - GUARDRAIL_CONFIG.bearer_token = bearer_token - - print("=" * 80) - print("Mock Bedrock Guardrail API Server") - print("=" * 80) - print(f"Server starting on: http://{host}:{port}") - print(f"Bearer Token: {bearer_token}") - print(f"Endpoint: POST /guardrail/{{id}}/version/{{version}}/apply") - print("=" * 80) - print("\nExample curl command:") - print( - f""" -curl -X POST "http://{host}:{port}/guardrail/test-guardrail/version/1/apply" \\ - -H "Authorization: Bearer {bearer_token}" \\ - -H "Content-Type: application/json" \\ - -d '{{ - "source": "INPUT", - "content": [ - {{ - "text": {{ - "text": "Hello, my email is test@example.com" - }} - }} - ] - }}' - """ - ) - print("=" * 80) - - uvicorn.run(app, host=host, port=port) diff --git a/cookbook/mock_prompt_management_server/README.md b/cookbook/mock_prompt_management_server/README.md deleted file mode 100644 index c13061cfe89..00000000000 --- a/cookbook/mock_prompt_management_server/README.md +++ /dev/null @@ -1,292 +0,0 @@ -# Mock Prompt Management Server - -A reference implementation of the [LiteLLM Generic Prompt Management API](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api). - -This FastAPI server demonstrates how to build a prompt management API that integrates with LiteLLM without requiring a PR to the LiteLLM repository. - -## Quick Start - -### 1. Install Dependencies - -```bash -pip install fastapi uvicorn pydantic -``` - -### 2. Start the Server - -```bash -python mock_prompt_management_server.py -``` - -The server will start on `http://localhost:8080` - -### 3. Test the Endpoint - -```bash -# Get a prompt -curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" - -# Get a prompt with authentication -curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" \ - -H "Authorization: Bearer test-token-12345" - -# List all prompts -curl "http://localhost:8080/prompts" - -# Get prompt variables -curl "http://localhost:8080/prompts/hello-world-prompt/variables" -``` - -## Using with LiteLLM - -### Configuration - -Create a `config.yaml` file: - -```yaml -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: openai/gpt-3.5-turbo - api_key: os.environ/OPENAI_API_KEY - -prompts: - - prompt_id: "hello-world-prompt" - litellm_params: - prompt_integration: "generic_prompt_management" - api_base: http://localhost:8080 - api_key: test-token-12345 -``` - -### Start LiteLLM Proxy - -```bash -litellm --config config.yaml -``` - -### Make a Request - -```bash -curl http://0.0.0.0:4000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ - -d '{ - "model": "gpt-3.5-turbo", - "prompt_id": "hello-world-prompt", - "prompt_variables": { - "domain": "data science", - "task": "analyzing customer behavior" - }, - "messages": [ - {"role": "user", "content": "Please help me get started"} - ] - }' -``` - -## Available Prompts - -The server includes several example prompts: - -| Prompt ID | Description | Variables | -|-----------|-------------|-----------| -| `hello-world-prompt` | Basic helpful assistant | `domain`, `task` | -| `code-review-prompt` | Code review assistant | `years_experience`, `language`, `code` | -| `customer-support-prompt` | Customer support agent | `company_name`, `customer_message` | -| `data-analysis-prompt` | Data analysis expert | `analysis_type`, `dataset_name`, `data` | -| `creative-writing-prompt` | Creative writing assistant | `genre`, `length`, `topic` | - -## Authentication - -The server supports optional Bearer token authentication. Valid tokens for testing: - -- `test-token-12345` -- `dev-token-67890` -- `prod-token-abcdef` - -If no `Authorization` header is provided, requests are allowed (for testing purposes). - -## API Endpoints - -### LiteLLM Spec Endpoints - -#### `GET /beta/litellm_prompt_management` - -Get a prompt by ID (required by LiteLLM). - -**Query Parameters:** -- `prompt_id` (required): The prompt ID -- `project_name` (optional): Project filter -- `slug` (optional): Slug filter -- `version` (optional): Version filter - -**Response:** -```json -{ - "prompt_id": "hello-world-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a helpful assistant specialized in {domain}." - }, - { - "role": "user", - "content": "Help me with: {task}" - } - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.7, - "max_tokens": 500 - } -} -``` - -### Convenience Endpoints (Not in LiteLLM Spec) - -#### `GET /health` - -Health check endpoint. - -#### `GET /prompts` - -List all available prompts. - -#### `GET /prompts/{prompt_id}/variables` - -Get all variables used in a prompt template. - -#### `POST /prompts` - -Create a new prompt (in-memory only, for testing). - -## Example: Full Integration Test - -### 1. Start the Mock Server - -```bash -python mock_prompt_management_server.py -``` - -### 2. Test with Python - -```python -from litellm import completion - -# The completion will: -# 1. Fetch the prompt from your API -# 2. Replace {domain} with "machine learning" -# 3. Replace {task} with "building a recommendation system" -# 4. Merge with your messages -# 5. Use the model and params from the prompt - -response = completion( - model="gpt-4", - prompt_id="hello-world-prompt", - prompt_variables={ - "domain": "machine learning", - "task": "building a recommendation system" - }, - messages=[ - {"role": "user", "content": "I have user behavior data from the past year."} - ], - # Configure the generic prompt manager - generic_prompt_config={ - "api_base": "http://localhost:8080", - "api_key": "test-token-12345", - } -) - -print(response.choices[0].message.content) -``` - -## Customization - -### Adding New Prompts - -Edit the `PROMPTS_DB` dictionary in `mock_prompt_management_server.py`: - -```python -PROMPTS_DB = { - "my-custom-prompt": { - "prompt_id": "my-custom-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a {role}." - }, - { - "role": "user", - "content": "{user_input}" - } - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.8, - "max_tokens": 1000 - } - } -} -``` - -### Using a Database - -Replace the `PROMPTS_DB` dictionary with database queries: - -```python -@app.get("/beta/litellm_prompt_management") -async def get_prompt(prompt_id: str): - # Fetch from database - prompt = await db.prompts.find_one({"prompt_id": prompt_id}) - - if not prompt: - raise HTTPException(status_code=404, detail="Prompt not found") - - return PromptResponse(**prompt) -``` - -### Adding Access Control - -Use the custom query parameters for access control: - -```python -@app.get("/beta/litellm_prompt_management") -async def get_prompt( - prompt_id: str, - project_name: Optional[str] = None, - user_id: Optional[str] = None, - authorization: Optional[str] = Header(None) -): - token = verify_api_key(authorization) - - # Check if user has access to this project - if not has_project_access(token, project_name): - raise HTTPException(status_code=403, detail="Access denied") - - # Fetch and return prompt - ... -``` - -## Production Considerations - -Before deploying to production: - -1. **Use a real database** instead of in-memory storage -2. **Implement proper authentication** with JWT tokens or API keys -3. **Add rate limiting** to prevent abuse -4. **Use HTTPS** for encrypted communication -5. **Add logging and monitoring** for observability -6. **Implement caching** for frequently accessed prompts -7. **Add versioning** for prompt management -8. **Implement access control** based on teams/users -9. **Add input validation** for all parameters -10. **Use environment variables** for configuration - -## Related Documentation - -- [Generic Prompt Management API Documentation](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api) -- [LiteLLM Prompt Management](https://docs.litellm.ai/docs/proxy/prompt_management) -- [Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api) - -## Questions? - -This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm). diff --git a/cookbook/mock_prompt_management_server/mock_prompt_management_server.py b/cookbook/mock_prompt_management_server/mock_prompt_management_server.py deleted file mode 100644 index 295a96e12a9..00000000000 --- a/cookbook/mock_prompt_management_server/mock_prompt_management_server.py +++ /dev/null @@ -1,390 +0,0 @@ -#!/usr/bin/env python3 -""" -Mock Prompt Management API Server - -This is a FastAPI server that implements the LiteLLM Generic Prompt Management API -for testing and demonstration purposes. - -Usage: - python mock_prompt_management_server.py - -The server will start on http://localhost:8080 - -Test the endpoint: - curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" -""" - -import os -import json -from typing import Any, Dict, List, Optional - -from fastapi import FastAPI, HTTPException, Header, Query, status -from fastapi.responses import JSONResponse -from pydantic import BaseModel, Field - -# ============================================================================ -# Response Models -# ============================================================================ - - -class MessageContent(BaseModel): - """A single message in the prompt template""" - - role: str = Field(..., description="Message role (system, user, assistant)") - content: str = Field( - ..., description="Message content with optional {variable} placeholders" - ) - - -class PromptResponse(BaseModel): - """Response format for the prompt management API""" - - prompt_id: str = Field(..., description="The ID of the prompt") - prompt_template: List[MessageContent] = Field( - ..., description="Array of messages in OpenAI format" - ) - prompt_template_model: Optional[str] = Field( - None, description="Optional model to use for this prompt" - ) - prompt_template_optional_params: Optional[Dict[str, Any]] = Field( - None, description="Optional parameters like temperature, max_tokens, etc." - ) - - -# ============================================================================ -# Mock Prompt Database -# ============================================================================ - -PROMPTS_DB = { - "hello-world-prompt": { - "prompt_id": "hello-world-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a helpful assistant specialized in {domain}.", - }, - {"role": "user", "content": "Help me with: {task}"}, - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": {"temperature": 0.7, "max_tokens": 500}, - }, - "code-review-prompt": { - "prompt_id": "code-review-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are an expert code reviewer with {years_experience} years of experience in {language}.", - }, - { - "role": "user", - "content": "Please review the following code for bugs, security issues, and best practices:\n\n{code}", - }, - ], - "prompt_template_model": "gpt-4-turbo", - "prompt_template_optional_params": { - "temperature": 0.3, - "max_tokens": 1500, - }, - }, - "customer-support-prompt": { - "prompt_id": "customer-support-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a friendly customer support agent for {company_name}. Always be professional, empathetic, and solution-oriented.", - }, - { - "role": "user", - "content": "Customer inquiry: {customer_message}", - }, - ], - "prompt_template_model": "gpt-3.5-turbo", - "prompt_template_optional_params": { - "temperature": 0.8, - "max_tokens": 800, - "top_p": 0.9, - }, - }, - "data-analysis-prompt": { - "prompt_id": "data-analysis-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a data scientist expert in {analysis_type} analysis.", - }, - { - "role": "user", - "content": "Analyze the following data and provide insights:\n\nDataset: {dataset_name}\nData: {data}", - }, - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.5, - "max_tokens": 2000, - }, - }, - "creative-writing-prompt": { - "prompt_id": "creative-writing-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a creative writer specializing in {genre} fiction.", - }, - { - "role": "user", - "content": "Write a {length} story about: {topic}", - }, - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.9, - "max_tokens": 3000, - "top_p": 0.95, - }, - }, -} - -# Valid API tokens for authentication (in production, use a secure token store) -VALID_API_TOKENS = { - "test-token-12345", - "dev-token-67890", - "prod-token-abcdef", -} - -# ============================================================================ -# FastAPI App -# ============================================================================ - -app = FastAPI( - title="Mock Prompt Management API", - description="A mock server implementing the LiteLLM Generic Prompt Management API", - version="1.0.0", -) - - -def verify_api_key(authorization: Optional[str] = Header(None)) -> bool: - """ - Verify the API key from the Authorization header. - - Args: - authorization: Authorization header (Bearer token) - - Returns: - True if valid, raises HTTPException if invalid - """ - if authorization is None: - # Allow requests without authentication for testing - return True - - # Extract token from "Bearer " - if not authorization.startswith("Bearer "): - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid authorization header format. Expected 'Bearer '", - ) - - token = authorization.replace("Bearer ", "").strip() - - if token not in VALID_API_TOKENS: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid API key", - ) - - return True - - -@app.get("/beta/litellm_prompt_management", response_model=PromptResponse) -async def get_prompt( - prompt_id: str = Query(..., description="The ID of the prompt to fetch"), - project_name: Optional[str] = Query( - None, description="Optional project name filter" - ), - slug: Optional[str] = Query(None, description="Optional slug filter"), - version: Optional[str] = Query(None, description="Optional version filter"), - authorization: Optional[str] = Header(None), -) -> PromptResponse: - """ - Get a prompt by ID with optional filtering. - - This endpoint implements the LiteLLM Generic Prompt Management API specification. - - Args: - prompt_id: The ID of the prompt to fetch - project_name: Optional project name for filtering - slug: Optional slug for filtering - version: Optional version for filtering - authorization: Optional Bearer token for authentication - - Returns: - PromptResponse with the prompt template and configuration - - Raises: - HTTPException: 401 if authentication fails, 404 if prompt not found - """ - # Verify authentication - verify_api_key(authorization) - - # Log the request parameters (useful for debugging) - print(f"Fetching prompt: {prompt_id}") - if project_name: - print(f" Project: {project_name}") - if slug: - print(f" Slug: {slug}") - if version: - print(f" Version: {version}") - - # Check if prompt exists - if prompt_id not in PROMPTS_DB: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Prompt '{prompt_id}' not found. Available prompts: {list(PROMPTS_DB.keys())}", - ) - - # Get the prompt from the database - prompt_data = PROMPTS_DB[prompt_id] - - # Optional: Apply filtering based on project_name, slug, or version - # In a real implementation, you might use these to filter prompts by access control - # or to fetch specific versions from your database - - return PromptResponse(**prompt_data) - - -@app.get("/health") -async def health_check(): - """Health check endpoint""" - return { - "status": "healthy", - "service": "mock-prompt-management-api", - "version": "1.0.0", - } - - -@app.get("/prompts") -async def list_prompts(authorization: Optional[str] = Header(None)): - """ - List all available prompts. - - This is a convenience endpoint (not part of the LiteLLM spec) for - discovering available prompts. - """ - # Verify authentication - verify_api_key(authorization) - - prompts_list = [ - { - "prompt_id": pid, - "model": p.get("prompt_template_model"), - "has_variables": any( - "{" in msg.get("content", "") for msg in p.get("prompt_template", []) - ), - } - for pid, p in PROMPTS_DB.items() - ] - - return {"prompts": prompts_list, "total": len(prompts_list)} - - -@app.get("/prompts/{prompt_id}/variables") -async def get_prompt_variables( - prompt_id: str, authorization: Optional[str] = Header(None) -): - """ - Get all variables in a prompt template. - - This is a convenience endpoint (not part of the LiteLLM spec) for - discovering what variables a prompt expects. - """ - # Verify authentication - verify_api_key(authorization) - - if prompt_id not in PROMPTS_DB: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Prompt '{prompt_id}' not found", - ) - - prompt_data = PROMPTS_DB[prompt_id] - variables = set() - - # Extract variables from the prompt template - import re - - for message in prompt_data["prompt_template"]: - content = message.get("content", "") - # Find all {variable} patterns - found_vars = re.findall(r"\{(\w+)\}", content) - variables.update(found_vars) - - return { - "prompt_id": prompt_id, - "variables": sorted(list(variables)), - "example_usage": { - "prompt_id": prompt_id, - "prompt_variables": {var: f"<{var}_value>" for var in variables}, - }, - } - - -@app.post("/prompts") -async def create_prompt( - prompt: PromptResponse, authorization: Optional[str] = Header(None) -): - """ - Create a new prompt (convenience endpoint for testing). - - This is NOT part of the LiteLLM spec - it's just for testing purposes. - """ - # Verify authentication - verify_api_key(authorization) - - if prompt.prompt_id in PROMPTS_DB: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=f"Prompt '{prompt.prompt_id}' already exists", - ) - - PROMPTS_DB[prompt.prompt_id] = prompt.dict() - - return { - "status": "created", - "prompt_id": prompt.prompt_id, - "message": "Prompt created successfully (in-memory only)", - } - - -# ============================================================================ -# Main -# ============================================================================ - -if __name__ == "__main__": - import uvicorn - - print("=" * 70) - print("Mock Prompt Management API Server") - print("=" * 70) - print(f"\nStarting server on http://localhost:8080") - print(f"\nAvailable prompts: {len(PROMPTS_DB)}") - for prompt_id in PROMPTS_DB.keys(): - print(f" - {prompt_id}") - print(f"\nValid API tokens: {len(VALID_API_TOKENS)}") - print(" - test-token-12345") - print(" - dev-token-67890") - print(" - prod-token-abcdef") - print("\nEndpoints:") - print(" GET /beta/litellm_prompt_management?prompt_id= (LiteLLM spec)") - print(" GET /health (health check)") - print(" GET /prompts (list all prompts)") - print( - " GET /prompts/{id}/variables (get prompt variables)" - ) - print(" POST /prompts (create prompt)") - print("\nExample usage:") - print( - ' curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"' - ) - print("\nPress CTRL+C to stop the server") - print("=" * 70) - - uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info") diff --git a/cookbook/nova_sonic_realtime.py b/cookbook/nova_sonic_realtime.py deleted file mode 100644 index 50844121cbf..00000000000 --- a/cookbook/nova_sonic_realtime.py +++ /dev/null @@ -1,292 +0,0 @@ -""" -Client script to test Nova Sonic realtime API through LiteLLM proxy. - -This script connects to LiteLLM proxy's realtime endpoint and enables -speech-to-speech conversation with Bedrock Nova Sonic. - -Prerequisites: -- LiteLLM proxy running with Bedrock configured -- pyaudio installed: pip install pyaudio -- websockets installed: pip install websockets - -Usage: - export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)" - python nova_sonic_realtime.py -""" - -import asyncio -import base64 -import json -import os -import pyaudio -import websockets -from typing import Optional - -# Bounded queue size for audio chunks (configurable via env to avoid unbounded memory) -AUDIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 10_000)) - -# Audio configuration (matching Nova Sonic requirements) -INPUT_SAMPLE_RATE = 16000 # Nova Sonic expects 16kHz input -OUTPUT_SAMPLE_RATE = 24000 # Nova Sonic outputs 24kHz -CHANNELS = 1 -FORMAT = pyaudio.paInt16 -CHUNK_SIZE = 1024 - -# LiteLLM proxy configuration -LITELLM_PROXY_URL = "ws://localhost:4000/v1/realtime?model=bedrock-sonic" -LITELLM_API_KEY = os.environ["LITELLM_MASTER_KEY"] - - -class RealtimeClient: - """Client for LiteLLM realtime API with audio support.""" - - def __init__(self, url: str, api_key: str): - self.url = url - self.api_key = api_key - self.ws: Optional[websockets.WebSocketClientProtocol] = None - self.is_active = False - self.audio_queue = asyncio.Queue(maxsize=AUDIO_QUEUE_MAXSIZE) - self.pyaudio = pyaudio.PyAudio() - self.input_stream = None - self.output_stream = None - - async def connect(self): - """Connect to LiteLLM proxy realtime endpoint.""" - print(f"Connecting to {self.url}...") - - headers = {} - if self.api_key: - headers["Authorization"] = f"Bearer {self.api_key}" - - self.ws = await websockets.connect( - self.url, - additional_headers=headers, - max_size=10 * 1024 * 1024, # 10MB max message size - ) - self.is_active = True - print("✓ Connected to LiteLLM proxy") - - async def send_session_update(self): - """Send session configuration.""" - session_update = { - "type": "session.update", - "session": { - "instructions": "You are a friendly assistant. Keep your responses short and conversational.", - "voice": "matthew", - "temperature": 0.8, - "max_response_output_tokens": 1024, - "modalities": ["text", "audio"], - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 500, - }, - }, - } - await self.ws.send(json.dumps(session_update)) - print("✓ Session configuration sent") - - async def receive_messages(self): - """Receive and process messages from the server.""" - try: - async for message in self.ws: - if not self.is_active: - break - - try: - data = json.loads(message) - event_type = data.get("type") - - if event_type == "session.created": - print(f"✓ Session created: {data.get('session', {}).get('id')}") - - elif event_type == "response.created": - print("🤖 Assistant is responding...") - - elif event_type == "response.text.delta": - # Print text transcription - delta = data.get("delta", "") - print(delta, end="", flush=True) - - elif event_type == "response.audio.delta": - # Queue audio for playback - audio_b64 = data.get("delta", "") - if audio_b64: - audio_bytes = base64.b64decode(audio_b64) - await self.audio_queue.put(audio_bytes) - - elif event_type == "response.text.done": - print() # New line after text - - elif event_type == "response.done": - print("✓ Response complete") - - elif event_type == "error": - print(f"❌ Error: {data.get('error', {})}") - - else: - # Debug: print other event types - print(f"[{event_type}]", end=" ") - - except json.JSONDecodeError: - print(f"Failed to parse message: {message[:100]}") - - except websockets.exceptions.ConnectionClosed: - print("\n✗ Connection closed") - except Exception as e: - print(f"\n✗ Error receiving messages: {e}") - finally: - self.is_active = False - - async def send_audio_chunk(self, audio_bytes: bytes): - """Send audio chunk to server.""" - if not self.is_active or not self.ws: - return - - audio_b64 = base64.b64encode(audio_bytes).decode("utf-8") - message = { - "type": "input_audio_buffer.append", - "audio": audio_b64, - } - await self.ws.send(json.dumps(message)) - - async def commit_audio_buffer(self): - """Commit the audio buffer to trigger processing.""" - if not self.is_active or not self.ws: - return - - message = {"type": "input_audio_buffer.commit"} - await self.ws.send(json.dumps(message)) - - async def capture_audio(self): - """Capture audio from microphone and send to server.""" - print("\n🎤 Starting audio capture...") - print("Speak into your microphone. Press Ctrl+C to stop.\n") - - self.input_stream = self.pyaudio.open( - format=FORMAT, - channels=CHANNELS, - rate=INPUT_SAMPLE_RATE, - input=True, - frames_per_buffer=CHUNK_SIZE, - ) - - try: - while self.is_active: - audio_data = self.input_stream.read( - CHUNK_SIZE, exception_on_overflow=False - ) - await self.send_audio_chunk(audio_data) - await asyncio.sleep(0.01) # Small delay to prevent overwhelming - except Exception as e: - print(f"Error capturing audio: {e}") - finally: - if self.input_stream: - self.input_stream.stop_stream() - self.input_stream.close() - - async def play_audio(self): - """Play audio responses from the server.""" - print("🔊 Starting audio playback...") - - self.output_stream = self.pyaudio.open( - format=FORMAT, - channels=CHANNELS, - rate=OUTPUT_SAMPLE_RATE, - output=True, - frames_per_buffer=CHUNK_SIZE, - ) - - try: - while self.is_active: - try: - audio_data = await asyncio.wait_for( - self.audio_queue.get(), timeout=0.1 - ) - if audio_data: - self.output_stream.write(audio_data) - except asyncio.TimeoutError: - continue - except Exception as e: - print(f"Error playing audio: {e}") - finally: - if self.output_stream: - self.output_stream.stop_stream() - self.output_stream.close() - - async def close(self): - """Close the connection and cleanup.""" - self.is_active = False - - if self.ws: - await self.ws.close() - - if self.input_stream: - self.input_stream.stop_stream() - self.input_stream.close() - - if self.output_stream: - self.output_stream.stop_stream() - self.output_stream.close() - - self.pyaudio.terminate() - print("\n✓ Connection closed") - - -async def main(): - """Main function to run the realtime client.""" - print("=" * 80) - print("Bedrock Nova Sonic Realtime Client") - print("=" * 80) - print() - - client = RealtimeClient(LITELLM_PROXY_URL, LITELLM_API_KEY) - - try: - # Connect to server - await client.connect() - - # Send session configuration - await client.send_session_update() - - # Wait a moment for session to be established - await asyncio.sleep(0.5) - - # Start tasks - receive_task = asyncio.create_task(client.receive_messages()) - capture_task = asyncio.create_task(client.capture_audio()) - playback_task = asyncio.create_task(client.play_audio()) - - # Wait for user to interrupt - await asyncio.gather( - receive_task, - capture_task, - playback_task, - return_exceptions=True, - ) - - except KeyboardInterrupt: - print("\n\n⚠ Interrupted by user") - except Exception as e: - print(f"\n❌ Error: {e}") - import traceback - - traceback.print_exc() - finally: - await client.close() - - -if __name__ == "__main__": - print("\nMake sure:") - print("1. LiteLLM proxy is running on port 4000") - print("2. Bedrock is configured in proxy_server_config.yaml") - print("3. AWS credentials are set") - print() - - try: - asyncio.run(main()) - except KeyboardInterrupt: - print("\n\nGoodbye!") diff --git a/cookbook/result.html b/cookbook/result.html deleted file mode 100644 index 0bd099bacc7..00000000000 --- a/cookbook/result.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - -
- - - - - - - - \ No newline at end of file diff --git a/cookbook/veo_video_generation.py b/cookbook/veo_video_generation.py deleted file mode 100644 index 8587032e0be..00000000000 --- a/cookbook/veo_video_generation.py +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/bin/env python3 -""" -Complete example for Veo video generation through LiteLLM proxy. - -This script demonstrates how to: -1. Generate videos using Google's Veo model -2. Poll for completion status -3. Download the generated video file - -Requirements: -- LiteLLM proxy running with Google AI Studio pass-through configured -- Google AI Studio API key with Veo access -""" - -import json -import os -import time -import requests -from typing import Optional - - -class VeoVideoGenerator: - """Complete Veo video generation client using LiteLLM proxy.""" - - def __init__( - self, - api_key: str, - base_url: str = "http://localhost:4000/gemini/v1beta", - ): - """ - Initialize the Veo video generator. - - Args: - base_url: Base URL for the LiteLLM proxy with Gemini pass-through - api_key: API key for LiteLLM proxy authentication - """ - self.base_url = base_url - self.api_key = api_key - self.headers = {"x-goog-api-key": api_key, "Content-Type": "application/json"} - - def generate_video(self, prompt: str) -> Optional[str]: - """ - Initiate video generation with Veo. - - Args: - prompt: Text description of the video to generate - - Returns: - Operation name if successful, None otherwise - """ - print(f"🎬 Generating video with prompt: '{prompt}'") - - url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning" - payload = {"instances": [{"prompt": prompt}]} - - try: - response = requests.post(url, headers=self.headers, json=payload) - response.raise_for_status() - - data = response.json() - operation_name = data.get("name") - - if operation_name: - print(f"✅ Video generation started: {operation_name}") - return operation_name - else: - print("❌ No operation name returned") - print(f"Response: {json.dumps(data, indent=2)}") - return None - - except requests.RequestException as e: - print(f"❌ Failed to start video generation: {e}") - if hasattr(e, "response") and e.response is not None: - try: - error_data = e.response.json() - print(f"Error details: {json.dumps(error_data, indent=2)}") - except: - print(f"Error response: {e.response.text}") - return None - - def wait_for_completion( - self, operation_name: str, max_wait_time: int = 600 - ) -> Optional[str]: - """ - Poll operation status until video generation is complete. - - Args: - operation_name: Name of the operation to monitor - max_wait_time: Maximum time to wait in seconds (default: 10 minutes) - - Returns: - Video URI if successful, None otherwise - """ - print("⏳ Waiting for video generation to complete...") - - operation_url = f"{self.base_url}/{operation_name}" - start_time = time.time() - poll_interval = 10 # Start with 10 seconds - - while time.time() - start_time < max_wait_time: - try: - print( - f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)" - ) - - response = requests.get(operation_url, headers=self.headers) - response.raise_for_status() - - data = response.json() - - # Check for errors - if "error" in data: - print("❌ Error in video generation:") - print(json.dumps(data["error"], indent=2)) - return None - - # Check if operation is complete - is_done = data.get("done", False) - - if is_done: - print("🎉 Video generation complete!") - - try: - # Extract video URI from nested response - video_uri = data["response"]["generateVideoResponse"][ - "generatedSamples" - ][0]["video"]["uri"] - print(f"📹 Video URI: {video_uri}") - return video_uri - except KeyError as e: - print(f"❌ Could not extract video URI: {e}") - print("Full response:") - print(json.dumps(data, indent=2)) - return None - - # Wait before next poll, with exponential backoff - time.sleep(poll_interval) - poll_interval = min(poll_interval * 1.2, 30) # Cap at 30 seconds - - except requests.RequestException as e: - print(f"❌ Error polling operation status: {e}") - time.sleep(poll_interval) - - print(f"⏰ Timeout after {max_wait_time} seconds") - return None - - def download_video( - self, video_uri: str, output_filename: str = "generated_video.mp4" - ) -> bool: - """ - Download the generated video file. - - Args: - video_uri: URI of the video to download (from Google's response) - output_filename: Local filename to save the video - - Returns: - True if download successful, False otherwise - """ - print(f"⬇️ Downloading video...") - print(f"Original URI: {video_uri}") - - # Convert Google URI to LiteLLM proxy URI - # Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media - if video_uri.startswith("files/"): - download_path = f"{video_uri}:download?alt=media" - else: - download_path = video_uri - - litellm_download_url = f"{self.base_url}/{download_path}" - print(f"Download URL: {litellm_download_url}") - - try: - # Download with streaming and redirect handling - response = requests.get( - litellm_download_url, - headers=self.headers, - stream=True, - allow_redirects=True, # Handle redirects automatically - ) - response.raise_for_status() - - # Save video file - with open(output_filename, "wb") as f: - downloaded_size = 0 - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) - downloaded_size += len(chunk) - - # Progress indicator for large files - if downloaded_size % (1024 * 1024) == 0: # Every MB - print( - f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB..." - ) - - # Verify file was created and has content - if os.path.exists(output_filename): - file_size = os.path.getsize(output_filename) - if file_size > 0: - print(f"✅ Video downloaded successfully!") - print(f"📁 Saved as: {output_filename}") - print(f"📏 File size: {file_size / (1024*1024):.2f} MB") - return True - else: - print("❌ Downloaded file is empty") - os.remove(output_filename) - return False - else: - print("❌ File was not created") - return False - - except requests.RequestException as e: - print(f"❌ Download failed: {e}") - if hasattr(e, "response") and e.response is not None: - print(f"Status code: {e.response.status_code}") - print(f"Response headers: {dict(e.response.headers)}") - return False - - def generate_and_download(self, prompt: str, output_filename: str = None) -> bool: - """ - Complete workflow: generate video and download it. - - Args: - prompt: Text description for video generation - output_filename: Output filename (auto-generated if None) - - Returns: - True if successful, False otherwise - """ - # Auto-generate filename if not provided - if output_filename is None: - timestamp = int(time.time()) - safe_prompt = "".join( - c for c in prompt[:30] if c.isalnum() or c in (" ", "-", "_") - ).rstrip() - output_filename = ( - f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4" - ) - - print("=" * 60) - print("🎬 VEO VIDEO GENERATION WORKFLOW") - print("=" * 60) - - # Step 1: Generate video - operation_name = self.generate_video(prompt) - if not operation_name: - return False - - # Step 2: Wait for completion - video_uri = self.wait_for_completion(operation_name) - if not video_uri: - return False - - # Step 3: Download video - success = self.download_video(video_uri, output_filename) - - if success: - print("=" * 60) - print("🎉 SUCCESS! Video generation complete!") - print(f"📁 Video saved as: {output_filename}") - print("=" * 60) - else: - print("=" * 60) - print("❌ FAILED! Video generation or download failed") - print("=" * 60) - - return success - - -def main(): - """ - Example usage of the VeoVideoGenerator. - - Configure these environment variables: - - LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta) - - LITELLM_API_KEY: API key for LiteLLM proxy authentication - """ - - # Configuration from environment or defaults - base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta") - api_key = os.environ["LITELLM_API_KEY"] - - print("🚀 Starting Veo Video Generation Example") - print(f"📡 Using LiteLLM proxy at: {base_url}") - - # Initialize generator - generator = VeoVideoGenerator(base_url=base_url, api_key=api_key) - - # Example prompts - try different ones! - example_prompts = [ - "A cat playing with a ball of yarn in a sunny garden", - "Ocean waves crashing against rocky cliffs at sunset", - "A bustling city street with people walking and cars passing by", - "A peaceful forest with sunlight filtering through the trees", - ] - - # Use first example or get from user - prompt = example_prompts[0] - print(f"🎬 Using prompt: '{prompt}'") - - # Generate and download video - success = generator.generate_and_download(prompt) - - if success: - print("\n✅ Example completed successfully!") - print("💡 Try modifying the prompt in the script for different videos!") - else: - print("\n❌ Example failed!") - print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key") - - # Troubleshooting tips - print("\n🔍 Troubleshooting:") - print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through") - print("2. Verify your Google AI Studio API key has Veo access") - print("3. Check that your prompt meets Veo's content guidelines") - print("4. Review the LiteLLM proxy logs for detailed error information") - - -if __name__ == "__main__": - main() diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json b/litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_all_metrics/grafana_dashboard.json similarity index 100% rename from cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json rename to litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_all_metrics/grafana_dashboard.json diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md b/litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_all_metrics/readme.md similarity index 100% rename from cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md rename to litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_all_metrics/readme.md diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_genai_otel/grafana_dashboard.json b/litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_genai_otel/grafana_dashboard.json similarity index 100% rename from cookbook/litellm_proxy_server/grafana_dashboard/dashboard_genai_otel/grafana_dashboard.json rename to litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_genai_otel/grafana_dashboard.json diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_genai_otel/readme.md b/litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_genai_otel/readme.md similarity index 100% rename from cookbook/litellm_proxy_server/grafana_dashboard/dashboard_genai_otel/readme.md rename to litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_genai_otel/readme.md diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_v2/grafana_dashboard.json b/litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_v2/grafana_dashboard.json similarity index 100% rename from cookbook/litellm_proxy_server/grafana_dashboard/dashboard_v2/grafana_dashboard.json rename to litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_v2/grafana_dashboard.json diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/readme.md b/litellm/integrations/prometheus_helpers/grafana_dashboards/readme.md similarity index 100% rename from cookbook/litellm_proxy_server/grafana_dashboard/readme.md rename to litellm/integrations/prometheus_helpers/grafana_dashboards/readme.md diff --git a/cookbook/skill_scanner/README.md b/litellm/proxy/discovery_endpoints/skill_scanner_adapter/README.md similarity index 94% rename from cookbook/skill_scanner/README.md rename to litellm/proxy/discovery_endpoints/skill_scanner_adapter/README.md index 2a68e116002..0e3ad5a726f 100644 --- a/cookbook/skill_scanner/README.md +++ b/litellm/proxy/discovery_endpoints/skill_scanner_adapter/README.md @@ -2,11 +2,11 @@ This standalone adapter runs NVIDIA SkillSpector on a separate machine. It is not installed in the LiteLLM proxy and does not require a shared Helm or Compose deployment -Start it with Python 3.12 or newer and uv. The script pins SkillSpector and its HTTP dependencies +Start it from the repository root with Python 3.12 or newer and uv. The script pins SkillSpector and its HTTP dependencies. The adapter is not included in the wheel or sdist ```sh export SKILL_SCANNER_API_KEY="$(openssl rand -hex 32)" -uv run server.py +uv run litellm/proxy/discovery_endpoints/skill_scanner_adapter/server.py ``` For another host, place the loopback service behind an HTTPS reverse proxy that forwards the Authorization header and enforces a 65 MiB request limit. Run it under an unprivileged account with a writable temporary directory and no production files or credentials. The scanner host needs Git and network access to requested sources and optional model providers. Apply network egress restrictions suitable for those sources diff --git a/cookbook/skill_scanner/server.py b/litellm/proxy/discovery_endpoints/skill_scanner_adapter/server.py similarity index 100% rename from cookbook/skill_scanner/server.py rename to litellm/proxy/discovery_endpoints/skill_scanner_adapter/server.py diff --git a/pyproject.toml b/pyproject.toml index 76bb3a54223..bf4e9e52445 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -341,6 +341,7 @@ features = ["extension-module"] profile = "release" editable-profile = "dev" include = [ + "litellm/integrations/prometheus_helpers/grafana_dashboards/**", "litellm/proxy/_experimental/out/**", "litellm/router_strategy/complexity_router/artifacts/*.json", "litellm/router_strategy/complexity_router/fuse_presets.json", @@ -353,6 +354,8 @@ include = [ { path = "model_prices_and_context_window.json", format = "sdist" }, ] exclude = [ + "litellm/proxy/discovery_endpoints/skill_scanner_adapter", + "litellm/proxy/discovery_endpoints/skill_scanner_adapter/**", "litellm/proxy/enterprise", "litellm/proxy/enterprise/**", "litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/guardrail_benchmarks", diff --git a/pyrightconfig.json b/pyrightconfig.json index 3a0812413c3..820c4758b46 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,7 +1,7 @@ { "include": ["litellm", "litellm_cli"], "ignore": [], - "exclude": ["**/node_modules", "**/__pycache__", "tests/e2e/claude_code", "tests/e2e_harness/claude_code", "tests/e2e/ui", "litellm/types/utils.py", "litellm/proxy/_types.py"], + "exclude": ["**/node_modules", "**/__pycache__", "tests/e2e/claude_code", "tests/e2e_harness/claude_code", "tests/e2e/ui", "litellm/types/utils.py", "litellm/proxy/_types.py", "litellm/proxy/discovery_endpoints/skill_scanner_adapter"], "executionEnvironments": [ { "root": "tests/e2e_harness", diff --git a/ruff.toml b/ruff.toml index 037b15dbfe9..f7624f07f51 100644 --- a/ruff.toml +++ b/ruff.toml @@ -28,6 +28,8 @@ lint.external = [ ] line-length = 120 +extend-exclude = ["litellm/proxy/discovery_endpoints/skill_scanner_adapter"] + format.exclude = ["**/enterprise/**"] # Was the top-level `exclude`. Scoped to lint so `ruff format` still formats these paths diff --git a/scripts/check_type_discipline.py b/scripts/check_type_discipline.py index cafec8d249b..52992281c99 100644 --- a/scripts/check_type_discipline.py +++ b/scripts/check_type_discipline.py @@ -1158,8 +1158,12 @@ def collect_paths(raw: Iterable[str]) -> Iterator[Path]: for item in raw: p = Path(item) if p.is_dir(): - yield from sorted(p.rglob("*.py")) - elif p.suffix == ".py": + yield from ( + path + for path in sorted(p.rglob("*.py")) + if "litellm/proxy/discovery_endpoints/skill_scanner_adapter/" not in path.as_posix() + ) + elif p.suffix == ".py" and "litellm/proxy/discovery_endpoints/skill_scanner_adapter/" not in p.as_posix(): yield p diff --git a/tests/code_coverage_tests/router_surface_check.py b/tests/code_coverage_tests/router_surface_check.py index 8eb7651f51c..672fa2dcaa4 100644 --- a/tests/code_coverage_tests/router_surface_check.py +++ b/tests/code_coverage_tests/router_surface_check.py @@ -67,7 +67,6 @@ SKIP_DIRS = { "litellm-rust", "ui", "docs", - "cookbook", "tests", "test", } diff --git a/tests/integration/security/test_skill_scanner.py b/tests/integration/security/test_skill_scanner.py index e18378463ee..d4e4c6763ae 100644 --- a/tests/integration/security/test_skill_scanner.py +++ b/tests/integration/security/test_skill_scanner.py @@ -8,7 +8,7 @@ import psutil import pytest from fastapi import HTTPException -from cookbook.skill_scanner import server +from litellm.proxy.discovery_endpoints.skill_scanner_adapter import server from tests.integration._support.process import _wait_until_gone, group_members, stop_processes diff --git a/tests/unit/cookbook/__init__.py b/tests/unit/cookbook/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/unit/cookbook/test_gpt_realtime_translate.py b/tests/unit/cookbook/test_gpt_realtime_translate.py deleted file mode 100644 index a40a3265600..00000000000 --- a/tests/unit/cookbook/test_gpt_realtime_translate.py +++ /dev/null @@ -1,141 +0,0 @@ -import asyncio -import base64 -import json -import wave -from pathlib import Path -from types import SimpleNamespace -from typing import Final, cast - -import pytest -from websockets.asyncio.client import ClientConnection - -from cookbook import gpt_realtime_translate as translate - - -@pytest.mark.asyncio -async def test_short_upload_waits_for_first_translated_audio(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.1) - audio: Final = bytes(480) - events: Final = iter( - ( - {"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()}, - {"type": "error", "error": {"message": "session closed"}}, - ) - ) - - async def recv() -> str: - event: Final = next(events) - if event["type"] == "session.output_audio.delta": - await asyncio.sleep(0.03) - return json.dumps(event) - - sender_finished: Final = asyncio.Event() - sender_finished.set() - output: Final = tmp_path / "translation.wav" - - result: Final = await translate.receive_translation( - cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished - ) - - assert result == 'Realtime API error: {"message": "session closed"}' - with wave.open(str(output), "rb") as rendered: - assert rendered.readframes(240) == audio - - -@pytest.mark.asyncio -async def test_long_upload_keeps_waiting_for_first_translated_audio_while_still_sending( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.05) - audio: Final = bytes(480) - pending: Final[asyncio.Queue[str]] = asyncio.Queue() - sender_finished: Final = asyncio.Event() - - async def speak_after_a_long_pause() -> None: - await asyncio.sleep(0.2) - await pending.put(json.dumps({"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()})) - await asyncio.sleep(0.02) - sender_finished.set() - await pending.put(json.dumps({"type": "session.closed"})) - - output: Final = tmp_path / "translation.wav" - speaker: Final = asyncio.create_task(speak_after_a_long_pause()) - - result: Final = await translate.receive_translation( - cast(ClientConnection, SimpleNamespace(recv=pending.get)), output, sender_finished - ) - await speaker - - assert result is None - with wave.open(str(output), "rb") as rendered: - assert rendered.readframes(240) == audio - - -@pytest.mark.asyncio -async def test_a_quiet_gap_after_the_sender_stops_keeps_the_rest_of_the_translation( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 1.0) - first: Final = bytes(480) - second: Final = bytes(range(240)) * 2 - events: Final = iter( - ( - {"type": "session.output_audio.delta", "delta": base64.b64encode(first).decode()}, - {"type": "session.output_audio.delta", "delta": base64.b64encode(second).decode()}, - {"type": "session.closed"}, - ) - ) - - async def recv() -> str: - event: Final = next(events) - if event.get("delta") == base64.b64encode(second).decode(): - await asyncio.sleep(0.3) - return json.dumps(event) - - sender_finished: Final = asyncio.Event() - sender_finished.set() - output: Final = tmp_path / "translation.wav" - - result: Final = await translate.receive_translation( - cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished - ) - - assert result is None - with wave.open(str(output), "rb") as rendered: - assert rendered.readframes(480) == first + second - - -@pytest.mark.asyncio -async def test_a_stream_that_never_closes_the_session_reports_an_error( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.05) - audio: Final = bytes(480) - pending: Final[asyncio.Queue[str]] = asyncio.Queue() - await pending.put(json.dumps({"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()})) - sender_finished: Final = asyncio.Event() - sender_finished.set() - - result: Final = await translate.receive_translation( - cast(ClientConnection, SimpleNamespace(recv=pending.get)), tmp_path / "translation.wav", sender_finished - ) - - assert result == "The translation stream went quiet for 0.05 s without closing the session" - - -@pytest.mark.asyncio -async def test_the_sender_closes_the_session_after_the_last_audio_chunk() -> None: - sent: Final[asyncio.Queue[str]] = asyncio.Queue() - finished: Final = asyncio.Event() - pcm: Final = bytes(translate.CHUNK_BYTES + 2) - - await translate.send_audio(cast(ClientConnection, SimpleNamespace(send=sent.put)), pcm, 0.0, finished) - - messages: Final = tuple(json.loads(sent.get_nowait()) for _ in range(sent.qsize())) - assert [message["type"] for message in messages] == [ - "session.input_audio_buffer.append", - "session.input_audio_buffer.append", - "session.close", - ] - assert b"".join(base64.b64decode(message["audio"]) for message in messages[:-1]) == pcm - assert finished.is_set() diff --git a/tests/unit/integrations/test_prometheus_metric_name_consistency.py b/tests/unit/integrations/test_prometheus_metric_name_consistency.py index 2c1def4c613..0303d7cde1c 100644 --- a/tests/unit/integrations/test_prometheus_metric_name_consistency.py +++ b/tests/unit/integrations/test_prometheus_metric_name_consistency.py @@ -28,7 +28,7 @@ from litellm.proxy.middleware.admission_control_middleware import create_prometh from litellm.proxy.middleware.in_flight_requests_middleware import InFlightRequestsMiddleware from litellm.tracing.exporter import lens_export_counters -_GRAFANA_DIR: Final = Path(__file__).parents[3] / "cookbook" / "litellm_proxy_server" / "grafana_dashboard" +_GRAFANA_DIR: Final = Path(__file__).parents[3] / "litellm" / "integrations" / "prometheus_helpers" / "grafana_dashboards" _ALL_METRICS_DASHBOARD: Final = _GRAFANA_DIR / "dashboard_all_metrics" / "grafana_dashboard.json" _LITELLM_DASHBOARDS: Final = (_ALL_METRICS_DASHBOARD, _GRAFANA_DIR / "dashboard_v2" / "grafana_dashboard.json") _METRIC_TOKEN_RE: Final = re.compile(r"\blitellm_[a-z0-9_]+") diff --git a/tests/unit/proxy/discovery_endpoints/test_skill_scanning.py b/tests/unit/proxy/discovery_endpoints/test_skill_scanning.py index 934620e5de8..028a185f9a0 100644 --- a/tests/unit/proxy/discovery_endpoints/test_skill_scanning.py +++ b/tests/unit/proxy/discovery_endpoints/test_skill_scanning.py @@ -97,7 +97,7 @@ def test_scanner_config_rejects_unsafe_credential_destinations(endpoint: str) -> def test_adapter_authentication_and_exact_remote_source(monkeypatch: pytest.MonkeyPatch) -> None: - from cookbook.skill_scanner import server + from litellm.proxy.discovery_endpoints.skill_scanner_adapter import server monkeypatch.setenv("SKILL_SCANNER_API_KEY", "adapter-secret") clock: Final = MagicMock() @@ -123,7 +123,7 @@ def test_adapter_authentication_and_exact_remote_source(monkeypatch: pytest.Monk def test_adapter_accepts_expanded_stored_skill_bytes(monkeypatch: pytest.MonkeyPatch) -> None: - from cookbook.skill_scanner import server + from litellm.proxy.discovery_endpoints.skill_scanner_adapter import server from litellm.proxy.anthropic_endpoints.claude_code_endpoints.skill_upload_archive import ( build_uploaded_skill_archive, ) diff --git a/tests/unit/test_check_type_discipline.py b/tests/unit/test_check_type_discipline.py index abb8c0d6a9f..01dff0a8b13 100644 --- a/tests/unit/test_check_type_discipline.py +++ b/tests/unit/test_check_type_discipline.py @@ -1036,6 +1036,19 @@ def test_worker_count_never_exceeds_the_cap(): assert checker._worker_count(100_000) <= checker.MAX_WORKERS +def test_standalone_scanner_adapter_is_outside_the_package_lint_scope(tmp_path: Path) -> None: + adapter: Final = tmp_path / "litellm" / "proxy" / "discovery_endpoints" / "skill_scanner_adapter" + adapter.mkdir(parents=True) + server: Final = adapter / "server.py" + server.write_text("value = []\n") + proxy: Final = tmp_path / "proxy.py" + proxy.write_text("value = []\n") + + assert tuple(checker.collect_paths([str(tmp_path)])) == (proxy,) + assert tuple(checker.collect_paths([str(server)])) == () + assert checker.scan_paths(tuple(checker.collect_paths([str(tmp_path)]))) + + def test_scan_paths_below_the_threshold_returns_every_violation(tmp_path): paths = _corpus(tmp_path, 3) assert checker._worker_count(len(paths)) == 1 From a6ce922f0762fa482f863b19b579690c87e91d64 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 11 Nov 2026 15:38:27 -0800 Subject: [PATCH 125/189] ci: remove CircleCI (#50950) --- .circleci/AGENTS.md | 7 - .circleci/config.yml | 1785 ----------------- .circleci/scripts/run_migration_tests.py | 116 -- .dockerignore | 1 - .github/AGENTS.md | 5 + .github/actions/detect-changes/action.yml | 2 +- .github/ci-coverage-allowlist.yml | 12 +- .github/scripts/assert_ci_coverage.py | 103 +- .../scripts/ci}/classify_changes.sh | 8 +- .../scripts/ci}/path_filter.sh | 9 +- .../scripts/ci}/prepare_replica_roles.py | 0 .../scripts/ci}/run_integration.sh | 19 +- .../scripts/ci}/stop_integration_processes.py | 0 .../scripts/ci}/verify_integration_browser.py | 4 +- .../scripts/ci}/wait_integration_services.py | 0 .github/scripts/detect_changes.sh | 2 +- .github/workflows/test-integration.yml | 16 +- .../test-mcp-dependency-resolution.yml | 2 +- ci_cd/cost_map_guard.py | 18 +- codecov.yaml | 2 +- helm/litellm/ci/postgres.yaml | 2 +- helm/litellm/ci/test-values.yaml | 2 +- tests/_vcr_conftest_common.py | 2 +- tests/e2e/AGENTS.md | 2 +- .../e2e/ui/fixtures/mock_llm_server/server.py | 3 +- tests/e2e/ui/integration.config.ts | 3 - tests/e2e/ui/run_e2e.sh | 2 +- .../e2e/ui/tests/auth/proxyLogoutUrl.spec.ts | 6 +- .../e2e/ui/tests/proxy-admin/license.spec.ts | 4 +- tests/integration/AGENTS.md | 14 +- tests/integration/README.md | 32 +- tests/integration/_support/redis_process.py | 2 +- tests/integration/conftest.py | 8 +- .../database/test_harness_lifecycle.py | 5 +- tests/unit/test_assert_ci_coverage.py | 169 +- ..._path_filter.py => test_ci_path_filter.py} | 30 +- tests/unit/test_circleci_checkout.py | 310 --- .../unit/test_circleci_run_migration_tests.py | 36 - tests/unit/test_circleci_rust_toolchain.py | 177 -- tests/unit/test_cost_map_guard.py | 29 + tests/unit/test_detect_changes.py | 6 +- tests/unit/test_vcr_conftest_common_banner.py | 2 +- 42 files changed, 294 insertions(+), 2663 deletions(-) delete mode 100644 .circleci/AGENTS.md delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/scripts/run_migration_tests.py create mode 100644 .github/AGENTS.md rename {.circleci/scripts => .github/scripts/ci}/classify_changes.sh (86%) rename {.circleci/scripts => .github/scripts/ci}/path_filter.sh (78%) rename {.circleci/scripts => .github/scripts/ci}/prepare_replica_roles.py (100%) rename {.circleci/scripts => .github/scripts/ci}/run_integration.sh (92%) rename {.circleci/scripts => .github/scripts/ci}/stop_integration_processes.py (100%) rename {.circleci/scripts => .github/scripts/ci}/verify_integration_browser.py (94%) rename {.circleci/scripts => .github/scripts/ci}/wait_integration_services.py (100%) rename tests/unit/{test_circleci_path_filter.py => test_ci_path_filter.py} (90%) delete mode 100644 tests/unit/test_circleci_checkout.py delete mode 100644 tests/unit/test_circleci_run_migration_tests.py delete mode 100644 tests/unit/test_circleci_rust_toolchain.py diff --git a/.circleci/AGENTS.md b/.circleci/AGENTS.md deleted file mode 100644 index b3591145a3e..00000000000 --- a/.circleci/AGENTS.md +++ /dev/null @@ -1,7 +0,0 @@ -# .circleci - -The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values - -Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base - -A step that checks out files from another commit fetches that commit with `--no-filter`, as `routing_parity` does, because the clone has no file contents and git would otherwise download each missing file in its own request, which took 210 to 896 seconds for the routing parity base diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 15129f1e5e3..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,1785 +0,0 @@ -version: 2.1 -parameters: - run_migration_tests: - type: boolean - default: false - migration_candidate_image: - type: string - default: "" - migration_baseline_image: - type: string - default: "ghcr.io/berriai/litellm-database:v1.102.0" - migration_source_sha: - type: string - default: "" - routing_parity_base: - type: string - default: "" -orbs: - codecov: codecov/codecov@4.0.1 - node: circleci/node@5.1.0 # Add this line to declare the node orb - win: circleci/windows@5.0 # Add Windows orb - -commands: - checkout_over_https: - parameters: - revision: - type: string - default: "" - shell: - type: string - default: /bin/bash -eo pipefail - steps: - - run: - name: Check out the public repository over HTTPS - shell: << parameters.shell >> - environment: - REQUESTED_REVISION: << parameters.revision >> - command: | - revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}" - [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1 - git init - git remote add origin https://github.com/BerriAI/litellm.git - for attempt in 1 2 3; do - git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break - [[ $attempt -lt 3 ]] || exit 1 - sleep 5 - done - skip_if_unrelated_changes: - parameters: - category: - type: enum - enum: ["backend", "client"] - default: "backend" - steps: - - run: - name: "Skip job when no << parameters.category >>-relevant files changed" - command: bash .circleci/scripts/path_filter.sh << parameters.category >> - setup_google_dns: - steps: - - run: - name: "Configure Google DNS" - command: | - # Backup original resolv.conf - sudo cp /etc/resolv.conf /etc/resolv.conf.backup - # Add both local and Google DNS servers - echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf - echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf - echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf - wait_for_service: - description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)" - parameters: - url: - type: string - timeout: - type: string - default: "60" - steps: - - run: - name: "Wait for << parameters.url >>" - command: | - TIMEOUT=<< parameters.timeout >> - URL="<< parameters.url >>" - ELAPSED=0 - echo "Waiting up to ${TIMEOUT}s for ${URL} ..." - if echo "$URL" | grep -q '^tcp://'; then - HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) - PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) - while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do - sleep 2; ELAPSED=$((ELAPSED+2)) - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi - done - else - while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do - sleep 2; ELAPSED=$((ELAPSED+2)) - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi - done - fi - echo "Service ready after ${ELAPSED}s" - install_helm: - steps: - - run: - name: Install Helm v3.17.3 - command: | - curl -sSLf -o /tmp/helm.tar.gz \ - https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz - echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c - - sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm - rm -f /tmp/helm.tar.gz - install_kind: - steps: - - run: - name: Install Kind v0.20.0 - command: | - curl -sSLf -o /tmp/kind \ - https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c - - chmod +x /tmp/kind - sudo mv /tmp/kind /usr/local/bin/kind - install_uv: - description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH." - steps: - - run: - name: Install uv (pinned 0.10.9) - command: | - curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh - echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh - rm -f /tmp/uv-install.sh - echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" - export PATH="$HOME/.local/bin:$PATH" - install_node: - description: "Install the Node.js version pinned in ui/litellm-dashboard/.nvmrc (24.19.0, which bundles npm 11.17.0) with checksum verification, and prepend it to PATH. Run this on any executor whose image does not already ship that version, or `npm ci` in ui/litellm-dashboard fails EBADENGINE against the engines floor. Installs into /opt/node rather than over /usr/local on purpose: cimg/python:*-browsers ships its own node there, and unpacking the tarball on top of it leaves npm 11.17 files merged with the image's npm 11.9 tree, which reports the new version and then exits 1 on `npm ci` with no error text at all. Requires checkout, which the .nvmrc drift check reads." - steps: - - run: - name: Install Node.js 24.19.0 - command: | - NODE_VERSION="24.19.0" - NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz" - NODE_EXPECTED_SHA="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647" - NVMRC_VERSION="$(tr -d '[:space:]' < ui/litellm-dashboard/.nvmrc)" - if [ "$NVMRC_VERSION" != "$NODE_VERSION" ]; then - echo "install_node: ui/litellm-dashboard/.nvmrc pins ${NVMRC_VERSION} but this command pins ${NODE_VERSION}; update NODE_VERSION and NODE_EXPECTED_SHA together" >&2 - exit 1 - fi - curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" - echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c - - sudo mkdir -p /opt/node - sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /opt/node --strip-components=1 - rm -f "/tmp/${NODE_TARBALL}" - echo 'export PATH="/opt/node/bin:$PATH"' >> "$BASH_ENV" - export PATH="/opt/node/bin:$PATH" - node --version - npm --version - install_rust: - description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself. Also restores the dev-profile cargo cache that save_cargo_target writes on main, minus the workspace crates' fingerprints so those always rebuild from the checked-out source." - steps: - - run: - name: Install Rust (rustup 1.28.2, toolchain 1.98.0) - command: | - case "$(uname -m)" in - x86_64) - RUSTUP_TRIPLE=x86_64-unknown-linux-gnu - RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c - ;; - aarch64) - RUSTUP_TRIPLE=aarch64-unknown-linux-gnu - RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c - ;; - *) - echo "install_rust: unsupported architecture $(uname -m)" >&2 - exit 1 - ;; - esac - curl -sSLf --retry 3 --retry-all-errors --retry-max-time 120 --connect-timeout 15 --max-time 60 -o /tmp/rustup-init \ - "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" - echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - - chmod +x /tmp/rustup-init - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0 - rm -f /tmp/rustup-init - echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" - echo 'export CARGO_INCREMENTAL=0' >> "$BASH_ENV" - export PATH="$HOME/.cargo/bin:$PATH" - rustc --version - cargo --version - { rustc -vV; cc --version; cat /etc/os-release; } > /tmp/cargo-build-env - - restore_cache: - keys: - - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}- - - run: - name: Force a rebuild of the workspace crates restored from the cargo cache - command: rm -rf litellm-rust/target/debug/.fingerprint/litellm-* - save_cargo_target: - steps: - - when: - condition: - equal: [main, << pipeline.git.branch >>] - steps: - - save_cache: - key: v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - paths: - - ~/.cargo/registry - - ~/project/litellm-rust/target/debug - start_postgres: - description: "Start a postgres-db container on port 5432 and wait until it accepts connections." - parameters: - db_name: - type: string - default: circle_test - image: - type: string - default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 - server_args: - type: string - default: "" - steps: - - run: - name: Start PostgreSQL - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=<< parameters.db_name >> \ - -p 5432:5432 \ - << parameters.image >> << parameters.server_args >> - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - start_redis: - description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys." - steps: - - run: - name: Start Redis - command: | - docker run -d \ - --name redis-cache \ - -p 6379:6379 \ - redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 - - wait_for_service: - url: tcp://localhost:6379 - timeout: "60" - install_codecov_cli: - steps: - - run: - name: Install Codecov CLI (pinned v11.3.1) - when: always - command: | - curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov - curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM - [ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ] - (cd /tmp && sha256sum -c codecov.SHA256SUM) - chmod +x /tmp/codecov - mkdir -p "$HOME/.local/bin" - mv /tmp/codecov "$HOME/.local/bin/codecov" - start_openai_record_replay_proxy: - description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay Redis cache hits. Cache misses call the live provider and store its response, so these legacy E2E jobs still require provider availability and may incur provider charges. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." - steps: - - run: - name: Start record/replay proxy - background: true - command: | - CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \ - RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \ - uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090 - - run: - name: Wait for record/replay proxy - command: | - for i in $(seq 1 30); do - if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then - echo "record/replay proxy is up" - exit 0 - fi - sleep 1 - done - echo "record/replay proxy did not become ready" >&2 - exit 1 - start_fake_openai_endpoint: - description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced." - steps: - - run: - name: Start fake OpenAI endpoint - background: true - command: | - uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 - - run: - name: Wait for fake OpenAI endpoint - command: | - for i in $(seq 1 30); do - if curl -sf http://localhost:8190/health >/dev/null 2>&1; then - echo "fake OpenAI endpoint is up" - exit 0 - fi - sleep 1 - done - echo "fake OpenAI endpoint did not become ready" >&2 - exit 1 - setup_litellm_enterprise_pip: - steps: - - run: - name: "Install local version of litellm-enterprise" - command: | - # litellm-enterprise is a uv workspace member and is already installed - # by the main `uv sync --all-groups --all-extras`. Do NOT run - # `uv sync --package litellm-enterprise` here — that overwrites the - # shared .venv and strips out dev/test deps (pytest, prisma, etc.). - uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" - install_windows_toolchain: - steps: - - run: - name: Install Rust and uv - no_output_timeout: 30m - command: | - $rustupInit = Join-Path $env:TEMP "rustup-init.exe" - $rustupVersion = "1.28.2" - $rustupUrl = "https://static.rust-lang.org/rustup/archive/$rustupVersion/x86_64-pc-windows-msvc/rustup-init.exe" - Invoke-WebRequest -Uri $rustupUrl -OutFile $rustupInit - $rustupExpected = "88d8258dcf6ae4f7a80c7d1088e1f36fa7025a1cfd1343731b4ee6f385121fc0" - $rustupActual = (Get-FileHash -Path $rustupInit -Algorithm SHA256).Hash.ToLower() - if ($rustupActual -ne $rustupExpected) { - throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual" - } - & $rustupInit -y --profile minimal --default-toolchain 1.98.0 - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - Remove-Item $rustupInit - $cargoBin = Join-Path $HOME ".cargo\bin" - $env:Path = "$cargoBin;$env:Path" - rustc --version - cargo --version - $installer = Join-Path $env:TEMP "uv-install.ps1" - Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer - $expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d" - $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() - if ($actual -ne $expected) { - throw "uv installer hash mismatch: expected $expected got $actual" - } - & $installer - Remove-Item $installer - $uvBin = Join-Path $HOME ".local\bin" - $env:Path = "$uvBin;$env:Path" - if (!(Test-Path $PROFILE)) { - New-Item -ItemType File -Force -Path $PROFILE | Out-Null - } - if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { - Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" - } - if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) { - Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`"" - } - setup_litellm_test_deps: - parameters: - checkout: - type: boolean - default: true - steps: - - when: - condition: << parameters.checkout >> - steps: - - checkout_over_https - - setup_google_dns - - install_uv - - install_rust - - restore_cache: - keys: - - v3-integration-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v3-integration-uv-cache-{{ checksum "uv.lock" }} - - save_cargo_target - -jobs: - windows_release_wheel: - executor: - name: win/default - shell: powershell.exe - size: xlarge - working_directory: ~/project - environment: - UV_PYTHON: "3.11" - CARGO_HTTP_MULTIPLEXING: "false" - CARGO_NET_RETRY: "5" - steps: - - checkout_over_https: - shell: bash.exe - - run: - name: Skip job when no windows-release-relevant files changed - shell: bash.exe - command: bash .circleci/scripts/path_filter.sh windows-release - - run: - name: Install Python 3.11.9 from python.org - no_output_timeout: 6m - command: | - $ProgressPreference = "SilentlyContinue" - $installer = Join-Path $env:TEMP "python-3.11.9-amd64.exe" - Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe -OutFile $installer -TimeoutSec 120 - $expected = "5ee42c4eee1e6b4464bb23722f90b45303f79442df63083f05322f1785f5fdde" - $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() - if ($actual -ne $expected) { - throw "Python installer hash mismatch: expected $expected got $actual" - } - $arguments = "/quiet InstallAllUsers=1 PrependPath=1 Include_test=0 Include_doc=0 Include_launcher=0 TargetDir=C:\Python311" - $process = Start-Process -FilePath $installer -ArgumentList $arguments -PassThru - if (-not $process.WaitForExit(300000)) { - $process.Kill() - throw "Python installer did not finish within 5 minutes" - } - if ($process.ExitCode -ne 0) { - throw "Python installer exited with $($process.ExitCode)" - } - Remove-Item $installer - & C:\Python311\python.exe --version - - install_windows_toolchain - - run: - name: Record the Rust build environment for the release cargo cache key - command: | - & "$HOME\.cargo\bin\rustc.exe" -vV | Out-File -Encoding ascii .cargo-build-env - - restore_cache: - keys: - - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}- - - run: - name: Force a rebuild of the workspace crates restored from the cargo cache - command: | - $fingerprints = "litellm-rust/target/release/.fingerprint" - if (Test-Path $fingerprints) { - Get-ChildItem -Path $fingerprints -Filter "litellm-*" | Remove-Item -Recurse -Force - } - - run: - name: Build the release wheel and install it under a worst-case MAX_PATH prefix - no_output_timeout: 30m - environment: - UV_HTTP_TIMEOUT: "300" - command: | - $env:Path = "C:\Python311;C:\Python311\Scripts;$HOME\.cargo\bin;$HOME\.local\bin;$env:Path" - uv build --wheel --out-dir dist - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - python tests/windows_tests/check_windows_wheel_install.py - - when: - condition: - equal: [main, << pipeline.git.branch >>] - steps: - - save_cache: - key: v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - paths: - - ~/.cargo/registry - - ~/project/litellm-rust/target/release - - base_sdk_install: - docker: - - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - steps: - - checkout_over_https - - setup_google_dns - - install_uv - - install_rust - - run: - name: Build the wheel - no_output_timeout: 30m - environment: - UV_HTTP_TIMEOUT: "300" - command: | - uv build --wheel --out-dir dist - - run: - name: Install the wheel with no extras and smoke-check it - environment: - UV_HTTP_TIMEOUT: "300" - command: | - uv venv /tmp/base-sdk --python 3.12 - VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl - /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py - - run: - name: Guard against MAX_PATH-busting packaged wheel paths - command: | - python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only - - helm_chart_testing: - machine: - image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker - resource_class: medium - working_directory: ~/project - - steps: - - checkout_over_https - - skip_if_unrelated_changes - - attach_workspace: - at: ~/project - - setup_google_dns - - install_helm - - install_kind - - # Install kubectl (pinned version with hardcoded checksum) - - run: - name: Install kubectl v1.31.4 - command: | - curl -sSLf -o /tmp/kubectl \ - https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl - echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c - - chmod +x /tmp/kubectl - sudo mv /tmp/kubectl /usr/local/bin/ - - # Create kind cluster - - run: - name: Create Kind Cluster - command: | - kind create cluster --name litellm-test - - - run: - name: Load Docker Database Image for helm tests - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - IMAGE_TAG=${CIRCLE_SHA1:-ci} - docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} - - - run: - name: Load Docker image into Kind - command: | - IMAGE_TAG=${CIRCLE_SHA1:-ci} - kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test - - - run: - name: Run helm lint - command: | - helm lint ./helm/litellm -f ./helm/litellm/ci/test-values.yaml - - - run: - name: Start PostgreSQL for the chart - command: | - kubectl apply -f ./helm/litellm/ci/postgres.yaml - kubectl rollout status deployment/litellm-ci-postgres --timeout=3m - - - run: - name: Run helm tests - command: | - IMAGE_TAG=${CIRCLE_SHA1:-ci} - helm install litellm ./helm/litellm -f ./helm/litellm/ci/test-values.yaml \ - --set image.repository=litellm-ci \ - --set image.tag=${IMAGE_TAG} \ - --set image.pullPolicy=Never \ - --wait --wait-for-jobs --timeout 10m || { - kubectl get pods -o wide - kubectl describe pods -l app.kubernetes.io/instance=litellm - kubectl logs -l app.kubernetes.io/component=proxy --tail=200 || true - kubectl logs -l app.kubernetes.io/component=migrations --tail=200 || true - exit 1 - } - - echo "Printing proxy pod logs..." - kubectl logs -l app.kubernetes.io/component=proxy --tail=200 - - helm test litellm --logs - - # Cleanup - - run: - name: Cleanup - command: | - kind delete cluster --name litellm-test - when: always # This ensures cleanup runs even if previous steps fail - - db_migration_disable_update_check: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: medium - working_directory: ~/project - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_fake_openai_endpoint - - start_postgres: - db_name: litellm_test - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Seed database with real schema - command: | - docker run -d \ - -p 4001:4000 \ - -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ - -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \ - --name schema-seed \ - --add-host=host.docker.internal:host-gateway \ - -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --use_prisma_db_push - - wait_for_service: - url: http://localhost:4001 - timeout: "300" - - run: - name: Stop schema seed container - command: docker stop schema-seed && docker rm schema-seed - - run: - name: Run Docker container with bad schema and disabled updates - command: | - docker run -d \ - -p 4000:4000 \ - -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \ - -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ - -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - -e DISABLE_SCHEMA_UPDATE="True" \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - --name my-app \ - --add-host=host.docker.internal:host-gateway \ - -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ - -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ - -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 - - wait_for_service: - url: http://localhost:4000 - timeout: "60" - - run: - name: Check container logs for expected message - command: | - echo "=== Printing Full Container Startup Logs ===" - LOG_OUTPUT="$(docker logs my-app 2>&1)" - printf '%s\n' "$LOG_OUTPUT" - echo "=== End of Full Container Startup Logs ===" - - if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then - echo "Expected message found in logs. Test passed." - else - echo "Expected message not found in logs. Test failed." - exit 1 - fi - - run: - name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -v \ - --junitxml=test-results/junit-2.xml \ - --durations=5" - no_output_timeout: 15m - - store_test_results: - path: test-results - - proxy_logging_guardrails_model_info_tests: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - start_openai_record_replay_proxy - - start_fake_openai_endpoint - # Second Docker Container Run with Different Config - # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license - - run: - name: Run Second Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ - -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE="bad-license" \ - -e LITELLM_LOG=ERROR \ - --add-host host.docker.internal:host-gateway \ - --name my-app-3 \ - -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 - - - run: - name: Start outputting logs for second container - command: docker logs -f my-app-3 - background: true - - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - - run: - name: Run second round of tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -v \ - --junitxml=test-results/junit-2.xml \ - --durations=5" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results - e2e_ui_build: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: large - working_directory: ~/project - steps: - - checkout_over_https - - skip_if_unrelated_changes: - category: client - - setup_google_dns - - install_node - - restore_cache: - keys: - - ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - - run: - name: Install dashboard dependencies - command: | - cd ui/litellm-dashboard - npm ci - - save_cache: - key: ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - paths: - - ui/litellm-dashboard/node_modules - - run: - name: Build UI bundle - command: | - cd ui/litellm-dashboard - npm run build - mkdir -p ../../ui-bundle - mv out ../../ui-bundle/out - find ../../ui-bundle/out -name '*.html' ! -name 'index.html' | while read -r f; do - d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" - done - - persist_to_workspace: - root: ~/project - paths: - - ui-bundle/out - - e2e_ui_testing: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: e2euser - POSTGRES_PASSWORD: e2epassword - POSTGRES_DB: litellm_e2e - resource_class: xlarge - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" - CI: "true" - # Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can - # assert the redirect. Set at job level so both the proxy boot step and the - # Playwright step (whose skip guard reads this) see the same value. Safe for - # the rest of the suite: nothing else performs a logout. - PROXY_LOGOUT_URL: "https://www.example.com" - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes: - category: client - - attach_workspace: - at: ~/project - - setup_google_dns - - install_node - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Python dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - - save_cache: - key: v1-uv-cache-{{ checksum "uv.lock" }} - paths: - - ~/.cache/uv - - restore_cache: - keys: - - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - - run: - name: Install Node dependencies and Playwright - # The cimg/python:3.12-browsers image already ships the Chromium system - # libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.). - # `--with-deps` triggers a redundant apt-get update + install that adds - # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors, - # so we install just the browser binary. - command: | - cd tests/e2e/ui - npm ci - npx playwright install chromium - - save_cache: - key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - paths: - - tests/e2e/ui/node_modules - - ~/.cache/ms-playwright - - run: - name: Use built UI bundle - command: | - rm -rf litellm/proxy/_experimental/out - cp -r ui-bundle/out litellm/proxy/_experimental/out - - wait_for_service: - url: tcp://localhost:5432 - timeout: "30" - - run: - name: Push Prisma schema - command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f tests/e2e/ui/fixtures/seed.sql - - run: - name: Start mock LLM server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py - background: true - - run: - name: Start mock Presidio server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py - background: true - - run: - name: Wait for mock Presidio server - command: | - for i in $(seq 1 30); do - if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi - sleep 1 - done - echo "Mock Presidio server never answered /health on port 8091" >&2 - exit 1 - - run: - name: Start LiteLLM proxy - environment: - MOCK_LLM_URL: "http://127.0.0.1:8090/v1" - DISABLE_SCHEMA_UPDATE: "true" - SERVER_ROOT_PATH: "" - # PROXY_LOGOUT_URL is inherited from the job-level environment so the - # proxy and proxyLogoutUrl.spec.ts agree on the logout target. - # LITELLM_LICENSE is forwarded from the project env so premium-gated - # UI flows can be exercised. license.spec.ts asserts the resulting - # JWT carries premium_user=true; if it ever stops being passed, that - # test fails loudly rather than silently regressing premium coverage. - command: | - LITELLM_LICENSE="$LITELLM_LICENSE" \ - uv run --no-sync python -m litellm.proxy.proxy_cli \ - --config tests/e2e/ui/fixtures/config.yml \ - --port 4000 - background: true - - run: - name: Wait for proxy to be ready - command: | - for i in $(seq 1 60); do - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true) - if [ "$HTTP_CODE" = "200" ]; then - echo "Proxy is ready" - exit 0 - fi - sleep 2 - done - echo "Proxy failed to start" - exit 1 - - run: - name: Run Playwright E2E tests - # Forward LITELLM_LICENSE so license.spec.ts can detect that the - # proxy was launched with a license and assert premium_user=true. - command: | - cd tests/e2e/ui - LITELLM_LICENSE="$LITELLM_LICENSE" \ - npx playwright test --config playwright.config.ts --workers=3 --retries=0 - no_output_timeout: 10m - - store_artifacts: - path: tests/e2e/ui/test-results - destination: e2e-test-results - - store_artifacts: - path: tests/e2e/ui/playwright-report - destination: e2e-playwright-report - - e2e_ui_testing_server_root_path: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: e2euser - POSTGRES_PASSWORD: e2epassword - POSTGRES_DB: litellm_e2e - resource_class: large - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" - CI: "true" - # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH - # is read both by the proxy at boot (to rewrite the built UI bundle in place) - # and by migration.serverRootPath.config.ts, which refuses to run without it. - SERVER_ROOT_PATH: "/litellm" - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes: - category: client - - attach_workspace: - at: ~/project - - setup_google_dns - - install_node - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Python dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - - save_cache: - key: v1-uv-cache-{{ checksum "uv.lock" }} - paths: - - ~/.cache/uv - - restore_cache: - keys: - - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - - run: - name: Install Node dependencies and Playwright - command: | - cd tests/e2e/ui - npm ci - npx playwright install chromium - - save_cache: - key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - paths: - - tests/e2e/ui/node_modules - - ~/.cache/ms-playwright - - run: - name: Use built UI bundle - command: | - rm -rf litellm/proxy/_experimental/out - cp -r ui-bundle/out litellm/proxy/_experimental/out - - wait_for_service: - url: tcp://localhost:5432 - timeout: "30" - - run: - name: Push Prisma schema - command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f tests/e2e/ui/fixtures/seed.sql - - run: - name: Start mock LLM server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py - background: true - - run: - name: Start mock Presidio server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py - background: true - - run: - name: Wait for mock Presidio server - command: | - for i in $(seq 1 30); do - if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi - sleep 1 - done - echo "Mock Presidio server never answered /health on port 8091" >&2 - exit 1 - - run: - name: Start LiteLLM proxy under a server root path - environment: - MOCK_LLM_URL: "http://127.0.0.1:8090/v1" - DISABLE_SCHEMA_UPDATE: "true" - # Output flows to this step's own log, so a boot crash is visible here - # rather than swallowed by a downstream readiness probe. - command: | - LITELLM_LICENSE="$LITELLM_LICENSE" \ - uv run --no-sync python -m litellm.proxy.proxy_cli \ - --config tests/e2e/ui/fixtures/config.yml \ - --port 4000 - background: true - - run: - name: Wait for prefixed proxy to be ready - command: | - for i in $(seq 1 60); do - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer $LITELLM_MASTER_KEY" http://127.0.0.1:4000/litellm/health 2>/dev/null || true) - if [ "$HTTP_CODE" = "200" ]; then - echo "Prefixed proxy is ready" - exit 0 - fi - sleep 2 - done - echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log" - exit 1 - - run: - name: Run migration smoke under SERVER_ROOT_PATH - command: | - cd tests/e2e/ui - LITELLM_LICENSE="$LITELLM_LICENSE" \ - npx playwright test --config migration.serverRootPath.config.ts - no_output_timeout: 10m - - store_artifacts: - path: tests/e2e/ui/test-results - destination: e2e-server-root-path-test-results - - store_artifacts: - path: tests/e2e/ui/playwright-report - destination: e2e-server-root-path-playwright-report - - build_docker_database_image: - parameters: - migration_qualification: - type: boolean - default: false - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - when: - condition: << parameters.migration_qualification >> - steps: - - checkout_over_https: - revision: << pipeline.parameters.migration_source_sha >> - - unless: - condition: << parameters.migration_qualification >> - steps: - - checkout_over_https - - skip_if_unrelated_changes - - - run: - name: Build Docker image - environment: - MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >> - command: | - if [ -n "$MIGRATION_CANDIDATE_IMAGE" ]; then - [[ "$MIGRATION_CANDIDATE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+@sha256:[0-9a-f]{64}$ ]] || exit 1 - docker pull "$MIGRATION_CANDIDATE_IMAGE" - docker tag "$MIGRATION_CANDIDATE_IMAGE" litellm-docker-database:ci - else - docker build \ - --label org.opencontainers.image.revision="$(git rev-parse HEAD)" \ - -t litellm-docker-database:ci \ - -f Dockerfile . - fi - python3 .circleci/scripts/run_migration_tests.py record-image - - - run: - name: Save Docker image to workspace root - command: | - docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst - - - persist_to_workspace: - root: . - paths: - - litellm-docker-database.tar.zst - - migration-image.json - - migration_startup_tests: - parameters: - suite: - type: enum - enum: [startup, recovery, legacy, upgrade, shaped] - baseline: - type: boolean - default: false - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - environment: - LITELLM_MIGRATION_TESTS: "1" - LITELLM_MIGRATION_TEST_IMAGE: litellm-docker-database:ci - LITELLM_MIGRATION_BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >> - MIGRATION_TEST_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres - MIGRATION_TEST_CONTAINER_ADMIN_URL: postgresql://postgres:postgres@host.docker.internal:5432/postgres - MIGRATION_TEST_OUTPUT: /tmp/migration-results - PYTHONPATH: tests/e2e - steps: - - checkout_over_https: - revision: << pipeline.parameters.migration_source_sha >> - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install test dependencies - command: uv sync --frozen --all-groups --all-extras --python 3.12 - - attach_workspace: - at: ~/project - - run: - name: Load the shared candidate and start PostgreSQL - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker run -d --name migration-postgres \ - -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \ - -p 5432:5432 \ - postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - when: - condition: << parameters.baseline >> - steps: - - run: - name: Pull the baseline release the upgrade starts from - environment: - BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >> - command: | - [[ "$BASELINE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+(@sha256:[0-9a-f]{64}|:v[0-9][0-9a-z.-]*)$ ]] || exit 1 - docker pull "$BASELINE_IMAGE" - - run: - name: Run migration startup regressions - environment: - MIGRATION_TEST_SUITE: << parameters.suite >> - MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >> - command: | - mkdir -p /tmp/migration-results - uv run --no-sync python .circleci/scripts/run_migration_tests.py - no_output_timeout: 15m - - store_test_results: - path: /tmp/migration-results/junit - - run: - name: Package migration diagnostics - when: always - command: | - mkdir -p /tmp/migration-artifacts - if [ -d /tmp/migration-results ]; then - tar -czf /tmp/migration-artifacts/diagnostics.tar.gz -C /tmp/migration-results . - if [ -f /tmp/migration-results/verdict.json ]; then - cp /tmp/migration-results/verdict.json /tmp/migration-artifacts/verdict.json - fi - fi - - store_artifacts: - path: /tmp/migration-artifacts - destination: migration-results - - run: - name: Remove migration test containers - when: always - command: | - docker ps -aq --filter label=litellm-migration-test=true | xargs -r docker rm -f - docker rm -f migration-postgres || true - - test_bad_database_url: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: medium - working_directory: ~/project - steps: - - checkout_over_https - - skip_if_unrelated_changes - - attach_workspace: - at: ~/project - - setup_google_dns - - start_postgres - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker tag litellm-docker-database:ci myapp:latest - - run: - name: Run Docker container with bad DATABASE_URL - command: | - set +e - docker run --name my-app \ - -p 4000:4000 \ - -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \ - -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ - myapp:latest \ - --port 4000 > docker_output.log 2>&1 - echo "$?" > docker_exit_code - set -e - - run: - name: Display Docker logs - command: cat docker_output.log - - run: - name: Proxy must refuse to serve on an unreachable database - command: | - fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; } - exit_code="$(cat docker_exit_code)" - [ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database" - grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server" - ! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state" - ! docker exec my-app true 2>/dev/null || fail "container is still running" - echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed." - - integration_contracts: - parameters: - suite: - type: string - mode: - type: enum - enum: [standard, replica] - default: standard - parallelism: - type: integer - default: 1 - resource_class: - type: enum - enum: [large, xlarge] - default: large - machine: - image: ubuntu-2204:2024.04.1 - resource_class: << parameters.resource_class >> - working_directory: ~/project - parallelism: << parameters.parallelism >> - steps: - - setup_litellm_test_deps - - when: - condition: - equal: [browser, << parameters.suite >>] - steps: - - install_node - - restore_cache: - keys: - - integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} - - run: - name: Install locked browser dependencies - command: | - cd ui/litellm-dashboard - npm ci - cd ../../tests/e2e/ui - npm ci - sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l \ - timeout --signal=TERM --kill-after=20s 6m node node_modules/@playwright/test/cli.js install-deps chromium - timeout --signal=TERM --kill-after=20s 3m node node_modules/@playwright/test/cli.js install chromium - - save_cache: - key: integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} - paths: - - ~/.npm - - ~/.cache/ms-playwright - - run: - name: Build the candidate dashboard - command: cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build - - when: - condition: - equal: [mcp, << parameters.suite >>] - steps: - - run: - name: Install MCP SDK1 peer - command: | - uv venv --python 3.12 .venv-mcp-peer - uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' - echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV" - - when: - condition: - equal: [sdk, << parameters.suite >>] - steps: - - run: - name: Install redis-server 7.2.16 - command: | - mkdir -p /tmp/redis-bin - cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)" - docker cp "${cid}:/usr/local/bin/redis-server" /tmp/redis-bin/redis-server - docker rm "$cid" - /tmp/redis-bin/redis-server --version | grep -q 'v=7.2.16' - echo 'export PATH="/tmp/redis-bin:$PATH"' >> "$BASH_ENV" - - start_postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - server_args: "-c max_connections=300 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - - start_redis - - run: - name: Run owned integration contracts - command: bash .circleci/scripts/run_integration.sh << parameters.suite >> << parameters.mode >> - no_output_timeout: 15m - - run: - name: Stop owned database and Redis - when: always - command: | - mkdir -p test-results/services-<< parameters.suite >>-<< parameters.mode >> - docker logs postgres-db > test-results/services-<< parameters.suite >>-<< parameters.mode >>/postgres.log 2>&1 || true - docker logs redis-cache > test-results/services-<< parameters.suite >>-<< parameters.mode >>/redis.log 2>&1 || true - docker rm -f postgres-db redis-cache - test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - - when: - condition: - not: - equal: [1, << parameters.parallelism >>] - steps: - - run: - name: Prepare this shard's execution record - command: | - results="test-results/integration-<< parameters.suite >>" - if [ "<< parameters.mode >>" = replica ]; then - results="$results-replica" - fi - mkdir -p "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX" - cp "$results/execution.json" "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX/execution.json" - if [ "<< parameters.suite >>" = mcp ]; then - mkdir -p "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage" - cp test-results/integration-mcp/coverage/data "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage/data.$CIRCLE_NODE_INDEX" - fi - - persist_to_workspace: - root: /tmp/integration-shards - paths: - - << parameters.suite >>-<< parameters.mode >> - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - - integration_shard_coverage: - parameters: - suite: - type: enum - enum: [management, accounting, database, extensions, providers, mcp] - mode: - type: enum - enum: [standard, replica] - default: standard - shards: - type: integer - docker: - - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: small - working_directory: ~/project - environment: - UV_PYTHON_DOWNLOADS: never - steps: - - checkout_over_https - - install_uv - - run: - name: Install the locked shard verifier and coverage dependencies - command: | - uv export --frozen --all-groups --all-extras --no-emit-project --no-emit-workspace --no-header --no-annotate \ - | awk '/^[^ ]/ { keep = $0 ~ /^(annotated-types|coverage|pydantic|pydantic-core|typing-extensions|typing-inspection)==/ } keep' \ - > /tmp/fan-in-requirements.txt - uv venv .venv --python 3.12 - uv pip install --no-config --python .venv --require-hashes -r /tmp/fan-in-requirements.txt - - attach_workspace: - at: /tmp/integration-shards - - run: - name: Verify every << parameters.suite >> case ran exactly once - command: | - .venv/bin/python -m tests.integration._support.sharding /tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >> \ - --shards << parameters.shards >> - - when: - condition: - equal: [mcp, << parameters.suite >>] - steps: - - run: - name: Combine coverage from every MCP shard - command: | - mkdir -p test-results/integration-mcp/coverage - export COVERAGE_FILE="$PWD/test-results/integration-mcp/coverage/data" - .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml /tmp/integration-shards/mcp-standard/*/coverage - .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > test-results/integration-mcp/coverage/coverage.txt - .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d test-results/integration-mcp/coverage/html - - store_artifacts: - path: test-results/integration-mcp/coverage - - routing_parity: - parameters: - suite: - type: string - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - setup_litellm_test_deps - - run: - name: Check out base product code - environment: - ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >> - command: | - [[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1 - git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE" - git rm -r -f --quiet litellm enterprise litellm-proxy-extras - git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras - git reset --quiet - test -f litellm/rust_bridge/_native.abi3.so - - start_postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - - start_redis - - run: - name: Run base side - command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity base - no_output_timeout: 15m - - run: - name: Stop base database and Redis - when: always - command: | - mkdir -p test-results/services-<< parameters.suite >>-parity-base - docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-base/postgres.log 2>&1 || true - docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-base/redis.log 2>&1 || true - docker rm -f postgres-db redis-cache - test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - - run: - name: Check out head product code - command: | - git rm -r -f --quiet litellm enterprise litellm-proxy-extras - git checkout "$CIRCLE_SHA1" -- litellm enterprise litellm-proxy-extras - git reset --quiet - test -f litellm/rust_bridge/_native.abi3.so - - start_postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - - start_redis - - run: - name: Run head side - command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity head - no_output_timeout: 15m - - run: - name: Stop head database and Redis - when: always - command: | - mkdir -p test-results/services-<< parameters.suite >>-parity-head - docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-head/postgres.log 2>&1 || true - docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-head/redis.log 2>&1 || true - docker rm -f postgres-db redis-cache - test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - - run: - name: Compare routing parity - command: PYTHONPATH="$PWD/tests" .venv/bin/python -m integration._support.routing check test-results/parity-<< parameters.suite >>/base test-results/parity-<< parameters.suite >>/head - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - - postgres_suite: - parameters: - test_path: - type: string - seed: - type: boolean - coverage_flag: - type: string - default: "" - timeout_minutes: - type: integer - workers: - type: integer - default: 1 - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - environment: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test - steps: - - checkout_over_https - - skip_if_unrelated_changes - - setup_litellm_test_deps: - checkout: false - - start_postgres: - db_name: litellm_test - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - - run: - name: Generate Prisma client - command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - when: - condition: << parameters.seed >> - steps: - - run: - name: Seed database schema - command: uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Run << parameters.test_path >> - command: | - mkdir -p test-results - coverage_args=() - worker_args=() - if [ "<< parameters.workers >>" -gt 1 ]; then - worker_args=(-n << parameters.workers >> --dist=worksteal) - fi - if [ -n "<< parameters.coverage_flag >>" ]; then - coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml) - fi - timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \ - << parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \ - --junitxml=test-results/junit.xml "${coverage_args[@]}" "${worker_args[@]}" - - when: - condition: << parameters.coverage_flag >> - steps: - - install_codecov_cli - - run: - name: Upload coverage - when: always - command: | - [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; } - codecov upload-process --disable-search --fail-on-error -f coverage.xml \ - -F << parameters.coverage_flag >> -C "$CIRCLE_SHA1" \ - -n "<< parameters.coverage_flag >>-${CIRCLE_BUILD_NUM}" --git-service github - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - -workflows: - migration_startup_scheduled: - triggers: - - schedule: - cron: "17 0,6,12,18 * * *" - filters: - branches: - only: main - jobs: - - build_docker_database_image: - migration_qualification: true - - migration_startup_tests: - name: migration-startup - suite: startup - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-recovery - suite: recovery - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-legacy-and-pooling - suite: legacy - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-upgrade - suite: upgrade - baseline: true - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-upgrade-shaped - suite: shaped - baseline: true - requires: [build_docker_database_image] - integration: - jobs: - - integration_contracts: - name: integration-<< matrix.suite >> - filters: &integration_default_run not pipeline.parameters.run_migration_tests and pipeline.parameters.routing_parity_base == "" - matrix: - parameters: - suite: [sdk, cost, browser] - parallelism: 1 - - integration_contracts: - name: integration-security - suite: security - parallelism: 1 - resource_class: xlarge - filters: *integration_default_run - - integration_contracts: - name: integration-management - suite: management - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-management-coverage - suite: management - shards: 4 - requires: [integration-management] - filters: *integration_default_run - - integration_contracts: - name: integration-accounting - suite: accounting - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-accounting-coverage - suite: accounting - shards: 4 - requires: [integration-accounting] - filters: *integration_default_run - - integration_contracts: - name: integration-providers - suite: providers - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-providers-coverage - suite: providers - shards: 4 - requires: - - integration-providers - filters: *integration_default_run - - integration_contracts: - name: integration-database - suite: database - parallelism: 2 - filters: *integration_default_run - - integration_shard_coverage: - name: integration-database-coverage - suite: database - shards: 2 - requires: [integration-database] - filters: *integration_default_run - - integration_contracts: - name: integration-mcp - suite: mcp - parallelism: 2 - filters: *integration_default_run - - integration_shard_coverage: - name: integration-mcp-coverage - suite: mcp - shards: 2 - requires: - - integration-mcp - filters: *integration_default_run - - integration_contracts: - name: integration-extensions - suite: extensions - parallelism: 8 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-extensions-coverage - suite: extensions - shards: 8 - requires: - - integration-extensions - filters: *integration_default_run - - integration_contracts: - name: integration-management-replica - suite: management - mode: replica - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-management-replica-coverage - suite: management - mode: replica - shards: 4 - requires: [integration-management-replica] - filters: *integration_default_run - - integration_contracts: - name: integration-database-replica - suite: database - mode: replica - parallelism: 2 - filters: *integration_default_run - - integration_shard_coverage: - name: integration-database-replica-coverage - suite: database - mode: replica - shards: 2 - requires: [integration-database-replica] - filters: *integration_default_run - - postgres_suite: - name: roi-database - test_path: tests/integration/database/test_roi_observed.py - seed: false - coverage_flag: roi-postgres - timeout_minutes: 10 - filters: *integration_default_run - - windows_release_wheel: - filters: *integration_default_run - - base_sdk_install: - filters: *integration_default_run - - build_docker_database_image: - filters: *integration_default_run - - proxy_logging_guardrails_model_info_tests: - requires: - - build_docker_database_image - filters: *integration_default_run - - db_migration_disable_update_check: - requires: - - build_docker_database_image - filters: *integration_default_run - - test_bad_database_url: - requires: - - build_docker_database_image - filters: *integration_default_run - - helm_chart_testing: - requires: - - build_docker_database_image - filters: *integration_default_run - - e2e_ui_build: - filters: *integration_default_run - - e2e_ui_testing: - requires: [e2e_ui_build] - filters: *integration_default_run - - e2e_ui_testing_server_root_path: - requires: [e2e_ui_build] - filters: *integration_default_run - - build_docker_database_image: - name: build_docker_database_image-migration - migration_qualification: true - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-startup - suite: startup - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-recovery - suite: recovery - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-legacy-and-pooling - suite: legacy - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-upgrade - suite: upgrade - baseline: true - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-upgrade-shaped - suite: shaped - baseline: true - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - routing_parity: - name: routing-parity-<< matrix.suite >> - matrix: - parameters: - suite: [management, accounting, database, providers, extensions, cost, mcp] - filters: pipeline.parameters.routing_parity_base != "" diff --git a/.circleci/scripts/run_migration_tests.py b/.circleci/scripts/run_migration_tests.py deleted file mode 100644 index 50d01f1d50b..00000000000 --- a/.circleci/scripts/run_migration_tests.py +++ /dev/null @@ -1,116 +0,0 @@ -from __future__ import annotations - -import json -import os -import re -import subprocess -import sys -from pathlib import Path -from typing import Final -from xml.etree import ElementTree - -SUITES: Final = { - "startup": (("test_startup.py",), 13), - "recovery": (("test_recovery.py",), 15), - "legacy": (("test_legacy.py", "test_pooling.py"), 11), - "upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5), - "shaped": (("test_shaped_database.py",), 1), -} - - -def successful_junit(path: Path, expected: int, exit_code: int) -> bool: - if exit_code != 0 or not path.is_file(): - return False - try: - root: Final = ElementTree.parse(path).getroot() - except ElementTree.ParseError: - return False - cases: Final = tuple(root.iter("testcase")) - identities: Final = frozenset((case.get("classname"), case.get("name")) for case in cases) - return len(cases) == len(identities) == expected and all( - not any(case.find(tag) is not None for tag in ("failure", "error", "skipped")) for case in cases - ) - - -def output(*command: str) -> str: - return subprocess.check_output(command, text=True, timeout=90).strip() - - -def record_image() -> None: - source: Final = output("git", "rev-parse", "HEAD") - image: Final = output("docker", "image", "inspect", "litellm-docker-database:ci", "--format", "{{.Id}}") - revision: Final = output( - "docker", - "image", - "inspect", - "litellm-docker-database:ci", - "--format", - '{{index .Config.Labels "org.opencontainers.image.revision"}}', - ) - assert re.fullmatch(r"[0-9a-f]{40}", source), "Invalid source revision" - assert revision == source, "Candidate image revision differs from the tested source" - Path("migration-image.json").write_text( - json.dumps( - { - "source_sha": source, - "image_id": image, - "candidate_image": os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""), - } - ) - ) - - -def main() -> int: - suite: Final = os.environ["MIGRATION_TEST_SUITE"] - files, expected = SUITES[suite] - metadata: Final = json.loads(Path("migration-image.json").read_text()) - assert metadata["source_sha"] == output("git", "rev-parse", "HEAD"), "Image and test source revisions differ" - assert metadata["image_id"] == output( - "docker", "image", "inspect", os.environ["LITELLM_MIGRATION_TEST_IMAGE"], "--format", "{{.Id}}" - ), "Loaded image differs from the build output" - assert metadata["candidate_image"] == os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""), "Wrong release candidate" - destination: Final = Path(os.environ["MIGRATION_TEST_OUTPUT"]) - junit: Final = destination / "junit" / "results.xml" - junit.parent.mkdir(parents=True, exist_ok=True) - result: Final = subprocess.run( - ( - sys.executable, - "-m", - "pytest", - *(f"tests/e2e/migrations/{name}" for name in files), - "-vv", - "--tb=short", - "--durations=10", - f"--junitxml={junit}", - "-o", - "addopts=", - "--reruns=0", - ), - check=False, - timeout=1200, - ) - passed: Final = successful_junit(junit, expected, result.returncode) - (destination / "verdict.json").write_text( - json.dumps( - { - **metadata, - "suite": suite, - "baseline_image": os.environ.get("LITELLM_MIGRATION_BASELINE_IMAGE", ""), - "expected_cases": expected, - "passed": passed, - "pytest_exit_code": result.returncode, - "test_revision": metadata["source_sha"], - "workflow_id": os.environ.get("CIRCLE_WORKFLOW_ID", ""), - "job_number": os.environ.get("CIRCLE_BUILD_NUM", ""), - }, - indent=2, - ) - ) - return 0 if passed else 1 - - -if __name__ == "__main__": - if sys.argv[1:] == ["record-image"]: - record_image() - else: - raise SystemExit(main()) diff --git a/.dockerignore b/.dockerignore index f0fdcf9d479..d8eb72a057f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ docs litellm/proxy/discovery_endpoints/skill_scanner_adapter -.circleci .github tests .git diff --git a/.github/AGENTS.md b/.github/AGENTS.md new file mode 100644 index 00000000000..14bd461a8a0 --- /dev/null +++ b/.github/AGENTS.md @@ -0,0 +1,5 @@ +# CI + +Integration groups and replica runs remain in `workflows/test-integration.yml`. Do not duplicate them in another workflow. + +CI helpers used by existing workflows live in `scripts/ci/`. Jobs that need merge bases check out full history. Removing a CI job does not count its tests as covered. List tests that no remaining job invokes explicitly in `ci-coverage-allowlist.yml`. diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index b0f9b72f0ee..59c6ab28c30 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -1,6 +1,6 @@ name: "Detect relevant changes" description: >- - Classify the pull request's changed files with .circleci/scripts/classify_changes.sh + Classify the pull request's changed files with .github/scripts/ci/classify_changes.sh and expose decision=run|skip for one category. backend means anything outside ui/, docs/ and markdown; ui means the dashboard sources alone. decision=skip lets callers short-circuit expensive steps while the job still completes successfully and satisfies diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index 76f8e3ef7e7..3a714187325 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -4,6 +4,16 @@ description: >- by a job nor listed here, so every entry below is a decision on the record. test_paths: + - reason: ran only in CircleCI, removed + paths: + - tests/e2e/migrations/test_legacy.py + - tests/e2e/migrations/test_pooling.py + - tests/e2e/migrations/test_recovery.py + - tests/e2e/migrations/test_rolling_upgrade.py + - tests/e2e/migrations/test_shaped_database.py + - tests/e2e/migrations/test_startup.py + - tests/e2e/migrations/test_upgrade.py + - tests/integration/database/test_roi_observed.py - reason: >- litellm.agent() end-to-end suite. It drives the real claude, codex, opencode and pi CLIs and deepagents against a live LiteLLM AI Gateway, so it needs those binaries on PATH plus @@ -19,7 +29,7 @@ test_paths: - tests/rust-python-harness - reason: >- Owned by the open pull request that moves the Rust/Python parity checks (#45135). It ran in - the CircleCI build_and_test job that globbed tests/test_*.py, which this migration deletes, + the retired build_and_test job that globbed tests/test_*.py, and that pull request gives it a new home paths: - tests/test_rust_python_harness.py diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index b442948c3ca..b443fd9e0d9 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -10,6 +10,7 @@ import warnings from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from functools import lru_cache +from itertools import chain from types import MappingProxyType from typing import Final @@ -17,7 +18,6 @@ import yaml REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] WORKFLOW_DIR = REPO_ROOT / ".github" / "workflows" -CIRCLECI_CONFIG = REPO_ROOT / ".circleci" / "config.yml" ALLOWLIST_FILE = REPO_ROOT / ".github" / "ci-coverage-allowlist.yml" TESTS_ROOT = REPO_ROOT / "tests" @@ -25,14 +25,18 @@ ALLOWLIST_KEYS = frozenset({"description", "test_paths", "dockerfiles"}) PATH_FILTER_KEYS = frozenset({"paths", "paths-ignore"}) TEST_PATH_KEYS = frozenset({"test-path", "test-paths", "test_path"}) DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"}) -TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bcircleci tests\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s") +TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s") IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b") TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+") IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)") DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*") COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE) GLOB_CHARS = frozenset("*?") -DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"}) +DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType( + { + "image-scan.yml": "tests/integration/database/image/", + } +) # Trees whose jobs are sharded with no catch-all bucket, so every child that holds # tests has to be named by some shard or it runs nowhere. A child listed here is @@ -115,9 +119,7 @@ def _scalars(node: object, key: str) -> tuple[Scalar, ...]: def _config_files() -> tuple[pathlib.Path, ...]: - workflows = tuple(sorted(path for path in WORKFLOW_DIR.iterdir() if path.suffix in (".yml", ".yaml"))) - circleci = (CIRCLECI_CONFIG,) if CIRCLECI_CONFIG.is_file() else () - return workflows + circleci + return tuple(sorted(path for path in WORKFLOW_DIR.iterdir() if path.suffix in (".yml", ".yaml"))) def _all_scalars() -> tuple[Scalar, ...]: @@ -243,7 +245,6 @@ def _describe(paths: tuple[str, ...]) -> str: return f"{len(paths)} test file(s) invoked by no job: {names}{suffix}" -GLOB_CALL_RE = re.compile(r'circleci tests glob "([^"]+)"') KEYWORD_RE = re.compile(r"-k\s+\\?[\"']([^\"'\\]+)") @@ -316,16 +317,14 @@ def _keyword_terms( def _slices() -> tuple[Slice, ...]: - if not CIRCLECI_CONFIG.exists(): - return () test_files: Final = frozenset(_test_files()) - jobs: Final = yaml.safe_load(CIRCLECI_CONFIG.read_text()).get("jobs", {}) return tuple( - Slice(job=job, globs=globs, named=named, required=required, excluded=excluded, understood=understood) - for job, body in jobs.items() + Slice(job=f"{path.name}:{job}", globs=globs, named=named, required=required, excluded=excluded, understood=understood) + for path in _config_files() + for job, body in yaml.safe_load(path.read_text()).get("jobs", {}).items() for text in ("\n".join(_strings(body)),) if "pytest" in text - for globs in (tuple(GLOB_CALL_RE.findall(text)),) + for globs in (tuple(token for token in TEST_TOKEN_RE.findall(text) if GLOB_CHARS & set(token)),) for named in (frozenset(TEST_TOKEN_RE.findall(text)) & test_files,) for required, excluded, understood in ( _keyword_terms(tuple(KEYWORD_RE.findall(text)), attributable=len(globs) < 2), @@ -352,16 +351,11 @@ def _matchable_names(relative_path: str) -> frozenset[str]: def _workflow_named_tokens() -> frozenset[str]: - """Test tokens a GitHub Actions job names directly. - - A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a - workflow names it, so the slice check has to credit those the same way the census does. - """ return _invoked_test_tokens( scalar for path in _config_files() - if path != CIRCLECI_CONFIG for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name) + if not KEYWORD_RE.search(scalar.value) ) @@ -566,13 +560,28 @@ def _owns(workflow: str, relative_path: str) -> bool: return owned_root is not None and relative_path.startswith(owned_root) +def _expression_suites(value: str) -> tuple[str, ...]: + suites: Final = chain.from_iterable(map(json.loads, re.findall(r"'(\[[^']*\])'", value))) + return tuple(suite for suite in suites if isinstance(suite, str) and "{" not in suite) + + +def _matrix_suites(value: object) -> tuple[str, ...]: + if isinstance(value, list): + return tuple(suite for suite in value if isinstance(suite, str)) + if isinstance(value, str): + return _expression_suites(value) + return () + + def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]: return frozenset[str]().union( *(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path)) ) -def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]: +def _integration_ownership( + repo_root: pathlib.Path = REPO_ROOT, *, allowlist: Allowlist | None = None +) -> tuple[frozenset[str], tuple[Finding, ...]]: runner: Final = repo_root / "tests/integration/run.py" if not runner.exists(): return frozenset(), () @@ -589,26 +598,19 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json" browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else () browser_paths: Final = frozenset(node.split("::", 1)[0] for node in browser_nodes) - circle_path: Final = repo_root / ".circleci/config.yml" - circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {} - circle_test_path_tokens: Final = _invoked_test_tokens( - scalar for scalar in _scalars(circle, "config.yml") if scalar.key == "test_path" - ) - steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ()) + integration_path: Final = repo_root / ".github/workflows/test-integration.yml" + integration: Final = yaml.safe_load(integration_path.read_text()) if integration_path.exists() else {} + integration_jobs: Final = integration.get("jobs", {}) invoked: Final = any( - ".circleci/scripts/run_integration.sh" in scalar.value - for scalar in _scalars(steps, "integration_contracts") - if scalar.key == "command" + ".github/scripts/ci/run_integration.sh" in scalar.value + for scalar in _scalars(integration_jobs, "integration") + if scalar.key == "run" ) scheduled: Final = frozenset( suite - for job in circle.get("workflows", {}).get("integration", {}).get("jobs", ()) - if isinstance(job, dict) and "integration_contracts" in job - for suite in job["integration_contracts"] - .get("matrix", {}) - .get("parameters", {}) - .get("suite", (job["integration_contracts"].get("suite"),)) - if isinstance(suite, str) + for body in integration_jobs.values() + if any(".github/scripts/ci/run_integration.sh" in scalar.value for scalar in _scalars(body, "integration")) + for suite in _matrix_suites(body.get("strategy", {}).get("matrix", {}).get("suite", ())) ) required: Final = (frozenset({"browser"}) if browser_paths else frozenset()) | frozenset( group @@ -630,19 +632,27 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens } ) gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values()) - circle_owned: Final = ( + invoked_github_files: Final = frozenset( + path for path in github_files if any(_token_covers(token, path) for token in gha_tokens) + ) + grouped: Final = ( frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files ) findings: Final = ( tuple( Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions") - for path in sorted(circle_owned) - if any(_token_covers(token, path) for token in gha_tokens) + for path in sorted(grouped) + if any( + _token_covers(token, path) + for workflow, tokens in workflow_tokens.items() + if workflow != "test-integration.yml" + for token in tokens + ) ) + tuple( Finding(path, "GitHub-owned integration contract has no invoking job") for path in sorted(github_files) - if not any(_token_covers(token, path) for token in gha_tokens | circle_test_path_tokens) + if path not in invoked_github_files and not (allowlist is not None and allowlist.covers_test(path)) ) + tuple( Finding(path, "GitHub-owned integration file is missing") @@ -658,6 +668,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens browser_commands: Final = tuple( scalar.value for path in (repo_root / ".github/workflows").glob("*.y*ml") + if path.name != "test-integration.yml" for scalar in _scalars(yaml.safe_load(path.read_text()), path.name) if scalar.key in {"run", "command"} ) @@ -690,14 +701,14 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens else () ) group_findings: Final = tuple( - Finding(group, "canonical integration group is not scheduled by CircleCI") + Finding(group, "canonical integration group is not scheduled by GitHub Actions") for group in sorted(required - scheduled) ) + tuple(Finding(path, "canonical node must have exactly one integration group") for path in sorted(ungrouped)) if not paths or not invoked or not scheduled: return frozenset(), findings + ( - Finding(str(runner.relative_to(repo_root)), "dedicated CircleCI runner is missing"), + Finding(str(runner.relative_to(repo_root)), "dedicated GitHub Actions runner is missing"), ) - return paths | browser_paths | github_files, findings + group_findings + browser_findings + exclusion_findings + return paths | browser_paths | invoked_github_files, findings + group_findings + browser_findings + exclusion_findings def main() -> int: @@ -709,7 +720,7 @@ def main() -> int: allowlist = _load_allowlist() scalars = _all_scalars() - integration_paths, ownership_findings = _integration_ownership() + integration_paths, ownership_findings = _integration_ownership(allowlist=allowlist) test_findings = _uncovered_tests( allowlist, _invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),), @@ -731,8 +742,8 @@ def main() -> int: ) if ownership_findings: browser_remedy = ( - " CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may " - "select them and the default Playwright config must exclude them." + " The browser group in test-integration.yml owns the integrationCritical Playwright contracts. " + "Other workflows and the default Playwright config must exclude them." if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings) else "" ) diff --git a/.circleci/scripts/classify_changes.sh b/.github/scripts/ci/classify_changes.sh similarity index 86% rename from .circleci/scripts/classify_changes.sh rename to .github/scripts/ci/classify_changes.sh index a33695d0dbe..82851fa0a82 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.github/scripts/ci/classify_changes.sh @@ -14,22 +14,22 @@ outside_cost_map_set=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue case "$file" in - litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .circleci/scripts/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock) + litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .github/scripts/ci/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock) has_redis_compat=true ;; esac case "$file" in *.md | *.mdx) : ;; - pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py) + pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .github/scripts/ci/classify_changes.sh | tests/unit/test_ci_path_filter.py | tests/unit/test_detect_changes.py) has_mcp_dependencies=true ;; esac case "$file" in - litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*) + litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .github/scripts/ci/* | .github/actions/*) has_windows_release=true ;; esac case "$file" in ui/* | tests/e2e/ui/*) has_client=true ;; docs/* | *.md | *.mdx) : ;; - .github/* | .circleci/*) has_ci=true; has_backend=true ;; + .github/*) has_ci=true; has_backend=true ;; *) has_backend=true ;; esac case "$file" in diff --git a/.circleci/scripts/path_filter.sh b/.github/scripts/ci/path_filter.sh similarity index 78% rename from .circleci/scripts/path_filter.sh rename to .github/scripts/ci/path_filter.sh index c3cf6e566ff..e8ac6e5b918 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.github/scripts/ci/path_filter.sh @@ -6,10 +6,12 @@ here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_full() { echo "path-filter[$category]: running job ($1)" + echo "decision=run" + [ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=run" >> "$GITHUB_OUTPUT" exit 0 } -[ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request" +[ "${GITHUB_EVENT_NAME:-}" = pull_request ] || run_full "not a pull request" candidate_bases="${PATH_FILTER_BASE_BRANCH:-main}" merge_base="" @@ -36,5 +38,6 @@ if [ "$decision" = run ]; then run_full "$category-relevant changes detected" fi -echo "path-filter[$category]: only unrelated changes detected; halting job as successful" -circleci-agent step halt +echo "path-filter[$category]: only unrelated changes detected" +echo "decision=skip" +[ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=skip" >> "$GITHUB_OUTPUT" diff --git a/.circleci/scripts/prepare_replica_roles.py b/.github/scripts/ci/prepare_replica_roles.py similarity index 100% rename from .circleci/scripts/prepare_replica_roles.py rename to .github/scripts/ci/prepare_replica_roles.py diff --git a/.circleci/scripts/run_integration.sh b/.github/scripts/ci/run_integration.sh similarity index 92% rename from .circleci/scripts/run_integration.sh rename to .github/scripts/ci/run_integration.sh index fb91bcce77f..ed405634796 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.github/scripts/ci/run_integration.sh @@ -30,7 +30,7 @@ egress_cgroup=litellm-integration cleanup() { original_status=$? trap - EXIT INT TERM - sudo .venv/bin/python .circleci/scripts/stop_integration_processes.py \ + sudo .venv/bin/python .github/scripts/ci/stop_integration_processes.py \ "$integration_identity" "$(id -u)" "$proxy_pid" "$peer_pid" "$upstream_pid" \ > "$results/process-cleanup.txt" 2>&1 || original_status=1 for owned_pid in "$peer_pid" "$proxy_pid" "$upstream_pid"; do @@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out" test -f "$LITELLM_UI_PATH/index.html" fi -export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')" +export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')" export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED" uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1 @@ -93,7 +93,7 @@ export INTEGRATION_PROXY_DATABASE_URL="" export INTEGRATION_PROXY_READ_REPLICA_URL="" export INTEGRATION_ROUTING="" if [ "$mode" = replica ] || [ "$mode" = parity ]; then - .venv/bin/python .circleci/scripts/prepare_replica_roles.py > "$results/prepare-replica-roles.log" 2>&1 + .venv/bin/python .github/scripts/ci/prepare_replica_roles.py > "$results/prepare-replica-roles.log" 2>&1 export INTEGRATION_PROXY_DATABASE_URL="postgresql://litellm_writer:litellm-writer@127.0.0.1:5432/circle_test" export INTEGRATION_PROXY_READ_REPLICA_URL="postgresql://litellm_reader:litellm-reader@127.0.0.1:5432/circle_test" fi @@ -202,7 +202,7 @@ start_proxy() { if [ "$isolated_workers" = false ]; then start_proxy 4000 proxy.log proxy_pid="$launched_pid" - .venv/bin/python .circleci/scripts/wait_integration_services.py + .venv/bin/python .github/scripts/ci/wait_integration_services.py curl --noproxy '*' -sSf -X POST "$INTEGRATION_PROXY_URL/config/update" \ -H "Authorization: Bearer $LITELLM_MASTER_KEY" -H 'Content-Type: application/json' \ -d '{"router_settings": {"num_retries": 0}}' > "$results/seed-router-settings.json" @@ -210,7 +210,7 @@ if [ "$isolated_workers" = false ]; then export INTEGRATION_PEER_URL=http://127.0.0.1:4001 start_proxy 4001 peer.log peer_pid="$launched_pid" - .venv/bin/python .circleci/scripts/wait_integration_services.py + .venv/bin/python .github/scripts/ci/wait_integration_services.py fi fi @@ -233,14 +233,13 @@ if [ "$suite" = browser ]; then E2E_UI_BASE_URL="$E2E_UI_BASE_URL" E2E_UI_ARTIFACT_DIR="$E2E_UI_ARTIFACT_DIR" \ LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" CI=true \ node tests/e2e/ui/node_modules/@playwright/test/cli.js test --config tests/e2e/ui/integration.config.ts - .venv/bin/python .circleci/scripts/verify_integration_browser.py "$results/browser-results.json" + .venv/bin/python .github/scripts/ci/verify_integration_browser.py "$results/browser-results.json" exit 0 fi shard_arguments=() -if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then - .venv/bin/python tests/integration/run.py "$suite" --list \ - | circleci tests split --split-by=timings --timings-type=filename > "$results/node-files.txt" +if [ -n "${INTEGRATION_SHARD_FILES:-}" ]; then + cp "$INTEGRATION_SHARD_FILES" "$results/node-files.txt" shard_arguments=(--shard-files "$results/node-files.txt") fi @@ -265,7 +264,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ "${license_env[@]}" \ INTEGRATION_SEED="$INTEGRATION_SEED" \ INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \ - INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \ + INTEGRATION_CASE_SHARDS="${INTEGRATION_CASE_SHARDS:-1}" INTEGRATION_CASE_SHARD_INDEX="${INTEGRATION_CASE_SHARD_INDEX:-0}" \ LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \ INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \ INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \ diff --git a/.circleci/scripts/stop_integration_processes.py b/.github/scripts/ci/stop_integration_processes.py similarity index 100% rename from .circleci/scripts/stop_integration_processes.py rename to .github/scripts/ci/stop_integration_processes.py diff --git a/.circleci/scripts/verify_integration_browser.py b/.github/scripts/ci/verify_integration_browser.py similarity index 94% rename from .circleci/scripts/verify_integration_browser.py rename to .github/scripts/ci/verify_integration_browser.py index 4468fadbcde..452a6cd43ae 100644 --- a/.circleci/scripts/verify_integration_browser.py +++ b/.github/scripts/ci/verify_integration_browser.py @@ -31,7 +31,7 @@ def main() -> None: result: Final = json.loads(Path(sys.argv[1]).read_text()) assert not result.get("errors"), result.get("errors") expected: Final = json.loads( - (Path(__file__).resolve().parents[2] / "tests/e2e/ui/tests/integrationCritical/expected.json").read_text() + (Path(__file__).resolve().parents[3] / "tests/e2e/ui/tests/integrationCritical/expected.json").read_text() ) assert expected and result["stats"]["expected"] == len(expected) assert all(result["stats"][name] == 0 for name in ("unexpected", "flaky", "skipped")) @@ -41,7 +41,7 @@ def main() -> None: suites: Final = TypeAdapter(list[BrowserSuite]).validate_python(result["suites"], strict=True) specs: Final = tuple(spec for suite in suites for spec in cases(suite)) - repository: Final = Path(__file__).resolve().parents[2] + repository: Final = Path(__file__).resolve().parents[3] report_root: Final = Path(result["config"]["rootDir"]) assert report_root.is_absolute(), "Playwright rootDir must be explicit" observed: Final = tuple( diff --git a/.circleci/scripts/wait_integration_services.py b/.github/scripts/ci/wait_integration_services.py similarity index 100% rename from .circleci/scripts/wait_integration_services.py rename to .github/scripts/ci/wait_integration_services.py diff --git a/.github/scripts/detect_changes.sh b/.github/scripts/detect_changes.sh index 2d427c92fb5..acc19be57f1 100755 --- a/.github/scripts/detect_changes.sh +++ b/.github/scripts/detect_changes.sh @@ -16,7 +16,7 @@ run_full() { } here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -classify="${here}/../../.circleci/scripts/classify_changes.sh" +classify="${here}/ci/classify_changes.sh" [ -n "${PR_NUMBER:-}" ] || run_full "not a pull_request event" [ -n "${REPO:-}" ] || run_full "no repository in the environment" diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 73159bd64d2..8be7b89e076 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -55,7 +55,6 @@ jobs: services: postgres: image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -81,6 +80,17 @@ jobs: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false + - name: Configure PostgreSQL for integration workers + env: + POSTGRES_CONTAINER: ${{ job.services.postgres.id }} + run: | + docker exec "$POSTGRES_CONTAINER" sh -c 'printf "%s\n" "max_connections = 1000" "shared_preload_libraries = pg_stat_statements" "pg_stat_statements.track = all" "pg_stat_statements.max = 20000" >> "$PGDATA/postgresql.conf"' + docker restart "$POSTGRES_CONTAINER" + for attempt in $(seq 1 30); do + if docker exec "$POSTGRES_CONTAINER" pg_isready -U postgres; then exit 0; fi + sleep 1 + done + exit 1 - name: Set up Python timeout-minutes: 3 @@ -155,8 +165,8 @@ jobs: LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }} run: | case "$SUITE" in - *-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;; - *) bash .circleci/scripts/run_integration.sh "$SUITE" ;; + *-replica) bash .github/scripts/ci/run_integration.sh "${SUITE%-replica}" replica ;; + *) bash .github/scripts/ci/run_integration.sh "$SUITE" ;; esac - name: Verify every case ran exactly once diff --git a/.github/workflows/test-mcp-dependency-resolution.yml b/.github/workflows/test-mcp-dependency-resolution.yml index 20efb351985..90589cf1e6f 100644 --- a/.github/workflows/test-mcp-dependency-resolution.yml +++ b/.github/workflows/test-mcp-dependency-resolution.yml @@ -28,7 +28,7 @@ on: - ".github/actions/cache-cargo-build/**" - ".github/scripts/detect_changes.sh" - ".github/scripts/uv_sync_with_retries.sh" - - ".circleci/scripts/classify_changes.sh" + - ".github/scripts/ci/classify_changes.sh" permissions: contents: read diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index d5fd755ba18..09138bd6c14 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -20,7 +20,7 @@ import json import os import subprocess import sys -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import Final @@ -28,8 +28,13 @@ from typing import Final COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" PINS_PATH: Final = "ci_cd/cost_map_pins.json" +PINS_PATHS: Final = ( + "litellm/scripts/ci_cd/cost_map_pins.json", + "scripts/ci_cd/cost_map_pins.json", + "ci_cd/cost_map_pins.json", +) GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH) -CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH) +CHECKED_PATHS: Final = (*GUARDED_PATHS, *PINS_PATHS) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync" TARGET_ENV: Final = "COST_MAP_GUARD_TARGET" @@ -193,7 +198,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]: - if PINS_PATH not in changed_files: + if not any(path in changed_files for path in PINS_PATHS): return () if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX): return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",) @@ -251,8 +256,11 @@ def _git(*args: str) -> str | None: return result.stdout if result.returncode == 0 else None -def snapshot(revision: str) -> Snapshot: - return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS)) +def snapshot(revision: str, *, read_git: Callable[[str, str], str | None] = _git) -> Snapshot: + return Snapshot( + *(read_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS), + next((text for path in PINS_PATHS if (text := read_git("show", f"{revision}:{path}")) is not None), ""), + ) def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None: diff --git a/codecov.yaml b/codecov.yaml index 521088868bd..419a28ce6c6 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -6,7 +6,7 @@ codecov: ignore: - "litellm-rust/**" -# Uploads are flagged per workflow tier (GHA) or "circleci". carryforward makes +# Uploads are flagged per workflow tier. carryforward makes # a re-upload of a flag replace its prior session instead of accumulating a # conflicting one, and lets a commit reuse a flag from its parent when that flag # was not re-uploaded. Required because the same commit can receive the diff --git a/helm/litellm/ci/postgres.yaml b/helm/litellm/ci/postgres.yaml index aa210e932b0..01472950b3b 100644 --- a/helm/litellm/ci/postgres.yaml +++ b/helm/litellm/ci/postgres.yaml @@ -1,4 +1,4 @@ -# Throwaway PostgreSQL for the CircleCI helm_chart_testing job. Applied with +# Throwaway PostgreSQL for the local chart smoke test. Applied with # kubectl before `helm install`; the chart itself ships no database. apiVersion: v1 kind: Secret diff --git a/helm/litellm/ci/test-values.yaml b/helm/litellm/ci/test-values.yaml index 40c68489ed3..850dd48ee63 100644 --- a/helm/litellm/ci/test-values.yaml +++ b/helm/litellm/ci/test-values.yaml @@ -1,4 +1,4 @@ -# Values for the CircleCI helm_chart_testing job: a monolith install on a +# Values for the local chart smoke test: a monolith install on a # kind cluster from the image built by the pipeline (image.repository / tag / # pullPolicy are passed with --set), backed by the throwaway PostgreSQL in # ci/postgres.yaml so the migrations Job and the proxy have a database. diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 1774442cddc..138e203aba9 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -102,7 +102,7 @@ def reset_vcr_diag_dir() -> None: pass -# CircleCI truncates a step's retrievable output to the last ~400 KB. The +# CI truncates a step's retrievable output to the last ~400 KB. The # diagnostic log is emitted right *before* the final pytest summary line but # *after* the VCR CLASSIFICATION SUMMARY, so an unbounded dump (the body/key # matchers log one block per *episode comparison*, even on an eventual HIT) diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 008a8f80cf0..ccdc616c243 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -30,7 +30,7 @@ deferred. It needs credentials Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family or behavior area. If you add a new folder, you must add a line here describing what kind of tests belong in it, so the layout stays self-describing. `gateway/` is the exception: it holds proxy configuration only and never tests -- `migrations/` - isolated Docker startup, concurrent migration, crash recovery, and legacy database compatibility. The CircleCI migration workflow enables `LITELLM_MIGRATION_TESTS=1`. These tests own their proxy containers and databases, so they do not use the shared proxy preflight or shared database cleanup +- `migrations/` - isolated Docker startup, concurrent migration, crash recovery, and legacy database compatibility. Set `LITELLM_MIGRATION_TESTS=1` to run this suite manually. No CI workflow invokes it. These tests own their proxy containers and databases, so they do not use the shared proxy preflight or shared database cleanup - `llm_translation/` - Azure realtime translation is explicitly skipped pending investigation of intermittent missing transcripts and remains unqualified. OpenAI translation and other Azure realtime cases remain enabled. LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown - `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation diff --git a/tests/e2e/ui/fixtures/mock_llm_server/server.py b/tests/e2e/ui/fixtures/mock_llm_server/server.py index 1e3dcecaff3..c2390abce34 100644 --- a/tests/e2e/ui/fixtures/mock_llm_server/server.py +++ b/tests/e2e/ui/fixtures/mock_llm_server/server.py @@ -130,8 +130,7 @@ async def systemone(request: Request): if __name__ == "__main__": # The port is overridable so two checkouts can run the harness at the same - # time; the default keeps every existing caller (run_e2e.sh, the CircleCI - # job, the e2e chart's sidecar) working untouched. + # time; the default keeps run_e2e.sh and the e2e chart's sidecar working untouched. # # The HOST is deliberately NOT configurable. Binding loopback is what makes # this reachable at 127.0.0.1:8090 from inside the proxy's own pod, which is diff --git a/tests/e2e/ui/integration.config.ts b/tests/e2e/ui/integration.config.ts index 55ccb860512..f40cd1d3b85 100644 --- a/tests/e2e/ui/integration.config.ts +++ b/tests/e2e/ui/integration.config.ts @@ -2,9 +2,6 @@ import { defineConfig, devices } from "@playwright/test"; import * as path from "path"; import { ARTIFACT_DIR, UI_BASE_URL } from "./constants"; -if (process.env.GITHUB_ACTIONS === "true") - throw new Error("Integration contracts are owned by CircleCI"); - export default defineConfig({ testDir: "./tests/integrationCritical", testMatch: "*.spec.ts", diff --git a/tests/e2e/ui/run_e2e.sh b/tests/e2e/ui/run_e2e.sh index 97cbe5fcf59..183b1e3dbc6 100755 --- a/tests/e2e/ui/run_e2e.sh +++ b/tests/e2e/ui/run_e2e.sh @@ -36,7 +36,7 @@ PROXY_LOG="" # Ports, overridable so two checkouts can run this harness at the same time -- # otherwise a second run aborts on "port 4000 is in use" and the only way out is # to stop someone else's stack. Defaults are the historical values, so an unset -# environment behaves exactly as before (CI, the CircleCI job and the docs all +# environment behaves exactly as before (CI and the docs # assume 4000/5432/8090). PROXY_PORT="${PROXY_PORT:-4000}" POSTGRES_PORT="${POSTGRES_PORT:-5432}" diff --git a/tests/e2e/ui/tests/auth/proxyLogoutUrl.spec.ts b/tests/e2e/ui/tests/auth/proxyLogoutUrl.spec.ts index 3a79ce82717..2fd8972b021 100644 --- a/tests/e2e/ui/tests/auth/proxyLogoutUrl.spec.ts +++ b/tests/e2e/ui/tests/auth/proxyLogoutUrl.spec.ts @@ -3,9 +3,9 @@ import { Role, users } from "../../fixtures/users"; import { logInThroughLoginPage } from "../../helpers/userOnboarding"; /** - * Runs as part of the standard e2e suite: both `run_e2e.sh` and the CircleCI - * `e2e_ui_testing` job boot the proxy with PROXY_LOGOUT_URL=https://www.example.com - * and export the same value to this Playwright process. The spec reads it to + * Runs as part of the standard e2e suite. `run_e2e.sh` boots the proxy with + * PROXY_LOGOUT_URL=https://www.example.com and exports the same value to + * this Playwright process. The spec reads it to * know where the browser is expected to land. * * The skip guard below is a safety net for environments that launch the proxy diff --git a/tests/e2e/ui/tests/proxy-admin/license.spec.ts b/tests/e2e/ui/tests/proxy-admin/license.spec.ts index 37a0e324f27..cfc098b56e4 100644 --- a/tests/e2e/ui/tests/proxy-admin/license.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/license.spec.ts @@ -4,11 +4,11 @@ import { ADMIN_STORAGE_PATH } from "../../constants"; /** * Sanity check that LITELLM_LICENSE is being forwarded to the proxy when set - * in the environment (e.g. CircleCI's `e2e_ui_testing` job). The login JWT's + * in the environment (e.g. `run_e2e.sh`). The login JWT's * `premium_user` claim is the same value the dashboard reads to enable * premium-gated UI surfaces (Team-BYOK switch, etc.), so asserting it here * catches any future regression where the env var stops being plumbed - * through `run_e2e.sh` / `.circleci/config.yml`. + * through `run_e2e.sh`. * * Skips locally when no license is configured. */ diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index e5068d24e88..b430ab553a5 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -26,20 +26,20 @@ directory that a `GROUPS` entry in `run.py` selects. There is no manifest and no tests. A product bug the test exposes is `pytest.skip("BUG: ")` at the top of the body, not a fix in the test and not a deletion. Needs no network, proxy, DB or Redis: `tests/unit` -CircleCI's integration groups run every file here except the OCI and sandbox tests, which no CI runs, and +GitHub Actions integration groups run every file here except the OCI and sandbox tests, which no CI runs, and the files in `GITHUB_FILES` in `run.py`. `test-integration.yml` runs the same groups through the same `run_integration.sh` on GitHub Actions, on main every three hours from 8am to 8pm Pacific Monday to Saturday, -and on a pull request when someone adds the `run-ci` label. Neither runs every pull request, so a new file here +and on a pull request when someone adds the `run-ci` label. These jobs do not run every pull request, so a new file here first runs on main after its PR merges, or earlier if the PR gets `run-ci`. Wanting a file to run before merge is no reason to change CI, and a new dedicated workflow for it is the same mistake as a shared shard. Write the file, leave `GITHUB_FILES` and every workflow alone, and tell the user it first runs on main. `GITHUB_FILES` is only for a file that -needs what CircleCI's groups cannot give it, like the images `image-scan.yml` builds or the plain pytest -run CircleCI's `roi-database` job gives `test_roi_observed.py`. Add a file there only in that case, +needs what GitHub Actions groups cannot give it, like the images `image-scan.yml` builds. `test_roi_observed.py` previously needed a dedicated plain pytest +job and now has no CI owner. It is listed explicitly in the coverage allowlist. Add a file to `GITHUB_FILES` only when needed, together with its dedicated job, and list a new dedicated GitHub workflow with the directory it owns in `DEDICATED_INTEGRATION_WORKFLOWS` in `.github/scripts/assert_ci_coverage.py`. The `ci-coverage` check -fails a file here that a GitHub workflow runs unless it is in `GITHUB_FILES` and that workflow owns its -directory (#46780) +fails a file here that another workflow selects unless that workflow owns its path in +`DEDICATED_INTEGRATION_WORKFLOWS` (#46780). ## Fixtures that boot a proxy @@ -50,4 +50,4 @@ on one xdist worker, so the proxy they share boots there once and stops when tha file. A test that requests one more of those fixtures than its neighbors gets its own group and can boot the proxy again on another worker, so keep that set the same across the tests that share a proxy. An unpinned session fixture boots a copy on every worker that runs one of its tests and keeps them all until -the node ends, and those copies ran CircleCI integration-extensions nodes out of memory +the node ends, and those copies ran GitHub Actions integration-extensions nodes out of memory diff --git a/tests/integration/README.md b/tests/integration/README.md index a1d224749b3..1f9399c8b22 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,24 +1,24 @@ # Integration contracts -These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite, and `.github/workflows/test-integration.yml` runs the same groups through `run_integration.sh` on GitHub Actions for `run-ci` pull requests and scheduled main runs. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls +These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. `.github/workflows/test-integration.yml` runs the groups through `.github/scripts/ci/run_integration.sh` for `run-ci` pull requests and scheduled main runs. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls -The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport +The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh`. They own temporary databases and script only the external provider transport. No remaining CI job invokes them. They are listed explicitly in `.github/ci-coverage-allowlist.yml`. -The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so CircleCI's `sdk` group installs `redis-server` before it runs them +The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so the GitHub Actions `sdk` group installs `redis-server` before it runs them The `cost` group is driven by the `CostTrackingTestCase` literals under `cost_calculation/cases/.py` (gathered by `cost_calculation/catalog.py`), with the scripted cost map in `cost_calculation/cost_map.py`. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL` -Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist. Every group run by more than one worker gives each worker a private database, Redis, scripted upstream process, fake provider and proxy, with a peer and read-only database role where required. There is no opt-in list and no shared proxy, upstream, database or Redis in that mode: `run_integration.sh` does not start them and does not hand their addresses to the run, so a test can only reach its own worker's services. The pytest controller owns these services and closes them after worker failures. Their cases use pytest-xdist load groups inside the files selected by CircleCI timing splits. Module and class fixtures are grouped automatically, so authors do not need scheduling markers. Function fixtures remain independently schedulable. Tests that own an independent mutable sink for every case should use function fixtures, so a shared module fixture does not serialize them or share pending observations. Nested harness probes discard the parent run's shard, worker and result settings, then supply their own selections. MCP keeps its explicit load groups. Automatic group suffixes are removed from JUnit and execution records to preserve case identities. Owned proxy helpers wait for a healthy response from every configured, live worker before yielding. Serial runs keep the supplied deployment, including routing parity runs. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate +Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist. Every group run by more than one worker gives each worker a private database, Redis, scripted upstream process, fake provider and proxy, with a peer and read-only database role where required. There is no opt-in list and no shared proxy, upstream, database or Redis in that mode: `run_integration.sh` does not start them and does not hand their addresses to the run, so a test can only reach its own worker's services. The pytest controller owns these services and closes them after worker failures. Their cases use pytest-xdist load groups within each group's selected files. Module and class fixtures are grouped automatically, so authors do not need scheduling markers. Function fixtures remain independently schedulable. Tests that own an independent mutable sink for every case should use function fixtures, so a shared module fixture does not serialize them or share pending observations. Nested harness probes discard the parent run's shard, worker and result settings, then supply their own selections. MCP keeps its explicit load groups. Automatic group suffixes are removed from JUnit and execution records to preserve case identities. Owned proxy helpers wait for a healthy response from every configured, live worker before yielding. Serial runs keep the supplied deployment, including routing parity runs. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate Management, MCP and accounting also require `INTEGRATION_PEER_URL`, `REDIS_HOST` and `REDIS_PORT`. Under several workers each worker gets a second directly addressed proxy sharing only that worker's stores, and a single-worker run uses the two proxies `run_integration.sh` starts. The test-only CLI wrapper supplies enterprise route entitlement, following the existing behavior suite's convention. It does not qualify license validation; run it with one worker and no reload -The generated lifecycle models use 20 examples, eight steps, generation and shrinking, with isolated resources per example. HTTP operation caps include generation and shrinking and exempt cleanup. Local qualification defaults to seed 4106601 and canonical order; CircleCI derives exploration and ordering seeds from the checked-out revision and workflow ID. The ordering seed shuffles files, scoped fixture configurations and cases within each configuration. Ordering keeps shared module and class configurations together where their dependencies permit it. Dependent fixtures with their own module parameters can still rebuild an outer configuration, with every case retained +The generated lifecycle models use 20 examples, eight steps, generation and shrinking, with isolated resources per example. HTTP operation caps include generation and shrinking and exempt cleanup. Local qualification defaults to seed 4106601 and canonical order; GitHub Actions derives exploration and ordering seeds from the checked-out revision and workflow ID. The ordering seed shuffles files, scoped fixture configurations and cases within each configuration. Ordering keeps shared module and class configurations together where their dependencies permit it. Dependent fixtures with their own module parameters can still rebuild an outer configuration, with every case retained -CircleCI uses its built-in `circleci tests split --split-by=timings --timings-type=filename` to assign test files using the uploaded JUnit timings. Each file stays on one node, preserving its module and parametrized fixtures. Jobs default to one node. Parallelism should use the smallest node count that keeps the complete job, including setup and cleanup, within twenty minutes. A group that already finishes within that target on one node stays on one node. Management, accounting, providers and management replica currently use four nodes. Extensions uses eight nodes; database and MCP use two, and fast suites use one. No job is configured above eight nodes. A single slow file can still determine a suite's runtime; split such a file along independent fixture boundaries rather than adding more nodes +GitHub Actions runs one job per suite and uses pytest-xdist workers inside that job. `INTEGRATION_WORKERS` controls the worker count, with defaults in `test-integration.yml`. Each worker owns its services. A slow file can still determine a suite's runtime, so split it along independent fixture boundaries rather than adding more workers -Every node collects the same candidate files before applying CircleCI's assigned file list. Its execution record retains the candidate inventory and the selected, passed and skipped cases. An empty assigned file list produces an empty shard, not a full-suite fallback. Every parallel group has a dependent census job that rejects an empty candidate inventory, missing shards, omitted cases and duplicate execution. Standard and replica runs keep separate workspace records +Every group records its candidate inventory and the selected, passed and skipped cases. A census step rejects an empty candidate inventory, omitted cases and duplicate execution. Standard and replica runs keep separate records -Future change-based selection belongs before timing splitting. Pass the selected group files or node IDs to `run.py --list` to list their files, then give those filenames to CircleCI. Run the same candidate selection on every node with `--shard-files --shards --shard-index `. The census compares execution with the candidate inventory, while full-suite runs remain the default. CircleCI determines scheduling from timing history; timing data never selects or omits candidates. Use `--seed` and `--order-seed` to reproduce test generation and order. The installed Hypothesis version, settings, seeds and collected order are recorded beside the execution manifest +Future change-based selection belongs before worker scheduling. Pass the selected group files or node IDs to `run.py --list` to list their files. An external shard assignment can use `INTEGRATION_SHARD_FILES` with `INTEGRATION_CASE_SHARDS` and `INTEGRATION_CASE_SHARD_INDEX`. An empty assigned file list produces an empty shard, not a full-suite fallback. The current GitHub Actions jobs use one shard with index zero and use pytest-xdist within it. Use `--seed` and `--order-seed` to reproduce test generation and order. The installed Hypothesis version, settings, seeds and collected order are recorded beside the execution manifest A future selector must explicitly skip a group when no candidates apply; omitting positional candidates means the full group. Keep full runs on the default branch and scheduled workflows, and select the full affected groups when shared fixtures, CI configuration or an unknown dependency changes @@ -30,9 +30,9 @@ Owned proxy workers configure a 32-thread executor before application startup, p Every owned listener's default port comes from `allocate_port()` in `_support/ports.py`. That covers owned proxies, scripted upstreams, fake providers, OTLP sinks, database relays and Redis, including peers such as `wire_server` that bind and keep their own sockets. Keeping those listeners out of the ephemeral range prevents them from taking a port Prisma selected with a bind-and-close probe before its query engine starts. Each pytest process claims a block of 200 ports between 10000 and 32767 by holding an exclusive lock file per block in the system temp directory, so two runs on one machine, such as parallel worktrees, never share a block, and the OS drops the lock when the process exits. The range sits above the fixed service ports and below the Linux and macOS ephemeral ranges, so a port-0 bind or an outbound connection never lands in a block. Ports inside a block go out in order and are never handed out twice in a run, a port that something outside the harness already holds is skipped, and a process that uses up its block claims the next free one. A test that restarts a server on the same port allocates that port up front instead of reading it back from the first server. TQ012 in `scripts/check_test_quality.py` rejects reading a port back from a port-0 bind anywhere under `tests/integration` -The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. `GITHUB_FILES` in `run.py` lists the files excluded from the `run_integration.sh` selection +The GitHub Actions workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. `GITHUB_FILES` in `run.py` lists the files excluded from the `run_integration.sh` selection -There is no hand-maintained list of test cases. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration//test_*.py` file in a scheduled group as owned by CircleCI +There is no hand-maintained list of test cases. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration//test_*.py` file in a scheduled group as owned by GitHub Actions Provider sentinels currently use the controlled server, not live recordings. The provider shard also runs the existing strict replay controls for changed requests, exhausted interactions, leftover interactions and no provider connection. Future recorded scenarios must use that replay-only implementation; missing recordings cannot fall back to a real provider. `GET /__observations` returns every request the upstream recorded since the last `DELETE /__observations`, and reading it never clears it. An autouse fixture resets it before each test, so a test that needs a clean slate between phases sends the DELETE itself. A check that a request never reached the upstream asserts on the same read that shows a request sent after it did reach the upstream, so the absence cannot pass on an empty or reset log @@ -46,20 +46,20 @@ Accounting cases compare persisted input and output cost components against lite Provider contracts exercise actual TCP requests with synthetic credentials and local protocol peers. The S3 verifier uses independently implemented equations, a published known-answer vector, a fixed signing clock and deliberately invalid signed requests. Bedrock cases clear ambient AWS credential sources and check the literal model path, loaded role references, STS requests and bearer-only behavior -CircleCI runs providers on four isolated nodes using its file timing splitter. Each node owns its services. The dependent providers coverage job verifies all four execution records against the candidate case inventory and rejects missing, duplicate, failed or incomplete execution. The node's assigned filenames are saved in `node-files.txt` beside its execution record +GitHub Actions runs the providers group with 32 pytest workers, each with its own services. The census step verifies the execution record against the candidate inventory and rejects missing, duplicate, failed or incomplete execution -Streaming checks send real HTTP transfer chunks, including one-byte partitions, fragmented tools, incomplete transfers and a cancellation barrier. They assert meaningful text, tool arguments, final usage and persisted cost. The Redis recovery case owns a separate database and Redis process, uses the supported one-second circuit-breaker recovery setting, waits for the real subscriber and verifies response data in Redis after restart. CircleCI reuses its existing Redis image for that extra process; it never pulls an image during tests +Streaming checks send real HTTP transfer chunks, including one-byte partitions, fragmented tools, incomplete transfers and a cancellation barrier. They assert meaningful text, tool arguments, final usage and persisted cost. The Redis recovery case owns a separate database and Redis process, uses the supported one-second circuit-breaker recovery setting, waits for the real subscriber and verifies response data in Redis after restart. GitHub Actions reuses its existing Redis image for that extra process; it never pulls an image during tests The `messages_endpoint/` directory holds `/v1/messages` endpoint contracts: native-provider backends under `providers/` (`anthropic`, `bedrock`, `gemini`) and the translation bridges (`responses_bridge`, `chat_bridge`) at the top level. It runs in the providers shard; `run.py` selects test files recursively under each scheduled directory -The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy or database. CircleCI starts a local Redis for this shard like the others, so SDK-side caching cases that need a real Redis server belong here too; a case that reaches the gateway belongs in one of the other shards +The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy or database. GitHub Actions starts a local Redis for this shard like the others, so SDK-side caching cases that need a real Redis server belong here too; a case that reaches the gateway belongs in one of the other shards -The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions. CircleCI runs it on eight xlarge nodes using its file timing splitter. Each node runs its assigned files with four pytest workers, each owning its database, Redis, upstream and proxy. The dependent coverage job verifies all eight execution records against the candidate case inventory. Tests must not assume a particular set of sibling files +The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions. GitHub Actions runs it with 48 pytest workers, each owning its database, Redis, upstream and proxy. The census step verifies execution against the candidate inventory. Tests must not assume a particular set of sibling files The security shard uses four pytest workers on an xlarge machine, each with its own isolated services like every other multi-worker group. Route sweeps use a new HTTP client for every call and share only the TLS verification context, so authentication and response cookies cannot cross callers. The canary rig supplies owned OpenAI and Anthropic endpoints and synthetic credentials and rejects attempted external egress The two mcp nodes own separate services and use the same case census as providers and extensions. Their coverage data is combined into one MCP report after both nodes finish. Each node runs the MCP gateway against SDK peers owned by each test (`_support/mcp.py`): streamable HTTP, SSE and stdio peers, an OpenAPI-spec app, and an OAuth 2.1 authorization-server double. Every peer records the requests it receives so a test can assert what reached the peer, not only what the proxy answered. The shard runs with `INTEGRATION_WORKERS` set and with `INTEGRATION_COVERAGE=1`, which starts every worker's proxy and peer under `coverage run --parallel-mode` limited to the MCP modules and stores `coverage.txt` plus an HTML report with the job artifacts. `mcp_coverage.toml` measures lines through `sys.monitoring`, because the CPython 3.12 trace function that branch measurement needs made a covered proxy take about four times as long to become ready. A test that fails because the product is wrong is skipped with `pytest.skip("BUG: ")` so the skip list in `execution.json` is the open MCP bug list -Browser contracts live in `tests/e2e/ui/tests/integrationCritical` and run only through `tests/e2e/ui/integration.config.ts`. The expected browser results are listed in `expected.json` in that directory and checked by `.circleci/scripts/verify_integration_browser.py`. The CircleCI browser shard builds the checked-out dashboard, starts the owned proxy with that build, and verifies one exact browser result without retries or skips. The default Playwright selection excludes this directory. The focused project flow asserts the submitted create and clear values, fresh SQL state and actual blocked/restored serving while preserving model restrictions +Browser contracts live in `tests/e2e/ui/tests/integrationCritical` and run only through `tests/e2e/ui/integration.config.ts`. The expected browser results are listed in `expected.json` in that directory and checked by `.github/scripts/ci/verify_integration_browser.py`. The GitHub Actions browser shard builds the checked-out dashboard, starts the owned proxy with that build, and verifies one exact browser result without retries or skips. The default Playwright selection excludes this directory. The focused project flow asserts the submitted create and clear values, fresh SQL state and actual blocked/restored serving while preserving model restrictions -Two always-on `-replica` CircleCI jobs (management, database) run their groups in replica mode, where every proxy connects through a real `litellm_writer` role and a real read-only `litellm_reader` role against the same PostgreSQL. Nothing is captured there: the job passes when the tests pass, and a write routed to the read-only reader fails the test that issued it. A deeper check runs on demand as the `routing_parity` workflow, triggered through the CircleCI API v2 pipeline endpoint on the PR branch with `{"parameters": {"routing_parity_base": "<40-hex merge-base sha>"}}`. The workflow fans out over the seven groups, and each `routing-parity-` job runs its own group twice against the same test harness, once with `litellm/`, `enterprise/`, and `litellm-proxy-extras/` checked out from the base revision and once from the head, with a pytest plugin snapshotting `pg_stat_statements` into `routing-observed.json` per side. The `check` step then compares the two observations and writes `routing-diff.txt`: a statement seen on both sides fails when its role set changed, globally or for the same test (per-test capture is skipped under xdist), unless it is listed in `tests/integration/routing/either_role.json`, where each entry names the statement and a one-line reason it legitimately runs on whichever role asks for it, printed under `== either role ==`. Queries seen on only one side are listed, never failed, `pg_stat_statements` evictions and a role that never ran a statement are failures +Two `-replica` GitHub Actions suites (management, database) run their groups in replica mode, where every proxy connects through a real `litellm_writer` role and a real read-only `litellm_reader` role against the same PostgreSQL. Nothing is captured there: the job passes when the tests pass, and a write routed to the read-only reader fails the test that issued it. Routing parity support remains in the local harness, but no CI workflow runs it. diff --git a/tests/integration/_support/redis_process.py b/tests/integration/_support/redis_process.py index 1a325d73659..494d13c750e 100644 --- a/tests/integration/_support/redis_process.py +++ b/tests/integration/_support/redis_process.py @@ -152,7 +152,7 @@ def owned_redis( ], text=True, ).strip() - assert host, "CircleCI owned Redis container has no address" + assert host, "CI-owned Redis container has no address" prefix: Final = ("docker", "exec", "redis-cache", "redis-server") output: Final = Path(inherited.get("INTEGRATION_RESULTS_DIR", str(directory))) output.mkdir(parents=True, exist_ok=True) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d605bd3d7d7..8c0c463c217 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -28,7 +28,6 @@ from tests.integration._support.provider import SharedProvider, shared_provider from tests.integration._support.routing import RoutingPlugin from tests.integration._support.sdk_models import build_sdk_models from tests.integration._support.workers import gateway_profile, start_worker_stacks, worker_stack -from tests.integration.run import GITHUB_FILES if TYPE_CHECKING: from tests.integration._support.actors import ActorSet, GatewayClient, ManagementDatabase, ScratchResources @@ -216,11 +215,6 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item for item in items if item.path.is_relative_to(root) and item.path.relative_to(root).parts[0] in OWNED_DIRECTORIES ) - circleci_only: Final = tuple( - item for item in owned if item.path.relative_to(root.parents[1]).as_posix() not in GITHUB_FILES - ) - if circleci_only and os.environ.get("GITHUB_ACTIONS") == "true": - raise pytest.UsageError("Integration contracts are owned by CircleCI") for item in owned: item.add_marker(pytest.mark.integration) config.stash[INVENTORY] = tuple(_case_identity(item.nodeid) for item in owned) @@ -230,7 +224,7 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item raise pytest.UsageError("Integration shard index must be within the positive shard count") files: Final = config.stash[SHARD_FILES] if count > 1 and files is None: - raise pytest.UsageError("Parallel integration runs require CircleCI's shard file selection") + raise pytest.UsageError("Parallel integration runs require a shard file selection") if files is not None: available: Final = frozenset(item.nodeid.split("::", 1)[0] for item in owned) for file in sorted(files - available - config.stash.get(FAILED_COLLECTIONS, frozenset())): diff --git a/tests/integration/database/test_harness_lifecycle.py b/tests/integration/database/test_harness_lifecycle.py index 4cb01dfeef6..724ba5412d2 100644 --- a/tests/integration/database/test_harness_lifecycle.py +++ b/tests/integration/database/test_harness_lifecycle.py @@ -79,8 +79,7 @@ def _nested_pytest_environment(environment: Mapping[str, str]) -> dict[str, str] "INTEGRATION_COVERAGE_FILE", "INTEGRATION_CASE_SHARDS", "INTEGRATION_CASE_SHARD_INDEX", - "CIRCLE_NODE_TOTAL", - "CIRCLE_NODE_INDEX", + "INTEGRATION_SHARD_FILES", "PYTEST_XDIST_WORKER", "PYTEST_XDIST_WORKER_COUNT", "PYTEST_XDIST_TESTRUNUID", @@ -750,8 +749,6 @@ def test_direct_and_dependent_consumers_reuse_the_same_module_parameter(tmp_path **os.environ, "INTEGRATION_CASE_SHARDS": "2", "INTEGRATION_CASE_SHARD_INDEX": "1", - "CIRCLE_NODE_TOTAL": "2", - "CIRCLE_NODE_INDEX": "1", } ), capture_output=True, diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index 87c56f1a42b..4127432bd3d 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -26,7 +26,25 @@ sys.modules[_spec.name] = coverage # @dataclass(slots=True) rebuilds via sys.mo _spec.loader.exec_module(coverage) -def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: Path) -> None: +def _write_group_workflow(root: Path, suites: tuple[str, ...] = ("management",)) -> Path: + workflow: Final = root / ".github/workflows/test-integration.yml" + workflow.parent.mkdir(parents=True, exist_ok=True) + workflow.write_text( + yaml.safe_dump( + { + "jobs": { + "integration": { + "strategy": {"matrix": {"suite": list(suites)}}, + "steps": [{"run": "bash .github/scripts/ci/run_integration.sh $SUITE"}], + } + } + } + ) + ) + return workflow + + +def test_integration_groups_require_exclusive_scheduled_github_owner(tmp_path: Path) -> None: test_path: Final = "tests/integration/management/test_contract.py" test_file: Final = tmp_path / test_path test_file.parent.mkdir(parents=True) @@ -37,39 +55,18 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: ) paths, findings = coverage._integration_ownership(tmp_path) assert not paths - assert [finding.detail for finding in findings] == ["dedicated CircleCI runner is missing"] - circle: Final = tmp_path / ".circleci/config.yml" - circle.parent.mkdir() - circle.write_text( - yaml.safe_dump( - { - "jobs": { - "integration_contracts": { - "steps": [{"run": {"command": "bash .circleci/scripts/run_integration.sh management"}}] - } - }, - "workflows": {"integration": {"jobs": [{"integration_contracts": {"suite": "management"}}]}}, - } - ) - ) + assert [finding.detail for finding in findings] == ["dedicated GitHub Actions runner is missing"] + _write_group_workflow(tmp_path) paths, findings = coverage._integration_ownership(tmp_path) assert paths == frozenset({test_path}) assert findings == () - configured: Final = yaml.safe_load(circle.read_text()) - configured["workflows"]["integration"]["jobs"] = [ - {"integration_contracts": {"matrix": {"parameters": {"suite": ["providers"]}}}} - ] - circle.write_text(yaml.safe_dump(configured)) + _write_group_workflow(tmp_path, ("providers",)) _, findings = coverage._integration_ownership(tmp_path) assert [(finding.subject, finding.detail) for finding in findings] == [ - ("management", "canonical integration group is not scheduled by CircleCI") + ("management", "canonical integration group is not scheduled by GitHub Actions") ] - configured["workflows"]["integration"]["jobs"][0]["integration_contracts"]["matrix"]["parameters"]["suite"] = [ - "management" - ] - circle.write_text(yaml.safe_dump(configured)) + _write_group_workflow(tmp_path) workflow: Final = tmp_path / ".github/workflows/test.yml" - workflow.parent.mkdir(parents=True) workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": "pytest tests/integration"}]}}})) _, findings = coverage._integration_ownership(tmp_path) assert [(finding.subject, finding.detail) for finding in findings] == [ @@ -98,12 +95,9 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: assert [(finding.subject, finding.detail) for finding in missing_invocation] == [ (github_path, "GitHub-owned integration contract has no invoking job") ] - circle_config: Final = yaml.safe_load(circle.read_text()) - circle_config["jobs"]["postgres_suite"] = {"parameters": {"test_path": {"type": "string"}}} - circle_config["workflows"]["integration"]["jobs"].append({"postgres_suite": {"test_path": github_path}}) - circle.write_text(yaml.safe_dump(circle_config)) - _, circle_invocation = coverage._integration_ownership(tmp_path) - assert circle_invocation == () + dedicated.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}})) + _, restored_invocation = coverage._integration_ownership(tmp_path) + assert restored_invocation == () def _write_integration_tree(root: Path, github_path: str) -> None: @@ -117,20 +111,24 @@ def _write_integration_tree(root: Path, github_path: str) -> None: 'GROUPS: Final = MappingProxyType({"management": ("management",)})\n' f"GITHUB_FILES: Final = frozenset({{{github_path!r}}})\n" ) - (root / ".circleci").mkdir() - (root / ".circleci/config.yml").write_text( - yaml.safe_dump( - { - "jobs": { - "integration_contracts": { - "steps": [{"run": {"command": "bash .circleci/scripts/run_integration.sh management"}}] - } - }, - "workflows": {"integration": {"jobs": [{"integration_contracts": {"suite": "management"}}]}}, - } - ) - ) - (root / ".github/workflows").mkdir(parents=True) + _write_group_workflow(root) + + +@pytest.mark.parametrize( + ("matrix", "expected"), + ( + (["management", "database"], ("management", "database")), + ([], ()), + (None, ()), + ("${{ inputs.suite }}", ()), + ( + "${{ fromJSON(inputs.suite == 'all' && '[\"management\",\"database\"]' || format('[\"{0}\"]', inputs.suite)) }}", + ("management", "database"), + ), + ), +) +def test_integration_matrix_accounts_for_default_suites(matrix: object, expected: tuple[str, ...]) -> None: + assert coverage._matrix_suites(matrix) == expected @pytest.mark.parametrize( @@ -172,20 +170,18 @@ def test_an_integration_file_outside_every_group_cannot_run_on_a_github_shard(tm assert not [finding for finding in before if finding.subject == ungrouped] -def test_circleci_postgres_suites_count_their_test_path_as_invoked() -> None: +def test_workflow_matrices_count_their_test_paths_as_invoked() -> None: config: Final = { - "workflows": { - "integration": { - "jobs": [ - {"postgres_suite": {"name": "proxy-behavior", "test_path": "tests/proxy_behavior", "seed": True}}, - { - "postgres_suite": { - "name": "roi-database", - "test_path": "tests/integration/database/test_roi_observed.py", - "seed": False, - } - }, - ] + "jobs": { + "postgres": { + "strategy": { + "matrix": { + "include": [ + {"test-path": "tests/proxy_behavior"}, + {"test-path": "tests/integration/database/test_roi_observed.py"}, + ] + } + } } } } @@ -194,15 +190,27 @@ def test_circleci_postgres_suites_count_their_test_path_as_invoked() -> None: ) -def test_every_circleci_postgres_suite_is_credited_on_the_repo_as_it_stands() -> None: - circle: Final = yaml.safe_load(coverage.CIRCLECI_CONFIG.read_text()) - paths: Final = tuple( - job["postgres_suite"]["test_path"] - for job in circle["workflows"]["integration"]["jobs"] - if isinstance(job, dict) and "postgres_suite" in job +def test_allowlisting_an_unrun_integration_contract_does_not_claim_it_runs(tmp_path: Path) -> None: + path: Final = "tests/integration/database/test_roi_observed.py" + _write_integration_tree(tmp_path, path) + paths, findings = coverage._integration_ownership(tmp_path) + assert path not in paths + assert [(finding.subject, finding.detail) for finding in findings] == [ + (path, "GitHub-owned integration contract has no invoking job") + ] + allowlist: Final = coverage.Allowlist( + test_paths=(coverage.AllowEntry(paths=(path,), reason="ran only in CircleCI, removed"),), dockerfiles=() ) - assert paths == ("tests/integration/database/test_roi_observed.py",) - assert set(paths) <= coverage._invoked_test_tokens(coverage._all_scalars()) + allowed_paths, allowed_findings = coverage._integration_ownership(tmp_path, allowlist=allowlist) + assert path not in allowed_paths + assert allowed_findings == () + (tmp_path / ".github/workflows/test-unit.yml").write_text( + yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {path}"}]}}}) + ) + _, wrong_owner = coverage._integration_ownership(tmp_path, allowlist=allowlist) + assert [(finding.subject, finding.detail) for finding in wrong_owner] == [ + (path, "GitHub-owned integration contract runs outside a dedicated integration workflow") + ] def test_an_ancestor_directory_covers_a_file_but_does_not_name_it(): @@ -419,7 +427,7 @@ def test_an_excluded_term_matching_only_an_inner_name_leaves_the_file_claimed(): assert slice_.claims("tests/x/test_router.py", frozenset({"test_cache_key"})) is True -def test_character_class_globs_match_the_letter_shards_circleci_uses(): +def test_character_class_globs_match_letter_shards(): # tests/local_testing is split by first letter; without character-class support every # file in it looks unglobbed, and the slice guard would report the whole directory. glob = "tests/local_testing/**/test_[a-mA-M]*.py" @@ -509,12 +517,15 @@ def test_a_workflow_that_names_a_file_clears_it_from_the_slice_check(): assert any(coverage._token_covers(token, "tests/unit/proxy/test_proxy_server_custom_logger.py") for token in named) -def test_the_slice_check_credits_only_workflows_never_the_circleci_config(): - named = coverage._workflow_named_tokens() - circleci_only = "tests/proxy_admin_ui_tests" - assert not any(coverage._token_covers(token, f"{circleci_only}/test_key_management.py") for token in named), ( - "a tree only CircleCI globs must not be credited to a workflow" - ) +def test_the_slice_check_ignores_configs_outside_the_workflow_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workflows: Final = tmp_path / "workflows" + workflows.mkdir() + (workflows / "test.yml").write_text("jobs:\n test:\n steps:\n - run: pytest tests/selected\n") + (tmp_path / "ignored.yml").write_text("jobs:\n test:\n steps:\n - run: pytest tests/unrun\n") + monkeypatch.setattr(coverage, "WORKFLOW_DIR", workflows) + assert coverage._workflow_named_tokens() == frozenset({"tests/selected"}) @pytest.mark.parametrize("selector", ("test_selected.py", "test_selected.py::test_redis_auth")) @@ -527,7 +538,6 @@ def test_a_workflow_does_not_credit_a_file_it_never_names( f"jobs:\n test:\n steps:\n - run: uv run pytest tests/local_testing/{selector}\n" ) monkeypatch.setattr(coverage, "WORKFLOW_DIR", workflows) - monkeypatch.setattr(coverage, "CIRCLECI_CONFIG", tmp_path / "circleci.yml") named: Final = coverage._workflow_named_tokens() assert named == frozenset({"tests/local_testing/test_selected.py"}) @@ -543,16 +553,14 @@ def test_slice_audit_rechecks_changed_and_new_files_between_calls( workflows.mkdir(parents=True) tests: Final = tmp_path / "tests/local_testing" tests.mkdir(parents=True) - circle: Final = tmp_path / ".circleci/config.yml" - circle.parent.mkdir() - circle.write_text( + (workflows / "test.yml").write_text( yaml.safe_dump( { "jobs": { selector: { "steps": [ { - "run": f'circleci tests glob "tests/local_testing/test_*.py" | xargs pytest -k "{selector}"' + "run": f'pytest tests/local_testing/test_*.py -k "{selector}"' } ] } @@ -564,7 +572,6 @@ def test_slice_audit_rechecks_changed_and_new_files_between_calls( monkeypatch.setattr(coverage, "REPO_ROOT", tmp_path) monkeypatch.setattr(coverage, "TESTS_ROOT", tmp_path / "tests") monkeypatch.setattr(coverage, "WORKFLOW_DIR", workflows) - monkeypatch.setattr(coverage, "CIRCLECI_CONFIG", circle) allowlist: Final = coverage.Allowlist((), ()) existing: Final = tests / "test_existing.py" existing.write_text("def test_selected(): pass\n") diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_ci_path_filter.py similarity index 90% rename from tests/unit/test_circleci_path_filter.py rename to tests/unit/test_ci_path_filter.py index 9f695960517..1e110b62434 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_ci_path_filter.py @@ -1,6 +1,6 @@ -"""Regression tests for CircleCI change-based job gating. +"""Regression tests for change-based job gating. -`.circleci/scripts/classify_changes.sh` is the pure decision function behind +`.github/scripts/ci/classify_changes.sh` is the pure decision function behind `path_filter.sh`: given the list of files a PR changed (on stdin) and a job category, it prints `run` or `skip`. The gating contract we lock in here: @@ -24,7 +24,7 @@ from pathlib import Path import pytest -SCRIPTS_DIR = Path(__file__).resolve().parents[2] / ".circleci" / "scripts" +SCRIPTS_DIR = Path(__file__).resolve().parents[2] / ".github" / "scripts" / "ci" SCRIPT = SCRIPTS_DIR / "classify_changes.sh" PATH_FILTER = SCRIPTS_DIR / "path_filter.sh" @@ -56,7 +56,7 @@ CI = [".github/workflows/test-unit.yml"] ("mcp-dependencies", ["litellm-proxy-extras/pyproject.toml"], "run"), ("mcp-dependencies", ["scripts/check_mcp_sdk_install.py"], "run"), ("mcp-dependencies", [".github/workflows/test-mcp-dependency-resolution.yml"], "run"), - ("mcp-dependencies", [".circleci/scripts/classify_changes.sh"], "run"), + ("mcp-dependencies", [".github/scripts/ci/classify_changes.sh"], "run"), ("mcp-dependencies", ["litellm/llms/openai/chat/gpt_transformation.py"], "skip"), ("mcp-dependencies", DOCS + CLIENT, "skip"), ("windows-release", ["litellm-rust/crates/inference/src/lib.rs"], "run"), @@ -65,7 +65,7 @@ CI = [".github/workflows/test-unit.yml"] ("windows-release", ["pyproject.toml"], "run"), ("windows-release", ["uv.lock"], "run"), ("windows-release", ["tests/windows_tests/check_windows_wheel_install.py"], "run"), - ("windows-release", [".circleci/config.yml"], "run"), + ("windows-release", [".github/scripts/ci/classify_changes.sh"], "run"), ("windows-release", ["litellm/main.py"], "skip"), ("windows-release", ["tests/unit/test_utils.py"], "skip"), ("windows-release", ["ui/litellm-dashboard/src/App.tsx"], "skip"), @@ -126,7 +126,7 @@ def test_classify_decisions(category: str, changed: list[str], expected: str) -> (".github/workflows/test-unit.yml", "run"), ("tests/integration/run.py", "run"), ("uv.lock", "run"), - (".circleci/config.yml", "skip"), + (".github/workflows/test-integration.yml", "skip"), ("litellm/router.py", "skip"), ("docs/my-website/redis.md", "skip"), ), @@ -158,7 +158,7 @@ def test_markdown_under_dot_github_is_still_docs() -> None: def test_ui_and_client_diverge_on_a_backend_only_change() -> None: - """`client` gates CircleCI's dashboard end-to-end jobs, which drive a real + """`client` gates the dashboard end-to-end jobs, which drive a real proxy and so must run on backend changes. `ui` gates the dashboard build and its unit tests, which cannot see the backend at all.""" assert classify("client", BACKEND) == "run" @@ -202,16 +202,9 @@ def _pr_repo(tmp_path: Path, feature_files: dict[str, str]) -> Path: def _run_path_filter(work: Path, tmp_path: Path, category: str, scripts_dir: Path, is_pr: bool = True): - bin_dir = tmp_path / "bin" - bin_dir.mkdir(exist_ok=True) - stub = bin_dir / "circleci-agent" - stub.write_text("#!/usr/bin/env bash\necho \"[stub] circleci-agent $*\"\nexit 0\n") - stub.chmod(0o755) env = dict(os.environ) - env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" - env.pop("CIRCLE_PULL_REQUEST", None) - if is_pr: - env["CIRCLE_PULL_REQUEST"] = "https://github.com/x/y/pull/1" + env["GITHUB_EVENT_NAME"] = "pull_request" if is_pr else "push" + env["GITHUB_OUTPUT"] = str(tmp_path / "output") return subprocess.run( ["bash", str(scripts_dir / "path_filter.sh"), category], cwd=work, @@ -221,11 +214,12 @@ def _run_path_filter(work: Path, tmp_path: Path, category: str, scripts_dir: Pat ) -def test_path_filter_halts_docs_only_pr(tmp_path: Path) -> None: +def test_path_filter_skips_docs_only_pr(tmp_path: Path) -> None: work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) result = _run_path_filter(work, tmp_path, "backend", SCRIPTS_DIR) assert result.returncode == 0 - assert "circleci-agent step halt" in result.stdout + assert "decision=skip" in result.stdout + assert (tmp_path / "output").read_text() == "decision=skip\n" def test_path_filter_runs_backend_pr(tmp_path: Path) -> None: diff --git a/tests/unit/test_circleci_checkout.py b/tests/unit/test_circleci_checkout.py deleted file mode 100644 index 347024838f7..00000000000 --- a/tests/unit/test_circleci_checkout.py +++ /dev/null @@ -1,310 +0,0 @@ -import os -import shutil -import subprocess -from collections.abc import Iterator, Mapping -from dataclasses import dataclass -from pathlib import Path -from typing import Final - -import pytest -import yaml -from pydantic import BaseModel, ConfigDict - -CIRCLECI: Final = Path(__file__).resolve().parents[2] / ".circleci" -CONFIG: Final = CIRCLECI / "config.yml" -PUBLIC_REPO: Final = "https://github.com/BerriAI/litellm.git" -LINUX_SHELL: Final = ("bash", "-eo", "pipefail", "-c") -WINDOWS_SHELL: Final = ("bash", "-c") -PRODUCT_DIRS: Final = ("litellm", "enterprise", "litellm-proxy-extras") - - -class _Run(BaseModel): - model_config = ConfigDict(frozen=True) - command: str - - -class _Step(BaseModel): - model_config = ConfigDict(frozen=True) - run: _Run - - -class _Command(BaseModel): - model_config = ConfigDict(frozen=True) - steps: tuple[_Step] - - -class _Commands(BaseModel): - model_config = ConfigDict(frozen=True) - checkout_over_https: _Command - - -class _NamedRun(BaseModel): - model_config = ConfigDict(frozen=True) - name: str = "" - command: str - - -class _JobStep(BaseModel): - model_config = ConfigDict(frozen=True) - run: _NamedRun | str | None = None - - -class _Job(BaseModel): - model_config = ConfigDict(frozen=True) - steps: tuple[_JobStep | str, ...] - - -class _Jobs(BaseModel): - model_config = ConfigDict(frozen=True) - routing_parity: _Job - - -class _Config(BaseModel): - model_config = ConfigDict(frozen=True) - commands: _Commands - jobs: _Jobs - - -class _TraceEvent(BaseModel): - model_config = ConfigDict(frozen=True) - event: str - name: str = "" - - -@dataclass(frozen=True, slots=True) -class _Upstream: - url: str - older: str - newer: str - - -def _step_name(step: object) -> str: - if isinstance(step, str): - return step - if isinstance(step, dict) and len(step) == 1: - return str(next(iter(step))) - return "" - - -def _step_names(node: object) -> Iterator[str]: - if isinstance(node, dict): - for key, value in node.items(): - if key in ("steps", "pre-steps", "post-steps") and isinstance(value, list): - yield from map(_step_name, value) - yield from _step_names(value) - if isinstance(node, list): - for item in node: - yield from _step_names(item) - - -def _config() -> _Config: - return _Config.model_validate(yaml.safe_load(CONFIG.read_text())) - - -def _routing_parity_step(name: str) -> str: - return next( - step.run.command - for step in _config().jobs.routing_parity.steps - if isinstance(step, _JobStep) and isinstance(step.run, _NamedRun) and step.run.name == name - ) - - -def _fetches(trace: Path) -> int: - events: Final = tuple(map(_TraceEvent.model_validate_json, trace.read_text().splitlines())) - return sum(event.event == "cmd_name" and event.name == "fetch" for event in events) - - -def _git(cwd: Path, *args: str) -> str: - return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout.strip() - - -def _commit(work: Path, files: Mapping[str, str], message: str) -> str: - for name, content in files.items(): - (work / name).parent.mkdir(parents=True, exist_ok=True) - (work / name).write_text(content) - _git(work, "add", "-A") - _git(work, "commit", "-q", "--allow-empty", "-m", message) - return _git(work, "rev-parse", "HEAD") - - -def _new_upstream(tmp_path: Path) -> tuple[Path, Path]: - remote: Final = tmp_path / "remote.git" - work: Final = tmp_path / "work" - work.mkdir() - _git(tmp_path, "init", "-q", "--bare", str(remote)) - _git(remote, "config", "uploadpack.allowFilter", "true") - _git(work, "init", "-q", "-b", "main") - _git(work, "config", "user.email", "t@t") - _git(work, "config", "user.name", "t") - return remote, work - - -def _publish_two_commits(tmp_path: Path) -> _Upstream: - remote, work = _new_upstream(tmp_path) - older: Final = _commit(work, {"app.py": "x = 1\n"}, "pipeline commit") - newer: Final = _commit(work, {"app.py": "x = 2\n"}, "main moved on") - _git(work, "push", "-q", str(remote), "main") - return _Upstream(url=remote.as_uri(), older=older, newer=newer) - - -def _publish_docs_only_pr(tmp_path: Path) -> _Upstream: - remote, work = _new_upstream(tmp_path) - main: Final = _commit(work, {"litellm_inference.py": "x\n"}, "base") - _git(work, "checkout", "-q", "-b", "litellm_feature") - head: Final = _commit(work, {"README.md": "# docs\n"}, "docs") - _git(work, "push", "-q", str(remote), "main", "litellm_feature") - return _Upstream(url=remote.as_uri(), older=main, newer=head) - - -def _product(version: str) -> dict[str, str]: - return {f"{directory}/code.py": f"{version} {directory}\n" for directory in PRODUCT_DIRS} - - -def _publish_parity_base(tmp_path: Path, base_on_main: bool) -> _Upstream: - remote, work = _new_upstream(tmp_path) - fork_point: Final = _commit(work, _product("fork point"), "fork point") - base: Final = _commit(work, _product("base"), "release") - _git(work, "branch", "-q", "stable") - _git(work, "reset", "-q", "--hard", base if base_on_main else fork_point) - head: Final = _commit(work, _product("head"), "pipeline commit") - _git(work, "push", "-q", str(remote), "main", "stable") - return _Upstream(url=remote.as_uri(), older=base, newer=head) - - -def _stub(bin_dir: Path, name: str, script: str) -> None: - bin_dir.mkdir(exist_ok=True) - (bin_dir / name).write_text(f"#!/usr/bin/env bash\n{script}\n") - (bin_dir / name).chmod(0o755) - - -def _job_env(tmp_path: Path, upstream: _Upstream) -> dict[str, str]: - return { - "PATH": f"{tmp_path / 'bin'}{os.pathsep}{os.environ['PATH']}", - "HOME": str(tmp_path), - "GIT_CONFIG_NOSYSTEM": "1", - "GIT_TERMINAL_PROMPT": "0", - "GIT_SSH_COMMAND": "false", - "GIT_CONFIG_COUNT": "1", - "GIT_CONFIG_KEY_0": f"url.{upstream.url}.insteadOf", - "GIT_CONFIG_VALUE_0": PUBLIC_REPO, - } - - -def _run_checkout( - tmp_path: Path, - upstream: _Upstream, - revisions: Mapping[str, str], - shell: tuple[str, ...] = LINUX_SHELL, -) -> subprocess.CompletedProcess[str]: - job: Final = tmp_path / "job" - job.mkdir() - command: Final = _config().commands.checkout_over_https - return subprocess.run( - [*shell, command.steps[0].run.command], - cwd=job, - env={**_job_env(tmp_path, upstream), **revisions}, - capture_output=True, - text=True, - ) - - -def test_every_clone_goes_over_https_so_no_job_needs_the_ssh_checkout_key() -> None: - names: Final = tuple(_step_names(yaml.safe_load(CONFIG.read_text()))) - assert "checkout_over_https" in names - assert "checkout" not in names - - -@pytest.mark.parametrize( - "revisions", - ( - {"CIRCLE_SHA1": "older"}, - {"CIRCLE_SHA1": "older", "REQUESTED_REVISION": ""}, - {"CIRCLE_SHA1": "newer", "REQUESTED_REVISION": "older"}, - ), -) -def test_a_job_checks_out_its_own_commit_from_the_public_https_url_after_main_moves( - tmp_path: Path, revisions: Mapping[str, str] -) -> None: - upstream: Final = _publish_two_commits(tmp_path) - shas: Final = {"older": upstream.older, "newer": upstream.newer, "": ""} - result: Final = _run_checkout(tmp_path, upstream, {name: shas[value] for name, value in revisions.items()}) - assert result.returncode == 0, result.stderr - assert _git(tmp_path / "job", "rev-parse", "HEAD") == upstream.older - - -@pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe")) -def test_a_revision_that_is_not_a_full_sha_is_refused_before_git_runs(tmp_path: Path, shell: tuple[str, ...]) -> None: - upstream: Final = _publish_two_commits(tmp_path) - result: Final = _run_checkout( - tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer, "REQUESTED_REVISION": "main"}, shell - ) - assert result.returncode != 0 - assert not (tmp_path / "job" / ".git").exists() - - -def test_a_docs_only_pr_job_still_skips_backend_work_after_the_checkout(tmp_path: Path) -> None: - upstream: Final = _publish_docs_only_pr(tmp_path) - _stub(tmp_path / "bin", "circleci-agent", 'echo "circleci-agent $*"') - checkout: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer}) - assert checkout.returncode == 0, checkout.stderr - path_filter: Final = subprocess.run( - ["bash", str(CIRCLECI / "scripts" / "path_filter.sh"), "backend"], - cwd=tmp_path / "job", - env={**_job_env(tmp_path, upstream), "CIRCLE_PULL_REQUEST": "https://github.com/BerriAI/litellm/pull/1"}, - capture_output=True, - text=True, - ) - assert path_filter.returncode == 0, path_filter.stderr - assert "circleci-agent step halt" in path_filter.stdout - - -@pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe")) -@pytest.mark.parametrize( - ("failing_step", "failures", "checkouts", "checked_out"), - (("fetch", 2, 1, True), ("fetch", 3, 0, False), ("checkout", 2, 3, True), ("checkout", 3, 3, False)), -) -def test_a_failed_fetch_or_blob_download_is_retried_and_the_step_stops_on_its_error_once_retries_run_out( - tmp_path: Path, shell: tuple[str, ...], failing_step: str, failures: int, checkouts: int, checked_out: bool -) -> None: - upstream: Final = _publish_two_commits(tmp_path) - calls: Final = tmp_path / "calls" - _stub(tmp_path / "bin", "sleep", f'echo sleep >> "{calls}"') - _stub( - tmp_path / "bin", - "git", - f'''echo "$1" >> "{calls}" -if [ "$1" = {failing_step} ] && [ "$(grep -c '^{failing_step}$' "{calls}")" -le {failures} ]; then exit 128; fi -exec "{shutil.which("git")}" "$@"''', - ) - result: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.older}, shell) - assert (result.returncode == 0) is checked_out, result.stderr - steps: Final = calls.read_text().splitlines() - assert steps.count("fetch") == min(failures + 1, 3) - assert steps.count("sleep") == steps.count("fetch") - 1 - assert steps.count("checkout") == checkouts - head: Final = subprocess.run(["git", "rev-parse", "HEAD"], cwd=tmp_path / "job", capture_output=True, text=True) - assert (head.stdout.strip() == upstream.older) is checked_out - - -@pytest.mark.parametrize("base_on_main", (True, False), ids=("main-commit", "release-branch-commit")) -def test_routing_parity_downloads_the_base_files_in_one_fetch_instead_of_one_per_missing_file( - tmp_path: Path, base_on_main: bool -) -> None: - upstream: Final = _publish_parity_base(tmp_path, base_on_main) - checkout: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer}) - assert checkout.returncode == 0, checkout.stderr - job: Final = tmp_path / "job" - (job / "litellm" / "rust_bridge").mkdir() - (job / "litellm" / "rust_bridge" / "_native.abi3.so").write_text("built for this job\n") - trace: Final = tmp_path / "trace.json" - result: Final = subprocess.run( - [*LINUX_SHELL, _routing_parity_step("Check out base product code")], - cwd=job, - env={**_job_env(tmp_path, upstream), "ROUTING_PARITY_BASE": upstream.older, "GIT_TRACE2_EVENT": str(trace)}, - capture_output=True, - text=True, - ) - assert result.returncode == 0, result.stderr - assert tuple((job / name).read_text() for name in _product("base")) == tuple(_product("base").values()) - assert _fetches(trace) == 1 - assert _git(job, "rev-parse", "HEAD") == upstream.newer diff --git a/tests/unit/test_circleci_run_migration_tests.py b/tests/unit/test_circleci_run_migration_tests.py deleted file mode 100644 index 30fe7383c07..00000000000 --- a/tests/unit/test_circleci_run_migration_tests.py +++ /dev/null @@ -1,36 +0,0 @@ -import importlib.util -from pathlib import Path -from typing import Final - -import pytest - -SCRIPT: Final = Path(__file__).resolve().parents[2] / ".circleci/scripts/run_migration_tests.py" -SPEC: Final = importlib.util.spec_from_file_location("migration_ci", SCRIPT) -assert SPEC is not None and SPEC.loader is not None -MODULE: Final = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(MODULE) - - -@pytest.mark.parametrize( - "xml,expected,exit_code,passed", - ( - ('', 2, 0, True), - ('', 2, 0, False), - ("", 1, 0, False), - ("", 1, 0, False), - ("", 1, 0, False), - ("", 1, 1, False), - ("", 1, 5, False), - ("", 1, 0, False), - ("', 2, 0, False), - ), -) -def test_only_a_complete_passing_suite_can_certify_an_image( - tmp_path: Path, xml: str | None, expected: int, exit_code: int, passed: bool -) -> None: - path: Final = tmp_path / "results.xml" - if xml is not None: - path.write_text(xml) - assert MODULE.successful_junit(path, expected, exit_code) is passed diff --git a/tests/unit/test_circleci_rust_toolchain.py b/tests/unit/test_circleci_rust_toolchain.py deleted file mode 100644 index 562fb5db032..00000000000 --- a/tests/unit/test_circleci_rust_toolchain.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Static guardrails for how CircleCI provisions Rust. - -The root package builds `litellm-rust` through maturin, so any job that runs -`uv sync` or `uv build` compiles the bridge. The `cimg/python` images ship no -Rust toolchain, and when cargo is missing maturin's `puccinialin` helper -quietly provisions one itself: it fetches `rustup-init` from the unversioned -`https://static.rust-lang.org/rustup/dist//` path with no checksum and -installs a floating `stable` toolchain. uv suppresses build-backend output on a -successful sync, so that happens with nothing in the job log to show for it, -and the compiler a job builds with changes whenever upstream publishes. - -Two invariants are pinned here: - - 1. No step list (job or reusable command) reaches a `uv sync` / `uv build` - without a Rust toolchain already provisioned ahead of it. That is the - `install_rust` command on Linux and `install_windows_toolchain` on Windows, - so the check accepts any command or step that installs a pinned rustup. A new job that syncs without one - falls back to the unpinned path, which is exactly the regression a static - check catches at PR time and a green CI run does not. - 2. Both installers pin what they download: an explicit rustup version, a - verified SHA-256, and the exact toolchain in `rust-toolchain.toml`. -""" - -from __future__ import annotations - -import re -from pathlib import Path -from typing import Final - -import pytest -import yaml - -REPO_ROOT = Path(__file__).resolve().parents[2] -CONFIG = REPO_ROOT / ".circleci" / "config.yml" -TOOLCHAIN: Final = REPO_ROOT / "rust-toolchain.toml" - -BUILDS_WORKSPACE = re.compile(r"\buv\s+(?:sync|build)\b") -RUSTUP_ARCHIVE_URL = re.compile(r"https://static\.rust-lang\.org/rustup/archive/\d+\.\d+\.\d+/") -EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?(\d+\.\d+\.\d+)\"?") -TOOLCHAIN_CHANNEL: Final = re.compile(r'^channel = "(\d+\.\d+\.\d+)"$', re.MULTILINE) - - -def _config() -> dict[str, object]: - return yaml.safe_load(CONFIG.read_text()) - - -def _step_text(step: object) -> str: - """Flatten one step into the shell text it runs, or '' for a command reference.""" - if isinstance(step, dict): - run = step.get("run") - if isinstance(run, str): - return run - if isinstance(run, dict): - command = run.get("command") - return command if isinstance(command, str) else "" - return "" - - -def _pinned_toolchain() -> str: - match: Final = TOOLCHAIN_CHANNEL.search(TOOLCHAIN.read_text()) - assert match is not None, "rust-toolchain.toml must pin an exact channel" - return match.group(1) - - -def _without_comments(text: str) -> str: - return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) - - -def _installs_pinned_rustup(step: object) -> bool: - text = _step_text(step) - return "rustup-init" in text and ("sha256sum" in text or "SHA256" in text) - - -def _provisioning_commands() -> frozenset[str]: - return frozenset( - name.removeprefix("command ") - for name, steps in _step_lists().items() - if name.startswith("command ") and any(_installs_pinned_rustup(step) for step in steps) - ) - - -def _provisions_rust(step: object, provisioning_commands: frozenset[str]) -> bool: - return (isinstance(step, str) and step in provisioning_commands) or _installs_pinned_rustup(step) - - -def _step_lists() -> dict[str, list[object]]: - config = _config() - lists: dict[str, list[object]] = {} - for kind in ("jobs", "commands"): - section = config.get(kind) - if not isinstance(section, dict): - continue - for name, body in section.items(): - steps = body.get("steps") if isinstance(body, dict) else None - if isinstance(steps, list): - lists[f"{kind[:-1]} {name}"] = steps - return lists - - -def _first_unprovisioned_build(steps: list[object], provisioning_commands: frozenset[str]) -> str | None: - """Return the shell text of the first workspace build reached without Rust, if any.""" - rust_ready = False - for step in steps: - if _provisions_rust(step, provisioning_commands): - rust_ready = True - text = _step_text(step) - if BUILDS_WORKSPACE.search(_without_comments(text)) and not rust_ready: - return text - return None - - -def test_step_lists_exist() -> None: - lists = _step_lists() - assert "command install_rust" in lists - building = { - name - for name, steps in lists.items() - if any(BUILDS_WORKSPACE.search(_without_comments(_step_text(s))) for s in steps) - } - assert "command setup_litellm_test_deps" in building, f"found {sorted(building)}" - - -def test_no_workspace_build_without_a_provisioned_rust_toolchain() -> None: - provisioning_commands: Final = _provisioning_commands() - assert {"install_rust", "install_windows_toolchain"} <= provisioning_commands - offenders = { - name: build - for name, steps in _step_lists().items() - if (build := _first_unprovisioned_build(steps, provisioning_commands)) is not None - } - assert not offenders, ( - "these CircleCI step lists run `uv sync`/`uv build` with no Rust toolchain provisioned first, " - "so maturin will download an unpinned rustup and a floating toolchain instead: " - f"{ {name: build.strip().splitlines()[0] for name, build in offenders.items()} }" - ) - - -@pytest.fixture(name="install_rust_command") -def _install_rust_command() -> str: - steps = _step_lists()["command install_rust"] - return "\n".join(_step_text(step) for step in steps) - - -def test_install_rust_pins_the_rustup_version_in_the_url(install_rust_command: str) -> None: - assert RUSTUP_ARCHIVE_URL.search(install_rust_command), ( - "install_rust must download rustup-init from a version-pinned /rustup/archive// URL; " - "the /rustup/dist/ path always serves whatever rustup is current" - ) - assert "/rustup/dist/" not in install_rust_command - - -def test_install_rust_verifies_the_installer_checksum(install_rust_command: str) -> None: - assert "sha256sum -c" in install_rust_command - assert re.search(r"RUSTUP_SHA256=[0-9a-f]{64}\b", install_rust_command), ( - "install_rust must compare the downloaded installer against a hardcoded SHA-256 " - "taken from rust-lang's published .sha256 sidecar" - ) - checksum_index = install_rust_command.index("sha256sum -c") - execute_index = install_rust_command.index("/tmp/rustup-init -y") - assert checksum_index < execute_index, "the checksum must be verified before the installer is executed" - - -def test_install_rust_pins_an_exact_toolchain_version(install_rust_command: str) -> None: - match: Final = EXACT_TOOLCHAIN.search(install_rust_command) - assert match is not None, ( - "install_rust must pin an exact toolchain version (e.g. 1.98.0); a channel name like " - "stable/beta/nightly makes the compiler drift with whatever upstream published that day" - ) - assert match.group(1) == _pinned_toolchain() - - -def test_windows_installer_matches_the_repo_toolchain() -> None: - windows_steps: Final = _step_lists()["command install_windows_toolchain"] - windows_command: Final = "\n".join(_step_text(step) for step in windows_steps) - match: Final = EXACT_TOOLCHAIN.search(windows_command) - assert match is not None - assert match.group(1) == _pinned_toolchain() diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index 2602149b623..ae3f2f6caa0 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -355,6 +355,35 @@ def test_checked_in_pins_hold_in_the_checked_in_cost_map() -> None: assert guard.pin_failures(pins, cost_map) == () +@pytest.mark.parametrize("pins_path", guard.PINS_PATHS) +@pytest.mark.parametrize("pins", (PINS, "", "not JSON")) +def test_relocated_pins_keep_the_same_validation(pins_path: str, pins: str) -> None: + cost_map: Final = _serialize(BASE_MAP) + blobs: Final = { + f"base:{guard.COST_MAP_PATH}": cost_map, + f"base:{guard.BACKUP_PATH}": cost_map, + f"head:{guard.COST_MAP_PATH}": cost_map, + f"head:{guard.BACKUP_PATH}": cost_map, + f"base:{guard.PINS_PATH}": PINS, + f"head:{pins_path}": pins, + } + + def read_git(_operation: str, blob: str) -> str | None: + return blobs.get(blob) + + base: Final = guard.snapshot("base", read_git=read_git) + head: Final = guard.snapshot("head", read_git=read_git) + assert base == _snapshot(BASE_MAP, pins=PINS) + assert head == _snapshot(BASE_MAP, pins=pins) + failures: Final = _failures(head, changed_files=(guard.PINS_PATH, pins_path), bot=False, base=base) + assert bool(failures) is (pins != PINS), failures + + +@pytest.mark.parametrize("pins_path", guard.PINS_PATHS) +def test_relocated_pins_cannot_be_changed_by_bots(pins_path: str) -> None: + assert guard.pin_author_failures("litellm_fix_pricing", "Bot", (pins_path,)) == (BOT_AUTHOR_PIN_FAILURE,) + + def _commit(repo: Path, cost_map: dict[str, object], message: str, pins: str | None = None) -> str: text = _serialize(cost_map) pins_file = repo / guard.PINS_PATH diff --git a/tests/unit/test_detect_changes.py b/tests/unit/test_detect_changes.py index 44a4bfdf448..8f41d6498f0 100644 --- a/tests/unit/test_detect_changes.py +++ b/tests/unit/test_detect_changes.py @@ -21,7 +21,7 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] SCRIPT = REPO_ROOT / ".github" / "scripts" / "detect_changes.sh" -CLASSIFIER = REPO_ROOT / ".circleci" / "scripts" / "classify_changes.sh" +CLASSIFIER = REPO_ROOT / ".github" / "scripts" / "ci" / "classify_changes.sh" UI_FILE = "ui/litellm-dashboard/src/components/Teams.tsx" BACKEND_FILE = "litellm/proxy/proxy_server.py" @@ -65,9 +65,9 @@ def _scripts_tree(tmp_path: Path, classifier_body: str | None = None) -> Path: """Copy the scripts into a throwaway tree, preserving their relative layout.""" root = tmp_path / "tree" (root / ".github" / "scripts").mkdir(parents=True) - (root / ".circleci" / "scripts").mkdir(parents=True) + (root / ".github" / "scripts" / "ci").mkdir(parents=True) shutil.copy(SCRIPT, root / ".github" / "scripts" / SCRIPT.name) - target = root / ".circleci" / "scripts" / CLASSIFIER.name + target = root / ".github" / "scripts" / "ci" / CLASSIFIER.name if classifier_body is None: shutil.copy(CLASSIFIER, target) else: diff --git a/tests/unit/test_vcr_conftest_common_banner.py b/tests/unit/test_vcr_conftest_common_banner.py index 30a65abba1d..515d031afa9 100644 --- a/tests/unit/test_vcr_conftest_common_banner.py +++ b/tests/unit/test_vcr_conftest_common_banner.py @@ -145,7 +145,7 @@ def test_banner_silent_when_vcr_disabled(monkeypatch, health_reset, patch_capaci # --------------------------------------------------------------------------- -# Diagnostic-log dedup + cap. CircleCI truncates step output to the last +# Diagnostic-log dedup + cap. GitHub Actions truncates step output to the last # ~400 KB; an unbounded diagnostic dump pushes the VCR classification summary # out of the retrievable window, so the dump must dedupe and cap. # --------------------------------------------------------------------------- From 8ee4ce84185bae9f4ed5bd652c30901c71f08fbd Mon Sep 17 00:00:00 2001 From: shrey Date: Wed, 11 Nov 2026 15:39:17 -0800 Subject: [PATCH 126/189] fix(proxy): stop request body fields from conflicting with admin-set deployment fields (#51254) --- litellm/proxy/auth/auth_utils.py | 95 +++++- tests/unit/proxy/auth/test_auth_utils.py | 300 +++++++++++++++++- .../proxy/test_credential_slot_registry.py | 2 - 3 files changed, 391 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 900e993cadb..7d0ca935cde 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -23,6 +23,10 @@ from litellm.constants import ( MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS, ) +from litellm.litellm_core_utils.clientside_credential_handler import ( + _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list of deployment credential fields + clientside_credential_keys, +) from litellm.litellm_core_utils.cloud_storage_security import REQUEST_S3_BUCKET_PARAMS, STORED_S3_BUCKET_PARAM from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider @@ -54,7 +58,12 @@ from litellm.types.router import ( server_owned_wif_fields_present, ) from litellm.types.router import reject_server_owned_wif_params as _reject_server_owned_wif_params -from litellm.types.utils import CustomPricingLiteLLMParams, LlmProviders, oauth_token_exchange_litellm_params +from litellm.types.utils import ( + OIDC_SECRET_REFERENCE_PREFIX, + CustomPricingLiteLLMParams, + LlmProviders, + oauth_token_exchange_litellm_params, +) from litellm.types.videos.main import DecodedVideoId @@ -380,6 +389,37 @@ def _without_route_named_vertex_target(request_body: Mapping[str, object], route return MappingProxyType({key: value for key, value in request_body.items() if key not in VERTEX_TARGET_PARAMS}) +_AZURE_ENTRA_CREDENTIAL_PARAMS: Final[tuple[str, ...]] = ( + "tenant_id", + "client_id", + "client_secret", + "azure_username", + "azure_password", + "azure_scope", +) +_MCP_OAUTH_CLIENT_FORM_FIELDS: Final[frozenset[str]] = frozenset(("client_id", "client_secret")) +_TRANSCRIPTION_SESSION_ROUTES: Final[frozenset[str]] = frozenset( + ( + "/v1/realtime/transcription_sessions", + "/realtime/transcription_sessions", + "/openai/v1/realtime/transcription_sessions", + ) +) +_TRANSCRIPTION_SESSION_FIELDS: Final[frozenset[str]] = frozenset(("client_secret",)) +_TOKEN_INTROSPECTION_ROUTE: Final = "/introspect" +_BANNED_PARAMS_ALLOWED_IN_METADATA: Final[frozenset[str]] = VERTEX_TARGET_PARAMS | frozenset( + _AZURE_ENTRA_CREDENTIAL_PARAMS +) + + +def _entra_params_owned_by_route(route: str | None) -> frozenset[str]: + if route in _TRANSCRIPTION_SESSION_ROUTES: + return _TRANSCRIPTION_SESSION_FIELDS + if route_manages_mcp_servers(route) or route == _TOKEN_INTROSPECTION_ROUTE: + return _MCP_OAUTH_CLIENT_FORM_FIELDS + return frozenset() + + _NESTED_CONFIG_KEYS: Final[tuple[str, ...]] = ("litellm_embedding_config", "extra_body") # Metadata containers that carry per-request configuration consumed by the @@ -483,6 +523,7 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( # caller-supplied value is the same exfil shape as # ``aws_web_identity_token`` on the Bedrock path. "azure_ad_token", + *_AZURE_ENTRA_CREDENTIAL_PARAMS, # Endpoint-targeting fields that retarget the outbound request or # an observability callback. An attacker-controlled value either # exfiltrates the request payload (incl. messages + admin-set @@ -548,8 +589,34 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( *sorted(CustomPricingLiteLLMParams.model_fields.keys()), ) +_SECRET_REFERENCE_PREFIXES: Final[tuple[str, ...]] = ("os.environ/", OIDC_SECRET_REFERENCE_PREFIX) +_SECRET_RESOLVING_PARAMS: Final[frozenset[str]] = frozenset( + (*_BANNED_REQUEST_BODY_PARAMS, *_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, *clientside_credential_keys) +) + + +def _reject_secret_references(body: Mapping[str, object]) -> None: + _reject_secret_reference_in(MappingProxyType({k: v for k, v in body.items() if k in _SECRET_RESOLVING_PARAMS})) + + +def _reject_secret_reference_in(params: Mapping[str, object]) -> None: + referencing: Final = next( + ( + param + for param, value in params.items() + if isinstance(value, str) and value.startswith(_SECRET_REFERENCE_PREFIXES) + ), + None, + ) + if referencing is not None: + raise ValueError( + f"Rejected Request: {referencing} holds a secret reference, which only a deployment or credential a " + "proxy admin configures can carry." + ) + + _BANNED_METADATA_PARAMS: Final[tuple[str, ...]] = tuple( - param for param in _BANNED_REQUEST_BODY_PARAMS if param not in VERTEX_TARGET_PARAMS + param for param in _BANNED_REQUEST_BODY_PARAMS if param not in _BANNED_PARAMS_ALLOWED_IN_METADATA ) @@ -707,6 +774,8 @@ def _check_banned_params( *, manages_deployments: bool = False, mcp_server_fields_allowed: bool = False, + route_owned_params: frozenset[str] = frozenset(), + rejects_secret_references: bool = True, banned_params: tuple[str, ...] = _BANNED_REQUEST_BODY_PARAMS, opt_in_groups: Callable[[], _OptInGroups], defer_model_checks: bool = False, @@ -726,9 +795,13 @@ def _check_banned_params( reject_server_owned_wif_params(wif_body) if not manages_deployments: reject_federated_credential_reference(body) + if rejects_secret_references: + _reject_secret_references(body) for param in banned_params: if param not in body: continue + if param in route_owned_params: + continue if general_settings.get("allow_client_side_credentials") is True: # Proxy-wide opt-in: every banned param is permitted, exit # entirely so the rest of the loop doesn't waste work. @@ -951,6 +1024,7 @@ def is_request_body_safe( llm_router, manages_deployments=manages_deployments, mcp_server_fields_allowed=route_manages_mcp_servers(route), + route_owned_params=_entra_params_owned_by_route(route), opt_in_groups=opt_in_groups, defer_model_checks=defer_model_checks, defer_s3_buckets=defer_s3_buckets, @@ -976,6 +1050,7 @@ def is_request_body_safe( llm_router, manages_deployments=manages_deployments, banned_params=_BANNED_METADATA_PARAMS, + rejects_secret_references=False, opt_in_groups=opt_in_groups, defer_model_checks=defer_model_checks, defer_s3_buckets=defer_s3_buckets, @@ -987,6 +1062,7 @@ def is_request_body_safe( llm_router, manages_deployments=manages_deployments, banned_params=_BANNED_METADATA_PARAMS, + rejects_secret_references=False, opt_in_groups=opt_in_groups, defer_model_checks=defer_model_checks, defer_s3_buckets=defer_s3_buckets, @@ -1020,15 +1096,30 @@ def is_request_body_safe( llm_router, manages_deployments=manages_deployments, banned_params=_BANNED_METADATA_PARAMS, + rejects_secret_references=False, opt_in_groups=opt_in_groups, defer_model_checks=defer_model_checks, defer_s3_buckets=defer_s3_buckets, ) for user_config_params in _user_config_litellm_params(request_body): reject_api_key_secret_reference(user_config_params) + _reject_user_config_secret_references(user_config_params) return True +def _reject_user_config_secret_references(litellm_params: Mapping[str, object]) -> None: + for params in (litellm_params, *_nested_configs(litellm_params), *_data_source_parameters(litellm_params)): + _reject_secret_reference_in(params) + + +def _data_source_parameters(litellm_params: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return tuple( + parameters + for source in _mappings_in(litellm_params.get("dataSources")) + if (parameters := _coerce_metadata_to_dict(source.get("parameters"))) is not None + ) + + def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None: if manages_deployments: reject_server_owned_wif_fields( diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 58a78aede3c..6a7bbb2f0ce 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -59,7 +59,7 @@ def test_token_exchange_settings_in_request_body_are_rejected(field: str) -> Non match="server-owned workload identity federation or OAuth token exchange parameter", ) as error: is_request_body_safe( - request_body={"model": "microsoft_365_copilot/chat", field: "attacker-chosen"}, + request_body={"model": "microsoft_365_copilot/chat", field: "caller-chosen"}, general_settings={"allow_client_side_credentials": True}, llm_router=None, model="microsoft_365_copilot/chat", @@ -151,6 +151,302 @@ def test_model_opt_in_cannot_allow_a_token_exchange_endpoint_in_a_request_body() ) +_AZURE_ENTRA_CREDENTIAL_PARAMS: Final = ( + "tenant_id", + "client_id", + "client_secret", + "azure_username", + "azure_password", + "azure_scope", +) + + +def _azure_entra_router(configurable_clientside_auth_params: tuple[str, ...] = ()) -> Router: + return Router( + model_list=[ + { + "model_name": "azure-group", + "litellm_params": { + "model": "azure/gpt-4o", + "api_base": "https://admin-endpoint.example.com", + "tenant_id": "admin-tenant", + "client_id": "admin-client", + "client_secret": "admin-secret", + "configurable_clientside_auth_params": list(configurable_clientside_auth_params), + }, + } + ] + ) + + +@pytest.mark.parametrize("param", _AZURE_ENTRA_CREDENTIAL_PARAMS) +@pytest.mark.parametrize("container", (None, "extra_body", "litellm_embedding_config")) +def test_azure_entra_credential_params_in_request_body_are_rejected(param: str, container: str | None) -> None: + override: Final = {param: "caller-chosen"} + with pytest.raises(ValueError, match=f"{param} is not allowed in request body"): + is_request_body_safe( + request_body={"model": "azure-group", **({container: override} if container else override)}, + general_settings={}, + llm_router=_azure_entra_router(), + model="azure-group", + route="/v1/chat/completions", + ) + + +@pytest.mark.parametrize("param", _AZURE_ENTRA_CREDENTIAL_PARAMS) +@pytest.mark.parametrize("metadata_key", ("metadata", "litellm_metadata")) +def test_azure_entra_credential_names_are_allowed_as_request_metadata(param: str, metadata_key: str) -> None: + assert ( + is_request_body_safe( + request_body={"model": "azure-group", metadata_key: {param: "caller-tag"}}, + general_settings={}, + llm_router=_azure_entra_router(), + model="azure-group", + route="/v1/chat/completions", + ) + is True + ) + + +def test_deployment_opt_in_allows_only_the_named_azure_entra_credential_param() -> None: + router: Final = _azure_entra_router(configurable_clientside_auth_params=("client_id",)) + + assert ( + is_request_body_safe( + request_body={"model": "azure-group", "client_id": "caller-client"}, + general_settings={}, + llm_router=router, + model="azure-group", + route="/v1/chat/completions", + ) + is True + ) + with pytest.raises(ValueError, match="client_secret"): + is_request_body_safe( + request_body={"model": "azure-group", "client_id": "caller-client", "client_secret": "caller-secret"}, + general_settings={}, + llm_router=router, + model="azure-group", + route="/v1/chat/completions", + ) + + +@pytest.mark.parametrize("param", (*_AZURE_ENTRA_CREDENTIAL_PARAMS, "api_key", "aws_access_key_id", "azure_ad_token")) +@pytest.mark.parametrize("reference", ("os.environ/LITELLM_MASTER_KEY", "oidc/env/LITELLM_MASTER_KEY")) +@pytest.mark.parametrize("container", (None, "extra_body")) +def test_secret_reference_in_a_credential_param_is_rejected_despite_client_side_opt_in( + param: str, reference: str, container: str | None +) -> None: + override: Final = {param: reference} + with pytest.raises(ValueError, match=f"{param} holds an? (oidc/ )?secret reference"): + is_request_body_safe( + request_body={"model": "azure-group", **({container: override} if container else override)}, + general_settings={"allow_client_side_credentials": True}, + llm_router=_azure_entra_router(), + model="azure-group", + route="/v1/chat/completions", + ) + + +def test_secret_reference_is_rejected_for_a_param_the_deployment_opted_in() -> None: + with pytest.raises(ValueError, match="client_id holds a secret reference"): + is_request_body_safe( + request_body={"model": "azure-group", "client_id": "os.environ/LITELLM_MASTER_KEY"}, + general_settings={}, + llm_router=_azure_entra_router(configurable_clientside_auth_params=("client_id",)), + model="azure-group", + route="/v1/chat/completions", + ) + + +@pytest.mark.parametrize( + "body", + ( + {"prompt": "os.environ/PATH is what in Python?"}, + {"input": "oidc/env/ tokens explained"}, + {"api_key": "sk-caller-own-key"}, + {"client_id": "an os.environ/ mention that is not a prefix"}, + ), +) +def test_values_that_are_not_secret_references_in_credential_params_are_allowed(body: Mapping[str, str]) -> None: + assert ( + is_request_body_safe( + request_body={"model": "azure-group", **body}, + general_settings={"allow_client_side_credentials": True}, + llm_router=_azure_entra_router(), + model="azure-group", + route="/v1/chat/completions", + ) + is True + ) + + +def test_deployment_management_body_may_carry_a_secret_reference() -> None: + assert ( + is_request_body_safe( + request_body={"model_name": "azure-group", "api_key": "os.environ/AZURE_API_KEY"}, + general_settings={}, + llm_router=None, + model="", + route="/model/new", + ) + is True + ) + + +@pytest.mark.parametrize("field", ("client_id", "client_secret")) +def test_mcp_oauth_token_form_may_carry_oauth_client_fields(field: str) -> None: + assert ( + is_request_body_safe( + request_body={"grant_type": "authorization_code", "code": "abc", field: "mcp-oauth-client"}, + general_settings={}, + llm_router=None, + model="", + route="/v1/mcp/server/oauth/abc123/token", + ) + is True + ) + + +@pytest.mark.parametrize("field", ("tenant_id", "azure_username", "azure_password", "azure_scope")) +def test_mcp_server_routes_still_reject_non_oauth_azure_entra_params(field: str) -> None: + with pytest.raises(ValueError, match=field): + is_request_body_safe( + request_body={"grant_type": "authorization_code", "code": "abc", field: "caller-chosen"}, + general_settings={}, + llm_router=None, + model="", + route="/v1/mcp/server/oauth/abc123/token", + ) + + +@pytest.mark.parametrize("body", ({"client_id": "gateway-1"}, {"client_id": "gateway-1", "client_secret": "gw-secret"})) +def test_token_introspection_form_may_carry_oauth_client_fields(body: Mapping[str, str]) -> None: + assert ( + is_request_body_safe( + request_body={"token": "llm_session_abc", **body}, + general_settings={}, + llm_router=None, + model="", + route="/introspect", + ) + is True + ) + + +@pytest.mark.parametrize("body", ({"tenant_id": "caller-tenant"}, {"client_id": "os.environ/LITELLM_MASTER_KEY"})) +def test_token_introspection_still_rejects_other_entra_params_and_secret_references(body: Mapping[str, str]) -> None: + with pytest.raises(ValueError, match=next(iter(body))): + is_request_body_safe( + request_body={"token": "llm_session_abc", **body}, + general_settings={}, + llm_router=None, + model="", + route="/introspect", + ) + + +_TRANSCRIPTION_SESSION_ROUTES: Final = ( + "/v1/realtime/transcription_sessions", + "/realtime/transcription_sessions", + "/openai/v1/realtime/transcription_sessions", +) + + +@pytest.mark.parametrize("route", _TRANSCRIPTION_SESSION_ROUTES) +def test_transcription_session_body_may_carry_its_client_secret_config(route: str) -> None: + assert ( + is_request_body_safe( + request_body={ + "model": "gpt-4o-transcribe", + "client_secret": {"expires_at": {"anchor": "created_at", "seconds": 600}}, + }, + general_settings={}, + llm_router=None, + model="", + route=route, + ) + is True + ) + + +@pytest.mark.parametrize("route", _TRANSCRIPTION_SESSION_ROUTES) +@pytest.mark.parametrize("body", ({"tenant_id": "caller-tenant"}, {"client_secret": "os.environ/LITELLM_MASTER_KEY"})) +def test_transcription_session_routes_still_reject_other_entra_params_and_secret_references( + route: str, body: Mapping[str, str] +) -> None: + with pytest.raises(ValueError, match=next(iter(body))): + is_request_body_safe( + request_body={"model": "gpt-4o-transcribe", **body}, + general_settings={}, + llm_router=None, + model="", + route=route, + ) + + +@pytest.mark.parametrize("param", _AZURE_ENTRA_CREDENTIAL_PARAMS) +@pytest.mark.parametrize("metadata_key", ("metadata", "litellm_metadata")) +@pytest.mark.parametrize("value", ("os.environ/HOME", "oidc/azure/team-a")) +def test_metadata_tags_whose_value_looks_like_a_secret_reference_are_allowed( + param: str, metadata_key: str, value: str +) -> None: + assert ( + is_request_body_safe( + request_body={"model": "azure-group", metadata_key: {param: value}}, + general_settings={}, + llm_router=_azure_entra_router(), + model="azure-group", + route="/v1/chat/completions", + ) + is True + ) + + +_USER_CONFIG_SECRET_REFERENCE_PARAMS: Final = ( + {"model": "openai/gpt-4o", "api_base": "https://caller.example.com", "api_key": "os.environ/LITELLM_MASTER_KEY"}, + {"model": "openai/gpt-4o", "organization": "os.environ/LITELLM_MASTER_KEY"}, + {"model": "openai/gpt-4o", "extra_body": {"api_key": "oidc/env/LITELLM_MASTER_KEY"}}, + {"model": "azure/gpt-4o", "dataSources": [{"parameters": {"key": "os.environ/LITELLM_MASTER_KEY"}}]}, +) + + +def _user_config(slot: str, litellm_params: Mapping[str, object]) -> Mapping[str, object]: + if slot == "default_litellm_params": + return {"default_litellm_params": dict(litellm_params)} + return {"model_list": [{"model_name": "caller-model", "litellm_params": dict(litellm_params)}]} + + +@pytest.mark.parametrize("litellm_params", _USER_CONFIG_SECRET_REFERENCE_PARAMS) +@pytest.mark.parametrize("slot", ("model_list", "default_litellm_params")) +def test_user_config_secret_references_are_rejected_despite_client_side_opt_in( + litellm_params: Mapping[str, object], slot: str +) -> None: + with pytest.raises(ValueError, match="holds a secret reference"): + is_request_body_safe( + request_body={"model": "caller-model", "user_config": _user_config(slot, litellm_params)}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="caller-model", + route="/v1/chat/completions", + ) + + +def test_user_config_with_literal_credentials_is_allowed_with_client_side_opt_in() -> None: + litellm_params: Final = {"model": "openai/gpt-4o", "api_base": "https://caller.example.com", "api_key": "sk-own"} + + assert ( + is_request_body_safe( + request_body={"model": "caller-model", "user_config": _user_config("model_list", litellm_params)}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="caller-model", + route="/v1/chat/completions", + ) + is True + ) + + @pytest.mark.parametrize( "param", sorted(ANTHROPIC_WIF_KWARGS_KEYS | OPENAI_WIF_KWARGS_KEYS | set(oauth_token_exchange_litellm_params)), @@ -5821,7 +6117,7 @@ def test_per_user_credential_slots_in_request_body_are_rejected(field: str) -> N connection by proxy-injected secret_fields; a body-supplied value could only spoof either.""" with pytest.raises(ValueError, match="Rejected Request") as error: is_request_body_safe( - request_body={"model": "github_copilot/gpt-4o", field: "attacker-chosen"}, + request_body={"model": "github_copilot/gpt-4o", field: "caller-chosen"}, general_settings={}, llm_router=None, model="github_copilot/gpt-4o", diff --git a/tests/unit/proxy/test_credential_slot_registry.py b/tests/unit/proxy/test_credential_slot_registry.py index 6332038feb8..47566b853a0 100644 --- a/tests/unit/proxy/test_credential_slot_registry.py +++ b/tests/unit/proxy/test_credential_slot_registry.py @@ -147,8 +147,6 @@ REQUEST_BODY_PARAM_CLASSIFICATION: Final[Mapping[str, Classification]] = Mapping "aws_access_key_id": Unplanted(), "aws_secret_access_key": Unplanted(), "aws_session_token": Unplanted(), - "azure_password": Unplanted(), - "client_secret": Unplanted(), "s3_access_key_id": Unplanted(), "s3_secret_access_key": Unplanted(), "valkey_password": Unplanted(), From cdcece4d329de10576ba9b79dcf4ec57d536438f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 11 Nov 2026 15:48:02 -0800 Subject: [PATCH 127/189] test(proxy): make the cancelled supervisor test deterministic (#51258) The test spawned a real Python child that installs a SIGTERM handler, slept 0.3s, then cancelled the supervisor and expected the child to exit 0. CI runs with --cov, and pytest-cov starts coverage in every spawned Python process, so the child is often still starting when SIGTERM lands and dies with -15. open_children now takes the process spawner the same way it takes the platform, and only needs the platform to signal and stop children. The test uses a fake child whose wait() blocks on an event and a fake platform that records stop_group, so it checks the shielded stop-then-wait with no real process and no sleep. It also checks the child is still being waited on, because anyio's wait_all_tasks_blocked on asyncio can return one loop tick before a task group finishes. --- .../proxy/rust_forward/platform/__init__.py | 16 +++- litellm/proxy/rust_forward/platform/posix.py | 6 +- .../proxy/rust_forward/platform/windows.py | 6 +- .../proxy/rust_forward/supervisor/runtime.py | 53 +++++++----- .../rust_forward/supervisor/test_runtime.py | 82 +++++++++++++++---- 5 files changed, 118 insertions(+), 45 deletions(-) diff --git a/litellm/proxy/rust_forward/platform/__init__.py b/litellm/proxy/rust_forward/platform/__init__.py index 4043adacdfb..b5a53d6efb0 100644 --- a/litellm/proxy/rust_forward/platform/__init__.py +++ b/litellm/proxy/rust_forward/platform/__init__.py @@ -24,6 +24,18 @@ class SpawnOptions: creationflags: int +class ChildProcess(Protocol): + @property + def pid(self) -> int: ... + + @property + def stdin(self) -> asyncio.StreamWriter | None: ... + + def send_signal(self, signum: int, /) -> None: ... + + async def wait(self) -> int: ... + + class Platform(Protocol): @property def name(self) -> str: ... @@ -46,9 +58,9 @@ class Platform(Protocol): def catch(self, loop: asyncio.AbstractEventLoop, on_signal: Callable[[signal.Signals], None]) -> None: ... - def signal_child(self, process: asyncio.subprocess.Process, signum: signal.Signals) -> None: ... + def signal_child(self, process: ChildProcess, signum: signal.Signals) -> None: ... - def stop_group(self, process: asyncio.subprocess.Process) -> None: ... + def stop_group(self, process: ChildProcess) -> None: ... def current_platform() -> Platform: diff --git a/litellm/proxy/rust_forward/platform/posix.py b/litellm/proxy/rust_forward/platform/posix.py index 0d0c6c96649..0cb71b01ae2 100644 --- a/litellm/proxy/rust_forward/platform/posix.py +++ b/litellm/proxy/rust_forward/platform/posix.py @@ -10,7 +10,7 @@ from typing import Final from pydantic import TypeAdapter from litellm.proxy.rust_forward.mode import RUST_HOP -from litellm.proxy.rust_forward.platform import PlatformListener, SpawnOptions +from litellm.proxy.rust_forward.platform import ChildProcess, PlatformListener, SpawnOptions from litellm.proxy.rust_forward.server import Server from litellm.proxy.server_listener import InheritedFd @@ -55,10 +55,10 @@ class PosixPlatform: for signum in self.relayed_signals: loop.add_signal_handler(signum, on_signal, signum) - def signal_child(self, process: asyncio.subprocess.Process, signum: signal.Signals) -> None: + def signal_child(self, process: ChildProcess, signum: signal.Signals) -> None: with contextlib.suppress(ProcessLookupError): process.send_signal(signum) - def stop_group(self, process: asyncio.subprocess.Process) -> None: + def stop_group(self, process: ChildProcess) -> None: with contextlib.suppress(ProcessLookupError): os.killpg(process.pid, signal.SIGTERM) diff --git a/litellm/proxy/rust_forward/platform/windows.py b/litellm/proxy/rust_forward/platform/windows.py index 163b96b9b20..841c9cbfd59 100644 --- a/litellm/proxy/rust_forward/platform/windows.py +++ b/litellm/proxy/rust_forward/platform/windows.py @@ -10,7 +10,7 @@ from typing import Final from pydantic import TypeAdapter from litellm.proxy.rust_forward.mode import RUST_HOP -from litellm.proxy.rust_forward.platform import PlatformListener, SpawnOptions +from litellm.proxy.rust_forward.platform import ChildProcess, PlatformListener, SpawnOptions from litellm.proxy.rust_forward.server import Server from litellm.proxy.server_listener import AssignedPort @@ -60,9 +60,9 @@ class WindowsPlatform: # A child in its own process group can only be sent CTRL_BREAK_EVENT, since Windows never delivers CTRL_C_EVENT # to a new group. Both children shut down on it as on Ctrl-C, but uvicorn force-exits only on a repeated SIGINT, # so a second Ctrl-C does not cut a Windows shutdown short the way it does on POSIX. - def signal_child(self, process: asyncio.subprocess.Process, signum: signal.Signals) -> None: + def signal_child(self, process: ChildProcess, signum: signal.Signals) -> None: self.stop_group(process) - def stop_group(self, process: asyncio.subprocess.Process) -> None: + def stop_group(self, process: ChildProcess) -> None: with contextlib.suppress(ProcessLookupError): process.send_signal(CTRL_BREAK_EVENT) diff --git a/litellm/proxy/rust_forward/supervisor/runtime.py b/litellm/proxy/rust_forward/supervisor/runtime.py index 7b727fc0d80..8a8bda6f7f2 100644 --- a/litellm/proxy/rust_forward/supervisor/runtime.py +++ b/litellm/proxy/rust_forward/supervisor/runtime.py @@ -14,7 +14,7 @@ from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStre from typing_extensions import assert_never from litellm.proxy.rust_forward.mode import RUST_HOP -from litellm.proxy.rust_forward.platform import Platform +from litellm.proxy.rust_forward.platform import ChildProcess, Platform from litellm.proxy.rust_forward.server import PythonServer from litellm.proxy.rust_forward.supervisor.children import ChildSpec, python_spec, rust_spec from litellm.proxy.rust_forward.supervisor.lifecycle import ( @@ -46,6 +46,27 @@ class Executor(Protocol): async def next_event(self) -> Event: ... +class ChildControl(Protocol): + def signal_child(self, process: ChildProcess, signum: signal.Signals) -> None: ... + + def stop_group(self, process: ChildProcess) -> None: ... + + +class Spawner(Protocol): + async def __call__(self, spec: ChildSpec, /) -> ChildProcess: ... + + +async def spawn_subprocess(spec: ChildSpec) -> asyncio.subprocess.Process: + return await asyncio.create_subprocess_exec( + *spec.argv, + stdin=asyncio.subprocess.PIPE, + env={**os.environ, **spec.env_overrides}, + pass_fds=spec.spawn.pass_fds, + start_new_session=spec.spawn.start_new_session, + creationflags=spec.spawn.creationflags, + ) + + async def run(platform: Platform, binary: str, command: Sequence[str], server: PythonServer) -> int | SpawnFailed: with platform.private_listener() as private: specs: Final = MappingProxyType( @@ -54,7 +75,7 @@ async def run(platform: Platform, binary: str, command: Sequence[str], server: P Child.RUST: rust_spec(platform, binary, private.port, server), } ) - async with open_children(platform, specs, private.port) as children: + async with open_children(platform, spawn_subprocess, specs, private.port) as children: platform.catch(asyncio.get_running_loop(), children.caught) return await drive(children, platform.stop_signals) @@ -100,14 +121,15 @@ def exit_like( @dataclass(frozen=True, slots=True) class Children: - platform: Platform + platform: ChildControl + spawner: Spawner specs: Mapping[Child, ChildSpec] private_port: int stack: contextlib.AsyncExitStack tasks: TaskGroup events_in: MemoryObjectSendStream[Event] events_out: MemoryObjectReceiveStream[Event] - processes: dict[Child, asyncio.subprocess.Process] # mutable-ok: the running children the stack owns + processes: dict[Child, ChildProcess] # mutable-ok: the running children the stack owns def caught(self, signum: signal.Signals) -> None: with contextlib.suppress(anyio.ClosedResourceError): @@ -115,7 +137,9 @@ class Children: async def spawn(self, child: Child) -> SpawnFailed | None: try: - process: Final = await self.stack.enter_async_context(_process(self.platform, self.specs[child])) + process: Final = await self.stack.enter_async_context( + _process(self.platform, self.spawner, self.specs[child]) + ) except OSError as error: return SpawnFailed(error) self.processes[child] = process @@ -137,28 +161,21 @@ class Children: # unbounded event buffer. @contextlib.asynccontextmanager async def open_children( - platform: Platform, specs: Mapping[Child, ChildSpec], private_port: int + platform: ChildControl, spawner: Spawner, specs: Mapping[Child, ChildSpec], private_port: int ) -> AsyncGenerator[Children]: events_in, events_out = anyio.create_memory_object_stream[Event](math.inf) with events_in, events_out: async with contextlib.AsyncExitStack() as stack: tasks: Final = await stack.enter_async_context(anyio.create_task_group()) - yield Children(platform, specs, private_port, stack, tasks, events_in, events_out, {}) + yield Children(platform, spawner, specs, private_port, stack, tasks, events_in, events_out, {}) # A child's stdin is a pipe only the supervisor holds open, so the child sees EOF as soon as the supervisor is gone, # however it went, and shuts down instead of outliving it. Stopping is shielded so a cancelled supervisor still waits # for the child it is stopping. @contextlib.asynccontextmanager -async def _process(platform: Platform, spec: ChildSpec) -> AsyncGenerator[asyncio.subprocess.Process]: - process: Final = await asyncio.create_subprocess_exec( - *spec.argv, - stdin=asyncio.subprocess.PIPE, - env={**os.environ, **spec.env_overrides}, - pass_fds=spec.spawn.pass_fds, - start_new_session=spec.spawn.start_new_session, - creationflags=spec.spawn.creationflags, - ) +async def _process(platform: ChildControl, spawner: Spawner, spec: ChildSpec) -> AsyncGenerator[ChildProcess]: + process: Final = await spawner(spec) try: yield process finally: @@ -169,9 +186,7 @@ async def _process(platform: Platform, spec: ChildSpec) -> AsyncGenerator[asynci process.stdin.close() -async def _report_exit( - child: Child, process: asyncio.subprocess.Process, events: MemoryObjectSendStream[Event] -) -> None: +async def _report_exit(child: Child, process: ChildProcess, events: MemoryObjectSendStream[Event]) -> None: with events: events.send_nowait(ChildExited(child, await process.wait())) diff --git a/tests/unit/proxy/rust_forward/supervisor/test_runtime.py b/tests/unit/proxy/rust_forward/supervisor/test_runtime.py index b84b1302e3c..c5d1bbc7a36 100644 --- a/tests/unit/proxy/rust_forward/supervisor/test_runtime.py +++ b/tests/unit/proxy/rust_forward/supervisor/test_runtime.py @@ -9,7 +9,7 @@ from typing import Final import anyio import pytest -from litellm.proxy.rust_forward.platform import Platform, current_platform +from litellm.proxy.rust_forward.platform import ChildProcess, Platform, current_platform from litellm.proxy.rust_forward.platform.windows import WindowsPlatform from litellm.proxy.rust_forward.supervisor import supervise from litellm.proxy.rust_forward.supervisor.children import ChildSpec @@ -23,7 +23,7 @@ from litellm.proxy.rust_forward.supervisor.lifecycle import ( Send, Spawn, ) -from litellm.proxy.rust_forward.supervisor.runtime import SpawnFailed, drive, exit_like, open_children +from litellm.proxy.rust_forward.supervisor.runtime import SpawnFailed, drive, exit_like, open_children, spawn_subprocess from tests.unit.proxy.rust_forward.supervisor._support import python_server @@ -129,34 +129,80 @@ def _children(argv: tuple[str, ...]) -> MappingProxyType[Child, ChildSpec]: return MappingProxyType({Child.PYTHON: spec, Child.RUST: spec}) -SLOW_TO_STOP: Final = ( - sys.executable, - "-c", - "import signal, time; signal.signal(signal.SIGTERM, lambda *_: (time.sleep(0.5), exit(0))); time.sleep(30)", -) +@dataclass(slots=True) +class HeldChild: + exits: anyio.Event + waiting: int = 0 + stdin: None = None + + @property + def pid(self) -> int: + raise NotImplementedError + + def send_signal(self, signum: int, /) -> None: + raise NotImplementedError + + async def wait(self) -> int: + self.waiting += 1 + try: + await self.exits.wait() + finally: + self.waiting -= 1 + return 0 + + +@dataclass(frozen=True, slots=True) +class StopRecorder: + stopped: list[ChildProcess] = field(default_factory=list) + + def signal_child(self, process: ChildProcess, signum: signal.Signals) -> None: + raise NotImplementedError + + def stop_group(self, process: ChildProcess) -> None: + self.stopped.append(process) -@POSIX_ONLY async def test_a_cancelled_supervisor_still_waits_for_the_child_it_stops() -> None: - with anyio.CancelScope() as scope: - async with open_children(current_platform(), _children(SLOW_TO_STOP), 1) as children: - assert await children.spawn(Child.RUST) is None - process: Final = children.processes[Child.RUST] - await anyio.sleep(0.3) - scope.cancel() - await anyio.sleep(1) - assert process.returncode == 0 + child: Final = HeldChild(anyio.Event()) + control: Final = StopRecorder() + supervisor: Final = anyio.CancelScope() + returned: Final = anyio.Event() + + async def spawn(_spec: ChildSpec) -> HeldChild: + return child + + async def supervise_one_child() -> None: + with supervisor: + async with open_children(control, spawn, _children(("held",)), 1) as children: + assert await children.spawn(Child.RUST) is None + await anyio.sleep_forever() + returned.set() + + async with anyio.create_task_group() as tasks: + tasks.start_soon(supervise_one_child) + await anyio.wait_all_tasks_blocked() + supervisor.cancel() + await anyio.wait_all_tasks_blocked() + while_child_runs: Final = (tuple(control.stopped), child.waiting, returned.is_set()) + child.exits.set() + await anyio.wait_all_tasks_blocked() + once_child_exits: Final = returned.is_set() + assert (while_child_runs, once_child_exits) == (((child,), 1, False), True) async def test_exiting_children_are_reported_as_events() -> None: - async with open_children(current_platform(), _children((sys.executable, "-c", "exit(7)")), 1) as children: + async with open_children( + current_platform(), spawn_subprocess, _children((sys.executable, "-c", "exit(7)")), 1 + ) as children: assert await children.spawn(Child.RUST) is None with anyio.fail_after(5): assert await children.next_event() == ChildExited(Child.RUST, 7) async def test_a_child_that_cannot_start_is_returned_as_a_failure(tmp_path: Path) -> None: - async with open_children(current_platform(), _children((str(tmp_path / "missing"),)), 1) as children: + async with open_children( + current_platform(), spawn_subprocess, _children((str(tmp_path / "missing"),)), 1 + ) as children: outcome: Final = await children.spawn(Child.RUST) assert isinstance(outcome, SpawnFailed) assert isinstance(outcome.error, FileNotFoundError) From 82471d4603029571e28719181a6bbeff28ff2706 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 11 Nov 2026 15:49:46 -0800 Subject: [PATCH 128/189] fix(responses): keep reasoning item IDs consistent across streaming events (#51243) * fix(responses): keep encrypted reasoning IDs consistent across stream events * fix(responses): preserve immutable stream events during affinity encoding * fix(responses): type affinity encoding input boundaries * fix(responses): use immutable affinity item updates --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../crates/inference-responses/src/ids.rs | 51 ++++- .../streaming_iterator.py | 9 +- litellm/responses/streaming_iterator.py | 59 ++---- litellm/responses/utils.py | 163 ++++++++++++--- .../test_streaming_iterator_transformation.py | 54 ++++- .../test_responses_websocket_all_providers.py | 71 ++++++- .../unit/responses/test_streaming_iterator.py | 192 +++++++++++++++++- .../test_encrypted_content_affinity_check.py | 87 +++++++- 8 files changed, 587 insertions(+), 99 deletions(-) diff --git a/litellm-rust/crates/inference-responses/src/ids.rs b/litellm-rust/crates/inference-responses/src/ids.rs index b51aea1559e..fbba8f949a0 100644 --- a/litellm-rust/crates/inference-responses/src/ids.rs +++ b/litellm-rust/crates/inference-responses/src/ids.rs @@ -11,7 +11,7 @@ //! | Id | Applied to | Decoded plaintext | //! |---|---|---| //! | `resp_` | the response `id` | `litellm:custom_llm_provider:{provider};model_id:{model_id};response_id:{id}` | -//! | `encitem_` | the `id` of an output item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` | +//! | `encitem_` | the `id` of a reasoning or compaction output item, or of any item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` | //! | `litellm_enc:;{content}` | that item's `encrypted_content` | `` is `model_id:{model_id}`, and the original content follows the `;` unencoded | //! | `cntr_` | `container_id` on output items, `code_interpreter_call`, and message annotations | `litellm:custom_llm_provider:{provider};model_id:{model_id};container_id:{id}` | //! @@ -25,8 +25,9 @@ //! - `encitem_` and `litellm_enc:` carry encrypted-content affinity: encrypted reasoning only //! decrypts under the API key that produced it, so the router pins replays to that deployment. //! They are stamped only when the router sets `encrypted_content_affinity_enabled` and a -//! deployment `model_id` is known (see [`IdScope`]). Items without an `id`, such as Codex's, -//! carry the deployment in the wrapped content alone. +//! deployment `model_id` is known (see [`IdScope`]). Reasoning and compaction items get an +//! `encitem_` id by type, so it matches the one Python streams before their content arrives. +//! Items without an `id`, such as Codex's, carry the deployment in the wrapped content alone. //! - `cntr_` lets container calls find the provider and deployment that own a container. A //! container id that already names a provider is left as it is. //! - On the way in, [`restore_input`] turns `encitem_` ids and `litellm_enc:` content in the @@ -51,6 +52,7 @@ const RESPONSE_PREFIX: &str = "resp_"; const ITEM_PREFIX: &str = "encitem_"; const CONTAINER_PREFIX: &str = "cntr_"; const WRAPPED_CONTENT_PREFIX: &str = "litellm_enc:"; +const ENCRYPTED_ITEM_TYPES: [&str; 2] = ["reasoning", "compaction"]; /// Python's non-strict `base64.b64decode`: characters outside the alphabet are skipped and /// non-zero trailing bits are accepted @@ -211,15 +213,23 @@ pub fn decode_container_id(id: &str) -> DecodedContainerId { } fn encode_encrypted_item(item: &mut Map, model_id: &str) { - let Some(Value::String(content)) = item.get_mut("encrypted_content") else { - return; + let wrapped = match item.get_mut("encrypted_content") { + Some(Value::String(content)) if !content.is_empty() => { + *content = wrap_encrypted_content(content, model_id); + true + } + _ => false, }; - if content.is_empty() { + let encrypted_type = item + .get("type") + .and_then(Value::as_str) + .is_some_and(|item_type| ENCRYPTED_ITEM_TYPES.contains(&item_type)); + if !wrapped && !encrypted_type { return; } - *content = wrap_encrypted_content(content, model_id); if let Some(Value::String(id)) = item.get_mut("id") && !id.is_empty() + && decode_encrypted_item_id(id).is_none() { *id = encrypted_item_id(model_id, id); } @@ -456,6 +466,33 @@ mod tests { ); } + #[rstest] + #[case::reasoning_without_ciphertext( + json!({"type": "reasoning", "id": "rs_0a1b", "summary": []}), + json!({"type": "reasoning", "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", "summary": []}) + )] + #[case::compaction( + json!({"type": "compaction", "id": "cmp_0a1b", "encrypted_content": "gAAAA"}), + json!({ + "type": "compaction", + "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOmNtcF8wYTFi", + "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA" + }) + )] + #[case::already_encoded( + json!({"type": "reasoning", "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", "summary": []}), + json!({"type": "reasoning", "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", "summary": []}) + )] + #[case::message( + json!({"type": "message", "id": "msg_1", "content": []}), + json!({"type": "message", "id": "msg_1", "content": []}) + )] + fn encodes_item_ids_by_type_under_affinity(#[case] item: Value, #[case] expected: Value) { + let mut encoded = response("resp_1", json!([item])); + encode_response(&mut encoded, &scope(Some("openai"), Some("deploy-1"), true)); + assert_eq!(Value::Array(encoded.output), json!([expected])); + } + #[rstest] #[case::affinity_disabled(scope(Some("openai"), Some("deploy-1"), false))] #[case::no_model_id(scope(Some("openai"), None, true))] diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 489ab8a317d..ea02e4830d2 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -187,7 +187,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.request_input: str | ResponseInputParam = request_input self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request self.custom_llm_provider: str | None = custom_llm_provider - self.litellm_metadata: dict | None = litellm_metadata or {} + self.litellm_metadata: dict[str, object] | None = litellm_metadata or {} self.tool_call_provider_fields: Final = tool_call_provider_fields # Store lightweight dict snapshots for stream_chunk_builder to reduce # repeated Pydantic attribute access in end-of-stream assembly. @@ -1331,7 +1331,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _stamp_sequence_number(self, event: BaseLiteLLMOpenAIResponseObject) -> BaseLiteLLMOpenAIResponseObject: stamped: Final = event.model_copy(update={"sequence_number": self._sequence_number}) self._sequence_number += 1 - return stamped + affinity_model_id: Final = ResponsesAPIRequestUtils.affinity_model_id(self.litellm_metadata) + return ( + stamped + if affinity_model_id is None + else ResponsesAPIRequestUtils.encode_stream_event_for_affinity(stamped, affinity_model_id) + ) async def __anext__( self, diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 347537277df..dca93436efa 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -498,11 +498,17 @@ class BaseResponsesAPIStreamingIterator: "type": ResponsesAPIStreamEvents.ERROR.value, } ) - openai_responses_api_chunk: Final = self.responses_api_provider_config.transform_streaming_response( + transformed_chunk: Final = self.responses_api_provider_config.transform_streaming_response( model=self.model, parsed_chunk=typed_chunk, logging_obj=self.logging_obj, ) + affinity_model_id: Final = ResponsesAPIRequestUtils.affinity_model_id(self.litellm_metadata) + openai_responses_api_chunk: Final = ( + transformed_chunk + if affinity_model_id is None + else ResponsesAPIRequestUtils.encode_stream_event_for_affinity(transformed_chunk, affinity_model_id) + ) # Only when the SSE JSON carries a response body (delta events do not). # Using getattr(..., "response") alone is unsafe with Mocks: they synthesize a @@ -571,27 +577,6 @@ class BaseResponsesAPIStreamingIterator: _stream_model_id, ) - # Wrap encrypted_content in streaming events (output_item.added, output_item.done) - if self.litellm_metadata and self.litellm_metadata.get("encrypted_content_affinity_enabled"): - openai_types = get_openai_response_types() - event_type: Final = getattr(openai_responses_api_chunk, "type", None) - if event_type in ( - openai_types.ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, - openai_types.ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, - ): - item: Final[object | None] = getattr(openai_responses_api_chunk, "item", None) - if item: - encrypted_content: Final = getattr(item, "encrypted_content", None) - if encrypted_content and isinstance(encrypted_content, str): - model_id: Final = _model_id_from_metadata(self.litellm_metadata) - if model_id: - wrapped_content: Final = ( - ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( - encrypted_content, model_id - ) - ) - setattr(item, "encrypted_content", wrapped_content) - # Store the completed response (also for incomplete/failed so logging still fires) _chunk_type: Final = getattr(openai_responses_api_chunk, "type", None) openai_types = get_openai_response_types() @@ -2493,9 +2478,6 @@ class _ResponsesWsTurn: ) -_RESPONSES_WS_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) - - def _ws_event_error(event: Mapping[str, object]) -> object: if event.get("type") == "error": return _error_event_error_object(event) @@ -2535,24 +2517,6 @@ def _restore_wrapped_ids_in_response_create(msg_obj: Mapping[str, object]) -> di return {**msg_obj, **top_fields, **restored_nested} -def _wrap_output_item_encrypted_content( - event_obj: Mapping[str, object], litellm_metadata: Mapping[str, object] -) -> dict[str, object] | None: - if not litellm_metadata.get("encrypted_content_affinity_enabled"): - return None - model_id: Final = _model_id_from_metadata(litellm_metadata) - item: Final = event_obj.get("item") - if model_id is None or not _is_json_object(item): - return None - encrypted_content: Final = item.get("encrypted_content") - if not isinstance(encrypted_content, str) or not encrypted_content: - return None - wrapped_content: Final = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( - encrypted_content=encrypted_content, model_id=model_id - ) - return {**event_obj, "item": {**item, "encrypted_content": wrapped_content}} - - class ResponsesWebSocketStreaming: """ Manages bidirectional WebSocket forwarding for the Responses API @@ -2758,9 +2722,12 @@ class ResponsesWebSocketStreaming: litellm_metadata=self.litellm_metadata, ) return json.dumps({**event_obj, "response": wrapped_response}) - if event_obj.get("type") not in _RESPONSES_WS_OUTPUT_ITEM_EVENT_TYPES: - return response_str - wrapped_event: Final = _wrap_output_item_encrypted_content(event_obj, self.litellm_metadata) + affinity_model_id: Final = ResponsesAPIRequestUtils.affinity_model_id(self.litellm_metadata) + wrapped_event: Final = ( + None + if affinity_model_id is None + else ResponsesAPIRequestUtils.affinity_encoded_stream_event(event_obj, affinity_model_id) + ) return response_str if wrapped_event is None else json.dumps(wrapped_event) async def backend_to_client(self) -> None: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 991cf6213c6..dd3bd1c7e35 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -13,7 +13,13 @@ from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire pay import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, decoded_json_object, iter_items +from litellm.litellm_core_utils.core_helpers import ( + as_object_list, + as_str_mapping, + attribute_or_none, + decoded_json_object, + iter_items, +) from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.responses.transformation import ( @@ -51,6 +57,19 @@ _PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None] ] = TypeAdapter(ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None) _REASONING_FIELDS_ONLY_ITS_ORIGIN_RESOLVES: Final = frozenset({"encrypted_content", "id", "status"}) +_ENCRYPTED_OUTPUT_ITEM_TYPES: Final = frozenset({"reasoning", "compaction"}) +_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) +_REASONING_ITEM_EVENT_TYPES: Final = frozenset( + { + "response.reasoning_summary_part.added", + "response.reasoning_summary_part.done", + "response.reasoning_summary_text.delta", + "response.reasoning_summary_text.done", + "response.reasoning_text.delta", + "response.reasoning_text.done", + } +) +_CONTENT_PART_EVENT_TYPES: Final = frozenset({"response.content_part.added", "response.content_part.done"}) class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object): @@ -133,6 +152,7 @@ def _as_input_text_part(part: object) -> object: _RequestInputT: Final = TypeVar("_RequestInputT") +_StreamEventT: Final = TypeVar("_StreamEventT", bound=BaseModel) def _provider_mapped_params( @@ -521,6 +541,95 @@ class ResponsesAPIRequestUtils: _decode_encrypted_item_id = decode_encrypted_item_id + @staticmethod + def affinity_model_id(litellm_metadata: Mapping[str, object] | None) -> str | None: + if not litellm_metadata or not litellm_metadata.get("encrypted_content_affinity_enabled"): + return None + model_info: Final = litellm_metadata.get("model_info") + model_id: Final = model_info.get("id") if _is_object_dict(model_info) else None + return model_id if isinstance(model_id, str) and model_id else None + + @staticmethod + def affinity_item_id(item_type: object, item_id: object, encrypted_content: object, model_id: str) -> str | None: + """The ``encitem_`` id an output item carries under encrypted-content affinity, or ``None`` to keep its id. + + Reasoning and compaction items qualify by type, so a stream encodes them before their + ``encrypted_content`` arrives and every event agrees with the final response. + """ + if not isinstance(item_id, str) or not item_id: + return None + if ResponsesAPIRequestUtils.decode_encrypted_item_id(item_id) is not None: + return None + encrypted_type: Final = isinstance(item_type, str) and item_type in _ENCRYPTED_OUTPUT_ITEM_TYPES + if not encrypted_type and not (isinstance(encrypted_content, str) and encrypted_content): + return None + return ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, item_id) + + @staticmethod + def _reasoning_event_item_id(event_type: object, part_type: object, item_id: object, model_id: str) -> str | None: + if not isinstance(event_type, str): + return None + scoped_to_reasoning: Final = event_type in _REASONING_ITEM_EVENT_TYPES or ( + event_type in _CONTENT_PART_EVENT_TYPES and part_type == "reasoning_text" + ) + if not scoped_to_reasoning: + return None + return ResponsesAPIRequestUtils.affinity_item_id("reasoning", item_id, None, model_id) + + @staticmethod + def _affinity_item_updates( + item_type: object, item_id: object, encrypted_content: object, model_id: str + ) -> Mapping[str, str]: + encoded_id: Final = ResponsesAPIRequestUtils.affinity_item_id(item_type, item_id, encrypted_content, model_id) + wrapped: Final = ( + ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id(encrypted_content, model_id) + if isinstance(encrypted_content, str) and encrypted_content + else None + ) + return MappingProxyType( + {key: value for key, value in (("id", encoded_id), ("encrypted_content", wrapped)) if value is not None} + ) + + @staticmethod + def encode_stream_event_for_affinity(event: _StreamEventT, model_id: str) -> _StreamEventT: + """The stream event with the item id and wrapped ``encrypted_content`` the final response carries.""" + event_type: Final = attribute_or_none(event, "type") + if isinstance(event_type, str) and event_type in _OUTPUT_ITEM_EVENT_TYPES: + item: Final = attribute_or_none(event, "item") + if not isinstance(item, BaseModel): + return event + item_updates: Final = ResponsesAPIRequestUtils._affinity_item_updates( + attribute_or_none(item, "type"), + attribute_or_none(item, "id"), + attribute_or_none(item, "encrypted_content"), + model_id, + ) + return event.model_copy(update={"item": item.model_copy(update=item_updates)}) if item_updates else event + part: Final = attribute_or_none(event, "part") + encoded_item_id: Final = ResponsesAPIRequestUtils._reasoning_event_item_id( + event_type, + part.get("type") if _is_object_dict(part) else attribute_or_none(part, "type"), + attribute_or_none(event, "item_id"), + model_id, + ) + return event if encoded_item_id is None else event.model_copy(update={"item_id": encoded_item_id}) + + @staticmethod + def affinity_encoded_stream_event(event: Mapping[str, object], model_id: str) -> dict[str, object] | None: + """The JSON stream event with the ids and wrapped content the final response carries, or ``None``.""" + event_type: Final = event.get("type") + item: Final = event.get("item") + if isinstance(event_type, str) and event_type in _OUTPUT_ITEM_EVENT_TYPES and _is_object_dict(item): + item_updates: Final = ResponsesAPIRequestUtils._affinity_item_updates( + item.get("type"), item.get("id"), item.get("encrypted_content"), model_id + ) + return {**event, "item": {**item, **item_updates}} if item_updates else None + part: Final = event.get("part") + encoded_item_id: Final = ResponsesAPIRequestUtils._reasoning_event_item_id( + event_type, part.get("type") if _is_object_dict(part) else None, event.get("item_id"), model_id + ) + return None if encoded_item_id is None else {**event, "item_id": encoded_item_id} + @staticmethod def wrap_encrypted_content_with_model_id(encrypted_content: str, model_id: str) -> str: """Wrap encrypted_content with model_id metadata for affinity routing. @@ -576,7 +685,7 @@ class ResponsesAPIRequestUtils: response: Union["ResponsesAPIResponse", dict[str, object]], model_id: str | None, ) -> Union["ResponsesAPIResponse", dict[str, object]]: - """Rewrite item IDs for output items that contain ``encrypted_content``. + """Rewrite item IDs for reasoning, compaction and other output items that contain ``encrypted_content``. Encodes ``model_id`` into the item ID so that follow-up requests can be routed back to the originating deployment without any cache lookup. @@ -593,40 +702,28 @@ class ResponsesAPIRequestUtils: else: output = getattr(response, "output", None) - if not isinstance(output, list): + items: Final = as_object_list(output) + if items is None: return response - for item in output: - if isinstance(item, dict): - item_id = item.get("id") - encrypted_content = item.get("encrypted_content") - - if encrypted_content and isinstance(encrypted_content, str): - # Always wrap encrypted_content with model_id for redundancy - item["encrypted_content"] = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( - encrypted_content, model_id + for item in items: + if _is_object_dict(item): + item.update( + ResponsesAPIRequestUtils._affinity_item_updates( + item.get("type"), item.get("id"), item.get("encrypted_content"), model_id ) - # Also encode the ID if present - if item_id and isinstance(item_id, str): - item["id"] = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, item_id) - else: - item_id = getattr(item, "id", None) - encrypted_content = getattr(item, "encrypted_content", None) - - if encrypted_content and isinstance(encrypted_content, str): - # Always wrap encrypted_content with model_id for redundancy - try: - item.encrypted_content = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( - encrypted_content, model_id - ) - except AttributeError: - pass - # Also encode the ID if present - if item_id and isinstance(item_id, str): - try: - item.id = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, item_id) - except AttributeError: - pass + ) + continue + for key, value in ResponsesAPIRequestUtils._affinity_item_updates( + attribute_or_none(item, "type"), + attribute_or_none(item, "id"), + attribute_or_none(item, "encrypted_content"), + model_id, + ).items(): + try: + setattr(item, key, value) + except AttributeError: + pass return response diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 1a6e3cc7832..0dce041cce8 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -81,14 +81,14 @@ class _FakeStreamWrapper: return self._chunks.pop(0) -def _build_iterator(chunks) -> LiteLLMCompletionStreamingIterator: +def _build_iterator(chunks, litellm_metadata: dict[str, object] | None = None) -> LiteLLMCompletionStreamingIterator: return LiteLLMCompletionStreamingIterator( model="claude-haiku-4-5", litellm_custom_stream_wrapper=_FakeStreamWrapper(chunks), request_input="What is the weather in San Francisco?", responses_api_request={}, custom_llm_provider="anthropic", - litellm_metadata={}, + litellm_metadata=litellm_metadata or {}, ) @@ -1774,6 +1774,56 @@ async def test_completed_output_follows_the_streamed_output_indexes(sync_mode: b assert [item.id for item in completed.response.output] == [event.item.id for event in added] +_REASONING_ITEM_EVENT_TYPES: Final = frozenset( + { + ResponsesAPIStreamEvents.RESPONSE_PART_ADDED, + ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, + ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DONE, + ResponsesAPIStreamEvents.REASONING_SUMMARY_PART_DONE, + } +) + + +def _reasoning_item_ids(events: list[BaseLiteLLMOpenAIResponseObject]) -> set[str]: + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + streamed_items: Final = { + event.item.id for event in events if getattr(getattr(event, "item", None), "type", None) == "reasoning" + } + scoped_events: Final = { + event.item_id for event in events if getattr(event, "type", None) in _REASONING_ITEM_EVENT_TYPES + } + completed_items: Final = {item.id for item in completed.response.output if item.type == "reasoning"} + return streamed_items | scoped_events | completed_items + + +@pytest.mark.parametrize( + "stream_chunks", + [_reasoning_then_text_chunks, _signed_thinking_and_text_in_one_chunk], + ids=["reasoning_then_text_chunks", "signed_thinking_and_text_in_one_chunk"], +) +@pytest.mark.parametrize( + "litellm_metadata, origin", + [({"encrypted_content_affinity_enabled": True, "model_info": {"id": "dep-1"}}, "dep-1"), ({}, None)], + ids=["affinity-on", "affinity-off"], +) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_reasoning_item_keeps_one_id_across_bridged_events_and_completed_output( + stream_chunks: StreamChunks, litellm_metadata: dict[str, object], origin: str | None, sync_mode: bool +): + events: Final = await _collect_events(_build_iterator(stream_chunks(), litellm_metadata), sync_mode) + + reasoning_ids: Final = _reasoning_item_ids(events) + assert len(reasoning_ids) == 1 + decoded: Final = ResponsesAPIRequestUtils.decode_encrypted_item_id(next(iter(reasoning_ids))) + assert (decoded or {}).get("model_id") == origin + message_ids: Final = {event.item.id for event in events if _is_message_item(event)} + assert message_ids + assert [ResponsesAPIRequestUtils.decode_encrypted_item_id(item_id) for item_id in message_ids] == [None] + + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_reasoning_only_stream_lists_no_message_item_it_never_announced(sync_mode: bool): diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index 9a73054b2e3..fe4ee0c4bbe 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -4692,6 +4692,22 @@ class TestNativeWebSocketEncryptedContentAffinity: backend_ws = MagicMock() backend_ws.recv = AsyncMock( side_effect=[ + json.dumps( + { + "type": "response.output_item.added", + "output_index": 0, + "item": {"type": "reasoning", "id": "rs_1", "summary": []}, + } + ), + json.dumps( + { + "type": "response.reasoning_summary_text.delta", + "item_id": "rs_1", + "output_index": 0, + "summary_index": 0, + "delta": "thinking", + } + ), json.dumps({"type": "response.output_item.done", "output_index": 0, "item": dict(reasoning_item)}), json.dumps( { @@ -4716,20 +4732,65 @@ class TestNativeWebSocketEncryptedContentAffinity: await handler.backend_to_client() wrapped_content = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("gAAAA-blob", "dep-1") - item_done = json.loads(websocket.send_text.await_args_list[0][0][0]) + added, summary_delta, item_done, completed = ( + json.loads(call[0][0]) for call in websocket.send_text.await_args_list[:4] + ) assert item_done["item"]["encrypted_content"] == wrapped_content - completed = json.loads(websocket.send_text.await_args_list[1][0][0]) assert completed["response"]["id"] == ResponsesAPIRequestUtils._build_responses_api_response_id( custom_llm_provider="openai", model_id="dep-1", response_id="resp_1" ) - assert completed["response"]["output"][0]["id"] == ResponsesAPIRequestUtils._build_encrypted_item_id( - "dep-1", "rs_1" - ) + assert { + added["item"]["id"], + summary_delta["item_id"], + item_done["item"]["id"], + completed["response"]["output"][0]["id"], + } == {ResponsesAPIRequestUtils._build_encrypted_item_id("dep-1", "rs_1")} assert completed["response"]["output"][0]["encrypted_content"] == wrapped_content await asyncio.sleep(0) logged = logging_obj.dispatch_success_handlers.await_args[0][0] assert logged[0]["response"]["id"] == completed["response"]["id"] + @pytest.mark.asyncio + @pytest.mark.parametrize( + "frame", + [ + {"note": "a frame without a type"}, + {"type": "response.output_text.delta", "item_id": "msg_1", "output_index": 1, "delta": "ok"}, + { + "type": "response.content_part.added", + "item_id": "msg_1", + "output_index": 1, + "content_index": 0, + "part": {"type": "output_text", "text": ""}, + }, + {"type": "response.output_item.done", "output_index": 1, "item": {"type": "message", "id": "msg_1"}}, + { + "type": "response.output_item.done", + "output_index": 0, + "item": {"type": "reasoning", "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXAtMTtpdGVtX2lkOnJzXzE="}, + }, + ], + ids=["no-type", "message-delta", "message-content-part", "message-item", "already-encoded-reasoning"], + ) + async def test_backend_to_client_forwards_events_affinity_does_not_own_unchanged(self, frame: dict[str, object]): + import websockets.exceptions # noqa: F401 (lazy submodule must be importable) + + raw_frame: Final = json.dumps(frame) + websocket: Final = MagicMock() + websocket.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.recv = AsyncMock(side_effect=[raw_frame, Exception("stop")]) + handler: Final = _make_streaming( + websocket=websocket, + backend_ws=backend_ws, + request_data={"litellm_metadata": dict(_AFFINITY_METADATA)}, + custom_llm_provider="openai", + ) + + await handler.backend_to_client() + + assert websocket.send_text.await_args_list[0][0][0] == raw_frame + @pytest.mark.asyncio async def test_backend_to_client_wraps_only_response_id_without_affinity(self): from unittest.mock import AsyncMock diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index ec14f67ce17..7fa5d6bfa80 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest import websockets.exceptions +from pydantic import BaseModel, TypeAdapter from pydantic_core import PydanticSerializationError import litellm @@ -43,6 +44,7 @@ from litellm.types.llms.openai import ( ) from contextlib import suppress from litellm.responses import streaming_iterator as streaming_module +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.utils import CallTypes from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from types import SimpleNamespace @@ -86,6 +88,7 @@ def _make_iterator( trailing_error: Optional[Exception] = None, config: Mock | None = None, request_data: dict | None = None, + litellm_metadata: dict[str, object] | None = None, ) -> ResponsesAPIStreamingIterator: async def aiter_bytes(): for evt in sse_events: @@ -102,7 +105,7 @@ def _make_iterator( model="gpt-4o-mini", responses_api_provider_config=config or _mock_config(), logging_obj=logging_obj, - litellm_metadata={}, + litellm_metadata=litellm_metadata or {}, custom_llm_provider="openai", request_data=request_data, ) @@ -561,6 +564,193 @@ async def test_event_without_a_type_reaches_the_caller_instead_of_ending_the_str assert untyped.model_extra == {"note": "hook output without a type"} +_REASONING_ITEM_ID: Final = "rs_affinity" +_MESSAGE_ITEM_ID: Final = "msg_affinity" +_STREAMED_REASONING_ITEM: Final = { + "id": _REASONING_ITEM_ID, + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "thinking"}], + "encrypted_content": "ciphertext", +} +_STREAMED_MESSAGE_ITEM: Final = { + "id": _MESSAGE_ITEM_ID, + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], +} +_REASONING_SCOPED_EVENT_TYPES: Final = ( + ResponsesAPIStreamEvents.CONTENT_PART_ADDED, + "response.reasoning_text.delta", + ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, +) + + +def _reasoning_response_body(status: str, output: list[dict[str, object]]) -> dict[str, object]: + return { + "id": "resp_affinity", + "object": "response", + "created_at": 1, + "status": status, + "model": "gpt-5-mini", + "output": output, + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + + +_REASONING_STREAM_EVENTS: Final = [ + _sse_event( + {"type": "response.created", "sequence_number": 0, "response": _reasoning_response_body("in_progress", [])} + ), + _sse_event( + { + "type": "response.output_item.added", + "sequence_number": 1, + "output_index": 0, + "item": {"id": _REASONING_ITEM_ID, "type": "reasoning", "summary": []}, + } + ), + _sse_event( + { + "type": "response.content_part.added", + "sequence_number": 2, + "item_id": _REASONING_ITEM_ID, + "output_index": 0, + "content_index": 0, + "part": {"type": "reasoning_text", "text": ""}, + } + ), + _sse_event( + { + "type": "response.reasoning_text.delta", + "sequence_number": 3, + "item_id": _REASONING_ITEM_ID, + "output_index": 0, + "content_index": 0, + "delta": "thinking", + } + ), + _sse_event( + { + "type": "response.reasoning_summary_text.delta", + "sequence_number": 4, + "item_id": _REASONING_ITEM_ID, + "output_index": 0, + "summary_index": 0, + "delta": "thinking", + } + ), + _sse_event( + {"type": "response.output_item.done", "sequence_number": 5, "output_index": 0, "item": _STREAMED_REASONING_ITEM} + ), + _sse_event( + { + "type": "response.output_text.delta", + "sequence_number": 6, + "item_id": _MESSAGE_ITEM_ID, + "output_index": 1, + "content_index": 0, + "delta": "ok", + } + ), + _sse_event( + {"type": "response.output_item.done", "sequence_number": 7, "output_index": 1, "item": _STREAMED_MESSAGE_ITEM} + ), + _sse_event( + { + "type": "response.completed", + "sequence_number": 8, + "response": _reasoning_response_body("completed", [_STREAMED_REASONING_ITEM, _STREAMED_MESSAGE_ITEM]), + } + ), +] + + +_ITEM_RECORD: Final = TypeAdapter(dict[str, object]) + + +def _item_record(item: object) -> dict[str, object]: + return _ITEM_RECORD.validate_python(item.model_dump(exclude_none=True) if isinstance(item, BaseModel) else item) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "litellm_metadata, origin", + [ + ({"encrypted_content_affinity_enabled": True, "model_info": {"id": "dep-1"}}, "dep-1"), + ({"encrypted_content_affinity_enabled": False, "model_info": {"id": "dep-1"}}, None), + ({"encrypted_content_affinity_enabled": True}, None), + ], + ids=["affinity-on", "affinity-off", "no-model-id"], +) +async def test_streamed_reasoning_item_keeps_one_id_and_replays_as_one_input_item( + litellm_metadata: dict[str, object], origin: str | None +): + iterator: Final = _make_iterator( + sse_events=_REASONING_STREAM_EVENTS, + logging_obj=_logging_obj_stub(), + config=OpenAIResponsesAPIConfig(), + litellm_metadata=litellm_metadata, + ) + + events: Final = [event async for event in iterator] + + item_events: Final = [ + _item_record(event.item) + for event in events + if event.type in (ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE) + ] + completed: Final = next( + event.response for event in events if event.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + output_items: Final = [_item_record(item) for item in completed.output] + scoped_item_ids: Final = {event.type: event.item_id for event in events if getattr(event, "item_id", None)} + expected_reasoning_id: Final = ( + _REASONING_ITEM_ID + if origin is None + else ResponsesAPIRequestUtils._build_encrypted_item_id(origin, _REASONING_ITEM_ID) + ) + assert {item["id"] for item in [*item_events, *output_items] if item["type"] == "reasoning"} | { + scoped_item_ids[event_type] for event_type in _REASONING_SCOPED_EVENT_TYPES + } == {expected_reasoning_id} + assert {item["id"] for item in [*item_events, *output_items] if item["type"] == "message"} | { + scoped_item_ids[ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA] + } == {_MESSAGE_ITEM_ID} + + done_reasoning: Final = next(item for item in reversed(item_events) if item["type"] == "reasoning") + completed_reasoning: Final = next(item for item in output_items if item["type"] == "reasoning") + assert done_reasoning["encrypted_content"] == completed_reasoning["encrypted_content"] + assert ResponsesAPIRequestUtils.unwrap_encrypted_content_with_model_id( + str(done_reasoning["encrypted_content"]) + ) == (origin, "ciphertext") + + replayed: Final = list( + { + record["id"]: {key: record[key] for key in ("id", "type", "encrypted_content")} + for record in (done_reasoning, completed_reasoning) + }.values() + ) + assert ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(replayed) == [ + {"id": _REASONING_ITEM_ID, "type": "reasoning", "encrypted_content": "ciphertext"} + ] + + +@pytest.mark.asyncio +async def test_output_item_added_without_an_item_streams_through_under_affinity(): + iterator: Final = _make_iterator( + sse_events=[ + _sse_event({"type": "response.output_item.added", "sequence_number": 0, "output_index": 0, "item": None}) + ], + logging_obj=_logging_obj_stub(), + config=OpenAIResponsesAPIConfig(), + litellm_metadata={"encrypted_content_affinity_enabled": True, "model_info": {"id": "dep-1"}}, + ) + + added: Final = await iterator.__anext__() + + assert (added.type, added.output_index, added.item) == (ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, 0, None) + + @pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS, ids=type) def test_sync_transport_error_before_any_output_raises_fallback_error(trailing_error): logging_obj: Final = _failure_tracking_logging_obj() diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index e908481ca2b..ae5acffab31 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -3,7 +3,7 @@ Tests for encrypted_content_affinity pre-call check. The mechanism works without any cache and supports two encoding strategies: -1. **Items with IDs**: item IDs for output items with `encrypted_content` are rewritten to +1. **Items with IDs**: IDs of reasoning, compaction and other `encrypted_content` output items are rewritten to `encitem_{base64("litellm:model_id:{model_id};item_id:{original_id}")}`. 2. **Items without IDs** (Codex): encrypted_content itself is wrapped with model_id metadata: @@ -18,10 +18,12 @@ The mechanism works without any cache and supports two encoding strategies: import datetime import time from collections.abc import Mapping +from dataclasses import dataclass from typing import Final from unittest.mock import AsyncMock, patch import pytest +from pydantic import TypeAdapter import litellm from litellm.integrations.custom_secret_manager import CustomSecretManager @@ -132,6 +134,13 @@ class TestEncryptedItemIdCodec: assert decoded["item_id"] == original_item_id +@dataclass(frozen=True, slots=True) +class _ReadOnlyReasoningItem: + id: str + type: str + encrypted_content: str + + class TestUpdateEncryptedContentItemIds: def test_rewrites_encrypted_items_in_dict_response(self): model_id = "deployment-1" @@ -157,6 +166,52 @@ class TestUpdateEncryptedContentItemIds: result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(response, None) assert result["output"][0]["id"] == "rs_xyz" + @pytest.mark.parametrize( + "response", + [ + {"id": "resp_failed", "status": "failed", "error": {"code": "server_error", "message": "boom"}}, + { + "id": "resp_failed", + "status": "failed", + "error": {"code": "server_error", "message": "boom"}, + "output": None, + }, + ], + ids=["output-absent", "output-none"], + ) + def test_passes_a_response_without_output_through_unchanged(self, response: dict[str, object]): + expected: Final = {**response} + result: Final = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response( + response, "deployment-1" + ) + assert result == expected + + def test_leaves_read_only_output_items_as_returned(self): + result: Final = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response( + {"id": "resp_123", "output": [_ReadOnlyReasoningItem("rs_xyz", "reasoning", "secret")]}, "deployment-1" + ) + assert result["output"] == [_ReadOnlyReasoningItem("rs_xyz", "reasoning", "secret")] + + @pytest.mark.parametrize( + "item, encoded", + [ + ({"id": "rs_xyz", "type": "reasoning", "summary": []}, True), + ({"id": "cmp_xyz", "type": "compaction", "encrypted_content": "secret"}, True), + ({"id": "ws_xyz", "type": "web_search_call"}, False), + ({"id": ResponsesAPIRequestUtils._build_encrypted_item_id("dep-0", "rs_xyz"), "type": "reasoning"}, False), + ], + ids=["reasoning-before-ciphertext", "compaction", "non-reasoning", "already-encoded"], + ) + def test_encodes_reasoning_and_compaction_ids_by_type(self, item: dict[str, object], encoded: bool): + result: Final = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response( + {"id": "resp_123", "output": [dict(item)]}, "deployment-1" + ) + assert result["output"][0]["id"] == ( + ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-1", str(item["id"])) + if encoded + else item["id"] + ) + class TestEncryptedContentWrapping: def test_wrap_and_unwrap_encrypted_content(self): @@ -219,10 +274,10 @@ class TestRestoreEncryptedContentItemIds: request_input = [ {"type": "message", "id": "msg_abc123", "role": "assistant"}, {"type": "reasoning", "id": encoded_id, "encrypted_content": "secret"}, + {"type": "item_reference", "id": encoded_id}, ] restored = ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(request_input) - assert restored[0]["id"] == "msg_abc123" - assert restored[1]["id"] == original_id + assert [item["id"] for item in restored] == ["msg_abc123", original_id, original_id] def test_unwraps_encrypted_content(self): """Test that wrapped encrypted_content is unwrapped before forwarding.""" @@ -715,6 +770,32 @@ async def test_encrypted_content_affinity_does_not_create_litellm_metadata_for_c assert len(result) == 1 +@pytest.mark.asyncio +async def test_item_reference_to_a_reasoning_item_returned_without_ciphertext_pins_its_deployment(): + returned: Final = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( + {"id": "resp_stored", "output": [{"type": "reasoning", "id": "rs_stored", "summary": []}]}, + "openai", + {"model_info": {"id": "dep-2"}, "encrypted_content_affinity_enabled": True}, + ) + returned_output: Final = TypeAdapter(list[dict[str, object]]).validate_python(returned["output"]) + deployments: Final = [ + {"model_info": {"id": "dep-1"}, "litellm_params": {"model": "openai/gpt-5.1-codex"}}, + {"model_info": {"id": "dep-2"}, "litellm_params": {"model": "openai/gpt-5.1-codex"}}, + ] + + pinned: Final = await EncryptedContentAffinityCheck().async_filter_deployments( + model="gpt-5.1-codex", + healthy_deployments=deployments, + messages=None, + request_kwargs={ + "input": [{"type": "item_reference", "id": returned_output[0]["id"]}], + "litellm_metadata": {}, + }, + ) + + assert pinned == [deployments[1]] + + @pytest.mark.asyncio async def test_encrypted_content_affinity_preserves_litellm_metadata_for_responses(): """ From 43daa2598e67a97253d7fcef2536aec6129030ea Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Wed, 11 Nov 2026 15:55:27 -0800 Subject: [PATCH 129/189] feat(guardrails): write guardrail changes to the audit log (#51118) * feat(guardrails): write guardrail changes to the audit log Creating, updating, patching or deleting a guardrail, and registering, approving or rejecting a team guardrail submission, now each write one row to LiteLLM_AuditLog with the guardrail before and after the change. Credentials the guardrails table encrypts at rest are redacted from those rows, and the routes take the litellm-changed-by header like key, team and model routes do. The Admin UI audit page gets a Guardrails table filter. Resolves LIT-10639 * fix(guardrails): audit an allowlisted guardrail snapshot and drop litellm-changed-by Guardrail audit rows now keep the guardrail id, name, info, status, team and the provider, mode and default_on settings, and name (never show) any other provider setting an update changed. Provider settings can hold credentials under any name, for example an apikey header, so no other value is written. The changer always comes from the caller's API key, so the litellm-changed-by header is gone. PUT audits the saved guardrail instead of the copy the live sync adds headers to. Resolves LIT-10639 * fix(guardrails): never let the audit snapshot fail a saved guardrail change The audit snapshot now reads guardrail settings stored as a JSON string and records empty settings when it cannot read them, so a saved approve or reject can no longer turn into a 500. Only the provider, mode and default_on settings are ever copied, so this cannot leak a credential. The create entry also records the active status the new guardrail is saved with. Resolves LIT-10639 * test(guardrails): run the guardrail audit tests through the real registry and in-memory handler The audit tests no longer replace GUARDRAIL_REGISTRY or the in-memory handler, and no test edits its inputs. Only the Prisma guardrails table is faked, returning exactly what the registry writes, so the PUT test now checks the real live sync adding x-api-key to the saved guardrail's headers. Resolves LIT-10639 --- litellm/proxy/_types.py | 1 + .../proxy/guardrails/guardrail_endpoints.py | 133 +++++++- .../guardrails/test_guardrail_endpoints.py | 292 +++++++++++++++++- .../components/logs/audit/AuditLogsTable.tsx | 1 + .../logs/audit/AuditLogsTableColumns.tsx | 1 + ui/litellm-dashboard/src/i18n/locales/en.json | 3 +- 6 files changed, 421 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 80819542d68..1342ae6bd84 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -211,6 +211,7 @@ class LitellmTableNames(str, enum.Enum): SSO_CONFIG_TABLE_NAME = "LiteLLM_SSOConfig" UI_SETTINGS_TABLE_NAME = "LiteLLM_UISettings" AGENT_TABLE_NAME = "LiteLLM_AgentsTable" + GUARDRAILS_TABLE_NAME = "LiteLLM_GuardrailsTable" from litellm.types.integrations.slack_alerting import ( diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index db4c30500a7..eaece1fe4c6 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -3,6 +3,7 @@ CRUD ENDPOINTS FOR GUARDRAILS """ import asyncio +import copy import inspect import json import os @@ -19,7 +20,7 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import AUDIT_ACTIONS, LitellmTableNames, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception @@ -53,6 +54,7 @@ from litellm.proxy.management_endpoints.common_utils import ( # noqa: F401 # l _user_has_admin_view, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export user_api_key_has_admin_view, ) +from litellm.proxy.management_helpers.audit_logs import create_object_audit_log, is_audit_logging_enabled from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import GuardrailsRepository from litellm.types.guardrails import ( @@ -112,6 +114,84 @@ def _reject_encrypted_litellm_params(litellm_params: object) -> None: ) +_AUDIT_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_AUDITED_GUARDRAIL_FIELDS: Final = ("guardrail_id", "guardrail_name", "guardrail_info", "status", "team_id") +_AUDITED_LITELLM_PARAMS: Final = ("guardrail", "mode", "default_on") +_NEW_GUARDRAIL_STATUS: Final = "active" + + +def _as_audit_mapping(value: object) -> Mapping[str, object]: + """Rows can hold JSON columns as strings, and an audit snapshot must never fail a change that is already saved.""" + candidate: Final = ( + value.model_dump(exclude_none=True) + if isinstance(value, BaseModel) + else _parse_json_field(value) + if isinstance(value, str) + else value + ) + try: + return _AUDIT_JSON_OBJECT.validate_python(candidate or {}) + except ValidationError: + return {} + + +def _litellm_params_of(guardrail: object) -> Mapping[str, object]: + return _as_audit_mapping(_as_audit_mapping(guardrail).get("litellm_params")) + + +def _guardrail_audit_snapshot(guardrail: object) -> Mapping[str, object]: + """A guardrail as written to the audit log. Provider settings can hold credentials under any name, so only the + provider, mode and default_on values are kept.""" + fields: Final = _as_audit_mapping(guardrail) + params: Final = _litellm_params_of(guardrail) + return { + **{key: fields[key] for key in _AUDITED_GUARDRAIL_FIELDS if fields.get(key) is not None}, + "litellm_params": {key: params[key] for key in _AUDITED_LITELLM_PARAMS if key in params}, + } + + +def _changed_litellm_param_names(before: object, after: object) -> tuple[str, ...]: + """Names, never values, of the provider settings an update changed beyond the ones the snapshot shows.""" + old: Final = _litellm_params_of(before) + new: Final = _litellm_params_of(after) + return tuple( + sorted( + key + for key in old.keys() | new.keys() + if key not in _AUDITED_LITELLM_PARAMS and old.get(key) != new.get(key) + ) + ) + + +async def _audit_guardrail_change( + *, + guardrail_id: str, + action: AUDIT_ACTIONS, + user_api_key_dict: UserAPIKeyAuth, + before: object = None, + after: object = None, +) -> None: + from litellm.proxy.proxy_server import litellm_proxy_admin_name + + if not is_audit_logging_enabled(): + return + changed: Final = _changed_litellm_param_names(before, after) if before is not None and after is not None else () + await create_object_audit_log( + object_id=guardrail_id, + action=action, + litellm_changed_by=None, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + table_name=LitellmTableNames.GUARDRAILS_TABLE_NAME, + before_value=None if before is None else safe_dumps(_guardrail_audit_snapshot(before)), + after_value=None + if after is None + else safe_dumps( + {**_guardrail_audit_snapshot(after), **({"changed_litellm_params": list(changed)} if changed else {})} + ), + ) + + def _guardrails_table(prisma_client: "PrismaClient") -> "TableActions[LiteLLM_GuardrailsTable]": return GuardrailsRepository(prisma_client).table @@ -479,6 +559,12 @@ async def create_guardrail( init_error, ) + await _audit_guardrail_change( + guardrail_id=str(guardrail_id), + action="created", + user_api_key_dict=user_api_key_dict, + after={**result, "status": _NEW_GUARDRAIL_STATUS}, + ) return result except HTTPException: raise @@ -579,6 +665,7 @@ async def update_guardrail( ) guardrail_name: Final = result.get("guardrail_name", "Unknown") + saved_guardrail: Final = copy.deepcopy(dict(result)) try: IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db( @@ -612,6 +699,13 @@ async def update_guardrail( update_error, ) + await _audit_guardrail_change( + guardrail_id=guardrail_id, + action="updated", + user_api_key_dict=user_api_key_dict, + before=existing_guardrail, + after=saved_guardrail, + ) return result except HTTPException as e: raise e @@ -687,6 +781,12 @@ async def delete_guardrail( delete_error, ) + await _audit_guardrail_change( + guardrail_id=guardrail_id, + action="deleted", + user_api_key_dict=user_api_key_dict, + before=existing_guardrail, + ) return result except HTTPException as e: raise e @@ -853,6 +953,12 @@ async def register_guardrail( "updated_at": now, }, ) + await _audit_guardrail_change( + guardrail_id=created.guardrail_id, + action="created", + user_api_key_dict=user_api_key_dict, + after=created, + ) return RegisterGuardrailResponse( guardrail_id=created.guardrail_id, guardrail_name=created.guardrail_name, @@ -1095,10 +1201,17 @@ async def approve_guardrail_submission( ) now: Final = datetime.now(timezone.utc) - await _guardrails_table(prisma_client).update( + approved: Final = await _guardrails_table(prisma_client).update( where={"guardrail_id": guardrail_id}, data={"status": "active", "reviewed_at": now, "updated_at": now}, ) + await _audit_guardrail_change( + guardrail_id=guardrail_id, + action="updated", + user_api_key_dict=user_api_key_dict, + before=row, + after=approved, + ) guardrail_info: Final = _parse_json_field(row.guardrail_info) if not litellm_params: @@ -1174,10 +1287,17 @@ async def reject_guardrail_submission( ) now: Final = datetime.now(timezone.utc) - await _guardrails_table(prisma_client).update( + rejected: Final = await _guardrails_table(prisma_client).update( where={"guardrail_id": guardrail_id}, data={"status": "rejected", "reviewed_at": now, "updated_at": now}, ) + await _audit_guardrail_change( + guardrail_id=guardrail_id, + action="updated", + user_api_key_dict=user_api_key_dict, + before=row, + after=rejected, + ) return { "guardrail_id": guardrail_id, "status": "rejected", @@ -1378,6 +1498,13 @@ async def patch_guardrail( update_error, ) + await _audit_guardrail_change( + guardrail_id=guardrail_id, + action="updated", + user_api_key_dict=user_api_key_dict, + before=existing_guardrail, + after=result, + ) return result except HTTPException as e: raise e diff --git a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py index f5e5cc32357..cb11acc99cb 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py @@ -1,10 +1,10 @@ import json import time -from collections.abc import Callable +from collections.abc import Awaitable, Callable, Mapping, Sequence from datetime import datetime from types import MappingProxyType -from typing import Dict, Final, List, Literal, Optional -from unittest.mock import AsyncMock +from typing import Final, List, Literal, Optional +from unittest.mock import AsyncMock, Mock import httpx import pytest @@ -48,9 +48,9 @@ from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_fil MOCK_ADMIN_USER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) from litellm.proxy.guardrails.guardrail_registry import ( - IN_MEMORY_GUARDRAIL_HANDLER, InMemoryGuardrailHandler, encrypt_guardrail_litellm_params, + guardrail_from_db_row, ) from litellm.types.guardrails import ( ApplyGuardrailRequest, @@ -1020,7 +1020,7 @@ def test_optional_params_not_returned_when_not_overridden(): """Test that optional_params is not returned when the config model doesn't override it""" from typing import Optional - from pydantic import BaseModel, Field + from pydantic import Field from litellm.proxy.guardrails.guardrail_endpoints import _get_fields_from_model from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel @@ -1080,7 +1080,7 @@ def test_optional_params_returned_when_properly_overridden(): @pytest.mark.asyncio async def test_bedrock_guardrail_prepare_request_with_api_key(): """Test _prepare_request method uses Bearer token when api_key is provided in data""" - from unittest.mock import Mock, patch + from unittest.mock import Mock from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockGuardrail, @@ -2029,6 +2029,286 @@ async def test_delete_guardrail_endpoint( assert "Failed to remove guardrail" in str(mock_logger.warning.call_args) +PLANTED_API_KEY: Final = "sk-planted-guardrail-api-key" +PLANTED_AUTH_HEADER: Final = "Bearer planted-guardrail-auth-header" +PLANTED_APIKEY_HEADER: Final = "planted-guardrail-apikey-header" +ROTATED_API_KEY: Final = "sk-planted-rotated-api-key" +PLANTED_SECRETS: Final = (PLANTED_API_KEY, ROTATED_API_KEY, PLANTED_AUTH_HEADER, PLANTED_APIKEY_HEADER, "litellm_enc::") +AUDIT_ADMIN: Final = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="security-admin", api_key="hashed-admin-key" +) +STORED_PARAMS: Final = MappingProxyType( + { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": "https://guardrails.example.com/validate", + "api_key": PLANTED_API_KEY, + "headers": {"Authorization": PLANTED_AUTH_HEADER, "apikey": PLANTED_APIKEY_HEADER, "X-Tenant": "acme"}, + } +) +STORED_GUARDRAIL_SNAPSHOT: Final = MappingProxyType( + { + "guardrail_id": "gr-audit", + "guardrail_name": "audited-guard", + "guardrail_info": {"description": "audited"}, + "status": "active", + "litellm_params": {"guardrail": "generic_guardrail_api", "mode": "pre_call", "default_on": False}, + } +) +SAVED_AT: Final = datetime(2026, 11, 11, 12, 0, 0) + + +def _requested_guardrail(**params: object) -> Guardrail: + return Guardrail( + guardrail_name="audited-guard", + litellm_params=LitellmParams(**{**STORED_PARAMS, **params}), + guardrail_info={"description": "audited"}, + ) + + +def _row_as_written(where: Mapping[str, object], data: Mapping[str, object]): + """The guardrails table row holding exactly what the registry wrote, as Prisma returns it.""" + from prisma.models import LiteLLM_GuardrailsTable + + return LiteLLM_GuardrailsTable( + **{"guardrail_id": where["guardrail_id"], "status": "active", "created_at": SAVED_AT, **data} + ) + + +def _stored_row(): + return _row_as_written( + {"guardrail_id": "gr-audit"}, + { + "guardrail_name": "audited-guard", + "litellm_params": json.dumps( + encrypt_guardrail_litellm_params(LitellmParams(**STORED_PARAMS).model_dump()), default=str + ), + "guardrail_info": json.dumps({"description": "audited"}), + "updated_at": SAVED_AT, + }, + ) + + +@pytest.fixture +def audit_prisma(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch) -> Mock: + """Audit logging on, with a prisma client whose LiteLLM_AuditLog inserts are recorded and whose guardrails + table holds one stored guardrail and returns whatever the registry writes.""" + import litellm + + monkeypatch.setattr(litellm, "store_audit_logs", True) + monkeypatch.setattr(litellm, "audit_log_callbacks", []) + monkeypatch.delenv("GENERIC_GUARDRAIL_API_BASE", raising=False) + mocker.patch("litellm.proxy.proxy_server.premium_user", True) + prisma: Final = Mock() + prisma.db.litellm_auditlog.create = AsyncMock() + prisma.db.litellm_guardrailstable.find_unique = AsyncMock(return_value=_stored_row()) + prisma.db.litellm_guardrailstable.update = AsyncMock(side_effect=_row_as_written) + prisma.db.litellm_guardrailstable.create = AsyncMock( + side_effect=lambda data: _row_as_written({"guardrail_id": "gr-audit"}, data) + ) + prisma.db.litellm_guardrailstable.delete = AsyncMock() + mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma) + return prisma + + +def _live_guardrails(mocker: MockerFixture, loaded: Sequence[object] = ()) -> InMemoryGuardrailHandler: + handler: Final = InMemoryGuardrailHandler() + for row in loaded: + handler.initialize_guardrail(guardrail=guardrail_from_db_row(row), source="db") + mocker.patch("litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", handler) + return handler + + +async def _written_audit_rows(prisma: Mock) -> tuple[Mapping[str, object], ...]: + from litellm.proxy.management_helpers.audit_logs import drain_audit_tasks + + await drain_audit_tasks() + insert: Final[AsyncMock] = prisma.db.litellm_auditlog.create + return tuple(MappingProxyType(call.kwargs["data"]) for call in insert.call_args_list) + + +def _audit_snapshot(row: Mapping[str, object], field: str) -> Mapping[str, object]: + value: Final = row[field] + assert isinstance(value, str), f"{field} should be a JSON string, got {value!r}" + return json.loads(value) + + +def _assert_no_planted_secret(row: Mapping[str, object]) -> None: + serialized: Final = json.dumps(dict(row), default=str) + for secret in PLANTED_SECRETS: + assert secret not in serialized, f"{secret!r} leaked into the audit row: {serialized}" + + +@pytest.mark.asyncio +async def test_create_guardrail_audit_row_records_the_caller_and_no_provider_secret( + mocker: MockerFixture, audit_prisma: Mock +): + _live_guardrails(mocker) + + await create_guardrail(CreateGuardrailRequest(guardrail=_requested_guardrail()), user_api_key_dict=AUDIT_ADMIN) + + rows: Final = await _written_audit_rows(audit_prisma) + assert [ + { + k: row.get(k) + for k in ("action", "table_name", "object_id", "changed_by", "changed_by_api_key", "before_value") + } + for row in rows + ] == [ + { + "action": "created", + "table_name": "LiteLLM_GuardrailsTable", + "object_id": "gr-audit", + "changed_by": "security-admin", + "changed_by_api_key": "hashed-admin-key", + "before_value": None, + } + ], rows + assert _audit_snapshot(rows[0], "updated_values") == STORED_GUARDRAIL_SNAPSHOT + _assert_no_planted_secret(rows[0]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["put", "patch"]) +async def test_update_guardrail_audit_row_shows_the_mode_change_and_names_only_the_rotated_key( + method: str, mocker: MockerFixture, audit_prisma: Mock +): + _live_guardrails(mocker, loaded=(_stored_row(),)) + + if method == "put": + await update_guardrail( + "gr-audit", + UpdateGuardrailRequest(guardrail=_requested_guardrail(mode="post_call", api_key=ROTATED_API_KEY)), + user_api_key_dict=AUDIT_ADMIN, + ) + else: + await patch_guardrail( + "gr-audit", + PatchGuardrailRequest(litellm_params={"api_key": ROTATED_API_KEY, "mode": "post_call"}), + user_api_key_dict=AUDIT_ADMIN, + ) + + rows: Final = await _written_audit_rows(audit_prisma) + assert [(r["action"], r["object_id"], r["changed_by"]) for r in rows] == [ + ("updated", "gr-audit", "security-admin") + ], rows + assert _audit_snapshot(rows[0], "before_value") == STORED_GUARDRAIL_SNAPSHOT + assert _audit_snapshot(rows[0], "updated_values") == { + **STORED_GUARDRAIL_SNAPSHOT, + "litellm_params": {"guardrail": "generic_guardrail_api", "mode": "post_call", "default_on": False}, + "changed_litellm_params": ["api_key"], + }, "only api_key changed, so headers the live guardrail adds for itself must not show up" + _assert_no_planted_secret(rows[0]) + + +@pytest.mark.asyncio +async def test_delete_guardrail_audit_row_keeps_the_deleted_guardrail_without_secrets( + mocker: MockerFixture, audit_prisma: Mock +): + _live_guardrails(mocker, loaded=(_stored_row(),)) + + await delete_guardrail(guardrail_id="gr-audit", user_api_key_dict=AUDIT_ADMIN) + + rows: Final = await _written_audit_rows(audit_prisma) + assert [(r["action"], r["object_id"], r.get("updated_values")) for r in rows] == [("deleted", "gr-audit", None)], ( + rows + ) + assert _audit_snapshot(rows[0], "before_value") == STORED_GUARDRAIL_SNAPSHOT + _assert_no_planted_secret(rows[0]) + + +@pytest.mark.asyncio +async def test_rejected_guardrail_update_writes_no_audit_row(mocker: MockerFixture, audit_prisma: Mock): + _live_guardrails(mocker, loaded=(_stored_row(),)) + + with pytest.raises(HTTPException) as exc_info: + await update_guardrail( + "gr-audit", + UpdateGuardrailRequest(guardrail=_requested_guardrail(mode="post_call", api_base=None)), + user_api_key_dict=AUDIT_ADMIN, + ) + + assert exc_info.value.status_code == 422, exc_info.value.detail + assert await _written_audit_rows(audit_prisma) == (), "a rolled-back update must not be audited" + + +def _team_guardrail_row(status: str, params_stored_as_json_string: bool = False): + encrypted: Final = encrypt_guardrail_litellm_params(dict(STORED_PARAMS)) + return _row_as_written( + {"guardrail_id": "team-gr"}, + { + "guardrail_name": "team-guard", + "litellm_params": json.dumps(json.dumps(encrypted) if params_stored_as_json_string else encrypted), + "guardrail_info": json.dumps({"description": "team"}), + "status": status, + "team_id": "team-1", + "updated_at": SAVED_AT, + }, + ) + + +def _team_guardrail_snapshot(status: str) -> Mapping[str, object]: + return MappingProxyType( + { + "guardrail_id": "team-gr", + "guardrail_name": "team-guard", + "guardrail_info": {"description": "team"}, + "status": status, + "team_id": "team-1", + "litellm_params": {"guardrail": "generic_guardrail_api", "mode": "pre_call", "default_on": False}, + } + ) + + +@pytest.mark.asyncio +async def test_register_guardrail_audit_row_records_the_pending_submission_without_secrets( + audit_prisma: Mock, +) -> None: + audit_prisma.db.litellm_guardrailstable.find_unique = AsyncMock(return_value=None) + audit_prisma.db.litellm_guardrailstable.create = AsyncMock(return_value=_team_guardrail_row("pending_review")) + request: Final = RegisterGuardrailRequest( + guardrail_name="team-guard", litellm_params=dict(STORED_PARAMS), team_id="team-1" + ) + + await register_guardrail(request, AUDIT_ADMIN) + + rows: Final = await _written_audit_rows(audit_prisma) + assert [(r["action"], r["object_id"]) for r in rows] == [("created", "team-gr")], rows + assert _audit_snapshot(rows[0], "updated_values") == _team_guardrail_snapshot("pending_review") + _assert_no_planted_secret(rows[0]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "review, new_status", [(approve_guardrail_submission, "active"), (reject_guardrail_submission, "rejected")] +) +@pytest.mark.parametrize("params_stored_as_json_string", [False, True], ids=["json_object", "json_string"]) +async def test_guardrail_submission_review_audit_row_shows_the_status_change( + review: Callable[[str, UserAPIKeyAuth], Awaitable[Mapping[str, object]]], + new_status: str, + params_stored_as_json_string: bool, + mocker: MockerFixture, + audit_prisma: Mock, +): + audit_prisma.db.litellm_guardrailstable.find_unique = AsyncMock( + return_value=_team_guardrail_row("pending_review", params_stored_as_json_string) + ) + audit_prisma.db.litellm_guardrailstable.update = AsyncMock( + return_value=_team_guardrail_row(new_status, params_stored_as_json_string) + ) + _live_guardrails(mocker) + + response: Final = await review("team-gr", AUDIT_ADMIN) + + assert response["status"] == new_status, response + rows: Final = await _written_audit_rows(audit_prisma) + assert [(r["action"], r["object_id"]) for r in rows] == [("updated", "team-gr")], rows + assert _audit_snapshot(rows[0], "before_value") == _team_guardrail_snapshot("pending_review") + assert _audit_snapshot(rows[0], "updated_values") == _team_guardrail_snapshot(new_status) + _assert_no_planted_secret(rows[0]) + + @pytest.mark.asyncio async def test_apply_guardrail_not_found(mocker): """ diff --git a/ui/litellm-dashboard/src/components/logs/audit/AuditLogsTable.tsx b/ui/litellm-dashboard/src/components/logs/audit/AuditLogsTable.tsx index b376a83f987..d99d754d206 100644 --- a/ui/litellm-dashboard/src/components/logs/audit/AuditLogsTable.tsx +++ b/ui/litellm-dashboard/src/components/logs/audit/AuditLogsTable.tsx @@ -56,6 +56,7 @@ const TABLE_OPTIONS: readonly FilterOption[] = [ { labelKey: "logs.audit.tableNames.organizations", value: "LiteLLM_OrganizationTable" }, { labelKey: "logs.audit.tableNames.models", value: "LiteLLM_ProxyModelTable" }, { labelKey: "logs.audit.tableNames.agents", value: "LiteLLM_AgentsTable" }, + { labelKey: "logs.audit.tableNames.guardrails", value: "LiteLLM_GuardrailsTable" }, ]; const ACTION_FILTER_ITEMS: readonly FilterOption[] = [ diff --git a/ui/litellm-dashboard/src/components/logs/audit/AuditLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/logs/audit/AuditLogsTableColumns.tsx index 9d22d09daa7..b0dca4ed6af 100644 --- a/ui/litellm-dashboard/src/components/logs/audit/AuditLogsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/logs/audit/AuditLogsTableColumns.tsx @@ -26,6 +26,7 @@ export const AUDIT_TABLE_NAME_DISPLAY: Record = { LiteLLM_OrganizationTable: "logs.audit.tableNames.organizations", LiteLLM_ProxyModelTable: "logs.audit.tableNames.models", LiteLLM_AgentsTable: "logs.audit.tableNames.agents", + LiteLLM_GuardrailsTable: "logs.audit.tableNames.guardrails", }; const ACTION_TONE: Record = { diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index 854a49fd546..8e15979afb1 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -14935,7 +14935,8 @@ "users": "Users", "organizations": "Organizations", "models": "Models", - "agents": "Agents" + "agents": "Agents", + "guardrails": "Guardrails" }, "drawer": { "notAvailable": "N/A", From a0e6cafbd8d317fb98ea032aea54c6aadf4c625e Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 11 Nov 2026 16:07:03 -0800 Subject: [PATCH 130/189] fix(llms-types): let the compact-2026-09-04 beta through on bedrock to match the headers config (#51272) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/llms-types/src/providers/anthropic/beta.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index 46c14a21c59..e3ad783c791 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -320,7 +320,6 @@ impl AnthropicBeta { Self::AfkMode20260131 | Self::CePlugins20260901 | Self::CeUserManagement20260713 - | Self::Compact20260904 | Self::Dreaming20260421 | Self::ExperimentalClaudeTag20261021 | Self::ExtendedCacheTtl20250411 @@ -403,7 +402,8 @@ impl AnthropicBeta { | Self::Output128k20250219 | Self::TokenEfficientTools20250219 | Self::WebSearch20250305 => &[BetaProvider::BedrockConverse, BetaProvider::Bedrock], - Self::MidConversationOutputConfig20260701 + Self::Compact20260904 + | Self::MidConversationOutputConfig20260701 | Self::MidConversationToolChanges20260701 => &[ BetaProvider::BedrockConverse, BetaProvider::BedrockMantle, From 59538ff1d99282ba973872c7689086ba5d54f188 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Wed, 11 Nov 2026 16:07:20 -0800 Subject: [PATCH 131/189] feat(messages): serve anthropic messages on rust only (#51119) * feat(messages): apply modify_params rewrites on rust * feat(messages): mint anthropic workload identity tokens on rust * feat(messages): apply prompt caching injection on rust * feat(messages): serve the easy python-only detectors on rust * feat(messages): retry rejected thinking signatures on rust * feat(messages): serve response cache reads on rust * feat(messages): retry rejected strict tools on rust * refactor(messages): delete the python path for anthropic messages Provider anthropic on /v1/messages is now RUST_REQUIRED, so it runs on Rust without LITELLM_RUST or litellm.rust(True) and never falls back to Python. A call the rules would send to Python (a MESSAGES_PYTHON_FEATURES feature or a detector that raises) gets UnsupportedParamsError naming the feature, raised from one place in litellm/messages/dispatch.py. Other providers keep the Python fallback. A call whose arguments the handler signature rejects still gets Python's TypeError. Deleted Python code that only served anthropic Messages: - workload identity in AnthropicMessagesConfig.validate/avalidate (subclasses were never eligible; the Rust host mints the token itself) - the anthropic Foundry beta-row and affinity-base branch in async_anthropic_messages_handler, and the "forward" mode of the beta filter that only that branch used The proxy MCP tools detector now uses the light tool precheck so a function tool never imports the gateway handler, since a raising detector now refuses. * feat(messages): forward message threads on rust The Rust path for anthropic /v1/messages dropped the thread body field, so a create or continue turn reached Anthropic without its thread or the message-threads-2026-08-12 beta. It now matches the Python handler from #50976: - the host forwards thread with the other body params - a non-null thread adds the message-threads beta - a well-formed continue turn keeps advisor blocks in history and derives no compaction, per-turn-control or tool-change beta from its messages - the recorded httpx_response carries the request that was sent, for JSON and streamed calls, so the proxy's thread ownership hook resends the create turn's betas on a continue as it does for Python Upstream error.details already reached the exception message unchanged, so thread_not_found relays and keeps the deployment out of cooldown on both paths. Parity cases cover create, continue, streaming, sync, retries, thread_not_found and the ownership hook round trip. * feat(messages): serve mock responses on rust * feat(messages): send caller clients and unsupported tls through python on rust * feat(messages): run pre-request hooks on rust * feat(messages): run agentic loop hooks on rust * feat(messages): run the safeguards classifier on rust * feat(messages): run registered interceptors on rust * fix(messages): match python stream cache gate, logging and interceptor order on rust Store a streamed Messages response in the cache only when Python would: a message_stop event line, no provider error frame and at least one frame with output. Leave dict results untouched by mark_rust_response so hidden params never reach result dicts, logging objects or hook digests. Ask interceptors whether they handle the call after the cache lookup, not at decode time, so a cache hit never calls can_handle. Emit the track_llm_api_timing service event for the provider exchange. Bill and log a stream superseded by a router fallback as the failure of its error frame, with the partial usage warning. Hand litellm_logging_obj to deployment pre-call hooks as @client does, so the autorouter baseline cache sees retried requests. * fix(messages): forward python-only json values to anthropic as python does Accept any max_tokens, sampling scalar and messages collection that Python main accepts, and carry values serde_json cannot hold (ints beyond 64 bits, NaN and infinities, strings with lone surrogates) as validated raw JSON leaves written back verbatim on the wire, so the upstream body matches Python's json.dumps * fix(messages): match python's body reads, error bodies, encoded heads and tls warnings on rust * fix(messages): log python's plain provider header names on rust streams * fix(messages): match python's request params, logged params and validation on rust Read a non-bool stream by Python truthiness and send it as given, log the caller's stream flag as Python's handler does, raise the raw Python errors main raises for malformed messages, tools, model_info and hook-set litellm_params or providers, read the tool custom-field drop from drops_unsupported_params, log the caller's nested fields that additional_drop_params strips and the Python values of raw JSON leaves, keep a missing stop_reason absent, and map provider-config rejections (missing max_tokens, unknown reasoning effort) as Python's HTTP handler maps them, with Python's text, effort order, model and provider * fix(messages): match python's tls curve names, redirects, auth token, header encoding and timeouts on rust * fix(messages): match python on hooked streams, empty cache writes and mcp cache hits * fix(messages): join already buffered stream reads and trim the env api key as python does on rust * fix(messages): deliver stream bytes read before a cut, keep wide reply numbers and follow python's redirect limit on rust A stream cut by a transport error now hands the caller the bytes already buffered before raising, as aiohttp does for a caller that reads them before the event loop sees the cut, so the frames and their usage reach the caller and the failure log. Provider replies keep integers wider than 64 bits exactly and read numbers that overflow a float as infinity, through the response, the response cache and the logged result. The provider client follows up to 20 redirects as httpx does, and running out raises Python's text. A provider_specific_header value the ascii codec cannot encode fails before the send like a caller extra header. The deployment success hook gets the stream as AnthropicMessagesStreamingResponse, as Python's handler hands it. * feat(messages): serve proxy mcp tools on rust * test(messages): cover hook, agentic and interceptor edge cases in the parity suite * fix(messages): keep provider raw-json marked strings, drop non-string tool_result texts and key clamped streams as python does on rust * fix(messages): match python's api key encoding, malformed api_base and string provider header errors on rust A header value Python's client cannot encode as ASCII now fails before the send whichever header carries it, so a caller api_key with a non-ASCII character raises Python's InternalServerError instead of a raw UnicodeEncodeError after the reply. A caller api_base without an http(s) scheme, or with whitespace, control or non-ASCII characters, is sent through Python's client at the URL Python builds, so its relative-URL and invalid-URL errors match, and a base urlsplit cannot split raises Python's bare ValueError before the send. A provider_specific_header of the wrong shape is scoped through Python's ProviderSpecificHeaderUtils first, so a plain string raises Python's AttributeError. * fix(messages): keep integer temperature and top_p digits a float cannot hold and raise python's typeerror for a string max_tokens while capping the thinking budget on rust temperature and top_p now deserialize through recognized::exact_float, which keeps an integer where float(n) != n (beyond 2**53) as an unrecognized value with its exact digits, so it is sent upstream and logged as the integer python forwards, and comparisons still go through scalar.rs cap_thinking_budget_to_max_tokens compares max_tokens <= 1024 first, so a max_tokens python cannot order against an int (a string, list or dict) raises TypeError "'<=' not supported between instances of 'str' and 'int'" in both the reasoning effort translation and the adaptive effort reshape. rust now returns ErrorDetail::UnorderableTypes there, the bridge raises it as a python TypeError marked as a late rejection, and map_failure hands it to the caller unmapped * test(messages): format the non-string tool_result parity helper * fix(messages): flag and log the call before python's pre-send raises on rust The bridge now hands settings() and map_failure() whether the call is async. A provider_specific_header of the wrong shape sets the aanthropic_messages flag before re-raising, as Python's handler has already flagged the call. A base urlsplit cannot split also logs the resolved model, provider and optional params first, as Python raises it only after its logger is updated. The TypeError raised while capping the thinking budget for a string max_tokens sets the flag too * feat(messages): serve the web search short-circuit override on rust through the intercept host call * fix(messages): give the web search entry kwargs its mutable-ok reason * test(messages): drop the body hook gap the rust short-circuit closes * fix(messages): name the partially unknown handler types on the short-circuit host call --- litellm-rust/Cargo.lock | 7 + .../crates/cache-response/src/service.rs | 4 + .../python_contract.json | 28 +- .../callbacks-legacy-python/src/adapter.rs | 392 +- .../callbacks-legacy-python/src/callbacks.rs | 22 +- .../crates/callbacks-legacy-python/src/lib.rs | 2 +- .../callbacks-legacy-python/src/logger.rs | 6 +- .../callbacks-legacy-python/src/mapping.rs | 3 + .../callbacks-legacy-python/src/python.rs | 6 + .../src/test_support.rs | 14 +- .../crates/gateway-inference/tests/caching.rs | 7 +- litellm-rust/crates/host-native/src/driver.rs | 7 + litellm-rust/crates/host-python/Cargo.toml | 2 +- .../crates/host-python/src/binding.rs | 4 +- litellm-rust/crates/host-python/src/driver.rs | 12 +- .../host-python/src/hooks/chain/dispatch.rs | 9 + litellm-rust/crates/host-python/src/lib.rs | 3 +- .../crates/host-python/src/marshal.rs | 182 +- .../crates/host-python/tests/support/mod.rs | 3 + litellm-rust/crates/host/Cargo.toml | 1 + litellm-rust/crates/host/src/interceptors.rs | 28 + litellm-rust/crates/host/src/lib.rs | 1 + litellm-rust/crates/host/src/lifecycle.rs | 15 +- .../crates/host/src/machine/context.rs | 10 +- litellm-rust/crates/host/src/protocol.rs | 6 +- litellm-rust/crates/host/src/raw_json.rs | 362 ++ litellm-rust/crates/http/src/config.rs | 20 +- litellm-rust/crates/http/src/error.rs | 4 + litellm-rust/crates/http/src/outbound.rs | 15 +- litellm-rust/crates/http/src/pool.rs | 28 +- litellm-rust/crates/http/src/tls.rs | 8 +- .../crates/inference-chat/tests/caching.rs | 1 + .../crates/inference-messages/Cargo.toml | 4 + .../inference-messages/src/constants.rs | 3 + .../crates/inference-messages/src/encoding.rs | 437 ++ .../crates/inference-messages/src/handler.rs | 968 ++++- .../inference-messages/src/host_answer.rs | 11 + .../crates/inference-messages/src/lib.rs | 141 +- .../inference-messages/src/pre_request.rs | 7 + .../crates/inference-messages/src/reads.rs | 394 ++ .../crates/inference-messages/src/route.rs | 99 + .../inference-messages/src/safeguards.rs | 248 ++ .../inference-messages/src/transport.rs | 59 + .../crates/inference-messages/src/types.rs | 43 +- .../src/workload_identity.rs | 11 + .../inference-messages/tests/caching.rs | 132 + .../tests/messages/gateway.rs | 99 + .../inference-messages/tests/messages/host.rs | 199 +- .../inference-messages/tests/messages/main.rs | 4 + .../tests/messages/pre_request.rs | 141 + .../tests/messages/response.rs | 119 +- .../tests/messages/retry.rs | 364 ++ .../tests/messages/stream.rs | 4 +- .../tests/messages/transport.rs | 256 ++ .../inference-messages/tests/support/mod.rs | 10 + .../inference-ocr/tests/ocr/lifecycle.rs | 1 + .../crates/inference-ocr/tests/ocr/machine.rs | 6 + litellm-rust/crates/inference/AGENTS.md | 1 + litellm-rust/crates/inference/Cargo.toml | 2 +- litellm-rust/crates/inference/src/caching.rs | 17 +- litellm-rust/crates/inference/src/context.rs | 13 +- litellm-rust/crates/inference/src/error.rs | 19 +- litellm-rust/crates/inference/src/outbound.rs | 17 +- .../src/formats/messages/request.rs | 64 +- .../src/providers/anthropic/beta.rs | 5 + .../crates/llms-types/src/recognized.rs | 149 + litellm-rust/crates/llms/Cargo.toml | 2 + .../crates/llms/src/anthropic/beta_headers.rs | 12 +- .../crates/llms/src/anthropic/common_utils.rs | 375 +- .../llms/src/anthropic/messages/handler.rs | 44 +- .../crates/llms/src/anthropic/messages/mod.rs | 1 + .../llms/src/anthropic/messages/scalar.rs | 113 + .../llms/src/anthropic/messages/thinking.rs | 210 +- .../src/anthropic/messages/transformation.rs | 660 ++- .../src/azure_ai/messages/transformation.rs | 25 +- .../llms/src/base_llm/messages/context.rs | 8 + .../crates/llms/src/base_llm/messages/mod.rs | 1 + .../src/base_llm/messages/normalization.rs | 26 +- .../llms/src/base_llm/messages/tools.rs | 194 + .../src/base_llm/messages/transformation.rs | 71 +- .../messages/transformation.rs | 27 +- .../anthropic_claude_transformation.rs | 18 +- .../bedrock_mantle/messages/transformation.rs | 31 +- .../src/deepseek/messages/transformation.rs | 27 +- .../src/edenai/messages/transformation.rs | 18 + litellm-rust/crates/llms/src/error.rs | 12 + .../src/minimax/messages/transformation.rs | 28 +- .../src/openrouter/messages/transformation.rs | 13 + .../src/tencent/messages/transformation.rs | 28 +- .../src/vertex_ai/messages/transformation.rs | 26 +- .../anthropic_messages_http_error_retry.json | 623 +++ .../llms/tests/messages_http_error_retry.rs | 138 + .../crates/python-bridge/src/cache/AGENTS.md | 2 +- .../crates/python-bridge/src/cache/mod.rs | 2 +- .../python-bridge/src/cache/python/AGENTS.md | 2 +- .../python-bridge/src/cache/python/host.rs | 77 +- .../python-bridge/src/cache/python/service.rs | 23 +- .../python-bridge/src/cache/selection.rs | 46 +- .../crates/python-bridge/src/errors.rs | 117 +- litellm-rust/crates/python-bridge/src/http.rs | 47 +- litellm-rust/crates/python-bridge/src/lib.rs | 3 + .../crates/python-bridge/src/marshal.rs | 15 +- .../src/routes/messages/agentic.rs | 172 + .../src/routes/messages/answer.rs | 29 + .../python-bridge/src/routes/messages/host.rs | 758 +++- .../python-bridge/src/routes/messages/mod.rs | 80 +- .../src/routes/messages/modify_params.rs | 152 + .../src/routes/messages/pre_request.rs | 35 + .../src/routes/messages/safeguards.rs | 183 + .../src/routes/messages/transport.rs | 275 ++ .../src/routes/messages/workload_identity.rs | 74 + .../crates/python-bridge/src/routes/mod.rs | 35 +- litellm-rust/crates/python-compat/src/lib.rs | 2 + .../crates/python-compat/src/reprlib.rs | 179 + litellm-rust/crates/router/src/lib.rs | 25 +- litellm/__init__.py | 4 +- litellm/anthropic_beta_headers_manager.py | 12 +- litellm/caching/caching_handler.py | 265 +- litellm/litellm_core_utils/core_helpers.py | 14 +- .../llms/anthropic/messages/native/handler.py | 52 +- .../messages/native/transformation.py | 23 +- litellm/llms/custom_httpx/http_handler.py | 4 + litellm/llms/custom_httpx/llm_http_handler.py | 16 +- .../llms/dashscope/messages/transformation.py | 4 +- litellm/messages/dispatch.py | 108 +- litellm/rust_bridge/AGENTS.md | 5 +- litellm/rust_bridge/_native.pyi | 2 + litellm/rust_bridge/catalog.py | 43 +- litellm/rust_bridge/detectors.py | 32 +- litellm/rust_bridge/execution.py | 6 +- litellm/rust_bridge/host/AGENTS.md | 2 +- litellm/rust_bridge/host/cache.py | 200 +- .../host/callbacks_legacy_python.py | 200 +- litellm/rust_bridge/host/http_transport.py | 25 + litellm/rust_bridge/host/routes/messages.py | 1255 +++++- litellm/rust_bridge/messages/detectors.py | 114 - litellm/rust_bridge/messages/entrypoints.py | 35 +- litellm/rust_bridge/response_metadata.py | 7 +- litellm/rust_bridge/runtime.py | 2 +- litellm/utils.py | 99 +- .../sdk/messages/test_native_parity.py | 3523 ++++++++++++++++- .../cache/test_python_cache.py | 140 +- tests/test_litellm_rust/messages/data.py | 4 - .../messages/test_body_hooks.py | 36 +- .../messages/test_dispatch.py | 635 ++- .../messages/test_workload_identity.py | 219 + .../support/recording_server.py | 40 +- tests/test_litellm_rust/support/served.py | 10 +- tests/test_litellm_rust/test_dispatch.py | 2 +- .../litellm_core_utils/test_core_helpers.py | 27 + .../test_litellm_logging.py | 3 +- .../test_anthropic_chat_transformation.py | 3 +- .../native/test_anthropic_messages_handler.py | 9 +- .../native/test_anthropic_messages_router.py | 16 +- .../test_anthropic_messages_transformation.py | 51 +- .../messages/native/test_response_cache.py | 50 +- .../anthropic/test_anthropic_common_utils.py | 38 - .../test_anthropic_prompt_cache_prediction.py | 2 + .../test_messages_http_error_retry_cases.py | 188 + .../test_anthropic_claude_transformation.py | 3 + .../custom_httpx/test_llm_http_handler.py | 33 +- .../hooks/test_autorouter_baseline_cache.py | 7 + .../unit/responses/mcp/test_tool_precheck.py | 21 +- .../test_context_compaction.py | 16 + .../router_strategy/test_complexity_router.py | 15 + .../test_fallback_event_handlers.py | 16 + .../rust_bridge/host/routes/test_messages.py | 1314 +++++- tests/unit/rust_bridge/host/test_cache.py | 119 +- .../host/test_callbacks_legacy_python.py | 155 +- tests/unit/rust_bridge/test_catalog.py | 25 +- tests/unit/rust_bridge/test_detectors.py | 161 +- tests/unit/rust_bridge/test_runtime.py | 56 +- tests/unit/test_private_usage_aliases.py | 2130 +++++----- tests/unit/test_router/test_router.py | 2 + .../test_router/test_router_helper_utils.py | 2 + 175 files changed, 19176 insertions(+), 2747 deletions(-) create mode 100644 litellm-rust/crates/host/src/raw_json.rs create mode 100644 litellm-rust/crates/inference-messages/src/encoding.rs create mode 100644 litellm-rust/crates/inference-messages/src/host_answer.rs create mode 100644 litellm-rust/crates/inference-messages/src/pre_request.rs create mode 100644 litellm-rust/crates/inference-messages/src/reads.rs create mode 100644 litellm-rust/crates/inference-messages/src/safeguards.rs create mode 100644 litellm-rust/crates/inference-messages/src/transport.rs create mode 100644 litellm-rust/crates/inference-messages/src/workload_identity.rs create mode 100644 litellm-rust/crates/inference-messages/tests/messages/gateway.rs create mode 100644 litellm-rust/crates/inference-messages/tests/messages/pre_request.rs create mode 100644 litellm-rust/crates/inference-messages/tests/messages/retry.rs create mode 100644 litellm-rust/crates/inference-messages/tests/messages/transport.rs create mode 100644 litellm-rust/crates/llms/src/anthropic/messages/scalar.rs create mode 100644 litellm-rust/crates/llms/src/base_llm/messages/tools.rs create mode 100644 litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json create mode 100644 litellm-rust/crates/llms/tests/messages_http_error_retry.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/messages/agentic.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/messages/answer.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/messages/modify_params.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/messages/pre_request.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/messages/safeguards.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/messages/transport.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/messages/workload_identity.rs create mode 100644 litellm-rust/crates/python-compat/src/reprlib.rs delete mode 100644 litellm/rust_bridge/messages/detectors.py create mode 100644 tests/test_litellm_rust/messages/test_workload_identity.py create mode 100644 tests/unit/llms/anthropic/test_messages_http_error_retry_cases.py diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ed658e074ee..2cdc9ee91c8 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4013,6 +4013,7 @@ dependencies = [ "litellm-coroutine", "litellm-secret-value", "rstest", + "serde", "serde_json", "thiserror 2.0.19", "tokio", @@ -4155,9 +4156,12 @@ name = "litellm-inference-messages" version = "0.1.0" dependencies = [ "bytes", + "flate2", "futures-util", + "http 1.4.2", "litellm-auth", "litellm-auth-aws", + "litellm-cache", "litellm-cache-memory", "litellm-cache-response", "litellm-core-utils", @@ -4178,6 +4182,7 @@ dependencies = [ "rstest", "serde", "serde_json", + "thiserror 2.0.19", "tokio", "tracing", "wiremock", @@ -4292,6 +4297,7 @@ dependencies = [ "base64 0.22.1", "bytes", "data-url", + "fancy-regex 0.19.2", "futures-util", "litellm-auth", "litellm-auth-aws", @@ -4307,6 +4313,7 @@ dependencies = [ "litellm-router-types", "litellm-secret-value", "litellm-secrets", + "litellm-tracing", "reqwest", "rstest", "serde", diff --git a/litellm-rust/crates/cache-response/src/service.rs b/litellm-rust/crates/cache-response/src/service.rs index 0bdf948ec48..ed4dccdc355 100644 --- a/litellm-rust/crates/cache-response/src/service.rs +++ b/litellm-rust/crates/cache-response/src/service.rs @@ -27,6 +27,10 @@ impl Default for ResponseCacheConfig { pub trait ResponseCacheService: Send + Sync { fn config(&self) -> &ResponseCacheConfig; + fn keyed_by_caller(&self) -> bool { + false + } + fn lookup<'a>( &'a self, request: &'a ResponseCacheRequest, diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index 8a212223523..a96e9f04bdc 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -11,7 +11,8 @@ "logger", "kwargs", "start_time", - "end_time" + "end_time", + "cache_hit" ], "update_logging": [ "logger", @@ -41,13 +42,24 @@ "logger", "status", "headers", - "body" + "body", + "sent" + ], + "retried_request": [ + "logger", + "logged_body", + "body", + "reasons" ], "record_api_window": [ "logger", "start", "end" ], + "times_provider_exchange": [ + "client", + "timeout" + ], "defers_async_logging": [ "logger" ], @@ -89,6 +101,12 @@ "start", "end" ], + "cache_hit_success": [ + "logger", + "response", + "start", + "end" + ], "enqueue_logging": [ "coroutine" ], @@ -113,14 +131,16 @@ ], "stream_opened": [ "logger", - "head" + "head", + "keeps_requested" ], "stamp_stream_head": [ "head", "logger", "kwargs", "start_time", - "end_time" + "end_time", + "cache_hit" ], "stream_success": [ "logger", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index f68541c5c7b..71ca89b0790 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -6,10 +6,10 @@ use crate::LoggingOperation; use litellm_host_python::PythonOwned; use litellm_host::{ - interceptors::{RawResponse, RequestContext, WireRequest}, + interceptors::{RawResponse, RequestContext, RetriedRequest, RetryReason, WireRequest}, lifecycle::{FailureOrigin, Timing, epoch_seconds}, }; -use litellm_host_python::{Error as HostError, HookStep, from_py, to_py}; +use litellm_host_python::{Error as HostError, HookStep, from_py, json_from_py, json_to_py, to_py}; use pyo3::{ exceptions::{PyBaseException, PyException}, gc::{PyTraverseError, PyVisit}, @@ -19,7 +19,7 @@ use pyo3::{ use serde_json::Value; use crate::{ - DeploymentHooks, LegacyCallbacks, PublicCall, PythonLogger, + DeploymentHooks, LegacyCallbacks, PublicCall, PythonLogger, SentRequest, deferred::{PendingLogging, PendingSuccess}, finalize, is_internal_call, python::{Logging, Streaming}, @@ -41,6 +41,7 @@ struct DeliveredStream { struct LoggedRequest { body: Py, headers: Py, + url: String, context: RequestContext, } @@ -59,6 +60,16 @@ pub struct LegacyLogging { cache_key: Option, /// When the provider request went out, until its window is recorded (Messages only). api_started: Option, + host_answered: bool, +} + +fn retry_reason(reason: &RetryReason) -> Value { + match reason { + RetryReason::ThinkingSignature => serde_json::json!({"kind": "thinking_signature"}), + RetryReason::StrictTools { dropped } => { + serde_json::json!({"kind": "strict_tools", "dropped": dropped}) + } + } } fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult> { @@ -90,6 +101,7 @@ impl LegacyLogging { internal: false, cache_key: None, api_started: None, + host_answered: false, } } @@ -171,6 +183,14 @@ impl LegacyLogging { Ok(context) } + fn times_provider_exchange(&self, py: Python<'_>) -> PyResult { + let client = self.call.lookup(py, "client")?; + let timeout = self.call.lookup(py, "timeout")?; + Logging::TimesProviderExchange + .call(py, (client, timeout))? + .extract() + } + /// Python times the provider exchange inside the async `AsyncHTTPHandler.post`; its sync /// client is untimed. fn close_api_window(&mut self, py: Python<'_>, end: f64) -> PyResult<()> { @@ -264,6 +284,7 @@ impl LegacyLogging { self.call.kwargs(), &self.start, &self.end, + self.messages_cache_hit(), )?; self.response .as_ref() @@ -271,6 +292,60 @@ impl LegacyLogging { .ok_or_else(|| HostError::MissingState.into()) } + fn messages_cache_hit(&self) -> bool { + self.operation == LoggingOperation::Messages && self.cache_key.is_some() + } + + fn flag_unsent_messages_call(&self, py: Python<'_>) -> PyResult<()> { + if self.operation != LoggingOperation::Messages + || self.request.is_some() + || self.cache_key.is_some() + || self.host_answered + { + return Ok(()); + } + let details = self.logger()?.object(py).getattr("model_call_details")?; + details + .call_method1("setdefault", ("litellm_params", PyDict::new(py)))? + .set_item("aanthropic_messages", self.asynchronous) + } + + fn unstreamed_stream_request(&self, py: Python<'_>) -> PyResult { + if self.operation != LoggingOperation::Messages + || self.stream.is_some() + || self.messages_cache_hit() + { + return Ok(false); + } + let Some(stream) = self.call.kwargs().bind(py).get_item("stream")? else { + return Ok(false); + }; + if !stream.is_exact_instance_of::() || !stream.is_truthy()? { + return Ok(false); + } + let logger = self.logger()?.object(py); + logger.setattr("stream", true)?; + logger + .getattr("model_call_details")? + .set_item("stream", true)?; + Ok(true) + } + + fn dispatch_cache_hit(&self, py: Python<'_>) -> PyResult<()> { + let logger = self.logger()?; + let logged = Logging::CacheHitSuccess.call( + py, + (logger.object(py), &self.response, &self.start, &self.end), + ); + match logged { + Err(error) if error.is_instance_of::(py) => { + error.write_unraisable(py, Some(logger.object(py))); + Ok(()) + } + result => result.map(|_| ()), + } + } + fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> { match self.try_dispatch_success(py) { Err(error) if error.is_instance_of::(py) => { @@ -489,6 +564,10 @@ impl LegacyLogging { self.logger = Some(result.logger()?); self.call.set_kwargs(result.kwargs()?); if self.runs_deployment_hooks() { + self.call + .kwargs() + .bind(py) + .set_item("litellm_logging_obj", self.logger()?.object(py))?; return Ok(HookStep::Await( DeploymentHooks::before_call(py, self.call.kwargs(), self.call_type())?, Self::resume_begin, @@ -512,9 +591,13 @@ impl LegacyLogging { context, self.logged_request_params(context).as_ref(), )?; - let body = to_py(py, &wire.body)? - .into_bound(py) - .cast_into::()?; + let lossless = self.operation == LoggingOperation::Messages; + let body = if lossless { + json_to_py(py, &wire.body)? + } else { + to_py(py, &wire.body)?.into_bound(py) + } + .cast_into::()?; for (name, sent) in wire.body.as_object().into_iter().flatten() { if let Some(value) = self.caller_value(py, name, sent)? { body.set_item(name, value)?; @@ -534,6 +617,7 @@ impl LegacyLogging { self.request = Some(LoggedRequest { body: body.clone().unbind(), headers: headers.clone().unbind(), + url: wire.url.clone(), context: context.clone(), }); let input = self.logged_input(py, &body)?; @@ -545,7 +629,10 @@ impl LegacyLogging { &headers, &wire.url, )?; - if self.operation == LoggingOperation::Messages && self.asynchronous { + if self.operation == LoggingOperation::Messages + && self.asynchronous + && self.times_provider_exchange(py)? + { self.api_started = Some(epoch_seconds()); } let headers = headers @@ -553,7 +640,11 @@ impl LegacyLogging { .map(|(name, value)| Ok((name.extract::()?, value.extract::()?))) .collect::>>()?; Ok(HookStep::Ready(Box::new(WireRequest { - body: from_py(&body)?, + body: if lossless { + json_from_py(body.as_any())? + } else { + from_py(&body)? + }, headers, ..*wire }))) @@ -567,7 +658,11 @@ impl LegacyLogging { ) -> PyResult>> { self.end = Some(datetime(py, timing.end_time)?); self.response = Some(response); - if self.runs_deployment_hooks() { + self.flag_unsent_messages_call(py)?; + if self.unstreamed_stream_request(py)? { + return self.finalize(py); + } + if self.runs_deployment_hooks() && !self.messages_cache_hit() { return Ok(HookStep::Await( DeploymentHooks::after_success( py, @@ -588,7 +683,21 @@ impl LegacyLogging { ) -> PyResult> { use litellm_host::interceptors::{Cost, ResultSource}; + self.cache_key = match &facts.source { + ResultSource::Provider => None, + ResultSource::Cache { key } => Some(key.clone()), + ResultSource::Host => { + self.host_answered = true; + return Ok(HookStep::Ready(())); + } + }; let logger = self.logger()?.object(py); + if self.messages_cache_hit() { + logger + .getattr("model_call_details")? + .set_item("cache_hit", true)?; + return Ok(HookStep::Ready(())); + } let params = logger .getattr("litellm_params")? .cast_into::()? @@ -616,10 +725,6 @@ impl LegacyLogging { }; logger.setattr("_provider_reported_cost", reported_cost)?; details.call_method1("pop", ("response_cost", py.None()))?; - self.cache_key = match &facts.source { - ResultSource::Provider => None, - ResultSource::Cache { key } => Some(key.clone()), - }; if self.cache_key.is_some() { details.set_item("cache_hit", true)?; } else { @@ -644,8 +749,13 @@ impl LegacyLogging { // response, and logging replays it through `AnthropicConfig.transform_response`. self.close_api_window(py, epoch_seconds())?; if let Some(head) = &raw.head { + let sent = self.request.as_ref().map(|request| SentRequest { + url: &request.url, + headers: &request.headers, + body: &request.body, + }); self.logger()? - .record_response(py, head.status, &head.headers, &raw.body)?; + .record_response(py, head.status, &head.headers, &raw.body, sent)?; } return Ok(HookStep::Ready(())); } @@ -664,6 +774,40 @@ impl LegacyLogging { Ok(HookStep::Ready(())) } + pub(crate) fn request_retried( + &mut self, + py: Python<'_>, + retry: &RetriedRequest, + ) -> PyResult> { + let Some(request) = &self.request else { + return Ok(HookStep::Ready(())); + }; + let reasons: Vec = retry.reasons.iter().map(retry_reason).collect(); + let retried = Logging::RetriedRequest.call( + py, + ( + self.logger()?.object(py), + request.body.bind(py), + to_py(py, &retry.body)?, + to_py(py, &reasons)?, + ), + )?; + let now = epoch_seconds(); + if self.api_started.is_some() { + self.close_api_window(py, now)?; + self.api_started = Some(now); + } + Ok(HookStep::Await(retried.unbind(), Self::resume_retried)) + } + + fn resume_retried( + &mut self, + _py: Python<'_>, + result: PyResult>, + ) -> PyResult> { + result.map(|_| HookStep::Ready(())) + } + pub(crate) fn succeeded( &mut self, py: Python<'_>, @@ -674,6 +818,8 @@ impl LegacyLogging { self.response = Some(response.clone_ref(py)); match &self.stream { Some(stream) => self.stream_success(py, stream, false)?, + None if self.messages_cache_hit() => self.dispatch_cache_hit(py)?, + None if self.unstreamed_stream_request(py)? => {} None => self.dispatch_success(py)?, } Ok(HookStep::Ready(())) @@ -708,7 +854,11 @@ impl LegacyLogging { if self.stream_billing().is_none() { return Err(HostError::MissingState.into()); } - if let Some(key) = &self.cache_key { + if let Some(key) = self + .cache_key + .as_ref() + .filter(|_| !self.messages_cache_hit()) + { head.bind(py).set_item("cache_key", key)?; head.bind(py).set_item("cache_hit", true)?; } @@ -723,10 +873,18 @@ impl LegacyLogging { self.call.kwargs(), &self.start, datetime(py, epoch_seconds())?, + self.messages_cache_hit(), ), )?; } - Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?; + Streaming::Opened.call( + py, + ( + self.logger()?.object(py), + head.bind(py), + self.operation == LoggingOperation::Messages, + ), + )?; self.stream = Some(DeliveredStream { chunks: PyList::empty(py).unbind(), first_chunk: None, @@ -747,7 +905,7 @@ impl LegacyLogging { // A caller that goes away mid-stream cancels the call before it ends: Python's Messages // stream iterator still bills the chunks it read, straight away rather than deferred. fn bill_abandoned_stream(&mut self, py: Python<'_>) -> PyResult<()> { - if !self.asynchronous || self.end.is_some() { + if !self.asynchronous || self.end.is_some() || self.messages_cache_hit() { return Ok(()); } let Some(stream) = self.stream.take() else { @@ -918,6 +1076,24 @@ if asynchronous: }); } + #[rstest::rstest] + fn the_deployment_pre_call_hook_receives_the_logger_as_client_injects_it() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +kwargs = {'logger': logger} +logger.hooks['pre'] = lambda value: seen.append(value.get('litellm_logging_obj')) or 'awaitable' +seen = [] +", + ); + let (_, step) = begin(py, &locals, true); + assert!(awaits_deployment_hook(&step)); + run(py, &locals, c"assert seen == [logger]"); + }); + } + #[rstest::rstest] fn kwargs_returned_by_the_pre_call_hook_are_what_the_call_prepares() { Python::initialize(); @@ -1038,6 +1214,82 @@ assert finalized is replacement }); } + #[rstest] + #[case::unary(false)] + #[case::stream(true)] + fn a_host_answer_keeps_the_logger_as_set_up_and_a_streamed_one_is_finalized_unstreamed( + #[case] stream: bool, + ) { + use litellm_host::interceptors::{Cost, ExecutionFacts, ProviderIdentity, ResultSource}; + + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +kwargs = {'logger': logger} +response = object() +logger.hooks = {'pre': lambda kwargs: kwargs} +", + ); + local(&locals, "kwargs").set_item("stream", stream).unwrap(); + let (mut logging, step) = begin(py, &locals, true); + logging.operation = crate::LoggingOperation::Messages; + resume( + &mut logging, + step, + py, + Ok(local(&locals, "kwargs").unbind()), + ) + .unwrap(); + let facts = ExecutionFacts { + provider: ProviderIdentity { + model: "claude-sonnet-4-5".into(), + provider: "anthropic".into(), + }, + cost: Cost::Deferred, + source: ResultSource::Host, + }; + assert!(matches!( + logging.result_ready(py, &facts).unwrap(), + HookStep::Ready(()) + )); + let response = local(&locals, "response").unbind(); + let step = logging + .transform_response(py, response.clone_ref(py), TIMING) + .unwrap(); + assert_eq!(awaits_deployment_hook(&step), !stream); + if !stream { + return; + } + let HookStep::Ready(returned) = step else { + panic!("a host-answered stream is returned as is"); + }; + assert!(returned.bind(py).is(response.bind(py))); + logging + .on_event( + py, + PythonCallEvent::Succeeded { + timing: TIMING, + response: &response, + }, + ) + .unwrap(); + run( + py, + &locals, + c" +names = logger.names() +assert ('finalize', response) in logger.calls, names +assert 'update_logging' not in names, names +assert not [name for name in names if 'success' in name], names +assert logger.stream is True +assert logger.model_call_details['stream'] is True +", + ); + }); + } + #[rstest] #[case::pre_call(false)] #[case::post_call(true)] @@ -1129,7 +1381,9 @@ mod payload_tests { use std::ffi::CStr; use litellm_host::hooks::CallHooks; - use litellm_host::interceptors::{RawResponse, RequestContext, WireRequest}; + use litellm_host::interceptors::{ + RawResponse, RequestContext, RetriedRequest, RetryReason, WireRequest, + }; use litellm_host::lifecycle::ExecutionEvent; use litellm_host_python::{HookStep, PythonCallEvent, PythonOwned, to_py}; use litellm_secret_value::SecretValue; @@ -1296,6 +1550,61 @@ assert ('custom_llm_provider' in logger.update['litellm_params']) is in_params, json!({"type": "document_url", "document_url": source}) } + #[test] + fn a_retried_request_hands_the_logged_body_the_retried_body_and_the_reasons_to_logging() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace(py, PAYLOAD_LOGGER); + let mut logging = LegacyLogging { + logger: Some(PythonLogger::new(local(&locals, "logger").unbind())), + ..legacy_call(py, &locals, false) + }; + let context = RequestContext { + model: "model".into(), + custom_llm_provider: "provider".into(), + optional_params: json!({}), + secret_fields: Vec::new(), + api_key: None, + }; + let wire = WireRequest { + url: "https://provider.invalid/v1/messages".into(), + headers: Vec::new(), + body: json!({"messages": [{"role": "user", "content": "hi"}], "thinking": {"type": "enabled"}}), + }; + logging + .before_provider_request(py, Box::new(wire.clone()), &context) + .unwrap(); + let retried = RetriedRequest { + reasons: vec![ + RetryReason::ThinkingSignature, + RetryReason::StrictTools { dropped: 2 }, + ], + body: json!({"messages": [{"role": "user", "content": "hi"}]}), + }; + + let step = logging + .on_event( + py, + PythonCallEvent::Execution(ExecutionEvent::ProviderRequestRetried { + retry: retried, + }), + ) + .unwrap(); + + assert!(matches!(step, HookStep::Await(_, _))); + run( + py, + &locals, + c" +[(logged, body, reasons)] = [value for name, value in logger.calls if name == 'retried_request'] +assert logged is logger.pre['complete_input_dict'], logged +assert body == {'messages': [{'role': 'user', 'content': 'hi'}]}, body +assert reasons == [{'kind': 'thinking_signature'}, {'kind': 'strict_tools', 'dropped': 2}], reasons +", + ); + }); + } + fn before_provider_request(script: &CStr, body: Value) -> WireRequest { before_send_with_secrets(script, json!({}), body, &[]) } @@ -2018,6 +2327,53 @@ mod terminal_tests { .unwrap() } + #[rstest] + #[case::requested_stream( + c"kwargs = {'logger': logger, 'stream': True}", + true, + &["finalize"] + )] + #[case::unstreamed(c"kwargs = {'logger': logger, 'stream': False}", false, &["success_hook"])] + #[case::stream_flag_not_true(c"kwargs = {'logger': logger, 'stream': 'true'}", false, &["success_hook"])] + fn a_whole_messages_result_for_a_requested_stream_is_finalized_without_success_logging( + #[case] script: &CStr, + #[case] handed_back: bool, + #[case] expected: &[&str], + ) { + Python::initialize(); + Python::attach(|py| { + let locals = namespace(py, c"response = object()"); + run(py, &locals, script); + let mut logging = LegacyLogging { + operation: crate::LoggingOperation::Messages, + ..logged(py, &locals, true) + }; + let step = logging + .transform_public_response(py, local(&locals, "response").unbind(), TIMING) + .unwrap(); + assert_eq!(matches!(step, HookStep::Ready(_)), handed_back); + if handed_back { + assert!(matches!( + succeed(py, &locals, &mut logging), + HookStep::Ready(()) + )); + } + let names: Vec = local(&locals, "logger") + .call_method0("names") + .unwrap() + .extract() + .unwrap(); + assert_eq!(names, expected); + if handed_back { + run( + py, + &locals, + c"assert logger.stream is True and logger.model_call_details['stream'] is True", + ); + } + }); + } + #[rstest] #[case::sync_listened(false, c"", &["submit"])] #[case::async_listened( diff --git a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs index 093e86c43dc..9eed008910c 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs @@ -3,13 +3,19 @@ //! duplication. All of it expires with the legacy callback contract. use litellm_host::interceptors::{RequestContext, WireRequest}; -use litellm_host_python::to_py; +use litellm_host_python::json_to_py; use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict}; use serde_json::Value; use crate::logger::PythonLogger; use crate::python::{Logging, Wrapper}; +pub struct SentRequest<'a> { + pub url: &'a str, + pub headers: &'a Py, + pub body: &'a Py, +} + pub trait LegacyCallbacks { /// `Logging.update_from_kwargs`: what the logger is told about the request it is /// about to see, with consumed credentials redacted. @@ -41,6 +47,7 @@ pub trait LegacyCallbacks { status: u16, headers: &[(String, String)], body: &str, + request: Option>, ) -> PyResult<()>; /// Records one provider HTTP exchange's window, as Python's timed `AsyncHTTPHandler.post` @@ -117,9 +124,7 @@ impl LegacyCallbacks for PythonLogger { let redacted_kwargs = redact(py, kwargs, &secret_fields)?; let optional_params = redact( py, - &to_py(py, &context.optional_params)? - .into_bound(py) - .cast_into::()?, + &json_to_py(py, &context.optional_params)?.cast_into::()?, &secret_fields, )?; let params = PyDict::new(py); @@ -138,7 +143,7 @@ impl LegacyCallbacks for PythonLogger { } } for (name, value) in request_params.into_iter().flatten() { - params.set_item(name, to_py(py, value)?)?; + params.set_item(name, json_to_py(py, value)?)?; } Logging::Update.call( py, @@ -177,8 +182,13 @@ impl LegacyCallbacks for PythonLogger { status: u16, headers: &[(String, String)], body: &str, + request: Option>, ) -> PyResult<()> { - Logging::RecordResponse.call(py, (self.object(py), status, headers.to_vec(), body))?; + let request = request.map(|sent| (sent.url, sent.headers.bind(py), sent.body.bind(py))); + Logging::RecordResponse.call( + py, + (self.object(py), status, headers.to_vec(), body, request), + )?; Ok(()) } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/lib.rs b/litellm-rust/crates/callbacks-legacy-python/src/lib.rs index f453237b469..3a8c15ae54c 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/lib.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/lib.rs @@ -17,7 +17,7 @@ mod mapping; mod python; pub use adapter::LegacyLogging; pub use call::PublicCall; -pub(crate) use callbacks::{LegacyCallbacks, is_internal_call}; +pub(crate) use callbacks::{LegacyCallbacks, SentRequest, is_internal_call}; pub(crate) use logger::{DeploymentHooks, PythonLogger, finalize, setup}; pub use mapping::{CallBoundary, CallbackMapping, Dispatch, callback_mappings}; diff --git a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs index 38f3bf29828..3019d87a20b 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs @@ -77,8 +77,12 @@ pub fn finalize( kwargs: &Py, start: &Py, end: &Option>, + cache_hit: bool, ) -> PyResult<()> { - Wrapper::Finalize.call(py, (response, logger.object(py), kwargs, start, end))?; + Wrapper::Finalize.call( + py, + (response, logger.object(py), kwargs, start, end, cache_hit), + )?; Ok(()) } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs b/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs index cfa9e53aa7b..e321b93b03d 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs @@ -179,6 +179,9 @@ impl CallHooks for LegacyLogging { PythonCallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }) => { (AFTER.invoke)(self, py, raw) } + PythonCallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) => { + self.request_retried(py, &retry) + } PythonCallEvent::Succeeded { timing, response } => { (SUCCESS.invoke)(self, py, timing, response) } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs index 5b6faa3dbc2..58ddf7c708a 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs @@ -50,6 +50,10 @@ pub(crate) enum Logging { RecordResponse, #[strum(serialize = "record_api_window")] RecordApiWindow, + #[strum(serialize = "times_provider_exchange")] + TimesProviderExchange, + #[strum(serialize = "retried_request")] + RetriedRequest, #[strum(serialize = "defers_async_logging")] DefersAsync, #[strum(serialize = "claim_async_success_log")] @@ -62,6 +66,8 @@ pub(crate) enum Logging { SubmitSuccess, #[strum(serialize = "async_success_handler")] AsyncSuccessHandler, + #[strum(serialize = "cache_hit_success")] + CacheHitSuccess, #[strum(serialize = "mark_async_success_enqueued")] MarkAsyncSuccessEnqueued, #[strum(serialize = "enqueue_logging")] diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index ee16e0c0059..5b8b2bc4f96 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -53,7 +53,7 @@ def setup(call_type, args, kwargs, start, asynchronous): FAKES = { 'setup': setup, - 'finalize': lambda response, logger, kwargs, start, end: logger.record('finalize', response), + 'finalize': lambda response, logger, kwargs, start, end, cache_hit: logger.record('finalize', response), 'update_logging': lambda logger, kwargs, model, optional_params, litellm_params, provider: logger.update_from_kwargs( kwargs=kwargs, model=model, @@ -66,12 +66,17 @@ FAKES = { 'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call( original_response, api_key, additional_args ), - 'record_response': lambda logger, status, headers, body: logger.model_call_details.update( + 'record_response': lambda logger, status, headers, body, sent: logger.model_call_details.update( response_headers=dict(headers) ), + 'retried_request': lambda logger, logged_body, body, reasons: ( + logger.record('retried_request', (logged_body, body, reasons)), + 'awaitable', + )[1], 'record_api_window': lambda logger, start, end: logger.model_call_details.update( llm_api_duration_ms=(end - start) * 1000 ), + 'times_provider_exchange': lambda client, timeout: client is None, 'defers_async_logging': lambda logger: bool(getattr(logger, '_defer_async_logging', False)), 'claim_async_success_log': lambda logger: logger.claim_async_success_log(), 'defer_success': lambda logger, pending, response, start, end: ( @@ -86,6 +91,7 @@ FAKES = { )(error, ''.join(traceback.format_exception(error)), start, end), 'submit_success': lambda logger, response, start, end: logger.record('submit', (response, start, end)), 'async_success_handler': lambda logger, response, start, end: logger.async_success_handler(response, start, end), + 'cache_hit_success': lambda logger, response, start, end: logger.record('cache_hit_success', response), 'mark_async_success_enqueued': lambda logger: logger.record('mark_async_success_enqueued', None), 'enqueue_logging': lambda coroutine: coroutine.enqueue(), 'restore_context': lambda logger: logger.record('restore', None), @@ -96,8 +102,8 @@ FAKES = { 'success', response, call_type ), 'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type), - 'stream_opened': lambda logger, head: logger.record('stream_opened', head), - 'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head), + 'stream_opened': lambda logger, head, keeps_requested: logger.record('stream_opened', head), + 'stamp_stream_head': lambda head, logger, kwargs, start, end, cache_hit: logger.record('stamp_stream_head', head), 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk, stream_teardown: logger.record( 'stream_success', list(chunks) ), diff --git a/litellm-rust/crates/gateway-inference/tests/caching.rs b/litellm-rust/crates/gateway-inference/tests/caching.rs index 143bc1ba6db..030c8a66ee3 100644 --- a/litellm-rust/crates/gateway-inference/tests/caching.rs +++ b/litellm-rust/crates/gateway-inference/tests/caching.rs @@ -36,7 +36,12 @@ async fn all_inference_endpoints_share_native_cache( } else { "message_stop" }; - let events = format!("event: {terminal}\ndata: {{\"type\":\"{terminal}\"}}\n\n"); + let output = if is_responses { + "" + } else { + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hello\"}}\n\n" + }; + let events = format!("{output}event: {terminal}\ndata: {{\"type\":\"{terminal}\"}}\n\n"); let template = if stream { ResponseTemplate::new(200).set_body_raw(events.clone(), "text/event-stream") } else { diff --git a/litellm-rust/crates/host-native/src/driver.rs b/litellm-rust/crates/host-native/src/driver.rs index af3248848a2..bb1a7025e10 100644 --- a/litellm-rust/crates/host-native/src/driver.rs +++ b/litellm-rust/crates/host-native/src/driver.rs @@ -84,6 +84,13 @@ where reply.send(self.interceptors.after_provider_response(raw).await); Ok(()) } + HostRequest::Intercept(InterceptRequest::ProviderRequestRetried { + retry, + reply, + }) => { + reply.send(self.interceptors.provider_request_retried(retry).await); + Ok(()) + } HostRequest::Stream(StreamDelivery::Open(head, reply)) => { self.demand = Some(reply); return Ok(Boundary::Open(head)); diff --git a/litellm-rust/crates/host-python/Cargo.toml b/litellm-rust/crates/host-python/Cargo.toml index 0b0107d0548..1fa5fb38830 100644 --- a/litellm-rust/crates/host-python/Cargo.toml +++ b/litellm-rust/crates/host-python/Cargo.toml @@ -14,6 +14,7 @@ bytes.workspace = true futures-util.workspace = true parking_lot.workspace = true serde.workspace = true +serde_json.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["rt", "sync"] } @@ -23,4 +24,3 @@ pythonize = "0.29.0" [dev-dependencies] rstest.workspace = true -serde_json.workspace = true diff --git a/litellm-rust/crates/host-python/src/binding.rs b/litellm-rust/crates/host-python/src/binding.rs index fe6173dd100..97d6be88767 100644 --- a/litellm-rust/crates/host-python/src/binding.rs +++ b/litellm-rust/crates/host-python/src/binding.rs @@ -29,8 +29,8 @@ pub trait PythonBinding: PythonOwned { /// What the stream carries at hand-off, as the caller's stream receives it. /// The unary response after the deployment transform, before success hooks run. - fn deliver_response(&mut self, _py: Python<'_>, _response: &Bound<'_, PyAny>) -> PyResult<()> { - Ok(()) + fn deliver_response(&mut self, _py: Python<'_>, response: Py) -> PyResult> { + Ok(response) } fn encode_stream_head( diff --git a/litellm-rust/crates/host-python/src/driver.rs b/litellm-rust/crates/host-python/src/driver.rs index 052156cbc5d..ba0c174c875 100644 --- a/litellm-rust/crates/host-python/src/driver.rs +++ b/litellm-rust/crates/host-python/src/driver.rs @@ -310,6 +310,14 @@ where reply.send(Ok(())); Ok(()) } + HostRequest::Intercept(InterceptRequest::ProviderRequestRetried { retry, reply }) => { + self.notify(PythonCallEvent::Execution( + ExecutionEvent::ProviderRequestRetried { retry }, + )) + .await?; + reply.send(Ok(())); + Ok(()) + } HostRequest::Stream(StreamDelivery::Open(head, reply)) => { self.streaming = true; let head = { @@ -405,10 +413,10 @@ where }; let delivered = { let mut parts = self.parts(); - Python::attach(|py| parts.binding.deliver_response(py, response.bind(py))) + Python::attach(|py| parts.binding.deliver_response(py, response)) }; match delivered { - Ok(()) => self.succeeded(response).await, + Ok(response) => self.succeeded(response).await, Err(error) => self.failure(error, FailureOrigin::Host).await, } } diff --git a/litellm-rust/crates/host-python/src/hooks/chain/dispatch.rs b/litellm-rust/crates/host-python/src/hooks/chain/dispatch.rs index 4939995cefa..26d5a5692d2 100644 --- a/litellm-rust/crates/host-python/src/hooks/chain/dispatch.rs +++ b/litellm-rust/crates/host-python/src/hooks/chain/dispatch.rs @@ -241,6 +241,9 @@ fn retain_event(py: Python<'_>, event: PythonCallEvent<'_>) -> OwnedEvent { CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }) => { CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }) } + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) => { + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) + } CallEvent::Succeeded { timing, response } => CallEvent::Succeeded { timing, response: response.clone_ref(py), @@ -280,6 +283,12 @@ fn dispatch( py, CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }), ), + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) => hooks.on_event( + py, + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { + retry: retry.clone(), + }), + ), CallEvent::Succeeded { timing, response } => hooks.on_event( py, CallEvent::Succeeded { diff --git a/litellm-rust/crates/host-python/src/lib.rs b/litellm-rust/crates/host-python/src/lib.rs index a9ebae427d9..1cd4d65e29e 100644 --- a/litellm-rust/crates/host-python/src/lib.rs +++ b/litellm-rust/crates/host-python/src/lib.rs @@ -31,7 +31,8 @@ pub use gil::{PythonContext, attach_blocking, release_count, release_gil}; pub use handle::{Execution, ExecutionBody, ExecutionStep, StepTypes, Suspension}; pub use hooks::{HookChain, HookResume, HookStep, PythonCallEvent, PythonCallHooks, PythonRuntime}; pub use marshal::{ - Pythonized, from_py, from_py_argument, json_loads, json_object_field, panic_to_pyerr, to_py, + Pythonized, from_py, from_py_argument, json_from_py, json_loads, json_object_field, json_to_py, + panic_to_pyerr, to_py, }; pub use owned::PythonOwned; pub use runtime::{ diff --git a/litellm-rust/crates/host-python/src/marshal.rs b/litellm-rust/crates/host-python/src/marshal.rs index 465521cf9d2..a6468d38d83 100644 --- a/litellm-rust/crates/host-python/src/marshal.rs +++ b/litellm-rust/crates/host-python/src/marshal.rs @@ -4,9 +4,10 @@ use std::panic::{AssertUnwindSafe, catch_unwind}; use pyo3::exceptions::PyValueError; use pyo3::panic::PanicException; use pyo3::prelude::*; -use pyo3::types::PyBytes; +use pyo3::types::{PyBool, PyBytes, PyDict, PyFloat, PyInt, PyList, PyString, PyTuple}; use serde::Serialize; use serde::de::DeserializeOwned; +use serde_json::{Map, Number, Value}; /// Converts a `#[pyo3(from_py_with = ...)]` argument, reporting failures as `ValueError` /// so a bad argument reads as a bad argument rather than as whatever the conversion hit. @@ -71,6 +72,119 @@ pub fn panic_to_pyerr(payload: Box) -> PyErr { PanicException::new_err(message.to_string()) } +pub fn json_from_py(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(Value::Null); + } + if let Ok(flag) = value.cast::() { + return Ok(Value::Bool(flag.is_true())); + } + if let Ok(int) = value.cast::() { + if let Ok(number) = int.extract::() { + return Ok(Value::from(number)); + } + if let Ok(number) = int.extract::() { + return Ok(Value::from(number)); + } + return raw_leaf(value); + } + if let Ok(float) = value.cast::() { + return match Number::from_f64(float.value()) { + Some(number) => Ok(Value::Number(number)), + None => raw_leaf(value), + }; + } + if let Ok(text) = value.cast::() { + return json_string(text); + } + if let Ok(dict) = value.cast::() { + if !dict + .keys() + .iter() + .all(|key| key.is_instance_of::()) + { + return from_py(value); + } + return dict + .iter() + .map(|(key, item)| Ok((json_key(key.cast::()?)?, json_from_py(&item)?))) + .collect::>>() + .map(Value::Object); + } + if let Ok(list) = value.cast::() { + return list + .iter() + .map(|item| json_from_py(&item)) + .collect::>() + .map(Value::Array); + } + if let Ok(tuple) = value.cast::() { + return tuple + .iter() + .map(|item| json_from_py(&item)) + .collect::>() + .map(Value::Array); + } + from_py(value) +} + +pub fn json_to_py<'py>(py: Python<'py>, value: &Value) -> PyResult> { + Ok(match value { + Value::Null => py.None().into_bound(py), + Value::Bool(flag) => PyBool::new(py, *flag).to_owned().into_any(), + Value::Number(number) => match (number.as_i64(), number.as_u64()) { + (Some(int), _) => int.into_pyobject(py)?.into_any(), + (None, Some(int)) => int.into_pyobject(py)?.into_any(), + (None, None) => PyFloat::new(py, number.as_f64().unwrap_or(f64::NAN)).into_any(), + }, + Value::String(text) => match litellm_host::raw_json::decode(text) { + Some(raw) => py.import("json")?.call_method1("loads", (raw,))?, + None => PyString::new(py, text).into_any(), + }, + Value::Array(items) => PyList::new( + py, + items + .iter() + .map(|item| json_to_py(py, item)) + .collect::>>()?, + )? + .into_any(), + Value::Object(fields) => { + let dict = PyDict::new(py); + for (key, item) in fields { + dict.set_item( + json_to_py(py, &Value::String(key.clone()))?, + json_to_py(py, item)?, + )?; + } + dict.into_any() + } + }) +} + +fn json_key(text: &Bound<'_, PyString>) -> PyResult { + match json_string(text)? { + Value::String(key) => Ok(key), + _ => unreachable!("a Python str converts to a JSON string"), + } +} + +fn json_string(text: &Bound<'_, PyString>) -> PyResult { + match text.to_str() { + Ok(text) if !litellm_host::raw_json::is_marked(text) => Ok(Value::String(text.to_owned())), + _ => raw_leaf(text.as_any()), + } +} + +fn raw_leaf(value: &Bound<'_, PyAny>) -> PyResult { + let raw: String = value + .py() + .import("json")? + .call_method1("dumps", (value,))? + .extract()?; + Ok(Value::String(litellm_host::raw_json::encode(&raw))) +} + #[cfg(test)] mod tests { use serde::Serializer; @@ -131,6 +245,72 @@ mod tests { }); } + #[rstest::rstest] + #[case::plain( + c"{'a': [1, 2.5, None, True, 'x'], 'b': (1,)}", + r#"{"a":[1,2.5,null,true,"x"],"b":[1]}"# + )] + #[case::nan(c"{'t': float('nan')}", r#"{"t":NaN}"#)] + #[case::infinities(c"[float('inf'), -float('inf')]", "[Infinity,-Infinity]")] + #[case::wide_integers( + c"[2**64, -(2**70), 2**64 - 1, -(2**63)]", + "[18446744073709551616,-1180591620717411303424,18446744073709551615,-9223372036854775808]" + )] + #[case::lone_surrogates(c"{'k\\udc00': 'a\\ud800b \\u00e9'}", r#"{"k\udc00":"a\ud800b é"}"#)] + #[case::marked_text(c"['\\uf8fflitellm-raw-json:1']", r#"["litellm-raw-json:1"]"#)] + fn json_values_keep_what_serde_json_cannot_hold( + #[case] source: &std::ffi::CStr, + #[case] wire: &str, + ) { + crate::initialize_python(); + Python::attach(|py| { + let value = py.eval(source, None, None).unwrap(); + let converted = json_from_py(&value).unwrap(); + let json = py.import("json").unwrap(); + let loads = |text: &str| json.call_method1("loads", (text,)).unwrap(); + let wire_bytes = litellm_host::raw_json::to_vec(&converted).unwrap(); + assert!( + loads(std::str::from_utf8(&wire_bytes).unwrap()) + .eq(loads(wire)) + .unwrap() + || wire.contains("NaN") + ); + assert_eq!( + json.call_method1("dumps", (loads(std::str::from_utf8(&wire_bytes).unwrap()),)) + .unwrap() + .extract::() + .unwrap(), + json.call_method1("dumps", (loads(wire),)) + .unwrap() + .extract::() + .unwrap() + ); + let back = json_to_py(py, &converted).unwrap(); + assert_eq!( + json.call_method1("dumps", (back,)) + .unwrap() + .extract::() + .unwrap(), + json.call_method1("dumps", (loads(wire),)) + .unwrap() + .extract::() + .unwrap() + ); + }); + } + + #[test] + fn values_json_cannot_hold_keep_their_old_conversion() { + crate::initialize_python(); + Python::attach(|py| { + let value = py.eval(c"{1: 'a'}", None, None).unwrap(); + assert_eq!( + json_from_py(&value).unwrap_err().to_string(), + from_py::(&value).unwrap_err().to_string() + ); + }); + } + #[test] fn depythonize_preserves_python_exception_identity_and_traceback() { crate::initialize_python(); diff --git a/litellm-rust/crates/host-python/tests/support/mod.rs b/litellm-rust/crates/host-python/tests/support/mod.rs index 15e864c0718..f4f0903cc45 100644 --- a/litellm-rust/crates/host-python/tests/support/mod.rs +++ b/litellm-rust/crates/host-python/tests/support/mod.rs @@ -422,6 +422,9 @@ impl CallHooks for SyntheticHooks { PythonCallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }) => { format!("response:{}", raw.body) } + PythonCallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) => { + format!("retried:{}", retry.body) + } PythonCallEvent::Succeeded { response, .. } => { format!("succeeded:{}", response.bind(py)) } diff --git a/litellm-rust/crates/host/Cargo.toml b/litellm-rust/crates/host/Cargo.toml index e0ccab4e0ad..2c113bc644c 100644 --- a/litellm-rust/crates/host/Cargo.toml +++ b/litellm-rust/crates/host/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true litellm-secret-value.workspace = true futures-util.workspace = true litellm-coroutine.workspace = true +serde.workspace = true serde_json.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["sync"] } diff --git a/litellm-rust/crates/host/src/interceptors.rs b/litellm-rust/crates/host/src/interceptors.rs index 9398ea4bad6..8146448cd98 100644 --- a/litellm-rust/crates/host/src/interceptors.rs +++ b/litellm-rust/crates/host/src/interceptors.rs @@ -10,6 +10,18 @@ pub struct WireRequest { pub body: Value, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RetryReason { + ThinkingSignature, + StrictTools { dropped: usize }, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RetriedRequest { + pub reasons: Vec, + pub body: Value, +} + /// What the route knows about the request it is sending, for a host that logs it. The /// route owns these facts; a host reads them beside the wire request and never rewrites /// them. @@ -62,6 +74,7 @@ pub struct ProviderIdentity { pub enum ResultSource { Provider, Cache { key: String }, + Host, } /// What the call cost, settled once by the route. A host records it; it never re-derives it. @@ -79,6 +92,7 @@ impl Cost { pub fn settle(source: &ResultSource, reported: Option) -> Self { match (source, reported) { (ResultSource::Cache { .. }, _) => Self::CacheHit, + (ResultSource::Host, _) => Self::Deferred, (ResultSource::Provider, Some(amount)) => Self::Reported { amount }, (ResultSource::Provider, None) => Self::Deferred, } @@ -107,6 +121,13 @@ pub trait Interceptors: Send + Sync { &self, raw: RawResponse, ) -> impl Future> + Send; + + fn provider_request_retried( + &self, + _retry: RetriedRequest, + ) -> impl Future> + Send { + async { Ok(()) } + } } impl + ?Sized> Interceptors for &T { @@ -128,6 +149,13 @@ impl + ?Sized> Interceptors for &T { ) -> impl Future> + Send { (**self).after_provider_response(raw) } + + fn provider_request_retried( + &self, + retry: RetriedRequest, + ) -> impl Future> + Send { + (**self).provider_request_retried(retry) + } } impl Interceptors for () { diff --git a/litellm-rust/crates/host/src/lib.rs b/litellm-rust/crates/host/src/lib.rs index 76fce582fb3..26c53c89f97 100644 --- a/litellm-rust/crates/host/src/lib.rs +++ b/litellm-rust/crates/host/src/lib.rs @@ -13,3 +13,4 @@ pub mod lifecycle; pub mod machine; pub mod observation; pub mod protocol; +pub mod raw_json; diff --git a/litellm-rust/crates/host/src/lifecycle.rs b/litellm-rust/crates/host/src/lifecycle.rs index e6b31381494..16456ea52d0 100644 --- a/litellm-rust/crates/host/src/lifecycle.rs +++ b/litellm-rust/crates/host/src/lifecycle.rs @@ -6,7 +6,10 @@ use std::{ use futures_util::TryStreamExt; -use crate::{call::CallOutput, interceptors::RawResponse}; +use crate::{ + call::CallOutput, + interceptors::{RawResponse, RetriedRequest}, +}; /// Seconds since the Unix epoch, on one clock for every host. pub fn epoch_seconds() -> f64 { @@ -50,7 +53,7 @@ pub enum CallEvent { }, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub enum ExecutionEvent { ResultReady { facts: crate::interceptors::ExecutionFacts, @@ -58,6 +61,9 @@ pub enum ExecutionEvent { ProviderResponseReceived { raw: Raw, }, + ProviderRequestRetried { + retry: RetriedRequest, + }, } impl> CallEvent { @@ -76,6 +82,11 @@ impl> CallEvent { + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { + retry: retry.clone(), + }) + } Self::Succeeded { timing, .. } => CallEvent::Succeeded { timing: *timing, response: (), diff --git a/litellm-rust/crates/host/src/machine/context.rs b/litellm-rust/crates/host/src/machine/context.rs index 1e53605ed7b..dacb470e97f 100644 --- a/litellm-rust/crates/host/src/machine/context.rs +++ b/litellm-rust/crates/host/src/machine/context.rs @@ -5,7 +5,7 @@ use litellm_coroutine::Co; use super::coroutine::MachineFault; use crate::{ - interceptors::{RawResponse, RequestContext, WireRequest}, + interceptors::{RawResponse, RequestContext, RetriedRequest, WireRequest}, protocol::{HostRequest, InterceptRequest, Protocol, Reply, StreamDelivery}, }; @@ -119,6 +119,14 @@ where }) .await? } + + async fn provider_request_retried(&self, retry: RetriedRequest) -> Result<(), P::Error> { + self.0 + .request_reply(|reply| { + HostRequest::Intercept(InterceptRequest::ProviderRequestRetried { retry, reply }) + }) + .await? + } } pub struct StreamSender(Channel

); diff --git a/litellm-rust/crates/host/src/protocol.rs b/litellm-rust/crates/host/src/protocol.rs index c24557b0348..5ec566b6a65 100644 --- a/litellm-rust/crates/host/src/protocol.rs +++ b/litellm-rust/crates/host/src/protocol.rs @@ -2,7 +2,7 @@ use std::ops::ControlFlow; pub use litellm_coroutine::{Abandoned, Answer, Reply, reply}; -use crate::interceptors::{RawResponse, RequestContext, WireRequest}; +use crate::interceptors::{RawResponse, RequestContext, RetriedRequest, WireRequest}; pub trait Protocol: Send + Sync + 'static { type Request: Send + 'static; @@ -33,6 +33,10 @@ pub enum InterceptRequest { raw: RawResponse, reply: Reply>, }, + ProviderRequestRetried { + retry: RetriedRequest, + reply: Reply>, + }, } pub enum StreamDelivery { diff --git a/litellm-rust/crates/host/src/raw_json.rs b/litellm-rust/crates/host/src/raw_json.rs new file mode 100644 index 00000000000..fbf9bbc6ecb --- /dev/null +++ b/litellm-rust/crates/host/src/raw_json.rs @@ -0,0 +1,362 @@ +use std::{borrow::Cow, io}; + +use serde::Serialize; +use serde_json::ser::{CharEscape, CompactFormatter, Formatter}; + +const MARK: char = '\u{F8FF}'; +const PREFIX: &str = "\u{F8FF}litellm-raw-json:"; + +pub fn encode(raw: &str) -> String { + format!("{PREFIX}{raw}") +} + +pub fn decode(text: &str) -> Option<&str> { + text.strip_prefix(PREFIX).filter(|raw| is_scalar(raw)) +} + +pub fn is_marked(text: &str) -> bool { + text.starts_with(PREFIX) +} + +pub fn to_vec(value: &T) -> serde_json::Result> { + let mut out = Vec::with_capacity(128); + let mut serializer = serde_json::Serializer::with_formatter(&mut out, RawLeaves::default()); + value.serialize(&mut serializer)?; + Ok(out) +} + +pub fn mark_wide_numbers(text: &str) -> Cow<'_, str> { + let bytes = text.as_bytes(); + let mut out = String::new(); + let mut copied = 0; + let mut index = 0; + while index < bytes.len() { + match bytes[index] { + b'"' => { + let end = string_end(bytes, index + 1); + if let Some(escaped) = escaped_marked_string(&text[index..end]) { + out.push_str(&text[copied..index]); + out.push_str(&escaped); + copied = end; + } + index = end; + } + b'-' | b'0'..=b'9' => { + let end = bytes[index..] + .iter() + .position(|byte| { + !matches!(byte, b'0'..=b'9' | b'-' | b'+' | b'.' | b'e' | b'E') + }) + .map_or(bytes.len(), |length| index + length); + if let Some(raw) = wide_number(&text[index..end]) { + out.push_str(&text[copied..index]); + out.push_str(&serde_json::Value::String(encode(raw)).to_string()); + copied = end; + } + index = end; + } + _ => index += 1, + } + } + if copied == 0 { + return Cow::Borrowed(text); + } + out.push_str(&text[copied..]); + Cow::Owned(out) +} + +fn escaped_marked_string(token: &str) -> Option { + let inner = token.strip_prefix('"')?; + let escaped_mark = inner + .get(..6) + .is_some_and(|head| head.eq_ignore_ascii_case("\\uf8ff")); + if !inner.starts_with(MARK) && !escaped_mark { + return None; + } + let text = serde_json::from_str::(token).ok()?; + is_marked(&text) + .then(|| serde_json::Value::String(encode(&ascii_string_literal(&text))).to_string()) +} + +fn ascii_string_literal(text: &str) -> String { + let mut literal = String::with_capacity(text.len() + 2); + literal.push('"'); + for character in text.chars() { + match character { + '"' => literal.push_str("\\\""), + '\\' => literal.push_str("\\\\"), + '\n' => literal.push_str("\\n"), + '\r' => literal.push_str("\\r"), + '\t' => literal.push_str("\\t"), + ' '..='~' => literal.push(character), + _ => { + let mut units = [0; 2]; + for unit in character.encode_utf16(&mut units) { + literal.push_str(&format!("\\u{unit:04x}")); + } + } + } + } + literal.push('"'); + literal +} + +fn string_end(bytes: &[u8], mut index: usize) -> usize { + while index < bytes.len() { + match bytes[index] { + b'\\' => index += 2, + b'"' => return index + 1, + _ => index += 1, + } + } + index +} + +fn wide_number(raw: &str) -> Option<&str> { + if is_integer(raw) { + return (raw.parse::().is_err() && raw.parse::().is_err()).then_some(raw); + } + match raw.parse::() { + Ok(number) if number == f64::INFINITY => Some("Infinity"), + Ok(number) if number == f64::NEG_INFINITY => Some("-Infinity"), + _ => None, + } +} + +fn is_scalar(raw: &str) -> bool { + matches!(raw, "NaN" | "Infinity" | "-Infinity") || is_integer(raw) || is_string_literal(raw) +} + +fn is_integer(raw: &str) -> bool { + let digits = raw.strip_prefix('-').unwrap_or(raw); + match digits.as_bytes() { + [b'0'] => true, + [b'1'..=b'9', rest @ ..] => rest.iter().all(u8::is_ascii_digit), + _ => false, + } +} + +fn is_string_literal(raw: &str) -> bool { + let Some(inner) = raw + .strip_prefix('"') + .and_then(|rest| rest.strip_suffix('"')) + else { + return false; + }; + let mut bytes = inner.bytes(); + while let Some(byte) = bytes.next() { + match byte { + b'\\' => match bytes.next() { + Some(b'"' | b'\\' | b'/' | b'b' | b'f' | b'n' | b'r' | b't') => {} + Some(b'u') => { + if !(0..4).all(|_| bytes.next().is_some_and(|hex| hex.is_ascii_hexdigit())) { + return false; + } + } + _ => return false, + }, + b'"' => return false, + 0x20..=0x7e => {} + _ => return false, + } + } + true +} + +fn unescaped(escape: CharEscape) -> char { + match escape { + CharEscape::Quote => '"', + CharEscape::ReverseSolidus => '\\', + CharEscape::Solidus => '/', + CharEscape::Backspace => '\u{8}', + CharEscape::FormFeed => '\u{c}', + CharEscape::LineFeed => '\n', + CharEscape::CarriageReturn => '\r', + CharEscape::Tab => '\t', + CharEscape::AsciiControl(byte) => char::from(byte), + } +} + +#[derive(Default)] +enum Text { + #[default] + Closed, + Opened, + Plain, + Marked(String), +} + +#[derive(Default)] +struct RawLeaves { + key: bool, + text: Text, +} + +impl Formatter for RawLeaves { + fn begin_object_key( + &mut self, + writer: &mut W, + first: bool, + ) -> io::Result<()> { + self.key = true; + CompactFormatter.begin_object_key(writer, first) + } + + fn end_object_key(&mut self, writer: &mut W) -> io::Result<()> { + self.key = false; + CompactFormatter.end_object_key(writer) + } + + fn begin_string(&mut self, _writer: &mut W) -> io::Result<()> { + self.text = Text::Opened; + Ok(()) + } + + fn write_string_fragment( + &mut self, + writer: &mut W, + fragment: &str, + ) -> io::Result<()> { + match &mut self.text { + Text::Opened if fragment.starts_with(MARK) => { + self.text = Text::Marked(fragment.to_owned()); + Ok(()) + } + Text::Opened => { + self.text = Text::Plain; + writer.write_all(b"\"")?; + writer.write_all(fragment.as_bytes()) + } + Text::Marked(held) => { + held.push_str(fragment); + Ok(()) + } + Text::Plain | Text::Closed => writer.write_all(fragment.as_bytes()), + } + } + + fn write_char_escape( + &mut self, + writer: &mut W, + escape: CharEscape, + ) -> io::Result<()> { + match &mut self.text { + Text::Opened => { + self.text = Text::Plain; + writer.write_all(b"\"")?; + CompactFormatter.write_char_escape(writer, escape) + } + Text::Marked(held) => { + held.push(unescaped(escape)); + Ok(()) + } + Text::Plain | Text::Closed => CompactFormatter.write_char_escape(writer, escape), + } + } + + fn end_string(&mut self, writer: &mut W) -> io::Result<()> { + match std::mem::take(&mut self.text) { + Text::Opened => writer.write_all(b"\"\""), + Text::Plain | Text::Closed => writer.write_all(b"\""), + Text::Marked(held) => match decode(&held) { + Some(raw) if !self.key || raw.starts_with('"') => writer.write_all(raw.as_bytes()), + _ => serde_json::to_writer(writer, &held).map_err(io::Error::other), + }, + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + + use super::*; + + #[rstest] + #[case::nan("NaN")] + #[case::infinity("Infinity")] + #[case::negative_infinity("-Infinity")] + #[case::wide_integer("18446744073709551616")] + #[case::negative_wide_integer("-1180591620717411303424")] + #[case::zero("0")] + #[case::surrogate_string(r#""sys \ud800 lone""#)] + #[case::escaped_string(r#""a\"b\\c\n\u00e9""#)] + fn scalar_texts_are_written_back_verbatim(#[case] raw: &str) { + let body = json!({"leaf": encode(raw), "list": [encode(raw)]}); + assert_eq!( + String::from_utf8(to_vec(&body).unwrap()).unwrap(), + format!(r#"{{"leaf":{raw},"list":[{raw}]}}"#) + ); + } + + #[rstest] + #[case::structure("1, \"injected\": true")] + #[case::object("{}")] + #[case::leading_zero("012")] + #[case::float("1.5")] + #[case::unquoted_quote(r#""a"b""#)] + #[case::non_ascii("\"\u{e9}\"")] + #[case::bad_escape(r#""\x""#)] + #[case::short_unicode_escape(r#""\u12""#)] + #[case::empty("")] + fn other_marked_texts_stay_strings(#[case] raw: &str) { + let marked = encode(raw); + assert_eq!(decode(&marked), None); + assert_eq!( + to_vec(&json!([marked])).unwrap(), + serde_json::to_vec(&json!([marked])).unwrap() + ); + } + + #[test] + fn keys_take_only_string_texts() { + let body = Value::Object( + [ + (encode(r#""k\udc00""#), json!(1)), + (encode("NaN"), json!(2)), + ] + .into_iter() + .collect(), + ); + assert_eq!( + String::from_utf8(to_vec(&body).unwrap()).unwrap(), + format!( + r#"{{"k\udc00":1,{}:2}}"#, + serde_json::to_string(&encode("NaN")).unwrap() + ) + ); + } + + #[rstest] + #[case::wide_integer(r#"{"a":18446744073709551616}"#, json!({"a": encode("18446744073709551616")}))] + #[case::negative_wide_integer(r#"[-12345678901234567890123]"#, json!([encode("-12345678901234567890123")]))] + #[case::overflowing_float(r#"{"a":[1e400,-1E+400]}"#, json!({"a": [encode("Infinity"), encode("-Infinity")]}))] + #[case::numbers_that_fit(r#"{"a":[18446744073709551615,-9223372036854775808,-0.0,5e-324,1E2,0.30000000000000004]}"#, json!({"a": [18_446_744_073_709_551_615_u64, i64::MIN, -0.0, 5e-324, 100.0, 0.300_000_000_000_000_04]}))] + #[case::provider_marked_text("{\"a\":\"\u{F8FF}litellm-raw-json:42\"}", json!({"a": encode(r#""\uf8fflitellm-raw-json:42""#)}))] + #[case::provider_escaped_mark(r#"{"\uF8FFlitellm-raw-json:1":["\uf8fflitellm-raw-json:\"x\"\n\u00e9"]}"#, json!({encode(r#""\uf8fflitellm-raw-json:1""#): [encode(r#""\uf8fflitellm-raw-json:\"x\"\n\u00e9""#)]}))] + #[case::mark_not_leading(r#"["x\uf8fflitellm-raw-json:NaN"]"#, json!(["x\u{F8FF}litellm-raw-json:NaN"]))] + #[case::numbers_inside_strings(r#"{"18446744073709551616":"1e400 \" 18446744073709551616"}"#, json!({"18446744073709551616": "1e400 \" 18446744073709551616"}))] + fn wide_numbers_are_read_as_marked_texts(#[case] text: &str, #[case] read: Value) { + assert_eq!( + serde_json::from_str::(&mark_wide_numbers(text)).unwrap(), + read + ); + } + + #[test] + fn a_text_without_wide_numbers_is_borrowed() { + assert!(matches!( + mark_wide_numbers(r#"{"a":[1,2.5,"18446744073709551616"]}"#), + Cow::Borrowed(_) + )); + } + + #[rstest] + #[case::plain(json!({"a": "text", "b": ["", "\"quoted\"\n", "\u{F8FF}"], "c": 1.5, "d": null}))] + #[case::mark_inside(json!(["x\u{F8FF}litellm-raw-json:NaN"]))] + #[case::control(json!(["\u{1}\t"]))] + fn unmarked_documents_serialize_as_serde_json_does(#[case] body: Value) { + assert_eq!(to_vec(&body).unwrap(), serde_json::to_vec(&body).unwrap()); + } +} diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index 5e5f2d66fa2..8e9a7000e69 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -94,6 +94,8 @@ impl From<&HttpSettings> for Resolution { } } +const MAX_REDIRECTS: usize = 20; + impl TryFrom<&HttpClientConfig> for reqwest::ClientBuilder { type Error = Error; @@ -101,7 +103,9 @@ impl TryFrom<&HttpClientConfig> for reqwest::ClientBuilder { let base = reqwest::Client::builder() .tls_backend_preconfigured(rustls::ClientConfig::try_from(config)?) .connect_timeout(config.connect_timeout) - .pool_idle_timeout(config.pool_idle_timeout); + .pool_idle_timeout(config.pool_idle_timeout) + .referer(false) + .redirect(reqwest::redirect::Policy::limited(MAX_REDIRECTS)); let with_keepalive = match config.tcp_keepalive { None => base, Some(keepalive) => base @@ -186,17 +190,23 @@ mod tests { assert_eq!(resolution.unsupported, []); } - #[test] - fn unsupported_ecdh_curve_keeps_the_defaults_and_is_reported() { + #[rstest] + #[case::openssl_lacks_in_rustls("secp521r1")] + #[case::nist_name("P-256")] + #[case::nist_p384("P-384")] + #[case::sec_name("secp256r1")] + #[case::lowercase("x25519")] + #[case::padded(" X25519 ")] + fn unsupported_ecdh_curve_keeps_the_defaults_and_is_reported(#[case] curve: &str) { let settings = HttpSettings { - ssl_ecdh_curve: Some("secp521r1".into()), + ssl_ecdh_curve: Some(curve.into()), ..HttpSettings::default() }; let resolution = Resolution::from(&settings); assert_eq!(resolution.config.key_exchange_group, None); assert_eq!( resolution.unsupported, - [Unsupported::EcdhCurve("secp521r1".into())] + [Unsupported::EcdhCurve(curve.into())] ); } diff --git a/litellm-rust/crates/http/src/error.rs b/litellm-rust/crates/http/src/error.rs index 9505aacf44b..d9fdfb5486f 100644 --- a/litellm-rust/crates/http/src/error.rs +++ b/litellm-rust/crates/http/src/error.rs @@ -251,6 +251,10 @@ impl Error { Repr::ComputedHeader(name.into()).into() } + pub fn request_body(error: serde_json::Error) -> Self { + Repr::RequestBody(error).into() + } + pub fn unsignable_body() -> Self { Repr::UnsignableBody.into() } diff --git a/litellm-rust/crates/http/src/outbound.rs b/litellm-rust/crates/http/src/outbound.rs index 20b0b1be972..b11041151af 100644 --- a/litellm-rust/crates/http/src/outbound.rs +++ b/litellm-rust/crates/http/src/outbound.rs @@ -27,7 +27,20 @@ pub fn json( body: &impl Serialize, timeout: Option, ) -> Result { - let body = serde_json::to_vec(body).map_err(Repr::RequestBody)?; + json_bytes( + url, + headers, + serde_json::to_vec(body).map_err(Repr::RequestBody)?, + timeout, + ) +} + +pub fn json_bytes( + url: &str, + headers: &[(String, String)], + body: Vec, + timeout: Option, +) -> Result { let url = Url::parse(url).map_err(Repr::ParseUrl)?; let with_headers = headers.iter().fold( http::Request::post(url.as_str()), diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index 3b20bbda607..c56c5de4a11 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -16,6 +16,7 @@ use crate::{ #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum ClientVariant { Provider, + Undecoded, NoRedirect, Media, UnpinnedMedia, @@ -103,6 +104,7 @@ impl HttpClientPool { ) -> reqwest::ClientBuilder { match variant { ClientVariant::Provider => builder, + ClientVariant::Undecoded => builder.no_gzip().no_deflate().no_brotli().no_zstd(), ClientVariant::NoRedirect | ClientVariant::UnpinnedMedia | ClientVariant::WebSocket => { builder.redirect(reqwest::redirect::Policy::none()) } @@ -132,7 +134,9 @@ fn effective(config: &HttpClientConfig, variant: ClientVariant) -> HttpClientCon }, ..config.clone() }, - ClientVariant::Provider | ClientVariant::NoRedirect => config.clone(), + ClientVariant::Provider | ClientVariant::Undecoded | ClientVariant::NoRedirect => { + config.clone() + } } } @@ -367,6 +371,28 @@ mod tests { assert!(request.contains("user-agent: litellm-test/9"), "{request}"); } + #[tokio::test] + async fn provider_client_follows_twenty_redirects_without_a_referer() { + let (address, _, requests) = serve("HTTP/1.1 307 Temporary Redirect").await; + let followed = pool() + .client(&config("a"), ClientVariant::Provider) + .unwrap() + .get(format!("http://{address}/start")) + .timeout(Duration::from_secs(5)) + .send() + .await; + assert!(followed.is_err()); + let requests = requests.lock().unwrap().clone(); + assert_eq!(requests.len(), 21, "{requests:?}"); + assert!(requests[1].starts_with("GET /elsewhere "), "{requests:?}"); + assert!( + requests + .iter() + .all(|request| !request.to_ascii_lowercase().contains("referer:")), + "{requests:?}" + ); + } + #[tokio::test] async fn no_redirect_variant_returns_the_redirect_instead_of_following_it() { let (address, _, _) = serve("HTTP/1.1 302 Found").await; diff --git a/litellm-rust/crates/http/src/tls.rs b/litellm-rust/crates/http/src/tls.rs index a83b0018b2b..47a1429f1a4 100644 --- a/litellm-rust/crates/http/src/tls.rs +++ b/litellm-rust/crates/http/src/tls.rs @@ -23,10 +23,10 @@ impl FromStr for KeyExchangeGroup { type Err = Unsupported; fn from_str(name: &str) -> Result { - match name.trim().to_ascii_lowercase().as_str() { - "x25519" => Ok(Self::X25519), - "prime256v1" | "secp256r1" | "p-256" => Ok(Self::Secp256r1), - "secp384r1" | "p-384" => Ok(Self::Secp384r1), + match name { + "X25519" => Ok(Self::X25519), + "prime256v1" => Ok(Self::Secp256r1), + "secp384r1" => Ok(Self::Secp384r1), _ => Err(Unsupported::EcdhCurve(name.to_owned())), } } diff --git a/litellm-rust/crates/inference-chat/tests/caching.rs b/litellm-rust/crates/inference-chat/tests/caching.rs index dd229d3d657..d8d3a8956c8 100644 --- a/litellm-rust/crates/inference-chat/tests/caching.rs +++ b/litellm-rust/crates/inference-chat/tests/caching.rs @@ -117,6 +117,7 @@ async fn the_same_route_entrypoint_reports_facts_with_or_without_caching( assert!(caching); assert!(!key.is_empty()); } + ResultSource::Host => panic!("the host never answers a chat call"), } let summaries = traces.summaries("litellm.route"); assert_eq!(summaries.len(), 2); diff --git a/litellm-rust/crates/inference-messages/Cargo.toml b/litellm-rust/crates/inference-messages/Cargo.toml index 0c5be52550d..0ca371c3ce7 100644 --- a/litellm-rust/crates/inference-messages/Cargo.toml +++ b/litellm-rust/crates/inference-messages/Cargo.toml @@ -8,7 +8,9 @@ repository.workspace = true [dependencies] litellm-secret-value.workspace = true bytes.workspace = true +flate2 = { workspace = true, features = ["zlib-rs"] } futures-util.workspace = true +http.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } litellm-auth-aws.workspace = true litellm-cache-response.workspace = true @@ -30,9 +32,11 @@ tracing.workspace = true [dev-dependencies] litellm-error.workspace = true litellm-framer = { workspace = true, features = ["sse"] } +litellm-cache.workspace = true litellm-cache-memory.workspace = true litellm-host-native.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-inference-testing.workspace = true rstest.workspace = true +thiserror.workspace = true wiremock.workspace = true diff --git a/litellm-rust/crates/inference-messages/src/constants.rs b/litellm-rust/crates/inference-messages/src/constants.rs index c0b023be7c7..5fae6e45dad 100644 --- a/litellm-rust/crates/inference-messages/src/constants.rs +++ b/litellm-rust/crates/inference-messages/src/constants.rs @@ -2,3 +2,6 @@ /// seconds. Mirrors the Python Anthropic Messages default. The per-request /// timeout from the caller still overrides this on the request builder. pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600; + +pub(crate) const MESSAGES_HTTP_ERROR_ATTEMPTS: usize = 2; +pub(crate) const MESSAGES_STREAM_ERROR_BODY_READ_SECS: u64 = 5; diff --git a/litellm-rust/crates/inference-messages/src/encoding.rs b/litellm-rust/crates/inference-messages/src/encoding.rs new file mode 100644 index 00000000000..c315deb1a33 --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/encoding.rs @@ -0,0 +1,437 @@ +use bytes::Bytes; +use flate2::{Decompress, DecompressError, FlushDecompress, Status}; +use futures_util::StreamExt; +use reqwest::header::{CONTENT_ENCODING, HeaderMap}; + +type BoxError = Box; + +const MAX_WINDOW_BITS: u8 = 15; +const MAX_DECOMPRESS_SIZE: usize = 1 << 16; + +#[derive(Debug)] +pub struct DecodingError(String); + +impl std::fmt::Display for DecodingError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for DecodingError {} + +impl From for DecodingError { + fn from(error: DecompressError) -> Self { + Self(match (error.needs_dictionary(), error.message()) { + (Some(_), _) => "Error 2 while decompressing data".to_owned(), + (None, Some(message)) => format!("Error -3 while decompressing data: {message}"), + (None, None) => "Error -3 while decompressing data: invalid input data".to_owned(), + }) + } +} + +enum Codec { + Gzip, + Deflate, +} + +enum FirstAttempt { + Unread, + Reading(Vec), + Over, +} + +struct Inflater { + zlib: Decompress, + first_attempt: FirstAttempt, + pending: Vec, + saturated: bool, + stalled: bool, + finished: bool, +} + +impl Inflater { + fn new(codec: Codec) -> Self { + let (zlib, first_attempt) = match codec { + Codec::Gzip => (Decompress::new_gzip(MAX_WINDOW_BITS), FirstAttempt::Over), + Codec::Deflate => (Decompress::new(true), FirstAttempt::Unread), + }; + Self { + zlib, + first_attempt, + pending: Vec::new(), + saturated: false, + stalled: false, + finished: false, + } + } + + fn push(&mut self, data: &[u8]) { + self.first_attempt = match self.first_attempt { + FirstAttempt::Unread => FirstAttempt::Reading(data.to_vec()), + FirstAttempt::Reading(_) | FirstAttempt::Over => FirstAttempt::Over, + }; + if self.finished { + return; + } + self.pending.extend_from_slice(data); + self.stalled = false; + } + + fn has_work(&self) -> bool { + !self.finished && (self.saturated || (!self.pending.is_empty() && !self.stalled)) + } + + fn step(&mut self) -> Result, DecodingError> { + match (self.inflate(), &mut self.first_attempt) { + (Err(_), FirstAttempt::Reading(first)) => { + self.pending = std::mem::take(first); + self.first_attempt = FirstAttempt::Over; + self.zlib = Decompress::new_with_window_bits(false, MAX_WINDOW_BITS); + self.saturated = false; + self.stalled = false; + self.step() + } + (inflated, attempt) => { + if self.pending.is_empty() && !self.saturated { + *attempt = FirstAttempt::Over; + } + inflated + } + } + } + + fn inflate(&mut self) -> Result, DecodingError> { + let mut output = Vec::with_capacity(MAX_DECOMPRESS_SIZE); + let (read, written) = (self.zlib.total_in(), self.zlib.total_out()); + let status = self + .zlib + .decompress_vec(&self.pending, &mut output, FlushDecompress::Sync)?; + self.pending.drain(..(self.zlib.total_in() - read) as usize); + self.finished = matches!(status, Status::StreamEnd); + if self.finished { + self.pending.clear(); + } + self.saturated = !self.finished && output.len() == MAX_DECOMPRESS_SIZE; + self.stalled = self.zlib.total_in() == read && self.zlib.total_out() == written; + Ok(output) + } +} + +struct Decoder(Vec); + +impl Decoder { + fn detect(headers: &HeaderMap) -> Option { + let inflaters: Vec = header_pairs(headers) + .into_iter() + .filter(|(name, _)| name.as_str() == CONTENT_ENCODING.as_str()) + .flat_map(|(_, value)| { + value + .split(',') + .map(|coding| coding.trim().to_lowercase()) + .collect::>() + }) + .filter_map(|coding| match coding.as_str() { + "gzip" => Some(Codec::Gzip), + "deflate" => Some(Codec::Deflate), + _ => None, + }) + .rev() + .map(Inflater::new) + .collect(); + (!inflaters.is_empty()).then_some(Self(inflaters)) + } + + fn feed(&mut self, chunk: &[u8]) { + self.0[0].push(chunk); + } + + fn next(&mut self) -> Result, DecodingError> { + self.pull(self.0.len() - 1) + .map(|decoded| decoded.map(Bytes::from)) + } + + fn pull(&mut self, layer: usize) -> Result>, DecodingError> { + loop { + if self.0[layer].has_work() { + let decoded = self.0[layer].step()?; + if !decoded.is_empty() { + return Ok(Some(decoded)); + } + continue; + } + let Some(outer) = layer.checked_sub(1) else { + return Ok(None); + }; + match self.pull(outer)? { + Some(decoded) => self.0[layer].push(&decoded), + None => return Ok(None), + } + } + } +} + +pub(crate) fn header_pairs(headers: &HeaderMap) -> Vec<(String, String)> { + let utf8 = headers + .values() + .all(|value| std::str::from_utf8(value.as_bytes()).is_ok()); + headers + .iter() + .map(|(name, value)| { + let bytes = value.as_bytes(); + let text = if utf8 { + String::from_utf8_lossy(bytes).into_owned() + } else { + bytes.iter().copied().map(char::from).collect() + }; + (name.to_string(), text) + }) + .collect() +} + +pub(crate) fn decoded(response: reqwest::Response) -> reqwest::Response { + let Some(decoder) = Decoder::detect(response.headers()) else { + return response; + }; + let status = response.status(); + let version = response.version(); + let headers = response.headers().clone(); + let body = futures_util::stream::unfold( + Some((response.bytes_stream().boxed(), decoder)), + |state| async move { + let (mut body, mut decoder) = state?; + loop { + match decoder.next() { + Ok(Some(decoded)) => return Some((Ok(decoded), Some((body, decoder)))), + Ok(None) => {} + Err(error) => return Some((Err(BoxError::from(error)), None)), + } + match body.next().await? { + Ok(chunk) => decoder.feed(&chunk), + Err(error) => return Some((Err(BoxError::from(error)), None)), + } + } + }, + ); + let mut rebuilt = http::Response::new(reqwest::Body::wrap_stream(body)); + *rebuilt.status_mut() = status; + *rebuilt.version_mut() = version; + *rebuilt.headers_mut() = headers; + reqwest::Response::from(rebuilt) +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use flate2::{Compression, write::GzEncoder, write::ZlibEncoder}; + use futures_util::TryStreamExt; + use rstest::rstest; + + use reqwest::header::{HeaderMap, HeaderValue}; + + use super::{decoded, header_pairs}; + + fn encoded(encoding: &str, body: &[u8]) -> Vec { + match encoding { + "gzip" => { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + } + "deflate" => { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + } + _ => body.to_vec(), + } + } + + fn response(encoding: &str, body: Vec) -> reqwest::Response { + let length = body.len().to_string(); + let mut response = http::Response::new(reqwest::Body::from(body)); + let headers = response.headers_mut(); + headers.insert("content-encoding", encoding.parse().unwrap()); + headers.insert("content-length", length.parse().unwrap()); + reqwest::Response::from(response) + } + + #[rstest] + #[case::gzip("gzip")] + #[case::deflate("deflate")] + #[case::unknown("br")] + #[tokio::test] + async fn decoded_bodies_keep_the_head_the_provider_sent(#[case] encoding: &str) { + let body = br#"{"type":"message"}"#; + let wire = encoded(encoding, body); + let length = wire.len().to_string(); + let response = decoded(response(encoding, wire.clone())); + + assert_eq!(response.headers()["content-encoding"], encoding); + assert_eq!(response.headers()["content-length"], length.as_str()); + let expected = if encoding == "br" { + wire + } else { + body.to_vec() + }; + assert_eq!( + response.bytes().await.unwrap().as_ref(), + expected.as_slice() + ); + } + + fn raw_deflate(body: &[u8]) -> Vec { + let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + } + + fn chunked(encoding: &str, wire: Vec, size: usize) -> reqwest::Response { + let chunks: Vec, std::io::Error>> = + wire.chunks(size).map(|chunk| Ok(chunk.to_vec())).collect(); + let mut response = http::Response::new(reqwest::Body::wrap_stream( + futures_util::stream::iter(chunks), + )); + response + .headers_mut() + .insert("content-encoding", encoding.parse().unwrap()); + reqwest::Response::from(response) + } + + async fn read(response: reqwest::Response) -> Result, String> { + response + .bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|error| { + litellm_error::find::(&error) + .map_or_else(|| error.to_string(), ToString::to_string) + }) + } + + const BODY: &[u8] = br#"{"type":"message","content":[{"type":"text","text":"Paris"}]}"#; + + #[rstest] + #[case::padded_upper_case(" GZIP ", encoded("gzip", BODY))] + #[case::listed_with_an_unknown_coding("gzip, br", encoded("gzip", BODY))] + #[case::applied_twice("gzip, gzip", encoded("gzip", &encoded("gzip", BODY)))] + #[case::deflate_without_its_zlib_header("deflate", raw_deflate(BODY))] + #[case::gzip_with_data_past_its_end("gzip", [encoded("gzip", BODY), b"\x1f\x8b trailing".to_vec()].concat())] + #[tokio::test] + async fn bodies_decode_as_httpx_decodes_them( + #[case] encoding: &str, + #[case] wire: Vec, + #[values(7, 4096)] chunk: usize, + ) { + assert_eq!( + read(decoded(chunked(encoding, wire, chunk))).await.unwrap(), + BODY + ); + } + + #[rstest] + #[case::x_gzip_alias(" X-Gzip ")] + #[case::unknown_only("br, identity")] + #[tokio::test] + async fn codings_httpx_has_no_decoder_for_pass_through(#[case] encoding: &str) { + let wire = encoded("gzip", BODY); + assert_eq!( + read(decoded(chunked(encoding, wire.clone(), 4096))) + .await + .unwrap(), + wire + ); + } + + #[rstest] + #[case::not_deflate("deflate", b"not deflate at all".to_vec(), "Error -3 while decompressing data: invalid block type")] + #[case::bad_gzip_trailer( + "gzip", + { let wire = encoded("gzip", BODY); [&wire[..wire.len() - 8], &[0; 8]].concat() }, + "Error -3 while decompressing data: incorrect data check" + )] + #[case::not_gzip("gzip", b"\x1f\x8b\x08\x00not really gzip data at all".to_vec(), "Error -3 while decompressing data: invalid stored block lengths")] + #[case::no_gzip_magic("gzip", b"plain text".to_vec(), "Error -3 while decompressing data: incorrect header check")] + #[tokio::test] + async fn corrupt_bodies_fail_with_zlib_s_error( + #[case] encoding: &str, + #[case] wire: Vec, + #[case] expected: &str, + ) { + assert_eq!( + read(decoded(chunked(encoding, wire, 4096))) + .await + .unwrap_err(), + expected + ); + } + + #[rstest] + #[case::gzip_in_gzip("gzip, gzip")] + #[case::deflate_in_gzip("deflate, gzip")] + #[tokio::test] + async fn a_nested_bomb_is_decoded_in_bounded_reads(#[case] encoding: &str) { + let zeros = vec![0; 8 << 20]; + let inner = if encoding.starts_with("deflate") { + encoded("deflate", &zeros) + } else { + encoded("gzip", &zeros) + }; + let wire = encoded("gzip", &inner); + let sizes: Vec = decoded(chunked(encoding, wire, 4096)) + .bytes_stream() + .map_ok(|chunk| chunk.len()) + .try_collect() + .await + .unwrap(); + assert!( + sizes.iter().all(|size| *size <= super::MAX_DECOMPRESS_SIZE), + "{sizes:?}" + ); + assert_eq!(sizes.iter().sum::(), zeros.len()); + } + + #[rstest] + #[tokio::test] + async fn a_truncated_body_yields_what_it_holds_without_an_error() { + let wire = encoded("gzip", BODY); + let decoded_body = read(decoded(chunked( + "gzip", + wire[..wire.len() - 8].to_vec(), + 4096, + ))) + .await + .unwrap(); + assert_eq!(decoded_body, BODY); + } + + #[rstest] + #[case::ascii(&[b"plain".as_slice()], &["plain"])] + #[case::utf_8(&[b"plain".as_slice(), b"caf\xc3\xa9"], &["plain", "caf\u{e9}"])] + #[case::latin_1_once_any_value_is_not_utf_8( + &[b"caf\xc3\xa9".as_slice(), b"caf\xe9"], + &["caf\u{c3}\u{a9}", "caf\u{e9}"] + )] + fn header_values_decode_as_httpx_decodes_them( + #[case] values: &[&[u8]], + #[case] expected: &[&str], + ) { + let headers: HeaderMap = values + .iter() + .enumerate() + .map(|(index, value)| { + ( + format!("x-fz-{index}") + .parse::() + .unwrap(), + HeaderValue::from_bytes(value).unwrap(), + ) + }) + .collect(); + let decoded: Vec = header_pairs(&headers) + .into_iter() + .map(|(_, value)| value) + .collect(); + assert_eq!(decoded, expected); + } +} diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index c7a7d93bf98..4796a640e25 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -18,7 +18,7 @@ use litellm_llms::base_llm::{ messages::{ context::MessagesTransformContext, streaming::{self, ByteStream, StreamDecoder}, - transformation::BaseMessagesConfig, + transformation::{BaseMessagesConfig, FederatedEnvironment}, }, }; use litellm_llms_types::formats::messages::{ @@ -29,13 +29,21 @@ use litellm_tracing::ByteChunk; use serde_json::{Map, Value}; use super::{ - Error, MessagesBody, MessagesCall, MessagesCallResponse, MessagesRoute, + Delivered, Error, HostAnswer, MessagesBody, MessagesCall, MessagesCallResponse, MessagesRoute, + Transport, TransportBody, TransportRequest, TransportResponse, UnencodableHeader, + WorkloadIdentity, common_utils::{MessagesProvider, merge_messages_header_sources, string_headers}, - constants::MESSAGES_TIMEOUT_SECS, + constants::{ + MESSAGES_HTTP_ERROR_ATTEMPTS, MESSAGES_STREAM_ERROR_BODY_READ_SECS, MESSAGES_TIMEOUT_SECS, + }, route::Messages, types::invalid_request, }; -use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request}; +use litellm_inference::{ + caching::{CacheRequest, CallCache}, + context::CallContext, + outbound::raw_leaf_outbound_request, +}; struct ProviderCall { identity: ProviderIdentity, @@ -44,6 +52,12 @@ struct ProviderCall { signer: Option, timeout: Option, stream: bool, + whole_sse_frames: bool, + joined_reads: bool, + transform_context: MessagesTransformContext, + ascii_response_headers: bool, + unencodable_request_header: Option>, + ascii_request_headers: bool, } impl ProviderCall { @@ -52,16 +66,122 @@ impl ProviderCall { } } +pub(super) async fn cached( + route: &MessagesRoute, + provider: MessagesProvider, + model: &str, + context: &CallContext<'_, impl Interceptors>, +) -> Result, Error> { + let Some(cache) = caller_keyed(route) else { + return Ok(None); + }; + let identity = ProviderIdentity { + model: model.to_owned(), + provider: provider.as_str().to_string(), + }; + let Some((output, source)) = CallCache::::new( + Some(cache), + context.cache, + CacheRequest::from_wire(identity.clone(), None), + ) + .lookup() + .await + else { + return Ok(None); + }; + context + .result_ready(execution_facts(provider, identity, &output, &source)) + .await?; + Ok(Some(output)) +} + +pub(super) async fn mocked( + route: &MessagesRoute, + provider: MessagesProvider, + model: String, + context: &CallContext<'_, impl Interceptors>, + mock: Map, + stream: bool, +) -> MessagesCallResponse { + let output = MessagesCallResponse::Complete(MessagesBody::Unchecked(mock)); + let Some(cache) = caller_keyed(route).filter(|_| !stream) else { + return output; + }; + let identity = ProviderIdentity { + model, + provider: provider.as_str().to_string(), + }; + CallCache::::new( + Some(cache), + context.cache, + CacheRequest::from_wire(identity, None), + ) + .finish(output, &ResultSource::Provider) + .await +} + +pub(super) async fn answered( + route: &MessagesRoute, + host_answer: &dyn HostAnswer, + provider: MessagesProvider, + model: String, + stream: bool, + context: &CallContext<'_, impl Interceptors>, +) -> Result { + host_answer.answer(stream).await?; + deliver_answer(route, provider, model, stream, context).await +} + +pub(super) async fn deliver_answer( + route: &MessagesRoute, + provider: MessagesProvider, + model: String, + stream: bool, + context: &CallContext<'_, impl Interceptors>, +) -> Result { + let identity = ProviderIdentity { + model, + provider: provider.as_str().to_string(), + }; + let cache = CallCache::::new( + caller_keyed(route), + context.cache, + CacheRequest::from_wire(identity.clone(), None), + ); + context + .result_ready(ExecutionFacts { + provider: identity, + source: ResultSource::Host, + cost: Cost::Deferred, + }) + .await?; + let output = MessagesCallResponse::Complete(MessagesBody::Unchecked(Map::new())); + if stream { + return Ok(output); + } + Ok(cache.finish(output, &ResultSource::Host).await) +} + +fn caller_keyed(route: &MessagesRoute) -> Option<&litellm_cache_response::ScopedCache> { + route + .cache + .as_ref() + .filter(|cache| cache.service.keyed_by_caller()) +} + pub(super) async fn execute( route: &MessagesRoute, provider: MessagesProvider, model: String, call: MessagesCall, secrets: Secrets, + requested_stream: bool, context: &CallContext<'_, impl Interceptors>, ) -> Result { + let caller_keyed = caller_keyed(route).is_some(); let request = provider_call( &route.auth, + route.workload_identity.as_deref(), provider, model, call, @@ -71,38 +191,64 @@ pub(super) async fn execute( .boxed() .await?; let cache = CallCache::::from_wire( - route.cache.as_ref().filter(|_| request.cacheable()), + route + .cache + .as_ref() + .filter(|_| caller_keyed || request.cacheable()), context.cache, &request.identity, &request.wire, ); let identity = request.identity.clone(); - let (output, source) = match cache.lookup().await { + let hit = match caller_keyed { + true => None, + false => cache.lookup().await, + }; + let (output, source) = match hit { Some(hit) => hit, None => ( - call_provider(&route.http, request, context).await?, + call_provider(&route.http, route.transport.as_deref(), request, context).await?, ResultSource::Provider, ), }; - let reported = match &output { + context + .result_ready(execution_facts(provider, identity, &output, &source)) + .await?; + let output = match source { + ResultSource::Provider => { + super::safeguards::stamped(route.safeguards.as_ref(), output).await? + } + ResultSource::Cache { .. } | ResultSource::Host => output, + }; + if !requested_stream && matches!(output, MessagesCallResponse::Stream { .. }) { + return Ok(output); + } + Ok(cache.finish(output, &source).await) +} + +fn execution_facts( + provider: MessagesProvider, + identity: ProviderIdentity, + output: &MessagesCallResponse, + source: &ResultSource, +) -> ExecutionFacts { + let reported = match output { MessagesCallResponse::Complete(response) => match response { MessagesBody::Message(message) => provider.config().reported_cost(message), MessagesBody::Unchecked(_) => None, }, MessagesCallResponse::Stream { .. } => None, }; - context - .result_ready(ExecutionFacts { - provider: identity, - cost: Cost::settle(&source, reported), - source: source.clone(), - }) - .await?; - Ok(cache.finish(output, &source).await) + ExecutionFacts { + provider: identity, + cost: Cost::settle(source, reported), + source: source.clone(), + } } async fn provider_call( auth: &AuthServices, + workload_identity: Option<&dyn WorkloadIdentity>, provider: MessagesProvider, model: String, call: MessagesCall, @@ -129,14 +275,18 @@ async fn provider_call( let sanitized = config.shape_request( MessagesRequest { model, ..body }, - shaping.settings.reasoning_auto_summary, + shaping.settings.shape_options(), )?; - let transform_context = - MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params); - let without_listed = - without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?; - let trimmed = requested_optional_params(without_listed, provider, &transform_context)?; - let requested_params = serde_json::to_value(&trimmed.params).map_err(serialize_failure)?; + let transform_context = MessagesTransformContext { + drops_unsupported_params: shaping.settings.drops_unsupported_params, + ..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params) + }; + let dropped_keys = top_level_paths(&shaping.settings.additional_drop_params); + let without_keys = without_additional_drop_params(sanitized, &dropped_keys)?; + let requested = requested_optional_params(without_keys, provider, &transform_context)?; + let requested_params = serde_json::to_value(&requested.params).map_err(serialize_failure)?; + let trimmed = + without_additional_drop_params(requested, &shaping.settings.additional_drop_params)?; let scoped = get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str()); @@ -144,35 +294,58 @@ async fn provider_call( extra_headers.unwrap_or_default(), scoped, ])))?; - let validated = config.validate_environment( + let validated = match config.validate_federated_environment( forwarded, api_key.as_deref(), &trimmed.model, &trimmed.params, &connection_params, &env_lookup, - )?; + )? { + FederatedEnvironment::Validated(validated) => validated, + FederatedEnvironment::Unauthenticated { + headers, + missing, + federate, + } => match workload_identity { + Some(identity) => match identity + .token(&connection_params, api_base.as_deref()) + .await? + { + Some(token) => federate(headers, token), + None => return Err(missing.into()), + }, + None => return Err(missing.into()), + }, + }; + let tool_custom_field = config.tool_custom_field(&trimmed.model); + let requested_output_format = trimmed.params.output_format.clone(); let body = config.request_body( - config.transform_anthropic_messages_request(trimmed, &transform_context)?, + tool_custom_field.apply( + config.transform_anthropic_messages_request(trimmed, &transform_context)?, + shaping.settings.drops_unsupported_params, + ), &validated.headers, shaping.settings.drop_params, )?; - let environment = ValidatedEnvironment { - headers: config.request_headers( - with_python_default_headers(validated.headers, config.default_headers()), - &body, - ), - auth: validated.auth, - }; - let url = config.get_complete_url( api_base.as_deref(), &body.model, &connection_params, - body.params.stream == Some(true), + body.params.streams(), &env_lookup, )?; + let environment = ValidatedEnvironment { + headers: config.headers_for_url( + config.request_headers( + with_python_default_headers(validated.headers, config.default_headers()), + &with_requested_output_format(&body, requested_output_format), + ), + &url, + ), + auth: validated.auth, + }; let request_context = RequestContext { model: body.model.clone(), @@ -197,19 +370,10 @@ async fn provider_call( request_context, ) .await?; - let stream = match wire.body.get("stream") { - None | Some(Value::Null) => false, - Some(Value::Bool(stream)) => *stream, - Some(value) => { - return Err(Error::invalid_request( - litellm_llms::ErrorDetail::InvalidValue { - field: "stream", - expected: "a boolean", - actual: value.clone(), - }, - )); - } - }; + let stream = wire + .body + .get("stream") + .is_some_and(litellm_llms_types::recognized::python_truthy); Ok(ProviderCall { identity, wire, @@ -217,9 +381,33 @@ async fn provider_call( signer: authenticated.signer, timeout: timeout.or_else(|| shaping.settings.request_timeout()), stream, + whole_sse_frames: shaping.settings.whole_sse_frames, + joined_reads: shaping.settings.joined_reads, + transform_context, + ascii_response_headers: shaping.settings.ascii_response_headers, + unencodable_request_header: shaping.settings.unencodable_request_header, + ascii_request_headers: shaping.settings.ascii_request_headers, }) } +fn with_requested_output_format( + body: &MessagesRequest, + requested: Option, +) -> std::borrow::Cow<'_, MessagesRequest> { + match requested { + Some(output_format) if body.params.output_format.is_none() => { + std::borrow::Cow::Owned(MessagesRequest { + params: MessagesOptionalParams { + output_format: Some(output_format), + ..body.params.clone() + }, + ..body.clone() + }) + } + _ => std::borrow::Cow::Borrowed(body), + } +} + /// Python sends its header dict through `httpx.Headers`, which folds names that differ only in /// case into one lowercase header whose values are joined with `", "`. fn joined_header_values(headers: Vec<(String, String)>) -> Vec<(String, String)> { @@ -278,6 +466,14 @@ fn requested_optional_params( }) } +fn top_level_paths(paths: &[String]) -> Vec { + paths + .iter() + .filter(|path| !path.contains(['.', '['])) + .cloned() + .collect() +} + fn without_additional_drop_params( request: MessagesRequest, paths: &[String], @@ -295,52 +491,57 @@ fn without_additional_drop_params( }) } +enum Upstream { + Native(reqwest::Response), + Hosted(TransportResponse), +} + async fn call_provider( http: &litellm_http::Client, + transport: Option<&dyn Transport>, request: ProviderCall, context: &CallContext<'_, impl Interceptors>, ) -> Result { + let timeout = request + .timeout + .unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS)); + let response = match send_recovering(http, transport, &request, timeout, context).await? { + Upstream::Native(response) => response, + Upstream::Hosted(response) => return hosted_response(response, request, context).await, + }; let ProviderCall { identity, - wire, provider, - signer, - timeout, stream, + whole_sse_frames, + joined_reads, + .. } = request; let provider_name = provider.as_str(); - log_request_body(provider_name, stream, &wire.body); - let timeout = timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS)); - let response = send( - http, - Authenticated { - headers: wire.headers, - signer, - }, - &wire.url, - &wire.body, - timeout, - stream, - ) - .await?; - if !response.status().is_success() { - return Err(provider_error(response, timeout).await); - } let config = provider.config(); + let head = ResponseHead { + status: response.status().as_u16(), + headers: super::encoding::header_pairs(response.headers()), + }; if stream { - return Ok(streaming_response( + context + .raw_response_received(RawResponse::new(String::new()).with_head(head)) + .await?; + let streamed = streaming_response( response, config.stream_decoder(), provider_name, timeout, - )); + joined_reads, + ); + return Ok(if whole_sse_frames { + framed(streamed) + } else { + streamed + }); } - let head = ResponseHead { - status: response.status().as_u16(), - headers: litellm_http::request::header_pairs(response.headers()), - }; let status = head.status; - let text = response.text().await.map_err(litellm_http::Error::from)?; + let text = read_body(response, timeout).await?; log_response_body(&text); context .raw_response_received(RawResponse::new(text.clone()).with_head(head)) @@ -353,6 +554,197 @@ async fn call_provider( decode_response(config, &identity.model, &text).map(MessagesCallResponse::Complete) } +async fn hosted_response( + response: TransportResponse, + request: ProviderCall, + context: &CallContext<'_, impl Interceptors>, +) -> Result { + let TransportResponse { + status, + headers, + delivered, + } = response; + let ProviderCall { + identity, + provider, + whole_sse_frames, + .. + } = request; + let provider_name = provider.as_str(); + let config = provider.config(); + let text = match delivered { + Delivered::Complete(text) => text, + Delivered::Stream(chunks) => { + context + .raw_response_received(RawResponse::new(String::new()).with_head(ResponseHead { + status, + headers: headers.clone(), + })) + .await?; + let streamed = + hosted_streaming_response(headers, chunks, config.stream_decoder(), provider_name); + return Ok(if whole_sse_frames { + framed(streamed) + } else { + streamed + }); + } + Delivered::Rejected { error, .. } => return Err(error), + }; + log_response_body(&text); + context + .raw_response_received( + RawResponse::new(text.clone()).with_head(ResponseHead { status, headers }), + ) + .await?; + if serde_json::from_str::(&text).is_err() { + return Err(litellm_http::Error::upstream(status, text).into()); + } + decode_response(config, &identity.model, &text).map(MessagesCallResponse::Complete) +} + +fn hosted_request( + authenticated: Authenticated, + url: &str, + body: &Value, + stream: bool, +) -> Result { + if authenticated.signer.is_none() { + return Ok(TransportRequest { + url: url.to_owned(), + headers: authenticated.headers, + body: TransportBody::Json(body.clone()), + stream, + }); + } + let names: Vec = authenticated + .headers + .iter() + .map(|(name, _)| name.clone()) + .collect(); + let signed = raw_leaf_outbound_request(authenticated, url, body, None)?; + let headers = signed + .headers() + .iter() + .filter_map(|(name, value)| { + let spelled = names + .iter() + .find(|known| known.eq_ignore_ascii_case(name.as_str())) + .cloned() + .unwrap_or_else(|| name.to_string()); + Some((spelled, value.to_str().ok()?.to_owned())) + }) + .collect(); + let bytes = signed + .body() + .and_then(reqwest::Body::as_bytes) + .map(Bytes::copy_from_slice) + .unwrap_or_default(); + Ok(TransportRequest { + url: signed.url().to_string(), + headers, + body: TransportBody::Signed(bytes), + stream, + }) +} + +async fn send_recovering( + http: &litellm_http::Client, + transport: Option<&dyn Transport>, + request: &ProviderCall, + timeout: Duration, + context: &CallContext<'_, impl Interceptors>, +) -> Result { + let config = request.provider.config(); + let mut body = std::borrow::Cow::Borrowed(&request.wire.body); + let mut attempt = 1; + loop { + log_request_body(request.provider.as_str(), request.stream, &body); + if let Some(value) = &request.unencodable_request_header { + return Err(Error::transport(UnencodableHeader::Sent(value.clone()))); + } + // The credential header built from the caller's api_key never passes the caller header + // check, so it is checked on the wire headers as Python's client encodes them. + if request.ascii_request_headers + && let Some((_, value)) = request + .wire + .headers + .iter() + .find(|(_, value)| !value.is_ascii()) + { + return Err(Error::transport(UnencodableHeader::Sent( + value.as_bytes().to_vec(), + ))); + } + let authenticated = Authenticated { + headers: request.wire.headers.clone(), + signer: request.signer.clone(), + }; + let response = match transport { + None => Upstream::Native( + send(http, authenticated, &request.wire.url, &body, timeout).await?, + ), + Some(transport) => Upstream::Hosted( + transport + .send(hosted_request( + authenticated, + &request.wire.url, + &body, + request.stream, + )?) + .await?, + ), + }; + if let Upstream::Native(response) = &response + && request.ascii_response_headers + && let Some(unencodable) = UnencodableHeader::received(response.headers()) + { + return Err(Error::transport(unencodable)); + } + let failure = match response { + Upstream::Native(response) if response.status().is_success() => { + return Ok(Upstream::Native(response)); + } + Upstream::Native(response) => { + provider_failure(response, timeout, request.stream).await? + } + Upstream::Hosted(TransportResponse { + status, + headers, + delivered: Delivered::Rejected { text, error }, + }) => { + log_error_body(status, &text); + ProviderFailure { + status, + text, + headers, + raised: Some(error), + } + } + hosted @ Upstream::Hosted(_) => return Ok(hosted), + }; + let retried = if attempt == MESSAGES_HTTP_ERROR_ATTEMPTS { + None + } else { + config.retry_after_http_error( + failure.status, + &failure.text, + &body, + &request.transform_context, + ) + }; + let Some(retried) = retried else { + return Err(failure.into()); + }; + attempt += 1; + tracing::debug!( + "Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt {attempt}/{MESSAGES_HTTP_ERROR_ATTEMPTS})." + ); + body = std::borrow::Cow::Owned(retried.body.clone()); + context.provider_request_retried(retried).await?; + } +} + fn serialize_failure(err: serde_json::Error) -> Error { Error::invalid_request(litellm_llms::ErrorDetail::failed( "Anthropic messages request serialization", @@ -360,27 +752,37 @@ fn serialize_failure(err: serde_json::Error) -> Error { )) } -/// Python's httpx timeout bounds each phase and each read, never the whole exchange. A complete -/// response keeps the client's overall deadline; a stream is bounded until its head arrives and -/// then per read (see [`timed_chunks`]), so a long stream that keeps sending is not cut off. async fn send( http: &litellm_http::Client, authenticated: Authenticated, url: &str, body: &Value, timeout: Duration, - stream: bool, ) -> Result { - let request = outbound_request(authenticated, url, body, (!stream).then_some(timeout))?; - if !stream { - return Ok(litellm_inference::outbound::send(request, http).await?); - } + let request = raw_leaf_outbound_request(authenticated, url, body, None)?; tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http)) .await .map_err(litellm_http::Error::read_timeout)? + .map(super::encoding::decoded) .map_err(Error::from) } +async fn read_body( + mut response: reqwest::Response, + timeout: Duration, +) -> Result { + let expected = response.content_length(); + let mut body = Vec::new(); + while let Some(chunk) = tokio::time::timeout(timeout, response.chunk()) + .await + .map_err(litellm_http::Error::read_timeout)? + .map_err(|error| litellm_http::Error::body_read(error, body.len() as u64, expected))? + { + body.extend_from_slice(&chunk); + } + Ok(String::from_utf8_lossy(&body).into_owned()) +} + /// The stream's body, each read bounded by `timeout` as httpx bounds each read. fn timed_chunks( response: reqwest::Response, @@ -402,30 +804,65 @@ fn timed_chunks( ) } -/// The provider's error, its body read under `timeout`: a stream has no overall deadline once -/// its head arrives, so an error body that stalls would otherwise hold the call forever. -async fn provider_error(response: reqwest::Response, timeout: Duration) -> Error { - let status = response.status().as_u16(); - let headers = litellm_http::request::header_pairs(response.headers()); - match tokio::time::timeout(timeout, response.text()).await { - Ok(Ok(text)) => { - log_error_body(status, &text); - litellm_http::Error::upstream_with_headers(status, text, headers).into() +struct ProviderFailure { + status: u16, + text: String, + headers: Vec<(String, String)>, + raised: Option, +} + +impl From for Error { + fn from(failure: ProviderFailure) -> Self { + match failure.raised { + Some(raised) => raised, + None => litellm_http::Error::upstream_with_headers( + failure.status, + failure.text, + failure.headers, + ) + .into(), } - Ok(Err(error)) => litellm_http::Error::from(error).into(), - Err(elapsed) => litellm_http::Error::read_timeout(elapsed).into(), } } +async fn provider_failure( + response: reqwest::Response, + timeout: Duration, + stream: bool, +) -> Result { + let status = response.status().as_u16(); + let headers = super::encoding::header_pairs(response.headers()); + let text = if stream { + tokio::time::timeout( + Duration::from_secs(MESSAGES_STREAM_ERROR_BODY_READ_SECS), + read_body(response, timeout), + ) + .await + .ok() + .and_then(Result::ok) + .unwrap_or_default() + } else { + read_body(response, timeout).await? + }; + log_error_body(status, &text); + Ok(ProviderFailure { + status, + text, + headers, + raised: None, + }) +} + fn decode_response( config: &dyn BaseMessagesConfig, model: &str, text: &str, ) -> Result { - let response = match serde_json::from_str::(text) { + let marked = litellm_host::raw_json::mark_wide_numbers(text); + let response = match serde_json::from_str::(&marked) { Ok(response) => response, Err(err) => { - let Ok(mut body) = serde_json::from_str::>(text) else { + let Ok(mut body) = serde_json::from_str::>(&marked) else { return Err(Error::invalid_response(litellm_llms::ErrorDetail::invalid( "messages response JSON", err, @@ -435,11 +872,34 @@ fn decode_response( return Ok(MessagesBody::Unchecked(body)); } }; - config + let message = config .transform_anthropic_messages_response(model, response) - .map(|message| MessagesBody::Message(Box::new(message))) .map_err(Error::from) - .map_err(Error::after_send) + .map_err(Error::after_send)?; + if text_has_stop_reason(text) { + return Ok(MessagesBody::Message(Box::new(message))); + } + match serde_json::to_value(message).map_err(serialize_failure)? { + Value::Object(mut body) => { + body.remove("stop_reason"); + Ok(MessagesBody::Unchecked(body)) + } + _ => unreachable!("a message serializes as a JSON object"), + } +} + +#[derive(serde::Deserialize)] +struct StopReasonKey { + #[serde(default, deserialize_with = "key_present")] + stop_reason: bool, +} + +fn key_present<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + >::deserialize(deserializer).map(|_| true) +} + +fn text_has_stop_reason(text: &str) -> bool { + serde_json::from_str::(text).is_ok_and(|key| key.stop_reason) } fn streaming_response( @@ -447,14 +907,15 @@ fn streaming_response( decoder: Option, provider: &'static str, timeout: Duration, + joined: bool, ) -> MessagesCallResponse { - let headers = response - .headers() - .iter() - .filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string()))) - .collect(); - let chunks = timed_chunks(response, timeout) - .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); + let headers = super::encoding::header_pairs(response.headers()); + let chunks = timed_chunks(response, timeout); + let chunks = match (joined, &decoder) { + (true, None) => super::reads::joined_reads(chunks), + _ => chunks.boxed(), + } + .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); let chunks = match decoder { None => chunks.map_err(Error::from).boxed(), Some(decode) => decoded_chunks( @@ -469,6 +930,73 @@ fn streaming_response( } } +fn hosted_streaming_response( + headers: Vec<(String, String)>, + chunks: BoxStream<'static, Result>, + decoder: Option, + provider: &'static str, +) -> MessagesCallResponse { + let chunks = chunks.inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); + let chunks = match decoder { + None => chunks.boxed(), + Some(decode) => decoded_chunks( + chunks.map_err(std::io::Error::other).boxed(), + decode, + provider, + ), + }; + MessagesCallResponse::Stream { + head: super::route::MessagesStreamHead { headers }, + chunks, + } +} + +fn framed(response: MessagesCallResponse) -> MessagesCallResponse { + let MessagesCallResponse::Stream { head, chunks } = response else { + return response; + }; + MessagesCallResponse::Stream { + head, + chunks: whole_sse_frames(chunks), + } +} + +fn whole_sse_frames( + chunks: BoxStream<'static, Result>, +) -> BoxStream<'static, Result> { + futures_util::stream::try_unfold( + (Some(chunks), Vec::new()), + |(chunks, mut pending)| async move { + let Some(mut chunks) = chunks else { + return Ok(None); + }; + while let Some(chunk) = chunks.try_next().await? { + pending.extend_from_slice(&chunk); + let end = complete_frames_end(&pending); + if end > 0 { + let rest = pending.split_off(end); + return Ok(Some((Bytes::from(pending), (Some(chunks), rest)))); + } + } + Ok((!pending.is_empty()).then(|| (Bytes::from(pending), (None, Vec::new())))) + }, + ) + .boxed() +} + +fn complete_frames_end(pending: &[u8]) -> usize { + [b"\r\n\r\n".as_slice(), b"\n\n", b"\r\r"] + .iter() + .filter_map(|delimiter| { + pending + .windows(delimiter.len()) + .rposition(|window| window == *delimiter) + .map(|at| at + delimiter.len()) + }) + .max() + .unwrap_or(0) +} + fn decoded_chunks( bytes: ByteStream, decode: StreamDecoder, @@ -529,6 +1057,14 @@ mod tests { async fn provider_call_with_secrets( call: MessagesCall, secrets: &(dyn Lookup + Send + Sync), + ) -> Result { + provider_call_federating(call, secrets, None).await + } + + async fn provider_call_federating( + call: MessagesCall, + secrets: &(dyn Lookup + Send + Sync), + workload_identity: Option<&dyn WorkloadIdentity>, ) -> Result { let provider = resolve_llm_provider( &call.body.model, @@ -542,6 +1078,7 @@ mod tests { let context = CallContext::new(&interceptors, litellm_inference::CallOptions::default()); provider_call( &AuthServices::default(), + workload_identity, config, model, call, @@ -735,6 +1272,178 @@ mod tests { assert!(error.is_missing_credential(), "{error:?}"); } + struct ScriptedIdentity { + minted: Result, ()>, + asked: std::sync::Mutex, Option)>>, + } + + impl ScriptedIdentity { + fn new(minted: Result, ()>) -> Self { + Self { + minted, + asked: std::sync::Mutex::new(Vec::new()), + } + } + + fn asked(&self) -> Vec<(Option, Option)> { + self.asked.lock().unwrap().clone() + } + } + + impl WorkloadIdentity for ScriptedIdentity { + fn token<'a>( + &'a self, + litellm_params: &'a litellm_router_types::LitellmParams, + api_base: Option<&'a str>, + ) -> futures_util::future::BoxFuture<'a, Result, litellm_auth::Error>> + { + self.asked.lock().unwrap().push(( + litellm_params + .extra + .get("anthropic_federation_rule_id") + .cloned(), + api_base.map(str::to_owned), + )); + let minted = self.minted; + Box::pin(async move { + minted + .map(|token| token.map(SecretValue::new)) + .map_err(|()| litellm_auth::Error::InvalidHeader) + }) + } + } + + fn federated_call( + shaping: MessagesShaping, + api_key: Option<&str>, + extra_headers: Option, + ) -> MessagesCall { + MessagesCall { + body: body( + json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), + ), + api_key: api_key.map(str::to_owned), + api_base: Some("https://anthropic.test".into()), + custom_llm_provider: Some("anthropic".into()), + litellm_params: crate::litellm_params( + serde_json::from_value( + json!({"model": "claude-test", "anthropic_federation_rule_id": "fdrl_1"}), + ) + .unwrap(), + ) + .unwrap(), + extra_headers: extra_headers.map(|headers| serde_json::from_value(headers).unwrap()), + provider_specific_header: None, + timeout: None, + shaping, + } + } + + fn sent_headers<'a>(call: &'a ProviderCall, names: &[&str]) -> Vec<(&'a str, &'a str)> { + call.wire + .headers + .iter() + .filter(|(name, _)| names.contains(&name.as_str())) + .map(|(name, value)| (name.as_str(), value.as_str())) + .collect() + } + + #[rstest] + #[case::alone(None, &[("anthropic-beta", "oauth-2025-04-20"), ("authorization", "Bearer sk-ant-oat01-minted")])] + #[case::beside_a_caller_beta( + Some(json!({"anthropic-beta": "files-api-2025-04-14"})), + &[("anthropic-beta", "files-api-2025-04-14,oauth-2025-04-20"), ("authorization", "Bearer sk-ant-oat01-minted")] + )] + #[tokio::test] + async fn a_minted_federation_token_is_sent_as_an_oauth_bearer( + shaping: MessagesShaping, + #[case] extra_headers: Option, + #[case] expected: &[(&str, &str)], + ) { + let identity = ScriptedIdentity::new(Ok(Some("sk-ant-oat01-minted"))); + let call = provider_call_federating( + federated_call(shaping, None, extra_headers), + &|_: &str| None, + Some(&identity), + ) + .await + .unwrap(); + let mut sent = sent_headers( + &call, + &[ + "anthropic-beta", + "authorization", + "x-api-key", + "anthropic-dangerous-direct-browser-access", + ], + ); + sent.sort_unstable(); + assert_eq!( + (sent.as_slice(), identity.asked()), + ( + expected, + vec![(Some(json!("fdrl_1")), Some("https://anthropic.test".into()))] + ) + ); + } + + #[rstest] + #[case::static_key(Some("sk-static"), None, &[("x-api-key", "sk-static")])] + #[case::caller_credential( + None, + Some(json!({"x-api-key": "sk-caller"})), + &[("x-api-key", "sk-caller")] + )] + #[tokio::test] + async fn a_configured_credential_is_never_federated( + shaping: MessagesShaping, + #[case] api_key: Option<&str>, + #[case] extra_headers: Option, + #[case] expected: &[(&str, &str)], + ) { + let identity = ScriptedIdentity::new(Ok(Some("sk-ant-oat01-minted"))); + let call = provider_call_federating( + federated_call(shaping, api_key, extra_headers), + &|_: &str| None, + Some(&identity), + ) + .await + .unwrap(); + assert_eq!( + ( + sent_headers(&call, &["authorization", "x-api-key"]), + identity.asked() + ), + (expected.to_vec(), Vec::new()) + ); + } + + #[rstest] + #[case::nothing_minted(Ok(None), true)] + #[case::minting_failed(Err(()), false)] + #[tokio::test] + async fn an_unminted_federation_token_fails_the_call( + shaping: MessagesShaping, + #[case] minted: Result, ()>, + #[case] missing_credential: bool, + ) { + let identity = ScriptedIdentity::new(minted); + let Err(error) = provider_call_federating( + federated_call(shaping, None, None), + &|_: &str| None, + Some(&identity), + ) + .await + else { + panic!("a call without a minted token fails"); + }; + assert_eq!( + error.is_missing_credential(), + missing_credential, + "{error:?}" + ); + } + #[rstest] #[case::nothing_configured(None, None, None)] #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))] @@ -1037,7 +1746,7 @@ mod tests { .await .unwrap(); assert_eq!(call.wire.url, format!("{base}{path}")); - let signed = outbound_request( + let signed = raw_leaf_outbound_request( Authenticated { headers: call.wire.headers, signer: call.signer, @@ -1081,6 +1790,7 @@ mod tests { Some(streaming::event_stream), "test", Duration::from_secs(5), + false, ) else { panic!("a streaming response returns chunks"); }; @@ -1092,4 +1802,48 @@ mod tests { } assert!(chunks.next().await.is_none()); } + + #[rstest] + #[case::no_delimiter(b"data: a".as_slice(), 0)] + #[case::lf(b"data: a\n\ndata: b", 9)] + #[case::crlf(b"data: a\r\n\r\ndata: b", 11)] + #[case::cr(b"data: a\r\rdata: b", 9)] + #[case::last_of_several(b"data: a\n\ndata: b\n\ndata: c", 18)] + fn complete_frames_end_is_after_the_last_delimiter(#[case] pending: &[u8], #[case] end: usize) { + assert_eq!(complete_frames_end(pending), end); + } + + #[rstest] + #[case::split_frame( + vec![&b"data: a\n"[..], b"\ndata: ", b"b\n\n"], + vec![&b"data: a\n\n"[..], b"data: b\n\n"] + )] + #[case::several_frames_in_one_chunk( + vec![&b"data: a\n\ndata: b\n\nda"[..], b"ta: c\n\n"], + vec![&b"data: a\n\ndata: b\n\n"[..], b"data: c\n\n"] + )] + #[case::unterminated_tail_is_flushed( + vec![&b"data: a\n\nda"[..], b"ta: b"], + vec![&b"data: a\n\n"[..], b"data: b"] + )] + #[tokio::test] + async fn whole_sse_frames_regroups_chunks_at_frame_boundaries( + #[case] chunks: Vec<&'static [u8]>, + #[case] expected: Vec<&'static [u8]>, + ) { + let upstream = futures_util::stream::iter( + chunks + .into_iter() + .map(|chunk| Ok::<_, Error>(Bytes::from_static(chunk))), + ) + .boxed(); + let framed: Vec = whole_sse_frames(upstream).try_collect().await.unwrap(); + assert_eq!( + framed, + expected + .into_iter() + .map(Bytes::from_static) + .collect::>() + ); + } } diff --git a/litellm-rust/crates/inference-messages/src/host_answer.rs b/litellm-rust/crates/inference-messages/src/host_answer.rs new file mode 100644 index 00000000000..07d183406ea --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/host_answer.rs @@ -0,0 +1,11 @@ +use futures_util::future::BoxFuture; + +use crate::Error; + +pub trait HostAnswer: Send + Sync { + fn answer(&self, stream: bool) -> BoxFuture<'_, Result<(), Error>>; + + fn intercept(&self, stream: bool) -> BoxFuture<'_, Result> { + Box::pin(async move { self.answer(stream).await.map(|()| true) }) + } +} diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index 25ebeaacb49..cd312ee61b1 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -1,25 +1,41 @@ mod common_utils; mod constants; +mod encoding; mod handler; +mod host_answer; +mod pre_request; +mod reads; pub mod route; +mod safeguards; +mod transport; mod types; +mod workload_identity; use futures_util::FutureExt; use litellm_auth::AuthServices; use litellm_host::interceptors::Interceptors; use litellm_inference::context::CallContext; use litellm_inference::provider::resolve_llm_provider; +use litellm_llms_types::recognized::Recognized; use litellm_secrets::source::SecretSource; use std::sync::Arc; use crate::common_utils::messages_provider; pub use common_utils::merge_messages_header_sources; +pub use encoding::DecodingError; +pub use host_answer::HostAnswer; pub use litellm_inference::Error; +pub use pre_request::PreRequestHooks; +pub use safeguards::Safeguards; +pub use transport::{ + Delivered, Transport, TransportBody, TransportRequest, TransportResponse, UnencodableHeader, +}; pub use types::{ MessagesBody, MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params, messages_body, }; +pub use workload_identity::WorkloadIdentity; #[derive(Clone)] pub struct MessagesRoute { @@ -27,6 +43,11 @@ pub struct MessagesRoute { auth: Arc, secrets: Arc, cache: Option, + workload_identity: Option>, + transport: Option>, + pre_request_hooks: Option>, + safeguards: Option>, + host_answer: Option>, } impl MessagesRoute { @@ -40,6 +61,51 @@ impl MessagesRoute { auth, secrets, cache: None, + workload_identity: None, + transport: None, + pre_request_hooks: None, + safeguards: None, + host_answer: None, + } + } + + #[must_use] + pub fn with_workload_identity(self, workload_identity: Arc) -> Self { + Self { + workload_identity: Some(workload_identity), + ..self + } + } + + #[must_use] + pub fn with_transport(self, transport: Arc) -> Self { + Self { + transport: Some(transport), + ..self + } + } + + #[must_use] + pub fn with_pre_request_hooks(self, hooks: Arc) -> Self { + Self { + pre_request_hooks: Some(hooks), + ..self + } + } + + #[must_use] + pub fn with_safeguards(self, safeguards: Arc) -> Self { + Self { + safeguards: Some(safeguards), + ..self + } + } + + #[must_use] + pub fn with_host_answer(self, host_answer: Arc) -> Self { + Self { + host_answer: Some(host_answer), + ..self } } @@ -67,7 +133,7 @@ impl MessagesRoute { model = %call.body.model, provider, resolved_model, - stream = call.body.params.stream == Some(true), + stream = call.body.params.streams(), outcome ))] async fn run( @@ -76,24 +142,69 @@ impl MessagesRoute { context: CallContext<'_, impl Interceptors>, ) -> Result { litellm_inference::diagnostic::call(async { - let provider = resolve_llm_provider( - &call.body.model, - call.custom_llm_provider.as_deref(), - "messages", - )?; - let config = messages_provider(provider.provider, provider.model).ok_or_else(|| { - Error::InvalidProvider(<&str>::from(provider.provider).to_string()) - })?; - litellm_inference::diagnostic::provider( - provider.model, - <&'static str>::from(provider.provider), - ); - let model = provider.model.to_owned(); + let requested_stream = call.body.params.streams(); + let client_streams = call.body.params.stream == Some(Recognized::Known(true)); + let call = match &self.pre_request_hooks { + None => call, + Some(hooks) => { + let (config, model, _) = Self::provider(&call)?; + if let Some(output) = handler::cached(self, config, &model, &context).await? { + return Ok(output); + } + hooks.run(call).await? + } + }; + let (config, model, provider) = Self::provider(&call)?; + litellm_inference::diagnostic::provider(&model, provider); + if self.pre_request_hooks.is_none() + && let Some(output) = handler::cached(self, config, &model, &context).await? + { + return Ok(output); + } + if let Some(host_answer) = self + .host_answer + .as_deref() + .filter(|_| call.shaping.settings.intercepted) + && host_answer.intercept(requested_stream).await? + { + return handler::deliver_answer(self, config, model, requested_stream, &context) + .await; + } + if let Some(mock) = call.shaping.settings.mock_response { + return Ok( + handler::mocked(self, config, model, &context, mock, requested_stream).await, + ); + } + if let Some(host_answer) = self + .host_answer + .as_deref() + .filter(|_| call.shaping.settings.mcp_gateway) + { + let stream = call.body.params.streams(); + return handler::answered(self, host_answer, config, model, stream, &context).await; + } let secrets = self.secrets.resolve(config.config().secret_names()).await?; - handler::execute(self, config, model, call, secrets, &context) + handler::execute(self, config, model, call, secrets, client_streams, &context) .boxed() .await }) .await } + + fn provider( + call: &MessagesCall, + ) -> Result<(common_utils::MessagesProvider, String, &'static str), Error> { + let provider = resolve_llm_provider( + &call.body.model, + call.custom_llm_provider.as_deref(), + "messages", + )?; + let config = messages_provider(provider.provider, provider.model) + .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?; + Ok(( + config, + provider.model.to_owned(), + <&'static str>::from(provider.provider), + )) + } } diff --git a/litellm-rust/crates/inference-messages/src/pre_request.rs b/litellm-rust/crates/inference-messages/src/pre_request.rs new file mode 100644 index 00000000000..949dcd6585b --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/pre_request.rs @@ -0,0 +1,7 @@ +use futures_util::future::BoxFuture; + +use super::{Error, MessagesCall}; + +pub trait PreRequestHooks: Send + Sync { + fn run(&self, call: MessagesCall) -> BoxFuture<'_, Result>; +} diff --git a/litellm-rust/crates/inference-messages/src/reads.rs b/litellm-rust/crates/inference-messages/src/reads.rs new file mode 100644 index 00000000000..0e127a42a13 --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/reads.rs @@ -0,0 +1,394 @@ +use std::{collections::VecDeque, sync::Arc}; + +use bytes::Bytes; +use futures_util::{FutureExt, Stream, StreamExt, stream::BoxStream}; +use std::sync::Mutex; +use tokio::{sync::Notify, task::JoinHandle}; + +const READ_SIZE: usize = 16 * 1024; +const LOW_WATER: usize = 64 * 1024; +const HIGH_WATER: usize = 2 * LOW_WATER; +const SETTLE_YIELDS: usize = 64; + +struct Buffer { + chunks: VecDeque, + size: usize, + quiet: bool, + failure: Option, + ended: bool, +} + +impl Buffer { + fn take(&mut self) -> Bytes { + let mut joined = Vec::new(); + while let Some(mut chunk) = self.chunks.pop_front() { + let room = READ_SIZE - joined.len(); + if chunk.len() > room { + let rest = chunk.split_off(room); + self.chunks.push_front(rest); + } + if joined.is_empty() && (chunk.len() == READ_SIZE || self.chunks.is_empty()) { + self.size -= chunk.len(); + return chunk; + } + joined.extend_from_slice(&chunk); + self.size -= chunk.len(); + if joined.len() == READ_SIZE { + break; + } + } + Bytes::from(joined) + } +} + +struct Reads { + buffer: Mutex>, + received: Notify, + drained: Notify, +} + +enum Read { + Chunk(Bytes), + Failure(E), + End, +} + +impl Reads { + fn readable(&self) -> bool { + let buffer = self.buffer(); + buffer.failure.is_some() + || buffer.ended + || (!buffer.chunks.is_empty() && (buffer.quiet || buffer.size >= READ_SIZE)) + } + + fn quiet(&self) { + self.buffer().quiet = true; + self.received.notify_one(); + } + + fn read(&self) -> Option> { + let mut buffer = self.buffer(); + if !buffer.chunks.is_empty() { + let chunk = buffer.take(); + drop(buffer); + self.drained.notify_one(); + return Some(Read::Chunk(chunk)); + } + if let Some(failure) = buffer.failure.take() { + return Some(Read::Failure(failure)); + } + buffer.ended.then_some(Read::End) + } + + fn buffer(&self) -> std::sync::MutexGuard<'_, Buffer> { + self.buffer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +struct Reader(JoinHandle<()>); + +impl Drop for Reader { + fn drop(&mut self) { + self.0.abort(); + } +} + +pub(crate) fn joined_reads( + chunks: impl Stream> + Send + 'static, +) -> BoxStream<'static, Result> { + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return chunks.boxed(); + }; + let reads = Arc::new(Reads { + buffer: Mutex::new(Buffer { + chunks: VecDeque::new(), + size: 0, + quiet: false, + failure: None, + ended: false, + }), + received: Notify::new(), + drained: Notify::new(), + }); + let reader = Reader(runtime.spawn(fill(chunks.boxed(), Arc::clone(&reads)))); + futures_util::stream::unfold(Some((reads, reader)), |state| async move { + let (reads, reader) = state?; + loop { + let received = reads.received.notified(); + if reads.readable() { + break; + } + received.await; + } + let read = reads.read()?; + match read { + Read::Chunk(chunk) => Some((Ok(chunk), Some((reads, reader)))), + Read::Failure(failure) => Some((Err(failure), None)), + Read::End => None, + } + }) + .boxed() +} + +async fn fill(mut chunks: BoxStream<'static, Result>, reads: Arc>) { + loop { + let next = match settled_next(&mut chunks).await { + Some(next) => next, + None => { + reads.quiet(); + chunks.next().await + } + }; + let paused = { + let mut buffer = reads.buffer(); + buffer.quiet = false; + match next { + Some(Ok(chunk)) => { + buffer.size += chunk.len(); + if !chunk.is_empty() { + buffer.chunks.push_back(chunk); + } + } + Some(Err(failure)) => { + buffer.failure = Some(failure); + buffer.ended = true; + } + None => buffer.ended = true, + } + if buffer.ended { + drop(buffer); + reads.received.notify_one(); + return; + } + buffer.size > HIGH_WATER + }; + reads.received.notify_one(); + if paused { + reads.quiet(); + loop { + let drained = reads.drained.notified(); + if reads.buffer().size < LOW_WATER { + break; + } + drained.await; + } + } + } +} + +async fn settled_next( + chunks: &mut BoxStream<'static, Result>, +) -> Option>> { + for _ in 0..SETTLE_YIELDS { + if let Some(next) = chunks.next().now_or_never() { + return Some(next); + } + tokio::task::yield_now().await; + } + None +} + +#[cfg(test)] +mod tests { + use std::{ + pin::Pin, + sync::atomic::{AtomicUsize, Ordering}, + task::{Context, Poll}, + time::Duration, + }; + + use futures_util::{FutureExt, StreamExt, TryStreamExt}; + use tokio::sync::mpsc; + + use super::*; + + struct Upstream { + receiver: mpsc::UnboundedReceiver>, + pulled: Arc, + } + + impl Stream for Upstream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let next = self.receiver.poll_recv(cx); + if let Poll::Ready(Some(_)) = next { + self.pulled.fetch_add(1, Ordering::SeqCst); + } + next + } + } + + type Item = Result; + + fn upstream() -> ( + mpsc::UnboundedSender, + BoxStream<'static, Item>, + Arc, + ) { + let (sender, receiver) = mpsc::unbounded_channel(); + let pulled = Arc::new(AtomicUsize::new(0)); + let reads = joined_reads(Upstream { + receiver, + pulled: Arc::clone(&pulled), + }); + (sender, reads, pulled) + } + + async fn settled() { + tokio::time::sleep(Duration::from_millis(20)).await; + } + + #[rstest::rstest] + #[tokio::test] + async fn chunks_that_arrived_together_are_read_as_one() { + let (sender, mut reads, _) = upstream(); + sender.send(Ok(Bytes::from("event: a\n\n"))).unwrap(); + sender.send(Ok(Bytes::from("event: b\n\n"))).unwrap(); + sender.send(Ok(Bytes::from("event: c\n\n"))).unwrap(); + settled().await; + assert_eq!( + reads.try_next().await.unwrap().unwrap(), + "event: a\n\nevent: b\n\nevent: c\n\n" + ); + sender.send(Ok(Bytes::from("event: d\n\n"))).unwrap(); + settled().await; + assert_eq!(reads.try_next().await.unwrap().unwrap(), "event: d\n\n"); + drop(sender); + assert!(reads.try_next().await.unwrap().is_none()); + } + + #[rstest::rstest] + #[tokio::test] + async fn chunks_still_being_handed_over_join_the_read() { + let handed_over = futures_util::stream::unfold(0, |sent| async move { + tokio::task::yield_now().await; + (sent < 50).then(|| (Ok::<_, &str>(Bytes::from("event: ping\n\n")), sent + 1)) + }); + let read: Vec = joined_reads(handed_over) + .map_ok(|chunk| chunk.len()) + .try_collect() + .await + .unwrap(); + assert_eq!(read, [50 * "event: ping\n\n".len()]); + } + + #[rstest::rstest] + #[tokio::test] + async fn a_read_returns_the_first_chunk_without_waiting_for_more() { + let (sender, mut reads, _) = upstream(); + let first = tokio::spawn(async move { reads.try_next().await }); + settled().await; + sender.send(Ok(Bytes::from("event: a\n\n"))).unwrap(); + assert_eq!(first.await.unwrap().unwrap().unwrap(), "event: a\n\n"); + } + + #[rstest::rstest] + #[tokio::test] + async fn reads_are_cut_at_sixteen_kibibytes() { + let (sender, reads, _) = upstream(); + sender.send(Ok(Bytes::from(vec![b'a'; 10_000]))).unwrap(); + sender.send(Ok(Bytes::from(vec![b'b'; 30_000]))).unwrap(); + drop(sender); + settled().await; + let sizes: Vec = reads + .map_ok(|chunk| chunk.len()) + .try_collect() + .await + .unwrap(); + assert_eq!(sizes, [READ_SIZE, READ_SIZE, 40_000 - 2 * READ_SIZE]); + } + + #[rstest::rstest] + #[tokio::test] + async fn the_bytes_still_buffered_are_read_before_a_failure() { + let (sender, mut reads, _) = upstream(); + sender.send(Ok(Bytes::from("event: a\n\n"))).unwrap(); + sender.send(Ok(Bytes::from("event: b\n\n"))).unwrap(); + sender.send(Err("reset")).unwrap(); + settled().await; + assert_eq!( + reads.try_next().await.unwrap().unwrap(), + "event: a\n\nevent: b\n\n" + ); + assert_eq!(reads.try_next().await.unwrap_err(), "reset"); + assert!(reads.next().await.is_none()); + } + + #[rstest::rstest] + #[tokio::test] + async fn reading_pauses_above_the_high_water_mark_until_below_the_low_one() { + let (sender, mut reads, pulled) = upstream(); + std::iter::repeat_with(|| Bytes::from(vec![b'a'; READ_SIZE])) + .take(12) + .try_for_each(|chunk| sender.send(Ok(chunk))) + .unwrap(); + settled().await; + assert_eq!(pulled.load(Ordering::SeqCst), 9); + let read: Vec = (&mut reads).take(5).try_collect().await.unwrap(); + assert_eq!(read.len(), 5); + settled().await; + assert_eq!(pulled.load(Ordering::SeqCst), 9); + reads.try_next().await.unwrap(); + settled().await; + assert_eq!(pulled.load(Ordering::SeqCst), 12); + } + + async fn chunked_upstream(frames: usize) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0_u8; 4096]; + let _ = socket.read(&mut request).await.unwrap(); + let frame = "event: ping\ndata: {\"type\": \"ping\"}\n\n"; + let body: String = + std::iter::repeat_n(format!("{:x}\r\n{frame}\r\n", frame.len()), frames).collect(); + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\nconnection: close\r\n\r\n{body}0\r\n\r\n" + ); + socket.write_all(response.as_bytes()).await.unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + }); + format!("http://{address}") + } + + #[rstest::rstest] + #[case::fifty(50)] + #[case::a_thousand(1000)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn http_chunks_written_together_are_read_together(#[case] frames: usize) { + let url = chunked_upstream(frames).await; + let response = litellm_http::Client::for_test() + .get(url) + .send() + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + let sizes: Vec = joined_reads(response.bytes_stream()) + .map_ok(|chunk| chunk.len()) + .try_collect() + .await + .unwrap(); + let total = frames * "event: ping\ndata: {\"type\": \"ping\"}\n\n".len(); + let expected: Vec = std::iter::repeat_n(READ_SIZE, total / READ_SIZE) + .chain((!total.is_multiple_of(READ_SIZE)).then_some(total % READ_SIZE)) + .collect(); + assert_eq!(sizes, expected); + } + + #[rstest::rstest] + fn without_a_runtime_the_chunks_pass_through() { + let chunks = + futures_util::stream::iter([Ok::<_, &str>(Bytes::from("a")), Ok(Bytes::from("b"))]); + let read: Vec = joined_reads(chunks) + .try_collect() + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(read, [Bytes::from("a"), Bytes::from("b")]); + } +} diff --git a/litellm-rust/crates/inference-messages/src/route.rs b/litellm-rust/crates/inference-messages/src/route.rs index 5f06d16d49f..fa850b23bf2 100644 --- a/litellm-rust/crates/inference-messages/src/route.rs +++ b/litellm-rust/crates/inference-messages/src/route.rs @@ -74,4 +74,103 @@ impl litellm_inference::caching::StreamCachable for Messages { fn bytes(chunk: &Self::Chunk) -> &[u8] { chunk.as_ref() } + + fn storable_stream(text: &str) -> bool { + let lines: Vec<&str> = sse_lines(text).collect(); + lines.contains(&"event: message_stop") + && !carries_provider_error(&lines) + && lines.iter().any(|line| line_carries_output(line)) + } +} + +fn sse_lines(text: &str) -> impl Iterator { + let mut rest = Some(text); + std::iter::from_fn(move || { + let current = rest?; + match current.find(['\r', '\n']) { + Some(at) => { + let skip = if current[at..].starts_with("\r\n") { + 2 + } else { + 1 + }; + rest = Some(¤t[at + skip..]); + Some(¤t[..at]) + } + None => { + rest = None; + (!current.is_empty()).then_some(current) + } + } + }) +} + +fn sse_data(line: &str, stripped: bool) -> Option> { + let data = line.strip_prefix("data:")?; + let data = if stripped { + data.trim_matches(|c: char| matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c')) + } else { + data + }; + match serde_json::from_str(data) { + Ok(serde_json::Value::Object(map)) => Some(map), + _ => None, + } +} + +fn carries_provider_error(lines: &[&str]) -> bool { + lines + .iter() + .filter_map(|line| sse_data(line, true)) + .find(|event| event.get("type").and_then(serde_json::Value::as_str) == Some("error")) + .is_some_and(|event| event.get("error").is_some_and(serde_json::Value::is_object)) +} + +fn line_carries_output(line: &str) -> bool { + let Some(event) = sse_data(line, false) else { + return false; + }; + let block = match event.get("type").and_then(serde_json::Value::as_str) { + Some("content_block_start") => event.get("content_block"), + Some("content_block_delta") => event.get("delta"), + _ => None, + }; + block + .and_then(serde_json::Value::as_object) + .is_some_and(|fields| { + fields.iter().any(|(name, value)| { + name != "type" + && !matches!(value, serde_json::Value::Null) + && value.as_str() != Some("") + && value.as_array().is_none_or(|items| !items.is_empty()) + }) + }) +} + +#[cfg(test)] +mod tests { + use litellm_inference::caching::StreamCachable; + use rstest::rstest; + + use super::Messages; + + const TEXT: &str = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Paris\"}}\n\n"; + const EMPTY: &str = "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"\"}}\n\n"; + const STOP: &str = "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"; + + #[rstest] + #[case::complete(format!("{TEXT}{STOP}"), true)] + #[case::crlf_lines(format!("{TEXT}{STOP}").replace('\n', "\r\n"), true)] + #[case::done_sentinel(format!("{TEXT}{STOP}data: [DONE]\n\n"), true)] + #[case::data_only_frames(format!("{TEXT}{STOP}").replace("event: message_stop\n", ""), false)] + #[case::event_without_space(format!("{TEXT}{STOP}").replace("event: message_stop", "event:message_stop"), false)] + #[case::provider_error(format!("{TEXT}{STOP}event: error\ndata: {{\"type\":\"error\",\"error\":{{\"type\":\"overloaded_error\"}}}}\n\n"), false)] + #[case::error_without_body(format!("{TEXT}{STOP}data: {{\"type\":\"error\",\"error\":\"overloaded\"}}\n\n"), true)] + #[case::no_output(format!("{EMPTY}{STOP}"), false)] + fn a_stream_is_stored_by_python_s_message_stop_error_and_output_tests( + #[case] text: String, + #[case] stored: bool, + ) { + assert_eq!(Messages::storable_stream(&text), stored); + } } diff --git a/litellm-rust/crates/inference-messages/src/safeguards.rs b/litellm-rust/crates/inference-messages/src/safeguards.rs new file mode 100644 index 00000000000..78e4a752536 --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/safeguards.rs @@ -0,0 +1,248 @@ +use std::{collections::VecDeque, sync::Arc}; + +use bytes::Bytes; +use futures_util::{StreamExt, TryStreamExt, future::BoxFuture, stream::BoxStream}; +use serde_json::Value; + +use super::{Error, MessagesBody, MessagesCallResponse}; + +const SAFEGUARD_RESULTS: &str = "safeguard_results"; + +pub trait Safeguards: Send + Sync { + fn results<'a>( + &'a self, + response: &'a MessagesBody, + ) -> BoxFuture<'a, Result, Error>>; + + fn open_stream(&self) -> BoxFuture<'_, Result>; + + fn frames(&self, chunk: Bytes) -> BoxFuture<'_, Result, Error>>; + + fn flush(&self) -> BoxFuture<'_, Result, Error>>; +} + +pub(super) async fn stamped( + safeguards: Option<&Arc>, + output: MessagesCallResponse, +) -> Result { + let Some(safeguards) = safeguards else { + return Ok(output); + }; + match output { + MessagesCallResponse::Complete(response) => { + let results = safeguards.results(&response).await?; + Ok(MessagesCallResponse::Complete(match results { + Some(results) => with_results(response, results), + None => response, + })) + } + MessagesCallResponse::Stream { head, chunks } => { + if !safeguards.open_stream().await? { + return Ok(MessagesCallResponse::Stream { head, chunks }); + } + Ok(MessagesCallResponse::Stream { + head, + chunks: stamped_frames(Arc::clone(safeguards), chunks), + }) + } + } +} + +fn with_results(response: MessagesBody, results: Value) -> MessagesBody { + match response { + MessagesBody::Message(mut message) => { + message.extra.insert(SAFEGUARD_RESULTS.to_owned(), results); + MessagesBody::Message(message) + } + MessagesBody::Unchecked(mut body) => { + body.insert(SAFEGUARD_RESULTS.to_owned(), results); + MessagesBody::Unchecked(body) + } + } +} + +struct Framing { + chunks: Option>>, + safeguards: Arc, + ready: VecDeque, +} + +fn stamped_frames( + safeguards: Arc, + chunks: BoxStream<'static, Result>, +) -> BoxStream<'static, Result> { + let framing = Framing { + chunks: Some(chunks), + safeguards, + ready: VecDeque::new(), + }; + futures_util::stream::try_unfold(framing, |mut framing| async move { + loop { + if let Some(frame) = framing.ready.pop_front() { + return Ok(Some((frame, framing))); + } + let Some(mut chunks) = framing.chunks.take() else { + return Ok(None); + }; + let frames = match chunks.try_next().await? { + Some(chunk) => { + framing.chunks = Some(chunks); + framing.safeguards.frames(chunk).await? + } + None => framing.safeguards.flush().await?, + }; + framing.ready.extend(frames); + } + }) + .boxed() +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use litellm_llms_types::formats::messages::MessagesResponse; + use rstest::rstest; + use serde_json::{Map, json}; + + use super::*; + use crate::route::MessagesStreamHead; + + struct Scripted { + results: Option, + streams: bool, + fed: Mutex>, + } + + impl Scripted { + fn new(results: Option, streams: bool) -> Arc { + Arc::new(Self { + results, + streams, + fed: Mutex::new(Vec::new()), + }) + } + } + + impl Safeguards for Scripted { + fn results<'a>( + &'a self, + _: &'a MessagesBody, + ) -> BoxFuture<'a, Result, Error>> { + Box::pin(async move { Ok(self.results.clone()) }) + } + + fn open_stream(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { Ok(self.streams) }) + } + + fn frames(&self, chunk: Bytes) -> BoxFuture<'_, Result, Error>> { + Box::pin(async move { + self.fed.lock().unwrap().push(chunk.clone()); + Ok(chunk + .split_inclusive(|byte| *byte == b';') + .map(Bytes::copy_from_slice) + .collect()) + }) + } + + fn flush(&self) -> BoxFuture<'_, Result, Error>> { + Box::pin(async move { Ok(vec![Bytes::from_static(b"tail")]) }) + } + } + + fn message() -> MessagesBody { + let response: MessagesResponse = serde_json::from_value(json!({ + "id": "msg_1", "type": "message", "role": "assistant", "model": "m", + "content": [], "stop_reason": "end_turn", "stop_sequence": null, + })) + .unwrap(); + MessagesBody::Message(Box::new(response)) + } + + fn stream(chunks: &[&'static [u8]]) -> MessagesCallResponse { + let chunks: Vec> = chunks + .iter() + .map(|chunk| Ok(Bytes::from_static(chunk))) + .collect(); + MessagesCallResponse::Stream { + head: MessagesStreamHead { + headers: Vec::new(), + }, + chunks: futures_util::stream::iter(chunks).boxed(), + } + } + + async fn collected(output: MessagesCallResponse) -> Vec { + let MessagesCallResponse::Stream { chunks, .. } = output else { + panic!("expected a stream"); + }; + chunks.try_collect().await.unwrap() + } + + #[rstest] + #[case::message_gains_the_results(message(), Some(json!([{"type": "dangerous_tool_use"}])))] + #[case::unchecked_gains_the_results( + MessagesBody::Unchecked(Map::new()), + Some(json!([{"type": "dangerous_tool_use"}])) + )] + #[case::declined_leaves_it(message(), None)] + #[tokio::test] + async fn a_complete_response_carries_what_the_classifier_answered( + #[case] response: MessagesBody, + #[case] results: Option, + ) { + let safeguards: Arc = Scripted::new(results.clone(), false); + let MessagesCallResponse::Complete(stamped) = stamped( + Some(&safeguards), + MessagesCallResponse::Complete(response.clone()), + ) + .await + .unwrap() else { + panic!("expected a complete response"); + }; + let body = serde_json::to_value(&stamped).unwrap(); + assert_eq!(body.get(SAFEGUARD_RESULTS), results.as_ref()); + let mut unstamped = body; + unstamped.as_object_mut().unwrap().remove(SAFEGUARD_RESULTS); + assert_eq!(unstamped, serde_json::to_value(&response).unwrap()); + } + + #[tokio::test] + async fn a_stream_is_reframed_through_the_classifier_and_flushed_at_its_end() { + let scripted = Scripted::new(None, true); + let safeguards: Arc = scripted.clone(); + let frames = collected( + stamped(Some(&safeguards), stream(&[b"a;b", b";c;"])) + .await + .unwrap(), + ) + .await; + assert_eq!( + frames, + ["a;", "b", ";", "c;", "tail"] + .map(|frame| Bytes::from_static(frame.as_bytes())) + .to_vec() + ); + assert_eq!(scripted.fed.lock().unwrap().len(), 2); + } + + #[rstest] + #[case::declined(Some(Scripted::new(None, false) as Arc))] + #[case::not_classified(None)] + #[tokio::test] + async fn a_stream_the_classifier_does_not_answer_passes_through( + #[case] safeguards: Option>, + ) { + let frames = collected( + stamped(safeguards.as_ref(), stream(&[b"a;b", b";c;"])) + .await + .unwrap(), + ) + .await; + assert_eq!( + frames, + [b"a;b".as_slice(), b";c;"].map(Bytes::from_static).to_vec() + ); + } +} diff --git a/litellm-rust/crates/inference-messages/src/transport.rs b/litellm-rust/crates/inference-messages/src/transport.rs new file mode 100644 index 00000000000..1353592447b --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/transport.rs @@ -0,0 +1,59 @@ +use bytes::Bytes; +use futures_util::{future::BoxFuture, stream::BoxStream}; +use serde_json::Value; + +use crate::Error; + +pub enum TransportBody { + Json(Value), + Signed(Bytes), +} + +pub struct TransportRequest { + pub url: String, + pub headers: Vec<(String, String)>, + pub body: TransportBody, + pub stream: bool, +} + +pub enum Delivered { + Complete(String), + Stream(BoxStream<'static, Result>), + Rejected { text: String, error: Error }, +} + +pub struct TransportResponse { + pub status: u16, + pub headers: Vec<(String, String)>, + pub delivered: Delivered, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum UnencodableHeader { + Sent(Vec), + Received(Vec), +} + +impl std::fmt::Display for UnencodableHeader { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a header value is not ASCII, which Python's HTTP client cannot encode") + } +} + +impl std::error::Error for UnencodableHeader {} + +impl UnencodableHeader { + pub(crate) fn received(headers: &reqwest::header::HeaderMap) -> Option { + headers + .values() + .find(|value| !value.as_bytes().is_ascii()) + .map(|value| Self::Received(value.as_bytes().to_vec())) + } +} + +pub trait Transport: Send + Sync { + fn send( + &self, + request: TransportRequest, + ) -> BoxFuture<'static, Result>; +} diff --git a/litellm-rust/crates/inference-messages/src/types.rs b/litellm-rust/crates/inference-messages/src/types.rs index 772ea6f7292..9dbc171ecb0 100644 --- a/litellm-rust/crates/inference-messages/src/types.rs +++ b/litellm-rust/crates/inference-messages/src/types.rs @@ -2,7 +2,10 @@ use std::time::Duration; use bytes::Bytes; use litellm_host::call::CallOutput; -use litellm_llms::{ErrorDetail, base_llm::messages::context::MessagesModelCapabilities}; +use litellm_llms::{ + ErrorDetail, + base_llm::messages::context::{MessagesModelCapabilities, MessagesShapeOptions}, +}; use litellm_llms_types::{ formats::messages::{MessagesRequest, MessagesResponse}, headers::ProviderSpecificHeaders, @@ -71,9 +74,38 @@ pub struct MessagesSettings { pub additional_drop_params: Vec, #[serde(default)] pub request_timeout: Option, + #[serde(default)] + pub mock_response: Option>, + #[serde(default)] + pub whole_sse_frames: bool, + #[serde(default)] + pub joined_reads: bool, + #[serde(default)] + pub presanitized: bool, + #[serde(default)] + pub drops_unsupported_params: bool, + #[serde(default)] + pub intercepted: bool, + #[serde(default)] + pub mcp_gateway: bool, + #[serde(default)] + pub ascii_response_headers: bool, + #[serde(default)] + pub unencodable_request_header: Option>, + /// Python's HTTP client encodes every header value it sends as ASCII, so a value it cannot + /// encode fails the call before the send. + #[serde(default)] + pub ascii_request_headers: bool, } impl MessagesSettings { + pub fn shape_options(&self) -> MessagesShapeOptions { + MessagesShapeOptions { + reasoning_auto_summary: self.reasoning_auto_summary, + presanitized: self.presanitized, + } + } + pub fn request_timeout(&self) -> Option { self.request_timeout .and_then(|secs| Duration::try_from_secs_f64(secs).ok()) @@ -140,7 +172,8 @@ mod tests { "drop_params": true, "reasoning_auto_summary": true, "additional_drop_params": ["metadata.user_id", "thinking"], - "request_timeout": 1800.0 + "request_timeout": 1800.0, + "drops_unsupported_params": true }), MessagesShaping { settings: MessagesSettings { @@ -148,6 +181,8 @@ mod tests { reasoning_auto_summary: true, additional_drop_params: vec!["metadata.user_id".to_string(), "thinking".to_string()], request_timeout: Some(1800.0), + drops_unsupported_params: true, + ..MessagesSettings::default() }, capabilities: MessagesModelCapabilities { supports_reasoning: true, @@ -185,6 +220,10 @@ mod tests { serialized["reasoning_auto_summary"], json!(expected.settings.reasoning_auto_summary) ); + assert_eq!( + serialized["drops_unsupported_params"], + json!(expected.settings.drops_unsupported_params) + ); assert_eq!( serialized["additional_drop_params"], json!(expected.settings.additional_drop_params) diff --git a/litellm-rust/crates/inference-messages/src/workload_identity.rs b/litellm-rust/crates/inference-messages/src/workload_identity.rs new file mode 100644 index 00000000000..56445de6eea --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/workload_identity.rs @@ -0,0 +1,11 @@ +use futures_util::future::BoxFuture; +use litellm_router_types::LitellmParams; +use litellm_secret_value::SecretValue; + +pub trait WorkloadIdentity: Send + Sync { + fn token<'a>( + &'a self, + litellm_params: &'a LitellmParams, + api_base: Option<&'a str>, + ) -> BoxFuture<'a, Result, litellm_auth::Error>>; +} diff --git a/litellm-rust/crates/inference-messages/tests/caching.rs b/litellm-rust/crates/inference-messages/tests/caching.rs index 787501a8fed..ff83f4e41e1 100644 --- a/litellm-rust/crates/inference-messages/tests/caching.rs +++ b/litellm-rust/crates/inference-messages/tests/caching.rs @@ -301,3 +301,135 @@ async fn provider_reported_cost_is_delivered_once_and_never_charged_for_a_cache_ assert!(matches!(cached.source, ResultSource::Cache { .. })); assert_eq!(cached.cost, Cost::CacheHit); } + +#[derive(Default)] +struct CallerKeyed { + config: ResponseCacheConfig, + entry: std::sync::Mutex>, +} + +impl ResponseCacheService for CallerKeyed { + fn config(&self) -> &ResponseCacheConfig { + &self.config + } + + fn keyed_by_caller(&self) -> bool { + true + } + + fn lookup<'a>( + &'a self, + _: &'a litellm_cache_response::ResponseCacheRequest, + _: Duration, + ) -> std::pin::Pin< + Box< + dyn std::future::Future, litellm_cache::Error>> + + Send + + 'a, + >, + > { + Box::pin(async move { Ok(self.entry.lock().unwrap().clone()) }) + } + + fn store<'a>( + &'a self, + _: &'a litellm_cache_response::ResponseCacheRequest, + value: Value, + _: Duration, + ) -> std::pin::Pin< + Box> + Send + 'a>, + > { + Box::pin(async move { + *self.entry.lock().unwrap() = Some(value); + Ok(()) + }) + } +} + +#[tokio::test] +async fn a_caller_keyed_hit_is_served_before_the_request_is_built_or_intercepted() { + use litellm_inference_messages::{MessagesCall, MessagesCallResponse}; + + let response = json!({"id": "msg_caller", "type": "message", "role": "assistant", "model": "native-test", + "content": [{"type": "text", "text": "answer"}], "stop_reason": "end_turn", "stop_sequence": null, + "usage": {"input_tokens": 3, "output_tokens": 2}}); + let upstream = support::upstream([support::json_response(response)]).await; + let hooks = ChangingHooks::default(); + let route = support::messages_route(litellm_inference_testing::no_secrets()).with_cache( + ScopedCache::new(Arc::new(CallerKeyed::default()), CacheScope::Shared), + ); + let request = |api_key: Option<&str>| { + MessagesCall { + body: serde_json::from_value(json!({"model": "native-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})).unwrap(), + api_key: api_key.map(Into::into), api_base: Some(upstream.uri()), custom_llm_provider: Some("anthropic".into()), + litellm_params: Default::default(), extra_headers: None, provider_specific_header: None, + timeout: None, shaping: Default::default(), + } + }; + let first = route + .execute(request(Some("key")), &hooks, None) + .await + .unwrap(); + let second = route.execute(request(None), &hooks, None).await.unwrap(); + let (MessagesCallResponse::Complete(first), MessagesCallResponse::Complete(second)) = + (first, second) + else { + panic!("expected Messages responses"); + }; + assert_eq!(first, second); + assert_eq!(support::received(&upstream).await.len(), 1); + assert_eq!(hooks.calls.load(Ordering::SeqCst), 1); + let facts = hooks.facts.lock().unwrap(); + let [provider, cached] = facts.as_slice() else { + panic!("expected provider and cache results"); + }; + assert_eq!(provider.source, ResultSource::Provider); + assert!(matches!(cached.source, ResultSource::Cache { .. })); + assert_eq!(cached.cost, Cost::CacheHit); +} + +#[rstest] +#[case::unary(false, true)] +#[case::streamed(true, false)] +#[tokio::test] +async fn a_mock_is_stored_under_the_caller_key_only_when_unstreamed( + #[case] stream: bool, + #[case] stored: bool, +) { + use litellm_inference_messages::{ + MessagesBody, MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, + }; + + let mock = json!({"id": "msg_mock", "type": "message", "role": "assistant", "model": "native-test", + "content": [{"type": "text", "text": "mocked"}], "stop_reason": "end_turn", "stop_sequence": null, + "usage": {"input_tokens": 3, "output_tokens": 2}}); + let upstream = support::upstream([]).await; + let hooks = ChangingHooks::default(); + let service = Arc::new(CallerKeyed::default()); + let route = support::messages_route(litellm_inference_testing::no_secrets()) + .with_cache(ScopedCache::new(service.clone(), CacheScope::Shared)); + let call = MessagesCall { + body: serde_json::from_value(json!({"model": "native-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16, "stream": stream})).unwrap(), + api_key: Some("key".into()), api_base: Some(upstream.uri()), custom_llm_provider: Some("anthropic".into()), + litellm_params: Default::default(), extra_headers: None, provider_specific_header: None, + timeout: None, + shaping: MessagesShaping { + settings: MessagesSettings { + mock_response: serde_json::from_value(mock.clone()).unwrap(), + ..Default::default() + }, + ..Default::default() + }, + }; + + let output = route.execute(call, &hooks, None).await.unwrap(); + + let MessagesCallResponse::Complete(MessagesBody::Unchecked(body)) = output else { + panic!("expected the mock as a whole response"); + }; + assert_eq!(Value::Object(body), mock); + assert!(support::received(&upstream).await.is_empty()); + assert_eq!(hooks.calls.load(Ordering::SeqCst), 0); + assert!(hooks.facts.lock().unwrap().is_empty()); + assert_eq!(service.entry.lock().unwrap().is_some(), stored); +} diff --git a/litellm-rust/crates/inference-messages/tests/messages/gateway.rs b/litellm-rust/crates/inference-messages/tests/messages/gateway.rs new file mode 100644 index 00000000000..d6e591c723c --- /dev/null +++ b/litellm-rust/crates/inference-messages/tests/messages/gateway.rs @@ -0,0 +1,99 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +use futures_util::future::BoxFuture; +use litellm_inference_messages::{HostAnswer, MessagesRoute}; +use rstest::rstest; + +use super::*; + +#[derive(Default)] +struct CountingGateway(AtomicUsize); + +impl HostAnswer for CountingGateway { + fn answer(&self, _stream: bool) -> BoxFuture<'_, Result<(), Error>> { + self.0.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(()) }) + } +} + +fn gateway_call(call: MessagesCall, api_base: String, mcp_gateway: bool) -> MessagesCall { + MessagesCall { + api_key: Some("sk-ant-test".into()), + api_base: Some(api_base), + shaping: MessagesShaping { + settings: MessagesSettings { + mcp_gateway, + ..MessagesSettings::default() + }, + ..MessagesShaping::default() + }, + ..call + } +} + +async fn run_through(route: MessagesRoute, call: MessagesCall) -> Result { + let host = LocalMessagesHost::new(call); + litellm_host_native::in_process::run_hosted( + route.machine(host.request()?, None), + host.runtime(), + ) + .await +} + +#[rstest] +#[tokio::test] +async fn a_call_the_host_hands_to_the_mcp_gateway_never_reaches_the_provider(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let gateway = Arc::new(CountingGateway::default()); + let route = + messages_route(Arc::new(RecordingSecrets::empty())).with_host_answer(gateway.clone()); + + let output = run_through(route, gateway_call(call, upstream.uri(), true)) + .await + .unwrap(); + + let MessagesOutput::Complete(MessagesBody::Unchecked(body)) = output else { + panic!("the host answers in place of the provider"); + }; + assert!(body.is_empty()); + assert_eq!(gateway.0.load(Ordering::SeqCst), 1); + assert!(received(&upstream).await.is_empty()); +} + +#[rstest] +#[tokio::test] +async fn a_call_the_host_keeps_reaches_the_provider_past_the_gateway(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let gateway = Arc::new(CountingGateway::default()); + let route = + messages_route(Arc::new(RecordingSecrets::empty())).with_host_answer(gateway.clone()); + + let output = run_through(route, gateway_call(call, upstream.uri(), false)) + .await + .unwrap(); + + assert!(matches!( + output, + MessagesOutput::Complete(MessagesBody::Message(_)) + )); + assert_eq!(gateway.0.load(Ordering::SeqCst), 0); + only_request(&upstream).await; +} + +#[rstest] +#[tokio::test] +async fn a_mock_answers_before_the_gateway(call: MessagesCall) { + let gateway = Arc::new(CountingGateway::default()); + let route = + messages_route(Arc::new(RecordingSecrets::empty())).with_host_answer(gateway.clone()); + let mut mocked = gateway_call(call, UNREACHABLE_BASE.into(), true); + mocked.shaping.settings.mock_response = Some(object(json!({"mocked": true}))); + + let output = run_through(route, mocked).await.unwrap(); + + let MessagesOutput::Complete(MessagesBody::Unchecked(body)) = output else { + panic!("the mock is the call's response"); + }; + assert_eq!(body, object(json!({"mocked": true}))); + assert_eq!(gateway.0.load(Ordering::SeqCst), 0); +} diff --git a/litellm-rust/crates/inference-messages/tests/messages/host.rs b/litellm-rust/crates/inference-messages/tests/messages/host.rs index 063e9d18738..7254830a2b0 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/host.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/host.rs @@ -3,7 +3,7 @@ use litellm_host::lifecycle::ExecutionEvent; use std::sync::Mutex; use litellm_host::{ - interceptors::{ExecutionFacts, RequestContext, ResultSource, WireRequest}, + interceptors::{ExecutionFacts, RawResponse, RequestContext, ResultSource, WireRequest}, lifecycle::CallEvent, }; use litellm_inference_messages::{MessagesBody, MessagesCallResponse, route::Messages}; @@ -46,19 +46,23 @@ impl RecordingHost { Self::new(call, Box::new(Ok)) } - fn raw_responses(&self) -> Vec { + fn raw_heads(&self) -> Vec { self.events .lock() .unwrap() .iter() .filter_map(|event| match event { CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }) => { - Some(raw.body.clone()) + Some(raw.clone()) } _ => None, }) .collect() } + + fn raw_responses(&self) -> Vec { + self.raw_heads().into_iter().map(|raw| raw.body).collect() + } } impl RecordingHost { @@ -137,7 +141,7 @@ async fn rejected_results_are_not_delivered_or_cached( let response = if streaming { ResponseTemplate::new(200).set_body_raw( - "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hi\"}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", "text/event-stream", ) } else { @@ -291,22 +295,23 @@ async fn what_before_send_returns_is_what_the_provider_receives(call: MessagesCa } #[rstest] -#[case::enable(false, json!(true), Some(true))] -#[case::disable(true, json!(false), Some(false))] -#[case::null(true, Value::Null, Some(false))] -#[case::invalid(false, json!("true"), None)] +#[case::enable(false, json!(true), true)] +#[case::disable(true, json!(false), false)] +#[case::null(true, Value::Null, false)] +#[case::truthy_string(false, json!("false"), true)] +#[case::falsy_integer(true, json!(0), false)] #[tokio::test] async fn response_mode_follows_the_intercepted_request( call: MessagesCall, traces: TraceCapture, #[case] original_stream: bool, #[case] rewritten_stream: Value, - #[case] expected_stream: Option, + #[case] expected_stream: bool, ) { use futures_util::TryStreamExt; let sse = "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"; - let response = if expected_stream == Some(true) { + let response = if expected_stream { ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream") } else { message_response() @@ -332,14 +337,14 @@ async fn response_mode_follows_the_intercepted_request( .await?; match output { MessagesCallResponse::Stream { chunks, .. } => { - assert_eq!(expected_stream, Some(true)); + assert!(expected_stream); assert_eq!( chunks.try_collect::>().await?.concat(), sse.as_bytes() ); } MessagesCallResponse::Complete(message) => { - assert_eq!(expected_stream, Some(false)); + assert!(!expected_stream); assert_eq!( message, MessagesBody::Message(serde_json::from_value(message_body()).unwrap()) @@ -352,18 +357,12 @@ async fn response_mode_follows_the_intercepted_request( let summaries = traces.summaries("litellm.route"); assert_eq!(summaries.len(), 1); - let Some(expected_stream) = expected_stream else { - assert!(result.unwrap_err().is_request()); - assert!(received(&upstream).await.is_empty()); - assert_eq!(summaries[0]["outcome"], "failure"); - return; - }; result.unwrap(); assert_eq!( only_request(&upstream).await.json()["stream"], rewritten_stream ); - assert_eq!(host.raw_responses().len(), usize::from(!expected_stream)); + assert_eq!(host.raw_responses().len(), 1); assert_eq!(summaries[0]["stream"], expected_stream); assert_eq!(summaries[0]["outcome"], "success"); } @@ -405,14 +404,9 @@ async fn the_raw_upstream_text_is_emitted_once_for_a_message(call: MessagesCall) } #[rstest] -#[case::upstream_error(ResponseTemplate::new(500).set_body_string("boom"))] -#[case::stream(ResponseTemplate::new(200).set_body_raw("event: message_stop\ndata: {}\n\n", "text/event-stream"))] #[tokio::test] -async fn no_raw_response_is_emitted_for_a_stream_or_a_failure( - call: MessagesCall, - #[case] response: ResponseTemplate, -) { - let upstream = upstream([response]).await; +async fn no_raw_response_is_emitted_for_a_failure(call: MessagesCall) { + let upstream = upstream([ResponseTemplate::new(500).set_body_string("boom")]).await; let host = RecordingHost::passthrough(authenticated( with_fields(call, json!({"stream": true})), upstream.uri(), @@ -424,6 +418,32 @@ async fn no_raw_response_is_emitted_for_a_stream_or_a_failure( assert!(host.raw_responses().is_empty()); } +#[rstest] +#[tokio::test] +async fn a_stream_emits_its_head_without_a_body_before_any_chunk(call: MessagesCall) { + let upstream = upstream([ResponseTemplate::new(200) + .insert_header("request-id", "req_stream") + .set_body_raw("event: message_stop\ndata: {}\n\n", "text/event-stream")]) + .await; + let host = RecordingHost::passthrough(authenticated( + with_fields(call, json!({"stream": true})), + upstream.uri(), + )); + + let _ = run_through(&host).await; + + let [raw] = <[RawResponse; 1]>::try_from(host.raw_heads()) + .unwrap_or_else(|raws| panic!("expected one raw response, got {}", raws.len())); + assert_eq!(raw.body, ""); + let head = raw.head.expect("the stream's head is forwarded"); + assert_eq!(head.status, 200); + assert!( + head.headers + .iter() + .any(|(name, value)| name == "request-id" && value == "req_stream") + ); +} + /// Python logs `optional_params` after its requested-param gate, so a dropped param must /// not resurface in callbacks. #[rstest] @@ -456,6 +476,38 @@ async fn the_request_context_carries_the_shaped_params_without_model_or_messages assert_eq!(optional_params, json!({"max_tokens": 16})); } +#[rstest] +#[tokio::test] +async fn the_request_context_keeps_the_nested_fields_additional_drop_params_strip( + call: MessagesCall, +) { + let upstream = upstream([message_response()]).await; + let tool = json!({"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}); + let host = RecordingHost::passthrough(authenticated( + MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + additional_drop_params: vec!["tools[*].input_examples".into()], + ..MessagesSettings::default() + }, + ..MessagesShaping::default() + }, + ..with_fields(call, json!({"tools": [tool.clone()]})) + }, + upstream.uri(), + )); + + run_through(&host).await.expect("messages call succeeds"); + + let [optional_params] = <[Value; 1]>::try_from(host.optional_params.into_inner().unwrap()) + .unwrap_or_else(|seen| panic!("before_provider_request runs once, saw {}", seen.len())); + assert_eq!(optional_params["tools"], json!([tool])); + assert_eq!( + only_request(&upstream).await.json()["tools"][0].get("input_examples"), + None + ); +} + #[rstest] #[tokio::test] async fn the_request_context_leaves_out_a_listed_param_the_model_refuses(call: MessagesCall) { @@ -534,3 +586,98 @@ async fn the_request_context_carries_the_requested_reasoning_not_its_translation ) ); } + +#[derive(Default)] +struct RecordingAnswer(Mutex>); + +impl litellm_inference_messages::HostAnswer for RecordingAnswer { + fn answer(&self, stream: bool) -> futures_util::future::BoxFuture<'_, Result<(), Error>> { + self.0.lock().unwrap().push(stream); + Box::pin(async { Ok(()) }) + } +} + +#[rstest] +#[case::unary(false)] +#[case::stream(true)] +#[tokio::test] +async fn an_intercepted_call_is_answered_by_the_host_before_the_mock_and_the_provider( + call: MessagesCall, + #[case] streaming: bool, +) { + let upstream = upstream([message_response()]).await; + let answer = Arc::new(RecordingAnswer::default()); + let route = messages_route(no_secrets()).with_host_answer(answer.clone()); + let mut request = authenticated( + with_fields(call, json!({"stream": streaming})), + upstream.uri(), + ); + request.shaping.settings.intercepted = true; + request.shaping.settings.mock_response = Some(object(json!({"id": "mocked"}))); + let host = RecordingHost::passthrough(request); + + let output = route + .execute(host.request().unwrap(), &host, None) + .await + .unwrap(); + + assert!(matches!( + output, + MessagesCallResponse::Complete(MessagesBody::Unchecked(body)) if body.is_empty() + )); + assert_eq!(*answer.0.lock().unwrap(), vec![streaming]); + assert!(received(&upstream).await.is_empty()); + let facts = host.facts.lock().unwrap(); + assert_eq!(facts.len(), 1); + assert_eq!(facts[0].source, ResultSource::Host); +} + +#[rstest] +#[tokio::test] +async fn a_call_no_interceptor_owns_never_asks_the_host(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let answer = Arc::new(RecordingAnswer::default()); + let route = messages_route(no_secrets()).with_host_answer(answer.clone()); + let host = RecordingHost::passthrough(authenticated(call, upstream.uri())); + + route + .execute(host.request().unwrap(), &host, None) + .await + .unwrap(); + + assert!(answer.0.lock().unwrap().is_empty()); + assert_eq!(received(&upstream).await.len(), 1); +} + +#[derive(Default)] +struct DecliningAnswer(Mutex>); + +impl litellm_inference_messages::HostAnswer for DecliningAnswer { + fn answer(&self, _stream: bool) -> futures_util::future::BoxFuture<'_, Result<(), Error>> { + panic!("a declined interception never asks for an answer") + } + + fn intercept(&self, stream: bool) -> futures_util::future::BoxFuture<'_, Result> { + self.0.lock().unwrap().push(stream); + Box::pin(async { Ok(false) }) + } +} + +#[rstest] +#[tokio::test] +async fn an_interception_no_interceptor_takes_reaches_the_provider(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let answer = Arc::new(DecliningAnswer::default()); + let route = messages_route(no_secrets()).with_host_answer(answer.clone()); + let mut request = authenticated(call, upstream.uri()); + request.shaping.settings.intercepted = true; + let host = RecordingHost::passthrough(request); + + route + .execute(host.request().unwrap(), &host, None) + .await + .unwrap(); + + assert_eq!(*answer.0.lock().unwrap(), vec![false]); + assert_eq!(received(&upstream).await.len(), 1); +} diff --git a/litellm-rust/crates/inference-messages/tests/messages/main.rs b/litellm-rust/crates/inference-messages/tests/messages/main.rs index fe203287766..1d515e87b90 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/main.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/main.rs @@ -19,12 +19,16 @@ use wiremock::ResponseTemplate; mod support; use support::*; +mod gateway; mod host; +mod pre_request; mod request; mod response; +mod retry; mod secrets; mod stream; mod tencent; +mod transport; const MODEL: &str = "claude-sonnet-4-5"; diff --git a/litellm-rust/crates/inference-messages/tests/messages/pre_request.rs b/litellm-rust/crates/inference-messages/tests/messages/pre_request.rs new file mode 100644 index 00000000000..13a632bdecd --- /dev/null +++ b/litellm-rust/crates/inference-messages/tests/messages/pre_request.rs @@ -0,0 +1,141 @@ +use futures_util::future::BoxFuture; +use litellm_inference_messages::PreRequestHooks; +use rstest::rstest; + +use super::*; + +struct Hooks { + rewrite: Box Result + Send + Sync>, + runs: Mutex, +} + +impl PreRequestHooks for Hooks { + fn run(&self, call: MessagesCall) -> BoxFuture<'_, Result> { + *self.runs.lock().unwrap() += 1; + let rewritten = (self.rewrite)(call); + Box::pin(async move { rewritten }) + } +} + +fn hooks( + rewrite: impl Fn(MessagesCall) -> Result + Send + Sync + 'static, +) -> Arc { + Arc::new(Hooks { + rewrite: Box::new(rewrite), + runs: Mutex::new(0), + }) +} + +async fn run_hooked(hooks: Arc, call: MessagesCall) -> Result { + let host = LocalMessagesHost::new(call); + let route = messages_route(Arc::new(RecordingSecrets::empty())).with_pre_request_hooks(hooks); + litellm_host_native::in_process::run_hosted( + route.machine(host.request()?, None), + host.runtime(), + ) + .await +} + +fn mocked(call: MessagesCall, mock: Map) -> MessagesCall { + MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + mock_response: Some(mock), + ..call.shaping.settings + }, + ..call.shaping + }, + ..call + } +} + +fn routed(call: MessagesCall, base: String) -> MessagesCall { + MessagesCall { + api_key: Some("sk-test".into()), + api_base: Some(base), + ..call + } +} + +#[rstest] +#[tokio::test] +async fn the_provider_receives_the_call_the_hooks_leave(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let hooks = hooks(|call| { + Ok(with_fields( + call, + json!({"system": "hooked", "max_tokens": 7}), + )) + }); + + run_hooked(hooks.clone(), routed(call, upstream.uri())) + .await + .expect("messages call succeeds"); + + let sent: Value = only_request(&upstream).await.body_json().unwrap(); + assert_eq!(sent["system"], "hooked"); + assert_eq!(sent["max_tokens"], 7); + assert_eq!(*hooks.runs.lock().unwrap(), 1); +} + +#[rstest] +#[case::hook_sets_the_mock(false, true, true)] +#[case::hook_keeps_the_callers_mock(true, false, true)] +#[case::hook_clears_the_callers_mock(true, true, false)] +#[tokio::test] +async fn the_mock_is_read_from_the_hooked_call( + call: MessagesCall, + #[case] called_with_mock: bool, + #[case] hook_rewrites_mock: bool, + #[case] mocked_response: bool, +) { + let upstream = upstream([message_response()]).await; + let mock = object(json!({"id": "mocked"})); + let call = routed(call, upstream.uri()); + let call = match called_with_mock { + true => mocked(call, mock.clone()), + false => call, + }; + let hooked_mock = (!called_with_mock).then(|| mock.clone()); + let hooks = hooks(move |call| { + Ok(match (hook_rewrites_mock, &hooked_mock) { + (false, _) => call, + (true, Some(mock)) => mocked(call, mock.clone()), + (true, None) => MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + mock_response: None, + ..call.shaping.settings + }, + ..call.shaping + }, + ..call + }, + }) + }); + + let output = run_hooked(hooks, call) + .await + .expect("messages call succeeds"); + + let mocked_output = matches!( + output, + MessagesOutput::Complete(MessagesBody::Unchecked(ref body)) if *body == mock + ); + assert_eq!(mocked_output, mocked_response); + assert_eq!(received(&upstream).await.is_empty(), mocked_response); +} + +#[rstest] +#[tokio::test] +async fn a_failing_hook_fails_the_call_before_the_provider(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let hooks = hooks(|_| Err(Error::InvalidProvider("refused".into()))); + + let Err(error) = run_hooked(hooks, routed(call, upstream.uri())).await else { + panic!("the hook's failure fails the call"); + }; + + assert!(matches!(error, Error::InvalidProvider(ref provider) if provider == "refused")); + assert!(received(&upstream).await.is_empty()); +} diff --git a/litellm-rust/crates/inference-messages/tests/messages/response.rs b/litellm-rust/crates/inference-messages/tests/messages/response.rs index f978da5e995..7c2f3250acf 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/response.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/response.rs @@ -2,7 +2,9 @@ use litellm_host::{ interceptors::{ExecutionFacts, ResultSource}, lifecycle::ExecutionEvent, }; -use litellm_inference_messages::{MessagesBody, MessagesCallResponse, messages_body}; +use litellm_inference_messages::{ + MessagesBody, MessagesCallResponse, UnencodableHeader, messages_body, +}; use litellm_inference_testing::{ RecordingSecrets, http_config, no_secrets, provider_http, resources, }; @@ -153,6 +155,37 @@ async fn the_message_passes_through_losslessly(call: MessagesCall) { assert_eq!(serde_json::to_value(&message).unwrap(), upstream_body); } +#[rstest] +#[case::without_stop_reason(json!({"stop_sequence": null}))] +#[case::without_stop_fields(json!({}))] +#[tokio::test] +async fn a_message_without_stop_reason_keeps_it_absent(call: MessagesCall, #[case] stop: Value) { + let mut upstream_body = json!({ + "id": "msg_3", "type": "message", "role": "assistant", "model": MODEL, + "content": [{"type": "text", "text": "hi"}], + "usage": {"input_tokens": 1, "output_tokens": 1} + }); + upstream_body + .as_object_mut() + .unwrap() + .extend(stop.as_object().unwrap().clone()); + let upstream = upstream([json_response(upstream_body.clone())]).await; + + let output = run(MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect("messages call succeeds"); + + let MessagesOutput::Complete(MessagesBody::Unchecked(body)) = output else { + panic!("a message without stop_reason is returned as sent"); + }; + upstream_body["stop_sequence"] = Value::Null; + assert_eq!(Value::Object(body), upstream_body); +} + #[rstest] #[tokio::test] async fn a_json_error_envelope_is_kept_verbatim(call: MessagesCall) { @@ -310,6 +343,88 @@ async fn a_provider_slower_than_the_timeout_fails_the_call(call: MessagesCall) { assert_eq!(error.upstream_response(), None); } +fn with_settings(call: MessagesCall, settings: MessagesSettings) -> MessagesCall { + MessagesCall { + shaping: MessagesShaping { + settings, + ..call.shaping + }, + ..call + } +} + +#[rstest] +#[case::unary(false)] +#[case::stream(true)] +#[tokio::test] +async fn a_non_ascii_response_header_fails_the_call_when_python_reads_heads_as_ascii( + call: MessagesCall, + #[case] stream: bool, +) { + let upstream = upstream([message_response().insert_header("x-fz-unicode", "café")]).await; + let call = with_fields( + MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }, + json!({"stream": stream}), + ); + let settings = MessagesSettings { + ascii_response_headers: true, + ..MessagesSettings::default() + }; + + let error = run(with_settings(call, settings)) + .await + .expect_err("the head cannot be encoded"); + + assert_eq!( + litellm_error::find::(&error), + Some(&UnencodableHeader::Received("café".as_bytes().to_vec())) + ); +} + +#[rstest] +#[tokio::test] +async fn a_non_ascii_response_header_is_served_when_python_keeps_raw_heads(call: MessagesCall) { + let upstream = upstream([message_response().insert_header("x-fz-unicode", "café")]).await; + + let message = run_message(MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await; + + assert_eq!(message.id, "msg_1"); +} + +#[rstest] +#[tokio::test] +async fn an_unencodable_request_header_fails_before_anything_is_sent(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let call = MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }; + let settings = MessagesSettings { + unencodable_request_header: Some(b"h\xed\xa0\x80".to_vec()), + ..MessagesSettings::default() + }; + + let error = run(with_settings(call, settings)) + .await + .expect_err("the header cannot be encoded"); + + assert_eq!( + litellm_error::find::(&error), + Some(&UnencodableHeader::Sent(b"h\xed\xa0\x80".to_vec())) + ); + assert!(received(&upstream).await.is_empty()); +} + #[rstest] #[tokio::test] async fn a_transport_error_does_not_echo_the_provider_url(call: MessagesCall) { @@ -365,7 +480,7 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes } #[rstest] -#[case::mistyped_param(json!({"model": MODEL, "messages": [], "max_tokens": "16"}))] +#[case::mistyped_param(json!({"model": MODEL, "messages": "hello", "max_tokens": 16}))] #[case::missing_messages(json!({"model": MODEL, "max_tokens": 16}))] fn a_body_that_does_not_parse_is_an_invalid_request(#[case] raw: Value) { let error = messages_body(object(raw)).expect_err("the body is rejected"); diff --git a/litellm-rust/crates/inference-messages/tests/messages/retry.rs b/litellm-rust/crates/inference-messages/tests/messages/retry.rs new file mode 100644 index 00000000000..e48482dc8b0 --- /dev/null +++ b/litellm-rust/crates/inference-messages/tests/messages/retry.rs @@ -0,0 +1,364 @@ +use futures_util::TryStreamExt; +use litellm_host::{ + interceptors::{RetriedRequest, RetryReason}, + lifecycle::{CallEvent, ExecutionEvent}, +}; +use litellm_inference_messages::MessagesCallResponse; +use litellm_inference_testing::RecordingSecrets; +use litellm_llms::base_llm::messages::context::MessagesModelCapabilities; +use rstest::rstest; + +use super::*; + +const INVALID_SIGNATURE: &str = "messages.1.content.0: Invalid `signature` in `thinking` block"; +const BOUND_ELSEWHERE: &str = "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation."; + +fn rejection(status: u16, message: &str) -> ResponseTemplate { + status_response( + status, + json!({"type": "error", "error": {"type": "invalid_request_error", "message": message}}), + ) +} + +fn sse_response() -> ResponseTemplate { + ResponseTemplate::new(200).set_body_raw( + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + "text/event-stream", + ) +} + +fn replay(call: MessagesCall, api_base: String, stream: bool, thinking: Value) -> MessagesCall { + MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(api_base), + ..with_fields( + call, + json!({ + "max_tokens": 4096, + "stream": stream, + "thinking": thinking, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": "sig"}, + {"type": "text", "text": "hello"} + ]}, + {"role": "assistant", "content": [{"type": "redacted_thinking", "data": "opaque"}]}, + {"role": "user", "content": "again"} + ] + }), + ) + } +} + +fn enabled() -> Value { + json!({"type": "enabled", "budget_tokens": 2048}) +} + +async fn execute( + host: &RecordingCall, +) -> Result<(MessagesCallResponse, Vec), Error> { + let route = messages_route(Arc::new(RecordingSecrets::empty())); + let response = route + .execute(host.request()?, host, Some(host.events.0.sender.clone())) + .await?; + match response { + MessagesCallResponse::Stream { head, chunks } => Ok(( + MessagesCallResponse::Stream { + head, + chunks: Box::pin(futures_util::stream::empty()), + }, + chunks.try_collect().await?, + )), + complete => Ok((complete, Vec::new())), + } +} + +fn observed_retries(host: &RecordingCall) -> Vec { + host.events + .0 + .lock() + .unwrap() + .iter() + .filter_map(|event| match event { + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) => { + Some(retry.clone()) + } + _ => None, + }) + .collect() +} + +#[rstest] +#[case::invalid_signature(INVALID_SIGNATURE)] +#[case::missing_signature( + "messages.1.content.0.thinking.signature.str: Input should be a valid string" +)] +#[case::empty_thinking("messages.1.content.0.thinking: each thinking block must contain thinking")] +#[case::unpinned_binding_mismatch(BOUND_ELSEWHERE)] +#[tokio::test] +async fn a_rejected_thinking_replay_is_sent_once_more_without_thinking( + call: MessagesCall, + #[case] message: &str, + #[values(false, true)] stream: bool, + #[values("anthropic", "azure_ai", "deepseek", "minimax")] provider: &str, +) { + let success = if stream { + sse_response() + } else { + message_response() + }; + let upstream = upstream([rejection(400, message), success]).await; + let host = RecordingCall::::new(replay( + MessagesCall { + custom_llm_provider: Some(provider.into()), + ..call + }, + upstream.uri(), + stream, + enabled(), + )); + + let (response, chunks) = execute(&host).await.expect("the retry succeeds"); + + let [first, second] = <[_; 2]>::try_from(received(&upstream).await).unwrap(); + assert_eq!(first.json()["thinking"], enabled()); + assert_eq!( + second.json()["messages"], + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "text", "text": "hello"}]}, + {"role": "user", "content": "again"} + ]) + ); + let mut expected = first.json(); + expected.as_object_mut().unwrap().shift_remove("thinking"); + expected["messages"] = second.json()["messages"].clone(); + assert_eq!(second.json(), expected); + assert_eq!(first.header("x-hook"), second.header("x-hook")); + let expected_retry = RetriedRequest { + reasons: vec![RetryReason::ThinkingSignature], + body: second.json(), + }; + assert_eq!(*host.retried.lock().unwrap(), vec![expected_retry.clone()]); + assert_eq!(observed_retries(&host), vec![expected_retry]); + assert_eq!( + matches!(response, MessagesCallResponse::Stream { .. }), + stream + ); + assert_eq!(chunks.is_empty(), !stream); +} + +#[rstest] +#[tokio::test] +async fn a_second_rejection_is_the_error_the_caller_sees( + call: MessagesCall, + #[values(false, true)] stream: bool, +) { + let upstream = upstream([ + rejection(400, INVALID_SIGNATURE), + rejection( + 400, + "messages.0.content.0: Invalid `signature` in `thinking` block, again", + ), + ]) + .await; + let host = RecordingCall::::new(replay(call, upstream.uri(), stream, enabled())); + + let error = execute(&host).await.err().expect("the retry fails"); + + assert_eq!(received(&upstream).await.len(), 2); + let (status, body) = error.upstream_response().unwrap(); + assert_eq!(status, 400); + assert!(body.contains("again"), "{body}"); + assert_eq!(host.retried.lock().unwrap().len(), 1); +} + +#[rstest] +#[case::not_a_400(429, INVALID_SIGNATURE, json!({"type": "adaptive"}))] +#[case::unrelated_400(400, "max_tokens: 4096 > 64000", json!({"type": "adaptive"}))] +#[case::signature_without_thinking(400, "Invalid `signature` in `tool_use` block", json!({"type": "adaptive"}))] +#[case::caller_pinned_binding( + 400, + BOUND_ELSEWHERE, + json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}) +)] +#[tokio::test] +async fn other_failures_reach_the_caller_without_a_retry( + call: MessagesCall, + #[case] status: u16, + #[case] message: &str, + #[case] thinking: Value, +) { + let upstream = upstream([rejection(status, message), message_response()]).await; + let adaptive = MessagesShaping { + capabilities: MessagesModelCapabilities { + supports_adaptive_thinking: true, + ..MessagesModelCapabilities::default() + }, + ..MessagesShaping::default() + }; + let host = RecordingCall::::new(replay( + MessagesCall { + shaping: adaptive, + ..call + }, + upstream.uri(), + false, + thinking.clone(), + )); + + let error = execute(&host).await.err().expect("the call fails"); + + let sent = only_request(&upstream).await; + assert_eq!(sent.json()["thinking"], thinking); + assert_eq!(error.upstream_response().unwrap().0, status); + assert!(host.retried.lock().unwrap().is_empty()); + assert!(observed_retries(&host).is_empty()); +} + +const STRICT_REJECTION: &str = "Too many strict tools (21). The maximum is 20."; + +fn strict_tools( + call: MessagesCall, + api_base: String, + stream: bool, + drop_params: bool, +) -> MessagesCall { + MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(api_base), + shaping: MessagesShaping { + settings: MessagesSettings { + drop_params, + drops_unsupported_params: drop_params, + ..MessagesSettings::default() + }, + ..MessagesShaping::default() + }, + ..with_fields( + call, + json!({ + "max_tokens": 64, + "stream": stream, + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + {"name": "a", "input_schema": {"type": "object"}, "strict": true}, + {"name": "b", "input_schema": {"type": "object"}, "strict": false}, + {"name": "c", "input_schema": {"type": "object"}} + ] + }), + ) + } +} + +#[rstest] +#[tokio::test] +async fn a_rejected_strict_tool_is_sent_once_more_without_strict_under_drop_params( + call: MessagesCall, + #[values(false, true)] stream: bool, +) { + let success = if stream { + sse_response() + } else { + message_response() + }; + let upstream = upstream([rejection(400, STRICT_REJECTION), success]).await; + let host = RecordingCall::::new(strict_tools(call, upstream.uri(), stream, true)); + + let (response, _) = execute(&host).await.expect("the retry succeeds"); + + let [first, second] = <[_; 2]>::try_from(received(&upstream).await).unwrap(); + assert_eq!(first.json()["tools"][0]["strict"], json!(true)); + let mut expected = first.json(); + expected["tools"] = json!([ + {"name": "a", "input_schema": {"type": "object"}}, + {"name": "b", "input_schema": {"type": "object"}}, + {"name": "c", "input_schema": {"type": "object"}} + ]); + assert_eq!(second.json(), expected); + let expected_retry = RetriedRequest { + reasons: vec![RetryReason::StrictTools { dropped: 1 }], + body: second.json(), + }; + assert_eq!(*host.retried.lock().unwrap(), vec![expected_retry.clone()]); + assert_eq!(observed_retries(&host), vec![expected_retry]); + assert_eq!( + matches!(response, MessagesCallResponse::Stream { .. }), + stream + ); +} + +#[rstest] +#[tokio::test] +async fn a_rejected_strict_tool_reaches_the_caller_without_drop_params(call: MessagesCall) { + let upstream = upstream([rejection(400, STRICT_REJECTION), message_response()]).await; + let host = RecordingCall::::new(strict_tools(call, upstream.uri(), false, false)); + + let error = execute(&host).await.err().expect("the call fails"); + + assert_eq!( + only_request(&upstream).await.json()["tools"][0]["strict"], + json!(true) + ); + assert_eq!(error.upstream_response().unwrap().0, 400); + assert!(observed_retries(&host).is_empty()); +} + +#[rstest] +#[tokio::test] +async fn strict_tools_and_a_thinking_replay_share_the_one_retry(call: MessagesCall) { + let upstream = upstream([ + rejection(400, STRICT_REJECTION), + rejection(400, INVALID_SIGNATURE), + message_response(), + ]) + .await; + let host = RecordingCall::::new(replay( + strict_tools(call, upstream.uri(), false, true), + upstream.uri(), + false, + enabled(), + )); + + let error = execute(&host) + .await + .err() + .expect("the second rejection is raised"); + + let [first, second] = <[_; 2]>::try_from(received(&upstream).await).unwrap(); + assert_eq!(first.json()["thinking"], enabled()); + assert_eq!(second.json()["thinking"], enabled()); + assert_eq!( + second.json()["tools"][0], + json!({"name": "a", "input_schema": {"type": "object"}}) + ); + assert_eq!(received(&upstream).await.len(), 2); + assert!(error.upstream_response().unwrap().1.contains("signature")); +} + +#[rstest] +#[tokio::test] +async fn a_truthy_global_drop_params_that_is_not_true_does_not_drop_strict(call: MessagesCall) { + let upstream = upstream([rejection(400, STRICT_REJECTION), message_response()]).await; + let strict = strict_tools(call, upstream.uri(), false, true); + let host = RecordingCall::::new(MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + drops_unsupported_params: false, + ..strict.shaping.settings.clone() + }, + ..strict.shaping.clone() + }, + ..strict + }); + + let error = execute(&host).await.err().expect("the call fails"); + + assert_eq!( + only_request(&upstream).await.json()["tools"][0]["strict"], + json!(true) + ); + assert_eq!(error.upstream_response().unwrap().0, 400); + assert!(observed_retries(&host).is_empty()); +} diff --git a/litellm-rust/crates/inference-messages/tests/messages/stream.rs b/litellm-rust/crates/inference-messages/tests/messages/stream.rs index 99ba08495aa..c9738fa2a0d 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/stream.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/stream.rs @@ -451,7 +451,7 @@ async fn stalling_error_upstream() -> (String, JoinHandle<()>) { #[rstest] #[tokio::test] -async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) { +async fn a_stalled_stream_error_body_reads_as_empty(call: MessagesCall) { let (base, connection) = stalling_error_upstream().await; let host = RecordingStreamHost::new( MessagesCall { @@ -467,7 +467,7 @@ async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) { .expect_err("a stalled error body fails the call"); assert!(error.dispatched(), "{error:?}"); - assert_eq!(error.upstream_response(), None); + assert_eq!(error.upstream_response(), Some((429, ""))); assert!(host.seen.into_inner().unwrap().is_empty()); tokio::time::timeout(Duration::from_secs(5), connection) .await diff --git a/litellm-rust/crates/inference-messages/tests/messages/transport.rs b/litellm-rust/crates/inference-messages/tests/messages/transport.rs new file mode 100644 index 00000000000..f561b09e55f --- /dev/null +++ b/litellm-rust/crates/inference-messages/tests/messages/transport.rs @@ -0,0 +1,256 @@ +use std::collections::VecDeque; + +use bytes::Bytes; +use futures_util::{StreamExt, TryStreamExt, future::BoxFuture}; +use litellm_inference_messages::{ + Delivered, MessagesCallResponse, Transport, TransportBody, TransportRequest, TransportResponse, +}; +use litellm_inference_testing::RecordingSecrets; +use rstest::rstest; + +use super::*; + +const INVALID_SIGNATURE: &str = "messages.1.content.0: Invalid `signature` in `thinking` block"; +const UNREACHABLE_BASE: &str = "http://127.0.0.1:9"; + +#[derive(Debug, thiserror::Error)] +#[error("raised by the host client")] +struct Raised; + +enum Scripted { + Complete(Value), + Stream(&'static [&'static str]), + Rejected(u16, &'static str), + Failed, +} + +struct Sent { + url: String, + headers: Vec<(String, String)>, + body: Value, + stream: bool, +} + +#[derive(Default)] +struct ScriptedTransport { + script: Mutex>, + sent: Mutex>, +} + +impl ScriptedTransport { + fn new(script: impl IntoIterator) -> Arc { + Arc::new(Self { + script: Mutex::new(script.into_iter().collect()), + sent: Mutex::default(), + }) + } + + fn sent(&self) -> std::sync::MutexGuard<'_, Vec> { + self.sent.lock().unwrap() + } +} + +impl Transport for ScriptedTransport { + fn send( + &self, + request: TransportRequest, + ) -> BoxFuture<'static, Result> { + let TransportBody::Json(body) = request.body else { + panic!("an unsigned request hands the host its JSON body"); + }; + self.sent.lock().unwrap().push(Sent { + url: request.url, + headers: request.headers, + body, + stream: request.stream, + }); + let next = self + .script + .lock() + .unwrap() + .pop_front() + .expect("a scripted answer"); + Box::pin(async move { + let (status, delivered) = match next { + Scripted::Complete(body) => (200, Delivered::Complete(body.to_string())), + Scripted::Stream(frames) => ( + 200, + Delivered::Stream( + futures_util::stream::iter( + frames.iter().map(|frame| Ok(Bytes::from_static(frame.as_bytes()))), + ) + .boxed(), + ), + ), + Scripted::Rejected(status, message) => ( + status, + Delivered::Rejected { + text: json!({"type": "error", "error": {"type": "invalid_request_error", "message": message}}) + .to_string(), + error: Error::transport(Raised), + }, + ), + Scripted::Failed => return Err(Error::transport(Raised)), + }; + Ok(TransportResponse { + status, + headers: vec![("request-id".into(), "req_1".into())], + delivered, + }) + }) + } +} + +fn hosted_call(call: MessagesCall, fields: Value) -> MessagesCall { + MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(UNREACHABLE_BASE.into()), + ..with_fields(call, fields) + } +} + +async fn execute( + transport: Arc, + call: MessagesCall, +) -> Result<(MessagesCallResponse, Vec), Error> { + let host = RecordingCall::::new(call); + let route = messages_route(Arc::new(RecordingSecrets::empty())).with_transport(transport); + match route + .execute(host.request()?, &host, Some(host.events.0.sender.clone())) + .await? + { + MessagesCallResponse::Stream { head, chunks } => Ok(( + MessagesCallResponse::Stream { + head, + chunks: Box::pin(futures_util::stream::empty()), + }, + chunks.try_collect().await?, + )), + complete => Ok((complete, Vec::new())), + } +} + +#[rstest] +#[tokio::test] +async fn the_host_transport_sends_the_wire_request_instead_of_the_native_client( + call: MessagesCall, +) { + let transport = ScriptedTransport::new([Scripted::Complete(message_body())]); + + let (response, _) = execute(transport.clone(), hosted_call(call, json!({}))) + .await + .expect("the host transport answers"); + + let MessagesCallResponse::Complete(MessagesBody::Message(message)) = response else { + panic!("a complete message"); + }; + assert_eq!(message.id, "msg_1"); + let sent = transport.sent(); + let [request] = sent.as_slice() else { + panic!("one request"); + }; + assert_eq!(request.url, format!("{UNREACHABLE_BASE}/v1/messages")); + assert!(!request.stream); + assert_eq!(request.body["model"], MODEL); + assert!( + request + .headers + .iter() + .any(|(name, value)| name == "x-api-key" && value == "sk-ant"), + "{:?}", + request.headers + ); +} + +#[rstest] +#[tokio::test] +async fn a_host_stream_reaches_the_caller_chunk_by_chunk(call: MessagesCall) { + let frames: &'static [&'static str] = &[ + "event: message_start\ndata: {\"type\":\"message_start\"}\n\n", + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + ]; + let transport = ScriptedTransport::new([Scripted::Stream(frames)]); + + let (response, chunks) = execute( + transport.clone(), + hosted_call(call, json!({"stream": true})), + ) + .await + .expect("the host stream opens"); + + let MessagesCallResponse::Stream { head, .. } = response else { + panic!("a stream"); + }; + assert_eq!( + head.headers, + vec![("request-id".to_string(), "req_1".to_string())] + ); + assert_eq!( + chunks, + frames + .iter() + .map(|frame| Bytes::from_static(frame.as_bytes())) + .collect::>() + ); + assert!(transport.sent()[0].stream); +} + +fn replayed_thinking() -> Value { + json!({ + "max_tokens": 4096, + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": "sig"}, + {"type": "text", "text": "hello"} + ]}, + {"role": "user", "content": "again"} + ] + }) +} + +#[rstest] +#[tokio::test] +async fn a_recoverable_rejection_is_retried_through_the_host_transport(call: MessagesCall) { + let transport = ScriptedTransport::new([ + Scripted::Rejected(400, INVALID_SIGNATURE), + Scripted::Complete(message_body()), + ]); + + execute(transport.clone(), hosted_call(call, replayed_thinking())) + .await + .expect("the retry succeeds"); + + let sent = transport.sent(); + let [first, second] = sent.as_slice() else { + panic!("two requests"); + }; + assert!(first.body.get("thinking").is_some()); + assert!(second.body.get("thinking").is_none()); +} + +#[rstest] +#[case::final_rejection(vec![Scripted::Rejected(429, "slow down")])] +#[case::spent_retry(vec![ + Scripted::Rejected(400, INVALID_SIGNATURE), + Scripted::Rejected(400, INVALID_SIGNATURE), +])] +#[case::client_failure(vec![Scripted::Failed])] +#[tokio::test] +async fn the_caller_gets_the_error_the_host_transport_raised( + call: MessagesCall, + #[case] script: Vec, +) { + let attempts = script.len(); + let transport = ScriptedTransport::new(script); + + let error = execute(transport.clone(), hosted_call(call, replayed_thinking())) + .await + .err() + .expect("the call fails"); + + assert!(litellm_error::find::(&error).is_some(), "{error:?}"); + assert_eq!(error.upstream_response(), None); + assert_eq!(transport.sent().len(), attempts); +} diff --git a/litellm-rust/crates/inference-messages/tests/support/mod.rs b/litellm-rust/crates/inference-messages/tests/support/mod.rs index 3cecc865070..9115b6e89bb 100644 --- a/litellm-rust/crates/inference-messages/tests/support/mod.rs +++ b/litellm-rust/crates/inference-messages/tests/support/mod.rs @@ -117,6 +117,7 @@ pub struct RecordingCall { pub events: Arc, pub chunks: Mutex>, pub head: Mutex>, + pub retried: Mutex>, } #[derive(Default)] @@ -167,6 +168,7 @@ impl RecordingCall

{ events: Arc::new(CallEvents::default()), chunks: Mutex::new(Vec::new()), head: Mutex::new(None), + retried: Mutex::new(Vec::new()), } } } @@ -195,6 +197,14 @@ impl litellm_host::interceptors::Intercepto ) -> Result<(), P::Error> { Ok(()) } + + async fn provider_request_retried( + &self, + retry: litellm_host::interceptors::RetriedRequest, + ) -> Result<(), P::Error> { + self.retried.lock().unwrap().push(retry); + Ok(()) + } } impl

RecordingCall

diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs b/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs index 9f59bb9a2a2..509704340fe 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs @@ -22,6 +22,7 @@ pub(crate) fn event_name(event: &CallEvent) -> &'static str { CallEvent::Execution(ExecutionEvent::ResultReady { .. }) => "result_ready", CallEvent::Started { .. } => "started", CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { .. }) => "response", + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { .. }) => "retried", CallEvent::Succeeded { .. } => "success", CallEvent::Failed { .. } => "failure", CallEvent::Cancelled { .. } => "cancelled", diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/machine.rs b/litellm-rust/crates/inference-ocr/tests/ocr/machine.rs index c7a5d58ad3d..9da62e6ccf4 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/machine.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/machine.rs @@ -66,6 +66,11 @@ async fn drive_until( reply.send(host.after_provider_response(raw).await); Ok(()) } + HostRequest::Intercept(InterceptRequest::ProviderRequestRetried { reply, .. }) => { + ops.push("retried"); + reply.send(Ok(())); + Ok(()) + } }; if let Err(failure) = answer { break machine.interrupt(failure).await.map(completed); @@ -86,6 +91,7 @@ async fn drive_until_notified(machine: &mut OcrMachine, host: &LocalOcrHost, sto MachineStep::Suspended(HostRequest::HostCall(op)) => host.handle_host_call(op).await.unwrap(), MachineStep::Suspended(HostRequest::Intercept(InterceptRequest::BeforeProviderRequest { wire, reply, .. })) => reply.send(*wire), MachineStep::Suspended(HostRequest::Intercept(InterceptRequest::AfterProviderResponse { reply, .. })) => reply.send(Ok(())), + MachineStep::Suspended(HostRequest::Intercept(InterceptRequest::ProviderRequestRetried { reply, .. })) => reply.send(Ok(())), MachineStep::Suspended(HostRequest::Stream(stream)) => match stream { litellm_host::protocol::StreamDelivery::Open(head, _) => match head {}, litellm_host::protocol::StreamDelivery::Chunk(chunk, _) => match chunk {}, diff --git a/litellm-rust/crates/inference/AGENTS.md b/litellm-rust/crates/inference/AGENTS.md index bcbc98a209e..bc9be4825ea 100644 --- a/litellm-rust/crates/inference/AGENTS.md +++ b/litellm-rust/crates/inference/AGENTS.md @@ -81,3 +81,4 @@ - Cache lookup follows provider preparation, credential resolution, and the request interceptor - keys describe the effective provider URL, authenticated headers, and rewritten body - signed requests bypass caching until the signing identity has a stable cache representation +- A service that keys entries by the caller's arguments (`keyed_by_caller`, the Python host's legacy cache) is looked up before provider preparation, as Python's `@client` wrapper does, and a hit skips request building and the request interceptor diff --git a/litellm-rust/crates/inference/Cargo.toml b/litellm-rust/crates/inference/Cargo.toml index 807a3ea85d3..168b9a569ea 100644 --- a/litellm-rust/crates/inference/Cargo.toml +++ b/litellm-rust/crates/inference/Cargo.toml @@ -12,6 +12,7 @@ litellm-framer.workspace = true tokio-util = { version = "0.7", features = ["codec"] } litellm-secrets.workspace = true litellm-core-utils.workspace = true +litellm-error.workspace = true litellm-host.workspace = true bytes.workspace = true futures-util.workspace = true @@ -29,7 +30,6 @@ time.workspace = true url.workspace = true [dev-dependencies] -litellm-error.workspace = true litellm-cache-memory.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-llms = { workspace = true, features = ["test-support"] } diff --git a/litellm-rust/crates/inference/src/caching.rs b/litellm-rust/crates/inference/src/caching.rs index d9b5b891015..fa9ceb6bd73 100644 --- a/litellm-rust/crates/inference/src/caching.rs +++ b/litellm-rust/crates/inference/src/caching.rs @@ -61,6 +61,10 @@ pub trait StreamCachable: Cachable { fn replay(data: Bytes) -> Option>; fn bytes(chunk: &Self::Chunk) -> &[u8]; + + fn storable_stream(text: &str) -> bool { + successful_stream(text, Self::TERMINAL_EVENT) + } } #[derive(Serialize, Deserialize)] @@ -238,12 +242,19 @@ impl CallCache

{ identity: &ProviderIdentity, wire: &WireRequest, ) -> Self { + Self::new( + cache, + policy, + CacheRequest::from_wire(identity.clone(), Some(wire)), + ) + } + + pub fn new(cache: Option<&ScopedCache>, policy: CachePolicy, request: CacheRequest) -> Self { let session = cache.and_then(|cache| { if !policy.enabled() { return None; } let options = cache.options(Some(policy)); - let request = CacheRequest::from_wire(identity.clone(), Some(wire)); Some(CacheSession { request: options.request( &cache.service.config().namespace, @@ -281,7 +292,7 @@ impl CallCache

{ P::Response: Serialize, { let Some(session) = self.session.filter(|session| { - *source == ResultSource::Provider && session.request.controls.writes() + !matches!(source, ResultSource::Cache { .. }) && session.request.controls.writes() }) else { return output; }; @@ -322,7 +333,7 @@ fn capture_stream( None => { if let Some(data) = captured && let Ok(text) = String::from_utf8(data) - && successful_stream(&text, P::TERMINAL_EVENT) + && P::storable_stream(&text) && let Ok(entry) = serde_json::to_value(ResponseEnvelope::new( P::SURFACE, CachedOutput::::Stream(text), diff --git a/litellm-rust/crates/inference/src/context.rs b/litellm-rust/crates/inference/src/context.rs index ef2635d24a1..39b259a4c97 100644 --- a/litellm-rust/crates/inference/src/context.rs +++ b/litellm-rust/crates/inference/src/context.rs @@ -1,6 +1,6 @@ use litellm_cache_response::CachePolicy; use litellm_host::{ - interceptors::{ExecutionFacts, Interceptors, RawResponse, ResultSource}, + interceptors::{ExecutionFacts, Interceptors, RawResponse, ResultSource, RetriedRequest}, lifecycle::{CallEvent, ExecutionEvent}, observation::ObservationSender, }; @@ -57,4 +57,15 @@ impl<'a, I: Interceptors> CallContext<'a, I> { .await .map_err(Error::post_call) } + + pub async fn provider_request_retried(&self, retry: RetriedRequest) -> Result<(), Error> { + if let Some(observers) = &self.observers { + observers.emit(CallEvent::Execution( + ExecutionEvent::ProviderRequestRetried { + retry: retry.clone(), + }, + )); + } + self.interceptors.provider_request_retried(retry).await + } } diff --git a/litellm-rust/crates/inference/src/error.rs b/litellm-rust/crates/inference/src/error.rs index 4135424d6ef..5f6e1b3f70d 100644 --- a/litellm-rust/crates/inference/src/error.rs +++ b/litellm-rust/crates/inference/src/error.rs @@ -24,6 +24,8 @@ pub enum Error { Secret(#[from] SecretError), #[error("the host driver failed")] Host(#[source] MachineFault), + #[error("the host transport failed")] + Transport(#[source] litellm_error::Source), #[error("post-call hook failed")] PostCallHook(#[source] Arc), } @@ -47,6 +49,10 @@ impl Error { Self::PostCallHook(Arc::new(error)) } + pub fn transport(cause: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Transport(litellm_error::Source::new(cause)) + } + /// The caller's request is what is wrong, as opposed to the environment, the wire, or /// the provider's answer. pub fn is_request(&self) -> bool { @@ -55,7 +61,11 @@ impl Error { Self::Llm(error) => error.is_request(), Self::Auth(error) => error.is_request(), Self::Http(error) => error.is_request(), - Self::Declined(_) | Self::Secret(_) | Self::Host(_) | Self::PostCallHook(_) => false, + Self::Declined(_) + | Self::Secret(_) + | Self::Host(_) + | Self::Transport(_) + | Self::PostCallHook(_) => false, } } @@ -70,6 +80,7 @@ impl Error { | Self::Headers(_) | Self::Secret(_) | Self::Host(_) + | Self::Transport(_) | Self::PostCallHook(_) => false, } } @@ -85,6 +96,7 @@ impl Error { | Self::Http(_) | Self::Secret(_) | Self::Host(_) + | Self::Transport(_) | Self::PostCallHook(_) => false, } } @@ -100,6 +112,7 @@ impl Error { | Self::Http(_) | Self::Secret(_) | Self::Host(_) + | Self::Transport(_) | Self::PostCallHook(_) => None, } } @@ -121,7 +134,7 @@ impl Error { match self { Self::Llm(error) => error.dispatched(), Self::Http(error) => error.dispatched(), - Self::Host(_) | Self::PostCallHook(_) => true, + Self::Host(_) | Self::Transport(_) | Self::PostCallHook(_) => true, Self::InvalidProvider(_) | Self::UnresolvedProvider { .. } | Self::Declined(_) @@ -142,6 +155,7 @@ impl Error { | Self::Http(_) | Self::Secret(_) | Self::Host(_) + | Self::Transport(_) | Self::PostCallHook(_) => None, } } @@ -193,6 +207,7 @@ impl Error { | Self::Headers(_) | Self::Secret(_) | Self::Host(_) + | Self::Transport(_) | Self::PostCallHook(_) => None, } } diff --git a/litellm-rust/crates/inference/src/outbound.rs b/litellm-rust/crates/inference/src/outbound.rs index b0cc71d425d..87fae0bbe9e 100644 --- a/litellm-rust/crates/inference/src/outbound.rs +++ b/litellm-rust/crates/inference/src/outbound.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use litellm_http::outbound::{json, signed}; +use litellm_http::outbound::{json, json_bytes, signed}; use litellm_llms::base_llm::auth::Authenticated; use serde_json::Value; @@ -34,3 +34,18 @@ pub fn outbound_request( Some(signer) => signed(request, &signer), } } + +pub fn raw_leaf_outbound_request( + authenticated: Authenticated, + url: &str, + body: &Value, + timeout: Option, +) -> Result { + let Authenticated { headers, signer } = authenticated; + let bytes = litellm_host::raw_json::to_vec(body).map_err(litellm_http::Error::request_body)?; + let request = json_bytes(url, &headers, bytes, timeout)?; + match signer { + None => Ok(request), + Some(signer) => signed(request, &signer), + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/request.rs b/litellm-rust/crates/llms-types/src/formats/messages/request.rs index d30d21e74e7..2e53b3adc5f 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/request.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/request.rs @@ -320,7 +320,7 @@ impl ThinkingConfig { #[macro_rules_attribute::apply(crate::wire_type)] pub struct MessagesRequest { pub model: String, - pub messages: Vec, + pub messages: Vec>, #[serde(flatten)] pub params: MessagesOptionalParams, } @@ -329,7 +329,7 @@ pub struct MessagesRequest { #[derive(Default)] pub struct MessagesOptionalParams { #[serde(skip_serializing_if = "Option::is_none")] - pub max_tokens: Option, + pub max_tokens: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub system: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -337,13 +337,21 @@ pub struct MessagesOptionalParams { #[serde(skip_serializing_if = "Option::is_none")] pub stop_sequences: Option>, #[serde(skip_serializing_if = "Option::is_none")] - pub stream: Option, + pub stream: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::recognized::exact_float" + )] + pub temperature: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::recognized::exact_float" + )] + pub top_p: Option>, #[serde(skip_serializing_if = "Option::is_none")] - pub temperature: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub top_p: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub top_k: Option, + pub top_k: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub tools: Option>>, #[serde(skip_serializing_if = "Option::is_none")] @@ -374,6 +382,12 @@ pub struct MessagesOptionalParams { pub extra: Map, } +impl MessagesOptionalParams { + pub fn streams(&self) -> bool { + self.stream.as_ref().is_some_and(Recognized::is_truthy) + } +} + impl Message { pub fn blocks(&self) -> &[ContentBlock] { match &self.content { @@ -447,8 +461,8 @@ mod tests { assert_eq!( ( - request.params.max_tokens, - request.params.stream, + request.params.max_tokens.clone(), + request.params.stream.clone(), request .params .extra @@ -456,11 +470,39 @@ mod tests { .map(String::as_str) .collect::>(), ), - (Some(16_u64), Some(true), vec!["safeguards"]) + ( + Some(Recognized::Known(16_u64)), + Some(Recognized::Known(true)), + vec!["safeguards"] + ) ); assert_eq!(serde_json::to_value(request).unwrap(), body); } + #[rstest] + #[case::numeric_string(json!({"max_tokens": "1024", "temperature": "0.5", "top_p": "x", "top_k": "5"}))] + #[case::string_stream(json!({"stream": "true"}))] + #[case::integer_stream(json!({"stream": 0}))] + #[case::floats(json!({"max_tokens": 1024.0, "top_k": 1.5}))] + #[case::booleans(json!({"max_tokens": true, "temperature": false}))] + #[case::wide_integers(json!({"max_tokens": 18_446_744_073_709_551_615_u64, "top_k": 9_223_372_036_854_775_808_u64}))] + #[case::nested(json!({"temperature": [1], "top_p": {"p": 1}}))] + fn scalar_params_of_any_type_round_trip_unchanged(#[case] params: Value) { + let body = Value::Object( + [ + ("model".to_string(), json!("m")), + ( + "messages".to_string(), + json!([{"role": "user", "content": "hi"}]), + ), + ] + .into_iter() + .chain(params.as_object().unwrap().clone()) + .collect(), + ); + assert_eq!(round_trip::(&body), body); + } + #[test] fn text_constructor_serializes_as_a_text_block() { assert_eq!( diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index e3ad783c791..d1c839e4363 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -317,6 +317,11 @@ impl AnthropicBeta { BetaProvider::VertexAi, BetaProvider::Databricks, ], + Self::Compact20260904 => &[ + BetaProvider::BedrockConverse, + BetaProvider::BedrockMantle, + BetaProvider::Databricks, + ], Self::AfkMode20260131 | Self::CePlugins20260901 | Self::CeUserManagement20260713 diff --git a/litellm-rust/crates/llms-types/src/recognized.rs b/litellm-rust/crates/llms-types/src/recognized.rs index 0545389afbc..2c389103aef 100644 --- a/litellm-rust/crates/llms-types/src/recognized.rs +++ b/litellm-rust/crates/llms-types/src/recognized.rs @@ -17,6 +17,25 @@ where as serde::Deserialize<'de>>::deserialize(deserializer).map(Some) } +/// `deserialize_with` for an optional float Python forwards as the caller sent it: an integer +/// stays `Unrecognized` with its exact digits, as Python sends, logs and prints it, and `null` +/// stays absent as before. +pub fn exact_float<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = ::deserialize(deserializer)?; + if value.is_null() { + return Ok(None); + } + if matches!(&value, Value::Number(number) if !number.is_f64()) { + return Ok(Some(Recognized::Unrecognized(value))); + } + serde_json::from_value(value) + .map(Some) + .map_err(serde::de::Error::custom) +} + impl Recognized { pub fn known(&self) -> Option<&T> { match self { @@ -24,6 +43,60 @@ impl Recognized { Self::Unrecognized(_) => None, } } + + pub fn map_known(self, f: impl FnOnce(T) -> T) -> Self { + match self { + Self::Known(value) => Self::Known(f(value)), + unrecognized => unrecognized, + } + } +} + +impl Recognized { + pub fn is_truthy(&self) -> bool { + match self { + Self::Known(flag) => *flag, + Self::Unrecognized(other) => python_truthy(other), + } + } +} + +pub fn python_truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64().is_none_or(|number| number != 0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(fields) => !fields.is_empty(), + } +} + +pub fn map_known_runs( + items: Vec>, + mut f: impl FnMut(Vec) -> Vec, +) -> Vec> { + let mut mapped = Vec::with_capacity(items.len()); + let mut run = Vec::new(); + for item in items { + match item { + Recognized::Known(value) => run.push(value), + unrecognized => { + if !run.is_empty() { + mapped.extend( + f(std::mem::take(&mut run)) + .into_iter() + .map(Recognized::Known), + ); + } + mapped.push(unrecognized); + } + } + } + if !run.is_empty() { + mapped.extend(f(run).into_iter().map(Recognized::Known)); + } + mapped } #[cfg(test)] @@ -32,6 +105,82 @@ mod tests { use rstest::rstest; use serde_json::json; + #[rstest] + #[case::known_true(Recognized::Known(true), true)] + #[case::known_false(Recognized::Known(false), false)] + #[case::string_true(Recognized::Unrecognized(json!("true")), true)] + #[case::string_false(Recognized::Unrecognized(json!("false")), true)] + #[case::empty_string(Recognized::Unrecognized(json!("")), false)] + #[case::zero(Recognized::Unrecognized(json!(0)), false)] + #[case::negative_zero(Recognized::Unrecognized(json!(-0.0)), false)] + #[case::one(Recognized::Unrecognized(json!(1)), true)] + #[case::null(Recognized::Unrecognized(Value::Null), false)] + #[case::empty_list(Recognized::Unrecognized(json!([])), false)] + #[case::list(Recognized::Unrecognized(json!([0])), true)] + #[case::empty_object(Recognized::Unrecognized(json!({})), false)] + #[case::object(Recognized::Unrecognized(json!({"a": 0})), true)] + fn flags_are_read_by_python_truthiness(#[case] flag: Recognized, #[case] expected: bool) { + assert_eq!(flag.is_truthy(), expected); + } + + #[test] + fn runs_of_known_items_are_mapped_around_unrecognized_ones() { + let items = vec![ + Recognized::Known(1), + Recognized::Known(2), + Recognized::Unrecognized(json!("x")), + Recognized::Known(3), + ]; + let mut runs = Vec::new(); + let mapped = map_known_runs(items, |run: Vec| { + runs.push(run.clone()); + run.into_iter() + .filter(|item| *item != 2) + .map(|item| item * 10) + .collect() + }); + assert_eq!(runs, vec![vec![1, 2], vec![3]]); + assert_eq!( + mapped, + vec![ + Recognized::Known(10), + Recognized::Unrecognized(json!("x")), + Recognized::Known(30), + ] + ); + assert_eq!( + map_known_runs(Vec::>::new(), |run| run), + vec![] + ); + } + + #[derive(serde::Deserialize)] + struct Sampling { + #[serde(default, deserialize_with = "exact_float")] + temperature: Option>, + } + + #[rstest] + #[case::float(json!({"temperature": 0.5}), Some(Recognized::Known(0.5)))] + #[case::small_int(json!({"temperature": 2}), Some(Recognized::Unrecognized(json!(2))))] + #[case::zero(json!({"temperature": 0}), Some(Recognized::Unrecognized(json!(0))))] + #[case::integral_float(json!({"temperature": 1.0}), Some(Recognized::Known(1.0)))] + #[case::largest_exact_int(json!({"temperature": 9_007_199_254_740_992_i64}), Some(Recognized::Unrecognized(json!(9_007_199_254_740_992_i64))))] + #[case::inexact_int(json!({"temperature": 1_152_921_504_606_846_977_i64}), Some(Recognized::Unrecognized(json!(1_152_921_504_606_846_977_i64))))] + #[case::inexact_negative_int(json!({"temperature": -9_223_372_036_854_775_807_i64}), Some(Recognized::Unrecognized(json!(-9_223_372_036_854_775_807_i64))))] + #[case::inexact_unsigned_int(json!({"temperature": 18_446_744_073_709_551_615_u64}), Some(Recognized::Unrecognized(json!(18_446_744_073_709_551_615_u64))))] + #[case::string(json!({"temperature": "0.5"}), Some(Recognized::Unrecognized(json!("0.5"))))] + #[case::null(json!({"temperature": null}), None)] + #[case::absent(json!({}), None)] + fn integers_keep_their_digits(#[case] value: Value, #[case] expected: Option>) { + assert_eq!( + serde_json::from_value::(value) + .unwrap() + .temperature, + expected + ); + } + #[rstest] #[case::known(json!(7), Recognized::Known(7))] #[case::wrong_type(json!("7"), Recognized::Unrecognized(json!("7")))] diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 79ee17e8b97..d59cd1be1d2 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -23,9 +23,11 @@ litellm-http.workspace = true litellm-secrets.workspace = true litellm-python-compat.workspace = true litellm-router-types.workspace = true +litellm-tracing.workspace = true base64.workspace = true bytes.workspace = true data-url = "0.3.2" +fancy-regex.workspace = true futures-util.workspace = true reqwest.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/llms/src/anthropic/beta_headers.rs b/litellm-rust/crates/llms/src/anthropic/beta_headers.rs index 7842b478dce..4a3575c9bbc 100644 --- a/litellm-rust/crates/llms/src/anthropic/beta_headers.rs +++ b/litellm-rust/crates/llms/src/anthropic/beta_headers.rs @@ -1,5 +1,5 @@ use litellm_http::request::{with_header, without_headers}; -use litellm_llms_types::providers::anthropic::{BETA_HEADER, BetaProvider, BetaSet}; +use litellm_llms_types::providers::anthropic::{AnthropicBeta, BETA_HEADER, BetaProvider, BetaSet}; use crate::{anthropic::common_utils::existing_betas, base_llm::auth::Headers}; @@ -33,6 +33,16 @@ impl BetaPolicy { } } +pub fn accepted_or_unlisted(betas: BetaSet, provider: BetaProvider) -> BetaSet { + betas + .into_iter() + .filter_map(|beta| match beta { + AnthropicBeta::Other(_) => Some(beta), + known => known.on(provider), + }) + .collect() +} + #[cfg(test)] mod tests { use crate::anthropic::beta_headers::BetaPolicy; diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs index 0854820358b..cadc5711474 100644 --- a/litellm-rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -1,4 +1,7 @@ +use std::sync::LazyLock; + use crate::base_llm::messages::context::MessagesModelCapabilities; +use fancy_regex::Regex; use litellm_auth::CredentialPlacement; use litellm_secret_value::SecretValue; @@ -22,7 +25,7 @@ use serde_json::Value; use crate::{ anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX, - base_llm::auth::{AuthScheme, Headers}, + base_llm::auth::{AuthScheme, Headers, ValidatedEnvironment}, }; pub const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; @@ -80,10 +83,21 @@ impl<'a> OauthToken<'a> { } pub fn into_auth(self) -> AuthScheme { - AuthScheme::Credential { - placement: CredentialPlacement::Bearer, - secret: SecretValue::new(self.0), - } + oauth_bearer(SecretValue::new(self.0)) + } +} + +fn oauth_bearer(secret: SecretValue) -> AuthScheme { + AuthScheme::Credential { + placement: CredentialPlacement::Bearer, + secret, + } +} + +pub fn minted_oauth_environment(headers: Headers, token: SecretValue) -> ValidatedEnvironment { + ValidatedEnvironment { + headers: with_oauth_companions(headers, &[], BrowserAccess::Omitted), + auth: oauth_bearer(token), } } @@ -92,11 +106,23 @@ pub fn get_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, ) -> Option { - resolve_non_empty(api_key, env_lookup, &[ANTHROPIC_API_KEY_ENV]) + nonempty_secret(api_key.map(str::to_owned)) + .or_else(|| nonempty_secret(env_lookup(ANTHROPIC_API_KEY_ENV))) } pub fn get_auth_token(env_lookup: &dyn Fn(&str) -> Option) -> Option { - resolve_non_empty(None, env_lookup, &[ANTHROPIC_AUTH_TOKEN_ENV]) + nonempty_secret(env_lookup(ANTHROPIC_AUTH_TOKEN_ENV)) +} + +/// Python's `normalize_nonempty_secret_str`: `str.strip()` also strips the ASCII separators. +fn nonempty_secret(value: Option) -> Option { + value + .map(|value| { + value + .trim_matches(|c: char| c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)) + .to_owned() + }) + .filter(|value| !value.is_empty()) } /// Python's `AnthropicModelInfo.get_auth_header`, naming the credential instead of building @@ -232,14 +258,24 @@ pub fn forwarded_oauth_bearer(headers: &[(String, String)]) -> Option Headers { - merge_beta_headers( +#[derive(Clone, Copy, PartialEq, Eq)] +enum BrowserAccess { + Granted, + Omitted, +} + +fn with_oauth_companions(headers: Headers, dropped: &[&str], browser: BrowserAccess) -> Headers { + let headers = merge_beta_headers( without_headers(headers, dropped), BetaSet::from_iter([AnthropicBeta::Oauth20250420]), - ) - .into_iter() - .chain([(DIRECT_BROWSER_ACCESS_HEADER.to_string(), "true".to_string())]) - .collect() + ); + match browser { + BrowserAccess::Granted => headers + .into_iter() + .chain([(DIRECT_BROWSER_ACCESS_HEADER.to_string(), "true".to_string())]) + .collect(), + BrowserAccess::Omitted => headers, + } } pub fn optionally_handle_anthropic_oauth(headers: Headers, api_key: Option<&str>) -> OauthHandling { @@ -247,13 +283,17 @@ pub fn optionally_handle_anthropic_oauth(headers: Headers, api_key: Option<&str> forwarded_oauth_bearer(&headers).map(|token| SecretValue::new(token.as_str())) { return OauthHandling::Bearer { - headers: with_oauth_companions(headers, &[API_KEY_HEADER, AUTHORIZATION]), + headers: with_oauth_companions( + headers, + &[API_KEY_HEADER, AUTHORIZATION], + BrowserAccess::Granted, + ), token, }; } if let Some(token) = api_key.and_then(OauthToken::parse) { return OauthHandling::Bearer { - headers: with_oauth_companions(headers, &[API_KEY_HEADER]), + headers: with_oauth_companions(headers, &[API_KEY_HEADER], BrowserAccess::Granted), token: SecretValue::new(token.as_str()), }; } @@ -278,12 +318,19 @@ pub fn has_advisor_tool(tools: Option<&[Recognized]>) -> bool { .any(|tool| matches!(tool, Recognized::Known(MessagesTool::Advisor { .. }))) } -pub fn requires_native_compaction_beta(compaction: Option<&Value>, messages: &[Message]) -> bool { +pub fn requires_native_compaction_beta( + compaction: Option<&Value>, + messages: &[Recognized], +) -> bool { compaction.is_some() - || messages.iter().flat_map(Message::blocks).any(|block| { - block.is_type(ContentBlockType::Compaction) - && block.signature.as_deref().is_some_and(|s| !s.is_empty()) - }) + || messages + .iter() + .filter_map(Recognized::known) + .flat_map(Message::blocks) + .any(|block| { + block.is_type(ContentBlockType::Compaction) + && block.signature.as_deref().is_some_and(|s| !s.is_empty()) + }) } fn is_blank(text: Option<&str>) -> bool { @@ -650,6 +697,113 @@ pub fn flatten_unencrypted_web_search_results(messages: Vec) -> Vec bool { + let lower = error_text.to_lowercase(); + if !lower.contains("thinking") { + return false; + } + (lower.contains("signature") && (lower.contains("invalid") || lower.contains("valid string"))) + || lower.contains("must contain thinking") +} + +pub fn is_anthropic_thinking_binding_mismatch_error(error_text: &str) -> bool { + error_text + .to_lowercase() + .contains("bound to a different conversation") +} + +pub fn anthropic_request_errors_on_thinking_binding_mismatch(request_body: &Value) -> bool { + request_body + .pointer("/thinking/block_binding/prefix_mismatch_behavior") + .and_then(Value::as_str) + == Some("error") +} + +fn is_thinking_value_block(block: &Value) -> bool { + matches!( + block.get("type").and_then(Value::as_str), + Some("thinking" | "redacted_thinking") + ) +} + +fn without_thinking_value_blocks(message: Value) -> Option { + let Value::Object(mut message) = message else { + return Some(message); + }; + if let Some(Value::Array(content)) = message.get_mut("content") { + content.retain(|block| !is_thinking_value_block(block)); + if content.is_empty() { + return None; + } + } + Some(Value::Object(message)) +} + +pub fn strip_thinking_blocks_from_anthropic_messages_request(request_body: Value) -> Value { + let Value::Object(mut body) = request_body else { + return request_body; + }; + if let Some(Value::Array(messages)) = body.get_mut("messages") { + *messages = std::mem::take(messages) + .into_iter() + .filter_map(without_thinking_value_blocks) + .collect(); + } + body.shift_remove("thinking"); + Value::Object(body) +} + +static CLAUDE_STRICT_TOOLS_REJECTION: LazyLock = LazyLock::new(|| { + Regex::new(concat!( + r"Too many strict tools \(\d+\)", + r"|Schemas contains too many (?:optional parameters|parameters with union types)", + r"|[Cc]ompiled grammar (?:is too large|size)", + r"|tools\.\d+\.custom(?:\.strict)?: ", + r"|Grammar compilation timed out", + )) + .expect("valid regex") +}); + +pub fn is_claude_strict_tools_rejection(status: u16, error_text: &str) -> bool { + status == 400 + && CLAUDE_STRICT_TOOLS_REJECTION + .is_match(error_text) + .unwrap_or(false) +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ToolsWithoutStrict { + pub tools: Vec, + pub dropped_count: usize, +} + +pub fn tools_without_strict(tools: Option<&Value>) -> Option { + let tools = tools?.as_array()?; + if !tools.iter().all(Value::is_object) { + return None; + } + let dropped_count = tools + .iter() + .filter(|tool| tool.get("strict") == Some(&Value::Bool(true))) + .count(); + if dropped_count == 0 { + return None; + } + Some(ToolsWithoutStrict { + tools: tools + .iter() + .cloned() + .map(|mut tool| { + if let Value::Object(fields) = &mut tool { + fields.shift_remove("strict"); + } + tool + }) + .collect(), + dropped_count, + }) +} + #[cfg(test)] mod tests { use crate::base_llm::messages::context::SupportedEffortTiers; @@ -668,7 +822,7 @@ mod tests { serde_json::from_value(value).unwrap() } - fn history(messages: Value) -> Vec { + fn history(messages: Value) -> Vec> { serde_json::from_value(messages).unwrap() } @@ -1636,6 +1790,11 @@ mod tests { &[("ANTHROPIC_AUTH_TOKEN", "env-token")], Some(("Authorization", "env-token")), )] + #[case::auth_token_stripped_of_whitespace( + None, + &[("ANTHROPIC_AUTH_TOKEN", " env-token \t\n\u{1f}")], + Some(("Authorization", "env-token")), + )] #[case::auth_token_when_the_env_key_is_blank( None, &[("ANTHROPIC_API_KEY", " \t"), ("ANTHROPIC_AUTH_TOKEN", "env-token")], @@ -1647,7 +1806,19 @@ mod tests { &[("ANTHROPIC_API_KEY", OAUTH_BEARER)], Some(("Authorization", OAUTH_TOKEN)), )] + #[case::env_key_stripped_of_whitespace( + None, + &[("ANTHROPIC_API_KEY", " sk-env \t\n\u{1f}")], + Some(("x-api-key", "sk-env")), + )] + #[case::api_key_param_stripped_of_separators(Some("\u{1c}sk-param "), &[], Some(("x-api-key", "sk-param")))] + #[case::separator_only_env_key_falls_to_the_auth_token( + None, + &[("ANTHROPIC_API_KEY", "\u{1c}\u{1f}"), ("ANTHROPIC_AUTH_TOKEN", "env-token")], + Some(("Authorization", "env-token")), + )] #[case::no_credentials(None, &[], None)] + #[case::separator_only_auth_token(None, &[("ANTHROPIC_AUTH_TOKEN", "\u{1c}\u{1f}")], None)] #[case::blank_everything(Some(""), &[("ANTHROPIC_API_KEY", " "), ("ANTHROPIC_AUTH_TOKEN", " \t")], None)] fn auth_header_prefers_the_key_then_the_auth_token( #[case] api_key: Option<&str>, @@ -1974,4 +2145,166 @@ mod tests { fn the_messages_url_follows_python_ensure_api_route(#[case] api_base: &str, #[case] url: &str) { assert_eq!(complete_anthropic_url(Some(api_base), &|_| None), url); } + + fn shared_cases(table: &str) -> Vec<(String, String, bool)> { + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/anthropic_messages_http_error_retry.json" + ))) + .unwrap(); + fixture[table] + .as_array() + .unwrap() + .iter() + .map(|case| { + ( + case["name"].as_str().unwrap().to_string(), + case["error_text"].as_str().unwrap().to_string(), + case["expected"].as_bool().unwrap(), + ) + }) + .collect() + } + + #[rstest] + #[case::invalid_thinking_block( + "invalid_thinking_block_error", + is_anthropic_invalid_thinking_block_error + )] + #[case::thinking_binding_mismatch( + "thinking_binding_mismatch_error", + is_anthropic_thinking_binding_mismatch_error + )] + fn error_texts_match_the_cases_shared_with_python( + #[case] table: &str, + #[case] recognizes: fn(&str) -> bool, + ) { + let cases = shared_cases(table); + assert!(!cases.is_empty()); + let mismatched: Vec<&String> = cases + .iter() + .filter(|(_, error_text, expected)| recognizes(error_text) != *expected) + .map(|(name, _, _)| name) + .collect(); + assert_eq!(mismatched, Vec::<&String>::new()); + } + + #[rstest] + #[case::pinned(json!({"thinking": {"block_binding": {"prefix_mismatch_behavior": "error"}}}), true)] + #[case::other_behavior(json!({"thinking": {"block_binding": {"prefix_mismatch_behavior": "drop"}}}), false)] + #[case::string_binding(json!({"thinking": {"block_binding": "strict"}}), false)] + #[case::no_thinking(json!({"messages": []}), false)] + #[case::not_an_object(json!(["thinking"]), false)] + fn a_pinned_binding_is_the_caller_asking_for_the_error( + #[case] body: Value, + #[case] expected: bool, + ) { + assert_eq!( + anthropic_request_errors_on_thinking_binding_mismatch(&body), + expected + ); + } + + #[rstest] + #[case::mixed_turn( + json!({"model": "m", "thinking": {"type": "enabled"}, "messages": [ + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "t", "signature": "s"}, + {"type": "text", "text": "a"}, + {"type": "redacted_thinking", "data": "d"} + ]} + ]}), + json!({"model": "m", "messages": [{"role": "assistant", "content": [{"type": "text", "text": "a"}]}]}) + )] + #[case::thinking_only_and_empty_turns_are_dropped( + json!({"messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "redacted_thinking", "data": "d"}]}, + {"role": "assistant", "content": []}, + {"role": "user", "content": "again"} + ]}), + json!({"messages": [{"role": "user", "content": "hi"}, {"role": "user", "content": "again"}]}) + )] + #[case::non_object_entries_are_kept( + json!({"messages": ["raw", {"role": "user", "content": ["raw", {"type": "thinking"}]}]}), + json!({"messages": ["raw", {"role": "user", "content": ["raw"]}]}) + )] + #[case::messages_not_a_list( + json!({"messages": "hi", "thinking": {"type": "adaptive"}}), + json!({"messages": "hi"}) + )] + fn retrying_strips_thinking_from_the_replay_and_the_request( + #[case] body: Value, + #[case] expected: Value, + ) { + assert_eq!( + strip_thinking_blocks_from_anthropic_messages_request(body), + expected + ); + } + + #[test] + fn stripping_keeps_the_order_of_the_remaining_fields() { + let stripped = strip_thinking_blocks_from_anthropic_messages_request( + json!({"model": "m", "thinking": {}, "messages": [], "max_tokens": 1}), + ); + let keys: Vec<&str> = stripped + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_eq!(keys, ["model", "messages", "max_tokens"]); + } + + fn shared_table(table: &str) -> Vec { + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/anthropic_messages_http_error_retry.json" + ))) + .unwrap(); + fixture[table].as_array().unwrap().clone() + } + + #[test] + fn strict_tools_rejections_match_the_cases_shared_with_python() { + let cases = shared_table("claude_strict_tools_rejection"); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + is_claude_strict_tools_rejection(status, case["error_text"].as_str().unwrap()) + != case["expected"].as_bool().unwrap() + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } + + #[test] + fn tools_without_strict_matches_the_cases_shared_with_python() { + let cases = shared_table("tools_without_strict"); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let expected = case["expected"] + .as_object() + .map(|expected| ToolsWithoutStrict { + tools: expected["tools"].as_array().unwrap().clone(), + dropped_count: usize::try_from(expected["dropped_count"].as_u64().unwrap()) + .unwrap(), + }); + tools_without_strict(Some(&case["tools"])) != expected + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } + + #[test] + fn absent_tools_have_no_strict_to_drop() { + assert_eq!(tools_without_strict(None), None); + } } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/handler.rs b/litellm-rust/crates/llms/src/anthropic/messages/handler.rs index 54f924ddec6..46a16e7d546 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/handler.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/handler.rs @@ -3,7 +3,7 @@ use litellm_llms_types::{ AdaptiveThinking, EnabledThinking, Message, MessagesOptionalParams, MessagesRequest, ThinkingConfig, ThinkingDisplay, }, - recognized::Recognized, + recognized::{Recognized, map_known_runs}, }; use serde_json::{Value, json}; @@ -13,14 +13,19 @@ use crate::{ flatten_unencrypted_web_search_results, sanitize_tool_use_ids, strip_empty_content_blocks, strip_provider_specific_fields, }, + base_llm::messages::context::MessagesShapeOptions, }; pub fn shape_anthropic_messages_request( request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { Ok(MessagesRequest { - messages: sanitize_anthropic_messages(request.messages), + messages: if options.presanitized { + map_known_runs(request.messages, strip_provider_specific_fields) + } else { + map_known_runs(request.messages, sanitize_anthropic_messages) + }, params: MessagesOptionalParams { metadata: request .params @@ -28,7 +33,10 @@ pub fn shape_anthropic_messages_request( .as_ref() .map(validate_anthropic_api_metadata) .transpose()?, - thinking: with_reasoning_auto_summary(request.params.thinking, reasoning_auto_summary), + thinking: with_reasoning_auto_summary( + request.params.thinking, + options.reasoning_auto_summary, + ), ..request.params }, ..request @@ -299,7 +307,10 @@ mod tests { "thinking": {"type": "enabled", "budget_tokens": 1024}, "safeguards": [{"type": "dangerous_tool_use"}] })), - true, + MessagesShapeOptions { + reasoning_auto_summary: true, + presanitized: false, + }, ) .unwrap(); assert_eq!( @@ -315,4 +326,27 @@ mod tests { }) ); } + + #[test] + fn presanitized_shaping_keeps_blocks_and_strips_only_provider_fields() { + let messages = json!([{"role": "user", "content": [ + {"type": "text", "text": " ", "cache_control": {"type": "ephemeral"}}, + {"type": "tool_result", "tool_use_id": "functions.Bash:0", "content": "ok", "provider_specific_fields": {"x": 1}} + ]}]); + let sanitized = shape_anthropic_messages_request( + request(json!({"model": "m", "messages": messages})), + MessagesShapeOptions { + reasoning_auto_summary: false, + presanitized: true, + }, + ) + .unwrap(); + assert_eq!( + serde_json::to_value(sanitized).unwrap(), + json!({"model": "m", "messages": [{"role": "user", "content": [ + {"type": "text", "text": " ", "cache_control": {"type": "ephemeral"}}, + {"type": "tool_result", "tool_use_id": "functions.Bash:0", "content": "ok"} + ]}]}) + ); + } } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/mod.rs b/litellm-rust/crates/llms/src/anthropic/messages/mod.rs index 597a1d819d9..4fa24a3f0f5 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/mod.rs @@ -1,3 +1,4 @@ pub mod handler; +mod scalar; pub mod thinking; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/anthropic/messages/scalar.rs b/litellm-rust/crates/llms/src/anthropic/messages/scalar.rs new file mode 100644 index 00000000000..db9aa01d177 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/messages/scalar.rs @@ -0,0 +1,113 @@ +use litellm_host::raw_json; +use litellm_llms_types::recognized::Recognized; +use litellm_python_compat::{ + Value as PythonValue, + json::{from_json, loads}, + repr::to_str, +}; +use serde_json::Value; + +pub(crate) fn python_value(value: &Value) -> PythonValue { + match value { + Value::String(text) => raw_json::decode(text) + .and_then(|raw| loads(raw).ok()) + .unwrap_or_else(|| PythonValue::Str(text.clone())), + Value::Array(items) => PythonValue::List(items.iter().map(python_value).collect()), + Value::Object(entries) => PythonValue::Dict( + entries + .iter() + .map(|(key, item)| (PythonValue::Str(key.clone()), python_value(item))) + .collect(), + ), + Value::Null | Value::Bool(_) | Value::Number(_) => from_json(value.clone()), + } +} + +pub(crate) fn python_number(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(f64::from(u8::from(*flag))), + Value::Number(number) => number.as_f64(), + Value::String(text) => match raw_json::decode(text)? { + "NaN" => Some(f64::NAN), + "Infinity" => Some(f64::INFINITY), + "-Infinity" => Some(f64::NEG_INFINITY), + raw => raw.parse().ok(), + }, + _ => None, + } +} + +pub(crate) fn is_one(value: &Recognized) -> bool { + match value { + Recognized::Known(number) => *number == 1.0, + Recognized::Unrecognized(other) => python_number(other) == Some(1.0), + } +} + +pub(crate) fn python_str(value: &Recognized) -> String { + match value { + Recognized::Known(known) => serde_json::to_value(known) + .map(|known| to_str(&from_json(known))) + .unwrap_or_default(), + Recognized::Unrecognized(other) => unrecognized_str(other), + } +} + +fn unrecognized_str(value: &Value) -> String { + match value { + Value::Null => "None".into(), + Value::Bool(true) => "True".into(), + Value::Bool(false) => "False".into(), + Value::String(text) => match raw_json::decode(text) { + Some("NaN") => "nan".into(), + Some("Infinity") => "inf".into(), + Some("-Infinity") => "-inf".into(), + Some(raw) => raw.trim_matches('"').into(), + None => text.clone(), + }, + other => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::int(json!(3), Some(3.0))] + #[case::float(json!(1.5), Some(1.5))] + #[case::bool(json!(true), Some(1.0))] + #[case::wide(json!(raw_json::encode("18446744073709551616")), Some(18_446_744_073_709_551_616.0))] + #[case::negative_infinity(json!(raw_json::encode("-Infinity")), Some(f64::NEG_INFINITY))] + #[case::string(json!("1"), None)] + #[case::marked_string(json!(raw_json::encode("\"1\"")), None)] + #[case::null(Value::Null, None)] + fn numbers_compare_as_python_compares_them( + #[case] value: Value, + #[case] expected: Option, + ) { + assert_eq!(python_number(&value), expected); + } + + #[test] + fn nan_is_a_number_unequal_to_everything() { + let nan = json!(raw_json::encode("NaN")); + assert!(python_number(&nan).is_some_and(f64::is_nan)); + assert!(!is_one(&Recognized::Unrecognized(nan))); + } + + #[rstest] + #[case::known(Recognized::Known(0.5), "0.5")] + #[case::known_wide(Recognized::Known(1e16), "1e+16")] + #[case::int(Recognized::Unrecognized(json!(0)), "0")] + #[case::bool(Recognized::Unrecognized(json!(false)), "False")] + #[case::string(Recognized::Unrecognized(json!("0.5")), "0.5")] + #[case::nan(Recognized::Unrecognized(json!(raw_json::encode("NaN"))), "nan")] + #[case::wide(Recognized::Unrecognized(json!(raw_json::encode("-1180591620717411303424"))), "-1180591620717411303424")] + fn values_print_as_python_prints_them(#[case] value: Recognized, #[case] expected: &str) { + assert_eq!(python_str(&value), expected); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs index 0ddbc915154..c6f7c1679ef 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs @@ -1,17 +1,20 @@ +use litellm_host::raw_json; use litellm_llms_types::{ formats::{ chat_completions::ReasoningEffort, messages::{ - AdaptiveThinking, DisabledThinking, EffortLevel, MessagesOptionalParams, - MessagesRequest, OutputConfig, ThinkingConfig, ThinkingDisplay, + AdaptiveThinking, DisabledThinking, EffortLevel, EnabledThinking, + MessagesOptionalParams, MessagesRequest, OutputConfig, ThinkingConfig, ThinkingDisplay, }, }, recognized::Recognized, }; -use litellm_python_compat::{json::from_json, repr::repr, truthy::truthy}; +use litellm_python_compat::{ + Value as PythonValue, json::from_json, repr::repr, reprlib::reprlib, truthy::truthy, +}; use serde_json::Value; -use strum::VariantArray; +use super::scalar::{is_one, python_number, python_str, python_value}; use crate::base_llm::messages::context::{ MessagesModelCapabilities, ThinkingBudgets, ThinkingContext, }; @@ -56,13 +59,61 @@ fn unmapped_effort(effort: &Value) -> Error { Error::InvalidRequest(crate::ErrorDetail::InvalidChoice { field: "reasoning effort", actual: repr(&from_json(effort.clone())), - choices: ReasoningEffort::VARIANTS - .iter() - .map(|effort| <&'static str>::from(*effort)) - .collect(), + choices: vec!["minimal", "low", "medium", "high", "xhigh", "max", "none"], }) } +fn invalid_adaptive_effort(effort: &Value) -> Error { + Error::InvalidRequest(crate::ErrorDetail::InvalidReasoningEffort { + actual: repr(&from_json(effort.clone())), + }) +} + +const NON_STRING_EFFORT_REASON: &str = "it is not a string. Send reasoning_effort as a level name such as 'high' to set a reasoning level."; +const NON_NUMERIC_BUDGET_REASON: &str = + "it is not a number. Send budget_tokens as an integer to set a reasoning level."; + +fn warn_dropped_reasoning( + param: &str, + value: &PythonValue, + model: &str, + reason: &str, + drop_params_gated: bool, +) { + let gate = if drop_params_gated { + " (drop_params=True)" + } else { + "" + }; + litellm_tracing::warn!( + "Dropping {param}={} for model={}{gate} because {reason}", + reprlib(value), + repr(&PythonValue::Str(model.to_owned())) + ); +} + +fn warn_non_string_reasoning_effort(effort: &Value, model: &str) { + let value = python_value(effort); + let requested = match &value { + PythonValue::Str(_) => return, + PythonValue::Dict(entries) => entries + .iter() + .find(|(key, _)| *key == PythonValue::Str("effort".to_owned())) + .map(|(_, level)| level), + other => Some(other), + }; + if matches!(requested, None | Some(PythonValue::None)) { + return; + } + warn_dropped_reasoning( + "reasoning_effort", + &value, + model, + NON_STRING_EFFORT_REASON, + false, + ); +} + fn unsupported_effort(level: EffortLevel, model: &str) -> Error { Error::InvalidRequest(crate::ErrorDetail::UnsupportedValue { field: "effort", @@ -82,11 +133,56 @@ fn output_effort(effort: ReasoningEffort) -> Option { } } -fn fit_budget_to_max_tokens(budget_tokens: u64, max_tokens: Option) -> Option { - let Some(max_tokens) = max_tokens else { - return Some(budget_tokens); +/// `cap_thinking_budget_to_max_tokens`, whose first comparison (`max_tokens <= 1024`) raises +/// `TypeError` for a `max_tokens` Python cannot order against an int, such as a string. +fn fit_budget_to_max_tokens( + budget_tokens: u64, + max_tokens: Option<&Recognized>, +) -> Result>, Error> { + let max_tokens = match max_tokens { + None => return Ok(Some(Recognized::Known(budget_tokens))), + Some(Recognized::Known(max_tokens)) => *max_tokens, + Some(Recognized::Unrecognized(other)) => { + return fit_budget_to_python_number(budget_tokens, other); + } }; - (max_tokens > ANTHROPIC_MIN_THINKING_BUDGET_TOKENS).then(|| budget_tokens.min(max_tokens - 1)) + Ok((max_tokens > ANTHROPIC_MIN_THINKING_BUDGET_TOKENS) + .then(|| Recognized::Known(budget_tokens.min(max_tokens - 1)))) +} + +fn enabled_thinking(budget_tokens: Recognized) -> ThinkingConfig { + ThinkingConfig::Enabled(EnabledThinking { + budget_tokens: Some(budget_tokens), + ..EnabledThinking::default() + }) +} + +fn fit_budget_to_python_number( + budget_tokens: u64, + max_tokens: &Value, +) -> Result>, Error> { + let Some(max_tokens) = python_number(max_tokens) else { + if max_tokens.is_null() { + return Ok(Some(Recognized::Known(budget_tokens))); + } + return Err(Error::InvalidRequest( + crate::ErrorDetail::UnorderableTypes { + op: "<=", + left: python_type_name(max_tokens), + right: "int", + }, + )); + }; + if max_tokens <= ANTHROPIC_MIN_THINKING_BUDGET_TOKENS as f64 { + return Ok(None); + } + if (budget_tokens as f64) < max_tokens { + return Ok(Some(Recognized::Known(budget_tokens))); + } + Ok(Some(Recognized::Unrecognized( + serde_json::Number::from_f64(max_tokens - 1.0) + .map_or_else(|| Value::String(raw_json::encode("NaN")), Value::Number), + ))) } fn known_thinking(request: &MessagesRequest) -> Option<&ThinkingConfig> { @@ -161,10 +257,17 @@ fn translate_reasoning_effort( }; let effort = match reasoning_effort { Recognized::Known(effort) => effort, - Recognized::Unrecognized(value @ Value::String(_)) => { - return Err(unmapped_effort(&value)); + Recognized::Unrecognized(value) if matches!(python_value(&value), PythonValue::Str(_)) => { + return Err(if context.capabilities.supports_adaptive_thinking { + invalid_adaptive_effort(&value) + } else { + unmapped_effort(&value) + }); + } + Recognized::Unrecognized(value) => { + warn_non_string_reasoning_effort(&value, &request.model); + return Ok(request); } - Recognized::Unrecognized(_) => return Ok(request), }; let (Some(level), Some(budget)) = ( output_effort(effort), @@ -203,10 +306,20 @@ fn translate_reasoning_effort( ..request }); } - let Some(budget) = fit_budget_to_max_tokens(budget, request.params.max_tokens) else { + let Some(budget) = fit_budget_to_max_tokens(budget, request.params.max_tokens.as_ref())? else { + litellm_tracing::warn!( + "Dropping `thinking` mapped from reasoning_effort={} for model={}: max_tokens={} is too small to fit the minimum thinking budget.", + <&'static str>::from(effort), + request.model, + request + .params + .max_tokens + .as_ref() + .map_or_else(|| "None".to_owned(), python_str) + ); return Ok(request); }; - let enabled = ThinkingConfig::enabled(budget); + let enabled = enabled_thinking(budget); Ok(MessagesRequest { params: MessagesOptionalParams { thinking: Some( @@ -340,6 +453,15 @@ fn translate_legacy_thinking_for_adaptive_model( crate::ErrorDetail::NonNumericThinkingBudget { actual }, )); } + if let (None, Some(Recognized::Unrecognized(dropped))) = (budget, &enabled.budget_tokens) { + warn_dropped_reasoning( + "thinking.budget_tokens", + &python_value(dropped), + &request.model, + NON_NUMERIC_BUDGET_REASON, + true, + ); + } let thinking = ThinkingConfig::Adaptive(AdaptiveThinking { display: kept_display(enabled.display.as_ref(), context), extra: carried_thinking_keys(&enabled.extra, context), @@ -396,11 +518,20 @@ fn translate_adaptive_effort_for_non_adaptive_model( } else { None }; + let thinking = match budget { + Some(budget) => fit_budget_to_max_tokens(budget, request.params.max_tokens.as_ref())? + .map(|budget| Recognized::Known(enabled_thinking(budget))), + None => None, + }; + if thinking.is_none() { + litellm_tracing::warn!( + "Dropping adaptive `thinking`/`output_config.effort` for model={}: the model does not support extended thinking, or max_tokens is too small to fit the minimum thinking budget.", + request.model + ); + } Ok(MessagesRequest { params: MessagesOptionalParams { - thinking: budget - .and_then(|budget| fit_budget_to_max_tokens(budget, request.params.max_tokens)) - .map(|budget| Recognized::Known(ThinkingConfig::enabled(budget))), + thinking, output_config: without_effort(request.params.output_config), ..request.params }, @@ -418,7 +549,8 @@ fn drop_incompatible_temperature_for_thinking( let pinned = request .params .temperature - .is_some_and(|temperature| temperature != 1.0); + .as_ref() + .is_some_and(|temperature| !is_one(temperature)); let thinking_enabled = matches!(known_thinking(&request), Some(ThinkingConfig::Enabled(_))); let effort_enabled = known_effort(&request).is_some(); if !pinned || !(thinking_enabled || effort_enabled) { @@ -451,7 +583,7 @@ mod tests { use super::*; use crate::base_llm::messages::context::SupportedEffortTiers; - const EFFORT_CHOICES: &str = "'none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'"; + const EFFORT_CHOICES: &str = "'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'none'"; fn request(fields: Value) -> MessagesRequest { let mut body = serde_json::json!({"model": "claude", "messages": [{"role": "user", "content": "Hello"}]}); @@ -819,17 +951,17 @@ mod tests { #[case::invalid_on_adaptive_model( opus_4_7(), serde_json::json!({"max_tokens": 1024, "reasoning_effort": "invalid"}), - format!("Unmapped reasoning effort: 'invalid'. Must be one of: {EFFORT_CHOICES}.") + format!("Invalid reasoning_effort: 'invalid'. Must be one of: {EFFORT_CHOICES}") )] #[case::disabled_on_adaptive_model( opus_4_7(), serde_json::json!({"max_tokens": 1024, "reasoning_effort": "disabled"}), - format!("Unmapped reasoning effort: 'disabled'. Must be one of: {EFFORT_CHOICES}.") + format!("Invalid reasoning_effort: 'disabled'. Must be one of: {EFFORT_CHOICES}") )] #[case::empty_on_adaptive_model( opus_4_7(), serde_json::json!({"max_tokens": 1024, "reasoning_effort": ""}), - format!("Unmapped reasoning effort: ''. Must be one of: {EFFORT_CHOICES}.") + format!("Invalid reasoning_effort: ''. Must be one of: {EFFORT_CHOICES}") )] #[case::xhigh_without_xhigh_tier_on_4_6( sonnet_4_6(), @@ -1434,6 +1566,34 @@ mod tests { ); } + #[rstest] + #[case::reasoning_effort(json!({"max_tokens": "8192", "reasoning_effort": "high"}), "str")] + #[case::caller_thinking( + json!({"max_tokens": "8192", "reasoning_effort": "low", "thinking": {"type": "enabled", "budget_tokens": 2048}}), + "str" + )] + #[case::adaptive_interface( + json!({"max_tokens": "8192", "thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}), + "str" + )] + #[case::list(json!({"max_tokens": [8192], "reasoning_effort": "high"}), "list")] + fn an_unorderable_max_tokens_raises_python_type_error_while_capping_the_budget( + haiku_4_5: MessagesModelCapabilities, + #[case] fields: Value, + #[case] left: &'static str, + ) { + assert_eq!( + translate(haiku_4_5, fields), + Err(Error::InvalidRequest( + crate::ErrorDetail::UnorderableTypes { + op: "<=", + left, + right: "int", + } + )) + ); + } + #[rstest] #[case::summarized("summarized", false, Some("summarized"))] #[case::updates_kept("updates", true, Some("updates"))] diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index 18789e5b2be..bbb9b4e4c17 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -1,29 +1,45 @@ use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::{RetriedRequest, RetryReason}; +use litellm_http::request::{with_header, without_headers}; use litellm_llms_types::{ formats::messages::{ - ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message, + ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message, MessageContent, MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay, }, - providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS}, - recognized::Recognized, + providers::anthropic::{AnthropicBeta, BETA_HEADER, BetaProvider, BetaSet, DEFAULT_HEADERS}, + recognized::{Recognized, map_known_runs}, }; use litellm_router_types::LitellmParams; use serde_json::{Map, Value, json}; -use super::{handler::shape_anthropic_messages_request, thinking::translate_thinking}; -use crate::base_llm::messages::context::MessagesTransformContext; +use super::{ + handler::shape_anthropic_messages_request, + scalar::{is_one, python_str, python_value}, + thinking::translate_thinking, +}; +use crate::anthropic::beta_headers::accepted_or_unlisted; +use crate::base_llm::messages::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + tools::ToolCustomField, +}; use crate::{ Error, anthropic::common_utils::{ ANTHROPIC_API_BASE_ENV, ANTHROPIC_API_KEY_ENV, ANTHROPIC_AUTH_TOKEN_ENV, - ANTHROPIC_BASE_URL_ENV, OauthHandling, complete_anthropic_url, get_auth_header, - has_advisor_tool, has_anthropic_credential, is_tool_search_used, merge_beta_headers, - optionally_handle_anthropic_oauth, requires_native_compaction_beta, strip_advisor_blocks, - strip_encrypted_reasoning_blocks, + ANTHROPIC_BASE_URL_ENV, OauthHandling, + anthropic_request_errors_on_thinking_binding_mismatch, complete_anthropic_url, + existing_betas, get_auth_header, has_advisor_tool, has_anthropic_credential, + is_anthropic_invalid_thinking_block_error, is_anthropic_thinking_binding_mismatch_error, + is_claude_strict_tools_rejection, is_tool_search_used, merge_beta_headers, + minted_oauth_environment, optionally_handle_anthropic_oauth, + requires_native_compaction_beta, strip_advisor_blocks, strip_encrypted_reasoning_blocks, + strip_thinking_blocks_from_anthropic_messages_request, tools_without_strict, }, base_llm::{ auth::AuthScheme, - messages::transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment}, + messages::transformation::{ + BaseMessagesConfig, FederatedEnvironment, Headers, ValidatedEnvironment, + }, }, }; @@ -35,9 +51,9 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn get_complete_url( @@ -59,6 +75,12 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { transform_messages_request(request, &context.keeping_display_updates()) } + fn tool_custom_field(&self, model: &str) -> ToolCustomField { + ToolCustomField::Rewrite { + hoist_defer_loading: is_claude_model(model), + } + } + fn secret_names(&self) -> &'static [&'static str] { &[ ANTHROPIC_API_KEY_ENV, @@ -79,31 +101,22 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { _litellm_params: &LitellmParams, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { - let headers = match optionally_handle_anthropic_oauth(headers, api_key) { - OauthHandling::Bearer { headers, token } => { - return Ok(ValidatedEnvironment { - headers, - auth: AuthScheme::Credential { - placement: CredentialPlacement::Bearer, - secret: token, - }, - }); - } - OauthHandling::Untouched(headers) => headers, - }; - if has_anthropic_credential(&headers) { - return Ok(ValidatedEnvironment { - headers, - auth: AuthScheme::Forwarded, - }); + match anthropic_environment(headers, api_key, env_lookup) { + FederatedEnvironment::Validated(validated) => Ok(validated), + FederatedEnvironment::Unauthenticated { missing, .. } => Err(missing), } - let auth = get_auth_header(api_key, env_lookup).ok_or(Error::Auth( - litellm_auth::Error::MissingApiKey { - provider: "Anthropic", - environment_variable: ANTHROPIC_API_KEY_ENV, - }, - ))?; - Ok(ValidatedEnvironment { headers, auth }) + } + + fn validate_federated_environment( + &self, + headers: Headers, + api_key: Option<&str>, + _model: &str, + _optional_params: &MessagesOptionalParams, + _litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + Ok(anthropic_environment(headers, api_key, env_lookup)) } fn default_headers(&self) -> &'static [(&'static str, &'static str)] { @@ -116,6 +129,157 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { provider_feature_betas(request, BetaProvider::Anthropic), ) } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + + fn headers_for_url(&self, headers: Headers, url: &str) -> Headers { + if !is_foundry_url(url) { + return headers; + } + let betas = existing_betas(&headers); + if betas.is_empty() { + return headers; + } + let kept = accepted_or_unlisted(betas, BetaProvider::AzureAi); + if kept.is_empty() { + return without_headers(headers, &[BETA_HEADER]); + } + with_header(headers, BETA_HEADER, kept.to_string()) + } +} + +const FOUNDRY_HOST_SUFFIXES: [&str; 3] = [ + "services.ai.azure.com", + "openai.azure.com", + "cognitiveservices.azure.com", +]; + +fn is_foundry_url(url: &str) -> bool { + let absolute = if url.contains("://") { + url.to_string() + } else { + format!("https://{url}") + }; + url::Url::parse(&absolute) + .ok() + .and_then(|parsed| parsed.host_str().map(str::to_ascii_lowercase)) + .is_some_and(|host| { + FOUNDRY_HOST_SUFFIXES + .iter() + .any(|suffix| host == *suffix || host.ends_with(&format!(".{suffix}"))) + }) +} + +pub(crate) fn is_claude_model(model: &str) -> bool { + model.to_lowercase().contains("claude") +} + +pub fn should_retry_anthropic_messages_on_http_error( + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, +) -> bool { + if status != 400 { + return false; + } + let drops_rejected_strict_tools = context.drops_unsupported_params + && is_claude_strict_tools_rejection(status, error_text) + && tools_without_strict(request_body.get("tools")).is_some(); + if drops_rejected_strict_tools { + return true; + } + let caller_asked_for_this_error = + anthropic_request_errors_on_thinking_binding_mismatch(request_body) + && is_anthropic_thinking_binding_mismatch_error(error_text); + !caller_asked_for_this_error && is_anthropic_invalid_thinking_block_error(error_text) +} + +pub fn transform_anthropic_messages_request_on_http_error( + status: u16, + error_text: &str, + request_body: Value, +) -> RetriedRequest { + let mut reasons = Vec::new(); + let mut body = if status == 400 && is_anthropic_invalid_thinking_block_error(error_text) { + reasons.push(RetryReason::ThinkingSignature); + strip_thinking_blocks_from_anthropic_messages_request(request_body) + } else { + request_body + }; + let without_strict = if is_claude_strict_tools_rejection(status, error_text) { + tools_without_strict(body.get("tools")) + } else { + None + }; + if let (Some(without_strict), Value::Object(fields)) = (without_strict, &mut body) { + reasons.push(RetryReason::StrictTools { + dropped: without_strict.dropped_count, + }); + fields.insert("tools".to_owned(), Value::Array(without_strict.tools)); + } + RetriedRequest { reasons, body } +} + +pub fn retry_anthropic_messages_after_http_error( + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, +) -> Option { + should_retry_anthropic_messages_on_http_error(status, error_text, request_body, context).then( + || { + transform_anthropic_messages_request_on_http_error( + status, + error_text, + request_body.clone(), + ) + }, + ) +} + +fn anthropic_environment( + headers: Headers, + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> FederatedEnvironment { + let headers = match optionally_handle_anthropic_oauth(headers, api_key) { + OauthHandling::Bearer { headers, token } => { + return FederatedEnvironment::Validated(ValidatedEnvironment { + headers, + auth: AuthScheme::Credential { + placement: CredentialPlacement::Bearer, + secret: token, + }, + }); + } + OauthHandling::Untouched(headers) => headers, + }; + if has_anthropic_credential(&headers) { + return FederatedEnvironment::Validated(ValidatedEnvironment { + headers, + auth: AuthScheme::Forwarded, + }); + } + match get_auth_header(api_key, env_lookup) { + Some(auth) => FederatedEnvironment::Validated(ValidatedEnvironment { headers, auth }), + None => FederatedEnvironment::Unauthenticated { + headers, + missing: Error::Auth(litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: ANTHROPIC_API_KEY_ENV, + }), + federate: minted_oauth_environment, + }, + } } pub(crate) fn transform_messages_request( @@ -125,6 +289,7 @@ pub(crate) fn transform_messages_request( if request.params.max_tokens.is_none() { return Err(Error::MissingField("max_tokens")); } + let request = without_unstructured_output_format(request, context); let request = drop_unsupported_params(request, context)?; let request = translate_thinking(request, &context.thinking)?; let context_management = request @@ -132,13 +297,20 @@ pub(crate) fn transform_messages_request( .context_management .clone() .map(map_openai_context_management_to_anthropic); - let messages = if has_advisor_tool(request.params.tools.as_deref()) { + let messages = if has_advisor_tool(request.params.tools.as_deref()) + || continues_message_thread(&request.params) + { request.messages } else { - strip_advisor_blocks(request.messages) + map_known_runs(request.messages, strip_advisor_blocks) + }; + let messages = if context.drops_unsupported_params { + drop_non_string_tool_result_texts(messages) + } else { + messages }; Ok(MessagesRequest { - messages: strip_encrypted_reasoning_blocks(messages), + messages: map_known_runs(messages, strip_encrypted_reasoning_blocks), params: MessagesOptionalParams { context_management, extra: without_null_safeguards(request.params.extra), @@ -148,6 +320,84 @@ pub(crate) fn transform_messages_request( }) } +fn without_unstructured_output_format( + request: MessagesRequest, + context: &MessagesTransformContext, +) -> MessagesRequest { + if !context.drops_unsupported_params + || request + .params + .output_format + .as_ref() + .is_none_or(Value::is_object) + { + return request; + } + MessagesRequest { + params: MessagesOptionalParams { + output_format: None, + ..request.params + }, + ..request + } +} + +fn is_python_str(value: &Value) -> bool { + match value { + Value::String(text) => { + litellm_host::raw_json::decode(text).is_none_or(|raw| raw.starts_with('"')) + } + _ => false, + } +} + +fn is_non_string_text(part: &Value) -> bool { + part.get("type").and_then(Value::as_str) == Some("text") + && part.get("text").is_some_and(|text| !is_python_str(text)) +} + +fn without_non_string_texts(content: &mut Value) { + if let Value::Array(parts) = content { + parts.retain(|part| !is_non_string_text(part)); + } +} + +fn drop_non_string_tool_result_texts( + messages: Vec>, +) -> Vec> { + messages + .into_iter() + .map(|message| match message { + Recognized::Known(mut message) => { + if message.role == "user" + && let MessageContent::Blocks(blocks) = &mut message.content + { + blocks + .iter_mut() + .filter(|block| block.is_type(ContentBlockType::ToolResult)) + .filter_map(|block| block.content.as_mut()) + .for_each(without_non_string_texts); + } + Recognized::Known(message) + } + Recognized::Unrecognized(mut message) => { + if message.get("role").and_then(Value::as_str) == Some("user") + && let Some(Value::Array(blocks)) = message.get_mut("content") + { + blocks + .iter_mut() + .filter(|block| { + block.get("type").and_then(Value::as_str) == Some("tool_result") + }) + .filter_map(|block| block.get_mut("content")) + .for_each(without_non_string_texts); + } + Recognized::Unrecognized(message) + } + }) + .collect() +} + fn without_null_safeguards(extra: Map) -> Map { extra .into_iter() @@ -193,22 +443,45 @@ fn mapped_host_betas(params: &MessagesOptionalParams) -> BetaSet { .collect() } +fn continues_message_thread(params: &MessagesOptionalParams) -> bool { + let Some(Value::Object(thread)) = params.extra.get("thread") else { + return false; + }; + thread.get("type").and_then(Value::as_str) == Some("continue") + && thread + .get("previous_message_id") + .is_none_or(Value::is_string) +} + +fn is_threaded(params: &MessagesOptionalParams) -> bool { + params + .extra + .get("thread") + .is_some_and(|thread| !thread.is_null()) +} + fn feature_betas(request: &MessagesRequest) -> BetaSet { let params = &request.params; let tools = params.tools.as_deref(); + let scanned_messages: &[Recognized] = if continues_message_thread(params) { + &[] + } else { + &request.messages + }; [ - requires_native_compaction_beta(params.compaction.as_ref(), &request.messages) + requires_native_compaction_beta(params.compaction.as_ref(), scanned_messages) .then_some(AnthropicBeta::Compact20260904), uses_structured_output(params).then_some(AnthropicBeta::StructuredOutputs20251113), (params.speed == Some(Recognized::Known(Speed::Fast))) .then_some(AnthropicBeta::FastMode20260201), - messages_carry_output_config(&request.messages) + messages_carry_output_config(scanned_messages) .then_some(AnthropicBeta::PerTurnControl20260701), + is_threaded(params).then_some(AnthropicBeta::MessageThreads20260812), uses_thinking_display_updates(params) .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818), uses_thinking_block_binding(params) .then_some(AnthropicBeta::ThinkingBindingControls20260801), - messages_carry_tool_changes(&request.messages) + messages_carry_tool_changes(scanned_messages) .then_some(AnthropicBeta::MidConversationToolChanges20260701), has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301), is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120), @@ -254,9 +527,10 @@ fn carries_safeguards(params: &MessagesOptionalParams) -> bool { .is_some_and(|value| !value.is_null()) } -fn messages_carry_output_config(messages: &[Message]) -> bool { +fn messages_carry_output_config(messages: &[Recognized]) -> bool { messages .iter() + .filter_map(Recognized::known) .any(|message| message.extra.contains_key("output_config")) } @@ -301,9 +575,10 @@ fn is_tool_reference_change(block: &ContentBlock) -> bool { == Some("tool_reference") } -fn messages_carry_tool_changes(messages: &[Message]) -> bool { +fn messages_carry_tool_changes(messages: &[Recognized]) -> bool { messages .iter() + .filter_map(Recognized::known) .filter(|message| message.role == "system") .flat_map(Message::blocks) .any(is_tool_reference_change) @@ -334,7 +609,7 @@ pub fn drop_unsupported_params( let params = request.params; let speed = match ¶ms.speed { Some(speed) if !capabilities.supports_speed => { - reject("speed", format!("'{}'", speed_text(speed)), "")?; + reject("speed", speed_repr(speed), "")?; None } _ => params.speed.clone(), @@ -346,21 +621,21 @@ pub fn drop_unsupported_params( }); } let temperature = match params.temperature { - Some(temperature) if temperature != 1.0 => { + Some(temperature) if !is_one(&temperature) => { reject( "temperature", - json!(temperature).to_string(), + python_str(&temperature), "Only temperature=1 is supported. ", )?; None } temperature => temperature, }; - if let Some(top_p) = params.top_p { - reject("top_p", json!(top_p).to_string(), "")?; + if let Some(top_p) = ¶ms.top_p { + reject("top_p", python_str(top_p), "")?; } - if let Some(top_k) = params.top_k { - reject("top_k", json!(top_k).to_string(), "")?; + if let Some(top_k) = ¶ms.top_k { + reject("top_k", python_str(top_k), "")?; } Ok(MessagesRequest { params: MessagesOptionalParams { @@ -374,11 +649,10 @@ pub fn drop_unsupported_params( }) } -fn speed_text(speed: &Recognized) -> String { +fn speed_repr(speed: &Recognized) -> String { match speed { - Recognized::Known(speed) => <&'static str>::from(*speed).to_string(), - Recognized::Unrecognized(Value::String(text)) => text.clone(), - Recognized::Unrecognized(other) => other.to_string(), + Recognized::Known(speed) => format!("'{}'", <&'static str>::from(*speed)), + Recognized::Unrecognized(other) => litellm_python_compat::repr::repr(&python_value(other)), } } @@ -437,6 +711,7 @@ mod tests { use super::*; use crate::anthropic::common_utils::ENCRYPTED_REASONING_SIGNATURE_PREFIX; + use litellm_http::request::header_values; type Env = &'static [(&'static str, &'static str)]; @@ -679,6 +954,21 @@ mod tests { json!({"speed": "fast"}), "claude does not support speed='fast'. To drop unsupported params, set `litellm.drop_params = True`." )] + #[case::integer_temperature_keeps_no_decimal( + sampling_removed(), + json!({"temperature": 0}), + "claude does not support temperature=0. Only temperature=1 is supported. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::integer_top_p( + sampling_removed(), + json!({"top_p": 1}), + "claude does not support top_p=1. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::non_string_speed_is_reprd( + unmapped(), + json!({"speed": 2}), + "claude does not support speed=2. To drop unsupported params, set `litellm.drop_params = True`." + )] #[case::speed_ahead_of_sampling_params( sampling_removed(), json!({"speed": "fast", "temperature": 0.5}), @@ -696,6 +986,28 @@ mod tests { assert_eq!(detail.to_string(), message); } + #[rstest] + #[case::string_dropped(json!("json"), true, None)] + #[case::string_kept_without_drop_params(json!("json"), false, Some(json!("json")))] + #[case::object_kept(json!({"type": "json_schema"}), true, Some(json!({"type": "json_schema"})))] + fn a_non_object_output_format_is_dropped_under_drop_params( + #[case] output_format: Value, + #[case] drops_unsupported_params: bool, + #[case] expected: Option, + ) { + let context = MessagesTransformContext { + drops_unsupported_params, + ..MessagesTransformContext::with_lookup(unmapped(), false, &no_env) + }; + let transformed = ANTHROPIC_MESSAGES_CONFIG + .transform_anthropic_messages_request( + request(json!({"output_format": output_format})), + &context, + ) + .unwrap(); + assert_eq!(transformed.params.output_format, expected); + } + #[rstest] #[case::compaction_threshold( json!([{"type": "compaction", "compact_threshold": 200000}]), @@ -802,6 +1114,40 @@ mod tests { ); } + #[rstest] + #[case::continue_turn(json!({"type": "continue", "previous_message_id": "msg_1"}), false)] + #[case::continue_without_a_previous_message_id(json!({"type": "continue"}), false)] + #[case::continue_with_extra_keys(json!({"type": "continue", "previous_message_id": "msg_1", "note": 1}), false)] + #[case::create_turn(json!({"type": "create"}), true)] + #[case::continue_with_a_null_previous_message_id(json!({"type": "continue", "previous_message_id": null}), true)] + #[case::continue_with_a_numeric_previous_message_id(json!({"type": "continue", "previous_message_id": 1}), true)] + #[case::not_an_object(json!("continue"), true)] + fn advisor_history_is_kept_on_a_thread_continue_turn( + #[case] thread: Value, + #[case] stripped: bool, + unmapped: MessagesModelCapabilities, + ) { + let messages = if stripped { + json!([ + {"role": "user", "content": "Build a worker pool."}, + {"role": "assistant", "content": [ + {"type": "text", "text": "Let me consult the advisor."}, + {"type": "text", "text": "Here is the implementation."} + ]} + ]) + } else { + advisor_history() + }; + assert_eq!( + transform( + json!({"messages": advisor_history(), "thread": thread.clone()}), + unmapped, + false + ), + Ok(body(json!({"messages": messages, "thread": thread}))) + ); + } + #[rstest] fn bridge_minted_reasoning_is_stripped_from_the_wire(unmapped: MessagesModelCapabilities) { let messages = json!([ @@ -864,6 +1210,7 @@ mod tests { keeps_display_updates: false, }, drop_params: true, + drops_unsupported_params: false, } ); return; @@ -1165,6 +1512,55 @@ mod tests { ]}), &[] )] + #[case::thread_create(json!({"thread": {"type": "create"}}), &["message-threads-2026-08-12"])] + #[case::thread_continue( + json!({"thread": {"type": "continue", "previous_message_id": "msg_1"}}), + &["message-threads-2026-08-12"] + )] + #[case::thread_of_any_shape(json!({"thread": "opaque"}), &["message-threads-2026-08-12"])] + #[case::null_thread(json!({"thread": null}), &[])] + #[case::thread_create_still_scans_history( + json!({ + "thread": {"type": "create"}, + "messages": [ + {"role": "assistant", "content": [{"type": "compaction", "content": "summary", "signature": "sig"}]}, + {"role": "user", "content": "hi", "output_config": {"effort": "low"}}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ], + }), + &[ + "compact-2026-09-04", + "message-threads-2026-08-12", + "mid-conversation-tool-changes-2026-07-01", + "per-turn-control-2026-07-01", + ] + )] + #[case::thread_continue_skips_history( + json!({ + "thread": {"type": "continue", "previous_message_id": "msg_1"}, + "messages": [ + {"role": "assistant", "content": [{"type": "compaction", "content": "summary", "signature": "sig"}]}, + {"role": "user", "content": "hi", "output_config": {"effort": "low"}}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ], + }), + &["message-threads-2026-08-12"] + )] + #[case::thread_continue_keeps_param_betas( + json!({ + "thread": {"type": "continue", "previous_message_id": "msg_1"}, + "compaction": {}, + "speed": "fast", + }), + &["compact-2026-09-04", "fast-mode-2026-02-01", "message-threads-2026-08-12"] + )] + #[case::malformed_continue_scans_history( + json!({ + "thread": {"type": "continue", "previous_message_id": 7}, + "messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}], + }), + &["message-threads-2026-08-12", "per-turn-control-2026-07-01"] + )] fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) { assert_eq!(feature_betas(&request(fields)), betas(expected)); } @@ -1352,4 +1748,158 @@ mod tests { .collect(); assert_eq!(undeclared, Vec::<&String>::new()); } + + const FOUNDRY: &str = "https://team.services.ai.azure.com/anthropic/v1/messages"; + + #[rstest] + #[case::drops_a_beta_foundry_lacks( + FOUNDRY, + "advisor-tool-2026-03-01,context-1m-2025-08-07", + Some("context-1m-2025-08-07") + )] + #[case::forwards_unknown_betas_sorted( + FOUNDRY, + "zzz,files-api-2025-04-14,aaa", + Some("aaa,files-api-2025-04-14,zzz") + )] + #[case::removes_the_header_when_nothing_is_left(FOUNDRY, "advisor-tool-2026-03-01", None)] + #[case::cognitive_services_host( + "team.cognitiveservices.azure.com/anthropic", + "advisor-tool-2026-03-01", + None + )] + #[case::anthropic_host_is_untouched( + "https://api.anthropic.com/v1/messages", + "advisor-tool-2026-03-01", + Some("advisor-tool-2026-03-01") + )] + #[case::lookalike_host_is_untouched( + "https://notservices.ai.azure.com.example/v1/messages", + "advisor-tool-2026-03-01", + Some("advisor-tool-2026-03-01") + )] + fn foundry_urls_filter_betas_through_the_azure_ai_row( + #[case] url: &str, + #[case] betas: &str, + #[case] expected: Option<&str>, + ) { + let filtered = ANTHROPIC_MESSAGES_CONFIG + .headers_for_url(headers(&[("x-a", "1"), (BETA_HEADER, betas)]), url); + assert_eq!( + header_values(&filtered, BETA_HEADER).collect::>(), + expected.into_iter().collect::>() + ); + assert_eq!(header_values(&filtered, "x-a").collect::>(), ["1"]); + } + + #[rstest] + fn foundry_url_without_betas_keeps_the_headers() { + let sent = headers(&[("x-a", "1")]); + assert_eq!( + ANTHROPIC_MESSAGES_CONFIG.headers_for_url(sent.clone(), FOUNDRY), + sent + ); + } + + #[test] + fn the_http_error_retry_matches_the_cases_shared_with_python() { + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/anthropic_messages_http_error_retry.json" + ))) + .unwrap(); + let table = &fixture["http_error_retry"]; + let cases = table["cases"].as_array().unwrap(); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let mut body = table["request"].clone(); + body["thinking"] = case["thinking"].clone(); + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + let error_text = case["error_text"].as_str().unwrap(); + let retried = should_retry_anthropic_messages_on_http_error( + status, + error_text, + &body, + &MessagesTransformContext::default(), + ); + let stripped = case["stripped"].as_bool().unwrap(); + let transformed = transform_anthropic_messages_request_on_http_error( + status, + error_text, + body.clone(), + ); + let expected = if stripped { + RetriedRequest { + reasons: vec![RetryReason::ThinkingSignature], + body: table["stripped_request"].clone(), + } + } else { + RetriedRequest { + reasons: Vec::new(), + body, + } + }; + retried != case["retried"].as_bool().unwrap() || transformed != expected + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } + + #[test] + fn the_strict_tools_retry_matches_the_cases_shared_with_python() { + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/anthropic_messages_http_error_retry.json" + ))) + .unwrap(); + let replay = &fixture["http_error_retry"]; + let table = &fixture["strict_tools_retry"]; + let cases = table["cases"].as_array().unwrap(); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let mut body = replay["request"].clone(); + body["thinking"] = case["thinking"].clone(); + body["tools"] = table["tools"].clone(); + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + let error_text = case["error_text"].as_str().unwrap(); + let context = MessagesTransformContext { + drops_unsupported_params: case["drops_unsupported_params"].as_bool().unwrap(), + ..MessagesTransformContext::default() + }; + let mut reasons = Vec::new(); + let mut expected = if case["stripped"].as_bool().unwrap() { + reasons.push(RetryReason::ThinkingSignature); + replay["stripped_request"].clone() + } else { + body.clone() + }; + expected["tools"] = if case["strict_dropped"].as_bool().unwrap() { + reasons.push(RetryReason::StrictTools { + dropped: usize::try_from(table["dropped"].as_u64().unwrap()).unwrap(), + }); + table["tools_without_strict"].clone() + } else { + table["tools"].clone() + }; + let retried = should_retry_anthropic_messages_on_http_error( + status, error_text, &body, &context, + ); + let transformed = + transform_anthropic_messages_request_on_http_error(status, error_text, body); + retried != case["retried"].as_bool().unwrap() + || transformed + != RetriedRequest { + reasons, + body: expected, + } + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } } diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index d13cfa81a94..fb50d229bc7 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -1,3 +1,5 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; +use litellm_host::interceptors::RetriedRequest; use litellm_http::request::{has_bearer_auth, has_header}; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest}, @@ -5,6 +7,7 @@ use litellm_llms_types::{ }; use litellm_router_types::LitellmParams; use litellm_secret_value::SecretValue; +use serde_json::Value; use crate::{ Error, @@ -21,7 +24,7 @@ use crate::{ base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, messages::{ - context::MessagesTransformContext, + context::{MessagesShapeOptions, MessagesTransformContext}, normalization::{normalize_system_role_messages, strip_cache_control_scope}, transformation::BaseMessagesConfig, }, @@ -36,12 +39,28 @@ pub const AZURE_ANTHROPIC_MESSAGES_CONFIG: AzureAnthropicMessagesConfig = AzureAnthropicMessagesConfig; impl BaseMessagesConfig for AzureAnthropicMessagesConfig { + fn tool_custom_field(&self, model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: crate::anthropic::messages::transformation::is_claude_model(model), + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn get_complete_url( diff --git a/litellm-rust/crates/llms/src/base_llm/messages/context.rs b/litellm-rust/crates/llms/src/base_llm/messages/context.rs index 3d8b0f41d8c..e5593d3b216 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/context.rs @@ -23,6 +23,12 @@ impl SupportedEffortTiers { } } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MessagesShapeOptions { + pub reasoning_auto_summary: bool, + pub presanitized: bool, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct MessagesModelCapabilities { #[serde(default)] @@ -123,6 +129,7 @@ pub struct ThinkingContext { pub struct MessagesTransformContext { pub thinking: ThinkingContext, pub drop_params: bool, + pub drops_unsupported_params: bool, } impl MessagesTransformContext { pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self { @@ -142,6 +149,7 @@ impl MessagesTransformContext { keeps_display_updates: false, }, drop_params, + drops_unsupported_params: false, } } diff --git a/litellm-rust/crates/llms/src/base_llm/messages/mod.rs b/litellm-rust/crates/llms/src/base_llm/messages/mod.rs index e9591daa9da..5c1563c7a23 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/mod.rs @@ -2,4 +2,5 @@ pub mod context; pub mod mid_conversation_system; pub mod normalization; pub mod streaming; +pub mod tools; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs b/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs index 7e24da2b5eb..ee2f4dcaeef 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs @@ -1,6 +1,9 @@ -use litellm_llms_types::formats::messages::{ - CacheControl, ContentBlock, Message, MessageContent, MessagesOptionalParams, MessagesRequest, - SystemPrompt, +use litellm_llms_types::{ + formats::messages::{ + CacheControl, ContentBlock, Message, MessageContent, MessagesOptionalParams, + MessagesRequest, SystemPrompt, + }, + recognized::{Recognized, map_known_runs}, }; use crate::{ @@ -22,15 +25,22 @@ pub fn normalize_system_role_messages( let leading_count = request .messages .iter() - .take_while(|message| is_system_role_message(message)) + .take_while(|message| message.known().is_some_and(is_system_role_message)) .count(); let mut messages = request.messages.into_iter(); - let hoisted: Vec = messages.by_ref().take(leading_count).collect(); - let remaining: Vec = messages.collect(); + let hoisted: Vec = messages + .by_ref() + .take(leading_count) + .filter_map(|message| match message { + Recognized::Known(message) => Some(message), + Recognized::Unrecognized(_) => None, + }) + .collect(); + let remaining: Vec> = messages.collect(); let remaining = if supports_mid_conversation_system { remaining } else { - convert_mid_conversation_system_turns(remaining) + map_known_runs(remaining, convert_mid_conversation_system_turns) }; let system = if hoisted.is_empty() { request.params.system @@ -94,7 +104,7 @@ pub fn strip_cache_control_scope(request: MessagesRequest) -> MessagesRequest { messages: request .messages .into_iter() - .map(strip_scope_from_message) + .map(|message| message.map_known(strip_scope_from_message)) .collect(), params: MessagesOptionalParams { system: request.params.system.map(strip_scope_from_system), diff --git a/litellm-rust/crates/llms/src/base_llm/messages/tools.rs b/litellm-rust/crates/llms/src/base_llm/messages/tools.rs new file mode 100644 index 00000000000..49113872c56 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/messages/tools.rs @@ -0,0 +1,194 @@ +use litellm_llms_types::{ + formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool}, + recognized::Recognized, +}; +use serde_json::{Map, Value}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ToolCustomField { + Keep, + Rewrite { hoist_defer_loading: bool }, +} + +impl ToolCustomField { + pub fn apply(self, request: MessagesRequest, drop_params: bool) -> MessagesRequest { + match self { + Self::Keep => request, + Self::Rewrite { + hoist_defer_loading, + } => tools_without_custom_field(request, hoist_defer_loading, drop_params), + } + } +} + +fn tools_without_custom_field( + request: MessagesRequest, + hoist: bool, + drop_custom: bool, +) -> MessagesRequest { + let tools = request.params.tools.map(|tools| { + tools + .into_iter() + .map(|tool| tool_without_custom_field(tool, hoist, drop_custom)) + .collect() + }); + MessagesRequest { + params: MessagesOptionalParams { + tools, + ..request.params + }, + ..request + } +} + +fn tool_without_custom_field( + tool: Recognized, + hoist: bool, + drop_custom: bool, +) -> Recognized { + let Ok(Value::Object(fields)) = serde_json::to_value(&tool) else { + return tool; + }; + let Some(custom) = fields.get("custom").cloned() else { + return tool; + }; + let defer_loading = hoist + .then(|| hoistable_defer_loading(&fields, &custom)) + .flatten(); + let without_custom = fields + .iter() + .filter(|(key, _)| *key != "custom") + .map(|(key, value)| (key.clone(), value.clone())); + let rebuilt: Map = match defer_loading { + None if drop_custom => without_custom.collect(), + None => return tool, + Some(defer_loading) => { + let unhoisted: Map = custom + .as_object() + .into_iter() + .flatten() + .filter(|(key, _)| *key != "defer_loading") + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + let kept_custom = (!unhoisted.is_empty() && !drop_custom) + .then(|| ("custom".to_string(), Value::Object(unhoisted))); + without_custom + .chain([("defer_loading".to_string(), Value::Bool(defer_loading))]) + .chain(kept_custom) + .collect() + } + }; + serde_json::from_value(Value::Object(rebuilt)).unwrap_or(tool) +} + +fn hoistable_defer_loading(tool: &Map, custom: &Value) -> Option { + if tool.contains_key("defer_loading") { + return None; + } + custom.as_object()?.get("defer_loading")?.as_bool() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + const WEATHER: &str = "get_weather"; + + fn tool(fields: Value) -> Value { + merged( + json!({"name": WEATHER, "input_schema": {"type": "object"}}), + fields, + ) + } + + fn tools_after(policy: ToolCustomField, tools: Value, drop_custom: bool) -> Value { + let request: MessagesRequest = serde_json::from_value(json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": [tools] + })) + .unwrap(); + serde_json::to_value(policy.apply(request, drop_custom)).unwrap()["tools"][0].clone() + } + + fn merged(base: Value, fields: Value) -> Value { + let mut merged = base.as_object().unwrap().clone(); + merged.extend(fields.as_object().unwrap().clone()); + Value::Object(merged) + } + + const HOIST: ToolCustomField = ToolCustomField::Rewrite { + hoist_defer_loading: true, + }; + const NO_HOIST: ToolCustomField = ToolCustomField::Rewrite { + hoist_defer_loading: false, + }; + + #[rstest] + #[case::hoists_defer_loading( + HOIST, + json!({"custom": {"defer_loading": true}}), + false, + json!({"defer_loading": true}) + )] + #[case::keeps_other_custom_keys( + HOIST, + json!({"custom": {"defer_loading": false, "note": "kept"}}), + false, + json!({"defer_loading": false, "custom": {"note": "kept"}}) + )] + #[case::drops_other_custom_keys_with_drop_params( + HOIST, + json!({"custom": {"defer_loading": true, "note": "dropped"}}), + true, + json!({"defer_loading": true}) + )] + #[case::keeps_an_opaque_custom( + HOIST, + json!({"custom": "opaque"}), + false, + json!({"custom": "opaque"}) + )] + #[case::drops_an_opaque_custom_with_drop_params( + HOIST, + json!({"custom": "opaque"}), + true, + json!({}) + )] + #[case::own_defer_loading_wins( + HOIST, + json!({"defer_loading": false, "custom": {"defer_loading": true}}), + false, + json!({"defer_loading": false, "custom": {"defer_loading": true}}) + )] + #[case::not_hoisted_off_claude_hosts( + NO_HOIST, + json!({"custom": {"defer_loading": true}}), + false, + json!({"custom": {"defer_loading": true}}) + )] + fn tool_custom_field_follows_the_python_handler( + #[case] policy: ToolCustomField, + #[case] fields: Value, + #[case] drop_custom: bool, + #[case] expected: Value, + ) { + assert_eq!( + tools_after(policy, tool(fields), drop_custom), + tool(expected) + ); + } + + #[rstest] + fn keep_leaves_the_custom_field() { + let custom = tool(json!({"custom": {"defer_loading": true}})); + assert_eq!( + tools_after(ToolCustomField::Keep, custom.clone(), true), + custom + ); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs index 744ce89afdb..13c021c2aec 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs @@ -1,19 +1,33 @@ +use litellm_host::interceptors::RetriedRequest; use litellm_llms_types::formats::messages::{ MessagesOptionalParams, MessagesRequest, MessagesResponse, }; use litellm_router_types::LitellmParams; +use litellm_secret_value::SecretValue; use serde_json::Value; -use super::context::MessagesTransformContext; +use super::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + tools::ToolCustomField, +}; pub use crate::base_llm::auth::{Headers, ValidatedEnvironment}; use crate::{Error, base_llm::messages::streaming::StreamDecoder}; +pub enum FederatedEnvironment { + Validated(ValidatedEnvironment), + Unauthenticated { + headers: Headers, + missing: Error, + federate: fn(Headers, SecretValue) -> ValidatedEnvironment, + }, +} + pub trait BaseMessagesConfig: Sync { fn shape_request( &self, request: MessagesRequest, - _reasoning_auto_summary: bool, + _options: MessagesShapeOptions, ) -> Result { Ok(request) } @@ -35,6 +49,10 @@ pub trait BaseMessagesConfig: Sync { Ok(request) } + fn tool_custom_field(&self, _model: &str) -> ToolCustomField { + ToolCustomField::Keep + } + fn transform_anthropic_messages_response( &self, _model: &str, @@ -62,6 +80,26 @@ pub trait BaseMessagesConfig: Sync { env_lookup: &dyn Fn(&str) -> Option, ) -> Result; + fn validate_federated_environment( + &self, + headers: Headers, + api_key: Option<&str>, + model: &str, + optional_params: &MessagesOptionalParams, + litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + self.validate_environment( + headers, + api_key, + model, + optional_params, + litellm_params, + env_lookup, + ) + .map(FederatedEnvironment::Validated) + } + /// `None` relays the upstream bytes untouched, which is right for every host that already /// speaks Anthropic SSE. A host on another wire returns the decoder that lifts its frames /// into Anthropic stream events, and the route re-encodes those as Anthropic SSE. @@ -77,6 +115,10 @@ pub trait BaseMessagesConfig: Sync { headers } + fn headers_for_url(&self, headers: Headers, _url: &str) -> Headers { + headers + } + fn request_body( &self, request: MessagesRequest, @@ -91,6 +133,16 @@ pub trait BaseMessagesConfig: Sync { fn wire_body(&self, body: Value) -> Value { body } + + fn retry_after_http_error( + &self, + _status: u16, + _error_text: &str, + _request_body: &Value, + _context: &MessagesTransformContext, + ) -> Option { + None + } } #[cfg(test)] @@ -149,9 +201,10 @@ mod tests { } #[rstest] - #[case::disabled(false)] - #[case::enabled(true)] - fn default_shaping_preserves_provider_policy_inputs(#[case] reasoning_auto_summary: bool) { + fn default_shaping_preserves_provider_policy_inputs( + #[values(false, true)] reasoning_auto_summary: bool, + #[values(false, true)] presanitized: bool, + ) { let request: MessagesRequest = serde_json::from_value(serde_json::json!({ "model": "test-model", "metadata": {"user_id": 7, "extra": "keep"}, @@ -160,7 +213,13 @@ mod tests { })) .unwrap(); assert_eq!( - DefaultsConfig.shape_request(request.clone(), reasoning_auto_summary), + DefaultsConfig.shape_request( + request.clone(), + MessagesShapeOptions { + reasoning_auto_summary, + presanitized + } + ), Ok(request) ); assert_eq!( diff --git a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs index 5b89345a1c5..4bf30b56f00 100644 --- a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs @@ -1,3 +1,5 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; +use litellm_host::interceptors::RetriedRequest; use std::sync::LazyLock; use litellm_auth::{AwsParams, CredentialPlacement, ParamKind, ParamSpec}; @@ -21,7 +23,10 @@ use crate::{ }, base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, - messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + messages::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + transformation::BaseMessagesConfig, + }, }, bedrock::common_utils::validated_region, }; @@ -50,12 +55,28 @@ pub const BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG: BedrockClaudePlatformMessages BedrockClaudePlatformMessagesConfig; impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { + fn tool_custom_field(&self, model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: crate::anthropic::messages::transformation::is_claude_model(model), + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn validate_environment( diff --git a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs index 1d380c917f6..fa037974342 100644 --- a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs @@ -1,9 +1,11 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_core_utils::decline::Decline; +use litellm_host::interceptors::RetriedRequest; use std::convert::Infallible; use crate::{ anthropic::messages::handler::shape_anthropic_messages_request, - base_llm::messages::context::MessagesTransformContext, + base_llm::messages::context::{MessagesShapeOptions, MessagesTransformContext}, }; use futures_util::StreamExt; use litellm_auth::AwsParams; @@ -103,12 +105,22 @@ fn invoke_url( } impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn get_complete_url( diff --git a/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs index f139b93e259..0e67ea1a139 100644 --- a/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs @@ -1,3 +1,5 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; +use litellm_host::interceptors::RetriedRequest; use std::sync::LazyLock; use litellm_auth::{AwsParams, CredentialPlacement}; @@ -17,6 +19,7 @@ use serde_json::Value; use crate::{ Error, anthropic::{ + beta_headers::accepted_or_unlisted, common_utils::{existing_betas, merge_beta_headers}, messages::{ handler::shape_anthropic_messages_request, @@ -26,7 +29,7 @@ use crate::{ base_llm::{ auth::AuthScheme, messages::{ - context::MessagesTransformContext, + context::{MessagesShapeOptions, MessagesTransformContext}, transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment}, }, }, @@ -56,6 +59,16 @@ pub const BEDROCK_MANTLE_MESSAGES_CONFIG: BedrockMantleAnthropicMessagesConfig = BedrockMantleAnthropicMessagesConfig; impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig { + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn validate_environment( &self, headers: Headers, @@ -129,9 +142,9 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig { fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn wire_body(&self, body: Value) -> Value { @@ -191,14 +204,10 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig { .filter_map(Value::as_str) .filter_map(|beta| beta.parse::().ok()) .collect(); - let caller: BetaSet = existing_betas(&headers) - .union(body_betas) - .into_iter() - .filter_map(|beta| match beta { - AnthropicBeta::Other(_) => Some(beta), - _ => beta.on(BetaProvider::BedrockMantle), - }) - .collect(); + let caller = accepted_or_unlisted( + existing_betas(&headers).union(body_betas), + BetaProvider::BedrockMantle, + ); let safeguards: BetaSet = request .params .extra diff --git a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs index 0dbcea01439..ee2aec9efb9 100644 --- a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs @@ -1,4 +1,6 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::RetriedRequest; use litellm_secret_value::SecretValue; use litellm_core_utils::settings::resolve_non_empty; @@ -21,7 +23,10 @@ use crate::{ }, base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, - messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + messages::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + transformation::BaseMessagesConfig, + }, }, }; @@ -42,12 +47,28 @@ pub const DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG: DeepSeekAnthropicMessagesConfig = DeepSeekAnthropicMessagesConfig; impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig { + fn tool_custom_field(&self, _model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: false, + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn get_complete_url( diff --git a/litellm-rust/crates/llms/src/edenai/messages/transformation.rs b/litellm-rust/crates/llms/src/edenai/messages/transformation.rs index d5944484522..d4f577e09a9 100644 --- a/litellm-rust/crates/llms/src/edenai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/edenai/messages/transformation.rs @@ -1,4 +1,6 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::RetriedRequest; use litellm_secret_value::SecretValue; use litellm_core_utils::settings::resolve_non_empty; @@ -33,6 +35,22 @@ pub struct EdenAIAnthropicMessagesConfig; pub const EDENAI_MESSAGES_CONFIG: EdenAIAnthropicMessagesConfig = EdenAIAnthropicMessagesConfig; impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig { + fn tool_custom_field(&self, _model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: false, + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn validate_environment( &self, headers: Headers, diff --git a/litellm-rust/crates/llms/src/error.rs b/litellm-rust/crates/llms/src/error.rs index 2975ceee3cb..77ee1f1036f 100644 --- a/litellm-rust/crates/llms/src/error.rs +++ b/litellm-rust/crates/llms/src/error.rs @@ -168,6 +168,10 @@ pub enum ErrorDetail { actual: String, choices: Vec<&'static str>, }, + #[error( + "Invalid reasoning_effort: {actual}. Must be one of: 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'none'" + )] + InvalidReasoningEffort { actual: String }, #[error("{field}='{value}' is not supported by this model. Got model: {model}")] UnsupportedValue { field: &'static str, @@ -187,6 +191,14 @@ pub enum ErrorDetail { "thinking.budget_tokens must be a number, got {actual}. To drop it, set `litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy" )] NonNumericThinkingBudget { actual: &'static str }, + /// Python's comparison of two operands it cannot order, which raises `TypeError` with this + /// text before any request is sent. + #[error("'{op}' not supported between instances of '{left}' and '{right}'")] + UnorderableTypes { + op: &'static str, + left: &'static str, + right: &'static str, + }, #[error( "Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({{LLM_PROVIDER}}_API_KEY) or via params" )] diff --git a/litellm-rust/crates/llms/src/minimax/messages/transformation.rs b/litellm-rust/crates/llms/src/minimax/messages/transformation.rs index dafe5fbd16b..2eaf182e657 100644 --- a/litellm-rust/crates/llms/src/minimax/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/minimax/messages/transformation.rs @@ -1,5 +1,8 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::RetriedRequest; use litellm_secret_value::SecretValue; +use serde_json::Value; use litellm_core_utils::settings::resolve_non_empty; use litellm_llms_types::{ @@ -19,7 +22,10 @@ use crate::{ }, base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, - messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + messages::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + transformation::BaseMessagesConfig, + }, }, }; @@ -31,12 +37,28 @@ pub struct MinimaxMessagesConfig; pub const MINIMAX_MESSAGES_CONFIG: MinimaxMessagesConfig = MinimaxMessagesConfig; impl BaseMessagesConfig for MinimaxMessagesConfig { + fn tool_custom_field(&self, _model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: false, + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn validate_environment( diff --git a/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs b/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs index eeb3d1cb7f8..dc09f4e4149 100644 --- a/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs @@ -1,5 +1,8 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::RetriedRequest; use litellm_secret_value::SecretValue; +use serde_json::Value; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{has_bearer_auth, without_headers}; @@ -31,6 +34,16 @@ pub const OPENROUTER_MESSAGES_CONFIG: OpenRouterAnthropicMessagesConfig = OpenRouterAnthropicMessagesConfig; impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig { + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn validate_environment( &self, headers: Headers, diff --git a/litellm-rust/crates/llms/src/tencent/messages/transformation.rs b/litellm-rust/crates/llms/src/tencent/messages/transformation.rs index 621a09f4fb0..b3ee28b7474 100644 --- a/litellm-rust/crates/llms/src/tencent/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/tencent/messages/transformation.rs @@ -1,5 +1,8 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::RetriedRequest; use litellm_secret_value::SecretValue; +use serde_json::Value; use litellm_core_utils::settings::resolve_non_empty; use litellm_llms_types::{ @@ -19,7 +22,10 @@ use crate::{ }, base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, - messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + messages::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + transformation::BaseMessagesConfig, + }, }, }; @@ -32,12 +38,28 @@ pub struct TencentAnthropicMessagesConfig; pub const TENCENT_MESSAGES_CONFIG: TencentAnthropicMessagesConfig = TencentAnthropicMessagesConfig; impl BaseMessagesConfig for TencentAnthropicMessagesConfig { + fn tool_custom_field(&self, _model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: false, + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn validate_environment( diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index 038617c2e10..674b9951d3e 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -1,8 +1,10 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::VertexParams; use litellm_auth_gcp::{ VertexConfig, get_vertex_ai_location, get_vertex_ai_project, get_vertex_ai_project_from_credentials, secret_names, }; +use litellm_host::interceptors::RetriedRequest; use litellm_http::request::{has_header, header_value, without_headers}; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig}, @@ -24,7 +26,7 @@ use crate::{ base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, messages::{ - context::MessagesTransformContext, + context::{MessagesShapeOptions, MessagesTransformContext}, normalization::{normalize_system_role_messages, strip_cache_control_scope}, transformation::BaseMessagesConfig, }, @@ -47,12 +49,28 @@ pub const VERTEX_ANTHROPIC_MESSAGES_CONFIG: VertexAiPartnerModelsAnthropicMessag VertexAiPartnerModelsAnthropicMessagesConfig; impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { + fn tool_custom_field(&self, model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: crate::anthropic::messages::transformation::is_claude_model(model), + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn get_complete_url( @@ -700,6 +718,7 @@ mod tests { ..ThinkingContext::default() }, drop_params: false, + drops_unsupported_params: false, }, ); assert_eq!(body["system"], json!("be terse")); @@ -717,6 +736,7 @@ mod tests { ..ThinkingContext::default() }, drop_params: false, + drops_unsupported_params: false, } } diff --git a/litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json b/litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json new file mode 100644 index 00000000000..843a1689056 --- /dev/null +++ b/litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json @@ -0,0 +1,623 @@ +{ + "invalid_thinking_block_error": [ + { + "name": "invalid_signature", + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "expected": true + }, + { + "name": "invalid_signature_in_an_error_envelope", + "error_text": "{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"messages.3.content.3: Invalid `signature` in `thinking` block\"},\"request_id\":\"req_011Ca2EtQDxp7x6RGUY2jVn9\"}", + "expected": true + }, + { + "name": "signature_not_a_string", + "error_text": "{\"message\":\"messages.2.content.0.thinking.signature.str: Input should be a valid string\"}", + "expected": true + }, + { + "name": "signature_not_a_string_bare", + "error_text": "messages.4.content.1.thinking.signature.str: Input should be a valid string", + "expected": true + }, + { + "name": "empty_thinking", + "error_text": "messages.1.content.0.thinking: each thinking block must contain thinking", + "expected": true + }, + { + "name": "bound_to_a_different_conversation", + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation.", + "expected": true + }, + { + "name": "uppercase", + "error_text": "MESSAGES.1: INVALID SIGNATURE IN THINKING BLOCK", + "expected": true + }, + { + "name": "signature_without_a_verdict", + "error_text": "thinking signature is malformed", + "expected": false + }, + { + "name": "invalid_without_thinking", + "error_text": "Invalid `signature` in `tool_use` block", + "expected": false + }, + { + "name": "model_not_found", + "error_text": "invalid_request_error: model not found", + "expected": false + }, + { + "name": "rate_limited", + "error_text": "rate limit exceeded", + "expected": false + }, + { + "name": "unrelated", + "error_text": "max_tokens: 4096 > 64000", + "expected": false + }, + { + "name": "empty", + "error_text": "", + "expected": false + } + ], + "thinking_binding_mismatch_error": [ + { + "name": "bound_elsewhere", + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation.", + "expected": true + }, + { + "name": "mixed_case", + "error_text": "The block is Bound To A Different Conversation.", + "expected": true + }, + { + "name": "plain_signature", + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "expected": false + } + ], + "http_error_retry": { + "request": { + "model": "claude-sonnet-4-5", + "max_tokens": 4096, + "messages": [ + { + "role": "user", + "content": "hi" + }, + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "plan", + "signature": "sig" + }, + { + "type": "text", + "text": "hello" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "redacted_thinking", + "data": "opaque" + } + ] + }, + { + "role": "user", + "content": "again" + } + ] + }, + "stripped_request": { + "model": "claude-sonnet-4-5", + "max_tokens": 4096, + "messages": [ + { + "role": "user", + "content": "hi" + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "hello" + } + ] + }, + { + "role": "user", + "content": "again" + } + ] + }, + "cases": [ + { + "name": "invalid_signature", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": true, + "stripped": true + }, + { + "name": "missing_signature", + "status": 400, + "error_text": "messages.1.content.0.thinking.signature.str: Input should be a valid string", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": true, + "stripped": true + }, + { + "name": "empty_thinking", + "status": 400, + "error_text": "messages.1.content.0.thinking: each thinking block must contain thinking", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": true, + "stripped": true + }, + { + "name": "unpinned_binding_mismatch", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation.", + "thinking": { + "type": "adaptive" + }, + "retried": true, + "stripped": true + }, + { + "name": "caller_pinned_binding", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation.", + "thinking": { + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": "error" + } + }, + "retried": false, + "stripped": true + }, + { + "name": "caller_pinned_binding_other_signature_error", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": "error" + } + }, + "retried": true, + "stripped": true + }, + { + "name": "not_a_400", + "status": 429, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": false, + "stripped": false + }, + { + "name": "unrelated_400", + "status": 400, + "error_text": "max_tokens: 4096 > 64000", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": false, + "stripped": false + }, + { + "name": "signature_without_thinking", + "status": 400, + "error_text": "Invalid `signature` in `tool_use` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": false, + "stripped": false + } + ] + }, + "claude_strict_tools_rejection": [ + { + "name": "too_many_strict_tools", + "status": 400, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "expected": true + }, + { + "name": "too_many_strict_tools_in_an_error_envelope", + "status": 400, + "error_text": "{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"Too many strict tools (24). The maximum is 20.\"}}", + "expected": true + }, + { + "name": "non_ascii_digits", + "status": 400, + "error_text": "Too many strict tools (٢١)", + "expected": true + }, + { + "name": "optional_parameters", + "status": 400, + "error_text": "Schemas contains too many optional parameters (30)", + "expected": true + }, + { + "name": "union_types", + "status": 400, + "error_text": "Schemas contains too many parameters with union types", + "expected": true + }, + { + "name": "compiled_grammar_too_large", + "status": 400, + "error_text": "The compiled grammar is too large", + "expected": true + }, + { + "name": "compiled_grammar_size", + "status": 400, + "error_text": "Compiled grammar size exceeds the limit", + "expected": true + }, + { + "name": "custom_strict", + "status": 400, + "error_text": "tools.3.custom.strict: Extra inputs are not permitted", + "expected": true + }, + { + "name": "custom", + "status": 400, + "error_text": "tools.0.custom: Extra inputs are not permitted", + "expected": true + }, + { + "name": "grammar_compilation_timed_out", + "status": 400, + "error_text": "Grammar compilation timed out", + "expected": true + }, + { + "name": "no_count", + "status": 400, + "error_text": "tools.0.strict: too many strict tools", + "expected": false + }, + { + "name": "empty_count", + "status": 400, + "error_text": "Too many strict tools ()", + "expected": false + }, + { + "name": "lowercase_grammar_timeout", + "status": 400, + "error_text": "grammar compilation timed out", + "expected": false + }, + { + "name": "custom_without_separator", + "status": 400, + "error_text": "tools.0.custom.strict", + "expected": false + }, + { + "name": "uppercase_schemas", + "status": 400, + "error_text": "SCHEMAS CONTAINS TOO MANY OPTIONAL PARAMETERS", + "expected": false + }, + { + "name": "not_a_400", + "status": 429, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "expected": false + }, + { + "name": "empty", + "status": 400, + "error_text": "", + "expected": false + } + ], + "tools_without_strict": [ + { + "name": "strict_dropped_from_every_tool", + "tools": [ + { + "name": "a", + "strict": true, + "input_schema": {} + }, + { + "name": "b", + "strict": false + }, + { + "name": "c" + } + ], + "expected": { + "tools": [ + { + "name": "a", + "input_schema": {} + }, + { + "name": "b" + }, + { + "name": "c" + } + ], + "dropped_count": 1 + } + }, + { + "name": "only_a_true_strict_counts", + "tools": [ + { + "name": "a", + "strict": 1 + }, + { + "name": "b", + "strict": "true" + } + ], + "expected": null + }, + { + "name": "no_strict", + "tools": [ + { + "name": "a" + } + ], + "expected": null + }, + { + "name": "empty", + "tools": [], + "expected": null + }, + { + "name": "non_object_tool", + "tools": [ + { + "name": "a", + "strict": true + }, + "b" + ], + "expected": null + }, + { + "name": "not_a_list", + "tools": { + "name": "a", + "strict": true + }, + "expected": null + }, + { + "name": "null", + "tools": null, + "expected": null + } + ], + "strict_tools_retry": { + "tools": [ + { + "name": "a", + "input_schema": { + "type": "object" + }, + "strict": true + }, + { + "name": "b", + "input_schema": { + "type": "object" + }, + "strict": false + }, + { + "name": "c", + "input_schema": { + "type": "object" + } + } + ], + "tools_without_strict": [ + { + "name": "a", + "input_schema": { + "type": "object" + } + }, + { + "name": "b", + "input_schema": { + "type": "object" + } + }, + { + "name": "c", + "input_schema": { + "type": "object" + } + } + ], + "dropped": 1, + "cases": [ + { + "name": "strict_rejection_under_drop_params", + "status": 400, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": false, + "strict_dropped": true + }, + { + "name": "strict_rejection_without_drop_params", + "status": 400, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": false, + "retried": false, + "stripped": false, + "strict_dropped": true + }, + { + "name": "compiled_grammar_under_drop_params", + "status": 400, + "error_text": "The compiled grammar is too large", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": false, + "strict_dropped": true + }, + { + "name": "strict_text_on_a_429", + "status": 429, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": false, + "stripped": false, + "strict_dropped": false + }, + { + "name": "unrelated_400_under_drop_params", + "status": 400, + "error_text": "max_tokens: 4096 > 64000", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": false, + "stripped": false, + "strict_dropped": false + }, + { + "name": "thinking_rejection_keeps_strict", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": true, + "strict_dropped": false + }, + { + "name": "thinking_rejection_without_drop_params_keeps_strict", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": false, + "retried": true, + "stripped": true, + "strict_dropped": false + }, + { + "name": "one_error_naming_both_without_drop_params", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": false, + "retried": true, + "stripped": true, + "strict_dropped": true + }, + { + "name": "pinned_binding_mismatch_with_a_strict_rejection_under_drop_params", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation. Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": "error" + } + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": true, + "strict_dropped": true + }, + { + "name": "pinned_binding_mismatch_with_a_strict_rejection_without_drop_params", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation. Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": "error" + } + }, + "drops_unsupported_params": false, + "retried": false, + "stripped": true, + "strict_dropped": true + } + ] + } +} diff --git a/litellm-rust/crates/llms/tests/messages_http_error_retry.rs b/litellm-rust/crates/llms/tests/messages_http_error_retry.rs new file mode 100644 index 00000000000..ac8ceac4ac6 --- /dev/null +++ b/litellm-rust/crates/llms/tests/messages_http_error_retry.rs @@ -0,0 +1,138 @@ +use litellm_host::interceptors::{RetriedRequest, RetryReason}; +use litellm_llms::{ + anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG, + azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG, + base_llm::messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + bedrock::{ + claude_platform::messages::transformation::BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG, + messages::invoke_transformations::anthropic_claude_transformation::BEDROCK_ANTHROPIC_MESSAGES_CONFIG, + }, + bedrock_mantle::messages::transformation::BEDROCK_MANTLE_MESSAGES_CONFIG, + deepseek::messages::transformation::DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG, + edenai::messages::transformation::EDENAI_MESSAGES_CONFIG, + minimax::messages::transformation::MINIMAX_MESSAGES_CONFIG, + openrouter::messages::transformation::OPENROUTER_MESSAGES_CONFIG, + tencent::messages::transformation::TENCENT_MESSAGES_CONFIG, + vertex_ai::messages::transformation::VERTEX_ANTHROPIC_MESSAGES_CONFIG, +}; +use rstest::rstest; +use serde_json::Value; + +#[rstest] +#[case::anthropic(&ANTHROPIC_MESSAGES_CONFIG)] +#[case::azure_ai(&AZURE_ANTHROPIC_MESSAGES_CONFIG)] +#[case::bedrock_invoke(&BEDROCK_ANTHROPIC_MESSAGES_CONFIG)] +#[case::bedrock_claude_platform(&BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG)] +#[case::bedrock_mantle(&BEDROCK_MANTLE_MESSAGES_CONFIG)] +#[case::deepseek(&DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG)] +#[case::edenai(&EDENAI_MESSAGES_CONFIG)] +#[case::minimax(&MINIMAX_MESSAGES_CONFIG)] +#[case::openrouter(&OPENROUTER_MESSAGES_CONFIG)] +#[case::tencent(&TENCENT_MESSAGES_CONFIG)] +#[case::vertex_ai(&VERTEX_ANTHROPIC_MESSAGES_CONFIG)] +fn every_messages_provider_retries_like_the_python_base_config( + #[case] config: &dyn BaseMessagesConfig, +) { + let fixture: Value = serde_json::from_str(include_str!( + "fixtures/anthropic_messages_http_error_retry.json" + )) + .unwrap(); + let table = &fixture["http_error_retry"]; + let cases = table["cases"].as_array().unwrap(); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let mut body = table["request"].clone(); + body["thinking"] = case["thinking"].clone(); + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + let expected = case["retried"].as_bool().unwrap().then(|| { + if case["stripped"].as_bool().unwrap() { + RetriedRequest { + reasons: vec![RetryReason::ThinkingSignature], + body: table["stripped_request"].clone(), + } + } else { + RetriedRequest { + reasons: Vec::new(), + body: body.clone(), + } + } + }); + config.retry_after_http_error( + status, + case["error_text"].as_str().unwrap(), + &body, + &MessagesTransformContext::default(), + ) != expected + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); +} + +#[rstest] +#[case::anthropic(&ANTHROPIC_MESSAGES_CONFIG)] +#[case::azure_ai(&AZURE_ANTHROPIC_MESSAGES_CONFIG)] +#[case::bedrock_invoke(&BEDROCK_ANTHROPIC_MESSAGES_CONFIG)] +#[case::bedrock_claude_platform(&BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG)] +#[case::bedrock_mantle(&BEDROCK_MANTLE_MESSAGES_CONFIG)] +#[case::deepseek(&DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG)] +#[case::edenai(&EDENAI_MESSAGES_CONFIG)] +#[case::minimax(&MINIMAX_MESSAGES_CONFIG)] +#[case::openrouter(&OPENROUTER_MESSAGES_CONFIG)] +#[case::tencent(&TENCENT_MESSAGES_CONFIG)] +#[case::vertex_ai(&VERTEX_ANTHROPIC_MESSAGES_CONFIG)] +fn every_messages_provider_drops_rejected_strict_tools_like_the_python_base_config( + #[case] config: &dyn BaseMessagesConfig, +) { + let fixture: Value = serde_json::from_str(include_str!( + "fixtures/anthropic_messages_http_error_retry.json" + )) + .unwrap(); + let replay = &fixture["http_error_retry"]; + let table = &fixture["strict_tools_retry"]; + let cases = table["cases"].as_array().unwrap(); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let mut body = replay["request"].clone(); + body["thinking"] = case["thinking"].clone(); + body["tools"] = table["tools"].clone(); + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + let expected = case["retried"].as_bool().unwrap().then(|| { + let mut reasons = Vec::new(); + let mut expected = if case["stripped"].as_bool().unwrap() { + reasons.push(RetryReason::ThinkingSignature); + replay["stripped_request"].clone() + } else { + body.clone() + }; + expected["tools"] = if case["strict_dropped"].as_bool().unwrap() { + reasons.push(RetryReason::StrictTools { + dropped: usize::try_from(table["dropped"].as_u64().unwrap()).unwrap(), + }); + table["tools_without_strict"].clone() + } else { + table["tools"].clone() + }; + RetriedRequest { + reasons, + body: expected, + } + }); + config.retry_after_http_error( + status, + case["error_text"].as_str().unwrap(), + &body, + &MessagesTransformContext { + drops_unsupported_params: case["drops_unsupported_params"].as_bool().unwrap(), + ..MessagesTransformContext::default() + }, + ) != expected + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); +} diff --git a/litellm-rust/crates/python-bridge/src/cache/AGENTS.md b/litellm-rust/crates/python-bridge/src/cache/AGENTS.md index e79562b8762..a95229c5bf4 100644 --- a/litellm-rust/crates/python-bridge/src/cache/AGENTS.md +++ b/litellm-rust/crates/python-bridge/src/cache/AGENTS.md @@ -2,6 +2,6 @@ This folder owns how Rust inference reaches the selected Python cache. Cache algorithms, storage protocols and response-cache semantics belong to their cache crates -`mod.rs` exposes the cache boundary to routes. `selection.rs` asks `litellm.rust_bridge.host.cache` for the live configured cache and owns route admission and inference protocol composition +`mod.rs` exposes the cache boundary to routes. `selection.rs` asks `litellm.rust_bridge.host.cache` for the live configured cache and owns route admission. A route's protocol carries cache operations through any host call type that converts from `CacheCall` `python/` delegates operations to the selected Python cache without discovering configuration. Routes use only the parent module's exports diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs index fe9e06b3d23..959c211b1d3 100644 --- a/litellm-rust/crates/python-bridge/src/cache/mod.rs +++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs @@ -2,7 +2,7 @@ mod python; mod selection; pub(crate) use python::{CacheCall, PythonCache}; -pub(crate) use selection::{Cached, Selection, configure, ensure_cache_disabled}; +pub(crate) use selection::{Selection, configure, ensure_cache_disabled}; use pyo3::{exceptions::PyRuntimeError, prelude::*}; diff --git a/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md b/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md index 3b516251f43..9aba11a3376 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md +++ b/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md @@ -1,6 +1,6 @@ # Python cache delegation -This directory lets Rust inference use a selected Python cache. `service.rs` implements the injected Rust response-cache service and yields typed cache operations. `host.rs` calls the Python cache's sync or async API and delivers the result back to Rust. +This directory lets Rust inference use a selected Python cache. `service.rs` implements the injected Rust response-cache service and yields typed cache operations. `host.rs` delivers each operation's result back to Rust. Asynchronous lookups and stream stores go through `litellm.rust_bridge.host.cache`, which reads with the `LLMCachingHandler` the `@client` wrapper uses, for the legacy function the route names, and writes with its dual cache. `configure` disables the cache for synchronous calls. Keep the shared inference protocol wrapper and adapter selection in the parent module. Receive the configured cache and prepared arguments from the parent module. Do not discover global configuration, choose native backends, or move inference to Python. Core remains independent of Python objects and cache implementation details diff --git a/litellm-rust/crates/python-bridge/src/cache/python/host.rs b/litellm-rust/crates/python-bridge/src/cache/python/host.rs index a8cbd0ce607..208f40a324b 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/host.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/host.rs @@ -1,6 +1,6 @@ use litellm_cache::Error; use litellm_host::protocol::Reply; -use litellm_host_python::{from_py, to_py}; +use litellm_host_python::{json_from_py, to_py}; use pyo3::{ gc::{PyTraverseError, PyVisit}, prelude::*, @@ -18,7 +18,9 @@ enum Pending { pub(crate) struct PythonCache { cache: Option>, + function: Option>, arguments: Option>, + read_handler: Option>, pending: Option, asynchronous: bool, deferred: bool, @@ -28,19 +30,27 @@ impl PythonCache { pub fn new(asynchronous: bool) -> Self { Self { cache: None, + function: None, arguments: None, + read_handler: None, pending: None, asynchronous, deferred: false, } } + pub(in crate::cache) fn asynchronous(&self) -> bool { + self.asynchronous + } + pub(in crate::cache) fn bind( &mut self, cache: Bound<'_, PyAny>, + function: &Bound<'_, PyAny>, arguments: &Bound<'_, PyDict>, ) { self.cache = Some(cache.unbind()); + self.function = Some(function.clone().unbind()); self.arguments = Some(arguments.clone().unbind()); } @@ -59,7 +69,7 @@ impl PythonCache { self.pending = Some(Pending::Lookup(reply)); ( if self.asynchronous { - "async_get_cache" + "lookup" } else { "get_cache" }, @@ -77,7 +87,7 @@ impl PythonCache { self.pending = Some(Pending::Store(reply)); ( if self.asynchronous { - "async_add_cache" + "store_stream" } else { "add_cache" }, @@ -86,9 +96,20 @@ impl PythonCache { } }; let result = match result { + Some(value) if self.asynchronous => py + .import(CACHE_HOST_MODULE)? + .getattr(method)? + .call((cache, value), Some(&arguments)), Some(value) => cache .bind(py) .call_method(method, (value,), Some(&arguments)), + None if self.asynchronous => py.import(CACHE_HOST_MODULE)?.getattr(method)?.call( + ( + cache, + self.function.as_ref().ok_or(HandleError::Unconfigured)?, + ), + Some(&arguments), + ), None => cache.bind(py).call_method(method, (), Some(&arguments)), } .map(Bound::unbind); @@ -98,7 +119,12 @@ impl PythonCache { self.resume(py, result) } - pub fn store_response(&mut self, py: Python<'_>, response: &Bound<'_, PyAny>) -> PyResult<()> { + pub fn store_response( + &mut self, + py: Python<'_>, + response: &Bound<'_, PyAny>, + verbatim: bool, + ) -> PyResult<()> { if !std::mem::take(&mut self.deferred) { return Ok(()); } @@ -106,11 +132,30 @@ impl PythonCache { return Err(HandleError::Unconfigured.into()); }; py.import(CACHE_HOST_MODULE)? - .getattr("store_response")? + .getattr(if verbatim { + "store_verbatim" + } else { + "store_response" + })? .call((cache, response), Some(&arguments.bind(py).copy()?))?; Ok(()) } + pub fn wrap_stream(&self, py: Python<'_>, stream: Py) -> PyResult> { + let (Some(cache), Some(function), Some(arguments)) = + (&self.cache, &self.function, &self.arguments) + else { + return Ok(stream); + }; + py.import(CACHE_HOST_MODULE)? + .getattr("wrap_stream")? + .call( + (cache, function, stream, &self.read_handler), + Some(&arguments.bind(py).copy()?), + ) + .map(Bound::unbind) + } + pub fn resume( &mut self, py: Python<'_>, @@ -125,11 +170,14 @@ impl PythonCache { }; match self.pending.take() { Some(Pending::Lookup(reply)) => { + if self.asynchronous { + self.read_handler = self.caching_handler(py).ok().flatten(); + } let value = result.map_err(Error::unavailable).and_then(|value| { if value.bind(py).is_none() { Ok(None) } else { - from_py(value.bind(py)) + json_from_py(value.bind(py)) .map(Some) .map_err(Error::invalid_entry) } @@ -144,15 +192,30 @@ impl PythonCache { Ok(None) } + fn caching_handler(&self, py: Python<'_>) -> PyResult>> { + let Some(arguments) = self.arguments.as_ref() else { + return Ok(None); + }; + let handler = py + .import(CACHE_HOST_MODULE)? + .getattr("caching_handler")? + .call((), Some(arguments.bind(py)))?; + Ok((!handler.is_none()).then(|| handler.unbind())) + } + pub fn close(&mut self) { self.pending = None; + self.read_handler = None; self.deferred = false; self.cache = None; + self.function = None; self.arguments = None; } pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.cache)?; - visit.call(&self.arguments) + visit.call(&self.function)?; + visit.call(&self.arguments)?; + visit.call(&self.read_handler) } } diff --git a/litellm-rust/crates/python-bridge/src/cache/python/service.rs b/litellm-rust/crates/python-bridge/src/cache/python/service.rs index 018c8647043..bd54accc0dc 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/service.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/service.rs @@ -55,11 +55,12 @@ struct PythonCacheService { config: ResponseCacheConfig, } -pub(in crate::cache) fn service>( +pub(in crate::cache) fn service( services: HostServices

, namespace: String, ) -> std::sync::Arc where + P::HostCall: From, P::Error: From + std::error::Error, { std::sync::Arc::new(PythonCacheService { @@ -71,14 +72,19 @@ where }) } -impl> ResponseCacheService for PythonCacheService

+impl ResponseCacheService for PythonCacheService

where + P::HostCall: From, P::Error: From + std::error::Error, { fn config(&self) -> &ResponseCacheConfig { &self.config } + fn keyed_by_caller(&self) -> bool { + true + } + fn lookup<'a>( &'a self, _: &'a ResponseCacheRequest, @@ -86,7 +92,7 @@ where ) -> Pin, Error>> + Send + 'a>> { Box::pin(async move { self.services - .call(|reply| CacheCall::Lookup { reply }) + .call(|reply| CacheCall::Lookup { reply }.into()) .await .map_err(Error::unavailable)?? .map(from_python) @@ -104,10 +110,13 @@ where let value = to_python(value)?; let unary = value.get(STREAM_EVENTS_KEY).is_none(); self.services - .call(|reply| CacheCall::Store { - value, - unary, - reply, + .call(|reply| { + CacheCall::Store { + value, + unary, + reply, + } + .into() }) .await .map_err(Error::unavailable)? diff --git a/litellm-rust/crates/python-bridge/src/cache/selection.rs b/litellm-rust/crates/python-bridge/src/cache/selection.rs index 78c1606ea40..6e4d222a1d0 100644 --- a/litellm-rust/crates/python-bridge/src/cache/selection.rs +++ b/litellm-rust/crates/python-bridge/src/cache/selection.rs @@ -6,17 +6,6 @@ use litellm_host::{ }; use pyo3::{prelude::*, types::PyDict}; -pub(crate) struct Cached

(std::marker::PhantomData

); - -impl Protocol for Cached

{ - type Request = (P::Request, Selection); - type Response = P::Response; - type Error = P::Error; - type HostCall = python::CacheCall; - type Chunk = P::Chunk; - type StreamHead = P::StreamHead; -} - enum Backend { Disabled, Python { namespace: String }, @@ -28,11 +17,12 @@ pub(crate) struct Selection { } impl Selection { - pub(crate) fn attach>( + pub(crate) fn attach( self, services: HostServices

, +} + +impl PythonTransport

{ + pub(super) fn new(services: HostServices

) -> Self { + Self { services } + } +} + +impl

Transport for PythonTransport

+where + P: Protocol, + P::HostCall: From, +{ + fn send( + &self, + request: TransportRequest, + ) -> BoxFuture<'static, Result> { + let services = self.services.clone(); + Box::pin(async move { + let sent = services + .call(|reply| TransportCall::Send { request, reply }.into()) + .await??; + let delivered = match sent.body { + SentBody::Text(text) => Delivered::Complete(text), + SentBody::Rejected { text, error } => Delivered::Rejected { text, error }, + SentBody::Stream => Delivered::Stream( + futures_util::stream::try_unfold(services, |services| async move { + let chunk = services + .call(|reply| TransportCall::Read { reply }.into()) + .await??; + Ok(chunk.map(|chunk| (chunk, services))) + }) + .boxed(), + ), + }; + Ok(TransportResponse { + status: sent.status, + headers: sent.headers, + delivered, + }) + }) + } +} + +#[derive(thiserror::Error)] +#[error("the Python HTTP client failed")] +struct HostTransportFailure { + error: Py, + raw: bool, +} + +impl fmt::Debug for HostTransportFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("HostTransportFailure") + } +} + +fn failed(py: Python<'_>, error: PyErr, raw: bool) -> Error { + Error::transport(HostTransportFailure { + error: error.into_value(py), + raw, + }) +} + +pub(super) fn python_failure( + py: Python<'_>, + error: &(dyn std::error::Error + 'static), +) -> Option<(PyErr, bool)> { + litellm_error::find::(error).map(|failure| { + ( + PyErr::from_value(failure.error.clone_ref(py).into_bound(py).into_any()), + failure.raw, + ) + }) +} + +enum Pending { + Send(Reply>), + Read(Reply, Error>>), +} + +#[derive(Default)] +pub(super) struct HostedTransport { + handler: Option>, + chunks: Option>, + pending: Option, +} + +impl HostedTransport { + pub(super) fn select(&mut self, py: Python<'_>, request: &Bound<'_, PyDict>) -> PyResult { + let handler = py + .import(ROUTE_HOST_MODULE)? + .getattr("transport_handler")? + .call1((request,))?; + self.handler = (!handler.is_none()).then(|| handler.unbind()); + Ok(self.handler.is_some()) + } + + pub(super) fn pending(&self) -> bool { + self.pending.is_some() + } + + pub(super) fn begin( + &mut self, + py: Python<'_>, + call: TransportCall, + request: &Bound<'_, PyDict>, + ) -> PyResult>> { + match call { + TransportCall::Send { + request: sent, + reply, + } => { + let handler = self + .handler + .as_ref() + .ok_or(litellm_host_python::Error::MissingState)?; + let body = match &sent.body { + TransportBody::Json(body) => json_to_py(py, body)?.unbind(), + TransportBody::Signed(bytes) => PyBytes::new(py, bytes).into_any().unbind(), + }; + let awaitable = py + .import(ROUTE_HOST_MODULE)? + .getattr("transport_send")? + .call1(( + handler, + request, + &sent.url, + to_py(py, &sent.headers)?, + body, + sent.stream, + ))?; + self.pending = Some(Pending::Send(reply)); + Ok(Some(awaitable.unbind())) + } + TransportCall::Read { reply } => { + let Some(chunks) = self.chunks.as_ref() else { + reply.send(Ok(None)); + return Ok(None); + }; + let awaitable = chunks.bind(py).call_method0("__anext__")?; + self.pending = Some(Pending::Read(reply)); + Ok(Some(awaitable.unbind())) + } + } + } + + pub(super) fn resume( + &mut self, + py: Python<'_>, + result: PyResult>, + ) -> PyResult>> { + let result = match result { + Err(error) if !error.is_instance_of::(py) => { + self.pending = None; + return Err(error); + } + result => result, + }; + match self.pending.take() { + Some(Pending::Send(reply)) => { + reply.send(match result { + Ok(sent) => self.sent(py, sent.bind(py))?, + Err(error) => Err(self.send_failure(py, error)?), + }); + } + Some(Pending::Read(reply)) => reply.send(match result { + Ok(chunk) => Ok(Some(Bytes::copy_from_slice( + chunk.bind(py).cast::()?.as_bytes(), + ))), + Err(error) if error.is_instance_of::(py) => { + self.chunks = None; + Ok(None) + } + Err(error) => { + self.chunks = None; + Err(failed(py, error, true)) + } + }), + None => return Err(litellm_host_python::Error::MissingState.into()), + } + Ok(None) + } + + fn send_failure(&self, py: Python<'_>, error: PyErr) -> PyResult { + let raw = py + .import(ROUTE_HOST_MODULE)? + .getattr("RawTransportFailure")?; + if error.value(py).is_instance(&raw)? { + let raised = error.value(py).getattr("error")?; + return Ok(failed(py, PyErr::from_value(raised), true)); + } + Ok(failed(py, error, false)) + } + + fn sent(&mut self, py: Python<'_>, sent: &Bound<'_, PyAny>) -> PyResult> { + let status: u16 = sent.getattr("status")?.extract()?; + let headers: Vec<(String, String)> = sent.getattr("headers")?.extract()?; + let text: Option = sent.getattr("text")?.extract()?; + let rejection = sent.getattr("rejection")?; + let chunks = sent.getattr("chunks")?; + let body = if !rejection.is_none() { + SentBody::Rejected { + text: text.unwrap_or_default(), + error: failed(py, PyErr::from_value(rejection), false), + } + } else if !chunks.is_none() { + self.chunks = Some(chunks.unbind()); + SentBody::Stream + } else { + SentBody::Text(text.unwrap_or_default()) + }; + Ok(Ok(Sent { + status, + headers, + body, + })) + } + + pub(super) fn close(&mut self) { + self.pending = None; + self.chunks = None; + self.handler = None; + } + + pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.handler)?; + visit.call(&self.chunks) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/workload_identity.rs b/litellm-rust/crates/python-bridge/src/routes/messages/workload_identity.rs new file mode 100644 index 00000000000..b8824d96584 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/workload_identity.rs @@ -0,0 +1,74 @@ +use std::fmt; + +use futures_util::future::BoxFuture; +use litellm_host_python::{PythonContext, attach_blocking, to_py}; +use litellm_inference_messages::WorkloadIdentity; +use litellm_router_types::{ExtraParams, LitellmParams}; +use litellm_secret_value::SecretValue; +use pyo3::{exceptions::PyBaseException, prelude::*}; + +const MODULE: &str = "litellm.rust_bridge.host.routes.messages"; + +pub(super) struct PythonWorkloadIdentity { + context: PythonContext, +} + +impl PythonWorkloadIdentity { + pub(super) fn new(context: PythonContext) -> Self { + Self { context } + } +} + +impl WorkloadIdentity for PythonWorkloadIdentity { + fn token<'a>( + &'a self, + litellm_params: &'a LitellmParams, + api_base: Option<&'a str>, + ) -> BoxFuture<'a, Result, litellm_auth::Error>> { + let params = litellm_params.extra.clone(); + let api_base = api_base.map(str::to_owned); + let context = self.context.clone(); + Box::pin(async move { + attach_blocking(context, move |py| { + mint(py, ¶ms, api_base).map_err(|error| raised(py, error)) + }) + .await + .map_err(|error| Python::attach(|py| raised(py, error))) + .and_then(|minted| minted) + .map(|token| token.map(SecretValue::new)) + }) + } +} + +fn mint( + py: Python<'_>, + params: &ExtraParams, + api_base: Option, +) -> PyResult> { + py.import(MODULE)? + .getattr("workload_identity_token")? + .call1((to_py(py, &**params)?, api_base))? + .extract() +} + +#[derive(thiserror::Error)] +#[error("Python workload identity federation failed")] +struct PythonWorkloadIdentityError(Py); + +impl fmt::Debug for PythonWorkloadIdentityError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("PythonWorkloadIdentityError") + } +} + +fn raised(py: Python<'_>, error: PyErr) -> litellm_auth::Error { + litellm_auth::Error::provider(PythonWorkloadIdentityError(error.into_value(py))) +} + +pub(crate) fn python_error( + py: Python<'_>, + error: &(dyn std::error::Error + 'static), +) -> Option { + litellm_error::find::(error) + .map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any())) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index f51c755c33e..9fcd792bdaa 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -166,6 +166,38 @@ fn run_public_call( hooks: impl PythonCallHooks + 'static, asynchronous: bool, ) -> PyResult> +where + H: PythonBinding + PythonHostCalls + 'static, + M: Machine + 'static, + M::Complete: Into::Response>>, +{ + run_public_call_after_policy( + py, + arguments, + start, + host, + hooks, + HookChain::new(), + asynchronous, + ) +} + +fn run_public_call_after_policy( + py: Python<'_>, + arguments: Py, + start: impl FnOnce( + Python<'_>, + &Bound<'_, PyDict>, + ::Request, + ) -> PyResult + + Send + + Sync + + 'static, + host: H, + hooks: impl PythonCallHooks + 'static, + after_policy: impl PythonCallHooks + 'static, + asynchronous: bool, +) -> PyResult> where H: PythonBinding + PythonHostCalls + 'static, M: Machine + 'static, @@ -179,7 +211,8 @@ where host, HookChain::new() .with(hooks) - .with(crate::preflight::SdkPolicy), + .with(crate::preflight::SdkPolicy) + .with(after_policy), arguments, CallOptions::new(asynchronous, execution_steps), ) diff --git a/litellm-rust/crates/python-compat/src/lib.rs b/litellm-rust/crates/python-compat/src/lib.rs index 2cc0eeac397..e23dbdb285e 100644 --- a/litellm-rust/crates/python-compat/src/lib.rs +++ b/litellm-rust/crates/python-compat/src/lib.rs @@ -7,6 +7,7 @@ //! |---|---| //! | [`literal`] | `ast.literal_eval(text)` | //! | [`repr`] | `repr(value)` and `str(value)` | +//! | [`reprlib`] | `reprlib.repr(value)` | //! | [`json`] | `json.dumps(value)` and `json.loads(text)` | //! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data | //! | [`truthy`] | `bool(value)` | @@ -29,6 +30,7 @@ pub mod literal; pub mod number; pub mod pickle; pub mod repr; +pub mod reprlib; pub mod truthy; mod value; diff --git a/litellm-rust/crates/python-compat/src/reprlib.rs b/litellm-rust/crates/python-compat/src/reprlib.rs new file mode 100644 index 00000000000..5d04b4e1ca7 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/reprlib.rs @@ -0,0 +1,179 @@ +use crate::{Value, repr::repr}; + +const MAX_LEVEL: usize = 6; +const MAX_TUPLE: usize = 6; +const MAX_LIST: usize = 6; +const MAX_SET: usize = 6; +const MAX_DICT: usize = 4; +const MAX_STRING: usize = 30; +const MAX_LONG: usize = 40; +const MAX_OTHER: usize = 30; +const FILL: &str = "..."; + +pub fn reprlib(value: &Value) -> String { + repr1(value, MAX_LEVEL) +} + +fn repr1(value: &Value, level: usize) -> String { + match value { + Value::Str(text) => repr_str(text), + Value::Int(_) => shortened(repr(value), MAX_LONG), + Value::Tuple(values) => repr_iterable(values, level, ("(", ")"), MAX_TUPLE, ","), + Value::List(values) => repr_iterable(values, level, ("[", "]"), MAX_LIST, ""), + Value::Set(values) if values.is_empty() => "set()".to_owned(), + Value::Set(values) => repr_iterable(&sorted(values), level, ("{", "}"), MAX_SET, ""), + Value::Dict(entries) => repr_dict(entries, level), + Value::None + | Value::Bool(_) + | Value::Float(_) + | Value::Complex { .. } + | Value::Bytes(_) => shortened(repr(value), MAX_OTHER), + } +} + +fn shortened(text: String, limit: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= limit { + return text; + } + let head = (limit - FILL.len()) / 2; + let tail = limit - FILL.len() - head; + let mut out: String = chars[..head].iter().collect(); + out.push_str(FILL); + out.extend(&chars[chars.len() - tail..]); + out +} + +fn repr_str(text: &str) -> String { + let chars: Vec = text.chars().collect(); + let quoted = repr(&Value::Str(chars.iter().take(MAX_STRING).collect())); + if quoted.chars().count() <= MAX_STRING { + return quoted; + } + let head = (MAX_STRING - FILL.len()) / 2; + let tail = MAX_STRING - FILL.len() - head; + let kept: String = chars[..head.min(chars.len())] + .iter() + .chain(&chars[chars.len().saturating_sub(tail)..]) + .collect(); + let quoted: Vec = repr(&Value::Str(kept)).chars().collect(); + let mut out: String = quoted[..head].iter().collect(); + out.push_str(FILL); + out.extend("ed[quoted.len() - tail..]); + out +} + +fn repr_iterable( + values: &[Value], + level: usize, + (left, right): (&str, &str), + max_items: usize, + trail: &str, +) -> String { + if level == 0 && !values.is_empty() { + return format!("{left}{FILL}{right}"); + } + let mut pieces: Vec = values + .iter() + .take(max_items) + .map(|value| repr1(value, level - 1)) + .collect(); + if values.len() > max_items { + pieces.push(FILL.to_owned()); + } + let trail = if values.len() == 1 { trail } else { "" }; + format!("{left}{}{trail}{right}", pieces.join(", ")) +} + +fn repr_dict(entries: &[(Value, Value)], level: usize) -> String { + if entries.is_empty() { + return "{}".to_owned(); + } + if level == 0 { + return format!("{{{FILL}}}"); + } + let mut pieces: Vec = sorted_entries(entries) + .into_iter() + .take(MAX_DICT) + .map(|(key, value)| format!("{}: {}", repr1(key, level - 1), repr1(value, level - 1))) + .collect(); + if entries.len() > MAX_DICT { + pieces.push(FILL.to_owned()); + } + format!("{{{}}}", pieces.join(", ")) +} + +fn sorted_entries(entries: &[(Value, Value)]) -> Vec<&(Value, Value)> { + let mut ordered: Vec<&(Value, Value)> = entries.iter().collect(); + if entries.iter().all(|(key, _)| matches!(key, Value::Str(_))) { + ordered.sort_by(|(left, _), (right, _)| match (left, right) { + (Value::Str(left), Value::Str(right)) => left.cmp(right), + _ => std::cmp::Ordering::Equal, + }); + } + ordered +} + +fn sorted(values: &[Value]) -> Vec { + let mut ordered = values.to_vec(); + if values.iter().all(|value| matches!(value, Value::Str(_))) { + ordered.sort_by(|left, right| match (left, right) { + (Value::Str(left), Value::Str(right)) => left.cmp(right), + _ => std::cmp::Ordering::Equal, + }); + } + ordered +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn list(values: impl IntoIterator) -> Value { + Value::List(values.into_iter().collect()) + } + + fn dict(entries: &[(&str, Value)]) -> Value { + Value::Dict( + entries + .iter() + .map(|(key, value)| (Value::from(*key), value.clone())) + .collect(), + ) + } + + fn nested(depth: usize) -> Value { + (0..depth).fold(list([Value::from(1)]), |inner, _| list([inner])) + } + + #[rstest] + #[case::none(Value::None, "None")] + #[case::bool(Value::Bool(true), "True")] + #[case::int(Value::from(3), "3")] + #[case::wide_int(Value::Int("1".repeat(45).parse().unwrap()), "111111111111111111...1111111111111111111")] + #[case::float(Value::Float(0.1), "0.1")] + #[case::long_float(Value::Float(1.0 / 3.0), "0.3333333333333333")] + #[case::nan(Value::Float(f64::NAN), "nan")] + #[case::short_str(Value::from("high"), "'high'")] + #[case::long_str( + Value::from("abcdefghijklmnopqrstuvwxyz0123456789"), + "'abcdefghijkl...xyz0123456789'" + )] + #[case::quoted_str( + Value::from("it's a fairly long string with a quote"), + "\"it's a fairl... with a quote\"" + )] + #[case::list(list([Value::from(1)]), "[1]")] + #[case::long_list(list((0..8).map(Value::from)), "[0, 1, 2, 3, 4, 5, ...]")] + #[case::one_tuple(Value::Tuple(vec![Value::from(1)]), "(1,)")] + #[case::sorted_dict(dict(&[("b", Value::from(2)), ("a", Value::from(1))]), "{'a': 1, 'b': 2}")] + #[case::long_dict(dict(&[("e", Value::None), ("d", Value::None), ("c", Value::None), ("b", Value::None), ("a", Value::None)]), "{'a': None, 'b': None, 'c': None, 'd': None, ...}")] + #[case::empty_dict(dict(&[]), "{}")] + #[case::deep(nested(6), "[[[[[[[...]]]]]]]")] + #[case::deep_dict(dict(&[("a", dict(&[("b", dict(&[("c", dict(&[("d", dict(&[("e", dict(&[("f", dict(&[("g", Value::None)]))]))]))]))]))]))]), "{'a': {'b': {'c': {'d': {'e': {'f': {...}}}}}}}")] + fn values_shorten_as_reprlib_shortens_them(#[case] value: Value, #[case] expected: &str) { + assert_eq!(reprlib(&value), expected); + } +} diff --git a/litellm-rust/crates/router/src/lib.rs b/litellm-rust/crates/router/src/lib.rs index b75c1a14446..6955b8a927a 100644 --- a/litellm-rust/crates/router/src/lib.rs +++ b/litellm-rust/crates/router/src/lib.rs @@ -31,6 +31,8 @@ impl Router { .model_list .iter() .map(|model| { + let drop_params = + drops_params_globally || is_on(model.litellm_params.drop_params.as_ref(), env); ( model.model_name.clone(), Deployment { @@ -45,8 +47,8 @@ impl Router { litellm_params: model.litellm_params.clone(), shaping: MessagesShaping { settings: MessagesSettings { - drop_params: drops_params_globally - || is_on(model.litellm_params.drop_params.as_ref(), env), + drop_params, + drops_unsupported_params: drop_params, ..MessagesSettings::default() }, ..MessagesShaping::default() @@ -113,15 +115,18 @@ mod tests { )) .unwrap(); + let settings = Router::from_config(&config, &|name: &str| { + (name == "DROP_PARAMS_FLAG").then(|| " Yes ".to_owned()) + }) + .get("public-model") + .unwrap() + .shaping + .settings + .clone(); + assert_eq!( - Router::from_config(&config, &|name: &str| (name == "DROP_PARAMS_FLAG") - .then(|| " Yes ".to_owned())) - .get("public-model") - .unwrap() - .shaping - .settings - .drop_params, - expected + (settings.drop_params, settings.drops_unsupported_params), + (expected, expected) ); } } diff --git a/litellm/__init__.py b/litellm/__init__.py index bfe0e2b5f3e..8a48b0812d9 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -27,7 +27,7 @@ from typing import ( from collections.abc import Mapping from litellm.types.integrations.datadog import DatadogInitParams from litellm.types.integrations.newrelic import NewRelicInitParams -from litellm.litellm_core_utils.core_helpers import drop_params_env_flag +from litellm.litellm_core_utils.core_helpers import drop_params_env_flag, env_flag from litellm.types.integrations.pointfive import PointFiveInitParams from litellm.types.integrations.zerobus import ZerobusInitParams from litellm._logging import ( @@ -227,7 +227,7 @@ token: Optional[str] = ( ) max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = drop_params_env_flag(os.environ, verbose_logger) -modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) +modify_params = env_flag(os.environ, "LITELLM_MODIFY_PARAMS", verbose_logger) bedrock_neutralize_orphaned_tool_blocks: bool = True use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index fd72bf7d2e7..b93c88cb935 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -49,7 +49,6 @@ _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig) _DECODED_JSON: Final = TypeAdapter(object) BetaPlacement: TypeAlias = Literal["header", "body"] -UnlistedBetas: TypeAlias = Literal["drop", "forward"] _HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only" @@ -281,13 +280,12 @@ def filter_and_transform_beta_headers( beta_headers: list[str], provider: str, placement: BetaPlacement = "header", - unlisted: UnlistedBetas = "drop", ) -> list[str]: """ Filter and transform beta headers based on provider's mapping configuration. This function: - 1. Only allows headers that are present in the provider's mapping keys, unless ``unlisted`` is "forward" + 1. Only allows headers that are present in the provider's mapping keys 2. Filters out headers with null values (unsupported) 3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool) @@ -295,7 +293,6 @@ def filter_and_transform_beta_headers( beta_headers: List of Anthropic beta header values provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai") placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field - unlisted: Whether a beta missing from the mapping is dropped or forwarded as sent Returns: List of filtered and transformed beta headers for the provider @@ -310,10 +307,6 @@ def filter_and_transform_beta_headers( for header in beta_headers: header = header.strip() - if header not in provider_mapping and unlisted == "forward": - filtered_headers.add(header) - continue - # Check if header is in the mapping if header not in provider_mapping: verbose_logger.debug( @@ -426,7 +419,6 @@ def update_headers_with_filtered_beta( headers: dict, provider: str, api_base: str | None = None, - unlisted: UnlistedBetas = "drop", keep: frozenset[str] = frozenset(), ) -> dict: """ @@ -437,7 +429,6 @@ def update_headers_with_filtered_beta( headers: Request headers dict (will be modified in place) provider: Provider name api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``) - unlisted: Whether a beta missing from the row is dropped or forwarded as sent keep: Betas the routed model accepts, forwarded as sent whatever the row says Returns: @@ -457,7 +448,6 @@ def update_headers_with_filtered_beta( *filter_and_transform_beta_headers( beta_headers=[b for b in beta_values if b not in keep], provider=resolve_beta_provider(provider, api_base), - unlisted=unlisted, ), *(b for b in beta_values if b in keep), ] diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index a3cf54916f1..1e89b700137 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -19,6 +19,7 @@ import datetime import inspect import time from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeVar from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError @@ -80,6 +81,19 @@ from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper EmbeddingCacheInputElement = str | list[int] | ChatCompletionFileObject +@dataclass(frozen=True, slots=True) +class CacheRead: + kwargs: dict[str, object] # mutable-ok: the @client call kwargs the embedding path keeps reading and rewriting + checked: bool = False + cached_result: object | None = None + model: str = "" + end_time: datetime.datetime | None = None + + @property + def hit(self) -> bool: + return self.end_time is not None + + class CachingHandlerResponse(LiteLLMBaseModel): """ This is the response object for the caching handler. We need to separate embedding cached responses and (completion / text_completion / transcription) cached responses @@ -313,6 +327,10 @@ def _request_cache_key(request_kwargs: Mapping[str, Any]) -> str | None: return request_kwargs.get("cache_key", None) +def _is_cache_miss(cached_result: object) -> bool: + return cached_result is None or isinstance(cached_result, list) + + def _set_cached_hidden_param(response: object, key: str, value: object) -> None: if isinstance(response, TextCompletionResponse): setattr(response.hidden_params, key, value) @@ -406,121 +424,146 @@ class LLMCachingHandler: Raises: None """ + if not self.reads_cache(model=model, kwargs=kwargs): + return None + args = args or () + final_embedding_cached_response: EmbeddingResponse | None = None + embedding_all_elements_cache_hit: bool = False + read: Final = await self.async_read_cache( + model=model, + original_function=original_function, + logging_obj=logging_obj, + call_type=call_type, + kwargs=kwargs, + args=args, + ) + kwargs = read.kwargs + cached_result: object | None = read.cached_result + if litellm.cache is not None and read.checked: + if read.end_time is not None: + cached_result = self._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=original_function.__name__, + kwargs=kwargs, + logging_obj=logging_obj, + model=read.model, + custom_llm_provider=kwargs.get("custom_llm_provider", None), + args=args, + ) + if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result): + # LOG SUCCESS + self._async_log_cache_hit_on_callbacks( + logging_obj=logging_obj, + cached_result=cached_result, + start_time=start_time, + end_time=read.end_time, + cache_hit=True, + ) + _set_cached_hidden_param(cached_result, "cache_key", self.hit_cache_key()) + return CachingHandlerResponse(cached_result=cached_result) + elif ( + call_type == CallTypes.aembedding.value + and cached_result is not None + and isinstance(cached_result, list) + and litellm.cache is not None + and not isinstance(litellm.cache.cache, S3Cache) # s3 doesn't support bulk writing. Exclude. + ): + ( + final_embedding_cached_response, + embedding_all_elements_cache_hit, + ) = self._process_async_embedding_cached_response( + final_embedding_cached_response=final_embedding_cached_response, + cached_result=cached_result, + kwargs=kwargs, + logging_obj=logging_obj, + start_time=start_time, + model=model, + ) + return CachingHandlerResponse( + final_embedding_cached_response=final_embedding_cached_response, + embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, + embedding_uncached_input=self.handle_kwargs_input_list_or_str(kwargs), + ) + + verbose_logger.debug("CACHE RESULT: %s", cached_result) + return CachingHandlerResponse( + cached_result=cached_result, + final_embedding_cached_response=final_embedding_cached_response, + ) + + _async_get_cache = async_get_cache + + def reads_cache( + self, + model: str, + kwargs: dict[str, Any], # pyright: ignore[reportExplicitAny] # mutable-ok: the @client call kwargs, read as the wrapper reads them + ) -> bool: if _is_response_cache_excluded(model=model, kwargs=kwargs): self.request_kwargs = _drop_logging_obj_from_kwargs(kwargs) - return None - - # Check if caching should be performed BEFORE doing expensive operations - if ( + return False + return ( (kwargs.get("caching", None) is None and litellm.cache is not None) or kwargs.get("caching", False) is True ) and ( not _cache_reads_disabled(kwargs) - ): # allow users to control returning cached responses from the completion function - args = args or () - final_embedding_cached_response: EmbeddingResponse | None = None - embedding_all_elements_cache_hit: bool = False - cached_result: object | None = None - kwargs = kwargs.copy() - ######################################################### - # Init cache timing metrics - ######################################################### - cache_check_start_time: Final = time.perf_counter() - cache_check_end_time: float | None = None - ######################################################### - parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - kwargs["parent_otel_span"] = parent_otel_span + ) # allow users to control returning cached responses from the completion function - if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function): - verbose_logger.debug("Checking Async Cache") - logging_obj.reset_caching_details() - cached_result = await self._retrieve_from_cache( - call_type=call_type, - kwargs=kwargs, - args=args, - ) - cache_check_end_time = time.perf_counter() + async def async_read_cache( + self, + model: str, + original_function: Callable[..., object], + logging_obj: LiteLLMLoggingObj, + call_type: str, + kwargs: Mapping[str, Any], # pyright: ignore[reportExplicitAny] # the @client call kwargs, read as the wrapper reads them + args: tuple[object, ...], + ) -> CacheRead: + cache_check_start_time: Final = time.perf_counter() + read_kwargs: Final = dict(kwargs) + read_kwargs["parent_otel_span"] = get_parent_otel_span_from_kwargs(read_kwargs) + if litellm.cache is None or not self._is_call_type_supported_by_cache(original_function=original_function): + return CacheRead(kwargs=read_kwargs) + verbose_logger.debug("Checking Async Cache") + logging_obj.reset_caching_details() + cached_result: Final = await self._retrieve_from_cache( + call_type=call_type, + kwargs=read_kwargs, + args=args, + ) + cache_check_end_time: Final = time.perf_counter() + if _is_cache_miss(cached_result): + return CacheRead(kwargs=read_kwargs, checked=True, cached_result=cached_result) + verbose_logger.debug("Cache Hit!") + end_time: Final = datetime.datetime.now() + provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, + custom_llm_provider=read_kwargs.get("custom_llm_provider", None), + api_base=read_kwargs.get("api_base", None), + api_key=read_kwargs.get("api_key", None), + ) + self._update_litellm_logging_obj_environment( + logging_obj=logging_obj, + model=provider_model, + kwargs=read_kwargs, + cached_result=cached_result, + is_async=True, + custom_llm_provider=custom_llm_provider, + cache_duration_ms=(cache_check_end_time - cache_check_start_time) * 1000, + ) + return CacheRead( + kwargs=read_kwargs, + checked=True, + cached_result=cached_result, + model=provider_model, + end_time=end_time, + ) - if cached_result is not None and not isinstance(cached_result, list): - verbose_logger.debug("Cache Hit!") - cache_hit: Final = True - end_time: Final = datetime.datetime.now() - provider_model, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, - custom_llm_provider=kwargs.get("custom_llm_provider", None), - api_base=kwargs.get("api_base", None), - api_key=kwargs.get("api_key", None), - ) - cache_duration_ms: Final = (cache_check_end_time - cache_check_start_time) * 1000 - self._update_litellm_logging_obj_environment( - logging_obj=logging_obj, - model=provider_model, - kwargs=kwargs, - cached_result=cached_result, - is_async=True, - custom_llm_provider=custom_llm_provider, - cache_duration_ms=cache_duration_ms, - ) - - call_type = original_function.__name__ - - cached_result = self._convert_cached_result_to_model_response( - cached_result=cached_result, - call_type=call_type, - kwargs=kwargs, - logging_obj=logging_obj, - model=provider_model, - custom_llm_provider=kwargs.get("custom_llm_provider", None), - args=args, - ) - if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result): - # LOG SUCCESS - self._async_log_cache_hit_on_callbacks( - logging_obj=logging_obj, - cached_result=cached_result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, - ) - cache_key: Final = ( - self.preset_cache_key - or self.request_kwargs.get("cache_key") - or litellm.cache.get_cache_key(**self.request_kwargs) - ) - _set_cached_hidden_param(cached_result, "cache_key", cache_key) - return CachingHandlerResponse(cached_result=cached_result) - elif ( - call_type == CallTypes.aembedding.value - and cached_result is not None - and isinstance(cached_result, list) - and litellm.cache is not None - and not isinstance(litellm.cache.cache, S3Cache) # s3 doesn't support bulk writing. Exclude. - ): - ( - final_embedding_cached_response, - embedding_all_elements_cache_hit, - ) = self._process_async_embedding_cached_response( - final_embedding_cached_response=final_embedding_cached_response, - cached_result=cached_result, - kwargs=kwargs, - logging_obj=logging_obj, - start_time=start_time, - model=model, - ) - return CachingHandlerResponse( - final_embedding_cached_response=final_embedding_cached_response, - embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, - embedding_uncached_input=self.handle_kwargs_input_list_or_str(kwargs), - ) - - verbose_logger.debug("CACHE RESULT: %s", cached_result) - return CachingHandlerResponse( - cached_result=cached_result, - final_embedding_cached_response=final_embedding_cached_response, - ) - # Caching disabled - return None to indicate no caching attempted - return None - - _async_get_cache = async_get_cache + def hit_cache_key(self) -> object: + if litellm.cache is None: + return None + return ( + self.preset_cache_key + or self.request_kwargs.get("cache_key") + or litellm.cache.get_cache_key(**self.request_kwargs) + ) def sync_get_cache( self, @@ -893,14 +936,14 @@ class LLMCachingHandler: _combine_cached_embedding_response_with_api_result = combine_cached_embedding_response_with_api_result - def _async_log_cache_hit_on_callbacks( + def async_log_cache_hit_on_callbacks( self, logging_obj: LiteLLMLoggingObj, cached_result: object, start_time: datetime.datetime, end_time: datetime.datetime, cache_hit: bool, - ): + ) -> None: """ Helper function to log the success of a cached result on callbacks @@ -941,6 +984,8 @@ class LLMCachingHandler: cache_hit=cache_hit, ) + _async_log_cache_hit_on_callbacks = async_log_cache_hit_on_callbacks + async def _retrieve_from_cache( self, call_type: str, kwargs: dict[str, object], args: tuple[object, ...] ) -> object | None: diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 71b20f92cf7..70cb517168c 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -259,19 +259,21 @@ def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool DROP_PARAMS_ENV_VAR: Final = "LITELLM_DROP_PARAMS" -def drop_params_env_flag(environ: Mapping[str, str], logger: logging.Logger) -> bool: - configured: Final = environ.get(DROP_PARAMS_ENV_VAR, "").strip() +def env_flag(environ: Mapping[str, str], name: str, logger: logging.Logger) -> bool: + configured: Final = environ.get(name, "").strip() if configured == "": return False - normalized: Final = normalize_drop_params(configured) + normalized: Final = lax_bool(configured) if normalized is None: - logger.warning( - "%s=%r is not a flag value, treating it as on. Set it to true or false", DROP_PARAMS_ENV_VAR, configured - ) + logger.warning("%s=%r is not a flag value, treating it as on. Set it to true or false", name, configured) return True return normalized +def drop_params_env_flag(environ: Mapping[str, str], logger: logging.Logger) -> bool: + return env_flag(environ, DROP_PARAMS_ENV_VAR, logger) + + def safe_divide( numerator: float, denominator: float, diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 08c3e0b48a8..16b03a711ac 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -467,6 +467,39 @@ def _short_circuit_safeguards_evaluator( return request_safeguards_evaluator(kwargs, messages) +def provider_after_pre_request_hooks( + model: str, custom_llm_provider: str | None, request_kwargs: Mapping[str, object] +) -> str | None: + if custom_llm_provider: + return custom_llm_provider + hook_params: Final = cast( # cast-ok: read as the hooks left it, so a non-mapping raises as Python's handler raises + Mapping[str, object], request_kwargs.get("litellm_params", {}) + ) + derived: Final = cast( # cast-ok: handed on unchecked, as Python's handler does + str | None, hook_params.get("custom_llm_provider") + ) + if derived: + return derived + try: + _, resolved, _, _ = litellm.get_llm_provider(model=model) + except Exception: # noqa: BLE001 # an unknown model keeps the provider the hooks left + return derived + return resolved + + +def tools_marked_for_cache_points( + tools: object, + request_kwargs: dict[str, object], # mutable-ok: pops the points and records the mark in the handler's kwargs +) -> Sequence[object] | None: + marked: Final = tools_marked_for_a_tool_config_point( + tools, request_kwargs.pop("cache_control_injection_points", None) + ) + if marked is None: + return None + AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1) + return list(marked) + + @client async def anthropic_messages( max_tokens: int, @@ -527,16 +560,7 @@ async def anthropic_messages( tool_choice = request_kwargs.pop("tool_choice", tool_choice) top_k = request_kwargs.pop("top_k", top_k) top_p = request_kwargs.pop("top_p", top_p) - # Propagate the provider derived inside pre-request hooks, if not already set. - # The litellm_params dict may have been overwritten by **kwargs in - # _execute_pre_request_hooks, so fall back to get_llm_provider() if needed. - if not custom_llm_provider: - custom_llm_provider = request_kwargs.get("litellm_params", {}).get("custom_llm_provider") - if not custom_llm_provider: - try: - _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) - except Exception: - pass + custom_llm_provider = provider_after_pre_request_hooks(model, custom_llm_provider, request_kwargs) # Remove litellm_params from kwargs (only needed for hooks) request_kwargs.pop("litellm_params", None) # Merge back any other modifications @@ -869,12 +893,8 @@ def anthropic_messages_handler( ) request_kwargs: Final = cast(dict[str, object], kwargs) # cast-ok: **kwargs of the handler is untyped - marked_tools: Final = tools_marked_for_a_tool_config_point( - local_vars.get("tools"), request_kwargs.pop("cache_control_injection_points", None) - ) - if marked_tools is not None: - AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1) - local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})}) + marked_tools: Final = tools_marked_for_cache_points(local_vars.get("tools"), request_kwargs) + local_vars.update({**request_kwargs, **({"tools": marked_tools} if marked_tools is not None else {})}) anthropic_messages_optional_request_params: Final = ( AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( params=local_vars, diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index b0558292271..c2c7244259b 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -105,7 +105,6 @@ _MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): - _workload_identity_eligible: ClassVar[bool] = True _report_reasoning_drops: ClassVar[bool] = True @property @@ -372,12 +371,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): self._apply_env_auth_header( headers, self._require_auth_header( - AnthropicModelInfo.get_auth_header( - api_key, - api_base=api_base, - litellm_params=litellm_params, - allow_workload_identity=self._allows_workload_identity, - ), + AnthropicModelInfo.get_auth_header(api_key, api_base=api_base), model=model, ), ) @@ -412,12 +406,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): self._apply_env_auth_header( oauth_headers, self._require_auth_header( - await AnthropicModelInfo.aget_auth_header( - oauth_api_key, - api_base=api_base, - litellm_params=litellm_params, - allow_workload_identity=self._allows_workload_identity, - ), + await AnthropicModelInfo.aget_auth_header(oauth_api_key, api_base=api_base), model=model, ), ) @@ -447,14 +436,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if merged_beta: headers["anthropic-beta"] = merged_beta - @property - def _allows_workload_identity(self) -> bool: - """Subclasses reuse this validate step for their own /v1/messages-compatible providers, so - eligibility is declared per class and never inherited.""" - from litellm.llms.anthropic.common_utils import config_allows_workload_identity - - return config_allows_workload_identity(self) - def finalize_messages_headers( self, headers: dict, # mutable-ok: out-param diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index a5b15a24a7f..b290a7abf3d 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -345,6 +345,10 @@ def _set_ciphers( ) +def rsa_key_exchange_and_chacha20_excluded() -> bool: + return _rsa_key_exchange_and_chacha20_excluded.is_set() + + def exclude_rsa_key_exchange_and_chacha20(excluded: bool) -> None: """ Turn RSA key exchange and TLS 1.2 ChaCha20 off or back on for new SSL contexts and every cached one, which clients diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index f65c4205944..68b13c6172f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -41,7 +41,6 @@ from litellm._internal_context import internal_sub_call from litellm._logging import redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import ( aload_beta_headers_config, - resolve_beta_provider, update_headers_with_filtered_beta, ) from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES @@ -82,7 +81,7 @@ from litellm.litellm_core_utils.realtime_errors import ( ) from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base -from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -2633,17 +2632,10 @@ class BaseLLMHTTPHandler: litellm_params.get("additional_drop_params") or (), anthropic_messages_optional_request_params, ) - affinity_api_base: Final = ( - AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base - ) - takes_another_beta_row: Final = ( - resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider - ) - provider_filters_betas: Final = anthropic_messages_provider_config.should_filter_anthropic_beta_headers() - filters_beta_headers: Final = provider_filters_betas or takes_another_beta_row + filters_beta_headers: Final = anthropic_messages_provider_config.should_filter_anthropic_beta_headers() if request_params.get("safeguards") is not None and filters_beta_headers: await aload_beta_headers_config() - default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) + default_affinity_header: Final = get_provider_affinity_header(api_base) ( headers, api_base, @@ -2666,8 +2658,6 @@ class BaseLLMHTTPHandler: headers = update_headers_with_filtered_beta( headers=headers, provider=custom_llm_provider, - api_base=affinity_api_base, - unlisted="drop" if provider_filters_betas else "forward", keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params), ) diff --git a/litellm/llms/dashscope/messages/transformation.py b/litellm/llms/dashscope/messages/transformation.py index 9d0d1db91c8..81f93a32166 100644 --- a/litellm/llms/dashscope/messages/transformation.py +++ b/litellm/llms/dashscope/messages/transformation.py @@ -2,7 +2,7 @@ DashScope Anthropic-compatible Messages transformation config. """ -from typing import ClassVar, Final +from typing import Final import litellm from litellm.exceptions import AuthenticationError @@ -25,8 +25,6 @@ class DashScopeAnthropicMessagesConfig(AnthropicMessagesConfig): ``dashscope.aliyuncs.com`` and ``{workspace}.{region}.maas.aliyuncs.com``. """ - _workload_identity_eligible: ClassVar[bool] = False - @property def custom_llm_provider(self) -> str | None: return "dashscope" diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index faa604a25ce..b974e677386 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -3,14 +3,14 @@ from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Itera from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable -from litellm.exceptions import BadRequestError +from litellm.exceptions import BadRequestError, UnsupportedParamsError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.anthropic.messages.native import handler as main from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.rust_bridge import detectors +from litellm.rust_bridge import catalog, detectors from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target +from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.dispatch import PublicDispatch -from litellm.rust_bridge.messages import detectors as route_detectors from litellm.rust_bridge.messages.entrypoints import ( NATIVE_AMESSAGES, NATIVE_MESSAGES, @@ -22,7 +22,6 @@ from litellm.rust_bridge.public_call import ( native_acall_hook, native_call, native_call_hook, - optional_sequence, optional_str, signature, ) @@ -66,7 +65,7 @@ def _native_kwargs(fields: Mapping[str, object], kwargs: Mapping[str, object]) - request_timeout=kwargs.get("request_timeout"), stream_timeout=kwargs.get("stream_timeout"), ), - stream=fields.get("stream") is True, + stream=bool(fields.get("stream")), custom_llm_provider=provider or "", ) if timeout is None: @@ -81,9 +80,7 @@ def _public_request( if fields is None: return None model: Final = fields.get("model") - messages: Final = optional_sequence(fields.get("messages")) - max_tokens: Final = fields.get("max_tokens") - if not isinstance(model, str) or messages is None or not isinstance(max_tokens, int): + if not isinstance(model, str): return None return native_call(legacy, args, _native_kwargs(fields, kwargs)) @@ -133,30 +130,20 @@ def messages_target(model: str, custom_llm_provider: str | None, model_info: obj return _route_target(model, resolved_model, resolved_provider, model_info, api_base, compacting=False) -_FEATURES: Final[detectors.Detectors] = MappingProxyType( +_NATIVE_COMMON_FEATURES: Final = frozenset( { - **{feature: detect for feature, detect in detectors.COMMON.items() if feature is not Feature.TIMEOUT_VARIANTS}, - Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity, - Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier, - Feature.RESPONSE_CACHE: detectors.response_cache, - Feature.MODIFY_PARAMS: detectors.modify_params, - Feature.WORKLOAD_IDENTITY: route_detectors.anthropic_workload_identity, - Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection, - Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection, - Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry, - Feature.TOOL_CUSTOM_FIELD: route_detectors.tool_custom_field, + Feature.TIMEOUT_VARIANTS, + Feature.LATEST_ALIAS, + Feature.MOCK_RESPONSE, + Feature.CALLER_CLIENT, + Feature.TLS_POLICY, + Feature.AGENTIC_LOOP_HOOK, + Feature.PROXY_MCP_TOOLS, } ) -_ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType( - { - **_FEATURES, - Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook, - Feature.FOUNDRY_BETA_ROW: route_detectors.foundry_beta_row, - Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception, - Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted, - Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry, - } +_FEATURES: Final[detectors.Detectors] = MappingProxyType( + {feature: detect for feature, detect in detectors.COMMON.items() if feature not in _NATIVE_COMMON_FEATURES} ) @@ -181,32 +168,87 @@ _DISPATCH: Final = PublicDispatch( _ADISPATCH: Final = PublicDispatch( route=Route.MESSAGES, request=lambda args, kwargs: _public_request(_AMESSAGES, args, kwargs), - context=lambda request: _context(request, _ASYNC_FEATURES), + context=lambda request: _context(request, _FEATURES), ) +def _unsupported_features(features: detectors.Detectors, request: NativeCall) -> tuple[str, ...]: + try: + return tuple(sorted(feature.value for feature in detectors.detected(features, request))) + except Exception: # noqa: BLE001 # a failing detector names nothing, and the call is refused all the same + return () + + +def _checked_target(request: NativeCall, model: str, provider: str | None) -> Target | None: + try: + return _target(request, model, provider) + except Exception: # noqa: BLE001 # the provider alone still decides whether the call has a Python path + return None + + +def _rust_only_refusal( + legacy: inspect.Signature, features: detectors.Detectors, args: tuple[object, ...], kwargs: Mapping[str, object] +) -> UnsupportedParamsError | None: + if bind(legacy, args, kwargs) is None: + return None + request: Final = native_call(legacy, args, kwargs) + requested_model: Final = request.resolved.get("model") + if not isinstance(requested_model, str): + return None + model, provider = _resolved(request) + context: Final = RouteContext( + Route.MESSAGES, provider=provider, model=requested_model, target=_checked_target(request, model, provider) + ) + if catalog.rollout(context) is not Rollout.RUST_REQUIRED: + return None + unsupported: Final = _unsupported_features(features, request) + return UnsupportedParamsError( + message=( + f"provider {provider} on /v1/messages runs only on the Rust path, which does not support " + f"{', '.join(unsupported) if unsupported else 'this call'}" + ), + llm_provider=provider, + model=requested_model, + ) + + +def _python_messages_handler( + *args: object, + **kwargs: object, # kwargs-ok: forwards the public call shape +) -> MessagesResult | Coroutine[object, object, MessagesResult]: + refusal: Final = _rust_only_refusal(_MESSAGES, _FEATURES, args, kwargs) + if refusal is not None: + raise refusal + return _PYTHON_MESSAGES(*args, **kwargs) + + +async def _python_anthropic_messages(*args: object, **kwargs: object) -> MessagesResult: # kwargs-ok: public call shape + refusal: Final = _rust_only_refusal(_AMESSAGES, _FEATURES, args, kwargs) + if refusal is not None: + raise refusal + return await _PYTHON_AMESSAGES(*args, **kwargs) + + def anthropic_messages_handler( *args: object, **kwargs: object, # kwargs-ok: preserve the public Anthropic Messages call shape ) -> MessagesResult | Coroutine[object, object, MessagesResult]: - python: Final = _PYTHON_MESSAGES if _FLAG.requested(kwargs): return anthropic_messages(**_FLAG.keywords(args, kwargs)) return _DISPATCH.run( args, _FLAG.dropped(kwargs), - python=python, + python=_python_messages_handler, binding=NATIVE_MESSAGES, native=native_call_hook, ) async def anthropic_messages(*args: object, **kwargs: object) -> MessagesResult: # kwargs-ok: public call shape - python: Final = _PYTHON_AMESSAGES return await _ADISPATCH.arun( args, _FLAG.dropped(kwargs), - python=python, + python=_python_anthropic_messages, binding=NATIVE_AMESSAGES, native=native_acall_hook, ) diff --git a/litellm/rust_bridge/AGENTS.md b/litellm/rust_bridge/AGENTS.md index a13f57fae4b..ca71fc59c28 100644 --- a/litellm/rust_bridge/AGENTS.md +++ b/litellm/rust_bridge/AGENTS.md @@ -21,8 +21,11 @@ - `tests/test_litellm_rust/test_public_entries.py` marks such a route `python=None`, so dropping a route's Python changes its entries, its rules and that field together - A detector reads the same inputs the legacy Python path reads, so the catalog and the fallback agree on which requests need Python - A route's context, its target as well as its features, reads `NativeCall.resolved`, so an argument counts the same whether it came by position, by keyword or as a signature default - - A route's sync and async dispatches read different detector maps when the legacy handlers differ: Messages runs pre-request hooks, web search interception and interceptors only on async calls, and the sync Responses handler runs the deployment hook itself + - A route's sync and async dispatches read different detector maps when the legacy handlers differ, as the sync Responses handler runs the deployment hook itself. Messages reads one map for both, since Rust serves its async-only pre-request hooks, web search short-circuit and interceptors through host calls - A detector that raises keeps the call on Python. `PublicDispatch` logs it and never fails the call over routing +- Messages for provider `anthropic` is `RUST_REQUIRED` and has no Python path + - a call the rules would send to Python (a `MESSAGES_PYTHON_FEATURES` feature or a raising detector) raises `UnsupportedParamsError` from the Python fallback in `litellm/messages/dispatch.py`, which names the detected features + - other Messages providers keep the Python fallback - `catalog.RULES` is rollout policy with no tests of its own. Tests pass the rules they need, so check these invariants by review whenever `RULES` changes # Public entries diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index ebbd45c7382..bde79aeac48 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -21,6 +21,7 @@ class NativeRedisStoreError(Exception): ... def trace_encode_error(message: str) -> bytes: ... def bundled_model_cost_map() -> bytes: ... def native_redis_breaker_states() -> list[str]: ... +def unsupported_tls_settings(ssl_security_level: str | None = None, ssl_ecdh_curve: str | None = None) -> list[str]: ... @final class NativeCostMapSource: def __new__(cls, url: str, *, use_http_settings: bool) -> NativeCostMapSource: ... @@ -732,6 +733,7 @@ __all__ = [ "responses", "trace_encode_error", "transcription", + "unsupported_tls_settings", ] @final diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index c493617003d..87f4c855561 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -32,8 +32,6 @@ class Target(str, Enum): class Feature(str, Enum): - SAFEGUARDS_CLASSIFIER = "safeguards_classifier" - """A non-Claude model with dangerous_tool_use safeguards, which the proxy classifier answers.""" MOCK_RESPONSE = "mock_response" """mock_response short-circuits before any provider call.""" PROXY_MCP_TOOLS = "proxy_mcp_tools" @@ -66,14 +64,6 @@ class Feature(str, Enum): """Extra body fields that Python transforms and merges into the provider request.""" PROVIDER_AFFINITY = "provider_affinity" """A provider affinity header that Python validates and populates from the session id.""" - FOUNDRY_BETA_ROW = "foundry_beta_row" - """A Foundry api_base, where Python drops the anthropic-beta values Foundry rejects.""" - PRE_REQUEST_HOOK = "pre_request_hook" - """A logger overriding async_pre_request_hook, which may rewrite the request.""" - WEBSEARCH_INTERCEPTION = "websearch_interception" - """A WebSearchInterceptionLogger that may answer without calling the provider.""" - MESSAGES_INTERCEPTOR = "messages_interceptor" - """A registered MessagesInterceptor that takes over the call.""" AGENTIC_LOOP_HOOK = "agentic_loop_hook" """A logger overriding async_should_run_agentic_loop, which may run follow-up calls.""" RESPONSE_CACHE = "response_cache" @@ -94,14 +84,6 @@ class Feature(str, Enum): """No static key, so Python mints a workload identity federation token.""" CACHE_CONTROL_INJECTION = "cache_control_injection" """Prompt caching injection that Python applies to messages and system.""" - STREAM_COST_INJECTION = "stream_cost_injection" - """litellm.include_cost_in_streaming_usage or the x-litellm-include-cost-in-streaming-usage header, where Python adds cost to streamed usage.""" - THINKING_SIGNATURE_RETRY = "thinking_signature_retry" - """Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature.""" - TOOL_CUSTOM_FIELD = "tool_custom_field" - """Tools carrying a non-spec custom object, which Python drops or hoists for the provider.""" - STRICT_TOOLS_RETRY = "strict_tools_retry" - """Strict tools under drop_params, which Python sends again without strict if Claude rejects their limits.""" CLIENT_HEADERS = "client_headers" """Proxy-forwarded headers that Python merges into extra_headers.""" DEPLOYMENT_HOOK = "deployment_hook" @@ -172,21 +154,14 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { Feature.MODEL_ALIAS_MAP, } -MESSAGES_PYTHON_FEATURES: Final = (COMMON_PYTHON_FEATURES - {Feature.TIMEOUT_VARIANTS}) | { - Feature.PROVIDER_AFFINITY, - Feature.FOUNDRY_BETA_ROW, - Feature.SAFEGUARDS_CLASSIFIER, - Feature.PRE_REQUEST_HOOK, - Feature.WEBSEARCH_INTERCEPTION, - Feature.MESSAGES_INTERCEPTOR, - Feature.RESPONSE_CACHE, - Feature.MODIFY_PARAMS, - Feature.WORKLOAD_IDENTITY, - Feature.CACHE_CONTROL_INJECTION, - Feature.STREAM_COST_INJECTION, - Feature.THINKING_SIGNATURE_RETRY, - Feature.TOOL_CUSTOM_FIELD, - Feature.STRICT_TOOLS_RETRY, +MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES - { + Feature.TIMEOUT_VARIANTS, + Feature.LATEST_ALIAS, + Feature.MOCK_RESPONSE, + Feature.CALLER_CLIENT, + Feature.TLS_POLICY, + Feature.AGENTIC_LOOP_HOOK, + Feature.PROXY_MCP_TOOLS, } RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { @@ -220,7 +195,7 @@ RULES: Final[Rules] = ( RouteRule(Route.OCR, Rollout.RUST_REQUIRED), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, targets=frozenset({Target.CHAT_COMPLETIONS, Target.RESPONSES})), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=MESSAGES_PYTHON_FEATURES), - RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN, providers=frozenset({"anthropic"})), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED, providers=frozenset({"anthropic"})), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY, features=RESPONSES_PYTHON_FEATURES), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), diff --git a/litellm/rust_bridge/detectors.py b/litellm/rust_bridge/detectors.py index c38c3ea7135..1dd3dc6b35c 100644 --- a/litellm/rust_bridge/detectors.py +++ b/litellm/rust_bridge/detectors.py @@ -2,10 +2,11 @@ from __future__ import annotations +import functools import os from collections.abc import Callable, Iterable, Mapping from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeAlias, cast # noqa: TID251 # tool lists stay caller-owned +from typing import Final, TypeAlias import httpx @@ -16,9 +17,6 @@ from litellm.litellm_core_utils.model_latest_aliases import is_model_latest_alia from litellm.rust_bridge.catalog import Feature from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str -if TYPE_CHECKING: - from litellm.responses.mcp.litellm_proxy_mcp_handler import ToolParam - Detector: TypeAlias = Callable[[NativeCall], bool] Detectors: TypeAlias = Mapping[Feature, Detector] @@ -69,12 +67,10 @@ def timeout_variants(request: NativeCall) -> bool: def proxy_mcp_tools(request: NativeCall) -> bool: """Python expands proxy MCP tools and may execute their tool calls.""" - from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler + from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway tools: Final = optional_sequence(request.resolved.get("tools")) - return tools is not None and LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway( - cast("Iterable[ToolParam]", tools) # cast-ok: the predicate skips entries that are not MCP tool dicts - ) + return tools is not None and may_name_litellm_mcp_gateway(tools) def prompt_management(request: NativeCall) -> bool: @@ -168,15 +164,27 @@ def response_cache(request: NativeCall) -> bool: return litellm.cache is not None and cache.get("no-cache") is not True -_NATIVE_ECDH_CURVES: Final = frozenset({"x25519", "prime256v1", "secp256r1", "p-256", "secp384r1", "p-384"}) +@functools.lru_cache(maxsize=32) +def _tls_unsupported(security_level: str | None, ecdh_curve: str | None) -> bool: + from litellm.rust_bridge import _native + + return bool(_native.unsupported_tls_settings(security_level, ecdh_curve)) def tls_policy(_request: NativeCall) -> bool: """Python applies TLS settings or ECDH curves the native client cannot configure.""" - curve: Final = os.getenv("SSL_ECDH_CURVE", litellm.ssl_ecdh_curve) - return bool(os.getenv("SSL_SECURITY_LEVEL", litellm.ssl_security_level)) or ( - bool(curve) and str(curve).strip().lower() not in _NATIVE_ECDH_CURVES + return unsupported_tls() + + +def unsupported_tls() -> bool: + configured: Final[tuple[object, object]] = ( + os.environ.get("SSL_SECURITY_LEVEL", litellm.ssl_security_level), + os.environ.get("SSL_ECDH_CURVE", litellm.ssl_ecdh_curve), ) + if any(value and not isinstance(value, str) for value in configured): # pyright: ignore[reportUnnecessaryIsInstance] # callers assign the ssl settings untyped + return True + security_level, ecdh_curve = (value if isinstance(value, str) else None for value in configured) + return _tls_unsupported(security_level, ecdh_curve) COMMON: Final[Detectors] = MappingProxyType( diff --git a/litellm/rust_bridge/execution.py b/litellm/rust_bridge/execution.py index 430de8d1ad6..a77fd162bf4 100644 --- a/litellm/rust_bridge/execution.py +++ b/litellm/rust_bridge/execution.py @@ -101,8 +101,10 @@ def synchronous(native: Callable[[_Call], object]) -> Callable[[_Call], object]: return lambda call: run(_execution(native(call)), SyncStream) -def asynchronous(native: Callable[[_Call], object]) -> Callable[[_Call], Awaitable[object]]: - return lambda call: drive(_execution(native(call)), AsyncStream) +def asynchronous( + native: Callable[[_Call], object], open_stream: Callable[[Execution, object], object] | None = None +) -> Callable[[_Call], Awaitable[object]]: + return lambda call: drive(_execution(native(call)), open_stream or AsyncStream) def _execution(value: object) -> Execution: diff --git a/litellm/rust_bridge/host/AGENTS.md b/litellm/rust_bridge/host/AGENTS.md index c249eb82ee0..da933f3b467 100644 --- a/litellm/rust_bridge/host/AGENTS.md +++ b/litellm/rust_bridge/host/AGENTS.md @@ -25,7 +25,7 @@ Python behavior Rust needs but does not own, behind one module per question. Rus - `logger`: where native diagnostics are emitted - `preflight`: the SDK globals the native preflight borrows - `secret_manager`: the configured secret manager, its native config and the error text Python's managers produce - - `http_transport`: which async transport is in use + - `http_transport`: which async transport is in use, and whether a call must send through Python's `AsyncHTTPHandler` - `timeouts`: timeout to seconds - `tokenizers`: bundled tiktoken files and Hugging Face tokenizer downloads - `randomness`: the process `random` generator the Python router draws from diff --git a/litellm/rust_bridge/host/cache.py b/litellm/rust_bridge/host/cache.py index bf2655f9f62..38a18b9be57 100644 --- a/litellm/rust_bridge/host/cache.py +++ b/litellm/rust_bridge/host/cache.py @@ -1,7 +1,8 @@ from __future__ import annotations -from collections.abc import Mapping -from typing import TYPE_CHECKING, Final +import weakref +from collections.abc import Callable, Mapping +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # bounded cast of the served response from pydantic import TypeAdapter @@ -9,9 +10,13 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_par if TYPE_CHECKING: from litellm.caching.caching import Cache + from litellm.litellm_core_utils.litellm_logging import Logging _RUST_RESPONSE_HEADER: Final = "x-litellm-rust" _HEADERS: Final = TypeAdapter(dict[str, object]) +_EVENTS: Final = TypeAdapter(list[str]) +_HITS: Final[weakref.WeakKeyDictionary[Logging, Mapping[str, object]]] = weakref.WeakKeyDictionary() +_UNCLAMPED: Final[weakref.WeakKeyDictionary[Logging, object]] = weakref.WeakKeyDictionary() def configured_cache() -> Cache | None: @@ -25,17 +30,14 @@ def _python_entry(response: Mapping[str, object]) -> Mapping[str, object]: headers: Final = hidden.get("additional_headers") if not isinstance(headers, Mapping) or _RUST_RESPONSE_HEADER not in headers: return response - return { - **response, - "_hidden_params": { - **hidden, - "additional_headers": { - name: value - for name, value in _HEADERS.validate_python(headers).items() - if name != _RUST_RESPONSE_HEADER - }, - }, + kept: Final = { + name: value for name, value in _HEADERS.validate_python(headers).items() if name != _RUST_RESPONSE_HEADER } + stored: Final = {name: value for name, value in hidden.items() if name != "additional_headers" or kept} | ( + {"additional_headers": kept} if kept else {} + ) + entry: Final = {name: value for name, value in response.items() if name != "_hidden_params"} + return {**entry, "_hidden_params": stored} if stored else entry def store_response( @@ -44,11 +46,183 @@ def store_response( /, **kwargs: object, # kwargs-ok: the cache call's own arguments ) -> None: - from litellm.caching.caching_handler import create_cache_write_task + from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output + if is_response_without_output(response): + return create_cache_write_task( lambda: cache.async_add_cache( # pyright: ignore[reportUnknownMemberType] # Cache.async_add_cache leaves its result untyped _python_entry(response), **kwargs, # pyright: ignore[reportArgumentType] # the cache call's own keyword arguments, as Python passes them ) ) + + +def store_verbatim( + cache: Cache, + response: Mapping[str, object], + /, + **kwargs: object, # kwargs-ok: the cache call's own arguments +) -> None: + from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output + + if is_response_without_output(response): + return + create_cache_write_task( + lambda: cache.async_add_cache( # pyright: ignore[reportUnknownMemberType] # Cache.async_add_cache leaves its result untyped + response, + **kwargs, # pyright: ignore[reportArgumentType] # the cache call's own keyword arguments, as Python passes them + ) + ) + + +async def store_stream( + cache: Cache, + entry: Mapping[str, object], + /, + **kwargs: object, # kwargs-ok: the cache call's own arguments +) -> None: + from litellm._logging import verbose_logger + from litellm.caching.caching_handler import LLMCachingHandler, create_cache_write_task + from litellm.types.caching import CACHED_STREAM_EVENTS_KEY + + events: Final = entry.get(CACHED_STREAM_EVENTS_KEY) + handler: Final = getattr(kwargs.get("litellm_logging_obj"), "llm_caching_handler", None) + preset_cache_key: Final = handler.preset_cache_key if isinstance(handler, LLMCachingHandler) else None + cache_key_override: Final[Mapping[str, object]] = ( + {"cache_key": preset_cache_key} if preset_cache_key is not None else {} + ) + + async def write() -> None: + try: + await cache.async_add_cache( # pyright: ignore[reportUnknownMemberType] # Cache.async_add_cache leaves its result untyped + {**entry, CACHED_STREAM_EVENTS_KEY: tuple(_EVENTS.validate_python(events))} + if isinstance(events, list) + else entry, + dynamic_cache_object=handler.dual_cache if isinstance(handler, LLMCachingHandler) else None, + **{**kwargs, **cache_key_override}, # pyright: ignore[reportArgumentType] # the cache call's own keyword arguments, as Python passes them + ) + except Exception as e: # noqa: BLE001 # a cache write must never surface as a client-visible stream error + verbose_logger.exception("Anthropic Messages stream cache write failed: %s", e) + + create_cache_write_task(write) + + +def caching_handler( + **kwargs: object, # kwargs-ok: the cache call's own arguments +) -> object | None: + from litellm.caching.caching_handler import LLMCachingHandler + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging) or not isinstance(logging_obj.llm_caching_handler, LLMCachingHandler): + return None + return logging_obj.llm_caching_handler + + +def wrap_stream( + _cache: Cache, + original_function: Callable[..., object], + stream: object, + read_handler: object | None, + /, + **kwargs: object, # kwargs-ok: the cache call's own arguments +) -> object: + from litellm.caching.caching_handler import LLMCachingHandler + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging): + return stream + handler: Final = ( + read_handler + if isinstance(read_handler, LLMCachingHandler) + else LLMCachingHandler( + original_function=original_function, request_kwargs=dict(kwargs), start_time=logging_obj.start_time + ) + ) + return handler.wrap_streaming_result_for_cache(result=stream, call_type=original_function.__name__) + + +def remember_unclamped(logging_obj: object, max_tokens: object) -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + + if isinstance(logging_obj, Logging): + _UNCLAMPED[logging_obj] = max_tokens + + +async def lookup( + _cache: Cache, + original_function: Callable[..., object], + /, + **kwargs: object, # kwargs-ok: the cache call's own arguments +) -> object | None: + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging): + return None + return await _read( + original_function, + logging_obj, + {**kwargs, "max_tokens": _UNCLAMPED.pop(logging_obj)} if logging_obj in _UNCLAMPED else kwargs, + ) + + +async def _read( + original_function: Callable[..., object], + logging_obj: Logging, + kwargs: dict[str, object], # mutable-ok: the read call's own keyword arguments, handed on as the handler takes them +) -> object | None: + from litellm.caching.caching_handler import LLMCachingHandler + + handler: Final = LLMCachingHandler( + original_function=original_function, request_kwargs=dict(kwargs), start_time=logging_obj.start_time + ) + logging_obj.llm_caching_handler = handler + model: Final = kwargs.get("model") + model_name: Final = model if isinstance(model, str) else "" + if not handler.reads_cache(model=model_name, kwargs=kwargs): + return None + read: Final = await handler.async_read_cache( + model=model_name, + original_function=original_function, + logging_obj=logging_obj, + call_type=original_function.__name__, + kwargs=kwargs, + args=(), + ) + if not read.hit: + return None + _remember(logging_obj, read.cached_result) + return read.cached_result + + +def _remember(logging_obj: Logging, cached: object) -> None: + if isinstance(cached, Mapping): + _HITS[logging_obj] = _HEADERS.validate_python(cached) + + +def _replace(response: object, cached: Mapping[str, object] | None) -> None: + if isinstance(response, dict) and cached is not None: + served: Final = cast( # cast-ok: a Messages response is the str-keyed dict the route serves + "dict[str, object]", response + ) + served.clear() + served.update(cached) + + +def restore_hit(response: object, logger: Logging) -> None: + from litellm.caching.caching_handler import ( + _set_cached_hidden_param, # pyright: ignore[reportPrivateUsage] # the @client cache hit's own hidden params + ) + + _replace(response, _HITS.pop(logger, None)) + _set_cached_hidden_param(response, "cache_hit", True) + + +def hit_cache_key(logger: Logging) -> object: + from litellm.caching.caching_handler import LLMCachingHandler + + handler: Final = logger.llm_caching_handler + return handler.hit_cache_key() if isinstance(handler, LLMCachingHandler) else None diff --git a/litellm/rust_bridge/host/callbacks_legacy_python.py b/litellm/rust_bridge/host/callbacks_legacy_python.py index ba557f769eb..19124fcffc0 100644 --- a/litellm/rust_bridge/host/callbacks_legacy_python.py +++ b/litellm/rust_bridge/host/callbacks_legacy_python.py @@ -15,7 +15,7 @@ import sys import traceback import uuid import weakref -from collections.abc import Awaitable, Callable, Coroutine, Mapping, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence from dataclasses import dataclass from typing import ( TYPE_CHECKING, @@ -27,6 +27,7 @@ from typing import ( import httpx from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging @@ -41,6 +42,7 @@ class MetadataUpdater(Protocol): kwargs: dict[str, object], start_time: datetime.datetime, end_time: datetime.datetime, + cache_hit: bool | None = None, ) -> None: ... @@ -97,6 +99,7 @@ def finalize( kwargs: dict[str, object], start_time: datetime.datetime, end_time: datetime.datetime, + cache_hit: bool, ) -> None: from litellm.litellm_core_utils.llm_response_utils import response_metadata @@ -104,6 +107,16 @@ def finalize( update: Final = cast( # cast-ok: legacy metadata function accepts concrete kwargs MetadataUpdater, response_metadata.update_response_metadata ) + if cache_hit: + from litellm.caching.caching_handler import ( + _set_cached_hidden_param, # pyright: ignore[reportPrivateUsage] # the @client cache hit's own hidden params + ) + from litellm.rust_bridge.host.cache import hit_cache_key, restore_hit + + restore_hit(response, logger) + update(response, logger, logger.model, kwargs, start_time, end_time, cache_hit=True) + _set_cached_hidden_param(response, "cache_key", hit_cache_key(logger)) + return update(response, logger, model if isinstance(model, str) else None, kwargs, start_time, end_time) cache_key: Final = logger.model_call_details.get("cache_key") if logger.model_call_details.get("cache_hit") is True and isinstance(cache_key, str): @@ -277,14 +290,76 @@ def post_call( logger.post_call(original_response=original_response, api_key=api_key, additional_args=additional_args) -def record_response(logger: Logging, status: int, headers: Sequence[tuple[str, str]], body: str) -> None: +class _UnreadBody(httpx.SyncByteStream, httpx.AsyncByteStream): + def __iter__(self) -> Iterator[bytes]: + return iter(()) + + async def __aiter__(self) -> AsyncIterator[bytes]: + for chunk in (): + yield chunk + + +def wire_headers(headers: Sequence[tuple[str, str]]) -> httpx.Headers: + return httpx.Headers([(name.encode(), value.encode()) for name, value in headers]) + + +def _sent_request(sent: tuple[str, Mapping[str, str], Mapping[str, object]]) -> httpx.Request: + url, headers, body = sent + return httpx.Request("POST", url, headers=list(headers.items()), content=json.dumps(body, default=str).encode()) + + +def record_response( + logger: Logging, + status: int, + headers: Sequence[tuple[str, str]], + body: str, + sent: tuple[str, Mapping[str, str], Mapping[str, object]] | None, +) -> None: """Stores the provider response the way Python's Messages handler does, for logging to replay.""" - response: Final = httpx.Response(status_code=status, headers=list(headers), content=body.encode()) + request: Final = None if sent is None else _sent_request(sent) + streamed: Final = sent is not None and sent[2].get("stream") is True + response: Final = ( + httpx.Response(status_code=status, stream=_UnreadBody(), request=request) + if streamed + else httpx.Response(status_code=status, content=body.encode(), request=request) + ) + response.headers = wire_headers(headers) details: Final = logger.model_call_details # the call's own logger: recording on it is the point details["httpx_response"] = response details["response_headers"] = dict(response.headers) +class _ThinkingSignatureRetry(TypedDict): + kind: ReadOnly[Literal["thinking_signature"]] + + +class _StrictToolsRetry(TypedDict): + kind: ReadOnly[Literal["strict_tools"]] + dropped: ReadOnly[int] + + +_RETRY_REASONS: Final = TypeAdapter(tuple[_ThinkingSignatureRetry | _StrictToolsRetry, ...]) + + +async def retried_request( + logger: Logging, + logged_body: dict[str, object], # mutable-ok: the retry rewrites the body pre_call logged, as Python's loop does + body: Mapping[str, object], + reasons: Sequence[Mapping[str, object]], +) -> None: + from litellm.llms.anthropic.common_utils import warn_claude_strict_tools_dropped + + if logger.baseline_cache_context is not None: + await logger.invalidate_baseline_cache_estimate("retried_request") + for reason in _RETRY_REASONS.validate_python(reasons): + if reason["kind"] == "strict_tools": + warn_claude_strict_tools_dropped(reason["dropped"]) + for name in logged_body.keys() - body.keys(): + del logged_body[name] # rebind-ok: the logged body is the one logging already holds + logged_body.update({name: value for name, value in body.items() if logged_body.get(name) != value}) + logger.model_call_details.update(logged_body) + + def record_api_window(logger: Logging, start: float, end: float) -> None: """Records the provider exchange's window the way Python's timed `AsyncHTTPHandler.post` does.""" from litellm.litellm_core_utils.logging_utils import ( @@ -296,6 +371,38 @@ def record_api_window(logger: Logging, start: float, end: float) -> None: start_time=datetime.datetime.fromtimestamp(start), # noqa: DTZ006 # the timed post records naive local times end_time=datetime.datetime.fromtimestamp(end), # noqa: DTZ006 # the timed post records naive local times ) + _log_api_window(logger, start, end) + + +def _log_api_window(logger: Logging, start: float, end: float) -> None: + from litellm._logging import verbose_logger + from litellm.litellm_core_utils.logging_utils import ( + _get_parent_otel_span_from_logging_obj, # pyright: ignore[reportPrivateUsage] # the timed post's own span lookup + _get_service_logger, # pyright: ignore[reportPrivateUsage] # the timed post's own service logger + ) + from litellm.types.services import ServiceTypes + + parent_otel_span: Final = _get_parent_otel_span_from_logging_obj(logger) + try: + service_logger: Final = _get_service_logger() + _ = asyncio.create_task( + service_logger.async_service_success_hook( # pyright: ignore[reportUnknownMemberType] # the hook's untyped event_metadata dict + service=ServiceTypes.LITELLM, + duration=end - start, + call_type="post <- track_llm_api_timing", + start_time=start, + end_time=end, + parent_otel_span=parent_otel_span, + ) + ) + except Exception as e: # noqa: BLE001 # the timed post logs and drops a service logging failure + verbose_logger.debug("Error in service logging: %s", e) + + +def times_provider_exchange(client: object, timeout: object) -> bool: + from litellm.rust_bridge.host import http_transport + + return not http_transport.sends_through_python(client, timeout) def defers_async_logging(logger: LoggingSurface) -> bool: @@ -369,6 +476,16 @@ def async_success_handler( return logger.async_success_handler(response, start, end, cache_hit=_cache_hit(logger)) +def cache_hit_success(logger: Logging, response: object, start: datetime.datetime, end: datetime.datetime) -> None: + from litellm.caching.caching_handler import LLMCachingHandler + + handler: Final = logger.llm_caching_handler + if isinstance(handler, LLMCachingHandler): + handler.async_log_cache_hit_on_callbacks( + logging_obj=logger, cached_result=response, start_time=start, end_time=end, cache_hit=True + ) + + def enqueue_logging(coroutine: Coroutine[object, object, None]) -> None: from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -414,7 +531,29 @@ def after_deployment_success(kwargs: dict[str, object], response: object, call_t hook: Final = cast( # cast-ok: bounded adapter for the untyped deployment hook DeploymentSuccessHook, utils.async_post_call_success_deployment_hook ) - return hook(kwargs, response, CallTypes(call_type)) + typed: Final = CallTypes(call_type) + return hook(kwargs, _messages_stream(response) if typed == CallTypes.anthropic_messages else response, typed) + + +def _messages_stream(response: object) -> object: + from litellm.rust_bridge.execution import AsyncStream + + if not isinstance(response, AsyncStream): + return response + from litellm.llms.anthropic.messages.native.streaming_iterator import ( + AnthropicMessagesStreamHiddenParams, + AnthropicMessagesStreamingResponse, + ) + + return AnthropicMessagesStreamingResponse( + completion_stream=cast( # cast-ok: the native stream yields the provider's SSE bytes + AsyncIterator[bytes], response + ), + hidden_params=cast( # cast-ok: the stream's stamped hidden params, shared as Python's handler shares them + AnthropicMessagesStreamHiddenParams, + response._hidden_params, # pyright: ignore[reportPrivateUsage] # the native stream's own hidden params + ), + ) def after_deployment_failure(kwargs: dict[str, object], error: Exception, call_type: str) -> Awaitable[None]: @@ -442,17 +581,25 @@ def stamp_stream_head( kwargs: dict[str, object], # mutable-ok: `finalize` takes the call kwargs as a dict start_time: datetime.datetime, end_time: datetime.datetime, + cache_hit: bool, ) -> None: + if kwargs.get("stream") is True: + logger.stream = True + logger.model_call_details["stream"] = True carrier: Final = _HiddenParamsCarrier(head) - finalize(carrier, logger, kwargs, start_time, end_time) + finalize(carrier, logger, kwargs, start_time, end_time, cache_hit) stamped: Final = carrier._hidden_params # pyright: ignore[reportPrivateUsage] # the carrier's own field if stamped is not head and isinstance(stamped, dict): head.update(cast("dict[str, object]", stamped)) # cast-ok: update_response_metadata writes a str-keyed dict -def stream_opened(logger: Logging, head: Mapping[str, object]) -> None: - logger.stream = True - logger.model_call_details["stream"] = True +def stream_opened(logger: Logging, head: Mapping[str, object], keeps_requested: bool = False) -> None: + requested: Final = logger.model_call_details.get("stream") + streamed: Final = requested if keeps_requested and requested else True + logger.stream = cast(bool, streamed) # cast-ok: Python's Messages handler logs the stream flag it sent + logger.model_call_details["stream"] = streamed + if "httpx_response" in logger.model_call_details: + return logger.model_call_details["response_headers"] = _STREAM_HEADERS.validate_python( head.get("additional_headers") or {} ) @@ -486,6 +633,12 @@ def stream_success( if first_chunk is not None: logger.completion_start_time = first_chunk logger.model_call_details["completion_start_time"] = first_chunk + superseded_error_event: Final = PassThroughStreamingHandler._error_frame_closed_at( # pyright: ignore[reportPrivateUsage] # the Messages stream's own close-at-frame check + EndpointType(endpoint_type), chunks + ) + if superseded_error_event is not None: + _log_superseded_stream_failure(logger, endpoint_type, request_body, chunks, superseded_error_event) + return build: Final = cast( # cast-ok: bounded adapter for the untyped pass-through logging builder StreamingLogBuilder, PassThroughStreamingHandler._route_streaming_logging_to_handler, # pyright: ignore[reportPrivateUsage] # the Messages stream iterator bills through the same builder @@ -518,6 +671,37 @@ def stream_success( enqueue_logging(coroutine) +def _log_superseded_stream_failure( + logger: Logging, + endpoint_type: str, + request_body: Mapping[str, object], + chunks: Sequence[bytes], + error_event: tuple[str, str, int], +) -> None: + from litellm._logging import verbose_proxy_logger + from litellm.llms.anthropic.common_utils import anthropic_error_frame_exception + from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler + from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType + + error_type, message, status_code = error_event + PassThroughStreamingHandler._record_partial_usage_for_failure( # pyright: ignore[reportPrivateUsage] # the superseded stream's own partial usage record + litellm_logging_obj=logger, + endpoint_type=EndpointType(endpoint_type), + request_body=dict(request_body), + raw_bytes=tuple(chunks), + stream_context=None, + ) + try: + enqueue_logging( + logger.dispatch_attempt_failure_handlers( # pyright: ignore[reportArgumentType] # the dispatch snapshots now, its coroutine never receives a sent value + anthropic_error_frame_exception(error_type, message, status_code, str(logger.model)), + traceback.format_exc(), + ) + ) + except Exception as e: # noqa: BLE001 # the stream's failure scheduling logs and drops its own failure + verbose_proxy_logger.error("Error scheduling stream failure logging: %s", e) + + def stream_failure( logger: Logging, endpoint_type: str, diff --git a/litellm/rust_bridge/host/http_transport.py b/litellm/rust_bridge/host/http_transport.py index 1bf0f890924..6155cfce2ea 100644 --- a/litellm/rust_bridge/host/http_transport.py +++ b/litellm/rust_bridge/host/http_transport.py @@ -1,7 +1,32 @@ from __future__ import annotations +from typing import Final + def uses_aiohttp_transport() -> bool: from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler return AsyncHTTPHandler.should_use_aiohttp_transport() + + +def _non_positive_timeout(timeout: object) -> bool: + import httpx + + seconds: Final = ( + (timeout.connect, timeout.read, timeout.write, timeout.pool) + if isinstance(timeout, httpx.Timeout) + else (timeout,) + ) + return any(isinstance(value, int | float) and value <= 0 for value in seconds) + + +def sends_through_python(client: object, timeout: object = None) -> bool: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, rsa_key_exchange_and_chacha20_excluded + from litellm.rust_bridge import detectors + + return ( + isinstance(client, AsyncHTTPHandler) + or detectors.unsupported_tls() + or rsa_key_exchange_and_chacha20_excluded() + or _non_positive_timeout(timeout) + ) diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index c6568264edb..48de9b0ac03 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -1,39 +1,99 @@ from __future__ import annotations +import asyncio import datetime import errno +import functools +import inspect import ipaddress +import json import os import socket -from collections.abc import Callable, Mapping, Sequence -from dataclasses import asdict, dataclass +from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence +from dataclasses import asdict, dataclass, replace from typing import Final, TypeAlias, cast # noqa: TID251 # narrows native payloads to public TypedDicts import httpx from aiohttp.http_exceptions import ContentLengthError, TransferEncodingError -from pydantic import TypeAdapter, ValidationError +from pydantic import JsonValue, TypeAdapter, ValidationError import litellm +from litellm.caching.caching_handler import LLMCachingHandler +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils import exception_mapping_utils -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, + drops_unsupported_params, + normalize_drop_params, + proxy_request_headers, + stream_cost_injection_requested, +) +from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header +from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.model_latest_aliases import is_model_latest_alias +from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout -from litellm.llms.anthropic.common_utils import AnthropicError, anthropic_pass_through_api_base +from litellm.litellm_core_utils.url_utils import ensure_api_route +from litellm.llms.anthropic.common_utils import ( + AnthropicError, + AnthropicModelInfo, + anthropic_pass_through_api_base, + strip_provider_specific_fields_from_anthropic_messages, +) from litellm.llms.anthropic.messages.native import handler +from litellm.llms.anthropic.messages.native import utils as native_utils +from litellm.llms.anthropic.messages.native.handler import ( + _bridge_target as _bridged, # pyright: ignore[reportPrivateUsage] # the handler's own bridge target +) +from litellm.llms.anthropic.messages.native.handler import ( + _BridgeTarget, # pyright: ignore[reportPrivateUsage] # the handler's own bridge target +) from litellm.llms.anthropic.messages.native.handler import ( _native_messages_config as native_messages_config, # pyright: ignore[reportPrivateUsage] # the handler's own config lookup ) +from litellm.llms.anthropic.messages.native.handler import ( + _resolve_requested_provider as _requested_provider, # pyright: ignore[reportPrivateUsage] # the handler's own provider resolution +) +from litellm.llms.anthropic.messages.native.handler import ( + _sets_own_entra_credential as _own_entra_credential, # pyright: ignore[reportPrivateUsage] # the handler's own credential check +) from litellm.llms.anthropic.messages.native.handler import ( _with_deployment_output_effort as _deployment_output_effort, # pyright: ignore[reportPrivateUsage] # the handler's own effort rounding ) +from litellm.llms.anthropic.messages.native.interceptors import MessagesInterceptor, get_messages_interceptors +from litellm.llms.anthropic.messages.native.utils import normalize_native_messages, prepare_native_messages +from litellm.llms.anthropic.messages.safeguards import ( + SafeguardsEvaluator, + StreamedSafeguardResults, + native_route_classifies, + with_safeguard_results, +) from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled +from litellm.llms.anthropic.wif import get_anthropic_wif_token +from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.custom_httpx.http_handler import get_ssl_configuration -from litellm.rust_bridge import failures -from litellm.rust_bridge.host import callbacks_legacy_python, http_transport -from litellm.rust_bridge.public_call import known_params, optional_str +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # its params dict is untyped + get_ssl_configuration, +) +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import ( + _merge_messages_header_sources as _merged_headers, # pyright: ignore[reportPrivateUsage] # the handler's own header merge +) +from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway +from litellm.rust_bridge import detectors, execution, failures +from litellm.rust_bridge.execution import AsyncStream +from litellm.rust_bridge.host import cache, callbacks_legacy_python, http_transport +from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead +from litellm.rust_bridge.public_call import NativeCall, known_params, optional_sequence, optional_str +from litellm.types.integrations.custom_logger import WEBSEARCH_CONVERTED_STREAM_KEY from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import CallTypes, ProviderSpecificHeader, anthropic_wif_litellm_params +from litellm.utils import clamp_max_tokens _METADATA: Final = TypeAdapter(dict[str, object]) _BODY_FIELDS: Final = frozenset( @@ -61,9 +121,12 @@ _BODY_FIELDS: Final = frozenset( "reasoning_effort", "safeguards", "service_tier", + "thread", } ) +WORKLOAD_IDENTITY_FIELDS: Final = anthropic_wif_litellm_params + @dataclass(frozen=True, slots=True) class MessagesSettings: @@ -71,6 +134,15 @@ class MessagesSettings: reasoning_auto_summary: bool additional_drop_params: Sequence[str] request_timeout: float | None + drops_unsupported_params: bool + model: str | None = None + mock_response: Mapping[str, object] | None = None + stream_cost_model: str | None = None + extra_headers: Mapping[str, object] | None = None + mcp_gateway: bool = False + ascii_response_headers: bool = False + joined_reads: bool = False + ascii_request_headers: bool = True def response(value: Mapping[str, object]) -> AnthropicMessagesResponse: @@ -90,13 +162,37 @@ def inferred_provider(model: str, api_base: str | None) -> str | None: return provider -def body_params(request: Mapping[str, object]) -> Mapping[str, object]: +def modify_max_tokens( + model: object, + args: tuple[object, ...], + kwargs: dict[str, object], # mutable-ok: the call's keyword view, rewritten in place as `@client` does +) -> None: + unclamped: Final = kwargs.get("max_tokens") + clamp_max_tokens(model, args, kwargs, CallTypes.anthropic_messages.value) + if kwargs.get("max_tokens") != unclamped: + cache.remember_unclamped(kwargs.get("litellm_logging_obj"), unclamped) + + +def _body_fields(request: Mapping[str, object]) -> Mapping[str, object]: params: Final = known_params(_BODY_FIELDS.difference(handler.additional_drop_paths(request)), request) if deployment_reasoning_effort_levels(request.get("model_info")) is None: return params return _with_deployment_effort(params, request) +def body_params(request: Mapping[str, object], asynchronous: bool, hooked: bool = False) -> Mapping[str, object]: + fields: Final = _body_fields(request) + if not detectors.cache_control_injection(NativeCall(args=(), kwargs=request, base={})): + return {**fields, "messages": request["messages"]} if hooked else fields + messages, system, tools = _prompt_cached(request, asynchronous, hooked) + return { + **fields, + "messages": messages, + **({"system": system} if "system" in fields else {}), + **({"tools": tools} if tools is not None and "tools" in fields else {}), + } + + def _with_deployment_effort(params: Mapping[str, object], request: Mapping[str, object]) -> Mapping[str, object]: """Python's handler rounds the requested effort to the levels the deployment declares in `model_info.reasoning_effort_levels`, the one place a deployment, not the model, decides.""" @@ -116,12 +212,309 @@ def _with_deployment_effort(params: Mapping[str, object], request: Mapping[str, ) +def cached_function() -> Callable[..., object]: + return cast( # cast-ok: the @client cache read names and checks the legacy handler it wraps + "Callable[..., object]", + handler.anthropic_messages, # noqa: TID251 # the @client cache read names and checks the legacy handler it wraps + ) + + +_ENTRY: Final = cast( # cast-ok: only the untyped handler's signature is read + Callable[..., object], + handler.anthropic_messages, # noqa: TID251 # reads the handler's signature, never calls it +) +_HANDLER: Final = cast( # cast-ok: only the untyped handler's signature is read + Callable[..., object], + handler.anthropic_messages_handler, # noqa: TID251 # reads the handler's signature, never calls it +) +_ENTRY_PARAMS: Final = frozenset(inspect.signature(_ENTRY).parameters) - {"kwargs"} +_HANDLER_PARAMS: Final = frozenset(inspect.signature(_HANDLER).parameters) - {"kwargs"} +_JsonBlocks: TypeAlias = list[dict[str, JsonValue]] +_Prepared: TypeAlias = tuple[_JsonBlocks, str | _JsonBlocks | None, Sequence[object] | None] + + +def _prompt_cached(request: Mapping[str, object], asynchronous: bool, hooked: bool) -> _Prepared: + model: Final = str(request["model"]) + provider: Final = optional_str(request.get("custom_llm_provider")) + api_base: Final = optional_str(request.get("api_base")) + tools: Final = cast(_JsonBlocks | None, request.get("tools")) # cast-ok: handed on unchecked, as the handler does + messages: Final = cast(_JsonBlocks, request["messages"]) # cast-ok: handed on unchecked, as the handler does + system: Final = cast( # cast-ok: handed on unchecked, as the handler does + str | _JsonBlocks | None, request.get("system") + ) + if not asynchronous: + return _handler_prepared( + request, + messages, + system, + model=model, + provider=provider, + tools=tools, + api_base=api_base, + presanitized=False, + ) + entry_kwargs: Final = {name: value for name, value in request.items() if name not in _ENTRY_PARAMS} + if hooked: + return _handler_prepared( + entry_kwargs, + messages, + system, + model=model, + provider=handler.provider_after_pre_request_hooks(model, provider, entry_kwargs), + tools=tools, + api_base=api_base, + presanitized=True, + ) + entry_messages, entry_system = prepare_native_messages( + messages, system, entry_kwargs, model=model, custom_llm_provider=provider, tools=tools, api_base=api_base + ) + return _handler_prepared( + entry_kwargs, + entry_messages, + entry_system, + model=model, + provider=handler.provider_after_pre_request_hooks(model, provider, entry_kwargs), + tools=tools, + api_base=api_base, + presanitized=True, + ) + + +def _handler_prepared( + call: Mapping[str, object], + messages: _JsonBlocks, + system: str | _JsonBlocks | None, + *, + model: str, + provider: str | None, + tools: _JsonBlocks | None, + api_base: str | None, + presanitized: bool, +) -> _Prepared: + handler_kwargs: Final = {name: value for name, value in call.items() if name not in _HANDLER_PARAMS} + prepared_messages, prepared_system = prepare_native_messages( + messages, + system, + handler_kwargs, + model=model, + custom_llm_provider=provider, + tools=tools, + api_base=api_base, + presanitized=presanitized, + ) + return prepared_messages, prepared_system, handler.tools_marked_for_cache_points(tools, handler_kwargs) + + +_execute_pre_request_hooks: Final = cast( # cast-ok: the handler's untyped pre-request hook loop + Callable[..., Awaitable[dict[str, object]]], + handler._execute_pre_request_hooks, # pyright: ignore[reportPrivateUsage] # the handler's own pre-request hook loop +) +_HOOKED_PARAMS: Final = ( + "tools", + "stream", + "metadata", + "stop_sequences", + "system", + "temperature", + "thinking", + "tool_choice", + "top_k", + "top_p", +) + + +def runs_pre_request_hooks() -> bool: + return detectors.overridden("async_pre_request_hook", litellm.callbacks) or any( + isinstance(callback, CustomLogger) and "async_pre_request_hook" in vars(callback) + for callback in litellm.callbacks + ) + + +def _unsupported(provider: str | None, model: str, unsupported: str) -> litellm.UnsupportedParamsError: + return litellm.UnsupportedParamsError( + message=( + f"provider {provider} on /v1/messages runs only on the Rust path, which does not support {unsupported}" + ), + llm_provider=provider, + model=model, + ) + + +async def pre_request_hooks(request: Mapping[str, object]) -> Mapping[str, object]: + model: Final = str(request["model"]) + provider: Final = optional_str(request.get("custom_llm_provider")) + tools: Final = cast(_JsonBlocks | None, request.get("tools")) # cast-ok: handed on unchecked, as the handler does + kwargs: Final = {name: value for name, value in request.items() if name not in _ENTRY_PARAMS} + messages, system = prepare_native_messages( + cast(_JsonBlocks, request["messages"]), # cast-ok: handed on unchecked, as the handler does + cast(str | _JsonBlocks | None, request.get("system")), # cast-ok: handed on unchecked, as the handler does + kwargs, + model=model, + custom_llm_provider=provider, + tools=tools, + api_base=optional_str(request.get("api_base")), + ) + named: Final = {**{name: request.get(name) for name in _ENTRY_PARAMS}, "messages": messages, "system": system} + hooked: Final = await _execute_pre_request_hooks( + model=model, + messages=messages, + tools=tools, + stream=cast(bool | None, named["stream"]), # cast-ok: handed on unchecked, as the handler does + custom_llm_provider=provider, + tool_choice=named["tool_choice"], + **kwargs, + ) + for name in _HOOKED_PARAMS: + named[name] = hooked.pop(name, named[name]) + hooked_provider: Final = handler.provider_after_pre_request_hooks(model, provider, hooked) + hooked.pop("litellm_params", None) + kwargs.update(hooked) + requested_provider: Final = handler.provider_after_pre_request_hooks(model, provider, {}) + functools.partial( + _HANDLER, + **{name: value for name, value in named.items() if name in _HANDLER_PARAMS - {"custom_llm_provider"}}, + custom_llm_provider=hooked_provider, + _litellm_messages_presanitized=True, + **kwargs, + ) + _python_handler_params({**named, **kwargs}, hooked_provider) + if hooked_provider != requested_provider: + raise _unsupported(requested_provider, model, f"a pre-request hook routing the call to {hooked_provider}") + return {**named, **kwargs} + + +def _intercepted_provider(request: Mapping[str, object]) -> str | None: + return handler.provider_after_pre_request_hooks( + str(request["model"]), optional_str(request.get("custom_llm_provider")), {} + ) + + +def has_interceptors() -> bool: + return bool(get_messages_interceptors()) + + +def short_circuits() -> bool: + from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger + + stock: Final[object] = WebSearchInterceptionLogger.try_short_circuit_search # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the logger annotates bare dicts + return any( + isinstance(callback, WebSearchInterceptionLogger) and type(callback).try_short_circuit_search is not stock # pyright: ignore[reportUnknownMemberType] # the logger annotates bare dicts + for callback in litellm.callbacks + ) + + +def interceptor(request: Mapping[str, object]) -> MessagesInterceptor | None: + tools: Final = cast(_JsonBlocks | None, request.get("tools")) # cast-ok: handed on unchecked, as the handler does + provider: Final = _intercepted_provider(request) + return next((owner for owner in get_messages_interceptors() if owner.can_handle(tools, provider)), None) + + +def _entry_messages( + request: Mapping[str, object], + kwargs: dict[str, object], # mutable-ok: the call's own kwargs, rewritten in place by the handler's normalizers + hooked: bool, +) -> tuple[_JsonBlocks, str | _JsonBlocks | None]: + messages: Final = cast(_JsonBlocks, request["messages"]) # cast-ok: handed on unchecked, as the handler does + system: Final = cast( # cast-ok: handed on unchecked, as the handler does + str | _JsonBlocks | None, request.get("system") + ) + if hooked: + return messages, system + return prepare_native_messages( + messages, + system, + kwargs, + model=str(request["model"]), + custom_llm_provider=optional_str(request.get("custom_llm_provider")), + tools=cast(_JsonBlocks | None, request.get("tools")), # cast-ok: handed on unchecked, as the handler does + api_base=optional_str(request.get("api_base")), + ) + + +def short_circuit(request: Mapping[str, object], stream: bool, hooked: bool) -> Awaitable[object]: + kwargs: Final = {name: value for name, value in request.items() if name not in _ENTRY_PARAMS} + messages, _ = _entry_messages(request, kwargs, hooked) + return handler._try_websearch_short_circuit( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownVariableType] # the handler's own short-circuit, which annotates bare dicts + model=str(request["model"]), + messages=messages, + tools=cast(_JsonBlocks | None, request.get("tools")), # cast-ok: handed on unchecked, as the handler does + custom_llm_provider=_intercepted_provider(request), + stream=stream or bool(kwargs.get(WEBSEARCH_CONVERTED_STREAM_KEY, False)), + kwargs={**kwargs, "metadata": request.get("metadata")}, + ) + + +def answer(owner: MessagesInterceptor, request: Mapping[str, object], stream: bool, hooked: bool) -> Awaitable[object]: + model: Final = str(request["model"]) + tools: Final = cast(_JsonBlocks | None, request.get("tools")) # cast-ok: handed on unchecked, as the handler does + kwargs: Final = {name: value for name, value in request.items() if name not in _ENTRY_PARAMS} + messages, system = _entry_messages(request, kwargs, hooked) + handle: Final = cast( # cast-ok: the interceptor answers with whatever the caller receives + Callable[..., Awaitable[object]], owner.handle + ) + return handle( + model=model, + messages=messages, + tools=tools, + stream=stream or kwargs.get("_websearch_interception_converted_stream", False), + max_tokens=cast(int, request["max_tokens"]), # cast-ok: admission requires an int max_tokens + custom_llm_provider=_intercepted_provider(request), + api_key=request.get("api_key"), + api_base=request.get("api_base"), + metadata=request.get("metadata"), + stop_sequences=request.get("stop_sequences"), + system=system, + temperature=request.get("temperature"), + thinking=request.get("thinking"), + tool_choice=request.get("tool_choice"), + top_k=request.get("top_k"), + top_p=request.get("top_p"), + **kwargs, + ) + + def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: from litellm.llms.anthropic.messages.native.streaming_iterator import ( anthropic_messages_stream_hidden_params, ) - return anthropic_messages_stream_hidden_params(httpx.Headers(list(headers))) + return MessagesStreamHead(anthropic_messages_stream_hidden_params(callbacks_legacy_python.wire_headers(headers))) + + +async def _unrequested_stream_delivered( + stream: object, + kwargs: dict[str, object], # mutable-ok: the call's own kwargs, as `@client` hands them on +) -> object: + try: + delivered: Final = await callbacks_legacy_python.after_deployment_success( + kwargs, stream, CallTypes.anthropic_messages.value + ) + function: Final = cached_function() + caching: Final = LLMCachingHandler( + original_function=function, + request_kwargs=kwargs, + start_time=datetime.datetime.now(), # noqa: DTZ005 # @client's naive start time + ) + store: Final = cast( # cast-ok: the handler leaves its original_function untyped + Callable[..., Awaitable[None]], caching.async_set_cache + ) + await store(result=delivered, original_function=function, kwargs=kwargs) + except BaseException: + from litellm.llms.anthropic.messages.native.streaming_iterator import aclose_if_supported + + await aclose_if_supported(stream) + raise + return delivered + + +def unrequested_stream( + head: object, + kwargs: dict[str, object], # mutable-ok: the call's own kwargs, as `@client` hands them on +) -> None: + if not isinstance(head, MessagesStreamHead): + return + deliver: Final = functools.partial(_unrequested_stream_delivered, kwargs=kwargs) + head.deliver = deliver # rebind-ok: the head carries the delivery back _UPSTREAM_ARGS: Final = TypeAdapter(tuple[int, str]) @@ -129,13 +522,17 @@ _UPSTREAM_HEADERS: Final = TypeAdapter(list[tuple[str, str]]) _BODY_CUT: Final = TypeAdapter(tuple[int, int | None, bool]) _ERROR_CLASS: TypeAlias = Callable[..., Exception] _validate_metadata: Final = cast( # cast-ok: the handler's untyped metadata validator - Callable[[dict[str, object]], object], handler.validate_anthropic_api_metadata + Callable[[object], object], handler.validate_anthropic_api_metadata +) +_mock_response: Final = cast( # cast-ok: the handler's mock builder takes untyped messages + Callable[..., Mapping[str, object]], native_utils.mock_response ) _exception_type: Final = cast( # cast-ok: the legacy public exception mapper is untyped Callable[..., Exception], exception_mapping_utils.exception_type ) _BODY_READ_TIMEOUT: Final = "Timeout on reading data from socket" # aiohttp's SocketTimeoutError text _UNREACHABLE: Final = "could not reach the provider: " +_TOO_MANY_REDIRECTS: Final = "upstream network error: error following redirect: too many redirects" _SEND_FAILED: Final = "upstream network error: error sending request: client error (SendRequest): " _DNS_FAILURE: Final = "failed to lookup address information: " _HTTPX_DNS_ERRNO: Final = { @@ -161,6 +558,12 @@ def _rejection(error: Exception, request: Mapping[str, object], request_provider _validate_metadata(_METADATA.validate_python(metadata)) except ValidationError as invalid: return invalid + provider_rejection: Final = getattr(error, "messages_provider_rejection", None) + if isinstance(provider_rejection, str): + _log_resolved_call(request, request_provider) + return _public_provider_error( + _config_error(400, provider_rejection, (), request, request_provider), error, request, request_provider + ) if getattr(error, "messages_unsupported_param", False): return litellm.UnsupportedParamsError(message=str(error)) return litellm.BadRequestError( @@ -251,7 +654,12 @@ def _aiohttp_connect_failure(url: httpx.URL, port: int, request: Mapping[str, ob def _transport_failure(error: Exception, request: Mapping[str, object], request_provider: str) -> str | None: """The text Python's HTTP client raises for a call that failed before any response arrived.""" + body_cut: Final = getattr(error, "messages_body_cut", None) + if body_cut is not None: + return str(_body_cut(*_BODY_CUT.validate_python(body_cut))) native: Final = str(error) + if native == _TOO_MANY_REDIRECTS: + return "Exceeded maximum allowed redirects." if native.startswith(_SEND_FAILED): reset: Final = native.endswith(f"Connection reset by peer (os error {errno.ECONNRESET})") closed: Final = native.endswith("connection closed before message completed") @@ -301,6 +709,20 @@ def _provider_error(error: Exception, request: Mapping[str, object], request_pro if failure is None: return None status, text, headers = failure + provider_error: Final = _config_error(status, text, headers, request, request_provider) + if transport is not None and isinstance(provider_error, BaseLLMException): + # Python's `_handle_error` gives a failure without a response the status 500. + provider_error.status_code_is_synthesized = True + return provider_error + + +def _config_error( + status: int, + text: str, + headers: Sequence[tuple[str, str]], + request: Mapping[str, object], + request_provider: str, +) -> Exception: api_base: Final = request.get("api_base") config: Final = native_messages_config( str(request["model"]).removeprefix(f"{request_provider}/"), @@ -308,17 +730,26 @@ def _provider_error(error: Exception, request: Mapping[str, object], request_pro request.get("model_info"), api_base if isinstance(api_base, str) else None, ) - provider_error: Final = ( - cast(_ERROR_CLASS, config.get_error_class)( # cast-ok: the config's untyped headers parameter - error_message=text, status_code=status, headers=dict(headers) - ) - if config is not None - else AnthropicError(status_code=status, message=text, headers=httpx.Headers(headers)) + if config is None: + return AnthropicError(status_code=status, message=text, headers=httpx.Headers(headers)) + return cast(_ERROR_CLASS, config.get_error_class)( # cast-ok: the config's untyped headers parameter + error_message=text, status_code=status, headers=dict(headers) ) - if transport is not None and isinstance(provider_error, BaseLLMException): - # Python's `_handle_error` gives a failure without a response the status 500. - provider_error.status_code_is_synthesized = True - return provider_error + + +def _public_provider_error( + provider_error: Exception, error: Exception, request: Mapping[str, object], request_provider: str +) -> Exception: + try: + return _exception_type( + model=str(request["model"]), + custom_llm_provider=request_provider, + original_exception=provider_error, + extra_kwargs=dict(request), + ) + except Exception as mapped: # noqa: BLE001 # exception_type raises the mapped public error + mapped.__context__ = error + return mapped def _log_resolved_call(request: Mapping[str, object], request_provider: str) -> None: @@ -331,20 +762,22 @@ def _log_resolved_call(request: Mapping[str, object], request_provider: str) -> logger, kwargs={name: value for name, value in request.items() if name != "metadata"}, model=str(request["model"]).removeprefix(f"{request_provider}/"), - optional_params=dict(body_params(request)), - litellm_params={"model_info": request.get("model_info"), **body_params(request)}, + optional_params=dict(_body_fields(request)), + litellm_params={"model_info": request.get("model_info"), **_body_fields(request)}, custom_llm_provider=request_provider, ) -def map_failure(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception: +def map_failure( + error: Exception, request: Mapping[str, object], request_provider: str, asynchronous: bool = False +) -> Exception: """Raises what Python's Messages handler raises for the same failure.""" if getattr(error, "messages_timeout", None) == "Body": # Python reads a stream's body outside its HTTP client's error mapping, so the transport's # own timeout reaches the caller unmapped. - return httpx.ReadTimeout(_BODY_READ_TIMEOUT) + return httpx.ReadTimeout(_BODY_READ_TIMEOUT if http_transport.uses_aiohttp_transport() else "") body_cut: Final = getattr(error, "messages_body_cut", None) - if body_cut is not None: + if body_cut is not None and request.get("stream") is True: # Python reads a stream's body outside its HTTP client's error mapping, so a body the # connection cut short reaches the caller as its transport raised it. return _body_cut(*_BODY_CUT.validate_python(body_cut)) @@ -356,6 +789,11 @@ def map_failure(error: Exception, request: Mapping[str, object], request_provide error.__context__ = chained # rebind-ok: Python chains the transport timeout as the exception context if getattr(error, "messages_late_rejection", False): _log_resolved_call(request, request_provider) + if getattr(error, "messages_python_raised", False): + # Python's request transform raises this builtin exception unmapped, e.g. the TypeError + # of comparing a string max_tokens while capping the thinking budget. + _flag_messages_call(request.get("litellm_logging_obj"), asynchronous) + return error if getattr(error, "messages_request_error", False): return _rejection(error, request, request_provider) if getattr(error, "messages_missing_credential", False): @@ -369,28 +807,759 @@ def map_failure(error: Exception, request: Mapping[str, object], request_provide return failures.map_native_failure( error, str(request["model"]), request_provider, request, optional_str(request.get("api_base")) ) - try: - return _exception_type( - model=str(request["model"]), - custom_llm_provider=request_provider, - original_exception=provider_error, - extra_kwargs=dict(request), - ) - except Exception as mapped: # noqa: BLE001 # exception_type raises the mapped public error - mapped.__context__ = error - return mapped + return _public_provider_error(provider_error, error, request, request_provider) def _drop_params(kwargs: Mapping[str, object]) -> bool: return bool(litellm.drop_params) or normalize_drop_params(kwargs.get("drop_params")) is True -def settings(kwargs: Mapping[str, object]) -> dict[str, object]: +def _python_handler_params(request: Mapping[str, object], custom_llm_provider: object) -> GenericLiteLLMParams: + return GenericLiteLLMParams( + **{name: value for name, value in request.items() if name not in _HANDLER_PARAMS}, + api_key=request.get("api_key"), + api_base=request.get("api_base"), + custom_llm_provider=custom_llm_provider, + ) + + +def _resolved_provider(request: Mapping[str, object]) -> str | None: + return optional_str(request.get("custom_llm_provider")) or inferred_provider( + str(request["model"]), optional_str(request.get("api_base")) + ) + + +def _scope_provider_specific_header(request: Mapping[str, object]) -> None: + scoped: Final = request.get("provider_specific_header") + if scoped is None: + return + ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=cast( # cast-ok: the handler hands the caller's value on unvalidated + ProviderSpecificHeader | None, scoped + ), + custom_llm_provider=_resolved_provider(request), + native_anthropic_messages=True, + ) + + +def _python_messages_url(api_base: str | None) -> str: + """`AnthropicMessagesConfig.get_complete_url`, untrimmed, whose `urlsplit` raises a bare + ValueError for a base it cannot split (``Invalid IPv6 URL``).""" + return ensure_api_route(AnthropicModelInfo.get_api_base(api_base) or "https://api.anthropic.com", "v1/messages") + + +def _raise_as_python_does(request: Mapping[str, object], asynchronous: bool) -> None: + iter(cast(Iterable[object], request.get("messages"))) # cast-ok: iterated unchecked, as Python's handler does + tools: Final = request.get("tools") + if tools is not None and not isinstance(tools, list | tuple) and has_interceptors(): + interceptor(request) + _python_handler_params(request, request.get("custom_llm_provider")) + # Python's handler scopes the caller's provider_specific_header before it builds any header, so + # a value of the wrong shape raises its own error (a str: "'str' object has no attribute 'get'"), + # and then builds the URL, unmapped. + logger: Final = request.get("litellm_logging_obj") + try: + _scope_provider_specific_header(request) + except Exception: + _flag_messages_call(logger, asynchronous) + raise + api_base: Final = request.get("api_base") + if not isinstance(api_base, str) or _resolved_provider(request) != "anthropic": + return + try: + _python_messages_url(api_base) + except ValueError: + _log_resolved_call(request, "anthropic") + _flag_messages_call(logger, asynchronous) + raise + + +def settings( + kwargs: Mapping[str, object], request: Mapping[str, object] | None = None, asynchronous: bool = False +) -> dict[str, object]: + if request is not None: + _raise_as_python_does(request, asynchronous) + aiohttp: Final = http_transport.uses_aiohttp_transport() + configured: Final = MessagesSettings( + drop_params=_drop_params(kwargs), + reasoning_auto_summary=is_reasoning_auto_summary_enabled(), + additional_drop_params=handler.additional_drop_paths(kwargs), + request_timeout=get_configured_request_timeout(), + drops_unsupported_params=drops_unsupported_params(kwargs.get("drop_params")), + ascii_response_headers=aiohttp, + joined_reads=aiohttp, + ) return asdict( - MessagesSettings( - drop_params=_drop_params(kwargs), - reasoning_auto_summary=is_reasoning_auto_summary_enabled(), - additional_drop_params=handler.additional_drop_paths(kwargs), - request_timeout=get_configured_request_timeout(), + configured if request is None else _with_call_settings(configured, request, kwargs.get("litellm_logging_obj")) + ) + + +def _flag_messages_call(logger: object, asynchronous: bool) -> None: + if not isinstance(logger, LiteLLMLoggingObj): + return + logger.model_call_details.setdefault("litellm_params", {})[CallTypes.aanthropic_messages.value] = asynchronous + + +@dataclass(frozen=True, slots=True) +class _Answered: + value: object + + def send(self, _value: object) -> execution.Complete: + return execution.Complete(self.value) + + def throw(self, error: BaseException) -> execution.Step: + raise error + + def close(self) -> None: + return None + + +def sync_answer(request: Mapping[str, object]) -> _Answered | None: + logger: Final = request.get("litellm_logging_obj") + mock: Final = _mock(request, logger, asynchronous=False) + if mock is not None: + _flag_messages_call(logger, asynchronous=False) + return _Answered(response(mock)) + served: Final = gateway(request, asynchronous=False) + return None if served is None else _Answered(served) + + +def _handler_params(request: Mapping[str, object]) -> GenericLiteLLMParams: + return GenericLiteLLMParams( + **{name: value for name, value in request.items() if name not in _HANDLER_PARAMS}, + api_key=optional_str(request.get("api_key")), + api_base=optional_str(request.get("api_base")), + custom_llm_provider=optional_str(request.get("custom_llm_provider")), + ) + + +def _requested_model(request: Mapping[str, object]) -> str: + model, _, _, _ = _requested_provider( + str(request["model"]), optional_str(request.get("custom_llm_provider")), _handler_params(request) + ) + return model + + +def _model_without_latest_alias(request: Mapping[str, object], model: str, custom_llm_provider: str | None) -> str: + if not is_model_latest_alias(model, custom_llm_provider): + return model + resolved, provider, _, _ = _requested_provider(model, custom_llm_provider, _handler_params(request)) + prefix, separator, _ = model.partition("/") + return f"{prefix}/{resolved}" if separator and prefix == provider else resolved + + +def _mock(request: Mapping[str, object], logger: object, asynchronous: bool) -> Mapping[str, object] | None: + mock: Final = request.get("mock_response") + if not mock or not isinstance(mock, str): + return None + _validate_metadata(request.get("metadata")) + messages: Final = request.get("messages") + max_tokens: Final = request.get("max_tokens") + model: Final = _requested_model(request) + try: + return dict( + _mock_response( + model=model, + messages=messages if isinstance(messages, list) else [], + max_tokens=max_tokens if isinstance(max_tokens, int) else 0, + mock_response=mock, + ) + ) + except Exception: + _flag_messages_call(logger, asynchronous) + raise + + +def _stream_cost_model(request: Mapping[str, object]) -> str | None: + if request.get("stream") is not True or not stream_cost_injection_requested(proxy_request_headers(request)): + return None + return _requested_model(request) + + +def _affinity_extra_headers( + request: Mapping[str, object], model: str, explicit_provider: str | None +) -> Mapping[str, object] | None: + api_base: Final = optional_str(request.get("api_base")) + anthropic_header: Final = get_provider_affinity_header(AnthropicModelInfo.get_api_base(api_base)) + other_header: Final = get_provider_affinity_header(api_base) + if anthropic_header is None and other_header is None: + return None + provider: Final = explicit_provider or inferred_provider(model, api_base) + default_header: Final = anthropic_header if provider == "anthropic" else other_header + if default_header is None: + return None + headers: Final = as_str_mapping(request.get("headers")) + extra_headers: Final = as_str_mapping(request.get("extra_headers")) + merged: Final = _merged_headers( + headers, + extra_headers, + ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=cast( # cast-ok: the handler hands the caller's value on unvalidated + ProviderSpecificHeader | None, request.get("provider_specific_header") + ), + custom_llm_provider=provider, + native_anthropic_messages=True, + ), + ) + added: Final = add_provider_affinity_header(merged, _handler_params(request), default_header) + affinity: Final = {name: value for name, value in added.items() if name not in merged} + if not affinity: + return None + return dict(_merged_headers(headers, extra_headers, affinity)) + + +def _with_call_settings( + configured: MessagesSettings, request: Mapping[str, object], logger: object +) -> MessagesSettings: + explicit_provider: Final = optional_str(request.get("custom_llm_provider")) + model: Final = _model_without_latest_alias(request, str(request["model"]), explicit_provider) + mock: Final = _mock(request, logger, asynchronous=True) + if mock is not None: + return replace(configured, model=model, mock_response=mock) + return replace( + configured, + model=model, + stream_cost_model=_stream_cost_model(request), + extra_headers=_affinity_extra_headers(request, model, explicit_provider), + mcp_gateway=_takes_mcp_gateway(request), + ) + + +def _takes_mcp_gateway(request: Mapping[str, object]) -> bool: + tools: Final = optional_sequence(request.get("tools")) + return not request.get("_skip_mcp_handler", False) and bool(tools) and may_name_litellm_mcp_gateway(tools) + + +MessagesResult: TypeAlias = AnthropicMessagesResponse | Iterator[bytes] | AsyncIterator[object] + + +def gateway( + request: Mapping[str, object], asynchronous: bool, hooked: bool = False +) -> Coroutine[object, object, MessagesResult] | None: + if not _takes_mcp_gateway(request): + return None + model: Final = str(request["model"]) + provider: Final = optional_str(request.get("custom_llm_provider")) + api_base: Final = optional_str(request.get("api_base")) + tools: Final = cast(_JsonBlocks, request.get("tools")) # cast-ok: handed on unchecked, as the handler does + messages: Final = cast(_JsonBlocks, request["messages"]) # cast-ok: handed on unchecked, as the handler does + system: Final = cast( # cast-ok: handed on unchecked, as the handler does + str | _JsonBlocks | None, request.get("system") + ) + if not asynchronous: + return _gateway(request, messages, system, model=model, provider=provider, tools=tools, presanitized=False) + entry_kwargs: Final = {name: value for name, value in request.items() if name not in _ENTRY_PARAMS} + entry_messages, entry_system = ( + (messages, system) + if hooked + else prepare_native_messages( + messages, system, entry_kwargs, model=model, custom_llm_provider=provider, tools=tools, api_base=api_base ) ) + return _gateway( + {**request, **entry_kwargs, "is_async": True}, + entry_messages, + entry_system, + model=model, + provider=handler.provider_after_pre_request_hooks(model, provider, entry_kwargs), + tools=tools, + presanitized=True, + ) + + +def _gateway( + call: Mapping[str, object], + messages: _JsonBlocks, + system: str | _JsonBlocks | None, + *, + model: str, + provider: str | None, + tools: _JsonBlocks, + presanitized: bool, +) -> Coroutine[object, object, MessagesResult]: + kwargs: Final = {name: value for name, value in call.items() if name not in _HANDLER_PARAMS} + api_key: Final = optional_str(call.get("api_key")) + api_base: Final = optional_str(call.get("api_base")) + prepared_messages, prepared_system = prepare_native_messages( + messages, + system, + kwargs, + model=model, + custom_llm_provider=provider, + tools=tools, + api_base=api_base, + presanitized=presanitized, + ) + metadata: Final = _validate_metadata(call.get("metadata")) + is_async: Final = kwargs.pop("is_async", False) + logging_obj: Final = kwargs.get("litellm_logging_obj") + litellm_params: Final = GenericLiteLLMParams( + **kwargs, api_key=api_key, api_base=api_base, custom_llm_provider=provider + ) + resolved_model, resolved_provider, dynamic_api_key, dynamic_api_base = _requested_provider( + model, provider, litellm_params + ) + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.model_call_details["agentic_loop_params"] = { + "model": model, + "custom_llm_provider": resolved_provider, + } + logging_obj.model_call_details.setdefault("litellm_params", {})[CallTypes.aanthropic_messages.value] = is_async + if kwargs.get("_websearch_interception_converted_stream", False): + logging_obj.model_call_details["websearch_interception_converted_stream"] = True + bridge: Final = _bridged( + _BridgeTarget(model=model, api_key=api_key, api_base=api_base), + _BridgeTarget( + model=resolved_model, + api_key=None if _own_entra_credential(litellm_params) else dynamic_api_key, + api_base=dynamic_api_base, + ), + resolved_provider, + ) + kwargs.pop("_skip_mcp_handler", False) + from litellm.llms.anthropic.messages.native.mcp_handler import anthropic_messages_with_mcp + + return anthropic_messages_with_mcp( + max_tokens=cast(int, call["max_tokens"]), # cast-ok: handed on unchecked, as the handler does + messages=prepared_messages, + model=resolved_model, + metadata=metadata, + stop_sequences=call.get("stop_sequences"), + stream=call.get("stream", False), + system=prepared_system, + temperature=call.get("temperature"), + thinking=call.get("thinking"), + tool_choice=call.get("tool_choice"), + tools=tools, + top_k=call.get("top_k"), + top_p=call.get("top_p"), + container=call.get("container"), + api_key=bridge.api_key, + api_base=bridge.api_base, + client=call.get("client"), + custom_llm_provider=resolved_provider, + **kwargs, + ) + + +def inject_stream_cost(frames: bytes, model: str, request: Mapping[str, object]) -> object: + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + logger: Final = request.get("litellm_logging_obj") + logging_obj: Final = logger if isinstance(logger, LiteLLMLoggingObj) else None + return ProxyBaseLLMRequestProcessing.process_chunk_with_cost_injection( + frames, + model, + logging_obj, + request_headers=proxy_request_headers( + as_str_mapping(logging_obj.model_call_details.get("litellm_params")) if logging_obj else None + ), + ) + + +def workload_identity_token(params: Mapping[str, object], api_base: str | None) -> str | None: + litellm_params: Final = GenericLiteLLMParams(**params) + return get_anthropic_wif_token(dict(litellm_params), api_base, "") + + +class RawTransportFailure(Exception): + def __init__(self, error: Exception) -> None: + super().__init__(str(error)) + self.error = error + + +@dataclass(frozen=True, slots=True) +class Sent: + status: int + headers: Sequence[tuple[str, str]] + text: str | None = None + chunks: AsyncIterator[bytes] | None = None + rejection: httpx.HTTPStatusError | None = None + + +def _unsendable_api_base(api_base: object) -> bool: + """A caller api_base the native URL parser rejects or reads otherwise than httpx and aiohttp + (no http(s) scheme, whitespace, control or non-ASCII characters): Python's client decides what + such a URL does, from a relative-URL error to a request on the host it finds.""" + if not isinstance(api_base, str) or not api_base: + return False + return not ( + api_base.isascii() + and api_base.isprintable() + and " " not in api_base + and api_base.lower().startswith(("http://", "https://")) + ) + + +def transport_handler(request: Mapping[str, object]) -> AsyncHTTPHandler | None: + client: Final = request.get("client") + if isinstance(client, AsyncHTTPHandler): + return client + if not http_transport.sends_through_python(client, request.get("timeout")) and not _unsendable_api_base( + request.get("api_base") + ): + return None + return get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) + + +def decoding_failure(message: str, request: Mapping[str, object], request_provider: str) -> Exception: + error: Final = httpx.DecodingError(message) + if request.get("stream") is True: + return error + return map_transport_failure(error, request, request_provider) + + +def unencodable_header(value: bytes, received: bool) -> Exception: + text: Final = value.decode("utf-8", "surrogateescape" if received else "surrogatepass") + try: + text.encode("ascii") + except UnicodeEncodeError as unencodable: + return unencodable + return ValueError(text) + + +def _sent_url(request: Mapping[str, object], url: str) -> str: + """The native URL builder trims the caller's api_base; Python sends it as given.""" + api_base: Final = request.get("api_base") + if not isinstance(api_base, str) or api_base == api_base.strip() or not _unsendable_api_base(api_base): + return url + try: + return _python_messages_url(api_base) if url == _python_messages_url(api_base.strip()) else url + except ValueError: # a base urlsplit cannot split keeps the URL the native builder made + return url + + +async def transport_send( + handler: AsyncHTTPHandler, + request: Mapping[str, object], + url: str, + headers: Sequence[tuple[str, str]], + body: object, + stream: bool, +) -> Sent: + try: + response: Final = await handler.post( # pyright: ignore[reportUnknownMemberType] # the handler's untyped dict params + url=_sent_url(request, url), + headers=dict(headers), + data=body if isinstance(body, bytes) else json.dumps(body), + stream=stream, + logging_obj=cast( # cast-ok: the call's logger, handed on as Python's handler hands it + "LiteLLMLoggingObj | None", request.get("litellm_logging_obj") + ), + timeout=cast( # cast-ok: the call's timeout as the handler resolved it for the post + "float | httpx.Timeout | None", request.get("timeout") + ), + ) + response.raise_for_status() + except httpx.HTTPStatusError as rejection: + try: + text: Final = rejection.response.text + except Exception as unreadable: # noqa: BLE001 # Python's retry check raises it from its except block, unmapped + raise RawTransportFailure(unreadable) from None + return Sent(rejection.response.status_code, rejection.response.headers.multi_items(), text, None, rejection) + if stream: + return Sent(response.status_code, response.headers.multi_items(), chunks=response.aiter_bytes()) + return Sent(response.status_code, response.headers.multi_items(), text=response.text) + + +def map_transport_failure(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception: + api_base: Final = request.get("api_base") + config: Final = native_messages_config( + str(request["model"]).removeprefix(f"{request_provider}/"), + request_provider, + request.get("model_info"), + api_base if isinstance(api_base, str) else None, + ) + try: + handler.base_llm_http_handler._handle_error(e=error, provider_config=config) # pyright: ignore[reportPrivateUsage, reportArgumentType] # the handler's own error mapping, which always raises + except Exception as provider_error: # noqa: BLE001 # _handle_error raises the provider exception it builds + provider_error.__context__ = error + if not isinstance(provider_error, BaseLLMException): + return provider_error + try: + return _exception_type( + model=str(request["model"]), + custom_llm_provider=request_provider, + original_exception=provider_error, + extra_kwargs=dict(request), + ) + except Exception as mapped: # noqa: BLE001 # exception_type raises the mapped public error + mapped.__context__ = provider_error + return mapped + return error + + +_AGENTIC_UNSENT_PARAMS: Final = frozenset({"model", "messages", "max_tokens"}) +_HANDLER_POPPED: Final = frozenset( + {"is_async", "_litellm_messages_presanitized", "_skip_mcp_handler", "cache_control_injection_points"} +) + + +def _logger(request: Mapping[str, object]) -> LiteLLMLoggingObj | None: + logger: Final = request.get("litellm_logging_obj") + return logger if isinstance(logger, LiteLLMLoggingObj) else None + + +def runs_agentic_hooks(request: Mapping[str, object]) -> bool: + logger: Final = _logger(request) + if logger is None: + return False + return BaseLLMHTTPHandler._has_agentic_completion_hook(logger) or bool( # pyright: ignore[reportPrivateUsage] # the handler's own agentic gate + request.get(WEBSEARCH_CONVERTED_STREAM_KEY, False) + ) + + +def record_agentic_loop(request: Mapping[str, object], custom_llm_provider: str) -> None: + logger: Final = _logger(request) + if logger is None: + return + logger.model_call_details["agentic_loop_params"] = { + "model": request["model"], + "custom_llm_provider": custom_llm_provider, + } + if request.get(WEBSEARCH_CONVERTED_STREAM_KEY, False): + logger.model_call_details["websearch_interception_converted_stream"] = True + + +@dataclass(frozen=True, slots=True) +class _AgenticCall: + model: str + messages: list[dict[str, object]] # mutable-ok: the handler's agentic hooks take a message list + config: BaseAnthropicMessagesConfig + params: dict[str, object] # mutable-ok: the handler's agentic hooks take a params dict + logger: LiteLLMLoggingObj + provider: str + kwargs: dict[str, object] # mutable-ok: the handler's agentic hooks take a kwargs dict + + +def _agentic_params( + sent: Mapping[str, object], requested: Mapping[str, object], request: Mapping[str, object] +) -> dict[str, object]: # mutable-ok: the handler's agentic hooks take a params dict + params: Final = {name: value for name, value in sent.items() if name not in _AGENTIC_UNSENT_PARAMS} + if "tools" not in params: + return params + caller_tools: Final = request.get("tools") + requested_tools: Final = requested.get("tools") + return {**params, "tools": caller_tools if caller_tools == requested_tools else requested_tools} + + +def _agentic_call( + request: Mapping[str, object], + sent: Mapping[str, object], + model: str, + custom_llm_provider: str, + requested: Mapping[str, object], + hooked: bool, +) -> _AgenticCall: + logger: Final = _logger(request) + api_base: Final = optional_str(request.get("api_base")) + config: Final = native_messages_config(model, custom_llm_provider, request.get("model_info"), api_base) + if logger is None or config is None: + raise ValueError("an agentic Messages call needs its logger and provider config") + kwargs: Final = { + name: value for name, value in request.items() if name not in _HANDLER_PARAMS and name not in _HANDLER_POPPED + } + messages, _, _ = _prompt_cached(request, asynchronous=True, hooked=hooked) + return _AgenticCall( + model=model, + messages=cast( # cast-ok: the handler hands its prepared messages to the hooks as dicts + list[dict[str, object]], strip_provider_specific_fields_from_anthropic_messages(messages) + ), + config=config, + params=_agentic_params(sent, requested, request), + logger=logger, + provider=custom_llm_provider, + kwargs=BaseLLMHTTPHandler._agentic_hook_kwargs( # pyright: ignore[reportPrivateUsage] # the handler's own follow-up kwargs + kwargs=kwargs, api_key=optional_str(request.get("api_key")), api_base=api_base + ), + ) + + +async def _failed_deployment(request: Mapping[str, object], error: Exception) -> None: + from litellm.utils import async_post_call_failure_deployment_hook + + try: + await async_post_call_failure_deployment_hook( + request_data=request, exception=error, call_type=CallTypes.anthropic_messages.value + ) + except asyncio.CancelledError: + raise + except BaseException: # noqa: BLE001, S110 # a failing hook never replaces the agentic loop's own failure, as in `@client` + pass + + +async def agentic_response( + response: object, + request: Mapping[str, object], + sent: Mapping[str, object], + model: str, + custom_llm_provider: str, + requested: Mapping[str, object], + hooked: bool, +) -> object: + try: + call: Final = _agentic_call(request, sent, model, custom_llm_provider, requested, hooked) + initial: Final = cast(AnthropicMessagesResponse, response) # cast-ok: the public response the handler finishes + return await handler.base_llm_http_handler._finalize_anthropic_messages_response( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownVariableType] # the handler's own agentic finish, typed with bare dicts + initial_response=initial, + model=call.model, + messages=call.messages, + anthropic_messages_provider_config=call.config, + anthropic_messages_optional_request_params=call.params, + logging_obj=call.logger, + custom_llm_provider=call.provider, + kwargs=call.kwargs, + ) + except Exception as error: + await _failed_deployment(request, error) + raise + + +def _agentic_stream(stream: AsyncStream, call: _AgenticCall, held: frozenset[str]) -> object: + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import AgenticAnthropicStreamingIterator + from litellm.llms.anthropic.messages.native.streaming_iterator import ( + AnthropicMessagesStreamHiddenParams, + AnthropicMessagesStreamingResponse, + ) + + return AnthropicMessagesStreamingResponse( + completion_stream=AgenticAnthropicStreamingIterator( + completion_stream=stream, + http_handler=handler.base_llm_http_handler, + model=call.model, + messages=call.messages, + anthropic_messages_provider_config=call.config, + anthropic_messages_optional_request_params=call.params, + logging_obj=call.logger, + custom_llm_provider=call.provider, + kwargs=call.kwargs, + hold_back=bool(held), + server_fulfilled_tool_names=held, + ), + hidden_params=cast( # cast-ok: the stream's stamped hidden params, shared as Python's handler shares them + AnthropicMessagesStreamHiddenParams, + stream._hidden_params, # pyright: ignore[reportPrivateUsage] # the native stream's own hidden params + ), + ) + + +def agentic_stream( + head: object, + request: Mapping[str, object], + sent: Mapping[str, object], + model: str, + custom_llm_provider: str, + requested: Mapping[str, object], + hooked: bool, +) -> None: + logger: Final = _logger(request) + if ( + not isinstance(head, MessagesStreamHead) + or logger is None + or not BaseLLMHTTPHandler._has_agentic_completion_hook(logger) # pyright: ignore[reportPrivateUsage] # the handler's own agentic gate + ): + return + call: Final = _agentic_call(request, sent, model, custom_llm_provider, requested, hooked) + held: Final = BaseLLMHTTPHandler._server_fulfilled_tools_in_request( # pyright: ignore[reportPrivateUsage] # the handler's own hold-back gate + logging_obj=logger, tools=call.params.get("tools") + ) + wrap: Final = functools.partial(_agentic_stream, call=call, held=held) + head.agentic = wrap # rebind-ok: the head carries the stream wrapper back + + +class MessagesSafeguards: + def __init__(self, request: Mapping[str, object]) -> None: + self._request: Final = request + self._frames: _StampedFrames | None = None + self._results: object = None + + async def _evaluator(self) -> SafeguardsEvaluator | None: + model, provider = _handler_model(self._request) + if not await handler.proxy_classifies_tool_uses(model, provider, self._request): + return None + messages: Final = normalize_native_messages(optional_sequence(self._request.get("messages")) or ()) + return await asyncio.to_thread( + handler._native_safeguards_evaluator, # pyright: ignore[reportPrivateUsage] # the handler's own evaluator build + self._request, + messages, + ) + + async def results(self, value: Mapping[str, object]) -> object: + evaluator: Final = await self._evaluator() + if evaluator is None: + return None + unstamped: Final = response(value) + stamped: Final = await with_safeguard_results(unstamped, evaluator) + if stamped is unstamped: + return None + self._results = stamped.get("safeguard_results") + return self._results + + def published(self, public: Mapping[str, object]) -> Mapping[str, object]: + return public if self._results is None else {**public, "safeguard_results": self._results} + + async def open_stream(self) -> bool: + evaluator: Final = await self._evaluator() + if evaluator is None: + return False + self._frames = _StampedFrames(StreamedSafeguardResults(evaluator)) + return True + + async def frames(self, chunk: bytes) -> Sequence[bytes]: + return await self._open_frames().feed(chunk) + + async def flush(self) -> Sequence[bytes]: + return await self._open_frames().flush() + + def _open_frames(self) -> _StampedFrames: + if self._frames is None: + raise RuntimeError("safeguards stream frames before the stream opened") + return self._frames + + +class _StampedFrames: + def __init__(self, safeguard_results: StreamedSafeguardResults) -> None: + self._safeguard_results: Final = safeguard_results + self._open_frame: Final = bytearray() + + async def feed(self, chunk: bytes) -> Sequence[bytes]: + from litellm.llms.anthropic.messages.native.streaming_iterator import ( + _SSE_FRAME_END, # pyright: ignore[reportPrivateUsage] # the handler's own SSE framing + _SSE_TERMINATOR_OVERLAP, # pyright: ignore[reportPrivateUsage] # the handler's own SSE framing + _ends_a_frame, # pyright: ignore[reportPrivateUsage] # the handler's own SSE framing + _stamped_sse_frame, # pyright: ignore[reportPrivateUsage] # the handler's own SSE framing + ) + + scan_from: Final = max(len(self._open_frame) - _SSE_TERMINATOR_OVERLAP, 0) + self._open_frame.extend(chunk) + if not _ends_a_frame(self._open_frame, scan_from): + return () + *frames, unterminated = _SSE_FRAME_END.split(self._open_frame) + self._open_frame[:] = unterminated + return tuple([bytes(await _stamped_sse_frame(frame, self._safeguard_results)) for frame in frames]) + + async def flush(self) -> Sequence[bytes]: + if not self._open_frame: + return () + from litellm.llms.anthropic.messages.native.streaming_iterator import ( + _stamped_sse_frame, # pyright: ignore[reportPrivateUsage] # the handler's own SSE framing + ) + + return (bytes(await _stamped_sse_frame(bytes(self._open_frame), self._safeguard_results)),) + + +def _handler_model(request: Mapping[str, object]) -> tuple[str, str | None]: + explicit_provider: Final = optional_str(request.get("custom_llm_provider")) + try: + model, provider, _, _ = litellm.get_llm_provider( + model=str(request["model"]), custom_llm_provider=explicit_provider + ) + except litellm.BadRequestError: + return str(request["model"]), explicit_provider + return model, provider + + +def safeguards(request: Mapping[str, object]) -> MessagesSafeguards | None: + if "safeguards" not in request or not native_route_classifies(str(request["model"]), request["safeguards"]): + return None + return MessagesSafeguards(request) diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py deleted file mode 100644 index 99555bb4852..00000000000 --- a/litellm/rust_bridge/messages/detectors.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Detectors for Python-only Anthropic Messages behavior.""" - -from __future__ import annotations - -from typing import Final - -import litellm -from litellm.anthropic_beta_headers_manager import is_foundry_api_base -from litellm.litellm_core_utils.core_helpers import ( - drops_unsupported_params, - proxy_request_headers, - stream_cost_injection_requested, -) -from litellm.llms.opencode.common_utils import is_opencode_api_base -from litellm.rust_bridge.detectors import api_base, overridden, present, streaming -from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str - - -def provider_affinity(request: NativeCall) -> bool: - return is_opencode_api_base(api_base(request)) - - -def foundry_beta_row(request: NativeCall) -> bool: - from litellm.llms.anthropic.common_utils import AnthropicModelInfo - - return is_foundry_api_base(AnthropicModelInfo.get_api_base(api_base(request))) - - -def anthropic_workload_identity(request: NativeCall) -> bool: - """Python mints an Anthropic federation token when no static credential is configured.""" - from litellm.llms.anthropic.common_utils import AnthropicModelInfo - from litellm.secret_managers.main import get_secret_str - - def configured(param: str, env: str) -> bool: - return present(request, param) or get_secret_str(env) is not None - - return ( - AnthropicModelInfo.get_api_key(optional_str(request.resolved.get("api_key"))) is None - and AnthropicModelInfo.get_auth_token() is None - and configured("anthropic_federation_rule_id", "ANTHROPIC_FEDERATION_RULE_ID") - and configured("anthropic_organization_id", "ANTHROPIC_ORGANIZATION_ID") - ) - - -_THINKING_BLOCKS: Final = frozenset({"thinking", "redacted_thinking"}) - - -def _replays_thinking(message: object) -> bool: - content: Final = optional_sequence((optional_mapping(message) or {}).get("content")) or () - return any((optional_mapping(block) or {}).get("type") in _THINKING_BLOCKS for block in content) - - -def thinking_signature_retry(request: NativeCall) -> bool: - """Python may strip replayed thinking blocks and retry after a rejected signature.""" - return any(_replays_thinking(message) for message in optional_sequence(request.resolved.get("messages")) or ()) - - -def tool_custom_field(request: NativeCall) -> bool: - tools: Final = optional_sequence(request.resolved.get("tools")) or () - return any("custom" in (optional_mapping(tool) or {}) for tool in tools) - - -def strict_tools_retry(request: NativeCall) -> bool: - tools: Final = optional_sequence(request.resolved.get("tools")) or () - return drops_unsupported_params(request.resolved.get("drop_params")) and any( - (optional_mapping(tool) or {}).get("strict") is True for tool in tools - ) - - -def stream_cost_injection(request: NativeCall) -> bool: - """Python adds response cost to streaming usage when the global setting or the request header asks for it.""" - return streaming(request) and stream_cost_injection_requested(proxy_request_headers(request.resolved)) - - -def pre_request_hook(_request: NativeCall) -> bool: - """The async Messages handler runs callbacks that can rewrite the request.""" - return overridden("async_pre_request_hook", litellm.callbacks) - - -def websearch_interception(_request: NativeCall) -> bool: - """The async Messages handler can answer a web-search request without an LLM call.""" - from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger - - return any(isinstance(callback, WebSearchInterceptionLogger) for callback in litellm.callbacks) - - -def _resolved_provider(request: NativeCall) -> str | None: - from litellm.exceptions import BadRequestError - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - - provider: Final = optional_str(request.resolved.get("custom_llm_provider")) - try: - return get_llm_provider(str(request.resolved["model"]), provider)[1] - except BadRequestError: - return provider - - -def intercepted(request: NativeCall) -> bool: - """The async Messages handler delegates matching tools to a registered interceptor.""" - from litellm.llms.anthropic.messages.native.interceptors import get_messages_interceptors - - tools: Final = optional_sequence(request.resolved.get("tools")) - if not tools: - return False - mappings: Final = tuple(mapping for mapping in map(optional_mapping, tools) if mapping is not None) - provider: Final = _resolved_provider(request) - return any(interceptor.can_handle(mappings, provider) for interceptor in get_messages_interceptors()) - - -def safeguards_classifier(request: NativeCall) -> bool: - """Python's proxy classifier evaluates dangerous tool use for non-Claude models.""" - from litellm.llms.anthropic.messages.safeguards import native_route_classifies - - return native_route_classifies(str(request.resolved["model"]), request.resolved.get("safeguards")) diff --git a/litellm/rust_bridge/messages/entrypoints.py b/litellm/rust_bridge/messages/entrypoints.py index 756d62b747a..ca195d99c62 100644 --- a/litellm/rust_bridge/messages/entrypoints.py +++ b/litellm/rust_bridge/messages/entrypoints.py @@ -1,6 +1,7 @@ from __future__ import annotations -from collections.abc import AsyncIterator, Awaitable, Iterator +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from dataclasses import dataclass from typing import Final, Protocol, cast # noqa: TID251 # validates dynamically loaded native callables from litellm.rust_bridge import execution @@ -9,6 +10,30 @@ from litellm.rust_bridge.public_call import NativeCall from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse +class MessagesStreamHead(dict[str, object]): + agentic: Callable[[execution.AsyncStream], object] | None = None + deliver: Callable[[object], Awaitable[object]] | None = None + + +@dataclass(frozen=True, slots=True) +class _Undelivered: + stream: object + deliver: Callable[[object], Awaitable[object]] + + +def _open_stream(native: execution.Execution, head: object) -> object: + stream: Final = execution.AsyncStream(native, head) + agentic: Final = head.agentic if isinstance(head, MessagesStreamHead) else None + opened: Final = stream if agentic is None else agentic(stream) + deliver: Final = head.deliver if isinstance(head, MessagesStreamHead) else None + return opened if deliver is None else _Undelivered(opened, deliver) + + +async def _delivered(opened: Awaitable[object]) -> object: + result: Final = await opened + return await result.deliver(result.stream) if isinstance(result, _Undelivered) else result + + class NativeMessages(Protocol): def __call__( self, @@ -33,8 +58,12 @@ def _messages_binding(value: object) -> NativeMessages | None: def _amessages_binding(value: object) -> NativeAmessages | None: if not callable(value): return None - native: Final = execution.asynchronous(value) - return cast("NativeAmessages", native) # cast-ok: the execution adapter returns the route's result + native: Final = execution.asynchronous(value, _open_stream) + + def amessages(request: NativeCall) -> Awaitable[object]: + return _delivered(native(request)) + + return cast("NativeAmessages", amessages) # cast-ok: the execution adapter returns the route's result NATIVE_MESSAGES: Final = NativeBinding("messages", validate=_messages_binding) diff --git a/litellm/rust_bridge/response_metadata.py b/litellm/rust_bridge/response_metadata.py index d2ddf61f7db..b5293bbe291 100644 --- a/litellm/rust_bridge/response_metadata.py +++ b/litellm/rust_bridge/response_metadata.py @@ -4,12 +4,15 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( _add_headers_to_response, # pyright: ignore[reportPrivateUsage] # reuse the proxy's identity-preserving response metadata writer get_hidden_params_dict, ) +from litellm.rust_bridge.catalog import Route ResultT: Final = TypeVar("ResultT") -def mark_rust_response(response: ResultT) -> ResultT: - cache_key: Final = get_hidden_params_dict(response).get("cache_key") +def mark_rust_response(response: ResultT, route: Route) -> ResultT: + if isinstance(response, dict): + return response # pyright: ignore[reportUnknownVariableType] # the caller's own dict, returned untouched + cache_key: Final = None if route is Route.MESSAGES else get_hidden_params_dict(response).get("cache_key") _add_headers_to_response( response, {"x-litellm-rust": "true", **({"x-litellm-cache-key": cache_key} if isinstance(cache_key, str) else {})}, diff --git a/litellm/rust_bridge/runtime.py b/litellm/rust_bridge/runtime.py index e22ebe73b08..6159d8de551 100644 --- a/litellm/rust_bridge/runtime.py +++ b/litellm/rust_bridge/runtime.py @@ -134,7 +134,7 @@ async def _aattempt_native( def _required(result: RustAttempt[ResultT], context: RouteContext) -> ResultT: if isinstance(result, RustHandled): - return mark_rust_response(result.value) + return mark_rust_response(result.value, context.route) _raise_required(result, _error_context(context)) diff --git a/litellm/utils.py b/litellm/utils.py index 2820638cd28..1d035215ca5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2087,6 +2087,39 @@ def _is_litellm_router_call(kwargs: Mapping[str, object]) -> bool: return any(isinstance(bucket, Mapping) and "model_group" in bucket for bucket in metadata_buckets) +def clamp_max_tokens( + model: object, + args: tuple[object, ...], + kwargs: dict[str, object], # mutable-ok: the @client wrapper rewrites the call's own kwargs + call_type: str, +) -> None: + if ( + kwargs.get("max_tokens", None) is not None + and model is not None + and litellm.modify_params is True + and call_type in (CallTypes.acompletion.value, CallTypes.completion.value, CallTypes.anthropic_messages.value) + ): + try: + base_model = model + if kwargs.get("hf_model_name", None) is not None: + base_model = f"huggingface/{kwargs.get('hf_model_name')}" + messages = None + if len(args) > 1: + messages = args[1] + elif kwargs.get("messages", None): + messages = kwargs["messages"] + kwargs["max_tokens"] = _get_modified_max_tokens()( + model=model, + base_model=base_model, + messages=messages, + user_max_tokens=kwargs.get("max_tokens"), + buffer_num=None, + buffer_perc=None, + ) + except Exception as e: + print_verbose(f"Error while checking max token limit: {e}") + + def client(original_function): from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit @@ -2203,38 +2236,7 @@ def client(original_function): verbose_logger.debug("Cache hit!") return caching_handler_response.cached_result - # CHECK MAX TOKENS - if ( - kwargs.get("max_tokens", None) is not None - and model is not None - and litellm.modify_params is True # user is okay with params being modified - and ( - call_type == CallTypes.acompletion.value - or call_type == CallTypes.completion.value - or call_type == CallTypes.anthropic_messages.value - ) - ): - try: - base_model = model - if kwargs.get("hf_model_name", None) is not None: - base_model = f"huggingface/{kwargs.get('hf_model_name')}" - messages = None - if len(args) > 1: - messages = args[1] - elif kwargs.get("messages", None): - messages = kwargs["messages"] - user_max_tokens: Final = kwargs.get("max_tokens") - modified_max_tokens: Final = _get_modified_max_tokens()( - model=model, - base_model=base_model, - messages=messages, - user_max_tokens=user_max_tokens, - buffer_num=None, - buffer_perc=None, - ) - kwargs["max_tokens"] = modified_max_tokens - except Exception as e: - print_verbose(f"Error while checking max token limit: {e}") + clamp_max_tokens(model, args, kwargs, call_type) # MODEL CALL result = original_function(*args, **kwargs) end_time = datetime.datetime.now() @@ -2499,38 +2501,7 @@ def client(original_function): if _llm_caching_handler.preset_cache_key is not None: logging_obj.litellm_params["preset_cache_key"] = _llm_caching_handler.preset_cache_key - # CHECK MAX TOKENS - if ( - kwargs.get("max_tokens", None) is not None - and model is not None - and litellm.modify_params is True # user is okay with params being modified - and ( - call_type == CallTypes.acompletion.value - or call_type == CallTypes.completion.value - or call_type == CallTypes.anthropic_messages.value - ) - ): - try: - base_model = model - if kwargs.get("hf_model_name", None) is not None: - base_model = f"huggingface/{kwargs.get('hf_model_name')}" - messages = None - if len(args) > 1: - messages = args[1] - elif kwargs.get("messages", None): - messages = kwargs["messages"] - user_max_tokens: Final = kwargs.get("max_tokens") - modified_max_tokens: Final = _get_modified_max_tokens()( - model=model, - base_model=base_model, - messages=messages, - user_max_tokens=user_max_tokens, - buffer_num=None, - buffer_perc=None, - ) - kwargs["max_tokens"] = modified_max_tokens - except Exception as e: - print_verbose(f"Error while checking max token limit: {e}") + clamp_max_tokens(model, args, kwargs, call_type) # MODEL CALL call_kwargs: Final = ( diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py index 41008f0d57f..224af4d5613 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py @@ -20,25 +20,68 @@ R request, S response, C callbacks, F covered by a fallback detector today) """ import asyncio +import contextlib import copy +import datetime +import gzip import json +import logging +import math import os import re -from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Iterable, Iterator, Mapping +import time +import uuid +from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Awaitable, + Callable, + Generator, + Iterable, + Iterator, + Mapping, + Sequence, +) from contextlib import contextmanager from dataclasses import asdict, dataclass, field, replace from types import MappingProxyType -from typing import Final, cast +from typing import ClassVar, Final, cast +import httpx import pytest from pydantic import JsonValue, TypeAdapter import litellm +from litellm._logging import verbose_logger +from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping +from litellm.caching.caching import Cache, CacheMode +from litellm.caching.caching_handler import ( + _PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # a primed entry must land before the measured call reads it +) +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger +from litellm.litellm_core_utils import logging_utils from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.anthropic.messages.native import interceptors as messages_interceptors +from litellm.llms.anthropic.messages.native.fake_stream_iterator import FakeAnthropicMessagesStreamIterator +from litellm.llms.anthropic.messages.native.interceptors import MessagesInterceptor +from litellm.llms.base_llm.search.transformation import SearchResponse +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.messages import dispatch +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.hooks.message_thread_ownership import MessageThreadOwnership +from litellm.proxy.utils import InternalUsageCache +from litellm.router_utils.cooldown_handlers import is_message_thread_not_found from litellm.rust_bridge import catalog -from litellm.rust_bridge.catalog import Route, RouteRule from litellm.rust_bridge.configuration import Rollout +from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging from tests.test_litellm_rust.support.isolation import isolated_callback_registries, rebound from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec @@ -51,8 +94,8 @@ OPUS_FAST: Final = "anthropic/claude-opus-5-5" API_KEY: Final = "sk-ant-parity-test" USER: Final[Mapping[str, JsonValue]] = MappingProxyType({"role": "user", "content": "What is the capital of France?"}) -_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _LOGGED: Final = TypeAdapter(dict[str, object]) +_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _TRANSPORT_HEADERS: Final = frozenset( {"accept", "accept-encoding", "connection", "content-length", "host", "user-agent"} ) @@ -82,6 +125,7 @@ _VOLATILE: Final = ( (re.compile(r"Timeout passed=[0-9.]+"), "Timeout passed="), (re.compile(r"127\.0\.0\.1:[0-9]+"), "127.0.0.1:"), ) +_SERVER_TOOL_ID: Final = re.compile(r"srvtoolu_[0-9a-f]{32}") TEXT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( { @@ -161,6 +205,7 @@ THINKING_REPLAY: Final[tuple[JsonValue, ...]] = ( {"role": "user", "content": "And Germany?"}, ) +_MODIFY_PARAMS: Final[Mapping[str, object]] = MappingProxyType({"modify_params": True}) NO_INPUT: Final[Mapping[str, JsonValue]] = MappingProxyType({}) PNG_PIXEL: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" @@ -283,6 +328,60 @@ SLOW_STREAM: Final = _events( ) +WIDE_NUMBERS_REPLY: Final = ( + b'{"id":"msg_01ParityWide","type":"message","role":"assistant","model":"claude-sonnet-4-5-20250929",' + b'"content":[{"type":"tool_use","id":"toolu_01ParityWide","name":"get_ids","input":{"big":18446744073709551616,' + b'"huge":12345678901234567890123,"negative":-12345678901234567890123,"neg_zero":-0.0,"overflow":1e400,' + b'"underflow":-1E+400,"exp":1E2,"tiny":5e-324,"precise":0.30000000000000004,"text":"1e400 18446744073709551616"}}],' + b'"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":9}}' +) +WIDE_INTEGER_REPLY: Final = ( + b'{"id":"msg_01ParityWideCached","type":"message","role":"assistant","model":"claude-sonnet-4-5-20250929",' + b'"content":[{"type":"tool_use","id":"toolu_01ParityWideCached","name":"get_ids","input":{"huge":12345678901234567890123}}],' + b'"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":9}}' +) + +MARKED_TEXTS_REPLY: Final = ( + '{"id":"msg_01ParityMarked","type":"message","role":"assistant","model":"claude-sonnet-4-5-20250929",' + '"content":[{"type":"text","text":"\\uf8fflitellm-raw-json:42"},{"type":"tool_use","id":"toolu_01ParityMarked",' + '"name":"get_ids","input":{"n":"\uf8fflitellm-raw-json:NaN","s":"\\uF8FFlitellm-raw-json:\\"x\\"",' + '"\\uf8fflitellm-raw-json:1":true,"big":18446744073709551616}}],' + '"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":9}}' +).encode() + + +def _non_string_tool_result_texts( + *texts: object, +) -> list[dict[str, object]]: # mutable-ok: the messages kwarg is a list of dicts + return [ + dict(USER), + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01Texts", "name": "get_ids", "input": {}}]}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01Texts", + "content": [{"type": "text", "text": text} for text in texts], + }, + {"type": "text", "text": "go on"}, + ], + }, + ] + + +_BAD_TRAILER_GZIP: Final = gzip.compress(json.dumps(dict(TEXT_RESPONSE)).encode(), mtime=0)[:-8] + bytes(8) +_NOT_GZIP: Final = b"\x1f\x8b\x08\x00not really gzip data at allnot really gzip data at all" + + +def _framed_bytes(events: tuple[tuple[str, object], ...]) -> tuple[bytes, ...]: + return tuple(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() for event, data in events) + + +def _encoded(body: bytes, encoding: str) -> ResponseSpec: + return ResponseSpec(body=body, headers={"content-encoding": encoding}) + + def _upstream_error(kind: str, message: str) -> Mapping[str, JsonValue]: return MappingProxyType({"type": "error", "error": {"type": kind, "message": message}}) @@ -295,6 +394,45 @@ def _stream(events: tuple[tuple[str, object], ...], chunk_delay: float = 0) -> R return ResponseSpec(body={}, events=events, chunk_delay=chunk_delay) +def _burst(events: tuple[tuple[str, object], ...], chunked: bool = True) -> ResponseSpec: + return ResponseSpec(body={}, events=events, burst=True, chunked=chunked) + + +def _chunked(events: tuple[tuple[str, object], ...], chunk_delay: float) -> ResponseSpec: + return ResponseSpec(body={}, events=events, chunk_delay=chunk_delay, chunked=True) + + +_PING: Final[tuple[str, JsonValue]] = ("ping", {"type": "ping"}) + +PINGED_STREAM: Final = _events( + _message_start("msg_01ParityPinged"), + *((_PING,) * 50), + *_text_block(0, "Pa", "ris"), + *_message_end("end_turn", 4), +) + +HEAVILY_PINGED_STREAM: Final = _events( + _message_start("msg_01ParityHeavilyPinged"), + *((_PING,) * 1000), + *_text_block(0, "Pa", "ris"), + *_message_end("end_turn", 4), +) + +LARGE_STREAM: Final = _events( + _message_start("msg_01ParityLarge"), + *_text_block(0, "a" * 15000, "b" * 15000, "c" * 15000), + *_message_end("end_turn", 4), +) + + +def _raw_stream(*frames: str) -> ResponseSpec: + return ResponseSpec(body="".join(f"{frame}\n\n" for frame in frames).encode()) + + +def _framed(events: tuple[tuple[str, object], ...], event_line: str = "event: ") -> tuple[str, ...]: + return tuple(f"{event_line}{event}\ndata: {json.dumps(data)}" for event, data in events) + + def _failure(status: int, kind: str, message: str, **headers: str) -> ResponseSpec: return ResponseSpec(body=dict(_upstream_error(kind, message)), status=status, headers=dict(headers)) @@ -305,6 +443,10 @@ CALLBACK_ASPECTS: Final = ( "logged_response_headers", "logged_api_key", "logged_call", + "logged_request", + "logged_sent_request", + "logged_model_parameters", + "logged_error", ) ASPECTS: Final = ("dispatch", "request", "outcome", "hidden_params", "logged", *CALLBACK_ASPECTS) @@ -319,12 +461,38 @@ class Case: env: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) settings: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) private_keys: bool = False + cache: Callable[[], Cache] | None = None + callbacks: tuple[Callable[[], CustomLogger], ...] = () + classifier: str | None = None + interceptor: Callable[[], MessagesInterceptor] | None = None gaps: Mapping[str, str | None] = field(default_factory=lambda: MappingProxyType({})) + loggers: Callable[[], Sequence[CustomLogger]] | None = None + gateway: bool = False + racy: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + partitioned: bool = False + nones: tuple[str, ...] = () + busy_before_reading: float = 0 + warned: bool = False + + def aspects(self) -> tuple[str, ...]: + return ( + *ASPECTS, + *(("cached",) if self.cache is not None else ()), + *(("warnings",) if self.warned else ()), + *(("hook_inputs",) if self.loggers else ()), + *(("classified",) if self.classifier is not None else ()), + *(("tool_runs",) if self.gateway else ()), + ) def params(self) -> Iterator[object]: - for aspect in ASPECTS: + for aspect in self.aspects(): gap = {**KNOWN_GAPS.get(self.name, {}), **self.gaps}.get(aspect) - marks = () if not gap else (pytest.mark.xfail(reason=gap, strict=True),) + race = self.racy.get(aspect) + marks = ( + (pytest.mark.skip(reason=race),) + if race + else (() if not gap else (pytest.mark.xfail(reason=gap, strict=True),)) + ) yield pytest.param(self, aspect, id=f"{self.name}-{aspect}", marks=marks) @@ -336,54 +504,48 @@ def _kw(**kwargs: object) -> Mapping[str, object]: return MappingProxyType(kwargs) +def _tls(**settings: object) -> Mapping[str, object]: + return MappingProxyType({**settings, "in_memory_llm_clients_cache": LLMClientCache()}) + + +E04_HTTPX_CHUNK_READS: Final = "E04: httpx yields each HTTP chunk whole from a 64 KiB socket read; hyper hands a large chunk over in pieces of its smaller read buffer" A10_NO_MAX_TOKENS: Final = "A10: native admission hands a call without an int max_tokens to Python, whose signature raises the TypeError; only dispatch differs" -S6_PRIVATE_KEYS: Final = "S6: native writes _hidden_params into the response dict" -S15_COMPLETE_DEADLINE: Final = ( - "S15: by design, native bounds a complete response by one overall deadline where Python bounds each read, " - "so a body trickling in slower than the timeout succeeds in Python and times out natively" +C_UNSENT_CALL_FLAG: Final = ( + "C: a call that fails natively before its provider request logs no aanthropic_messages flag, " + "which Python's handler sets before it shapes the request" ) -F_MOCK: Final = "F: mock_response is Python-only (detector mock_response)" -F_THINKING_RETRY: Final = ( - "F: retry without thinking after a rejected signature is Python-only (detector thinking_signature_retry)" + +F46_SEE_OTHER_CONTENT_TYPE: Final = ( + "F46: httpx keeps content-type on the GET that follows a 303; reqwest's redirect follower drops it with the body" +) +REDIRECTED_SENT_REQUEST: Final = ( + "a followed provider redirect: Python logs the redirected request (its URL, its body already streamed) on the " + "logged response, the native call logs the request it sent" ) -F_CACHE_INJECTION: Final = "F: prompt cache_control injection is Python-only (detector cache_control_injection)" -F_TOOL_CUSTOM: Final = "F: hoisting a tool's custom field is Python-only (detector tool_custom_field)" -F_STREAM_COST: Final = "F: cost injection into streamed usage is Python-only (detector stream_cost_injection)" KNOWN_GAPS: Final[Mapping[str, Mapping[str, str | None]]] = MappingProxyType( { - "response_private_keys": _gaps(outcome=S6_PRIVATE_KEYS), "missing_max_tokens": _gaps(dispatch=A10_NO_MAX_TOKENS), - "mock_response": _gaps( - logged_response_headers=F_MOCK, - request=F_MOCK, - outcome=F_MOCK, - hidden_params=F_MOCK, - logged=F_MOCK, - logged_metadata=F_MOCK, - logged_api_key=F_MOCK, - logged_call=F_MOCK, - ), - "mock_response_rate_limit": _gaps( - logged_response_headers=F_MOCK, - request=F_MOCK, - outcome=F_MOCK, - logged=F_MOCK, - logged_api_key=F_MOCK, - logged_call=F_MOCK, - ), - "thinking_signature_rejected_then_retried": _gaps( - request=F_THINKING_RETRY, - outcome=F_THINKING_RETRY, - hidden_params=F_THINKING_RETRY, - logged=F_THINKING_RETRY, - logged_metadata=F_THINKING_RETRY, - logged_response_headers=F_THINKING_RETRY, - ), - "cache_control_injection_points": _gaps(request=F_CACHE_INJECTION), - "tool_custom_field": _gaps(request=F_TOOL_CUSTOM), - "stream_cost_injection": _gaps(outcome=F_STREAM_COST), - "enable_prompt_caching": _gaps(request=F_CACHE_INJECTION), + **{ + name: _gaps(logged_call=C_UNSENT_CALL_FLAG) + for name in ( + "speed_on_model_without_speed", + "temperature_on_adaptive_model", + "top_k_on_adaptive_model", + "legacy_thinking_non_numeric_budget_on_adaptive_model", + "legacy_thinking_null_budget_on_adaptive_model", + "missing_api_key", + "tls_security_level_python_cannot_apply", + "max_tokens_none", + "reasoning_effort_unknown", + "output_config_effort_unknown", + "reasoning_effort_unknown_on_adaptive_model", + "reasoning_effort_xhigh_on_model_without_xhigh", + "int_temperature_on_adaptive_model", + "int_top_p_on_adaptive_model", + "int_speed_on_model_without_speed", + ) + }, } ) @@ -440,6 +602,7 @@ REQUEST_CASES: Final = ( "legacy_thinking_non_numeric_budget_dropped", _kw(thinking={"type": "enabled", "budget_tokens": [1]}, max_tokens=8192, drop_params=True), model=OPUS_ADAPTIVE, + warned=True, ), Case( "legacy_thinking_carries_extra_keys", @@ -657,6 +820,192 @@ RESPONSE_CASES: Final = ( ), ), Case("response_private_keys", private_keys=True), + Case( + "response_gzip_under_a_padded_mixed_case_alias", + upstream=(_encoded(gzip.compress(json.dumps(dict(TEXT_RESPONSE)).encode(), mtime=0), " X-Gzip "),), + ), + Case("response_deflate_body_that_is_not_deflate", upstream=(_encoded(b"not deflate at all", "deflate"),)), + Case("response_gzip_body_with_a_bad_trailer", upstream=(_encoded(_BAD_TRAILER_GZIP, "gzip"),)), + Case("response_gzip_body_that_is_not_gzip", upstream=(_encoded(_NOT_GZIP, "gzip"),)), + Case( + "stream_deflate_body_that_is_not_deflate", + _kw(stream=True), + upstream=(_encoded(b"not deflate at all", "deflate"),), + ), + Case( + "stream_gzip_listed_with_an_unknown_coding", + _kw(stream=True), + upstream=(_encoded(gzip.compress(b"".join(_framed_bytes(TEXT_STREAM)), mtime=0), "gzip, br"),), + ), +) + +INPUT_VALUE_CASES: Final = ( + Case("max_tokens_numeric_string", _kw(max_tokens="1024")), + Case("max_tokens_fraction", _kw(max_tokens=1.5)), + Case("max_tokens_whole_float", _kw(max_tokens=1024.0)), + Case("max_tokens_bool", _kw(max_tokens=True)), + Case("max_tokens_beyond_64_bits", _kw(max_tokens=2**64)), + Case("reasoning_effort_with_fractional_max_tokens", _kw(reasoning_effort="high", max_tokens=1500.5)), + Case("temperature_numeric_string", _kw(temperature="0.5")), + Case("temperature_integer_beyond_float_precision", _kw(temperature=2**60 + 1)), + Case("top_p_integer_beyond_float_precision", _kw(top_p=-9223372036854775807)), + Case( + "reasoning_effort_with_numeric_string_max_tokens", + _kw(reasoning_effort="high", max_tokens="8192"), + upstream=(), + ), + Case("top_k_beyond_64_bits", _kw(top_k=2**70)), + Case("messages_mapping", _kw(messages={"role": "user", "content": "hi"})), + Case("messages_string", _kw(messages="hi")), + Case("messages_with_non_object_entries", _kw(messages=[dict(USER), "stray", 7])), + Case("system_lone_surrogate", _kw(system="sys \ud800 lone")), + Case("metadata_user_id_lone_surrogate", _kw(metadata={"user_id": "user-\udc00"})), + Case("message_text_lone_surrogate", _kw(messages=[{"role": "user", "content": "a\ud800b \u00e9"}])), + Case("text_starting_with_the_raw_leaf_mark", _kw(system="\uf8fflitellm-raw-json:1")), + Case("thinking_budget_beyond_64_bits", _kw(thinking={"type": "enabled", "budget_tokens": 2**70}, max_tokens=2048)), + Case( + "tool_schema_integers_beyond_64_bits", + _kw(tools=[{**WEATHER_TOOL, "input_schema": {"type": "integer", "maximum": 2**70, "minimum": -(2**70)}}]), + ), + Case("temperature_nan", _kw(temperature=float("nan"))), + Case( + "tool_schema_infinities", + _kw( + tools=[ + {**WEATHER_TOOL, "input_schema": {"type": "number", "maximum": float("inf"), "minimum": -float("inf")}} + ] + ), + ), +) + +LOOSE_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"name": "lookup", "description": "lookup", "input_schema": {"type": "object", "properties": {}}} +) + + +class _StringStreamHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + kwargs["stream"] = "true" + return kwargs + + +class _IntegerStreamHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + kwargs["stream"] = 1 + return kwargs + + +class _StringParamsHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + kwargs["litellm_params"] = "not-a-mapping" + return kwargs + + +class _IntegerProviderHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + params: Final = kwargs.setdefault("litellm_params", {}) + assert isinstance(params, dict) + params["custom_llm_provider"] = 5 + return kwargs + + +ARGUMENT_TYPE_CASES: Final = ( + Case("stream_string_true", _kw(stream="true"), upstream=(_stream(TEXT_STREAM),)), + Case("stream_string_false", _kw(stream="false"), upstream=(_stream(TEXT_STREAM),)), + Case("stream_zero", _kw(stream=0)), + Case("stream_one", _kw(stream=1), upstream=(_stream(TEXT_STREAM),)), + Case("stream_one_sync", _kw(stream=1), upstream=(_stream(TEXT_STREAM),), sync=True), + Case( + "pre_request_hook_streams_on_a_string", + _kw(stream=True), + callbacks=(_StringStreamHook,), + upstream=(_stream(TEXT_STREAM),), + ), + Case("pre_request_hook_streams_on_an_integer", callbacks=(_IntegerStreamHook,), upstream=(_stream(TEXT_STREAM),)), + Case("pre_request_hook_sets_string_litellm_params", callbacks=(_StringParamsHook,), model="claude-sonnet-4-5"), + Case("pre_request_hook_sets_integer_provider", callbacks=(_IntegerProviderHook,), model="claude-sonnet-4-5"), + Case("model_info_string", _kw(model_info="not-a-mapping")), + Case("tools_mapping", _kw(tools=dict(LOOSE_TOOL)), upstream=()), + Case("messages_none", _kw(messages=None), nones=("messages",), upstream=()), + Case("max_tokens_none", _kw(max_tokens=None), nones=("max_tokens",), upstream=()), + Case("reasoning_effort_unknown", _kw(reasoning_effort="ultra"), upstream=()), + Case("reasoning_effort_unknown_on_adaptive_model", _kw(reasoning_effort="ultra"), model=OPUS_ADAPTIVE, upstream=()), + Case( + "reasoning_effort_xhigh_on_model_without_xhigh", + _kw(reasoning_effort="xhigh", max_tokens=8192), + model="anthropic/claude-opus-4-6", + upstream=(), + ), + Case("int_temperature_on_adaptive_model", _kw(temperature=0), model=OPUS_ADAPTIVE, upstream=()), + Case("int_top_p_on_adaptive_model", _kw(top_p=1), model=OPUS_ADAPTIVE, upstream=()), + Case("int_speed_on_model_without_speed", _kw(speed=2), upstream=()), + Case("string_output_format_dropped", _kw(output_format="json", drop_params=True)), + Case("string_output_format_kept_without_drop_params", _kw(output_format="json")), + Case("non_string_reasoning_effort_warns", _kw(reasoning_effort=3), warned=True), + Case( + "reasoning_effort_that_cannot_fit_max_tokens_warns", + _kw(reasoning_effort="high", max_tokens=500), + warned=True, + ), + Case( + "adaptive_thinking_that_cannot_fit_max_tokens_warns", + _kw(thinking={"type": "adaptive"}, output_config={"effort": "high"}, max_tokens=1024), + warned=True, + ), + Case( + "adaptive_thinking_on_model_without_reasoning_warns", + _kw(thinking={"type": "adaptive"}), + model="anthropic/claude-3-haiku-20240307", + warned=True, + ), + Case("output_config_effort_unknown", _kw(output_config={"effort": "ultra"})), + Case( + "global_drop_params_string_true", + _kw(tools=[{**LOOSE_TOOL, "custom": {"note": "opaque"}}]), + settings=_kw(drop_params="true"), + ), + Case( + "global_drop_params_string_false", + _kw(tools=[{**LOOSE_TOOL, "custom": {"note": "opaque"}}]), + settings=_kw(drop_params="false"), + ), + Case( + "request_drop_params_string_true", + _kw(drop_params="true", tools=[{**LOOSE_TOOL, "custom": {"note": "opaque"}}]), + ), + Case( + "additional_drop_params_tool_custom", + _kw( + additional_drop_params=["tools[*].custom"], + tools=[{**LOOSE_TOOL, "custom": {"defer_loading": True, "note": "x"}}], + ), + ), + Case( + "additional_drop_params_tool_input_examples", + _kw( + additional_drop_params=["tools[*].input_examples", "metadata.user_id", "fz.nope"], + metadata={"user_id": "u1"}, + tools=[{**LOOSE_TOOL, "input_examples": [{"q": "x"}]}], + ), + ), + Case( + "response_without_stop_reason", + upstream=(_ok({key: value for key, value in TEXT_RESPONSE.items() if key != "stop_reason"}),), + ), + Case( + "response_without_stop_fields", + upstream=( + _ok({key: value for key, value in TEXT_RESPONSE.items() if key not in ("stop_reason", "stop_sequence")}), + ), + ), ) FAILURE_CASES: Final = ( @@ -684,16 +1033,122 @@ FAILURE_CASES: Final = ( "response_trickles_slower_than_timeout", _kw(timeout=1), upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), chunk_delay=0.5, body_chunks=4),), - gaps=_gaps( - outcome=S15_COMPLETE_DEADLINE, - hidden_params=S15_COMPLETE_DEADLINE, - logged=S15_COMPLETE_DEADLINE, - logged_metadata=S15_COMPLETE_DEADLINE, - logged_response_headers=S15_COMPLETE_DEADLINE, + ), + Case( + "response_body_stalls_past_timeout", + _kw(timeout=0.5), + upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), chunk_delay=1.0, body_chunks=2),), + ), + Case( + "upstream_500_body_cut", + upstream=( + ResponseSpec(body=dict(_upstream_error("api_error", "cut")), status=500, body_chunks=2, cut_after=1), ), ), + Case("response_body_cut", upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), body_chunks=2, cut_after=1),)), + Case("gzip_response", upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), gzip=True),)), + Case("upstream_500_gzip", upstream=(replace(_failure(500, "api_error", "internal"), gzip=True),)), Case("missing_max_tokens", _kw(max_tokens=None), upstream=()), Case("missing_api_key", _kw(api_key=None), upstream=()), + Case( + "api_key_with_a_trailing_newline", + _kw(api_key=None), + env=MappingProxyType({"ANTHROPIC_API_KEY": f"{API_KEY}\n"}), + ), + Case( + "api_key_with_a_trailing_crlf_streamed", + _kw(api_key=None, stream=True), + upstream=(_stream(TEXT_STREAM),), + env=MappingProxyType({"ANTHROPIC_API_KEY": f"{API_KEY}\r\n"}), + ), + Case( + "api_key_padded_with_whitespace", + _kw(api_key=None), + env=MappingProxyType({"ANTHROPIC_API_KEY": f" {API_KEY}\t\n"}), + ), + Case( + "api_key_padded_with_separators", + _kw(api_key=None), + env=MappingProxyType({"ANTHROPIC_API_KEY": f"\x1f{API_KEY}\x1f"}), + ), + Case( + "api_key_of_separators_falls_to_the_auth_token", + _kw(api_key=None), + env=MappingProxyType({"ANTHROPIC_API_KEY": "\x1c\x1f\n", "ANTHROPIC_AUTH_TOKEN": "oauth-token"}), + ), + Case( + "auth_token_with_a_trailing_newline", + _kw(api_key=None), + env=MappingProxyType({"ANTHROPIC_AUTH_TOKEN": "oauth-token\n"}), + ), + Case("caller_api_key_padded_with_separators", _kw(api_key=f"\x1f{API_KEY}\x1f")), + Case("caller_api_key_with_a_trailing_newline", _kw(api_key=f"{API_KEY}\n")), + Case( + "auth_token_padded_with_whitespace", + _kw(api_key=None), + env=MappingProxyType({"ANTHROPIC_AUTH_TOKEN": " fz-env-auth-token "}), + ), + Case( + "provider_redirect_is_followed", + gaps=_gaps(logged_sent_request=REDIRECTED_SENT_REQUEST), + upstream=( + ResponseSpec(body=b"moved", status=307, headers={"location": "/redirected/v1/messages"}), + ResponseSpec(body=dict(TEXT_RESPONSE)), + ), + ), + Case( + "provider_redirect_chain_of_eleven_hops", + gaps=_gaps(logged_sent_request=REDIRECTED_SENT_REQUEST), + upstream=( + *( + ResponseSpec(body=b"moved", status=307, headers={"location": f"/hop{hop}/v1/messages"}) + for hop in range(11) + ), + ResponseSpec(body=dict(TEXT_RESPONSE)), + ), + ), + Case( + "provider_redirect_chain_past_twenty_hops", + upstream=tuple( + ResponseSpec(body=b"moved", status=307, headers={"location": f"/hop{hop}/v1/messages"}) for hop in range(21) + ), + ), + Case( + "provider_see_other_redirect", + gaps=_gaps(logged_sent_request=REDIRECTED_SENT_REQUEST, request=F46_SEE_OTHER_CONTENT_TYPE), + upstream=( + ResponseSpec(body=b"see other", status=303, headers={"location": "/other/v1/messages"}), + ResponseSpec(body=dict(TEXT_RESPONSE)), + ), + ), + Case("extra_header_lone_surrogate", _kw(extra_headers={"x-fz-surrogate": "h\ud800"}), upstream=()), + Case( + "provider_specific_header_lone_surrogate", + _kw( + provider_specific_header={ + "custom_llm_provider": "anthropic", + "extra_headers": {"x-fz-surrogate": "h\ud800"}, + } + ), + upstream=(), + ), + Case("extra_header_non_ascii", _kw(extra_headers={"x-fz-unicode": "caf\u00e9"}), upstream=()), + Case("caller_api_key_non_ascii", _kw(api_key="sk-ant-\u200djoined"), upstream=()), + Case("caller_api_key_non_ascii_stream", _kw(api_key="sk-ant-\u200djoined", stream=True), upstream=()), + Case("api_base_with_a_space_and_no_scheme", _kw(api_base="event: message_stop"), upstream=()), + Case("api_base_with_a_space_and_no_scheme_stream", _kw(api_base="data: [DONE]", stream=True), upstream=()), + Case("api_base_with_a_nul", _kw(api_base="a\x00b"), upstream=()), + Case("api_base_non_ascii_without_a_scheme", _kw(api_base="h\u00e9llo"), upstream=()), + Case("api_base_with_an_unclosed_ipv6_bracket", _kw(api_base_suffix="data: [DONE]"), upstream=()), + Case("provider_specific_header_as_a_string", _kw(provider_specific_header="anthropic"), upstream=()), + Case("response_header_latin_1", upstream=(_ok(**{"x-fz-unicode": "caf\u00e9"}),)), + Case("response_header_utf_8", upstream=(_ok(**{"x-fz-unicode": "caf\u00c3\u00a9"}),)), + Case( + "response_header_utf_8_httpx_transport", + upstream=(_ok(**{"x-fz-unicode": "caf\u00c3\u00a9"}),), + env=MappingProxyType({"DISABLE_AIOHTTP_TRANSPORT": "true"}), + settings=_tls(), + ), ) STREAM_CASES: Final = ( @@ -705,6 +1160,75 @@ STREAM_CASES: Final = ( upstream=(_stream(THINKING_STREAM),), ), Case("stream_error_event_mid_stream", _kw(stream=True), upstream=(_stream(ERROR_MID_STREAM),)), + Case("stream_events_in_one_write", _kw(stream=True), upstream=(_burst(TEXT_STREAM),), partitioned=True), + Case( + "stream_body_in_one_write", + _kw(stream=True), + upstream=(_burst(TEXT_STREAM, chunked=False),), + partitioned=True, + ), + Case( + "stream_events_in_chunks_written_apart", + _kw(stream=True), + upstream=(_chunked(TEXT_STREAM, chunk_delay=0.05),), + partitioned=True, + ), + Case( + "stream_events_written_apart", + _kw(stream=True), + upstream=(_stream(TEXT_STREAM, chunk_delay=0.05),), + partitioned=True, + ), + Case( + "stream_pings_after_message_start_in_one_write", + _kw(stream=True), + upstream=(_burst(PINGED_STREAM),), + partitioned=True, + ), + Case( + "stream_pings_after_message_start_written_apart", + _kw(stream=True), + upstream=(_stream(PINGED_STREAM, chunk_delay=0.01),), + partitioned=True, + ), + Case("stream_burst_past_one_read", _kw(stream=True), upstream=(_burst(LARGE_STREAM),), partitioned=True), + Case( + "stream_thousand_pings_in_one_write", + _kw(stream=True), + upstream=(_burst(HEAVILY_PINGED_STREAM),), + partitioned=True, + ), + Case( + "stream_error_event_in_one_write", + _kw(stream=True), + upstream=(_burst(ERROR_MID_STREAM),), + partitioned=True, + ), + Case( + "stream_events_in_one_write_httpx_transport", + _kw(stream=True), + upstream=(_burst(TEXT_STREAM),), + env=MappingProxyType({"DISABLE_AIOHTTP_TRANSPORT": "true"}), + settings=_tls(), + partitioned=True, + ), + Case( + "stream_burst_past_one_read_httpx_transport", + _kw(stream=True), + upstream=(_burst(LARGE_STREAM),), + env=MappingProxyType({"DISABLE_AIOHTTP_TRANSPORT": "true"}), + settings=_tls(), + partitioned=True, + gaps=_gaps(outcome=E04_HTTPX_CHUNK_READS), + ), + Case( + "stream_events_written_apart_httpx_transport", + _kw(stream=True), + upstream=(_stream(TEXT_STREAM, chunk_delay=0.05),), + env=MappingProxyType({"DISABLE_AIOHTTP_TRANSPORT": "true"}), + settings=_tls(), + partitioned=True, + ), Case( "stream_longer_than_timeout", _kw(stream=True, timeout=1), @@ -721,6 +1245,81 @@ STREAM_CASES: Final = ( upstream=(_failure(429, "rate_limit_error", "slow down", **{"retry-after": "3"}),), ), Case("stream_false_explicit", _kw(stream=False)), + Case( + "stream_upstream_500_body_cut", + _kw(stream=True), + upstream=( + ResponseSpec(body=dict(_upstream_error("api_error", "cut")), status=500, body_chunks=2, cut_after=1), + ), + ), + Case( + "stream_cut_after_four_events", + _kw(stream=True), + upstream=(replace(_stream(TEXT_STREAM), cut_after=4, cut_delay=0.1),), + ), + Case( + "stream_cut_after_message_start", + _kw(stream=True), + upstream=(replace(_stream(TEXT_STREAM), cut_after=1, cut_delay=0.1),), + ), + Case( + "stream_cut_after_events_written_apart", + _kw(stream=True), + upstream=(replace(_stream(TEXT_STREAM, chunk_delay=0.05), cut_after=6, cut_delay=0.1),), + ), + Case( + "stream_chunked_cut_after_four_events", + _kw(stream=True), + upstream=(replace(_chunked(TEXT_STREAM, chunk_delay=0), cut_after=4, cut_delay=0.1),), + ), + Case( + "stream_cut_while_the_caller_is_busy", + _kw(stream=True), + upstream=(replace(_chunked(TEXT_STREAM, chunk_delay=0), cut_after=4, cut_delay=0.1),), + busy_before_reading=0.4, + ), + Case( + "stream_cut_after_message_start_while_the_caller_is_busy", + _kw(stream=True), + upstream=(replace(_stream(TEXT_STREAM), cut_after=1, cut_delay=0.1),), + busy_before_reading=0.4, + ), + Case( + "stream_upstream_500_body_stalls", + _kw(stream=True, timeout=8), + upstream=( + ResponseSpec(body=dict(_upstream_error("api_error", "stalled")), status=500, chunk_delay=6, body_chunks=2), + ), + ), + Case( + "stream_upstream_429_body_stalls_past_timeout", + _kw(stream=True, timeout=1), + upstream=( + ResponseSpec( + body=dict(_upstream_error("rate_limit_error", "stalled")), status=429, chunk_delay=2, body_chunks=2 + ), + ), + ), + Case("stream_gzip_response", _kw(stream=True), upstream=(replace(_stream(TEXT_STREAM), gzip=True),)), + Case( + "stream_read_stalls_past_timeout_httpx_transport", + _kw(stream=True, timeout=0.5), + upstream=(_stream(SLOW_STREAM, chunk_delay=1.0),), + env=MappingProxyType({"DISABLE_AIOHTTP_TRANSPORT": "true"}), + settings=_tls(), + ), + Case( + "stream_response_header_utf_8", + _kw(stream=True), + upstream=(replace(_stream(TEXT_STREAM), headers={"x-fz-unicode": "caf\u00c3\u00a9"}),), + ), + Case( + "stream_response_header_utf_8_httpx_transport", + _kw(stream=True), + upstream=(replace(_stream(TEXT_STREAM), headers={"x-fz-unicode": "caf\u00c3\u00a9"}),), + env=MappingProxyType({"DISABLE_AIOHTTP_TRANSPORT": "true"}), + settings=_tls(), + ), ) SYNC_CASES: Final = ( @@ -729,47 +1328,2022 @@ SYNC_CASES: Final = ( Case("sync_upstream_429", upstream=(_failure(429, "rate_limit_error", "slow down"),), sync=True), ) -FALLBACK_CASES: Final = ( - Case("mock_response", _kw(mock_response="Hi from the mock"), upstream=()), - Case("mock_response_rate_limit", _kw(mock_response="litellm.RateLimitError"), upstream=()), - Case( - "thinking_replay", - _kw(messages=list(THINKING_REPLAY), thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096), - ), - Case( - "thinking_signature_rejected_then_retried", - _kw(messages=list(THINKING_REPLAY), thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096), - upstream=( - _failure(400, "invalid_request_error", "messages.1.content.0: Invalid `signature` in `thinking` block"), - _ok(), - ), - ), +SYSTEM_POINT: Final[Mapping[str, JsonValue]] = MappingProxyType({"location": "message", "role": "system"}) +LAST_MESSAGE_POINT: Final[Mapping[str, JsonValue]] = MappingProxyType({"location": "message", "index": -1}) +SYSTEM_BLOCKS: Final[tuple[JsonValue, ...]] = ( + {"type": "text", "text": "Answer in one word."}, + {"type": "text", "text": "Be terse."}, +) +MARKED_TURNS: Final[tuple[JsonValue, ...]] = ( + {"role": "user", "content": [{"type": "text", "text": "One.", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Two.", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "Three.", "cache_control": {"type": "ephemeral"}}]}, +) + +PROMPT_CACHING_CASES: Final = ( Case( "cache_control_injection_points", - _kw(system="Answer in one word.", cache_control_injection_points=[{"location": "message", "role": "system"}]), + _kw(system="Answer in one word.", cache_control_injection_points=[dict(SYSTEM_POINT)]), ), - Case("tool_custom_field", _kw(tools=[{**WEATHER_TOOL, "custom": {"defer_loading": True}}])), + Case("enable_prompt_caching", _kw(system="Answer in one word.", enable_prompt_caching=True)), Case( - "strict_tools_rejected_then_retried", - _kw(tools=[{**WEATHER_TOOL, "strict": True}], drop_params=True), - upstream=( - _failure(400, "invalid_request_error", "tools.0.strict: too many strict tools"), - _ok(), + "enable_prompt_caching_without_system", + _kw(messages=list(TOOL_TURNS), tools=[dict(WEATHER_TOOL)], enable_prompt_caching=True), + ), + Case( + "global_prompt_caching_with_ttl", + _kw(system=list(SYSTEM_BLOCKS)), + settings=MappingProxyType({"enable_anthropic_prompt_caching": True, "anthropic_prompt_caching_ttl": "1h"}), + ), + Case( + "global_prompt_caching_stands_down_on_client_marks", + _kw(system=[{"type": "text", "text": "Answer in one word.", "cache_control": {"type": "ephemeral"}}]), + settings=MappingProxyType({"enable_anthropic_prompt_caching": True}), + ), + Case( + "global_prompt_caching_stands_down_on_marked_tools", + _kw(tools=[{**WEATHER_TOOL, "cache_control": {"type": "ephemeral"}}]), + settings=MappingProxyType({"enable_anthropic_prompt_caching": True}), + ), + Case( + "enable_prompt_caching_false_overrides_nothing", + _kw(system="Answer in one word.", enable_prompt_caching=False), + ), + Case( + "injection_point_negative_index", + _kw( + messages=list(TOOL_TURNS), + cache_control_injection_points=[ + {"location": "message", "index": -2, "control": {"type": "ephemeral", "ttl": "1h"}} + ], ), ), - Case("modify_params", _kw(max_tokens=1), settings=MappingProxyType({"modify_params": True})), + Case( + "injection_point_string_index", + _kw(messages=list(TOOL_TURNS), cache_control_injection_points=[{"location": "message", "index": "1"}]), + ), + Case( + "injection_point_out_of_range_index", + _kw(messages=list(TOOL_TURNS), cache_control_injection_points=[{"location": "message", "index": 7}]), + ), + Case( + "injection_point_by_role", + _kw(messages=list(TOOL_TURNS), cache_control_injection_points=[{"location": "message", "role": "user"}]), + ), + Case( + "injection_points_capped_by_client_marks", + _kw( + messages=list(MARKED_TURNS), + system=list(SYSTEM_BLOCKS), + tools=[{**WEATHER_TOOL, "cache_control": {"type": "ephemeral"}}], + cache_control_injection_points=[dict(SYSTEM_POINT), dict(LAST_MESSAGE_POINT)], + ), + ), + Case( + "injection_points_beside_client_marks", + _kw( + messages=[*MARKED_TURNS[:2], {"role": "user", "content": "Three."}], + cache_control_injection_points=[dict(SYSTEM_POINT), dict(LAST_MESSAGE_POINT)], + system="Answer in one word.", + ), + ), + Case( + "tool_config_point_marks_the_last_tool", + _kw( + tools=[dict(WEATHER_TOOL), {**WEATHER_TOOL, "name": "get_time"}], + cache_control_injection_points=[{"location": "tool_config"}, dict(LAST_MESSAGE_POINT)], + ), + ), + Case( + "tool_config_point_without_tools", + _kw(cache_control_injection_points=[{"location": "tool_config"}]), + ), + Case( + "injection_marks_the_gateway_deployment", + _kw( + system="Answer in one word.", + enable_prompt_caching=True, + litellm_metadata={"user_api_key": "hashed"}, + model_info={"id": "deployment-1"}, + ), + ), + Case( + "injection_into_whitespace_only_turn", + _kw( + messages=[dict(USER), {"role": "assistant", "content": "Paris"}, {"role": "user", "content": " "}], + cache_control_injection_points=[dict(LAST_MESSAGE_POINT)], + ), + upstream=(_failure(400, "invalid_request_error", "messages: text content blocks must be non-empty"),), + ), + Case( + "injection_after_empty_blocks_are_stripped", + _kw( + messages=[ + dict(USER), + {"role": "assistant", "content": [{"type": "text", "text": ""}]}, + {"role": "user", "content": "And Germany?"}, + ], + cache_control_injection_points=[{"location": "message", "index": 1}], + ), + ), + Case( + "stream_with_prompt_caching", + _kw(stream=True, system="Answer in one word.", enable_prompt_caching=True), + upstream=(_stream(TEXT_STREAM),), + ), + Case( + "global_prompt_caching_beside_an_unmatched_point", + _kw( + system="Answer in one word.", cache_control_injection_points=[{"location": "message", "role": "assistant"}] + ), + settings=MappingProxyType({"enable_anthropic_prompt_caching": True}), + ), +) + +SYNC_PROMPT_CACHING_CASES: Final = tuple( + replace(case, name=f"sync_{case.name}", sync=True) for case in PROMPT_CACHING_CASES +) + +LATEST_ALIAS: Final = "anthropic/claude-opus-latest" +CUSTOM_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType({**WEATHER_TOOL, "custom": {"defer_loading": True}}) +COST_HEADER: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"headers": {"x-litellm-include-cost-in-streaming-usage": "true"}} +) + +SERVED_NATIVELY_CASES: Final = ( + Case("mock_response", _kw(mock_response="Hi from the mock"), upstream=()), + Case("mock_response_rate_limit", _kw(mock_response="litellm.RateLimitError"), upstream=()), + Case("mock_response_context_window", _kw(mock_response="litellm.ContextWindowExceededError"), upstream=()), + Case("mock_response_empty_calls_the_provider", _kw(mock_response="")), + Case("mock_response_bad_metadata", _kw(mock_response="Hi", metadata={"user_id": 123}), upstream=()), + Case("mock_response_stream", _kw(mock_response="Hi from the mock", stream=True), upstream=()), + Case("mock_response_stream_rate_limit", _kw(mock_response="litellm.RateLimitError", stream=True), upstream=()), + Case( + "mock_response_stream_empty_calls_the_provider", + _kw(mock_response="", stream=True), + upstream=(_stream(TEXT_STREAM),), + ), + Case("mock_response_cached", _kw(mock_response="Hi from the mock"), upstream=(), cache=Cache), + Case("mock_response_stream_cached", _kw(mock_response="Hi from the mock", stream=True), upstream=(), cache=Cache), + Case("mock_response_sync", _kw(mock_response="Hi from the mock"), upstream=(), sync=True), + Case("mock_response_sync_stream", _kw(mock_response="Hi from the mock", stream=True), upstream=(), sync=True), + Case("mock_response_sync_rate_limit", _kw(mock_response="litellm.RateLimitError"), upstream=(), sync=True), + Case("mock_response_sync_internal_error", _kw(mock_response="litellm.InternalServerError"), upstream=(), sync=True), + Case("mock_response_sync_bad_metadata", _kw(mock_response="Hi", metadata={"user_id": 123}), upstream=(), sync=True), + Case("mock_response_sync_empty_calls_the_provider", _kw(mock_response=""), sync=True), + Case("latest_alias", model=LATEST_ALIAS), + Case("latest_alias_bare_with_provider", _kw(custom_llm_provider="anthropic"), model="claude-opus-latest"), + Case("latest_alias_stream", _kw(stream=True), upstream=(_stream(TEXT_STREAM),), model=LATEST_ALIAS), + Case( + "latest_alias_upstream_400", + upstream=(_failure(400, "invalid_request_error", "max_tokens: too large"),), + model=LATEST_ALIAS, + ), + Case("tool_custom_field", _kw(tools=[dict(CUSTOM_TOOL)])), + Case( + "tool_custom_field_keeps_other_custom_keys", + _kw(tools=[{**WEATHER_TOOL, "custom": {"defer_loading": False, "note": "kept"}}]), + ), + Case( + "tool_custom_field_dropped_with_drop_params", + _kw(tools=[{**WEATHER_TOOL, "custom": {"defer_loading": True, "note": "dropped"}}], drop_params=True), + ), + Case("tool_custom_field_not_an_object", _kw(tools=[{**WEATHER_TOOL, "custom": "opaque"}])), + Case( + "tool_custom_field_under_own_defer_loading", + _kw(tools=[{**WEATHER_TOOL, "defer_loading": False, "custom": {"defer_loading": True}}]), + ), Case( "stream_cost_injection", _kw(stream=True), upstream=(_stream(TEXT_STREAM),), settings=MappingProxyType({"include_cost_in_streaming_usage": True}), ), - Case("enable_prompt_caching", _kw(system="Answer in one word.", enable_prompt_caching=True)), + Case( + "stream_cost_injection_from_proxy_header", + _kw(stream=True, proxy_server_request=dict(COST_HEADER)), + upstream=(_stream(TOOL_STREAM, chunk_delay=0.01),), + ), + Case( + "cache_off_by_default", + settings=MappingProxyType({"cache": Cache(mode=CacheMode.default_off)}), + ), + Case( + "cache_skips_messages_call_type", + settings=MappingProxyType({"cache": Cache(supported_call_types=["acompletion"])}), + ), +) + +FALLBACK_CASES: Final = ( + Case("modify_params", _kw(max_tokens=1), settings=_MODIFY_PARAMS), + Case("modify_params_clamps_max_tokens", _kw(max_tokens=200_000), settings=_MODIFY_PARAMS), + Case( + "modify_params_clamps_streamed_max_tokens", + _kw(stream=True, max_tokens=200_000), + upstream=(_stream(TEXT_STREAM),), + settings=_MODIFY_PARAMS, + ), + Case( + "modify_params_clamps_tool_turns", + _kw(messages=list(TOOL_TURNS), tools=[dict(WEATHER_TOOL)], max_tokens=200_000), + upstream=(_ok(TOOL_USE_RESPONSE),), + settings=_MODIFY_PARAMS, + ), + Case( + "modify_params_clamps_thinking", + _kw(thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=200_000), + upstream=(_ok(THINKING_RESPONSE),), + settings=_MODIFY_PARAMS, + ), + Case( + "modify_params_image_content", + _kw( + messages=[ + { + "role": "user", + "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_PIXEL}}, + {"type": "text", "text": "What is this?"}, + ], + } + ], + max_tokens=200_000, + ), + settings=_MODIFY_PARAMS, + ), + Case( + "modify_params_unknown_model", + _kw(max_tokens=200_000), + model="anthropic/claude-parity-unknown", + settings=_MODIFY_PARAMS, + ), + Case("modify_params_hf_model_name", _kw(max_tokens=200_000, hf_model_name="gpt2"), settings=_MODIFY_PARAMS), + Case( + "modify_params_cache_write_without_read", + _kw(max_tokens=200_000, cache={"no-cache": True}), + upstream=(_ok(), _ok()), + settings=_MODIFY_PARAMS, + cache=Cache, + ), + Case( + "modify_params_clamped_call_never_hits_the_cache", + _kw(max_tokens=200_000), + upstream=(_ok(), _ok()), + settings=_MODIFY_PARAMS, + cache=Cache, + ), + Case( + "modify_params_unclamped_call_hits_the_cache", + _kw(max_tokens=10), + upstream=(_ok(), _ok()), + settings=_MODIFY_PARAMS, + cache=Cache, + ), + Case("cache_write_without_read", _kw(cache={"no-cache": True}), upstream=(_ok(), _ok()), cache=Cache), + Case("reply_with_wide_numbers", upstream=(ResponseSpec(body=WIDE_NUMBERS_REPLY),)), + Case("reply_with_a_wide_integer_cached", upstream=(ResponseSpec(body=WIDE_INTEGER_REPLY),), cache=Cache), + Case("reply_with_raw_leaf_marked_texts", upstream=(ResponseSpec(body=MARKED_TEXTS_REPLY),)), + Case("reply_with_raw_leaf_marked_texts_cached", upstream=(ResponseSpec(body=MARKED_TEXTS_REPLY),), cache=Cache), + Case( + "modify_params_clamped_stream_hits_the_cache", + _kw(max_tokens=200_000, stream=True), + upstream=(_stream(TEXT_STREAM), _stream(TEXT_STREAM)), + settings=_MODIFY_PARAMS, + cache=Cache, + ), + Case( + "drop_params_drops_non_string_tool_result_texts", + _kw( + messages=_non_string_tool_result_texts(5, None, {"a": 1}, 8.5, 2**70, float("nan"), "kept"), + drop_params=True, + ), + ), + Case( + "global_drop_params_drops_every_tool_result_text", + _kw(messages=_non_string_tool_result_texts(5, None)), + settings=MappingProxyType({"drop_params": True}), + ), + Case( + "drop_params_keeps_string_tool_result_texts", + _kw(messages=_non_string_tool_result_texts("\uf8fflitellm-raw-json:1", "kept"), drop_params=True), + ), + Case("modify_params_sync", _kw(max_tokens=200_000), sync=True, settings=_MODIFY_PARAMS), Case("request_timeout_kwarg", _kw(request_timeout=30)), Case("stream_timeout_kwarg", _kw(stream=True, stream_timeout=30), upstream=(_stream(TEXT_STREAM),)), ) -CASES: Final = (*REQUEST_CASES, *RESPONSE_CASES, *FAILURE_CASES, *STREAM_CASES, *SYNC_CASES, *FALLBACK_CASES) +INVALID_SIGNATURE: Final = "messages.1.content.0: Invalid `signature` in `thinking` block" +BOUND_ELSEWHERE: Final = f"{INVALID_SIGNATURE}. The block is bound to a different conversation." +THINKING: Final[Mapping[str, JsonValue]] = MappingProxyType({"type": "enabled", "budget_tokens": 2048}) +BINDING_ERRORS: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} +) +THINKING_ONLY_REPLAY: Final[tuple[JsonValue, ...]] = ( + {"role": "user", "content": "What is the capital of France?"}, + { + "role": "assistant", + "content": [ + {"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, + {"type": "thinking", "thinking": "Paris.", "signature": "EqQBCkYIARgCKkB"}, + ], + }, + {"role": "assistant", "content": []}, + {"role": "user", "content": "And Germany?"}, +) + + +def _replay(**kwargs: object) -> Mapping[str, object]: + return _kw(messages=list(THINKING_REPLAY), thinking=dict(THINKING), max_tokens=4096, **kwargs) + + +def _rejected(message: str = INVALID_SIGNATURE, status: int = 400) -> ResponseSpec: + return _failure(status, "invalid_request_error", message) + + +THINKING_RETRY_CASES: Final = ( + Case("thinking_replay", _replay()), + Case("thinking_signature_rejected_then_retried", _replay(), upstream=(_rejected(), _ok())), + Case( + "thinking_signature_rejected_then_retried_stream", + _replay(stream=True), + upstream=(_rejected(), _stream(TEXT_STREAM)), + ), + Case( + "thinking_signature_missing_then_retried", + _replay(), + upstream=(_rejected("messages.1.content.0.thinking.signature.str: Input should be a valid string"), _ok()), + ), + Case( + "empty_thinking_rejected_then_retried_without_thinking_only_turns", + _kw(messages=list(THINKING_ONLY_REPLAY), thinking=dict(THINKING), max_tokens=4096), + upstream=(_rejected("messages.1.content.1.thinking: each thinking block must contain thinking"), _ok()), + ), + Case("thinking_signature_rejected_twice", _replay(), upstream=(_rejected(), _rejected())), + Case( + "thinking_signature_rejected_twice_stream", + _replay(stream=True), + upstream=(_rejected(), _rejected()), + ), + Case("thinking_signature_text_on_a_429_is_not_retried", _replay(), upstream=(_rejected(status=429),)), + Case( + "unrelated_400_is_not_retried", + _replay(), + upstream=(_rejected("max_tokens: 4096 > 64000, which is the maximum allowed"),), + ), + Case( + "binding_mismatch_retried_when_the_caller_did_not_pin_it", + _kw(messages=list(THINKING_REPLAY), thinking={"type": "adaptive"}, max_tokens=4096), + model=OPUS_ADAPTIVE, + upstream=(_rejected(BOUND_ELSEWHERE), _ok()), + ), + Case( + "binding_mismatch_raised_when_the_caller_pinned_it", + _kw(messages=list(THINKING_REPLAY), thinking=dict(BINDING_ERRORS), max_tokens=4096), + model=OPUS_ADAPTIVE, + upstream=(_rejected(BOUND_ELSEWHERE),), + ), + Case( + "invalid_signature_retried_even_when_the_caller_pinned_binding", + _kw(messages=list(THINKING_REPLAY), thinking=dict(BINDING_ERRORS), max_tokens=4096), + model=OPUS_ADAPTIVE, + upstream=(_rejected(), _ok()), + ), +) + + +def _default_off_cache() -> Cache: + return Cache(mode=CacheMode.default_off) + + +def _completion_only_cache() -> Cache: + return Cache(supported_call_types=["acompletion"]) + + +CACHE_CASES: Final = ( + Case("cache_hit", upstream=(_ok(), _ok()), cache=Cache), + Case("cache_hit_stream", _kw(stream=True), upstream=(_stream(TEXT_STREAM), _stream(TEXT_STREAM)), cache=Cache), + Case("cache_no_cache", _kw(cache={"no-cache": True}), upstream=(_ok(), _ok()), cache=Cache), + Case("cache_no_store", _kw(cache={"no-store": True}), upstream=(_ok(), _ok()), cache=Cache), + Case("cache_caching_false", _kw(caching=False), upstream=(_ok(), _ok()), cache=Cache), + Case( + "cache_ttl_and_namespace", _kw(cache={"ttl": 60, "namespace": "parity"}), upstream=(_ok(), _ok()), cache=Cache + ), + Case("cache_default_off", upstream=(_ok(), _ok()), cache=_default_off_cache), + Case( + "cache_default_off_use_cache", + _kw(cache={"use-cache": True}), + upstream=(_ok(), _ok()), + cache=_default_off_cache, + ), + Case("cache_unsupported_call_type", upstream=(_ok(), _ok()), cache=_completion_only_cache), + Case("cache_hit_with_metadata", _kw(metadata={"user_id": "user-123"}), upstream=(_ok(), _ok()), cache=Cache), + Case("cache_sync", sync=True, upstream=(_ok(), _ok()), cache=Cache), + *( + Case(name, _kw(stream=True), upstream=(_raw_stream(*frames), _raw_stream(*frames)), cache=Cache) + for name, frames in ( + ("cache_stream_data_only_frames", tuple(f"data: {json.dumps(data)}" for _, data in TEXT_STREAM)), + ("cache_stream_event_without_space", _framed(TEXT_STREAM, "event:")), + ("cache_stream_done_sentinel", (*_framed(TEXT_STREAM), "data: [DONE]")), + ( + "cache_stream_without_output", + _framed( + _events(_message_start("msg_01ParityEmpty"), *_text_block(0, ""), *_message_end("end_turn", 1)) + ), + ), + ( + "cache_stream_error_after_stop", + ( + *_framed(TEXT_STREAM), + *_framed((("error", dict(_upstream_error("overloaded_error", "Overloaded"))),)), + ), + ), + ) + ), +) + +STRICT_REJECTION: Final = "Too many strict tools (21). The maximum number of strict tools is 20." +STRICT_TOOLS: Final[tuple[JsonValue, ...]] = ( + {**WEATHER_TOOL, "strict": True}, + {**WEATHER_TOOL, "name": "get_time", "strict": False}, + dict(WEATHER_TOOL, name="get_tide"), +) + + +def _strict(**kwargs: object) -> Mapping[str, object]: + return _kw(tools=list(STRICT_TOOLS), **kwargs) + + +STRICT_TOOLS_RETRY_CASES: Final = ( + Case( + "strict_tools_rejected_then_retried", _strict(drop_params=True), upstream=(_rejected(STRICT_REJECTION), _ok()) + ), + Case( + "strict_tools_rejected_then_retried_stream", + _strict(drop_params=True, stream=True), + upstream=(_rejected(STRICT_REJECTION), _stream(TEXT_STREAM)), + ), + Case( + "strict_tools_rejected_then_retried_under_global_drop_params", + _strict(), + upstream=(_rejected("The compiled grammar is too large"), _ok()), + settings=MappingProxyType({"drop_params": True}), + ), + Case( + "strict_tools_custom_field_rejected_then_retried", + _strict(drop_params="true"), + upstream=(_rejected("tools.0.custom.strict: Extra inputs are not permitted"), _ok()), + ), + Case( + "strict_tools_rejected_without_drop_params", + _strict(), + upstream=(_rejected(STRICT_REJECTION),), + settings=MappingProxyType({"drop_params": False}), + ), + Case( + "strict_tools_rejected_under_a_truthy_global_drop_params_that_is_not_true", + _strict(), + upstream=(_rejected(STRICT_REJECTION),), + settings=MappingProxyType({"drop_params": 1}), + ), + Case( + "strict_rejection_without_a_strict_tool", + _kw(tools=[{**WEATHER_TOOL, "strict": False}], drop_params=True), + upstream=(_rejected(STRICT_REJECTION),), + ), + Case( + "strict_rejection_text_on_a_429_is_not_retried", + _strict(drop_params=True), + upstream=(_rejected(STRICT_REJECTION, status=429),), + ), + Case( + "strict_tools_rejected_twice", + _strict(drop_params=True), + upstream=(_rejected(STRICT_REJECTION), _rejected("Grammar compilation timed out")), + ), + Case( + "strict_tools_rejected_twice_stream", + _strict(drop_params=True, stream=True), + upstream=(_rejected(STRICT_REJECTION), _rejected(STRICT_REJECTION)), + ), + Case( + "strict_tools_and_thinking_strict_rejected_then_retried", + _replay(tools=list(STRICT_TOOLS), drop_params=True), + upstream=(_rejected(STRICT_REJECTION), _ok()), + ), + Case( + "strict_tools_and_thinking_signature_rejected_then_retried", + _replay(tools=list(STRICT_TOOLS), drop_params=True), + upstream=(_rejected(), _ok()), + ), + Case( + "strict_tools_and_thinking_each_rejected_spends_the_one_retry", + _replay(tools=list(STRICT_TOOLS), drop_params=True), + upstream=(_rejected(STRICT_REJECTION), _rejected()), + ), + Case( + "strict_tools_and_thinking_signature_then_strict_rejected", + _replay(tools=list(STRICT_TOOLS), drop_params=True), + upstream=(_rejected(), _rejected(STRICT_REJECTION), _ok()), + ), + Case( + "one_error_naming_both_drops_thinking_and_strict_without_drop_params", + _replay(tools=list(STRICT_TOOLS)), + upstream=(_rejected(f"{INVALID_SIGNATURE}. {STRICT_REJECTION}"), _ok()), + ), +) + +THREADS_BETA: Final = "message-threads-2026-08-12" +ADVISOR_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"type": "advisor_20260301", "name": "advisor", "model": "claude-opus-4-8"} +) +THREAD_CREATE: Final[Mapping[str, JsonValue]] = MappingProxyType({"type": "create"}) +THREAD_CONTINUE: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"type": "continue", "previous_message_id": "msg_01ParityText"} +) +ADVISOR_TURNS: Final[tuple[JsonValue, ...]] = ( + {"role": "user", "content": "Build a worker pool."}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Let me consult the advisor."}, + {"type": "server_tool_use", "id": "srvtoolu_01Parity", "name": "advisor", "input": {}}, + { + "type": "advisor_tool_result", + "tool_use_id": "srvtoolu_01Parity", + "content": {"type": "advisor_result", "text": "Use channels."}, + }, + {"type": "text", "text": "Here is the implementation."}, + ], + }, + {"role": "user", "content": "Now add a timeout.", "output_config": {"effort": "low"}}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + {"role": "user", "content": "Go on."}, +) +THREAD_NOT_FOUND: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "type": "error", + "error": { + "type": "not_found_error", + "message": "No thread state was found for the requested `previous_message_id`.", + "details": {"error_code": "thread_not_found"}, + }, + "request_id": "req_01Parity", + } +) + + +def _thread_not_found() -> ResponseSpec: + return ResponseSpec(body=dict(THREAD_NOT_FOUND), status=404) + + +THREAD_CASES: Final = ( + Case("thread_create", _kw(thread=dict(THREAD_CREATE), system="Answer in one word.")), + Case("thread_create_with_advisor_tool", _kw(thread=dict(THREAD_CREATE), tools=[dict(ADVISOR_TOOL)])), + Case("thread_create_scans_history", _kw(thread=dict(THREAD_CREATE), messages=list(ADVISOR_TURNS))), + Case("thread_continue", _kw(thread=dict(THREAD_CONTINUE))), + Case( + "thread_continue_keeps_history_and_skips_its_betas", + _kw(thread=dict(THREAD_CONTINUE), messages=list(ADVISOR_TURNS)), + ), + Case( + "thread_continue_keeps_param_betas", + _kw( + thread=dict(THREAD_CONTINUE), + messages=list(ADVISOR_TURNS), + compaction={}, + context_management={"edits": [{"type": "compact_20260112"}]}, + ), + ), + Case("thread_continue_without_previous_message_id", _kw(thread={"type": "continue"}, messages=list(ADVISOR_TURNS))), + Case( + "thread_continue_with_a_malformed_previous_message_id", + _kw(thread={"type": "continue", "previous_message_id": 7}, messages=list(ADVISOR_TURNS)), + ), + Case("thread_of_unknown_shape", _kw(thread="opaque", messages=list(ADVISOR_TURNS))), + Case("thread_none_is_left_out", _kw(thread=None)), + Case( + "thread_beside_a_client_beta", + _kw(thread=dict(THREAD_CONTINUE), extra_headers={"anthropic-beta": f"advisor-tool-2026-03-01,{THREADS_BETA}"}), + ), + Case( + "thread_dropped_by_additional_drop_params", _kw(thread=dict(THREAD_CREATE), additional_drop_params=["thread"]) + ), + Case("thread_create_stream", _kw(thread=dict(THREAD_CREATE), stream=True), upstream=(_stream(TEXT_STREAM),)), + Case( + "thread_continue_stream", + _kw(thread=dict(THREAD_CONTINUE), stream=True, messages=list(ADVISOR_TURNS)), + upstream=(_stream(TEXT_STREAM),), + ), + Case("thread_create_sync", _kw(thread=dict(THREAD_CREATE)), sync=True), + Case( + "thread_continue_sync_stream", + _kw(thread=dict(THREAD_CONTINUE), stream=True), + upstream=(_stream(TEXT_STREAM),), + sync=True, + ), + Case("thread_not_found", _kw(thread=dict(THREAD_CONTINUE)), upstream=(_thread_not_found(),)), + Case("thread_not_found_stream", _kw(thread=dict(THREAD_CONTINUE), stream=True), upstream=(_thread_not_found(),)), + Case("thread_not_found_sync", _kw(thread=dict(THREAD_CONTINUE)), upstream=(_thread_not_found(),), sync=True), + Case( + "thread_continue_retried_after_a_rejected_signature", + _replay(thread=dict(THREAD_CONTINUE)), + upstream=(_rejected(), _ok()), + ), +) + + +class _CallerHandler(AsyncHTTPHandler): + def __deepcopy__(self, memo: dict[int, object]) -> "_CallerHandler": + return _CallerHandler() + + +class _UncheckedHandler(_CallerHandler): + def __deepcopy__(self, memo: dict[int, object]) -> "_UncheckedHandler": + return _UncheckedHandler() + + async def post(self, url: str, **kwargs: object) -> httpx.Response: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: the test double ignores the rest of the post + return httpx.Response(503, text="unavailable") + + +class _UnreadHandler(_CallerHandler): + def __deepcopy__(self, memo: dict[int, object]) -> "_UnreadHandler": + return _UnreadHandler() + + async def post(self, url: str, **kwargs: object) -> httpx.Response: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: the test double ignores the rest of the post + return httpx.Response(500, stream=httpx.ByteStream(b"unread"), request=httpx.Request("POST", url)) + + +class _CallerHttpxClient(httpx.AsyncClient): + def __deepcopy__(self, memo: dict[int, object]) -> "_CallerHttpxClient": + return _CallerHttpxClient() + + +class _CallerSyncHandler(HTTPHandler): + def __deepcopy__(self, memo: dict[int, object]) -> "_CallerSyncHandler": + return _CallerSyncHandler() + + +def _caller(**kwargs: object) -> Mapping[str, object]: + return _kw(client=_CallerHandler(), **kwargs) + + +_TLS_LEVEL: Final = _tls(ssl_security_level="DEFAULT@SECLEVEL=1") +_TLS_CURVE: Final = _tls(ssl_ecdh_curve="secp521r1") + +TRANSPORT_CASES: Final = ( + Case("caller_client", _caller()), + Case("caller_client_stream", _caller(stream=True), upstream=(_stream(TEXT_STREAM),)), + Case( + "caller_client_upstream_429", + _caller(), + upstream=( + _failure(429, "rate_limit_error", "slow down", **{"retry-after": "7", "request-id": "req_01Parity"}), + ), + ), + Case( + "caller_client_stream_upstream_429", + _caller(stream=True), + upstream=(_failure(429, "rate_limit_error", "slow down", **{"retry-after": "3"}),), + ), + Case( + "caller_client_non_json_error", + _caller(), + upstream=(ResponseSpec(body=b"bad gateway", status=502),), + ), + Case("caller_client_invalid_json_success", _caller(), upstream=(ResponseSpec(body=b"not json", status=200),)), + Case("caller_client_timeout", _caller(timeout=0.5), upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), delay=1.5),)), + Case( + "caller_client_stream_read_stalls_past_timeout", + _caller(stream=True, timeout=0.5), + upstream=(_stream(SLOW_STREAM, chunk_delay=1.0),), + ), + Case( + "caller_client_thinking_signature_rejected_then_retried", + _kw(**_replay(), client=_CallerHandler()), + upstream=(_rejected(), _ok()), + ), + Case( + "caller_client_strict_tools_rejected_then_retried_stream", + _strict(drop_params=True, stream=True, client=_CallerHandler()), + upstream=(_rejected(STRICT_REJECTION), _stream(TEXT_STREAM)), + ), + Case("caller_client_sync", _caller(), sync=True), + Case("caller_client_post_returns_unchecked_failure", _kw(client=_UncheckedHandler()), upstream=()), + Case("caller_client_post_returns_unread_failure", _kw(client=_UnreadHandler(), stream=True), upstream=()), + Case("caller_httpx_client_is_ignored", _kw(client=_CallerHttpxClient())), + Case("caller_sync_handler_is_ignored", _kw(client=_CallerSyncHandler())), + Case("tls_security_level", settings=_TLS_LEVEL), + Case("tls_security_level_stream", _kw(stream=True), upstream=(_stream(TEXT_STREAM),), settings=_TLS_LEVEL), + Case("tls_ecdh_curve", settings=_TLS_CURVE), + Case("tls_ecdh_curve_x448", settings=_tls(ssl_ecdh_curve="X448")), + Case("tls_ecdh_curve_upstream_500", upstream=(_failure(500, "api_error", "internal"),), settings=_TLS_CURVE), + Case( + "tls_security_level_from_environment", + env=MappingProxyType({"SSL_SECURITY_LEVEL": "DEFAULT@SECLEVEL=1"}), + settings=_tls(), + ), + Case("tls_security_level_python_cannot_apply", upstream=(), settings=_tls(ssl_security_level="NOT-A-CIPHER")), + Case("tls_ecdh_curve_nist_name", settings=_tls(ssl_ecdh_curve="P-256")), + Case("tls_ecdh_curve_padded_lowercase", settings=_tls(ssl_ecdh_curve=" x25519 ")), + Case("negative_timeout", _kw(timeout=-1), upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), delay=0.3),)), + Case( + "negative_timeout_stream", + _kw(stream=True, timeout=-0.5), + upstream=(replace(_stream(TEXT_STREAM), delay=0.3),), + ), + Case( + "zero_timeout_httpx_transport", + _kw(timeout=0), + env=MappingProxyType({"DISABLE_AIOHTTP_TRANSPORT": "true"}), + settings=_tls(), + ), + Case( + "negative_timeout_httpx_transport", + _kw(timeout=-1), + env=MappingProxyType({"DISABLE_AIOHTTP_TRANSPORT": "true"}), + settings=_tls(), + ), +) + + +class _RewritingHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + kwargs["tools"] = [dict(WEATHER_TOOL)] + kwargs["tool_choice"] = {"type": "tool", "name": "get_weather"} + kwargs["temperature"] = 0.25 + kwargs["top_k"] = 7 + kwargs["system"] = "Answer as a forecaster." + kwargs["stop_sequences"] = ["STOP"] + kwargs["metadata"] = {"user_id": "hooked-user"} + kwargs["extra_headers"] = {"x-hooked": "1"} + return kwargs + + +class _InPlaceHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + messages.append({"role": "assistant", "content": [{"type": "text", "text": ""}]}) + messages.append({"role": "user", "content": "And Germany?"}) + return None + + +class _UnstreamingHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + return {**kwargs, "stream": not kwargs["stream"]} + + +class _MockingHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + return {**kwargs, "mock_response": "Hi from the hook"} + + +class _RaisingHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + raise ValueError("the hook refused the request") + + +class _ModelHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + return {**kwargs, "model": OPUS_ADAPTIVE} + + +class _CredentialHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + return {**kwargs, "api_key": "sk-ant-hooked"} + + +def _instance_hook() -> CustomLogger: + async def hook(model: str, messages: list[dict[str, object]], kwargs: dict[str, object]) -> dict[str, object]: + return {**kwargs, "temperature": 0.25, "metadata": {"user_id": "instance-hook"}} + + logger: Final = CustomLogger() + logger.async_pre_request_hook = hook # pyright: ignore[reportAttributeAccessIssue] # rebind-ok: an instance-level override, as a caller may install one + return logger + + +PRE_REQUEST_HOOK_CASES: Final = ( + Case("pre_request_hook_rewrites_the_request", callbacks=(_RewritingHook,), upstream=(_ok(TOOL_USE_RESPONSE),)), + Case("pre_request_hook_edits_messages_in_place", callbacks=(_InPlaceHook,)), + Case( + "pre_request_hook_after_prompt_caching", + _kw(cache_control_injection_points=[{"location": "message", "role": "user"}]), + callbacks=(_InPlaceHook,), + ), + Case( + "pre_request_hook_unstreams", + _kw(stream=True), + callbacks=(_UnstreamingHook,), + ), + Case( + "pre_request_hook_unstreams_behind_a_cache", + _kw(stream=True), + callbacks=(_UnstreamingHook,), + upstream=(_ok(), _ok()), + cache=Cache, + ), + Case("pre_request_hook_streams", callbacks=(_UnstreamingHook,), upstream=(_stream(TEXT_STREAM),)), + Case("pre_request_hook_sets_a_mock", callbacks=(_MockingHook,), upstream=()), + Case("pre_request_hook_raises", callbacks=(_RaisingHook,), upstream=()), + Case("pre_request_hook_set_on_the_instance", callbacks=(_instance_hook,)), + Case("pre_request_hook_names_the_model", callbacks=(_ModelHook,), upstream=()), + Case("pre_request_hook_names_a_credential", callbacks=(_CredentialHook,), upstream=()), + Case( + "pre_request_hook_two_hooks_chain", callbacks=(_RewritingHook, _InPlaceHook), upstream=(_ok(TOOL_USE_RESPONSE),) + ), + Case( + "pre_request_hook_behind_a_cache_hit", + callbacks=(_RewritingHook,), + upstream=(_ok(TOOL_USE_RESPONSE), _ok(TOOL_USE_RESPONSE)), + cache=Cache, + ), +) + +_OBJECTS: Final = TypeAdapter(list[dict[str, object]]) +_VOLATILE_HOOK_KWARGS: Final = frozenset({"litellm_call_id", "litellm_logging_obj"}) + + +def _content(response: object) -> list[dict[str, object]]: + if not isinstance(response, dict): + return [] + return _OBJECTS.validate_python(_OBJECT.validate_python(response).get("content") or []) + + +def _tool_calls(response: object) -> list[dict[str, object]]: + return [block for block in _content(response) if block.get("type") == "tool_use"] + + +class _FollowUpOnToolUse(CustomLogger): + """Answers each tool call with a canned result in one follow-up turn, recording what the hooks were handed.""" + + def __init__(self) -> None: + super().__init__() # pyright: ignore[reportUnknownMemberType] # CustomLogger takes untyped kwargs + self.seen: list[Mapping[str, object]] = [] + + async def async_should_run_agentic_loop( # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + self, + response: object, + model: str, + messages: list[dict[str, object]], + tools: list[dict[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict[str, object], + ) -> tuple[bool, dict[str, object]]: + calls: Final = _tool_calls(response) + self.seen.append( + { + "hook": "should_run", + "response": _OBJECT.validate_python({**response, "id": ""}) if isinstance(response, dict) else None, + "model": model, + "messages": _OBJECTS.validate_python(messages), + "tools": tools, + "stream": stream, + "custom_llm_provider": custom_llm_provider, + "kwargs": sorted(name for name in kwargs if name not in _VOLATILE_HOOK_KWARGS), + "api_key": kwargs.get("api_key"), + } + ) + return bool(calls), {"tool_calls": calls} + + async def async_build_agentic_loop_plan( # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + self, + tools: dict[str, object], + model: str, + messages: list[dict[str, object]], + response: object, + anthropic_messages_provider_config: object, + anthropic_messages_optional_request_params: dict[str, object], + logging_obj: Logging, + stream: bool, + kwargs: dict[str, object], + ) -> AgenticLoopPlan: + self.seen.append( + { + "hook": "build_plan", + "optional_params": anthropic_messages_optional_request_params, + "config": type(anthropic_messages_provider_config).__name__, + "agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params"), + "kwargs": sorted(name for name in kwargs if name not in _VOLATILE_HOOK_KWARGS), + } + ) + return self.plan(messages, response) + + def plan(self, messages: list[dict[str, object]], response: object) -> AgenticLoopPlan: + answers: Final = [ + {"type": "tool_result", "tool_use_id": call["id"], "content": "18C, sunny"} + for call in _tool_calls(response) + ] + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[ + *messages, + {"role": "assistant", "content": _content(response)}, + {"role": "user", "content": answers}, + ] + ), + ) + + +class _HeldBack(_FollowUpOnToolUse): + server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({"get_weather"}) + + +class _Terminates(_FollowUpOnToolUse): + def plan(self, messages: list[dict[str, object]], response: object) -> AgenticLoopPlan: + return AgenticLoopPlan(terminate=True, stop_reason="answered") + + +class _Overrides(_FollowUpOnToolUse): + def plan(self, messages: list[dict[str, object]], response: object) -> AgenticLoopPlan: + return AgenticLoopPlan(response_override={**TEXT_RESPONSE, "id": "msg_01Override"}) + + +class _Fails(_FollowUpOnToolUse): + async def async_build_agentic_loop_plan( # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + self, + tools: dict[str, object], + model: str, + messages: list[dict[str, object]], + response: object, + anthropic_messages_provider_config: object, + anthropic_messages_optional_request_params: dict[str, object], + logging_obj: Logging, + stream: bool, + kwargs: dict[str, object], + ) -> AgenticLoopPlan: + raise ValueError("the agentic plan failed") + + +class _WebSearch(WebSearchInterceptionLogger): + """The stock web search interception, recording what its pre-request hook leaves for its agentic hook.""" + + def __init__(self, enabled_providers: list[str] | None = None, max_agentic_loops: int | None = None) -> None: + super().__init__( # pyright: ignore[reportUnknownMemberType] # the logger annotates bare containers + enabled_providers=list(enabled_providers) if enabled_providers is not None else None, + max_agentic_loops=max_agentic_loops, + ) + self.seen: list[Mapping[str, object]] = [] + + async def async_should_run_agentic_loop( # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + self, + response: object, + model: str, + messages: list[dict[str, object]], + tools: list[dict[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict[str, object], + ) -> tuple[bool, dict[str, object]]: + self.seen.append( + { + "model": model, + "messages": _OBJECTS.validate_python(messages), + "tools": tools, + "stream": stream, + "custom_llm_provider": custom_llm_provider, + "kwargs": { + name: kwargs[name] + for name in sorted(kwargs) + if name == "max_agentic_loops" or name.startswith("_websearch_interception") + }, + "kwarg_names": sorted(name for name in kwargs if name not in _VOLATILE_HOOK_KWARGS), + } + ) + return await super().async_should_run_agentic_loop( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the logger annotates bare containers + response=response, + model=model, + messages=messages, + tools=tools, + stream=stream, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + ) + + +class _DeploymentHooks(CustomLogger): + def __init__(self) -> None: + super().__init__() # pyright: ignore[reportUnknownMemberType] # CustomLogger takes untyped kwargs + self.seen: list[Mapping[str, object]] = [] + + async def async_post_call_success_deployment_hook( # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + self, request_data: dict[str, object], response: object, call_type: object + ) -> object: + self.seen.append({"hook": "success", "call_type": str(call_type)}) + return None + + async def async_post_call_failure_deployment_hook( # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + self, request_data: Mapping[str, object], exception: Exception, call_type: object + ) -> None: + self.seen.append({"hook": "failure", "error": type(exception).__name__, "call_type": str(call_type)}) + + +def _loggers(*kinds: Callable[[], CustomLogger]) -> Callable[[], Sequence[CustomLogger]]: + return lambda: tuple(kind() for kind in kinds) + + +def _web_search() -> CustomLogger: + return _WebSearch(enabled_providers=["anthropic"], max_agentic_loops=2) + + +async def _search(*args: object, **kwargs: object) -> SearchResponse: # kwargs-ok: litellm.asearch's call shape + return SearchResponse.model_validate( + {"results": [{"title": "Paris", "url": "https://example.com/paris", "snippet": "Paris is the capital."}]} + ) + + +WEB_SEARCH_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"type": "web_search_20250305", "name": "web_search", "max_uses": 2} +) +WEB_SEARCH_CALL_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "id": "msg_01ParitySearch", + "content": [ + {"type": "tool_use", "id": "toolu_01Search", "name": "litellm_web_search", "input": {"query": "capital"}} + ], + "stop_reason": "tool_use", + } +) +_WEB_SEARCH_SETTINGS: Final[Mapping[str, object]] = MappingProxyType({"asearch": _search}) +_WEATHER: Final = _kw(tools=[dict(WEATHER_TOOL)]) + +AGENTIC_CASES: Final = ( + Case("agentic_loop_without_tool_use", _WEATHER, loggers=_loggers(_FollowUpOnToolUse)), + Case( + "agentic_loop_follow_up", + _WEATHER, + upstream=(_ok(TOOL_USE_RESPONSE), _ok()), + loggers=_loggers(_FollowUpOnToolUse), + ), + Case( + "agentic_loop_follow_up_with_request_params", + _kw( + tools=[dict(WEATHER_TOOL)], + system="Answer in one word.", + temperature=0.3, + metadata={"user_id": "user-123"}, + thinking={"type": "enabled", "budget_tokens": 2048}, + max_tokens=4096, + messages=[{"role": "user", "content": "Weather?", "provider_specific_fields": {"x": 1}}], + ), + upstream=(_ok(TOOL_USE_RESPONSE), _ok()), + loggers=_loggers(_FollowUpOnToolUse), + ), + *( + Case( + f"agentic_loop_follow_up_keeps_{name}", + _kw(**{"tools": [dict(WEATHER_TOOL)], **params}), + upstream=(_ok(TOOL_USE_RESPONSE), _ok()), + model=model, + loggers=_loggers(_FollowUpOnToolUse), + ) + for name, params, model in ( + ( + "output_format", + { + "output_format": { + "type": "json_schema", + "schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + }, + SONNET, + ), + ("reasoning_effort", {"reasoning_effort": "high", "max_tokens": 8192}, SONNET), + ("output_config_effort", {"output_config": {"effort": "medium"}}, OPUS_ADAPTIVE), + ( + "injected_cache_points", + {"cache_control_injection_points": [{"location": "message", "role": "user"}]}, + SONNET, + ), + ("context_management", {"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}, SONNET), + ("speed", {"speed": "fast"}, OPUS_FAST), + ("service_tier", {"service_tier": "auto"}, SONNET), + ("inference_geo", {"inference_geo": "us"}, SONNET), + ( + "container", + {"container": {"skills": [{"type": "anthropic", "skill_id": "pptx", "version": "latest"}]}}, + SONNET, + ), + ("custom_tool", {"tools": [{**WEATHER_TOOL, "type": "custom"}]}, SONNET), + ("metadata_extras", {"metadata": {"user_id": "user-123", "session": "dropped"}}, SONNET), + ) + ), + Case( + "agentic_loop_prices_the_final_round", + _kw(tools=[dict(WEATHER_TOOL)]), + upstream=(_ok(TOOL_USE_RESPONSE), _ok({**TEXT_RESPONSE, "usage": {"input_tokens": 900, "output_tokens": 300}})), + loggers=_loggers(_FollowUpOnToolUse), + ), + Case( + "agentic_loop_prices_the_final_round_stream", + _kw(stream=True, tools=[dict(WEATHER_TOOL)]), + upstream=( + _stream(TOOL_STREAM), + _stream( + _events(_message_start("msg_01ParityLate"), *_text_block(0, "Paris"), *_message_end("end_turn", 300)) + ), + ), + loggers=_loggers(_FollowUpOnToolUse), + ), + Case( + "agentic_loop_follow_up_stream", + _kw(stream=True, tools=[dict(WEATHER_TOOL)]), + upstream=(_stream(TOOL_STREAM), _stream(TEXT_STREAM)), + loggers=_loggers(_FollowUpOnToolUse), + ), + Case( + "agentic_loop_follow_up_stream_in_one_write", + _kw(stream=True, tools=[dict(WEATHER_TOOL)]), + upstream=(_burst(TOOL_STREAM), _burst(TEXT_STREAM)), + loggers=_loggers(_FollowUpOnToolUse), + partitioned=True, + ), + Case( + "agentic_loop_stream_without_tool_use_in_one_write", + _kw(stream=True, tools=[dict(WEATHER_TOOL)]), + upstream=(_burst(TEXT_STREAM),), + loggers=_loggers(_FollowUpOnToolUse), + partitioned=True, + ), + Case( + "agentic_loop_stream_without_tool_use", + _kw(stream=True, tools=[dict(WEATHER_TOOL)]), + upstream=(_stream(TEXT_STREAM),), + loggers=_loggers(_FollowUpOnToolUse), + ), + Case( + "agentic_loop_holds_back_a_server_fulfilled_tool", + _kw(stream=True, tools=[dict(WEATHER_TOOL)]), + upstream=(_stream(TOOL_STREAM), _stream(TEXT_STREAM)), + loggers=_loggers(_HeldBack), + ), + Case( + "agentic_loop_terminates", + _WEATHER, + upstream=(_ok(TOOL_USE_RESPONSE),), + loggers=_loggers(_Terminates), + ), + Case( + "agentic_loop_response_override", + _WEATHER, + upstream=(_ok(TOOL_USE_RESPONSE),), + loggers=_loggers(_Overrides), + ), + Case( + "agentic_loop_plan_fails", + _WEATHER, + upstream=(_ok(TOOL_USE_RESPONSE),), + loggers=_loggers(_Fails), + ), + Case( + "agentic_loop_follow_up_fails", + _WEATHER, + upstream=(_ok(TOOL_USE_RESPONSE), _failure(529, "overloaded_error", "Overloaded")), + loggers=_loggers(_FollowUpOnToolUse), + ), + Case( + "agentic_loop_plan_fails_runs_the_failure_deployment_hook_once", + _WEATHER, + upstream=(_ok(TOOL_USE_RESPONSE),), + loggers=_loggers(_Fails, _DeploymentHooks), + ), + Case( + "agentic_loop_follow_up_fails_runs_the_failure_deployment_hook_per_round", + _WEATHER, + upstream=(_ok(TOOL_USE_RESPONSE), _failure(529, "overloaded_error", "Overloaded")), + loggers=_loggers(_FollowUpOnToolUse, _DeploymentHooks), + ), + Case( + "agentic_loop_follow_up_runs_the_success_deployment_hook_per_round", + _WEATHER, + upstream=(_ok(TOOL_USE_RESPONSE), _ok()), + loggers=_loggers(_FollowUpOnToolUse, _DeploymentHooks), + ), + Case( + "pre_request_hook_raises_runs_the_failure_deployment_hook_once", + callbacks=(_RaisingHook,), + upstream=(), + loggers=_loggers(_DeploymentHooks), + ), + Case( + "agentic_loop_refused_past_max_loops", + _kw(tools=[dict(WEATHER_TOOL)], max_agentic_loops=1), + upstream=(_ok(TOOL_USE_RESPONSE), _ok(TOOL_USE_RESPONSE)), + loggers=_loggers(_FollowUpOnToolUse), + ), + Case( + "agentic_loop_upstream_error_skips_the_hooks", + _WEATHER, + upstream=(_failure(400, "invalid_request_error", "bad"),), + loggers=_loggers(_FollowUpOnToolUse), + ), + Case( + "web_search_interception", + _kw(tools=[dict(WEB_SEARCH_TOOL)]), + upstream=(_ok(WEB_SEARCH_CALL_RESPONSE), _ok()), + settings=_WEB_SEARCH_SETTINGS, + loggers=_loggers(_web_search), + ), + Case( + "web_search_interception_converted_stream", + _kw(stream=True, tools=[dict(WEB_SEARCH_TOOL)]), + upstream=(_ok(WEB_SEARCH_CALL_RESPONSE), _ok()), + settings=_WEB_SEARCH_SETTINGS, + loggers=_loggers(_web_search), + ), + Case( + "web_search_interception_without_a_search_call", + _kw(stream=True, tools=[dict(WEB_SEARCH_TOOL)]), + settings=_WEB_SEARCH_SETTINGS, + loggers=_loggers(_web_search), + ), + Case( + "web_search_interception_for_a_provider_it_skips", + _kw(tools=[dict(WEB_SEARCH_TOOL)]), + loggers=_loggers(_WebSearch), + ), + Case( + "web_search_interception_rewrites_the_messages_it_hands_the_loop", + _kw( + stream=True, + tools=[dict(WEB_SEARCH_TOOL)], + messages=[ + {"role": "user", "content": [{"type": "text", "text": ""}, {"type": "text", "text": "Capital?"}]} + ], + ), + upstream=(_ok(WEB_SEARCH_CALL_RESPONSE), _ok()), + settings=_WEB_SEARCH_SETTINGS, + loggers=_loggers(_web_search), + ), +) + +CLASSIFIER_MODEL: Final = "openai/parity-classifier" +CLASSIFIER_CALL_TYPE: Final = "acompletion" +NON_CLAUDE: Final = "anthropic/parity-tool-model" +DANGEROUS_TOOL_USE: Final[tuple[JsonValue, ...]] = ( + { + "type": "dangerous_tool_use", + "classifier_context": { + "permission_mode": "auto", + "platform": "linux", + "trusted_directories": ["/repo"], + "git_state": {"cwd": "/repo", "branch": "main", "dirty": True}, + "ignored": "not rendered", + }, + }, +) +FLAGGED: Final = json.dumps({"verdicts": {"toolu_01Parity": {"flagged": True, "explanation": "Sends the weather."}}}) +CLEARED: Final = json.dumps({"verdicts": {"toolu_01Parity": {"flagged": False}}}) +CUT_OFF_TOOL_STREAM: Final = (*TOOL_STREAM[:-2], *_message_end("max_tokens", 22)) +UPSTREAM_ANSWERED: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TOOL_USE_RESPONSE, + "safeguard_results": [{"type": "dangerous_tool_use", "status": {"type": "unsupported"}}], + } +) + + +def _safeguarded(**kwargs: object) -> Mapping[str, object]: + return _kw(safeguards=list(DANGEROUS_TOOL_USE), messages=list(TOOL_TURNS), tools=[dict(WEATHER_TOOL)], **kwargs) + + +SAFEGUARDS_CASES: Final = ( + Case( + "safeguards_classifier_flags_tool_use", + _safeguarded(litellm_metadata={"user_api_key_end_user_id": "end-user-1", "user_api_key": "hashed-key"}), + upstream=(_ok(TOOL_USE_RESPONSE),), + model=NON_CLAUDE, + classifier=FLAGGED, + ), + Case( + "safeguards_classifier_clears_tool_use", + _safeguarded(), + upstream=(_ok(TOOL_USE_RESPONSE),), + model=NON_CLAUDE, + classifier=CLEARED, + ), + Case( + "safeguards_classifier_unparsable_reply", + _safeguarded(), + upstream=(_ok(TOOL_USE_RESPONSE),), + model=NON_CLAUDE, + classifier="I cannot tell.", + ), + Case( + "safeguards_classifier_rejects_long_input", + _safeguarded(), + upstream=(_ok(TOOL_USE_RESPONSE),), + model=NON_CLAUDE, + classifier="litellm.ContextWindowExceededError", + ), + Case("safeguards_classifier_without_tool_use", _safeguarded(), model=NON_CLAUDE, classifier=FLAGGED), + Case( + "safeguards_classifier_upstream_answered", + _safeguarded(), + upstream=(_ok(UPSTREAM_ANSWERED),), + model=NON_CLAUDE, + classifier=FLAGGED, + ), + Case( + "safeguards_classifier_stream_tool_use", + _safeguarded(stream=True), + upstream=(_stream(TOOL_STREAM),), + model=NON_CLAUDE, + classifier=FLAGGED, + ), + Case( + "safeguards_classifier_stream_cut_off", + _safeguarded(stream=True), + upstream=(_stream(CUT_OFF_TOOL_STREAM),), + model=NON_CLAUDE, + classifier=FLAGGED, + ), + Case( + "safeguards_classifier_stream_text", + _safeguarded(stream=True), + upstream=(_stream(TEXT_STREAM),), + model=NON_CLAUDE, + classifier=FLAGGED, + ), + Case( + "safeguards_classifier_upstream_failure", + _safeguarded(), + upstream=(_failure(529, "overloaded_error", "Overloaded"),), + model=NON_CLAUDE, + classifier=FLAGGED, + ), + Case( + "safeguards_claude_model_is_not_classified", + _safeguarded(), + upstream=(_ok(TOOL_USE_RESPONSE),), + classifier=FLAGGED, + ), + Case("safeguards_classifier_sync", _safeguarded(), model=NON_CLAUDE, sync=True, classifier=FLAGGED), +) + +INTERCEPTED_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType({"type": "parity_intercept", "name": "intercept"}) +_ECHOED_KWARGS: Final = ( + "model", + "messages", + "tools", + "stream", + "max_tokens", + "custom_llm_provider", + "api_key", + "metadata", + "system", + "temperature", + "tool_choice", +) + + +class _Intercepting(MessagesInterceptor): + def can_handle(self, tools: Sequence[Mapping[str, object]] | None, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == "anthropic" and any( + tool.get("type") == INTERCEPTED_TOOL["type"] for tool in tools or () + ) + + +class _Answering(_Intercepting): + async def handle(self, **kwargs: object) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: the interceptor contract's keyword call + echoed: Final = {name: kwargs.get(name) for name in _ECHOED_KWARGS} + answer: Final = { + **TEXT_RESPONSE, + "id": "msg_01ParityIntercepted", + "content": [{"type": "text", "text": json.dumps(echoed, sort_keys=True, default=repr)}], + "extra_keys": sorted(name for name in kwargs if name not in _ECHOED_KWARGS), + } + if kwargs.get("stream"): + return FakeAnthropicMessagesStreamIterator( + cast(AnthropicMessagesResponse, answer) + ) # cast-ok: a synthetic Messages response + return answer + + +class _Delegating(_Intercepting): + async def handle(self, **kwargs: object) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: the interceptor contract's keyword call + from litellm.messages import anthropic_messages + + tools: Final = cast(list[Mapping[str, object]], kwargs["tools"]) # cast-ok: the caller's tools + return await anthropic_messages( + model=kwargs["model"], + messages=kwargs["messages"], + max_tokens=kwargs["max_tokens"], + tools=[tool for tool in tools if tool.get("type") != INTERCEPTED_TOOL["type"]], + api_key=kwargs["api_key"], + api_base=kwargs["api_base"], + stream=kwargs["stream"], + ) + + +class _Refusing(_Intercepting): + async def handle(self, **kwargs: object) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: the interceptor contract's keyword call + raise ValueError("the interceptor refused the call") + + +def _intercepted(**kwargs: object) -> Mapping[str, object]: + return _kw(tools=[dict(INTERCEPTED_TOOL), dict(WEATHER_TOOL)], **kwargs) + + +class _InterceptedToolHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + return {**kwargs, "tools": [dict(INTERCEPTED_TOOL), *cast(list[object], kwargs.get("tools") or [])]} + + +class _UninterceptedToolHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + return {**kwargs, "tools": [dict(WEATHER_TOOL)]} + + +class _DeploymentHookRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() # pyright: ignore[reportUnknownMemberType] # CustomLogger takes untyped kwargs + self.seen: list[Mapping[str, object]] = [] + + async def async_pre_call_deployment_hook(self, kwargs: dict[str, object], call_type: object) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + self.seen.append({"logging_obj": type(kwargs.get("litellm_logging_obj")).__name__, "call_type": str(call_type)}) + + +class _DeploymentResponseRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() # pyright: ignore[reportUnknownMemberType] # CustomLogger takes untyped kwargs + self.seen: list[Mapping[str, object]] = [] + + async def async_post_call_success_deployment_hook( # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + self, request_data: dict[str, object], response: object, call_type: object + ) -> object: + self.seen.append({"response": type(response).__name__, "call_type": str(call_type)}) + return None + + +DEPLOYMENT_HOOK_CASES: Final = ( + Case( + "deployment_hook_sees_the_stream_of_a_string_flag", + _kw(stream="true"), + upstream=(_stream(TEXT_STREAM),), + loggers=_loggers(_DeploymentResponseRecorder), + ), + Case( + "deployment_hook_sees_the_stream_of_an_integer_flag", + _kw(stream=1), + upstream=(_stream(TEXT_STREAM),), + loggers=_loggers(_DeploymentResponseRecorder), + ), + Case( + "deployment_hook_sees_the_stream_of_a_bool_flag", + _kw(stream=True), + upstream=(_stream(TEXT_STREAM),), + loggers=_loggers(_DeploymentResponseRecorder), + ), + Case( + "deployment_hook_sees_a_stream_a_hook_made", + upstream=(_stream(TEXT_STREAM),), + callbacks=(_UnstreamingHook,), + loggers=_loggers(_DeploymentResponseRecorder), + ), + Case( + "deployment_hook_sees_a_stream_a_hook_made_on_an_integer", + upstream=(_stream(TEXT_STREAM),), + callbacks=(_IntegerStreamHook,), + loggers=_loggers(_DeploymentResponseRecorder), + ), + Case("deployment_hook_sees_the_logging_object", loggers=_loggers(_DeploymentHookRecorder)), + Case( + "deployment_hook_sees_the_logging_object_stream", + _kw(stream=True), + upstream=(_stream(TEXT_STREAM),), + loggers=_loggers(_DeploymentHookRecorder), + ), +) + +INTERCEPTOR_CASES: Final = ( + Case("interceptor_answers", _intercepted(), upstream=(), interceptor=_Answering), + Case( + "pre_request_hook_adds_a_tool_an_interceptor_owns", + _kw(tools=[dict(WEATHER_TOOL)]), + upstream=(), + callbacks=(_InterceptedToolHook,), + interceptor=_Answering, + ), + Case( + "pre_request_hook_adds_a_tool_an_interceptor_owns_stream", + _kw(tools=[dict(WEATHER_TOOL)], stream=True), + upstream=(), + callbacks=(_InterceptedToolHook,), + interceptor=_Answering, + ), + Case( + "pre_request_hook_unstreams_a_call_an_interceptor_owns", + _intercepted(stream=True), + upstream=(), + callbacks=(_UnstreamingHook,), + interceptor=_Answering, + ), + Case( + "pre_request_hook_removes_the_tool_an_interceptor_owns", + _intercepted(), + callbacks=(_UninterceptedToolHook,), + interceptor=_Answering, + ), + Case( + "interceptor_answers_with_system_and_metadata", + _intercepted(system="Be brief.", metadata={"user_id": "user-123"}, temperature=0.5), + upstream=(), + interceptor=_Answering, + ), + Case("interceptor_answers_stream", _intercepted(stream=True), upstream=(), interceptor=_Answering), + Case( + "interceptor_wins_over_mock_response", + _intercepted(mock_response="Hi from the mock"), + upstream=(), + interceptor=_Answering, + ), + Case( + "interceptor_answers_bare_model", + _intercepted(custom_llm_provider="anthropic"), + upstream=(), + model="claude-sonnet-4-5", + interceptor=_Answering, + ), + Case("interceptor_delegates_to_the_public_api", _intercepted(), interceptor=_Delegating), + Case( + "interceptor_delegates_a_stream", + _intercepted(stream=True), + upstream=(_stream(TEXT_STREAM),), + interceptor=_Delegating, + ), + Case("interceptor_raises", _intercepted(), upstream=(), interceptor=_Refusing), + Case("interceptor_is_not_consulted_on_a_sync_call", _intercepted(), upstream=(), sync=True, interceptor=_Answering), + Case( + "pre_request_hook_is_not_run_on_a_sync_call_an_interceptor_owns", + _intercepted(), + upstream=(), + sync=True, + callbacks=(_RaisingHook,), + interceptor=_Answering, + ), + Case("pre_request_hook_is_not_run_on_a_sync_call", upstream=(), sync=True, callbacks=(_RaisingHook,)), + Case( + "interceptor_raises_runs_the_failure_deployment_hook_once", + _intercepted(), + upstream=(), + interceptor=_Refusing, + loggers=_loggers(_DeploymentHooks), + ), + Case("interceptor_answer_is_cached", _intercepted(), upstream=(), cache=Cache, interceptor=_Answering), + Case( + "interceptor_stream_answer_with_cache", + _intercepted(stream=True), + upstream=(), + cache=Cache, + interceptor=_Answering, + ), + Case("interceptor_ignores_other_calls", _kw(tools=[dict(WEATHER_TOOL)]), interceptor=_Answering), +) + +SHORT_CIRCUITED: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "id": "msg_01ParityShortCircuit", + "content": [{"type": "text", "text": "Paris is the capital."}], + } +) + + +class _ShortCircuit(WebSearchInterceptionLogger): + """A web search interception whose replaced short-circuit records its inputs and answers every call.""" + + answer: Mapping[str, JsonValue] | None = SHORT_CIRCUITED + + def __init__(self) -> None: + super().__init__() # pyright: ignore[reportUnknownMemberType] # the logger annotates bare containers + self.seen: list[Mapping[str, object]] = [] + + async def try_short_circuit_search( # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + self, + model: str, + messages: list[dict[str, object]], + tools: list[dict[str, object]] | None, + custom_llm_provider: str | None, + kwargs: Mapping[str, object] | None = None, + ) -> dict[str, object] | None: + passed: Final = kwargs or {} + self.seen.append( + { + "model": model, + "messages": _OBJECTS.validate_python(messages), + "tools": tools, + "custom_llm_provider": custom_llm_provider, + "kwargs": sorted(name for name in passed if name not in _VOLATILE_HOOK_KWARGS), + "metadata": passed.get("metadata"), + "safeguards": passed.get("safeguards"), + } + ) + return None if self.answer is None else _OBJECT.validate_python(copy.deepcopy(dict(self.answer))) + + +class _ShortCircuitPassing(_ShortCircuit): + answer = None + + +class _ShortCircuitToolUse(_ShortCircuit): + answer = TOOL_USE_RESPONSE + + +class _ShortCircuitRefusing(_ShortCircuit): + async def try_short_circuit_search( # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + self, + model: str, + messages: list[dict[str, object]], + tools: list[dict[str, object]] | None, + custom_llm_provider: str | None, + kwargs: Mapping[str, object] | None = None, + ) -> dict[str, object] | None: + await super().try_short_circuit_search(model, messages, tools, custom_llm_provider, kwargs) + raise ValueError("the short-circuit search failed") + + +def _searched(**kwargs: object) -> Mapping[str, object]: + return _kw(tools=[dict(WEB_SEARCH_TOOL)], metadata={"user_id": "parity-user"}, **kwargs) + + +SHORT_CIRCUIT_CASES: Final = ( + Case("short_circuit_answers", _searched(), upstream=(), loggers=_loggers(_ShortCircuit)), + Case("short_circuit_answers_stream", _searched(stream=True), upstream=(), loggers=_loggers(_ShortCircuit)), + Case("short_circuit_raises", _searched(), upstream=(), loggers=_loggers(_ShortCircuitRefusing)), + Case("short_circuit_raises_stream", _searched(stream=True), upstream=(), loggers=_loggers(_ShortCircuitRefusing)), + Case("short_circuit_passes_the_call_on", _searched(), loggers=_loggers(_ShortCircuitPassing)), + Case( + "short_circuit_passes_a_stream_on", + _searched(stream=True), + upstream=(_stream(TEXT_STREAM),), + loggers=_loggers(_ShortCircuitPassing), + ), + Case( + "short_circuit_first_answer_wins", + _searched(), + upstream=(), + loggers=_loggers(_ShortCircuitPassing, _ShortCircuit, _ShortCircuitRefusing), + ), + Case( + "short_circuit_answers_before_the_interceptor", + _intercepted(), + upstream=(), + loggers=_loggers(_ShortCircuit), + interceptor=_Answering, + ), + Case( + "short_circuit_passes_on_to_the_interceptor", + _intercepted(), + upstream=(), + loggers=_loggers(_ShortCircuitPassing), + interceptor=_Answering, + ), + Case( + "short_circuit_answer_with_cache", + _searched(), + upstream=(), + cache=Cache, + loggers=_loggers(_ShortCircuit), + ), + Case( + "short_circuit_answer_is_classified", + _safeguarded(), + upstream=(), + model=NON_CLAUDE, + classifier=FLAGGED, + loggers=_loggers(_ShortCircuitToolUse), + ), + Case( + "short_circuit_stream_answer_is_classified", + _safeguarded(stream=True), + upstream=(), + model=NON_CLAUDE, + classifier=FLAGGED, + loggers=_loggers(_ShortCircuitToolUse), + ), + Case("short_circuit_is_not_consulted_on_a_sync_call", _searched(), sync=True, loggers=_loggers(_ShortCircuit)), +) + +MCP_REFERENCE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "type": "mcp", + "server_label": "litellm", + "server_url": "litellm_proxy/mcp/deepwiki", + "require_approval": "never", + } +) +MCP_NEEDS_APPROVAL: Final[Mapping[str, JsonValue]] = MappingProxyType( + {key: value for key, value in MCP_REFERENCE.items() if key != "require_approval"} +) +WIKI_TOOL_CALL: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"type": "tool_use", "id": "toolu_01Wiki", "name": "read_wiki", "input": {"repo": "a/b"}} +) +WIKI_TOOL_USE_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "id": "msg_01ParityWiki", + "content": [{"type": "text", "text": "Reading the wiki."}, dict(WIKI_TOOL_CALL)], + "stop_reason": "tool_use", + } +) +WIKI_TOOL_STREAM: Final = _events( + _message_start("msg_01ParityWikiStream"), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "tool_use", "id": "toolu_01Wiki", "name": "read_wiki", "input": {}}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"repo": "a/b"}'}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + *_message_end("tool_use", 9), +) + + +def _wiki(**kwargs: object) -> Mapping[str, object]: + return _kw(**{"tools": [dict(MCP_REFERENCE)], **kwargs}) + + +class _McpToolHook(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + return {**kwargs, "tools": [dict(MCP_REFERENCE), *cast(list[object], kwargs.get("tools") or [])]} + + +_SHARED_LOGGER: Final = ( + "the gateway's follow-up call rewrites the shared Logging object while the first call's async success callback " + "is still queued, on Python and Rust alike, so the callback logs whichever call it saw" +) +SHARED_LOGGER_RACE: Final = MappingProxyType( + { + "logged_request": _SHARED_LOGGER, + "logged_metadata": _SHARED_LOGGER, + "logged_response_headers": _SHARED_LOGGER, + "logged_sent_request": _SHARED_LOGGER, + } +) + + +MCP_CASES: Final = ( + Case( + "mcp_tool_runs_then_answers", + _wiki(), + upstream=(_ok(WIKI_TOOL_USE_RESPONSE), _ok()), + gateway=True, + racy=SHARED_LOGGER_RACE, + ), + Case( + "mcp_tool_runs_then_answers_stream", + _wiki(stream=True), + upstream=(_ok(WIKI_TOOL_USE_RESPONSE), _ok()), + gateway=True, + racy=SHARED_LOGGER_RACE, + ), + Case("mcp_answers_without_a_tool", _wiki(), gateway=True), + Case( + "mcp_tool_beside_a_native_tool", + _wiki(tools=[dict(MCP_REFERENCE), dict(WEATHER_TOOL)], system="Be terse.", temperature=0.2), + upstream=(_ok(WIKI_TOOL_USE_RESPONSE), _ok()), + gateway=True, + racy=SHARED_LOGGER_RACE, + ), + Case( + "mcp_whole_gateway", + _wiki(tools=[{**MCP_REFERENCE, "server_url": "litellm_proxy"}]), + upstream=(_ok(WIKI_TOOL_USE_RESPONSE), _ok()), + gateway=True, + racy=SHARED_LOGGER_RACE, + ), + Case( + "mcp_tool_needs_approval", + _wiki(tools=[dict(MCP_NEEDS_APPROVAL)]), + upstream=(_ok(WIKI_TOOL_USE_RESPONSE),), + gateway=True, + ), + Case( + "mcp_tool_needs_approval_stream", + _wiki(tools=[dict(MCP_NEEDS_APPROVAL)], stream=True), + upstream=(_stream(WIKI_TOOL_STREAM),), + gateway=True, + ), + Case( + "mcp_follow_up_fails", + _wiki(), + upstream=(_ok(WIKI_TOOL_USE_RESPONSE), _failure(429, "rate_limit_error", "slow down")), + gateway=True, + racy=SHARED_LOGGER_RACE, + ), + Case( + "mcp_first_call_fails", + _wiki(), + upstream=(_failure(400, "invalid_request_error", "max_tokens: too large"),), + gateway=True, + ), + Case( + "mcp_metadata_and_litellm_metadata", + _wiki(metadata={"user_id": "user-123"}, litellm_metadata={"user_api_key": "hashed", "tags": ["t1"]}), + upstream=(_ok(WIKI_TOOL_USE_RESPONSE), _ok()), + gateway=True, + racy=SHARED_LOGGER_RACE, + ), + Case( + "mcp_unserved_proxy_path_passes_through", + _wiki(tools=[{**MCP_REFERENCE, "server_url": "https://gateway.example/mcp/unknown"}]), + gateway=True, + ), + Case( + "mcp_skip_flag_passes_the_reference_through", + _wiki(_skip_mcp_handler=True), + gateway=True, + ), + Case( + "mcp_tool_execution_raises", + _wiki(), + upstream=(_ok({**WIKI_TOOL_USE_RESPONSE, "content": [{**WIKI_TOOL_CALL, "input": {"repo": "boom"}}]}),), + gateway=True, + ), + Case( + "mcp_cache_hit", + _wiki(), + upstream=(_ok(WIKI_TOOL_USE_RESPONSE), _ok(), _ok(WIKI_TOOL_USE_RESPONSE), _ok()), + cache=Cache, + gateway=True, + ), + Case("mcp_sync", _wiki(), upstream=(), sync=True, gateway=True), + Case("mcp_mock_answers_first", _wiki(mock_response="Hi from the mock"), upstream=(), gateway=True), + Case("mcp_sync_mock_answers_first", _wiki(mock_response="Hi from the mock"), upstream=(), sync=True, gateway=True), + Case( + "pre_request_hook_adds_an_mcp_tool", + _kw(tools=[dict(WEATHER_TOOL)]), + upstream=(_ok(WIKI_TOOL_USE_RESPONSE), _ok()), + callbacks=(_McpToolHook,), + gateway=True, + racy=SHARED_LOGGER_RACE, + ), + Case( + "pre_request_hook_removes_the_mcp_tool", + _wiki(), + callbacks=(_UninterceptedToolHook,), + gateway=True, + ), + Case( + "interceptor_answers_before_the_mcp_gateway", + _kw(tools=[dict(INTERCEPTED_TOOL), dict(MCP_REFERENCE)]), + upstream=(), + interceptor=_Answering, + gateway=True, + ), +) + + +class _OrderedMemory(InMemoryCache): + def __init__(self) -> None: + super().__init__() + self.ops: list[str] = [] + + async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: records the backend writes in order + self.ops.append(f"set {key}") + await super().async_set_cache(key, value, **kwargs) # pyright: ignore[reportUnknownMemberType] # InMemoryCache leaves its signature untyped + + async def async_get_cache(self, key: str, **kwargs: object) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: records the backend reads in order + self.ops.append(f"get {key}") + return await super().async_get_cache(key, **kwargs) # pyright: ignore[reportUnknownMemberType] # InMemoryCache leaves its signature untyped + + +def _ordered_cache() -> Cache: + cache: Final = Cache() + cache.cache = _OrderedMemory() + return cache + + +EMPTY_CONTENT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType({**TEXT_RESPONSE, "content": []}) +EMPTY_TEXT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + {**TEXT_RESPONSE, "content": [{"type": "text", "text": ""}]} +) + +HOOKED_STREAM_AND_CACHE_CASES: Final = ( + Case( + "pre_request_hook_streams_runs_the_deployment_success_hook", + callbacks=(_UnstreamingHook,), + upstream=(_stream(TEXT_STREAM),), + loggers=_loggers(_DeploymentHooks), + ), + Case( + "pre_request_hook_streams_behind_a_cache", + callbacks=(_UnstreamingHook,), + upstream=(_stream(TEXT_STREAM), _stream(TEXT_STREAM)), + cache=Cache, + ), + Case( + "stream_one_runs_the_deployment_success_hook", + _kw(stream=1), + upstream=(_stream(TEXT_STREAM),), + loggers=_loggers(_DeploymentHooks), + ), + Case( + "stream_one_behind_a_cache", _kw(stream=1), upstream=(_stream(TEXT_STREAM), _stream(TEXT_STREAM)), cache=Cache + ), + Case( + "stream_one_sync_behind_a_cache", + _kw(stream=1), + upstream=(_stream(TEXT_STREAM), _stream(TEXT_STREAM)), + cache=Cache, + sync=True, + ), + Case( + "stream_string_true_behind_a_cache", + _kw(stream="true"), + upstream=(_stream(TEXT_STREAM), _stream(TEXT_STREAM)), + cache=Cache, + ), + Case( + "pre_request_hook_streams_on_an_integer_behind_a_cache", + callbacks=(_IntegerStreamHook,), + upstream=(_stream(TEXT_STREAM), _stream(TEXT_STREAM)), + cache=Cache, + ), + Case( + "cache_empty_content_is_not_stored", + upstream=(_ok(EMPTY_CONTENT_RESPONSE), _ok(EMPTY_CONTENT_RESPONSE)), + cache=Cache, + ), + Case( + "cache_empty_text_block_is_not_stored", + upstream=(_ok(EMPTY_TEXT_RESPONSE), _ok(EMPTY_TEXT_RESPONSE)), + cache=Cache, + ), + Case( + "mcp_cache_hit_stream", + _wiki(stream=True), + upstream=(_ok(WIKI_TOOL_USE_RESPONSE), _ok(), _ok(WIKI_TOOL_USE_RESPONSE), _ok()), + cache=Cache, + gateway=True, + ), + Case( + "safeguards_classifier_stream_cached", + _safeguarded(stream=True), + upstream=(_stream(TOOL_STREAM), _stream(TOOL_STREAM)), + model=NON_CLAUDE, + classifier=FLAGGED, + cache=_ordered_cache, + ), +) + +CASES: Final = ( + *REQUEST_CASES, + *INPUT_VALUE_CASES, + *ARGUMENT_TYPE_CASES, + *AGENTIC_CASES, + *PROMPT_CACHING_CASES, + *SYNC_PROMPT_CACHING_CASES, + *RESPONSE_CASES, + *FAILURE_CASES, + *STREAM_CASES, + *SYNC_CASES, + *SERVED_NATIVELY_CASES, + *THINKING_RETRY_CASES, + *STRICT_TOOLS_RETRY_CASES, + *FALLBACK_CASES, + *CACHE_CASES, + *THREAD_CASES, + *TRANSPORT_CASES, + *PRE_REQUEST_HOOK_CASES, + *SAFEGUARDS_CASES, + *INTERCEPTOR_CASES, + *SHORT_CIRCUIT_CASES, + *MCP_CASES, + *DEPLOYMENT_HOOK_CASES, + *HOOKED_STREAM_AND_CACHE_CASES, +) @dataclass(frozen=True, slots=True) @@ -780,6 +3354,11 @@ class Leg: hidden_params: Mapping[str, object] logged: Mapping[str, object] callback_kwargs: Mapping[str, object] + cached: tuple[str, ...] = () + warnings: tuple[str, ...] = () + hook_inputs: tuple[object, ...] = () + classified: tuple[Mapping[str, object], ...] = () + tool_runs: tuple[Mapping[str, object], ...] = () @dataclass(slots=True) @@ -806,7 +3385,6 @@ def native_messages() -> Iterator[FallbackProbe]: return _PYTHON_MESSAGES(*args, **kwargs) with ( - rebound(catalog, "RULES", (RouteRule(Route.MESSAGES, Rollout.RUST_OPT_OUT), *catalog.RULES)), rebound(dispatch, "_PYTHON_AMESSAGES", python_afallback), rebound(dispatch, "_PYTHON_MESSAGES", python_fallback), ): @@ -820,6 +3398,12 @@ def _environment(env: Mapping[str, str]) -> Generator[None]: *env, "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_FEDERATION_RULE_ID", + "ANTHROPIC_ORGANIZATION_ID", + "ANTHROPIC_IDENTITY_TOKEN", + "ANTHROPIC_IDENTITY_TOKEN_FILE", + "ANTHROPIC_IDENTITY_SOURCE", + "LITELLM_ANTHROPIC_WIF_ALLOWED_HOSTS", "LITELLM_REASONING_AUTO_SUMMARY", "LITELLM_RUST", ) @@ -843,6 +3427,48 @@ def _settings(settings: Mapping[str, object]) -> Generator[None]: setattr(litellm, name, value) +@contextmanager +def _classifier(reply: str | None, calls: list[Mapping[str, object]]) -> Generator[None]: + if reply is None: + yield + return + acompletion: Final = litellm.acompletion + + async def classify(**kwargs: object) -> object: # kwargs-ok: stands in for litellm.acompletion + calls.append( + { + "names": sorted(kwargs), + **{name: kwargs.get(name) for name in ("model", "messages", "max_tokens", "num_retries", "user")}, + "litellm_metadata": _mapping(kwargs.get("litellm_metadata")), + } + ) + return await acompletion(**kwargs, mock_response=reply) + + with ( + rebound(proxy_server, "general_settings", {"safeguards_classifier_model": CLASSIFIER_MODEL}), + rebound(litellm, "acompletion", classify), + ): + yield + + +@contextmanager +def _interceptor(interceptor: Callable[[], MessagesInterceptor] | None) -> Generator[None]: + if interceptor is None: + yield + return + with rebound(messages_interceptors, "_interceptors", [interceptor(), *messages_interceptors._interceptors]): # pyright: ignore[reportPrivateUsage] # registers a test interceptor ahead of the shipped ones + yield + + +@contextmanager +def _cache(cache: Callable[[], Cache] | None) -> Generator[None]: + if cache is None: + yield + return + with rebound(litellm, "cache", cache()): + yield + + def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]: # Deep: Python writes into the caller's own `litellm_metadata`, which must not leak into the # other leg. @@ -856,7 +3482,7 @@ def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]: "api_base": f"{server.base_url}{suffix}", **supplied, } - return {name: value for name, value in kwargs.items() if value is not None or name in ("safeguards",)} + return {name: value for name, value in kwargs.items() if value is not None or name in ("safeguards", *case.nones)} def _mapping(value: object) -> Mapping[str, object]: @@ -881,15 +3507,60 @@ def _error(error: BaseException, phase: str, chunks: Iterable[object] = ()) -> M "llm_provider": getattr(error, "llm_provider", None), "model": getattr(error, "model", None), "message": _normalized(str(error)), + "details": AnthropicExceptionMapping.embedded_error_details(str(error)), + "keeps_the_deployment_warm": is_message_thread_not_found(error), "headers": {name: header_items.get(name) for name in ("retry-after", "request-id") if name in header_items}, "partial": _stream_text(chunks), } +def _sent_request(response: object) -> Mapping[str, object] | None: + if not isinstance(response, httpx.Response): + return None + try: + sent: Final = response.request + except RuntimeError: + return {"status": response.status_code, "request": None} + return { + "status": response.status_code, + "request": { + "method": sent.method, + "path": sent.url.path, + "anthropic-beta": sent.headers.get("anthropic-beta"), + "body": _sent_body(sent), + }, + } + + +def _sent_body(sent: httpx.Request) -> object: + try: + sent_body: Final = sent.read() + except (httpx.RequestNotRead, httpx.StreamClosed): + return None + return json.loads(sent_body) if sent_body else None + + +def _partition(chunks: Iterable[object]) -> tuple[int, ...]: + return tuple(len(chunk if isinstance(chunk, bytes) else str(chunk).encode()) for chunk in chunks) + + def _stream_text(chunks: Iterable[object]) -> str: return b"".join(chunk if isinstance(chunk, bytes) else str(chunk).encode() for chunk in chunks).decode() +def _scrubbed(value: object) -> object: + """Masks the random server tool ids inside a response, leaving every other value as it was.""" + match value: + case str(): + return _SERVER_TOOL_ID.sub("srvtoolu_", value) + case dict(): + return {name: _scrubbed(item) for name, item in cast(dict[object, object], value).items()} + case list(): + return [_scrubbed(item) for item in cast(list[object], value)] + case _: + return value + + def _response(result: object, private_keys: bool) -> Mapping[str, object]: assert isinstance(result, dict), f"expected a dict response, got {type(result).__name__}" body: Final = { @@ -897,13 +3568,13 @@ def _response(result: object, private_keys: bool) -> Mapping[str, object]: for name, value in cast(dict[str, object], result).items() if private_keys or not name.startswith("_") } - return {"kind": "response", "body": _OBJECT.validate_python(body)} + return {"kind": "response", "body": _LOGGED.validate_python(_scrubbed(body))} def _hidden_params(result: object) -> Mapping[str, object]: hidden: Final = dict(_mapping(get_hidden_params_dict(result))) headers: Final = _mapping(hidden.pop("additional_headers", None)) - header_names: Final = sorted(name for name in headers if name not in ("x-litellm-rust", "x-litellm-cache-key")) + header_names: Final = sorted(name for name in headers if name != "x-litellm-rust") return {"keys": sorted(hidden), "additional_headers": header_names} @@ -949,32 +3620,42 @@ async def _invoke( if isinstance(result, dict): body = cast(Mapping[str, object], result) return _response(body, case.private_keys), _hidden_params(body) + time.sleep(case.busy_before_reading) # noqa: ASYNC251 # holds the event loop, as a caller's own work between the call and its first read does chunks, failure = await _consume_async(cast(AsyncIterator[object], result)) hidden: Final = _hidden_params(result) + partition: Final = {"partition": _partition(chunks)} if case.partitioned else {} if failure is not None: - return _error(failure, "stream", chunks), hidden + return {**_error(failure, "stream", chunks), **partition}, hidden return { "kind": "stream", "chunk_types": sorted({type(chunk).__name__ for chunk in chunks}), - "text": _stream_text(chunks), + "text": _SERVER_TOOL_ID.sub("srvtoolu_", _stream_text(chunks)), + **partition, }, hidden -async def _terminal_events(recorder: RecordingLogger) -> tuple[str, ...]: +async def _terminal_events(recorder: RecordingLogger, since: int = 0) -> tuple[str, ...]: for _ in range(100): await drain_logging() - if any(name in _TERMINAL_EVENTS for name in recorder.names): + if any(name in _TERMINAL_EVENTS for name in recorder.names[since:]): break await asyncio.sleep(0.05) await asyncio.sleep(0.2) await drain_logging() - return tuple(name for name in recorder.names if name in _TERMINAL_EVENTS) + return tuple(name for name in recorder.names[since:] if name in _TERMINAL_EVENTS) -def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping[str, object], Mapping[str, object]]: - terminal: Final = [event for event in recorder.events if event.name in _TERMINAL_EVENTS] +def _logged( + recorder: RecordingLogger, events: tuple[str, ...], since: int = 0 +) -> tuple[Mapping[str, object], Mapping[str, object]]: + terminal: Final = [ + event + for event in recorder.events[since:] + if event.name in _TERMINAL_EVENTS and event.call_type != CLASSIFIER_CALL_TYPE + ] + hooks: Final = tuple(sorted(set(recorder.names[since:]))) if not terminal: - return {"events": events}, {} + return {"events": events, "hooks": hooks}, {} details: Final = _mapping(terminal[-1].kwargs) standard: Final = _mapping(details.get("standard_logging_object")) error_class: Final = _mapping(standard.get("error_information")).get("error_class") @@ -983,35 +3664,139 @@ def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping metadata_keys: Final = _mapping(metadata).keys() logged: Final[Mapping[str, object]] = { "events": events, + "hooks": hooks, **{name: standard.get(name) for name in _COMPARED_LOG_FIELDS}, "error_class": error_class, } callback_kwargs: Final[Mapping[str, object]] = { "logged_metadata": sorted(metadata_keys) if isinstance(metadata, dict) else metadata, "logged_model_info": litellm_params.get("model_info"), - "logged_response_headers": bool(details.get("response_headers")), + "logged_response_headers": { + name: value for name, value in _mapping(details.get("response_headers")).items() if name.lower() != "date" + }, "logged_api_key": details.get("api_key"), "logged_call": { "custom_llm_provider": details.get("custom_llm_provider"), "model": details.get("model"), "stream": details.get("stream"), + "aanthropic_messages": litellm_params.get("aanthropic_messages"), }, + "logged_request": { + "messages": details.get("messages"), + "thinking": details.get("thinking"), + "tools": details.get("tools"), + "complete_input_dict": _mapping(details.get("additional_args")).get("complete_input_dict"), + }, + "logged_sent_request": _sent_request(details.get("httpx_response")), + "logged_model_parameters": standard.get("model_parameters"), + "logged_error": _logged_error(details, standard), } return logged, callback_kwargs +def _logged_error(details: Mapping[str, object], standard: Mapping[str, object]) -> Mapping[str, object]: + exception: Final = details.get("exception") + information: Final = _mapping(standard.get("error_information")) + return { + "error_str": _normalized(str(standard.get("error_str"))), + "error_message": _normalized(str(information.get("error_message"))), + "error_code": information.get("error_code"), + "llm_provider": information.get("llm_provider"), + "exception": None if exception is None else _normalized(str(exception)), + "exception_model": getattr(exception, "model", None), + "exception_provider": getattr(exception, "llm_provider", None), + } + + +async def _prime(case: Case, server: RecordingServer, recorder: RecordingLogger, native: bool) -> int: + if case.cache is None: + return 0 + outcome: Final = await _invoke(case, _kwargs(case, server), native=native) + assert case.sync or outcome[0]["kind"] != "error", outcome + await _terminal_events(recorder) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + return len(recorder.events) + + +WIKI_MCP_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "name": "read_wiki", + "description": "Read a repository wiki", + "inputSchema": {"type": "object", "properties": {"repo": {"type": "string"}}, "required": ["repo"]}, + } +) + + +@contextmanager +def _gateway(case: Case, runs: list[Mapping[str, object]]) -> Generator[None]: # mutable-ok: records the tool runs + if not case.gateway: + yield + return + from mcp.types import Tool as MCPTool + + from litellm.responses.mcp import tool_precheck + from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler + + split_mcp_tools: Final = LiteLLM_Proxy_MCP_Handler.split_mcp_tools + + async def served(names: Iterable[str]) -> frozenset[str]: + return frozenset({"deepwiki"}) & frozenset(names) + + async def split(tools: Iterable[Mapping[str, object]] | None) -> object: + return await split_mcp_tools(tools, served) + + async def listed(**_: object) -> tuple[list[MCPTool], list[str]]: + return [MCPTool.model_validate(dict(WIKI_MCP_TOOL))], ["deepwiki"] + + async def executed(**kwargs: object) -> list[dict[str, object]]: + calls: Final = cast(list[Mapping[str, object]], kwargs["tool_calls"]) + runs.append( + { + "tool_calls": [dict(call) for call in calls], + "tool_server_map": kwargs["tool_server_map"], + "request_tags": kwargs["request_tags"], + "guardrail_context": json.loads(json.dumps(kwargs["guardrail_context"], default=repr)), + } + ) + if any(call.get("input") == {"repo": "boom"} for call in calls): + raise RuntimeError("the wiki server went away") + return [{"tool_call_id": call.get("id"), "result": "9 sections", "name": call.get("name")} for call in calls] + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(tool_precheck, "_gateway_registry_loads", lambda: True) + patch.setattr(LiteLLM_Proxy_MCP_Handler, "split_mcp_tools", staticmethod(split)) + patch.setattr(LiteLLM_Proxy_MCP_Handler, "_get_mcp_tools_from_manager", staticmethod(listed)) + patch.setattr(LiteLLM_Proxy_MCP_Handler, "execute_tool_calls", staticmethod(executed)) + yield + + async def _leg(case: Case, server: RecordingServer, probe: FallbackProbe | None) -> Leg: - with isolated_callback_registries(), _environment(case.env), _settings(case.settings): - recorder: Final = RecordingLogger() - litellm.callbacks.append(recorder) + classified: Final[list[Mapping[str, object]]] = [] + runs: Final[list[Mapping[str, object]]] = [] + with ( + isolated_callback_registries(), + _environment(case.env), + _settings(case.settings), + _cache(case.cache), + _classifier(case.classifier, classified), + _interceptor(case.interceptor), + _gateway(case, runs), + ): for spec in case.upstream: server.enqueue(replace(spec)) request_count: Final = len(server.requests) + recorder: Final = RecordingLogger() + loggers: Final = () if case.loggers is None else case.loggers() + litellm.callbacks.extend(loggers) + litellm.callbacks.append(recorder) + litellm.callbacks.extend(callback() for callback in case.callbacks) fallbacks_before: Final = 0 if probe is None else probe.calls - outcome, hidden = await _invoke(case, _kwargs(case, server), native=probe is not None) - events: Final = await _terminal_events(recorder) + primed: Final = await _prime(case, server, recorder, native=probe is not None) + with _warnings() as warned: + outcome, hidden = await _invoke(case, _kwargs(case, server), native=probe is not None) + events: Final = await _terminal_events(recorder, primed) server.responses.clear() - logged, callback_kwargs = _logged(recorder, events) + logged, callback_kwargs = _logged(recorder, events, primed) return Leg( fell_back=probe is not None and probe.calls > fallbacks_before, requests=tuple( @@ -1029,9 +3814,40 @@ async def _leg(case: Case, server: RecordingServer, probe: FallbackProbe | None) hidden_params=hidden, logged=logged, callback_kwargs=callback_kwargs, + cached=() if litellm.cache is None else _cache_entries(litellm.cache), + warnings=tuple(warned), + hook_inputs=tuple(getattr(logger, "seen", ()) for logger in loggers), + classified=tuple(classified), + tool_runs=tuple(runs), ) +class _WarningLines(logging.Handler): + def __init__(self, lines: list[str]) -> None: # mutable-ok: the handler appends each warning as it is logged + super().__init__(logging.WARNING) + self.lines: Final = lines + + def emit(self, record: logging.LogRecord) -> None: + self.lines.append(record.getMessage()) + + +@contextmanager +def _warnings() -> Generator[list[str]]: # mutable-ok: filled while the call runs + lines: Final[list[str]] = [] + handler: Final = _WarningLines(lines) + verbose_logger.addHandler(handler) + try: + yield lines + finally: + verbose_logger.removeHandler(handler) + + +def _cache_entries(cache: Cache) -> tuple[str, ...]: + backend: Final = cache.cache + keys: Final = tuple(sorted(backend.cache_dict)) if isinstance(backend, InMemoryCache) else () + return (*keys, *backend.ops) if isinstance(backend, _OrderedMemory) else keys + + @pytest.fixture def parity_server(recording_server: RecordingServer) -> RecordingServer: recording_server.expected_requests = None @@ -1068,12 +3884,521 @@ async def test_native_messages_match_python( case "dispatch": assert not native.fell_back case "request": - assert native.requests == python.requests + assert _comparable(native.requests) == _comparable(python.requests) case "outcome": assert native.outcome == python.outcome case "hidden_params": assert native.hidden_params == python.hidden_params case "logged": assert native.logged == python.logged + case "cached": + assert native.cached == python.cached + case "warnings": + assert native.warnings == python.warnings + case "hook_inputs": + assert native.hook_inputs == python.hook_inputs + case "classified": + assert native.classified == python.classified + case "tool_runs": + assert native.tool_runs == python.tool_runs case _: - assert native.callback_kwargs.get(aspect) == python.callback_kwargs.get(aspect) + assert _comparable(native.callback_kwargs.get(aspect)) == _comparable(python.callback_kwargs.get(aspect)) + + +def _comparable(value: object) -> object: + match value: + case float() if math.isnan(value): + return "" + case Mapping(): + return {name: _comparable(item) for name, item in cast(Mapping[object, object], value).items()} + case list() | tuple(): + return [_comparable(item) for item in cast(Sequence[object], value)] + case _: + return value + + +@pytest.mark.parametrize( + "case", + [ + pytest.param(case, id=case.name) + for case in (*SERVED_NATIVELY_CASES, *AGENTIC_CASES, *SAFEGUARDS_CASES, *SHORT_CIRCUIT_CASES) + ], +) +def test_catalog_serves_the_case_natively(case: Case, parity_server: RecordingServer) -> None: + with ( + isolated_callback_registries(), + _environment(case.env), + _settings(case.settings), + _classifier(case.classifier, []), + ): + litellm.callbacks.extend(() if case.loggers is None else case.loggers()) + binding: Final = dispatch._MESSAGES if case.sync else dispatch._AMESSAGES # pyright: ignore[reportPrivateUsage] # the public binding the entry routes by + features: Final = dispatch._FEATURES # pyright: ignore[reportPrivateUsage] # the entry's own detectors + request: Final = dispatch._public_request(binding, (), _kwargs(case, parity_server)) # pyright: ignore[reportPrivateUsage] # the public binding the entry routes by + assert request is not None + context: Final = dispatch._context(request, features) # pyright: ignore[reportPrivateUsage] # the entry's own route context + assert catalog.rollout(context) is Rollout.RUST_REQUIRED, context.features + + +@pytest.mark.parametrize("case", [pytest.param(case, id=case.name) for case in TRANSPORT_CASES]) +def test_catalog_serves_the_transport_case_natively(case: Case, parity_server: RecordingServer) -> None: + with isolated_callback_registries(), _environment(case.env), _settings(case.settings): + request: Final = dispatch._public_request(dispatch._AMESSAGES, (), _kwargs(case, parity_server)) # pyright: ignore[reportPrivateUsage] # the public binding the async entry routes by + assert request is not None + context: Final = dispatch._context(request, dispatch._FEATURES) # pyright: ignore[reportPrivateUsage] # the async entry's own route context + assert catalog.rollout(context) is Rollout.RUST_REQUIRED, context.features + + +@dataclass(slots=True) +class _Exchanges: + posts: int = 0 + windows: int = 0 + + +async def _exchanges( + case: Case, server: RecordingServer, probe: FallbackProbe | None, monkeypatch: pytest.MonkeyPatch +) -> _Exchanges: + counted: Final = _Exchanges() + post: Final = AsyncHTTPHandler.post # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the handler's untyped dict params + record: Final = logging_utils._set_duration_in_model_call_details # pyright: ignore[reportPrivateUsage] # the one recorder of an exchange window + + async def posted( + handler: AsyncHTTPHandler, *args: object, **kwargs: object + ) -> object: # kwargs-ok: the handler's own post signature + counted.posts += 1 + return await post(handler, *args, **kwargs) # pyright: ignore[reportArgumentType] # forwarded unchanged + + def windowed(**kwargs: object) -> None: # kwargs-ok: the recorder's own keywords + counted.windows += 1 + record(**kwargs) # pyright: ignore[reportArgumentType] # forwarded unchanged + + with monkeypatch.context() as patch: + patch.setattr(AsyncHTTPHandler, "post", posted) + patch.setattr(logging_utils, "_set_duration_in_model_call_details", windowed) + await _leg(case, server, probe) + return counted + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", [pytest.param(case, id=case.name) for case in TRANSPORT_CASES if not case.sync]) +async def test_transport_cases_post_through_the_handler_python_posts_through( + case: Case, parity_server: RecordingServer, native_messages: FallbackProbe, monkeypatch: pytest.MonkeyPatch +) -> None: + python: Final = await _exchanges(case, parity_server, None, monkeypatch) + native: Final = await _exchanges(case, parity_server, native_messages, monkeypatch) + ignored: Final = case.name.endswith("_is_ignored") + assert python.posts == len(case.upstream) + assert native.posts == (0 if ignored else python.posts) + assert native.windows == python.windows + assert native_messages.calls == 0 + + +_COST_CASES: Final = tuple( + pytest.param(case, id=case.name) for case in SERVED_NATIVELY_CASES if case.name.startswith("stream_cost_injection") +) + + +def _proxy_relayed(text: str, request_headers: Mapping[str, object]) -> str: + return "".join( + str( + ProxyBaseLLMRequestProcessing.process_chunk_with_cost_injection( + frame + "\n\n", "proxy-model-alias", None, request_headers=request_headers + ) + ) + for frame in text.split("\n\n") + if frame + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", _COST_CASES) +async def test_both_lanes_carry_one_cost_per_usage_through_the_proxy_relay( + case: Case, parity_server: RecordingServer, native_messages: FallbackProbe +) -> None: + python, native = await _legs(case, parity_server, native_messages) + opted_in: Final = {"x-litellm-include-cost-in-streaming-usage": "true"} + python_text: Final = str(python.outcome["text"]) + native_text: Final = str(native.outcome["text"]) + assert '"cost"' in python_text + assert native_text == python_text + relayed: Final = _proxy_relayed(native_text, opted_in) + assert relayed == _proxy_relayed(python_text, opted_in) + assert all(frame.count('"cost"') <= 1 for frame in relayed.split("\n\n")) + + +def _proxied(kwargs: Mapping[str, object]) -> dict[str, object]: + logging_obj, data = litellm.utils.function_setup( + original_function="anthropic_messages", + rules_obj=litellm.utils.Rules(), + start_time=datetime.datetime.now(), + is_async_call=True, + litellm_call_id=str(uuid.uuid4()), + **kwargs, + ) + return {**data, "litellm_logging_obj": logging_obj} + + +async def _through_the_ownership_hook( + hook: MessageThreadOwnership, owner: UserAPIKeyAuth, data: dict[str, object], native: bool +) -> None: + acall: Final = _PUBLIC_AMESSAGES if native else _PYTHON_AMESSAGES + result: Final = await acall(**data) + if isinstance(result, dict): + await hook.async_post_call_success_hook(data, owner, cast(AnthropicMessagesResponse, result)) + return + async for _ in hook.async_post_call_streaming_iterator_hook(owner, cast(AsyncIterator[object], result), data): + pass + + +async def _thread_betas(server: RecordingServer, stream: bool, native: bool) -> tuple[str | None, str | None]: + hook: Final = MessageThreadOwnership(InternalUsageCache(dual_cache=DualCache())) + owner: Final = UserAPIKeyAuth(token="hashed-owner") + base: Final[dict[str, object]] = { + "model": SONNET, + "max_tokens": 64, + "messages": [dict(USER)], + "api_key": API_KEY, + "api_base": server.base_url, + } + server.enqueue(_stream(TEXT_STREAM) if stream else _ok()) + server.enqueue(_ok()) + sent: Final = len(server.requests) + with isolated_callback_registries(), _environment({}): + await _through_the_ownership_hook( + hook, + owner, + _proxied({**base, "thread": dict(THREAD_CREATE), "tools": [dict(ADVISOR_TOOL)], "stream": stream}), + native, + ) + replied: Final = "msg_01ParityStream" if stream else str(TEXT_RESPONSE["id"]) + continued: Final = _proxied({**base, "thread": {"type": "continue", "previous_message_id": replied}}) + hooked: Final = await hook.async_pre_call_hook(owner, DualCache(), continued, "anthropic_messages") + await _through_the_ownership_hook(hook, owner, continued if hooked is None else hooked, native) + await drain_logging() + server.responses.clear() + create, resumed = server.requests[sent:] + return create.headers.get("anthropic-beta"), resumed.headers.get("anthropic-beta") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", (False, True), ids=("json", "streamed")) +async def test_both_lanes_resend_a_thread_s_create_betas_through_the_ownership_hook( + stream: bool, parity_server: RecordingServer, native_messages: FallbackProbe +) -> None: + python: Final = await _thread_betas(parity_server, stream, native=False) + native: Final = await _thread_betas(parity_server, stream, native=True) + + assert native_messages.calls == 0 + assert native == python + assert python == ("advisor-tool-2026-03-01,message-threads-2026-08-12",) * 2 + + +_WEB_SEARCH_PRE_HOOK_CASES: Final = tuple( + pytest.param(case, id=case.name) + for case in AGENTIC_CASES + if case.name in {"web_search_interception", "web_search_interception_converted_stream"} +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", _WEB_SEARCH_PRE_HOOK_CASES) +async def test_both_lanes_hand_the_web_search_pre_hook_kwargs_to_the_agentic_loop( + case: Case, parity_server: RecordingServer, native_messages: FallbackProbe +) -> None: + python, native = await _legs(case, parity_server, native_messages) + + assert native_messages.calls == 0 + assert native.hook_inputs == python.hook_inputs + [seen] = python.hook_inputs + first: Final = cast(list[Mapping[str, object]], seen)[0] + converted: Final = case.name.endswith("_converted_stream") + assert first["kwargs"] == { + "max_agentic_loops": 2, + "_websearch_interception_emit_native_blocks": True, + **({"_websearch_interception_converted_stream": True} if converted else {}), + } + assert [tool.get("name") for tool in cast(list[Mapping[str, object]], first["tools"])] == ["litellm_web_search"] + assert first["stream"] is False + + +class _CountingAnswering(_Answering): + def __init__(self) -> None: + self.asked = 0 + + def can_handle(self, tools: Sequence[Mapping[str, object]] | None, custom_llm_provider: str | None) -> bool: + self.asked += 1 + return super().can_handle(tools, custom_llm_provider) + + +async def _interceptor_asks(server: RecordingServer, hooked: bool, native: bool) -> tuple[int, tuple[object, ...]]: + counting: Final = _CountingAnswering() + acall: Final = _PUBLIC_AMESSAGES if native else _PYTHON_AMESSAGES + kwargs: Final[dict[str, object]] = { + "model": SONNET, + "max_tokens": 64, + "messages": [dict(USER)], + "api_key": API_KEY, + "api_base": server.base_url, + "tools": [dict(WEATHER_TOOL)] if hooked else [dict(INTERCEPTED_TOOL), dict(WEATHER_TOOL)], + } + answers: Final[list[object]] = [] + with ( + isolated_callback_registries(), + _environment({}), + _cache(Cache), + _interceptor(lambda: counting), + ): + if hooked: + litellm.callbacks.append(_InterceptedToolHook()) + for _ in range(2): + answers.append(_response(await acall(**copy.deepcopy(kwargs)), private_keys=False)) + await drain_logging() + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + return counting.asked, tuple(answers) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("hooked", (False, True), ids=("plain", "pre_request_hooked")) +async def test_both_lanes_ask_the_interceptor_only_on_a_cache_miss( + hooked: bool, parity_server: RecordingServer, native_messages: FallbackProbe +) -> None: + python: Final = await _interceptor_asks(parity_server, hooked, native=False) + native: Final = await _interceptor_asks(parity_server, hooked, native=True) + + assert native_messages.calls == 0 + assert not parity_server.requests + assert native == python + assert python[0] == 1 + + +class _ServiceRecorder: + def __init__(self) -> None: + self.events: list[Mapping[str, object]] = [] # mutable-ok: appended per service hook + + async def async_service_success_hook(self, **kwargs: object) -> None: # kwargs-ok: the service hook's keyword call + self.events.append( + { + "service": kwargs.get("service"), + "call_type": kwargs.get("call_type"), + "timed": isinstance(kwargs.get("duration"), float), + "parent_otel_span": kwargs.get("parent_otel_span"), + } + ) + + +async def _llm_api_timings(server: RecordingServer, stream: bool, native: bool) -> tuple[Mapping[str, object], ...]: + recorder: Final = _ServiceRecorder() + server.enqueue(_stream(TEXT_STREAM) if stream else _ok()) + acall: Final = _PUBLIC_AMESSAGES if native else _PYTHON_AMESSAGES + with isolated_callback_registries(), _environment({}), rebound(logging_utils, "_service_logger", recorder): + result: Final = await acall( + model=SONNET, + max_tokens=64, + messages=[dict(USER)], + api_key=API_KEY, + api_base=server.base_url, + stream=stream, + ) + if not isinstance(result, dict): + await _consume_async(cast(AsyncIterator[object], result)) + await drain_logging() + await asyncio.sleep(0) + server.responses.clear() + return tuple(recorder.events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", (False, True), ids=("json", "streamed")) +async def test_both_lanes_time_the_provider_exchange_as_one_llm_service_event( + stream: bool, parity_server: RecordingServer, native_messages: FallbackProbe +) -> None: + python: Final = await _llm_api_timings(parity_server, stream, native=False) + native: Final = await _llm_api_timings(parity_server, stream, native=True) + + assert native_messages.calls == 0 + assert native == python + assert [event["call_type"] for event in python] == ["post <- track_llm_api_timing"] + + +class _AttemptRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() # pyright: ignore[reportUnknownMemberType] # CustomLogger takes untyped kwargs + self.seen: list[Mapping[str, object]] = [] + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + logged: Final = _LOGGED.validate_python(kwargs.get("standard_logging_object") or {}) + self.seen.append( + { + "event": "success", + "model_id": logged.get("model_id"), + "completion_tokens": logged.get("completion_tokens"), + } + ) + + async def async_log_failure_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # test double narrows the hook's types + logged: Final = _LOGGED.validate_python(kwargs.get("standard_logging_object") or {}) + self.seen.append( + { + "event": "failure", + "model_id": logged.get("model_id"), + "exception": type(kwargs.get("exception")).__name__, + } + ) + + +async def _stream_fallback_attempts( + server: RecordingServer, native: bool +) -> tuple[tuple[Mapping[str, object], ...], str]: + overloaded: Final = _events( + _message_start("msg_01ParityOverloaded"), ("error", dict(_upstream_error("overloaded_error", "Overloaded"))) + ) + server.enqueue(_stream(overloaded)) + server.enqueue(_stream(TEXT_STREAM)) + recorder: Final = _AttemptRecorder() + deployment: Final[Mapping[str, object]] = {"model": SONNET, "api_base": server.base_url, "api_key": API_KEY} + with contextlib.ExitStack() as stack: + stack.enter_context(isolated_callback_registries()) + stack.enter_context(_environment({})) + if not native: + stack.enter_context(rebound(litellm, "anthropic_messages", _PYTHON_AMESSAGES)) + litellm.callbacks.append(recorder) + router: Final = litellm.Router( + model_list=[ + {"model_name": "primary", "litellm_params": dict(deployment), "model_info": {"id": "dep-primary"}}, + {"model_name": "fallback", "litellm_params": dict(deployment), "model_info": {"id": "dep-fallback"}}, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + result: Final = await router.aanthropic_messages( + model="primary", max_tokens=64, messages=[dict(USER)], stream=True + ) + chunks, failure = await _consume_async(cast(AsyncIterator[object], result)) + assert failure is None + for _ in range(100): + await drain_logging() + if len(recorder.seen) >= 2: + break + await asyncio.sleep(0.05) + server.responses.clear() + return tuple(recorder.seen), _stream_text(chunks) + + +@pytest.mark.asyncio +async def test_both_lanes_log_a_stream_superseded_by_a_router_fallback_as_its_frame_s_failure( + parity_server: RecordingServer, native_messages: FallbackProbe +) -> None: + python: Final = await _stream_fallback_attempts(parity_server, native=False) + native: Final = await _stream_fallback_attempts(parity_server, native=True) + + assert native_messages.calls == 0 + assert native == python + assert sorted(str(event["event"]) for event in python[0]) == ["failure", "success"] + + +async def _router_stream( + server: RecordingServer, native: bool, upstream: tuple[ResponseSpec, ...] +) -> tuple[tuple[str, ...], tuple[Mapping[str, object], ...]]: + for response in upstream: + server.enqueue(response) + recorder: Final = _AttemptRecorder() + deployment: Final[Mapping[str, object]] = {"model": SONNET, "api_base": server.base_url, "api_key": API_KEY} + with contextlib.ExitStack() as stack: + stack.enter_context(isolated_callback_registries()) + stack.enter_context(_environment({})) + if not native: + stack.enter_context(rebound(litellm, "anthropic_messages", _PYTHON_AMESSAGES)) + litellm.callbacks.append(recorder) + router: Final = litellm.Router( + model_list=[ + {"model_name": "primary", "litellm_params": dict(deployment), "model_info": {"id": "dep-primary"}}, + {"model_name": "fallback", "litellm_params": dict(deployment), "model_info": {"id": "dep-fallback"}}, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + result: Final = await router.aanthropic_messages( + model="primary", max_tokens=64, messages=[dict(USER)], stream=True + ) + chunks, failure = await _consume_async(cast(AsyncIterator[object], result)) + assert failure is None + for _ in range(100): + await drain_logging() + if recorder.seen: + break + await asyncio.sleep(0.05) + server.responses.clear() + return tuple(_stream_text((chunk,)) for chunk in chunks), tuple(recorder.seen) + + +OVERLOADED_AFTER_PINGS: Final = _events( + _message_start("msg_01ParityPingedOverloaded"), + *((_PING,) * 50), + ("error", dict(_upstream_error("overloaded_error", "Overloaded"))), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "upstream", + [ + pytest.param((_burst(PINGED_STREAM),), id="pings_after_message_start"), + pytest.param((_burst(HEAVILY_PINGED_STREAM),), id="a_thousand_pings_after_message_start"), + pytest.param((_burst(OVERLOADED_AFTER_PINGS), _burst(TEXT_STREAM)), id="pings_then_overloaded_then_fallback"), + ], +) +async def test_both_lanes_stream_a_router_burst_in_the_same_chunks( + parity_server: RecordingServer, native_messages: FallbackProbe, upstream: tuple[ResponseSpec, ...] +) -> None: + python: Final = await _router_stream(parity_server, native=False, upstream=upstream) + native: Final = await _router_stream(parity_server, native=True, upstream=upstream) + + assert native_messages.calls == 0 + assert native == python + + +async def _abandoned_stream( + server: RecordingServer, native: bool, upstream: ResponseSpec +) -> tuple[tuple[str, ...], tuple[Mapping[str, object], ...]]: + server.enqueue(upstream) + recorder: Final = _AttemptRecorder() + acall: Final = _PUBLIC_AMESSAGES if native else _PYTHON_AMESSAGES + with contextlib.ExitStack() as stack: + stack.enter_context(isolated_callback_registries()) + stack.enter_context(_environment({})) + litellm.callbacks.append(recorder) + stream: Final = cast( + AsyncGenerator[object], + await acall( + model=SONNET, + max_tokens=64, + messages=[dict(USER)], + api_key=API_KEY, + api_base=server.base_url, + stream=True, + ), + ) + first: Final = await anext(stream) + await stream.aclose() + for _ in range(100): + await drain_logging() + if recorder.seen: + break + await asyncio.sleep(0.05) + server.responses.clear() + return (_stream_text((first,)),), tuple(recorder.seen) + + +@pytest.mark.asyncio +async def test_both_lanes_bill_a_stream_abandoned_after_its_first_chunk_alike( + parity_server: RecordingServer, native_messages: FallbackProbe +) -> None: + python: Final = await _abandoned_stream(parity_server, native=False, upstream=_burst(TEXT_STREAM)) + native: Final = await _abandoned_stream(parity_server, native=True, upstream=_burst(TEXT_STREAM)) + + assert native_messages.calls == 0 + assert native == python diff --git a/tests/test_litellm_rust/cache/test_python_cache.py b/tests/test_litellm_rust/cache/test_python_cache.py index d88feec7d5e..4bf0b389356 100644 --- a/tests/test_litellm_rust/cache/test_python_cache.py +++ b/tests/test_litellm_rust/cache/test_python_cache.py @@ -6,6 +6,7 @@ from pydantic import TypeAdapter import litellm from litellm.caching.caching import Cache, CacheMode +from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.caching_handler import ( _PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # await the existing background cache writer before the next request ) @@ -33,6 +34,14 @@ from tests.test_litellm_rust.support.routing import native_route pytestmark = pytest.mark.requires_rust_extension +_STORED: Final = TypeAdapter(dict[str, object]) + + +def _entries(storage: InMemoryCache) -> dict[str, object]: + return _STORED.validate_python( + storage.cache_dict # pyright: ignore[reportUnknownMemberType] # InMemoryCache leaves its dict untyped + ) + @pytest.mark.asyncio @pytest.mark.parametrize( @@ -110,7 +119,9 @@ async def test_legacy_cache_controls_and_backend_credential_key_semantics(record recording_server.expected_requests = 3 litellm.cache = Cache() await invoke("messages", recording_server, {"cache": {"no-store": True}}) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) await invoke("messages", recording_server, {}) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) await invoke("messages", recording_server, {}) assert len(recording_server.requests) == 2 await invoke("messages", recording_server, {"cache": {"no-cache": True}}) @@ -137,9 +148,11 @@ async def test_legacy_messages_replays_a_completed_stream(recording_server: Reco assert cache_key(first_stream) is None first: Final = await collect(first_stream) await recorder.wait_for_async("async_log_success_event") + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) second_stream: Final = await litellm.anthropic_messages(**parameters) - assert isinstance(cache_key(second_stream), str) - assert cache_key(second_stream) == get_hidden_params_dict(second_stream)["cache_key"] + assert cache_key(second_stream) is None + assert get_hidden_params_dict(second_stream)["cache_hit"] is True + assert isinstance(get_hidden_params_dict(second_stream)["cache_key"], str) second: Final = await collect(second_stream) assert payload(first) == payload(second) assert first == b"".join(recording_server.default_response.payloads()) @@ -174,13 +187,18 @@ async def test_rust_messages_uses_a_legacy_cache_without_python_inference( "api_base": recording_server.base_url, } first: Final = await invoke("messages", recording_server, parameters) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) second: Final = await invoke("messages", recording_server, parameters) - assert cache_key(second) - assert cache_key(first) is None assert payload(first) == payload(second) assert len(recording_server.requests) == 1 + assert cache_key(first) is None + assert cache_key(second) is None - await logger.wait_for_async("async_log_success_event", count=2) + successes: Final = await logger.wait_for_async("async_log_success_event", count=2) + assert [TypeAdapter(dict[str, object]).validate_python(event.kwargs).get("cache_hit") for event in successes] == [ + None, + True, + ] assert logger.names.count("async_log_success_event") == 2 assert "log_failure_event" not in logger.names assert "async_log_failure_event" not in logger.names @@ -210,7 +228,9 @@ async def test_legacy_cache_default_off_requires_opt_in_even_for_existing_entrie @pytest.mark.asyncio -async def test_legacy_cache_lookup_uses_backend_request_callback_semantics(recording_server: RecordingServer) -> None: +async def test_a_legacy_cache_hit_skips_the_pre_call_callback_that_would_rewrite_the_request( + recording_server: RecordingServer, +) -> None: class Rewrite(RecordingLogger): temperature = 0.1 @@ -222,15 +242,16 @@ async def test_legacy_cache_lookup_uses_backend_request_callback_semantics(recor litellm.cache = Cache() recording_server.expected_requests = 1 await invoke("messages", recording_server, {"callbacks": [logger]}) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) first_hit: Final = await invoke("messages", recording_server, {"callbacks": [logger]}) logger.temperature = 0.8 await invoke("messages", recording_server, {"callbacks": [logger]}) second_hit: Final = await invoke("messages", recording_server, {"callbacks": [logger]}) - assert logger.names.count("log_pre_api_call") == 4 + assert logger.names.count("log_pre_api_call") == 1 + assert cache_key(first_hit) is None + assert cache_key(second_hit) is None assert len(recording_server.requests) == recording_server.expected_requests assert recording_server.requests[0].body["temperature"] == 0.1 - assert isinstance(cache_key(first_hit), str) - assert isinstance(cache_key(second_hit), str) @pytest.mark.asyncio @@ -258,7 +279,7 @@ async def test_rust_responses_rejects_a_configured_python_cache( @pytest.mark.asyncio @pytest.mark.parametrize("cancel_lookup", (False, True)) @pytest.mark.parametrize("stream", (False, True), ids=("response", "stream")) -async def test_python_cache_operations_stay_in_the_rust_callers_task_except_the_unary_store( +async def test_python_cache_lookups_stay_in_the_rust_callers_task_and_stores_run_detached( recording_server: RecordingServer, cancel_lookup: bool, stream: bool, @@ -287,11 +308,8 @@ async def test_python_cache_operations_stay_in_the_rust_callers_task_except_the_ async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None: writer: Final = asyncio.current_task() - if stream: - assert writer is caller - else: - assert writer is not caller - assert writer in _PENDING_CACHE_WRITES + assert writer is not caller + assert writer in _PENDING_CACHE_WRITES await asyncio.sleep(0) storage.set_cache(key, value, **kwargs) @@ -314,7 +332,10 @@ async def test_python_cache_operations_stay_in_the_rust_callers_task_except_the_ second: Final = await invoke("messages", recording_server, {"stream": stream}) second_payload: Final = await collect(second) if stream else payload(second) assert first_payload == second_payload - assert cache_key(second) + assert cache_key(first) is None + assert cache_key(second) is None + if stream: + assert get_hidden_params_dict(second)["cache_key"] in _entries(storage) assert len(recording_server.requests) == 1 @@ -343,12 +364,12 @@ async def test_rust_unary_messages_store_keeps_the_hidden_params_stamped_after_t "model_id": "deployment", } assert get_hidden_params_dict(first)["additional_headers"] == { - "x-litellm-rust": "true", "x-litellm-model-group": "group", "x-litellm-attempted-retries": 0, } hit: Final = await invoke("messages", recording_server, {}) - assert get_hidden_params_dict(hit)["cache_key"] == cache_key(hit) + assert isinstance(get_hidden_params_dict(hit)["cache_key"], str) + assert cache_key(hit) is None assert len(recording_server.requests) == 1 @@ -389,6 +410,8 @@ async def test_messages_cache_stores_the_response_a_yielding_deployment_hook_ret async def test_rust_messages_legacy_cache_honors_request_namespaces( recording_server: RecordingServer, namespace_source: str ) -> None: + from litellm.caching.in_memory_cache import InMemoryCache + litellm.cache = Cache() recording_server.expected_requests = 2 first_options: Final = ( @@ -399,13 +422,20 @@ async def test_rust_messages_legacy_cache_honors_request_namespaces( if namespace_source == "cache" else {"metadata": {"redis_namespace": "second"}} ) + storage: Final = litellm.cache.cache + assert isinstance(storage, InMemoryCache) await invoke("messages", recording_server, first_options) - second: Final = await invoke("messages", recording_server, second_options) - first_hit: Final = await invoke("messages", recording_server, first_options) - second_hit: Final = await invoke("messages", recording_server, second_options) - assert cache_key(second) is None - assert cache_key(first_hit) is not None - assert cache_key(second_hit) is not None + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + first_keys: Final = set(_entries(storage)) + await invoke("messages", recording_server, second_options) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + second_keys: Final = set(_entries(storage)) - first_keys + assert len(recording_server.requests) == 2 + assert [key.split(":", 1)[0] for key in first_keys] == ["first"] + assert [key.split(":", 1)[0] for key in second_keys] == ["second"] + assert {key.split(":", 1)[1] for key in first_keys} == {key.split(":", 1)[1] for key in second_keys} + await invoke("messages", recording_server, first_options) + await invoke("messages", recording_server, second_options) assert len(recording_server.requests) == 2 @@ -420,9 +450,8 @@ async def test_rust_messages_legacy_semantic_cache_preserves_python_scope( first_options: Final = {"messages": [{"role": "user", "content": "hello"}]} second_options: Final = {"messages": [{"role": "user", "content": "hi"}]} first: Final = await invoke("messages", recording_server, first_options) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) second: Final = await invoke("messages", recording_server, second_options) - assert cache_key(first) is None - assert cache_key(second) is not None assert payload(first) == payload(second) assert len(recording_server.requests) == 1 @@ -443,3 +472,62 @@ async def test_legacy_cache_keeps_public_messages_responses_compatible( second_payload: Final = await collect(second) if stream else payload(second) assert second_payload == first_payload assert len(recording_server.requests) == 1 + + +_PER_CALL_HIDDEN_PARAMS: Final = frozenset( + {"_response_ms", "litellm_overhead_time_ms", "callback_duration_ms", "litellm_call_id"} +) + + +async def _write_then_read( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + writer: bool, + reader: bool, + stream: bool, +) -> tuple[object, object, object, int]: + from litellm.caching.in_memory_cache import InMemoryCache + + storage: Final = InMemoryCache() + monkeypatch.setattr(litellm, "cache", Cache(_backend=storage)) + requests_before: Final = len(recording_server.requests) + monkeypatch.setenv("LITELLM_RUST", "1" if writer else "0") + first: Final = await invoke("messages", recording_server, {"stream": stream}, native=writer) + if stream: + await collect(first) + else: + add_retry_headers_to_response( + _add_headers_to_response(first, {"x-litellm-model-group": "group"}), attempted_retries=0 + ) + get_hidden_params_dict(first)["model_id"] = "deployment" + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + (entry,) = _entries(storage).values() + stored: Final = TypeAdapter(dict[str, object]).validate_python(entry)["response"] + monkeypatch.setenv("LITELLM_RUST", "1" if reader else "0") + hit: Final = await invoke("messages", recording_server, {"stream": stream}, native=reader) + hidden: Final = { + name: value for name, value in get_hidden_params_dict(hit).items() if name not in _PER_CALL_HIDDEN_PARAMS + } + headers: Final = TypeAdapter(dict[str, object]).validate_python(hidden.pop("additional_headers", {})) + kept: Final = { + name: value for name, value in headers.items() if name not in {"x-litellm-rust", "x-litellm-cache-key"} + } + served: Final = await collect(hit) if stream else payload(hit) + return stored, served, {**hidden, "additional_headers": kept}, len(recording_server.requests) - requests_before + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", (False, True), ids=("response", "stream")) +@pytest.mark.parametrize(("writer", "reader"), ((True, False), (False, True)), ids=("rust_to_python", "python_to_rust")) +async def test_a_cache_entry_written_by_one_leg_is_stored_and_served_like_the_readers_own( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + writer: bool, + reader: bool, + stream: bool, +) -> None: + recording_server.expected_requests = 2 + crossed: Final = await _write_then_read(recording_server, monkeypatch, writer, reader, stream) + own: Final = await _write_then_read(recording_server, monkeypatch, reader, reader, stream) + assert crossed == own + assert crossed[3] == 1 diff --git a/tests/test_litellm_rust/messages/data.py b/tests/test_litellm_rust/messages/data.py index 46127c9237a..d11c901d7d9 100644 --- a/tests/test_litellm_rust/messages/data.py +++ b/tests/test_litellm_rust/messages/data.py @@ -1,7 +1,6 @@ from typing import Final import litellm -from tests.test_litellm_rust.support.gaps import gap from tests.test_litellm_rust.support.recording_server import RecordingServer from tests.test_litellm_rust.support.routing import call_public @@ -57,6 +56,3 @@ async def call(server: RecordingServer, *, asynchronous: bool = True, **kwargs: asynchronous=asynchronous, **kwargs, ) - - -BODY_HOOKS: Final = gap("native Messages does not run the handler body hooks") diff --git a/tests/test_litellm_rust/messages/test_body_hooks.py b/tests/test_litellm_rust/messages/test_body_hooks.py index 59f27d5dc04..bf333a17725 100644 --- a/tests/test_litellm_rust/messages/test_body_hooks.py +++ b/tests/test_litellm_rust/messages/test_body_hooks.py @@ -1,16 +1,22 @@ +import asyncio from collections.abc import Iterator from datetime import datetime from typing import Final import pytest +from pydantic import TypeAdapter import litellm +from litellm.caching.caching import Cache +from litellm.caching.caching_handler import ( + _PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # the first call's entry must land before the second reads it +) from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.anthropic.messages.native import interceptors from litellm.llms.anthropic.messages.native.interceptors.base import MessagesInterceptor -from tests.test_litellm_rust.messages.data import BODY_HOOKS, MESSAGES_RESPONSE, REQUEST, call +from tests.test_litellm_rust.messages.data import MESSAGES_RESPONSE, REQUEST, call from tests.test_litellm_rust.support.callback_recorder import drain_logging from tests.test_litellm_rust.support.body_hooks import LegacyAgenticLoop, PlannedAgenticLoop, agentic_hooks from tests.test_litellm_rust.support.isolation import rebound @@ -20,6 +26,8 @@ from tests.test_litellm_rust.support.routing import CALL_ID, assert_served_nativ pytestmark = pytest.mark.requires_rust_extension +_SENT: Final = TypeAdapter(dict[str, object]) + @pytest.fixture def server(server: RecordingServer) -> Iterator[RecordingServer]: @@ -27,7 +35,6 @@ def server(server: RecordingServer) -> Iterator[RecordingServer]: @pytest.mark.asyncio -@BODY_HOOKS async def test_body_pre_request_hook_rewrites_the_provider_request(server: RecordingServer) -> None: class Rewrite(CustomLogger): async def async_pre_request_hook( @@ -44,7 +51,27 @@ async def test_body_pre_request_hook_rewrites_the_provider_request(server: Recor @pytest.mark.asyncio -@BODY_HOOKS +async def test_body_pre_request_hook_runs_only_after_a_response_cache_miss(server: RecordingServer) -> None: + seen: Final[list[object]] = [] + + class Rewrite(CustomLogger): + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object]: + seen.append(kwargs.get("stop_sequences")) + return {**kwargs, "stop_sequences": ["HOOKED"]} + + with rebound(litellm, "callbacks", [Rewrite()]), rebound(litellm, "cache", Cache()): + await call(server, stop_sequences=["CALLER"]) + await asyncio.gather(*tuple(_PENDING_CACHE_WRITES)) + await call(server, stop_sequences=["CALLER"]) + + assert seen == [None] + assert len(server.requests) == 1 + assert _SENT.validate_python(server.requests[0].body).get("stop_sequences") == ["HOOKED"] + + +@pytest.mark.asyncio async def test_body_websearch_short_circuit_answers_without_the_provider(server: RecordingServer) -> None: class ShortCircuit(WebSearchInterceptionLogger): async def try_short_circuit_search( @@ -65,7 +92,6 @@ async def test_body_websearch_short_circuit_answers_without_the_provider(server: @pytest.mark.asyncio -@BODY_HOOKS async def test_body_interceptor_replaces_the_provider_call(server: RecordingServer) -> None: class Replace(MessagesInterceptor): def can_handle(self, tools: list[dict[str, object]] | None, custom_llm_provider: str | None) -> bool: @@ -82,7 +108,6 @@ async def test_body_interceptor_replaces_the_provider_call(server: RecordingServ @pytest.mark.asyncio -@BODY_HOOKS async def test_body_planned_agentic_loop_sends_the_followup_and_replaces_the_response(server: RecordingServer) -> None: loop: Final = PlannedAgenticLoop() server.enqueue(ResponseSpec(body=MESSAGES_RESPONSE)) @@ -105,7 +130,6 @@ async def test_body_planned_agentic_loop_sends_the_followup_and_replaces_the_res @pytest.mark.asyncio -@BODY_HOOKS async def test_body_legacy_agentic_loop_replaces_the_response(server: RecordingServer) -> None: loop: Final = LegacyAgenticLoop() diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index b580eca2e42..e26aafa5e55 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -1,5 +1,5 @@ import inspect -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from typing import Final, cast # noqa: TID251 # narrows legacy callable signatures for inspect import httpx @@ -8,22 +8,29 @@ import pytest import litellm from litellm.caching.caching import Cache from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS +from litellm.exceptions import UnsupportedParamsError from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger from litellm.llms.anthropic.messages.native import handler as python_messages +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.anthropic.messages.native import interceptors as messages_interceptors +from litellm.llms.anthropic.messages.native.interceptors import MessagesInterceptor from litellm.messages import dispatch from litellm.messages.dispatch import ( _ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch _DISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch ) from litellm.proxy import proxy_server +from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Feature, Route, RouteRule, Rules, Target +from litellm.rust_bridge.catalog import MESSAGES_PYTHON_FEATURES, RULES, Feature, Route, RouteRule, Rules, Target from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES, NativeAmessages, NativeMessages from litellm.rust_bridge.public_call import NativeCall from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse +from tests.test_litellm_rust.support.isolation import rebound +from tests.test_litellm_rust.support.routing import routed MESSAGES: Final = [{"role": "user", "content": "hi"}] PYTHON_RULES: Final[Rules] = () @@ -251,50 +258,6 @@ DANGEROUS_TOOL_USE: Final = [{"type": "dangerous_tool_use", "classifier_context" CLASSIFIER_SETTINGS: Final = {"safeguards_classifier_model": "gpt-oss-120b"} -@pytest.mark.parametrize( - ("model", "kwargs", "general_settings", "serves_python"), - ( - ("deepseek/deepseek-v4-flash", {"safeguards": DANGEROUS_TOOL_USE}, CLASSIFIER_SETTINGS, True), - ( - "deepseek-v4-flash", - {"safeguards": DANGEROUS_TOOL_USE, "custom_llm_provider": "deepseek"}, - CLASSIFIER_SETTINGS, - True, - ), - ("deepseek/deepseek-v4-flash", {"safeguards": DANGEROUS_TOOL_USE}, {}, False), - ("deepseek/deepseek-v4-flash", {}, CLASSIFIER_SETTINGS, False), - ("deepseek/deepseek-v4-flash", {"safeguards": [{"type": "other"}]}, CLASSIFIER_SETTINGS, False), - ("anthropic/claude-sonnet-4-5", {"safeguards": DANGEROUS_TOOL_USE}, CLASSIFIER_SETTINGS, False), - ), -) -def test_proxy_classified_safeguards_reach_the_feature_rule( - monkeypatch: pytest.MonkeyPatch, - model: str, - kwargs: Mapping[str, object], - general_settings: Mapping[str, object], - serves_python: bool, -) -> None: - monkeypatch.setattr(proxy_server, "general_settings", general_settings) - python_result: Final = response(model) - native_result: Final = response(model) - - def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: fallback - return python_result - - result: Final = _DISPATCH.run( - (16, MESSAGES, model), - kwargs, - python=python, - binding=messages_binding(lambda request: native_result), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=( - RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), - RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), - ), - ) - assert result is (python_result if serves_python else native_result) - - @pytest.mark.parametrize( ("args", "kwargs"), ( @@ -460,92 +423,15 @@ class _Agentic(CustomLogger): return False, {} +class _ShortCircuit(WebSearchInterceptionLogger): + async def try_short_circuit_search(self, *args: object, **kwargs: object) -> dict[str, object] | None: # pyright: ignore[reportIncompatibleMethodOverride] # test double only needs the override + return None + + ADVISOR_TOOL: Final = {"type": ANTHROPIC_ADVISOR_TOOL_TYPE, "name": "advisor", "model": "claude-opus-5-5"} STRICT_TOOL: Final = {"name": "a", "input_schema": {"type": "object"}, "strict": True} -@pytest.mark.parametrize( - ("feature", "model", "kwargs", "settings"), - ( - (Feature.MOCK_RESPONSE, "anthropic/claude-sonnet-4-5", {"mock_response": "hi"}, {}), - ( - Feature.PROXY_MCP_TOOLS, - "anthropic/claude-sonnet-4-5", - {"tools": [{"type": "mcp", "server_label": "gateway", "server_url": "litellm_proxy"}]}, - {}, - ), - (Feature.CALLER_CLIENT, "anthropic/claude-sonnet-4-5", {"client": object()}, {}), - (Feature.AGENTIC_LOOP_HOOK, "anthropic/claude-sonnet-4-5", {}, {"callbacks": [_Agentic()]}), - (Feature.RESPONSE_CACHE, "anthropic/claude-sonnet-4-5", {}, {"cache": Cache()}), - (Feature.TLS_POLICY, "anthropic/claude-sonnet-4-5", {}, {"ssl_security_level": "DEFAULT@SECLEVEL=1"}), - (Feature.MODIFY_PARAMS, "anthropic/claude-sonnet-4-5", {}, {"modify_params": True}), - ( - Feature.WORKLOAD_IDENTITY, - "anthropic/claude-sonnet-4-5", - {"anthropic_federation_rule_id": "rule-1", "anthropic_organization_id": "org-1"}, - {}, - ), - (Feature.CACHE_CONTROL_INJECTION, "anthropic/claude-sonnet-4-5", {"enable_prompt_caching": True}, {}), - ( - Feature.THINKING_SIGNATURE_RETRY, - "anthropic/claude-sonnet-4-5", - {"messages": [{"role": "assistant", "content": [{"type": "redacted_thinking", "data": "x"}]}]}, - {}, - ), - ( - Feature.TOOL_CUSTOM_FIELD, - "anthropic/claude-sonnet-4-5", - {"tools": [{"name": "a", "input_schema": {"type": "object"}, "custom": {"eager_input_streaming": True}}]}, - {}, - ), - ( - Feature.STREAM_COST_INJECTION, - "anthropic/claude-sonnet-4-5", - {"stream": True}, - {"include_cost_in_streaming_usage": True}, - ), - ), -) -def test_each_detected_feature_reaches_its_rule( - monkeypatch: pytest.MonkeyPatch, - feature: Feature, - model: str, - kwargs: Mapping[str, object], - settings: Mapping[str, object], -) -> None: - rules: Final[Rules] = ( - RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({feature})), - RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), - ) - python_result: Final = response(model) - native_result: Final = response(model) - - def served(call_kwargs: Mapping[str, object]) -> AnthropicMessagesResponse: - messages: Final = call_kwargs.get("messages", MESSAGES) - return _DISPATCH.run( - (16, messages, model), - {name: value for name, value in call_kwargs.items() if name != "messages"}, - python=lambda *call_args, **python_kwargs: python_result, - binding=messages_binding(lambda request: native_result), - native=lambda hook, request, call_args, native_kwargs: hook(request), - rules=rules, - ) - - monkeypatch.setattr(litellm, "callbacks", []) - monkeypatch.setattr(litellm, "cache", None) - monkeypatch.setattr(litellm, "ssl_security_level", None) - for setting in ("modify_params", "enable_anthropic_prompt_caching", "include_cost_in_streaming_usage"): - monkeypatch.setattr(litellm, setting, False) - for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE", "ANTHROPIC_FEDERATION_RULE_ID", "ANTHROPIC_ORGANIZATION_ID"): - monkeypatch.delenv(name, raising=False) - monkeypatch.setenv("ANTHROPIC_API_KEY", "") - monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) - assert served({}) is native_result - for name, value in settings.items(): - monkeypatch.setattr(litellm, name, value) - assert served(kwargs) is python_result - - @pytest.mark.parametrize( "kwargs", ( @@ -580,27 +466,103 @@ def test_proxy_request_shape_stays_native(kwargs: Mapping[str, object]) -> None: @pytest.mark.asyncio @pytest.mark.parametrize( - "kwargs", + "block", ( - pytest.param({"tools": [STRICT_TOOL]}, id="drop-params-unset"), - pytest.param({"drop_params": False, "tools": [STRICT_TOOL]}, id="drop-params-off"), - pytest.param( - {"drop_params": True, "tools": [{"name": "a", "input_schema": {"type": "object"}}]}, id="no-strict" - ), + pytest.param({"type": "thinking", "thinking": "x", "signature": "s"}, id="thinking"), + pytest.param({"type": "redacted_thinking", "data": "x"}, id="redacted-thinking"), ), ) -async def test_strict_tools_stay_native_unless_python_could_drop_strict( +async def test_replayed_thinking_is_served_natively_under_the_shipped_rules( + monkeypatch: pytest.MonkeyPatch, block: Mapping[str, object] +) -> None: + monkeypatch.setenv("LITELLM_RUST", "1") + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "cache", None) + replay: Final = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [block, {"type": "text", "text": "hello"}]}, + {"role": "user", "content": "again"}, + ] + native_result: Final = response() + python_result: Final = response() + + async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + return python_result + + async def native(request: NativeCall) -> AnthropicMessagesResponse: + return native_result + + served: Final = await _ADISPATCH.arun( + (16, replay, "anthropic/claude-sonnet-4-5"), + {"api_key": "sk-ant-test"}, + python=python, + binding=amessages_binding(native), + native=lambda hook, request, call_args, native_kwargs: hook(request), + rules=RULES, + ) + + assert served is native_result + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("kwargs", "global_drop_params"), + ( + pytest.param({"drop_params": True, "tools": [STRICT_TOOL]}, False, id="drop-params-kwarg"), + pytest.param({"tools": [STRICT_TOOL]}, True, id="drop-params-global"), + pytest.param({"tools": [STRICT_TOOL]}, False, id="drop-params-unset"), + ), +) +async def test_strict_tools_are_served_natively_under_the_shipped_rules( + monkeypatch: pytest.MonkeyPatch, kwargs: Mapping[str, object], global_drop_params: bool +) -> None: + monkeypatch.setenv("LITELLM_RUST", "1") + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "cache", None) + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + native_result: Final = response() + + async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + pytest.fail("strict tools stay on the native path") + + async def native(request: NativeCall) -> AnthropicMessagesResponse: + return native_result + + served: Final = await _ADISPATCH.arun( + (16, MESSAGES, "anthropic/claude-sonnet-4-5"), + kwargs, + python=python, + binding=amessages_binding(native), + native=lambda hook, request, call_args, native_kwargs: hook(request), + rules=RULES, + ) + + assert served is native_result + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kwargs", + ( + pytest.param({}, id="reads-and-writes"), + pytest.param({"cache": {"no-store": True}}, id="no-store"), + pytest.param({"cache": {"ttl": 60, "namespace": "team"}}, id="ttl-and-namespace"), + pytest.param({"stream": True}, id="stream"), + ), +) +async def test_a_configured_response_cache_stays_native( monkeypatch: pytest.MonkeyPatch, kwargs: Mapping[str, object] ) -> None: rules: Final[Rules] = ( - RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.STRICT_TOOLS_RETRY})), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=MESSAGES_PYTHON_FEATURES), RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), ) native_result: Final = response() - monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "cache", Cache()) async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub - pytest.fail("Python cannot drop strict here") + pytest.fail("Rust reads and writes the configured response cache") async def native(request: NativeCall) -> AnthropicMessagesResponse: return native_result @@ -617,45 +579,61 @@ async def test_strict_tools_stay_native_unless_python_could_drop_strict( assert served is native_result -ASYNC_ONLY_FEATURES: Final = ( - pytest.param(Feature.PRE_REQUEST_HOOK, {}, {"callbacks": [_Rewrite()]}, id="pre-request-hook"), - pytest.param( - Feature.FOUNDRY_BETA_ROW, - {"api_base": "https://team.services.ai.azure.com/anthropic"}, +@pytest.mark.asyncio +async def test_modify_params_stays_native(monkeypatch: pytest.MonkeyPatch) -> None: + rules: Final[Rules] = ( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=catalog.MESSAGES_PYTHON_FEATURES), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + ) + native_result: Final = response() + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "modify_params", True) + + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: rejected fallback + pytest.fail("modify_params must not fall back to Python") + + async def apython(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + pytest.fail("modify_params must not fall back to Python") + + async def native(request: NativeCall) -> AnthropicMessagesResponse: + return native_result + + call_args: Final = (200_000, MESSAGES, "anthropic/claude-sonnet-4-5") + served: Final = _DISPATCH.run( + call_args, {}, - id="foundry-beta-row", - ), - pytest.param( - Feature.WEBSEARCH_INTERCEPTION, {}, {"callbacks": [WebSearchInterceptionLogger()]}, id="websearch-interception" - ), - pytest.param( - Feature.MESSAGES_INTERCEPTOR, - {"model": "deepseek/deepseek-v4-flash", "tools": [ADVISOR_TOOL]}, + python=python, + binding=messages_binding(lambda request: native_result), + native=lambda hook, request, args, kwargs: hook(request), + rules=rules, + ) + aserved: Final = await _ADISPATCH.arun( + call_args, {}, - id="messages-interceptor", - ), - pytest.param( - Feature.STRICT_TOOLS_RETRY, {"drop_params": True, "tools": [STRICT_TOOL]}, {}, id="strict-tools-retry" - ), - pytest.param( - Feature.STRICT_TOOLS_RETRY, {"tools": [STRICT_TOOL]}, {"drop_params": True}, id="strict-tools-retry-global" - ), -) + python=apython, + binding=amessages_binding(native), + native=lambda hook, request, args, kwargs: hook(request), + rules=rules, + ) + + assert (served, aserved) == (native_result, native_result) @pytest.mark.asyncio -@pytest.mark.parametrize(("feature", "kwargs", "settings"), ASYNC_ONLY_FEATURES) -async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_python( - monkeypatch: pytest.MonkeyPatch, feature: Feature, kwargs: Mapping[str, object], settings: Mapping[str, object] -) -> None: +@pytest.mark.parametrize( + "kwargs", + ( + pytest.param({"mock_response": "hi"}, id="unary"), + pytest.param({"mock_response": "hi", "stream": True}, id="streamed"), + ), +) +async def test_mock_response_is_served_natively(monkeypatch: pytest.MonkeyPatch, kwargs: Mapping[str, object]) -> None: rules: Final[Rules] = ( - RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({feature})), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.MOCK_RESPONSE})), RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), ) python_result: Final = response() native_result: Final = response() - model: Final = str(kwargs.get("model", "anthropic/claude-sonnet-4-5")) - call_kwargs: Final = {name: value for name, value in kwargs.items() if name != "model"} async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub return python_result @@ -664,48 +642,331 @@ async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_ return native_result monkeypatch.setattr(litellm, "callbacks", []) - for name, value in settings.items(): - monkeypatch.setattr(litellm, name, value) served_async: Final = await _ADISPATCH.arun( - (16, MESSAGES, model), - call_kwargs, + (16, MESSAGES, "anthropic/claude-sonnet-4-5"), + kwargs, python=python, binding=amessages_binding(native), native=lambda hook, request, args, native_kwargs: hook(request), rules=rules, ) served_sync: Final = _DISPATCH.run( - (16, MESSAGES, model), - call_kwargs, + (16, MESSAGES, "anthropic/claude-sonnet-4-5"), + kwargs, python=lambda *call_args, **python_kwargs: python_result, binding=messages_binding(lambda request: native_result), native=lambda hook, request, args, native_kwargs: hook(request), rules=rules, ) - assert served_async is python_result + assert served_async is native_result assert served_sync is native_result +class _Claims(MessagesInterceptor): + def can_handle(self, tools: Sequence[Mapping[str, object]] | None, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == "anthropic" + + async def handle(self, **kwargs: object) -> AnthropicMessagesResponse: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: test double only needs the override + return response() + + @pytest.mark.asyncio -async def test_interceptor_feature_follows_the_resolved_provider() -> None: +async def test_a_call_a_registered_interceptor_owns_runs_on_rust( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str] +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(messages_interceptors, "_interceptors", [_Claims()]) + + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: rejected fallback + pytest.fail("Rust hands the call to the interceptor itself") + + with rebound(dispatch, "_PYTHON_AMESSAGES", python): + result: Final = await _public(True, "anthropic/claude-sonnet-4-5", {"tools": [ADVISOR_TOOL]}) + + assert isinstance(result, dict) + assert shipped_native == ["async"] + + +def test_a_federated_request_stays_native(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"): + monkeypatch.delenv(name, raising=False) rules: Final[Rules] = ( - RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.MESSAGES_INTERCEPTOR})), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.WORKLOAD_IDENTITY})), RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), ) native_result: Final = response() - async def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub - pytest.fail("anthropic serves the advisor tool itself") + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: rejected fallback + pytest.fail("Rust mints the federation token itself") - async def native(request: NativeCall) -> AnthropicMessagesResponse: - return native_result - - result: Final = await _ADISPATCH.arun( - (16, MESSAGES, "claude-sonnet-4-5"), - {"tools": [ADVISOR_TOOL], "custom_llm_provider": "anthropic"}, + served: Final = _DISPATCH.run( + (16, MESSAGES, "anthropic/claude-sonnet-4-5"), + {"anthropic_federation_rule_id": "rule-1", "anthropic_organization_id": "org-1"}, python=python, - binding=amessages_binding(native), - native=lambda hook, request, call_args, call_kwargs: hook(request), + binding=messages_binding(lambda request: native_result), + native=lambda hook, request, call_args, native_kwargs: hook(request), rules=rules, ) - assert result is native_result + assert served is native_result + + +def _shipped_native(calls: list[str]) -> None: # mutable-ok: records which native entry ran + def native(request: NativeCall) -> AnthropicMessagesResponse: + calls.append("sync") + return response() + + async def anative(request: NativeCall) -> AnthropicMessagesResponse: + calls.append("async") + return response() + + NATIVE_MESSAGES.override(native) + NATIVE_AMESSAGES.override(anative) + + +@pytest.fixture +def shipped_native() -> Iterator[list[str]]: + calls: Final[list[str]] = [] + _shipped_native(calls) + with routed(*RULES): + yield calls + NATIVE_MESSAGES.reset() + NATIVE_AMESSAGES.reset() + + +async def _public(asynchronous: bool, model: str, kwargs: Mapping[str, object]) -> object: + call: Final = {"max_tokens": 16, "messages": MESSAGES, "model": model, "api_key": "sk-test", **kwargs} + if asynchronous: + return await dispatch.anthropic_messages(**call) + return dispatch.anthropic_messages_handler(**call) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +async def test_anthropic_runs_on_rust_without_the_opt_in( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], asynchronous: bool +) -> None: + monkeypatch.setenv("LITELLM_RUST", "0") + monkeypatch.setattr(litellm, "callbacks", []) + + result: Final = await _public(asynchronous, "anthropic/claude-sonnet-4-5", {}) + + assert isinstance(result, dict) + assert shipped_native == ["async" if asynchronous else "sync"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +@pytest.mark.parametrize( + "model", ("anthropic/parity-tool-model", "anthropic/claude-sonnet-4-5"), ids=("non-claude", "claude") +) +async def test_proxy_classified_safeguards_run_on_rust( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], asynchronous: bool, model: str +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(proxy_server, "general_settings", CLASSIFIER_SETTINGS) + + result: Final = await _public(asynchronous, model, {"safeguards": DANGEROUS_TOOL_USE}) + + assert isinstance(result, dict) + assert shipped_native == ["async" if asynchronous else "sync"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +@pytest.mark.parametrize( + "thread", + ( + pytest.param({"type": "create"}, id="create"), + pytest.param({"type": "continue", "previous_message_id": "msg_1"}, id="continue"), + ), +) +async def test_a_message_thread_runs_on_rust( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], asynchronous: bool, thread: Mapping[str, str] +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + + result: Final = await _public(asynchronous, "anthropic/claude-sonnet-4-5", {"thread": dict(thread)}) + + assert isinstance(result, dict) + assert shipped_native == ["async" if asynchronous else "sync"] + + +@pytest.mark.asyncio +async def test_anthropic_runs_pre_request_hooks_on_rust( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str] +) -> None: + monkeypatch.setattr(litellm, "callbacks", [_Rewrite()]) + + result: Final = await _public(True, "anthropic/claude-sonnet-4-5", {}) + + assert isinstance(result, dict) + assert shipped_native == ["async"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +async def test_anthropic_runs_a_web_search_short_circuit_on_rust( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], asynchronous: bool +) -> None: + monkeypatch.setattr(litellm, "callbacks", [_ShortCircuit()]) + + result: Final = await _public(asynchronous, "anthropic/claude-sonnet-4-5", {}) + + assert isinstance(result, dict) + assert shipped_native == ["async" if asynchronous else "sync"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +@pytest.mark.parametrize( + ("kwargs", "settings"), + ( + pytest.param({"client": object()}, {}, id="caller-client"), + pytest.param({"client": AsyncHTTPHandler()}, {}, id="caller-async-handler"), + pytest.param({}, {"ssl_security_level": "DEFAULT@SECLEVEL=1"}, id="tls-security-level"), + pytest.param({}, {"ssl_ecdh_curve": "secp521r1"}, id="tls-ecdh-curve"), + ), +) +async def test_anthropic_serves_the_http_transport_features_on_rust( + monkeypatch: pytest.MonkeyPatch, + shipped_native: list[str], + asynchronous: bool, + kwargs: Mapping[str, object], + settings: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE"): + monkeypatch.delenv(name, raising=False) + for name, value in settings.items(): + monkeypatch.setattr(litellm, name, value) + + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: rejected fallback + pytest.fail("anthropic has no Python Messages path") + + with rebound(dispatch, "_PYTHON_MESSAGES", python), rebound(dispatch, "_PYTHON_AMESSAGES", python): + result: Final = await _public(asynchronous, "anthropic/claude-sonnet-4-5", kwargs) + + assert isinstance(result, dict) + assert shipped_native == ["async" if asynchronous else "sync"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +@pytest.mark.parametrize( + "callbacks", + ( + pytest.param([_Agentic()], id="agentic-loop"), + pytest.param([WebSearchInterceptionLogger(enabled_providers=["anthropic"])], id="websearch-interception"), + ), +) +async def test_anthropic_serves_agentic_loop_callbacks_on_rust( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], asynchronous: bool, callbacks: list[object] +) -> None: + monkeypatch.setattr(litellm, "callbacks", callbacks) + + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: rejected fallback + pytest.fail("anthropic has no Python Messages path") + + with rebound(dispatch, "_PYTHON_MESSAGES", python), rebound(dispatch, "_PYTHON_AMESSAGES", python): + result: Final = await _public(asynchronous, "anthropic/claude-sonnet-4-5", {}) + + assert isinstance(result, dict) + assert shipped_native == ["async" if asynchronous else "sync"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +@pytest.mark.parametrize( + "server_url", + ( + pytest.param("litellm_proxy", id="whole-gateway"), + pytest.param("litellm_proxy/mcp/deepwiki", id="one-server"), + pytest.param("https://gateway.example/mcp/deepwiki", id="proxy-path"), + ), +) +async def test_anthropic_serves_gateway_mcp_tools_on_rust( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], asynchronous: bool, server_url: str +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + tools: Final = [{"type": "mcp", "server_label": "gateway", "server_url": server_url, "require_approval": "never"}] + + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: rejected fallback + pytest.fail("anthropic has no Python Messages path") + + with rebound(dispatch, "_PYTHON_MESSAGES", python), rebound(dispatch, "_PYTHON_AMESSAGES", python): + result: Final = await _public(asynchronous, "anthropic/claude-sonnet-4-5", {"tools": tools}) + + assert isinstance(result, dict) + assert shipped_native == ["async" if asynchronous else "sync"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +async def test_other_providers_keep_python_for_the_same_features( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], asynchronous: bool +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + python_result: Final = response("deepseek/deepseek-v4-flash") + + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + return python_result + + async def apython(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + return python_result + + with rebound(dispatch, "_PYTHON_MESSAGES", python), rebound(dispatch, "_PYTHON_AMESSAGES", apython): + result: Final = await _public(asynchronous, "deepseek/deepseek-v4-flash", {"client": object()}) + + assert result is python_result + assert shipped_native == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "refused"), + ( + pytest.param("anthropic/claude-sonnet-4-5", True, id="anthropic"), + pytest.param("deepseek/deepseek-v4-flash", False, id="deepseek"), + ), +) +async def test_a_failing_detector_refuses_only_the_rust_only_provider( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], model: str, refused: bool +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + python_result: Final = response(model) + + def failing(request: NativeCall) -> bool: + raise RuntimeError("detector failed") + + async def apython(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + return python_result + + with ( + rebound(dispatch, "_FEATURES", {Feature.MOCK_RESPONSE: failing}), + rebound(dispatch, "_PYTHON_AMESSAGES", apython), + ): + if not refused: + assert await _public(True, model, {}) is python_result + return + with pytest.raises(UnsupportedParamsError, match="does not support this call"): + await _public(True, model, {}) + assert shipped_native == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +async def test_an_anthropic_call_its_signature_rejects_keeps_python_s_type_error( + monkeypatch: pytest.MonkeyPatch, shipped_native: list[str], asynchronous: bool +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + call: Final = {"messages": MESSAGES, "model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"} + + async def attempt() -> object: + if asynchronous: + return await dispatch.anthropic_messages(**call) + return dispatch.anthropic_messages_handler(**call) + + with pytest.raises(TypeError, match="max_tokens"): + await attempt() + + assert shipped_native == [] diff --git a/tests/test_litellm_rust/messages/test_workload_identity.py b/tests/test_litellm_rust/messages/test_workload_identity.py new file mode 100644 index 00000000000..d2c1a85de91 --- /dev/null +++ b/tests/test_litellm_rust/messages/test_workload_identity.py @@ -0,0 +1,219 @@ +import re +from collections.abc import Iterator, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.llms.base_llm.auth.token_exchange import default_token_exchange_engine +from tests.test_litellm_rust.messages.data import MESSAGES_EVENTS, MESSAGES_RESPONSE, call +from tests.test_litellm_rust.support.isolation import rebound +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec, relaxed +from tests.test_litellm_rust.support.routing import assert_served_natively + +pytestmark = pytest.mark.requires_rust_extension + +IDENTITY_TOKEN: Final = "eyJ.native-identity.jwt" +MINTED_TOKEN: Final = "sk-ant-oat01-native-minted" +FEDERATION_IDS: Final = {"anthropic_federation_rule_id": "fdrl_native", "anthropic_organization_id": "org-native"} +MINTED: Final = ResponseSpec(body={"access_token": MINTED_TOKEN, "token_type": "Bearer", "expires_in": 3600}) +EXCHANGE: Final = { + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "federation_rule_id": "fdrl_native", + "organization_id": "org-native", + "assertion": IDENTITY_TOKEN, +} +FEDERATION_ENV_NAMES: Final = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_FEDERATION_RULE_ID", + "ANTHROPIC_ORGANIZATION_ID", + "ANTHROPIC_IDENTITY_TOKEN", + "ANTHROPIC_IDENTITY_TOKEN_FILE", + "ANTHROPIC_IDENTITY_SOURCE", + "LITELLM_ANTHROPIC_WIF_ALLOWED_HOSTS", +) + + +@pytest.fixture +def server(server: RecordingServer) -> Iterator[RecordingServer]: + yield from relaxed(server) + + +@pytest.fixture +def federation(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + for name in FEDERATION_ENV_NAMES: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("ANTHROPIC_IDENTITY_TOKEN", IDENTITY_TOKEN) + monkeypatch.setenv("LITELLM_ANTHROPIC_WIF_ALLOWED_HOSTS", "127.0.0.1") + with ( + rebound(default_token_exchange_engine, "_entries", {}), + rebound(default_token_exchange_engine, "_shared_store", None), + ): + yield + + +def wire(server: RecordingServer) -> tuple[tuple[str, object, Mapping[str, str]], ...]: + return tuple((sent.path, sent.body, sent.headers) for sent in server.requests) + + +def assert_minted_bearer(headers: Mapping[str, str]) -> None: + assert headers.get("authorization") == f"Bearer {MINTED_TOKEN}" + assert "x-api-key" not in headers + assert "oauth-2025-04-20" in headers.get("anthropic-beta", "").split(",") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("federation") +@pytest.mark.parametrize( + ("kwargs", "env", "exchange"), + ( + pytest.param(FEDERATION_IDS, {}, EXCHANGE, id="ids-from-params"), + pytest.param( + {}, + {"ANTHROPIC_FEDERATION_RULE_ID": "fdrl_env", "ANTHROPIC_ORGANIZATION_ID": "org-env"}, + {**EXCHANGE, "federation_rule_id": "fdrl_env", "organization_id": "org-env"}, + id="ids-from-environment", + ), + pytest.param( + { + **FEDERATION_IDS, + "anthropic_federation_workspace_id": "wrkspc_native", + "anthropic_service_account_id": "svac_native", + }, + {}, + {**EXCHANGE, "workspace_id": "wrkspc_native", "service_account_id": "svac_native"}, + id="workspace-and-service-account", + ), + ), +) +async def test_workload_identity_mints_a_bearer_for_a_call_without_a_key( + server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + kwargs: Mapping[str, object], + env: Mapping[str, str], + exchange: Mapping[str, str], +) -> None: + for name, value in env.items(): + monkeypatch.setenv(name, value) + server.enqueue(MINTED) + + response: Final = await call(server, api_key=None, **kwargs) + + assert_served_natively(server, count=2) + (token_path, token_body, _), (messages_path, _, headers) = wire(server) + assert (token_path, token_body) == ("/v1/oauth/token", exchange) + assert messages_path == "/v1/messages" + assert_minted_bearer(headers) + assert isinstance(response, dict) + assert response["id"] == MESSAGES_RESPONSE["id"] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("federation") +async def test_workload_identity_keeps_the_caller_betas_and_headers(server: RecordingServer) -> None: + server.enqueue(MINTED) + + await call( + server, + api_key=None, + extra_headers={"anthropic-beta": "files-api-2025-04-14", "x-trace-id": "trace-1"}, + **FEDERATION_IDS, + ) + + assert_served_natively(server, count=2) + _, (_, _, headers) = wire(server) + assert_minted_bearer(headers) + assert "files-api-2025-04-14" in headers["anthropic-beta"].split(",") + assert headers["x-trace-id"] == "trace-1" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("federation") +async def test_workload_identity_streams_with_the_minted_bearer(server: RecordingServer) -> None: + server.enqueue(MINTED) + server.enqueue(ResponseSpec(body={}, events=MESSAGES_EVENTS)) + + stream: Final = await call(server, api_key=None, stream=True, **FEDERATION_IDS) + chunks: Final = [chunk async for chunk in stream] # pyright: ignore[reportGeneralTypeIssues] # the public call returns an async stream + + assert_served_natively(server, count=2) + _, (_, _, headers) = wire(server) + assert_minted_bearer(headers) + assert b"message_stop" in b"".join(chunk if isinstance(chunk, bytes) else str(chunk).encode() for chunk in chunks) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("federation") +async def test_a_static_key_shadows_workload_identity(server: RecordingServer) -> None: + await call(server, api_key="sk-ant-static", **FEDERATION_IDS) + + assert_served_natively(server) + ((path, _, headers),) = wire(server) + assert path == "/v1/messages" + assert headers.get("x-api-key") == "sk-ant-static" + assert "authorization" not in headers + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("federation") +@pytest.mark.parametrize( + ("kwargs", "env", "upstream", "message", "sent"), + ( + pytest.param( + FEDERATION_IDS, + {}, + (ResponseSpec(body={"error": "authentication_error"}, status=401),), + "Anthropic workload identity federation failed. The token endpoint returned HTTP 401", + ("/v1/oauth/token",), + id="exchange-denied", + ), + pytest.param( + FEDERATION_IDS, + {"LITELLM_ANTHROPIC_WIF_ALLOWED_HOSTS": None}, + (), + "Anthropic workload identity federation refused to use host", + (), + id="untrusted-exchange-host", + ), + pytest.param( + {"anthropic_identity_token": "oidc/env/ANTHROPIC_IDENTITY_TOKEN"}, + {}, + (), + "anthropic_identity_token is set, but anthropic_federation_rule_id and anthropic_organization_id", + (), + id="identity-token-without-ids", + ), + pytest.param( + {**FEDERATION_IDS, "anthropic_disable_workload_identity_federation": True}, + {}, + (), + "Missing Anthropic API Key", + (), + id="federation-disabled", + ), + ), +) +async def test_workload_identity_failures_raise_before_the_messages_call( + server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + kwargs: Mapping[str, object], + env: Mapping[str, str | None], + upstream: tuple[ResponseSpec, ...], + message: str, + sent: tuple[str, ...], +) -> None: + for name, value in env.items(): + if value is None: + monkeypatch.delenv(name) + else: + monkeypatch.setenv(name, value) + for spec in upstream: + server.enqueue(spec) + + with pytest.raises(litellm.AuthenticationError, match=re.escape(message)) as raised: + await call(server, api_key=None, **kwargs) + + assert raised.value.llm_provider == "anthropic" + assert_served_natively(server, count=len(sent)) + assert tuple(path for path, _, _ in wire(server)) == sent diff --git a/tests/test_litellm_rust/support/recording_server.py b/tests/test_litellm_rust/support/recording_server.py index 5743658abdf..2b5e6fe93a6 100644 --- a/tests/test_litellm_rust/support/recording_server.py +++ b/tests/test_litellm_rust/support/recording_server.py @@ -1,5 +1,6 @@ import asyncio import copy +import gzip import json import threading import time @@ -29,13 +30,31 @@ class ResponseSpec: events: tuple[tuple[str, object], ...] = () # Splits a non-stream body into this many writes, `chunk_delay` apart, so it trickles in. body_chunks: int = 1 + cut_after: int | None = None + cut_delay: float = 0 + gzip: bool = False + # Writes every stream event in one write, so they reach the client in one read. + burst: bool = False + # Sends each stream event as its own HTTP/1.1 chunk instead of under a Content-Length. + chunked: bool = False def payloads(self) -> tuple[bytes, ...]: if not self.events: - body: Final = self.body if isinstance(self.body, bytes) else json.dumps(self.body).encode() + raw: Final = self.body if isinstance(self.body, bytes) else json.dumps(self.body).encode() + body: Final = gzip.compress(raw, mtime=0) if self.gzip else raw size: Final = max(-(-len(body) // max(self.body_chunks, 1)), 1) return tuple(body[start : start + size] for start in range(0, len(body), size)) or (body,) - return tuple(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() for event, data in self.events) + frames: Final = tuple(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() for event, data in self.events) + if self.gzip: + return (gzip.compress(b"".join(frames), mtime=0),) + return (b"".join(frames),) if self.burst and not self.chunked else frames + + def writes(self) -> tuple[bytes, ...]: + payloads: Final = self.payloads() + if not self.chunked: + return payloads + chunks: Final = (*(b"%x\r\n%s\r\n" % (len(payload), payload) for payload in payloads), b"0\r\n\r\n") + return (b"".join(chunks),) if self.burst else chunks @dataclass @@ -84,15 +103,28 @@ def recording_service() -> Iterator[RecordingServer]: response: Final = responses.pop(0) if responses else copy.deepcopy(recording_server.default_response) if response.delay: time.sleep(response.delay) - payloads: Final = response.payloads() + payloads: Final = response.writes() + if response.chunked: + self.protocol_version = "HTTP/1.1" + self.close_connection = True self.send_response(response.status) self.send_header("Content-Type", "text/event-stream" if response.events else "application/json") - self.send_header("Content-Length", str(sum(len(payload) for payload in payloads))) + if response.chunked: + self.send_header("Transfer-Encoding", "chunked") + self.send_header("Connection", "close") + else: + self.send_header("Content-Length", str(sum(len(payload) for payload in payloads))) + if response.gzip: + self.send_header("Content-Encoding", "gzip") for name, value in response.headers.items(): self.send_header(name, value) self.end_headers() try: for index, payload in enumerate(payloads): + if index == response.cut_after: + time.sleep(response.cut_delay) + self.close_connection = True + break if index and response.chunk_delay: time.sleep(response.chunk_delay) self.wfile.write(payload) diff --git a/tests/test_litellm_rust/support/served.py b/tests/test_litellm_rust/support/served.py index 0890077f745..5dfaa2cfaeb 100644 --- a/tests/test_litellm_rust/support/served.py +++ b/tests/test_litellm_rust/support/served.py @@ -6,11 +6,13 @@ from tests.test_litellm_rust.support.recording_server import RecordingServer def assert_served_natively(server: RecordingServer, *responses: object, count: int = 1) -> None: assert len(server.requests) == (len(responses) or count) - if not responses: - assert not any( - request.headers.get("user-agent", "").startswith("python-httpx") for request in server.requests - ) + marked: Final = tuple(response for response in responses if not isinstance(response, dict)) + if not marked: + assert not any(request.headers.get("user-agent", "").startswith("python-httpx") for request in server.requests) for response in responses: + if all(response is not other for other in marked): + assert "x-litellm-rust" not in str(get_hidden_params_dict(response)) + for response in marked: additional_headers: Final = get_hidden_params_dict(response).get("additional_headers") assert isinstance(additional_headers, dict) assert additional_headers.get("x-litellm-rust") == "true" diff --git a/tests/test_litellm_rust/test_dispatch.py b/tests/test_litellm_rust/test_dispatch.py index 193866409a2..10cc05e0801 100644 --- a/tests/test_litellm_rust/test_dispatch.py +++ b/tests/test_litellm_rust/test_dispatch.py @@ -189,7 +189,7 @@ def test_rules_for_other_routes_and_constrained_python_rules_skip_projection() - RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, providers=frozenset({"other"})), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, models=frozenset({"other"})), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, targets=frozenset({Target.CHAT_COMPLETIONS})), - RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.STREAMING})), ), ids=("providers", "models", "targets", "features"), ) diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 02476545cae..ef2048998b1 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.core_helpers import ( bind_budget_reservation_to_callbacks, budget_reservation_from_metadata, drop_params_env_flag, + env_flag, drop_params_flag, get_litellm_metadata_from_kwargs, get_or_create_metadata_bucket, @@ -497,6 +498,32 @@ def test_drop_params_env_flag_keeps_a_non_flag_value_on_with_a_warning(configure ) +@pytest.mark.parametrize( + ("configured", "expected"), + [("", False), ("false", False), ("False", False), ("0", False), ("no", False), ("true", True), (" 1 ", True)], +) +def test_env_flag_parses_modify_params_as_a_bool(configured: str, expected: bool, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.WARNING, logger="env-flag-test"): + assert ( + env_flag({"LITELLM_MODIFY_PARAMS": configured}, "LITELLM_MODIFY_PARAMS", logging.getLogger("env-flag-test")) + is expected + ) + assert caplog.text == "" + + +def test_env_flag_treats_a_non_flag_value_as_on_with_a_warning(caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.WARNING, logger="env-flag-test"): + assert ( + env_flag({"LITELLM_MODIFY_PARAMS": "enabled"}, "LITELLM_MODIFY_PARAMS", logging.getLogger("env-flag-test")) + is True + ) + assert "LITELLM_MODIFY_PARAMS='enabled' is not a flag value, treating it as on" in caplog.text + + +def test_env_flag_is_off_when_unset(): + assert env_flag({}, "LITELLM_MODIFY_PARAMS", logging.getLogger("env-flag-test")) is False + + class TestIsExpectedClientError: def test_status_ranges(self): from litellm.litellm_core_utils.core_helpers import is_expected_client_error diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index c1d878d5b09..51fbcaa54bf 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -2376,6 +2376,7 @@ async def test_anthropic_messages_marks_litellm_params_async(): import litellm from litellm.integrations.custom_logger import CustomLogger + from litellm.llms.anthropic.messages.native import handler as python_messages captured = {} logged = asyncio.Event() @@ -2390,7 +2391,7 @@ async def test_anthropic_messages_marks_litellm_params_async(): original_callbacks = getattr(litellm, "callbacks", []) try: litellm.callbacks = [logger] - await litellm.anthropic_messages( + await python_messages.anthropic_messages( max_tokens=100, messages=[{"role": "user", "content": "Hey"}], model="anthropic/claude-sonnet-4-5", diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 54f55aba46a..f434b1f4d9f 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -15,6 +15,7 @@ from httpx import Headers from pydantic import BaseModel, ConfigDict import litellm +from litellm.llms.anthropic.messages.native import handler as python_messages from litellm.constants import ( ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, @@ -4509,7 +4510,7 @@ async def test_native_compaction_wire_roundtrip( return httpx.Response(200, json=body) route.mock(side_effect=respond) - call: Final = litellm.anthropic.messages.acreate if messages_api else litellm.acompletion + call: Final = python_messages.anthropic_messages if messages_api else litellm.acompletion params: Final = dict( model=f"{'openai/' if gateway else ''}anthropic/claude-sonnet-5", api_key="test", diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index ff150af324d..54b76756975 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -20,6 +20,7 @@ from pydantic import TypeAdapter, ValidationError from unittest.mock import AsyncMock, MagicMock, patch import litellm +from litellm.llms.anthropic.messages.native import handler as python_messages from litellm.anthropic_interface import messages from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -1470,7 +1471,7 @@ async def test_native_messages_strips_replayed_provider_specific_fields_from_wir patch.object(httpx.AsyncClient, "send", fake_send), pytest.raises(litellm.exceptions.InternalServerError), ): - await litellm.anthropic.messages.acreate( + await python_messages.anthropic_messages( max_tokens=100, messages=messages, model="anthropic/claude-haiku-4-5-20251001", @@ -2451,7 +2452,7 @@ async def test_anthropic_messages_with_extra_headers(): mock_client.post = AsyncMock(return_value=mock_response) # Call the handler with extra_headers and our mocked client - response = await litellm.anthropic.messages.acreate( + response = await python_messages.anthropic_messages( messages=messages, api_key=api_key, model="claude-haiku-4-5-20251001", @@ -2515,7 +2516,7 @@ async def test_anthropic_messages_with_thinking(): mock_client.post = AsyncMock(return_value=mock_response) # Call the handler with extra_headers and our mocked client - response = await litellm.anthropic.messages.acreate( + response = await python_messages.anthropic_messages( messages=messages, api_key=api_key, model="claude-haiku-4-5-20251001", @@ -2785,7 +2786,7 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s anthropic: Final = _RejectsStrictToolsOnce() tool: Final = {"name": "lookup_record", "input_schema": {"type": "object"}, "strict": True} - response: Final = await litellm.anthropic_messages( + response: Final = await python_messages.anthropic_messages( max_tokens=16, messages=[{"role": "user", "content": "Use lookup_record to find invoice 42"}], model="anthropic/claude-sonnet-4-6", diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py index 1d0d70d8e60..a6aade2e3cf 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py @@ -18,6 +18,7 @@ import respx import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.anthropic.messages.native import handler as python_messages from litellm.router import Router from litellm.types.llms.anthropic import ( AnthropicMessagesTextParam, @@ -220,6 +221,11 @@ def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) +@pytest.fixture(autouse=True) +def _python_messages(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "anthropic_messages", python_messages.anthropic_messages) + + @pytest.mark.asyncio async def test_router_aanthropic_messages_non_streaming_posts_anthropic_body(respx_mock): route: Final = respx_mock.post(url__regex=_ANTHROPIC_URL).mock( @@ -362,7 +368,7 @@ async def test_acreate_non_streaming_returns_dict_content_blocks( ): _set_fake_aws_env(monkeypatch) route: Final = respx_mock.post(url__regex=url).mock(return_value=httpx.Response(200, json=body)) - response: Final = await litellm.anthropic.messages.acreate( + response: Final = await python_messages.anthropic_messages( messages=[{"role": "user", "content": _PROMPT}], model=model, max_tokens=100, @@ -602,7 +608,7 @@ async def test_bedrock_converse_system_prompt_caching_returns_cache_tokens(respx }, ) ) - response: Final = await litellm.anthropic.messages.acreate( + response: Final = await python_messages.anthropic_messages( model=f"bedrock/converse/{_BEDROCK_SONNET}", messages=[{"role": "user", "content": "What are the key terms?"}], system=_cached_system(), @@ -632,7 +638,7 @@ async def test_bedrock_invoke_system_prompt_caching_returns_cache_tokens(respx_m }, ) ) - response: Final = await litellm.anthropic.messages.acreate( + response: Final = await python_messages.anthropic_messages( model=f"bedrock/invoke/{_BEDROCK_SONNET}", messages=[{"role": "user", "content": "What are the key terms?"}], system=_cached_system(), @@ -664,7 +670,7 @@ async def test_bedrock_streaming_message_start_carries_cache_usage_fields( route: Final = respx_mock.post(url__regex=url).mock( return_value=httpx.Response(200, content=body, headers={"content-type": "application/vnd.amazon.eventstream"}) ) - stream: Final = await litellm.anthropic.messages.acreate( + stream: Final = await python_messages.anthropic_messages( model=model, messages=[ { @@ -734,7 +740,7 @@ async def test_tool_search_forwards_deferred_tools_and_beta_header( }, ) ) - response: Final = await litellm.anthropic.messages.acreate( + response: Final = await python_messages.anthropic_messages( model="anthropic/claude-sonnet-4-5-20250929", messages=[{"role": "user", "content": prompt}], tools=[dict(tool) for tool in _tool_search_tools()], diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 97d5b0550d4..d86c25c2558 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -16,7 +16,6 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) -from litellm.llms.anthropic.wif import get_anthropic_wif_token from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig from litellm.llms.minimax.messages.transformation import MinimaxMessagesConfig @@ -27,12 +26,7 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass ) from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicFinishReason from litellm.types.router import GenericLiteLLMParams -from tests.unit.llms.anthropic.test_anthropic_wif import ( - ScriptedPoster, - make_engine, - token_response, - write_token_file, -) +from tests.unit.llms.anthropic.test_anthropic_wif import write_token_file _WIF_PARAMS: Final[dict] = { "anthropic_federation_rule_id": "fdrl_abc123", @@ -165,18 +159,6 @@ def test_messages_request_never_defers_tools_off_claude( assert request["tools"] == [{"name": "A", "input_schema": _EMPTY_SCHEMA} if drop_params else wrapped_tool] -def test_workload_identity_allowed_for_anthropic() -> None: - assert AnthropicMessagesConfig()._allows_workload_identity is True - - -def test_workload_identity_blocked_for_minimax() -> None: - assert MinimaxMessagesConfig()._allows_workload_identity is False - - -def test_workload_identity_blocked_for_tencent() -> None: - assert TencentAnthropicMessagesConfig()._allows_workload_identity is False - - def test_minimax_validate_environment_never_attaches_anthropic_wif_credential( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -228,37 +210,6 @@ def test_tencent_validate_environment_never_attaches_anthropic_wif_credential( ) -def test_wif_token_exchange_reaches_only_anthropic_not_minimax_or_tencent( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """get_anthropic_wif_token's engine parameter is the only DI seam in the WIF minting chain; - validate_anthropic_messages_environment always uses the module's default engine, so this - drives that seam directly with the exact litellm_params AnthropicModelInfo.get_auth_header - would receive from each config, proving MiniMax/Tencent never reach the token endpoint even - when a mint would otherwise succeed.""" - monkeypatch.setenv("ANTHROPIC_FEDERATION_RULE_ID", "fdrl_prod") - monkeypatch.setenv("ANTHROPIC_ORGANIZATION_ID", "org-prod-uuid") - monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) - token_file = write_token_file(tmp_path, "jwt-assertion-value") - litellm_params = {"anthropic_identity_token_file": str(token_file)} - poster = ScriptedPoster([token_response("sk-ant-oat01-canary")]) - engine = make_engine(poster) - - minted: Final = get_anthropic_wif_token( - litellm_params, - "https://api.anthropic.com", - "claude-sonnet-4-5", - engine, - ) - assert minted == "sk-ant-oat01-canary" - assert len(poster.requests) == 1 - - for config in (MinimaxMessagesConfig(), TencentAnthropicMessagesConfig()): - assert config._allows_workload_identity is False - - assert len(poster.requests) == 1 - - DANGEROUS_TOOL_USE_BETA: Final = "dangerous-tool-use-2026-09-03" AUTO_MODE_SAFEGUARDS: Final = [ {"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}} diff --git a/tests/unit/llms/anthropic/messages/native/test_response_cache.py b/tests/unit/llms/anthropic/messages/native/test_response_cache.py index ac104b55bde..bc26d5612df 100644 --- a/tests/unit/llms/anthropic/messages/native/test_response_cache.py +++ b/tests/unit/llms/anthropic/messages/native/test_response_cache.py @@ -99,9 +99,9 @@ async def test_non_streaming_request_is_served_from_cache(local_cache, request_k fake_handler = _CountingHandler([_anthropic_response("msg_1", "ALPHA"), _anthropic_response("msg_2", "BETA")]) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - first = await litellm.anthropic_messages(**request_kwargs) + first = await handler.anthropic_messages(**request_kwargs) await asyncio.sleep(0) - second = await litellm.anthropic_messages(**request_kwargs) + second = await handler.anthropic_messages(**request_kwargs) assert len(fake_handler.calls) == 1 assert first == second @@ -115,9 +115,9 @@ async def test_cache_key_separates_different_system_prompts(local_cache, request fake_handler = _CountingHandler([_anthropic_response("msg_1", "ALPHA"), _anthropic_response("msg_2", "BETA")]) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - first = await litellm.anthropic_messages(**request_kwargs, system="Always answer ALPHA") + first = await handler.anthropic_messages(**request_kwargs, system="Always answer ALPHA") await asyncio.sleep(0) - second = await litellm.anthropic_messages(**request_kwargs, system="Always answer BETA") + second = await handler.anthropic_messages(**request_kwargs, system="Always answer BETA") assert len(fake_handler.calls) == 2 assert first["content"][0]["text"] == "ALPHA" @@ -130,9 +130,9 @@ async def test_cache_key_separates_anthropic_native_params(local_cache, request_ fake_handler = _CountingHandler([_anthropic_response("msg_1", "ALPHA"), _anthropic_response("msg_2", "BETA")]) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - await litellm.anthropic_messages(**request_kwargs) + await handler.anthropic_messages(**request_kwargs) await asyncio.sleep(0) - await litellm.anthropic_messages(**request_kwargs, **anthropic_param) + await handler.anthropic_messages(**request_kwargs, **anthropic_param) assert len(fake_handler.calls) == 2 @@ -181,9 +181,9 @@ async def test_streaming_request_is_replayed_from_cache(local_cache, request_kwa fake_handler = _CountingHandler([_byte_stream(events), _byte_stream([b"event: never_used\n\n"])]) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - first = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + first = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) await asyncio.sleep(0) - second_stream = await litellm.anthropic_messages(**request_kwargs, stream=True) + second_stream = await handler.anthropic_messages(**request_kwargs, stream=True) second = await _collect(second_stream) assert len(fake_handler.calls) == 1 @@ -197,8 +197,8 @@ async def test_streaming_cache_is_not_shared_with_non_streaming(local_cache, req fake_handler = _CountingHandler([_byte_stream(STREAM_EVENTS), _anthropic_response("msg_2", "ALPHA")]) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) - non_streaming = await litellm.anthropic_messages(**request_kwargs) + await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) + non_streaming = await handler.anthropic_messages(**request_kwargs) assert len(fake_handler.calls) == 2 assert non_streaming["content"][0]["text"] == "ALPHA" @@ -219,11 +219,11 @@ async def test_stream_without_output_is_not_cached(local_cache, request_kwargs, ) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - empty = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + empty = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) await asyncio.sleep(0) - refilled = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + refilled = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) await asyncio.sleep(0) - replayed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + replayed = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) assert empty == empty_events assert refilled == STREAM_EVENTS @@ -239,8 +239,8 @@ async def test_failed_stream_is_not_cached(local_cache, request_kwargs, monkeypa fake_handler = _CountingHandler([_byte_stream(error_events), _byte_stream(STREAM_EVENTS)]) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - failed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) - replayed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + failed = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) + replayed = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) assert failed == error_events assert len(fake_handler.calls) == 2 @@ -260,9 +260,9 @@ async def test_multibyte_utf8_split_across_chunks_streams_and_caches(local_cache fake_handler = _CountingHandler([_byte_stream(chunks), _byte_stream([b"event: never_used\n\n"])]) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - first = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + first = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) await asyncio.sleep(0) - second = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + second = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) assert len(fake_handler.calls) == 1 assert first == chunks @@ -278,9 +278,9 @@ async def test_message_stop_split_across_chunks_still_caches(local_cache, reques fake_handler = _CountingHandler([_byte_stream(chunks), _byte_stream([b"event: never_used\n\n"])]) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - first = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + first = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) await asyncio.sleep(0) - second = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + second = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) assert len(fake_handler.calls) == 1 assert first == chunks @@ -296,8 +296,8 @@ async def test_error_event_split_across_chunks_is_not_cached(local_cache, reques fake_handler = _CountingHandler([_byte_stream(chunks), _byte_stream(STREAM_EVENTS)]) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - failed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) - replayed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + failed = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) + replayed = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) assert failed == chunks assert len(fake_handler.calls) == 2 @@ -309,11 +309,11 @@ async def test_abandoned_stream_is_not_cached(local_cache, request_kwargs, monke fake_handler = _CountingHandler([_byte_stream(STREAM_EVENTS), _byte_stream(STREAM_EVENTS)]) monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) - partial_stream = await litellm.anthropic_messages(**request_kwargs, stream=True) + partial_stream = await handler.anthropic_messages(**request_kwargs, stream=True) await partial_stream.__anext__() await partial_stream.aclose() - replayed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + replayed = await _collect(await handler.anthropic_messages(**request_kwargs, stream=True)) assert len(fake_handler.calls) == 2 assert replayed == STREAM_EVENTS @@ -488,7 +488,5 @@ def test_cache_writer_forwards_has_buffered_provider_output(request_kwargs): ) held_back = AnthropicMessagesStreamCacheWriter(stream=_HeldBackStream(), caching_handler=caching_handler) assert held_back.has_buffered_provider_output is True - replayable = AnthropicMessagesStreamCacheWriter( - stream=_byte_stream(STREAM_EVENTS), caching_handler=caching_handler - ) + replayable = AnthropicMessagesStreamCacheWriter(stream=_byte_stream(STREAM_EVENTS), caching_handler=caching_handler) assert replayable.has_buffered_provider_output is False diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 4e8dc69fd01..d105a7958b4 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -3266,23 +3266,6 @@ class TestWifLitellmParamsPlumbing: assert headers["authorization"] == f"Bearer {FAKE_MINTED_TOKEN}" assert len(poster.requests) == 1 - def test_messages_mints_from_litellm_params(self, wif_engine): - from litellm.llms.anthropic.messages.native.transformation import ( - AnthropicMessagesConfig, - ) - - poster, _ = wif_engine - headers, _ = AnthropicMessagesConfig().validate_anthropic_messages_environment( - headers={}, - model="claude-sonnet-4-5", - messages=[{"role": "user", "content": "Hello"}], - optional_params={}, - litellm_params=dict(WIF_PARAMS_ONLY), - ) - - assert headers["authorization"] == f"Bearer {FAKE_MINTED_TOKEN}" - assert len(poster.requests) == 1 - class TestWifTokenUrlParity: """Both credential tiers must derive the SAME clean token URL from any form of @@ -3344,27 +3327,6 @@ class TestWifAsyncSeam: assert poster.thread_ids == [poster.thread_ids[0]] assert poster.thread_ids[0] != threading.get_ident() - @pytest.mark.asyncio - async def test_avalidate_messages_environment_mints_off_loop(self, wif_async_engine, monkeypatch): - from litellm.llms.anthropic.messages.native.transformation import ( - AnthropicMessagesConfig, - ) - - poster, sync_calls = wif_async_engine - monkeypatch.setenv("WIF_PARAMS_TEST_TOKEN", "params-jwt") - - headers, _ = await AnthropicMessagesConfig().avalidate_anthropic_messages_environment( - headers={}, - model="claude-sonnet-4-5", - messages=[{"role": "user", "content": "Hello"}], - optional_params={}, - litellm_params=dict(WIF_PARAMS_ONLY), - ) - - assert headers["authorization"] == f"Bearer {FAKE_MINTED_TOKEN}" - assert sync_calls == [] - assert poster.thread_ids[0] != threading.get_ident() - @pytest.mark.asyncio async def test_avalidate_delegates_to_subclass_sync_override(self): """A provider subclass that only overrides the sync method must keep its diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index 6a946c2e75c..e07695fcd03 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -11,6 +11,7 @@ import respx from pydantic import JsonValue import litellm +from litellm.llms.anthropic.messages.native import handler as python_messages from litellm.caching.dual_cache import DualCache from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.anthropic.count_tokens import handler as count_handler @@ -165,6 +166,7 @@ async def test_environment_credential_matches_native_count_and_observed_scope( monkeypatch.setenv("LIT7658_PROVIDER_KEY", _KEY) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + monkeypatch.setattr(litellm, "anthropic_messages", python_messages.anthropic_messages) params: Final = { "model": f"anthropic/{_MODEL}", "api_key": "os.environ/LIT7658_PROVIDER_KEY", diff --git a/tests/unit/llms/anthropic/test_messages_http_error_retry_cases.py b/tests/unit/llms/anthropic/test_messages_http_error_retry_cases.py new file mode 100644 index 00000000000..a9edb2160f7 --- /dev/null +++ b/tests/unit/llms/anthropic/test_messages_http_error_retry_cases.py @@ -0,0 +1,188 @@ +import copy +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import httpx +import pytest +from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict + +import litellm +from litellm.llms.anthropic.common_utils import ( + ToolsWithoutStrict, + is_anthropic_invalid_thinking_block_error, + is_anthropic_thinking_binding_mismatch_error, + is_claude_strict_tools_rejection, + tools_without_strict, +) +from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig +from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig +from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig + +FIXTURE_PATH: Final = ( + Path(__file__).parents[4] / "litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json" +) + + +class _TextCase(TypedDict): + name: ReadOnly[str] + error_text: ReadOnly[str] + expected: ReadOnly[bool] + + +class _RetryCase(TypedDict): + name: ReadOnly[str] + status: ReadOnly[int] + error_text: ReadOnly[str] + thinking: ReadOnly[Mapping[str, object]] + retried: ReadOnly[bool] + stripped: ReadOnly[bool] + + +class _RetryTable(TypedDict): + request: ReadOnly[Mapping[str, object]] + stripped_request: ReadOnly[Mapping[str, object]] + cases: ReadOnly[tuple[_RetryCase, ...]] + + +class _StrictRejectionCase(TypedDict): + name: ReadOnly[str] + status: ReadOnly[int] + error_text: ReadOnly[str] + expected: ReadOnly[bool] + + +class _ToolsWithoutStrictExpected(TypedDict): + tools: ReadOnly[tuple[Mapping[str, object], ...]] + dropped_count: ReadOnly[int] + + +class _ToolsWithoutStrictCase(TypedDict): + name: ReadOnly[str] + tools: ReadOnly[object] + expected: ReadOnly[_ToolsWithoutStrictExpected | None] + + +class _StrictRetryCase(TypedDict): + name: ReadOnly[str] + status: ReadOnly[int] + error_text: ReadOnly[str] + thinking: ReadOnly[Mapping[str, object]] + drops_unsupported_params: ReadOnly[bool] + retried: ReadOnly[bool] + stripped: ReadOnly[bool] + strict_dropped: ReadOnly[bool] + + +class _StrictRetryTable(TypedDict): + tools: ReadOnly[tuple[Mapping[str, object], ...]] + tools_without_strict: ReadOnly[tuple[Mapping[str, object], ...]] + dropped: ReadOnly[int] + cases: ReadOnly[tuple[_StrictRetryCase, ...]] + + +class _Fixture(TypedDict): + invalid_thinking_block_error: ReadOnly[tuple[_TextCase, ...]] + thinking_binding_mismatch_error: ReadOnly[tuple[_TextCase, ...]] + http_error_retry: ReadOnly[_RetryTable] + claude_strict_tools_rejection: ReadOnly[tuple[_StrictRejectionCase, ...]] + tools_without_strict: ReadOnly[tuple[_ToolsWithoutStrictCase, ...]] + strict_tools_retry: ReadOnly[_StrictRetryTable] + + +FIXTURE: Final = TypeAdapter(_Fixture).validate_json(FIXTURE_PATH.read_text()) +RETRY: Final = FIXTURE["http_error_retry"] +STRICT_RETRY: Final = FIXTURE["strict_tools_retry"] + + +@pytest.mark.parametrize( + "case", [pytest.param(case, id=case["name"]) for case in FIXTURE["invalid_thinking_block_error"]] +) +def test_invalid_thinking_block_errors_match_the_shared_cases(case: _TextCase) -> None: + assert is_anthropic_invalid_thinking_block_error(case["error_text"]) is case["expected"] + + +@pytest.mark.parametrize( + "case", [pytest.param(case, id=case["name"]) for case in FIXTURE["thinking_binding_mismatch_error"]] +) +def test_binding_mismatch_errors_match_the_shared_cases(case: _TextCase) -> None: + assert is_anthropic_thinking_binding_mismatch_error(case["error_text"]) is case["expected"] + + +@pytest.mark.parametrize( + "case", [pytest.param(case, id=case["name"]) for case in FIXTURE["claude_strict_tools_rejection"]] +) +def test_strict_tools_rejections_match_the_shared_cases(case: _StrictRejectionCase) -> None: + assert is_claude_strict_tools_rejection(case["status"], case["error_text"]) is case["expected"] + + +@pytest.mark.parametrize("case", [pytest.param(case, id=case["name"]) for case in FIXTURE["tools_without_strict"]]) +def test_tools_without_strict_matches_the_shared_cases(case: _ToolsWithoutStrictCase) -> None: + expected: Final = case["expected"] + + assert tools_without_strict(copy.deepcopy(case["tools"])) == ( + None + if expected is None + else ToolsWithoutStrict(tools=tuple(expected["tools"]), dropped_count=expected["dropped_count"]) + ) + + +def _rejection(case: _RetryCase | _StrictRetryCase) -> httpx.HTTPStatusError: + request: Final = httpx.Request("POST", "https://api.anthropic.com/v1/messages") + return httpx.HTTPStatusError( + "rejected", + request=request, + response=httpx.Response(case["status"], text=case["error_text"], request=request), + ) + + +@pytest.mark.parametrize( + "config", + [AnthropicMessagesConfig(), DeepSeekAnthropicMessagesConfig()], + ids=["anthropic", "deepseek"], +) +@pytest.mark.parametrize("case", [pytest.param(case, id=case["name"]) for case in RETRY["cases"]]) +def test_the_http_error_retry_matches_the_shared_cases(config: BaseAnthropicMessagesConfig, case: _RetryCase) -> None: + body: Final = {**copy.deepcopy(RETRY["request"]), "thinking": case["thinking"]} + + retried: Final = config.should_retry_anthropic_messages_on_http_error( + e=_rejection(case), litellm_params={}, request_body=body + ) + transformed: Final = config.transform_anthropic_messages_request_on_http_error( + e=_rejection(case), request_data=copy.deepcopy(body) + ) + + assert retried is case["retried"] + assert transformed == (RETRY["stripped_request"] if case["stripped"] else body) + + +@pytest.mark.parametrize( + "config", + [AnthropicMessagesConfig(), DeepSeekAnthropicMessagesConfig()], + ids=["anthropic", "deepseek"], +) +@pytest.mark.parametrize("case", [pytest.param(case, id=case["name"]) for case in STRICT_RETRY["cases"]]) +def test_the_strict_tools_retry_matches_the_shared_cases( + monkeypatch: pytest.MonkeyPatch, config: BaseAnthropicMessagesConfig, case: _StrictRetryCase +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + body: Final = { + **copy.deepcopy(RETRY["request"]), + "thinking": case["thinking"], + "tools": copy.deepcopy(list(STRICT_RETRY["tools"])), + } + expected: Final = { + **copy.deepcopy(RETRY["stripped_request"] if case["stripped"] else body), + "tools": list(STRICT_RETRY["tools_without_strict"] if case["strict_dropped"] else STRICT_RETRY["tools"]), + } + + retried: Final = config.should_retry_anthropic_messages_on_http_error( + e=_rejection(case), litellm_params={"drop_params": case["drops_unsupported_params"]}, request_body=body + ) + transformed: Final = config.transform_anthropic_messages_request_on_http_error( + e=_rejection(case), request_data=copy.deepcopy(body) + ) + + assert retried is case["retried"] + assert transformed == expected diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 991a0c36f03..4d4a92f632e 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -4736,7 +4736,10 @@ async def test_bedrock_messages_fallback_to_anthropic_sends_the_tool_names_bedro unnamed_tool_extras: dict[str, object], sent_extras: dict[str, object], ) -> None: + from litellm.llms.anthropic.messages.native import handler as python_messages + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "anthropic_messages", python_messages.anthropic_messages) upstream: Final = BedrockDownAnthropicUp() respx_mock.post(url__regex=r"https://.*").mock(side_effect=upstream) router: Final = litellm.Router( diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 4d8ba26d8d8..08f38352735 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -19,6 +19,7 @@ from botocore.credentials import RefreshableCredentials from h2.events import ConnectionTerminated import litellm +from litellm.llms.anthropic.messages.native import handler as python_messages from litellm._logging import verbose_logger from litellm.integrations.code_interpreter_interception.handler import ( CodeInterpreterInterceptionLogger, @@ -6493,7 +6494,7 @@ async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_ upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(anthropic_answers)) try: - await litellm.anthropic.messages.acreate( + await python_messages.anthropic_messages( max_tokens=16, messages=[{"role": "user", "content": "hi"}], model="anthropic/claude-sonnet-4-6", @@ -6546,7 +6547,6 @@ async def _search_through(transport: httpx.MockTransport, is_async: bool) -> obj @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) -@pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) @pytest.mark.parametrize( ("api_base", "explicit_header", "expected_session"), [ @@ -6557,7 +6557,6 @@ async def _search_through(transport: httpx.MockTransport, is_async: bool) -> obj ) async def test_messages_opencode_session_header( stream: bool, - environment_base: str | None, api_base: str, explicit_header: str | None, expected_session: str | None, @@ -6565,8 +6564,6 @@ async def test_messages_opencode_session_header( ) -> None: monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) - if environment_base: - monkeypatch.setenv(environment_base, "https://opencode.ai/inference/go/anthropic") received: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() payload: Final = { "id": "msg_test", @@ -6594,11 +6591,11 @@ async def test_messages_opencode_session_header( upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) try: - result: Final = await litellm.anthropic.messages.acreate( + result: Final = await python_messages.anthropic_messages( model="anthropic/claude-haiku-5-5", messages=[{"role": "user", "content": "hello"}], max_tokens=128000, - api_base=None if environment_base and api_base == "https://opencode.ai/inference/go/anthropic" else api_base, + api_base=api_base, api_key="test-key", stream=stream, litellm_session_id="conversation-123", @@ -6737,20 +6734,6 @@ def _anthropic_messages_logging_obj(): @pytest.mark.parametrize( ("custom_llm_provider", "messages_config", "foundry_api_base", "forwarded_betas"), [ - pytest.param( - "anthropic", - "anthropic", - "https://my-resource.services.ai.azure.com/anthropic", - f"thinking-binding-controls-2026-08-01,{_BETA_THE_TABLE_DOES_NOT_LIST}", - id="anthropic-drops-what-the-azure-ai-row-rejects", - ), - pytest.param( - "anthropic", - "anthropic", - "https://my-resource.cognitiveservices.azure.com/anthropic", - f"thinking-binding-controls-2026-08-01,{_BETA_THE_TABLE_DOES_NOT_LIST}", - id="anthropic-on-a-cognitiveservices-host", - ), pytest.param( "azure_ai", "azure_ai", @@ -6775,7 +6758,6 @@ async def test_async_anthropic_messages_handler_picks_the_beta_row_by_deployment forwarded_betas: str, ) -> None: from litellm import anthropic_beta_headers_manager - from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig @@ -6783,7 +6765,6 @@ async def test_async_anthropic_messages_handler_picks_the_beta_row_by_deployment monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None) assert _BETA_THE_TABLE_DOES_NOT_LIST not in anthropic_beta_headers_manager._load_beta_headers_config()["azure_ai"] configs: Final = { - "anthropic": AnthropicMessagesConfig, "azure_ai": AzureAnthropicMessagesConfig, "pass_through": OpenAILikeAnthropicMessagesConfig, } @@ -6817,7 +6798,7 @@ async def test_first_foundry_request_with_betas_fetches_the_beta_table_off_the_e monkeypatch: pytest.MonkeyPatch, ) -> None: from litellm import anthropic_beta_headers_manager, anthropic_beta_headers_url - from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig + from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig monkeypatch.delenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", raising=False) monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False) @@ -6835,9 +6816,9 @@ async def test_first_foundry_request_with_betas_fetches_the_beta_table_off_the_e await BaseLLMHTTPHandler().async_anthropic_messages_handler( model="claude-sonnet-4-6", messages=[{"role": "user", "content": "Hello"}], - anthropic_messages_provider_config=AnthropicMessagesConfig(), + anthropic_messages_provider_config=AzureAnthropicMessagesConfig(), anthropic_messages_optional_request_params={"max_tokens": 8}, - custom_llm_provider="anthropic", + custom_llm_provider="azure_ai", litellm_params=GenericLiteLLMParams(api_base=foundry_api_base), logging_obj=_anthropic_messages_logging_obj(), api_key="sk-test", diff --git a/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py b/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py index 12aa850afb6..86a7e03db84 100644 --- a/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py +++ b/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py @@ -1150,3 +1150,10 @@ async def test_a_turn_routed_to_a_non_anthropic_tier_says_its_provider_is_unsupp captured: Final = _observation(await rig.capture.payload()) assert captured.observation.outcome == "uncertain" assert captured.observation.reason == "unsupported_selected_provider" + + +@pytest.fixture(autouse=True) +def _python_messages(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.messages.native import handler as python_messages + + monkeypatch.setattr(litellm, "anthropic_messages", python_messages.anthropic_messages) diff --git a/tests/unit/responses/mcp/test_tool_precheck.py b/tests/unit/responses/mcp/test_tool_precheck.py index bd37114ccea..04670f6b4ed 100644 --- a/tests/unit/responses/mcp/test_tool_precheck.py +++ b/tests/unit/responses/mcp/test_tool_precheck.py @@ -36,16 +36,6 @@ ANTHROPIC_FUNCTION_TOOL: Final = { "description": "No operation", "input_schema": {"type": "object", "properties": {}}, } -ANTHROPIC_MESSAGE_PAYLOAD: Final = { - "id": "msg_1", - "type": "message", - "role": "assistant", - "model": "claude-sonnet-5-5", - "content": [{"type": "text", "text": "ok"}], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": {"input_tokens": 1, "output_tokens": 1}, -} GATEWAY_MCP_TOOL: Final = {"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy"} GATEWAY_SERVER_MCP_TOOL: Final = {"type": "mcp", "server_label": "x", "server_url": "litellm_proxy/mcp/github"} GATEWAY_PATH_MCP_TOOL: Final = {"type": "mcp", "server_label": "x", "server_url": "https://gw.example.com/mcp/github"} @@ -193,23 +183,16 @@ def test_responses_with_gateway_mcp_tool_still_reaches_the_handler(): @pytest.mark.asyncio @pytest.mark.usefixtures("gateway_handler_unavailable") -async def test_anthropic_messages_with_function_tool_does_not_import_the_gateway_handler( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter -): - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - upstream: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( - return_value=httpx.Response(200, json=ANTHROPIC_MESSAGE_PAYLOAD) - ) - +async def test_anthropic_messages_with_function_tool_does_not_import_the_gateway_handler(): response: Final = await litellm.anthropic_interface.messages.acreate( model="anthropic/claude-sonnet-5-5", messages=[{"role": "user", "content": "hi"}], max_tokens=16, tools=[ANTHROPIC_FUNCTION_TOOL], api_key="sk-test", + mock_response="ok", ) - assert upstream.called assert response["content"][0]["text"] == "ok" diff --git a/tests/unit/router_strategy/complexity_router/test_context_compaction.py b/tests/unit/router_strategy/complexity_router/test_context_compaction.py index 850accf574b..3fed862ae50 100644 --- a/tests/unit/router_strategy/complexity_router/test_context_compaction.py +++ b/tests/unit/router_strategy/complexity_router/test_context_compaction.py @@ -503,3 +503,19 @@ async def test_retry_reuses_summary_or_terminal_cancellation(outcome: Literal["s assert calls.qsize() == 1 and stopped.is_set() finally: compaction_executor.reset(token) + + +@pytest.fixture(autouse=True) +def _python_messages(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.rust_bridge import catalog + from litellm.rust_bridge.configuration import Rollout + + monkeypatch.setattr( + catalog, + "RULES", + tuple( + rule + for rule in catalog.RULES + if not (rule.route is catalog.Route.MESSAGES and rule.rollout is Rollout.RUST_REQUIRED) + ), + ) diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 054c48bbb5e..0190a7e929d 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -15783,6 +15783,21 @@ class TestModalityRouting: @pytest.mark.usefixtures("local_model_cost_map") class TestHealthFallbackDispatch: + @pytest.fixture(autouse=True) + def _python_messages(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.rust_bridge import catalog + from litellm.rust_bridge.configuration import Rollout + + monkeypatch.setattr( + catalog, + "RULES", + tuple( + rule + for rule in catalog.RULES + if not (rule.route is catalog.Route.MESSAGES and rule.rollout is Rollout.RUST_REQUIRED) + ), + ) + @pytest.mark.asyncio @pytest.mark.parametrize("classifier", ("capability", "llm_v2")) @pytest.mark.parametrize("calibrated", (False, True), ids=("raw", "calibrated")) diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 47d58a3b869..1bfe88275c2 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -2630,3 +2630,19 @@ async def test_run_async_fallback_never_re_reserves_the_requested_model_group() with pytest.raises(Exception, match="model group 'primary' failed"): await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}]) assert checked == ["fallback"] + + +@pytest.fixture(autouse=True) +def _python_messages(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.rust_bridge import catalog + from litellm.rust_bridge.configuration import Rollout + + monkeypatch.setattr( + catalog, + "RULES", + tuple( + rule + for rule in catalog.RULES + if not (rule.route is catalog.Route.MESSAGES and rule.rollout is Rollout.RUST_REQUIRED) + ), + ) diff --git a/tests/unit/rust_bridge/host/routes/test_messages.py b/tests/unit/rust_bridge/host/routes/test_messages.py index aa7dcc74841..ed60951002d 100644 --- a/tests/unit/rust_bridge/host/routes/test_messages.py +++ b/tests/unit/rust_bridge/host/routes/test_messages.py @@ -1,10 +1,29 @@ +import datetime +import functools +import json +from collections.abc import Iterator, Mapping, Sequence from types import MappingProxyType from typing import Final +import httpx import pytest +from pydantic import ValidationError import litellm +from litellm.caching.llm_caching_handler import LLMClientCache from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.model_latest_aliases import get_model_latest_aliases +from litellm.llms.anthropic.messages.native import interceptors, mcp_handler +from litellm.llms.anthropic.messages.native.interceptors.base import MessagesInterceptor +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + exclude_rsa_key_exchange_and_chacha20, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # its params dict is untyped +) +from litellm.rust_bridge.host import callbacks_legacy_python from litellm.rust_bridge.host.routes import messages as messages_host from litellm.rust_bridge.host.routes.messages import response @@ -30,11 +49,26 @@ def test_response_is_a_detached_public_messages_dict() -> None: assert "_hidden_params" not in native +@pytest.mark.parametrize( + ("disabled", "joined"), + [pytest.param("true", False, id="httpx_transport"), pytest.param("false", True, id="aiohttp_transport")], +) +def test_settings_join_buffered_stream_reads_only_on_the_aiohttp_transport( + monkeypatch: pytest.MonkeyPatch, disabled: str, joined: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", disabled) + + assert messages_host.settings({})["joined_reads"] is joined + + def test_settings_project_caller_configuration_without_resolving_a_model(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "drop_params", False) monkeypatch.setattr(litellm, "reasoning_auto_summary", True) monkeypatch.setattr(litellm, "request_timeout", 1800) monkeypatch.setattr(litellm, "request_timeout_explicitly_set", True) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False) projected: Final = messages_host.settings({"drop_params": "true", "additional_drop_params": ["metadata.user_id"]}) @@ -43,9 +77,44 @@ def test_settings_project_caller_configuration_without_resolving_a_model(monkeyp "reasoning_auto_summary": True, "additional_drop_params": ("metadata.user_id",), "request_timeout": 1800.0, + "drops_unsupported_params": True, + "model": None, + "mock_response": None, + "stream_cost_model": None, + "extra_headers": None, + "mcp_gateway": False, + "ascii_response_headers": True, + "joined_reads": True, + "ascii_request_headers": True, } +@pytest.mark.parametrize( + ("global_drop_params", "kwargs", "drop_params", "drops_unsupported_params"), + [ + pytest.param(True, {}, True, True, id="global-true"), + pytest.param(1, {}, True, False, id="global-truthy-non-bool"), + pytest.param(False, {"drop_params": "True"}, True, True, id="kwarg-flag-string"), + pytest.param(False, {"drop_params": 1}, True, True, id="kwarg-one"), + pytest.param(False, {"drop_params": "maybe"}, False, False, id="kwarg-not-a-flag"), + pytest.param(False, {}, False, False, id="unset"), + ], +) +def test_strict_tools_retry_reads_drop_params_as_the_python_retry_check_does( + monkeypatch: pytest.MonkeyPatch, + global_drop_params: object, + kwargs: Mapping[str, object], + drop_params: bool, + drops_unsupported_params: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + + projected: Final = messages_host.settings(kwargs) + + assert projected["drop_params"] is drop_params + assert projected["drops_unsupported_params"] is drops_unsupported_params + + @pytest.mark.parametrize( ("timeout", "explicitly_set", "expected"), [ @@ -170,7 +239,7 @@ def test_body_params_round_effort_to_the_levels_the_deployment_declares( "model_info": {"id": "deployment-1", "reasoning_effort_levels": levels}, } - assert messages_host.body_params(request)["output_config"] == {"effort": expected} + assert messages_host.body_params(request, asynchronous=True)["output_config"] == {"effort": expected} def test_body_params_keep_the_requested_effort_without_deployment_levels() -> None: @@ -181,7 +250,7 @@ def test_body_params_keep_the_requested_effort_without_deployment_levels() -> No "output_config": {"effort": "max"}, } - assert messages_host.body_params(request)["output_config"] == {"effort": "max"} + assert messages_host.body_params(request, asynchronous=True)["output_config"] == {"effort": "max"} @pytest.mark.parametrize( @@ -203,7 +272,28 @@ def test_body_params_derive_no_effort_from_a_param_the_deployment_lists( "model_info": {"id": "deployment-1", "reasoning_effort_levels": ["low", "medium", "high"]}, } - assert set(messages_host.body_params(request)).difference({"messages", "model"}) == expected_keys + assert set(messages_host.body_params(request, asynchronous=True)).difference({"messages", "model"}) == expected_keys + + +@pytest.mark.parametrize( + ("thread", "expected"), + [ + ({"type": "create"}, {"type": "create"}), + ({"type": "continue", "previous_message_id": "msg_1"}, {"type": "continue", "previous_message_id": "msg_1"}), + (None, None), + ], +) +def test_body_params_forward_the_thread_like_the_python_handler( + thread: dict[str, str] | None, expected: dict[str, str] | None +) -> None: + request: Final[dict[str, object]] = { + "model": "claude-sonnet-4-5", + "messages": [], + "max_tokens": 8, + "thread": thread, + } + + assert messages_host.body_params(request, asynchronous=True).get("thread") == expected def _marked(phase: str | None = None, *args: object) -> ValueError: @@ -213,13 +303,40 @@ def _marked(phase: str | None = None, *args: object) -> ValueError: return error -def test_a_stalled_stream_body_reaches_the_caller_as_the_transport_timeout() -> None: +@pytest.mark.parametrize( + ("disabled", "text"), ((None, "Timeout on reading data from socket"), ("true", "")), ids=("aiohttp", "httpx") +) +def test_a_stalled_stream_body_reaches_the_caller_as_the_transport_timeout( + monkeypatch: pytest.MonkeyPatch, disabled: str | None, text: str +) -> None: import httpx + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + if disabled is None: + monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False) + else: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", disabled) + mapped: Final = messages_host.map_failure(_marked("Body"), {"model": _ADAPTIVE_OPUS}, "anthropic") assert isinstance(mapped, httpx.ReadTimeout) - assert str(mapped) == "Timeout on reading data from socket" + assert str(mapped) == text + + +@pytest.mark.parametrize( + ("value", "received", "text"), + ( + ("h\ud800".encode("utf-8", "surrogatepass"), False, "can't encode character '\\ud800' in position 1"), + ("caf\u00e9".encode(), True, "can't encode character '\\xe9' in position 3"), + (b"caf\xe9", True, "can't encode character '\\udce9' in position 3"), + ), + ids=("sent-surrogate", "received-utf-8", "received-latin-1"), +) +def test_an_unencodable_header_is_the_error_pythons_ascii_codec_raises(value: bytes, received: bool, text: str) -> None: + error: Final = messages_host.unencodable_header(value, received) + + assert isinstance(error, UnicodeEncodeError) + assert text in str(error) def test_a_native_read_timeout_maps_to_the_public_timeout_with_python_text() -> None: @@ -365,12 +482,47 @@ def test_a_stream_body_cut_reaches_the_caller_as_the_transport_raised_it( native: Final = RuntimeError("upstream network error") native.messages_body_cut = cut # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets - mapped: Final = messages_host.map_failure(native, {"model": _ADAPTIVE_OPUS}, "anthropic") + mapped: Final = messages_host.map_failure(native, {"model": _ADAPTIVE_OPUS, "stream": True}, "anthropic") assert type(mapped) is getattr(httpx, error_class) assert str(mapped) == text +@pytest.mark.parametrize( + ("aiohttp", "cut", "text"), + ( + ( + True, + (19, 100, False), + "Response payload is not completed: ", + ), + ( + False, + (19, 100, False), + "peer closed connection without sending complete message body (received 19 bytes, expected 100)", + ), + ), +) +def test_a_complete_body_cut_is_a_synthesized_server_error( + monkeypatch: pytest.MonkeyPatch, + aiohttp: bool, + cut: tuple[int, int | None, bool], + text: str, +) -> None: + from litellm.rust_bridge.host import http_transport + + monkeypatch.setattr(http_transport, "uses_aiohttp_transport", lambda: aiohttp) + native: Final = RuntimeError("upstream network error") + native.messages_body_cut = cut # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + + mapped: Final = messages_host.map_failure(native, {"model": _ADAPTIVE_OPUS}, "anthropic") + + assert type(mapped) is litellm.InternalServerError + assert mapped.status_code == 500 + assert text in str(mapped) + + _TRANSPORT: Final = "error sending request: client error" @@ -445,3 +597,1153 @@ def test_a_failure_before_any_response_maps_like_the_python_transport_error( f"litellm.InternalServerError: AnthropicException - {text}. Handle with `litellm.InternalServerError`." ) assert "This looks like a bug in LiteLLM" not in str(mapped) + + +_CALL: Final = MappingProxyType({"messages": [{"role": "user", "content": "hi"}], "max_tokens": 8}) +_OPENCODE: Final = "https://opencode.ai/inference/go/anthropic/v1" +_COST_HEADER: Final = MappingProxyType({"headers": {"x-litellm-include-cost-in-streaming-usage": "true"}}) + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected"), + [ + ( + "anthropic/claude-opus-latest", + None, + f"anthropic/{get_model_latest_aliases()['anthropic']['claude-opus-latest']}", + ), + ("claude-opus-latest", "anthropic", get_model_latest_aliases()["anthropic"]["claude-opus-latest"]), + ("anthropic/claude-sonnet-4-5", None, "anthropic/claude-sonnet-4-5"), + ], +) +def test_settings_resolve_a_latest_alias_keeping_the_provider_prefix( + model: str, custom_llm_provider: str | None, expected: str +) -> None: + settings: Final = messages_host.settings({}, {**_CALL, "model": model, "custom_llm_provider": custom_llm_provider}) + + assert settings["model"] == expected + + +@pytest.mark.parametrize( + ("extra", "expected"), + [ + ({"api_base": _OPENCODE, "session_id": "s-1"}, {"x-opencode-session": "s-1"}), + ( + {"api_base": _OPENCODE, "session_id": "s-1", "headers": {"x-trace": "t"}}, + {"x-trace": "t", "x-opencode-session": "s-1"}, + ), + ({"api_base": _OPENCODE, "session_id": "s-1", "extra_headers": {"x-opencode-session": "mine"}}, None), + ({"api_base": "https://other.example/v1", "session_id": "s-1"}, None), + ({"api_base": _OPENCODE, "session_id": "s-1", "mock_response": "hi"}, None), + ], +) +def test_settings_add_only_the_affinity_header_the_caller_did_not_send( + extra: dict[str, object], expected: dict[str, str] | None +) -> None: + settings: Final = messages_host.settings({}, {**_CALL, "model": "anthropic/claude-sonnet-4-5", **extra}) + + assert settings["extra_headers"] == expected + + +@pytest.mark.parametrize("variable", ["ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) +def test_settings_add_the_affinity_header_for_an_anthropic_base_from_the_environment( + monkeypatch: pytest.MonkeyPatch, variable: str +) -> None: + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.setenv(variable, _OPENCODE) + + settings: Final = messages_host.settings({}, {**_CALL, "model": "anthropic/claude-sonnet-4-5", "session_id": "s-1"}) + + assert settings["extra_headers"] == {"x-opencode-session": "s-1"} + + +@pytest.mark.parametrize( + ("extra", "expected"), + [ + ({"stream": True, "proxy_server_request": dict(_COST_HEADER)}, "claude-sonnet-4-5"), + ({"proxy_server_request": dict(_COST_HEADER)}, None), + ({"stream": True}, None), + ({"stream": True, "proxy_server_request": dict(_COST_HEADER), "mock_response": "hi"}, None), + ], +) +def test_settings_name_the_cost_model_only_for_an_opted_in_stream( + extra: dict[str, object], expected: str | None +) -> None: + settings: Final = messages_host.settings({}, {**_CALL, "model": "anthropic/claude-sonnet-4-5", **extra}) + + assert settings["stream_cost_model"] == expected + + +def test_settings_carry_the_mock_the_python_handler_returns() -> None: + settings: Final = messages_host.settings( + {}, {**_CALL, "model": "anthropic/claude-sonnet-4-5", "mock_response": "hi"} + ) + + assert isinstance(settings["mock_response"], dict) + assert settings["mock_response"]["content"] == [{"type": "text", "text": "hi"}] + + +def test_settings_infer_no_provider_off_an_affinity_host(monkeypatch: pytest.MonkeyPatch) -> None: + def unexpected(model: str, api_base: str | None) -> str | None: + pytest.fail("the provider is only inferred for an affinity api_base") + + monkeypatch.setattr(messages_host, "inferred_provider", unexpected) + + settings: Final = messages_host.settings({}, {**_CALL, "model": "claude-sonnet-4-5"}) + + assert settings["model"] == "claude-sonnet-4-5" + assert settings["extra_headers"] is None + + +def test_body_params_leave_the_messages_to_the_route_for_a_call_without_prompt_caching( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False) + + fields: Final = messages_host.body_params( + { + "model": "anthropic/claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hi"}], + "system": "Be terse.", + "enable_prompt_caching": False, + "cache_control_injection_points": [], + }, + asynchronous=True, + ) + + assert fields == {"system": "Be terse."} + + +@pytest.mark.parametrize("asynchronous", [True, False]) +def test_body_params_mark_the_turns_tools_and_deployment_the_handler_marks( + monkeypatch: pytest.MonkeyPatch, asynchronous: bool +) -> None: + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False) + gateway: Final[dict[str, object]] = {} + caller_metadata: Final = {"user_id": "u"} + tool: Final = {"name": "get_weather", "input_schema": {"type": "object"}} + + fields: Final = messages_host.body_params( + { + "model": "anthropic/claude-sonnet-4-5", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "text", "text": ""}]}, + {"role": "user", "content": "again"}, + ], + "system": "Be terse.", + "tools": [tool], + "metadata": caller_metadata, + "litellm_metadata": gateway, + "model_info": {"id": "deployment-1"}, + "cache_control_injection_points": [ + {"location": "message", "role": "system"}, + {"location": "message", "index": -1, "control": {"type": "ephemeral", "ttl": "1h"}}, + {"location": "tool_config"}, + ], + }, + asynchronous=asynchronous, + ) + + assert fields == { + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + { + "role": "user", + "content": [{"type": "text", "text": "again", "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + }, + ], + "system": [{"type": "text", "text": "Be terse.", "cache_control": {"type": "ephemeral"}}], + "tools": [{**tool, "cache_control": {"type": "ephemeral"}}], + "metadata": caller_metadata, + } + assert gateway == {"litellm_gateway_injected_cache": "deployment-1"} + assert caller_metadata == {"user_id": "u"} + + +@pytest.mark.parametrize( + ("asynchronous", "expected_system"), + [ + (True, [{"type": "text", "text": "Be terse.", "cache_control": {"type": "ephemeral"}}]), + (False, "Be terse."), + ], +) +def test_body_params_prepare_prompt_caching_as_the_entry_that_ran( + monkeypatch: pytest.MonkeyPatch, asynchronous: bool, expected_system: object +) -> None: + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + + fields: Final = messages_host.body_params( + { + "model": "anthropic/claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hi"}], + "system": "Be terse.", + "cache_control_injection_points": [{"location": "message", "role": "assistant"}], + }, + asynchronous=asynchronous, + ) + + assert fields["system"] == expected_system + + +@pytest.mark.parametrize( + ("modify_params", "args", "kwargs", "expected"), + [ + pytest.param(True, (), {"max_tokens": 200_000}, 64_000, id="clamped-to-model-output-limit"), + pytest.param(True, (), {"max_tokens": 1}, 1, id="under-the-limit"), + pytest.param(False, (), {"max_tokens": 200_000}, 200_000, id="modify-params-off"), + pytest.param("true", (), {"max_tokens": 200_000}, 200_000, id="truthy-but-not-true"), + pytest.param(True, (200_000,), {"max_tokens": 200_000}, 200_000, id="positional-model-is-not-a-model"), + ], +) +def test_modify_max_tokens_clamps_the_keyword_view_as_the_client_wrapper_does( + monkeypatch: pytest.MonkeyPatch, + modify_params: object, + args: tuple[object, ...], + kwargs: dict[str, object], + expected: int, +) -> None: + monkeypatch.setattr(litellm, "modify_params", modify_params) + model: Final = args[0] if args else "anthropic/claude-sonnet-4-5" + view: Final = {**kwargs, "messages": [{"role": "user", "content": "hi"}]} + + messages_host.modify_max_tokens(model, args, view) + + assert view["max_tokens"] == expected + + +def test_workload_identity_token_mints_with_the_params_python_validates(monkeypatch: pytest.MonkeyPatch) -> None: + minted: Final[list[tuple[dict[str, object], str | None, str]]] = [] + + def mint(litellm_params: dict[str, object], api_base: str | None, model: str) -> str: + federation = {name: value for name, value in litellm_params.items() if name.startswith("anthropic_")} + minted.append(({name: value for name, value in federation.items() if value is not None}, api_base, model)) + return "sk-ant-oat01-minted" + + monkeypatch.setattr(messages_host, "get_anthropic_wif_token", mint) + + token: Final = messages_host.workload_identity_token( + {"anthropic_federation_rule_id": "fdrl_1", "anthropic_issuer_ttl_seconds": "60"}, + "https://gateway.test", + ) + + assert (token, minted) == ( + "sk-ant-oat01-minted", + [({"anthropic_federation_rule_id": "fdrl_1", "anthropic_issuer_ttl_seconds": 60}, "https://gateway.test", "")], + ) + + +def test_workload_identity_token_is_none_without_a_federation_rule(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("ANTHROPIC_FEDERATION_RULE_ID", "ANTHROPIC_ORGANIZATION_ID", "ANTHROPIC_IDENTITY_TOKEN"): + monkeypatch.delenv(name, raising=False) + + assert messages_host.workload_identity_token({}, None) is None + + +def test_workload_identity_fields_are_the_federation_litellm_params() -> None: + assert set(messages_host.WORKLOAD_IDENTITY_FIELDS) >= { + "anthropic_federation_rule_id", + "anthropic_organization_id", + "anthropic_disable_workload_identity_federation", + } + + +class _Handler(AsyncHTTPHandler): + def __init__(self, answer: httpx.Response | Exception) -> None: + super().__init__() # pyright: ignore[reportUnknownMemberType] # the handler's untyped concurrent_limit + self.answer: Final = answer + self.posted: list[Mapping[str, object]] = [] + + async def post(self, url: str, **kwargs: object) -> httpx.Response: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: records the post's keywords + self.posted.append({"url": url, **kwargs}) + if isinstance(self.answer, Exception): + raise self.answer + return self.answer + + +_URL: Final = "https://api.anthropic.com/v1/messages" +_REQUEST: Final = httpx.Request("POST", _URL) + + +@pytest.fixture +def no_tls_settings(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "ssl_security_level", None) + monkeypatch.setattr(litellm, "ssl_ecdh_curve", None) + + +@pytest.mark.usefixtures("no_tls_settings") +@pytest.mark.parametrize( + "client", (None, object(), httpx.AsyncClient(), HTTPHandler()), ids=("none", "object", "httpx", "sync-handler") +) +def test_the_native_client_stands_in_for_any_client_python_ignores(client: object) -> None: + assert messages_host.transport_handler({"client": client}) is None + + +@pytest.mark.usefixtures("no_tls_settings") +def test_a_caller_async_handler_is_the_handler_python_posts_through() -> None: + client: Final = AsyncHTTPHandler() + + assert messages_host.transport_handler({"client": client}) is client + + +@pytest.mark.usefixtures("no_tls_settings") +@pytest.mark.parametrize( + "setting", ({"ssl_security_level": "DEFAULT@SECLEVEL=1"}, {"ssl_ecdh_curve": "secp521r1"}), ids=("level", "curve") +) +def test_tls_the_native_client_cannot_apply_posts_through_pythons_anthropic_handler( + monkeypatch: pytest.MonkeyPatch, setting: Mapping[str, str] +) -> None: + for name, value in setting.items(): + monkeypatch.setattr(litellm, name, value) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + handler: Final = messages_host.transport_handler({"client": object()}) + + assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) + assert callbacks_legacy_python.times_provider_exchange(object(), None) is False + + +@pytest.fixture +def fips_cipher_exclusion() -> Iterator[None]: + exclude_rsa_key_exchange_and_chacha20(True) + yield + exclude_rsa_key_exchange_and_chacha20(False) + + +@pytest.mark.usefixtures("no_tls_settings", "fips_cipher_exclusion") +def test_fips_cipher_exclusion_posts_through_pythons_anthropic_handler(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + handler: Final = messages_host.transport_handler({"client": None}) + + assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) + assert callbacks_legacy_python.times_provider_exchange(None, None) is False + + +@pytest.mark.usefixtures("no_tls_settings") +def test_only_a_native_post_is_timed_by_the_native_call() -> None: + assert callbacks_legacy_python.times_provider_exchange(None, None) is True + assert callbacks_legacy_python.times_provider_exchange(None, 30.0) is True + assert callbacks_legacy_python.times_provider_exchange(AsyncHTTPHandler(), None) is False + + +@pytest.mark.usefixtures("no_tls_settings") +@pytest.mark.parametrize( + "timeout", + (-1.0, 0, -0.5, httpx.Timeout(10.0, connect=0)), + ids=("negative", "zero", "fraction", "httpx-zero-connect"), +) +def test_a_non_positive_timeout_posts_through_pythons_anthropic_handler( + monkeypatch: pytest.MonkeyPatch, timeout: object +) -> None: + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + handler: Final = messages_host.transport_handler({"client": None, "timeout": timeout}) + + assert handler is get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) + assert callbacks_legacy_python.times_provider_exchange(None, timeout) is False + + +@pytest.mark.usefixtures("no_tls_settings") +@pytest.mark.parametrize("timeout", (None, 0.5, httpx.Timeout(10.0)), ids=("unset", "positive", "httpx")) +def test_a_positive_timeout_keeps_the_native_client(timeout: object) -> None: + assert messages_host.transport_handler({"client": None, "timeout": timeout}) is None + + +@pytest.mark.asyncio +async def test_transport_send_posts_what_the_python_handler_posts() -> None: + logger: Final = object() + timeout: Final = httpx.Timeout(30.0, connect=2.0) + handler: Final = _Handler( + httpx.Response(200, json={"id": "msg_1"}, headers={"request-id": "req_1"}, request=_REQUEST) + ) + + sent: Final = await messages_host.transport_send( + handler, + {"litellm_logging_obj": logger, "timeout": timeout}, + _URL, + [("x-api-key", "sk-ant"), ("anthropic-version", "2023-06-01")], + {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "héllo"}]}, + False, + ) + + assert handler.posted == [ + { + "url": _URL, + "headers": {"x-api-key": "sk-ant", "anthropic-version": "2023-06-01"}, + "data": json.dumps({"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "héllo"}]}), + "stream": False, + "logging_obj": logger, + "timeout": timeout, + } + ] + assert (sent.status, sent.text, sent.chunks, sent.rejection) == (200, '{"id":"msg_1"}', None, None) + assert ("request-id", "req_1") in sent.headers + + +@pytest.mark.asyncio +async def test_transport_send_hands_signed_bytes_on_unchanged() -> None: + handler: Final = _Handler(httpx.Response(200, json={}, request=_REQUEST)) + + await messages_host.transport_send(handler, {}, _URL, [], b'{"signed":true}', False) + + assert handler.posted[0]["data"] == b'{"signed":true}' + assert handler.posted[0]["timeout"] is None + + +@pytest.mark.asyncio +async def test_transport_send_leaves_a_stream_for_the_route_to_read() -> None: + handler: Final = _Handler( + httpx.Response(200, stream=httpx.ByteStream(b"event: message_stop\n\n"), request=_REQUEST) + ) + + sent: Final = await messages_host.transport_send(handler, {}, _URL, [], {}, True) + + assert sent.chunks is not None + assert sent.text is None + assert [chunk async for chunk in sent.chunks] == [b"event: message_stop\n\n"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("checked", (True, False), ids=("handler-raised", "status-checked")) +async def test_transport_send_reports_a_rejection_with_the_status_error(checked: bool) -> None: + rejected: Final = httpx.Response(429, text="slow down", headers={"retry-after": "7"}, request=_REQUEST) + error: Final = httpx.HTTPStatusError("rate limited", request=_REQUEST, response=rejected) + handler: Final = _Handler(error if checked else rejected) + + sent: Final = await messages_host.transport_send(handler, {}, _URL, [], {}, False) + + assert (sent.status, sent.text) == (429, "slow down") + assert ("retry-after", "7") in sent.headers + assert isinstance(sent.rejection, httpx.HTTPStatusError) + assert sent.rejection.response is rejected + + +@pytest.mark.asyncio +async def test_a_rejection_python_cannot_read_reaches_the_caller_as_raised() -> None: + unread: Final = httpx.Response(500, stream=httpx.ByteStream(b"unread"), request=_REQUEST) + + with pytest.raises(messages_host.RawTransportFailure) as raised: + await messages_host.transport_send(_Handler(unread), {}, _URL, [], {}, True) + + assert isinstance(raised.value.error, httpx.ResponseNotRead) + assert isinstance(raised.value.error.__context__, httpx.HTTPStatusError) + + +@pytest.mark.asyncio +async def test_a_handler_failure_is_raised_for_the_route_to_map() -> None: + failure: Final = httpx.ConnectError("refused") + + with pytest.raises(httpx.ConnectError) as raised: + await messages_host.transport_send(_Handler(failure), {}, _URL, [], {}, False) + + assert raised.value is failure + + +@pytest.mark.parametrize( + ("error", "expected", "status"), + ( + pytest.param( + litellm.Timeout( + message="Read timed out.", model="default-model-name", llm_provider="litellm-httpx-handler" + ), + litellm.Timeout, + 408, + id="timeout", + ), + pytest.param( + httpx.ConnectError("All connection attempts failed"), litellm.InternalServerError, 500, id="connect" + ), + pytest.param( + httpx.HTTPStatusError( + "unavailable", + request=_REQUEST, + response=httpx.Response(503, text="Unavailable", request=_REQUEST), + ), + litellm.ServiceUnavailableError, + 503, + id="status", + ), + ), +) +def test_a_handler_failure_maps_as_the_python_handler_maps_it( + error: Exception, expected: type[Exception], status: int +) -> None: + mapped: Final = messages_host.map_transport_failure( + error, {"model": "anthropic/claude-sonnet-4-5", "api_base": None}, "anthropic" + ) + + assert isinstance(mapped, expected) + assert getattr(mapped, "status_code", None) == status + assert mapped.__context__ is not None + assert mapped.__context__.__context__ is error + + +class _Hook(CustomLogger): + rewrite: Mapping[str, object] = MappingProxyType({}) + seen: tuple[tuple[str, list[dict[str, object]], dict[str, object]], ...] = () + + async def async_pre_request_hook( + self, model: str, messages: list[dict[str, object]], kwargs: dict[str, object] + ) -> dict[str, object] | None: + self.seen = (*self.seen, (model, list(messages), dict(kwargs))) + return {**kwargs, **self.rewrite} + + +def _hook(rewrite: Mapping[str, object]) -> _Hook: + hook: Final = _Hook() + hook.rewrite = rewrite + return hook + + +def _hook_request(**overrides: object) -> dict[str, object]: + return { + "max_tokens": 64, + "messages": [{"role": "user", "content": [{"type": "text", "text": ""}, {"type": "text", "text": "hi"}]}], + "model": "anthropic/claude-sonnet-4-5", + "metadata": None, + "stop_sequences": None, + "stream": False, + "system": None, + "temperature": None, + "thinking": None, + "tool_choice": None, + "tools": None, + "top_k": None, + "top_p": None, + "api_key": "sk-test", + "api_base": None, + "client": None, + "custom_llm_provider": None, + "litellm_call_id": "call-1", + **overrides, + } + + +def _instance_hook() -> CustomLogger: + logger: Final = CustomLogger() + logger.async_pre_request_hook = _hook({}).async_pre_request_hook # pyright: ignore[reportAttributeAccessIssue] # rebind-ok: an instance-level override + return logger + + +@pytest.mark.parametrize( + ("callbacks", "expected"), + [ + pytest.param([_hook({})], True, id="override"), + pytest.param([CustomLogger()], False, id="base-logger"), + pytest.param([_instance_hook()], True, id="instance-override"), + pytest.param(["not-a-logger"], False, id="not-a-logger"), + ], +) +def test_runs_pre_request_hooks_needs_a_global_override( + monkeypatch: pytest.MonkeyPatch, callbacks: list[object], expected: bool +) -> None: + monkeypatch.setattr(litellm, "callbacks", callbacks) + + assert messages_host.runs_pre_request_hooks() is expected + + +@pytest.mark.asyncio +async def test_pre_request_hooks_hand_the_handler_what_the_python_entry_hands_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + hook: Final = _hook({"temperature": 0.2, "tools": [{"name": "t"}], "extra_headers": {"x-hooked": "1"}}) + monkeypatch.setattr(litellm, "callbacks", [hook]) + + hooked: Final = await messages_host.pre_request_hooks(_hook_request(temperature=0.9)) + + assert hook.seen == ( + ( + "anthropic/claude-sonnet-4-5", + [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + { + "tools": None, + "stream": False, + "litellm_params": {"custom_llm_provider": "anthropic"}, + "tool_choice": None, + "litellm_call_id": "call-1", + }, + ), + ) + assert hooked["temperature"] == 0.2 + assert hooked["tools"] == [{"name": "t"}] + assert hooked["extra_headers"] == {"x-hooked": "1"} + assert hooked["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + assert hooked["custom_llm_provider"] is None + assert "litellm_params" not in hooked + + +@pytest.mark.asyncio +@pytest.mark.parametrize("name", ["model", "max_tokens", "messages", "api_key", "api_base", "custom_llm_provider"]) +async def test_pre_request_hooks_naming_a_handler_argument_raise_the_python_type_error( + monkeypatch: pytest.MonkeyPatch, name: str +) -> None: + monkeypatch.setattr(litellm, "callbacks", [_hook({name: "hooked"})]) + + with pytest.raises(TypeError, match=f"functools.partial\\(\\) got multiple values for keyword argument '{name}'"): + await messages_host.pre_request_hooks(_hook_request()) + + +@pytest.mark.asyncio +async def test_pre_request_hooks_refuse_a_route_to_another_provider(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "callbacks", [_hook({"litellm_params": {"custom_llm_provider": "openai"}})]) + + with pytest.raises(litellm.UnsupportedParamsError, match="routing the call to openai") as refused: + await messages_host.pre_request_hooks(_hook_request()) + + assert refused.value.llm_provider == "anthropic" + + +@pytest.mark.asyncio +async def test_pre_request_hooks_keep_the_callers_provider_over_the_hook(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "callbacks", [_hook({"litellm_params": {"custom_llm_provider": "openai"}})]) + + hooked: Final = await messages_host.pre_request_hooks(_hook_request(custom_llm_provider="anthropic")) + + assert hooked["custom_llm_provider"] == "anthropic" + + +def test_body_params_hand_hooked_messages_on_as_already_sanitized() -> None: + hooked: Final = [{"role": "user", "content": [{"type": "text", "text": ""}]}] + + fields: Final = messages_host.body_params( + {"model": "anthropic/claude-sonnet-4-5", "messages": hooked}, asynchronous=True, hooked=True + ) + + assert fields == {"messages": hooked} + + +class _Agentic(CustomLogger): + async def async_should_run_agentic_loop(self, *args: object, **kwargs: object) -> tuple[bool, dict[str, object]]: # pyright: ignore[reportIncompatibleMethodOverride] # test double only needs the override + return False, {} + + +def _agentic_logger(callbacks: Sequence[CustomLogger]) -> Logging: + logger: Final = Logging( + model="anthropic/claude-sonnet-4-5", + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="agentic", + function_id="agentic", + dynamic_success_callbacks=list(callbacks), + ) + return logger + + +@pytest.mark.parametrize( + ("callbacks", "dynamic", "request_extra", "expected"), + [ + pytest.param([], [], {}, False, id="no-hook"), + pytest.param([_Agentic()], [], {}, True, id="global-hook"), + pytest.param([], [_Agentic()], {}, True, id="call-hook"), + pytest.param([], [], {"_websearch_interception_converted_stream": True}, True, id="converted-stream"), + ], +) +def test_runs_agentic_hooks_gates_as_the_python_handler_does( + monkeypatch: pytest.MonkeyPatch, + callbacks: list[object], + dynamic: Sequence[CustomLogger], + request_extra: Mapping[str, object], + expected: bool, +) -> None: + monkeypatch.setattr(litellm, "callbacks", callbacks) + request: Final = {"model": "anthropic/claude-sonnet-4-5", "litellm_logging_obj": _agentic_logger(dynamic)} + + assert messages_host.runs_agentic_hooks({**request, **request_extra}) is expected + assert messages_host.runs_agentic_hooks({"model": "anthropic/claude-sonnet-4-5"}) is False + + +def test_record_agentic_loop_names_the_requested_model_for_follow_up_calls() -> None: + logger: Final = _agentic_logger([]) + messages_host.record_agentic_loop( + { + "model": "anthropic/claude-sonnet-4-5", + "litellm_logging_obj": logger, + "_websearch_interception_converted_stream": True, + }, + "anthropic", + ) + + details: Final = logger.model_call_details + assert details["agentic_loop_params"] == { + "model": "anthropic/claude-sonnet-4-5", + "custom_llm_provider": "anthropic", + } + assert details["websearch_interception_converted_stream"] is True + + +def test_agentic_stream_tags_the_head_only_when_a_hook_runs(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.rust_bridge.messages.entrypoints import MessagesStreamHead + + request: Final = { + "model": "anthropic/claude-sonnet-4-5", + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "hi", "provider_specific_fields": {"x": 1}}]} + ], + "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}], + "api_key": "sk-test", + "litellm_call_id": "agentic", + "litellm_logging_obj": _agentic_logger([]), + } + sent: Final = { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16, + "stream": True, + "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}], + } + requested: Final = {"max_tokens": 16, "stream": True, "tools": request["tools"]} + head: Final = MessagesStreamHead() + + monkeypatch.setattr(litellm, "callbacks", []) + messages_host.agentic_stream(head, request, sent, "claude-sonnet-4-5", "anthropic", requested, hooked=False) + assert head.agentic is None + + monkeypatch.setattr(litellm, "callbacks", [_Agentic()]) + messages_host.agentic_stream(head, request, sent, "claude-sonnet-4-5", "anthropic", requested, hooked=False) + agentic: Final = head.agentic + assert isinstance(agentic, functools.partial) + bound: Final[Mapping[str, object]] = agentic.keywords + call: Final = bound["call"] + assert isinstance(call, messages_host._AgenticCall) # pyright: ignore[reportPrivateUsage] # the partial binds the handler call + assert call.messages == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + assert call.params == {"stream": True, "tools": request["tools"]} + assert call.params["tools"] is request["tools"] + assert call.kwargs["api_key"] == "sk-test" + assert "tools" not in call.kwargs + + +_DANGEROUS_TOOL_USE: Final = [{"type": "dangerous_tool_use"}] +_CLASSIFIER_SETTINGS: Final = {"safeguards_classifier_model": "openai/classifier"} +_NO_TOOL_USES: Final[dict[str, object]] = {} +_AVAILABLE: Final = ({"type": "dangerous_tool_use", "status": {"type": "available", "tool_uses": _NO_TOOL_USES}},) + + +@pytest.mark.parametrize( + ("request_fields", "general_settings", "classifies"), + ( + pytest.param( + {"model": "anthropic/parity-tool-model", "safeguards": _DANGEROUS_TOOL_USE}, + _CLASSIFIER_SETTINGS, + True, + id="non-claude", + ), + pytest.param( + {"model": "anthropic/claude-sonnet-4-5", "safeguards": _DANGEROUS_TOOL_USE}, + _CLASSIFIER_SETTINGS, + False, + id="claude", + ), + pytest.param( + {"model": "anthropic/parity-tool-model", "safeguards": _DANGEROUS_TOOL_USE}, {}, False, id="unconfigured" + ), + pytest.param({"model": "anthropic/parity-tool-model"}, _CLASSIFIER_SETTINGS, False, id="not-requested"), + pytest.param( + {"model": "anthropic/parity-tool-model", "safeguards": [{"type": "other"}]}, + _CLASSIFIER_SETTINGS, + False, + id="other-safeguard", + ), + ), +) +def test_safeguards_classify_only_what_the_proxy_answers( + monkeypatch: pytest.MonkeyPatch, + request_fields: Mapping[str, object], + general_settings: Mapping[str, object], + classifies: bool, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + + assert (messages_host.safeguards(request_fields) is not None) is classifies + + +@pytest.mark.asyncio +async def test_safeguards_frames_split_and_stamp_like_the_python_stream(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", _CLASSIFIER_SETTINGS) + evaluated: Final[list[object]] = [] + + async def evaluate(_evaluator: SafeguardsEvaluator, tool_uses: object) -> object: + evaluated.append(tool_uses) + return _AVAILABLE + + monkeypatch.setattr(SafeguardsEvaluator, "evaluate", evaluate) + classifier: Final = messages_host.safeguards( + { + "model": "anthropic/parity-tool-model", + "safeguards": _DANGEROUS_TOOL_USE, + "messages": [{"role": "user", "content": "hi"}], + } + ) + assert classifier is not None + assert await classifier.open_stream() + + opened: Final = await classifier.frames( + b'event: ping\ndata: {"type": "ping"}\n\nevent: message_delta\ndata: {"type": ' + ) + closed: Final = await classifier.frames(b'"message_delta", "delta": {"stop_reason": "end_turn"}}\n\ntail') + flushed: Final = await classifier.flush() + + assert opened == (b'event: ping\ndata: {"type": "ping"}\n\n',) + assert closed == ( + b'event: message_delta\ndata: {"type": "message_delta", "delta": {"stop_reason": "end_turn", ' + b'"safeguard_results": [{"type": "dangerous_tool_use", "status": {"type": "available", "tool_uses": {}}}]}}\n\n', + ) + assert flushed == (b"tail",) + assert evaluated == [()] + + +@pytest.mark.asyncio +async def test_safeguards_publish_the_results_python_stamped(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", _CLASSIFIER_SETTINGS) + stamped: Final = ({"type": "dangerous_tool_use", "status": {"type": "unsupported"}},) + + async def evaluate(_evaluator: SafeguardsEvaluator, tool_uses: object) -> object: + return stamped + + monkeypatch.setattr(SafeguardsEvaluator, "evaluate", evaluate) + request: Final = { + "model": "anthropic/parity-tool-model", + "safeguards": _DANGEROUS_TOOL_USE, + "messages": [{"role": "user", "content": "hi"}], + } + unanswered: Final = messages_host.safeguards(request) + answered: Final = messages_host.safeguards(request) + assert unanswered is not None and answered is not None + + assert await unanswered.results({"content": [], "safeguard_results": []}) is None + assert unanswered.published({"id": "msg"}) == {"id": "msg"} + assert await answered.results({"content": []}) is stamped + assert answered.published({"id": "msg", "safeguard_results": [stamped[0]]}) == { + "id": "msg", + "safeguard_results": stamped, + } + + +class _Claims(MessagesInterceptor): + def __init__(self, provider: str) -> None: + self.provider = provider + self.asked: list[tuple[object, str | None]] = [] + + def can_handle(self, tools: Sequence[Mapping[str, object]] | None, custom_llm_provider: str | None) -> bool: + self.asked.append((tools, custom_llm_provider)) + return custom_llm_provider == self.provider + + async def handle(self, **kwargs: object) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # kwargs-ok: the interceptor contract's keyword call + return kwargs + + +_TOOLS: Final = [{"type": "custom_tool", "name": "t"}] + + +@pytest.mark.parametrize( + ("request_fields", "provider"), + ( + pytest.param({"model": "anthropic/claude-sonnet-4-5"}, "anthropic", id="prefixed"), + pytest.param({"model": "claude-sonnet-4-5", "custom_llm_provider": "anthropic"}, "anthropic", id="explicit"), + pytest.param({"model": "not-a-known-model"}, None, id="unresolved"), + ), +) +def test_interceptor_asks_with_the_tools_and_the_provider_the_handler_resolves( + monkeypatch: pytest.MonkeyPatch, request_fields: Mapping[str, object], provider: str | None +) -> None: + declined: Final = _Claims("other") + claims: Final = _Claims("anthropic") + monkeypatch.setattr(interceptors, "_interceptors", [declined, claims]) + + chosen: Final = messages_host.interceptor({**request_fields, "tools": _TOOLS}) + + assert chosen is (claims if provider == "anthropic" else None) + assert declined.asked == [(_TOOLS, provider)] + + +@pytest.mark.asyncio +async def test_answer_hands_the_interceptor_the_call_as_the_handler_does() -> None: + owner: Final = _Claims("anthropic") + request: Final = { + "model": "anthropic/claude-sonnet-4-5", + "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": ""}]}], + "max_tokens": 16, + "tools": _TOOLS, + "stream": None, + "system": None, + "client": object(), + "custom_llm_provider": None, + "litellm_call_id": "call-1", + } + + handed: Final = await messages_host.answer(owner, request, stream=False, hooked=False) + + assert isinstance(handed, dict) + assert handed["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + assert handed["custom_llm_provider"] == "anthropic" + assert handed["stream"] is False + assert handed["litellm_call_id"] == "call-1" + assert "client" not in handed + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("stream", "converted", "expected"), + ( + pytest.param(True, False, True, id="caller-stream"), + pytest.param(False, True, True, id="converted-stream"), + pytest.param(False, False, False, id="unary"), + ), +) +async def test_answer_hands_a_hooked_call_on_with_the_callers_stream( + stream: bool, converted: bool, expected: bool +) -> None: + hooked_messages: Final = [{"role": "user", "content": [{"type": "text", "text": ""}]}] + request: Final = { + "model": "anthropic/claude-sonnet-4-5", + "messages": hooked_messages, + "max_tokens": 16, + "tools": _TOOLS, + "stream": False, + "system": "hooked", + "custom_llm_provider": None, + **({"_websearch_interception_converted_stream": True} if converted else {}), + } + + handed: Final = await messages_host.answer(_Claims("anthropic"), request, stream=stream, hooked=True) + + assert isinstance(handed, dict) + assert handed["messages"] is hooked_messages + assert handed["system"] == "hooked" + assert handed["stream"] is expected + + +MCP_REFERENCE: Final = MappingProxyType( + {"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy/mcp/deepwiki", "require_approval": "never"} +) + + +@pytest.mark.parametrize( + ("extra", "expected"), + [ + pytest.param({"tools": [dict(MCP_REFERENCE)]}, True, id="gateway-reference"), + pytest.param({"tools": [{**MCP_REFERENCE, "server_url": "litellm_proxy"}]}, True, id="whole-gateway"), + pytest.param({"tools": [dict(MCP_REFERENCE)], "_skip_mcp_handler": True}, False, id="gateway-follow-up"), + pytest.param({"tools": [{"name": "a", "input_schema": {"type": "object"}}]}, False, id="native-tool"), + pytest.param({"tools": []}, False, id="no-tools"), + pytest.param({"tools": [dict(MCP_REFERENCE)], "mock_response": "hi"}, False, id="mock-answers-first"), + ], +) +def test_settings_hand_the_call_to_the_mcp_gateway_where_the_python_handler_does( + extra: dict[str, object], expected: bool +) -> None: + settings: Final = messages_host.settings({}, {**_CALL, "model": "anthropic/claude-sonnet-4-5", **extra}) + + assert settings["mcp_gateway"] is expected + + +def test_gateway_is_none_for_a_call_the_handler_serves_itself() -> None: + assert messages_host.gateway({**_CALL, "model": "anthropic/claude-sonnet-4-5"}, True) is None + + +def _gateway_request() -> dict[str, object]: + return { + **_CALL, + "model": "anthropic/claude-sonnet-4-5", + "messages": [{"role": "user", "content": [{"type": "text", "text": ""}, {"type": "text", "text": "hi"}]}], + "tools": [dict(MCP_REFERENCE)], + "metadata": {"user_id": "u-1", "session": "dropped"}, + "stream": True, + "temperature": 0.2, + "litellm_metadata": {"tags": ["t1"]}, + } + + +def test_sync_gateway_hands_the_mcp_run_what_the_python_handler_hands_it(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.messages.native import handler + + sentinel: Final = object() + handed: Final[list[dict[str, object]]] = [] + + def run(**kwargs: object) -> object: + handed.append(kwargs) + return sentinel + + monkeypatch.setattr(mcp_handler, "anthropic_messages_with_mcp", run) + request: Final = _gateway_request() + + assert handler.anthropic_messages_handler(**_gateway_request()) is sentinel # noqa: TID251 # the Python handler under comparison + assert messages_host.gateway(request, False) is sentinel + assert handed[1] == handed[0] + assert handed[1]["model"] == "claude-sonnet-4-5" + assert handed[1]["custom_llm_provider"] == "anthropic" + assert handed[1]["metadata"] == {"user_id": "u-1"} + assert handed[1]["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + assert "_skip_mcp_handler" not in handed[1] + + +def test_async_gateway_tells_the_logger_the_agentic_loop_params_as_the_handler_does( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + + sentinel: Final = object() + handed: Final[list[dict[str, object]]] = [] + + def run(**kwargs: object) -> object: + handed.append(kwargs) + return sentinel + + monkeypatch.setattr(mcp_handler, "anthropic_messages_with_mcp", run) + logger: Final = Logging( + model="anthropic/claude-sonnet-4-5", + messages=[], + stream=False, + call_type="aanthropic_messages", + start_time=None, + litellm_call_id="call-1", + function_id="fn", + ) + + assert messages_host.gateway({**_gateway_request(), "litellm_logging_obj": logger}, True) is sentinel + assert logger.model_call_details["agentic_loop_params"] == { + "model": "anthropic/claude-sonnet-4-5", + "custom_llm_provider": "anthropic", + } + assert logger.model_call_details["litellm_params"]["aanthropic_messages"] is True + assert "is_async" not in handed[0] + assert handed[0]["litellm_logging_obj"] is logger + + +def test_async_gateway_hands_hooked_messages_on_as_already_sanitized(monkeypatch: pytest.MonkeyPatch) -> None: + handed: Final[list[dict[str, object]]] = [] + + def run(**kwargs: object) -> object: + handed.append(kwargs) + return None + + monkeypatch.setattr(mcp_handler, "anthropic_messages_with_mcp", run) + hooked: Final = [ + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]} + ] + + messages_host.gateway({**_gateway_request(), "messages": hooked}, True, hooked=True) + + assert handed[0]["messages"] == hooked + + +def test_sync_answer_serves_the_gateway_after_a_mock(monkeypatch: pytest.MonkeyPatch) -> None: + sentinel: Final = object() + + def run(**_: object) -> object: + return sentinel + + monkeypatch.setattr(mcp_handler, "anthropic_messages_with_mcp", run) + + served: Final = messages_host.sync_answer(_gateway_request()) + mocked: Final = messages_host.sync_answer({**_gateway_request(), "mock_response": "hi"}) + + assert served is not None and served.value is sentinel + assert mocked is not None and isinstance(mocked.value, dict) + assert messages_host.sync_answer({**_CALL, "model": "anthropic/claude-sonnet-4-5"}) is None + + +@pytest.mark.parametrize( + ("extra", "raised", "text"), + [ + pytest.param({"messages": None}, TypeError, "'NoneType' object is not iterable", id="messages-none"), + pytest.param({"messages": 5}, TypeError, "'int' object is not iterable", id="messages-int"), + pytest.param({"tools": {"name": "t"}}, AttributeError, "'str' object has no attribute 'get'", id="tools-map"), + pytest.param({"model_info": "not-a-mapping"}, ValidationError, "model_info", id="model-info-str"), + pytest.param({"custom_llm_provider": 5}, ValidationError, "custom_llm_provider", id="provider-int"), + ], +) +def test_settings_raise_what_the_python_handler_raises_for_arguments_of_the_wrong_type( + extra: dict[str, object], raised: type[Exception], text: str +) -> None: + with pytest.raises(raised, match=text): + messages_host.settings({}, {**_CALL, "model": "anthropic/claude-sonnet-4-5", **extra}) + + +def test_settings_leave_well_typed_tools_to_the_interceptors_after_the_cache(monkeypatch: pytest.MonkeyPatch) -> None: + asked: Final[list[object]] = [] + + def can_handle(tools: object, custom_llm_provider: object) -> bool: + asked.append(tools) + return False + + advisor: Final = interceptors.get_messages_interceptors()[0] + monkeypatch.setattr(advisor, "can_handle", can_handle) + + messages_host.settings({}, {**_CALL, "model": "anthropic/claude-sonnet-4-5", "tools": [{"name": "t"}]}) + + assert asked == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("hook_params", "raised", "text"), + [ + pytest.param("not-a-mapping", AttributeError, "'str' object has no attribute 'get'", id="params-str"), + pytest.param({"custom_llm_provider": 5}, ValidationError, "custom_llm_provider", id="provider-int"), + ], +) +async def test_pre_request_hooks_raise_what_the_python_handler_raises_for_the_hooks_params( + monkeypatch: pytest.MonkeyPatch, hook_params: object, raised: type[Exception], text: str +) -> None: + monkeypatch.setattr(litellm, "callbacks", [_hook({"litellm_params": hook_params})]) + + with pytest.raises(raised, match=text): + await messages_host.pre_request_hooks(_hook_request(model="claude-sonnet-4-5")) + + +def test_a_provider_config_rejection_maps_as_the_python_http_handler_maps_it() -> None: + logger: Final = Logging( + model="anthropic/claude-sonnet-4-5", + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="rejected", + function_id="rejected", + ) + rejected: Final = ValueError("missing required field: max_tokens") + rejected.messages_request_error = True # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + rejected.messages_provider_rejection = "max_tokens is required for Anthropic /v1/messages API" # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + + mapped: Final = messages_host.map_failure( + rejected, + {"model": "anthropic/claude-sonnet-4-5", "messages": [], "litellm_logging_obj": logger}, + "anthropic", + ) + + assert isinstance(mapped, litellm.BadRequestError) + assert str(mapped) == ( + "litellm.BadRequestError: AnthropicException - max_tokens is required for Anthropic /v1/messages API" + ) + assert mapped.model == "anthropic/claude-sonnet-4-5" # pyright: ignore[reportUnknownMemberType] # the public error leaves model untyped + assert mapped.llm_provider == "anthropic" # pyright: ignore[reportUnknownMemberType] # the public error leaves llm_provider untyped + assert logger.model_call_details["custom_llm_provider"] == "anthropic" + + +def test_short_circuits_needs_a_replaced_web_search_short_circuit(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger + + class _ShortCircuit(WebSearchInterceptionLogger): + async def try_short_circuit_search(self, *args: object, **kwargs: object) -> dict[str, object] | None: # pyright: ignore[reportIncompatibleMethodOverride] # test double only needs the override + return None + + monkeypatch.setattr(litellm, "callbacks", [CustomLogger()]) + assert not messages_host.short_circuits() + monkeypatch.setattr(litellm, "callbacks", [WebSearchInterceptionLogger()]) + assert not messages_host.short_circuits() + monkeypatch.setattr(litellm, "callbacks", [_ShortCircuit()]) + assert messages_host.short_circuits() diff --git a/tests/unit/rust_bridge/host/test_cache.py b/tests/unit/rust_bridge/host/test_cache.py index c2b87cca1dc..9b58524a444 100644 --- a/tests/unit/rust_bridge/host/test_cache.py +++ b/tests/unit/rust_bridge/host/test_cache.py @@ -1,12 +1,22 @@ import asyncio +import datetime +from collections.abc import AsyncIterator, Callable from typing import Final import pytest import litellm from litellm.caching.caching import Cache -from litellm.caching.caching_handler import _PENDING_CACHE_WRITES -from litellm.rust_bridge.host.cache import configured_cache, store_response +from litellm.caching.caching_handler import _PENDING_CACHE_WRITES, LLMCachingHandler +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.anthropic.messages.native.response_cache import AnthropicMessagesStreamCacheWriter +from litellm.rust_bridge.host.cache import ( + caching_handler, + configured_cache, + store_response, + store_verbatim, + wrap_stream, +) from litellm.types.caching import LiteLLMCacheType @@ -26,9 +36,9 @@ def test_configured_cache_preserves_identity_and_observes_runtime_replacement(mo _REQUEST: Final = {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}]} -async def _stored(response: dict[str, object]) -> object: +async def _stored(response: dict[str, object], store: Callable[..., None] = store_response) -> object: cache: Final = Cache(type=LiteLLMCacheType.LOCAL) - store_response(cache, response, **_REQUEST) + store(cache, response, **_REQUEST) await asyncio.gather(*_PENDING_CACHE_WRITES) return await cache.async_get_cache(**_REQUEST) @@ -51,3 +61,104 @@ async def test_store_response_drops_only_the_rust_marker_header_from_the_cached_ async def test_store_response_keeps_an_entry_without_the_rust_marker_unchanged() -> None: response: Final = {"id": "msg_2", "_hidden_params": {"additional_headers": {"llm_provider-x": "1"}}} assert await _stored(response) == response + + +async def _events() -> AsyncIterator[bytes]: + yield b"event: message_stop\ndata: {}\n\n" + + +async def anthropic_messages() -> None: + return None + + +def _logging() -> Logging: + return Logging( + model="claude-sonnet-4-5", + messages=[], + stream=True, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="messages", + function_id="messages", + ) + + +def test_wrap_stream_hands_the_stream_to_the_cache_writer_the_client_wrapper_uses( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cache: Final = Cache(type=LiteLLMCacheType.LOCAL, supported_call_types=["anthropic_messages"]) + monkeypatch.setattr(litellm, "cache", cache) + stream: Final = _events() + + wrapped: Final = wrap_stream(cache, anthropic_messages, stream, None, **_REQUEST, litellm_logging_obj=_logging()) + + assert isinstance(wrapped, AnthropicMessagesStreamCacheWriter) + assert wrapped.stream is stream + + +def test_wrap_stream_leaves_a_stream_the_cache_would_not_store(monkeypatch: pytest.MonkeyPatch) -> None: + cache: Final = Cache(type=LiteLLMCacheType.LOCAL, supported_call_types=["anthropic_messages"]) + monkeypatch.setattr(litellm, "cache", cache) + stream: Final = _events() + + assert ( + wrap_stream( + cache, + anthropic_messages, + stream, + None, + **_REQUEST, + cache={"no-store": True}, + litellm_logging_obj=_logging(), + ) + is stream + ) + + +@pytest.mark.asyncio +async def test_store_verbatim_keeps_the_rust_marker_header_in_the_cached_entry() -> None: + response: Final[dict[str, object]] = { + "id": "msg_3", + "_hidden_params": {"additional_headers": {"x-litellm-rust": "true", "llm_provider-x": "1"}}, + } + assert await _stored(response, store_verbatim) == response + + +@pytest.mark.asyncio +@pytest.mark.parametrize("store", [store_response, store_verbatim]) +@pytest.mark.parametrize("content", [[], [{"type": "text", "text": ""}]]) +async def test_a_message_without_output_is_never_stored( + store: Callable[..., None], content: list[dict[str, object]] +) -> None: + assert await _stored({"id": "msg_4", "type": "message", "content": content}, store) is None + + +def test_wrap_stream_writes_through_the_handler_that_read_the_cache_over_a_later_call_s( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cache: Final = Cache(type=LiteLLMCacheType.LOCAL, supported_call_types=["anthropic_messages"]) + monkeypatch.setattr(litellm, "cache", cache) + logging_obj: Final = _logging() + read: Final = LLMCachingHandler( + original_function=anthropic_messages, request_kwargs=dict(_REQUEST), start_time=datetime.datetime.now() + ) + logging_obj.llm_caching_handler = LLMCachingHandler( + original_function=anthropic_messages, request_kwargs={}, start_time=datetime.datetime.now() + ) + + wrapped: Final = wrap_stream( + cache, anthropic_messages, _events(), read, **_REQUEST, litellm_logging_obj=logging_obj + ) + + assert isinstance(wrapped, AnthropicMessagesStreamCacheWriter) + assert wrapped.caching_handler is read + + +def test_caching_handler_is_the_one_the_logging_object_holds() -> None: + logging_obj: Final = _logging() + assert caching_handler(litellm_logging_obj=logging_obj) is None + handler: Final = LLMCachingHandler( + original_function=anthropic_messages, request_kwargs={}, start_time=datetime.datetime.now() + ) + logging_obj.llm_caching_handler = handler + assert caching_handler(litellm_logging_obj=logging_obj) is handler diff --git a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py index cb594767908..2a448df6e5a 100644 --- a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py @@ -4,7 +4,7 @@ import inspect from collections.abc import Mapping from pathlib import Path from types import MappingProxyType -from typing import Final +from typing import Final, cast # noqa: TID251 # a duck-typed logger double stands in for Logging import httpx import pytest @@ -45,6 +45,20 @@ def test_native_stream_headers_reach_spend_callbacks() -> None: assert logger.model_call_details["response_headers"] == {"llm_provider-request-id": "req_native"} +def test_recorded_messages_stream_keeps_the_provider_header_names() -> None: + logger: Final = _supplied_logger() + sent: Final[tuple[str, Mapping[str, str], Mapping[str, object]]] = ( + "http://upstream/v1/messages", + {}, + {"stream": True}, + ) + legacy.record_response(logger, 200, [("request-id", "req_native")], "", sent) + legacy.stream_opened(logger, {"additional_headers": {"llm_provider-request-id": "req_native"}}) + + assert logger.stream is True + assert logger.model_call_details["response_headers"] == {"request-id": "req_native"} + + def test_setup_reuses_a_supplied_logger() -> None: supplied: Final = _supplied_logger() result: Final = setup( @@ -73,7 +87,7 @@ def test_cache_hit_finalization_preserves_execution_provider_attribution(explici ) prepared.logger.model_call_details.update({"cache_hit": True, "cache_key": "cached-response"}) response: Final = ModelResponse(model="cache-test-model") - legacy.finalize(response, prepared.logger, prepared.kwargs, now, now) + legacy.finalize(response, prepared.logger, prepared.kwargs, now, now, False) assert prepared.logger.model_call_details["custom_llm_provider"] == "azure" assert prepared.logger.model_call_details["model"] == "resolved-cache-model" assert prepared.logger.litellm_params["metadata"]["user_api_key"] == "key-hash" @@ -303,7 +317,7 @@ def _messages_logger() -> Logging: def test_native_provider_response_is_recorded_for_logging_replay() -> None: logger: Final = _messages_logger() - legacy.record_response(logger, 200, (("request-id", "req_native"),), '{"id": "msg_native"}') + legacy.record_response(logger, 200, (("request-id", "req_native"),), '{"id": "msg_native"}', None) recorded: Final[object] = logger.model_call_details["httpx_response"] assert isinstance(recorded, httpx.Response) @@ -312,6 +326,51 @@ def test_native_provider_response_is_recorded_for_logging_replay() -> None: assert logger.model_call_details["response_headers"]["request-id"] == "req_native" +def test_a_non_ascii_provider_header_is_recorded_as_httpx_decodes_it() -> None: + logger: Final = _messages_logger() + + legacy.record_response(logger, 200, (("x-fz-unicode", "caf\u00e9"),), '{"id": "msg_native"}', None) + + assert logger.model_call_details["response_headers"]["x-fz-unicode"] == "caf\u00e9" + + +_HTTPX_RESPONSE: Final = TypeAdapter(httpx.Response, config={"arbitrary_types_allowed": True}) +_SENT_HEADERS: Final = MappingProxyType({"x-api-key": "sk-ant", "anthropic-beta": "message-threads-2026-08-12"}) + + +def test_native_provider_response_carries_the_request_that_was_sent() -> None: + logger: Final = _messages_logger() + body: Final = {"model": "claude-sonnet-4-5", "thread": {"type": "create"}} + + legacy.record_response( + logger, 200, (), '{"id": "msg_native"}', ("https://api.anthropic.com/v1/messages", _SENT_HEADERS, body) + ) + + recorded: Final = _HTTPX_RESPONSE.validate_python(logger.model_call_details["httpx_response"]) + assert recorded.request.method == "POST" + assert str(recorded.request.url) == "https://api.anthropic.com/v1/messages" + assert recorded.request.headers["anthropic-beta"] == "message-threads-2026-08-12" + assert recorded.request.content == b'{"model": "claude-sonnet-4-5", "thread": {"type": "create"}}' + assert recorded.content == b'{"id": "msg_native"}' + + +def test_native_stream_response_is_recorded_unread_like_the_python_stream() -> None: + logger: Final = _messages_logger() + + legacy.record_response( + logger, + 200, + (("request-id", "req_native"),), + "", + ("https://api.anthropic.com/v1/messages", _SENT_HEADERS, {"stream": True}), + ) + + recorded: Final = _HTTPX_RESPONSE.validate_python(logger.model_call_details["httpx_response"]) + assert recorded.request.headers["anthropic-beta"] == "message-threads-2026-08-12" + with pytest.raises(httpx.ResponseNotRead): + _ = recorded.content + + def test_native_provider_exchange_window_is_recorded_like_the_timed_post() -> None: logger: Final = _messages_logger() logger.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed"}} @@ -323,6 +382,76 @@ def test_native_provider_exchange_window_is_recorded_like_the_timed_post() -> No assert metadata["llm_api_timing_windows"] == ((1_700_000_000.0, 1_700_000_000.25),) +@pytest.mark.asyncio +async def test_a_retried_request_rewrites_the_logged_body_in_place_like_the_python_retry_loop() -> None: + logger: Final = _messages_logger() + metadata: Final = {"user_id": "u-1"} + replayed: Final = [ + {"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "s"}]}, + {"role": "user", "content": "again"}, + ] + logged: Final[dict[str, object]] = { + "model": "claude-sonnet-4-5", + "messages": replayed, + "metadata": metadata, + "thinking": {"type": "enabled", "budget_tokens": 1024}, + } + logger.model_call_details.update(logged) + retried: Final = { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "again"}], + "metadata": {"user_id": "u-1"}, + } + + await legacy.retried_request(logger, logged, retried, ({"kind": "thinking_signature"},)) + + assert logged == retried + assert logged["metadata"] is metadata + assert logger.model_call_details["messages"] == [{"role": "user", "content": "again"}] + assert logger.model_call_details["thinking"] == {"type": "enabled", "budget_tokens": 1024} + + +class _BaselineLogger: + def __init__(self) -> None: + self.baseline_cache_context: object = object() + self.model_call_details: dict[str, object] = {} + self.invalidated: list[str] = [] + + async def invalidate_baseline_cache_estimate(self, reason: str) -> None: + self.invalidated.append(reason) + + +@pytest.mark.asyncio +async def test_a_retried_request_invalidates_the_baseline_cache_estimate() -> None: + logger: Final = _BaselineLogger() + + await legacy.retried_request(cast(Logging, logger), {"messages": []}, {"messages": []}, ()) # cast-ok: duck-typed + + assert logger.invalidated == ["retried_request"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("reasons", "warned"), + [ + pytest.param(({"kind": "thinking_signature"},), [], id="thinking_signature"), + pytest.param(({"kind": "strict_tools", "dropped": 2},), [2], id="strict_tools"), + pytest.param(({"kind": "thinking_signature"}, {"kind": "strict_tools", "dropped": 3}), [3], id="both_reasons"), + ], +) +async def test_a_retry_that_dropped_strict_tools_warns_with_the_dropped_count( + monkeypatch: pytest.MonkeyPatch, reasons: tuple[Mapping[str, object], ...], warned: list[int] +) -> None: + from litellm.llms.anthropic import common_utils + + warnings: Final[list[int]] = [] + monkeypatch.setattr(common_utils, "warn_claude_strict_tools_dropped", warnings.append) + + await legacy.retried_request(_messages_logger(), {"messages": []}, {"messages": []}, reasons) + + assert warnings == warned + + def test_stream_head_is_stamped_in_place_with_the_call_hidden_params() -> None: logger: Final = _messages_logger() head: Final[dict[str, object]] = {"additional_headers": {"request-id": "req_native"}} @@ -334,6 +463,7 @@ def test_stream_head_is_stamped_in_place_with_the_call_hidden_params() -> None: {"model": "anthropic/claude-sonnet-4-5", "litellm_call_id": "messages"}, started, started + datetime.timedelta(milliseconds=5), + False, ) assert head["additional_headers"] == {"request-id": "req_native", "llm_provider-request-id": "req_native"} @@ -403,3 +533,22 @@ async def test_a_proxy_stream_abandoned_mid_cancel_bills_after_the_disconnect_cl await asyncio.sleep(0) assert events == ["disconnect recorded", "billed"] + + +@pytest.mark.parametrize( + ("requested", "keeps_requested", "expected"), + [ + pytest.param("false", True, "false", id="messages-keeps-a-truthy-string"), + pytest.param(1, True, 1, id="messages-keeps-a-truthy-int"), + pytest.param(False, True, True, id="messages-hook-streamed"), + pytest.param("false", False, True, id="other-routes-stream-true"), + ], +) +def test_a_stream_logs_the_flag_python_logs(requested: object, keeps_requested: bool, expected: object) -> None: + logger: Final = _supplied_logger() + logger.model_call_details["stream"] = requested + + legacy.stream_opened(logger, {}, keeps_requested) + + assert logger.model_call_details["stream"] == expected + assert logger.stream == expected diff --git a/tests/unit/rust_bridge/test_catalog.py b/tests/unit/rust_bridge/test_catalog.py index 1994dd00709..56fcad14851 100644 --- a/tests/unit/rust_bridge/test_catalog.py +++ b/tests/unit/rust_bridge/test_catalog.py @@ -20,16 +20,10 @@ from litellm.rust_bridge.public_call import NativeCall @pytest.mark.parametrize( - ("model", "expected"), - ( - ("anthropic/claude-opus-latest", Rollout.PYTHON_ONLY), - (f"anthropic/{get_model_latest_aliases()['anthropic']['claude-opus-latest']}", Rollout.RUST_OPT_IN), - ), + "model", + ("anthropic/claude-opus-latest", f"anthropic/{get_model_latest_aliases()['anthropic']['claude-opus-latest']}"), ) -def test_latest_alias_messages_falls_back_without_changing_the_pinned_model_rollout( - model: str, - expected: Rollout, -) -> None: +def test_latest_alias_messages_stays_native_like_the_pinned_model(model: str) -> None: request: Final = NativeCall(args=(), kwargs={"model": model}, base={}) context: Final = RouteContext( Route.MESSAGES, @@ -41,7 +35,7 @@ def test_latest_alias_messages_falls_back_without_changing_the_pinned_model_roll RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=catalog.MESSAGES_PYTHON_FEATURES), RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN, providers=frozenset({"anthropic"})), ) - assert catalog.rollout(context, rules) is expected + assert catalog.rollout(context, rules) is Rollout.RUST_OPT_IN @pytest.fixture(autouse=True) @@ -74,7 +68,7 @@ def test_missing_rule_stays_on_python_even_when_rust_is_enabled(monkeypatch: pyt (RouteContext(Route.MESSAGES, target=Target.MESSAGES), Decision.RUST_REQUIRED), (RouteContext(Route.MESSAGES, target=Target.CHAT_COMPLETIONS), Decision.PYTHON), (RouteContext(Route.MESSAGES), Decision.PYTHON), - (RouteContext(Route.OCR, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), Decision.PYTHON), + (RouteContext(Route.OCR, features=frozenset({Feature.STREAMING})), Decision.PYTHON), (RouteContext(Route.OCR), Decision.RUST_REQUIRED), ), ) @@ -88,7 +82,7 @@ def test_first_matching_rule_respects_every_constraint(context: RouteContext, ex ), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED, targets=frozenset({Target.MESSAGES})), - RouteRule(Route.OCR, Rollout.PYTHON_ONLY, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), + RouteRule(Route.OCR, Rollout.PYTHON_ONLY, features=frozenset({Feature.STREAMING})), RouteRule(Route.OCR, Rollout.RUST_REQUIRED), ) @@ -152,6 +146,7 @@ def test_empty_constraints_match_nothing() -> None: ) assert catalog.decision(RouteContext(Route.OCR), rules) is Decision.PYTHON - assert catalog.decision( - RouteContext(Route.OCR, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), rules - ) is Decision.PYTHON + assert ( + catalog.decision(RouteContext(Route.OCR, features=frozenset({Feature.STREAMING})), rules) + is Decision.PYTHON + ) diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index fccc01dd8e6..7ad8069758e 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -1,4 +1,5 @@ from collections.abc import Callable, Mapping +import ssl from copy import deepcopy from datetime import datetime from functools import reduce @@ -14,10 +15,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.rust_bridge import detectors from litellm.rust_bridge.chat_completions import detectors as chat_detectors -from litellm.rust_bridge.messages import detectors as messages_detectors from litellm.rust_bridge.public_call import NativeCall from litellm.rust_bridge.responses import detectors as responses_detectors -from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES +from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES GATEWAY_TOOL: Final = {"type": "mcp", "server_label": "gateway", "server_url": "litellm_proxy"} REMOTE_TOOL: Final = {"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com/sse"} @@ -31,9 +31,6 @@ MAPPED_FILE: Final = { } MAPPED_FILE_INPUT: Final = [{"role": "user", "content": [{"type": "input_file", "file_id": "litellm-file"}]}] CACHED_TEXT: Final = {"type": "input_text", "text": "hi", "cache_control": {"type": "ephemeral"}} -THINKING_TURN: Final = {"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "s"}]} -REDACTED: Final = {"type": "redacted_thinking", "data": "x"} -STRICT_TOOL: Final = {"name": "b", "input_schema": {"type": "object"}, "strict": True} TOOL_SEARCH_TYPE: Final = sorted(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES)[0] @@ -63,25 +60,6 @@ def test_floating_alias_requires_python(model: str, provider: str | None, expect assert detectors.latest_alias(call(model=model, custom_llm_provider=provider)) is expected -@pytest.mark.parametrize( - ("model", "provider", "tools", "expected"), - ( - ("deepseek/test-model", None, [{"type": ANTHROPIC_ADVISOR_TOOL_TYPE}], True), - ("anthropic/test-model", None, [{"type": ANTHROPIC_ADVISOR_TOOL_TYPE}], False), - ("test-model", "anthropic", [{"type": ANTHROPIC_ADVISOR_TOOL_TYPE}], False), - ("unresolved-model", None, [{"type": ANTHROPIC_ADVISOR_TOOL_TYPE}], True), - ("deepseek/test-model", None, [{"type": "function", "name": "f"}], False), - ("deepseek/test-model", None, ["not-a-tool"], False), - ("deepseek/test-model", None, [], False), - ("deepseek/test-model", None, None, False), - ), -) -def test_messages_interception_matches_tools_and_resolved_provider( - model: str, provider: str | None, tools: object, expected: bool -) -> None: - assert messages_detectors.intercepted(call(model=model, custom_llm_provider=provider, tools=tools)) is expected - - def _wrap_nested_level(value: object, depth: int) -> object: return {"nested": value} if depth % 2 else [value] @@ -136,8 +114,6 @@ def _wrap_nested_level(value: object, depth: int) -> object: (responses_detectors.provider_affinity, {"provider_affinity_header": None, "session_id": "session-1"}, False), (responses_detectors.provider_affinity, {}, False), (responses_detectors.provider_affinity, {"api_base": "https://opencode.ai/inference/go/openai/v1"}, True), - (messages_detectors.provider_affinity, {"api_base": "https://opencode.ai/inference/go/anthropic/v1"}, True), - (messages_detectors.provider_affinity, {"api_base": "https://other.example/v1"}, False), (detectors.streaming, {"stream": True}, True), (detectors.streaming, {"stream": False}, False), (detectors.timeout_variants, {"request_timeout": 5}, True), @@ -196,36 +172,6 @@ def _wrap_nested_level(value: object, depth: int) -> object: (detectors.cache_control_injection, {"enable_prompt_caching": True}, True), (detectors.cache_control_injection, {"cache_control_injection_points": [{"location": "message"}]}, True), (detectors.cache_control_injection, {"cache_control_injection_points": []}, False), - (messages_detectors.thinking_signature_retry, {"messages": [THINKING_TURN]}, True), - ( - messages_detectors.thinking_signature_retry, - {"messages": [{"role": "assistant", "content": [REDACTED]}]}, - True, - ), - (messages_detectors.thinking_signature_retry, {"messages": [{"role": "assistant", "content": "done"}]}, False), - ( - messages_detectors.thinking_signature_retry, - {"messages": [{"role": "user", "content": [{"type": "text"}]}]}, - False, - ), - (messages_detectors.tool_custom_field, {"tools": [{"name": "a"}, {"name": "b", "custom": {}}]}, True), - (messages_detectors.tool_custom_field, {"tools": [{"name": "a", "defer_loading": True}]}, False), - (messages_detectors.tool_custom_field, {}, False), - (messages_detectors.strict_tools_retry, {"drop_params": True, "tools": [{"name": "a"}, STRICT_TOOL]}, True), - ( - messages_detectors.strict_tools_retry, - {"drop_params": True, "tools": [{**STRICT_TOOL, "strict": False}]}, - False, - ), - ( - messages_detectors.strict_tools_retry, - {"additional_drop_params": ["metadata.user_id"], "tools": [STRICT_TOOL]}, - False, - ), - (messages_detectors.strict_tools_retry, {"tools": [STRICT_TOOL]}, False), - (messages_detectors.strict_tools_retry, {"drop_params": False, "tools": [STRICT_TOOL]}, False), - (messages_detectors.strict_tools_retry, {"drop_params": True, "tools": [{"name": "a"}]}, False), - (messages_detectors.strict_tools_retry, {"drop_params": True}, False), (detectors.drop_params, {"drop_params": True}, True), (detectors.drop_params, {"drop_params": False}, False), (detectors.drop_params, {"drop_params": " True "}, True), @@ -323,8 +269,6 @@ def test_input_rewrite_preserves_caller_input(input_value: object, expected: boo (detectors.modify_params, "modify_params", True, {}), (detectors.retries_fallbacks, "num_retries", 3, {}), (detectors.cache_control_injection, "enable_anthropic_prompt_caching", True, {}), - (messages_detectors.stream_cost_injection, "include_cost_in_streaming_usage", True, {"stream": True}), - (messages_detectors.stream_cost_injection, "include_cost_in_streaming_usage", "true", {"stream": True}), (chat_detectors.chat_retries_fallbacks, "model_fallbacks", [{"gpt-5": ["gpt-5-mini"]}], {}), (chat_detectors.chat_retries_fallbacks, "num_retries", 3, {}), (chat_detectors.model_alias_map, "model_alias_map", {"gpt-5": "openai/gpt-5"}, {}), @@ -347,30 +291,6 @@ def test_global_settings_are_detected( assert detector(call(**kwargs)) -def test_stream_cost_injection_needs_a_stream(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) - assert not messages_detectors.stream_cost_injection(call()) - - -def test_stream_cost_injection_reads_the_request_header_the_proxy_reads(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) - opted_in: Final = {"proxy_server_request": {"headers": {"x-litellm-include-cost-in-streaming-usage": "true"}}} - assert messages_detectors.stream_cost_injection(call(stream=True, **opted_in)) - assert not messages_detectors.stream_cost_injection(call(**opted_in)) - assert not messages_detectors.stream_cost_injection(call(stream=True, proxy_server_request={"headers": {}})) - - -def test_foundry_beta_row_reads_the_api_base_the_anthropic_handler_reads(monkeypatch: pytest.MonkeyPatch) -> None: - for name in ("ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"): - monkeypatch.delenv(name, raising=False) - assert messages_detectors.foundry_beta_row(call(api_base="https://team.services.ai.azure.com/anthropic")) - assert messages_detectors.foundry_beta_row(call(base_url="https://team.cognitiveservices.azure.com/anthropic")) - assert not messages_detectors.foundry_beta_row(call(api_base="https://api.anthropic.com")) - assert not messages_detectors.foundry_beta_row(call()) - monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://team.services.ai.azure.com/anthropic") - assert messages_detectors.foundry_beta_row(call()) - - def test_model_alias_map_applies_only_to_the_requested_model(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_alias_map", {"fast": "openai/gpt-5-mini"}) assert chat_detectors.model_alias_map(call(model="fast")) @@ -413,20 +333,6 @@ def test_openai_workload_identity_applies_only_without_a_static_key(monkeypatch: assert not responses_detectors.openai_workload_identity(call()) -def test_anthropic_workload_identity_applies_only_without_a_static_key(monkeypatch: pytest.MonkeyPatch) -> None: - for name in ( - "ANTHROPIC_API_KEY", - "ANTHROPIC_AUTH_TOKEN", - "ANTHROPIC_FEDERATION_RULE_ID", - "ANTHROPIC_ORGANIZATION_ID", - ): - monkeypatch.delenv(name, raising=False) - federation: Final = {"anthropic_federation_rule_id": "rule-1", "anthropic_organization_id": "org-1"} - assert messages_detectors.anthropic_workload_identity(call(**federation)) - assert not messages_detectors.anthropic_workload_identity(call(**federation, api_key="sk-ant-static")) - assert not messages_detectors.anthropic_workload_identity(call(anthropic_federation_rule_id="rule-1")) - - class _Logger(CustomLogger): pass @@ -436,10 +342,6 @@ class _Rewrite(CustomLogger): return kwargs -class _InheritedRewrite(_Rewrite): - pass - - class _Agentic(CustomLogger): async def async_should_run_agentic_loop(self, *args: object, **kwargs: object) -> tuple[bool, dict]: # pyright: ignore[reportIncompatibleMethodOverride, reportMissingTypeArgument] # test double only needs the override return False, {} @@ -460,10 +362,6 @@ class _Deployment(CustomLogger): @pytest.mark.parametrize( ("detector", "callbacks", "expected"), ( - (messages_detectors.pre_request_hook, [_Rewrite()], True), - (messages_detectors.pre_request_hook, [_InheritedRewrite()], True), - (messages_detectors.pre_request_hook, [_Logger(), _Agentic()], False), - (messages_detectors.pre_request_hook, ["not-a-logger"], False), (detectors.agentic_loop_hook, [_Agentic()], True), (detectors.agentic_loop_hook, [_Logger(), _Rewrite()], False), (detectors.agentic_loop_hook, [_ChatAgentic()], False), @@ -501,7 +399,6 @@ def _logging_with(callback: CustomLogger) -> Logging: (detectors.agentic_loop_hook, {"success_callback": [_Agentic()]}, True), (detectors.agentic_loop_hook, {"litellm_logging_obj": _logging_with(_Agentic())}, True), (detectors.agentic_loop_hook, {"success_callback": [_Logger()]}, False), - (messages_detectors.pre_request_hook, {"success_callback": [_Rewrite()]}, False), (chat_detectors.chat_agentic_loop_hook, {"litellm_logging_obj": _logging_with(_ChatAgentic())}, True), (responses_detectors.deployment_hook, {"success_callback": [_Deployment()]}, True), ), @@ -509,7 +406,6 @@ def _logging_with(callback: CustomLogger) -> Logging: "success-callback", "logging-object", "plain-logger", - "pre-request-reads-globals-only", "chat-agentic-logging-object", "deployment-success-callback", ), @@ -524,15 +420,6 @@ def test_callback_detectors_read_the_request_callbacks_the_handler_reads( assert detector(call(**kwargs)) is expected -def test_websearch_interception_needs_the_interception_logger(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger - - monkeypatch.setattr(litellm, "callbacks", [_Rewrite()]) - assert not messages_detectors.websearch_interception(call()) - monkeypatch.setattr(litellm, "callbacks", [WebSearchInterceptionLogger()]) - assert messages_detectors.websearch_interception(call()) - - @pytest.mark.parametrize( ("cache", "kwargs", "expected"), ( @@ -551,7 +438,14 @@ def test_response_cache_is_skipped_only_by_no_cache( @pytest.mark.parametrize( ("environment", "setting", "expected"), - ((None, None, False), ("DEFAULT@SECLEVEL=1", None, True), (None, "DEFAULT@SECLEVEL=1", True)), + ( + (None, None, False), + ("DEFAULT@SECLEVEL=1", None, True), + (None, "DEFAULT@SECLEVEL=1", True), + (None, "ECDHE-RSA-AES128-GCM-SHA256", False), + (None, "ECDHE-RSA-AES128-GCM-SHA256:RC4-SHA", True), + ("HIGH", None, False), + ), ) def test_tls_policy_reads_the_security_level_environment_then_the_setting( monkeypatch: pytest.MonkeyPatch, environment: str | None, setting: str | None, expected: bool @@ -567,7 +461,19 @@ def test_tls_policy_reads_the_security_level_environment_then_the_setting( @pytest.mark.parametrize( ("environment", "setting", "expected"), - ((None, "X25519", False), (None, " P-256 ", False), ("secp521r1", None, True), (None, "secp521r1", True)), + ( + (None, "X25519", False), + (None, "prime256v1", False), + (None, "secp384r1", False), + (None, " P-256 ", True), + (None, "P-384", True), + (None, "secp256r1", True), + (None, "x25519", True), + ("secp521r1", None, True), + (None, "secp521r1", True), + ("", "secp521r1", False), + (None, "", False), + ), ) def test_tls_policy_flags_only_curves_native_tls_lacks( monkeypatch: pytest.MonkeyPatch, environment: str | None, setting: str | None, expected: bool @@ -579,3 +485,24 @@ def test_tls_policy_flags_only_curves_native_tls_lacks( monkeypatch.setenv("SSL_ECDH_CURVE", environment) monkeypatch.setattr(litellm, "ssl_ecdh_curve", setting) assert detectors.tls_policy(call()) is expected + + +@pytest.mark.parametrize(("setting", "value"), (("ssl_security_level", 1), ("ssl_ecdh_curve", object()))) +def test_tls_policy_hands_an_undecodable_setting_to_python_without_raising( + monkeypatch: pytest.MonkeyPatch, setting: str, value: object +) -> None: + for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "ssl_security_level", None) + monkeypatch.setattr(litellm, "ssl_ecdh_curve", None) + monkeypatch.setattr(litellm, setting, value) + assert detectors.tls_policy(call()) is True + + +def test_tls_policy_ignores_a_live_ssl_context(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "ssl_security_level", None) + monkeypatch.setattr(litellm, "ssl_ecdh_curve", None) + monkeypatch.setattr(litellm, "ssl_verify", ssl.create_default_context()) + assert detectors.tls_policy(call()) is False diff --git a/tests/unit/rust_bridge/test_runtime.py b/tests/unit/rust_bridge/test_runtime.py index 75cdfde087b..548591a2903 100644 --- a/tests/unit/rust_bridge/test_runtime.py +++ b/tests/unit/rust_bridge/test_runtime.py @@ -216,9 +216,12 @@ async def test_python_fallback_does_not_claim_rust_execution(missing: bool) -> N @pytest.mark.parametrize("shape", ("model", "dict")) @pytest.mark.parametrize("asynchronous", (False, True)) @pytest.mark.parametrize("cache_key", (None, "test-cache-key")) +@pytest.mark.parametrize("route", (Route.MESSAGES, Route.OCR)) async def test_native_response_marker_reaches_caller_with_existing_metadata( - shape: str, asynchronous: bool, cache_key: str | None + shape: str, asynchronous: bool, cache_key: str | None, route: Route ) -> None: + context: Final = RouteContext(route, provider="anthropic", model="model") + required: Final = (RouteRule(route, Rollout.RUST_REQUIRED, providers=frozenset({"anthropic"})),) hidden: Final = { "additional_headers": {"x-request-id": "upstream"}, "response_cost": 0.01, @@ -232,6 +235,47 @@ async def test_native_response_marker_reaches_caller_with_existing_metadata( bound: Final[bindings.NativeBinding[Callable[[], object]]] = bindings.NativeBinding("ocr", validate=lambda _: None) bound.override(lambda: response) + def python() -> object: + pytest.fail("native success must not fall back") + + async def anative(fn: Callable[[], object]) -> object: + return fn() + + async def apython() -> object: + return python() + + result: Final = ( + await runtime.arun(context, binding=bound, native=anative, python=apython, rules=required) + if asynchronous + else runtime.run(context, binding=bound, native=lambda fn: fn(), python=python, rules=required) + ) + assert result is response + assert get_hidden_params_dict(result) == ( + { + "response_cost": 0.01, + "additional_headers": {"x-request-id": "upstream"}, + **({"cache_key": cache_key} if cache_key is not None else {}), + } + if shape == "dict" + else { + "response_cost": 0.01, + "additional_headers": { + "x-request-id": "upstream", + "x-litellm-rust": "true", + **({"x-litellm-cache-key": cache_key} if cache_key is not None and route is not Route.MESSAGES else {}), + }, + **({"cache_key": cache_key} if cache_key is not None else {}), + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True)) +async def test_native_dict_response_reaches_caller_without_hidden_params_key(asynchronous: bool) -> None: + response: Final[dict[str, object]] = {"content": "native"} + bound: Final[bindings.NativeBinding[Callable[[], object]]] = bindings.NativeBinding("ocr", validate=lambda _: None) + bound.override(lambda: response) + def python() -> object: pytest.fail("native success must not fall back") @@ -249,15 +293,7 @@ async def test_native_response_marker_reaches_caller_with_existing_metadata( ) ) assert result is response - assert get_hidden_params_dict(result) == { - "response_cost": 0.01, - "additional_headers": { - "x-request-id": "upstream", - "x-litellm-rust": "true", - **({"x-litellm-cache-key": cache_key} if cache_key is not None else {}), - }, - **({"cache_key": cache_key} if cache_key is not None else {}), - } + assert result == {"content": "native"} class ScriptedStreamExecution: diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index 6ceea94d843..44b5876a078 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -1223,6 +1223,13 @@ ALIAS_CASES: Final = ( False, ), ("litellm.caching.caching_handler", "LLMCachingHandler", "_async_get_cache", "async_get_cache", False), + ( + "litellm.caching.caching_handler", + "LLMCachingHandler", + "_async_log_cache_hit_on_callbacks", + "async_log_cache_hit_on_callbacks", + False, + ), ( "litellm.caching.caching_handler", "LLMCachingHandler", @@ -1957,7 +1964,13 @@ ALIAS_CASES: Final = ( "get_tags_from_request_kwargs", False, ), - ("litellm.litellm_core_utils.add_retry_fallback_headers", "HiddenParamsAsyncIteratorWrapper", "_inner", "inner", True), + ( + "litellm.litellm_core_utils.add_retry_fallback_headers", + "HiddenParamsAsyncIteratorWrapper", + "_inner", + "inner", + True, + ), ("litellm.litellm_core_utils.add_retry_fallback_headers", "", "_HiddenParamsHost", "HiddenParamsHost", False), ( "litellm.litellm_core_utils.batch_utils", @@ -2301,1837 +2314,1837 @@ LLMS_ALIAS_CASES: Final = ( ) PROXY_ALIAS_CASES: Final = ( ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - '', - '_is_mcp_admitted_user_subject', - 'is_mcp_admitted_user_subject', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "", + "_is_mcp_admitted_user_subject", + "is_mcp_admitted_user_subject", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_mcp_auth_header_from_headers', - 'get_mcp_auth_header_from_headers', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_mcp_auth_header_from_headers", + "get_mcp_auth_header_from_headers", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_mcp_server_auth_headers_from_headers', - 'get_mcp_server_auth_headers_from_headers', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_mcp_server_auth_headers_from_headers", + "get_mcp_server_auth_headers_from_headers", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_mcp_servers_from_access_groups', - 'get_mcp_servers_from_access_groups', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_mcp_servers_from_access_groups", + "get_mcp_servers_from_access_groups", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_get_oauth2_headers_from_headers', - 'get_oauth2_headers_from_headers', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_get_oauth2_headers_from_headers", + "get_oauth2_headers_from_headers", False, ), ( - 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', - 'MCPRequestHandler', - '_safe_get_headers_from_scope', - 'safe_get_headers_from_scope', + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", + "MCPRequestHandler", + "_safe_get_headers_from_scope", + "safe_get_headers_from_scope", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_bridge_mint_error_response', - 'bridge_mint_error_response', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_bridge_mint_error_response", + "bridge_mint_error_response", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_extract_user_id_from_request', - 'extract_user_id_from_request', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_extract_user_id_from_request", + "extract_user_id_from_request", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_finish_bridge_mint', - 'finish_bridge_mint', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_finish_bridge_mint", + "finish_bridge_mint", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_prepare_bridge_mint', - 'prepare_bridge_mint', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_prepare_bridge_mint", + "prepare_bridge_mint", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_prepare_bridge_refresh', - 'prepare_bridge_refresh', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_prepare_bridge_refresh", + "prepare_bridge_refresh", False, ), ( - 'litellm.proxy.mcp_server.bridge_token_flow', - '', - '_reload_active_user_by_id', - 'reload_active_user_by_id', + "litellm.proxy.mcp_server.bridge_token_flow", + "", + "_reload_active_user_by_id", + "reload_active_user_by_id", False, ), ( - 'litellm.proxy.mcp_server.byok_oauth_endpoints', - '', - '_user_id_from_session_cookie', - 'user_id_from_session_cookie', + "litellm.proxy.mcp_server.byok_oauth_endpoints", + "", + "_user_id_from_session_cookie", + "user_id_from_session_cookie", False, ), ( - 'litellm.proxy.mcp_server.db', - '', - '_decode_oauth_payload', - 'decode_oauth_payload', + "litellm.proxy.mcp_server.db", + "", + "_decode_oauth_payload", + "decode_oauth_payload", False, ), ( - 'litellm.proxy.mcp_server.discoverable_endpoints', - '', - '_raise_if_not_oauth2', - 'raise_if_not_oauth2', + "litellm.proxy.mcp_server.discoverable_endpoints", + "", + "_raise_if_not_oauth2", + "raise_if_not_oauth2", False, ), ( - 'litellm.proxy.mcp_server.mcp_context', - '', - '_mcp_active_toolset_id', - 'mcp_active_toolset_id', + "litellm.proxy.mcp_server.mcp_context", + "", + "_mcp_active_toolset_id", + "mcp_active_toolset_id", False, ), ( - 'litellm.proxy.mcp_server.mcp_context', - '', - '_mcp_gateway_initialize_instructions', - 'mcp_gateway_initialize_instructions', + "litellm.proxy.mcp_server.mcp_context", + "", + "_mcp_gateway_initialize_instructions", + "mcp_gateway_initialize_instructions", False, ), ( - 'litellm.proxy.mcp_server.mcp_context', - '', - '_mcp_gateway_server_name', - 'mcp_gateway_server_name', + "litellm.proxy.mcp_server.mcp_context", + "", + "_mcp_gateway_server_name", + "mcp_gateway_server_name", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES', - 'UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES", + "UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_caller_authorization_fans_out', - 'caller_authorization_fans_out', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_caller_authorization_fans_out", + "caller_authorization_fans_out", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_client_forwarded_authorization_headers', - 'client_forwarded_authorization_headers', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_client_forwarded_authorization_headers", + "client_forwarded_authorization_headers", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_resolve_openapi_tool_auth', - 'resolve_openapi_tool_auth', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_resolve_openapi_tool_auth", + "resolve_openapi_tool_auth", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - '', - '_should_strip_caller_authorization', - 'should_strip_caller_authorization', + "litellm.proxy.mcp_server.mcp_server_manager", + "", + "_should_strip_caller_authorization", + "should_strip_caller_authorization", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_build_mcp_server_table', - 'build_mcp_server_table', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_build_mcp_server_table", + "build_mcp_server_table", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_build_stdio_env', - 'build_stdio_env', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_build_stdio_env", + "build_stdio_env", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_create_mcp_client', - 'create_mcp_client', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_create_mcp_client", + "create_mcp_client", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_ensure_upstream_initialize_instructions_cached', - 'ensure_upstream_initialize_instructions_cached', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_ensure_upstream_initialize_instructions_cached", + "ensure_upstream_initialize_instructions_cached", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_extract_subject_token', - 'extract_subject_token', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_extract_subject_token", + "extract_subject_token", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_get_mcp_server_from_tool_name', - 'get_mcp_server_from_tool_name', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_get_mcp_server_from_tool_name", + "get_mcp_server_from_tool_name", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_get_tools_from_server', - 'get_tools_from_server', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_get_tools_from_server", + "get_tools_from_server", False, ), ( - 'litellm.proxy.mcp_server.mcp_server_manager', - 'MCPServerManager', - '_is_server_accessible_from_ip', - 'is_server_accessible_from_ip', + "litellm.proxy.mcp_server.mcp_server_manager", + "MCPServerManager", + "_is_server_accessible_from_ip", + "is_server_accessible_from_ip", False, ), ( - 'litellm.proxy.mcp_server.oauth2_token_cache', - '', - '_compute_per_user_token_ttl', - 'compute_per_user_token_ttl', + "litellm.proxy.mcp_server.oauth2_token_cache", + "", + "_compute_per_user_token_ttl", + "compute_per_user_token_ttl", False, ), ( - 'litellm.proxy.mcp_server.oauth_utils', - '', - '_redact_mcp_resource_url', - 'redact_mcp_resource_url', + "litellm.proxy.mcp_server.oauth_utils", + "", + "_redact_mcp_resource_url", + "redact_mcp_resource_url", False, ), ( - 'litellm.proxy.mcp_server.server', - '', - '_redact_mcp_resource_url', - 'redact_mcp_resource_url', + "litellm.proxy.mcp_server.server", + "", + "_redact_mcp_resource_url", + "redact_mcp_resource_url", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_OPENAPI_TOOL_NAME_MAX_LEN', - 'OPENAPI_TOOL_NAME_MAX_LEN', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_OPENAPI_TOOL_NAME_MAX_LEN", + "OPENAPI_TOOL_NAME_MAX_LEN", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_request_auth_header', - 'request_auth_header', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_request_auth_header", + "request_auth_header", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_request_extra_headers', - 'request_extra_headers', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_request_extra_headers", + "request_extra_headers", False, ), ( - 'litellm.proxy.mcp_server.openapi_to_mcp_generator', - '', - '_request_resolved_auth_headers', - 'request_resolved_auth_headers', + "litellm.proxy.mcp_server.openapi_to_mcp_generator", + "", + "_request_resolved_auth_headers", + "request_resolved_auth_headers", False, ), ( - 'litellm.proxy.mcp_server.semantic_tool_filter', - 'SemanticMCPToolFilter', - '_extract_tool_info', - 'extract_tool_info', + "litellm.proxy.mcp_server.semantic_tool_filter", + "SemanticMCPToolFilter", + "_extract_tool_info", + "extract_tool_info", False, ), ( - 'litellm.proxy.mcp_server.server', - '', - '_apply_toolset_scope', - 'apply_toolset_scope', + "litellm.proxy.mcp_server.server", + "", + "_apply_toolset_scope", + "apply_toolset_scope", False, ), ( - 'litellm.proxy.mcp_server.server_resolution', - 'MCPServerRegistry', - '_build_mcp_server_table', - 'build_mcp_server_table', + "litellm.proxy.mcp_server.server_resolution", + "MCPServerRegistry", + "_build_mcp_server_table", + "build_mcp_server_table", False, ), ( - 'litellm.proxy.mcp_server.server_resolution', - 'MCPServerRegistry', - '_is_server_accessible_from_ip', - 'is_server_accessible_from_ip', + "litellm.proxy.mcp_server.server_resolution", + "MCPServerRegistry", + "_is_server_accessible_from_ip", + "is_server_accessible_from_ip", False, ), ( - 'litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace', - '', - '_get_prisma_client', - 'get_prisma_client', + "litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace", + "", + "_get_prisma_client", + "get_prisma_client", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_cache_access_object', - 'cache_access_object', + "litellm.proxy.auth.auth_checks", + "", + "_cache_access_object", + "cache_access_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_cache_key_object', - 'cache_key_object', + "litellm.proxy.auth.auth_checks", + "", + "_cache_key_object", + "cache_key_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_cache_team_object', - 'cache_team_object', + "litellm.proxy.auth.auth_checks", + "", + "_cache_team_object", + "cache_team_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_can_object_call_model', - 'can_object_call_model', + "litellm.proxy.auth.auth_checks", + "", + "_can_object_call_model", + "can_object_call_model", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_check_end_user_budget', - 'check_end_user_budget', + "litellm.proxy.auth.auth_checks", + "", + "_check_end_user_budget", + "check_end_user_budget", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_check_model_access_helper', - 'check_model_access_helper', + "litellm.proxy.auth.auth_checks", + "", + "_check_model_access_helper", + "check_model_access_helper", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_check_team_member_model_access', - 'check_team_member_model_access', + "litellm.proxy.auth.auth_checks", + "", + "_check_team_member_model_access", + "check_team_member_model_access", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_copy_user_api_key_auth_for_cache', - 'copy_user_api_key_auth_for_cache', + "litellm.proxy.auth.auth_checks", + "", + "_copy_user_api_key_auth_for_cache", + "copy_user_api_key_auth_for_cache", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_delete_cache_access_object', - 'delete_cache_access_object', + "litellm.proxy.auth.auth_checks", + "", + "_delete_cache_access_object", + "delete_cache_access_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_delete_cache_key_object', - 'delete_cache_key_object', + "litellm.proxy.auth.auth_checks", + "", + "_delete_cache_key_object", + "delete_cache_key_object", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_fetch_key_object_from_db_with_reconnect', - 'fetch_key_object_from_db_with_reconnect', + "litellm.proxy.auth.auth_checks", + "", + "_fetch_key_object_from_db_with_reconnect", + "fetch_key_object_from_db_with_reconnect", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_agent_ids_from_access_groups', - 'get_agent_ids_from_access_groups', + "litellm.proxy.auth.auth_checks", + "", + "_get_agent_ids_from_access_groups", + "get_agent_ids_from_access_groups", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_mcp_server_ids_from_access_groups', - 'get_mcp_server_ids_from_access_groups', + "litellm.proxy.auth.auth_checks", + "", + "_get_mcp_server_ids_from_access_groups", + "get_mcp_server_ids_from_access_groups", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_models_from_access_groups', - 'get_models_from_access_groups', + "litellm.proxy.auth.auth_checks", + "", + "_get_models_from_access_groups", + "get_models_from_access_groups", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_team_object_from_cache', - 'get_team_object_from_cache', + "litellm.proxy.auth.auth_checks", + "", + "_get_team_object_from_cache", + "get_team_object_from_cache", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_get_user_role', - 'get_user_role', + "litellm.proxy.auth.auth_checks", + "", + "_get_user_role", + "get_user_role", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_is_model_cost_zero', - 'is_model_cost_zero', + "litellm.proxy.auth.auth_checks", + "", + "_is_model_cost_zero", + "is_model_cost_zero", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_is_user_proxy_admin', - 'is_user_proxy_admin', + "litellm.proxy.auth.auth_checks", + "", + "_is_user_proxy_admin", + "is_user_proxy_admin", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_key_access_group_grants_model', - 'key_access_group_grants_model', + "litellm.proxy.auth.auth_checks", + "", + "_key_access_group_grants_model", + "key_access_group_grants_model", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_organization_max_budget_check', - 'organization_max_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_organization_max_budget_check", + "organization_max_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_team_max_budget_check', - 'team_max_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_team_max_budget_check", + "team_max_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_team_member_max_budget_alert_check', - 'team_member_max_budget_alert_check', + "litellm.proxy.auth.auth_checks", + "", + "_team_member_max_budget_alert_check", + "team_member_max_budget_alert_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_virtual_key_max_budget_alert_check', - 'virtual_key_max_budget_alert_check', + "litellm.proxy.auth.auth_checks", + "", + "_virtual_key_max_budget_alert_check", + "virtual_key_max_budget_alert_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_virtual_key_max_budget_check', - 'virtual_key_max_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_virtual_key_max_budget_check", + "virtual_key_max_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks', - '', - '_virtual_key_soft_budget_check', - 'virtual_key_soft_budget_check', + "litellm.proxy.auth.auth_checks", + "", + "_virtual_key_soft_budget_check", + "virtual_key_soft_budget_check", False, ), ( - 'litellm.proxy.auth.auth_checks_organization', - '', - '_user_is_org_admin', - 'user_is_org_admin', + "litellm.proxy.auth.auth_checks_organization", + "", + "_user_is_org_admin", + "user_is_org_admin", False, ), ( - 'litellm.proxy.auth.auth_exception_handler', - 'UserAPIKeyAuthExceptionHandler', - '_handle_authentication_error', - 'handle_authentication_error', + "litellm.proxy.auth.auth_exception_handler", + "UserAPIKeyAuthExceptionHandler", + "_handle_authentication_error", + "handle_authentication_error", False, ), ( - 'litellm.proxy.auth.auth_utils', - '', - '_get_request_ip_address', - 'get_request_ip_address', + "litellm.proxy.auth.auth_utils", + "", + "_get_request_ip_address", + "get_request_ip_address", False, ), ( - 'litellm.proxy.auth.resolvers.store', - 'IdentityStore', - '_principal_from_key', - 'principal_from_key', + "litellm.proxy.auth.resolvers.store", + "IdentityStore", + "_principal_from_key", + "principal_from_key", False, ), ( - 'litellm.proxy.auth.route_checks', - 'RouteChecks', - '_get_request_method', - 'get_request_method', + "litellm.proxy.auth.route_checks", + "RouteChecks", + "_get_request_method", + "get_request_method", False, ), ( - 'litellm.proxy.auth.route_checks', - 'RouteChecks', - '_is_assistants_api_request', - 'is_assistants_api_request', + "litellm.proxy.auth.route_checks", + "RouteChecks", + "_is_assistants_api_request", + "is_assistants_api_request", False, ), ( - 'litellm.proxy.auth.route_checks', - 'RouteChecks', - '_is_wildcard_pattern', - 'is_wildcard_pattern', + "litellm.proxy.auth.route_checks", + "RouteChecks", + "_is_wildcard_pattern", + "is_wildcard_pattern", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_enforce_key_and_fallback_model_access', - 'enforce_key_and_fallback_model_access', + "litellm.proxy.auth.user_api_key_auth", + "", + "_enforce_key_and_fallback_model_access", + "enforce_key_and_fallback_model_access", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_fetch_global_spend_with_event_coordination', - 'fetch_global_spend_with_event_coordination', + "litellm.proxy.auth.user_api_key_auth", + "", + "_fetch_global_spend_with_event_coordination", + "fetch_global_spend_with_event_coordination", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_get_bearer_token', - 'get_bearer_token', + "litellm.proxy.auth.user_api_key_auth", + "", + "_get_bearer_token", + "get_bearer_token", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_run_centralized_common_checks', - 'run_centralized_common_checks', + "litellm.proxy.auth.user_api_key_auth", + "", + "_run_centralized_common_checks", + "run_centralized_common_checks", False, ), ( - 'litellm.proxy.auth.user_api_key_auth', - '', - '_user_api_key_auth_builder', - 'user_api_key_auth_builder', + "litellm.proxy.auth.user_api_key_auth", + "", + "_user_api_key_auth_builder", + "user_api_key_auth_builder", False, ), ( - 'litellm.proxy.common_request_processing', - '', - '_is_azure_model_router_request', - 'is_azure_model_router_request', + "litellm.proxy.common_request_processing", + "", + "_is_azure_model_router_request", + "is_azure_model_router_request", False, ), ( - 'litellm.proxy.common_request_processing', - '', - '_should_return_raw_model_name', - 'should_return_raw_model_name', + "litellm.proxy.common_request_processing", + "", + "_should_return_raw_model_name", + "should_return_raw_model_name", False, ), ( - 'litellm.proxy.common_request_processing', - 'ProxyBaseLLMRequestProcessing', - '_finalize_streaming_generator_cleanup', - 'finalize_streaming_generator_cleanup', + "litellm.proxy.common_request_processing", + "ProxyBaseLLMRequestProcessing", + "_finalize_streaming_generator_cleanup", + "finalize_streaming_generator_cleanup", False, ), ( - 'litellm.proxy.common_request_processing', - 'ProxyBaseLLMRequestProcessing', - '_handle_llm_api_exception', - 'handle_llm_api_exception', + "litellm.proxy.common_request_processing", + "ProxyBaseLLMRequestProcessing", + "_handle_llm_api_exception", + "handle_llm_api_exception", False, ), ( - 'litellm.proxy.common_request_processing', - 'ProxyBaseLLMRequestProcessing', - '_process_chunk_with_cost_injection', - 'process_chunk_with_cost_injection', + "litellm.proxy.common_request_processing", + "ProxyBaseLLMRequestProcessing", + "_process_chunk_with_cost_injection", + "process_chunk_with_cost_injection", False, ), ( - 'litellm.proxy.common_utils.callback_utils', - '', - '_CALLBACK_VAR_ENCRYPTED_PREFIX', - 'CALLBACK_VAR_ENCRYPTED_PREFIX', + "litellm.proxy.common_utils.callback_utils", + "", + "_CALLBACK_VAR_ENCRYPTED_PREFIX", + "CALLBACK_VAR_ENCRYPTED_PREFIX", False, ), ( - 'litellm.proxy.common_utils.config_sync_pubsub', - '', - '_ConfigSyncPubSub', - 'ConfigSyncPubSub', + "litellm.proxy.common_utils.config_sync_pubsub", + "", + "_ConfigSyncPubSub", + "ConfigSyncPubSub", False, ), ( - 'litellm.proxy.common_utils.config_sync_pubsub', - '', - '_pubsub_capable_client', - 'pubsub_capable_client', + "litellm.proxy.common_utils.config_sync_pubsub", + "", + "_pubsub_capable_client", + "pubsub_capable_client", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_ALGO_AES_GCM', - 'ALGO_AES_GCM', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_ALGO_AES_GCM", + "ALGO_AES_GCM", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_ENCRYPTION_ALGORITHM_SETTING', - 'ENCRYPTION_ALGORITHM_SETTING', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_ENCRYPTION_ALGORITHM_SETTING", + "ENCRYPTION_ALGORITHM_SETTING", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_V2_GCM_PREFIX', - 'V2_GCM_PREFIX', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_V2_GCM_PREFIX", + "V2_GCM_PREFIX", False, ), ( - 'litellm.proxy.common_utils.encrypt_decrypt_utils', - '', - '_get_salt_key', - 'get_salt_key', + "litellm.proxy.common_utils.encrypt_decrypt_utils", + "", + "_get_salt_key", + "get_salt_key", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_read_request_body', - 'read_request_body', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_read_request_body", + "read_request_body", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_safe_get_request_headers', - 'safe_get_request_headers', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_safe_get_request_headers", + "safe_get_request_headers", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_safe_get_request_query_params', - 'safe_get_request_query_params', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_safe_get_request_query_params", + "safe_get_request_query_params", False, ), ( - 'litellm.proxy.common_utils.http_parsing_utils', - '', - '_safe_set_request_parsed_body', - 'safe_set_request_parsed_body', + "litellm.proxy.common_utils.http_parsing_utils", + "", + "_safe_set_request_parsed_body", + "safe_set_request_parsed_body", False, ), ( - 'litellm.proxy.common_utils.realtime_utils', - '', - '_realtime_request_body', - 'realtime_request_body', + "litellm.proxy.common_utils.realtime_utils", + "", + "_realtime_request_body", + "realtime_request_body", False, ), ( - 'litellm.proxy.db.db_spend_update_writer', - 'DBSpendUpdateWriter', - '_commit_daily_tag_spend_to_db', - 'commit_daily_tag_spend_to_db', + "litellm.proxy.db.db_spend_update_writer", + "DBSpendUpdateWriter", + "_commit_daily_tag_spend_to_db", + "commit_daily_tag_spend_to_db", False, ), ( - 'litellm.proxy.db.db_spend_update_writer', - 'DBSpendUpdateWriter', - '_commit_daily_tag_spend_to_db_with_redis', - 'commit_daily_tag_spend_to_db_with_redis', + "litellm.proxy.db.db_spend_update_writer", + "DBSpendUpdateWriter", + "_commit_daily_tag_spend_to_db_with_redis", + "commit_daily_tag_spend_to_db_with_redis", False, ), ( - 'litellm.proxy.db.db_spend_update_writer', - 'DBSpendUpdateWriter', - '_handle_spend_update_failure', - 'handle_spend_update_failure', + "litellm.proxy.db.db_spend_update_writer", + "DBSpendUpdateWriter", + "_handle_spend_update_failure", + "handle_spend_update_failure", False, ), ( - 'litellm.proxy.db.db_transaction_queue.redis_update_buffer', - 'RedisUpdateBuffer', - '_should_commit_spend_updates_to_redis', - 'should_commit_spend_updates_to_redis', + "litellm.proxy.db.db_transaction_queue.redis_update_buffer", + "RedisUpdateBuffer", + "_should_commit_spend_updates_to_redis", + "should_commit_spend_updates_to_redis", False, ), ( - 'litellm.proxy.db.log_db_metrics', - '', - '_is_exception_related_to_db', - 'is_exception_related_to_db', + "litellm.proxy.db.log_db_metrics", + "", + "_is_exception_related_to_db", + "is_exception_related_to_db", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.azure.base', - '', - '_RESPONSES_API_CALL_TYPES', - 'RESPONSES_API_CALL_TYPES', + "litellm.proxy.guardrails.guardrail_hooks.azure.base", + "", + "_RESPONSES_API_CALL_TYPES", + "RESPONSES_API_CALL_TYPES", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.cisco_ai_defense.cisco_ai_defense_mcp', - '', - '_CiscoAIDefenseMcpMixin', - 'CiscoAIDefenseMcpMixin', + "litellm.proxy.guardrails.guardrail_hooks.cisco_ai_defense.cisco_ai_defense_mcp", + "", + "_CiscoAIDefenseMcpMixin", + "CiscoAIDefenseMcpMixin", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base', - '', - '_compile_marker', - 'compile_marker', + "litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base", + "", + "_compile_marker", + "compile_marker", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base', - '', - '_count_signals', - 'count_signals', + "litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base", + "", + "_count_signals", + "count_signals", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base', - '', - '_word_boundary_match', - 'word_boundary_match', + "litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base", + "", + "_word_boundary_match", + "word_boundary_match", False, ), ( - 'litellm.proxy.guardrails.guardrail_hooks.presidio', - '', - '_OPTIONAL_PresidioPIIMasking', - 'OPTIONAL_PresidioPIIMasking', + "litellm.proxy.guardrails.guardrail_hooks.presidio", + "", + "_OPTIONAL_PresidioPIIMasking", + "OPTIONAL_PresidioPIIMasking", False, ), ( - 'litellm.proxy.health_check', - '', - '_clean_endpoint_data', - 'clean_endpoint_data', + "litellm.proxy.health_check", + "", + "_clean_endpoint_data", + "clean_endpoint_data", False, ), ( - 'litellm.proxy.health_check', - '', - '_update_litellm_params_for_health_check', - 'update_litellm_params_for_health_check', + "litellm.proxy.health_check", + "", + "_update_litellm_params_for_health_check", + "update_litellm_params_for_health_check", False, ), ( - 'litellm.proxy.health_endpoints._health_endpoints', - '', - '_convert_health_check_to_dict', - 'convert_health_check_to_dict', + "litellm.proxy.health_endpoints._health_endpoints", + "", + "_convert_health_check_to_dict", + "convert_health_check_to_dict", False, ), ( - 'litellm.proxy.health_endpoints._health_endpoints', - '', - '_save_background_health_checks_to_db', - 'save_background_health_checks_to_db', + "litellm.proxy.health_endpoints._health_endpoints", + "", + "_save_background_health_checks_to_db", + "save_background_health_checks_to_db", False, ), ( - 'litellm.proxy.hooks.azure_content_safety', - '', - '_PROXY_AzureContentSafety', - 'PROXY_AzureContentSafety', + "litellm.proxy.hooks.azure_content_safety", + "", + "_PROXY_AzureContentSafety", + "PROXY_AzureContentSafety", False, ), ( - 'litellm.proxy.hooks.batch_rate_limiter', - '', - '_PROXY_BatchRateLimiter', - 'PROXY_BatchRateLimiter', + "litellm.proxy.hooks.batch_rate_limiter", + "", + "_PROXY_BatchRateLimiter", + "PROXY_BatchRateLimiter", False, ), ( - 'litellm.proxy.hooks.batch_redis_get', - '', - '_PROXY_BatchRedisRequests', - 'PROXY_BatchRedisRequests', + "litellm.proxy.hooks.batch_redis_get", + "", + "_PROXY_BatchRedisRequests", + "PROXY_BatchRedisRequests", False, ), ( - 'litellm.proxy.hooks.cache_control_check', - '', - '_PROXY_CacheControlCheck', - 'PROXY_CacheControlCheck', + "litellm.proxy.hooks.cache_control_check", + "", + "_PROXY_CacheControlCheck", + "PROXY_CacheControlCheck", False, ), ( - 'litellm.proxy.hooks.dynamic_rate_limiter', - '', - '_PROXY_DynamicRateLimitHandler', - 'PROXY_DynamicRateLimitHandler', + "litellm.proxy.hooks.dynamic_rate_limiter", + "", + "_PROXY_DynamicRateLimitHandler", + "PROXY_DynamicRateLimitHandler", False, ), ( - 'litellm.proxy.hooks.dynamic_rate_limiter_v3', - '', - '_PROXY_DynamicRateLimitHandlerV3', - 'PROXY_DynamicRateLimitHandlerV3', + "litellm.proxy.hooks.dynamic_rate_limiter_v3", + "", + "_PROXY_DynamicRateLimitHandlerV3", + "PROXY_DynamicRateLimitHandlerV3", False, ), ( - 'litellm.proxy.hooks.max_budget_per_session_limiter', - '', - '_PROXY_MaxBudgetPerSessionHandler', - 'PROXY_MaxBudgetPerSessionHandler', + "litellm.proxy.hooks.max_budget_per_session_limiter", + "", + "_PROXY_MaxBudgetPerSessionHandler", + "PROXY_MaxBudgetPerSessionHandler", False, ), ( - 'litellm.proxy.hooks.max_iterations_limiter', - '', - '_PROXY_MaxIterationsHandler', - 'PROXY_MaxIterationsHandler', + "litellm.proxy.hooks.max_iterations_limiter", + "", + "_PROXY_MaxIterationsHandler", + "PROXY_MaxIterationsHandler", False, ), ( - 'litellm.proxy.hooks.model_max_budget_limiter', - '', - '_PROXY_VirtualKeyModelMaxBudgetLimiter', - 'PROXY_VirtualKeyModelMaxBudgetLimiter', + "litellm.proxy.hooks.model_max_budget_limiter", + "", + "_PROXY_VirtualKeyModelMaxBudgetLimiter", + "PROXY_VirtualKeyModelMaxBudgetLimiter", False, ), ( - 'litellm.proxy.hooks.parallel_request_limiter', - '', - '_PROXY_MaxParallelRequestsHandler', - 'PROXY_MaxParallelRequestsHandler', + "litellm.proxy.hooks.parallel_request_limiter", + "", + "_PROXY_MaxParallelRequestsHandler", + "PROXY_MaxParallelRequestsHandler", False, ), ( - 'litellm.proxy.hooks.parallel_request_limiter_v3', - '', - '_PROXY_MaxParallelRequestsHandler_v3', - 'PROXY_MaxParallelRequestsHandler_v3', + "litellm.proxy.hooks.parallel_request_limiter_v3", + "", + "_PROXY_MaxParallelRequestsHandler_v3", + "PROXY_MaxParallelRequestsHandler_v3", False, ), ( - 'litellm.proxy.hooks.parallel_request_limiter_v3', - '_PROXY_MaxParallelRequestsHandler_v3', - '_create_rate_limit_descriptors', - 'create_rate_limit_descriptors', + "litellm.proxy.hooks.parallel_request_limiter_v3", + "_PROXY_MaxParallelRequestsHandler_v3", + "_create_rate_limit_descriptors", + "create_rate_limit_descriptors", False, ), ( - 'litellm.proxy.hooks.prompt_injection_detection', - '', - '_OPTIONAL_PromptInjectionDetection', - 'OPTIONAL_PromptInjectionDetection', + "litellm.proxy.hooks.prompt_injection_detection", + "", + "_OPTIONAL_PromptInjectionDetection", + "OPTIONAL_PromptInjectionDetection", False, ), ( - 'litellm.proxy.hooks.proxy_track_cost_callback', - '', - '_ProxyDBLogger', - 'ProxyDBLogger', + "litellm.proxy.hooks.proxy_track_cost_callback", + "", + "_ProxyDBLogger", + "ProxyDBLogger", False, ), ( - 'litellm.proxy.hooks.sensitive_data_routing', - '', - '_PROXY_SensitiveDataRoutingHandler', - 'PROXY_SensitiveDataRoutingHandler', + "litellm.proxy.hooks.sensitive_data_routing", + "", + "_PROXY_SensitiveDataRoutingHandler", + "PROXY_SensitiveDataRoutingHandler", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_add_guardrails_from_key_or_team_metadata', - 'add_guardrails_from_key_or_team_metadata', + "litellm.proxy.litellm_pre_call_utils", + "", + "_add_guardrails_from_key_or_team_metadata", + "add_guardrails_from_key_or_team_metadata", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_get_dynamic_logging_metadata', - 'get_dynamic_logging_metadata', + "litellm.proxy.litellm_pre_call_utils", + "", + "_get_dynamic_logging_metadata", + "get_dynamic_logging_metadata", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_get_metadata_variable_name', - 'get_metadata_variable_name', + "litellm.proxy.litellm_pre_call_utils", + "", + "_get_metadata_variable_name", + "get_metadata_variable_name", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - '', - '_get_validated_callback_metadata', - 'get_validated_callback_metadata', + "litellm.proxy.litellm_pre_call_utils", + "", + "_get_validated_callback_metadata", + "get_validated_callback_metadata", False, ), ( - 'litellm.proxy.litellm_pre_call_utils', - 'LiteLLMProxyRequestSetup', - '_merge_tags', - 'merge_tags', + "litellm.proxy.litellm_pre_call_utils", + "LiteLLMProxyRequestSetup", + "_merge_tags", + "merge_tags", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_check_disable_global_guardrails_caller_permission', - 'check_disable_global_guardrails_caller_permission', + "litellm.proxy.management_endpoints.common_utils", + "", + "_check_disable_global_guardrails_caller_permission", + "check_disable_global_guardrails_caller_permission", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_check_passthrough_routes_caller_permission', - 'check_passthrough_routes_caller_permission', + "litellm.proxy.management_endpoints.common_utils", + "", + "_check_passthrough_routes_caller_permission", + "check_passthrough_routes_caller_permission", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_set_object_metadata_field', - 'set_object_metadata_field', + "litellm.proxy.management_endpoints.common_utils", + "", + "_set_object_metadata_field", + "set_object_metadata_field", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_team_member_has_permission', - 'team_member_has_permission', + "litellm.proxy.management_endpoints.common_utils", + "", + "_team_member_has_permission", + "team_member_has_permission", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_update_metadata_fields', - 'update_metadata_fields', + "litellm.proxy.management_endpoints.common_utils", + "", + "_update_metadata_fields", + "update_metadata_fields", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_upsert_budget_and_membership', - 'upsert_budget_and_membership', + "litellm.proxy.management_endpoints.common_utils", + "", + "_upsert_budget_and_membership", + "upsert_budget_and_membership", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_user_has_admin_privileges', - 'user_has_admin_privileges', + "litellm.proxy.management_endpoints.common_utils", + "", + "_user_has_admin_privileges", + "user_has_admin_privileges", False, ), ( - 'litellm.proxy.management_endpoints.common_utils', - '', - '_user_has_admin_view', - 'user_api_key_has_admin_view', + "litellm.proxy.management_endpoints.common_utils", + "", + "_user_has_admin_view", + "user_api_key_has_admin_view", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_clear_hashicorp_vault_state', - 'clear_hashicorp_vault_state', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_clear_hashicorp_vault_state", + "clear_hashicorp_vault_state", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_get_current_env_values', - 'get_current_env_values', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_get_current_env_values", + "get_current_env_values", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_parse_config_value', - 'parse_config_value', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_parse_config_value", + "parse_config_value", False, ), ( - 'litellm.proxy.management_endpoints.config_override_endpoints', - '', - '_set_env_vars', - 'set_env_vars', + "litellm.proxy.management_endpoints.config_override_endpoints", + "", + "_set_env_vars", + "set_env_vars", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_calculate_key_rotation_time', - 'calculate_key_rotation_time', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_calculate_key_rotation_time", + "calculate_key_rotation_time", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_check_permissions_caller_permission', - 'check_permissions_caller_permission', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_check_permissions_caller_permission", + "check_permissions_caller_permission", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_get_caller_team_role', - 'get_caller_team_role', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_get_caller_team_role", + "get_caller_team_role", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_list_key_helper', - 'list_key_helper', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_list_key_helper", + "list_key_helper", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_persist_deleted_verification_tokens', - 'persist_deleted_verification_tokens', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_persist_deleted_verification_tokens", + "persist_deleted_verification_tokens", False, ), ( - 'litellm.proxy.management_endpoints.key_management_endpoints', - '', - '_rotate_master_key', - 'rotate_master_key', + "litellm.proxy.management_endpoints.key_management_endpoints", + "", + "_rotate_master_key", + "rotate_master_key", False, ), ( - 'litellm.proxy.management_endpoints.mcp_management_endpoints', - '', - '_inherit_credentials_from_existing_server', - 'inherit_credentials_from_existing_server', + "litellm.proxy.management_endpoints.mcp_management_endpoints", + "", + "_inherit_credentials_from_existing_server", + "inherit_credentials_from_existing_server", False, ), ( - 'litellm.proxy.management_endpoints.model_management_endpoints', - '', - '_add_model_to_db', - 'add_model_to_db', + "litellm.proxy.management_endpoints.model_management_endpoints", + "", + "_add_model_to_db", + "add_model_to_db", False, ), ( - 'litellm.proxy.management_endpoints.model_management_endpoints', - '', - '_add_team_model_to_db', - 'add_team_model_to_db', + "litellm.proxy.management_endpoints.model_management_endpoints", + "", + "_add_team_model_to_db", + "add_team_model_to_db", False, ), ( - 'litellm.proxy.management_endpoints.model_management_endpoints', - '', - '_deduplicate_litellm_router_models', - 'deduplicate_litellm_router_models', + "litellm.proxy.management_endpoints.model_management_endpoints", + "", + "_deduplicate_litellm_router_models", + "deduplicate_litellm_router_models", False, ), ( - 'litellm.proxy.management_endpoints.organization_endpoints', - '', - '_verify_org_access', - 'verify_org_access', + "litellm.proxy.management_endpoints.organization_endpoints", + "", + "_verify_org_access", + "verify_org_access", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_all_names_per_competitor', - 'build_all_names_per_competitor', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_all_names_per_competitor", + "build_all_names_per_competitor", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_comparison_blocked_words', - 'build_comparison_blocked_words', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_comparison_blocked_words", + "build_comparison_blocked_words", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_competitor_guardrail_definitions', - 'build_competitor_guardrail_definitions', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_competitor_guardrail_definitions", + "build_competitor_guardrail_definitions", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_name_blocked_words', - 'build_name_blocked_words', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_name_blocked_words", + "build_name_blocked_words", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_recommendation_blocked_words', - 'build_recommendation_blocked_words', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_recommendation_blocked_words", + "build_recommendation_blocked_words", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_build_refinement_prompt', - 'build_refinement_prompt', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_build_refinement_prompt", + "build_refinement_prompt", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_clean_competitor_line', - 'clean_competitor_line', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_clean_competitor_line", + "clean_competitor_line", False, ), ( - 'litellm.proxy.management_endpoints.policy_endpoints.endpoints', - '', - '_parse_variations_response', - 'parse_variations_response', + "litellm.proxy.management_endpoints.policy_endpoints.endpoints", + "", + "_parse_variations_response", + "parse_variations_response", False, ), ( - 'litellm.proxy.management_endpoints.team_endpoints', - '', - '_cleanup_members_with_roles', - 'cleanup_members_with_roles', + "litellm.proxy.management_endpoints.team_endpoints", + "", + "_cleanup_members_with_roles", + "cleanup_members_with_roles", False, ), ( - 'litellm.proxy.management_endpoints.team_endpoints', - '', - '_refresh_cached_team', - 'refresh_cached_team', + "litellm.proxy.management_endpoints.team_endpoints", + "", + "_refresh_cached_team", + "refresh_cached_team", False, ), ( - 'litellm.proxy.management_endpoints.team_endpoints', - 'TeamMemberBudgetHandler', - '_clean_team_member_fields', - 'clean_team_member_fields', + "litellm.proxy.management_endpoints.team_endpoints", + "TeamMemberBudgetHandler", + "_clean_team_member_fields", + "clean_team_member_fields", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - '', - '_sso_return_to_redirect', - 'sso_return_to_redirect', + "litellm.proxy.management_endpoints.ui_sso", + "", + "_sso_return_to_redirect", + "sso_return_to_redirect", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_delete_pkce_verifier', - 'delete_pkce_verifier', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_delete_pkce_verifier", + "delete_pkce_verifier", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_get_cli_state', - 'get_cli_state', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_get_cli_state", + "get_cli_state", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_get_user_email_and_id_from_result', - 'get_user_email_and_id_from_result', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_get_user_email_and_id_from_result", + "get_user_email_and_id_from_result", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_pkce_token_exchange', - 'pkce_token_exchange', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_pkce_token_exchange", + "pkce_token_exchange", False, ), ( - 'litellm.proxy.management_endpoints.ui_sso', - 'SSOAuthenticationHandler', - '_validate_return_to', - 'validate_return_to', + "litellm.proxy.management_endpoints.ui_sso", + "SSOAuthenticationHandler", + "_validate_return_to", + "validate_return_to", False, ), ( - 'litellm.proxy.management_helpers.object_permission_utils', - '', - '_get_allow_all_keys_server_ids', - 'get_allow_all_keys_server_ids', + "litellm.proxy.management_helpers.object_permission_utils", + "", + "_get_allow_all_keys_server_ids", + "get_allow_all_keys_server_ids", False, ), ( - 'litellm.proxy.management_helpers.object_permission_utils', - '', - '_get_team_allowed_mcp_servers', - 'get_team_allowed_mcp_servers', + "litellm.proxy.management_helpers.object_permission_utils", + "", + "_get_team_allowed_mcp_servers", + "get_team_allowed_mcp_servers", False, ), ( - 'litellm.proxy.management_helpers.object_permission_utils', - '', - '_set_object_permission', - 'set_object_permission', + "litellm.proxy.management_helpers.object_permission_utils", + "", + "_set_object_permission", + "set_object_permission", False, ), ( - 'litellm.proxy.openai_files_endpoints.common_utils', - '', - '_is_base64_encoded_unified_file_id', - 'is_base64_encoded_unified_file_id', + "litellm.proxy.openai_files_endpoints.common_utils", + "", + "_is_base64_encoded_unified_file_id", + "is_base64_encoded_unified_file_id", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints', - '', - '_extract_model_from_bedrock_endpoint', - 'extract_model_from_bedrock_endpoint', + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints", + "", + "_extract_model_from_bedrock_endpoint", + "extract_model_from_bedrock_endpoint", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints', - 'BaseOpenAIPassThroughHandler', - '_base_openai_pass_through_handler', - 'base_openai_pass_through_handler', + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints", + "BaseOpenAIPassThroughHandler", + "_base_openai_pass_through_handler", + "base_openai_pass_through_handler", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler', - 'AnthropicPassthroughLoggingHandler', - '_build_complete_streaming_response', - 'build_complete_streaming_response', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler", + "AnthropicPassthroughLoggingHandler", + "_build_complete_streaming_response", + "build_complete_streaming_response", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler', - 'AnthropicPassthroughLoggingHandler', - '_build_usage_only_response_from_chunks', - 'build_usage_only_response_from_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler", + "AnthropicPassthroughLoggingHandler", + "_build_usage_only_response_from_chunks", + "build_usage_only_response_from_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler', - 'AnthropicPassthroughLoggingHandler', - '_handle_logging_anthropic_collected_chunks', - 'handle_logging_anthropic_collected_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler", + "AnthropicPassthroughLoggingHandler", + "_handle_logging_anthropic_collected_chunks", + "handle_logging_anthropic_collected_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler', - 'AssemblyAIPassthroughLoggingHandler', - '_get_assembly_base_url_from_region', - 'get_assembly_base_url_from_region', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler", + "AssemblyAIPassthroughLoggingHandler", + "_get_assembly_base_url_from_region", + "get_assembly_base_url_from_region", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler', - 'AssemblyAIPassthroughLoggingHandler', - '_get_assembly_region_from_url', - 'get_assembly_region_from_url', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler", + "AssemblyAIPassthroughLoggingHandler", + "_get_assembly_region_from_url", + "get_assembly_region_from_url", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler', - 'AssemblyAIPassthroughLoggingHandler', - '_should_log_request', - 'should_log_request', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler", + "AssemblyAIPassthroughLoggingHandler", + "_should_log_request", + "should_log_request", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler', - '', - '_is_openai_compatible_url', - 'is_openai_compatible_url', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler", + "", + "_is_openai_compatible_url", + "is_openai_compatible_url", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler', - 'OpenAIPassthroughLoggingHandler', - '_handle_logging_openai_collected_chunks', - 'handle_logging_openai_collected_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler", + "OpenAIPassthroughLoggingHandler", + "_handle_logging_openai_collected_chunks", + "handle_logging_openai_collected_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler', - 'VertexPassthroughLoggingHandler', - '_handle_logging_vertex_collected_chunks', - 'handle_logging_vertex_collected_chunks', + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler", + "VertexPassthroughLoggingHandler", + "_handle_logging_vertex_collected_chunks", + "handle_logging_vertex_collected_chunks", False, ), ( - 'litellm.proxy.pass_through_endpoints.pass_through_endpoints', - 'HttpPassThroughEndpointHelpers', - '_init_kwargs_for_pass_through_endpoint', - 'init_kwargs_for_pass_through_endpoint', + "litellm.proxy.pass_through_endpoints.pass_through_endpoints", + "HttpPassThroughEndpointHelpers", + "_init_kwargs_for_pass_through_endpoint", + "init_kwargs_for_pass_through_endpoint", False, ), ( - 'litellm.proxy.pass_through_endpoints.pass_through_endpoints', - 'HttpPassThroughEndpointHelpers', - '_update_stream_param_based_on_request_body', - 'update_stream_param_based_on_request_body', + "litellm.proxy.pass_through_endpoints.pass_through_endpoints", + "HttpPassThroughEndpointHelpers", + "_update_stream_param_based_on_request_body", + "update_stream_param_based_on_request_body", False, ), ( - 'litellm.proxy.pass_through_endpoints.streaming_handler', - 'PassThroughStreamingHandler', - '_route_streaming_logging_to_handler', - 'route_streaming_logging_to_handler', + "litellm.proxy.pass_through_endpoints.streaming_handler", + "PassThroughStreamingHandler", + "_route_streaming_logging_to_handler", + "route_streaming_logging_to_handler", False, ), ( - 'litellm.proxy.pass_through_endpoints.success_handler', - 'PassThroughEndpointLogging', - '_handle_logging', - 'handle_logging', + "litellm.proxy.pass_through_endpoints.success_handler", + "PassThroughEndpointLogging", + "_handle_logging", + "handle_logging", False, ), ( - 'litellm.proxy.policy_engine.policy_registry', - 'PolicyRegistry', - '_parse_policy', - 'parse_policy', + "litellm.proxy.policy_engine.policy_registry", + "PolicyRegistry", + "_parse_policy", + "parse_policy", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_apply_uvicorn_max_requests_jitter', - 'apply_uvicorn_max_requests_jitter', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_apply_uvicorn_max_requests_jitter", + "apply_uvicorn_max_requests_jitter", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_configure_dev_reload', - 'configure_dev_reload', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_configure_dev_reload", + "configure_dev_reload", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_echo_litellm_version', - 'echo_litellm_version', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_echo_litellm_version", + "echo_litellm_version", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_get_default_unvicorn_init_args', - 'get_default_unvicorn_init_args', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_get_default_unvicorn_init_args", + "get_default_unvicorn_init_args", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_get_loop_type', - 'get_loop_type', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_get_loop_type", + "get_loop_type", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_init_granian_server', - 'init_granian_server', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_init_granian_server", + "init_granian_server", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_init_hypercorn_server', - 'init_hypercorn_server', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_init_hypercorn_server", + "init_hypercorn_server", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_is_port_in_use', - 'is_port_in_use', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_is_port_in_use", + "is_port_in_use", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_maybe_setup_prometheus_multiproc_dir', - 'maybe_setup_prometheus_multiproc_dir', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_maybe_setup_prometheus_multiproc_dir", + "maybe_setup_prometheus_multiproc_dir", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_config_validation', - 'run_config_validation', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_config_validation", + "run_config_validation", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_gunicorn_server', - 'run_gunicorn_server', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_gunicorn_server", + "run_gunicorn_server", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_health_check', - 'run_health_check', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_health_check", + "run_health_check", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_ollama_serve', - 'run_ollama_serve', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_ollama_serve", + "run_ollama_serve", False, ), ( - 'litellm.proxy.proxy_cli', - 'ProxyInitializationHelpers', - '_run_test_chat_completion', - 'run_test_chat_completion', + "litellm.proxy.proxy_cli", + "ProxyInitializationHelpers", + "_run_test_chat_completion", + "run_test_chat_completion", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_build_redis_usage_cache', - 'build_redis_usage_cache', + "litellm.proxy.proxy_server", + "", + "_build_redis_usage_cache", + "build_redis_usage_cache", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_ensure_spend_counter_initialized', - 'ensure_spend_counter_initialized', + "litellm.proxy.proxy_server", + "", + "_ensure_spend_counter_initialized", + "ensure_spend_counter_initialized", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_ensure_window_spend_counter_initialized', - 'ensure_window_spend_counter_initialized', + "litellm.proxy.proxy_server", + "", + "_ensure_window_spend_counter_initialized", + "ensure_window_spend_counter_initialized", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_environment_has_redis_connection_target', - 'environment_has_redis_connection_target', + "litellm.proxy.proxy_server", + "", + "_environment_has_redis_connection_target", + "environment_has_redis_connection_target", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_get_model_group_info', - 'get_model_group_info', + "litellm.proxy.proxy_server", + "", + "_get_model_group_info", + "get_model_group_info", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_increment_spend_counter_cache', - 'increment_spend_counter_cache', + "litellm.proxy.proxy_server", + "", + "_increment_spend_counter_cache", + "increment_spend_counter_cache", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_initialize_shared_aiohttp_session', - 'initialize_shared_aiohttp_session', + "litellm.proxy.proxy_server", + "", + "_initialize_shared_aiohttp_session", + "initialize_shared_aiohttp_session", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_invalidate_spend_counter', - 'invalidate_spend_counter', + "litellm.proxy.proxy_server", + "", + "_invalidate_spend_counter", + "invalidate_spend_counter", False, ), ( - 'litellm.proxy.proxy_server', - '', - '_title', - 'title', + "litellm.proxy.proxy_server", + "", + "_title", + "title", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_add_deployment_locked', - 'add_deployment_locked', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_add_deployment_locked", + "add_deployment_locked", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_decrypt_and_set_db_env_variables', - 'decrypt_and_set_db_env_variables', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_decrypt_and_set_db_env_variables", + "decrypt_and_set_db_env_variables", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_decrypt_db_variables', - 'decrypt_db_variables', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_decrypt_db_variables", + "decrypt_db_variables", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_encrypt_env_variables', - 'encrypt_env_variables', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_encrypt_env_variables", + "encrypt_env_variables", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_encrypt_env_variables_for_db', - 'encrypt_env_variables_for_db', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_encrypt_env_variables_for_db", + "encrypt_env_variables_for_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_get_models_from_db', - 'get_models_from_db', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_get_models_from_db", + "get_models_from_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_init_cache', - 'init_cache', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_init_cache", + "init_cache", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_init_semantic_filter_settings_in_db', - 'init_semantic_filter_settings_in_db', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_init_semantic_filter_settings_in_db", + "init_semantic_filter_settings_in_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyConfig', - '_serve_pass_through_endpoints', - 'serve_pass_through_endpoints', + "litellm.proxy.proxy_server", + "ProxyConfig", + "_serve_pass_through_endpoints", + "serve_pass_through_endpoints", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_add_proxy_budget_to_db', - 'add_proxy_budget_to_db', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_add_proxy_budget_to_db", + "add_proxy_budget_to_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_attach_router_to_prompt_injection_detectors', - 'attach_router_to_prompt_injection_detectors', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_attach_router_to_prompt_injection_detectors", + "attach_router_to_prompt_injection_detectors", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_get_transaction_buffer_redis_cache', - 'get_transaction_buffer_redis_cache', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_get_transaction_buffer_redis_cache", + "get_transaction_buffer_redis_cache", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_init_coordination_redis_from_db', - 'init_coordination_redis_from_db', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_init_coordination_redis_from_db", + "init_coordination_redis_from_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_init_dd_tracer', - 'init_dd_tracer', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_init_dd_tracer", + "init_dd_tracer", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_init_pyroscope', - 'init_pyroscope', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_init_pyroscope", + "init_pyroscope", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_initialize_jwt_auth', - 'initialize_jwt_auth', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_initialize_jwt_auth", + "initialize_jwt_auth", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_initialize_semantic_tool_filter', - 'initialize_semantic_tool_filter', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_initialize_semantic_tool_filter", + "initialize_semantic_tool_filter", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_initialize_startup_logging', - 'initialize_startup_logging', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_initialize_startup_logging", + "initialize_startup_logging", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_setup_prisma_client', - 'setup_prisma_client', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_setup_prisma_client", + "setup_prisma_client", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_sync_ui_settings_to_general_settings', - 'sync_ui_settings_to_general_settings', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_sync_ui_settings_to_general_settings", + "sync_ui_settings_to_general_settings", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_update_default_team_member_budget', - 'update_default_team_member_budget', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_update_default_team_member_budget", + "update_default_team_member_budget", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_validate_redis_transaction_buffer_config', - 'validate_redis_transaction_buffer_config', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_validate_redis_transaction_buffer_config", + "validate_redis_transaction_buffer_config", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warm_global_spend_cache', - 'warm_global_spend_cache', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warm_global_spend_cache", + "warm_global_spend_cache", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warn_budget_without_db', - 'warn_budget_without_db', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warn_budget_without_db", + "warn_budget_without_db", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warn_fail_closed_rate_limits_without_redis', - 'warn_fail_closed_rate_limits_without_redis', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warn_fail_closed_rate_limits_without_redis", + "warn_fail_closed_rate_limits_without_redis", False, ), ( - 'litellm.proxy.proxy_server', - 'ProxyStartupEvent', - '_warn_if_mock_testing_params_enabled', - 'warn_if_mock_testing_params_enabled', + "litellm.proxy.proxy_server", + "ProxyStartupEvent", + "_warn_if_mock_testing_params_enabled", + "warn_if_mock_testing_params_enabled", False, ), ( - 'litellm.proxy.spend_tracking.spend_management_endpoints', - '', - '_get_spend_report_for_time_range', - 'get_spend_report_for_time_range', + "litellm.proxy.spend_tracking.spend_management_endpoints", + "", + "_get_spend_report_for_time_range", + "get_spend_report_for_time_range", False, ), ( - 'litellm.proxy.spend_tracking.spend_management_endpoints', - '', - '_is_admin_view_safe', - 'is_admin_view_safe', + "litellm.proxy.spend_tracking.spend_management_endpoints", + "", + "_is_admin_view_safe", + "is_admin_view_safe", False, ), ( - 'litellm.proxy.spend_tracking.spend_tracking_utils', - '', - '_is_master_key', - 'is_master_key', + "litellm.proxy.spend_tracking.spend_tracking_utils", + "", + "_is_master_key", + "is_master_key", False, ), ( - 'litellm.proxy.spend_tracking.spend_tracking_utils', - '', - '_sanitize_error_information_for_spend_logs', - 'sanitize_error_information_for_spend_logs', + "litellm.proxy.spend_tracking.spend_tracking_utils", + "", + "_sanitize_error_information_for_spend_logs", + "sanitize_error_information_for_spend_logs", False, ), ( - 'litellm.proxy.utils', - '', - '_cache_user_row', - 'cache_user_row', + "litellm.proxy.utils", + "", + "_cache_user_row", + "cache_user_row", False, ), ( - 'litellm.proxy.utils', - '', - '_check_and_merge_model_level_guardrails', - 'check_and_merge_model_level_guardrails', + "litellm.proxy.utils", + "", + "_check_and_merge_model_level_guardrails", + "check_and_merge_model_level_guardrails", False, ), ( - 'litellm.proxy.utils', - '', - '_get_docs_url', - 'get_docs_url', + "litellm.proxy.utils", + "", + "_get_docs_url", + "get_docs_url", False, ), ( - 'litellm.proxy.utils', - '', - '_get_openapi_url', - 'get_openapi_url', + "litellm.proxy.utils", + "", + "_get_openapi_url", + "get_openapi_url", False, ), ( - 'litellm.proxy.utils', - '', - '_get_projected_spend_over_limit', - 'get_projected_spend_over_limit', + "litellm.proxy.utils", + "", + "_get_projected_spend_over_limit", + "get_projected_spend_over_limit", False, ), ( - 'litellm.proxy.utils', - '', - '_get_redoc_url', - 'get_redoc_url', + "litellm.proxy.utils", + "", + "_get_redoc_url", + "get_redoc_url", False, ), ( - 'litellm.proxy.utils', - '', - '_hash_token_if_needed', - 'hash_token_if_needed', + "litellm.proxy.utils", + "", + "_hash_token_if_needed", + "hash_token_if_needed", False, ), ( - 'litellm.proxy.utils', - '', - '_is_projected_spend_over_limit', - 'is_projected_spend_over_limit', + "litellm.proxy.utils", + "", + "_is_projected_spend_over_limit", + "is_projected_spend_over_limit", False, ), ( - 'litellm.proxy.utils', - '', - '_is_valid_team_configs', - 'is_valid_team_configs', + "litellm.proxy.utils", + "", + "_is_valid_team_configs", + "is_valid_team_configs", False, ), ( - 'litellm.proxy.utils', - '', - '_monitor_spend_logs_queue', - 'monitor_spend_logs_queue', + "litellm.proxy.utils", + "", + "_monitor_spend_logs_queue", + "monitor_spend_logs_queue", False, ), ( - 'litellm.proxy.utils', - '', - '_premium_user_check', - 'premium_user_check', + "litellm.proxy.utils", + "", + "_premium_user_check", + "premium_user_check", False, ), ( - 'litellm.proxy.utils', - '', - '_raise_failed_update_spend_exception', - 'raise_failed_update_spend_exception', + "litellm.proxy.utils", + "", + "_raise_failed_update_spend_exception", + "raise_failed_update_spend_exception", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_arelease_max_parallel_requests_on_disconnect', - 'arelease_max_parallel_requests_on_disconnect', + "litellm.proxy.utils", + "ProxyLogging", + "_arelease_max_parallel_requests_on_disconnect", + "arelease_max_parallel_requests_on_disconnect", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_callback_capabilities', - 'callback_capabilities', + "litellm.proxy.utils", + "ProxyLogging", + "_callback_capabilities", + "callback_capabilities", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_convert_mcp_hook_response_to_kwargs', - 'convert_mcp_hook_response_to_kwargs', + "litellm.proxy.utils", + "ProxyLogging", + "_convert_mcp_hook_response_to_kwargs", + "convert_mcp_hook_response_to_kwargs", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_create_mcp_request_object_from_kwargs', - 'create_mcp_request_object_from_kwargs', + "litellm.proxy.utils", + "ProxyLogging", + "_create_mcp_request_object_from_kwargs", + "create_mcp_request_object_from_kwargs", False, ), ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_fire_deferred_stream_logging', - 'fire_deferred_stream_logging', + "litellm.proxy.utils", + "ProxyLogging", + "_fire_deferred_stream_logging", + "fire_deferred_stream_logging", False, ), ) @@ -5188,17 +5201,16 @@ LLMS_FORWARDER_CASES: Final = ( ) PROXY_FORWARDER_CASES: Final = ( ( - 'litellm.proxy.utils', - 'ProxyLogging', - '_convert_mcp_to_llm_format', - 'convert_mcp_to_llm_format', - 'instance', + "litellm.proxy.utils", + "ProxyLogging", + "_convert_mcp_to_llm_format", + "convert_mcp_to_llm_format", + "instance", False, ), ) - def _get_owner(module_path: str, owner_name: str) -> object: module: Final = import_module(module_path or "litellm") if module_path == "litellm.llms.openai_like.dynamic_config" and owner_name == "JSONProviderConfig": @@ -5292,14 +5304,10 @@ def _make_private_override( def recorded_args(args: tuple[object, ...]) -> tuple[object, ...]: return args[1:] if descriptor in ("instance", "classmethod") else args - def sync_override( - *args: object, **kwargs: object - ) -> tuple[object, tuple[object, ...], dict[str, object]]: + def sync_override(*args: object, **kwargs: object) -> tuple[object, tuple[object, ...], dict[str, object]]: return result, recorded_args(args), kwargs - async def async_override( - *args: object, **kwargs: object - ) -> tuple[object, tuple[object, ...], dict[str, object]]: + async def async_override(*args: object, **kwargs: object) -> tuple[object, tuple[object, ...], dict[str, object]]: return result, recorded_args(args), kwargs implementation: Final = async_override if is_async else sync_override diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index ba928bfb8c3..273bebd98e5 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -20104,11 +20104,13 @@ def test_router_sends_an_image_to_the_text_only_leg_when_the_vision_leg_is_cooli async def test_anthropic_caller_rate_limit_does_not_block_other_credentials( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, surface: str, credential_kind: str ) -> None: + from litellm.llms.anthropic.messages.native import handler as python_messages from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.router import AllowedFailsPolicy monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr("time.time", lambda: 1767225600.0) + monkeypatch.setattr(litellm, "anthropic_messages", python_messages.anthropic_messages) def upstream(request: httpx.Request) -> httpx.Response: credential: Final = request.headers.get("authorization", request.headers.get("x-api-key", "")) diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 8a5d164fdff..d2b5d2b911b 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -750,9 +750,11 @@ async def test_retried_call_settles_deployment_io_tokens_to_the_successful_attem respx_mock: respx.MockRouter, ): from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + from litellm.llms.anthropic.messages.native import handler as python_messages from litellm.responses.streaming_iterator import drain_logging_tasks monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "anthropic_messages", python_messages.anthropic_messages) litellm.in_memory_llm_clients_cache.flush_cache() provider: Final = respx_mock.post(provider_url).mock( side_effect=[ From b4227652c3bea59863cedb9e3510851f39afd9f7 Mon Sep 17 00:00:00 2001 From: kenneth-berri Date: Wed, 11 Nov 2026 16:07:33 -0800 Subject: [PATCH 132/189] fix(health): log Test Connection spend with the mode's call type and model id (#50987) * fix(health): price connection tests using their resolved mode Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(health): avoid deployment pricing on alternate probes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(health): preserve pricing for equivalent model names Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): avoid authentication when comparing health check models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): cover health probe pricing without a deployment model Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): pin the clock in the rpm rate limit test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../health_check_helpers.py | 7 +- litellm/main.py | 19 ++ litellm/proxy/health_check.py | 35 ++- .../health_endpoints/_health_endpoints.py | 5 +- .../unit/proxy/test_health_check_functions.py | 221 +++++++++++++++++- tests/unit/test_main.py | 58 +++++ .../test_router_max_parallel_requests.py | 76 ++++-- 7 files changed, 392 insertions(+), 29 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 2c7d00552b4..03a10806f91 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -9,6 +9,8 @@ from collections.abc import Awaitable, Callable, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal +from pydantic import TypeAdapter + from litellm.llms.base_llm.ocr.transformation import DocumentType from litellm.rust_bridge import runtime from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_HEALTH_CHECK_DOCUMENT @@ -190,7 +192,10 @@ class HealthCheckHelpers: from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth _metadata_variable_name: Final = "litellm_metadata" - litellm_metadata: Final = HealthCheckHelpers._get_metadata_for_health_check_call() + litellm_metadata: Final[Mapping[str, object]] = { + **HealthCheckHelpers._get_metadata_for_health_check_call(), + "model_info": TypeAdapter(dict[str, object]).validate_python(model_params).get("model_info") or {}, + } model_params[_metadata_variable_name] = litellm_metadata model_params = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( data=model_params, diff --git a/litellm/main.py b/litellm/main.py index 239a85cef4c..e4110455e24 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -9512,6 +9512,7 @@ async def ahealth_check( from litellm.litellm_core_utils.cached_imports import get_litellm_logging_class from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers, default_health_check_mode from litellm.litellm_core_utils.health_check_utils import OPTIONAL_STR + from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS # Use cached import helper to lazy-load Logging class (only loads when function is called) Logging: Final = get_litellm_logging_class() @@ -9556,6 +9557,24 @@ async def ahealth_check( mode = mode or default_health_check_mode( requested_model=requested_model, model=model, custom_llm_provider=custom_llm_provider ) + litellm_logging_obj.model = requested_model + litellm_logging_obj.call_type = { + "completion": "atext_completion", + "embedding": "aembedding", + "audio_speech": "aspeech", + "audio_transcription": "atranscription", + "image_generation": "aimage_generation", + "image_edit": "aimage_edit", + "video_generation": "avideo_generation", + "rerank": "arerank", + "realtime": "arealtime", + "responses": "aresponses", + "anthropic_messages": "anthropic_messages", + "ocr": "aocr", + "decisions": "asystemone", + "extraction": "aextraction", + "batch": ("alist_batches" if custom_llm_provider in LIST_BATCHES_SUPPORTED_PROVIDERS else "acompletion"), + }.get("decisions" if is_decisions_model_mode(mode) else mode, "acompletion") if "*" in model: return await HealthCheckHelpers.ahealth_check_wildcard_models( model=model, diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index de2ed4fd4ba..550a3f66f3b 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -26,6 +26,7 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_PROMPT, HEALTH_CHECK_TIMEOUT_SECONDS, ) +from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, is_model_capability_field from litellm.router_strategy.auto_router.auto_router_model_naming import ( @@ -831,6 +832,24 @@ _PROBE_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +def _same_health_check_model(deployment_model: str | None, probe_params: Mapping[str, object]) -> bool: + model: Final = probe_params.get("model") + if model == deployment_model: + return True + if not isinstance(model, str) or deployment_model is None: + return False + provider_param: Final = probe_params.get("custom_llm_provider") + provider: Final = provider_param if isinstance(provider_param, str) else None + try: + deployment_name, deployment_provider = resolve_model_and_provider_for_metadata( + model=deployment_model, custom_llm_provider=provider + ) + probe_name, probe_provider = resolve_model_and_provider_for_metadata(model=model, custom_llm_provider=provider) + except litellm.exceptions.BadRequestError: + return False + return (deployment_name, deployment_provider) == (probe_name, probe_provider) + + def update_litellm_params_for_health_check( model_info: dict, litellm_params: dict, deployment_model: str | None = None ) -> dict: @@ -895,6 +914,15 @@ def update_litellm_params_for_health_check( if mode == "audio_speech": litellm_params["voice"] = model_info.get("health_check_voice", "alloy") + logging_model_info: Final = _MODEL_INFO_ADAPTER.validate_python( + _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model_info") or {} + ) + probe_logging_model_info: Final = ( + logging_model_info + if _same_health_check_model(base_model_owner, _PROBE_PARAMS_ADAPTER.validate_python(litellm_params)) + else {key: value for key, value in logging_model_info.items() if key == "id"} + ) + # Handle Bedrock region routing format: bedrock/region/model # This is needed because health checks bypass get_llm_provider() for the model param # Issue #15807: Without this, health checks send "region/model" as the model ID to AWS @@ -941,7 +969,12 @@ def update_litellm_params_for_health_check( model_id_params: Final = {} if model_id_pin is None else {"model_id": model_id_pin} return { key: value - for key, value in {**audio_output_params, **model_id_params, **probe_params}.items() + for key, value in { + **audio_output_params, + **model_id_params, + **probe_params, + **({"model_info": probe_logging_model_info} if "model_info" in probe_params else {}), + }.items() if not is_model_capability_field(key) } diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 1211c56975b..7c7a1240c82 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -2695,7 +2695,10 @@ async def test_model_connection( stored_model: Final = _OBJECT_MAPPING.validate_python(config_litellm_params).get("model") litellm_params = update_litellm_params_for_health_check( model_info=dict(probe_model_info), - litellm_params=litellm_params, + litellm_params={ + **litellm_params, + "model_info": _OBJECT_MAPPING.validate_python(resolved_model_info or {}), + }, deployment_model=stored_model if isinstance(stored_model, str) else None, ) diff --git a/tests/unit/proxy/test_health_check_functions.py b/tests/unit/proxy/test_health_check_functions.py index 79eec44958f..fb26b9706f6 100644 --- a/tests/unit/proxy/test_health_check_functions.py +++ b/tests/unit/proxy/test_health_check_functions.py @@ -4,10 +4,19 @@ from datetime import datetime, timedelta, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +import respx +from pydantic import TypeAdapter +from starlette.requests import Request -from litellm.proxy._types import UserAPIKeyAuth +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.chatgpt import authenticator as chatgpt_authenticator +from litellm.proxy import proxy_server +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.db.health_check_latest import LatestHealthCheckRow +from litellm.proxy.health_check import _same_health_check_model, update_litellm_params_for_health_check from litellm.proxy.health_endpoints._health_endpoints import ( _aggregate_health_check_results, _build_model_param_to_info_mapping, @@ -16,8 +25,10 @@ from litellm.proxy.health_endpoints._health_endpoints import ( _save_health_check_results_if_changed, _save_health_check_to_db, latest_health_checks_endpoint, + test_model_connection as probe_model_connection, ) from litellm.proxy.utils import PrismaClient +from litellm.types.utils import StandardLoggingPayload @pytest.fixture @@ -946,5 +957,213 @@ def test_filter_deployments_to_model_groups(): assert filter_deployments_to_model_groups(model_list, frozenset()) == () +@pytest.mark.asyncio +@pytest.mark.parametrize( + "pricing", [{"base_model": "gpt-5.4"}, {"input_cost_per_token": 0.02, "output_cost_per_token": 0.03}] +) +@pytest.mark.parametrize("deployment_id", [None, "alternate-probe-deployment"]) +@pytest.mark.parametrize("override", ["health_check_model", "health_check_params", "same_model"]) +async def test_alternate_health_check_model_uses_its_own_price( + pricing: dict[str, object], deployment_id: str | None, override: str, monkeypatch: pytest.MonkeyPatch +) -> None: + events: Final[asyncio.Queue[StandardLoggingPayload]] = asyncio.Queue() + + class SpendCapture(CustomLogger): + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + events.put_nowait(TypeAdapter(StandardLoggingPayload).validate_python(kwargs["standard_logging_object"])) + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "_async_success_callback", [SpendCapture()]) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr( + proxy_server, + "prisma_client", + PrismaClient( + database_url="postgresql://test:test@localhost/test", proxy_logging_obj=proxy_server.proxy_logging_obj + ), + ) + monkeypatch.setitem( + litellm.model_cost, + "health-probe", + { + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00002, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + monkeypatch.setitem( + litellm.model_cost, + "gpt-5.4", + {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "litellm_provider": "openai", "mode": "chat"}, + ) + if deployment_id is not None: + monkeypatch.setitem( + litellm.model_cost, deployment_id, {**pricing, "litellm_provider": "openai", "mode": "chat"} + ) + params: Final = {"api_base": "https://health-test.example/v1", "api_key": "test-key"} + model_info: Final = { + **pricing, + **( + {"health_check_params": {"model": "openai/health-probe"}} + if override == "health_check_params" + else {"health_check_model": "openai/health-probe"} + ), + **({"id": deployment_id} if deployment_id is not None else {}), + } + payload: Final = { + "id": "chatcmpl-health-probe", + "object": "chat.completion", + "created": 1, + "model": "health-probe", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + with respx.mock: + respx.post("https://health-test.example/v1/chat/completions").mock( + return_value=httpx.Response(200, json=payload) + ) + await litellm.acompletion( + model="openai/health-probe", + messages=[{"role": "user", "content": "test"}], + litellm_metadata={"model_info": model_info} if override == "same_model" else {}, + **params, + ) + normal: Final = await asyncio.wait_for(events.get(), timeout=10) + result: Final = await probe_model_connection( + request=Request({"type": "http", "headers": []}), + mode="chat", + litellm_params={"model": "openai/health-probe" if override == "same_model" else "openai/gpt-5.4", **params}, + model_info=model_info, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert result["status"] == "success", result + health: Final = await asyncio.wait_for(events.get(), timeout=10) + + assert health["call_type"] == normal["call_type"] + assert health["model"] == normal["model"] + assert health["response_cost"] == normal["response_cost"] + assert health["response_cost"] > 0 + assert health["model_id"] == (deployment_id or "") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("stored_model", "requested_model"), + [("openai/health-probe", "health-probe"), ("health-probe", "openai/health-probe")], +) +async def test_equivalent_health_check_model_names_keep_deployment_prices( + stored_model: str, requested_model: str, monkeypatch: pytest.MonkeyPatch +) -> None: + events: Final[asyncio.Queue[StandardLoggingPayload]] = asyncio.Queue() + + class SpendCapture(CustomLogger): + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + events.put_nowait(TypeAdapter(StandardLoggingPayload).validate_python(kwargs["standard_logging_object"])) + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "_async_success_callback", [SpendCapture()]) + monkeypatch.setitem( + litellm.model_cost, + "health-probe", + { + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00002, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + params: Final = { + "api_base": "https://equivalent-health.example/v1", + "api_key": "test-key", + "custom_llm_provider": "openai", + } + deployment_id: Final = "priced-health-deployment" + model_info: Final = { + "id": deployment_id, + "mode": "chat", + "input_cost_per_token": 0.02, + "output_cost_per_token": 0.03, + } + router: Final = litellm.Router( + model_list=[ + { + "model_name": "priced-health", + "litellm_params": {"model": stored_model, **params}, + "model_info": model_info, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr( + proxy_server, + "prisma_client", + PrismaClient( + database_url="postgresql://test:test@localhost/test", proxy_logging_obj=proxy_server.proxy_logging_obj + ), + ) + payload: Final = { + "id": "chatcmpl-equivalent-health", + "object": "chat.completion", + "created": 1, + "model": "health-probe", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + with respx.mock: + respx.post("https://equivalent-health.example/v1/chat/completions").mock( + return_value=httpx.Response(200, json=payload) + ) + result: Final = await probe_model_connection( + request=Request({"type": "http", "headers": []}), + mode="chat", + litellm_params={"model": requested_model, "custom_llm_provider": "openai"}, + model_info={"id": deployment_id}, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert result["status"] == "success", result + health: Final = await asyncio.wait_for(events.get(), timeout=10) + + assert health["response_cost"] == 10 * model_info["input_cost_per_token"] + 5 * model_info["output_cost_per_token"] + assert health["model"] == "health-probe" + assert health["model_id"] == deployment_id + + +def test_health_check_without_deployment_model_keeps_probe_but_drops_deployment_prices() -> None: + deployment_info: Final = { + "id": "probe-deployment", + "base_model": "openai/deployment-model", + "input_cost_per_token": 0.02, + "output_cost_per_token": 0.03, + } + probe_model: Final = "openai/health-probe" + + probe_params: Final = update_litellm_params_for_health_check( + model_info={"mode": "chat", "health_check_model": probe_model}, + litellm_params={"model_info": deployment_info}, + ) + + assert probe_params["model"] == probe_model + assert probe_params["model_info"] == {"id": deployment_info["id"]} + + +@pytest.mark.parametrize(("model", "expected"), [("health-probe", True), ("other-health-probe", False)]) +def test_chatgpt_health_check_model_comparison_does_not_access_credentials( + model: str, expected: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + def forbidden_auth_file_access(*args: object, **kwargs: object) -> None: + pytest.fail("Health-check model comparison accessed the ChatGPT auth file") + + monkeypatch.setattr(chatgpt_authenticator, "open", forbidden_auth_file_access, raising=False) + + assert ( + _same_health_check_model("chatgpt/health-probe", {"model": model, "custom_llm_provider": "chatgpt"}) is expected + ) + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 97df9cdbfbf..ddecf43de91 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -52,6 +52,7 @@ from litellm.types.utils import ( Delta, ModelResponseStream, StandardCallbackDynamicParams, + StandardLoggingPayload, StreamingChoices, TextCompletionResponse, Usage, @@ -103,6 +104,63 @@ def preserve_litellm_completion_state(monkeypatch: pytest.MonkeyPatch) -> None: WHITE_PNG: Final = (Path(__file__).parents[1] / "white_100x100.png").read_bytes() +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["image_generation", "embedding"]) +@pytest.mark.parametrize("deployment_id", [None, "health-deployment"]) +async def test_health_check_logs_the_same_cost_and_deployment_as_normal_calls( + mode: str, deployment_id: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + events: Final[asyncio.Queue[StandardLoggingPayload]] = asyncio.Queue() + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setitem( + litellm.model_cost, + "standard/1024-x-1024/health-image", + {"input_cost_per_image": 0.25, "litellm_provider": "openai", "mode": "image_generation"}, + ) + + class SpendCapture(CustomLogger): + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + events.put_nowait(TypeAdapter(StandardLoggingPayload).validate_python(kwargs["standard_logging_object"])) + + monkeypatch.setattr(litellm, "_async_success_callback", [SpendCapture()]) + model: Final = "openai/health-image" if mode == "image_generation" else "openai/text-embedding-3-small" + payload: Final = ( + {"created": 1, "data": [{"url": "https://example.com/image.png"}]} + if mode == "image_generation" + else { + "object": "list", + "data": [{"object": "embedding", "embedding": [0.1], "index": 0}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 10, "total_tokens": 10}, + } + ) + params: Final = { + "model": model, + "api_base": "https://health-test.example/v1", + "api_key": "test-key", + **({"model_info": {"id": deployment_id}} if deployment_id else {}), + **({"size": "1024x1024", "quality": "standard", "n": 1} if mode == "image_generation" else {}), + } + with respx.mock: + respx.post(url__regex=r"https://health-test.example/v1/.*").mock(return_value=httpx.Response(200, json=payload)) + if mode == "image_generation": + await litellm.aimage_generation(**params, prompt="test") + else: + await litellm.aembedding(**params, input=["test"]) + normal: Final = await asyncio.wait_for(events.get(), timeout=10) + result: Final = await litellm.ahealth_check(model_params=params.copy(), mode=mode, prompt="test") + assert "error" not in result, result + health: Final = await asyncio.wait_for(events.get(), timeout=10) + + assert health["call_type"] == normal["call_type"] + assert health["model"] == normal["model"] + assert health["response_cost"] == normal["response_cost"] + assert health["response_cost"] > 0 + assert health["model_id"] == (deployment_id or "") + + @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize( diff --git a/tests/unit/test_router/test_router_max_parallel_requests.py b/tests/unit/test_router/test_router_max_parallel_requests.py index 1b350f31f23..ef73e4983a9 100644 --- a/tests/unit/test_router/test_router_max_parallel_requests.py +++ b/tests/unit/test_router/test_router_max_parallel_requests.py @@ -1,11 +1,43 @@ from __future__ import annotations +import datetime from typing import Final +from unittest.mock import patch + +import pytest import litellm -import pytest from litellm.utils import calculate_max_parallel_requests +_ROUTER_CLOCK: Final = "litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime" +_FIRST_MINUTE: Final = datetime.datetime(2026, 1, 1, 12, 0, 59, tzinfo=datetime.timezone.utc) +_NEXT_MINUTE: Final = datetime.datetime(2026, 1, 1, 12, 1, 0, tzinfo=datetime.timezone.utc) + + +def _rpm_limited_router() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "limited", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + "rpm": 1, + }, + } + ], + routing_strategy="usage-based-routing-v2", + enable_pre_call_checks=True, + num_retries=0, + ) + + +_RPM_LIMITED_REQUEST: Final = { + "model": "limited", + "messages": [{"role": "user", "content": "rate limit"}], + "mock_response": "response", +} + @pytest.mark.parametrize( ("max_parallel_requests", "tpm", "rpm", "default_max_parallel_requests", "expected"), @@ -37,32 +69,26 @@ def test_scenario( @pytest.mark.asyncio async def test_max_parallel_requests_rpm_rate_limiting() -> None: - router: Final = litellm.Router( - model_list=[ - { - "model_name": "limited", - "litellm_params": { - "model": "openai/gpt-4o-mini", - "api_key": "test-key", - "rpm": 1, - }, - } - ], - routing_strategy="usage-based-routing-v2", - enable_pre_call_checks=True, - num_retries=0, - ) - request: Final = { - "model": "limited", - "messages": [{"role": "user", "content": "rate limit"}], - "mock_response": "response", - } + router: Final = _rpm_limited_router() - first_response: Final = await router.acompletion(**request) + with patch(_ROUTER_CLOCK, return_value=_FIRST_MINUTE): + first_response: Final = await router.acompletion(**_RPM_LIMITED_REQUEST) - assert first_response.choices[0].message.content == "response" - with pytest.raises(litellm.RateLimitError): - await router.acompletion(**request) + assert first_response.choices[0].message.content == "response" + with pytest.raises(litellm.RateLimitError): + await router.acompletion(**_RPM_LIMITED_REQUEST) + + +@pytest.mark.asyncio +async def test_rpm_limit_resets_in_the_next_minute() -> None: + router: Final = _rpm_limited_router() + + with patch(_ROUTER_CLOCK, return_value=_FIRST_MINUTE): + await router.acompletion(**_RPM_LIMITED_REQUEST) + with patch(_ROUTER_CLOCK, return_value=_NEXT_MINUTE): + next_minute_response: Final = await router.acompletion(**_RPM_LIMITED_REQUEST) + + assert next_minute_response.choices[0].message.content == "response" @pytest.mark.asyncio From 22252073e7f0b0c18236b10ac4cc5d6060d04992 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:19:34 -0800 Subject: [PATCH 133/189] feat(bedrock): search knowledge bases by image and return image hits (#51063) * feat(bedrock): search knowledge bases by image and return image hits vector_stores.search and asearch take a query_image (base64 jpeg, png, gif or webp), and query is optional when it is sent. Bedrock sends it as an IMAGE retrievalQuery, and a caller's own retrievalQuery in extra_body still wins Bedrock IMAGE hits now come back as image content items instead of being dropped. The media type comes from the source file mime type metadata, then the data URI label, then application/octet-stream A request with both query and query_image, with neither, or with a malformed image gets a 400. So does query_image on a provider without image search, unless drop_params is on. The proxy accepts query_image in place of query on /v1/vector_stores/{id}/search Spend logs redact image data when prompts are not stored. Emulated file_search skips image-only hits, RAG rerank maps its indices to the text documents it ranked, and the dashboard tester renders image hits * fix(vector-stores): keep image hits to image searches and reject a caller retrievalQuery Text searches on a multimodal Bedrock knowledge base return the same text hits as before, so chat, RAG and logging payloads stay unchanged. Image hits come back only when the caller sent query_image A caller retrievalQuery now answers 400 pointing at query_image, or is dropped under drop_params, instead of replacing the built query unchecked A file_search chat tool that carries query_image or rewrite_query searches again instead of failing, and a non-string query next to an image answers 400 instead of 500. query_image is keyword-only on search and asearch * fix(vector-stores): redact query images and keep store search params out of image checks * fix(vector-stores): redact images nested in extra_body and keep a store's extra_body retrievalQuery out of searches * fix(vector_stores): keep a caller's image query through store and router params A deployment's retrievalQuery no longer reaches a routed search, a caller's query fields survive a store's own extra_body, and an extra_body image stops before the provider. file_search forwards rewrite_query, both retrieval query spellings are redacted, and the cost error stops echoing the response when message logging is off * fix(cost): leave image search responses out of the cost error text * fix(vector_stores): accept line-wrapped base64 in query_image data Bedrock takes inlineContent wrapped with \n or \r\n line breaks, as plain base64 writes it, so the check ignores only those before decoding and the data goes to Bedrock as sent * fix(vector_stores): drop rewrite_query false from the chat file_search tool False asks for the default search every store already runs, and MongoDB rejects the field outright, so forwarding it made a MongoDB chat lose the store's context for a no-op flag. True is still forwarded. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 21 +- .../vector_store_pre_call_hook.py | 3 +- litellm/litellm_core_utils/redact_messages.py | 37 +++- .../bedrock/vector_stores/transformation.py | 116 ++++++++-- litellm/proxy/route_llm_request.py | 5 +- .../proxy/vector_store_endpoints/endpoints.py | 14 +- litellm/router_rust/invoke/endpoints.py | 13 +- .../integrations/rag/bedrock_knowledgebase.py | 12 +- litellm/types/vector_stores.py | 26 ++- litellm/vector_stores/main.py | 35 ++- litellm/vector_stores/search_query.py | 205 ++++++++++++++++++ .../vector_stores/vector_store_registry.py | 27 +-- .../test_vector_store_pre_call_hook.py | 34 ++- .../test_redact_messages.py | 63 ++++++ ...est_bedrock_vector_store_transformation.py | 156 +++++++++++++ .../test_mongodb_transformation.py | 37 ++++ tests/unit/proxy/test_route_llm_request.py | 4 + .../test_vector_store_endpoints.py | 80 +++++++ tests/unit/test_cost_calculator.py | 36 +++ ...test_endpoints_characterization_generic.py | 23 ++ tests/unit/vector_stores/test_main.py | 141 ++++++++++++ .../test_vector_store_registry.py | 23 ++ 22 files changed, 1036 insertions(+), 75 deletions(-) create mode 100644 litellm/vector_stores/search_query.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 233ff48e8a3..a37160b4528 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -64,6 +64,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( select_cost_metric_for_model, ) from litellm.litellm_core_utils.llm_response_utils.response_metadata import response_ms_since_logging_start +from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.llms.anthropic.cost_calculation import ( cost_per_token as anthropic_cost_per_token, ) @@ -1665,6 +1666,14 @@ def _response_time_ms_for_cost( return litellm_logging_obj.get_response_ms() +def _model_is_none_error(completion_response_text: str, litellm_logging_obj: LitellmLoggingObject | None) -> ValueError: + message: Final = "Model is None and does not exist in passed completion_response." + model_call_details: Final = {} if litellm_logging_obj is None else litellm_logging_obj.model_call_details + if should_redact_message_logging(model_call_details) or model_call_details.get("query_image") is not None: + return ValueError(message) + return ValueError(f"{message} Passed completion_response={completion_response_text}, model=None") + + def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None: breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown total: Final[object] = None if breakdown is None else breakdown.get("total_cost") @@ -2013,9 +2022,7 @@ def completion_cost( else: if model is None: - raise ValueError( - f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" - ) + raise _model_is_none_error(f"{completion_response}", litellm_logging_obj) if len(messages) > 0: prompt_tokens = token_counter(model=model, messages=messages) elif len(prompt) > 0: @@ -2029,9 +2036,7 @@ def completion_cost( return A2ACostCalculator.calculate_a2a_cost(litellm_logging_obj=litellm_logging_obj) if model is None: - raise ValueError( - f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" - ) + raise _model_is_none_error(f"{completion_response}", litellm_logging_obj) if custom_llm_provider is None: try: model, custom_llm_provider, _, _ = litellm.get_llm_provider( @@ -2276,9 +2281,7 @@ def completion_cost( return get_replicate_completion_pricing(completion_response, response_time_ms) if model is None: - raise ValueError( - f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" - ) + raise _model_is_none_error(f"{completion_response}", litellm_logging_obj) if custom_llm_provider is not None and custom_llm_provider == "vertex_ai": # Calculate the prompt characters + response characters diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index f40a166e67d..80341cb282d 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -35,6 +35,7 @@ from litellm.types.vector_stores import ( VectorStoreSearchResponse, VectorStoreSearchResult, ) +from litellm.vector_stores.search_query import store_search_params if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -327,7 +328,7 @@ class VectorStorePreCallHook(CustomLogger): "query": query, "custom_llm_provider": custom_llm_provider, "metadata": request_metadata, - **litellm_params_for_vector_store, + **store_search_params(litellm_params_for_vector_store), }, ) except Exception as search_error: diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 3760d7f3059..530339f1db2 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -34,6 +34,7 @@ from litellm.llms.vertex_ai.common_utils import ( from litellm.secret_managers.main import str_to_bool from litellm.types.router import BaselineRouteStamp from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.vector_stores import RETRIEVAL_QUERY_PARAMS if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -86,8 +87,8 @@ def _redacted_entries_for_opted_out_logger(model_call_details: Mapping[str, obje "messages": [{"role": "user", "content": REDACTED_BY_LITELLM}], "prompt": "", "input": "", - **({"instructions": REDACTED_BY_LITELLM} if model_call_details.get("instructions") is not None else {}), - **({"optional_params": _with_redacted_instructions(optional_params)} if optional_params is not None else {}), + **_request_field_redactions(model_call_details), + **({"optional_params": _with_redacted_request_fields(optional_params)} if optional_params is not None else {}), **({"litellm_params": redacted_litellm_params(litellm_params)} if litellm_params is not None else {}), **( {"standard_logging_object": _redact_standard_logging_object(standard_logging_object)} @@ -464,8 +465,24 @@ def _redact_responses_api_output_dict(output_items: Sequence[object], redacted_s return [_redacted_responses_api_output_item(item, redacted_str) for item in output_items] -def _with_redacted_instructions(params: Mapping[str, object]) -> Mapping[str, object]: - return params if params.get("instructions") is None else {**params, "instructions": REDACTED_BY_LITELLM} +_REDACTED_REQUEST_FIELDS: Final = ("instructions", "query_image", *RETRIEVAL_QUERY_PARAMS) +_NESTED_REQUEST_FIELDS: Final = ("extra_body", "additional_args", "complete_input_dict") + + +def _request_field_redactions(params: Mapping[str, object]) -> Mapping[str, object]: + return { + **{field: REDACTED_BY_LITELLM for field in _REDACTED_REQUEST_FIELDS if params.get(field) is not None}, + **{ + field: _with_redacted_request_fields(nested) + for field in _NESTED_REQUEST_FIELDS + if (nested := _as_mapping(params.get(field))) is not None + }, + } + + +def _with_redacted_request_fields(params: Mapping[str, object]) -> Mapping[str, object]: + redactions: Final = _request_field_redactions(params) + return {**params, **redactions} if redactions else params def _redacted_responses_api_response( @@ -478,7 +495,7 @@ def _redacted_responses_api_response( if isinstance(output, list) else output, } - return _with_redacted_instructions(redacted) if redact_instructions else redacted + return _with_redacted_request_fields(redacted) if redact_instructions else redacted def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[str, object]: @@ -502,7 +519,7 @@ def _redact_standard_logging_object( standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}] model_parameters: Final = _as_mapping(standard_logging_object.get("model_parameters")) if model_parameters is not None and redact_instructions: - standard_logging_object["model_parameters"] = _with_redacted_instructions(model_parameters) + standard_logging_object["model_parameters"] = _with_redacted_request_fields(model_parameters) response: Final = standard_logging_object.get("response") if response is not None: @@ -599,7 +616,7 @@ def redacted_litellm_params(params: Mapping[str, object], *, redact_request_body """ request: Final = _as_mapping(params.get("proxy_server_request")) return { - **(_with_redacted_instructions(params) if redact_request_body else params), + **(_with_redacted_request_fields(params) if redact_request_body else params), **{ key: _redacted_baseline_metadata(value) for key, value in params.items() @@ -613,7 +630,7 @@ def redacted_litellm_params(params: Mapping[str, object], *, redact_request_body } -def perform_redaction(model_call_details: dict, result, redact_streaming_responses: bool = True): +def perform_redaction(model_call_details: dict[str, object], result, redact_streaming_responses: bool = True): """ Performs the actual redaction on the logging object and result. @@ -638,8 +655,8 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons if redact_streaming_responses: model_call_details.update( { - **({"instructions": REDACTED_BY_LITELLM} if model_call_details.get("instructions") is not None else {}), - **({"optional_params": _with_redacted_instructions(optional_params)} if optional_params else {}), + **_request_field_redactions(model_call_details), + **({"optional_params": _with_redacted_request_fields(optional_params)} if optional_params else {}), } ) model_call_details.pop(SERVED_OUTPUT_TEXTS_KEY, None) diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index afcbe61cd75..b4cac70fee2 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -1,20 +1,25 @@ +import re from collections.abc import Mapping from copy import deepcopy +from itertools import product +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, runtime_checkable from urllib.parse import urlparse import httpx from pydantic import TypeAdapter -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import decoded_json_object, keyword_arguments +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import decoded_json_object, drops_unsupported_params, keyword_arguments from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import BedrockError from litellm.types.integrations.rag.bedrock_knowledgebase import ( BedrockKBContent, + BedrockKBImageFormat, BedrockKBResponse, BedrockKBRetrievalConfiguration, BedrockKBRetrievalQuery, @@ -22,11 +27,14 @@ from litellm.types.integrations.rag.bedrock_knowledgebase import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( + RETRIEVAL_QUERY_PARAMS, VECTOR_STORE_OPENAI_PARAMS, BaseVectorStoreAuthCredentials, VectorStoreIndexEndpoints, VectorStoreResultContent, + VectorStoreSearchImageMediaType, VectorStoreSearchOptionalRequestParams, + VectorStoreSearchQueryImage, VectorStoreSearchResponse, VectorStoreSearchResult, ) @@ -70,6 +78,64 @@ class _KBResponseFields(TypedDict, total=False, extra_items=object): pass +_BEDROCK_IMAGE_FORMATS: Final[Mapping[VectorStoreSearchImageMediaType, BedrockKBImageFormat]] = MappingProxyType( + {"image/jpeg": "jpeg", "image/png": "png", "image/gif": "gif", "image/webp": "webp"} +) +_BASE64_DATA_URI: Final = re.compile(r"data:(?P[^;,]*)[^,]*;base64,(?P.*)", re.DOTALL) +_SOURCE_FILE_MIME_TYPE_KEY: Final = "x-amz-bedrock-kb-source-file-mime-type" +_UNKNOWN_MEDIA_TYPE: Final = "application/octet-stream" +_RETRIEVAL_QUERY_UNSUPPORTED: Final = ( + "bedrock vector stores do not take retrievalQuery. Send query, or query_image to search by image. To drop it, " + "set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" +) + + +class _QueryImageParams(TypedDict, total=False): + query_image: ReadOnly[VectorStoreSearchQueryImage | None] + + +def _retrieval_query(query: str, params: _QueryImageParams) -> BedrockKBRetrievalQuery: + query_image: Final = params.get("query_image") + if query_image is None: + return BedrockKBRetrievalQuery(text=query) + return BedrockKBRetrievalQuery( + type="IMAGE", + image={"format": _BEDROCK_IMAGE_FORMATS[query_image["media_type"]], "inlineContent": query_image["data"]}, + ) + + +def _image_content(byte_content: str, metadata: Mapping[str, object]) -> VectorStoreResultContent: + data_uri: Final = _BASE64_DATA_URI.fullmatch(byte_content) + declared_media_type: Final = data_uri.group("media_type") if data_uri is not None else "" + source_file_media_type: Final = metadata.get(_SOURCE_FILE_MIME_TYPE_KEY) + return VectorStoreResultContent( + type="image", + source={ + "type": "base64", + "media_type": ( + source_file_media_type + if isinstance(source_file_media_type, str) and source_file_media_type.startswith("image/") + else declared_media_type or _UNKNOWN_MEDIA_TYPE + ), + "data": data_uri.group("data") if data_uri is not None else byte_content, + }, + ) + + +def _result_content( + content: BedrockKBContent | None, metadata: Mapping[str, object], include_images: bool +) -> VectorStoreResultContent | None: + if content is None: + return None + text: Final = content.get("text") + if text is not None: + return VectorStoreResultContent(text=text, type="text") + byte_content: Final = content.get("byteContent") + if not include_images or content.get("type") != "IMAGE" or not byte_content: + return None + return _image_content(byte_content, metadata) + + class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): """Vector store configuration for AWS Bedrock Knowledge Bases.""" @@ -95,7 +161,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): } def get_supported_openai_params(self, model: str) -> list[VECTOR_STORE_OPENAI_PARAMS]: - return ["filters", "max_num_results", "ranking_options"] + return ["filters", "max_num_results", "ranking_options", "query_image"] def _map_operator_to_aws(self, operator: str) -> str: """ @@ -250,8 +316,14 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url: Final = join_api_base(api_base, f"{encoded_vector_store_id}/retrieve") + caller_params: Final = decoded_json_object(litellm_params) + caller_sources: Final = self._caller_sources(extra_body=extra_body, litellm_params=caller_params) + if self._caller_value(caller_sources, RETRIEVAL_QUERY_PARAMS) is not None and not drops_unsupported_params( + caller_params.get("drop_params") + ): + raise BadRequestError(message=_RETRIEVAL_QUERY_UNSUPPORTED, model="", llm_provider="bedrock") request_body: Final[dict[str, object]] = { - "retrievalQuery": BedrockKBRetrievalQuery(text=query), + "retrievalQuery": _retrieval_query(query, vector_store_search_optional_params), } retrieval_config: Final[object] = ( @@ -283,7 +355,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): ] = filters if retrieval_config: request_body["retrievalConfiguration"] = cast(BedrockKBRetrievalConfiguration, retrieval_config) - user_context: Final = self._user_context(extra_body=extra_body, litellm_params=litellm_params) + user_context: Final = self._user_context(caller_sources) if user_context is not None: request_body["userContext"] = user_context @@ -291,19 +363,18 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): return url, request_body @staticmethod - def _user_context( + def _caller_sources( extra_body: Mapping[str, object] | None, litellm_params: Mapping[str, object] - ) -> BedrockKBUserContext | None: - sources: Final = tuple(source for source in (extra_body, litellm_params) if isinstance(source, Mapping)) - found: Final = next( - ( - source[key] - for source in sources - for key in ("userContext", "user_context") - if source.get(key) is not None - ), - None, - ) + ) -> tuple[Mapping[str, object], ...]: + return tuple(source for source in (extra_body, litellm_params) if isinstance(source, Mapping)) + + @staticmethod + def _caller_value(sources: tuple[Mapping[str, object], ...], keys: tuple[str, ...]) -> object: + return next((source[key] for source, key in product(sources, keys) if source.get(key) is not None), None) + + @staticmethod + def _user_context(sources: tuple[Mapping[str, object], ...]) -> BedrockKBUserContext | None: + found: Final = BedrockVectorStoreConfig._caller_value(sources, ("userContext", "user_context")) return None if found is None else cast(BedrockKBUserContext, found) def sign_request( @@ -375,15 +446,14 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): **keyword_arguments(_DECODED_JSON.validate_python(response.json()), BedrockKBResponse) ) ) + include_images: Final = litellm_logging_obj.model_call_details.get("query_image") is not None results: Final[list[VectorStoreSearchResult]] = [] for item in response_data.get("retrievalResults", []) or []: - content: BedrockKBContent | None = item.get("content") - text = content.get("text") if content else None - if text is None: + metadata = item.get("metadata", {}) or {} + result_content = _result_content(item.get("content"), metadata, include_images) + if result_content is None: continue - # Extract metadata and use helper functions - metadata = item.get("metadata", {}) or {} file_id = self._get_file_id_from_metadata(metadata) filename = self._get_filename_from_metadata(metadata) attributes = self._get_attributes_from_metadata(metadata) @@ -391,7 +461,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): results.append( VectorStoreSearchResult( score=item.get("score"), - content=[VectorStoreResultContent(text=text, type="text")], + content=[result_content], file_id=file_id, filename=filename, attributes=attributes, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index b1ebd042f3d..5810a2f71ab 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -280,7 +280,10 @@ REQUIRED_ONE_OF_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, str]]] = Map ) BODY_PARAM_ALTERNATIVES_BY_ROUTE: Final[Mapping[str, Mapping[str, tuple[str, ...]]]] = MappingProxyType( - {"acreate_interaction": MappingProxyType({"input": ("continuation_token",)})} + { + "acreate_interaction": MappingProxyType({"input": ("continuation_token",)}), + "avector_store_search": MappingProxyType({"query": ("query_image",)}), + } ) JSON_OBJECT_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index a6f9ba2ef14..0065f8e4002 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -36,6 +36,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( from litellm.repositories.table_repositories import ManagedVectorStoreIndexRepository from litellm.types.utils import LlmProviders from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse +from litellm.vector_stores.search_query import store_search_params, with_caller_query_params from litellm.vector_stores.utils import VectorStoreWriteBody, callers_metadata_snapshot from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry @@ -113,8 +114,8 @@ def build_request_data_from_managed_vector_store( if key in vector_store } ) - litellm_params: Final = vector_store.get("litellm_params") or MappingProxyType({}) - return MappingProxyType({**top_level, **litellm_params}) + litellm_params: Final[Mapping[str, object]] = vector_store.get("litellm_params") or MappingProxyType({}) + return MappingProxyType({**top_level, **store_search_params(litellm_params)}) async def _update_request_data_with_litellm_managed_vector_store_registry( @@ -187,8 +188,13 @@ async def vector_store_search( caller_body: Final = JSON_OBJECT_ADAPTER.validate_python(data) # Check for legacy vector store registry (non-managed vector stores) - data = await _update_request_data_with_litellm_managed_vector_store_registry( - data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict + data = dict( # rebind-ok: pre-existing rebinding of the request body + with_caller_query_params( + await _update_request_data_with_litellm_managed_vector_store_registry( + data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict + ), + caller_body.get("extra_body"), + ) ) reject_caller_opensearch_index_name( caller_body=caller_body, diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index fd3d055f707..c170aaab6e6 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -53,6 +53,7 @@ from litellm.router_strategy.complexity_router.context_compaction import ( surface_for_call, ) from litellm.router_utils.endpoint_eligibility import endpoint_for_call, endpoint_selection +from litellm.vector_stores.search_query import store_search_params if TYPE_CHECKING: from litellm.caching.dual_cache import DualCache @@ -854,6 +855,16 @@ async def compacted_attempt( return await compact_to_fit(ctx, deployment, call.kwargs, _compaction_surface(core, plan)) +_VECTOR_STORE_SEARCH_OPS: Final = frozenset({"vector_stores.main.asearch", "vector_stores.main.search"}) + + +def _deployment_params( + plan: Plan, deployment: Mapping[str, object] +) -> dict[str, object]: # mutable-ok: the attempt's own copy of the deployment params + params: Final = as_dict(deployment.get("litellm_params")) + return as_dict(store_search_params(params)) if plan.host_op in _VECTOR_STORE_SEARCH_OPS else params + + def attempt_call( core: EndpointCore, plan: Plan, @@ -869,7 +880,7 @@ def attempt_call( shaping: Final = core.shaping kwargs: Final = without(request, *ROUTING_KEYS) named: Final = {name: kwargs.pop(name) for name in plan.named if name in kwargs} - data: Final = as_dict(deployment.get("litellm_params")) + data: Final = _deployment_params(plan, deployment) model_name: Final = str(data.get("model")) cache: Final = core.routing_cache caching: Final = shaping.cache_responses diff --git a/litellm/types/integrations/rag/bedrock_knowledgebase.py b/litellm/types/integrations/rag/bedrock_knowledgebase.py index 7288eca03b4..f472cdc078f 100644 --- a/litellm/types/integrations/rag/bedrock_knowledgebase.py +++ b/litellm/types/integrations/rag/bedrock_knowledgebase.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Literal, TypeAlias from typing_extensions import ReadOnly, TypedDict @@ -114,10 +114,20 @@ class BedrockKBRetrievalConfiguration(TypedDict, total=False): vectorSearchConfiguration: BedrockKBVectorSearchConfiguration | None +BedrockKBImageFormat: TypeAlias = Literal["png", "jpeg", "gif", "webp"] + + +class BedrockKBInputImage(TypedDict): + format: ReadOnly[BedrockKBImageFormat] + inlineContent: ReadOnly[str] + + class BedrockKBRetrievalQuery(TypedDict, total=False): """Query structure for retrieval.""" text: str | None + type: ReadOnly[Literal["TEXT", "IMAGE"]] + image: ReadOnly[BedrockKBInputImage] class BedrockKBGuardrailConfiguration(TypedDict, total=False): diff --git a/litellm/types/vector_stores.py b/litellm/types/vector_stores.py index 2259339e479..b154d161e13 100644 --- a/litellm/types/vector_stores.py +++ b/litellm/types/vector_stores.py @@ -2,12 +2,14 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime from enum import Enum -from typing import Any, Literal +from typing import Any, Final, Literal, TypeAlias from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.base import LiteLLMBaseModel +RETRIEVAL_QUERY_PARAMS: Final = ("retrievalQuery", "retrieval_query") + class SupportedVectorStoreIntegrations(str, Enum): """Supported vector store integrations.""" @@ -74,11 +76,29 @@ class VectorStoreInfoRequest(LiteLLMBaseModel): vector_store_id: str +VectorStoreSearchImageMediaType: TypeAlias = Literal["image/jpeg", "image/png", "image/gif", "image/webp"] + + +class VectorStoreSearchQueryImage(TypedDict): + """Base64 image to search a multimodal vector store with, in place of a text query""" + + type: ReadOnly[Literal["base64"]] + media_type: ReadOnly[VectorStoreSearchImageMediaType] + data: ReadOnly[str] + + +class VectorStoreResultImageSource(TypedDict): + type: ReadOnly[Literal["base64"]] + media_type: ReadOnly[str] + data: ReadOnly[str] + + class VectorStoreResultContent(TypedDict, total=False): """Content of a vector store result""" text: str | None type: str | None + source: ReadOnly[VectorStoreResultImageSource] class VectorStoreSearchResult(TypedDict, total=False): @@ -117,6 +137,7 @@ class VectorStoreSearchOptionalRequestParams(TypedDict, total=False): max_num_results: int | None ranking_options: dict | None rewrite_query: bool | None + query_image: ReadOnly[VectorStoreSearchQueryImage | None] class VectorStoreSearchRequest(VectorStoreSearchOptionalRequestParams, total=False): @@ -320,6 +341,7 @@ VECTOR_STORE_OPENAI_PARAMS = Literal[ "max_num_results", "ranking_options", "rewrite_query", + "query_image", ] @@ -330,6 +352,7 @@ class VectorStoreToolParams: filters: dict | None = None max_num_results: int | None = None ranking_options: dict | None = None + rewrite_query: bool | None = None def to_dict(self) -> dict: """Convert to dict, excluding None values""" @@ -339,6 +362,7 @@ class VectorStoreToolParams: "filters": self.filters, "max_num_results": self.max_num_results, "ranking_options": self.ranking_options, + "rewrite_query": self.rewrite_query or None, }.items() if v is not None } diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 9f53accfa0b..88bc477249b 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -13,6 +13,7 @@ import httpx import litellm from litellm.constants import request_timeout +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.vector_store.transformation import ( @@ -28,10 +29,12 @@ from litellm.types.vector_stores import ( VectorStoreFileCounts, VectorStoreResultContent, VectorStoreSearchOptionalRequestParams, + VectorStoreSearchQueryImage, VectorStoreSearchResponse, VectorStoreSearchResult, ) from litellm.utils import ProviderConfigManager, client, load_credentials_from_list +from litellm.vector_stores.search_query import searched_query, sent_query_image from litellm.vector_stores.utils import VectorStoreRequestUtils if TYPE_CHECKING: @@ -289,7 +292,7 @@ def create( @client async def asearch( vector_store_id: str, - query: str | builtins.list[str], + query: str | builtins.list[str] | None = None, filters: dict | None = None, max_num_results: int | None = None, ranking_options: dict | None = None, @@ -303,6 +306,8 @@ async def asearch( # LiteLLM specific params, custom_llm_provider: str | None = None, router: "Router | None" = None, + *, + query_image: VectorStoreSearchQueryImage | None = None, **kwargs, ) -> VectorStoreSearchResponse: """ @@ -329,6 +334,7 @@ async def asearch( max_num_results=max_num_results, ranking_options=ranking_options, rewrite_query=rewrite_query, + query_image=query_image, extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, @@ -362,7 +368,7 @@ async def asearch( @client def search( vector_store_id: str, - query: str | builtins.list[str], + query: str | builtins.list[str] | None = None, filters: dict | None = None, max_num_results: int | None = None, ranking_options: dict | None = None, @@ -376,6 +382,8 @@ def search( # LiteLLM specific params, custom_llm_provider: str | None = None, router: "Router | None" = None, + *, + query_image: VectorStoreSearchQueryImage | None = None, **kwargs, ) -> VectorStoreSearchResponse | Coroutine[object, object, VectorStoreSearchResponse]: """ @@ -383,11 +391,12 @@ def search( Args: vector_store_id: The ID of the vector store to search. - query: A query string or array for the search. + query: A query string or array for the search. Optional when query_image is sent. filters: Optional filter to apply based on file attributes. max_num_results: Maximum number of results to return (1-50, default 10). ranking_options: Optional ranking options for search. rewrite_query: Whether to rewrite the natural language query for vector search. + query_image: A base64 image to search with in place of query, on providers that support it. Returns: VectorStoreSearchResponse containing the search results. @@ -444,21 +453,31 @@ def search( local_vars.update(kwargs) + search_query: Final = searched_query( + query=query, + query_image=sent_query_image(query_image, extra_body), + custom_llm_provider=custom_llm_provider, + supports_query_image="query_image" in vector_store_provider_config.get_supported_openai_params(model=""), + drop_params=drops_unsupported_params(local_vars.get("drop_params")), + ) + # Get VectorStoreSearchOptionalRequestParams with only valid parameters vector_store_search_optional_params: Final[VectorStoreSearchOptionalRequestParams] = ( VectorStoreRequestUtils.get_requested_vector_store_search_optional_param( - local_vars, + {**local_vars, "query_image": search_query.image}, vector_store_provider_config=vector_store_provider_config, ) ) + sent_extra_body: Final = as_str_mapping(extra_body) + # Pre Call logging litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=api_type, optional_params={ "vector_store_id": vector_store_id, - "query": query, + "query": search_query.query, **vector_store_search_optional_params, }, litellm_params={ @@ -471,7 +490,7 @@ def search( response: Final = base_llm_http_handler.vector_store_search_handler( vector_store_id=vector_store_id, - query=query, + query=search_query.query, vector_store_search_optional_params=vector_store_search_optional_params, vector_store_provider_config=vector_store_provider_config, custom_llm_provider=custom_llm_provider, @@ -479,7 +498,9 @@ def search( logging_obj=litellm_logging_obj, embedding_executor=embedding_executor, extra_headers=extra_headers, - extra_body=extra_body, + extra_body=extra_body + if sent_extra_body is None + else dict(_items_without_key(sent_extra_body, "query_image")), timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), diff --git a/litellm/vector_stores/search_query.py b/litellm/vector_stores/search_query.py new file mode 100644 index 00000000000..b4ec0b6707b --- /dev/null +++ b/litellm/vector_stores/search_query.py @@ -0,0 +1,205 @@ +import base64 +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Generic, Literal, TypeAlias, TypeVar + +from pydantic import TypeAdapter, ValidationError +from pydantic_core import ErrorDetails +from typing_extensions import assert_never + +import litellm +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping +from litellm.types.vector_stores import RETRIEVAL_QUERY_PARAMS, VectorStoreSearchQueryImage + +_QUERY_IMAGE: Final = TypeAdapter(VectorStoreSearchQueryImage) +_Query: Final = TypeVar("_Query", bound=str | Sequence[str]) +_CALLER_QUERY_PARAMS: Final = ("query_image", *RETRIEVAL_QUERY_PARAMS) +_BASE64_LINE_BREAKS: Final = str.maketrans("", "", "\r\n") + + +@dataclass(frozen=True, slots=True) +class TextSearchQuery(Generic[_Query]): + query: _Query + image: None = None + + +@dataclass(frozen=True, slots=True) +class ImageSearchQuery: + image: VectorStoreSearchQueryImage + query: Literal[""] = "" + + +@dataclass(frozen=True, slots=True) +class QueryAndImageBothSent: + pass + + +@dataclass(frozen=True, slots=True) +class NoQuerySent: + pass + + +@dataclass(frozen=True, slots=True) +class InvalidQueryImage: + problems: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class QueryImageUnsupported: + custom_llm_provider: str + + +@dataclass(frozen=True, slots=True) +class QueryImageDroppedWithoutQuery: + custom_llm_provider: str + + +SearchQuery: TypeAlias = TextSearchQuery[_Query] | ImageSearchQuery +SearchQueryError: TypeAlias = ( + QueryAndImageBothSent | NoQuerySent | InvalidQueryImage | QueryImageUnsupported | QueryImageDroppedWithoutQuery +) + + +def _has_text(query: object) -> bool: + if isinstance(query, str): + return bool(query.strip()) + items: Final = as_object_list(query) + if items is None: + return True + return any(not isinstance(item, str) or item.strip() for item in items) + + +def _problem(detail: ErrorDetails) -> str: + location: Final = ".".join(("query_image", *(str(part) for part in detail["loc"]))) + return f"{location}: {detail['msg']}" + + +def _is_base64(data: str) -> bool: + try: + base64.b64decode(data.translate(_BASE64_LINE_BREAKS), validate=True) + except ValueError: + return False + return True + + +def _validated_query_image(query_image: object) -> ImageSearchQuery | InvalidQueryImage: + try: + image: Final = _QUERY_IMAGE.validate_python(query_image) + except ValidationError as error: + return InvalidQueryImage( + problems=tuple(_problem(detail) for detail in error.errors(include_url=False, include_input=False)) + ) + if not _is_base64(image["data"]): + return InvalidQueryImage(problems=("query_image.data: not valid base64",)) + return ImageSearchQuery(image=image) + + +def sent_query_image(query_image: object, extra_body: object) -> object: + if query_image is not None: + return query_image + body: Final = as_str_mapping(extra_body) + return None if body is None else body.get("query_image") + + +def _without_retrieval_query(params: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in params.items() if key not in RETRIEVAL_QUERY_PARAMS} + + +def store_search_params(litellm_params: Mapping[str, object]) -> Mapping[str, object]: + extra_body: Final = as_str_mapping(litellm_params.get("extra_body")) + return MappingProxyType( + { + **_without_retrieval_query(litellm_params), + **({} if extra_body is None else {"extra_body": _without_retrieval_query(extra_body)}), + } + ) + + +def with_caller_query_params(request: Mapping[str, object], caller_extra_body: object) -> Mapping[str, object]: + caller: Final = as_str_mapping(caller_extra_body) + store: Final = as_str_mapping(request.get("extra_body")) + if caller is None or store is None: + return request + query_params: Final = {key: caller[key] for key in _CALLER_QUERY_PARAMS if key in caller} + return {**request, "extra_body": {**store, **query_params}} + + +def resolve_search_query( + query: _Query | None, + query_image: object, + custom_llm_provider: str, + supports_query_image: bool, + drop_params: bool, +) -> SearchQuery[_Query] | SearchQueryError: + if query_image is None: + return NoQuerySent() if query is None else TextSearchQuery(query=query) + text_query: Final = query if query is not None and _has_text(query) else None + if not supports_query_image and not drop_params: + return QueryImageUnsupported(custom_llm_provider=custom_llm_provider) + if not supports_query_image: + return ( + QueryImageDroppedWithoutQuery(custom_llm_provider=custom_llm_provider) + if text_query is None + else TextSearchQuery(query=text_query) + ) + if text_query is not None: + return QueryAndImageBothSent() + return _validated_query_image(query_image) + + +def _error_message(error: SearchQueryError) -> str: + match error: + case QueryAndImageBothSent(): + return "Send either query or query_image, not both. Leave query empty when you send query_image." + case NoQuerySent(): + return "query or query_image is required." + case InvalidQueryImage(problems=problems): + return ( + f"Invalid query_image: {'; '.join(problems)}. Expected " + '{"type": "base64", "media_type": "image/jpeg" | "image/png" | "image/gif" | "image/webp", ' + '"data": ""}.' + ) + case QueryImageUnsupported(custom_llm_provider=provider): + return ( + f"{provider} vector stores do not support query_image. To drop it, set `litellm.drop_params=True` " + "or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ) + case QueryImageDroppedWithoutQuery(custom_llm_provider=provider): + return ( + f"{provider} vector stores do not support query_image and drop_params dropped it, " + "which leaves no query to search with. Send query." + ) + case _: + assert_never(error) + + +def searched_query( + query: _Query | None, + query_image: object, + custom_llm_provider: str, + supports_query_image: bool, + drop_params: bool, +) -> SearchQuery[_Query]: + resolved: Final = resolve_search_query( + query=query, + query_image=query_image, + custom_llm_provider=custom_llm_provider, + supports_query_image=supports_query_image, + drop_params=drop_params, + ) + match resolved: + case TextSearchQuery() | ImageSearchQuery(): + return resolved + case ( + QueryAndImageBothSent() + | NoQuerySent() + | InvalidQueryImage() + | QueryImageUnsupported() + | QueryImageDroppedWithoutQuery() + ): + raise litellm.BadRequestError( + message=_error_message(resolved), model=None, llm_provider=custom_llm_provider + ) + case _: + assert_never(resolved) diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py index 2adaf17ac44..31fc111edab 100644 --- a/litellm/vector_stores/vector_store_registry.py +++ b/litellm/vector_stores/vector_store_registry.py @@ -1,6 +1,7 @@ # litellm/proxy/vector_stores/vector_store_registry.py import json from collections.abc import Iterable, Mapping, Sequence +from dataclasses import fields, replace from datetime import datetime, timezone from itertools import chain from typing import ( @@ -8,20 +9,18 @@ from typing import ( Any, # noqa: TID251 # untyped non_default_params dict is the only source of the unknown key type Final, cast, # noqa: TID251 # untyped non_default_params dict is the only source of the unknown key type - get_args, ) from pydantic import ConfigDict, TypeAdapter, ValidationError from typing_extensions import TypedDict from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import remove_items_at_indices +from litellm.litellm_core_utils.core_helpers import decoded_json_object, remove_items_at_indices from litellm.repositories.table_repositories import ( ManagedVectorStoreIndexRepository, ManagedVectorStoresRepository, ) from litellm.types.vector_stores import ( - VECTOR_STORE_OPENAI_PARAMS, LiteLLM_ManagedVectorStore, LiteLLM_ManagedVectorStoreIndex, LiteLLM_ManagedVectorStoreListResponse, @@ -150,19 +149,15 @@ class VectorStoreRegistry: self.vector_store_ids_to_vector_store_map: dict[str, LiteLLM_ManagedVectorStore] = {} def _extract_tool_params(self, tool: dict) -> VectorStoreToolParams: - """ - Extract supported parameters from a tool definition. - - Dynamically extracts all parameters defined in VECTOR_STORE_OPENAI_PARAMS. - """ - # Get the list of supported param names from the Literal type - declared_params: Final[tuple[object, ...]] = get_args(VECTOR_STORE_OPENAI_PARAMS) - supported_params: Final = tuple(param for param in declared_params if isinstance(param, str)) - - # Extract only the params that exist in the tool - kwargs: Final = {param: tool.get(param) for param in supported_params if param in tool} - - return VectorStoreToolParams(**kwargs) + tool_fields: Final = decoded_json_object(tool) + return replace( + VectorStoreToolParams(), + **{ + field.name: tool_fields[field.name] + for field in fields(VectorStoreToolParams) + if field.name in tool_fields + }, + ) def get_vector_store_ids_to_run(self, non_default_params: dict, tools: object = None) -> list[str]: """ diff --git a/tests/unit/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py b/tests/unit/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py index 726ce2f7f1d..48d726aa4df 100644 --- a/tests/unit/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py +++ b/tests/unit/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py @@ -2,7 +2,7 @@ import logging from collections.abc import Callable, Iterable, Iterator, Mapping from dataclasses import dataclass, field from types import MappingProxyType -from typing import Literal, Protocol +from typing import Final, Literal, Protocol import pytest from fastapi import HTTPException @@ -157,6 +157,38 @@ async def _run_hook( ) +@pytest.mark.asyncio +@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"]) +async def test_a_store_configured_with_a_raw_retrieval_query_still_runs_its_chat_search( + monkeypatch: pytest.MonkeyPatch, retrieval_query_param: str +) -> None: + monkeypatch.setattr( + litellm, + "vector_store_registry", + VectorStoreRegistry( + vector_stores=[ + LiteLLM_ManagedVectorStore( + vector_store_id="vs-raw", + custom_llm_provider="bedrock", + litellm_params={"aws_region_name": "us-west-2", retrieval_query_param: {"text": "from the store"}}, + ) + ], + ), + ) + router: Final = RecordingRouter() + + await _run_hook( + VectorStorePreCallHook(proxy_runtime=FakeProxyRuntime(router=router)), + ["vs-raw"], + FakeLoggingObj({}), + ) + + assert len(router.calls) == 1 + assert router.calls[0]["query"] == "what is litellm?" + assert router.calls[0]["aws_region_name"] == "us-west-2" + assert retrieval_query_param not in router.calls[0] + + @pytest.mark.asyncio async def test_hook_searches_through_the_injected_router_with_the_request_metadata( registry_with: RegisterStores, diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 8e4bed02d7e..888eb9c9507 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -2011,6 +2011,69 @@ def test_request_body_redaction_replaces_instructions_only_when_sent() -> None: assert "instructions" not in chat_body +def _vector_store_search_details(query_image: Mapping[str, str]) -> dict[str, object]: + return { + "query_image": query_image, + "optional_params": {"query_image": query_image}, + "litellm_params": {"proxy_server_request": {"body": {"query_image": query_image}}}, + "standard_logging_object": {"model_parameters": {"query_image": query_image}, "messages": [], "response": {}}, + } + + +def test_redaction_removes_a_vector_store_query_image_from_every_logged_copy() -> None: + details: Final = _vector_store_search_details({"type": "base64", "media_type": "image/jpeg", "data": SECRET}) + + perform_redaction(details, result=None) + + assert details["query_image"] == REDACTED_BY_LITELLM + assert SECRET not in json.dumps(details, default=str) + + +def test_redaction_removes_a_query_image_sent_inside_extra_body() -> None: + image: Final = {"type": "base64", "media_type": "image/jpeg", "data": SECRET} + details: Final[dict[str, object]] = { + "optional_params": {"extra_body": {"query_image": image}}, + "litellm_params": {"proxy_server_request": {"body": {"extra_body": {"query_image": image}}}}, + } + + perform_redaction(details, result=None) + + assert SECRET not in json.dumps(details, default=str) + + +@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"]) +def test_redaction_removes_a_raw_retrieval_query_sent_inside_extra_body(retrieval_query_param: str) -> None: + retrieval_query: Final = {"type": "IMAGE", "image": {"format": "jpeg", "inlineContent": SECRET}} + details: Final[dict[str, object]] = { + "litellm_params": {"proxy_server_request": {"body": {"extra_body": {retrieval_query_param: retrieval_query}}}}, + } + + perform_redaction(details, result=None) + + assert SECRET not in json.dumps(details, default=str) + + +def test_redaction_removes_the_image_from_the_provider_request_copy() -> None: + retrieval_query: Final = {"type": "IMAGE", "image": {"format": "jpeg", "inlineContent": SECRET}} + details: Final[dict[str, object]] = { + "additional_args": {"complete_input_dict": {"retrievalQuery": retrieval_query, "knowledgeBaseId": "kb"}}, + } + + perform_redaction(details, result=None) + + assert SECRET not in json.dumps(details, default=str) + assert details["additional_args"]["complete_input_dict"]["knowledgeBaseId"] == "kb" + + +def test_one_callback_opting_out_redacts_a_query_image_only_for_itself() -> None: + details: Final = _vector_store_search_details({"type": "base64", "media_type": "image/jpeg", "data": SECRET}) + + redacted: Final = redacted_model_call_details_for_custom_logger(details, TestCustomLogger(message_logging=False)) + + assert SECRET not in json.dumps(redacted, default=str) + assert SECRET in json.dumps(details, default=str) + + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_redaction_responses_api_stream(): diff --git a/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py b/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py index e69fb34038f..e20dfaffcc6 100644 --- a/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py +++ b/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py @@ -6,6 +6,7 @@ import httpx import pytest import respx +import litellm from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.vector_stores.transformation import BedrockVectorStoreConfig @@ -378,6 +379,161 @@ async def test_bedrock_search_with_credentials_managed_registry( assert search_response["data"][0]["content"][0]["text"] == "Mock search result" +def _image_search_body(media_type: str, extra_body: dict[str, object] | None) -> dict[str, object]: + config: Final = BedrockVectorStoreConfig() + mock_log: Final = MagicMock() + mock_log.model_call_details = {} + _, body = config.transform_search_vector_store_request( + vector_store_id="kb123", + query="", + vector_store_search_optional_params={ + "max_num_results": 3, + "query_image": {"type": "base64", "media_type": media_type, "data": "aW1hZ2UtYnl0ZXM="}, + }, + api_base="https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases", + litellm_logging_obj=mock_log, + litellm_params={}, + extra_body=extra_body, + ) + return body + + +@pytest.mark.parametrize( + ("media_type", "bedrock_format"), + [("image/jpeg", "jpeg"), ("image/png", "png"), ("image/gif", "gif"), ("image/webp", "webp")], +) +def test_transform_search_request_sends_query_image_as_an_image_retrieval_query( + media_type: str, bedrock_format: str +) -> None: + body: Final = _image_search_body(media_type, extra_body=None) + + assert body["retrievalQuery"] == { + "type": "IMAGE", + "image": {"format": bedrock_format, "inlineContent": "aW1hZ2UtYnl0ZXM="}, + } + assert body["retrievalConfiguration"] == {"vectorSearchConfiguration": {"numberOfResults": 3}} + + +CALLER_RETRIEVAL_QUERY: Final = {"type": "IMAGE", "image": {"format": "png", "inlineContent": "Y2FsbGVy"}} + + +@pytest.mark.parametrize("key", ["retrievalQuery", "retrieval_query"]) +@pytest.mark.parametrize("source", ["extra_body", "litellm_params"]) +def test_transform_search_request_rejects_a_callers_retrieval_query(key: str, source: str) -> None: + caller_fields: Final = {key: CALLER_RETRIEVAL_QUERY} + + with pytest.raises(litellm.BadRequestError) as exc_info: + _search_body( + extra_body=caller_fields if source == "extra_body" else None, + litellm_params=caller_fields if source == "litellm_params" else {}, + ) + + assert exc_info.value.status_code == 400 + assert "query_image" in str(exc_info.value) + + +def test_transform_search_request_drops_a_callers_retrieval_query_when_drop_params_is_on() -> None: + body: Final = _search_body( + extra_body={"retrievalQuery": CALLER_RETRIEVAL_QUERY}, litellm_params={"drop_params": True} + ) + + assert body["retrievalQuery"] == {"text": "hello"} + + +QUERY_IMAGE: Final = {"type": "base64", "media_type": "image/jpeg", "data": "/9j/4AAQSkZJRg=="} +IMAGE_HIT_METADATA: Final = { + "x-amz-bedrock-kb-source-uri": "s3://kb-images/shapes/red-circle.jpg", + "x-amz-bedrock-kb-source-file-mime-type": "image/jpeg", + "x-amz-bedrock-kb-source-file-modality": "IMAGE", +} +TEXT_HIT_METADATA: Final = {"x-amz-bedrock-kb-source-uri": "s3://kb-docs/shapes.txt"} +MIXED_RETRIEVAL_RESULTS: Final = [ + { + "content": {"type": "IMAGE", "byteContent": "data:image/png;base64,/9j/4AAQSkZJRg=="}, + "metadata": IMAGE_HIT_METADATA, + "location": {"type": "S3", "s3Location": {"uri": "s3://kb-images/shapes/red-circle.jpg"}}, + "score": 0.998, + }, + {"content": {"type": "TEXT", "text": "a red circle"}, "metadata": TEXT_HIT_METADATA, "score": 0.5}, + {"content": {"type": "ROW", "row": []}, "metadata": {}, "score": 0.4}, + {"content": {"type": "AUDIO", "byteContent": "data:audio/mpeg;base64,SUQz"}, "metadata": {}, "score": 0.3}, +] +TEXT_HIT: Final = { + "score": 0.5, + "content": [{"text": "a red circle", "type": "text"}], + "file_id": "s3://kb-docs/shapes.txt", + "filename": "shapes.txt", + "attributes": TEXT_HIT_METADATA, +} + + +def _search_response( + retrieval_results: list[dict[str, object]], query: str = "", query_image: dict[str, str] | None = QUERY_IMAGE +) -> dict[str, object]: + config: Final = BedrockVectorStoreConfig() + mock_log: Final = MagicMock() + mock_log.model_call_details = {"query": query, "query_image": query_image} + return config.transform_search_vector_store_response( + response=httpx.Response(200, json={"retrievalResults": retrieval_results}), + litellm_logging_obj=mock_log, + ) + + +def test_transform_search_response_to_an_image_search_returns_image_hits_next_to_text_hits() -> None: + response: Final = _search_response(MIXED_RETRIEVAL_RESULTS) + + assert response["search_query"] == "" + assert response["data"] == [ + { + "score": 0.998, + "content": [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/jpeg", "data": "/9j/4AAQSkZJRg=="}, + } + ], + "file_id": "s3://kb-images/shapes/red-circle.jpg", + "filename": "red-circle.jpg", + "attributes": IMAGE_HIT_METADATA, + }, + TEXT_HIT, + ] + + +def test_transform_search_response_to_a_text_search_returns_only_text_hits() -> None: + response: Final = _search_response(MIXED_RETRIEVAL_RESULTS, query="red circle", query_image=None) + + assert response["search_query"] == "red circle" + assert response["data"] == [TEXT_HIT] + + +@pytest.mark.parametrize( + ("byte_content", "metadata", "media_type", "data"), + [ + ("data:image/webp;base64,UklGRg==", {}, "image/webp", "UklGRg=="), + ("data:image/png;base64,iVBORw0K", {"x-amz-bedrock-kb-source-file-mime-type": ""}, "image/png", "iVBORw0K"), + ( + "data:image/png;base64,iVBORw0K", + {"x-amz-bedrock-kb-source-file-mime-type": "application/pdf"}, + "image/png", + "iVBORw0K", + ), + ("/9j/4AAQ", {"x-amz-bedrock-kb-source-file-mime-type": "image/jpeg"}, "image/jpeg", "/9j/4AAQ"), + ("/9j/4AAQ", {}, "application/octet-stream", "/9j/4AAQ"), + ], +) +def test_transform_search_response_image_hit_media_type_falls_back_to_the_data_uri( + byte_content: str, metadata: dict[str, object], media_type: str, data: str +) -> None: + response: Final = _search_response( + [{"content": {"type": "IMAGE", "byteContent": byte_content}, "metadata": metadata, "score": 0.9}] + ) + + assert response["data"][0]["content"] == [ + {"type": "image", "source": {"type": "base64", "media_type": media_type, "data": data}} + ] + + class _SearchLogging: def __init__(self, query: str) -> None: self.model_call_details: Final = {"query": query} diff --git a/tests/unit/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/unit/llms/mongodb/vector_stores/test_mongodb_transformation.py index 9de473fb1f0..d7d9b21ed20 100644 --- a/tests/unit/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/unit/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -220,3 +220,40 @@ async def test_public_sdk_preserves_http_errors_response_and_timeout( else: assert await search() == RESULT executor.call.assert_called_once_with("embedding-alias", "travel policy", {}) + + +QUERY_IMAGE: Final = {"type": "base64", "media_type": "image/jpeg", "data": "/9j/4AAQSkZJRg=="} + + +def _search_with_extra_body(extra_body: dict[str, object], **search_kwargs: object) -> VectorStoreSearchResponse: + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=RESULT) + + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + return litellm.vector_stores.search( + **BASE_PARAMS, + vector_store_id="policy_index", + query="travel policy", + custom_llm_provider="mongodb", + extra_body=extra_body, + _direct_vector_store_embedding_executor=RecordingEmbeddingExecutor(), + client=HTTPHandler(client=transport), + **search_kwargs, + ) + + +@pytest.mark.parametrize("drop_params_source", ["request", "global"]) +def test_search_with_drop_params_drops_an_extra_body_query_image( + drop_params_source: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params_source == "global") + drop_kwargs: Final = {"drop_params": True} if drop_params_source == "request" else {} + + assert _search_with_extra_body({"query_image": QUERY_IMAGE}, **drop_kwargs) == RESULT + + +def test_search_with_drop_params_still_rejects_other_extra_body_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + + with pytest.raises(litellm.BadRequestError, match="does not support extra_body overrides"): + _search_with_extra_body({"query_image": QUERY_IMAGE, "other": "value"}) diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 75a17d49d3a..fa53f422f36 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -1899,6 +1899,10 @@ def test_raise_if_required_body_param_missing_names_each_missing_param(route_typ ("aspeech", {"model": "gpt-4o-mini-tts", "input": "hi", "voice": "alloy"}), ("amoderation", {"model": "omni-moderation-latest", "input": ""}), ("asearch", {"model": "perplexity-search", "query": "litellm"}), + ( + "avector_store_search", + {"vector_store_id": "kb1", "query_image": {"type": "base64", "media_type": "image/png", "data": "iVBO"}}, + ), ( "acreate_batch", {"input_file_id": "file-abc", "endpoint": "/v1/chat/completions", "completion_window": "24h"}, diff --git a/tests/unit/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/unit/proxy/vector_store_endpoints/test_vector_store_endpoints.py index fa4a1656bc3..2d0f3679144 100644 --- a/tests/unit/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/unit/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -1,5 +1,6 @@ import json from datetime import datetime, timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -15,8 +16,10 @@ from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, ) from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +from litellm.vector_stores.search_query import with_caller_query_params from litellm.proxy.vector_store_endpoints.endpoints import ( _update_request_data_with_litellm_managed_vector_store_registry, + build_request_data_from_managed_vector_store, index_create, index_list, ) @@ -3103,6 +3106,83 @@ def test_vector_store_search_rejects_caller_opensearch_index_name_only_on_opense assert fake_search.await_count == expected_searches +def test_vector_store_search_accepts_a_query_image_without_a_query(): + from fastapi.testclient import TestClient + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + from litellm.types.vector_stores import VectorStoreSearchResponse + + query_image: Final = {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="} + store: Final = { + "vector_store_id": "kb", + "custom_llm_provider": "bedrock", + "litellm_params": {"vector_store_id": "kb", "custom_llm_provider": "bedrock"}, + } + registry: Final = MagicMock() + registry.get_litellm_managed_vector_store_from_registry.return_value = store + fake_search: Final = AsyncMock( + return_value=VectorStoreSearchResponse(object="vector_store.search_results.page", search_query="", data=[]) + ) + mock_auth: Final = UserAPIKeyAuth(user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER.value) + original_overrides: Final = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: mock_auth + try: + with ( + patch("litellm.vector_stores.main.asearch", new=fake_search), # test-quality-ok: the search boundary under test + patch.object(litellm, "vector_store_registry", registry), # test-quality-ok: seeds the store under test + patch( # test-quality-ok: store access is not under test, so the request reaches the search boundary + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), + ), + ): + router: Final = litellm.Router(model_list=[]) + with patch("litellm.proxy.proxy_server.llm_router", router): # test-quality-ok: router built over the fake search + response: Final = TestClient(app).post("/v1/vector_stores/kb/search", json={"query_image": query_image}) + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 200, response.json() + assert fake_search.await_args.kwargs["query_image"] == query_image + + +@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"]) +def test_a_stored_raw_retrieval_query_never_reaches_the_search_request(retrieval_query_param: str): + request_data: Final = build_request_data_from_managed_vector_store( + { + "vector_store_id": "kb", + "custom_llm_provider": "bedrock", + "litellm_params": {"aws_region_name": "us-west-2", retrieval_query_param: {"text": "from the store"}}, + } + ) + + assert request_data == {"custom_llm_provider": "bedrock", "aws_region_name": "us-west-2"} + + +@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"]) +def test_a_raw_retrieval_query_in_a_stores_extra_body_never_reaches_the_search_request(retrieval_query_param: str): + request_data: Final = build_request_data_from_managed_vector_store( + { + "vector_store_id": "kb", + "custom_llm_provider": "bedrock", + "litellm_params": {"extra_body": {retrieval_query_param: {"text": "from the store"}, "kept": "value"}}, + } + ) + + assert request_data == {"custom_llm_provider": "bedrock", "extra_body": {"kept": "value"}} + + +@pytest.mark.parametrize("caller_param", ["query_image", "retrievalQuery", "retrieval_query"]) +def test_a_callers_query_survives_a_stores_own_extra_body(caller_param: str): + caller_value: Final = {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="} + request_data: Final = with_caller_query_params( + {"custom_llm_provider": "bedrock", "extra_body": {"retrievalConfiguration": "from the store"}}, + {caller_param: caller_value, "retrievalConfiguration": "from the caller", "other": "from the caller"}, + ) + + assert request_data["extra_body"] == {"retrievalConfiguration": "from the store", caller_param: caller_value} + + class TestConfigOwnedVectorStores: """Stores declared under ``vector_store_registry`` in config.yaml are owned by the config file""" diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index ad1e15330d2..71efa1352e4 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -11900,3 +11900,39 @@ def test_per_call_custom_prices_keep_the_regional_and_speed_multipliers( ) assert response._hidden_params["response_cost"] == pytest.approx((1000 * 2e-06 + 1000 * 4e-06) * multiplier) + + +@pytest.mark.parametrize( + ("message_logging_off", "query_image", "echoes_response"), + [ + (True, None, False), + (False, None, True), + (False, {"type": "base64", "media_type": "image/jpeg", "data": "cXVlcnk="}, False), + ], +) +def test_cost_failure_for_an_unpriced_response_echoes_only_a_text_search_response_with_message_logging_on( + message_logging_off: bool, + query_image: Mapping[str, str] | None, + echoes_response: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", message_logging_off) + image_data: Final = "c2VjcmV0LWltYWdlLWJ5dGVz" + logging_obj: Final = Logging( + model="", + messages=[], + stream=False, + call_type="avector_store_search", + start_time=datetime.datetime.now(), + litellm_call_id="unpriced-search", + function_id="f", + ) + logging_obj.model_call_details["query_image"] = query_image + + with pytest.raises(ValueError, match="Model is None") as exc_info: + completion_cost( + completion_response={"data": [{"content": [{"source": {"data": image_data}}]}]}, + litellm_logging_obj=logging_obj, + ) + + assert (image_data in str(exc_info.value)) is echoes_response diff --git a/tests/unit/test_router/core/test_endpoints_characterization_generic.py b/tests/unit/test_router/core/test_endpoints_characterization_generic.py index f7669bff80c..9a78f5d4fb1 100644 --- a/tests/unit/test_router/core/test_endpoints_characterization_generic.py +++ b/tests/unit/test_router/core/test_endpoints_characterization_generic.py @@ -431,6 +431,29 @@ async def test_a_plain_response_id_without_a_model_fails_with_type_error(make_co assert host.calls == [] +@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"]) +@pytest.mark.parametrize("endpoint", ["avector_store_search", "vector_store_search"]) +async def test_a_deployments_raw_retrieval_query_never_reaches_a_routed_vector_store_search( + make_core: MakeCore, host: EndpointHost, endpoint: str, retrieval_query_param: str +): + from_deployment: Final = {"text": "from the deployment"} + core: Final = make_core( + model_list=[ + deployment( + "g", + "d1", + **{retrieval_query_param: from_deployment, "extra_body": {retrieval_query_param: from_deployment, "kept": 1}}, + ) + ] + ) + + await _call(core, endpoint, {"model": "g", "vector_store_id": "v", "query": "q"}) + + sent: Final = host.calls[0].kwargs + assert retrieval_query_param not in sent + assert sent["extra_body"] == {"kept": 1} + + # --- container ids ------------------------------------------------------------------------ ENCODED_CONTAINER_ID: Final = ResponsesAPIRequestUtils.build_container_id("azure", "d2", "cntr_upstream") diff --git a/tests/unit/vector_stores/test_main.py b/tests/unit/vector_stores/test_main.py index 21b1eee3a3e..b89806e6377 100644 --- a/tests/unit/vector_stores/test_main.py +++ b/tests/unit/vector_stores/test_main.py @@ -120,6 +120,147 @@ def test_search_forwards_top_level_user_context_to_bedrock_retrieve(): assert posted["retrievalQuery"] == {"text": "q"} +QUERY_IMAGE: Final = {"type": "base64", "media_type": "image/jpeg", "data": "/9j/4AAQSkZJRg=="} + + +def _search_with_recorded_post(**search_kwargs: object) -> MagicMock: + client: Final = MagicMock(spec=HTTPHandler) + client.post.return_value = MagicMock( + status_code=200, + json=MagicMock( + return_value=( + {"object": "vector_store.search_results.page", "search_query": "", "data": []} + if search_kwargs.get("custom_llm_provider") == "openai" + else {"retrievalResults": []} + ) + ), + ) + credentials: Final = ( + {"api_key": "sk-test"} + if search_kwargs.get("custom_llm_provider") == "openai" + else {"aws_region_name": "us-west-2", "aws_access_key_id": "test-key-id", "aws_secret_access_key": "secret"} + ) + search(vector_store_id="kb123", client=client, litellm_logging_obj=MagicMock(), **credentials, **search_kwargs) + return client + + +@pytest.mark.parametrize("query", [None, "", " ", ["", " "]]) +def test_search_by_query_image_sends_a_bedrock_image_retrieval_query(query: str | list[str] | None): + query_kwargs: Final = {} if query is None else {"query": query} + + client: Final = _search_with_recorded_post(custom_llm_provider="bedrock", query_image=QUERY_IMAGE, **query_kwargs) + + posted: Final = json.loads(client.post.call_args.kwargs["data"]) + assert posted["retrievalQuery"] == { + "type": "IMAGE", + "image": {"format": "jpeg", "inlineContent": "/9j/4AAQSkZJRg=="}, + } + + +@pytest.mark.parametrize( + ("search_kwargs", "message"), + [ + ( + {"custom_llm_provider": "bedrock", "query": "red circle", "query_image": QUERY_IMAGE}, + "Send either query or query_image, not both.", + ), + ({"custom_llm_provider": "bedrock"}, "query or query_image is required."), + ( + {"custom_llm_provider": "bedrock", "query": 5, "query_image": QUERY_IMAGE}, + "Send either query or query_image, not both.", + ), + ( + {"custom_llm_provider": "bedrock", "query": [1, 2], "query_image": QUERY_IMAGE}, + "Send either query or query_image, not both.", + ), + ( + {"custom_llm_provider": "openai", "query_image": QUERY_IMAGE}, + "openai vector stores do not support query_image. To drop it", + ), + ( + {"custom_llm_provider": "openai", "query": "red circle", "query_image": QUERY_IMAGE}, + "openai vector stores do not support query_image. To drop it", + ), + ( + {"custom_llm_provider": "openai", "query": "", "query_image": QUERY_IMAGE, "drop_params": True}, + "drop_params dropped it", + ), + ], +) +def test_search_rejects_an_unusable_query_with_a_bad_request(search_kwargs: dict[str, object], message: str): + with pytest.raises(litellm.BadRequestError) as exc_info: + _search_with_recorded_post(**search_kwargs) + + assert exc_info.value.status_code == 400 + assert message in str(exc_info.value) + + +def test_search_rejects_an_invalid_query_image_without_echoing_its_data(): + image_data: Final = "c2VjcmV0LWltYWdlLWJ5dGVz" + + with pytest.raises(litellm.BadRequestError) as exc_info: + _search_with_recorded_post( + custom_llm_provider="bedrock", + query_image={"type": "base64", "media_type": "image/tiff", "data": image_data}, + ) + + assert exc_info.value.status_code == 400 + assert "query_image.media_type" in str(exc_info.value) + assert image_data not in str(exc_info.value) + + +@pytest.mark.parametrize("data", ["not base64!", "/9j/4AAQ SkZJRg==", "/9j/4AAQ\tSkZJRg=="]) +def test_search_rejects_query_image_data_that_is_not_base64(data: str): + with pytest.raises(litellm.BadRequestError) as exc_info: + _search_with_recorded_post( + custom_llm_provider="bedrock", + query_image={"type": "base64", "media_type": "image/jpeg", "data": data}, + ) + + assert exc_info.value.status_code == 400 + assert "query_image.data: not valid base64" in str(exc_info.value) + assert data not in str(exc_info.value) + + +@pytest.mark.parametrize("line_break", ["\n", "\r\n"]) +def test_search_sends_line_wrapped_query_image_data_to_bedrock_as_sent(line_break: str): + data: Final = line_break.join(("/9j/4AAQ", "SkZJRg==")) + + client: Final = _search_with_recorded_post( + custom_llm_provider="bedrock", + query_image={"type": "base64", "media_type": "image/jpeg", "data": data}, + ) + + posted: Final = json.loads(client.post.call_args.kwargs["data"]) + assert posted["retrievalQuery"]["image"]["inlineContent"] == data + + +def test_search_by_a_query_image_sent_in_extra_body_sends_a_bedrock_image_retrieval_query(): + client: Final = _search_with_recorded_post(custom_llm_provider="bedrock", extra_body={"query_image": QUERY_IMAGE}) + + posted: Final = json.loads(client.post.call_args.kwargs["data"]) + assert posted["retrievalQuery"] == { + "type": "IMAGE", + "image": {"format": "jpeg", "inlineContent": "/9j/4AAQSkZJRg=="}, + } + + +@pytest.mark.parametrize("drop_params_source", ["request", "global"]) +def test_search_drops_query_image_for_a_provider_without_image_search_when_drop_params_is_on( + drop_params_source: str, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "drop_params", drop_params_source == "global") + drop_kwargs: Final = {"drop_params": True} if drop_params_source == "request" else {} + + client: Final = _search_with_recorded_post( + custom_llm_provider="openai", query="red circle", query_image=QUERY_IMAGE, **drop_kwargs + ) + + posted: Final = json.loads(client.post.call_args.kwargs["data"]) + assert posted["query"] == "red circle" + assert "query_image" not in posted + + PROXY_STAMPED_METADATA: Final = { "team": "llmproxy", "user_api_key": "hashed-key", diff --git a/tests/unit/vector_stores/test_vector_store_registry.py b/tests/unit/vector_stores/test_vector_store_registry.py index a6e28f258cb..588dbf35678 100644 --- a/tests/unit/vector_stores/test_vector_store_registry.py +++ b/tests/unit/vector_stores/test_vector_store_registry.py @@ -497,6 +497,29 @@ def test_vector_store_ids_to_run_reads_every_store_id_a_malformed_vector_store_i assert VectorStoreRegistry().get_vector_store_ids_to_run(non_default_params={}, tools=tools) == ["vs_requested"] +@pytest.mark.parametrize( + ("tool_field", "tool_value", "searched_params"), + [ + ("query_image", {"type": "base64", "media_type": "image/jpeg", "data": "/9j/"}, {"max_num_results": 3}), + ("rewrite_query", True, {"max_num_results": 3, "rewrite_query": True}), + ("rewrite_query", False, {"max_num_results": 3}), + ], +) +def test_file_search_tool_passes_its_store_only_the_fields_a_store_search_takes( + tool_field: str, tool_value: object, searched_params: dict[str, object] +) -> None: + now: Final = datetime.now(timezone.utc) + registry: Final = VectorStoreRegistry( + [LiteLLM_ManagedVectorStore(vector_store_id="vs_kb", created_at=now, updated_at=now)] + ) + tools: Final = [{"type": "file_search", "vector_store_ids": ["vs_kb"], "max_num_results": 3, tool_field: tool_value}] + + to_run: Final = registry.pop_vector_stores_to_run(non_default_params={}, tools=tools) + + assert [store["vector_store_id"] for store in to_run] == ["vs_kb"] + assert to_run[0]["litellm_params"] == searched_params + + @pytest.mark.parametrize( "vector_store_ids", ["vs_allowed", ["vs_allowed", 12345], [12345], ["vs_allowed"]], From 0608a41a077dfe6f79e8dcdc1668cbb2702e622b Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 11 Nov 2026 16:22:14 -0800 Subject: [PATCH 134/189] fix(fips): hash MCP advisory-lock keys with SHA-256 in fips_mode and compute the OCI body digest as a security hash (#51165) * fix(fips): hash MCP advisory-lock keys with SHA-256 in fips_mode and compute the OCI body digest as a security hash * test(fips): move the lock-key tests into test_mcp_env_vars.py and type the OCI digest test --- litellm/llms/oci/common_utils.py | 12 ++---- litellm/proxy/mcp_server/db.py | 24 ++++++----- tests/unit/llms/oci/test_oci_common_utils.py | 14 ++++++ .../proxy/mcp_server/test_mcp_env_vars.py | 43 +++++++++++++++++++ 4 files changed, 75 insertions(+), 18 deletions(-) diff --git a/litellm/llms/oci/common_utils.py b/litellm/llms/oci/common_utils.py index fe8d0e1c1e7..a545fc26481 100644 --- a/litellm/llms/oci/common_utils.py +++ b/litellm/llms/oci/common_utils.py @@ -97,15 +97,11 @@ class OCIRequestWrapper: def sha256_base64(data: bytes) -> str: - # SHA-256 is used here to compute the x-content-sha256 header required by the - # OCI HTTP signing specification (RSA-SHA256 request signing), not for password - # or secret hashing. This is the correct and mandated algorithm for this purpose. + # Computes the x-content-sha256 header required by the OCI HTTP signing + # specification. The RSA-SHA256 request signature covers this header, so the + # digest is a security use of the hash. # See: https://docs.oracle.com/en-us/iaas/Content/API/Concepts/signingrequests.htm - # - # ``usedforsecurity=False`` declares non-security intent to static analyzers - # (CodeQL ``py/weak-sensitive-data-hashing``) — without it the request body - # gets flagged as "password-like data" via taint tracking. - digest: Final = hashlib.sha256(data, usedforsecurity=False).digest() # noqa: S324 + digest: Final = hashlib.sha256(data).digest() return base64.b64encode(digest).decode() diff --git a/litellm/proxy/mcp_server/db.py b/litellm/proxy/mcp_server/db.py index f29acc3b380..08e1ffd5333 100644 --- a/litellm/proxy/mcp_server/db.py +++ b/litellm/proxy/mcp_server/db.py @@ -35,6 +35,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: F401 # encrypt_value_helper, get_salt_key, ) +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.mcp_server.oauth_identity_binding import ( RefreshTokenPresented, credential_binding_matches, @@ -690,15 +691,22 @@ async def find_mcp_server_identifier_conflict( ) +def _advisory_lock_key(name: str) -> int: + """Signed 64-bit Postgres advisory-lock key for ``name``: BLAKE2b-64, or the + first 8 bytes of SHA-256 when general_settings.fips_mode is on.""" + digest: Final = ( + hashlib.sha256(name.encode()).digest()[:8] + if fips_mode_active() + else hashlib.blake2b(name.encode(), digest_size=8).digest() + ) + return int.from_bytes(digest, "big", signed=True) + + def _mcp_identifier_lock_keys(*identifiers: str | None) -> tuple[int, ...]: """Deterministic advisory-lock keys for the lowercased identifiers, sorted so concurrent requests for the same pair always lock in the same order.""" return tuple( - int.from_bytes( - hashlib.blake2b(f"mcp_identifier:{normalized}".encode(), digest_size=8).digest(), - "big", - signed=True, - ) + _advisory_lock_key(f"mcp_identifier:{normalized}") for normalized in sorted(frozenset(value.lower() for value in identifiers if value)) ) @@ -2515,11 +2523,7 @@ async def merge_user_env_vars( so an admin retiring a user-scoped variable also clears its stored value. """ allowed: Final = set(allowed_names) - lock_key: Final = int.from_bytes( - hashlib.blake2b(f"{user_id}:{server_id}".encode(), digest_size=8).digest(), - "big", - signed=True, - ) + lock_key: Final = _advisory_lock_key(f"{user_id}:{server_id}") async with _db_transaction_manager(prisma_client) as tx: await tx.execute_raw("SELECT pg_advisory_xact_lock($1::bigint)", lock_key) row: Final[prisma_db_models.LiteLLM_MCPUserEnvVars | None] = await tx.litellm_mcpuserenvvars.find_unique( diff --git a/tests/unit/llms/oci/test_oci_common_utils.py b/tests/unit/llms/oci/test_oci_common_utils.py index e66645c4dcd..34e66629120 100644 --- a/tests/unit/llms/oci/test_oci_common_utils.py +++ b/tests/unit/llms/oci/test_oci_common_utils.py @@ -54,6 +54,20 @@ def test_sha256_base64_known_value(): assert sha256_base64(data) == expected +def test_sha256_base64_asks_for_a_security_digest(monkeypatch: pytest.MonkeyPatch) -> None: + import hashlib + + real_sha256: Final = hashlib.sha256 + + def security_only_sha256(data: bytes = b"", *, usedforsecurity: bool = True) -> "hashlib._Hash": + assert usedforsecurity, "usedforsecurity=False lets a provider outside the FIPS module serve the digest" + return real_sha256(data) + + monkeypatch.setattr(hashlib, "sha256", security_only_sha256) + + assert sha256_base64(b"hello") == "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ=" + + def test_sha256_base64_empty(): result = sha256_base64(b"") assert isinstance(result, str) diff --git a/tests/unit/proxy/mcp_server/test_mcp_env_vars.py b/tests/unit/proxy/mcp_server/test_mcp_env_vars.py index 57ac6c0be2d..b3aebe49968 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_env_vars.py +++ b/tests/unit/proxy/mcp_server/test_mcp_env_vars.py @@ -1748,3 +1748,46 @@ async def test_missing_user_env_vars_error_renders_in_mcp_call_tool(): assert "CorporateDB" in text assert "CORP_USERNAME" in text assert "fill_env_vars=srv-99" in text + + +@pytest.fixture +def blake2b_refused(monkeypatch: pytest.MonkeyPatch) -> None: + """hashlib.blake2b as FIPS Python builds ship it: the call raises.""" + import hashlib + from typing import NoReturn + + def blake2b(*_args: object, **_kwargs: object) -> NoReturn: + raise ValueError("blake2b is blocked when usedforsecurity=True") + + monkeypatch.setattr(hashlib, "blake2b", blake2b) + + +def test_identifier_lock_keys_are_unchanged_without_fips_mode() -> None: + from litellm.proxy.mcp_server.db import _mcp_identifier_lock_keys + + assert _mcp_identifier_lock_keys("GitHub", None, "gh") == (-1720930101670657375, -1498181317228972674) + + +@pytest.mark.usefixtures("fips_mode", "blake2b_refused") +def test_fips_mode_identifier_lock_keys_are_truncated_sha256() -> None: + import hashlib + + from litellm.proxy.mcp_server.db import _mcp_identifier_lock_keys + + assert _mcp_identifier_lock_keys("GitHub", None, "gh") == tuple( + int.from_bytes(hashlib.sha256(name).digest()[:8], "big", signed=True) + for name in (b"mcp_identifier:gh", b"mcp_identifier:github") + ) + assert _mcp_identifier_lock_keys("GitHub") == (-7731150984991732978,) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode", "blake2b_refused", "env_vars_salt_key") +async def test_fips_mode_merge_user_env_vars_takes_its_lock_without_blake2b() -> None: + from litellm.proxy.mcp_server.db import merge_user_env_vars + + merged: Final = await merge_user_env_vars( + _transactional_env_vars_prisma(), "alice", "srv-1", {"TOKEN": "abc"}, allowed_names=["TOKEN"] + ) + + assert merged == {"TOKEN": "abc"} From 072290e78f278c3a3caaadb61c76e754c3afafbf Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 11 Nov 2026 16:26:58 -0800 Subject: [PATCH 135/189] test(proxy): await background spend counters before asserting (#51249) * test(proxy): wait for child readiness before supervisor cancellation * test(proxy): sync supervisor cancellation test on a socketpair handshake Replace the readiness marker poll and timed sleeps with one-byte socketpair messages. The child reports its SIGTERM handler is installed, reports the stop, and holds its exit until the parent releases it, so the assertion only passes when the cancelled supervisor waits for the child. A pytest-timeout marker bounds a deadlock. * test(proxy): drive supervisor cancellation cleanup with a controlled child Replace the real child and socket handshake with a scripted process returned by asyncio.create_subprocess_exec and a stop observed through os.killpg. The child only exits after the stop and a checkpoint, so returncode is 0 only when the cancelled supervisor stops the child and waits for it without being cancelled. * test(proxy): start the supervisor cancellation only once the reporter waits The scripted child signals when a wait begins, and the test cancels only after the exit reporter is blocked on the stop, so cancellation reaches it without relying on task start order. * test(proxy): await background failure spend counters before asserting Failure spend counters finish in a background task when the counter store is slower than the wait budget, so the tests read stale values on a slow runner. Hold the counter store on a release event with a zero wait budget, capture the background task, and await it before asserting the counters. --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../hooks/test_proxy_track_cost_callback.py | 135 ++++++++++++++---- 1 file changed, 110 insertions(+), 25 deletions(-) diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index dcec934188f..1e5ad21fc5d 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -1,11 +1,11 @@ import asyncio import json import logging -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping from datetime import datetime, timedelta, timezone from pathlib import Path from types import MappingProxyType, SimpleNamespace -from typing import Final, cast +from typing import Final, ParamSpec, cast from unittest.mock import AsyncMock, MagicMock, create_autospec, patch import httpx @@ -29,6 +29,7 @@ from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction from litellm.proxy.db.spend_log_tool_index import response_tool_call_names from litellm.proxy.hooks.proxy_track_cost_callback import ( ProxyDBLogger, + _IncrementSpendCounters, _charge_request_cost, _get_budget_reservation_from_metadata, _should_track_cost_callback, @@ -2970,7 +2971,42 @@ async def _counter_values(counters: SpendCounterStore) -> dict[str, float | None return {key: counters.get_local(key) for key in _LIVE_COUNTERS} +_P = ParamSpec("_P") + + +def _held_until( + release: asyncio.Event, counting: asyncio.Future[asyncio.Task[object]], store: Callable[_P, Awaitable[None]] +) -> Callable[_P, Awaitable[None]]: + async def held(*args: _P.args, **kwargs: _P.kwargs) -> None: + if (task := asyncio.current_task()) is not None and not counting.done(): + counting.set_result(task) + await release.wait() + await store(*args, **kwargs) + + return held + + +async def _fail_request_and_finish_counting_in_background( + store: _IncrementSpendCounters, failure: Mapping[str, object], committed: CommittedBudgetSpend | None = None +) -> None: + release: Final = asyncio.Event() + counting: Final[asyncio.Future[asyncio.Task[object]]] = asyncio.get_running_loop().create_future() + held: Final = _held_until(release, counting, store) + try: + await _fail_request( + ProxyDBLogger(spend_counters=lambda: held, failure_counter_wait_seconds=0), failure, committed + ) + background: Final = await counting + release.set() + await background + finally: + release.set() + if counting.done(): + await asyncio.wait((counting.result(),)) + + @pytest.mark.asyncio +@pytest.mark.timeout(30) @pytest.mark.parametrize( ("failure", "charged"), [ @@ -2982,7 +3018,6 @@ async def _counter_values(counters: SpendCounterStore) -> dict[str, float | None 0.125, ), (_failure_with_billed_attempts(), 1.25), - ({}, 0.0), ], ) async def test_failure_spend_rows_raise_the_live_spend_counters_by_their_cost( @@ -2990,10 +3025,12 @@ async def test_failure_spend_rows_raise_the_live_spend_counters_by_their_cost( ): """#35568: budget checks read the live counters, so a failure row's cost has to land there as well as in the spend log, or an over-budget key keeps passing until the counter is next repaired from the database.""" + from litellm.proxy.proxy_server import increment_spend_counters + for key, spend in _LIVE_COUNTERS.items(): live_spend_counters.set_local(key, spend) - await _fail_request(ProxyDBLogger(), failure) + await _fail_request_and_finish_counting_in_background(increment_spend_counters, failure) assert await _counter_values(live_spend_counters) == pytest.approx( {key: spend + charged for key, spend in _LIVE_COUNTERS.items()} @@ -3001,39 +3038,83 @@ async def test_failure_spend_rows_raise_the_live_spend_counters_by_their_cost( @pytest.mark.asyncio +async def test_a_failure_that_cost_nothing_leaves_the_live_spend_counters_alone(live_spend_counters: SpendCounterStore): + for key, spend in _LIVE_COUNTERS.items(): + live_spend_counters.set_local(key, spend) + + await _fail_request(ProxyDBLogger(), {}) + + assert await _counter_values(live_spend_counters) == pytest.approx(_LIVE_COUNTERS) + + +_COMMITTED_COUNTERS: Final = MappingProxyType( + { + "spend:committed:key:hashed-key": 0.2, + "spend:committed:team:team-1": 0.3, + "spend:committed:user:user-1": 0.4, + } +) +_BUDGETED_KEY_AND_TEAM: Final = CommittedBudgetSpend( + counter_keys=EntitySpendCounterKeys.of( + token="hashed-key", team_id="team-1", user_id=None, org_id=None, end_user_id=None + ) +) + + +def _committed_counter_values(counters: SpendCounterStore) -> dict[str, float | None]: + return {key: counters.get_local(key) for key in _COMMITTED_COUNTERS} + + +def _committed_counters_after_charging_key_and_team(charged: float) -> dict[str, float]: + return { + "spend:committed:key:hashed-key": 0.2 + charged, + "spend:committed:team:team-1": 0.3 + charged, + "spend:committed:user:user-1": 0.4, + } + + +@pytest.mark.asyncio +@pytest.mark.timeout(30) @pytest.mark.parametrize( ("failure", "charged"), [ ({"combined_usage_object": Usage(prompt_tokens=30), "response_cost": 0.125}, 0.125), (_failure_with_billed_attempts(), 1.25), - ({"standard_logging_object": {"cost_finalized": True, "response_cost": 0.125}}, 0.125), ], - ids=["recovered_stream_cost", "billed_retries", "completed_cost"], + ids=["recovered_stream_cost", "billed_retries"], ) async def test_a_failed_charge_moves_only_the_committed_counters_auth_named_as_budgeted( live_spend_counters: SpendCounterStore, failure: Mapping[str, object], charged: float ): """The remaining budget gauges read the committed counters, so a charge missing from them shows as budget the request never left behind, until the next reseed from the database.""" - budgeted: Final = EntitySpendCounterKeys.of( - token="hashed-key", team_id="team-1", user_id=None, org_id=None, end_user_id=None - ) - starting: Final = { - "spend:committed:key:hashed-key": 0.2, - "spend:committed:team:team-1": 0.3, - "spend:committed:user:user-1": 0.4, - } - for key, spend in starting.items(): + from litellm.proxy.proxy_server import increment_spend_counters + + for key, spend in _COMMITTED_COUNTERS.items(): live_spend_counters.set_local(key, spend) - await _fail_request(ProxyDBLogger(), failure, CommittedBudgetSpend(counter_keys=budgeted)) + await _fail_request_and_finish_counting_in_background(increment_spend_counters, failure, _BUDGETED_KEY_AND_TEAM) - assert {key: live_spend_counters.get_local(key) for key in starting} == pytest.approx( - { - "spend:committed:key:hashed-key": 0.2 + charged, - "spend:committed:team:team-1": 0.3 + charged, - "spend:committed:user:user-1": 0.4, - } + assert _committed_counter_values(live_spend_counters) == pytest.approx( + _committed_counters_after_charging_key_and_team(charged) + ) + + +@pytest.mark.asyncio +async def test_a_failed_completed_charge_moves_only_the_committed_counters_auth_named_as_budgeted( + live_spend_counters: SpendCounterStore, +): + for key, spend in _COMMITTED_COUNTERS.items(): + live_spend_counters.set_local(key, spend) + + await _fail_request( + ProxyDBLogger(), + {"standard_logging_object": {"cost_finalized": True, "response_cost": 0.125}}, + _BUDGETED_KEY_AND_TEAM, + ) + + assert _committed_counter_values(live_spend_counters) == pytest.approx( + _committed_counters_after_charging_key_and_team(0.125) ) @@ -3067,10 +3148,11 @@ async def test_failure_spend_rows_leave_the_live_spend_counters_alone_while_spen @pytest.mark.asyncio +@pytest.mark.timeout(30) async def test_a_failing_spend_counter_store_is_still_sent_every_failure_spend_row(): counters: Final = AsyncMock(side_effect=RuntimeError("redis unavailable")) - await _fail_request(ProxyDBLogger(spend_counters=lambda: counters), _failure_with_billed_attempts()) + await _fail_request_and_finish_counting_in_background(counters, _failure_with_billed_attempts()) assert [call.kwargs["response_cost"] for call in counters.call_args_list] == [0.5, 0.25, 0.5] @@ -3141,6 +3223,7 @@ async def test_a_failure_spend_row_lands_in_the_key_budget_window_as_of_the_requ @pytest.mark.asyncio +@pytest.mark.timeout(30) async def test_a_billed_attempt_on_a_deployment_the_router_dropped_debits_every_attributed_access_group( live_spend_counters: SpendCounterStore, monkeypatch: pytest.MonkeyPatch ): @@ -3159,8 +3242,10 @@ async def test_a_billed_attempt_on_a_deployment_the_router_dropped_debits_every_ ), ) - await _fail_request( - ProxyDBLogger(), + from litellm.proxy.proxy_server import increment_spend_counters + + await _fail_request_and_finish_counting_in_background( + increment_spend_counters, { **_failure_with_billed_attempts(), "metadata": {MODEL_ACCESS_GROUP_METADATA_KEY: ["pool", "premium"], "model_info": {"id": "dep-b"}}, From fb207a88a9dab30a4df548f257cd218e9787face Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:43:40 +0000 Subject: [PATCH 136/189] chore(databricks): price claude-fable-5-2 from the proprietary pricing page (#51232) * chore(databricks): price claude-fable-5-2 from the proprietary pricing page * test(databricks): list claude-fable-5-2 among the rows with published cache DBU rates --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 30 +++++++++++-------- model_prices_and_context_window.json | 30 +++++++++++-------- .../test_databricks_cost_calculator.py | 2 ++ 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b6eec1c9574..810fd22a64f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -99949,17 +99949,20 @@ ] }, "databricks/databricks-claude-fable-5-2": { - "cache_creation_input_token_cost": 8e-06, - "cache_read_input_token_cost": 8e-06, - "input_cost_per_token": 8e-06, + "cache_creation_input_token_cost": 9.99999e-06, + "cache_creation_input_token_cost_above_1hr": 1.599997e-05, + "cache_read_input_token_cost": 1.9999e-07, + "input_cost_per_token": 8.00002e-06, + "input_dbu_cost_per_token": 0.000114286, "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "metadata": {"notes":"Claude Fable 5.2 on Databricks is available only through Unity Gateway with the model service name system.ai.claude-fable-5-2, and Databricks hosts it through Amazon Bedrock. The Databricks pricing page lists no row for it yet, so input and output are inherited from the catalog Bedrock row anthropic.claude-fable-5-2, no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "metadata": {"notes":"Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields."}, "mode": "chat", - "output_cost_per_token": 4e-05, + "output_cost_per_token": 4.000003e-05, + "output_dbu_cost_per_token": 0.000571429, "prompt_cache_min_tokens": 512.0, - "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, "supports_anthropic_thinking_payload": true, "supports_assistant_prefill": false, @@ -99974,17 +99977,20 @@ "thinking_always_on": true }, "databricks/system.ai.claude-fable-5-2": { - "cache_creation_input_token_cost": 8e-06, - "cache_read_input_token_cost": 8e-06, - "input_cost_per_token": 8e-06, + "cache_creation_input_token_cost": 9.99999e-06, + "cache_creation_input_token_cost_above_1hr": 1.599997e-05, + "cache_read_input_token_cost": 1.9999e-07, + "input_cost_per_token": 8.00002e-06, + "input_dbu_cost_per_token": 0.000114286, "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "metadata": {"notes":"Claude Fable 5.2 on Databricks is available only through Unity Gateway with the model service name system.ai.claude-fable-5-2, and Databricks hosts it through Amazon Bedrock. The Databricks pricing page lists no row for it yet, so input and output are inherited from the catalog Bedrock row anthropic.claude-fable-5-2, no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "metadata": {"notes":"Claude Fable 5.2 on Databricks is served through Unity Gateway with the model service name system.ai.claude-fable-5-2, hosted through Amazon Bedrock. Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields."}, "mode": "chat", - "output_cost_per_token": 4e-05, + "output_cost_per_token": 4.000003e-05, + "output_dbu_cost_per_token": 0.000571429, "prompt_cache_min_tokens": 512.0, - "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, "supports_anthropic_thinking_payload": true, "supports_assistant_prefill": false, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b6eec1c9574..810fd22a64f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -99949,17 +99949,20 @@ ] }, "databricks/databricks-claude-fable-5-2": { - "cache_creation_input_token_cost": 8e-06, - "cache_read_input_token_cost": 8e-06, - "input_cost_per_token": 8e-06, + "cache_creation_input_token_cost": 9.99999e-06, + "cache_creation_input_token_cost_above_1hr": 1.599997e-05, + "cache_read_input_token_cost": 1.9999e-07, + "input_cost_per_token": 8.00002e-06, + "input_dbu_cost_per_token": 0.000114286, "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "metadata": {"notes":"Claude Fable 5.2 on Databricks is available only through Unity Gateway with the model service name system.ai.claude-fable-5-2, and Databricks hosts it through Amazon Bedrock. The Databricks pricing page lists no row for it yet, so input and output are inherited from the catalog Bedrock row anthropic.claude-fable-5-2, no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "metadata": {"notes":"Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields."}, "mode": "chat", - "output_cost_per_token": 4e-05, + "output_cost_per_token": 4.000003e-05, + "output_dbu_cost_per_token": 0.000571429, "prompt_cache_min_tokens": 512.0, - "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, "supports_anthropic_thinking_payload": true, "supports_assistant_prefill": false, @@ -99974,17 +99977,20 @@ "thinking_always_on": true }, "databricks/system.ai.claude-fable-5-2": { - "cache_creation_input_token_cost": 8e-06, - "cache_read_input_token_cost": 8e-06, - "input_cost_per_token": 8e-06, + "cache_creation_input_token_cost": 9.99999e-06, + "cache_creation_input_token_cost_above_1hr": 1.599997e-05, + "cache_read_input_token_cost": 1.9999e-07, + "input_cost_per_token": 8.00002e-06, + "input_dbu_cost_per_token": 0.000114286, "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "metadata": {"notes":"Claude Fable 5.2 on Databricks is available only through Unity Gateway with the model service name system.ai.claude-fable-5-2, and Databricks hosts it through Amazon Bedrock. The Databricks pricing page lists no row for it yet, so input and output are inherited from the catalog Bedrock row anthropic.claude-fable-5-2, no '*_dbu_cost_per_token' fields are published, and cache tokens bill at the input rate until Databricks publishes cache DBU rates."}, + "metadata": {"notes":"Claude Fable 5.2 on Databricks is served through Unity Gateway with the model service name system.ai.claude-fable-5-2, hosted through Amazon Bedrock. Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields."}, "mode": "chat", - "output_cost_per_token": 4e-05, + "output_cost_per_token": 4.000003e-05, + "output_dbu_cost_per_token": 0.000571429, "prompt_cache_min_tokens": 512.0, - "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_adaptive_thinking": true, "supports_anthropic_thinking_payload": true, "supports_assistant_prefill": false, diff --git a/tests/unit/llms/databricks/test_databricks_cost_calculator.py b/tests/unit/llms/databricks/test_databricks_cost_calculator.py index e9bc75119aa..b968191383d 100644 --- a/tests/unit/llms/databricks/test_databricks_cost_calculator.py +++ b/tests/unit/llms/databricks/test_databricks_cost_calculator.py @@ -44,6 +44,8 @@ PUBLISHED_DBU_PER_MILLION: Final = { "databricks/databricks-claude-opus-5-5": ("57.143", "285.714", "71.429", "2.857"), "databricks/databricks-claude-fable-5-1": ("142.858", "714.286", "178.572", "3.572"), "databricks/databricks-claude-fable-5": ("142.858", "714.286", "178.572", "14.286"), + "databricks/databricks-claude-fable-5-2": ("114.286", "571.429", "142.857", "2.857"), + "databricks/system.ai.claude-fable-5-2": ("114.286", "571.429", "142.857", "2.857"), "databricks/databricks-claude-opus-5": ("71.429", "357.143", "89.286", "7.143"), "databricks/databricks-claude-opus-4-8": ("71.429", "357.143", "89.286", "7.143"), "databricks/databricks-claude-opus-4-7": ("71.429", "357.143", "89.286", "7.143"), From 72319cc3501dc513e4d3fa8c991deee882b0f6f1 Mon Sep 17 00:00:00 2001 From: Rick <26716961+Bytechoreographer@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:52:55 +0800 Subject: [PATCH 137/189] feat(azure_ai): support gpt-image edits on Azure AI Foundry (#49650) * feat(azure_ai): support gpt-image edits on Azure AI Foundry Foundry hosts gpt-image deployments on the same /openai/deployments/{deployment}/images/edits route as Azure OpenAI, but the azure_ai image-edit factory only knew MAI and FLUX, so gpt-image-2 and the gpt-image-2.5 models raised a ValueError before any request was sent Reuse AzureImageEditConfig for the request, URL, and multipart handling, and resolve the api_base and credentials with the azure_ai helpers Co-Authored-By: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Co-Authored-By: Claude Code * fix(azure_ai): default gpt-image edits to an edit-capable api-version Foundry gpt-image edits now fall back to 2025-04-01-preview when no api_version is set, since the shared Azure default 2025-02-01-preview answers 404 on /images/edits. The config returns merged headers instead of updating the caller's dict, and the tests check the request each gpt-image model sends * fix(azure_ai): ignore the Azure OpenAI api-version env for Foundry gpt-image edits The proxy CLI always sets AZURE_API_VERSION to 2025-02-01-preview, which Foundry answers with 404 on image edits. gpt-image edits now read only litellm_params, AZURE_AI_API_VERSION, and litellm.api_version before the 2025-04-01-preview default, matching the other azure_ai edit configs * fix(azure_ai): route custom Foundry deployments, require a Foundry api_base, and honor caller auth for gpt-image edits A deployment name the edit factory does not recognize now uses the gpt-image route instead of raising, since Foundry deployment names are free-form. A gpt-image edit with no Foundry api_base raises like the MAI and FLUX configs instead of borrowing AZURE_API_BASE. The shared edit handler passes the call's extra_headers to validate_environment, so a caller's api-key or Authorization header is sent alone * fix(azure_ai): keep the shared edit handler unchanged and send caller auth headers like azure_ai chat --------- Co-authored-by: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Co-authored-by: Claude Code Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/azure_ai/image_edit/__init__.py | 5 +- .../azure_ai/image_edit/gpt_transformation.py | 65 ++++++ .../test_gpt_image_edit_transformation.py | 193 ++++++++++++++++++ 3 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 litellm/llms/azure_ai/image_edit/gpt_transformation.py create mode 100644 tests/unit/llms/azure_ai/image_edit/test_gpt_image_edit_transformation.py diff --git a/litellm/llms/azure_ai/image_edit/__init__.py b/litellm/llms/azure_ai/image_edit/__init__.py index fda9335a5d6..1001edd2b9e 100644 --- a/litellm/llms/azure_ai/image_edit/__init__.py +++ b/litellm/llms/azure_ai/image_edit/__init__.py @@ -9,12 +9,14 @@ from litellm.llms.azure_ai.image_generation.mai_transformation import ( from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from .flux2_transformation import AzureFoundryFlux2ImageEditConfig +from .gpt_transformation import AzureFoundryGPTImageEditConfig from .mai_transformation import AzureFoundryMAIImageEditConfig from .transformation import AzureFoundryFluxImageEditConfig __all__ = [ "AzureFoundryFlux2ImageEditConfig", "AzureFoundryFluxImageEditConfig", + "AzureFoundryGPTImageEditConfig", "AzureFoundryMAIImageEditConfig", ] @@ -26,6 +28,7 @@ def get_azure_ai_image_edit_config(model: str) -> BaseImageEditConfig: - MAI models use /mai/v1/images/edits with multipart form data - FLUX 2 models use JSON with base64 image - FLUX 1 models use multipart/form-data + - Every other deployment, gpt-image or custom-named, uses /openai/deployments/{deployment}/images/edits """ if AzureFoundryMAIImageGenerationConfig.is_mai_model(model): return AzureFoundryMAIImageEditConfig() @@ -39,4 +42,4 @@ def get_azure_ai_image_edit_config(model: str) -> BaseImageEditConfig: if model_normalized == "" or "flux" in model_normalized: return AzureFoundryFluxImageEditConfig() - raise ValueError(f"Model {model} is not supported for Azure AI image editing.") + return AzureFoundryGPTImageEditConfig() diff --git a/litellm/llms/azure_ai/image_edit/gpt_transformation.py b/litellm/llms/azure_ai/image_edit/gpt_transformation.py new file mode 100644 index 00000000000..2fd120e2a56 --- /dev/null +++ b/litellm/llms/azure_ai/image_edit/gpt_transformation.py @@ -0,0 +1,65 @@ +from collections.abc import Mapping +from typing import Final + +import litellm +from litellm.llms.azure.image_edit.transformation import AzureImageEditConfig +from litellm.llms.azure_ai.common_utils import ( + AzureFoundryModelInfo, + get_azure_ai_auth_headers, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams + +AZURE_AI_GPT_IMAGE_EDIT_DEFAULT_API_VERSION: Final = "2025-04-01-preview" + + +class AzureFoundryGPTImageEditConfig(AzureImageEditConfig): + """ + Azure AI Foundry gpt-image edit config. + + Foundry serves gpt-image deployments on the same + ``/openai/deployments/{deployment}/images/edits`` route as Azure OpenAI, so the + request and URL handling is shared with ``AzureImageEditConfig``. Only the + credential source and the default api-version are Foundry-specific. + """ + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + api_key: str | None = None, + litellm_params: Mapping[str, object] | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: llm_http_handler adds extra_headers to the returned dict in place + params: Final = GenericLiteLLMParams(**(litellm_params or {})) + return { + **headers, + **get_azure_ai_auth_headers( + api_key=AzureFoundryModelInfo.get_api_key(params.api_key or api_key), + litellm_params=litellm_params, + api_key_header="api-key", + ), + } + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: Mapping[str, object], + ) -> str: + foundry_api_base: Final = AzureFoundryModelInfo.get_api_base(api_base) + if foundry_api_base is None: + raise ValueError( + "Azure AI API base is required. Set AZURE_AI_API_BASE environment variable or pass api_base parameter." + ) + api_version: Final = ( + litellm_params.get("api_version") + or get_secret_str("AZURE_AI_API_VERSION") + or litellm.api_version + or AZURE_AI_GPT_IMAGE_EDIT_DEFAULT_API_VERSION + ) + return super().get_complete_url( # pyright: ignore[reportUnknownMemberType] # bare dict in base + model=model, + api_base=foundry_api_base, + litellm_params={**litellm_params, "api_version": api_version}, + ) diff --git a/tests/unit/llms/azure_ai/image_edit/test_gpt_image_edit_transformation.py b/tests/unit/llms/azure_ai/image_edit/test_gpt_image_edit_transformation.py new file mode 100644 index 00000000000..6f4a91c7675 --- /dev/null +++ b/tests/unit/llms/azure_ai/image_edit/test_gpt_image_edit_transformation.py @@ -0,0 +1,193 @@ +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.llms.azure_ai.image_edit.gpt_transformation import ( + AzureFoundryGPTImageEditConfig, +) +from litellm.llms.custom_httpx.http_handler import HTTPHandler + +FOUNDRY_API_BASE: Final = "https://example.services.ai.azure.com" +DEPLOYMENT_EDIT_URL: Final = ( + f"{FOUNDRY_API_BASE}/openai/deployments/gpt-image-2.5-flare/images/edits?api-version=2025-04-01-preview" +) + + +def test_gpt_image_edit_url_uses_foundry_deployment_route_and_configured_version(): + url: Final = AzureFoundryGPTImageEditConfig().get_complete_url( + model="gpt-image-2.5-flare", + api_base=FOUNDRY_API_BASE, + litellm_params={"api_version": "2025-06-01-preview"}, + ) + + assert url == f"{FOUNDRY_API_BASE}/openai/deployments/gpt-image-2.5-flare/images/edits?api-version=2025-06-01-preview" + + +def test_gpt_image_edit_url_defaults_to_an_edit_capable_version_on_a_proxy(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("AZURE_AI_API_VERSION", raising=False) + monkeypatch.setenv("AZURE_API_VERSION", litellm.AZURE_DEFAULT_API_VERSION) + monkeypatch.setattr(litellm, "api_version", None) + + url: Final = AzureFoundryGPTImageEditConfig().get_complete_url( + model="gpt-image-2.5-flare", + api_base=FOUNDRY_API_BASE, + litellm_params={}, + ) + + assert url == DEPLOYMENT_EDIT_URL + + +def test_gpt_image_edit_url_uses_azure_ai_api_base_and_version_env(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AZURE_AI_API_BASE", FOUNDRY_API_BASE) + monkeypatch.setenv("AZURE_AI_API_VERSION", "2025-06-01-preview") + monkeypatch.setenv("AZURE_API_VERSION", litellm.AZURE_DEFAULT_API_VERSION) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "api_version", None) + + url: Final = AzureFoundryGPTImageEditConfig().get_complete_url( + model="gpt-image-2.5-flare", + api_base=None, + litellm_params={}, + ) + + assert url == f"{FOUNDRY_API_BASE}/openai/deployments/gpt-image-2.5-flare/images/edits?api-version=2025-06-01-preview" + + +def test_gpt_image_edit_url_uses_v1_surface_for_preview_version(): + url: Final = AzureFoundryGPTImageEditConfig().get_complete_url( + model="gpt-image-2.5-flare", + api_base=FOUNDRY_API_BASE, + litellm_params={"api_version": "preview"}, + ) + + assert url == f"{FOUNDRY_API_BASE}/openai/v1/images/edits?api-version=preview" + + +def test_gpt_image_edit_sends_api_key_header_without_changing_caller_headers(): + caller_headers: Final = {"x-trace": "abc"} + + headers: Final = AzureFoundryGPTImageEditConfig().validate_environment( + caller_headers, + "gpt-image-2.5-flare", + api_key="test-key", + ) + + assert headers == {"x-trace": "abc", "api-key": "test-key"} + assert caller_headers == {"x-trace": "abc"} + + +def test_gpt_image_edit_prefers_litellm_params_api_key(): + headers: Final = AzureFoundryGPTImageEditConfig().validate_environment( + {}, + "gpt-image-2.5-flare", + api_key="positional-key", + litellm_params={"api_key": "params-key"}, + ) + + assert headers["api-key"] == "params-key" + + +def test_gpt_image_edit_reads_azure_ai_api_key_env(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AZURE_AI_API_KEY", "env-key") + monkeypatch.setattr(litellm, "api_key", None) + + headers: Final = AzureFoundryGPTImageEditConfig().validate_environment( + {}, + "gpt-image-2.5-flare", + ) + + assert headers["api-key"] == "env-key" + + +def test_gpt_image_edit_falls_back_to_entra_token(monkeypatch: pytest.MonkeyPatch, no_ambient_azure_credentials: None): + monkeypatch.delenv("AZURE_AI_API_KEY", raising=False) + monkeypatch.setattr(litellm, "api_key", None) + + headers: Final = AzureFoundryGPTImageEditConfig().validate_environment( + {}, + "gpt-image-2.5-flare", + litellm_params={"azure_ad_token": "entra-token"}, + ) + + assert headers == {"Authorization": "Bearer entra-token"} + + +def test_gpt_image_edit_strips_model_from_deployment_multipart(): + data: Final = AzureFoundryGPTImageEditConfig().finalize_image_edit_request_data( + {"model": "gpt-image-2.5-flare", "prompt": "Add a hat"}, + DEPLOYMENT_EDIT_URL, + ) + + assert data == {"prompt": "Add a hat"} + + +def test_gpt_image_edit_keeps_model_on_v1_multipart(): + data: Final = AzureFoundryGPTImageEditConfig().finalize_image_edit_request_data( + {"model": "gpt-image-2.5-flare", "prompt": "Add a hat"}, + f"{FOUNDRY_API_BASE}/openai/v1/images/edits?api-version=preview", + ) + + assert data == {"model": "gpt-image-2.5-flare", "prompt": "Add a hat"} + + +def _edit_through_fake_foundry(model: str, **kwargs: object) -> httpx.Request: + captured: Final[list[httpx.Request]] = [] # mutable-ok: the fake Foundry endpoint records what litellm sent + + def respond(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response(200, json={"created": 1, "data": [{"b64_json": "aW1n"}]}) + + response: Final = litellm.image_edit( + model=f"azure_ai/{model}", + image=b"image", + prompt="Add a hat", + api_base=FOUNDRY_API_BASE, + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + **kwargs, + ) + + assert response.data[0]["b64_json"] == "aW1n" + assert len(captured) == 1 + return captured[0] + + +@pytest.mark.parametrize( + "model", + [ + "gpt-image-1", + "gpt-image-2", + "gpt-image-2.5-flare", + "gpt-image-2.5-flare-2026-09-08", + "gpt-image-2.5-sunburst", + "gpt-image-2.5-sunburst-2026-09-08", + "my-image-editor", + ], +) +def test_gpt_image_edit_end_to_end_posts_multipart_to_deployment_route(monkeypatch: pytest.MonkeyPatch, model: str): + monkeypatch.delenv("AZURE_AI_API_VERSION", raising=False) + monkeypatch.setenv("AZURE_API_VERSION", litellm.AZURE_DEFAULT_API_VERSION) + monkeypatch.setattr(litellm, "api_version", None) + + request: Final = _edit_through_fake_foundry(model, api_key="test-key") + + assert str(request.url) == ( + f"{FOUNDRY_API_BASE}/openai/deployments/{model}/images/edits?api-version=2025-04-01-preview" + ) + assert request.headers["api-key"] == "test-key" + assert b'name="prompt"' in request.content + assert b'name="model"' not in request.content + + +def test_gpt_image_edit_requires_a_foundry_api_base_instead_of_the_azure_openai_one(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("AZURE_AI_API_BASE", raising=False) + monkeypatch.setenv("AZURE_API_BASE", "https://other-resource.openai.azure.com") + monkeypatch.setattr(litellm, "api_base", None) + + with pytest.raises(ValueError, match="Azure AI API base is required"): + AzureFoundryGPTImageEditConfig().get_complete_url( + model="gpt-image-2.5-flare", + api_base=None, + litellm_params={}, + ) From 194c984cff053fc35235d7491a6ad5c59e685564 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:54:10 +0000 Subject: [PATCH 138/189] fix(proxy): keep a cost settings update from being undone by an in-flight config reload (#51273) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cost_tracking_settings.py | 61 ++++++++----------- .../test_cost_tracking_settings.py | 56 ++++++++++++++++- 2 files changed, 79 insertions(+), 38 deletions(-) diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index 7e6aec84292..dcc6a614a54 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -253,6 +253,7 @@ async def update_cost_discount_config( ``` """ from litellm.proxy.proxy_server import ( + DB_RECONCILE_LOCK, prisma_client, proxy_config, store_model_in_db, @@ -301,21 +302,13 @@ async def update_cost_discount_config( ) try: - # Load existing config - config: Final = await proxy_config.get_config() - - # Ensure litellm_settings exists - if "litellm_settings" not in config: - config["litellm_settings"] = {} - - # Update cost_discount_config - config["litellm_settings"]["cost_discount_config"] = cost_discount_config - - # Save the updated config to DB - await proxy_config.save_config(new_config=config) - - # Update in-memory litellm.cost_discount_config - litellm.cost_discount_config = cost_discount_config + async with DB_RECONCILE_LOCK: + config: Final = await proxy_config.get_config() + if "litellm_settings" not in config: + config["litellm_settings"] = {} + config["litellm_settings"]["cost_discount_config"] = cost_discount_config + await proxy_config.save_config(new_config=config) + litellm.cost_discount_config = cost_discount_config verbose_proxy_logger.info("Updated cost_discount_config: %s", cost_discount_config) @@ -397,6 +390,7 @@ async def update_cost_margin_config( ``` """ from litellm.proxy.proxy_server import ( + DB_RECONCILE_LOCK, prisma_client, proxy_config, store_model_in_db, @@ -475,21 +469,13 @@ async def update_cost_margin_config( ) try: - # Load existing config - config: Final = await proxy_config.get_config() - - # Ensure litellm_settings exists - if "litellm_settings" not in config: - config["litellm_settings"] = {} - - # Update cost_margin_config - config["litellm_settings"]["cost_margin_config"] = cost_margin_config - - # Save the updated config to DB - await proxy_config.save_config(new_config=config) - - # Update in-memory litellm.cost_margin_config - litellm.cost_margin_config = cost_margin_config + async with DB_RECONCILE_LOCK: + config: Final = await proxy_config.get_config() + if "litellm_settings" not in config: + config["litellm_settings"] = {} + config["litellm_settings"]["cost_margin_config"] = cost_margin_config + await proxy_config.save_config(new_config=config) + litellm.cost_margin_config = cost_margin_config verbose_proxy_logger.info("Updated cost_margin_config: %s", cost_margin_config) @@ -534,6 +520,7 @@ async def update_block_requests_for_models_without_pricing( request: BlockUnpricedModelsRequest, ) -> BlockUnpricedModelsResponse: from litellm.proxy.proxy_server import ( + DB_RECONCILE_LOCK, prisma_client, proxy_config, store_model_in_db, @@ -552,13 +539,13 @@ async def update_block_requests_for_models_without_pricing( ) try: - config = await proxy_config.get_config() - if "litellm_settings" not in config: - config["litellm_settings"] = {} - config["litellm_settings"]["block_requests_for_models_without_pricing"] = request.enabled - await proxy_config.save_config(new_config=config) - - litellm.block_requests_for_models_without_pricing = request.enabled + async with DB_RECONCILE_LOCK: + config = await proxy_config.get_config() + if "litellm_settings" not in config: + config["litellm_settings"] = {} + config["litellm_settings"]["block_requests_for_models_without_pricing"] = request.enabled + await proxy_config.save_config(new_config=config) + litellm.block_requests_for_models_without_pricing = request.enabled verbose_proxy_logger.info("Updated block_requests_for_models_without_pricing: %s", request.enabled) return BlockUnpricedModelsResponse(enabled=request.enabled) diff --git a/tests/unit/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/unit/proxy/management_endpoints/test_cost_tracking_settings.py index b93e23dcb94..8b06e6545a1 100644 --- a/tests/unit/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/unit/proxy/management_endpoints/test_cost_tracking_settings.py @@ -4,6 +4,7 @@ Tests for cost tracking settings management endpoints. Tests the GET and PATCH endpoints for managing cost discount configuration. """ +import asyncio from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch @@ -15,7 +16,11 @@ from pydantic import ValidationError import litellm from litellm._internal_context import pinned_billing_time from litellm.proxy._types import CostEstimateRequest -from litellm.proxy.management_endpoints.cost_tracking_settings import router +from litellm.proxy.management_endpoints.cost_tracking_settings import ( + router, + update_cost_discount_config, + update_cost_margin_config, +) from litellm.proxy.proxy_server import app client = TestClient(app) @@ -762,6 +767,55 @@ class TestEstimateCostOnPremProvider: +class TestCostSettingsPatchAgainstConfigReload: + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("setting", "update", "stale", "fresh"), + [ + ( + "cost_margin_config", + update_cost_margin_config, + {"openai": {"percentage": 0.1, "fixed_amount": 0.001}}, + {"global": 1}, + ), + ("cost_discount_config", update_cost_discount_config, {"openai": 0.05}, {}), + ], + ) + async def test_patch_is_not_undone_by_a_reload_that_read_the_previous_row(self, setting, update, stale, fresh): + from litellm.proxy import proxy_server + + reload_has_read_the_row = asyncio.Event() + release_reload = asyncio.Event() + + async def reload_holding_the_previous_row() -> None: + async with proxy_server.DB_RECONCILE_LOCK: + reload_has_read_the_row.set() + await release_reload.wait() + proxy_server.ProxyConfig()._apply_litellm_settings_db_values({setting: stale}) + + mock_proxy_config = AsyncMock() + mock_proxy_config.get_config = AsyncMock(return_value={"litellm_settings": {setting: stale}}) + mock_proxy_config.save_config = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch("litellm.proxy.proxy_server.DB_RECONCILE_LOCK", asyncio.Lock()), + patch.object(litellm, setting, stale), + ): + reload = asyncio.create_task(reload_holding_the_previous_row()) + await reload_has_read_the_row.wait() + patched = asyncio.create_task(update(fresh, user_api_key_dict=MagicMock())) + for _ in range(20): + await asyncio.sleep(0) + release_reload.set() + await asyncio.gather(reload, patched) + + assert patched.result()["values"] == fresh + assert getattr(litellm, setting) == fresh + + class TestBlockRequestsForModelsWithoutPricing: """Test suite for the block_requests_for_models_without_pricing toggle endpoints""" From 935a1b8cf686b24616212ccfe18f4552bdad4045 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:55:05 +0000 Subject: [PATCH 139/189] chore(cost-map): source the fireworks deepseek v4 flash deprecation and name its replacement (#51285) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 ++++++ model_prices_and_context_window.json | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 810fd22a64f..1472d10c12e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28593,6 +28593,7 @@ "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-08-14", + "deprecation_source": "https://docs.fireworks.ai/api-reference/list-models.md", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -28600,6 +28601,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", + "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -29065,6 +29067,7 @@ "fireworks_ai/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-08-14", + "deprecation_source": "https://docs.fireworks.ai/api-reference/list-models.md", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -29072,6 +29075,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", + "successor_model": "fireworks_ai/deepseek-v4-flash-0731", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -83589,6 +83593,7 @@ "fireworks_ai/accounts/fireworks/routers/spot-deepseek-v4-flash": { "cache_read_input_token_cost": 2.1e-08, "deprecation_date": "2026-08-14", + "deprecation_source": "https://docs.fireworks.ai/api-reference/list-models.md", "input_cost_per_token": 1.05e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -83596,6 +83601,7 @@ "mode": "chat", "output_cost_per_token": 2.1e-07, "source": "https://api.fireworks.ai/v1/serverless/models", + "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 810fd22a64f..1472d10c12e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28593,6 +28593,7 @@ "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-08-14", + "deprecation_source": "https://docs.fireworks.ai/api-reference/list-models.md", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -28600,6 +28601,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", + "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -29065,6 +29067,7 @@ "fireworks_ai/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, "deprecation_date": "2026-08-14", + "deprecation_source": "https://docs.fireworks.ai/api-reference/list-models.md", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -29072,6 +29075,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", + "successor_model": "fireworks_ai/deepseek-v4-flash-0731", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -83589,6 +83593,7 @@ "fireworks_ai/accounts/fireworks/routers/spot-deepseek-v4-flash": { "cache_read_input_token_cost": 2.1e-08, "deprecation_date": "2026-08-14", + "deprecation_source": "https://docs.fireworks.ai/api-reference/list-models.md", "input_cost_per_token": 1.05e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -83596,6 +83601,7 @@ "mode": "chat", "output_cost_per_token": 2.1e-07, "source": "https://api.fireworks.ai/v1/serverless/models", + "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, From 9c41214b0cad08524b458696b02a46dfb4df7f99 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:57:46 +0000 Subject: [PATCH 140/189] test(guardrails): expect the content_filter finish on a blocked chat stream (#51281) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/observability/test_guardrail_effects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 5d3ab4474ad..4f1ab570d29 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -1637,7 +1637,7 @@ def test_chat_pre_call_denial_streams_content_filter(gateway: Gateway, tmp_path: assert lines[-1] == "data: [DONE]", response.text chunks: Final = tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1]) assert chunks[0]["choices"][0]["delta"]["content"] == _RESPONSES_DENIAL, chunks - assert chunks[-1]["choices"][0]["finish_reason"] == "stop", chunks + assert chunks[-1]["choices"][0]["finish_reason"] == "content_filter", chunks @pytest.mark.covers("other.observability.guardrails.chat_pre_call_denial_returns_content_filter") From e959e3313263c04d692a15bfda727dadefe5a311 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 11 Nov 2026 16:58:08 -0800 Subject: [PATCH 141/189] fix(llms-types): drop the duplicate compact-2026-09-04 arm that fails rust-lint on main (#51280) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/llms-types/src/providers/anthropic/beta.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index d1c839e4363..e3ad783c791 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -317,11 +317,6 @@ impl AnthropicBeta { BetaProvider::VertexAi, BetaProvider::Databricks, ], - Self::Compact20260904 => &[ - BetaProvider::BedrockConverse, - BetaProvider::BedrockMantle, - BetaProvider::Databricks, - ], Self::AfkMode20260131 | Self::CePlugins20260901 | Self::CeUserManagement20260713 From 74d76fd758fbba675f1a3c5370dcff49b42dc20a Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Wed, 11 Nov 2026 16:58:52 -0800 Subject: [PATCH 142/189] perf(router): list a group's deployments once per registry change on the acompletion path (#50749) * perf(router): list a group's deployments once per registry change on the acompletion path * perf(router): drop the new docstrings * perf(router): keep the counter cache's mutable-ok on its annotation * perf(router): define the counter cache's alias in the registry --- litellm/router_rust/core.py | 9 ++++- litellm/router_rust/invoke/endpoints.py | 2 ++ .../router_rust/invoke/prompt_management.py | 6 ++-- litellm/router_rust/surface/model_info.py | 25 ++++++++----- litellm/router_rust/surface/registry.py | 35 ++++++++++++++++++- ...st_registry_characterization_model_info.py | 15 ++++++++ 6 files changed, 79 insertions(+), 13 deletions(-) diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index 69724f0b9fe..3d8627cd2b7 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -1351,6 +1351,9 @@ class _EndpointPort: def group_deployments(self, model_name: str) -> Sequence[Mapping[str, object]] | None: return self._core._hosted._core_get_model_list(model_name=model_name) # pyright: ignore[reportPrivateUsage] # see the class docstring + def group_members(self, model_name: str) -> Sequence[tuple[object, object]] | None: + return self._core._hosted._group_members(model_name) # pyright: ignore[reportPrivateUsage] # see the class docstring + def routed_name(self, model: object) -> str: return self._core.routed_name(model) @@ -1683,7 +1686,11 @@ class RustCore( try: selector: Final = self._prefetch_usage_selector(model, request) - deployments: Final = self._hosted._core_get_model_list(model_name=model) or [] # pyright: ignore[reportPrivateUsage] # see __init__ + deployments: Final = ( + self._hosted._core_get_model_list(model_name=model) or [] # pyright: ignore[reportPrivateUsage] # see __init__ + if isinstance(selector, LowestTPMLoggingHandler_v2) + else [] + ) except Exception: # noqa: BLE001 # the batched read is an optimisation: the reads stay apart yield return diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index c170aaab6e6..5c1205c70ca 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -122,6 +122,8 @@ class EndpointCore(Protocol): """``_core_get_model_list(model_name=...)`` on the host registry.""" ... + def group_members(self, model_name: str) -> Sequence[tuple[object, object]] | None: ... + def routed_name(self, model: object) -> str: """``RustCore.routed_name``: what a ``model`` that is no string raises, or the name it routes under.""" ... diff --git a/litellm/router_rust/invoke/prompt_management.py b/litellm/router_rust/invoke/prompt_management.py index 62bd77e1b48..74a7991b06c 100644 --- a/litellm/router_rust/invoke/prompt_management.py +++ b/litellm/router_rust/invoke/prompt_management.py @@ -21,10 +21,10 @@ _PROMPT_MANAGEMENT_PARAMS: Final = frozenset( def is_prompt_management_model(core: EndpointCore, model: str) -> bool: """`_is_prompt_management_model`: one deployment, whose model's provider prefix is a known custom-logger-compatible callback.""" - model_list: Final = core.group_deployments(model_name=model) - if model_list is None or len(model_list) != 1: + members: Final = core.group_members(model) + if members is None or len(members) != 1: return False - litellm_model: Final = as_dict(model_list[0].get("litellm_params")).get("model") + litellm_model: Final = members[0][1] if not isinstance(litellm_model, str) or "/" not in litellm_model: return False known: Final = core.host.dynamic().known_custom_logger_compatible_callbacks diff --git a/litellm/router_rust/surface/model_info.py b/litellm/router_rust/surface/model_info.py index 508d2bb1145..dd67e252370 100644 --- a/litellm/router_rust/surface/model_info.py +++ b/litellm/router_rust/surface/model_info.py @@ -35,6 +35,7 @@ from typing_extensions import overload import litellm from litellm._internal_context import service_target from litellm._logging import verbose_router_logger +from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE from litellm.litellm_core_utils.add_retry_fallback_headers import ( apply_quality_router_decision_headers, apply_remaining_usage_headers, @@ -74,6 +75,7 @@ from litellm.types.utils import ModelInfo as ModelMapInfo from litellm.utils import get_utc_datetime, is_generalized_model_info if TYPE_CHECKING: + from litellm.router_rust.surface.registry import GroupCounters from litellm.rust_bridge._native import ( RouterCall, RouterCoreHandle, @@ -255,9 +257,12 @@ class HostModelInfo: """Mixed into ``HostRegistry``, which holds the model list, the indexes and the host.""" _core_usage_store: RouterUsage + _group_counters_cache: dict[str, "GroupCounters"] # mutable-ok: by model group def _live_handle(self) -> "RouterCoreHandle | None": ... # the registry's: the handle ``RustCore`` set + def _group_members(self, model_group: str) -> tuple[tuple[object, object], ...] | None: ... # the registry's + def _core_handle(self) -> "RouterCoreHandle": handle: Final = self._live_handle() if handle is None: @@ -798,17 +803,21 @@ class HostModelInfo: """This minute's tpm and rpm counters of every deployment in ``model_group`` (can be a wildcard route), under the model sent to ``litellm.completion()``, as the usage writes name them.""" current_minute: Final = get_utc_datetime().strftime("%H-%M") # use the same timezone regardless of system clock - model_list: Final = self._core_get_model_list(model_name=model_group) - if model_list is None: # no matching deployments + members: Final = self._group_members(model_group) + if members is None: # no matching deployments return None - named: Final = tuple( - (model.get("model_info", {}).get("id"), model["litellm_params"].get("model")) for model in model_list - ) - deployments: Final = tuple(pair for pair in named if pair[0] is not None and pair[1] is not None) - return tuple( - tuple(UsageCounter(id, litellm_model, current_minute, kind) for id, litellm_model in deployments) + cached: Final = self._group_counters_cache.get(model_group) + if cached is not None and cached[0] is members and cached[1] == current_minute: + return cached[2] + deployments: Final = tuple(pair for pair in members if pair[0] is not None and pair[1] is not None) + counters: Final = tuple( + tuple(UsageCounter(id, litellm_model, current_minute, kind) for id, litellm_model in deployments) # pyright: ignore[reportArgumentType] # what the listing names, as the Python core passed it for kind in ("tpm", "rpm") ) + if len(self._group_counters_cache) >= DEFAULT_MAX_LRU_CACHE_SIZE: + self._group_counters_cache.clear() + self._group_counters_cache[model_group] = (members, current_minute, counters) + return counters async def _core_get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: """Current (tpm, rpm) usage of ``model_group``.""" diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py index 59d3d7389be..18bc493b878 100644 --- a/litellm/router_rust/surface/registry.py +++ b/litellm/router_rust/surface/registry.py @@ -96,7 +96,7 @@ from litellm.litellm_core_utils.routing_groups import apply_routing_group_priori from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.redis.stores.connection import connect -from litellm.redis.stores.router_usage import RouterUsage, RouterUsageStore +from litellm.redis.stores.router_usage import RouterUsage, RouterUsageStore, UsageCounter from litellm.responses.bridge_check import keep_registered_mode from litellm.router_api import RouterRegistry, deployment_id from litellm.router_rust.invoke.shaping import ROUTER_CLIENTS_TARGET @@ -697,6 +697,21 @@ def _index_key(value: object) -> str | None: return value if isinstance(value, str) else None +GroupMembers: TypeAlias = tuple[tuple[object, object], ...] +GroupCounters: TypeAlias = tuple[object, str, tuple[tuple[UsageCounter, ...], ...]] + + +def _members_of(model_list: Sequence[Mapping[str, object]] | None) -> GroupMembers | None: + if model_list is None: + return None + return tuple((_field(entry, "model_info", "id"), _field(entry, "litellm_params", "model")) for entry in model_list) + + +def _field(entry: Mapping[str, object], section: str, key: str) -> object: + held: Final = entry.get(section) + return held.get(key) if isinstance(held, Mapping) else None + + def _round_trips(value: object) -> bool: """Whether JSON reads ``value`` back equal to itself.""" try: @@ -787,6 +802,8 @@ class HostRegistry(HostModelInfo): # `zero_cost()`: the per-deployment zero-cost lookup the auth checks fill, emptied with the # group info caches. self._zero_cost_cache: dict[str, bool] = {} # mutable-ok: the auth checks write into it + self._group_members_cache: dict[str, tuple[object, GroupMembers | None]] = {} # mutable-ok: by model group + self._group_counters_cache = {} # The routing cache usage reads come from; RustCore sets it to the one its Rust core writes. self._core_cache: DualCache = DualCache() # The tpm/rpm store usage reads come from; RustCore sets it to the one its Rust core writes. @@ -939,6 +956,7 @@ class HostRegistry(HostModelInfo): self._core_cached_deployment_model_info.cache_clear() # pyright: ignore[reportFunctionMemberAccess] # an lru_cache self.cached_deployment_tracks_usage.cache_clear() # pyright: ignore[reportFunctionMemberAccess] # an lru_cache self._zero_cost_cache.clear() + self._group_members_cache.clear() self._routing_group_rows = None def set_routing_groups(self, groups: Sequence[RoutingGroup]) -> None: @@ -1887,6 +1905,21 @@ class HostRegistry(HostModelInfo): return sum(len(held(member)) for member in group.models) > 1 + def _group_members(self, model_group: str) -> GroupMembers | None: + # An alias or a wildcard name reads state the registry version does not cover: those list every time. + handle: Final = self._live_handle() + if handle is None or _index_key(model_group) is None or model_group in (self._core_model_group_alias or {}): + return _members_of(self._core_get_model_list(model_name=model_group)) + stamp: Final = (handle, handle.registry_version(), self._registry_version) + cached: Final = self._group_members_cache.get(model_group) + if cached is not None and cached[0] == stamp: + return cached[1] + members: Final = _members_of(self._core_get_model_list(model_name=model_group)) + if len(self._group_members_cache) >= DEFAULT_MAX_LRU_CACHE_SIZE: + self._group_members_cache.clear() + self._group_members_cache[model_group] = (stamp, members) + return members + def _core_get_model_list_from_model_alias( self, model_name: str | None = None ) -> list[UntypedEntry]: # mutable-ok: a fresh list diff --git a/tests/unit/test_router/core/test_registry_characterization_model_info.py b/tests/unit/test_router/core/test_registry_characterization_model_info.py index 578183d7f02..b613aed041d 100644 --- a/tests/unit/test_router/core/test_registry_characterization_model_info.py +++ b/tests/unit/test_router/core/test_registry_characterization_model_info.py @@ -8,6 +8,7 @@ import pytest import litellm from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.router_api import ChangeSet from litellm.router_core import RouterCore from litellm.types.router import Deployment, RouterCacheEnum from litellm.utils import get_utc_datetime @@ -481,6 +482,20 @@ async def test_group_usage_sums_each_deployments_counters_for_this_minute(make_c assert await core.health().usage("al") == (34, 3) +@pytest.mark.asyncio +async def test_group_usage_follows_deployments_added_and_removed_after_a_read(make_core: MakeCore) -> None: + core = _usage_core(make_core) + await _serve(core, "a", "openai/gpt-4o", 30) + assert await core.health().usage("gpt") == (30, 1) + + core.apply(ChangeSet(upsert=[typed("gpt", "c", model="openai/gpt-4o-mini", tpm=100, rpm=10)])) + await _serve(core, "c", "openai/gpt-4o-mini", 5) + assert await core.health().usage("gpt") == (35, 2) + + core.apply(ChangeSet(delete=["a"])) + assert await core.health().usage("gpt") == (5, 1) + + @pytest.mark.asyncio async def test_io_token_usage_sums_input_and_output_counters(make_core: MakeCore) -> None: core = _usage_core(make_core) From 208a5863a522a6a1968d58482c1792ee85a832ab Mon Sep 17 00:00:00 2001 From: collieai-dev Date: Thu, 12 Nov 2026 02:05:12 +0100 Subject: [PATCH 143/189] feat(guardrails): add the CollieAi guardrail integration (#49224) * feat(guardrails): add the CollieAi guardrail integration * refactor(guardrails): tighten the CollieAi wrapper and its tests The loader takes its importer as a parameter, so a test covers the fallback for a package that is too old, and the install hint names the Python the package needs The config model is frozen, the Garden preset's phases and default_on are pinned by a test, and comments the code already says are gone * fix(guardrails): use the generated Collieai provider key in the dashboard The dashboard builds provider keys from the proxy's provider list, so collieai becomes Collieai. The static CollieAi key came first in the provider map, which left saved CollieAi guardrails without their display name and logo. The map, the Garden preset and the card now use Collieai, and a test covers the lookup The preset expectation is a named value now, which keeps the dashboard under its inline object lint budget * refactor(guardrails): drop the CollieAi wrapper's function docstrings AGENTS.md keeps comments to complex logic and tool input, and these only restated what each function returns * fix(guardrails): declare CollieAi's phases, read COLLIEAI_API_KEY and plate its logo The dashboard offered every mode for CollieAi, and a guardrail saved with a mode the package refuses was reported as created but never loaded. The wrapper now declares pre_call and post_call, so the form offers only those api_key falls back to COLLIEAI_API_KEY as its description says, and the navy logo gets a plate in dark mode like DeepKeep's The tests now also cover an entry's own name, phase and default_on, the key fallback, and the modes the dashboard offers * fix(guardrails): reject CollieAi guardrails the package refuses and stop env-keyed rebuilds A Garden create whose key or settings the package refuses now answers 400 and saves nothing, instead of saving a guardrail that checks nothing. Every setting on the entry now reaches the package, like the class-path route. DB guardrails keyed with an os.environ/ reference are no longer rebuilt on every DB poll. The Garden card says which calls CollieAi checks. * fix(guardrails): load CollieAi stream_scope and import errors as setup errors, default the Garden card off Pass stream_scope into the package constructor inside the setup-error wrap so a refused scope answers 400 on create and stops a config-file proxy at startup. Drop settings cleared to null so the package default applies. Say an installed package failed to load, with the error, instead of asking to install it, and keep any import-time exception from breaking the guardrail routes. Make api_key optional in the form since COLLIEAI_API_KEY is the fallback, and open the Garden card with Always On off like the other partner presets. * fix(guardrails): leave CollieAi form fields empty and share one guardrail setup error The Garden form no longer pre-fills api_base, deadline_ms or forward_conversation_id, so a create saves only what the admin typed and the package keeps its own defaults. Setup failures raise GuardrailSetupError from custom_guardrail, the missing-package hint covers the Docker image, and the card copy no longer repeats the package's eligibility rules. * test(guardrails): cover env-reference api_base in the DB poll comparison --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 4 + .../proxy/guardrails/guardrail_endpoints.py | 4 +- .../guardrail_hooks/collieai/__init__.py | 55 +++ .../guardrail_hooks/collieai/collieai.py | 94 +++++ .../proxy/guardrails/guardrail_registry.py | 29 +- litellm/types/guardrails.py | 1 + .../guardrails/guardrail_hooks/collieai.py | 47 +++ .../guardrail_hooks/test_collieai.py | 362 ++++++++++++++++++ .../guardrails/test_guardrail_registry.py | 32 ++ .../public/assets/logos/collieai.svg | 6 + .../_components/guardrail_garden_configs.ts | 6 + .../_components/guardrail_garden_data.test.ts | 31 ++ .../_components/guardrail_garden_data.ts | 10 + .../_components/guardrail_info_helpers.tsx | 3 + .../src/lib/logoTreatments.test.ts | 1 + .../src/lib/logoTreatments.ts | 1 + 16 files changed, 678 insertions(+), 8 deletions(-) create mode 100644 litellm/proxy/guardrails/guardrail_hooks/collieai/__init__.py create mode 100644 litellm/proxy/guardrails/guardrail_hooks/collieai/collieai.py create mode 100644 litellm/types/proxy/guardrails/guardrail_hooks/collieai.py create mode 100644 tests/unit/proxy/guardrails/guardrail_hooks/test_collieai.py create mode 100644 ui/litellm-dashboard/public/assets/logos/collieai.svg diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index ae469a104f8..fafbdb1f95a 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -386,6 +386,10 @@ def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuard return metadata.get("guardrails") or [] +class GuardrailSetupError(Exception): + pass + + class CustomGuardrail(CustomLogger): mcp_block_behavior: Literal["tool_result", "raise"] | None = None # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path. diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index eaece1fe4c6..f6591ca00d5 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -18,7 +18,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import CustomGuardrail, GuardrailSetupError from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import AUDIT_ACTIONS, LitellmTableNames, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -540,7 +540,7 @@ async def create_guardrail( verbose_proxy_logger.info( "Immediate sync: Successfully initialized guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) - except (ValueError, TypeError, CustomCodeCompilationError) as init_error: + except (ValueError, TypeError, CustomCodeCompilationError, GuardrailSetupError) as init_error: # Configuration error — roll back the DB write so the guardrail isn't orphaned if prisma_client is not None: try: diff --git a/litellm/proxy/guardrails/guardrail_hooks/collieai/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/collieai/__init__.py new file mode 100644 index 00000000000..c0ea6335741 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/collieai/__init__.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from litellm.integrations.custom_guardrail import GuardrailSetupError +from litellm.types.guardrails import SupportedGuardrailIntegrations +from litellm.types.proxy.guardrails.guardrail_hooks.collieai import DEFAULT_API_BASE + +from .collieai import CollieAiGuardrail + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import Guardrail, LitellmParams + +_NO_EXTRAS: Final[Mapping[str, object]] = MappingProxyType({}) + + +def initialize_guardrail( + litellm_params: LitellmParams, + guardrail: Guardrail, + guardrail_cls: Callable[..., CustomGuardrail] = CollieAiGuardrail, +) -> CustomGuardrail: + import litellm + from litellm.secret_managers.main import get_secret_str + + extras: Final = MappingProxyType( + {key: value for key, value in (litellm_params.model_extra or _NO_EXTRAS).items() if value is not None} + ) + try: + _callback: Final = guardrail_cls( + **{ + **extras, + "api_base": litellm_params.api_base or DEFAULT_API_BASE, + "api_key": litellm_params.api_key or get_secret_str("COLLIEAI_API_KEY"), + "guardrail_name": guardrail["guardrail_name"], + "event_hook": litellm_params.mode, + "default_on": litellm_params.default_on, + "stream_scope": litellm_params.stream_scope, + } + ) + except Exception as setup_error: + raise GuardrailSetupError(str(setup_error)) from setup_error + litellm.logging_callback_manager.add_litellm_callback(_callback) + return _callback + + +guardrail_initializer_registry: Final = { + SupportedGuardrailIntegrations.COLLIEAI.value: initialize_guardrail, +} + +guardrail_class_registry: Final = { + SupportedGuardrailIntegrations.COLLIEAI.value: CollieAiGuardrail, +} diff --git a/litellm/proxy/guardrails/guardrail_hooks/collieai/collieai.py b/litellm/proxy/guardrails/guardrail_hooks/collieai/collieai.py new file mode 100644 index 00000000000..5ea1d147f91 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/collieai/collieai.py @@ -0,0 +1,94 @@ +"""CollieAi as a LiteLLM guardrail, backed by the ``collieai-litellm-guardrail`` PyPI package. + +Install: ``pip install "collieai-litellm-guardrail>=1.0.0"`` +Source: https://github.com/CollieAi/collieai-litellm-guardrail +""" + +from __future__ import annotations + +import importlib +from collections.abc import Callable +from typing import Final + +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.proxy.guardrails.guardrail_hooks.collieai import CollieAiGuardrailConfigModel + +PACKAGE_MODULE: Final = "collieai_guardrail.guardrail" +MISSING_PACKAGE_MESSAGE: Final = ( + "collieai-litellm-guardrail>=1.0.0 (Python 3.11 or newer) is required for the CollieAi guardrail. " + 'Install it with: pip install "collieai-litellm-guardrail>=1.0.0". ' + "The LiteLLM Docker image does not include it, " + "see https://docs.collieai.io/integrations/litellm to add it to an image." +) + + +def has_readiness_guard(package: object) -> bool: + return getattr(package, "require_guardrail", None) is not None + + +def package_guardrail(package: object) -> type[CustomGuardrail] | None: + guardrail: Final[object] = getattr(package, "CollieAIGuardrail", None) + if not has_readiness_guard(package) or not isinstance(guardrail, type): + return None + return guardrail if issubclass(guardrail, CustomGuardrail) else None + + +def unavailable_package(message: str, cause: BaseException | None) -> type[CustomGuardrail]: + class UnavailablePackage(CustomGuardrail): + def __init__(self, **kwargs: object) -> None: # kwargs-ok: mirrors the package constructor, only raises + raise ImportError(message) from cause + + return UnavailablePackage + + +def missing_package(import_error: ImportError | None) -> type[CustomGuardrail]: + return unavailable_package(MISSING_PACKAGE_MESSAGE, import_error) + + +def broken_package(load_error: Exception) -> type[CustomGuardrail]: + return unavailable_package( + "collieai-litellm-guardrail is installed but failed to load with this LiteLLM version " + f"({type(load_error).__name__}: {load_error}). " + 'Upgrade it with: pip install -U "collieai-litellm-guardrail>=1.0.0"', + load_error, + ) + + +def is_the_package_missing(import_error: ImportError) -> bool: + package_root: Final = PACKAGE_MODULE.partition(".")[0] + return isinstance(import_error, ModuleNotFoundError) and (import_error.name or "").partition(".")[0] == package_root + + +def load_base(import_module: Callable[[str], object] = importlib.import_module) -> type[CustomGuardrail]: + try: + package: Final = import_module(PACKAGE_MODULE) + except ImportError as import_error: + return missing_package(import_error) if is_the_package_missing(import_error) else broken_package(import_error) + except Exception as load_error: # noqa: BLE001 # a broken third-party install must not break the guardrail routes + return broken_package(load_error) + return package_guardrail(package) or missing_package(None) + + +_Base: Final = load_base() + + +class CollieAiGuardrail(_Base): + @staticmethod + def get_config_model() -> type[CollieAiGuardrailConfigModel]: + return CollieAiGuardrailConfigModel + + @classmethod + def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: # mutable-ok: CustomGuardrail's signature + return [GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call] + + +__all__ = ( + "MISSING_PACKAGE_MESSAGE", + "PACKAGE_MODULE", + "CollieAiGuardrail", + "has_readiness_guard", + "load_base", + "missing_package", + "package_guardrail", +) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index cafd79626ef..dd4d8f49010 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -685,6 +685,24 @@ def _configure_callback_scoping( _ParamsT = TypeVar("_ParamsT", bound=BaseModel) +SECRET_REFERENCE_PARAMS: Final = ("api_key", "api_base") + + +def resolve_secret_reference(value: str | None) -> str | None: + return str(get_secret(value)) if value and value.startswith("os.environ/") else value + + +def _with_resolved_secret_references(params: Mapping[str, object]) -> Mapping[str, object]: + return { + **params, + **{ + key: resolve_secret_reference(value) + for key in SECRET_REFERENCE_PARAMS + if isinstance(value := params.get(key), str) + }, + } + + def parse_tolerant_litellm_params( litellm_params_data: Mapping[str, object], guardrail_name: str, @@ -788,11 +806,8 @@ class InMemoryGuardrailHandler: ) litellm_params.category_thresholds = lakera_category_thresholds - if litellm_params.api_key and litellm_params.api_key.startswith("os.environ/"): - litellm_params.api_key = str(get_secret(litellm_params.api_key)) - - if litellm_params.api_base and litellm_params.api_base.startswith("os.environ/"): - litellm_params.api_base = str(get_secret(litellm_params.api_base)) + litellm_params.api_key = resolve_secret_reference(litellm_params.api_key) + litellm_params.api_base = resolve_secret_reference(litellm_params.api_base) guardrail_type: Final = litellm_params.guardrail @@ -1068,7 +1083,9 @@ class InMemoryGuardrailHandler: return params.model_dump() if isinstance(params, dict): try: - return parse_tolerant_litellm_params(with_tolerated_stream_scope(params), guardrail_name).model_dump() + return _with_resolved_secret_references( + parse_tolerant_litellm_params(with_tolerated_stream_scope(params), guardrail_name).model_dump() + ) except ValidationError as e: verbose_proxy_logger.warning( "Could not normalize guardrail litellm_params for comparison; treating the guardrail as changed. Error: %s", diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index e311f2ee5f9..8765db30ed5 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -163,6 +163,7 @@ class SupportedGuardrailIntegrations(Enum): AIRIA = "airia" WINGBACK = "wingback" DECISION_MODEL = "decision_model" + COLLIEAI = "collieai" class Role(Enum): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/collieai.py b/litellm/types/proxy/guardrails/guardrail_hooks/collieai.py new file mode 100644 index 00000000000..784bbfc9d0d --- /dev/null +++ b/litellm/types/proxy/guardrails/guardrail_hooks/collieai.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import Final + +from pydantic import ConfigDict, Field + +from litellm.types.llms.base import LiteLLMBaseModel + +from .base import GuardrailConfigModel + +DEFAULT_API_BASE: Final = "https://app.collieai.io/v1/integrations/litellm" + + +class CollieAiGuardrailConfigModelOptionalParams(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + deadline_ms: int | None = Field( + default=None, + gt=0, + description=( + "Each phase's whole deadline in milliseconds: connection, send, reply and parse. " + "Leave empty for the package default." + ), + ) + forward_conversation_id: bool | None = Field( + default=None, + description=( + "Forward the caller's x-collieai-conversation-id header, and only that header. Leave empty to keep it off." + ), + ) + + +class CollieAiGuardrailConfigModel(GuardrailConfigModel[CollieAiGuardrailConfigModelOptionalParams]): + model_config = ConfigDict(frozen=True) + + api_key: str | None = Field( + default=None, + description="CollieAi project API key, starting with clai_. Leave empty to use COLLIEAI_API_KEY.", + ) + api_base: str | None = Field( + default=None, + description="CollieAi's LiteLLM integration base URL. Leave empty for CollieAi's hosted service.", + ) + + @staticmethod + def ui_friendly_name() -> str: + return "CollieAi" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_collieai.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_collieai.py new file mode 100644 index 00000000000..40629355a57 --- /dev/null +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_collieai.py @@ -0,0 +1,362 @@ +from __future__ import annotations + +import importlib.util +from types import ModuleType, SimpleNamespace +from typing import Final + +import httpx +import pytest +import respx +from fastapi import HTTPException + +import litellm +from litellm.caching.dual_cache import DualCache +from litellm.exceptions import GuardrailRaisedException +from litellm.integrations.custom_guardrail import CustomGuardrail, GuardrailSetupError +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.guardrails.guardrail_endpoints import ( + CreateGuardrailRequest, + create_guardrail, + get_guardrail_ui_settings, + get_provider_specific_params, +) +from litellm.proxy.guardrails.guardrail_hooks.collieai import CollieAiGuardrail, initialize_guardrail +from litellm.proxy.guardrails.guardrail_hooks.collieai.collieai import ( + MISSING_PACKAGE_MESSAGE, + PACKAGE_MODULE, + has_readiness_guard, + load_base, + package_guardrail, +) +from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler +from litellm.proxy.utils import ProxyLogging +from litellm.types.guardrails import Guardrail, LitellmParams +from litellm.types.proxy.guardrails.guardrail_hooks.collieai import ( + DEFAULT_API_BASE, + CollieAiGuardrailConfigModel, + CollieAiGuardrailConfigModelOptionalParams, +) + +PACKAGE_INSTALLED: Final = importlib.util.find_spec("collieai_guardrail") is not None +API_KEY: Final = "clai_" + "0" * 40 +CHAT_REQUEST: Final = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "hello"}]} + + +class _RecordingGuardrail(CustomGuardrail): + def __init__( + self, + *, + api_base: str | None = None, + api_key: str | None = None, + guardrail_name: str | None = None, + event_hook: object = None, + default_on: bool | None = None, + stream_scope: object = None, + **settings: object, + ) -> None: + super().__init__(guardrail_name=guardrail_name, default_on=bool(default_on), stream_scope=stream_scope) + self.given = { + "api_base": api_base, + "api_key": api_key, + "guardrail_name": guardrail_name, + "event_hook": event_hook, + "default_on": default_on, + "stream_scope": stream_scope, + **settings, + } + + +class _RefusesStreamScope(_RecordingGuardrail): + def apply_stream_scope(self, stream_scope: object) -> None: + if stream_scope not in (None, "both"): + raise RuntimeError("enforcement_mode block needs stream_scope unset or both") + super().apply_stream_scope(stream_scope) + + +def _params(**extras: object) -> LitellmParams: + return LitellmParams( + guardrail="collieai", mode=["pre_call", "post_call"], default_on=True, api_key=API_KEY, **extras + ) + + +def _guardrail(litellm_params: LitellmParams) -> Guardrail: + return Guardrail(guardrail_name="collieai", litellm_params=litellm_params) + + +def _init(litellm_params: LitellmParams) -> _RecordingGuardrail: + callback: Final = initialize_guardrail( + litellm_params, _guardrail(litellm_params), guardrail_cls=_RecordingGuardrail + ) + assert isinstance(callback, _RecordingGuardrail) + return callback + + +@pytest.fixture(autouse=True) +def _isolate_callbacks(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + + +def test_hands_the_entry_and_every_setting_to_the_package() -> None: + callback: Final = _init( + _params( + api_base="https://collieai.internal.example/v1/integrations/litellm", + enforcement_mode="block", + deadline_ms=2000, + network_margin_ms=300, + max_body_bytes=1024, + forward_conversation_id=True, + summary_interval_s=30, + synthetic_interval_s=0, + workspace_id="not-a-collieai-setting", + stream_scope="non_streaming", + ) + ) + + assert callback.given == { + "api_base": "https://collieai.internal.example/v1/integrations/litellm", + "api_key": API_KEY, + "guardrail_name": "collieai", + "event_hook": ["pre_call", "post_call"], + "default_on": True, + "stream_scope": "non_streaming", + "enforcement_mode": "block", + "deadline_ms": 2000, + "network_margin_ms": 300, + "max_body_bytes": 1024, + "forward_conversation_id": True, + "summary_interval_s": 30, + "synthetic_interval_s": 0, + "workspace_id": "not-a-collieai-setting", + } + assert litellm.callbacks == [callback] + + +def test_points_at_app_collieai_io_unless_told_otherwise() -> None: + callback: Final = _init(_params()) + + # The hosted service's base URL, from https://docs.collieai.io/integrations/litellm (2026-10-29) + assert callback.given["api_base"] == DEFAULT_API_BASE == "https://app.collieai.io/v1/integrations/litellm" + assert set(callback.given) == {"api_base", "api_key", "guardrail_name", "event_hook", "default_on", "stream_scope"} + + +def test_a_cleared_optional_setting_leaves_the_package_its_default() -> None: + callback: Final = _init(_params(deadline_ms=None, forward_conversation_id=None, network_margin_ms=300)) + + assert "deadline_ms" not in callback.given + assert "forward_conversation_id" not in callback.given + assert callback.given["network_margin_ms"] == 300 + + +def test_a_stream_scope_the_package_refuses_is_a_setup_error_and_adds_no_callback() -> None: + litellm_params: Final = _params(enforcement_mode="block", stream_scope="streaming") + + with pytest.raises(GuardrailSetupError, match="stream_scope"): + initialize_guardrail(litellm_params, _guardrail(litellm_params), guardrail_cls=_RefusesStreamScope) + + assert litellm.callbacks == [] + + +def test_hands_each_entry_its_own_name_phase_and_default_on() -> None: + litellm_params: Final = LitellmParams(guardrail="collieai", mode="pre_call", default_on=False, api_key=API_KEY) + callback: Final = initialize_guardrail( + litellm_params, + Guardrail(guardrail_name="collieai-prompts", litellm_params=litellm_params), + guardrail_cls=_RecordingGuardrail, + ) + + assert isinstance(callback, _RecordingGuardrail) + assert (callback.given["guardrail_name"], callback.given["event_hook"], callback.given["default_on"]) == ( + "collieai-prompts", + "pre_call", + False, + ) + + +def test_takes_the_key_from_collieai_api_key_only_when_the_entry_has_none(monkeypatch: pytest.MonkeyPatch) -> None: + from_environment: Final = "clai_" + "1" * 40 + monkeypatch.setenv("COLLIEAI_API_KEY", from_environment) + without_key: Final = LitellmParams(guardrail="collieai", mode=["pre_call", "post_call"], default_on=True) + + assert _init(without_key).given["api_key"] == from_environment + assert _init(_params()).given["api_key"] == API_KEY + + +def test_an_untouched_ui_form_leaves_every_setting_to_the_package() -> None: + model: Final = CollieAiGuardrailConfigModel(api_key=API_KEY) + optional: Final = CollieAiGuardrailConfigModelOptionalParams() + callback: Final = _init( + _params(**model.model_dump(exclude={"api_key", "optional_params"}), **optional.model_dump()) + ) + + assert callback.given["api_base"] == DEFAULT_API_BASE + assert set(callback.given) == {"api_base", "api_key", "guardrail_name", "event_hook", "default_on", "stream_scope"} + + +@pytest.mark.asyncio +async def test_ui_offers_collieai_fields_without_the_package() -> None: + assert CollieAiGuardrail.get_config_model() is CollieAiGuardrailConfigModel + + fields: Final = (await get_provider_specific_params())["collieai"] + + assert fields["ui_friendly_name"] == "CollieAi" + assert fields["api_key"]["required"] is False + assert set(fields["optional_params"]["fields"]) == {"deadline_ms", "forward_conversation_id"} + assert [name for name in ("api_key", "api_base") if "default_value" in fields[name]] == [] + assert [name for name, field in fields["optional_params"]["fields"].items() if "default_value" in field] == [] + + +@pytest.mark.asyncio +async def test_ui_offers_only_the_two_phases_collieai_checks() -> None: + settings: Final = await get_guardrail_ui_settings() + + assert settings.supported_modes_by_provider["collieai"] == ["pre_call", "post_call"] + + +def test_a_package_without_the_readiness_guard_is_too_old() -> None: + assert not has_readiness_guard(ModuleType("collieai_guardrail.guardrail")) + assert has_readiness_guard(SimpleNamespace(require_guardrail=object())) + + +def test_only_a_current_package_s_guardrail_class_is_taken() -> None: + guard: Final = object() + + assert package_guardrail(SimpleNamespace(require_guardrail=guard, CollieAIGuardrail=_RecordingGuardrail)) is ( + _RecordingGuardrail + ) + assert package_guardrail(SimpleNamespace(CollieAIGuardrail=_RecordingGuardrail)) is None + assert package_guardrail(SimpleNamespace(require_guardrail=guard, CollieAIGuardrail=object())) is None + assert package_guardrail(SimpleNamespace(require_guardrail=guard, CollieAIGuardrail=ValueError)) is None + + +def test_builds_on_the_package_only_when_it_is_current() -> None: + current: Final = SimpleNamespace(require_guardrail=object(), CollieAIGuardrail=_RecordingGuardrail) + too_old: Final = SimpleNamespace(CollieAIGuardrail=_RecordingGuardrail) + + assert load_base({PACKAGE_MODULE: current}.__getitem__) is _RecordingGuardrail + with pytest.raises(ImportError, match="pip install") as raised: + load_base({PACKAGE_MODULE: too_old}.__getitem__)(guardrail_name="collieai") + + assert str(raised.value) == MISSING_PACKAGE_MESSAGE + assert raised.value.__cause__ is None + + +def test_without_the_package_building_the_guardrail_names_what_to_install() -> None: + cause: Final = ModuleNotFoundError("No module named 'collieai_guardrail'", name="collieai_guardrail") + + def import_module(name: str) -> object: + raise cause + + with pytest.raises(ImportError, match="pip install") as raised: + load_base(import_module)(guardrail_name="collieai") + + assert str(raised.value) == MISSING_PACKAGE_MESSAGE + assert raised.value.__cause__ is cause + + +@pytest.mark.parametrize( + "cause", + [ + ModuleNotFoundError("No module named 'litellm.gone'", name="litellm.gone"), + ImportError("cannot import name 'Moved' from 'litellm.types'", name="litellm.types"), + ImportError("cannot import name 'Observer' from 'collieai_guardrail.observer'", name="collieai_guardrail.observer"), + AttributeError("module 'litellm' has no attribute 'moved'"), + ], +) +def test_an_installed_package_that_fails_to_load_says_so_instead_of_asking_to_install_it(cause: Exception) -> None: + def import_module(name: str) -> object: + raise cause + + with pytest.raises(ImportError, match="installed but failed to load") as raised: + load_base(import_module)(guardrail_name="collieai") + + assert str(cause) in str(raised.value) + assert MISSING_PACKAGE_MESSAGE not in str(raised.value) + assert raised.value.__cause__ is cause + + +@pytest.mark.skipif(PACKAGE_INSTALLED, reason="exercises the missing-package fallback") +def test_missing_package_fails_at_config_load_with_install_hint() -> None: + with pytest.raises(GuardrailSetupError, match="pip install") as raised: + InMemoryGuardrailHandler().initialize_guardrail(_guardrail(_params())) + + assert not isinstance(raised.value, (ValueError, TypeError)) + assert litellm.callbacks == [] + + +@pytest.mark.asyncio +async def test_creating_a_guardrail_the_package_refuses_answers_400_and_removes_the_row(mocker) -> None: + from unittest.mock import AsyncMock + + litellm_params: Final = { + "guardrail": "collieai", + "mode": ["pre_call", "post_call"], + "default_on": True, + "api_key": "sk-not-a-collieai-key", + } + row: Final = {"guardrail_name": "collieai-refused", "litellm_params": litellm_params, "guardrail_info": {}} + registry: Final = mocker.Mock(add_guardrail_to_db=AsyncMock(return_value={**row, "guardrail_id": "refused"})) + mocker.patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", registry) + mocker.patch("litellm.proxy.proxy_server.prisma_client", mocker.Mock()) + mocker.patch("litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", InMemoryGuardrailHandler()) + delete_row: Final = mocker.patch("litellm.proxy.guardrails.guardrail_endpoints._delete_guardrail_row", AsyncMock()) + + with pytest.raises(HTTPException) as refused: + await create_guardrail( + CreateGuardrailRequest(guardrail=row), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert refused.value.status_code == 400 + delete_row.assert_awaited_once() + assert litellm.callbacks == [] + + +def _load(monkeypatch: pytest.MonkeyPatch, **extras: object) -> CustomGuardrail: + """The guardrail as the proxy loads `guardrail: collieai` from its config, with no synthetic check at start. Block + mode also needs the package's readiness guard among the proxy's callbacks, which the config file loads first""" + from collieai_guardrail.guardrail import require_guardrail + + block: Final = extras.get("enforcement_mode") == "block" + monkeypatch.setattr(litellm, "callbacks", [require_guardrail] if block else []) + InMemoryGuardrailHandler().initialize_guardrail(_guardrail(_params(synthetic_interval_s=0, **extras))) + loaded: Final = [callback for callback in litellm.callbacks if isinstance(callback, CollieAiGuardrail)] + assert len(loaded) == 1 + return loaded[0] + + +async def _pre_call() -> dict[str, object]: + data: Final = {**CHAT_REQUEST, "metadata": {}} + await ProxyLogging(user_api_key_cache=DualCache()).pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"), data=data, call_type="acompletion" + ) + return data + + +@pytest.mark.skipif(not PACKAGE_INSTALLED, reason="needs collieai-litellm-guardrail") +@pytest.mark.asyncio +@respx.mock +async def test_block_refuses_the_call_when_collieai_is_unreachable(monkeypatch: pytest.MonkeyPatch) -> None: + route: Final = respx.post(url__startswith=DEFAULT_API_BASE).mock(side_effect=httpx.ConnectError("refused")) + _load(monkeypatch, enforcement_mode="block") + + with pytest.raises((GuardrailRaisedException, HTTPException)) as refused: + await _pre_call() + + assert route.called + assert getattr(refused.value, "status_code", None) == 503 + + +@pytest.mark.skipif(not PACKAGE_INSTALLED, reason="needs collieai-litellm-guardrail") +@pytest.mark.asyncio +@respx.mock +async def test_monitor_lets_the_call_go_on_and_records_why(monkeypatch: pytest.MonkeyPatch) -> None: + route: Final = respx.post(url__startswith=DEFAULT_API_BASE).mock(side_effect=httpx.ConnectError("refused")) + _load(monkeypatch) + + data: Final = await _pre_call() + + assert route.called + metadata: Final = data["metadata"] + assert isinstance(metadata, dict) + [record] = metadata["standard_logging_guardrail_information"] + assert (record["guardrail_provider"], record["guardrail_status"]) == ("collieai", "guardrail_failed_to_respond") diff --git a/tests/unit/proxy/guardrails/test_guardrail_registry.py b/tests/unit/proxy/guardrails/test_guardrail_registry.py index 7eabc24e090..bf328264bab 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_registry.py +++ b/tests/unit/proxy/guardrails/test_guardrail_registry.py @@ -649,6 +649,38 @@ def test_unchanged_db_params_do_not_register_as_changed(): assert handler._has_guardrail_params_changed(gid, new) is False +@pytest.mark.parametrize("field", ("api_key", "api_base")) +def test_db_poll_does_not_rebuild_a_guardrail_whose_secret_field_is_an_env_reference(monkeypatch, field): + from litellm.proxy.guardrails import guardrail_registry as registry_module + + built_with: Final[list[str | None]] = [] + + def _initializer(litellm_params, guardrail): + built_with.append(getattr(litellm_params, field)) + return CustomGuardrail( + guardrail_name=guardrail["guardrail_name"], event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + + monkeypatch.setitem(registry_module.guardrail_initializer_registry, "env_ref_test", _initializer) + monkeypatch.setenv("ENV_REF_TEST_VALUE", "resolved-value") + row: Final = { + "guardrail_id": "env-ref", + "guardrail_name": "env-ref", + "litellm_params": { + "guardrail": "env_ref_test", + "mode": "pre_call", + field: "os.environ/ENV_REF_TEST_VALUE", + }, + } + handler: Final = InMemoryGuardrailHandler() + handler.initialize_guardrail(guardrail=dict(row), source="db") + + handler.sync_guardrail_from_db(dict(row)) + handler.sync_guardrail_from_db(dict(row)) + + assert built_with == ["resolved-value"] + + def test_db_poll_does_not_reinitialize_config_guardrail_without_default_on(): handler = InMemoryGuardrailHandler() guardrail_id: Final = "config-default-on-guardrail" diff --git a/ui/litellm-dashboard/public/assets/logos/collieai.svg b/ui/litellm-dashboard/public/assets/logos/collieai.svg new file mode 100644 index 00000000000..dff072d0c30 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/collieai.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_configs.ts b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_configs.ts index e532b5bfd74..7a9060c5d02 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_configs.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_configs.ts @@ -393,4 +393,10 @@ export const GUARDRAIL_PRESETS: Record = { mode: ["pre_call", "post_call"], defaultOn: false, }, + collieai: { + provider: "Collieai", + guardrailNameSuggestion: "CollieAi", + mode: ["pre_call", "post_call"], + defaultOn: false, + }, }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts index 20725e73e37..9839db91376 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts @@ -6,6 +6,13 @@ import { LITELLM_CONTENT_FILTER_CARDS, PARTNER_GUARDRAIL_CARDS, } from "./guardrail_garden_data"; +import { + getGuardrailLogoAndName, + getGuardrailProviders, + guardrailLogoMap, + populateGuardrailProviderMap, + populateGuardrailProviders, +} from "./guardrail_info_helpers"; const EXPECTED_PARTNER_LOGO_FILES: Record = { presidio: "microsoft_azure.svg", @@ -39,6 +46,7 @@ const EXPECTED_PARTNER_LOGO_FILES: Record = { conduct: "conduct.png", wingback: "wingback.png", airia: "airia.svg", + collieai: "collieai.svg", }; describe("guardrail_garden_data logos", () => { @@ -100,3 +108,26 @@ describe("decision model presets", () => { } }); }); + +describe("collieai preset", () => { + it("opens the CollieAi card on both halves of the call, off until a request or Always On asks for it", () => { + const expectedPreset = { + provider: "Collieai", + guardrailNameSuggestion: "CollieAi", + mode: ["pre_call", "post_call"], + defaultOn: false, + }; + expect(GUARDRAIL_PRESETS["collieai"]).toEqual(expectedPreset); + }); + + it("uses the provider key the proxy's provider list generates, so saved guardrails keep their name and logo", () => { + const providerParams = { collieai: { ui_friendly_name: "CollieAi" } }; + populateGuardrailProviders(providerParams); + populateGuardrailProviderMap(providerParams); + const provider = GUARDRAIL_PRESETS["collieai"].provider; + + expect(getGuardrailLogoAndName("collieai")).toEqual({ logo: guardrailLogoMap.CollieAi, displayName: "CollieAi" }); + expect(Object.keys(getGuardrailProviders())).toContain(provider); + expect(PARTNER_GUARDRAIL_CARDS.find((card) => card.id === "collieai")?.providerKey).toBe(provider); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts index dc6acffa13c..9cb9521bb9d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts @@ -717,6 +717,16 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ tags: ["Security", "Prompt Injection", "PII", "Secrets", "Redaction"], providerKey: "Airia", }, + { + id: "collieai", + name: "CollieAi", + description: + "CollieAi checks chat completion text against your CollieAi policy before and after the model call and records each finding in LiteLLM's guardrail log. It observes by default. Block mode refuses the calls the policy blocks and every call it cannot check.", + category: "partner", + logo: guardrailLogoMap["CollieAi"], + tags: ["Security", "Prompt Injection", "PII", "Policy", "Monitoring"], + providerKey: "Collieai", + }, ]; export const ALL_CARDS = [...LITELLM_CONTENT_FILTER_CARDS, ...DECISION_MODEL_CARDS, ...PARTNER_GUARDRAIL_CARDS]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx index d0253875735..bb773bbbb7f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx @@ -9,6 +9,7 @@ import conductLogo from "../../../../../public/assets/logos/conduct.png"; import aporiaLogo from "../../../../../public/assets/logos/aporia.png"; import bedrockLogo from "../../../../../public/assets/logos/bedrock.svg"; import catoNetworksLogo from "../../../../../public/assets/logos/cato_networks.svg"; +import collieaiLogo from "../../../../../public/assets/logos/collieai.svg"; import ciscoLogo from "../../../../../public/assets/logos/cisco.png"; import deepkeepLogo from "../../../../../public/assets/logos/deepkeep.svg"; import enkryptAiLogo from "../../../../../public/assets/logos/enkrypt_ai.avif"; @@ -94,6 +95,7 @@ export const guardrail_provider_map: Record = { "LLM Shield Proxy": "llm_shield_proxy", Conduct: "conduct", DecisionModel: "decision_model", + Collieai: "collieai", }; // Search aliases matched alongside the provider label in the add-guardrail provider combobox @@ -332,6 +334,7 @@ export const guardrailLogoMap = { "Conduct Guard": conductLogo.src, Wingback: wingbackLogo.src, "Airia Guardrail": airiaLogo.src, + CollieAi: collieaiLogo.src, "Decision Model": litellmLogo.src, } satisfies Record; diff --git a/ui/litellm-dashboard/src/lib/logoTreatments.test.ts b/ui/litellm-dashboard/src/lib/logoTreatments.test.ts index 828ea72a57f..6ae74738b4d 100644 --- a/ui/litellm-dashboard/src/lib/logoTreatments.test.ts +++ b/ui/litellm-dashboard/src/lib/logoTreatments.test.ts @@ -14,6 +14,7 @@ describe("logoTreatmentFor", () => { it("marks a multicolor dark mark for a plate instead of inversion", () => { expect(logoTreatmentFor("/ui/assets/logos/fireworks.svg")).toBe("plate"); + expect(logoTreatmentFor("/ui/assets/logos/collieai.svg")).toBe("plate"); }); it("plates a dark mark with a light knockout, which inversion would flatten away", () => { diff --git a/ui/litellm-dashboard/src/lib/logoTreatments.ts b/ui/litellm-dashboard/src/lib/logoTreatments.ts index 4cd6f65a613..f8b4ecc05b1 100644 --- a/ui/litellm-dashboard/src/lib/logoTreatments.ts +++ b/ui/litellm-dashboard/src/lib/logoTreatments.ts @@ -36,6 +36,7 @@ const TREATMENT_BY_ASSET: Readonly> = { "aiml_api.svg": "plate", "akto.svg": "plate", "aws.svg": "plate", + "collieai.svg": "plate", "deepkeep.svg": "plate", "fireworks.svg": "plate", "llm_guard.png": "plate", From 6116adeb7415fc44b0b82aaa3cdd67b7225c733d Mon Sep 17 00:00:00 2001 From: nuernber Date: Wed, 11 Nov 2026 17:06:15 -0800 Subject: [PATCH 144/189] fix(anthropic_messages): bound the detached pump so it cannot outlive its stream (#51268) * fix(anthropic): time-limit /v1/messages pump waits on a stalled client The upstream pump waited on client_detached with no timeout, both when the relay queue was full and when waiting for the relay to consume a queued upstream error. client_detached is only set by the relay generator's finally block, so a client that resets its connection while stalled could leave the pump parked forever, holding its request, logging object and collected chunks. Both waits now give up after ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS (default 120), treat the client as gone, and fall through to the existing detached billing path. * fix(anthropic): bound the detached /v1/messages drain and end a relay whose pump gave up A provider that stalled after the client left kept the detached pump, the open provider stream and every chunk it had buffered for billing alive until the provider read timed out, if it ever did. Pumps that never read another chunk after the disconnect are not counted against the detached-drain cap, so nothing limited them A detached drain still reading ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS (default 600) after the client left now bills what it read and closes the provider stream. The deadline starts when the relay closes or when the client stall timeout gives up on the client, so it also covers a read that was already stalled Once the stall timeout gives up, the pump stops feeding the relay. A client that resumed reading then waited on an empty queue forever. The relay now ends with the incomplete-stream error once the pump is gone. The stall timeout is read with get_env_float so fractional values parse Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): end a stream cleanly when the pump gives up on its last put When the upstream had already finished and only the end-of-stream sentinel was left, a client that resumed after the stall timeout got every chunk through message_stop and then an extra incomplete-stream error event. The relay now only adds that error when the client has not already received a message_stop or error event The new test pins the case where the stall timeout fires on the final sentinel put, so the pump exits with the request billed and a resumed client still sees a clean, complete stream Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): give a stalled /v1/messages client 600 s before the pump gives up on it The 120 s default could cut off a client that was only slow. 600 s matches the usual request_timeout, so a stalled client and a stalled provider get the same budget. SSE keepalive pings do not change this, because they block on the same stalled write. A stuck pump now lives at most about 20 minutes, which keeps only a handful alive at once Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic_messages): keep queued chunks when dropping an unconsumed upstream error _exception_left_unconsumed emptied the whole relay queue to check whether the provider error was still in it, so a client that resumed after the error wait expired lost every chunk buffered ahead of the error. Re-queue everything but the error, add a regression test for the resumed client, and type the new test helpers. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 6 + .../messages/native/streaming_iterator.py | 130 ++++++-- tests/documentation_tests/test_env_keys.py | 2 + .../native/test_streaming_iterator.py | 302 ++++++++++++++++++ 4 files changed, 413 insertions(+), 27 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 5d0eab30b90..966a5b71b8c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -736,6 +736,12 @@ ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE: Final = get_env_int( # upstream stream immediately, instead of continuing to drain for the real # terminal usage. ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS: Final = get_env_int("ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 100) +ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS: Final = get_env_float( + "ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS", 600.0 +) +ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS: Final = get_env_float( + "ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS", 600.0 +) LOGGING_WORKER_CONCURRENCY: Final = get_env_int("LOGGING_WORKER_CONCURRENCY", 100) # Must be above 0 LOGGING_WORKER_MAX_QUEUE_SIZE: Final = get_env_int("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000) LOGGING_WORKER_MAX_TIME_PER_COROUTINE: Final = get_env_float("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0) diff --git a/litellm/llms/anthropic/messages/native/streaming_iterator.py b/litellm/llms/anthropic/messages/native/streaming_iterator.py index 5a7633698ec..b166b822b4f 100644 --- a/litellm/llms/anthropic/messages/native/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/streaming_iterator.py @@ -1,17 +1,22 @@ import asyncio +import contextlib import json +import math import re from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence from contextvars import ContextVar from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable +import anyio import httpx from pydantic import TypeAdapter, ValidationError from typing_extensions import TypedDict from litellm.constants import ( + ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS, ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS, + ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS, ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE, ) from litellm.litellm_core_utils.core_helpers import process_response_headers @@ -43,6 +48,41 @@ _UPSTREAM_PUMP_TASKS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: stdl _DETACHED_STREAM_DRAINS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: bounded strong-ref set, detached drains +class _PumpExited: + pass + + +_PUMP_EXITED: Final = _PumpExited() + + +def _start_detached_drain_deadline(drain_deadline: anyio.CancelScope) -> None: + if drain_deadline.deadline == math.inf: + drain_deadline.deadline = ( # rebind-ok: anyio cancel scopes are armed by setting their deadline + anyio.current_time() + ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS + ) + + +async def _next_relay_item( + queue: "asyncio.Queue[bytes | None | BaseException]", pump_task: "asyncio.Task[None]" +) -> bytes | None | BaseException | _PumpExited: + """The next queued item, or ``_PUMP_EXITED`` once the pump is gone and will never send another.""" + with contextlib.suppress(asyncio.QueueEmpty): + return queue.get_nowait() + if pump_task.done(): + return _PUMP_EXITED + getter: Final = asyncio.ensure_future(queue.get()) + try: + await asyncio.wait(frozenset((getter, pump_task)), return_when=asyncio.FIRST_COMPLETED) + finally: + if not getter.done(): + getter.cancel() + if getter.done() and not getter.cancelled(): + return getter.result() + with contextlib.suppress(asyncio.QueueEmpty): + return queue.get_nowait() + return _PUMP_EXITED + + def is_message_stop_chunk(chunk: object) -> bool: if isinstance(chunk, dict): return chunk.get("type") == "message_stop" @@ -204,7 +244,11 @@ def _try_claim_detached_drain_slot() -> bool: def _exception_left_unconsumed(queue: "asyncio.Queue[bytes | None | BaseException]", exc: BaseException) -> bool: + """Drop ``exc`` from ``queue`` if the client never consumed it, keeping the chunks queued ahead of it.""" remaining: Final = tuple(queue.get_nowait() for _ in range(queue.qsize())) + for item in remaining: + if item is not exc: + queue.put_nowait(item) return any(item is exc for item in remaining) @@ -606,8 +650,12 @@ class BaseAnthropicMessagesStreamingIterator: keeps a single ``collected_chunks`` copy for billing, and the number of such post-disconnect drains running at once is capped so client behavior can't create unbounded worker state; over the cap the pump bills what it - has rather than draining further. Detached-drain lifetime is otherwise - bounded by the upstream stream/read timeout. + has rather than draining further. A drain still reading + ``ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS`` after the + client left, a stalled provider for one, bills what it has and closes the + provider stream too, so it cannot hold its buffered response forever. A + relay whose pump exited without an end-of-stream sentinel ends the stream + with an incomplete-stream error instead of waiting on an empty queue. An upstream failure (Bedrock read / decode / chunk-conversion error) that happens while the client is still connected is forwarded through @@ -622,23 +670,33 @@ class BaseAnthropicMessagesStreamingIterator: maxsize=ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE ) client_detached: Final = asyncio.Event() + drain_deadline: Final = anyio.CancelScope() - pump_task: Final = asyncio.create_task(self._pump_upstream_to_queue(completion_stream, queue, client_detached)) + pump_task: Final = asyncio.create_task( + self._pump_upstream_to_queue(completion_stream, queue, client_detached, drain_deadline) + ) _UPSTREAM_PUMP_TASKS.add(pump_task) pump_task.add_done_callback(_UPSTREAM_PUMP_TASKS.discard) reached_end = False # rebind-ok: flipped once the relay consumes the end-of-stream sentinel + relayed_terminal_event = False # rebind-ok: flipped once a message_stop or error event reaches the client try: while True: - item = await queue.get() + item = await _next_relay_item(queue, pump_task) + if isinstance(item, _PumpExited): + if not relayed_terminal_event: + yield _incomplete_stream_error_sse_event() + break if item is None: reached_end = True break if isinstance(item, BaseException): raise item + relayed_terminal_event = relayed_terminal_event or _is_terminal_stream_chunk(item) yield item finally: client_detached.set() + _start_detached_drain_deadline(drain_deadline) if not reached_end: self._dispatch_pending_deferred_logging() @@ -720,13 +778,18 @@ class BaseAnthropicMessagesStreamingIterator: put_task: Final = asyncio.ensure_future(queue.put(item)) detached_task: Final = asyncio.ensure_future(client_detached.wait()) try: - await asyncio.wait(frozenset((put_task, detached_task)), return_when=asyncio.FIRST_COMPLETED) + await asyncio.wait( + frozenset((put_task, detached_task)), + timeout=ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS, + return_when=asyncio.FIRST_COMPLETED, + ) finally: if not detached_task.done(): detached_task.cancel() if put_task.done() and not put_task.cancelled(): return True put_task.cancel() + client_detached.set() return False async def _pump_upstream_to_queue( @@ -734,6 +797,7 @@ class BaseAnthropicMessagesStreamingIterator: completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | Mapping[str, object]], queue: "asyncio.Queue[bytes | None | BaseException]", client_detached: "asyncio.Event", + drain_deadline: anyio.CancelScope, ) -> None: """Drain the whole upstream into ``queue`` (backpressured) and bill once. @@ -750,32 +814,41 @@ class BaseAnthropicMessagesStreamingIterator: collected_chunks: Final[list[bytes]] = [] # mutable-ok: SSE billing buffer appended to across the drain saw_terminal_event = False # rebind-ok: accumulates across the upstream loop draining_detached = False # rebind-ok: set once this pump claims a detached-drain slot + over_drain_cap = False # rebind-ok: set when no detached-drain slot is free try: - async for chunk in completion_stream: - if self.completion_start_time is None: - self.completion_start_time = datetime.now() - saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk) - encoded_chunk = self._convert_chunk_to_sse_format(chunk) - collected_chunks.append(encoded_chunk) - if not client_detached.is_set(): - await self._enqueue_for_client(queue, client_detached, encoded_chunk) - continue - if not draining_detached: - if not _try_claim_detached_drain_slot(): - verbose_proxy_logger.warning( - "async_sse_wrapper: detached-drain cap (%d) reached; billing %d partial " - "chunks and aborting the upstream stream to stop provider billing", - ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS, - len(collected_chunks), - ) - await self._bill_collected_chunks(collected_chunks, stream_teardown=True) - await self._abort_upstream(completion_stream) - return - draining_detached = True + with drain_deadline: + async for chunk in completion_stream: + if self.completion_start_time is None: + self.completion_start_time = datetime.now() + saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk) + encoded_chunk = self._convert_chunk_to_sse_format(chunk) + collected_chunks.append(encoded_chunk) + if not client_detached.is_set(): + if not await self._enqueue_for_client(queue, client_detached, encoded_chunk): + _start_detached_drain_deadline(drain_deadline) + continue + if not draining_detached: + if not _try_claim_detached_drain_slot(): + over_drain_cap = True + break + draining_detached = True except Exception as exc: # noqa: BLE001 # upstream errors are handled/forwarded by _handle_pump_upstream_error await self._handle_pump_upstream_error(queue, client_detached, collected_chunks, exc) return + if over_drain_cap or drain_deadline.cancelled_caught: + verbose_proxy_logger.warning( + "async_sse_wrapper: %s; billing %d partial chunks and aborting the upstream stream to stop " + "provider billing", + f"detached-drain cap ({ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS}) reached" + if over_drain_cap + else f"detached drain still running {ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS}s " + "after the client left", + len(collected_chunks), + ) + await self._bill_collected_chunks(collected_chunks, stream_teardown=True) + await self._abort_upstream(completion_stream) + return if client_detached.is_set(): await self._bill_collected_chunks(collected_chunks, stream_teardown=True) return @@ -814,7 +887,10 @@ class BaseAnthropicMessagesStreamingIterator: exception=described, ) if not client_detached.is_set() and await self._enqueue_for_client(queue, client_detached, described): - await client_detached.wait() + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for( + client_detached.wait(), timeout=ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS + ) if not _exception_left_unconsumed(queue, described): return await self._fire_detached_failure_hook(described) diff --git a/tests/documentation_tests/test_env_keys.py b/tests/documentation_tests/test_env_keys.py index 2a7da58f39c..72386877704 100644 --- a/tests/documentation_tests/test_env_keys.py +++ b/tests/documentation_tests/test_env_keys.py @@ -38,6 +38,8 @@ EXCLUDED_ROLLOUT_FLAGS = { # These are advanced settings with sensible defaults that most users should not modify EXCLUDED_INTERNAL_TUNING_VARS = { "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", + "ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS", + "ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS", "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", "DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY", } diff --git a/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py index 39f7a4a05cb..ded599512d0 100644 --- a/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py @@ -1,5 +1,7 @@ import asyncio +import gc import json +import tracemalloc from collections.abc import AsyncIterator, Mapping, Sequence from datetime import datetime from typing import Final, Literal @@ -1921,3 +1923,303 @@ async def test_safeguarded_sse_stream_reports_upstream_buffering_and_closes_the_ assert SafeguardedAnthropicSSEStream( _chunked(b"", 1), _safeguards_evaluator(_OneVerdictClassifier("{}")) ).has_buffered_provider_output is False + + +def _finite_stream(chunks: int) -> AsyncIterator[dict[str, object]]: + async def _stream() -> AsyncIterator[dict[str, object]]: + yield {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}} + for i in range(chunks): + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"t{i}"}} + yield {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 42}} + yield {"type": "message_stop"} + + return _stream() + + +async def _wait_until_no_live_pumps() -> None: + for _ in range(300): + if not any(not task.done() for task in streaming_iterator_module._UPSTREAM_PUMP_TASKS): + return + await asyncio.sleep(0.01) + raise AssertionError("pump task is still alive") + + +@pytest.mark.asyncio +async def test_pump_exits_when_relay_generator_is_never_closed_and_queue_is_full(monkeypatch: pytest.MonkeyPatch): + """ + Regression: a client that resets its connection while the relay queue is full + can leave the relay generator unfinalized, so client_detached is never set. + The pump must give up on the stalled client after the stall timeout instead of + waiting forever, then bill what it drained. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 2) + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS", 0.05) + + iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("pump_stall_full_queue"), request_body={}) + gen = iterator.async_sse_wrapper(_finite_stream(50)) + await gen.__anext__() + + await _wait_until_no_live_pumps() + + assert iterator.logging_call_count == 1 + assert any(c.startswith(b"event: message_stop\n") for c in iterator.logged_chunks) + + +@pytest.mark.asyncio +async def test_pump_keeps_streaming_to_slow_client_that_stays_within_stall_timeout(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 2) + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS", 5) + + iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("pump_slow_client"), request_body={}) + received = [] + async for chunk in iterator.async_sse_wrapper(_finite_stream(20)): + received.append(chunk) + await asyncio.sleep(0.01) + + assert any(c.startswith(b"event: message_stop\n") for c in received) + assert len(received) == 23 + + +@pytest.mark.asyncio +async def test_pump_exits_when_queued_upstream_error_is_never_consumed_and_relay_is_never_closed( + monkeypatch: pytest.MonkeyPatch, +): + """ + Regression: the pump waits for the relay to consume a queued upstream error. + If the relay generator is never finalized, that wait must time out and fire + the detached failure hook instead of hanging forever. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS", 0.05) + upstream_errored = asyncio.Event() + + async def _failing_stream() -> AsyncIterator[dict[str, object]]: + yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}} + upstream_errored.set() + raise _ProviderStreamError("mid-stream failure", status_code=500) + + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("pump_stall_queued_error", _FailureRecorder()), request_body={} + ) + detached_hook = _DetachedFailureRecorder() + iterator.litellm_logging_obj.on_detached_stream_failure = detached_hook + + gen = iterator.async_sse_wrapper(_failing_stream()) + await gen.__anext__() + await upstream_errored.wait() + + await _wait_until_no_live_pumps() + + assert len(detached_hook.exceptions) == 1 + + +@pytest.mark.asyncio +async def test_slow_client_that_resumes_after_an_unconsumed_upstream_error_still_gets_its_buffered_chunks( + monkeypatch: pytest.MonkeyPatch, +): + """ + Regression: when the pump gave up waiting for a stalled client to consume a queued provider + error, it emptied the whole relay queue to check whether the error was still in it, so a client + that resumed lost every chunk buffered ahead of the error. Only the error may be dropped. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS", 0) + deltas: Final = tuple( + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"t{i}"}} for i in range(3) + ) + + async def _failing_stream() -> AsyncIterator[dict[str, object]]: + yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}} + for delta in deltas: + yield delta + raise _ProviderStreamError("mid-stream failure", status_code=500) + + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("resumed_client_queued_error", _FailureRecorder()), request_body={} + ) + detached_hook = _DetachedFailureRecorder() + iterator.litellm_logging_obj.on_detached_stream_failure = detached_hook + + gen = iterator.async_sse_wrapper(_failing_stream()) + first_frame: Final = await gen.__anext__() + await _wait_until_no_live_pumps() + + async def _read_rest() -> list[bytes]: + return [frame async for frame in gen] + + rest: Final = await asyncio.wait_for(_read_rest(), timeout=5) + + assert first_frame.startswith(b"event: message_start\n") + assert rest == [ + *(iterator._convert_chunk_to_sse_format(delta) for delta in deltas), + _incomplete_stream_error_sse_event(), + ] + assert len(detached_hook.exceptions) == 1 + + +class _BilledChunkCountIterator(BaseAnthropicMessagesStreamingIterator): + def __init__(self, litellm_logging_obj: LiteLLMLoggingObj, billed_chunk_counts: list[int]): + super().__init__(litellm_logging_obj=litellm_logging_obj, request_body={}) + self._billed_chunk_counts: Final = billed_chunk_counts + + async def _handle_streaming_logging(self, collected_chunks: list[bytes], *, stream_teardown: bool = False) -> None: + self._billed_chunk_counts.append(len(collected_chunks)) + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_releases_stalled_upstream_after_client_disconnect(monkeypatch: pytest.MonkeyPatch): + """ + Regression: when the provider stalls after the client disconnects, the detached pump kept the + provider stream open and every chunk it had buffered for billing alive until the provider read + timed out, so each such request leaked its whole response. The drain must stop at its time + limit, bill what it read, close the provider stream and free the buffer. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS", 0) + requests: Final = 5 + chunks_before_stall: Final = 50 + payload: Final = "x" * 65536 + response_bytes: Final = chunks_before_stall * len(payload) + provider_stalled: Final = asyncio.Event() + upstream_closed: Final = tuple(asyncio.Event() for _ in range(requests)) + billed_chunk_counts: Final[list[int]] = [] + + async def _stalling_stream(closed: asyncio.Event) -> AsyncIterator[dict[str, object]]: + try: + yield {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}} + for _ in range(chunks_before_stall): + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": payload}} + await provider_stalled.wait() + finally: + closed.set() + + gc.collect() + tracemalloc.start() + try: + baseline: Final = tracemalloc.get_traced_memory()[0] + for index, closed in enumerate(upstream_closed): + iterator = _BilledChunkCountIterator(_make_logging_obj(f"stalled_drain_{index}"), billed_chunk_counts) + gen = iterator.async_sse_wrapper(_stalling_stream(closed)) + await gen.__anext__() + await gen.aclose() + try: + await asyncio.wait_for(asyncio.gather(*(closed.wait() for closed in upstream_closed)), timeout=5) + except asyncio.TimeoutError: + pass + gc.collect() + retained: Final = tracemalloc.get_traced_memory()[0] - baseline + closed_while_stalled: Final = sum(closed.is_set() for closed in upstream_closed) + billed_while_stalled: Final = tuple(billed_chunk_counts) + finally: + tracemalloc.stop() + provider_stalled.set() + await asyncio.gather(*tuple(streaming_iterator_module._UPSTREAM_PUMP_TASKS), return_exceptions=True) + + assert closed_while_stalled == requests, f"only {closed_while_stalled}/{requests} stalled provider streams closed" + assert retained < response_bytes, f"{retained} bytes still held after {requests} disconnected stalled streams" + assert billed_while_stalled == (chunks_before_stall + 1,) * requests + + +@pytest.mark.asyncio +async def test_slow_client_that_resumes_after_the_stall_timeout_gets_an_error_instead_of_hanging( + monkeypatch: pytest.MonkeyPatch, +): + """ + Regression: once the pump gave up on a stalled client it stopped feeding the relay without an + end-of-stream sentinel, so a client that started reading again drained the queue and then + waited forever. It must get the incomplete-stream error and an ended stream. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 1) + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS", 0) + iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("resumed_slow_client"), request_body={}) + gen = iterator.async_sse_wrapper(_finite_stream(20)) + first_frame: Final = await gen.__anext__() + await _wait_until_no_live_pumps() + + async def _read_rest() -> list[bytes]: + return [frame async for frame in gen] + + rest: Final = await asyncio.wait_for(_read_rest(), timeout=5) + + assert first_frame.startswith(b"event: message_start\n") + assert rest[-1] == _incomplete_stream_error_sse_event() + assert any(c.startswith(b"event: message_stop\n") for c in iterator.logged_chunks) + + + +@pytest.mark.asyncio +async def test_pump_closes_a_stalled_upstream_after_giving_up_on_a_client_that_never_closes( + monkeypatch: pytest.MonkeyPatch, +): + """ + Regression: a client that stops reading without closing its relay is detached by the stall + timeout, and the pump then drains on its own. If the provider stalls too, nothing else ever + ends that drain, so the drain time limit has to start when the stall timeout gives up. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 1) + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS", 0) + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_DETACHED_STREAM_DRAIN_TIMEOUT_SECONDS", 0) + provider_stalled: Final = asyncio.Event() + upstream_closed: Final = asyncio.Event() + + async def _stalling_stream() -> AsyncIterator[dict[str, object]]: + try: + yield {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}} + for i in range(5): + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"t{i}"}} + await provider_stalled.wait() + finally: + upstream_closed.set() + + iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("client_never_closes"), request_body={}) + gen = iterator.async_sse_wrapper(_stalling_stream()) + await gen.__anext__() + try: + await asyncio.wait_for(upstream_closed.wait(), timeout=5) + except asyncio.TimeoutError: + pass + closed_while_stalled: Final = upstream_closed.is_set() + provider_stalled.set() + await _wait_until_no_live_pumps() + + assert closed_while_stalled, "the stalled provider stream stayed open after the client was given up on" + assert iterator.logging_call_count == 1 + + +@pytest.mark.asyncio +async def test_pump_gives_up_on_the_end_sentinel_and_a_resumed_client_still_gets_a_clean_complete_stream( + monkeypatch: pytest.MonkeyPatch, +): + """ + Regression: when the upstream had already finished and only the end-of-stream sentinel was + left, a stalled client with a full relay queue kept the pump waiting on that last put. The + pump must give up at the stall timeout with the request already billed, and a client that + resumes must still get every chunk, ending at message_stop with no error event after it. + """ + content_chunks: Final = 3 + remaining_events: Final = content_chunks + 2 + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", remaining_events) + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_CLIENT_STALL_TIMEOUT_SECONDS", 0) + client_stopped_reading: Final = asyncio.Event() + + async def _stream_that_finishes_after_the_client_stalls() -> AsyncIterator[dict[str, object]]: + yield {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}} + await client_stopped_reading.wait() + for i in range(content_chunks): + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"t{i}"}} + yield {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 42}} + yield {"type": "message_stop"} + + iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("blocked_end_sentinel"), request_body={}) + gen = iterator.async_sse_wrapper(_stream_that_finishes_after_the_client_stalls()) + first_frame: Final = await gen.__anext__() + client_stopped_reading.set() + await _wait_until_no_live_pumps() + billed_before_resume: Final = iterator.logging_call_count + + async def _read_rest() -> list[bytes]: + return [frame async for frame in gen] + + frames: Final = [first_frame, *await asyncio.wait_for(_read_rest(), timeout=5)] + + assert billed_before_resume == 1 + assert len(frames) == remaining_events + 1 + assert frames[-1].startswith(b"event: message_stop\n") + assert not any(frame.startswith(b"event: error\n") for frame in frames) From a5d04ec9ac5688c67efad617d99bd752853ca709 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 11 Nov 2026 17:10:01 -0800 Subject: [PATCH 145/189] fix(auth): allow video retrieval after budget exhaustion (#51227) * fix(auth): allow paid video retrieval after budget exhaustion * test(auth): check video retrieval boundary on registered route * fix(auth): exempt video retrieval in post-custom-auth checks * style(auth): format custom video budget exemption --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/proxy/auth/auth_checks.py | 17 ++- litellm/proxy/auth/user_api_key_auth.py | 13 +- litellm/proxy/common_request_processing.py | 7 +- ...st_auth_checks_object_access_and_lookup.py | 67 +++++++++ .../auth/test_custom_auth_end_user_budget.py | 93 ++++++++++++ .../unit/proxy/auth/test_user_api_key_auth.py | 109 ++++++++++++++ .../test_user_api_key_auth_request_flow.py | 142 +++++++++++++++++- .../proxy/test_common_request_processing.py | 43 ++++++ 8 files changed, 485 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index fa331b37950..c7b698b7f18 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1081,6 +1081,15 @@ BUDGET_ENFORCED_SIDE_EFFECT_ROUTES: Final = frozenset( } ) +VIDEO_RETRIEVAL_ROUTES: Final = frozenset( + { + "/videos/{video_id}", + "/v1/videos/{video_id}", + "/videos/{video_id}/content", + "/v1/videos/{video_id}/content", + } +) + MCP_DISCOVERY_ROUTES: Final = frozenset({"/mcp-rest/tools/list", "/v1/mcp/tools"}) MCP_ZERO_SPEND_JSONRPC_METHODS: Final = frozenset({"initialize", "notifications/initialized", "ping", "tools/list"}) @@ -1097,6 +1106,10 @@ def is_mcp_discovery_request(route: str, request_body: Mapping[str, object]) -> return isinstance(method, str) and method in MCP_ZERO_SPEND_JSONRPC_METHODS +def is_video_retrieval_request(route: str, http_method: str | None) -> bool: + return http_method == "GET" and RouteChecks.check_route_access(route=route, allowed_routes=VIDEO_RETRIEVAL_ROUTES) + + class ModelMaxBudgetHolder(Protocol): @property def model_max_budget(self) -> Mapping[str, object] | None: ... @@ -1126,8 +1139,9 @@ def request_skips_budget_checks( llm_router: Router | None, *, request_body: Mapping[str, object] | None = None, + http_method: str | None = None, ) -> bool: - if route_skips_budget_checks(route=route): + if route_skips_budget_checks(route=route) or is_video_retrieval_request(route=route, http_method=http_method): return True if request_body is not None and is_mcp_discovery_request(route=route, request_body=request_body): return True @@ -1189,6 +1203,7 @@ async def common_checks( model=_model, llm_router=llm_router, request_body=_typed_request_body(request_body), + http_method=RouteChecks.get_request_method(request), ) ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 1b2e4e791e3..ff6e7f84bed 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -85,6 +85,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo is_model_cost_zero, is_user_proxy_admin, is_valid_fallback_model, + is_video_retrieval_request, jwt_key_mapping_cache_key, key_access_group_grants_model, key_model_aliases_for_auth_check, @@ -2680,7 +2681,11 @@ async def user_api_key_auth_builder( team_id=valid_token.team_id, ) skip_budget_checks = request_skips_budget_checks( - route=route, model=model, llm_router=llm_router, request_body=request_data + route=route, + model=model, + llm_router=llm_router, + request_body=request_data, + http_method=RouteChecks.get_request_method(request), ) # Fetch project object for JWT path if project_id is set @@ -3198,6 +3203,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e model=await budget_served_model(model, request_data, request, llm_router, valid_token), llm_router=llm_router, request_body=request_data, + http_method=RouteChecks.get_request_method(request), ) # Check 3. Check if user is in their team budget @@ -4299,6 +4305,7 @@ async def _should_skip_budget_checks( model=(model if sub_call else await budget_served_model(model, request_data, request, llm_router, valid_token)), llm_router=llm_router, request_body=request_data, + http_method=RouteChecks.get_request_method(request), ) @@ -4976,7 +4983,9 @@ async def _run_post_custom_auth_checks( # spend some other model accrued. The JWT and virtual-key paths already skip # every budget check for these; this path did not, so the same request could # be refused under custom auth and served under the other two. - skip_budget_checks: Final = ( + skip_budget_checks: Final = is_video_retrieval_request( + route=route, http_method=RouteChecks.get_request_method(request) + ) or ( is_model_cost_zero( model=await budget_served_model(current_model, request_data, request, llm_router, valid_token), llm_router=llm_router, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index e585cd7b1ff..3efdbfd30e2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -111,6 +111,7 @@ from litellm.proxy.auth.auth_checks import ( tag_max_budget_check_for_tags, ) from litellm.proxy.auth.auth_utils import check_response_size_is_safe, get_request_route +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.callback_utils import ( get_logging_caching_headers, @@ -1048,11 +1049,14 @@ async def _enforce_guardrail_added_tag_budgets( llm_router: Router | None, user_api_key_dict: UserAPIKeyAuth, proxy_logging_obj: ProxyLogging, + http_method: str | None = None, ) -> None: added_tags: Final = tuple( tag for tag in get_tags_from_request_body(request_body=data) if tag not in tags_before_guardrails ) - if not added_tags or request_skips_budget_checks(route=route, model=_request_model(data), llm_router=llm_router): + if not added_tags or request_skips_budget_checks( + route=route, model=_request_model(data), llm_router=llm_router, http_method=http_method + ): return from litellm.proxy.proxy_server import prisma_client, user_api_key_cache @@ -2864,6 +2868,7 @@ class ProxyBaseLLMRequestProcessing: llm_router=llm_router, user_api_key_dict=user_api_key_dict, proxy_logging_obj=proxy_logging_obj, + http_method=RouteChecks.get_request_method(request), ) if route_type == "aget_responses": attach_post_call_pipelines_to_retrieval( diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index e7bcc696c0d..4cbe5320aa1 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -7286,6 +7286,45 @@ async def test_model_discovery_route_bypasses_user_budget(): assert result is True +@pytest.mark.parametrize( + "method, route, expect_blocked", + [ + ("GET", "/v1/videos/video_abc", False), + ("GET", "/videos/video_abc/content", False), + ("POST", "/v1/videos", True), + ("POST", "/v1/videos/video_abc/remix", True), + ("GET", "/v1/videos", True), + ], +) +@pytest.mark.asyncio +async def test_exhausted_team_budget_spares_only_video_retrieval(method: str, route: str, expect_blocked: bool) -> None: + from litellm.proxy.auth.auth_checks import common_checks + + request: Final = MagicMock() + request.method = method + + async def run() -> bool: + return await common_checks( + request_body={}, + team_object=LiteLLM_TeamTable(team_id="test-team", spend=0.4, max_budget=0.3), + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route=route, + llm_router=None, + proxy_logging_obj=AsyncMock(), + valid_token=UserAPIKeyAuth(token="test-token", team_id="test-team"), + request=request, + ) + + if expect_blocked: + with pytest.raises(litellm.BudgetExceededError): + await run() + else: + assert await run() is True + + @pytest.mark.parametrize("route", ["/health/services", "/auto_router/test_routing"]) @pytest.mark.asyncio async def test_side_effectful_info_route_still_enforces_budget(route: str) -> None: @@ -11202,6 +11241,34 @@ def test_request_skips_budget_checks_for_non_spending_requests( ) +@pytest.mark.parametrize( + "route, http_method, expected", + [ + ("/v1/videos/video_abc", "GET", True), + ("/videos/video_abc", "GET", True), + ("/v1/videos/video_abc/content", "GET", True), + ("/videos/video_abc/content", "GET", True), + ("/v1/videos/video_abc", "POST", False), + ("/v1/videos/video_abc/content", None, False), + ("/v1/videos", "GET", False), + ("/v1/videos", "POST", False), + ("/v1/videos/video_abc/remix", "POST", False), + ("/v1/videos/edits", "POST", False), + ("/v1/videos/video_abc/remix", "GET", False), + ("/v1/chat/completions", "GET", False), + ], +) +def test_request_skips_budget_checks_only_for_paid_video_retrieval( + route: str, http_method: str | None, expected: bool, free_and_paid_router: litellm.Router +) -> None: + assert ( + request_skips_budget_checks( + route=route, model="paid-model", llm_router=free_and_paid_router, http_method=http_method + ) + is expected + ) + + def _agent_model_ceiling_resolver( models: frozenset[str] | None, ) -> tuple[CeilingResolver, list[str]]: diff --git a/tests/unit/proxy/auth/test_custom_auth_end_user_budget.py b/tests/unit/proxy/auth/test_custom_auth_end_user_budget.py index e75a77325c6..2a5ee88dbf9 100644 --- a/tests/unit/proxy/auth/test_custom_auth_end_user_budget.py +++ b/tests/unit/proxy/auth/test_custom_auth_end_user_budget.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -6,15 +7,24 @@ from starlette.requests import Request from starlette.routing import Route import litellm +from litellm.caching.dual_cache import DualCache from litellm.proxy._types import ( LiteLLM_BudgetTable, LiteLLM_EndUserTable, + Litellm_EntityType, + ProxyErrorTypes, + ProxyException, UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import ( _run_post_custom_auth_checks, update_valid_token_with_end_user_params, ) +from litellm.proxy.hooks.model_max_budget_limiter import ( + PROXY_VirtualKeyModelMaxBudgetLimiter, + model_budget_spend_cache_key, +) +from litellm.types.videos.utils import encode_video_id_with_provider @pytest.mark.asyncio @@ -513,3 +523,86 @@ async def test_custom_auth_judges_the_zero_cost_waiver_by_the_key_alias_target( result: Final = await run() assert result.token == "test_token" limiter.is_end_user_within_model_budget.assert_not_awaited() + + +_VIDEO_ID: Final = encode_video_id_with_provider(video_id="video_raw", provider="openai", model_id="sora-2") + + +async def _post_custom_auth_video_request( + valid_token: UserAPIKeyAuth, method: str, template: str, request_data: dict[str, object] +) -> UserAPIKeyAuth: + limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + await limiter.dual_cache.async_set_cache( + key=model_budget_spend_cache_key( + entity_type=Litellm_EntityType.KEY, entity_id=valid_token.token, budget_model="sora-2", budget_duration="1d" + ), + value=0.4, + ) + path: Final = template.format(video_id=_VIDEO_ID) + http_request: Final = Request( + { + "type": "http", + "method": method, + "path": path, + "headers": [], + "query_string": b"", + "route": Route(template, endpoint=lambda request: None), + } + ) + with ( + patch("litellm.proxy.proxy_server.llm_router", None), + patch("litellm.proxy.proxy_server.model_max_budget_limiter", limiter), + patch("litellm.proxy.proxy_server.general_settings", {}), + ): + return await _run_post_custom_auth_checks( + valid_token=valid_token, + request=http_request, + request_data=request_data, + route=path, + parent_otel_span=None, + ) + + +def _exhausted_model_budget_token() -> UserAPIKeyAuth: + return UserAPIKeyAuth(token="test_token", model_max_budget={"sora-2": {"max_budget": 0.3, "budget_duration": "1d"}}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "template", + ["/v1/videos/{video_id}", "/videos/{video_id}", "/v1/videos/{video_id}/content", "/videos/{video_id}/content"], +) +async def test_custom_auth_exhausted_model_budget_still_retrieves_an_already_paid_video(template: str) -> None: + result: Final = await _post_custom_auth_video_request( + _exhausted_model_budget_token(), "GET", template, {"video_id": _VIDEO_ID} + ) + + assert result.token == "test_token" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method, template, request_data", + [ + ("POST", "/v1/videos", {"model": "sora-2"}), + ("POST", "/v1/videos/{video_id}/remix", {"video_id": _VIDEO_ID}), + ("GET", "/v1/videos", {"model": "sora-2"}), + ], +) +async def test_custom_auth_exhausted_model_budget_still_blocks_video_requests_other_than_retrieval( + method: str, template: str, request_data: dict[str, object] +) -> None: + with pytest.raises(litellm.BudgetExceededError): + await _post_custom_auth_video_request(_exhausted_model_budget_token(), method, template, request_data) + + +@pytest.mark.asyncio +async def test_custom_auth_video_retrieval_still_rejects_an_expired_token() -> None: + expired_token: Final = _exhausted_model_budget_token().model_copy( + update={"expires": datetime(2000, 1, 1, tzinfo=timezone.utc)} + ) + + with pytest.raises(ProxyException) as exc_info: + await _post_custom_auth_video_request(expired_token, "GET", "/v1/videos/{video_id}", {"video_id": _VIDEO_ID}) + + assert exc_info.value.type == ProxyErrorTypes.expired_key diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index 7746fc96673..d39c8cb8c85 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -2360,6 +2360,115 @@ async def test_jwt_path_skips_user_model_budget_only_for_zero_spend_mcp_methods( assert user_model_budget_check.await_count == (1 if expect_user_model_budget_check else 0) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method, path, expect_rejected", + [ + ("GET", "/v1/videos/{video_id}", False), + ("GET", "/videos/{video_id}/content", False), + ("POST", "/v1/videos", True), + ("POST", "/v1/videos/{video_id}/remix", True), + ], +) +async def test_jwt_path_spares_only_video_retrieval_from_an_exhausted_user_model_budget( + monkeypatch: pytest.MonkeyPatch, method: str, path: str, expect_rejected: bool +) -> None: + from litellm.caching.dual_cache import DualCache + from litellm.proxy._types import JWTAuthBuilderResult, LiteLLM_JWTAuth, Litellm_EntityType, ProxyErrorTypes + from litellm.proxy.auth import user_api_key_auth as user_api_key_auth_module + from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_builder + from litellm.proxy.hooks.model_max_budget_limiter import ( + PROXY_VirtualKeyModelMaxBudgetLimiter, + model_budget_spend_cache_key, + ) + from litellm.types.videos.utils import encode_video_id_with_provider + + video_id: Final = encode_video_id_with_provider(video_id="video_raw", provider="openai", model_id="sora-2") + jwt_result: Final = JWTAuthBuilderResult( + is_proxy_admin=False, + team_object=None, + user_object=LiteLLM_UserTable( + user_id="jwt-user", + model_max_budget={"sora-2": {"budget_limit": 0.3, "time_period": "1d"}}, + ), + end_user_object=None, + org_object=None, + token="fake.jwt.token", + team_id=None, + user_id="jwt-user", + user_email=None, + end_user_id=None, + org_id=None, + team_membership=None, + jwt_claims={}, + agent_id=None, + ) + + class _VerifiedJWTAuthManager(JWTAuthManager): + @staticmethod + async def auth_builder( + api_key: object, + jwt_handler: object, + request_data: object, + general_settings: object, + route: object, + prisma_client: object, + user_api_key_cache: object, + parent_otel_span: object, + proxy_logging_obj: object, + request_headers: object = None, + request_method: object = None, + ) -> JWTAuthBuilderResult: + return jwt_result + + limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + await limiter.dual_cache.async_set_cache( + key=model_budget_spend_cache_key( + entity_type=Litellm_EntityType.USER, entity_id="jwt-user", budget_model="sora-2", budget_duration="1d" + ), + value=0.4, + ) + jwt_handler: Final = JWTHandler() + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + mock_request: Final = MagicMock() + mock_request.url.path = path.format(video_id=video_id) + mock_request.method = method + mock_request.headers = {"authorization": "Bearer fake.jwt.token"} + mock_request.query_params = {} + request_data: Final[dict[str, object]] = {"model": "sora-2"} if path == "/v1/videos" else {"video_id": video_id} + + monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {"enable_jwt_auth": True}) + monkeypatch.setattr(litellm.proxy.proxy_server, "premium_user", True) + monkeypatch.setattr(litellm.proxy.proxy_server, "jwt_handler", jwt_handler) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", None) + monkeypatch.setattr(litellm.proxy.proxy_server, "model_max_budget_limiter", limiter) + monkeypatch.setattr(user_api_key_auth_module, "JWTAuthManager", _VerifiedJWTAuthManager) + + async def authenticate() -> str | None: + result: Final = await user_api_key_auth_builder( + request=mock_request, + api_key="Bearer fake.jwt.token", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data=request_data, + ) + return result.user_id + + if not expect_rejected: + assert await authenticate() == "jwt-user" + return + with pytest.raises(ProxyException) as exc_info: + await authenticate() + assert exc_info.value.type == ProxyErrorTypes.budget_exceeded + + def _qa_websocket(messages): async def receive(): return {"type": "websocket.connect"} diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 5824b5fccab..67a76906dc8 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -13,8 +13,8 @@ from functools import partial from io import BytesIO from pathlib import Path from textwrap import dedent -from types import SimpleNamespace -from typing import Final +from types import MappingProxyType, SimpleNamespace +from typing import Final, Literal from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -98,6 +98,7 @@ from litellm.proxy.auth.user_api_key_auth import ( ) from litellm.proxy.pass_through_endpoints import pass_through_endpoints as pass_through from litellm.proxy.spend_tracking.carried_budget_state import carried_budget_metadata +from litellm.types.videos.utils import encode_video_id_with_provider from tests.unit.proxy.db.fake_prisma_engine import engine_call from tests._master_key import MASTER_KEY @@ -13956,6 +13957,143 @@ async def test_exhausted_key_budget_spares_only_zero_spend_mcp_discovery( assert result.spend == 5.0 +_VIDEO_ID: Final = encode_video_id_with_provider(video_id="video_raw", provider="openai", model_id="sora-2") +_VIDEO_RETRIEVAL_ROUTES: Final = ( + f"/v1/videos/{_VIDEO_ID}", + f"/videos/{_VIDEO_ID}", + f"/v1/videos/{_VIDEO_ID}/content", + f"/videos/{_VIDEO_ID}/content", +) +_EXPIRED_BEFORE_ANY_TEST_RUN: Final = datetime(2000, 1, 1, tzinfo=timezone.utc) +_VIDEO_SPENDING_REQUESTS: Final = ( + ("POST", "/v1/videos", MappingProxyType({"model": "sora-2"})), + ("POST", f"/v1/videos/{_VIDEO_ID}/remix", MappingProxyType({"video_id": _VIDEO_ID})), + ("POST", "/v1/videos/edits", MappingProxyType({"model": "sora-2"})), + ("POST", f"/v1/videos/{_VIDEO_ID}", MappingProxyType({"video_id": _VIDEO_ID})), + ("GET", "/v1/videos", MappingProxyType({"model": "sora-2"})), +) + + +async def _authenticate_video_request( + key: UserAPIKeyAuth, api_key: str, method: str, route: str, request_data: dict[str, object] +) -> UserAPIKeyAuth: + user_api_key_cache: Final = DualCache() + await cache_key_object( + hashed_token=key.token or "", + user_api_key_obj=key, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=None, + ) + model_budget_limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + await model_budget_limiter.dual_cache.async_set_cache( + key=model_budget_spend_cache_key( + entity_type=Litellm_EntityType.KEY, entity_id=key.token, budget_model="sora-2", budget_duration="1d" + ), + value=0.4, + ) + request: Final = Request( + { + "type": "http", + "method": method, + "path": route, + "headers": [(b"authorization", f"Bearer {api_key}".encode())], + "query_string": b"", + } + ) + proxy_logging_obj: Final = MagicMock() + proxy_logging_obj.budget_alerts = AsyncMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + with ( + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.llm_router", None), + patch("litellm.proxy.proxy_server.llm_model_list", []), + patch("litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache), + patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj), + patch("litellm.proxy.proxy_server.model_max_budget_limiter", model_budget_limiter), + patch( + "litellm.proxy.auth.user_api_key_auth.virtual_key_max_budget_alert_check", + new_callable=AsyncMock, + ), + ): + resolved: Final = await user_api_key_auth_builder( + request=request, + api_key=f"Bearer {api_key}", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data=request_data, + ) + await _authorize_authenticated_request(resolved, request, request_data, route, api_key) + return resolved + + +def _exhausted_video_key(budget: Literal["max_budget", "model_max_budget"]) -> tuple[UserAPIKeyAuth, str]: + from litellm.proxy.utils import hash_token + + api_key: Final = f"sk-exhausted-video-{budget}" + if budget == "max_budget": + return UserAPIKeyAuth(token=hash_token(api_key), max_budget=0.3, spend=0.4), api_key + return ( + UserAPIKeyAuth( + token=hash_token(api_key), model_max_budget={"sora-2": {"max_budget": 0.3, "budget_duration": "1d"}} + ), + api_key, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("budget", ["max_budget", "model_max_budget"]) +@pytest.mark.parametrize("route", _VIDEO_RETRIEVAL_ROUTES) +async def test_exhausted_key_budget_still_retrieves_an_already_paid_video( + budget: Literal["max_budget", "model_max_budget"], route: str +) -> None: + key, api_key = _exhausted_video_key(budget) + + result: Final = await _authenticate_video_request(key, api_key, "GET", route, {"video_id": _VIDEO_ID}) + + assert result.token == key.token + + +@pytest.mark.asyncio +@pytest.mark.parametrize("budget", ["max_budget", "model_max_budget"]) +@pytest.mark.parametrize("method, route, request_data", _VIDEO_SPENDING_REQUESTS) +async def test_exhausted_key_budget_still_blocks_video_requests_other_than_retrieval( + budget: Literal["max_budget", "model_max_budget"], method: str, route: str, request_data: Mapping[str, object] +) -> None: + key, api_key = _exhausted_video_key(budget) + + with pytest.raises(ProxyException) as exc_info: + await _authenticate_video_request(key, api_key, method, route, dict(request_data)) + + assert exc_info.value.type == ProxyErrorTypes.budget_exceeded + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "overrides, expected_type", + [ + ({"expires": _EXPIRED_BEFORE_ANY_TEST_RUN}, ProxyErrorTypes.expired_key), + ({"blocked": True}, ProxyErrorTypes.auth_error), + ({"models": ["other-model"]}, ProxyErrorTypes.key_model_access_denied), + ], +) +@pytest.mark.parametrize("route", (_VIDEO_RETRIEVAL_ROUTES[0], _VIDEO_RETRIEVAL_ROUTES[2])) +async def test_video_retrieval_on_exhausted_key_keeps_key_access_controls( + overrides: dict[str, object], expected_type: ProxyErrorTypes, route: str +) -> None: + exhausted_key, api_key = _exhausted_video_key("max_budget") + + with pytest.raises(ProxyException) as exc_info: + await _authenticate_video_request( + exhausted_key.model_copy(update=overrides), api_key, "GET", route, {"video_id": _VIDEO_ID} + ) + + assert exc_info.value.type == expected_type + + @pytest.mark.asyncio @pytest.mark.parametrize( "requested_model, key_aliases, team_model_aliases, expect_blocked", diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index ccfac3d4e30..bbc77b8645b 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -1020,6 +1020,49 @@ class TestProxyBaseLLMRequestProcessing: ) tag_budget_check.assert_not_awaited() + @pytest.mark.asyncio + @pytest.mark.parametrize( + "http_method, route, route_type, expect_rejected", + [ + ("GET", "/v1/videos/video_abc", "avideo_status", False), + ("GET", "/videos/video_abc/content", "avideo_content", False), + ("POST", "/v1/videos", "avideo_generation", True), + ("POST", "/v1/videos/video_abc/remix", "avideo_remix", True), + ("GET", "/v1/videos", "avideo_list", True), + ], + ) + async def test_common_processing_pre_call_logic_spares_only_video_retrieval_from_exhausted_guardrail_tags( + self, monkeypatch: pytest.MonkeyPatch, http_method: str, route: str, route_type: str, expect_rejected: bool + ) -> None: + processing_obj, mock_request, mock_proxy_logging_obj, mock_proxy_config, tag_budget_check = ( + self._guardrail_tag_budget_harness( + monkeypatch, + request_body={"model": "paid-model", "metadata": {"tags": ["existing-tag"]}}, + guardrail_tags=["guardrail-tag"], + route=route, + ) + ) + tag_budget_check.side_effect = litellm.BudgetExceededError(current_cost=0.4, max_budget=0.3) + mock_request.method = http_method + + async def pre_call() -> None: + await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=mock_proxy_config, + route_type=route_type, + llm_router=self._router_with_free_and_paid_models(), + ) + + if not expect_rejected: + await pre_call() + return + with pytest.raises(ProxyException) as exc_info: + await pre_call() + assert exc_info.value.type == ProxyErrorTypes.budget_exceeded + @pytest.mark.asyncio async def test_enforce_guardrail_added_tag_budgets_raises_budget_exceeded_proxy_exception(self, monkeypatch): from litellm.proxy.common_request_processing import _enforce_guardrail_added_tag_budgets From 0bc103911e8b1b4e0e77f8b9cbe81cfbef728875 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 11 Nov 2026 17:12:51 -0800 Subject: [PATCH 146/189] fix(proxy): bill anthropic pass-through calls at the deployment's custom price (#51267) * fix(proxy): bill anthropic pass-through calls at the deployment's custom price The /anthropic route never attached a router deployment's model_info, so spend fell back to the shared cost-map key, which carries no per-deployment prices. Attach the model_info of the one deployment in the requested group that serves that exact Anthropic model and carries custom pricing, so streaming and non-streaming calls, and dated snapshot responses, bill at it. Fixes #49596 * fix(proxy): validate deployment rows before reading anthropic pass-through pricing Read the untyped router rows and request body through a pydantic adapter so the pricing lookup adds no basedpyright unknown-type errors. * test(proxy): drive the real anthropic pass-through relay when asserting the deployment price Stub only the upstream http client and assert the x-litellm-response-cost the route returns, instead of doubling create_pass_through_route. * fix(proxy): match an anthropic pass-through deployment by its configured or resolved model name get_llm_provider rewrites -latest aliases, so a deployment configured with the alias the caller sends never matched and lost its custom price. * fix(proxy): price an anthropic pass-through call by the model its custom_body forwards The relay forwards custom_body when the caller sends one, so resolving the deployment from the outer model let a caller bill an expensive call at a cheaper deployment's price. * fix(proxy): only price an anthropic pass-through call with a deployment the caller's team may use A teamless key could pick up a team-owned deployment's price because the group lookup does not filter by team when the key has none. * fix(proxy): resolve anthropic pass-through pricing from the body forwarded after pre-call hooks The route stamped the deployment price before the pass-through ran its pre-call hooks, so a hook that swapped the model billed the original model's deployment. The route now hands the relay a resolver, and the relay applies it to the final forwarded body, which already reflects the custom_body envelope and every hook. * refactor(proxy): clear pass-through request state keys in one helper Keeps create_pass_through_route within the C901 ceiling now that the deployment pricing resolver adds a fourth request.state key. --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../llm_passthrough_endpoints.py | 71 ++++- .../pass_through_endpoints.py | 31 ++- .../pass_through_endpoints.py | 11 +- ...t_anthropic_passthrough_logging_handler.py | 68 +++++ .../test_llm_pass_through_endpoints.py | 250 ++++++++++++++++++ 5 files changed, 423 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 321c921b5bd..782b5698f3c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -177,7 +177,9 @@ from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, + LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, + DeploymentPricingResolver, WebSocketSessionMeter, ) from litellm.types.passthrough_endpoints.tinyfish import ( @@ -188,7 +190,12 @@ from litellm.types.passthrough_endpoints.tinyfish import ( ) from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials from litellm.types.router import LiteLLMParamsTypedDict -from litellm.types.utils import CallTypes, LlmProviders +from litellm.types.utils import ( + DEPLOYMENT_SCOPED_PRICING_FIELDS, + SERVER_DERIVED_PRICING_FIELDS, + CallTypes, + LlmProviders, +) from litellm.types.vector_stores import LiteLLM_ManagedVectorStore, LiteLLM_ManagedVectorStoreIndex from litellm.utils import ProviderConfigManager, with_named_credential @@ -248,6 +255,58 @@ def _models_served_by_group(llm_router: litellm.Router, model_group: str) -> fro ) +_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_DEPLOYMENT_PRICING_FIELDS: Final[frozenset[str]] = SERVER_DERIVED_PRICING_FIELDS | DEPLOYMENT_SCOPED_PRICING_FIELDS + + +def _object_mapping(value: object) -> Mapping[str, object]: + return _OBJECT_MAPPING.validate_python(value if isinstance(value, Mapping) else {}) + + +def _anthropic_deployment_model_info(deployment: object, model: str) -> Mapping[str, object] | None: + fields: Final = _object_mapping(deployment) + params: Final = _object_mapping(fields.get("litellm_params")) + backend: Final = params.get("model") + provider_override: Final = params.get("custom_llm_provider") + if not isinstance(backend, str): + return None + try: + backend_model, provider, _, _ = get_llm_provider( + model=backend, custom_llm_provider=provider_override if isinstance(provider_override, str) else None + ) + except litellm.BadRequestError: + return None + if provider != LlmProviders.ANTHROPIC.value or model not in (backend_model, backend.removeprefix(f"{provider}/")): + return None + return { + **_object_mapping(fields.get("model_info")), + **{field: value for field in _DEPLOYMENT_PRICING_FIELDS if (value := params.get(field)) is not None}, + } + + +def anthropic_deployment_pricing_model_info( + llm_router: litellm.Router | None, request_body: Mapping[str, object], team_id: str | None +) -> Mapping[str, object] | None: + """ + The pass-through selects no deployment, so ``request_body`` is the body it forwards. Only a single deployment in + that model's group serving that exact Anthropic model can say what it costs: with several, any pick could bill + another deployment's negotiated price. + """ + model: Final = request_body.get("model") + if llm_router is None or not isinstance(model, str) or not model: + return None + candidates: Final = tuple( + model_info + for deployment in llm_router.registry().deployments(model, team_id=team_id) or () + if llm_router.registry().includes(deployment, team_id=team_id) + and (model_info := _anthropic_deployment_model_info(deployment, model)) is not None + ) + if len(candidates) != 1: + return None + (only_match,) = candidates + return only_match if any(only_match.get(field) is not None for field in _DEPLOYMENT_PRICING_FIELDS) else None + + def is_passthrough_request_streaming(request_body: object) -> bool: """ Returns True if the request is streaming. @@ -1236,6 +1295,16 @@ async def anthropic_proxy_route( ## check for streaming is_streaming_request: Final = await is_streaming_request_fn(request) + from litellm.proxy.proxy_server import llm_router + + setattr( + request.state, + LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, + DeploymentPricingResolver( + resolve=partial(anthropic_deployment_pricing_model_info, llm_router, team_id=user_api_key_dict.team_id) + ), + ) + ## CREATE PASS-THROUGH auth_header: Final = await AnthropicModelInfo.aget_auth_header( anthropic_api_key or None, allow_workload_identity=True diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 00e2aeb9e6e..8a728a7335a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -167,8 +167,10 @@ from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, + LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, + DeploymentPricingResolver, EndpointType, PassthroughStandardLoggingPayload, WebSocketSessionMeter, @@ -448,6 +450,20 @@ async def chat_completion_pass_through_endpoint( ) +_PASS_THROUGH_REQUEST_STATE_KEYS: Final = ( + LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, + LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, + LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, + LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, +) + + +def _clear_pass_through_request_state(state: object) -> None: + for key in _PASS_THROUGH_REQUEST_STATE_KEYS: + if hasattr(state, key): + delattr(state, key) + + def _is_provider_payload(request: Request) -> bool: return is_opaque_pass_through_request(get_request_route(request), request.headers.get("content-type", "")) @@ -798,6 +814,14 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): ) if isinstance(deployment_model_info, Mapping): _metadata["model_info"] = dict(deployment_model_info) + deployment_pricing: Final = getattr(_request_state, LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, None) + priced_model_info: Final = ( + deployment_pricing.resolve(server_marker_free_body) + if isinstance(deployment_pricing, DeploymentPricingResolver) + else None + ) + if priced_model_info is not None: + _metadata["model_info"] = dict(priced_model_info) kwargs: Final = { "litellm_params": { @@ -2536,12 +2560,7 @@ def create_pass_through_route( keep_target_query=keep_target_query, ) finally: - if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY): - delattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY) - if hasattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY): - delattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY) - if hasattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY): - delattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY) + _clear_pass_through_request_state(request.state) # The upstream withholds its response headers until its first token, so # the whole time-to-first-token is spent inside _relay with nothing on diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index b6bab77ab33..8a2b2a8f99c 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime from enum import Enum @@ -20,6 +20,15 @@ LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY: Final = "litellm_pass_through_raw_body" # `model_info` of the router deployment a provider route (e.g. Vertex) resolved for this request. LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY: Final = "litellm_pass_through_deployment_model_info" +# Resolves that `model_info` from the body actually forwarded, after the `custom_body` envelope and pre-call hooks. +LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY: Final = "litellm_pass_through_deployment_pricing" + + +@dataclass(frozen=True, slots=True) +class DeploymentPricingResolver: + resolve: Callable[[Mapping[str, object]], Mapping[str, object] | None] + + # Attribute set on the FastAPI endpoint function of every user-defined pass-through # route. Auth reads it off the dispatched endpoint (``request.scope["endpoint"]``) to # decide whether a request body ``model`` names an upstream model rather than a diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index ad12716ef2a..df011a8296f 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -3102,3 +3102,71 @@ def test_anthropic_passthrough_stream_bills_a_bedrock_guardrail_trace_whose_json ) (entry,) = _recorded_guardrail_entries(mock_logging_obj) assert entry["guardrail_usage"] == {"topicPolicyUnits": 3, "contentPolicyUnits": 3} + + +_DEPLOYMENT_PRICING: Final = {"input_cost_per_token": 3e-03, "output_cost_per_token": 7e-03} + + +def _deployment_priced_logging_obj(monkeypatch: pytest.MonkeyPatch, stream: bool) -> LiteLLMLoggingObj: + monkeypatch.setattr(litellm, "model_cost", {"sonnet-dep": {"litellm_provider": "anthropic", **_DEPLOYMENT_PRICING}}) + logging_obj: Final = LiteLLMLoggingObj( + model="claude-sonnet-4-5", + messages=[], + stream=stream, + call_type="pass_through_endpoint", + start_time=datetime(2025, 1, 1), + litellm_call_id="deployment-priced", + function_id="1", + ) + litellm_params: Final = {"metadata": {"model_info": {"id": "sonnet-dep", **_DEPLOYMENT_PRICING}}} + logging_obj.litellm_params = litellm_params + logging_obj.model_call_details["litellm_params"] = litellm_params + logging_obj.model_call_details["custom_llm_provider"] = "anthropic" + logging_obj.model_call_details["stream"] = stream + return logging_obj + + +def _deployment_price_of(usage: litellm.Usage) -> float: + return ( + usage.prompt_tokens * _DEPLOYMENT_PRICING["input_cost_per_token"] + + usage.completion_tokens * _DEPLOYMENT_PRICING["output_cost_per_token"] + ) + + +def test_a_response_naming_a_dated_snapshot_is_billed_at_the_resolved_deployment_price( + monkeypatch, mock_httpx_response +): + logging_obj: Final = _deployment_priced_logging_obj(monkeypatch, stream=False) + + result: Final = AnthropicPassthroughLoggingHandler.anthropic_passthrough_handler( + httpx_response=mock_httpx_response, + response_body=mock_httpx_response.json(), + logging_obj=logging_obj, + url_route="/v1/messages", + result="success", + start_time=datetime(2025, 1, 1), + end_time=datetime(2025, 1, 1), + cache_hit=False, + request_body={"model": "claude-sonnet-4-5", "messages": []}, + ) + + assert result["result"].usage.prompt_tokens > 0 + assert result["kwargs"]["response_cost"] == pytest.approx(_deployment_price_of(result["result"].usage)) + + +def test_a_stream_naming_a_dated_snapshot_is_billed_at_the_resolved_deployment_price(monkeypatch, all_chunks): + logging_obj: Final = _deployment_priced_logging_obj(monkeypatch, stream=True) + + result: Final = AnthropicPassthroughLoggingHandler.handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + request_body={"model": "claude-sonnet-4-5", "stream": True, "messages": []}, + endpoint_type=MagicMock(), + start_time=datetime(2025, 1, 1), + all_chunks=all_chunks, + end_time=datetime(2025, 1, 1), + ) + + assert result["result"].usage.prompt_tokens > 0 + assert result["kwargs"]["response_cost"] == pytest.approx(_deployment_price_of(result["result"].usage)) diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 7f118fb3186..ca04787e592 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -7256,6 +7256,256 @@ class TestAnthropicProxyRoute: assert thread_ids and thread_ids[0] != threading.get_ident() +_HAIKU_PRICING: Final = MappingProxyType({"input_cost_per_token": 1e-07, "output_cost_per_token": 5e-07}) + + +def _anthropic_deployment( + deployment_id: str, + model_name: str = "claude-haiku-5-5", + backend: str = "anthropic/claude-haiku-5-5", + model_info_pricing: Mapping[str, float] = _HAIKU_PRICING, + params_pricing: Mapping[str, float] = MappingProxyType({}), + team_id: str | None = None, +) -> dict[str, object]: + return { + "model_name": model_name, + "litellm_params": {"model": backend, "api_key": "sk-ant-test", **params_pricing}, + "model_info": {"id": deployment_id, **model_info_pricing, **({"team_id": team_id} if team_id else {})}, + } + + +def _passthrough_pricing( + monkeypatch: pytest.MonkeyPatch, + *deployments: dict[str, object], + request_body: Mapping[str, object] = MappingProxyType({"model": "claude-haiku-5-5"}), + team_id: str | None = None, +) -> Mapping[str, object] | None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + anthropic_deployment_pricing_model_info, + ) + + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) + router: Final = litellm.Router(model_list=[*deployments]) + return anthropic_deployment_pricing_model_info(router, request_body, team_id) + + +def test_anthropic_passthrough_prices_with_the_one_deployment_serving_the_requested_model(monkeypatch): + model_info: Final = _passthrough_pricing(monkeypatch, _anthropic_deployment("haiku-dep")) + + assert model_info is not None + assert model_info["id"] == "haiku-dep" + assert {key: model_info[key] for key in _HAIKU_PRICING} == _HAIKU_PRICING + + +def test_anthropic_passthrough_reads_custom_pricing_set_in_litellm_params(monkeypatch): + model_info: Final = _passthrough_pricing( + monkeypatch, + _anthropic_deployment("haiku-dep", model_info_pricing=MappingProxyType({}), params_pricing=_HAIKU_PRICING), + ) + + assert model_info is not None + assert model_info["id"] == "haiku-dep" + assert {key: model_info[key] for key in _HAIKU_PRICING} == _HAIKU_PRICING + + +def test_anthropic_passthrough_prices_a_deployment_configured_with_the_alias_the_caller_sends(monkeypatch): + model_info: Final = _passthrough_pricing( + monkeypatch, + _anthropic_deployment( + "haiku-latest-dep", model_name="claude-haiku-latest", backend="anthropic/claude-haiku-latest" + ), + request_body={"model": "claude-haiku-latest"}, + ) + + assert model_info is not None + assert model_info["id"] == "haiku-latest-dep" + + +@pytest.mark.parametrize( + ("caller_team_id", "priced"), + [ + pytest.param("team-a", True, id="owning-team"), + pytest.param("team-b", False, id="other-team"), + pytest.param(None, False, id="teamless-key"), + ], +) +def test_anthropic_passthrough_prices_a_team_deployment_only_for_its_team(monkeypatch, caller_team_id, priced): + model_info: Final = _passthrough_pricing( + monkeypatch, _anthropic_deployment("team-a-haiku-dep", team_id="team-a"), team_id=caller_team_id + ) + + assert (model_info is not None) is priced + + +@pytest.mark.parametrize( + "deployments", + [ + pytest.param( + ( + _anthropic_deployment("haiku-dep-a"), + _anthropic_deployment("haiku-dep-b", model_info_pricing={"input_cost_per_token": 9e-07}), + ), + id="two-deployments-could-bill-either-price", + ), + pytest.param( + (_anthropic_deployment("opus-dep", backend="anthropic/claude-opus-5-5"),), + id="group-serves-a-different-upstream-model", + ), + pytest.param( + (_anthropic_deployment("bedrock-dep", backend="bedrock/claude-haiku-5-5"),), + id="deployment-is-not-anthropic", + ), + pytest.param( + (_anthropic_deployment("other-group-dep", model_name="haiku-alias"),), + id="model-group-not-requested", + ), + pytest.param( + (_anthropic_deployment("unpriced-dep", model_info_pricing=MappingProxyType({})),), + id="deployment-has-no-custom-pricing", + ), + ], +) +def test_anthropic_passthrough_attaches_no_deployment_pricing_without_one_priced_match(monkeypatch, deployments): + assert _passthrough_pricing(monkeypatch, *deployments) is None + + +_RELAY_HAIKU_PRICING: Final = MappingProxyType({"input_cost_per_token": 1e-03, "output_cost_per_token": 2e-03}) +_RELAY_OPUS_PRICING: Final = MappingProxyType({"input_cost_per_token": 5e-03, "output_cost_per_token": 6e-03}) +_RELAY_MESSAGES: Final = ({"role": "user", "content": "hi"},) + + +async def _hook_passes_body_through(*, data: dict[str, object], **_: object) -> dict[str, object]: + return data + + +async def _hook_upgrades_to_opus(*, data: dict[str, object], **_: object) -> dict[str, object]: + return {**data, "model": "claude-opus-5-5"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_body", "pre_call_hook", "forwarded_model", "pricing"), + [ + pytest.param( + {"model": "claude-haiku-5-5", "messages": [*_RELAY_MESSAGES]}, + _hook_passes_body_through, + "claude-haiku-5-5", + _RELAY_HAIKU_PRICING, + id="requested-model", + ), + pytest.param( + {"model": "claude-haiku-5-5", "custom_body": {"model": "claude-opus-5-5", "messages": [*_RELAY_MESSAGES]}}, + _hook_passes_body_through, + "claude-opus-5-5", + _RELAY_OPUS_PRICING, + id="custom-body-envelope", + ), + pytest.param( + {"model": "claude-haiku-5-5", "messages": [*_RELAY_MESSAGES]}, + _hook_upgrades_to_opus, + "claude-opus-5-5", + _RELAY_OPUS_PRICING, + id="pre-call-hook-rewrite", + ), + ], +) +async def test_anthropic_proxy_route_bills_the_custom_price_of_the_forwarded_model( + monkeypatch, request_body, pre_call_hook, forwarded_model, pricing +): + for name in ( + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_API_BASE", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_FEDERATION_RULE_ID", + "ANTHROPIC_ORGANIZATION_ID", + "ANTHROPIC_IDENTITY_TOKEN_FILE", + "ANTHROPIC_IDENTITY_TOKEN", + "SERVER_ROOT_PATH", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test") + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_router", + litellm.Router( + model_list=[ + _anthropic_deployment("haiku-dep", model_info_pricing=_RELAY_HAIKU_PRICING), + _anthropic_deployment( + "opus-dep", + model_name="claude-opus-5-5", + backend="anthropic/claude-opus-5-5", + model_info_pricing=_RELAY_OPUS_PRICING, + ), + ] + ), + ) + upstream_response: Final = httpx.Response( + 200, + json={ + "id": "msg_relay", + "type": "message", + "role": "assistant", + "model": forwarded_model, + "content": [{"type": "text", "text": "pong"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 18, "output_tokens": 4}, + }, + request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), + ) + httpx_client: Final = MagicMock() + httpx_client.build_request = MagicMock(return_value=MagicMock()) + httpx_client.send = AsyncMock(return_value=upstream_response) + client_wrapper: Final = MagicMock() + client_wrapper.client = httpx_client + body: Final = json.dumps(request_body).encode() + + async def receive() -> dict[str, object]: + return {"type": "http.request", "body": body, "more_body": False} + + request: Final = Request( + { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "https", + "path": "/anthropic/v1/messages", + "raw_path": b"/anthropic/v1/messages", + "root_path": "", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + "client": ("127.0.0.1", 51234), + "server": ("proxy.local", 4000), + "state": {}, + }, + receive, + ) + + with ( + patch( # test-quality-ok: the upstream http client is the boundary, it answers with a real Anthropic body + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client", + return_value=client_wrapper, + ), + patch( # test-quality-ok: the relay calls these hooks, and they need a db this test has no use for + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_logging_obj, + ): + mock_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) + mock_logging_obj.post_call_success_hook = AsyncMock() + mock_logging_obj.post_call_failure_hook = AsyncMock() + mock_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + response: Final = await anthropic_proxy_route( + endpoint="v1/messages", + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-caller-virtual-key"), + ) + + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx( + 18 * pricing["input_cost_per_token"] + 4 * pricing["output_cost_per_token"] + ) + + class TestAnthropicProxyRouteCallerAuthHeaders: """Regression for a caller credential riding upstream next to a server-owned one. From d1933c92c036eb37aa3b53f78342401de6111129 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 11 Nov 2026 17:39:10 -0800 Subject: [PATCH 147/189] ci(unit): split proxy-infra-root in two, isolate subprocess tests, make its file exclusions take effect (#51316) pytest --ignore is a no-op for a file the shell glob already expanded, so the 25 files proxy-infra-root listed under --ignore still ran there as well as in their own shards (805 duplicate tests, 18% of the shard's time). Switch them to --deselect, which pytest applies to explicit arguments, and teach assert_ci_coverage.py to read --deselect and [..] character-class globs. Split the rest by first letter into proxy-infra-root-a-p and proxy-infra-root-q-z, and give the three files that spawn a child python (test_component_allowlists, test_prometheus_metrics_server, test_proxy_cli) their own 2-worker proxy-subprocess shard so the children stop competing with 4 xdist workers on a 4 vCPU runner. Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/scripts/assert_ci_coverage.py | 6 +- .github/workflows/test-unit.yml | 79 ++++++++++++++++++--------- codecov.yaml | 6 +- tests/unit/test_assert_ci_coverage.py | 26 +++++++++ 4 files changed, 86 insertions(+), 31 deletions(-) diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index b443fd9e0d9..9dcae329b59 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -27,11 +27,11 @@ TEST_PATH_KEYS = frozenset({"test-path", "test-paths", "test_path"}) DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"}) TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s") IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b") -TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+") -IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)") +TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?\[\]-]+") +IGNORE_ARG_RE: Final = re.compile(r"--(?:ignore(?:-glob)?|deselect)[= ](\S+)") DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*") COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE) -GLOB_CHARS = frozenset("*?") +GLOB_CHARS = frozenset("*?[") DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType( { "image-scan.yml": "tests/integration/database/image/", diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 6f980b458fc..f36c232791a 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -673,34 +673,31 @@ jobs: job-timeout-minutes: 60 dist: loadscope - - shard: proxy-infra-root + - shard: proxy-infra-root-a-p test-path: |- - tests/unit/proxy/test_*.py - --ignore=tests/unit/proxy/test_aproxy_startup.py - --ignore=tests/unit/proxy/test_credential_slot_registry.py - --ignore=tests/unit/proxy/test_custom_callback_input.py - --ignore=tests/unit/proxy/test_custom_logger_s3_gcs.py - --ignore=tests/unit/proxy/test_custom_tokenizer_bug.py - --ignore=tests/unit/proxy/test_db_schema_changes.py - --ignore=tests/unit/proxy/test_deprecated_key_grace_period.py - --ignore=tests/unit/proxy/test_get_favicon.py - --ignore=tests/unit/proxy/test_get_image.py - --ignore=tests/unit/proxy/test_prisma_client_backoff_retry.py - --ignore=tests/unit/proxy/test_prompt_test_endpoint.py - --ignore=tests/unit/proxy/test_proxy_config_unit_test.py - --ignore=tests/unit/proxy/test_proxy_custom_auth.py - --ignore=tests/unit/proxy/test_proxy_reject_logging.py - --ignore=tests/unit/proxy/test_proxy_server.py - --ignore=tests/unit/proxy/test_proxy_setting_guardrails.py - --ignore=tests/unit/proxy/test_proxy_token_counter.py - --ignore=tests/unit/proxy/test_proxy_utils.py - --ignore=tests/unit/proxy/test_reducto_ocr_route.py - --ignore=tests/unit/proxy/test_response_polling_pre_call_checks.py - --ignore=tests/unit/proxy/test_server_root_path.py - --ignore=tests/unit/proxy/test_ui_path_detection.py - --ignore=tests/unit/proxy/test_unit_test_proxy_hooks.py - --ignore=tests/unit/proxy/test_update_spend.py - --ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py + tests/unit/proxy/test_[_a-p]*.py + --deselect=tests/unit/proxy/test_aproxy_startup.py + --deselect=tests/unit/proxy/test_component_allowlists.py + --deselect=tests/unit/proxy/test_credential_slot_registry.py + --deselect=tests/unit/proxy/test_custom_callback_input.py + --deselect=tests/unit/proxy/test_custom_logger_s3_gcs.py + --deselect=tests/unit/proxy/test_custom_tokenizer_bug.py + --deselect=tests/unit/proxy/test_db_schema_changes.py + --deselect=tests/unit/proxy/test_deprecated_key_grace_period.py + --deselect=tests/unit/proxy/test_get_favicon.py + --deselect=tests/unit/proxy/test_get_image.py + --deselect=tests/unit/proxy/test_prisma_client_backoff_retry.py + --deselect=tests/unit/proxy/test_prometheus_metrics_server.py + --deselect=tests/unit/proxy/test_prompt_test_endpoint.py + --deselect=tests/unit/proxy/test_proxy_cli.py + --deselect=tests/unit/proxy/test_proxy_config_unit_test.py + --deselect=tests/unit/proxy/test_proxy_custom_auth.py + --deselect=tests/unit/proxy/test_proxy_reject_logging.py + --deselect=tests/unit/proxy/test_proxy_server.py + --deselect=tests/unit/proxy/test_proxy_setting_guardrails.py + --deselect=tests/unit/proxy/test_proxy_token_counter.py + --deselect=tests/unit/proxy/test_proxy_utils.py + --deselect=tests/unit/proxy/test_unit_test_proxy_hooks.py python-version: "3.12" workers: 4 reruns: 0 @@ -708,6 +705,34 @@ jobs: job-timeout-minutes: 60 dist: loadscope + - shard: proxy-infra-root-q-z + test-path: |- + tests/unit/proxy/test_[q-z]*.py + --deselect=tests/unit/proxy/test_reducto_ocr_route.py + --deselect=tests/unit/proxy/test_response_polling_pre_call_checks.py + --deselect=tests/unit/proxy/test_server_root_path.py + --deselect=tests/unit/proxy/test_ui_path_detection.py + --deselect=tests/unit/proxy/test_update_spend.py + --deselect=tests/unit/proxy/test_zero_cost_model_budget_bypass.py + python-version: "3.12" + workers: 4 + reruns: 0 + timeout-minutes: 20 + job-timeout-minutes: 60 + dist: loadscope + + - shard: proxy-subprocess + test-path: |- + tests/unit/proxy/test_component_allowlists.py + tests/unit/proxy/test_prometheus_metrics_server.py + tests/unit/proxy/test_proxy_cli.py + python-version: "3.12" + workers: 2 + reruns: 0 + timeout-minutes: 20 + job-timeout-minutes: 60 + dist: loadscope + - shard: caching-local test-path: >- tests/unit/caching diff --git a/codecov.yaml b/codecov.yaml index 419a28ce6c6..d671d883120 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -58,7 +58,11 @@ flag_management: carryforward: false - name: proxy-infra carryforward: false - - name: proxy-infra-root + - name: proxy-infra-root-a-p + carryforward: false + - name: proxy-infra-root-q-z + carryforward: false + - name: proxy-subprocess carryforward: false - name: caching-local carryforward: false diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index 4127432bd3d..0a50bef385d 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -343,6 +343,32 @@ def test_a_file_its_only_shard_ignores_is_not_covered_by_that_shards_glob() -> N assert selection.covers("tests/unit/proxy/test_update_spend.py") is False +def test_a_deselected_file_is_treated_like_an_ignored_one() -> None: + # --ignore is a no-op for a file the shell glob already expanded, --deselect is what drops it + scalars: Final = ( + coverage.Scalar( + key="test-path", + value="tests/unit/proxy/test_[q-z]*.py\n--deselect=tests/unit/proxy/test_update_spend.py", + ), + ) + assert coverage._invoked_test_tokens(scalars) == frozenset({"tests/unit/proxy/test_[q-z]*.py"}) + (selection,) = coverage._invoked_selections(scalars) + assert selection.covers("tests/unit/proxy/test_route_llm_request.py") is True + assert selection.covers("tests/unit/proxy/test_update_spend.py") is False + + +def test_a_character_class_glob_in_a_test_path_is_one_token_that_names_only_its_letters() -> None: + (selection,) = coverage._invoked_selections( + (coverage.Scalar(key="test-path", value="tests/unit/proxy/test_[_a-p]*.py"),) + ) + assert selection.included == frozenset({"tests/unit/proxy/test_[_a-p]*.py"}) + assert selection.covers("tests/unit/proxy/test__types.py") is True + assert selection.covers("tests/unit/proxy/test_caching_routes.py") is True + assert selection.covers("tests/unit/proxy/test_route_llm_request.py") is False + assert coverage._token_names("tests/unit/proxy/test_[_a-p]*.py", "tests/unit/proxy/test_admin_mcp.py") is True + assert coverage._token_names("tests/unit/proxy/test_[_a-p]*.py", "tests/unit/proxy/test_route_llm_request.py") is False + + def test_check_shards_passes_on_the_repo_as_it_stands(capsys): assert coverage._check_shards() == 0 From 801e139c35af5eca132e8761cb11be19621218c4 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 11 Nov 2026 17:39:27 -0800 Subject: [PATCH 148/189] fix(gemini): attribute /gemini batchGenerateContent batches to the calling key (#51279) * fix(gemini): attribute /gemini batchGenerateContent batches to the calling key A 200 from models/{model}:batchGenerateContent through the Gemini pass-through now registers a managed batch row for the calling key, team and tags, so CheckBatchCost prices the output file on completion instead of the job landing as a $0 pass_through_endpoint row with model unknown. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(gemini): unit regression for pass-through batch registration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): file batch created through /gemini is registered for the calling key Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(gemini): bill pass-through batches through the configured gemini deployment The managed row now points at the gemini/ deployment CheckBatchCost fetches the output with, not the first router id for the model name, and a model with no gemini deployment logs a warning instead of registering a row that can never be billed. A key without a user keeps user_id unset rather than default-user. Typed the registration helper and handler so the lint gates stay flat. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../batch_attribution.py | 86 +++++++++++- .../gemini_passthrough_logging_handler.py | 114 +++++++++++++++- .../pass_through_endpoints/success_handler.py | 1 + .../test_gemini_passthrough_batch_wire.py | 124 +++++++++++++++++ .../test_batch_attribution.py | 124 ++++++++++++++++- ...test_gemini_passthrough_logging_handler.py | 129 ++++++++++++++++-- 6 files changed, 559 insertions(+), 19 deletions(-) create mode 100644 tests/integration/providers/test_gemini_passthrough_batch_wire.py diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/batch_attribution.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/batch_attribution.py index cc3b0c429ff..e936959a085 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/batch_attribution.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/batch_attribution.py @@ -6,13 +6,36 @@ background poll that has no request to read them from. """ import asyncio -from collections.abc import Mapping, Sequence -from typing import Final +import base64 +from collections.abc import Coroutine, Mapping, Sequence +from functools import partial +from typing import TYPE_CHECKING, Final, Literal, Protocol, runtime_checkable from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.safe_json_dumps import strip_null_bytes +from litellm.types.utils import LiteLLMBatch, SpecialEnums + +if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth + + +@runtime_checkable +class BatchRegistry(Protocol): + """The managed-files hook's write path, the one method passthrough batch attribution needs from it.""" + + def store_unified_object_id( + self, + unified_object_id: str, + file_object: LiteLLMBatch, + litellm_parent_otel_span: None, + model_object_id: str, + file_purpose: Literal["batch"], + user_api_key_dict: "UserAPIKeyAuth", + request_tags: Sequence[str] | None, + persist_attribution: bool, + ) -> Coroutine[object, object, None]: ... def optional_str(value: object) -> str | None: @@ -88,3 +111,62 @@ def log_batch_registration_result( unified_object_id, model_object_id, ) + + +def unified_batch_object_id(model_id: str, batch_id: str) -> str: + """The managed object id CheckBatchCost decodes back into the deployment and the provider batch id.""" + unified: Final = SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format(model_id, batch_id) + return base64.urlsafe_b64encode(unified.encode()).decode().rstrip("=") + + +def register_passthrough_batch( + provider: str, + unified_object_id: str, + batch_object: LiteLLMBatch, + model_object_id: str, + request_metadata: Mapping[str, object], +) -> None: + """Record a batch created through a passthrough create for the calling key, team and tags, so + CheckBatchCost prices it on completion. Fire and forget, the passthrough response is already on its way.""" + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.proxy_server import proxy_logging_obj + + managed_files_hook: Final = proxy_logging_obj.get_proxy_hook("managed_files") + if not isinstance(managed_files_hook, BatchRegistry): + verbose_proxy_logger.warning( + "Managed files hook not available, cannot store %s batch %s for cost tracking", provider, model_object_id + ) + return + user_api_key_dict: Final = UserAPIKeyAuth( + user_id=optional_str(request_metadata.get("user_api_key_user_id")), + api_key=optional_str(request_metadata.get("user_api_key")), + team_id=optional_str(request_metadata.get("user_api_key_team_id")), + user_role=LitellmUserRoles.CUSTOMER, + spend=0.0, + models=[], + metadata={}, + permissions={}, + model_max_budget={}, + model_spend={}, + ) + task: Final[asyncio.Task[None]] = asyncio.create_task( + managed_files_hook.store_unified_object_id( + unified_object_id=unified_object_id, + file_object=batch_object, + litellm_parent_otel_span=None, + model_object_id=model_object_id, + file_purpose="batch", + user_api_key_dict=user_api_key_dict, + request_tags=request_tags_from_metadata(request_metadata), + persist_attribution=True, + ) + ) + task.add_done_callback( + partial( + log_batch_registration_result, + provider=provider, + unified_object_id=unified_object_id, + model_object_id=model_object_id, + is_batch_create=True, + ) + ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py index d449f62af2c..24a11c653d7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py @@ -1,8 +1,10 @@ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -10,11 +12,16 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.gemini.videos.transformation import GeminiVideoConfig from litellm.llms.vertex_ai.common_utils import get_vertex_model_id_from_url from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( + register_passthrough_batch, + unified_batch_object_id, +) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( GeminiSseChunkParser, VertexPassthroughLoggingHandler, ) from litellm.types.utils import ( + Choices, ModelResponse, TextCompletionResponse, ) @@ -28,6 +35,10 @@ else: EndpointType = Any +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) + + class GeminiPassthroughLoggingHandler: @staticmethod def gemini_passthrough_handler( @@ -53,6 +64,14 @@ class GeminiPassthroughLoggingHandler: custom_llm_provider="gemini", vertex_location=None, ) + if "batchGenerateContent" in url_route: + return GeminiPassthroughLoggingHandler._batch_create_handler( + httpx_response=httpx_response, + logging_obj=logging_obj, + url_route=url_route, + start_time=start_time, + kwargs=_JSON_OBJECT.validate_python(kwargs), + ) if "predictLongRunning" in url_route: model = GeminiPassthroughLoggingHandler.extract_model_from_url(url_route) @@ -127,6 +146,99 @@ class GeminiPassthroughLoggingHandler: "kwargs": kwargs, } + @staticmethod + def gemini_deployment_id(model: str) -> str | None: + """The configured ``gemini/`` deployment CheckBatchCost bills a pass-through batch through, or nothing.""" + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + return None + deployments: Final = tuple( + _JSON_OBJECT.validate_python(deployment) + for deployment in llm_router.registry().deployments(model_name=model) or () + ) + deployment_ids: Final = ( + _JSON_OBJECT.validate_python(deployment.get("model_info") or _EMPTY).get("id") + for deployment in deployments + if _JSON_OBJECT.validate_python(deployment.get("litellm_params") or _EMPTY).get("model") + == f"gemini/{model}" + ) + return next((deployment_id for deployment_id in deployment_ids if isinstance(deployment_id, str)), None) + + @staticmethod + def _batch_create_handler( + httpx_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + url_route: str, + start_time: datetime, + kwargs: Mapping[str, object], + ) -> PassThroughEndpointLoggingTypedDict: + """A 200 from ``models/{model}:batchGenerateContent`` registers the job for the calling key, so + CheckBatchCost prices its output file when it completes. Anything else is logged as the plain + $0 pass-through row it always was.""" + from litellm._uuid import uuid + from litellm.llms.gemini.batches.transformation import GeminiBatchesConfig + + model: Final = GeminiPassthroughLoggingHandler.extract_model_from_url(url_route) + if httpx_response.status_code != 200 or "name" not in _JSON_OBJECT.validate_json(httpx_response.content): + return {"result": None, "kwargs": dict(kwargs, model=model, custom_llm_provider="gemini")} + batch: Final = GeminiBatchesConfig().transform_create_batch_response( + model=model, raw_response=httpx_response, logging_obj=logging_obj, litellm_params=MappingProxyType({}) + ) + deployment_id: Final = GeminiPassthroughLoggingHandler.gemini_deployment_id(model) + if deployment_id is None: + verbose_proxy_logger.warning( + "No gemini/%s deployment is configured, so batch %s created through the pass-through cannot be billed", + model, + batch.id, + ) + return { + "result": None, + "kwargs": dict(kwargs, model=model, custom_llm_provider="gemini", batch_id=batch.id), + } + unified_object_id: Final = unified_batch_object_id(deployment_id, batch.id) + litellm_params: Final = _JSON_OBJECT.validate_python(kwargs.get("litellm_params") or _EMPTY) + request_metadata: Final = _JSON_OBJECT.validate_python(litellm_params.get("metadata") or _EMPTY) + register_passthrough_batch("Gemini", unified_object_id, batch, batch.id, request_metadata) + + litellm_model_response: Final = ModelResponse() + litellm_model_response.id = str(uuid.uuid4()) + litellm_model_response.model = model + litellm_model_response.object = "batch" + litellm_model_response.created = int(start_time.timestamp()) + litellm_model_response.choices = [ + Choices( + finish_reason="stop", + index=0, + message={ + "role": "assistant", + "content": f"Batch job {batch.id} created and is pending. Spend lands when the batch completes.", + "tool_calls": None, + "function_call": None, + "provider_specific_fields": { + "batch_job_id": batch.id, + "batch_job_state": batch.status, + "unified_object_id": unified_object_id, + }, + }, + ) + ] + logging_obj.model = model # rebind-ok: passthrough attribution records the resolved Gemini model + logging_obj.model_call_details.update( + {"model": model, "custom_llm_provider": "gemini", "response_cost": 0.0, "batch_id": batch.id} + ) + return { + "result": litellm_model_response, + "kwargs": dict( + kwargs, + response_cost=0.0, + model=model, + custom_llm_provider="gemini", + batch_id=batch.id, + unified_object_id=unified_object_id, + ), + } + @staticmethod def _handle_logging_gemini_collected_chunks( litellm_logging_obj: LiteLLMLoggingObj, diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 4552ac98624..2375cd4dc69 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -159,6 +159,7 @@ class PassThroughEndpointLogging: self.TRACKED_GEMINI_ROUTES = [ "generateContent", "streamGenerateContent", + "batchGenerateContent", "predictLongRunning", ] diff --git a/tests/integration/providers/test_gemini_passthrough_batch_wire.py b/tests/integration/providers/test_gemini_passthrough_batch_wire.py new file mode 100644 index 00000000000..f3b7344c514 --- /dev/null +++ b/tests/integration/providers/test_gemini_passthrough_batch_wire.py @@ -0,0 +1,124 @@ +"""A file-based batch created through the /gemini pass-through is registered for the calling key, so +CheckBatchCost can price it, and its spend row names the model instead of ``unknown``.""" + +import json +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +from litellm.proxy.utils import hash_token + +pytestmark = pytest.mark.timeout(240) + +_API_KEY: Final = "scripted-gemini-key" +_MODEL: Final = "gemini-3.8-flash" +_BATCH_ID: Final = "wire7h3x9batch" +_INPUT_FILE: Final = "files/wireinput123" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_OPERATION: Final[dict[str, JsonValue]] = { + "name": f"batches/{_BATCH_ID}", + "metadata": { + "@type": "type.googleapis.com/google.ai.generativelanguage.v1main.GenerateContentBatch", + "model": f"models/{_MODEL}", + "displayName": "wire-batch", + "inputConfig": {"fileName": _INPUT_FILE}, + "createTime": "2026-11-11T23:15:29.000000Z", + "updateTime": "2026-11-11T23:15:29.000000Z", + "state": "BATCH_STATE_PENDING", + }, +} + + +def _respond(request: Request) -> Reply: + assert request.target.split("?")[0].endswith(f"/v1beta/models/{_MODEL}:batchGenerateContent"), request.target + return Reply(body=json.dumps(_OPERATION).encode()) + + +@pytest.fixture(scope="module") +def rig() -> Iterator[Gateway]: + with gateway_from_environment() as gateway: + yield gateway + + +@pytest.fixture(scope="module") +def wire() -> Iterator[Wire]: + with wire_server(_respond) as served: + yield served + + +def _config(directory: Path) -> Path: + path: Final = directory / "gemini-passthrough-batch.yaml" + path.write_text( + "model_list:\n" + f" - model_name: {_MODEL}\n" + " litellm_params:\n" + f" model: gemini/{_MODEL}\n" + " api_key: os.environ/GEMINI_API_KEY\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + "router_settings:\n" + " disable_cooldowns: true\n" + ) + return path + + +@pytest.fixture(scope="module") +def proxy(rig: Gateway, wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("gemini-passthrough-batch") + overrides: Final = {"GEMINI_API_BASE": wire.url, "GEMINI_API_KEY": _API_KEY} + with owned_proxy(rig, directory, overrides, config=_config(directory), workers=2) as owned: + yield owned + + +def test_file_batch_created_through_gemini_passthrough_is_registered_for_the_calling_key( + proxy: Gateway, wire: Wire +) -> None: + wire.drain() + with proxy.scenario() as scenario: + key: Final = scenario.key() + body: Final = {"batch": {"displayName": "wire-batch", "inputConfig": {"fileName": _INPUT_FILE}}} + response: Final = httpx.post( + f"{str(proxy.client.base_url).rstrip('/')}/gemini/v1beta/models/{_MODEL}:batchGenerateContent", + headers={"x-goog-api-key": key}, + json=body, + timeout=30, + ) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content) == _OPERATION, response.text + upstream: Final = wire.drain() + assert [(request.method, request.target.split("?")[0]) for request in upstream] == [ + ("POST", f"/v1beta/models/{_MODEL}:batchGenerateContent") + ], upstream + assert f"key={_API_KEY}" in upstream[0].target, upstream[0].target + assert key not in upstream[0].target and key.encode() not in upstream[0].body, upstream[0] + assert key not in upstream[0].headers.values(), upstream[0].headers + + registered: Final = eventually( + lambda: proxy.rows( + 'SELECT api_key, file_purpose, status FROM "LiteLLM_ManagedObjectTable" WHERE model_object_id = %s', + (_BATCH_ID,), + ), + lambda rows: len(rows) == 1, + seconds=30, + ) + assert registered[0]["api_key"] == hash_token(key), registered + assert registered[0]["file_purpose"] == "batch", registered + + spend_rows: Final = eventually( + lambda: proxy.rows( + 'SELECT model, spend FROM "LiteLLM_SpendLogs" WHERE api_key = %s AND call_type = %s', + (hash_token(key), "pass_through_endpoint"), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert str(spend_rows[0]["model"]).endswith(_MODEL), spend_rows + assert float(spend_rows[0]["spend"]) == 0.0, spend_rows diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_batch_attribution.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_batch_attribution.py index 1f7acd0723f..4214c7fac3f 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_batch_attribution.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_batch_attribution.py @@ -102,8 +102,16 @@ class TestRequestTagsFromMetadata: ("https://api.anthropic.com/v1/messages/batches/msgbatch_1", "/v1/messages/batches", False), # a proxied base with a path prefix still resolves, because this is a suffix match ("https://gateway.internal/anthropic/v1/messages/batches", "/v1/messages/batches", True), - ("https://aiplatform.googleapis.com/v1/projects/p/locations/l/batchPredictionJobs", "batchPredictionJobs", True), - ("https://aiplatform.googleapis.com/v1/projects/p/locations/l/batchPredictionJobs/9", "batchPredictionJobs", False), + ( + "https://aiplatform.googleapis.com/v1/projects/p/locations/l/batchPredictionJobs", + "batchPredictionJobs", + True, + ), + ( + "https://aiplatform.googleapis.com/v1/projects/p/locations/l/batchPredictionJobs/9", + "batchPredictionJobs", + False, + ), ], ) def test_is_collection_route(url_route, suffix, expected): @@ -179,3 +187,115 @@ class TestLogBatchRegistrationResult: logger.info.assert_not_called() logger.error.assert_called_once() + + +def _pending_batch(batch_id: str): + from litellm.types.utils import LiteLLMBatch + + return LiteLLMBatch( + id=batch_id, + object="batch", + status="validating", + completion_window="24h", + created_at=1762905329, + endpoint="/v1/chat/completions", + input_file_id="files/wireinput123", + ) + + +def test_unified_batch_object_id_decodes_to_the_deployment_and_provider_batch_id(): + import base64 + + from litellm.proxy.openai_files_endpoints.common_utils import ( + get_batch_id_from_unified_batch_id, + get_model_id_from_unified_batch_id, + ) + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import unified_batch_object_id + + unified = unified_batch_object_id("gemini/gemini-3.8-flash", "wire7h3x9batch") + padded = unified + "=" * (-len(unified) % 4) + decoded = base64.urlsafe_b64decode(padded).decode() + assert get_model_id_from_unified_batch_id(decoded) == "gemini/gemini-3.8-flash" + assert get_batch_id_from_unified_batch_id(decoded) == "wire7h3x9batch" + + +def test_register_passthrough_batch_stores_the_calling_key_team_and_tags(): + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( + register_passthrough_batch, + ) + + hook = MagicMock() + hook.store_unified_object_id = AsyncMock() + batch_object = _pending_batch("b1") + metadata = { + "user_api_key": "hashed-key", + "user_api_key_user_id": "user-7", + "user_api_key_team_id": "team-3", + "tags": ["pipeline:nightly"], + } + + async def run(): + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as proxy_logging: + proxy_logging.get_proxy_hook.return_value = hook + register_passthrough_batch("Gemini", "uoi", batch_object, "b1", metadata) + await asyncio.sleep(0) + await asyncio.sleep(0) + + asyncio.run(run()) + hook.store_unified_object_id.assert_awaited_once() + stored = hook.store_unified_object_id.await_args.kwargs + assert stored["unified_object_id"] == "uoi" + assert stored["model_object_id"] == "b1" + assert stored["file_object"] is batch_object + assert stored["file_purpose"] == "batch" + assert stored["persist_attribution"] is True + assert stored["request_tags"] == ("pipeline:nightly",) + assert stored["user_api_key_dict"].api_key == "hashed-key" + assert stored["user_api_key_dict"].user_id == "user-7" + assert stored["user_api_key_dict"].team_id == "team-3" + + +def test_register_passthrough_batch_leaves_the_user_unset_for_a_key_without_one(): + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( + register_passthrough_batch, + ) + + hook = MagicMock() + hook.store_unified_object_id = AsyncMock() + + async def run(): + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as proxy_logging: + proxy_logging.get_proxy_hook.return_value = hook + register_passthrough_batch("Gemini", "uoi", _pending_batch("b1"), "b1", {"user_api_key": "hashed-key"}) + await asyncio.sleep(0) + await asyncio.sleep(0) + + asyncio.run(run()) + stored = hook.store_unified_object_id.await_args.kwargs + assert stored["user_api_key_dict"].user_id is None + assert stored["user_api_key_dict"].api_key == "hashed-key" + + +def test_register_passthrough_batch_without_a_managed_files_hook_stores_nothing(): + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( + register_passthrough_batch, + ) + + async def run(): + with ( + patch("litellm.proxy.proxy_server.proxy_logging_obj") as proxy_logging, + patch( + "litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution.verbose_proxy_logger" + ) as logger, + ): + proxy_logging.get_proxy_hook.return_value = None + register_passthrough_batch("Gemini", "uoi", _pending_batch("b1"), "b1", {}) + await asyncio.sleep(0) + return logger.warning.call_args + + warned = asyncio.run(run()) + assert warned is not None and "Gemini" in warned.args and "b1" in warned.args, warned diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py index 31d95984996..bb50a33cf17 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py @@ -1,4 +1,5 @@ import asyncio +import base64 import json from collections.abc import Iterator from datetime import datetime @@ -10,6 +11,8 @@ import pytest import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.proxy.openai_files_endpoints.common_utils import get_model_id_from_unified_batch_id from litellm.proxy.pass_through_endpoints.llm_provider_handlers.gemini_passthrough_logging_handler import ( GeminiPassthroughLoggingHandler, ) @@ -145,12 +148,8 @@ class TestGeminiPassthroughLoggingHandler: assert model == "gemini-1.5-pro" @patch("litellm.completion_cost") - @patch( - "litellm.litellm_core_utils.litellm_logging.get_standard_logging_object_payload" - ) - def test_gemini_passthrough_handler_success( - self, mock_get_standard_logging, mock_completion_cost - ): + @patch("litellm.litellm_core_utils.litellm_logging.get_standard_logging_object_payload") + def test_gemini_passthrough_handler_success(self, mock_get_standard_logging, mock_completion_cost): """Test successful cost tracking for Gemini generateContent endpoint""" # Arrange mock_completion_cost.return_value = 0.000045 @@ -278,9 +277,7 @@ class TestGeminiPassthroughLoggingHandler: "litellm.proxy.pass_through_endpoints.llm_provider_handlers.gemini_passthrough_logging_handler.litellm.completion_cost", return_value=0.000050, ) - async def test_pass_through_success_handler_gemini_routing( - self, mock_completion_cost - ): + async def test_pass_through_success_handler_gemini_routing(self, mock_completion_cost): """Test that the success handler correctly routes Gemini requests to the Gemini handler""" handler = PassThroughEndpointLogging() @@ -344,11 +341,7 @@ class TestGeminiPassthroughLoggingHandler: # Request body with durationSeconds request_body = { - "instances": [ - { - "prompt": "A close up of two people staring at a cryptic drawing on a wall," - } - ], + "instances": [{"prompt": "A close up of two people staring at a cryptic drawing on a wall,"}], "parameters": {"durationSeconds": 8}, } @@ -592,3 +585,111 @@ def test_gemini_passthrough_bills_the_list_price_when_the_model_info_base_model_ recorded: Final = _cost_the_logger_records_for_gemini(stream=stream, model_info={"base_model": base_model}) assert recorded == pytest.approx(7 * _LIST_INPUT_RATE + 1 * _LIST_OUTPUT_RATE), stream + + +_BATCH_OPERATION: Final = { + "name": "batches/wire7h3x9batch", + "metadata": { + "@type": "type.googleapis.com/google.ai.generativelanguage.v1main.GenerateContentBatch", + "model": "models/gemini-3.8-flash", + "inputConfig": {"fileName": "files/wireinput123"}, + "createTime": "2026-11-11T23:15:29.000000Z", + "updateTime": "2026-11-11T23:15:29.000000Z", + "state": "BATCH_STATE_PENDING", + }, +} +_BATCH_ROUTE: Final = "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:batchGenerateContent" + + +def _decoded(unified_object_id: str) -> str: + return base64.urlsafe_b64decode(unified_object_id + "=" * (-len(unified_object_id) % 4)).decode() + + +def _gemini_router(model: str) -> litellm.Router: + return litellm.Router( + model_list=[ + {"model_name": model, "litellm_params": {"model": f"vertex_ai/{model}", "vertex_project": "p"}}, + { + "model_name": model, + "litellm_params": {"model": f"gemini/{model}", "api_key": "scripted"}, + "model_info": {"id": "gemini-deployment-1"}, + }, + ] + ) + + +def _batch_create_through_passthrough( + status_code: int, body: dict[str, object], router: litellm.Router | None +) -> tuple[PassThroughEndpointLoggingTypedDict, MagicMock]: + hook = MagicMock() + hook.store_unified_object_id = AsyncMock() + logging_obj = MagicMock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {} + kwargs = {"litellm_params": {"metadata": {"user_api_key": "hashed-key", "user_api_key_team_id": "team-3"}}} + response = httpx.Response(status_code, json=body, request=httpx.Request("POST", _BATCH_ROUTE)) + + async def run(): + with ( + patch("litellm.proxy.proxy_server.proxy_logging_obj") as proxy_logging, + patch("litellm.proxy.proxy_server.llm_router", router), + ): + proxy_logging.get_proxy_hook.return_value = hook + outcome = GeminiPassthroughLoggingHandler.gemini_passthrough_handler( + httpx_response=response, + response_body=body, + logging_obj=logging_obj, + url_route=_BATCH_ROUTE, + result="", + start_time=datetime(2026, 11, 11, 23, 15, 29), + end_time=datetime(2026, 11, 11, 23, 15, 30), + cache_hit=False, + request_body={"batch": {"inputConfig": {"fileName": "files/wireinput123"}}}, + **kwargs, + ) + await asyncio.sleep(0) + await asyncio.sleep(0) + return outcome + + return asyncio.run(run()), hook + + +def test_batch_generate_content_is_a_tracked_gemini_route(): + assert PassThroughEndpointLogging().is_gemini_route(_BATCH_ROUTE, "gemini") is True + + +def test_a_created_gemini_batch_is_registered_against_the_gemini_deployment_at_zero_cost(): + outcome, hook = _batch_create_through_passthrough(200, _BATCH_OPERATION, _gemini_router("gemini-3.8-flash")) + + stored = hook.store_unified_object_id.await_args.kwargs + assert stored["model_object_id"] == "wire7h3x9batch" + assert get_model_id_from_unified_batch_id(_decoded(stored["unified_object_id"])) == "gemini-deployment-1" + assert stored["file_purpose"] == "batch" + assert stored["file_object"].status == "validating" + assert stored["user_api_key_dict"].api_key == "hashed-key" + assert stored["user_api_key_dict"].team_id == "team-3" + assert stored["unified_object_id"] == outcome["kwargs"]["unified_object_id"] + assert outcome["kwargs"]["model"] == "gemini-3.8-flash" + assert outcome["kwargs"]["custom_llm_provider"] == "gemini" + assert outcome["kwargs"]["batch_id"] == "wire7h3x9batch" + assert outcome["kwargs"]["response_cost"] == 0.0 + assert outcome["result"].object == "batch" + + +def test_a_created_gemini_batch_without_a_gemini_deployment_registers_nothing_but_names_the_model(): + outcome, hook = _batch_create_through_passthrough(200, _BATCH_OPERATION, _gemini_router("gemini-other")) + + hook.store_unified_object_id.assert_not_awaited() + assert outcome["result"] is None + assert outcome["kwargs"]["model"] == "gemini-3.8-flash" + assert outcome["kwargs"]["batch_id"] == "wire7h3x9batch" + + +def test_a_rejected_gemini_batch_create_registers_nothing(): + outcome, hook = _batch_create_through_passthrough( + 400, {"error": {"code": 400, "message": "bad file"}}, _gemini_router("gemini-3.8-flash") + ) + + hook.store_unified_object_id.assert_not_awaited() + assert outcome["result"] is None + assert outcome["kwargs"]["model"] == "gemini-3.8-flash" + assert "batch_id" not in outcome["kwargs"] From 23354ad08fdbd398aaecd4bb2680f049729efc46 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 11 Nov 2026 17:59:21 -0800 Subject: [PATCH 149/189] ci(unit): deselect test_unit_test_proxy_hooks from the q-z shard it actually matches, keep the dead proxy-infra-root codecov flag pinned (#51324) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-unit.yml | 2 +- codecov.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f36c232791a..914d984e541 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -697,7 +697,6 @@ jobs: --deselect=tests/unit/proxy/test_proxy_setting_guardrails.py --deselect=tests/unit/proxy/test_proxy_token_counter.py --deselect=tests/unit/proxy/test_proxy_utils.py - --deselect=tests/unit/proxy/test_unit_test_proxy_hooks.py python-version: "3.12" workers: 4 reruns: 0 @@ -712,6 +711,7 @@ jobs: --deselect=tests/unit/proxy/test_response_polling_pre_call_checks.py --deselect=tests/unit/proxy/test_server_root_path.py --deselect=tests/unit/proxy/test_ui_path_detection.py + --deselect=tests/unit/proxy/test_unit_test_proxy_hooks.py --deselect=tests/unit/proxy/test_update_spend.py --deselect=tests/unit/proxy/test_zero_cost_model_budget_bypass.py python-version: "3.12" diff --git a/codecov.yaml b/codecov.yaml index d671d883120..2fa8ea04f0e 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -58,6 +58,8 @@ flag_management: carryforward: false - name: proxy-infra carryforward: false + - name: proxy-infra-root + carryforward: false - name: proxy-infra-root-a-p carryforward: false - name: proxy-infra-root-q-z From 6b98361765895948ea6007de48071cccf4f22f2a Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:04:20 +0000 Subject: [PATCH 150/189] test(integration): remove the openapi docs gate detached worktree test that cannot run offline (#51300) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../configuration/test_openapi_docs_gate.py | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 tests/integration/configuration/test_openapi_docs_gate.py diff --git a/tests/integration/configuration/test_openapi_docs_gate.py b/tests/integration/configuration/test_openapi_docs_gate.py deleted file mode 100644 index 1afa450c67a..00000000000 --- a/tests/integration/configuration/test_openapi_docs_gate.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Final - -import openapi_docs_gate as gate -import pytest - - -def test_base_counts_imports_native_extension_from_detached_worktree(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("UV_OFFLINE", "1") - expected_counts: Final = dict(gate.count_by_rule(gate.head_violations())) - assert expected_counts - assert dict(gate.base_counts("HEAD")) == expected_counts From e7fd0d6cbaa02930a3973b290e73d1fd417ca0a8 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:12:28 +0000 Subject: [PATCH 151/189] feat(gemini): add gemini-decision-1, the Decisions API model priced at $0.10 per 1M input tokens (#51215) * feat(gemini): add gemini-decision-1, the Decisions API model priced at $0.10 per 1M input tokens * fix(cost-map): give gemini-decision-1 its Tier 1 rpm and tpm so test_get_model_info_gemini passes --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 33 +++++++++++++++++++ model_prices_and_context_window.json | 33 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1472d10c12e..f76ec44b0fe 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -100792,5 +100792,38 @@ "metadata": {"comment":"5 credits per 1,000 characters @ $0.01 per credit = $0.00005 per character; 1 credit minimum not modeled"}, "mode": "audio_speech", "source": "https://docs.dev.runwayml.com/guides/pricing/" + }, + "gemini/gemini-decision-1": { + "input_cost_per_token": 1e-07, + "litellm_provider": "gemini", + "max_input_tokens": 262144, + "mode": "decisions", + "output_cost_per_token": 0.0, + "rpm": 1500, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/decisions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_computer_use": false, + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_tool_choice": false, + "supports_url_context": false, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": false, + "tpm": 1000000 } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1472d10c12e..f76ec44b0fe 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -100792,5 +100792,38 @@ "metadata": {"comment":"5 credits per 1,000 characters @ $0.01 per credit = $0.00005 per character; 1 credit minimum not modeled"}, "mode": "audio_speech", "source": "https://docs.dev.runwayml.com/guides/pricing/" + }, + "gemini/gemini-decision-1": { + "input_cost_per_token": 1e-07, + "litellm_provider": "gemini", + "max_input_tokens": 262144, + "mode": "decisions", + "output_cost_per_token": 0.0, + "rpm": 1500, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/decisions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_computer_use": false, + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_tool_choice": false, + "supports_url_context": false, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": false, + "tpm": 1000000 } } From d40dedf23bb88aa7911bd29e2be3aea19f1ae669 Mon Sep 17 00:00:00 2001 From: jesus-berri Date: Wed, 11 Nov 2026 18:22:02 -0800 Subject: [PATCH 152/189] feat(proxy): run spend log cleanup from a DB-backed scheduled job table (#50131) * feat(proxy): run spend log cleanup from a DB-backed scheduled job table Spend log retention ran from an in-process APScheduler timer that reset on every restart and needed Redis for leader election, so pods that restart more often than the interval never cleaned up at all. Add LiteLLM_ScheduledJobRun, a generic table with one row per job run. Every pod polls it, plans the next run idempotently and claims a due run under a transaction-scoped advisory lock, so exactly one pod runs each occurrence. Runs carry a lease, a status and a log. Intervals count from the last finished run and survive restarts. Schedule edits cancel and replace the pending run. The retention and size-check jobs share a lock group so they never overlap, and the Redis pod lock is gone. * fix(proxy): plan an earlier cron edit from now, and cover the job runner Moving the cleanup cron earlier queued tomorrow's run because the next occurrence was planned after the pending run being cancelled. Plan from finished runs only. Register LiteLLM_ScheduledJobRun as a span table, keep the advisory lock query inside the tick's span, move the non-string cron startup test to the schedule parser, and add unit tests for the runner and job wiring. * test(proxy): annotate new spend log cleanup test locals as Final * test(proxy): type the parameters of the new spend log cleanup tests * test(proxy): leave an hour of margin when moving a cron earlier so an hour boundary cannot flake it * fix(proxy): reject a cron that never fires instead of failing every tick A crontab like 0 0 31 2 * parses but has no next fire time, so next_run_at raised on every tick and the job was never scheduled again. cron_schedule now returns InvalidSchedule for it, which logs and keeps the pending run like any other bad config * refactor(proxy): drop the informational log column from LiteLLM_ScheduledJobRun Nothing read the column back. Run outcomes stay in status, and the reasons already go to the proxy logs, so JobResult becomes a SUCCEEDED or FAILED literal and the cleanup returns its RunOutcome instead of a summary report --- .../migration.sql | 23 + .../litellm_proxy_extras/schema.prisma | 25 + litellm/constants.py | 4 + litellm/integrations/otel/model/spans.py | 1 + .../db_transaction_queue/spend_log_cleanup.py | 133 +++-- litellm/proxy/db/scheduled_job_runs.py | 454 ++++++++++++++++++ litellm/proxy/proxy_server.py | 189 ++------ litellm/proxy/schema.prisma | 25 + schema.prisma | 25 + .../database/test_cleanup_transactions.py | 1 - .../database/test_scheduled_job_runs.py | 294 ++++++++++++ .../unit/proxy/db/test_scheduled_job_runs.py | 355 ++++++++++++++ .../proxy/proxy_server/test_proxy_config.py | 188 +------- ...test_proxy_server_endpoints_and_startup.py | 75 +-- tests/unit/proxy/test_spend_log_cleanup.py | 195 +++----- 15 files changed, 1417 insertions(+), 570 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_scheduled_job_runs/migration.sql create mode 100644 litellm/proxy/db/scheduled_job_runs.py create mode 100644 tests/integration/database/test_scheduled_job_runs.py create mode 100644 tests/unit/proxy/db/test_scheduled_job_runs.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_scheduled_job_runs/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_scheduled_job_runs/migration.sql new file mode 100644 index 00000000000..4e9b168921c --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_scheduled_job_runs/migration.sql @@ -0,0 +1,23 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TYPE "ScheduledJobRunStatus" AS ENUM ('PENDING', 'RUNNING', 'SUCCEEDED', 'FAILED', 'CANCELLED'); + +CREATE TABLE IF NOT EXISTS "LiteLLM_ScheduledJobRun" ( + "id" TEXT NOT NULL, + "job_name" TEXT NOT NULL, + "lock_group" TEXT NOT NULL, + "schedule" TEXT NOT NULL, + "scheduled_for" TIMESTAMPTZ(3) NOT NULL, + "status" "ScheduledJobRunStatus" NOT NULL DEFAULT 'PENDING', + "claimed_by" TEXT, + "claimed_at" TIMESTAMPTZ(3), + "lease_expires_at" TIMESTAMPTZ(3), + "finished_at" TIMESTAMPTZ(3), + "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_ScheduledJobRun_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_ScheduledJobRun_job_name_status_idx" ON "LiteLLM_ScheduledJobRun"("job_name", "status"); + +CREATE INDEX IF NOT EXISTS "LiteLLM_ScheduledJobRun_lock_group_status_idx" ON "LiteLLM_ScheduledJobRun"("lock_group", "status"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 5a5d56cc6e2..4314a23e011 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1155,6 +1155,31 @@ enum JobStatus { INACTIVE } +model LiteLLM_ScheduledJobRun { + id String @id @default(uuid()) + job_name String + lock_group String + schedule String + scheduled_for DateTime @client.Timestamptz(3) + status ScheduledJobRunStatus @default(PENDING) + claimed_by String? + claimed_at DateTime? @client.Timestamptz(3) + lease_expires_at DateTime? @client.Timestamptz(3) + finished_at DateTime? @client.Timestamptz(3) + created_at DateTime @default(now()) @client.Timestamptz(3) + + @@index([job_name, status]) + @@index([lock_group, status]) +} + +enum ScheduledJobRunStatus { + PENDING + RUNNING + SUCCEEDED + FAILED + CANCELLED +} + model LiteLLM_ManagedFileTable { id String @id @default(uuid()) unified_file_id String @unique // The base64 encoded unified file ID diff --git a/litellm/constants.py b/litellm/constants.py index 966a5b71b8c..066b682362d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1931,6 +1931,10 @@ SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = get_env_float("SPEND_LOG_CLEANU SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = get_env_int("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", 100000) SPEND_LOG_SIZE_CHECK_JOB_NAME: Final = "spend_log_size_check" SPEND_LOG_SIZE_CHECK_INTERVAL: Final = os.getenv("SPEND_LOG_SIZE_CHECK_INTERVAL", "1d") +SCHEDULED_JOB_POLL_INTERVAL_SECONDS: Final = get_env_int("SCHEDULED_JOB_POLL_INTERVAL_SECONDS", 30) +SCHEDULED_JOB_TICK_TIMEOUT_SECONDS: Final = get_env_float("SCHEDULED_JOB_TICK_TIMEOUT_SECONDS", 10.0) +SCHEDULED_JOB_LEASE_GRACE_SECONDS: Final = get_env_float("SCHEDULED_JOB_LEASE_GRACE_SECONDS", 300.0) +SCHEDULED_JOB_RUN_HISTORY_LIMIT: Final = get_env_int("SCHEDULED_JOB_RUN_HISTORY_LIMIT", 1000) SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES: Final = get_env_int("SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES", 10) SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = get_env_float( "SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", 5.0 diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 8f9e2c8c6db..ad275f64096 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -328,6 +328,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_PromptTable", "LiteLLM_HealthCheckTable", "LiteLLM_SearchToolsTable", + "LiteLLM_ScheduledJobRun", "LiteLLM_SSOConfig", "LiteLLM_ManagedVectorStoreIndexTable", "LiteLLM_CacheConfig", diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index 29f727ea41b..7ca3075732f 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -2,7 +2,7 @@ import asyncio import math import re import time -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from contextvars import ContextVar from dataclasses import dataclass from datetime import datetime, timedelta, timezone @@ -11,8 +11,8 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, TypeAdapter from litellm._logging import verbose_proxy_logger -from litellm.caching import RedisCache from litellm.constants import ( + SCHEDULED_JOB_LEASE_GRACE_SECONDS, SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS, SPEND_LOG_CLEANUP_BATCH_SIZE, SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS, @@ -21,6 +21,8 @@ from litellm.constants import ( SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP, SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS, SPEND_LOG_RUN_LOOPS, + SPEND_LOG_SIZE_CHECK_INTERVAL, + SPEND_LOG_SIZE_CHECK_JOB_NAME, SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES, ) from litellm.litellm_core_utils.duration_parser import duration_in_seconds @@ -35,6 +37,14 @@ from litellm.proxy.db.db_transaction_queue.spend_logs_partition_manager import ( bounded_tx, trusted_sql, ) +from litellm.proxy.db.scheduled_job_runs import ( + InvalidSchedule, + JobResult, + Schedule, + ScheduledJob, + cron_schedule, + interval_schedule, +) from litellm.proxy.utils import PrismaClient from litellm.types.llms.base import LiteLLMBaseModel @@ -395,7 +405,8 @@ class SpendLogCleanup: When LiteLLM_SpendLogs is range-partitioned, expired data is reclaimed by dropping whole partitions (instant, frees disk immediately). Otherwise it falls back to deleting logs in batches. - Uses PodLockManager to ensure only one pod runs cleanup in multi-pod deployments. + One pod runs it at a time, through the LiteLLM_ScheduledJobRun claim of SPEND_LOG_RETENTION_JOB and + SPEND_LOG_SIZE_CHECK_JOB, which share a lock group so the two never overlap either. Every run is bounded so it can never monopolise the database: a wall-clock budget shared across all tables, a per-table batch cap, and a Postgres @@ -419,7 +430,6 @@ class SpendLogCleanup: def __init__( self, general_settings=None, - redis_cache: RedisCache | None = None, partition_manager: SpendLogsPartitionManager | None = None, *, clock: Callable[[], float] = time.monotonic, @@ -431,10 +441,6 @@ class SpendLogCleanup: self.general_settings = general_settings or default_settings self._refresh_bounds() - from litellm.proxy.proxy_server import proxy_logging_obj - - pod_lock_manager: Final = proxy_logging_obj.db_spend_update_writer.pod_lock_manager - self.pod_lock_manager = pod_lock_manager verbose_proxy_logger.info( "SpendLogCleanup initialized: batch_size=%s max_batches=%s run_budget=%ss batch_timeout=%ss", self.batch_size, @@ -1420,31 +1426,26 @@ class SpendLogCleanup: return verbose_proxy_logger.warning(message, outcome, elapsed_seconds, per_table, storage) - async def cleanup_old_spend_logs(self, prisma_client: PrismaClient) -> None: - """ - The scheduled retention run: every age rule, plus the size rule when one cron drives both. - If pod_lock_manager is available, ensures only one pod runs cleanup. - If no pod_lock_manager, runs cleanup without distributed locking. - """ - await self._run(prisma_client, age_rules=True, size_rule=self._size_pass_runs_with_age_pass()) + async def cleanup_old_spend_logs(self, prisma_client: PrismaClient) -> RunOutcome: + """The scheduled retention run: every age rule, plus the size rule when one cron drives both.""" + return await self._run(prisma_client, age_rules=True, size_rule=self._size_pass_runs_with_age_pass()) - async def cleanup_spend_logs_by_size(self, prisma_client: PrismaClient) -> None: + async def cleanup_spend_logs_by_size(self, prisma_client: PrismaClient) -> RunOutcome: """The size-check run scheduled on maximum_spend_logs_size_check_interval: the size rule only.""" - await self._run(prisma_client, age_rules=False, size_rule=True) + return await self._run(prisma_client, age_rules=False, size_rule=True) - async def _run(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> None: + async def _run(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> RunOutcome: """ - One pass at a time per pod. The age job and the size job fire from separate schedules, and the pod - lock only exists with Redis, so this is what keeps them apart in a pod without it. The later one waits - rather than skips, because the other job applies a different rule and would not do its work for it. + One pass at a time per pod, so a pass started outside the job claim (a manual call) cannot overlap a + scheduled one in the same pod. The later one waits rather than skips, because the other job applies a + different rule and would not do its work for it. """ if _PASS_LOCK.locked(): verbose_proxy_logger.info("Another cleanup pass is running in this pod; waiting for it to finish") async with _PASS_LOCK: - await self._run_passes(prisma_client, age_rules, size_rule) + return await self._run_passes(prisma_client, age_rules, size_rule) - async def _run_passes(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> None: - lock_acquired = False + async def _run_passes(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> RunOutcome: run_started_at: Final = self._clock() progress: Final = _RunProgress() progress_token: Final = _run_progress.set(progress) @@ -1471,29 +1472,12 @@ class SpendLogCleanup: and active_size_rule is None ): SpendLogCleanupMetrics.record_run("skipped_disabled") - return + return "skipped_disabled" if delete_spend_logs and self.retention_seconds is None: verbose_proxy_logger.error("Retention seconds is None, cannot proceed with cleanup") SpendLogCleanupMetrics.record_run("skipped_disabled") - return - - # If we have a pod lock manager, try to acquire the lock - if self.pod_lock_manager and self.pod_lock_manager.redis_cache: - lock_acquired = ( - await self.pod_lock_manager.acquire_lock( - cronjob_id=SPEND_LOG_CLEANUP_JOB_NAME, - ) - or False - ) - verbose_proxy_logger.info( - "Lock acquisition attempt: %s at %s", "successful" if lock_acquired else "failed", datetime.now() - ) - - if not lock_acquired: - verbose_proxy_logger.info("Another pod is already running cleanup") - SpendLogCleanupMetrics.record_run("skipped_locked") - return + return "skipped_disabled" deadline: Final = self._clock() + self.run_budget_seconds size_group: Final = int(active_size_rule is not None) @@ -1583,10 +1567,63 @@ class SpendLogCleanup: e, ) SpendLogCleanupMetrics.record_run("aborted") - return # Return after error handling + return "aborted" + else: + return outcome finally: _run_progress.reset(progress_token) - # Only release the lock if it was actually acquired - if lock_acquired and self.pod_lock_manager and self.pod_lock_manager.redis_cache: - await self.pod_lock_manager.release_lock(cronjob_id=SPEND_LOG_CLEANUP_JOB_NAME) - verbose_proxy_logger.info("Released cleanup lock") + + +def retention_schedule(settings: Mapping[str, object]) -> Schedule | InvalidSchedule: + cron: Final = settings.get("maximum_spend_logs_cleanup_cron") + if cron: + return cron_schedule("maximum_spend_logs_cleanup_cron", cron) + return interval_schedule( + "maximum_spend_logs_retention_interval", settings.get("maximum_spend_logs_retention_interval", "1d") + ) + + +def size_check_schedule(settings: Mapping[str, object]) -> Schedule | InvalidSchedule: + return interval_schedule( + "maximum_spend_logs_size_check_interval", + settings.get("maximum_spend_logs_size_check_interval", SPEND_LOG_SIZE_CHECK_INTERVAL), + ) + + +def _live_settings() -> Mapping[str, object]: + from litellm.proxy.proxy_server import general_settings + + return general_settings # pyright: ignore[reportUnknownVariableType] # general_settings is an untyped module dict + + +def _cleanup_lease() -> timedelta: + run_budget: Final = SpendLogCleanup(general_settings=_live_settings()).run_budget_seconds + return timedelta(seconds=run_budget + SCHEDULED_JOB_LEASE_GRACE_SECONDS) + + +def _job_result(outcome: RunOutcome) -> JobResult: + return "FAILED" if outcome == "aborted" else "SUCCEEDED" + + +async def _run_retention(prisma_client: PrismaClient) -> JobResult: + return _job_result(await SpendLogCleanup().cleanup_old_spend_logs(prisma_client)) + + +async def _run_size_check(prisma_client: PrismaClient) -> JobResult: + return _job_result(await SpendLogCleanup().cleanup_spend_logs_by_size(prisma_client)) + + +SPEND_LOG_RETENTION_JOB: Final = ScheduledJob( + name=SPEND_LOG_CLEANUP_JOB_NAME, + lock_group=SPEND_LOG_CLEANUP_JOB_NAME, + schedule=lambda: retention_schedule(_live_settings()), + lease=_cleanup_lease, + run=_run_retention, +) +SPEND_LOG_SIZE_CHECK_JOB: Final = ScheduledJob( + name=SPEND_LOG_SIZE_CHECK_JOB_NAME, + lock_group=SPEND_LOG_CLEANUP_JOB_NAME, + schedule=lambda: size_check_schedule(_live_settings()), + lease=_cleanup_lease, + run=_run_size_check, +) diff --git a/litellm/proxy/db/scheduled_job_runs.py b/litellm/proxy/db/scheduled_job_runs.py new file mode 100644 index 00000000000..f38ab375345 --- /dev/null +++ b/litellm/proxy/db/scheduled_job_runs.py @@ -0,0 +1,454 @@ +import asyncio +import os +import socket +import uuid +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from functools import lru_cache +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias + +from apscheduler.triggers import cron # pyright: ignore[reportMissingTypeStubs, reportUnknownVariableType] # untyped +from pydantic import AwareDatetime, BaseModel, ConfigDict, TypeAdapter + +from litellm._logging import verbose_proxy_logger +from litellm.constants import ( + SCHEDULED_JOB_RUN_HISTORY_LIMIT, + SCHEDULED_JOB_TICK_TIMEOUT_SECONDS, +) +from litellm.litellm_core_utils.duration_parser import duration_in_seconds +from litellm.proxy.db.db_span import db_span +from litellm.proxy.db.db_transaction_queue.spend_logs_partition_manager import bounded_tx, trusted_sql + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +_TABLE: Final = "LiteLLM_ScheduledJobRun" +_FIRE_TIME: Final = TypeAdapter[datetime](AwareDatetime) + + +@dataclass(frozen=True, slots=True) +class IntervalSchedule: + seconds: float + + @property + def spec(self) -> str: + return f"interval:{self.seconds:g}s" + + +class FireTimes(Protocol): + def get_next_fire_time(self, previous_fire_time: datetime | None, now: datetime) -> datetime | None: ... + + +class JobScheduler(Protocol): + """The slice of APScheduler's AsyncIOScheduler that registers a poll, which ships no type information.""" + + def get_job(self, job_id: str) -> object | None: ... + + def remove_job(self, job_id: str) -> None: ... + + def add_job( + self, + func: Callable[["PrismaClient | None"], Awaitable[None]], + trigger: str, + *, + args: Sequence[object], + id: str, + replace_existing: bool, + max_instances: int, + misfire_grace_time: int, + seconds: float, + ) -> object: ... + + +@dataclass(frozen=True, slots=True) +class CronSchedule: + expression: str + trigger: FireTimes + + @property + def spec(self) -> str: + return f"cron:{self.expression}" + + +@dataclass(frozen=True, slots=True) +class InvalidSchedule: + reason: str + + +Schedule: TypeAlias = IntervalSchedule | CronSchedule + + +def interval_schedule(setting_name: str, value: object) -> Schedule | InvalidSchedule: + if not isinstance(value, str): + return InvalidSchedule(f"{setting_name} must be a duration string like '1d', got {value!r}") + try: + seconds: Final = duration_in_seconds(value) + timedelta(seconds=seconds) + except (ValueError, OverflowError): + return InvalidSchedule(f"{setting_name} is not a valid duration: {value!r}") + if seconds <= 0: + return InvalidSchedule(f"{setting_name} must be positive, got {value!r}") + return IntervalSchedule(seconds=seconds) + + +def _cron_trigger(expression: str) -> FireTimes: + return cron.CronTrigger.from_crontab(expression, timezone=timezone.utc) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped + + +def cron_schedule(setting_name: str, value: object) -> Schedule | InvalidSchedule: + if not isinstance(value, str): + return InvalidSchedule(f"{setting_name} must be a crontab string, got {value!r}") + try: + trigger: Final = _cron_trigger(value) + except (ValueError, TypeError): + return InvalidSchedule(f"{setting_name} is not a valid crontab expression: {value!r}") + if trigger.get_next_fire_time(None, datetime.now(timezone.utc)) is None: + return InvalidSchedule(f"{setting_name} never fires: {value!r}") + return CronSchedule(expression=value, trigger=trigger) + + +JobResult: TypeAlias = Literal["SUCCEEDED", "FAILED"] + + +@dataclass(frozen=True, slots=True) +class ScheduledJob: + """ + A job that runs on exactly one pod per occurrence. + + Jobs sharing a lock_group never run at the same time, on any pod. schedule and lease are read on every tick, + so a settings change applies without re-registering the job. + """ + + name: str + lock_group: str + schedule: Callable[[], Schedule | InvalidSchedule] + lease: Callable[[], timedelta] + run: Callable[["PrismaClient"], Awaitable[JobResult]] + + +class PendingRun(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str + schedule: str + scheduled_for: AwareDatetime + + +class JobState(BaseModel): + model_config = ConfigDict(frozen=True) + + now: AwareDatetime + group_running: bool + job_running: bool + last_finished_at: AwareDatetime | None + last_scheduled_for: AwareDatetime | None + + +@dataclass(frozen=True, slots=True) +class NewRun: + id: str + schedule: str + scheduled_for: datetime + + +@dataclass(frozen=True, slots=True) +class TickPlan: + cancel: tuple[str, ...] + insert: NewRun | None + claim: str | None + + +def next_run_at(schedule: Schedule, state: JobState) -> datetime: + """ + When the next occurrence is due. + + An interval counts from the last finished run, so a restart never resets it and a missed run happens once, right + away. A cron fires at its next time after both now and the last occurrence, so an outage collapses into the one + occurrence that was already pending. + """ + match schedule: + case IntervalSchedule(seconds=seconds): + if state.last_finished_at is None: + return state.now + return state.last_finished_at + timedelta(seconds=seconds) + case CronSchedule(trigger=trigger): + anchor: Final = max(state.now, state.last_scheduled_for or state.now) + return _FIRE_TIME.validate_python(trigger.get_next_fire_time(None, anchor + timedelta(microseconds=1))) + + +def plan_tick( + schedule: Schedule | InvalidSchedule, state: JobState, pending: tuple[PendingRun, ...], new_id: str +) -> TickPlan: + """ + Decide one tick from the job's rows. Pure, so every pod reaches the same decision from the same rows. + + A pending row planned from a different schedule is cancelled and replaced, so a settings change applies at once. + With an invalid schedule nothing is replanned, and a pending row from the last valid one still runs. + """ + if isinstance(schedule, InvalidSchedule): + return TickPlan(cancel=(), insert=None, claim=_claimable(pending[:1], state)) + current: Final = tuple(run for run in pending if run.schedule == schedule.spec) + keep: Final = current[:1] + cancel: Final = tuple(run.id for run in pending if run not in keep) + if keep or state.job_running: + return TickPlan(cancel=cancel, insert=None, claim=_claimable(keep, state)) + new_run: Final = NewRun(id=new_id, schedule=schedule.spec, scheduled_for=next_run_at(schedule, state)) + claim: Final = new_run.id if new_run.scheduled_for <= state.now and not state.group_running else None + return TickPlan(cancel=cancel, insert=new_run, claim=claim) + + +def _claimable(pending: tuple[PendingRun, ...], state: JobState) -> str | None: + if not pending or state.group_running or pending[0].scheduled_for > state.now: + return None + return pending[0].id + + +def default_worker_id() -> str: + return f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}" + + +class _Claimed(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str + job_name: str + + +_STATE: Final = TypeAdapter(list[JobState]) +_PENDING: Final = TypeAdapter(list[PendingRun]) +_CLAIMED: Final = TypeAdapter(list[_Claimed]) + +_REAP_SQL: Final = f""" + UPDATE "{_TABLE}" + SET "status" = 'FAILED', "finished_at" = now() + WHERE "lock_group" = $1 AND "status" = 'RUNNING' AND "lease_expires_at" < now() + RETURNING "id", "job_name" +""" +_STATE_SQL: Final = f""" + SELECT now() AS "now", + EXISTS (SELECT 1 FROM "{_TABLE}" WHERE "lock_group" = $2 AND "status" = 'RUNNING') AS "group_running", + EXISTS (SELECT 1 FROM "{_TABLE}" WHERE "job_name" = $1 AND "status" = 'RUNNING') AS "job_running", + (SELECT max("finished_at") FROM "{_TABLE}" + WHERE "job_name" = $1 AND "status" IN ('SUCCEEDED', 'FAILED')) AS "last_finished_at", + (SELECT max("scheduled_for") FROM "{_TABLE}" + WHERE "job_name" = $1 AND "status" NOT IN ('CANCELLED', 'PENDING')) AS "last_scheduled_for" +""" +_PENDING_SQL: Final = f""" + SELECT "id", "schedule", "scheduled_for" FROM "{_TABLE}" + WHERE "job_name" = $1 AND "status" = 'PENDING' + ORDER BY "scheduled_for", "created_at" +""" +_CANCEL_SQL: Final = f""" + UPDATE "{_TABLE}" SET "status" = 'CANCELLED', "finished_at" = now() + WHERE "id" = $1 AND "status" = 'PENDING' +""" +_INSERT_SQL: Final = f""" + INSERT INTO "{_TABLE}" ("id", "job_name", "lock_group", "schedule", "scheduled_for") + VALUES ($1, $2, $3, $4, $5::timestamptz) +""" +_CLAIM_SQL: Final = f""" + UPDATE "{_TABLE}" + SET "status" = 'RUNNING', "claimed_by" = $2, "claimed_at" = now(), + "lease_expires_at" = now() + $3::double precision * interval '1 second' + WHERE "id" = $1 AND "status" = 'PENDING' + RETURNING "id", "job_name" +""" +_FINISH_SQL: Final = f""" + UPDATE "{_TABLE}" SET "status" = $3::"ScheduledJobRunStatus", "finished_at" = now() + WHERE "id" = $1 AND "claimed_by" = $2 AND "status" = 'RUNNING' +""" +_PRUNE_SQL: Final = f""" + DELETE FROM "{_TABLE}" WHERE "id" IN ( + SELECT "id" FROM "{_TABLE}" WHERE "job_name" = $1 AND "finished_at" IS NOT NULL + ORDER BY "finished_at" DESC OFFSET $2 + ) +""" + + +class ScheduledJobRunner: + """ + Polls one ScheduledJob's rows in LiteLLM_ScheduledJobRun. Every pod runs the same tick. + + Each tick runs in one transaction under a transaction-scoped advisory lock on the job's lock_group, so only one + pod plans and claims for a group at a time and a pod that dies mid-tick releases it. The claimed run executes + outside any transaction under a lease. A pod that dies mid-run leaves its row RUNNING until the lease expires, + when the next tick on any pod marks it FAILED and plans the next occurrence. + """ + + def __init__( + self, + job: ScheduledJob, + *, + worker_id: str | None = None, + new_id: Callable[[], str] = lambda: str(uuid.uuid4()), + ) -> None: + self.job = job + self.worker_id = worker_id or default_worker_id() + self._new_id = new_id + self._running = False + + async def tick(self, prisma_client: "PrismaClient | None") -> None: + """ + Plan, claim and run whatever is due. Awaits the run, so graceful shutdown sees it as an in-flight job. + + Register it with max_instances=2: the extra instance lets the polls that land during a long run return at + once instead of APScheduler warning that it skipped them. + """ + if prisma_client is None or self._running: + return + self._running = True + try: + claimed: Final = await self._safe_plan_and_claim(prisma_client) + if claimed is not None: + await self._execute(prisma_client, claimed) + finally: + self._running = False + + async def _safe_plan_and_claim(self, prisma_client: "PrismaClient") -> str | None: + try: + return await self._plan_and_claim(prisma_client) + except Exception as e: # noqa: BLE001 - a failed tick is retried by the next one + verbose_proxy_logger.warning("Scheduled job %s tick failed: %s: %s", self.job.name, type(e).__name__, e) + return None + + async def _plan_and_claim(self, prisma_client: "PrismaClient") -> str | None: + schedule: Final = self.job.schedule() + if isinstance(schedule, InvalidSchedule): + _log_invalid_schedule(self.job.name, schedule.reason) + timeout_ms: Final = int(SCHEDULED_JOB_TICK_TIMEOUT_SECONDS * 1000) + async with db_span("scheduled_job_tick", _TABLE), bounded_tx(prisma_client, timeout_ms) as tx: + await tx.execute_raw(trusted_sql(f"SET LOCAL statement_timeout = {timeout_ms}")) + await tx.execute_raw(trusted_sql(f"SET LOCAL lock_timeout = {timeout_ms}")) + locked: Final = _LOCKED.validate_python( + await tx.query_raw(trusted_sql(_GROUP_LOCK_SQL), f"litellm_scheduled_job:{self.job.lock_group}") + ) + if not locked or not locked[0].locked: + return None + for reaped in _CLAIMED.validate_python(await tx.query_raw(trusted_sql(_REAP_SQL), self.job.lock_group)): + verbose_proxy_logger.warning( + "Scheduled job %s run %s lost its lease; marked FAILED", reaped.job_name, reaped.id + ) + state: Final = _STATE.validate_python( + await tx.query_raw(trusted_sql(_STATE_SQL), self.job.name, self.job.lock_group) + )[0] + pending: Final = tuple( + _PENDING.validate_python(await tx.query_raw(trusted_sql(_PENDING_SQL), self.job.name)) + ) + plan: Final = plan_tick(schedule, state, pending, self._new_id()) + for run_id in plan.cancel: + await tx.execute_raw(trusted_sql(_CANCEL_SQL), run_id) + if plan.insert is not None: + await tx.execute_raw( + trusted_sql(_INSERT_SQL), + plan.insert.id, + self.job.name, + self.job.lock_group, + plan.insert.schedule, + plan.insert.scheduled_for.isoformat(), + ) + if plan.claim is None: + return None + claimed: Final = _CLAIMED.validate_python( + await tx.query_raw( + trusted_sql(_CLAIM_SQL), plan.claim, self.worker_id, self.job.lease().total_seconds() + ) + ) + return claimed[0].id if claimed else None + + async def _execute(self, prisma_client: "PrismaClient", run_id: str) -> None: + verbose_proxy_logger.info("Scheduled job %s run %s claimed by %s", self.job.name, run_id, self.worker_id) + try: + result: Final = await self._run_job(prisma_client) + except asyncio.CancelledError: + await asyncio.shield(self._finish(prisma_client, run_id, "FAILED")) + raise + await self._finish(prisma_client, run_id, result) + next_run_id: Final = await self._safe_plan_and_claim(prisma_client) + if next_run_id is not None: + await self._execute(prisma_client, next_run_id) + + async def _run_job(self, prisma_client: "PrismaClient") -> JobResult: + try: + return await self.job.run(prisma_client) + except Exception: # noqa: BLE001 - the failure is recorded on the run row instead of escaping the task + verbose_proxy_logger.exception("Scheduled job %s raised", self.job.name) + return "FAILED" + + async def _finish(self, prisma_client: "PrismaClient", run_id: str, result: JobResult) -> None: + timeout_ms: Final = int(SCHEDULED_JOB_TICK_TIMEOUT_SECONDS * 1000) + try: + async with db_span("scheduled_job_finish", _TABLE), bounded_tx(prisma_client, timeout_ms) as tx: + await tx.execute_raw(trusted_sql(f"SET LOCAL statement_timeout = {timeout_ms}")) + updated: Final = await tx.execute_raw(trusted_sql(_FINISH_SQL), run_id, self.worker_id, result) + await tx.execute_raw(trusted_sql(_PRUNE_SQL), self.job.name, SCHEDULED_JOB_RUN_HISTORY_LIMIT) + except Exception as e: # noqa: BLE001 - the lease expiry marks the row when this write cannot + verbose_proxy_logger.warning( + "Scheduled job %s run %s could not record %s: %s: %s", + self.job.name, + run_id, + result, + type(e).__name__, + e, + ) + return + if updated == 0: + verbose_proxy_logger.warning( + "Scheduled job %s run %s finished after its lease expired; its %s result was not recorded", + self.job.name, + run_id, + result, + ) + return + verbose_proxy_logger.info("Scheduled job %s run %s %s", self.job.name, run_id, result) + + +class _Locked(BaseModel): + model_config = ConfigDict(frozen=True) + + locked: bool + + +_LOCKED: Final = TypeAdapter(list[_Locked]) +_GROUP_LOCK_SQL: Final = 'SELECT pg_try_advisory_xact_lock(hashtext($1)) AS "locked"' + + +@lru_cache(maxsize=64) +def _log_invalid_schedule(job_name: str, reason: str) -> None: + verbose_proxy_logger.error( + "Scheduled job %s has an invalid schedule and will not be replanned: %s", job_name, reason + ) + + +def register_poll( + scheduler: JobScheduler, + prisma_client: "PrismaClient | None", + job_id: str, + job: ScheduledJob, + *, + wanted: bool, + poll_seconds: float, + misfire_grace_time: int, +) -> None: + """ + Make every pod poll job, or stop polling it. A poll that already exists is kept: its runner reads the schedule + on every tick, so a schedule edit needs no re-registration. + """ + registered: Final = scheduler.get_job(job_id) is not None + if registered and not wanted: + scheduler.remove_job(job_id) + verbose_proxy_logger.info("Stopped polling scheduled job %s", job.name) + return + if registered or not wanted: + return + scheduler.add_job( + ScheduledJobRunner(job).tick, + "interval", + args=(prisma_client,), + id=job_id, + replace_existing=True, + max_instances=2, + misfire_grace_time=misfire_grace_time, + seconds=poll_seconds, + ) + verbose_proxy_logger.info("Polling scheduled job %s every %ss", job.name, poll_seconds) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d8320667a3a..0c34f77890f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -277,8 +277,6 @@ try: import yaml from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.schedulers.base import STATE_STOPPED - from apscheduler.triggers.base import BaseTrigger - from apscheduler.triggers.interval import IntervalTrigger except ImportError as e: raise ImportError(f"Missing dependency {e}. Run `pip install 'litellm[proxy]'`") @@ -370,8 +368,8 @@ from litellm.constants import ( REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG, + SCHEDULED_JOB_POLL_INTERVAL_SECONDS, SPEND_CAPTURE_RATE_CHECK_JOB_ID, - SPEND_LOG_SIZE_CHECK_INTERVAL, USER_SPEND_ALERTS_JOB_ID, WEEKLY_SPEND_REPORT_JOB_ID, ) @@ -617,7 +615,6 @@ from litellm.proxy.common_utils.scheduled_job_stagger import ( apply_scheduled_job_stagger, attach_job_timing_logger, parse_stagger_settings, - stagger_trigger, ) from litellm.proxy.common_utils.slow_gc_logger import log_slow_gc_passes from litellm.proxy.common_utils.swagger_utils import ERROR_RESPONSES @@ -671,7 +668,10 @@ from litellm.proxy.container_endpoints.endpoints import router as container_rout from litellm.proxy.credential_endpoints.endpoints import router as credential_router from litellm.proxy.db.create_views import SupportsRawQueries from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager -from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup +from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import ( + SPEND_LOG_RETENTION_JOB, + SPEND_LOG_SIZE_CHECK_JOB, +) from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( build_window_spend_transaction, ) @@ -697,6 +697,7 @@ from litellm.proxy.db.request_error_tracking import ( flush_request_errors, request_error_accumulator, ) +from litellm.proxy.db.scheduled_job_runs import JobScheduler, register_poll from litellm.proxy.db.spend_counter_reseed import END_USER_COUNTER_PREFIX, SpendCounterReseed from litellm.proxy.discovery_endpoints import ( agent_skills_discovery_router, @@ -5911,6 +5912,28 @@ def _wants_cleanup_jobs(settings: Mapping[str, object]) -> bool: return _wants_retention_job(settings) or _wants_size_check_job(settings) +_SPEND_LOG_CLEANUP_POLLS: Final = ( + (_SPEND_LOG_CLEANUP_JOB_ID, _wants_retention_job, SPEND_LOG_RETENTION_JOB), + (_SPEND_LOG_SIZE_CHECK_JOB_ID, _wants_size_check_job, SPEND_LOG_SIZE_CHECK_JOB), +) + + +def _register_spend_log_cleanup_jobs( + job_scheduler: JobScheduler, db: PrismaClient | None, settings: Mapping[str, object] +) -> None: + """Poll LiteLLM_ScheduledJobRun for each cleanup job the settings ask for. Every pod polls, one claims each run.""" + for job_id, wanted, job in _SPEND_LOG_CLEANUP_POLLS: + register_poll( + job_scheduler, + db, + job_id, + job, + wanted=wanted(settings), + poll_seconds=SCHEDULED_JOB_POLL_INTERVAL_SECONDS, + misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, + ) + + @lru_cache(maxsize=4096) def _log_ignored_cost_map_copy(model_id: str, fields: tuple[str, ...]) -> None: verbose_proxy_logger.warning( @@ -8488,97 +8511,10 @@ class ProxyConfig: ) async def _reschedule_spend_log_cleanup_job(self): - """ - Reschedule the spend log cleanup job based on current general_settings. - This is called when maximum_spend_logs_retention_period is updated dynamically. - If the retention period is None, the job will be removed. - """ - global scheduler, general_settings, prisma_client + """Add or remove the cleanup polls when the settings that ask for them change.""" if scheduler is None: return - - from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import ( - SpendLogCleanup, - ) - - if not _wants_retention_job(general_settings): - if scheduler.get_job(_SPEND_LOG_CLEANUP_JOB_ID) is not None: - scheduler.remove_job(_SPEND_LOG_CLEANUP_JOB_ID) - verbose_proxy_logger.info("Removed existing spend log cleanup job") - else: - trigger: Final = self._spend_log_cleanup_trigger() - if trigger is not None: - scheduler.add_job( - SpendLogCleanup().cleanup_old_spend_logs, - trigger, - args=[prisma_client], - id=_SPEND_LOG_CLEANUP_JOB_ID, - replace_existing=True, - misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, - ) - verbose_proxy_logger.info("Spend log cleanup rescheduled with trigger: %s", trigger) - - if not _wants_size_check_job(general_settings): # pyright: ignore[reportUnknownArgumentType] # general_settings is an untyped module dict - if scheduler.get_job(_SPEND_LOG_SIZE_CHECK_JOB_ID) is not None: # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped - scheduler.remove_job(_SPEND_LOG_SIZE_CHECK_JOB_ID) # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped - verbose_proxy_logger.info("Removed existing spend log size check job") - return - size_trigger: Final = self._spend_log_size_check_trigger() - if size_trigger is None: - return - scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped - SpendLogCleanup().cleanup_spend_logs_by_size, - size_trigger, - args=[prisma_client], - id=_SPEND_LOG_SIZE_CHECK_JOB_ID, - replace_existing=True, - misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, - ) - verbose_proxy_logger.info("Spend log size check rescheduled with trigger: %s", size_trigger) - - def _spend_log_size_check_trigger(self) -> object | None: - return self._staggered_interval_trigger( - _SPEND_LOG_SIZE_CHECK_JOB_ID, - "maximum_spend_logs_size_check_interval", - general_settings.get("maximum_spend_logs_size_check_interval", SPEND_LOG_SIZE_CHECK_INTERVAL), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # general_settings is an untyped module dict - ) - - @staticmethod - def _staggered_interval_trigger(job_id: str, setting_name: str, interval: object) -> object | None: - if not isinstance(interval, str): - verbose_proxy_logger.error("Invalid %s value: %r", setting_name, interval) - return None - # this runs against a started scheduler, which the startup stagger sweep - # cannot reach, so the offset is applied here or the job reconverges across - # replicas the first time an admin edits the settings - try: - interval_seconds: Final = duration_in_seconds(interval) - return stagger_trigger( - job_id=job_id, - trigger=IntervalTrigger(seconds=interval_seconds), - period_seconds=interval_seconds, - settings=parse_stagger_settings(general_settings), # pyright: ignore[reportUnknownArgumentType] # general_settings is an untyped module dict - ) - except (ValueError, OverflowError): - verbose_proxy_logger.error("Invalid %s value: %r", setting_name, interval) - return None - - def _spend_log_cleanup_trigger(self) -> object | None: - cleanup_cron: Final[object] = general_settings.get("maximum_spend_logs_cleanup_cron") - if cleanup_cron: - from apscheduler.triggers.cron import CronTrigger - - try: - cron_trigger: Final[BaseTrigger] = CronTrigger.from_crontab(cleanup_cron) - except (ValueError, TypeError, AttributeError): - verbose_proxy_logger.error("Invalid maximum_spend_logs_cleanup_cron value: %s", cleanup_cron) - return None - return cron_trigger - return self._staggered_interval_trigger( - _SPEND_LOG_CLEANUP_JOB_ID, - "maximum_spend_logs_retention_interval", - general_settings.get("maximum_spend_logs_retention_interval", "1d"), - ) + _register_spend_log_cleanup_jobs(scheduler, prisma_client, general_settings) # pyright: ignore[reportUnknownArgumentType, reportArgumentType] # apscheduler is untyped async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | str | None) -> None: global general_settings @@ -12026,70 +11962,7 @@ class ProxyStartupEvent: ### SPEND LOG CLEANUP ### cleanup_settings: Final = _current_general_settings() - if _wants_size_check_job(cleanup_settings): - size_check_interval: Final = cleanup_settings.get( - "maximum_spend_logs_size_check_interval", SPEND_LOG_SIZE_CHECK_INTERVAL - ) - try: - if not isinstance(size_check_interval, str): - raise TypeError(size_check_interval) - size_check_seconds: Final = duration_in_seconds(size_check_interval) - scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped - SpendLogCleanup().cleanup_spend_logs_by_size, - "interval", - seconds=size_check_seconds + random.randint(0, 60), - args=[prisma_client], - id=_SPEND_LOG_SIZE_CHECK_JOB_ID, - replace_existing=True, - misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, - ) - verbose_proxy_logger.info("Spend log size check scheduled every %s", size_check_interval) - except (TypeError, ValueError, OverflowError): - verbose_proxy_logger.error( - "Invalid maximum_spend_logs_size_check_interval value: %r", size_check_interval - ) - if _wants_retention_job(cleanup_settings): - spend_log_cleanup: Final = SpendLogCleanup() - cleanup_cron: Final = cleanup_settings.get("maximum_spend_logs_cleanup_cron") - - if cleanup_cron and not isinstance(cleanup_cron, str): - verbose_proxy_logger.error("Invalid maximum_spend_logs_cleanup_cron value: %r", cleanup_cron) - elif isinstance(cleanup_cron, str) and cleanup_cron: - from apscheduler.triggers.cron import CronTrigger - - try: - cron_trigger: Final = CronTrigger.from_crontab(cleanup_cron) - scheduler.add_job( - spend_log_cleanup.cleanup_old_spend_logs, - cron_trigger, - args=[prisma_client], - id=_SPEND_LOG_CLEANUP_JOB_ID, - replace_existing=True, - misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, - ) - verbose_proxy_logger.info("Spend log cleanup scheduled with cron: %s", cleanup_cron) - except ValueError: - verbose_proxy_logger.error("Invalid maximum_spend_logs_cleanup_cron value: %s", cleanup_cron) - else: - # Interval-based scheduling (existing behavior) - retention_interval: Final = cleanup_settings.get("maximum_spend_logs_retention_interval", "1d") - try: - if not isinstance(retention_interval, str): - raise ValueError(retention_interval) - interval_seconds: Final = duration_in_seconds(retention_interval) - scheduler.add_job( - spend_log_cleanup.cleanup_old_spend_logs, - "interval", - seconds=interval_seconds + random.randint(0, 60), - args=[prisma_client], - id=_SPEND_LOG_CLEANUP_JOB_ID, - replace_existing=True, - misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, - ) - except (ValueError, OverflowError): - verbose_proxy_logger.error( - "Invalid maximum_spend_logs_retention_interval value: %r", retention_interval - ) + _register_spend_log_cleanup_jobs(scheduler, prisma_client, cleanup_settings) # pyright: ignore[reportUnknownArgumentType, reportArgumentType] # apscheduler is untyped if _wants_cleanup_jobs(cleanup_settings): proxy_config.record_cleanup_schedule_attempt(cleanup_settings) ### CHECK BATCH COST ### diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 5a5d56cc6e2..4314a23e011 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1155,6 +1155,31 @@ enum JobStatus { INACTIVE } +model LiteLLM_ScheduledJobRun { + id String @id @default(uuid()) + job_name String + lock_group String + schedule String + scheduled_for DateTime @client.Timestamptz(3) + status ScheduledJobRunStatus @default(PENDING) + claimed_by String? + claimed_at DateTime? @client.Timestamptz(3) + lease_expires_at DateTime? @client.Timestamptz(3) + finished_at DateTime? @client.Timestamptz(3) + created_at DateTime @default(now()) @client.Timestamptz(3) + + @@index([job_name, status]) + @@index([lock_group, status]) +} + +enum ScheduledJobRunStatus { + PENDING + RUNNING + SUCCEEDED + FAILED + CANCELLED +} + model LiteLLM_ManagedFileTable { id String @id @default(uuid()) unified_file_id String @unique // The base64 encoded unified file ID diff --git a/schema.prisma b/schema.prisma index 5a5d56cc6e2..4314a23e011 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1155,6 +1155,31 @@ enum JobStatus { INACTIVE } +model LiteLLM_ScheduledJobRun { + id String @id @default(uuid()) + job_name String + lock_group String + schedule String + scheduled_for DateTime @client.Timestamptz(3) + status ScheduledJobRunStatus @default(PENDING) + claimed_by String? + claimed_at DateTime? @client.Timestamptz(3) + lease_expires_at DateTime? @client.Timestamptz(3) + finished_at DateTime? @client.Timestamptz(3) + created_at DateTime @default(now()) @client.Timestamptz(3) + + @@index([job_name, status]) + @@index([lock_group, status]) +} + +enum ScheduledJobRunStatus { + PENDING + RUNNING + SUCCEEDED + FAILED + CANCELLED +} + model LiteLLM_ManagedFileTable { id String @id @default(uuid()) unified_file_id String @unique // The base64 encoded unified file ID diff --git a/tests/integration/database/test_cleanup_transactions.py b/tests/integration/database/test_cleanup_transactions.py index 202657d180e..be2487d5498 100644 --- a/tests/integration/database/test_cleanup_transactions.py +++ b/tests/integration/database/test_cleanup_transactions.py @@ -202,7 +202,6 @@ async def test_size_rule_deletes_the_oldest_rows_until_under_the_target_and_does "maximum_spend_logs_cleanup_batch_size": 100, } ) - cleanup.pod_lock_manager = None async def size_check() -> None: database: Final = Prisma(datasource={"url": scoped_url}) diff --git a/tests/integration/database/test_scheduled_job_runs.py b/tests/integration/database/test_scheduled_job_runs.py new file mode 100644 index 00000000000..913111600bf --- /dev/null +++ b/tests/integration/database/test_scheduled_job_runs.py @@ -0,0 +1,294 @@ +import asyncio +import time +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import TYPE_CHECKING, Final, cast + +import psycopg +import pytest +from integration._support.database import scratch_database +from prisma import Prisma +from psycopg.rows import dict_row + +from litellm.proxy.db.scheduled_job_runs import ( + JobResult, + Schedule, + ScheduledJob, + ScheduledJobRunner, + cron_schedule, + interval_schedule, +) + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +_MIGRATION: Final = next( + Path(__file__) + .resolve() + .parents[3] + .joinpath("litellm-proxy-extras/litellm_proxy_extras/migrations") + .glob("*_scheduled_job_runs/migration.sql") +) +_PODS: Final = 4 + + +@dataclass(frozen=True) +class _Connection: + db: Prisma + + +@dataclass(frozen=True) +class _Database: + url: str + pods: tuple["PrismaClient", ...] + + def rows(self, where: str = "TRUE") -> list[dict[str, object]]: + with psycopg.connect(self.url, row_factory=dict_row) as connection: + return connection.execute( + f'SELECT * FROM "LiteLLM_ScheduledJobRun" WHERE {where} ORDER BY "created_at", "scheduled_for"' + ).fetchall() + + def execute(self, statement: str, parameters: tuple[object, ...] = ()) -> None: + with psycopg.connect(self.url, autocommit=True) as connection: + connection.execute(statement, parameters) + + +class _Job: + """A job whose runs are counted and can be held open until the test releases them.""" + + def __init__(self, name: str = "job", lock_group: str = "group", schedule: str = "1h") -> None: + self.name = name + self.lock_group = lock_group + self.schedule_value = schedule + self.started = 0 + self.release = asyncio.Event() + self.release.set() + self.outcome: Callable[[], Awaitable[JobResult]] = self._succeed + + async def _succeed(self) -> JobResult: + return "SUCCEEDED" + + async def _run(self, _prisma_client: object) -> JobResult: + self.started += 1 + await self.release.wait() + return await self.outcome() + + def _schedule(self) -> Schedule: + parsed: Final = ( + cron_schedule("cron", self.schedule_value) + if " " in self.schedule_value + else interval_schedule("interval", self.schedule_value) + ) + assert isinstance(parsed, Schedule) + return parsed + + def definition(self) -> ScheduledJob: + return ScheduledJob( + name=self.name, + lock_group=self.lock_group, + schedule=self._schedule, + lease=lambda: timedelta(minutes=5), + run=self._run, + ) + + +@asynccontextmanager +async def _scratch() -> AsyncIterator[_Database]: + with scratch_database("scheduled_jobs") as url: + with psycopg.connect(url, autocommit=True) as setup: + setup.execute(_MIGRATION.read_text()) + connections: Final = tuple(_Connection(Prisma(datasource={"url": url})) for _ in range(_PODS)) + for connection in connections: + await connection.db.connect() + try: + yield _Database( + url=url, + pods=tuple( + cast("PrismaClient", connection) # cast-ok: the runner only reaches the client through .db + for connection in connections + ), + ) + finally: + for connection in connections: + await connection.db.disconnect() + + +def _runners(job: _Job) -> tuple[ScheduledJobRunner, ...]: + return tuple(ScheduledJobRunner(job.definition(), worker_id=f"pod-{index}") for index in range(_PODS)) + + +async def _tick_all(runners: tuple[ScheduledJobRunner, ...], database: _Database) -> None: + await asyncio.gather(*(runner.tick(pod) for runner, pod in zip(runners, database.pods))) + + +async def _until(condition: Callable[[], bool], seconds: float = 10) -> None: + deadline: Final = time.monotonic() + seconds + while not condition(): + assert time.monotonic() < deadline, "condition never held" + await asyncio.sleep(0.02) + + +async def test_every_pod_ticks_but_exactly_one_runs_and_the_next_run_is_planned_one_interval_after_it_finished() -> ( + None +): + async with _scratch() as database: + job: Final = _Job() + job.release.clear() + runners: Final = _runners(job) + ticks: Final = asyncio.gather(*(runner.tick(pod) for runner, pod in zip(runners, database.pods))) + await _until(lambda: job.started == 1) + await asyncio.sleep(0.5) + running: Final = database.rows("\"status\" = 'RUNNING'") + assert job.started == 1 + assert len(running) == 1 + job.release.set() + await ticks + + finished: Final = database.rows("\"status\" = 'SUCCEEDED'") + pending: Final = database.rows("\"status\" = 'PENDING'") + assert len(finished) == 1 + assert finished[0]["claimed_by"] == running[0]["claimed_by"] + assert len(pending) == 1 + assert pending[0]["scheduled_for"] == finished[0]["finished_at"] + timedelta(hours=1) + + await _tick_all(runners, database) + assert job.started == 1, "a run was started before the next occurrence was due" + assert len(database.rows("\"status\" = 'PENDING'")) == 1 + + +async def test_jobs_in_one_lock_group_never_run_at_the_same_time_on_different_pods() -> None: + async with _scratch() as database: + first: Final = _Job(name="first") + second: Final = _Job(name="second") + first.release.clear() + first_runner: Final = ScheduledJobRunner(first.definition(), worker_id="pod-a") + second_runner: Final = ScheduledJobRunner(second.definition(), worker_id="pod-b") + first_tick: Final = asyncio.create_task(first_runner.tick(database.pods[0])) + await _until(lambda: first.started == 1) + + await second_runner.tick(database.pods[1]) + assert second.started == 0 + assert [row["status"] for row in database.rows("\"job_name\" = 'second'")] == ["PENDING"] + + first.release.set() + await first_tick + await second_runner.tick(database.pods[1]) + assert second.started == 1 + + +async def test_a_run_that_raises_is_recorded_as_failed_and_the_next_run_is_still_planned() -> None: + async with _scratch() as database: + job: Final = _Job() + + async def explode() -> JobResult: + raise RuntimeError("disk on fire") + + job.outcome = explode + await _tick_all(_runners(job), database) + + failed: Final = database.rows("\"status\" = 'FAILED'") + assert len(failed) == 1 + assert len(database.rows("\"status\" = 'PENDING'")) == 1 + + +async def test_a_run_held_by_a_dead_pod_is_failed_once_its_lease_expires_and_the_schedule_carries_on() -> None: + async with _scratch() as database: + database.execute( + 'INSERT INTO "LiteLLM_ScheduledJobRun" ("id", "job_name", "lock_group", "schedule", "scheduled_for", ' + '"status", "claimed_by", "claimed_at", "lease_expires_at") ' + "VALUES ('dead', 'job', 'group', 'interval:3600s', now() - interval '2 hours', 'RUNNING', 'dead-pod', " + "now() - interval '2 hours', now() - interval '1 second')" + ) + job: Final = _Job() + await _tick_all(_runners(job), database) + + (reaped,) = database.rows("\"id\" = 'dead'") + assert reaped["status"] == "FAILED" + (pending,) = database.rows("\"status\" = 'PENDING'") + assert pending["scheduled_for"] == reaped["finished_at"] + timedelta(hours=1) + assert job.started == 0 + + +async def test_a_run_held_by_a_live_pod_is_left_alone_and_nothing_else_starts() -> None: + async with _scratch() as database: + job: Final = _Job() + job.release.clear() + runners: Final = _runners(job) + holder: Final = asyncio.create_task(runners[0].tick(database.pods[0])) + await _until(lambda: job.started == 1) + + await asyncio.gather(*(runner.tick(pod) for runner, pod in zip(runners[1:], database.pods[1:]))) + assert [row["status"] for row in database.rows()] == ["RUNNING"] + + job.release.set() + await holder + assert job.started == 1 + + +async def test_an_interval_already_overdue_when_the_pods_restart_runs_on_the_first_tick() -> None: + async with _scratch() as database: + database.execute( + 'INSERT INTO "LiteLLM_ScheduledJobRun" ("id", "job_name", "lock_group", "schedule", "scheduled_for", ' + '"status", "finished_at") ' + "VALUES ('yesterday', 'job', 'group', 'interval:86400s', now() - interval '2 days', 'SUCCEEDED', " + "now() - interval '2 days')" + ) + job: Final = _Job(schedule="1d") + await _tick_all(_runners(job), database) + + assert job.started == 1 + assert [row["status"] for row in database.rows("\"id\" <> 'yesterday'")] == ["SUCCEEDED", "PENDING"] + + +async def test_a_schedule_change_replaces_the_pending_run_planned_from_the_old_schedule() -> None: + async with _scratch() as database: + job: Final = _Job(schedule="0 4 * * *") + runners: Final = _runners(job) + await _tick_all(runners, database) + (old,) = database.rows("\"status\" = 'PENDING'") + assert old["scheduled_for"].astimezone(timezone.utc).hour == 4 + + job.schedule_value = "0 5 * * *" + await _tick_all(runners, database) + + assert database.rows(f"\"id\" = '{old['id']}'")[0]["status"] == "CANCELLED" + (replacement,) = database.rows("\"status\" = 'PENDING'") + assert replacement["schedule"] == "cron:0 5 * * *" + assert replacement["scheduled_for"].astimezone(timezone.utc).hour == 5 + assert replacement["scheduled_for"] > datetime.now(timezone.utc) + assert job.started == 0 + + +async def test_moving_a_cron_earlier_plans_the_earlier_occurrence_instead_of_skipping_a_day() -> None: + async with _scratch() as database: + now: Final = datetime.now(timezone.utc) + job: Final = _Job(schedule=f"0 {(now + timedelta(hours=3)).hour} * * *") + runners: Final = _runners(job) + await _tick_all(runners, database) + (old,) = database.rows("\"status\" = 'PENDING'") + + job.schedule_value = f"0 {(now + timedelta(hours=2)).hour} * * *" + await _tick_all(runners, database) + + (replacement,) = database.rows("\"status\" = 'PENDING'") + assert replacement["scheduled_for"] < old["scheduled_for"] + assert replacement["scheduled_for"] - datetime.now(timezone.utc) < timedelta(hours=2) + + +async def test_a_run_cancelled_by_shutdown_is_recorded_as_failed_instead_of_holding_its_lease() -> None: + async with _scratch() as database: + job: Final = _Job() + job.release.clear() + runner: Final = ScheduledJobRunner(job.definition(), worker_id="pod-shutting-down") + tick: Final = asyncio.create_task(runner.tick(database.pods[0])) + await _until(lambda: job.started == 1) + + tick.cancel() + with pytest.raises(asyncio.CancelledError): + await tick + + (cancelled,) = database.rows() + assert cancelled["status"] == "FAILED" diff --git a/tests/unit/proxy/db/test_scheduled_job_runs.py b/tests/unit/proxy/db/test_scheduled_job_runs.py new file mode 100644 index 00000000000..9555a5ea61c --- /dev/null +++ b/tests/unit/proxy/db/test_scheduled_job_runs.py @@ -0,0 +1,355 @@ +import asyncio +import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Final, cast + +import pytest +from apscheduler.schedulers.asyncio import AsyncIOScheduler + +from litellm.proxy.db.scheduled_job_runs import ( + InvalidSchedule, + JobResult, + JobScheduler, + JobState, + NewRun, + PendingRun, + Schedule, + ScheduledJob, + ScheduledJobRunner, + TickPlan, + cron_schedule, + interval_schedule, + next_run_at, + plan_tick, + register_poll, +) + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +NOW: Final = datetime(2026, 11, 4, 12, 0, tzinfo=timezone.utc) + + +def _schedule(value: str) -> Schedule: + parsed: Final = cron_schedule("cron", value) if " " in value else interval_schedule("interval", value) + assert not isinstance(parsed, InvalidSchedule) + return parsed + + +def _state( + *, + group_running: bool = False, + job_running: bool = False, + last_finished_at: datetime | None = None, + last_scheduled_for: datetime | None = None, +) -> JobState: + return JobState( + now=NOW, + group_running=group_running, + job_running=job_running, + last_finished_at=last_finished_at, + last_scheduled_for=last_scheduled_for, + ) + + +def _pending(run_id: str, schedule: str, scheduled_for: datetime) -> PendingRun: + return PendingRun(id=run_id, schedule=schedule, scheduled_for=scheduled_for) + + +@pytest.mark.parametrize( + ("schedule", "state", "expected"), + [ + pytest.param("1h", _state(), NOW, id="interval-never-run-is-due-now"), + pytest.param( + "1h", + _state(last_finished_at=NOW - timedelta(hours=3)), + NOW - timedelta(hours=2), + id="interval-counts-from-last-finish-so-a-restart-cannot-postpone-it", + ), + pytest.param( + "1h", + _state(last_finished_at=NOW - timedelta(minutes=10)), + NOW + timedelta(minutes=50), + id="interval-not-yet-elapsed", + ), + pytest.param("0 4 * * *", _state(), datetime(2026, 11, 5, 4, tzinfo=timezone.utc), id="cron-next-fire"), + pytest.param( + "0 12 * * *", + _state(last_scheduled_for=NOW), + datetime(2026, 11, 5, 12, tzinfo=timezone.utc), + id="cron-never-replans-the-occurrence-it-just-had", + ), + pytest.param( + "0 4 * * *", + _state(last_scheduled_for=NOW - timedelta(days=3)), + datetime(2026, 11, 5, 4, tzinfo=timezone.utc), + id="cron-outage-collapses-to-one-next-fire", + ), + ], +) +def test_next_run_at(schedule: str, state: JobState, expected: datetime) -> None: + assert next_run_at(_schedule(schedule), state) == expected + + +DUE: Final = NOW - timedelta(minutes=1) +LATER: Final = NOW + timedelta(hours=1) + + +@pytest.mark.parametrize( + ("schedule", "state", "pending", "expected"), + [ + pytest.param( + "1h", + _state(), + (), + TickPlan(cancel=(), insert=NewRun(id="new", schedule="interval:3600s", scheduled_for=NOW), claim="new"), + id="first-ever-tick-plans-and-claims", + ), + pytest.param( + "1h", + _state(), + (_pending("a", "interval:3600s", DUE),), + TickPlan(cancel=(), insert=None, claim="a"), + id="due-pending-is-claimed", + ), + pytest.param( + "1h", + _state(), + (_pending("a", "interval:3600s", LATER),), + TickPlan(cancel=(), insert=None, claim=None), + id="future-pending-waits", + ), + pytest.param( + "1h", + _state(group_running=True), + (_pending("a", "interval:3600s", DUE),), + TickPlan(cancel=(), insert=None, claim=None), + id="due-pending-waits-while-its-lock-group-runs", + ), + pytest.param( + "1h", + _state(group_running=True), + (), + TickPlan(cancel=(), insert=NewRun(id="new", schedule="interval:3600s", scheduled_for=NOW), claim=None), + id="a-sibling-run-still-lets-the-next-run-be-planned", + ), + pytest.param( + "1h", + _state(group_running=True, job_running=True), + (), + TickPlan(cancel=(), insert=None, claim=None), + id="nothing-is-planned-while-the-job-itself-runs", + ), + pytest.param( + "2h", + _state(last_finished_at=NOW - timedelta(minutes=30)), + (_pending("old", "interval:3600s", NOW + timedelta(minutes=30)),), + TickPlan( + cancel=("old",), + insert=NewRun(id="new", schedule="interval:7200s", scheduled_for=NOW + timedelta(minutes=90)), + claim=None, + ), + id="schedule-change-replaces-the-pending-run", + ), + pytest.param( + "1h", + _state(), + (_pending("a", "interval:3600s", DUE), _pending("b", "interval:3600s", LATER)), + TickPlan(cancel=("b",), insert=None, claim="a"), + id="a-duplicate-pending-run-is-cancelled", + ), + ], +) +def test_plan_tick(schedule: str, state: JobState, pending: tuple[PendingRun, ...], expected: TickPlan) -> None: + assert plan_tick(_schedule(schedule), state, pending, "new") == expected + + +def test_an_invalid_schedule_replans_nothing_but_still_runs_what_the_last_valid_one_planned() -> None: + pending: Final = (_pending("a", "interval:3600s", DUE),) + plan: Final = plan_tick(InvalidSchedule("bad cron"), _state(), pending, "new") + assert plan == TickPlan(cancel=(), insert=None, claim="a") + + +@pytest.mark.parametrize("value", ["not a cron", "0 0 * * * *", "0 0 31 2 *", "0 0 30 2 *", 5, None]) +def test_cron_schedule_rejects_what_is_not_a_crontab(value: object) -> None: + assert isinstance(cron_schedule("maximum_spend_logs_cleanup_cron", value), InvalidSchedule) + + +@pytest.mark.parametrize("value", ["soon", "99999999999d", "0s", 86400, None]) +def test_interval_schedule_rejects_what_is_not_a_positive_duration(value: object) -> None: + assert isinstance(interval_schedule("maximum_spend_logs_retention_interval", value), InvalidSchedule) + + +class _ScriptedDatabase: + """Answers the runner's statements the way Postgres would for one due run, and records every write.""" + + def __init__(self, *, locked: bool = True, finish_updates: int = 1, broken: bool = False) -> None: + self.locked = locked + self.finish_updates = finish_updates + self.broken = broken + self.writes: tuple[tuple[str, tuple[object, ...]], ...] = () + self.db = self + + @asynccontextmanager + async def tx(self, timeout: timedelta) -> AsyncIterator["_ScriptedDatabase"]: + if self.broken: + raise ConnectionError("database unreachable") + yield self + + async def query_raw(self, sql: str, *args: object) -> list[dict[str, object]]: + if "pg_try_advisory_xact_lock" in sql: + return [{"locked": self.locked}] + if "SELECT now()" in sql: + return [ + { + "now": NOW, + "group_running": False, + "job_running": False, + "last_finished_at": None, + "last_scheduled_for": None, + } + ] + if "'RUNNING', \"claimed_by\"" in sql: + self.writes = (*self.writes, ("claim", args)) + return [{"id": args[0], "job_name": "job"}] + return [] + + async def execute_raw(self, sql: str, *args: object) -> int: + if sql.lstrip().startswith("SET LOCAL"): + return 0 + if 'SET "status" = $3' in sql: + self.writes = (*self.writes, ("finish", args)) + self.locked = False + return self.finish_updates + if sql.lstrip().startswith("INSERT"): + self.writes = (*self.writes, ("insert", args)) + return 1 + + def client(self) -> "PrismaClient": + return cast("PrismaClient", self) # cast-ok: the runner only reaches the client through .db + + +def _finishes(database: _ScriptedDatabase) -> tuple[tuple[object, ...], ...]: + return tuple(args for kind, args in database.writes if kind == "finish") + + +class _Job: + def __init__(self, result: JobResult | BaseException = "SUCCEEDED") -> None: + self.result = result + self.runs = 0 + self.release = asyncio.Event() + self.release.set() + + async def run(self, _prisma_client: object) -> JobResult: + self.runs += 1 + await self.release.wait() + if isinstance(self.result, BaseException): + raise self.result + return self.result + + def runner(self) -> ScheduledJobRunner: + definition: Final = ScheduledJob( + name="job", + lock_group="group", + schedule=lambda: _schedule("1h"), + lease=lambda: timedelta(minutes=5), + run=self.run, + ) + return ScheduledJobRunner(definition, worker_id="pod-a", new_id=lambda: "run-1") + + +async def test_a_due_run_is_claimed_by_this_pod_run_and_recorded_as_succeeded() -> None: + database: Final = _ScriptedDatabase() + job: Final = _Job("SUCCEEDED") + await job.runner().tick(database.client()) + + assert job.runs == 1 + assert ("claim", ("run-1", "pod-a", 300.0)) in database.writes + assert _finishes(database) == (("run-1", "pod-a", "SUCCEEDED"),) + + +@pytest.mark.parametrize( + "result", + [pytest.param("FAILED", id="reported"), pytest.param(RuntimeError("disk on fire"), id="raised")], +) +async def test_a_failed_run_is_recorded_as_failed(result: JobResult | BaseException) -> None: + database: Final = _ScriptedDatabase() + await _Job(result).runner().tick(database.client()) + assert _finishes(database) == (("run-1", "pod-a", "FAILED"),) + + +async def test_a_pod_that_does_not_get_the_group_lock_neither_plans_nor_runs() -> None: + database: Final = _ScriptedDatabase(locked=False) + job: Final = _Job() + await job.runner().tick(database.client()) + assert job.runs == 0 + assert database.writes == () + + +async def test_an_unreachable_database_skips_the_tick_instead_of_raising_into_the_scheduler() -> None: + job: Final = _Job() + await job.runner().tick(_ScriptedDatabase(broken=True).client()) + assert job.runs == 0 + + +async def test_a_poll_landing_while_this_pod_still_runs_the_job_does_not_touch_the_database() -> None: + database: Final = _ScriptedDatabase() + job: Final = _Job() + job.release.clear() + runner: Final = job.runner() + first: Final = asyncio.create_task(runner.tick(database.client())) + while job.runs == 0: + await asyncio.sleep(0) + writes_during_run: Final = database.writes + + await runner.tick(database.client()) + assert database.writes == writes_during_run + + job.release.set() + await first + assert job.runs == 1 + + +async def test_a_run_cancelled_mid_flight_records_failed_and_still_cancels() -> None: + database: Final = _ScriptedDatabase() + job: Final = _Job() + job.release.clear() + tick: Final = asyncio.create_task(job.runner().tick(database.client())) + while job.runs == 0: + await asyncio.sleep(0) + tick.cancel() + with pytest.raises(asyncio.CancelledError): + await tick + assert _finishes(database) == (("run-1", "pod-a", "FAILED"),) + + +async def test_a_result_that_arrives_after_the_lease_expired_is_reported_not_recorded( + caplog: pytest.LogCaptureFixture, +) -> None: + database: Final = _ScriptedDatabase(finish_updates=0) + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await _Job().runner().tick(database.client()) + assert any("after its lease expired" in record.getMessage() for record in caplog.records) + + +async def test_register_poll_adds_a_wanted_poll_once_and_removes_it_when_no_longer_wanted() -> None: + scheduler: Final = AsyncIOScheduler() + scheduler.start(paused=True) + job_scheduler: Final = cast(JobScheduler, scheduler) # cast-ok: apscheduler ships no type information + definition: Final = _Job().runner().job + + def register(wanted: bool) -> None: + register_poll(job_scheduler, None, "poll-id", definition, wanted=wanted, poll_seconds=30, misfire_grace_time=60) + + try: + register(True) + first: Final = scheduler.get_job("poll-id") + assert first is not None + register(True) + assert scheduler.get_job("poll-id") is first, "re-registering replaced the live poll and its runner" + register(False) + assert scheduler.get_job("poll-id") is None + finally: + scheduler.shutdown(wait=False) diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 91ff818da4c..9302b16f4f5 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -19,7 +19,7 @@ import sys from collections.abc import Awaitable, Callable, Mapping from contextlib import nullcontext from dataclasses import dataclass -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path from types import MappingProxyType, SimpleNamespace from typing import Any, Dict, Final @@ -4717,30 +4717,9 @@ async def test_ProxyConfig__reschedule_spend_log_cleanup_job_no_scheduler(monkey assert snapshot == {"raised": False, "called": True, "scheduler_was": "none"} -@pytest.mark.asyncio -async def test_ProxyConfig__reschedule_spend_log_cleanup_job_invalid_cron(monkeypatch): - fake_scheduler = MagicMock() - fake_scheduler.remove_job = MagicMock() - fake_scheduler.add_job = MagicMock() - monkeypatch.setattr("litellm.proxy.proxy_server.scheduler", fake_scheduler) - monkeypatch.setattr( - "litellm.proxy.proxy_server.general_settings", - { - "maximum_spend_logs_retention_period": "1d", - "maximum_spend_logs_cleanup_cron": "INVALID CRON STRING", - }, - ) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) - pc = ProxyConfig() - # Invalid cron is caught and logged — does not raise outward. - await pc._reschedule_spend_log_cleanup_job() - # But add_job should not have been called for the invalid cron path. - assert fake_scheduler.add_job.call_count == 0 - - @pytest.mark.asyncio async def test_ProxyConfig__reschedule_spend_log_cleanup_job_health_check_retention(monkeypatch): - fake_scheduler = MagicMock() + fake_scheduler = MagicMock(**{"get_job.return_value": None}) monkeypatch.setattr("litellm.proxy.proxy_server.scheduler", fake_scheduler) monkeypatch.setattr( "litellm.proxy.proxy_server.general_settings", @@ -4889,7 +4868,7 @@ async def test_ProxyConfig__reschedule_spend_log_cleanup_job_daily_tag_spend_ret await pc._reschedule_spend_log_cleanup_job() job = real_scheduler.get_job("spend_log_cleanup_job") assert job is not None, "daily tag spend retention alone did not schedule the cleanup job" - assert job.func.__name__ == "cleanup_old_spend_logs" + assert job.func.__name__ == "tick" finally: real_scheduler.shutdown(wait=False) @@ -4924,33 +4903,6 @@ async def test_ProxyConfig__update_general_settings_schedules_cleanup_when_db_ro real_scheduler.shutdown(wait=False) -@pytest.mark.asyncio -async def test_ProxyConfig__update_general_settings_retries_a_failed_schedule_once_per_settings_value( - monkeypatch, caplog -): - """An unparseable cron leaves no job behind; reloads must not retry it every tick, only when the - cron or a retention value changes.""" - real_scheduler = _paused_scheduler(monkeypatch) - pc = ProxyConfig() - bad_cron = {"maximum_daily_tag_spend_retention_period": "90d", "maximum_spend_logs_cleanup_cron": "not a cron"} - pc.settings.apply_db_row("general_settings", bad_cron) - monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings) - try: - with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): - for _ in range(3): - await pc._update_general_settings(bad_cron) - assert real_scheduler.get_job("spend_log_cleanup_job") is None - cron_errors = [r for r in caplog.records if "maximum_spend_logs_cleanup_cron" in r.getMessage()] - assert len(cron_errors) == 1, f"invalid cron was retried on every reload: {len(cron_errors)} error lines" - - await pc._update_general_settings({**bad_cron, "maximum_spend_logs_cleanup_cron": "* * * * *"}) - job = real_scheduler.get_job("spend_log_cleanup_job") - assert job is not None, "a corrected cron did not schedule cleanup" - assert "minute='*'" in str(job.trigger) - finally: - real_scheduler.shutdown(wait=False) - - @pytest.mark.asyncio async def test_ProxyConfig__update_general_settings_retries_a_schedule_that_raised(monkeypatch): """A transient add_job failure must not be remembered as a completed attempt; the next @@ -4969,29 +4921,6 @@ async def test_ProxyConfig__update_general_settings_retries_a_schedule_that_rais real_scheduler.shutdown(wait=False) -@pytest.mark.asyncio -async def test_ProxyConfig__update_general_settings_retries_a_failed_replacement_of_the_live_job(monkeypatch): - """A cron change whose add_job raised keeps the old job running, so the next reload with the - same settings must try the replacement again instead of leaving the new cron unapplied.""" - real_scheduler = _scheduler_whose_first_add_job_raises(monkeypatch) - pc = ProxyConfig() - pc.settings.load_yaml({"maximum_daily_tag_spend_retention_period": "90d"}) - monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings) - real_scheduler.raised = True - await pc._reschedule_spend_log_cleanup_job() - real_scheduler.raised = False - try: - new_cron = {"maximum_spend_logs_cleanup_cron": "0 3 * * *"} - await pc._update_general_settings(new_cron) - assert "hour='3'" not in str(real_scheduler.get_job("spend_log_cleanup_job").trigger), "old job was lost" - await pc._update_general_settings(new_cron) - assert "hour='3'" in str(real_scheduler.get_job("spend_log_cleanup_job").trigger), ( - "failed replacement was not retried on the next sync" - ) - finally: - real_scheduler.shutdown(wait=False) - - @pytest.mark.asyncio async def test_ProxyConfig__update_general_settings_leaves_a_changed_db_schedule_to_startup_while_scheduler_is_stopped( monkeypatch, @@ -5027,34 +4956,6 @@ async def test_ProxyConfig__update_general_settings_leaves_first_registration_to assert real_scheduler.get_jobs() == [], "DB sync registered the cleanup job before the scheduler started" -@pytest.mark.asyncio -async def test_ProxyConfig__update_general_settings_runtime_interval_job_carries_the_stagger_offset(monkeypatch): - """Once the scheduler is running the sync owns registration and the job it adds is staggered.""" - from apscheduler.schedulers.asyncio import AsyncIOScheduler - from apscheduler.triggers.interval import IntervalTrigger - - from litellm.proxy.common_utils.scheduled_job_stagger import _OffsetTrigger - - real_scheduler = AsyncIOScheduler() - real_scheduler.start(paused=True) - monkeypatch.setattr("litellm.proxy.proxy_server.scheduler", real_scheduler) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) - pc = ProxyConfig() - pc.settings.load_yaml({"scheduled_job_stagger": {"offsets": {"spend_log_cleanup_job": 120}}}) - monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings) - try: - await pc._update_general_settings({"maximum_daily_tag_spend_retention_period": "90d"}) - jobs = real_scheduler.get_jobs() - assert [job.id for job in jobs] == ["spend_log_cleanup_job"] - trigger: Final = jobs[0].trigger - assert isinstance(trigger, _OffsetTrigger), repr(trigger) - assert trigger.offset == timedelta(seconds=120) - assert isinstance(trigger.base, IntervalTrigger) - assert trigger.base.interval == timedelta(days=1) - finally: - real_scheduler.shutdown(wait=False) - - @pytest.mark.asyncio @pytest.mark.parametrize( "bad_schedule", @@ -5094,89 +4995,6 @@ async def test_ProxyConfig__update_general_settings_keeps_the_live_cleanup_job_w real_scheduler.shutdown(wait=False) -@pytest.mark.asyncio -async def test_ProxyConfig__update_general_settings_logs_an_overflowing_interval_once(monkeypatch, caplog): - """An interval that parses but overflows the trigger must keep the live job and log one - error, not a traceback on every sync.""" - from apscheduler.schedulers.asyncio import AsyncIOScheduler - - real_scheduler = AsyncIOScheduler() - real_scheduler.start(paused=True) - monkeypatch.setattr("litellm.proxy.proxy_server.scheduler", real_scheduler) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) - pc = ProxyConfig() - monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings) - try: - await pc._update_general_settings({"maximum_daily_tag_spend_retention_period": "90d"}) - old_trigger = real_scheduler.get_job("spend_log_cleanup_job").trigger - overflowing = { - "maximum_daily_tag_spend_retention_period": "90d", - "maximum_spend_logs_retention_interval": "99999999999d", - } - with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): - for _ in range(5): - await pc._update_general_settings(overflowing) - errors = [record for record in caplog.records if record.levelno >= logging.ERROR] - assert len(errors) == 1, [record.getMessage() for record in errors] - assert real_scheduler.get_job("spend_log_cleanup_job").trigger is old_trigger - finally: - real_scheduler.shutdown(wait=False) - - -@pytest.mark.asyncio -async def test_ProxyConfig__update_general_settings_reschedules_when_only_the_cron_changes(monkeypatch): - real_scheduler = _paused_scheduler(monkeypatch) - pc = ProxyConfig() - pc.settings.load_yaml({"maximum_daily_tag_spend_retention_period": "90d"}) - monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings) - await pc._reschedule_spend_log_cleanup_job() - try: - interval_job = real_scheduler.get_job("spend_log_cleanup_job") - assert interval_job is not None and "hour='3'" not in str(interval_job.trigger) - - await pc._update_general_settings({"maximum_spend_logs_cleanup_cron": "0 3 * * *"}) - cron_job = real_scheduler.get_job("spend_log_cleanup_job") - assert "hour='3'" in str(cron_job.trigger), "cron-only change did not reschedule" - - await pc._update_general_settings({"maximum_spend_logs_cleanup_cron": "0 3 * * *"}) - assert real_scheduler.get_job("spend_log_cleanup_job") is cron_job, "unchanged cron replaced the job" - finally: - real_scheduler.shutdown(wait=False) - - -@pytest.mark.asyncio -async def test_ProxyConfig__update_general_settings_reschedules_a_cron_edit_the_reload_path_already_applied( - monkeypatch, -): - """The periodic reload applies the DB row through _update_config_from_db before - _update_general_settings snapshots the previous schedule, so a cron edited in the DB must - still replace the live job's trigger.""" - from apscheduler.schedulers.asyncio import AsyncIOScheduler - - real_scheduler = AsyncIOScheduler() - real_scheduler.start(paused=True) - monkeypatch.setattr("litellm.proxy.proxy_server.scheduler", real_scheduler) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) - pc = ProxyConfig() - monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings) - try: - first_row = {"maximum_daily_tag_spend_retention_period": "90d", "maximum_spend_logs_cleanup_cron": "0 3 * * *"} - pc.settings.apply_db_row("general_settings", first_row) - await pc._update_general_settings(first_row) - assert "hour='3'" in str(real_scheduler.get_job("spend_log_cleanup_job").trigger) - - edited_row = {**first_row, "maximum_spend_logs_cleanup_cron": "0 5 * * *"} - pc.settings.apply_db_row("general_settings", edited_row) - await pc._update_general_settings(edited_row) - assert "hour='5'" in str(real_scheduler.get_job("spend_log_cleanup_job").trigger), "DB cron edit was ignored" - - pc.settings.apply_db_row("general_settings", edited_row) - await pc._update_general_settings(edited_row) - assert "hour='5'" in str(real_scheduler.get_job("spend_log_cleanup_job").trigger) - finally: - real_scheduler.shutdown(wait=False) - - # --------------------------------------------------------------------------- # ProxyConfig._update_general_settings # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 4f96af7f02c..3d4d8f74c10 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -6,8 +6,6 @@ import json import logging import os import re -import socket -import subprocess import sys import time import types @@ -19,9 +17,6 @@ from typing import Final from unittest import mock from unittest.mock import AsyncMock, MagicMock, call, create_autospec, mock_open, patch -from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stubs - -import click import fastapi.routing import httpx import pytest @@ -34,7 +29,6 @@ from starlette.datastructures import URL import litellm import litellm.proxy.proxy_server as proxy_server_module -from litellm._internal_context import current_service_target from litellm._logging import turn_on_json, verbose_proxy_logger from litellm.caching.caching import RedisCache from litellm.caching.dual_cache import DualCache @@ -56,16 +50,17 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.login_throttle import LoginThrottle from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.banner import LITELLM_BANNER from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendUpdateQueue from litellm.proxy.hooks.parallel_request_limiter_v3 import RequestRateLimiterStash from litellm.proxy.proxy_server import app, aws_error_type_exception_handler, initialize, openai_exception_handler from litellm.litellm_core_utils.model_cost_table import publish -from litellm.proxy.common_utils.banner import LITELLM_BANNER -from tests._master_key import MASTER_KEY from litellm.redis.stores.connection import connect from litellm.redis.stores.rate_limits import RateLimitScope, RateLimitStore from litellm.redis.stores.spend_counters import SpendCounterStore from litellm.rust_bridge._native import NativeRedisStoreError +from tests._master_key import MASTER_KEY +from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stubs from tests.unit.redis.stores.spend_counters_double import FakeSpendCounters PUBLICLY_KNOWN_KEY: Final = "sk-" + "1234" @@ -1057,43 +1052,6 @@ async def test_initialize_scheduled_jobs_registers_cleanup_when_retention_lives_ scheduler.shutdown(wait=False) -@pytest.mark.asyncio -async def test_initialize_scheduled_jobs_does_not_fall_back_to_the_interval_for_a_non_string_cron(monkeypatch): - """A truthy non-string cron is invalid, so startup must log it and register no cleanup job - rather than silently pruning on the default interval the admin never configured.""" - monkeypatch.delenv("DISABLE_PRISMA_SCHEMA_UPDATE", raising=False) - monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False) - from apscheduler.schedulers.asyncio import AsyncIOScheduler - - from litellm.proxy.proxy_server import ProxyStartupEvent - from litellm.proxy.utils import ProxyLogging - - mock_prisma_client = MagicMock() - mock_proxy_logging = MagicMock(spec=ProxyLogging) - mock_proxy_logging.slack_alerting_instance = MagicMock() - mock_proxy_logging.db_spend_update_writer = MagicMock() - settings = {"maximum_daily_tag_spend_retention_period": "30d", "maximum_spend_logs_cleanup_cron": 5} - scheduler = AsyncIOScheduler() - try: - with ( - patch("litellm.proxy.proxy_server.proxy_config", _mock_scheduled_proxy_config()), - patch("litellm.proxy.proxy_server.store_model_in_db", False), - patch("litellm.proxy.proxy_server.general_settings", settings), - patch("litellm.proxy.proxy_server.AsyncIOScheduler", return_value=scheduler), - ): - await ProxyStartupEvent.initialize_scheduled_background_jobs( - general_settings=settings, - prisma_client=mock_prisma_client, - proxy_budget_rescheduler_min_time=1, - proxy_budget_rescheduler_max_time=2, - proxy_batch_write_at=5, - proxy_logging_obj=mock_proxy_logging, - ) - assert scheduler.get_job("spend_log_cleanup_job") is None, "invalid cron fell back to the interval" - finally: - scheduler.shutdown(wait=False) - - @pytest.mark.asyncio async def test_initialize_scheduled_jobs_uses_configured_config_reload_interval(monkeypatch): """ @@ -1756,7 +1714,7 @@ async def test_aaaproxy_startup_master_key(mock_prisma, monkeypatch, tmp_path): with open(config_path, "w") as f: yaml.dump(test_config, f) - print(f"SET ENV VARIABLE - CONFIG_FILE_PATH, str(config_path): {str(config_path)}") + print(f"SET ENV VARIABLE - CONFIG_FILE_PATH, str(config_path): {config_path!s}") # Second setting of CONFIG_FILE_PATH to a different value monkeypatch.setenv("CONFIG_FILE_PATH", str(config_path)) print(f"config_path: {config_path}") @@ -1775,7 +1733,7 @@ async def test_aaaproxy_startup_master_key(mock_prisma, monkeypatch, tmp_path): yaml.dump(empty_config, f) monkeypatch.setenv("LITELLM_MASTER_KEY", test_env_master_key) - print("test_env_master_key: {}".format(test_env_master_key)) + print(f"test_env_master_key: {test_env_master_key}") async with proxy_startup_event(app): from litellm.proxy.proxy_server import master_key @@ -2098,7 +2056,7 @@ def test_team_info_masking(): all_teams_config=[team1_info], ) - print("Got exception: {}".format(exc_info.value)) + print(f"Got exception: {exc_info.value}") assert "secret-test-key" not in str(exc_info.value) assert "public-test-key" not in str(exc_info.value) @@ -2110,7 +2068,6 @@ async def test_get_all_team_models(): """ from unittest.mock import AsyncMock, MagicMock - from litellm.proxy._types import LiteLLM_TeamTable from litellm.proxy.proxy_server import get_all_team_models # Mock team data @@ -4092,7 +4049,7 @@ async def test_load_config_warns_per_worker_login_counters_without_general_setti Redis must hear that its counters are per worker even when the config has no general_settings.""" import logging - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy.auth.login_throttle import warn_login_counters_are_per_worker from litellm.proxy.proxy_server import ProxyConfig @@ -4252,7 +4209,6 @@ async def test_load_environment_variables_blocks_dangerous_keys(): Test that _load_environment_variables rejects dangerous env var keys like PATH, LD_PRELOAD, PYTHONPATH, etc. """ - import logging from litellm.proxy.proxy_server import ProxyConfig @@ -4334,7 +4290,7 @@ async def test_write_config_to_file(monkeypatch): """ Do not write config to file if store_model_in_db is True """ - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import AsyncMock, patch from litellm.proxy.proxy_server import ProxyConfig @@ -4382,7 +4338,7 @@ async def test_write_config_to_file_when_store_model_in_db_false(monkeypatch): """ Test that config IS written to file when store_model_in_db is False """ - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import patch from litellm.proxy.proxy_server import ProxyConfig @@ -4479,7 +4435,7 @@ async def test_async_data_generator_midstream_error(): try: async for data in async_data_generator(mock_response, mock_user_api_key_dict, mock_request_data): yielded_data.append(data) - except Exception as e: + except Exception: # If there's an exception, that's also part of what we want to test pass @@ -5581,7 +5537,7 @@ async def test_add_router_settings_from_db_config_merge_logic(): This tests how router settings from config file and database are combined, including scenarios where nested dictionaries should be properly merged. """ - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import AsyncMock, MagicMock from litellm.proxy.proxy_server import ProxyConfig @@ -6753,7 +6709,7 @@ async def test_init_sso_settings_in_db_error_handling(): """ Test that _init_sso_settings_in_db handles database errors gracefully. """ - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import AsyncMock, MagicMock from litellm.proxy.proxy_server import ProxyConfig @@ -6883,7 +6839,6 @@ async def test_init_hashicorp_vault_config_override_retries_on_transport_error() invocations.append(None) if len(invocations) == 1: raise prisma.errors.ClientNotConnectedError() - return None # No config in DB → function returns early after retry. mock_prisma_client = MagicMock() mock_prisma_client.db.litellm_configoverrides.find_unique = AsyncMock(side_effect=_flaky_find_unique) @@ -7031,9 +6986,7 @@ def test_root_redirect_when_docs_url_not_root_and_redirect_url_set(monkeypatch): routes_to_remove = [] for route in app.routes: if hasattr(route, "path") and route.path == "/": - if hasattr(route, "methods") and "GET" in route.methods: - routes_to_remove.append(route) - elif not hasattr(route, "methods"): # Catch-all routes + if hasattr(route, "methods") and "GET" in route.methods or not hasattr(route, "methods"): routes_to_remove.append(route) for route in routes_to_remove: @@ -9869,7 +9822,7 @@ async def test_reseed_locks_dict_is_bounded(): long-lived deployments with high counter-key churn. Inserting more than the cap evicts the oldest entries. """ - import litellm.constants as constants + from litellm import constants from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed orig_locks = SpendCounterReseed._locks.copy() diff --git a/tests/unit/proxy/test_spend_log_cleanup.py b/tests/unit/proxy/test_spend_log_cleanup.py index 114c970e163..01289c5c37e 100644 --- a/tests/unit/proxy/test_spend_log_cleanup.py +++ b/tests/unit/proxy/test_spend_log_cleanup.py @@ -22,16 +22,20 @@ from litellm.constants import ( ) from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import ( SPEND_LOG_CLEANUP_BOUND_SETTINGS, + SPEND_LOG_RETENTION_JOB, + SPEND_LOG_SIZE_CHECK_JOB, SizeCleanupState, SpendLogCleanup, SpendLogStorage, StorageProbe, TableCleanupResult, advance_size_state, + retention_schedule, ) from litellm.proxy.db.db_transaction_queue.spend_log_cleanup_metrics import ( SpendLogCleanupMetrics, ) +from litellm.proxy.db.scheduled_job_runs import InvalidSchedule, ScheduledJob from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -276,17 +280,8 @@ async def test_cleanup_old_spend_logs_batch_deletion(): # Wire up mocks mock_prisma_client.db = mock_db - # Mock Redis cache and pod_lock_manager - mock_redis_cache = MagicMock() - mock_pod_lock_manager = MagicMock() - mock_pod_lock_manager.redis_cache = mock_redis_cache - mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) - mock_pod_lock_manager.release_lock = AsyncMock() - - # Run cleanup with mocked pod_lock_manager test_settings = {"maximum_spend_logs_retention_period": "7d"} cleaner = SpendLogCleanup(general_settings=test_settings) - cleaner.pod_lock_manager = mock_pod_lock_manager assert cleaner._should_delete_spend_logs() is True await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -325,17 +320,8 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff(): mock_db.execute_raw = AsyncMock(return_value=0) mock_prisma_client.db = mock_db - # Mock Redis cache and pod_lock_manager - mock_redis_cache = MagicMock() - mock_pod_lock_manager = MagicMock() - mock_pod_lock_manager.redis_cache = mock_redis_cache - mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) - mock_pod_lock_manager.release_lock = AsyncMock() - - # Run cleanup with mocked pod_lock_manager test_settings = {"maximum_spend_logs_retention_period": "24h"} cleaner = SpendLogCleanup(general_settings=test_settings) - cleaner.pod_lock_manager = mock_pod_lock_manager assert cleaner._should_delete_spend_logs() is True await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -371,8 +357,6 @@ async def test_cleanup_drops_partitions_when_enabled_and_partitioned(): }, partition_manager=partition_manager, ) - cleaner.pod_lock_manager = MagicMock() - cleaner.pod_lock_manager.redis_cache = None await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -408,8 +392,6 @@ async def test_cleanup_uses_delete_when_partitioning_not_enabled(): general_settings={"maximum_spend_logs_retention_period": "7d"}, partition_manager=partition_manager, ) - cleaner.pod_lock_manager = MagicMock() - cleaner.pod_lock_manager.redis_cache = None await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -442,8 +424,6 @@ async def test_cleanup_uses_delete_when_not_partitioned(): }, partition_manager=partition_manager, ) - cleaner.pod_lock_manager = MagicMock() - cleaner.pod_lock_manager.redis_cache = None await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -468,32 +448,6 @@ async def test_cleanup_old_spend_logs_no_retention_period(): mock_prisma_client.db.execute_raw.assert_not_called() -@pytest.mark.asyncio -async def test_lock_not_released_when_not_acquired(): - """ - Lock release should be skipped when _should_delete_spend_logs returns False - before the lock is ever acquired. - """ - mock_prisma_client = MagicMock() - _wire_tx(mock_prisma_client.db) - mock_prisma_client.db.execute_raw = AsyncMock() - - mock_redis_cache = MagicMock() - mock_pod_lock_manager = MagicMock() - mock_pod_lock_manager.redis_cache = mock_redis_cache - mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) - mock_pod_lock_manager.release_lock = AsyncMock() - - # No retention setting → _should_delete_spend_logs() returns False before lock is acquired - cleaner = SpendLogCleanup(general_settings={}) - cleaner.pod_lock_manager = mock_pod_lock_manager - - await cleaner.cleanup_old_spend_logs(mock_prisma_client) - - mock_pod_lock_manager.acquire_lock.assert_not_called() - mock_pod_lock_manager.release_lock.assert_not_called() - - @pytest.mark.asyncio async def test_integer_retention_treated_as_days(): """ @@ -719,7 +673,6 @@ async def test_cleanup_uses_logger_exception_for_full_traceback(monkeypatch): _wire_tx(mock_prisma_client.db) # Force the outer try/except to fire by making _should_delete_spend_logs raise. cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) - cleaner.pod_lock_manager = None def boom(): raise RuntimeError("simulated prisma timeout") @@ -738,9 +691,8 @@ async def test_cleanup_uses_logger_exception_for_full_traceback(monkeypatch): @pytest.mark.asyncio -async def test_cleanup_releases_lock_after_persistent_batch_failures(monkeypatch): - """Even when batch deletion aborts due to consecutive failures, the pod lock - must still be released so the next scheduled run isn't permanently blocked.""" +async def test_a_run_whose_batches_keep_failing_reports_aborted(monkeypatch: pytest.MonkeyPatch) -> None: + """An aborted run is what the scheduled job records as FAILED, so the abort must reach the report.""" import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 2) @@ -748,23 +700,12 @@ async def test_cleanup_releases_lock_after_persistent_batch_failures(monkeypatch mock_prisma_client = MagicMock() _wire_tx(mock_prisma_client.db) - mock_db = MagicMock() - _wire_tx(mock_db) - mock_db.execute_raw = AsyncMock(side_effect=TimeoutError("DB down")) - mock_prisma_client.db = mock_db - - mock_pod_lock_manager = MagicMock() - mock_pod_lock_manager.redis_cache = MagicMock() - mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) - mock_pod_lock_manager.release_lock = AsyncMock() + mock_prisma_client.db.execute_raw = AsyncMock(side_effect=TimeoutError("DB down")) cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) - cleaner.pod_lock_manager = mock_pod_lock_manager + outcome: Final = await cleaner.cleanup_old_spend_logs(mock_prisma_client) - await cleaner.cleanup_old_spend_logs(mock_prisma_client) - - # Cleanup didn't crash; the abort-after-failures path returned cleanly. - mock_pod_lock_manager.release_lock.assert_awaited_once() + assert outcome == "aborted" def test_cleanup_batch_size_env_var(monkeypatch): @@ -801,7 +742,6 @@ def _mock_prisma_for_retention(side_effect: list) -> "MagicMock": async def test_spend_logs_retention_alone_does_not_touch_the_session_rollup(): client = _mock_prisma_for_retention([0, 0]) cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(client) tables = [call[0][0] for call in client.db.execute_raw.call_args_list] assert any('"LiteLLM_SpendLogs"' in sql for sql in tables) @@ -815,7 +755,6 @@ async def test_spend_logs_retention_alone_does_not_touch_the_session_rollup(): async def test_session_retention_alone_cleans_both_session_rollups_and_the_daily_rollup(): client = _mock_prisma_for_retention([0, 0, 0]) cleaner = SpendLogCleanup(general_settings={"maximum_autorouter_session_retention_period": "365d"}) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(client) calls = client.db.execute_raw.call_args_list tables = [call[0][0] for call in calls] @@ -830,7 +769,6 @@ async def test_session_retention_alone_cleans_both_session_rollups_and_the_daily async def test_health_check_retention_alone_cleans_only_the_health_check_table(): client = _mock_prisma_for_retention([0]) cleaner = SpendLogCleanup(general_settings={"maximum_health_check_retention_period": "30d"}) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(client) tables = [call[0][0] for call in client.db.execute_raw.call_args_list] assert len(tables) == 1 @@ -846,7 +784,6 @@ async def test_health_check_retention_alone_cleans_only_the_health_check_table() async def test_daily_tag_spend_retention_alone_prunes_only_that_table_by_calendar_day(): client = _mock_prisma_for_retention([0]) cleaner = SpendLogCleanup(general_settings={"maximum_daily_tag_spend_retention_period": "90d"}) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(client) tables = [call[0][0] for call in client.db.execute_raw.call_args_list] assert len(tables) == 1 @@ -859,7 +796,6 @@ async def test_daily_tag_spend_retention_alone_prunes_only_that_table_by_calenda async def test_spend_logs_retention_alone_keeps_daily_tag_spend_forever(): client = _mock_prisma_for_retention([0, 0]) cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(client) tables = [call[0][0] for call in client.db.execute_raw.call_args_list] assert not any('"LiteLLM_DailyTagSpend"' in sql for sql in tables) @@ -875,7 +811,6 @@ async def test_each_retention_key_cuts_off_at_its_own_horizon(): "maximum_health_check_retention_period": "30d", } ) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(client) cutoffs = { ( @@ -903,7 +838,6 @@ async def test_each_retention_key_cuts_off_at_its_own_horizon(): async def test_no_retention_keys_means_no_cleanup_at_all(): client = _mock_prisma_for_retention([]) cleaner = SpendLogCleanup(general_settings={}) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(client) assert client.db.execute_raw.await_count == 0 @@ -981,7 +915,6 @@ async def test_run_budget_is_shared_across_tables_not_granted_per_table(): }, clock=clock, ) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -1014,7 +947,6 @@ async def test_cleanup_groups_share_budget_so_health_checks_still_get_a_delete() }, clock=clock, ) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -1145,7 +1077,6 @@ async def test_a_bound_changed_after_construction_reaches_the_next_run( every dashboard change inert until the process restarts.""" settings = {"maximum_spend_logs_retention_period": "7d"} cleaner = SpendLogCleanup(general_settings=settings) - cleaner.pod_lock_manager = None assert getattr(cleaner, attribute) != expected settings[setting_name] = setting_value @@ -1163,7 +1094,6 @@ async def test_a_bound_cleared_after_construction_falls_back_to_its_default(clea spells the clear as an explicit None or as an absent key.""" settings = {"maximum_spend_logs_retention_period": "7d", "maximum_spend_logs_cleanup_batch_size": 137} cleaner = SpendLogCleanup(general_settings=settings) - cleaner.pod_lock_manager = None assert cleaner.batch_size == 137 if cleared_to_none: @@ -1203,32 +1133,6 @@ async def test_remaining_rows_probe_is_capped_so_it_cannot_scan_the_table(): assert mock_db.query_raw.call_args[0][2] == SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP -@pytest.mark.asyncio -async def test_a_run_skipped_because_another_pod_holds_the_lock_is_reported(): - """Operators need to tell "nothing to do" apart from "someone else is doing - it", so a lock-skipped run is recorded under its own outcome.""" - recorded: list[str] = [] - original_record_run = SpendLogCleanupMetrics.record_run - - mock_prisma_client = MagicMock() - _wire_tx(mock_prisma_client.db) - - cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) - cleaner.pod_lock_manager = MagicMock() - cleaner.pod_lock_manager.redis_cache = MagicMock() - cleaner.pod_lock_manager.acquire_lock = AsyncMock(return_value=False) - cleaner.pod_lock_manager.release_lock = AsyncMock() - - SpendLogCleanupMetrics.record_run = classmethod(lambda cls, outcome: recorded.append(outcome)) - try: - await cleaner.cleanup_old_spend_logs(mock_prisma_client) - finally: - SpendLogCleanupMetrics.record_run = original_record_run - - assert recorded == ["skipped_locked"] - cleaner.pod_lock_manager.release_lock.assert_not_awaited() - - @pytest.mark.asyncio async def test_the_outstanding_rows_probe_carries_a_statement_timeout(): """ @@ -1521,10 +1425,6 @@ async def test_a_cancelled_run_records_aborted_and_logs_its_progress_before_re_r mock_prisma_client.db.execute_raw = _execute_raw cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) - cleaner.pod_lock_manager = MagicMock() - cleaner.pod_lock_manager.redis_cache = MagicMock() - cleaner.pod_lock_manager.acquire_lock = AsyncMock(return_value=True) - cleaner.pod_lock_manager.release_lock = AsyncMock() run = asyncio.ensure_future(cleaner.cleanup_old_spend_logs(mock_prisma_client)) await asyncio.wait_for(third_batch_reached.wait(), timeout=5) @@ -1534,7 +1434,6 @@ async def test_a_cancelled_run_records_aborted_and_logs_its_progress_before_re_r assert _runs_recorded("aborted") == aborted_runs_before + 1 assert {outcome: _runs_recorded(outcome) for outcome in _OTHER_OUTCOMES} == other_runs_before - cleaner.pod_lock_manager.release_lock.assert_awaited_once() mock_logger.exception.assert_not_called() (error_call,) = mock_logger.error.call_args_list rendered = error_call[0][0] % error_call[0][1:] @@ -1554,7 +1453,6 @@ async def test_progress_reported_for_a_cancelled_run_is_that_run_only(monkeypatc _wire_tx(mock_prisma_client.db) mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[150, 0, 0]) cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(mock_prisma_client) mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[150, asyncio.CancelledError()]) @@ -1568,7 +1466,7 @@ async def test_progress_reported_for_a_cancelled_run_is_that_run_only(monkeypatc @pytest.mark.asyncio async def test_a_size_pass_that_fires_while_an_age_pass_is_running_in_the_same_pod_waits_for_it(monkeypatch): - """Without Redis there is no pod lock, and the two jobs are scheduled on two SpendLogCleanup instances""" + """Two passes started in one pod, on two SpendLogCleanup instances, must not overlap""" import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module mock_logger = MagicMock() @@ -1593,7 +1491,6 @@ async def test_a_size_pass_that_fires_while_an_age_pass_is_running_in_the_same_p size_client.db = size_db age_cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) - age_cleaner.pod_lock_manager = None size_cleaner = _size_cleaner({}) slow_run = asyncio.ensure_future(age_cleaner.cleanup_old_spend_logs(slow_client)) @@ -1643,7 +1540,6 @@ async def test_spend_logs_backlog_cannot_starve_tool_index_cleanup(): }, clock=clock, ) - cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -1672,7 +1568,6 @@ async def test_run_that_leaves_backlog_logs_a_warning_summary_naming_each_table( "maximum_spend_logs_cleanup_max_batches": 2, } ) - cleaner.pod_lock_manager = None with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -1697,7 +1592,6 @@ async def test_run_that_drains_every_table_logs_the_summary_at_info_not_warning( mock_prisma_client.db = mock_db cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) - cleaner.pod_lock_manager = None with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -1776,7 +1670,6 @@ def _wire_size_probes( def _size_cleaner(settings: dict, **kwargs) -> SpendLogCleanup: cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_size": "1GB", **settings}, **kwargs) - cleaner.pod_lock_manager = None return cleaner @@ -2267,3 +2160,71 @@ async def test_size_state_round_trips_through_its_litellm_config_row_and_a_lost_ with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): await cleaner._save_size_state(prisma_client, _far_deadline(), _RUN_STATE) assert any("Could not save the spend-log size cleanup state" in record.getMessage() for record in caplog.records) + + +@pytest.mark.parametrize( + ("settings", "expected_spec"), + [ + ({}, "interval:86400s"), + ({"maximum_spend_logs_retention_interval": "6h"}, "interval:21600s"), + ( + {"maximum_spend_logs_retention_interval": "6h", "maximum_spend_logs_cleanup_cron": "0 4 * * *"}, + "cron:0 4 * * *", + ), + ], +) +def test_the_retention_job_follows_the_cleanup_cron_when_set_and_the_retention_interval_otherwise( + settings: dict[str, str], expected_spec: str +) -> None: + schedule: Final = retention_schedule(settings) + assert not isinstance(schedule, InvalidSchedule) + assert schedule.spec == expected_spec + + +@pytest.mark.parametrize("cron", [5, ["0 4 * * *"], "not a cron"]) +def test_an_invalid_cleanup_cron_is_rejected_instead_of_falling_back_to_the_retention_interval(cron: object) -> None: + schedule: Final = retention_schedule( + {"maximum_spend_logs_cleanup_cron": cron, "maximum_spend_logs_retention_interval": "1h"} + ) + assert isinstance(schedule, InvalidSchedule) + + +async def test_the_retention_job_records_an_aborted_cleanup_as_failed(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module + + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 1) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"maximum_spend_logs_retention_period": "7d"}) + mock_prisma_client: Final = MagicMock() + _wire_tx(mock_prisma_client.db) + mock_prisma_client.db.execute_raw = AsyncMock(side_effect=TimeoutError("DB down")) + + result: Final = await SPEND_LOG_RETENTION_JOB.run(mock_prisma_client) + + assert result == "FAILED" + + +@pytest.mark.parametrize("job", [SPEND_LOG_RETENTION_JOB, SPEND_LOG_SIZE_CHECK_JOB], ids=["retention", "size"]) +async def test_a_cleanup_job_with_no_rule_configured_succeeds_without_touching_the_database( + monkeypatch: pytest.MonkeyPatch, job: ScheduledJob +) -> None: + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + mock_prisma_client: Final = MagicMock() + mock_prisma_client.db.execute_raw = AsyncMock() + + result: Final = await job.run(mock_prisma_client) + + assert result == "SUCCEEDED" + mock_prisma_client.db.execute_raw.assert_not_awaited() + + +def test_a_cleanup_lease_outlasts_the_configured_run_budget(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"maximum_spend_logs_cleanup_run_budget": "10m"}) + for job in (SPEND_LOG_RETENTION_JOB, SPEND_LOG_SIZE_CHECK_JOB): + assert job.lease().total_seconds() > 600 + + +def test_the_size_check_job_follows_the_size_check_interval(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"maximum_spend_logs_size_check_interval": "6h"}) + schedule: Final = SPEND_LOG_SIZE_CHECK_JOB.schedule() + assert not isinstance(schedule, InvalidSchedule) + assert schedule.spec == "interval:21600s" From b5a70cbc1878c26ea9443f6db9a8d4517793d553 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 12 Nov 2026 10:23:01 +0800 Subject: [PATCH 153/189] feat(ui): add Activity tabs to resource details (#50630) * feat(ui): add Activity tabs to resource details * fix(ui): preserve Activity dates and model breakdowns * fix(ui): clarify team activity visibility * fix(ui): skip unused entity breakdown in model activity --- litellm/proxy/_lazy_openapi_snapshot.json | 12 + .../daily_activity_routes.py | 3 +- .../test_daily_activity_routes.py | 20 ++ .../view_users/MemberRoleSelect.tsx | 36 +++ .../user_info_view.integration.test.tsx | 36 +++ .../_components/view_users/user_info_view.tsx | 51 ++--- .../ResourceActivity.integration.test.tsx | 207 ++++++++++++++++++ .../components/UsagePage/ResourceActivity.tsx | 117 ++++++++++ .../components/UsagePage/dailyActivityApi.ts | 1 + .../model_info/ExistingCredentialDialog.tsx | 39 ++++ .../model_info_view.integration.test.tsx | 37 ++++ .../src/components/model_info_view.tsx | 38 ++-- .../networking.dailyActivity.test.ts | 8 + .../src/components/networking.tsx | 5 +- .../src/components/team/TeamInfo.tsx | 6 + .../team/tabVisibilityUtils.test.ts | 8 +- .../src/components/team/tabVisibilityUtils.ts | 9 +- .../components/templates/key_info_view.tsx | 8 + ui/litellm-dashboard/src/i18n/locales/en.json | 6 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 14 ++ .../src/lib/wildcard_routes.ts | 2 +- 21 files changed, 605 insertions(+), 58 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/MemberRoleSelect.tsx create mode 100644 ui/litellm-dashboard/src/components/UsagePage/ResourceActivity.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/UsagePage/ResourceActivity.tsx create mode 100644 ui/litellm-dashboard/src/components/model_info/ExistingCredentialDialog.tsx diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 36e2cb46e90..b72beea0ba9 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -5146,6 +5146,18 @@ "type": "integer" } }, + { + "description": "Include per-entity usage and key counts.", + "in": "query", + "name": "include_entity_breakdown", + "required": false, + "schema": { + "default": true, + "description": "Include per-entity usage and key counts.", + "title": "Include Entity Breakdown", + "type": "boolean" + } + }, { "in": "query", "name": "agent_ids", diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index b5af673585c..2b5307fc696 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -273,6 +273,7 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver, api_key_limit: Annotated[ int, Query(ge=1, le=constants.USAGE_TOP_API_KEYS_MAX) ] = constants.USAGE_TOP_API_KEYS_DEFAULT, + include_entity_breakdown: Annotated[bool, Query(description="Include per-entity usage and key counts.")] = True, ) -> SpendAnalyticsPaginatedResponse: try: resolved: ResolvedScope = await _resolved_scope( @@ -286,7 +287,7 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver, repository, resolved.scope, entity_metadata_field=resolved.entity_metadata, - include_entity_breakdown=resolver.include_entity_breakdown, + include_entity_breakdown=resolver.include_entity_breakdown and include_entity_breakdown, include_guardrail_breakdown=resolver.include_guardrail_breakdown, api_key_limit=api_key_limit, ) diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index 23ad9efdc70..d0efca7ea55 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -655,6 +655,26 @@ def test_aggregated_routes_return_scoped_results( assert len(body["results"]) == 2, response.text +@pytest.mark.parametrize("model", (None, "popular")) +def test_aggregated_activity_can_omit_entities_without_changing_chart_data( + daily_activity_client: tuple[TestClient, _FakeRepository], model: str | None +) -> None: + client, _ = daily_activity_client + params: Final = {**_DATE_PARAMS, **({"model": model} if model else {})} + full: Final = client.get("/user/daily/activity/aggregated", params=params) + compact: Final = client.get( + "/user/daily/activity/aggregated", params={**params, "include_entity_breakdown": "false"} + ) + assert full.status_code == compact.status_code == 200 + body: Final = full.json() + assert all(day["breakdown"]["entities"] for day in body["results"]) + assert compact.json() == { + **body, + "metadata": {**body["metadata"], "entity_total_api_keys": None}, + "results": [{**day, "breakdown": {**day["breakdown"], "entities": {}}} for day in body["results"]], + } + + def _ptu_sized_router( model_group: str, shares: Mapping[str, int] = MappingProxyType({"team-a": 30, "team-b": 20}) ) -> Router: diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/MemberRoleSelect.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/MemberRoleSelect.tsx new file mode 100644 index 00000000000..bea5df6314b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/MemberRoleSelect.tsx @@ -0,0 +1,36 @@ +import type { ParseKeys } from "i18next"; +import { useTranslation } from "react-i18next"; +import { Field, FieldLabel } from "@/components/ui/field"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { SimpleTooltip } from "@/components/ui/tooltip"; + +const ADD_TEAM_ROLE_FIELD_ID = "add-team-role"; + +const MEMBER_ROLE_OPTIONS: ReadonlyArray<{ value: string; hintKey: ParseKeys }> = [ + { value: "user", hintKey: "users.userInfo.memberRoles.userHint" }, + { value: "admin", hintKey: "users.userInfo.memberRoles.adminHint" }, +]; + +export function MemberRoleSelect({ value, onChange }: { value: string; onChange: (value: string) => void }) { + const { t } = useTranslation(); + return ( + + {t("users.userInfo.addTeamDialog.memberRole")} + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx index d434987b873..e78e4abe42c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx @@ -10,6 +10,7 @@ import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event" import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; import { Profiler } from "react"; import UserInfoView from "./user_info_view"; +import { EMPTY_DAILY_ACTIVITY_RESPONSE } from "@/components/UsagePage/dailyActivityApi"; import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; import type { AutoRouterBenchmarksResponse } from "@/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks"; @@ -122,6 +123,41 @@ describe("UserInfoView add-to-team form", () => { mockListMCPTools.mockResolvedValue({ tools: [] }); }); + it("loads only the selected user's Activity, after opening the tab", async () => { + mockUserDailyActivityAggregatedCall.mockResolvedValue(EMPTY_DAILY_ACTIVITY_RESPONSE); + render(); + const activityTab = await screen.findByRole("tab", { name: "Activity" }); + expect(mockUserDailyActivityAggregatedCall).not.toHaveBeenCalled(); + + fireEvent.click(activityTab); + + expect(await screen.findByText("No daily details available")).toBeInTheDocument(); + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledWith( + "user", + expect.objectContaining({ entityIds: [defaultProps.userId], apiKey: undefined }), + ); + vi.stubGlobal("requestIdleCallback", (callback: () => void) => { + callback(); + return 0; + }); + fireEvent.click(screen.getByRole("button", { name: /\d.* - .*\d/ })); + fireEvent.click(screen.getByRole("button", { name: /Last 7 Days/i })); + fireEvent.click(screen.getByRole("button", { name: "Apply" })); + await waitFor(() => expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledTimes(2)); + const selected = vi.mocked(mockUserDailyActivityAggregatedCall).mock.calls.at(-1)![1]; + fireEvent.click(screen.getByRole("tab", { name: "Overview" })); + fireEvent.click(activityTab); + await waitFor(() => expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledTimes(3)); + expect(mockUserDailyActivityAggregatedCall).toHaveBeenLastCalledWith( + "user", + expect.objectContaining({ + startTime: selected.startTime, + endTime: selected.endTime, + }), + ); + vi.unstubAllGlobals(); + }); + const setup = () => userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); const openAddTeam = async (user: ReturnType, anchorTeam = "Alpha Team") => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx index 89267f121bc..ca2c4264213 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx @@ -1,4 +1,5 @@ -import type { ParseKeys } from "i18next"; +import ResourceActivity from "@/components/UsagePage/ResourceActivity"; +import { MemberRoleSelect } from "./MemberRoleSelect"; import React, { useState } from "react"; import { useTranslation } from "react-i18next"; import { Button } from "@/components/ui/button"; @@ -19,7 +20,6 @@ import { teamMemberDeleteCall, Member, } from "@/components/networking"; -import { SimpleTooltip } from "@/components/ui/tooltip"; import { Field, FieldGroup, FieldLabel } from "@/components/ui/field"; import { Combobox, @@ -29,7 +29,6 @@ import { ComboboxItem, ComboboxList, } from "@/components/ui/combobox"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { hasProxyWideSpendView, rolesWithWriteAccess } from "@/utils/roles"; import { teamDetailHref } from "@/utils/entityLinks"; import { BadgeLink } from "@/components/shared/BadgeLink"; @@ -69,18 +68,12 @@ interface TeamDisplayInfo { } const ADD_TEAM_FIELD_ID = "add-team-team"; -const ADD_TEAM_ROLE_FIELD_ID = "add-team-role"; interface TeamOption { team_id: string; team_alias: string; } -const MEMBER_ROLE_OPTIONS: ReadonlyArray<{ value: string; hintKey: ParseKeys }> = [ - { value: "user", hintKey: "users.userInfo.memberRoles.userHint" }, - { value: "admin", hintKey: "users.userInfo.memberRoles.adminHint" }, -]; - export default function UserInfoView({ userId, onClose, @@ -94,7 +87,7 @@ export default function UserInfoView({ const { t } = useTranslation(); const { premiumUser, userId: signedInUserId } = useAuthorized(); const canViewAutoRouterUsage = hasProxyWideSpendView(userRole); - const canViewSavings = canViewAutoRouterUsage || (Boolean(userId.trim()) && userId === signedInUserId); + const canViewUserUsage = canViewAutoRouterUsage || (Boolean(userId.trim()) && userId === signedInUserId); const activityDateRange = useActivityDateRange(); const [userData, setUserData] = useState(null); const [teamDetails, setTeamDetails] = useState([]); @@ -107,7 +100,7 @@ export default function UserInfoView({ const [invitationLinkData, setInvitationLinkData] = useState(null); const [baseUrl, setBaseUrl] = useState(null); const [activeTab, setActiveTab] = useState(initialTab === 1 ? "details" : "overview"); - const hiddenSavingsTab = activeTab === "savings" && !canViewSavings; + const hiddenUsageTab = (activeTab === "savings" || activeTab === "activity") && !canViewUserUsage; const hiddenRouterTab = activeTab === "auto-router-usage" && !canViewAutoRouterUsage; const [copiedStates, setCopiedStates] = useState>({}); const [isTeamsExpanded, setIsTeamsExpanded] = useState(false); @@ -490,7 +483,7 @@ export default function UserInfoView({ /> setActiveTab(String(v))} className="gap-0" > @@ -501,7 +494,12 @@ export default function UserInfoView({ {t("users.userInfo.tabs.details")} - {canViewSavings && ( + {canViewUserUsage && ( + + {t("shared.resourceActivity.title")} + + )} + {canViewUserUsage && ( {t("users.userInfo.tabs.savings")} @@ -513,6 +511,12 @@ export default function UserInfoView({ )} + {canViewUserUsage && ( + + + + )} + {/* Overview Panel */}

@@ -731,7 +735,7 @@ export default function UserInfoView({ )} - {canViewSavings && ( + {canViewUserUsage && ( {activeTab === "savings" && (userId.trim() ? ( @@ -829,24 +833,7 @@ export default function UserInfoView({ - - {t("users.userInfo.addTeamDialog.memberRole")} - - +
diff --git a/ui/litellm-dashboard/src/components/UsagePage/ResourceActivity.integration.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/ResourceActivity.integration.test.tsx new file mode 100644 index 00000000000..d567445e35e --- /dev/null +++ b/ui/litellm-dashboard/src/components/UsagePage/ResourceActivity.integration.test.tsx @@ -0,0 +1,207 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import * as networking from "@/components/networking"; +import { EMPTY_DAILY_ACTIVITY_METADATA, type DailyActivityAggregatedResponse } from "./dailyActivityApi"; +import ResourceActivity from "./ResourceActivity"; + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: vi.fn() })); + +const auth = { + accessToken: "test-session", + userId: "user-1", + userRole: "Admin", +} as ReturnType; + +const response: DailyActivityAggregatedResponse = { + results: [ + { + date: "2026-11-01", + metrics: { + spend: 12.34, + api_requests: 42, + successful_requests: 40, + failed_requests: 2, + total_tokens: 4200, + prompt_tokens: 3000, + completion_tokens: 1200, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + breakdown: {}, + }, + ], + metadata: { + ...EMPTY_DAILY_ACTIVITY_METADATA, + total_spend: 12.34, + total_api_requests: 42, + total_successful_requests: 40, + total_failed_requests: 2, + total_tokens: 4200, + }, +}; + +describe("ResourceActivity", () => { + afterEach(() => vi.useRealTimers()); + + it.each(["team", "user"] as const)("shows the returned model breakdown for a %s", async (scope) => { + vi.mocked(networking.dailyActivityAggregatedCall).mockResolvedValue({ + ...response, + results: [ + { + ...response.results[0], + breakdown: { + models: { + "example-model": { metrics: response.results[0].metrics }, + }, + }, + }, + ], + }); + render(); + expect(await screen.findByRole("cell", { name: "example-model" })).toBeInTheDocument(); + }); + + it("shows matching recorded models and their totals for a wildcard, in one request", async () => { + vi.mocked(networking.dailyActivityAggregatedCall).mockResolvedValue({ + ...response, + metadata: { ...response.metadata, total_spend: 124.68 }, + results: [ + { + ...response.results[0], + breakdown: { + models: { + "openai/model-one": { metrics: response.results[0].metrics }, + "openai/model-two": { metrics: response.results[0].metrics }, + "anthropic/other-model": { metrics: { ...response.results[0].metrics, spend: 100 } }, + }, + }, + }, + ], + }); + render(); + expect(await screen.findByText("$24.68")).toBeInTheDocument(); + expect(screen.getByText("openai/model-one")).toBeInTheDocument(); + expect(screen.getByText("openai/model-two")).toBeInTheDocument(); + expect(screen.queryByText("anthropic/other-model")).not.toBeInTheDocument(); + expect(networking.dailyActivityAggregatedCall).toHaveBeenCalledExactlyOnceWith( + "user", + expect.objectContaining({ model: undefined, includeEntityBreakdown: false }), + ); + }); + + it("keeps wildcard usage within the viewer's scope and shows empty when no model matches", async () => { + vi.mocked(useAuthorized).mockReturnValue({ ...auth, userRole: "Internal User" }); + render(); + expect(await screen.findByText("No daily details available")).toBeInTheDocument(); + expect(screen.queryByText("$12.34")).not.toBeInTheDocument(); + expect(networking.dailyActivityAggregatedCall).toHaveBeenCalledWith( + "user", + expect.objectContaining({ model: undefined, entityIds: [auth.userId] }), + ); + }); + + it("includes today's UTC bucket for a team when the local date is still yesterday", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2026-11-09T01:00:00Z")); + vi.spyOn(networking, "formatDate").mockImplementation((date) => + new Date(date.getTime() - 8 * 60 * 60 * 1000).toISOString().slice(0, 10), + ); + render(); + expect(await screen.findByText("$12.34")).toBeInTheDocument(); + const request = vi.mocked(networking.dailyActivityAggregatedCall).mock.calls[0][1]; + expect(request.endTime.getFullYear()).toBe(2026); + expect(request.endTime.getMonth()).toBe(10); + expect(request.endTime.getDate()).toBe(9); + }); + + beforeEach(() => { + vi.restoreAllMocks(); + vi.mocked(useAuthorized).mockReturnValue(auth); + vi.spyOn(networking, "dailyActivityAggregatedCall").mockResolvedValue(response); + }); + + it.each([ + { scope: "key", id: "key-1", entity: "user", filter: { apiKey: "key-1", entityIds: null } }, + { scope: "team", id: "team-1", entity: "team", filter: { entityIds: ["team-1"] } }, + { scope: "user", id: "user-2", entity: "user", filter: { entityIds: ["user-2"] } }, + { scope: "model", id: "example-model", entity: "user", filter: { model: "example-model", entityIds: null } }, + ] as const)("shows existing Activity metrics filtered to the $scope", async ({ scope, id, entity, filter }) => { + render(); + + expect(await screen.findByText("$12.34")).toBeInTheDocument(); + expect(screen.getByText("42")).toBeInTheDocument(); + const expectedRequest = { + ...filter, + includeEntityBreakdown: scope === "model" ? false : undefined, + startTime: expect.any(Date), + endTime: expect.any(Date), + }; + expect(networking.dailyActivityAggregatedCall).toHaveBeenCalledWith( + entity, + expect.objectContaining(expectedRequest), + ); + }); + + it("keeps a non-admin key view within that user's spend scope", async () => { + vi.mocked(useAuthorized).mockReturnValue({ ...auth, userRole: "Internal User" }); + render(); + + expect(await screen.findByText("$12.34")).toBeInTheDocument(); + expect(screen.getByText("Showing your usage only.")).toBeInTheDocument(); + expect(networking.dailyActivityAggregatedCall).toHaveBeenCalledWith( + "user", + expect.objectContaining({ entityIds: ["user-1"], apiKey: "key-1" }), + ); + }); + + it("explains that team Activity is limited to the keys the viewer can access", async () => { + vi.mocked(useAuthorized).mockReturnValue({ ...auth, userRole: "Internal User" }); + render(); + + expect(await screen.findByText("$12.34")).toBeInTheDocument(); + expect(screen.getByText("Showing usage for the team keys you have permission to view.")).toBeInTheDocument(); + expect(networking.dailyActivityAggregatedCall).toHaveBeenCalledWith( + "team", + expect.objectContaining({ entityIds: ["team-1"] }), + ); + }); + + it("does not query another user's spend for a non-admin", () => { + vi.mocked(useAuthorized).mockReturnValue({ ...auth, userRole: "Internal User" }); + render(); + + expect(networking.dailyActivityAggregatedCall).not.toHaveBeenCalled(); + expect(screen.queryByText("$12.34")).not.toBeInTheDocument(); + }); + + it("shows a retryable error instead of zero usage when the request fails", async () => { + vi.mocked(networking.dailyActivityAggregatedCall).mockRejectedValueOnce(new Error("Activity unavailable")); + render(); + + expect(await screen.findByRole("alert")).toHaveTextContent("Activity unavailable"); + expect(screen.queryByText("$12.34")).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: /retry/i })); + expect(await screen.findByText("$12.34")).toBeInTheDocument(); + }); + + it("replaces the previous resource's data when navigating to another resource", async () => { + const { rerender } = render(); + expect(await screen.findByText("$12.34")).toBeInTheDocument(); + + vi.mocked(networking.dailyActivityAggregatedCall).mockResolvedValue({ + results: [], + metadata: EMPTY_DAILY_ACTIVITY_METADATA, + }); + rerender(); + + expect(await screen.findByText("No daily details available")).toBeInTheDocument(); + expect(screen.queryByText("$12.34")).not.toBeInTheDocument(); + await waitFor(() => + expect(networking.dailyActivityAggregatedCall).toHaveBeenLastCalledWith( + "team", + expect.objectContaining({ entityIds: ["team-2"] }), + ), + ); + }); +}); diff --git a/ui/litellm-dashboard/src/components/UsagePage/ResourceActivity.tsx b/ui/litellm-dashboard/src/components/UsagePage/ResourceActivity.tsx new file mode 100644 index 00000000000..f0eb6cf66a3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/UsagePage/ResourceActivity.tsx @@ -0,0 +1,117 @@ +"use client"; + +import { useMemo } from "react"; +import { useTranslation } from "react-i18next"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { + useActivityDateRange, + type ActivityDateRange, +} from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange"; +import { benchmarksWindow } from "@/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks"; +import { useAggregatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity"; +import { getTopModels } from "@/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageAggregations"; +import { ActivityMetrics, ModelSection, processActivityData } from "@/components/activity_metrics"; +import { dailyActivityAggregatedCall } from "@/components/networking"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import { ChartLoader } from "@/components/shared/chart_loader"; +import { ErrorState } from "@/components/shared/ErrorState"; +import { hasProxyWideSpendView, spendScopeUserId } from "@/utils/roles"; +import { EMPTY_DAILY_ACTIVITY_METADATA, toDailyData, type DailyActivityRequest } from "./dailyActivityApi"; +import { keyDetailFromResponse, overallUsageMetrics, reportingKeyId } from "./keyActivityData"; +import { matchesWildcard } from "@/lib/wildcard_routes"; + +interface ResourceActivityProps { + scope: "key" | "team" | "user" | "model"; + id: string; + activity?: ActivityDateRange; +} + +export default function ResourceActivity({ scope, id, activity }: ResourceActivityProps) { + const { t } = useTranslation(); + const { accessToken, userId, userRole } = useAuthorized(); + const localActivity = useActivityDateRange(); + const { dateValue, onDateChange } = activity ?? localActivity; + const wildcard = scope === "model" && id.includes("*"); + const entity = scope === "team" ? "team" : "user"; + const entityId = scope === "team" || scope === "user" ? id : spendScopeUserId(userRole, userId); + const apiKey = scope === "key" ? reportingKeyId(id) : undefined; + const canView = scope !== "user" || hasProxyWideSpendView(userRole) || id === userId; + const hasAccess = accessToken && id && canView; + const scopedToViewer = scope !== "user" && !hasProxyWideSpendView(userRole); + const request: DailyActivityRequest | null = + hasAccess && dateValue.from && dateValue.to + ? { + accessToken, + startTime: dateValue.from, + endTime: + scope === "team" ? new Date(`${benchmarksWindow(dateValue, new Date()).end_date}T12:00:00`) : dateValue.to, + entityIds: entityId ? [entityId] : null, + apiKey, + apiKeyLimit: 1, + includeEntityBreakdown: scope === "model" ? false : undefined, + model: scope === "model" && !wildcard ? id : undefined, + includeCurrentUtcDay: true, + } + : null; + const { data, loading, failed, errorMessage, reload } = useAggregatedDailyActivity({ + fetch: () => dailyActivityAggregatedCall(entity, request as DailyActivityRequest), + enabled: request !== null, + deps: [accessToken, scope, id, entityId, dateValue.from, dateValue.to], + }); + const results = useMemo(() => toDailyData(data), [data]); + const metrics = useMemo( + () => + (apiKey ? keyDetailFromResponse(data, apiKey, []) : undefined) ?? { + ...overallUsageMetrics(results, data.metadata ?? EMPTY_DAILY_ACTIVITY_METADATA), + top_models: + scope === "team" || scope === "user" + ? getTopModels(results, "models", Infinity).map(({ key, ...model }) => ({ model: key, ...model })) + : [], + }, + [data, results, apiKey, scope], + ); + const modelMetrics = useMemo( + () => + wildcard + ? Object.fromEntries( + Object.entries(processActivityData({ results }, "models", [])).filter(([model]) => + matchesWildcard(id, model), + ), + ) + : {}, + [results, wildcard, id], + ); + + if (!request) return null; + + const content = (() => { + if (loading) return ; + if (failed) return ; + if (data.results.length === 0 || (wildcard && Object.keys(modelMetrics).length === 0)) { + return

{t("usage.keyActivity.noDailyDetails")}

; + } + return wildcard ? ( + + ) : ( + + ); + })(); + + return ( +
+
+ {t("shared.scopedSavingsTab.bucketedByUtc")} + +
+ {scopedToViewer && ( +

+ {t(scope === "team" ? "shared.resourceActivity.teamScope" : "shared.resourceActivity.ownUsage")} +

+ )} + {scope === "model" && ( +

{t("shared.resourceActivity.modelScope", { model: id })}

+ )} + {content} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/UsagePage/dailyActivityApi.ts b/ui/litellm-dashboard/src/components/UsagePage/dailyActivityApi.ts index c3201b563b1..4c8ff5dc087 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/dailyActivityApi.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/dailyActivityApi.ts @@ -26,6 +26,7 @@ export interface DailyActivityRequest { model?: string | null; includeCurrentUtcDay?: boolean; apiKeyLimit?: number; + includeEntityBreakdown?: boolean; } export const EMPTY_DAILY_ACTIVITY_METADATA: DailyActivityMetadata = { diff --git a/ui/litellm-dashboard/src/components/model_info/ExistingCredentialDialog.tsx b/ui/litellm-dashboard/src/components/model_info/ExistingCredentialDialog.tsx new file mode 100644 index 00000000000..fae378afbcc --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_info/ExistingCredentialDialog.tsx @@ -0,0 +1,39 @@ +import { useTranslation } from "react-i18next"; +import type { CredentialItem } from "@/components/networking"; +import { CredentialIdentity } from "@/components/shared/CredentialIdentity"; +import { credentialLabelsByName } from "@/components/shared/credentialOptions"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; + +export function ExistingCredentialDialog({ + open, + onClose, + credentialName, + credentials, +}: { + open: boolean; + onClose: () => void; + credentialName: string; + credentials: CredentialItem[]; +}) { + const { t } = useTranslation(); + return ( + !next && onClose()}> + + + {t("modelInfoView.existingCredentialDialog.title")} + + + + + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx index 80ade2182a1..62043d2daed 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx @@ -4,6 +4,7 @@ import userEvent from "@testing-library/user-event"; import React, { ReactNode } from "react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import ModelInfoView from "./model_info_view"; +import { EMPTY_DAILY_ACTIVITY_RESPONSE } from "./UsagePage/dailyActivityApi"; import { toast } from "@/lib/toast"; import * as networking from "./networking"; import { useUserLookup } from "@/app/(dashboard)/hooks/users/useUsers"; @@ -20,6 +21,7 @@ vi.mock("../../utils/dataUtils", () => ({ })); vi.mock("./networking", () => ({ + dailyActivityAggregatedCall: vi.fn(), credentialGetCall: vi.fn(), credentialListCall: vi.fn(), getGuardrailsList: vi.fn(), @@ -198,6 +200,41 @@ describe("ModelInfoView", () => { const wrapper = ({ children }: { children: ReactNode }) => React.createElement(QueryClientProvider, { client: queryClient }, children); + it("loads Activity for the underlying model only when its tab is opened", async () => { + vi.mocked(networking.dailyActivityAggregatedCall).mockReset().mockResolvedValue(EMPTY_DAILY_ACTIVITY_RESPONSE); + render(, { wrapper }); + const activityTab = await screen.findByRole("tab", { name: "Activity" }); + expect(networking.dailyActivityAggregatedCall).not.toHaveBeenCalled(); + + fireEvent.click(activityTab); + + expect(await screen.findByText("No daily details available")).toBeInTheDocument(); + expect(networking.dailyActivityAggregatedCall).toHaveBeenCalledWith( + "user", + expect.objectContaining({ model: defaultModelData.litellm_params.model }), + ); + vi.stubGlobal("requestIdleCallback", (callback: () => void) => { + callback(); + return 0; + }); + fireEvent.click(screen.getByRole("button", { name: /\d.* - .*\d/ })); + fireEvent.click(screen.getByRole("button", { name: /Last 7 Days/i })); + fireEvent.click(screen.getByRole("button", { name: "Apply" })); + await waitFor(() => expect(networking.dailyActivityAggregatedCall).toHaveBeenCalledTimes(2)); + const selected = vi.mocked(networking.dailyActivityAggregatedCall).mock.calls.at(-1)![1]; + fireEvent.click(screen.getByRole("tab", { name: "Overview" })); + fireEvent.click(activityTab); + await waitFor(() => expect(networking.dailyActivityAggregatedCall).toHaveBeenCalledTimes(3)); + expect(networking.dailyActivityAggregatedCall).toHaveBeenLastCalledWith( + "user", + expect.objectContaining({ + startTime: selected.startTime, + endTime: selected.endTime, + }), + ); + vi.unstubAllGlobals(); + }); + it("should render", async () => { render(, { wrapper }); await waitFor(() => { diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 226cc06e881..4f3692492b8 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -1,3 +1,5 @@ +import ResourceActivity from "@/components/UsagePage/ResourceActivity"; +import { useActivityDateRange } from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange"; import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import { useModelHub, useModelsInfo } from "@/app/(dashboard)/hooks/models/useModels"; import { useQueryClient } from "@tanstack/react-query"; @@ -31,8 +33,7 @@ import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import DeleteResourceModal from "./common_components/DeleteResourceModal"; import EditAutoRouterModal from "./edit_auto_router/edit_auto_router_modal"; import ReuseCredentialsModal from "./model_add/reuse_credentials"; -import { credentialLabelsByName } from "./shared/credentialOptions"; -import { CredentialIdentity } from "./shared/CredentialIdentity"; +import { ExistingCredentialDialog } from "./model_info/ExistingCredentialDialog"; import { toast } from "@/lib/toast"; import { CredentialItem, @@ -78,6 +79,7 @@ export default function ModelInfoView({ }: ModelInfoViewProps) { const { t } = useTranslation(); const queryClient = useQueryClient(); + const activityDateRange = useActivityDateRange(); const [localModelData, setLocalModelData] = useState(null); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [deleteLoading, setDeleteLoading] = useState(false); @@ -615,6 +617,9 @@ export default function ModelInfoView({ {t("modelInfoView.tabs.overview")} + + {t("shared.resourceActivity.title")} + {t("modelInfoView.tabs.rawJson")} @@ -751,6 +756,10 @@ export default function ModelInfoView({ + + + + 
@@ -803,25 +812,12 @@ export default function ModelInfoView({
           setIsCredentialModalOpen={setIsCredentialModalOpen}
         />
       ) : (
-         !open && setIsCredentialModalOpen(false)}>
-          
-            
-              {t("modelInfoView.existingCredentialDialog.title")}
-            
-            
-            
-              
-            
-          
-        
+         setIsCredentialModalOpen(false)}
+          credentialName={modelData.litellm_params.litellm_credential_name ?? ""}
+          credentials={credentialsList}
+        />
       )}
 
       {isUpdateCredentialsModalOpen && accessToken && (
diff --git a/ui/litellm-dashboard/src/components/networking.dailyActivity.test.ts b/ui/litellm-dashboard/src/components/networking.dailyActivity.test.ts
index 6821d26fb33..09ac797c841 100644
--- a/ui/litellm-dashboard/src/components/networking.dailyActivity.test.ts
+++ b/ui/litellm-dashboard/src/components/networking.dailyActivity.test.ts
@@ -41,6 +41,14 @@ afterEach(() => {
 });
 
 describe("dailyActivityAggregatedCall", () => {
+  it.each([undefined, true, false])("sends the entity breakdown choice %s without changing defaults", async (value) => {
+    const mockFetch = captureFetch();
+    await dailyActivityAggregatedCall("user", req({ includeEntityBreakdown: value }));
+    expect(requestedUrl(mockFetch).searchParams.get("include_entity_breakdown")).toBe(
+      value === undefined ? null : String(value),
+    );
+  });
+
   it.each<[DailyActivityEntity, string]>([
     ["user", "user_id"],
     ["team", "team_ids"],
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx
index 7be345de09a..32e2d0357f6 100644
--- a/ui/litellm-dashboard/src/components/networking.tsx
+++ b/ui/litellm-dashboard/src/components/networking.tsx
@@ -1395,7 +1395,10 @@ export const dailyActivityAggregatedCall = (
 ): Promise =>
   apiClient.get(`/${entity}/daily/activity/aggregated`, {
     accessToken: req.accessToken,
-    query: dailyActivityQuery(entity, req, req.apiKeyLimit === undefined ? {} : { api_key_limit: req.apiKeyLimit }),
+    query: dailyActivityQuery(entity, req, {
+      api_key_limit: req.apiKeyLimit,
+      include_entity_breakdown: req.includeEntityBreakdown,
+    }),
   });
 
 export const subscriptionUsageCall = (
diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx
index 69fac298b0f..039a9da71d0 100644
--- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx
+++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx
@@ -1,3 +1,4 @@
+import ResourceActivity from "@/components/UsagePage/ResourceActivity";
 import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
 import type { components } from "@/lib/http/schema";
 import useCan from "@/app/(dashboard)/hooks/useCan";
@@ -1607,6 +1608,11 @@ const TeamInfoView: React.FC = ({
), }, + { + key: TEAM_INFO_TAB_KEYS.ACTIVITY, + label: t(TEAM_INFO_TAB_LABELS[TEAM_INFO_TAB_KEYS.ACTIVITY]), + children: , + }, { key: TEAM_INFO_TAB_KEYS.MY_USER, label: t(TEAM_INFO_TAB_LABELS[TEAM_INFO_TAB_KEYS.MY_USER]), diff --git a/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.test.ts b/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.test.ts index e115499dcb0..d59004ace74 100644 --- a/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.test.ts +++ b/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.test.ts @@ -23,13 +23,19 @@ describe("team_info_tabs", () => { describe("getTeamInfoVisibleTabs", () => { it("returns overview, my user, and virtual keys when user cannot edit team", () => { const tabs = getTeamInfoVisibleTabs(false); - expect(tabs).toEqual([TEAM_INFO_TAB_KEYS.OVERVIEW, TEAM_INFO_TAB_KEYS.MY_USER, TEAM_INFO_TAB_KEYS.VIRTUAL_KEYS]); + expect(tabs).toEqual([ + TEAM_INFO_TAB_KEYS.OVERVIEW, + TEAM_INFO_TAB_KEYS.ACTIVITY, + TEAM_INFO_TAB_KEYS.MY_USER, + TEAM_INFO_TAB_KEYS.VIRTUAL_KEYS, + ]); }); it("returns all tabs when user can edit team", () => { const tabs = getTeamInfoVisibleTabs(true); expect(tabs).toEqual([ TEAM_INFO_TAB_KEYS.OVERVIEW, + TEAM_INFO_TAB_KEYS.ACTIVITY, TEAM_INFO_TAB_KEYS.MY_USER, TEAM_INFO_TAB_KEYS.VIRTUAL_KEYS, TEAM_INFO_TAB_KEYS.MEMBERS, diff --git a/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.ts b/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.ts index 7e9c68a3fcf..a0616b7c174 100644 --- a/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.ts +++ b/ui/litellm-dashboard/src/components/team/tabVisibilityUtils.ts @@ -7,6 +7,7 @@ import type { ParseKeys } from "i18next"; export const TEAM_INFO_TAB_KEYS = { OVERVIEW: "overview", + ACTIVITY: "activity", MY_USER: "my-user", VIRTUAL_KEYS: "virtual-keys", MEMBERS: "members", @@ -15,6 +16,7 @@ export const TEAM_INFO_TAB_KEYS = { } as const; export const TEAM_INFO_TAB_LABELS: Record = { + [TEAM_INFO_TAB_KEYS.ACTIVITY]: "shared.resourceActivity.title", [TEAM_INFO_TAB_KEYS.OVERVIEW]: "teams.teamInfo.tabs.overview", [TEAM_INFO_TAB_KEYS.MY_USER]: "teams.teamInfo.tabs.myUser", [TEAM_INFO_TAB_KEYS.VIRTUAL_KEYS]: "teams.teamInfo.tabs.virtualKeys", @@ -29,7 +31,12 @@ export const TEAM_INFO_TAB_LABELS: Record = { * - Members, Member Permissions, Settings: only when canEditTeam is true */ export function getTeamInfoVisibleTabs(canEditTeam: boolean): readonly string[] { - const baseTabs = [TEAM_INFO_TAB_KEYS.OVERVIEW, TEAM_INFO_TAB_KEYS.MY_USER, TEAM_INFO_TAB_KEYS.VIRTUAL_KEYS]; + const baseTabs = [ + TEAM_INFO_TAB_KEYS.OVERVIEW, + TEAM_INFO_TAB_KEYS.ACTIVITY, + TEAM_INFO_TAB_KEYS.MY_USER, + TEAM_INFO_TAB_KEYS.VIRTUAL_KEYS, + ]; if (canEditTeam) { return [ ...baseTabs, diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 120770f2001..c580165b842 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -1,3 +1,4 @@ +import ResourceActivity from "@/components/UsagePage/ResourceActivity"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { useModelAccessGroupNames } from "@/app/(dashboard)/hooks/models/useModels"; import { useProjects } from "@/app/(dashboard)/hooks/projects/useProjects"; @@ -746,6 +747,9 @@ export default function KeyInfoView({ {t("keyDetail.keyInfoView.tabs.overview")} + + {t("shared.resourceActivity.title")} + {t("keyDetail.keyInfoView.tabs.savings")} @@ -950,6 +954,10 @@ export default function KeyInfoView({
+ + + + {/* Savings Panel. No keepMounted: this tab sweeps the daily rollup, and staying mounted would fire that request on every key page open for people who never look at it. */} diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index 8e15979afb1..b41302e1402 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -8610,6 +8610,12 @@ } }, "shared": { + "resourceActivity": { + "title": "Activity", + "ownUsage": "Showing your usage only.", + "teamScope": "Showing usage for the team keys you have permission to view.", + "modelScope": "Usage for {{model}} across deployments of this model." + }, "credentialOptions": { "none": "None" }, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 11c06527413..a446cf87857 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -54218,6 +54218,8 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + /** @description Include per-entity usage and key counts. */ + include_entity_breakdown?: boolean; agent_ids?: string | null; start_date?: string | null; end_date?: string | null; @@ -59906,6 +59908,8 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + /** @description Include per-entity usage and key counts. */ + include_entity_breakdown?: boolean; end_user_ids?: string | null; start_date?: string | null; end_date?: string | null; @@ -60963,6 +60967,8 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + /** @description Include per-entity usage and key counts. */ + include_entity_breakdown?: boolean; end_user_ids?: string | null; start_date?: string | null; end_date?: string | null; @@ -70582,6 +70588,8 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + /** @description Include per-entity usage and key counts. */ + include_entity_breakdown?: boolean; organization_ids?: string | null; start_date?: string | null; end_date?: string | null; @@ -76854,6 +76862,8 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + /** @description Include per-entity usage and key counts. */ + include_entity_breakdown?: boolean; start_date?: string | null; end_date?: string | null; model?: string | null; @@ -77544,6 +77554,8 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + /** @description Include per-entity usage and key counts. */ + include_entity_breakdown?: boolean; team_ids?: string | null; start_date?: string | null; end_date?: string | null; @@ -79967,6 +79979,8 @@ export interface operations { parameters: { query?: { api_key_limit?: number; + /** @description Include per-entity usage and key counts. */ + include_entity_breakdown?: boolean; /** @description Start date in YYYY-MM-DD format */ start_date?: string | null; /** @description End date in YYYY-MM-DD format */ diff --git a/ui/litellm-dashboard/src/lib/wildcard_routes.ts b/ui/litellm-dashboard/src/lib/wildcard_routes.ts index 607e5d106a3..1527e43e558 100644 --- a/ui/litellm-dashboard/src/lib/wildcard_routes.ts +++ b/ui/litellm-dashboard/src/lib/wildcard_routes.ts @@ -7,7 +7,7 @@ export type WildcardRoute = { readonly model: string; readonly sourceGroup: stri // A linear glob scan rather than a RegExp: patterns are admin-controlled model_name values, and a // backtracking regex built from one ("a*a*a*...") can freeze another admin's dashboard. -const matchesWildcard = (pattern: string, name: string): boolean => { +export const matchesWildcard = (pattern: string, name: string): boolean => { const parts = pattern.split("*"); const head = parts[0]; const tail = parts[parts.length - 1]; From 26971a8f1f8ea60a81dc03c3a7cef747fc4112ff Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 12 Nov 2026 10:23:02 +0800 Subject: [PATCH 154/189] feat(alerts): add persisted alerts and status APIs (#50658) * feat(alerts): add persisted alerts and status APIs * fix(alerts): preserve string IDs and complete backend wiring * feat(alerts): support whole-gateway alert scope --- .github/workflows/test-unit.yml | 1 + backend/routes/allowlist.py | 2 + .../20261109013000_add_alerts/migration.sql | 22 ++ .../litellm_proxy_extras/schema.prisma | 17 ++ litellm/integrations/otel/model/spans.py | 1 + litellm/models/alert.py | 131 +++++++++ litellm/proxy/alerting_endpoints/alerts.py | 56 ++++ litellm/proxy/deps/repositories.py | 9 + litellm/proxy/proxy_server.py | 2 + litellm/proxy/schema.prisma | 17 ++ litellm/repositories/alert_repository.py | 63 ++++ schema.prisma | 17 ++ tests/integration/database/test_alerts.py | 214 ++++++++++++++ tests/unit/models/test_alert.py | 68 +++++ .../unit/proxy/alerting_endpoints/__init__.py | 0 .../proxy/alerting_endpoints/test_alerts.py | 161 +++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 273 ++++++++++++++++++ 17 files changed, 1054 insertions(+) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261109013000_add_alerts/migration.sql create mode 100644 litellm/models/alert.py create mode 100644 litellm/proxy/alerting_endpoints/alerts.py create mode 100644 litellm/repositories/alert_repository.py create mode 100644 tests/integration/database/test_alerts.py create mode 100644 tests/unit/models/test_alert.py create mode 100644 tests/unit/proxy/alerting_endpoints/__init__.py create mode 100644 tests/unit/proxy/alerting_endpoints/test_alerts.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 914d984e541..e334ffee64a 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -551,6 +551,7 @@ jobs: tests/unit/proxy/management_helpers tests/unit/proxy/list_api tests/unit/proxy/analytics_endpoints + tests/unit/proxy/alerting_endpoints tests/unit/proxy/decisions_endpoints tests/unit/proxy/memory tests/unit/proxy/agent_endpoints diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index 2d3ccf40175..d6a9bf20296 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -110,6 +110,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( "/v1/rust_control_plane/", # Alerting / email / IP allowlist "/alerting/", + "/alerts/", "/email/", "/add/allowed_ip", "/delete/allowed_ip", @@ -154,6 +155,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset( "/", "/routes", "/lens", + "/alerts", "/openapi.json", "/docs", "/docs/oauth2-redirect", diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261109013000_add_alerts/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261109013000_add_alerts/migration.sql new file mode 100644 index 00000000000..da623a0c6c6 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261109013000_add_alerts/migration.sql @@ -0,0 +1,22 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_Alert" ( + "id" BIGSERIAL PRIMARY KEY, + "detection_key" VARCHAR(64) NOT NULL, + "type" VARCHAR(32) NOT NULL, + "resource_type" VARCHAR(16) NOT NULL, + "resource_id" VARCHAR(1024) NOT NULL, + "status" VARCHAR(16) NOT NULL DEFAULT 'open', + "window_start" TIMESTAMP(3) NOT NULL, + "window_end" TIMESTAMP(3) NOT NULL, + "evidence" JSONB NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_by" VARCHAR(1024), + CHECK ("status" IN ('open', 'resolved', 'dismissed')), + CHECK ("window_end" > "window_start"), + CHECK (octet_length("evidence"::text) <= 8192) +); + +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_Alert_detection_key_key" ON "LiteLLM_Alert"("detection_key"); +CREATE INDEX IF NOT EXISTS "LiteLLM_Alert_status_id_idx" ON "LiteLLM_Alert"("status", "id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 4314a23e011..c02b3d19fa9 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2207,3 +2207,20 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +model LiteLLM_Alert { + id BigInt @id @default(autoincrement()) + detection_key String @unique @client.VarChar(64) + type String @client.VarChar(32) + resource_type String @client.VarChar(16) + resource_id String @client.VarChar(1024) + status String @default("open") @client.VarChar(16) + window_start DateTime + window_end DateTime + evidence Json + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + updated_by String? @client.VarChar(1024) + + @@index([status, id]) +} diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index ad275f64096..b8ebb2d6c6f 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -260,6 +260,7 @@ _PRISMA_VIEWS: Final[frozenset[str]] = frozenset( ) _PRISMA_MODELS: Final[frozenset[str]] = frozenset( ( + "LiteLLM_Alert", "LiteLLM_BudgetTable", "LiteLLM_CredentialsTable", "LiteLLM_ProxyModelTable", diff --git a/litellm/models/alert.py b/litellm/models/alert.py new file mode 100644 index 00000000000..c9a26c6aff9 --- /dev/null +++ b/litellm/models/alert.py @@ -0,0 +1,131 @@ +from datetime import date, datetime, timezone +from typing import Annotated, Final, Literal, Protocol, TypeAlias + +from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, field_validator, model_validator +from typing_extensions import Self + +AlertType: TypeAlias = Literal["spend", "traffic", "latency", "reliability", "model_deprecation"] +AlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] +AlertResourceType: TypeAlias = Literal["gateway", "key", "team", "user", "model"] +MAX_ALERT_ID: Final = 2**63 - 1 + + +def _validate_alert_id(value: str) -> str: + if int(value) > MAX_ALERT_ID: + raise ValueError("Alert ID exceeds the supported range") + return value + + +AlertID: TypeAlias = Annotated[ + str, Field(pattern=r"^[1-9][0-9]{0,18}$", max_length=19), AfterValidator(_validate_alert_id) +] + + +class MetricEvidence(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + value: float = Field(ge=0, allow_inf_nan=False, description="Observed metric value in the alert window.") + baseline: float = Field(ge=0, allow_inf_nan=False, description="Comparison value in the same unit.") + unit: Literal["usd", "requests", "milliseconds", "ratio"] = Field(description="Unit for both values.") + sample_size: int = Field(ge=0, le=MAX_ALERT_ID, description="Number of requests in the alert window.") + + +class DeprecationEvidence(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + deprecation_date: date = Field(description="Date the affected model retires.") + replacement_model: str | None = Field( + default=None, min_length=1, max_length=1024, description="Suggested replacement." + ) + + +class AlertDetection(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + type: AlertType = Field(description="Built-in alert category.") + resource_type: AlertResourceType = Field(description="Kind of affected resource.") + resource_id: str = Field( + min_length=1, + max_length=1024, + description="Resource ID. Whole-gateway alerts use 'gateway'. Keys use their SHA-256 hash, never a secret.", + ) + window_start: AwareDatetime = Field( + description="Inclusive start of the observed window, normalized to UTC milliseconds." + ) + window_end: AwareDatetime = Field( + description="Exclusive end of the observed window, normalized to UTC milliseconds." + ) + evidence: MetricEvidence | DeprecationEvidence = Field(description="Small saved snapshot explaining the alert.") + + @field_validator("window_start", "window_end") + @classmethod + def normalize_window(cls, value: datetime) -> datetime: + return value.astimezone(timezone.utc).replace(microsecond=value.microsecond // 1000 * 1000) + + @model_validator(mode="after") + def validate_detection(self) -> Self: + if self.window_end <= self.window_start: + raise ValueError("window_end must be after window_start") + if self.resource_type == "gateway" and self.resource_id != "gateway": + raise ValueError("Whole-gateway alerts must use resource_id 'gateway'") + if self.resource_type == "key" and ( + len(self.resource_id) != 64 or any(c not in "0123456789abcdef" for c in self.resource_id) + ): + raise ValueError("Key resource IDs must be SHA-256 hashes") + if self.type == "model_deprecation": + if not isinstance(self.evidence, DeprecationEvidence) or self.resource_type != "model": + raise ValueError("Model deprecations require a model resource and deprecation evidence") + return self + expected_unit: Final = { + "spend": "usd", + "traffic": "requests", + "latency": "milliseconds", + "reliability": "ratio", + }[self.type] + if not isinstance(self.evidence, MetricEvidence) or self.evidence.unit != expected_unit: + raise ValueError("Evidence unit must match the alert type") + if self.type == "reliability" and (self.evidence.value > 1 or self.evidence.baseline > 1): + raise ValueError("Error ratios must be between zero and one") + return self + + def identity(self) -> tuple[str, ...]: + return ( + self.type, + self.resource_type, + self.resource_id, + self.window_start.astimezone(timezone.utc).isoformat(), + self.window_end.astimezone(timezone.utc).isoformat(), + ) + + +class Alert(AlertDetection): + id: AlertID = Field(description="Opaque alert ID, also used as a pagination cursor.") + status: AlertStatus = Field(description="Current review status.") + created_at: AwareDatetime = Field(description="When the alert was first saved.") + updated_at: AwareDatetime = Field(description="When its status last changed.") + updated_by: str | None = Field(default=None, max_length=1024, description="User who last changed the status.") + + +class AlertUpdate(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: AlertStatus = Field(description="Mark resolved, dismiss, or reopen the alert.") + + +class AlertPage(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[Alert, ...] = Field(description="Alerts ordered newest first.") + next_cursor: AlertID | None = Field( + default=None, description="Pass as cursor to fetch the next page, or null at the end." + ) + + +class AlertStore(Protocol): + async def record(self, detection: AlertDetection) -> Alert: ... + + async def list(self, status: AlertStatus, limit: int, cursor: int | None) -> AlertPage: ... + + async def get(self, alert_id: int) -> Alert | None: ... + + async def set_status(self, alert_id: int, status: AlertStatus, actor: str) -> Alert | None: ... diff --git a/litellm/proxy/alerting_endpoints/alerts.py b/litellm/proxy/alerting_endpoints/alerts.py new file mode 100644 index 00000000000..fa586ca7a25 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/alerts.py @@ -0,0 +1,56 @@ +from typing import Annotated, Final, TypeAlias + +from fastapi import APIRouter, Depends, HTTPException, Path, Query + +from litellm.models.alert import Alert, AlertID, AlertPage, AlertStatus, AlertStore, AlertUpdate +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.deps.repositories import AlertRepositoryDep +from litellm.types.proxy.auth.user_roles import LitellmUserRoles + +router: Final = APIRouter(tags=["alerts"]) +Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] +AlertPath: TypeAlias = Annotated[AlertID, Path(description="Alert ID returned by GET /alerts.")] + + +def _authorized_store(user: UserAPIKeyAuth, repository: AlertStore | None, *, write: bool) -> AlertStore: + if user.user_role != LitellmUserRoles.PROXY_ADMIN and ( + write or user.user_role != LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ): + raise HTTPException(status_code=403, detail="Only gateway admins can manage alerts") + if repository is None: + raise HTTPException(status_code=503, detail="Alerts require a database connection") + return repository + + +def _found(alert: Alert | None) -> Alert: + if alert is None: + raise HTTPException(status_code=404, detail="Alert not found") + return alert + + +@router.get("/alerts", response_model=AlertPage, summary="List saved alerts") +async def list_alerts( + user: Auth, + repository: AlertRepositoryDep, + status: Annotated[AlertStatus, Query(description="Status to list.")] = "open", + limit: Annotated[int, Query(ge=1, le=100, description="Maximum alerts per page.")] = 25, + cursor: Annotated[AlertID | None, Query(description="next_cursor from the preceding page.")] = None, +) -> AlertPage: + """Read saved alerts, newest first. Gateway admins and view-only admins can read. No usage queries run.""" + store: Final = _authorized_store(user, repository, write=False) + return await store.list(status, limit, int(cursor) if cursor is not None else None) + + +@router.get("/alerts/{alert_id}", response_model=Alert, summary="Read an alert") +async def get_alert(alert_id: AlertPath, user: Auth, repository: AlertRepositoryDep) -> Alert: + """Return the saved evidence and status. Requires a gateway admin or view-only admin.""" + store: Final = _authorized_store(user, repository, write=False) + return _found(await store.get(int(alert_id))) + + +@router.patch("/alerts/{alert_id}", response_model=Alert, summary="Change an alert's status") +async def update_alert(alert_id: AlertPath, data: AlertUpdate, user: Auth, repository: AlertRepositoryDep) -> Alert: + """Resolve, dismiss, or reopen an alert. Only gateway admins can write. Returns the updated alert.""" + store: Final = _authorized_store(user, repository, write=True) + return _found(await store.set_status(int(alert_id), data.status, user.user_id or "proxy-admin")) diff --git a/litellm/proxy/deps/repositories.py b/litellm/proxy/deps/repositories.py index 503314640c2..6e54658d13d 100644 --- a/litellm/proxy/deps/repositories.py +++ b/litellm/proxy/deps/repositories.py @@ -2,6 +2,7 @@ from typing import Annotated, TypeAlias from fastapi import Depends +from litellm.models.alert import AlertStore from litellm.models.deployment_warning import DeploymentWarningIgnoreStore from litellm.models.fallback import RouterFallbackStore from litellm.models.gateway_activity import GatewayActivityStore @@ -9,6 +10,7 @@ from litellm.models.plugin_upload import PluginUploadStore from litellm.models.skill_submission import SkillSubmissionStore from litellm.models.user_banner import UserBannerStore from litellm.proxy.deps.database import DatabaseDep +from litellm.repositories.alert_repository import AlertRepository from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository from litellm.repositories.gateway_activity_repository import GatewayActivityRepository @@ -58,6 +60,13 @@ SkillSubmissionRepositoryDep: TypeAlias = Annotated[ ] +async def get_alert_repository(database: DatabaseDep) -> AlertStore | None: + return None if database is None else AlertRepository(database.prisma) + + +AlertRepositoryDep: TypeAlias = Annotated[AlertStore | None, Depends(get_alert_repository)] + + async def get_router_fallback_repository(database: DatabaseDep) -> RouterFallbackStore | None: return None if database is None else ConfigRepository(database.prisma, use_writer=True) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0c34f77890f..b1a746fbefd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -428,6 +428,7 @@ from litellm.messages.dispatch import messages_target from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling +from litellm.proxy.alerting_endpoints.alerts import router as alerts_router from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( router as model_deprecation_alerts_router, ) @@ -22591,6 +22592,7 @@ app.include_router(router_settings_router) app.include_router(fallback_management_router) app.include_router(cache_settings_router) app.include_router(model_deprecation_alerts_router) +app.include_router(alerts_router) app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(gateway_request_router) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 4314a23e011..c02b3d19fa9 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2207,3 +2207,20 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +model LiteLLM_Alert { + id BigInt @id @default(autoincrement()) + detection_key String @unique @client.VarChar(64) + type String @client.VarChar(32) + resource_type String @client.VarChar(16) + resource_id String @client.VarChar(1024) + status String @default("open") @client.VarChar(16) + window_start DateTime + window_end DateTime + evidence Json + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + updated_by String? @client.VarChar(1024) + + @@index([status, id]) +} diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py new file mode 100644 index 00000000000..1f02d3cdd81 --- /dev/null +++ b/litellm/repositories/alert_repository.py @@ -0,0 +1,63 @@ +import hashlib +import json +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # Prisma exposes generated table actions dynamically + +from litellm.models.alert import Alert, AlertDetection, AlertPage, AlertStatus +from litellm.repositories.prisma_protocols import PrismaRecord, TableActions + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +def _alert(row: PrismaRecord) -> Alert: + values: Final = row.dict() + return Alert.model_validate( + {**{key: value for key, value in values.items() if key != "detection_key"}, "id": str(values["id"])} + ) + + +class AlertRepository: + def __init__(self, prisma_client: "PrismaClient") -> None: + self._table = cast( # cast-ok: the generated Prisma client delegates typed table actions dynamically + TableActions[PrismaRecord], prisma_client.writer_db.litellm_alert + ) + + async def record(self, detection: AlertDetection) -> Alert: + from prisma import Json + from prisma.errors import UniqueViolationError + + key: Final = hashlib.sha256(json.dumps(detection.identity()).encode()).hexdigest() + try: + row: Final = await self._table.create( + data={ + **detection.model_dump(exclude={"evidence"}), + "detection_key": key, + "evidence": Json(detection.evidence.model_dump(mode="json")), + } + ) + return _alert(row) + except UniqueViolationError: + existing: Final = await self._table.find_unique(where={"detection_key": key}) + if existing is None: + raise + return _alert(existing) + + async def list(self, status: AlertStatus, limit: int, cursor: int | None) -> AlertPage: + size: Final = max(1, min(limit, 100)) + rows: Final = await self._table.find_many( + where={"status": status, **({"id": {"lt": cursor}} if cursor is not None else {})}, + order={"id": "desc"}, + take=size + 1, + ) + return AlertPage( + data=tuple(_alert(row) for row in rows[:size]), + next_cursor=str(rows[size - 1].dict()["id"]) if len(rows) > size else None, + ) + + async def get(self, alert_id: int) -> Alert | None: + row: Final = await self._table.find_unique(where={"id": alert_id}) + return None if row is None else _alert(row) + + async def set_status(self, alert_id: int, status: AlertStatus, actor: str) -> Alert | None: + row: Final = await self._table.update(where={"id": alert_id}, data={"status": status, "updated_by": actor}) + return None if row is None else _alert(row) diff --git a/schema.prisma b/schema.prisma index 4314a23e011..c02b3d19fa9 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2207,3 +2207,20 @@ model LiteLLM_SubscriptionAccountTable { @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key") } + +model LiteLLM_Alert { + id BigInt @id @default(autoincrement()) + detection_key String @unique @client.VarChar(64) + type String @client.VarChar(32) + resource_type String @client.VarChar(16) + resource_id String @client.VarChar(1024) + status String @default("open") @client.VarChar(16) + window_start DateTime + window_end DateTime + evidence Json + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + updated_by String? @client.VarChar(1024) + + @@index([status, id]) +} diff --git a/tests/integration/database/test_alerts.py b/tests/integration/database/test_alerts.py new file mode 100644 index 00000000000..b3054edba02 --- /dev/null +++ b/tests/integration/database/test_alerts.py @@ -0,0 +1,214 @@ +import asyncio +import os +import uuid +from collections.abc import AsyncIterator +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Final + +import pytest +import pytest_asyncio + +from litellm.models.alert import Alert, AlertDetection, AlertPage, AlertType +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.alert_repository import AlertRepository +from tests.integration._support.client import Gateway +from tests.integration._support.database import read_rows, write_rows + + +@dataclass(frozen=True, slots=True) +class Alerts: + repository: AlertRepository + resource_id: str + + def detection(self, kind: AlertType = "spend", day: int = 0) -> AlertDetection: + start: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + timedelta(days=day) + evidence: Final = ( + {"deprecation_date": "2026-03-01", "replacement_model": "example/replacement"} + if kind == "model_deprecation" + else { + "value": 0.5 if kind == "reliability" else 20, + "baseline": 0.1 if kind == "reliability" else 2, + "unit": {"spend": "usd", "traffic": "requests", "latency": "milliseconds", "reliability": "ratio"}[ + kind + ], + "sample_size": 100, + } + ) + return AlertDetection.model_validate( + { + "type": kind, + "resource_type": "model", + "resource_id": self.resource_id, + "window_start": start, + "window_end": start + timedelta(days=1), + "evidence": evidence, + } + ) + + +@pytest_asyncio.fixture(loop_scope="function") +async def alerts() -> AsyncIterator[Alerts]: + resource_id: Final = f"alerts-test-{uuid.uuid4().hex}" + client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + yield Alerts(AlertRepository(client), resource_id) + finally: + try: + write_rows('DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s', (resource_id,)) + finally: + await client.disconnect() + + +@pytest.mark.parametrize("kind", ("spend", "traffic", "latency", "reliability", "model_deprecation")) +@pytest.mark.asyncio +async def test_each_alert_type_round_trips_through_api_and_status_changes( + alerts: Alerts, gateway: Gateway, kind: AlertType +) -> None: + detection: Final = alerts.detection(kind) + saved: Final = await alerts.repository.record(detection) + response: Final = gateway.request("GET", f"/alerts/{saved.id}") + assert response.status_code == 200, response.text + assert Alert.model_validate_json(response.content) == saved + assert saved.status == "open" + assert saved.model_dump(include=set(AlertDetection.model_fields)) == detection.model_dump() + for status in ("resolved", "dismissed", "open"): + update: Final = gateway.request("PATCH", f"/alerts/{saved.id}", {"status": status}) + assert update.status_code == 200, update.text + changed: Final = Alert.model_validate_json(update.content) + assert changed.status == status + assert changed.updated_by + assert changed.evidence == saved.evidence + assert changed.created_at == saved.created_at + assert changed.updated_at >= saved.updated_at + assert await alerts.repository.get(int(saved.id)) == changed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ("spend", "traffic", "latency", "reliability")) +async def test_gateway_alert_round_trips_without_colliding_with_resource_alerts( + alerts: Alerts, gateway: Gateway, kind: AlertType +) -> None: + detection: Final = AlertDetection.model_validate( + {**alerts.detection(kind).model_dump(), "resource_type": "gateway", "resource_id": "gateway"} + ) + try: + saved: Final = await alerts.repository.record(detection) + scoped: Final = await alerts.repository.record(detection.model_copy(update={"resource_type": "model"})) + assert scoped.id != saved.id + detail: Final = gateway.request("GET", f"/alerts/{saved.id}") + assert detail.status_code == 200, detail.text + assert Alert.model_validate_json(detail.content) == saved + listed: Final = gateway.request("GET", "/alerts", params={"limit": "2"}) + assert listed.status_code == 200, listed.text + assert AlertPage.model_validate_json(listed.content).data == (scoped, saved) + update: Final = gateway.request("PATCH", f"/alerts/{saved.id}", {"status": "dismissed"}) + assert update.status_code == 200, update.text + dismissed: Final = Alert.model_validate_json(update.content) + assert dismissed.status == "dismissed" + assert await alerts.repository.record(detection) == dismissed + assert await alerts.repository.get(int(scoped.id)) == scoped + finally: + write_rows( + 'DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s AND type=%s AND window_start=%s AND window_end=%s', + ("gateway", kind, detection.window_start, detection.window_end), + ) + + +@pytest.mark.asyncio +async def test_concurrent_detection_retries_preserve_dismissal_and_original_evidence(alerts: Alerts) -> None: + detection: Final = alerts.detection() + duplicates: Final = await asyncio.gather(*(alerts.repository.record(detection) for _ in range(10))) + assert len({alert.id for alert in duplicates}) == 1 + dismissed: Final = await alerts.repository.set_status(int(duplicates[0].id), "dismissed", "reviewer") + revised: Final = AlertDetection.model_validate( + {**detection.model_dump(), "evidence": {"value": 100, "baseline": 2, "unit": "usd", "sample_size": 100}} + ) + retried: Final = await alerts.repository.record(revised) + assert retried == dismissed + assert retried.evidence == detection.evidence + assert read_rows('SELECT count(*) AS count FROM "LiteLLM_Alert" WHERE resource_id=%s', (alerts.resource_id,)) == [ + {"count": 1} + ] + + +@pytest.mark.asyncio +async def test_keyset_pages_survive_status_changes_and_do_not_include_newer_alerts( + alerts: Alerts, gateway: Gateway +) -> None: + oldest: Final = await alerts.repository.record(alerts.detection(day=0)) + middle: Final = await alerts.repository.record(alerts.detection(day=1)) + newest: Final = await alerts.repository.record(alerts.detection(day=2)) + first: Final = gateway.request("GET", "/alerts", params={"limit": "2"}) + assert first.status_code == 200, first.text + page: Final = AlertPage.model_validate_json(first.content) + assert tuple(alert.id for alert in page.data) == (newest.id, middle.id) + assert page.next_cursor == middle.id + await alerts.repository.set_status(int(middle.id), "resolved", "reviewer") + late: Final = await alerts.repository.record(alerts.detection(day=3)) + second: Final = gateway.request("GET", "/alerts", params={"limit": "2", "cursor": middle.id}) + assert second.status_code == 200, second.text + remainder: Final = AlertPage.model_validate_json(second.content) + assert oldest.id in tuple(alert.id for alert in remainder.data) + assert not {newest.id, middle.id, late.id}.intersection(alert.id for alert in remainder.data) + resolved: Final = gateway.request("GET", "/alerts", params={"status": "resolved", "limit": "1"}) + assert resolved.status_code == 200, resolved.text + assert AlertPage.model_validate_json(resolved.content).data[0].id == middle.id + + +@pytest.mark.asyncio +async def test_repository_caps_page_size_and_follows_to_end(alerts: Alerts) -> None: + for day in range(102): + await alerts.repository.record(alerts.detection(day=day)) + page: Final = await alerts.repository.list("open", 100_000, None) + assert len(page.data) == 100 + assert page.next_cursor == page.data[-1].id + next_page: Final = await alerts.repository.list("open", 100, int(page.next_cursor)) + ours: Final = tuple(alert.id for alert in next_page.data if alert.resource_id == alerts.resource_id) + assert len(ours) == 2 + assert not set(ours).intersection(alert.id for alert in page.data) + empty: Final = await alerts.repository.list("dismissed", 25, None) + assert all(alert.status == "dismissed" for alert in empty.data) + + +@pytest.mark.asyncio +async def test_live_auth_denies_nonadmins_and_viewers_cannot_write(alerts: Alerts, gateway: Gateway) -> None: + saved: Final = await alerts.repository.record(alerts.detection()) + with gateway.scenario() as scenario: + for role in ("internal_user", "proxy_admin_viewer"): + user: Final = scenario.user(user_role=role) + key: Final = scenario.key(user_id=user) + for path in ("/alerts", f"/alerts/{saved.id}"): + response: Final = gateway.request("GET", path, key=key) + assert response.status_code in ((200,) if role == "proxy_admin_viewer" else (401, 403)), response.text + write: Final = gateway.request("PATCH", f"/alerts/{saved.id}", {"status": "dismissed"}, key=key) + assert write.status_code in (401, 403), write.text + assert await alerts.repository.get(int(saved.id)) == saved + + +@pytest.mark.asyncio +async def test_unknown_alert_returns_none_for_read_and_write(alerts: Alerts) -> None: + assert await alerts.repository.get(2**63 - 1) is None + assert await alerts.repository.set_status(2**63 - 1, "dismissed", "reviewer") is None + + +@pytest.mark.asyncio +async def test_windows_round_trip_at_database_precision_and_timezone(alerts: Alerts) -> None: + start: Final = datetime(2026, 1, 1, microsecond=123456, tzinfo=timezone.utc) + detection: Final = AlertDetection.model_validate( + {**alerts.detection().model_dump(), "window_start": start, "window_end": start + timedelta(seconds=1)} + ) + saved: Final = await alerts.repository.record(detection) + assert saved.window_start == start.replace(microsecond=123000) + assert saved.window_end == saved.window_start + timedelta(seconds=1) + offset: Final = timezone(timedelta(hours=8)) + shifted: Final = AlertDetection.model_validate( + { + **detection.model_dump(), + "window_start": start.astimezone(offset), + "window_end": (start + timedelta(seconds=1)).astimezone(offset), + } + ) + assert await alerts.repository.record(shifted) == saved diff --git a/tests/unit/models/test_alert.py b/tests/unit/models/test_alert.py new file mode 100644 index 00000000000..cf915c530b8 --- /dev/null +++ b/tests/unit/models/test_alert.py @@ -0,0 +1,68 @@ +from collections.abc import Mapping +from datetime import timedelta, timezone +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.models.alert import AlertDetection + +DETECTION: Final = { + "type": "spend", + "resource_type": "team", + "resource_id": "example-team", + "window_start": "2026-01-08T00:00:00Z", + "window_end": "2026-01-09T00:00:00Z", + "evidence": {"value": 20, "baseline": 2, "unit": "usd", "sample_size": 50}, +} + + +@pytest.mark.parametrize( + "change", + ( + {"resource_id": "x" * 1025}, + {"resource_id": ""}, + {"resource_type": "key", "resource_id": "sk-example-not-a-real-key"}, + {"resource_type": "gateway", "resource_id": "another-gateway"}, + { + "resource_type": "gateway", + "resource_id": "gateway", + "type": "model_deprecation", + "evidence": {"deprecation_date": "2026-02-01"}, + }, + {"window_start": "2026-01-08T00:00:00"}, + {"window_start": "2026-01-08T00:00:00.123456Z", "window_end": "2026-01-08T00:00:00.123457Z"}, + {"window_end": "2026-01-08T00:00:00Z"}, + {"window_end": "2026-01-07T00:00:00Z"}, + {"type": "custom"}, + {"type": "latency"}, + {"type": "model_deprecation"}, + {"evidence": {"value": float("inf"), "baseline": 2, "unit": "usd", "sample_size": 1}}, + {"evidence": {"value": 2, "baseline": float("nan"), "unit": "usd", "sample_size": 1}}, + {"evidence": {"value": -1, "baseline": 2, "unit": "usd", "sample_size": 1}}, + {"evidence": {"value": 2, "baseline": 2, "unit": "usd", "sample_size": -1}}, + {"evidence": {"value": 2, "baseline": 2, "unit": "usd", "sample_size": 1, "unbounded": "extra"}}, + {"type": "reliability", "evidence": {"value": 1.1, "baseline": 0, "unit": "ratio", "sample_size": 1}}, + {"type": "reliability", "evidence": {"value": 0, "baseline": 1.1, "unit": "ratio", "sample_size": 1}}, + { + "resource_type": "model", + "type": "model_deprecation", + "evidence": {"deprecation_date": "2026-02-01", "replacement_model": "x" * 1025}, + }, + ), +) +def test_invalid_or_unbounded_detection_is_rejected(change: Mapping[str, object]) -> None: + with pytest.raises(ValidationError): + AlertDetection.model_validate({**DETECTION, **change}) + + +def test_identity_uses_instants_not_timezone_or_evidence() -> None: + original: Final = AlertDetection.model_validate(DETECTION) + revised: Final = original.model_copy( + update={ + "window_start": original.window_start.astimezone(timezone(timedelta(hours=8))), + "window_end": original.window_end.astimezone(timezone(timedelta(hours=8))), + "evidence": original.evidence.model_copy(update={"value": 30}), + } + ) + assert original.identity() == revised.identity() diff --git a/tests/unit/proxy/alerting_endpoints/__init__.py b/tests/unit/proxy/alerting_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/alerting_endpoints/test_alerts.py b/tests/unit/proxy/alerting_endpoints/test_alerts.py new file mode 100644 index 00000000000..15dfad975fc --- /dev/null +++ b/tests/unit/proxy/alerting_endpoints/test_alerts.py @@ -0,0 +1,161 @@ +from collections.abc import Iterator +from datetime import datetime, timezone +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pydantic import BaseModel, ConfigDict + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.alerting_endpoints.alerts import router +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.deps.database import get_database +from litellm.repositories.database import Database +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from tests._support.native_database import native_database + + +class StoredAlert(BaseModel): + model_config = ConfigDict(frozen=True) + id: int = 5 + detection_key: str = "digest" + type: str = "spend" + resource_type: str = "team" + resource_id: str = "example-team" + status: str = "open" + window_start: datetime = datetime(2026, 1, 8, tzinfo=timezone.utc) + window_end: datetime = datetime(2026, 1, 9, tzinfo=timezone.utc) + evidence: object = {"value": 20, "baseline": 2, "unit": "usd", "sample_size": 50} + created_at: datetime = datetime(2026, 1, 9, tzinfo=timezone.utc) + updated_at: datetime = datetime(2026, 1, 9, tzinfo=timezone.utc) + updated_by: str | None = None + + +@pytest.fixture +def table() -> MagicMock: + table: Final = MagicMock() + table.find_many = AsyncMock(return_value=[StoredAlert()]) + table.find_unique = AsyncMock(return_value=None) + table.update = AsyncMock(return_value=None) + return table + + +@pytest.fixture +def app(table: MagicMock) -> FastAPI: + app: Final = FastAPI() + app.include_router(router) + prisma: Final = MagicMock() + prisma.writer_db.litellm_alert = table + app.dependency_overrides[get_database] = lambda: Database(prisma, native_database()) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN + ) + return app + + +@pytest.fixture +def client(app: FastAPI) -> Iterator[TestClient]: + with TestClient(app) as client: + yield client + + +@pytest.mark.parametrize( + "role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.TEAM, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) +) +@pytest.mark.parametrize("method,path", (("GET", "/alerts"), ("GET", "/alerts/5"), ("PATCH", "/alerts/5"))) +def test_permissions_precede_database_access( + app: FastAPI, table: MagicMock, role: LitellmUserRoles, method: str, path: str +) -> None: + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role) + app.dependency_overrides[get_database] = lambda: None + with TestClient(app) as client: + response: Final = client.request(method, path, json={"status": "dismissed"} if method == "PATCH" else None) + assert response.status_code == (503 if role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY and method == "GET" else 403) + table.find_many.assert_not_awaited() + table.find_unique.assert_not_awaited() + table.update.assert_not_awaited() + + +@pytest.mark.parametrize( + "query", ("limit=0", "limit=101", "cursor=0", "cursor=9223372036854775808", "cursor=oops", "status=invalid") +) +def test_invalid_page_never_reads_database(client: TestClient, table: MagicMock, query: str) -> None: + assert client.get(f"/alerts?{query}").status_code == 422 + table.find_many.assert_not_awaited() + + +@pytest.mark.parametrize("alert_id", ("0", "-1", "01", "1.0", "oops", "9223372036854775808", "1" * 20)) +@pytest.mark.parametrize("method", ("GET", "PATCH")) +def test_invalid_id_never_accesses_database(client: TestClient, table: MagicMock, alert_id: str, method: str) -> None: + response: Final = client.request( + method, f"/alerts/{alert_id}", json={"status": "resolved"} if method == "PATCH" else None + ) + assert response.status_code == 422 + table.find_unique.assert_not_awaited() + table.update.assert_not_awaited() + + +@pytest.mark.parametrize("alert_id", (9007199254740993, 9223372036854775807)) +def test_large_ids_remain_exact_in_reads_updates_and_pagination( + client: TestClient, table: MagicMock, alert_id: int +) -> None: + row: Final = StoredAlert(id=alert_id) + table.find_unique.return_value = row + table.update.return_value = row.model_copy(update={"status": "resolved", "updated_by": "admin-user"}) + table.find_many.return_value = [row, StoredAlert(id=alert_id - 1)] + detail: Final = client.get(f"/alerts/{alert_id}") + assert detail.status_code == 200, detail.text + assert detail.json()["id"] == str(alert_id) + table.find_unique.assert_awaited_once_with(where={"id": alert_id}) + updated: Final = client.patch(f"/alerts/{alert_id}", json={"status": "resolved"}) + assert updated.status_code == 200, updated.text + assert updated.json()["id"] == str(alert_id) + assert updated.json()["status"] == "resolved" + table.update.assert_awaited_once_with( + where={"id": alert_id}, data={"status": "resolved", "updated_by": "admin-user"} + ) + page: Final = client.get("/alerts?limit=1") + assert page.status_code == 200, page.text + assert page.json()["next_cursor"] == str(alert_id) + following: Final = client.get("/alerts", params={"cursor": page.json()["next_cursor"], "limit": 1}) + assert following.status_code == 200, following.text + table.find_many.assert_awaited_with(where={"status": "open", "id": {"lt": alert_id}}, order={"id": "desc"}, take=2) + + +def test_openapi_keeps_ids_and_cursors_as_strings(client: TestClient) -> None: + spec: Final = client.get("/openapi.json").json() + paths: Final = spec["paths"] + for method in ("get", "patch"): + parameter: Final = next(p for p in paths["/alerts/{alert_id}"][method]["parameters"] if p["name"] == "alert_id") + assert parameter["schema"]["type"] == "string" + cursor: Final = next(p for p in paths["/alerts"]["get"]["parameters"] if p["name"] == "cursor") + assert {s["type"] for s in cursor["schema"]["anyOf"]} == {"string", "null"} + schemas: Final = spec["components"]["schemas"] + assert schemas["Alert"]["properties"]["id"]["type"] == "string" + assert {s["type"] for s in schemas["AlertPage"]["properties"]["next_cursor"]["anyOf"]} == {"string", "null"} + + +def test_viewer_reads_saved_alert_with_string_id(app: FastAPI, table: MagicMock) -> None: + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ) + with TestClient(app) as client: + response: Final = client.get("/alerts?status=open&limit=1") + assert response.status_code == 200, response.text + expected: Final = StoredAlert().model_dump(mode="json", exclude={"detection_key"}) + assert response.json() == {"data": [{**expected, "id": "5"}], "next_cursor": None} + + +@pytest.mark.parametrize("method", ("GET", "PATCH")) +def test_missing_alert_returns_not_found(client: TestClient, method: str) -> None: + response: Final = client.request(method, "/alerts/5", json={"status": "resolved"} if method == "PATCH" else None) + assert response.status_code == 404 + assert response.json() == {"detail": "Alert not found"} + + +@pytest.mark.parametrize("body", ({"status": "unknown"}, {"status": "open", "evidence": {}}, {})) +def test_status_update_rejects_other_changes(client: TestClient, table: MagicMock, body: object) -> None: + assert client.patch("/alerts/5", json=body).status_code == 422 + table.update.assert_not_awaited() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a446cf87857..3d680bf333d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1002,6 +1002,50 @@ export interface paths { patch?: never; trace?: never; }; + "/alerts": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List saved alerts + * @description Read saved alerts, newest first. Gateway admins and view-only admins can read. No usage queries run. + */ + get: operations["list_alerts_alerts_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerts/{alert_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Read an alert + * @description Return the saved evidence and status. Requires a gateway admin or view-only admin. + */ + get: operations["get_alert_alerts__alert_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** + * Change an alert's status + * @description Resolve, dismiss, or reopen an alert. Only gateway admins can write. Returns the updated alert. + */ + patch: operations["update_alert_alerts__alert_id__patch"]; + trace?: never; + }; "/anthropic/{endpoint}": { parameters: { query?: never; @@ -27106,12 +27150,99 @@ export interface components { /** Url */ url: string; }; + /** Alert */ + Alert: { + /** + * Created At + * Format: date-time + * @description When the alert was first saved. + */ + created_at: string; + /** + * Evidence + * @description Small saved snapshot explaining the alert. + */ + evidence: components["schemas"]["MetricEvidence"] | components["schemas"]["DeprecationEvidence"]; + /** + * Id + * @description Opaque alert ID, also used as a pagination cursor. + */ + id: string; + /** + * Resource Id + * @description Resource ID. Whole-gateway alerts use 'gateway'. Keys use their SHA-256 hash, never a secret. + */ + resource_id: string; + /** + * Resource Type + * @description Kind of affected resource. + * @enum {string} + */ + resource_type: "gateway" | "key" | "team" | "user" | "model"; + /** + * Status + * @description Current review status. + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + /** + * Type + * @description Built-in alert category. + * @enum {string} + */ + type: "spend" | "traffic" | "latency" | "reliability" | "model_deprecation"; + /** + * Updated At + * Format: date-time + * @description When its status last changed. + */ + updated_at: string; + /** + * Updated By + * @description User who last changed the status. + */ + updated_by?: string | null; + /** + * Window End + * Format: date-time + * @description Exclusive end of the observed window, normalized to UTC milliseconds. + */ + window_end: string; + /** + * Window Start + * Format: date-time + * @description Inclusive start of the observed window, normalized to UTC milliseconds. + */ + window_start: string; + }; + /** AlertPage */ + AlertPage: { + /** + * Data + * @description Alerts ordered newest first. + */ + data: components["schemas"]["Alert"][]; + /** + * Next Cursor + * @description Pass as cursor to fetch the next page, or null at the end. + */ + next_cursor?: string | null; + }; /** * AlertType * @description Enum for alert types and management event types * @enum {string} */ AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed"; + /** AlertUpdate */ + AlertUpdate: { + /** + * Status + * @description Mark resolved, dismiss, or reopen the alert. + * @enum {string} + */ + status: "open" | "resolved" | "dismissed"; + }; /** AllowedVectorStoreIndexItem */ AllowedVectorStoreIndexItem: { /** Index Name */ @@ -33055,6 +33186,20 @@ export interface components { model: string; status: components["schemas"]["DeprecationAlertStatus"]; }; + /** DeprecationEvidence */ + DeprecationEvidence: { + /** + * Deprecation Date + * Format: date + * @description Date the affected model retires. + */ + deprecation_date: string; + /** + * Replacement Model + * @description Suggested replacement. + */ + replacement_model?: string | null; + }; /** DeprecationTagUsage */ DeprecationTagUsage: { /** Api Key */ @@ -39714,6 +39859,30 @@ export interface components { } & { [key: string]: unknown; }; + /** MetricEvidence */ + MetricEvidence: { + /** + * Baseline + * @description Comparison value in the same unit. + */ + baseline: number; + /** + * Sample Size + * @description Number of requests in the alert window. + */ + sample_size: number; + /** + * Unit + * @description Unit for both values. + * @enum {string} + */ + unit: "usd" | "requests" | "milliseconds" | "ratio"; + /** + * Value + * @description Observed metric value in the alert window. + */ + value: number; + }; /** MetricWithMetadata */ MetricWithMetadata: { /** Api Key Breakdown */ @@ -54531,6 +54700,110 @@ export interface operations { }; }; }; + list_alerts_alerts_get: { + parameters: { + query?: { + /** @description Status to list. */ + status?: "open" | "resolved" | "dismissed"; + /** @description Maximum alerts per page. */ + limit?: number; + /** @description next_cursor from the preceding page. */ + cursor?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AlertPage"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_alert_alerts__alert_id__get: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Alert ID returned by GET /alerts. */ + alert_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Alert"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_alert_alerts__alert_id__patch: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Alert ID returned by GET /alerts. */ + alert_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AlertUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Alert"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; anthropic_proxy_route_anthropic__endpoint__get: { parameters: { query?: never; From 701639ea2272a72a5e8dd4fa8dc5d3e84bcb5a92 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 12 Nov 2026 10:23:02 +0800 Subject: [PATCH 155/189] feat(alerts): detect daily changes in recorded usage (#50666) * feat(alerts): detect daily changes in usage for gateway, models, teams, users and keys * fix(alerts): drop unused suppressions * fix(alerts): satisfy strict lint for model usage timing * fix(alerts): skip only resources that time out or hold invalid data * fix(alerts): log an unfinished daily sweep at info level * fix(alerts): keep the deployment-wide run every five minutes * fix(alerts): record alert and sweep timestamps in UTC * fix(alerts): repair detector tests and tracing * refactor(alerts): build the key index in a migration instead of after startup * refactor(alerts): build the key index after startup again A concurrent index build inside the boot migration can exceed the 600 second migrate deploy deadline on a large LiteLLM_DailyUserSpend and stop the proxy from starting. This reverts 393db6e4e0a. * fix(alerts): keep keys pending, bound the accounting wait, resume unfinished scopes Keys waiting for their index no longer mark the day complete. Only deliveries created during the evaluated day hold back its sweep. A day that did not finish restarts the next day from where it stopped instead of a hash boundary that never rotated non-hex IDs. * refactor(alerts): drop the pending delivery wait, alerts are best effort on flushed usage * test(alerts): cover detector persistence and recovery * fix(alerts): recognize Prisma statement timeouts --- .../migration.sql | 21 ++ .../request_log_indexes.py | 1 + .../litellm_proxy_extras/schema.prisma | 19 ++ litellm/accounting/projection.py | 4 + litellm/integrations/otel/model/spans.py | 1 + litellm/models/performance_alert.py | 217 +++++++++++++++ litellm/proxy/_types.py | 7 + litellm/proxy/db/model_usage_rollup.py | 9 +- litellm/proxy/proxy_server.py | 20 ++ litellm/proxy/schema.prisma | 19 ++ .../spend_tracking/performance_alerts.py | 89 ++++++ litellm/repositories/alert_repository.py | 38 ++- .../performance_alert_repository.py | 263 ++++++++++++++++++ schema.prisma | 19 ++ .../database/test_performance_alerts.py | 217 +++++++++++++++ tests/unit/accounting/test_projection.py | 2 + tests/unit/models/test_performance_alert.py | 166 +++++++++++ .../unit/proxy/db/test_model_usage_rollup.py | 46 ++- .../test_model_insights_endpoints.py | 1 + .../spend_tracking/test_performance_alerts.py | 170 +++++++++++ ...test_proxy_server_endpoints_and_startup.py | 1 + .../test_performance_alert_repository.py | 159 +++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 + 23 files changed, 1488 insertions(+), 7 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261109090000_add_performance_alerts/migration.sql create mode 100644 litellm/models/performance_alert.py create mode 100644 litellm/proxy/spend_tracking/performance_alerts.py create mode 100644 litellm/repositories/performance_alert_repository.py create mode 100644 tests/integration/database/test_performance_alerts.py create mode 100644 tests/unit/models/test_performance_alert.py create mode 100644 tests/unit/proxy/spend_tracking/test_performance_alerts.py create mode 100644 tests/unit/repositories/test_performance_alert_repository.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261109090000_add_performance_alerts/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261109090000_add_performance_alerts/migration.sql new file mode 100644 index 00000000000..071e4177761 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261109090000_add_performance_alerts/migration.sql @@ -0,0 +1,21 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_PerformanceAlertState" ( + "id" INTEGER PRIMARY KEY, + "owner" VARCHAR(32) NOT NULL, + "next_run_at" TIMESTAMPTZ(3) NOT NULL, + "day" VARCHAR(10), + "scope" VARCHAR(16), + "resume_after" VARCHAR(1024) NOT NULL DEFAULT '', + "wrapped" BOOLEAN NOT NULL DEFAULT false, + "checked" INTEGER NOT NULL DEFAULT 0, + "skipped" INTEGER NOT NULL DEFAULT 0, + "start_scope" VARCHAR(16), + "start_after" VARCHAR(1024) NOT NULL DEFAULT '', + "completed_at" TIMESTAMP(3), + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +ALTER TABLE "LiteLLM_DailyModelUsage" + ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py b/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py index 4c856c741d3..91eddf46782 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py +++ b/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py @@ -59,6 +59,7 @@ class RequestLogIndex: REQUEST_LOG_INDEXES: Final = ( RequestLogIndex("LiteLLM_SpendLogs", "LiteLLM_SpendLogs_api_key_startTime_idx", '("api_key", "startTime")'), RequestLogIndex("LiteLLM_SpendLogs", "LiteLLM_SpendLogs_litellm_call_id_idx", '("litellm_call_id")'), + RequestLogIndex("LiteLLM_DailyUserSpend", "LiteLLM_DailyUserSpend_api_key_date_idx", '("api_key", "date")'), ) _IDENTIFIER_MAX_BYTES: Final = 63 diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index c02b3d19fa9..70d0536cbed 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -897,6 +897,7 @@ model LiteLLM_DailyUserSpend { @@index([date]) @@index([user_id, date]) @@index([api_key]) + @@index([api_key, date]) @@index([model]) @@index([mcp_namespaced_tool_name]) @@index([endpoint]) @@ -1383,6 +1384,8 @@ model LiteLLM_DailyModelUsage { request_count BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -2224,3 +2227,19 @@ model LiteLLM_Alert { @@index([status, id]) } + +model LiteLLM_PerformanceAlertState { + id Int @id + owner String @client.VarChar(32) + next_run_at DateTime @client.Timestamptz(3) + day String? @client.VarChar(10) + scope String? @client.VarChar(16) + resume_after String @default("") @client.VarChar(1024) + wrapped Boolean @default(false) + checked Int @default(0) + skipped Int @default(0) + start_scope String? @client.VarChar(16) + start_after String @default("") @client.VarChar(1024) + completed_at DateTime? + updated_at DateTime @default(now()) @updatedAt +} diff --git a/litellm/accounting/projection.py b/litellm/accounting/projection.py index 9d04d39884e..d1e461efca9 100644 --- a/litellm/accounting/projection.py +++ b/litellm/accounting/projection.py @@ -231,6 +231,8 @@ def _model_contributions(row: Mapping[str, object], metadata: Mapping[str, objec model: Final = _text(row.get("model")) or "unknown" caller: Final = not origin success: Final = row.get("status") == "success" + duration: Final = row.get("request_duration_ms") + timed: Final = caller and success and isinstance(duration, int) and duration >= 0 return ( Contribution( "LiteLLM_DailyModelUsage", @@ -248,6 +250,8 @@ def _model_contributions(row: Mapping[str, object], metadata: Mapping[str, objec ("request_count", int(caller)), ("successful_requests", int(caller and success)), ("failed_requests", int(caller and not success)), + ("total_response_time_ms", _number(duration) if timed else 0), + ("timed_requests", int(timed)), ), ), ) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index b8ebb2d6c6f..a2d55aaaa06 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -261,6 +261,7 @@ _PRISMA_VIEWS: Final[frozenset[str]] = frozenset( _PRISMA_MODELS: Final[frozenset[str]] = frozenset( ( "LiteLLM_Alert", + "LiteLLM_PerformanceAlertState", "LiteLLM_BudgetTable", "LiteLLM_CredentialsTable", "LiteLLM_ProxyModelTable", diff --git a/litellm/models/performance_alert.py b/litellm/models/performance_alert.py new file mode 100644 index 00000000000..37497b3742e --- /dev/null +++ b/litellm/models/performance_alert.py @@ -0,0 +1,217 @@ +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta, timezone +from typing import Final, Literal, Protocol, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field + +from litellm.models.alert import MAX_ALERT_ID, AlertDetection, AlertType, MetricEvidence + +PerformanceScope: TypeAlias = Literal["gateway", "model", "team", "user", "key"] +SCOPES: Final[tuple[PerformanceScope, ...]] = ("gateway", "model", "team", "user", "key") +BASELINE_DAYS: Final = 7 + +MIN_ACTIVE_DAYS: Final = 4 +SPEND_MULTIPLIER: Final = 3 +SPEND_MIN_INCREASE_USD: Final = 50 +TRAFFIC_SPIKE_MULTIPLIER: Final = 10 +TRAFFIC_SPIKE_MIN_INCREASE: Final = 1000 +TRAFFIC_DROP_RATIO: Final = 0.2 +TRAFFIC_DROP_MIN_BASELINE: Final = 500 +LATENCY_MULTIPLIER: Final = 3 +LATENCY_MIN_INCREASE_MS: Final = 500 +LATENCY_MIN_TIMED_REQUESTS: Final = 100 +RELIABILITY_MULTIPLIER: Final = 3 +RELIABILITY_MIN_INCREASE: Final = 0.02 +RELIABILITY_MIN_REQUESTS: Final = 500 + + +class ResourceWindow(BaseModel): + """One resource's totals for the evaluated day, and summed over the seven days before it.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + resource_id: str = Field(min_length=1, max_length=1024) + spend: float = Field(allow_inf_nan=False) + base_spend: float = Field(allow_inf_nan=False) + requests: int = Field(ge=0, le=MAX_ALERT_ID) + base_requests: int = Field(ge=0, le=MAX_ALERT_ID) + failed: int = Field(ge=0, le=MAX_ALERT_ID) + base_failed: int = Field(ge=0, le=MAX_ALERT_ID) + response_ms: int = Field(ge=0, le=MAX_ALERT_ID) + timed: int = Field(ge=0, le=MAX_ALERT_ID) + base_response_ms: int = Field(ge=0, le=MAX_ALERT_ID) + base_timed: int = Field(ge=0, le=MAX_ALERT_ID) + active_days: int = Field(ge=0, le=BASELINE_DAYS) + timed_days: int = Field(ge=0, le=BASELINE_DAYS) + exists: bool + + +class SweepState(BaseModel): + """Where the daily sweep stands. Scopes are walked in ID order. When the previous day stopped partway, that scope + starts where it stopped and wraps around, so an unfinished sweep does not leave out the same resources every day.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + day: date + scope: PerformanceScope = "gateway" + cursor: str = Field(default="", max_length=1024) + wrapped: bool = False + checked: int = Field(default=0, ge=0) + skipped: int = Field(default=0, ge=0) + completed: bool = False + start_scope: PerformanceScope | None = None + start_after: str = Field(default="", max_length=1024) + + @property + def start(self) -> str: + return self.start_after if self.scope == self.start_scope else "" + + +@dataclass(frozen=True, slots=True) +class Claim: + owner: str + state: SweepState | None + + +class SweepOwnershipLostError(Exception): + pass + + +class ResourceUnreadableError(Exception): + """A page that timed out or held invalid data. Retried alone, then stepped over. + Any other database error ends the run so the next run resumes from the same place.""" + + +class PerformanceAlertStore(Protocol): + async def claim(self, interval: timedelta) -> Claim | None: ... + async def key_index_ready(self) -> bool: ... + async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState: ... + async def skip_next(self, owner: str, state: SweepState) -> SweepState: ... + + +def target_day(now: datetime) -> date: + """The last complete UTC day, switching to a new day at 01:00 UTC so late accounting has time to land.""" + return (now.astimezone(timezone.utc) - timedelta(hours=1)).date() - timedelta(days=1) + + +def new_sweep(day: date, previous: SweepState | None = None) -> SweepState: + if previous is None or previous.completed or previous.scope == "gateway": + return SweepState(day=day) + return SweepState(day=day, start_scope=previous.scope, start_after=previous.cursor) + + +def next_scope(state: SweepState) -> SweepState: + position: Final = SCOPES.index(state.scope) + if position + 1 == len(SCOPES): + return state.model_copy(update={"cursor": "", "wrapped": False, "completed": True}) + moved: Final = state.model_copy(update={"scope": SCOPES[position + 1], "wrapped": False}) + return moved.model_copy(update={"cursor": moved.start}) + + +def advance(state: SweepState, resource_ids: Sequence[str], limit: int, skipped: int = 0) -> SweepState: + counted: Final = state.model_copy( + update={"checked": state.checked + len(resource_ids) - skipped, "skipped": state.skipped + skipped} + ) + if state.scope == "gateway": + return next_scope(counted) + if len(resource_ids) == limit: + return counted.model_copy(update={"cursor": resource_ids[-1]}) + if state.wrapped or not state.start: + return next_scope(counted) + return counted.model_copy(update={"cursor": "", "wrapped": True}) + + +def _ratio(numerator: float, denominator: float) -> float: + return numerator / denominator if denominator else 0.0 + + +def _alert( + scope: PerformanceScope, + window: ResourceWindow, + day: date, + kind: AlertType, + value: float, + baseline: float, + sample_size: int, +) -> AlertDetection: + start: Final = datetime.combine(day, time.min, timezone.utc) + unit: Final = {"spend": "usd", "traffic": "requests", "latency": "milliseconds", "reliability": "ratio"}[kind] + return AlertDetection.model_validate( + { + "type": kind, + "resource_type": scope, + "resource_id": "gateway" if scope == "gateway" else window.resource_id, + "window_start": start, + "window_end": start + timedelta(days=1), + "evidence": MetricEvidence.model_validate( + {"value": value, "baseline": baseline, "unit": unit, "sample_size": sample_size} + ), + } + ) + + +def _spend_spike(window: ResourceWindow) -> tuple[float, float] | None: + baseline: Final = window.base_spend / BASELINE_DAYS + triggered: Final = ( + window.active_days >= MIN_ACTIVE_DAYS + and baseline > 0 + and window.spend >= SPEND_MULTIPLIER * baseline + and window.spend - baseline >= SPEND_MIN_INCREASE_USD + ) + return (window.spend, baseline) if triggered else None + + +def _traffic_change(window: ResourceWindow) -> tuple[float, float] | None: + baseline: Final = window.base_requests / BASELINE_DAYS + spike: Final = ( + window.active_days >= MIN_ACTIVE_DAYS + and window.requests >= TRAFFIC_SPIKE_MULTIPLIER * baseline + and window.requests - baseline >= TRAFFIC_SPIKE_MIN_INCREASE + ) + drop: Final = ( + window.exists + and window.active_days == BASELINE_DAYS + and baseline >= TRAFFIC_DROP_MIN_BASELINE + and window.requests < TRAFFIC_DROP_RATIO * baseline + ) + return (window.requests, baseline) if spike or drop else None + + +def _slower_responses(window: ResourceWindow) -> tuple[float, float] | None: + current: Final = _ratio(window.response_ms, window.timed) + baseline: Final = _ratio(window.base_response_ms, window.base_timed) + triggered: Final = ( + window.timed >= LATENCY_MIN_TIMED_REQUESTS + and window.timed_days >= MIN_ACTIVE_DAYS + and baseline > 0 + and current >= LATENCY_MULTIPLIER * baseline + and current - baseline >= LATENCY_MIN_INCREASE_MS + ) + return (current, baseline) if triggered else None + + +def _more_failures(window: ResourceWindow) -> tuple[float, float] | None: + current: Final = min(1.0, _ratio(window.failed, window.requests)) + baseline: Final = min(1.0, _ratio(window.base_failed, window.base_requests)) + triggered: Final = ( + window.active_days >= MIN_ACTIVE_DAYS + and window.requests >= RELIABILITY_MIN_REQUESTS + and current >= RELIABILITY_MULTIPLIER * baseline + and current - baseline >= RELIABILITY_MIN_INCREASE + ) + return (current, baseline) if triggered else None + + +def detect(scope: PerformanceScope, window: ResourceWindow, day: date) -> tuple[AlertDetection, ...]: + checks: Final[tuple[tuple[AlertType, tuple[float, float] | None, int], ...]] = ( + ("spend", _spend_spike(window), window.requests), + ("traffic", _traffic_change(window), window.requests), + ("latency", _slower_responses(window), window.timed), + ("reliability", _more_failures(window), window.requests), + ) + return tuple( + _alert(scope, window, day, kind, found[0], found[1], sample_size) + for kind, found, sample_size in checks + if found is not None + ) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 1342ae6bd84..ee99895e209 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2967,6 +2967,13 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): "every replica. On by default; set to tune the window, pin a job, or turn it off." ), ) + performance_alerts_enabled: bool = Field( + False, + description=( + "Check yesterday's usage for the gateway, models, teams, users and keys against the week before, " + "and save unusual spend, traffic, response time and failure rate changes as alerts. Off by default." + ), + ) spend_capture_rate_check: SpendCaptureRateCheckSettings | None = Field( None, description=( diff --git a/litellm/proxy/db/model_usage_rollup.py b/litellm/proxy/db/model_usage_rollup.py index 1a5f15fe294..6b8f5bfa70e 100644 --- a/litellm/proxy/db/model_usage_rollup.py +++ b/litellm/proxy/db/model_usage_rollup.py @@ -69,6 +69,7 @@ class ModelUsageTransaction: completion_tokens: int successful: bool counts_as_request: bool = True + duration_ms: int | None = None def model_usage_task_type(request_tags: str) -> str: @@ -108,6 +109,8 @@ def build_model_usage_transaction(payload: SpendLogsPayload) -> ModelUsageTransa if date is None or (origin and origin != BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN): return None model: Final = payload["model"] or "unknown" + successful: Final = payload["status"] == "success" + duration: Final = payload["request_duration_ms"] return ModelUsageTransaction( key=ModelUsageKey( date=date, @@ -119,8 +122,9 @@ def build_model_usage_transaction(payload: SpendLogsPayload) -> ModelUsageTransa spend=payload["spend"], prompt_tokens=payload["prompt_tokens"], completion_tokens=payload["completion_tokens"], - successful=payload["status"] == "success", + successful=successful, counts_as_request=not origin, + duration_ms=duration if not origin and successful and isinstance(duration, int) and duration >= 0 else None, ) @@ -157,6 +161,7 @@ async def flush_model_usage_transactions( requests = tuple(entry for entry in entries if entry.counts_as_request) successful = sum(1 for entry in requests if entry.successful) failed = len(requests) - successful + durations = tuple(entry.duration_ms for entry in requests if entry.duration_ms is not None) key_fields = { "date": key.date, "model_group": key.model_group, @@ -174,6 +179,8 @@ async def flush_model_usage_transactions( "request_count": len(requests), "successful_requests": successful, "failed_requests": failed, + "total_response_time_ms": sum(durations), + "timed_requests": len(durations), }, ) return diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b1a746fbefd..014d39455de 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11919,6 +11919,26 @@ class ProxyStartupEvent: read_general_settings=_current_general_settings, ) + from litellm.proxy.spend_tracking.performance_alerts import RUN_INTERVAL, run_performance_alerts + from litellm.repositories.performance_alert_repository import PerformanceAlertRepository + + performance_alert_repository: Final = PerformanceAlertRepository(prisma_client) + + async def run_performance_alerts_job() -> None: + if _current_general_settings().get("performance_alerts_enabled") is True: + await run_performance_alerts(performance_alert_repository) + + scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # APScheduler does not provide typed job methods + run_performance_alerts_job, + "interval", + seconds=RUN_INTERVAL.total_seconds(), + id="performance_alerts", + replace_existing=True, + max_instances=1, + coalesce=True, + misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, + ) + ### PTU DAILY ROLLUP ### from litellm.proxy.spend_tracking.ptu_feature_flag import ( is_ptu_cost_attribution_enabled, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index c02b3d19fa9..70d0536cbed 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -897,6 +897,7 @@ model LiteLLM_DailyUserSpend { @@index([date]) @@index([user_id, date]) @@index([api_key]) + @@index([api_key, date]) @@index([model]) @@index([mcp_namespaced_tool_name]) @@index([endpoint]) @@ -1383,6 +1384,8 @@ model LiteLLM_DailyModelUsage { request_count BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -2224,3 +2227,19 @@ model LiteLLM_Alert { @@index([status, id]) } + +model LiteLLM_PerformanceAlertState { + id Int @id + owner String @client.VarChar(32) + next_run_at DateTime @client.Timestamptz(3) + day String? @client.VarChar(10) + scope String? @client.VarChar(16) + resume_after String @default("") @client.VarChar(1024) + wrapped Boolean @default(false) + checked Int @default(0) + skipped Int @default(0) + start_scope String? @client.VarChar(16) + start_after String @default("") @client.VarChar(1024) + completed_at DateTime? + updated_at DateTime @default(now()) @updatedAt +} diff --git a/litellm/proxy/spend_tracking/performance_alerts.py b/litellm/proxy/spend_tracking/performance_alerts.py new file mode 100644 index 00000000000..bb9a47d02c5 --- /dev/null +++ b/litellm/proxy/spend_tracking/performance_alerts.py @@ -0,0 +1,89 @@ +import asyncio +import time +from collections.abc import Awaitable, Callable +from datetime import datetime, timedelta, timezone +from typing import Final + +from litellm._logging import verbose_proxy_logger +from litellm.models.performance_alert import ( + Claim, + PerformanceAlertStore, + ResourceUnreadableError, + SweepOwnershipLostError, + SweepState, + new_sweep, + target_day, +) +from litellm.repositories.performance_alert_repository import KEY_DATE_INDEX + +RUN_INTERVAL: Final = timedelta(minutes=5) +RUN_BUDGET_SECONDS: Final = 10.0 +PAGE_SIZE: Final = 25 +PAUSE_SECONDS: Final = 0.1 + + +async def _step(store: PerformanceAlertStore, owner: str, state: SweepState) -> SweepState: + """One page, falling back to a single resource and then stepping over it, so one bad resource + cannot hold up the rest of the sweep.""" + try: + return await store.evaluate_page(owner, state, PAGE_SIZE) + except ResourceUnreadableError: + pass + try: + return await store.evaluate_page(owner, state, 1) + except ResourceUnreadableError: + verbose_proxy_logger.warning( + "Performance alerts skipped one %s it could not read within its limits", state.scope + ) + return await store.skip_next(owner, state) + + +def _start(claim: Claim, now: datetime) -> SweepState: + day: Final = target_day(now) + if claim.state is not None and claim.state.day == day: + return claim.state + if claim.state is not None and not claim.state.completed: + verbose_proxy_logger.info( + "Performance alerts did not finish %s: stopped in %s after %d resources, %d skipped", + claim.state.day, + claim.state.scope, + claim.state.checked, + claim.state.skipped, + ) + return new_sweep(day, claim.state) + + +async def run_performance_alerts( + store: PerformanceAlertStore, + interval: timedelta = RUN_INTERVAL, + now: Callable[[], datetime] = lambda: datetime.now(timezone.utc), + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, +) -> None: + from prisma.errors import PrismaError + from pydantic import ValidationError + + try: + claim: Final = await store.claim(interval) + if claim is None: + return + started: Final = _start(claim, now()) + if started.completed: + return + deadline: Final = clock() + RUN_BUDGET_SECONDS + keys_ready: Final = await store.key_index_ready() + state = started # rebind-ok: each page returns the next saved position + while not state.completed and clock() < deadline: + if state.scope == "key" and not keys_ready: + verbose_proxy_logger.info("Performance alerts waiting for %s before checking keys", KEY_DATE_INDEX) + break + state = await _step(store, claim.owner, state) + await sleep(PAUSE_SECONDS) + if state.completed: + verbose_proxy_logger.info( + "Performance alerts checked %d resources for %s, %d skipped", state.checked, state.day, state.skipped + ) + except SweepOwnershipLostError: + verbose_proxy_logger.info("Performance alerts run ended, another worker took over") + except (PrismaError, ValidationError, TimeoutError) as exc: + verbose_proxy_logger.warning("Performance alerts run ended early (%s)", type(exc).__name__) diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py index 1f02d3cdd81..299b1a67bdc 100644 --- a/litellm/repositories/alert_repository.py +++ b/litellm/repositories/alert_repository.py @@ -1,6 +1,12 @@ import hashlib import json -from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # Prisma exposes generated table actions dynamically +from collections.abc import Sequence +from typing import ( + TYPE_CHECKING, + Final, + Protocol, + cast, # noqa: TID251 # Prisma exposes generated table actions dynamically +) from litellm.models.alert import Alert, AlertDetection, AlertPage, AlertStatus from litellm.repositories.prisma_protocols import PrismaRecord, TableActions @@ -9,6 +15,34 @@ if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient +class AlertTransaction(Protocol): + async def execute_raw(self, query: str, *args: object) -> int: ... + + +def _detection_key(detection: AlertDetection) -> str: + return hashlib.sha256(json.dumps(detection.identity()).encode()).hexdigest() + + +async def record_alerts(transaction: AlertTransaction, detections: Sequence[AlertDetection]) -> None: + if not detections: + return + await transaction.execute_raw( + """INSERT INTO "LiteLLM_Alert" + (detection_key, type, resource_type, resource_id, window_start, window_end, evidence, created_at, updated_at) + SELECT detection_key, type, resource_type, resource_id, window_start, window_end, evidence, + now() AT TIME ZONE 'UTC', now() AT TIME ZONE 'UTC' + FROM jsonb_to_recordset($1::jsonb) AS rows(detection_key text, type text, resource_type text, + resource_id text, window_start timestamp, window_end timestamp, evidence jsonb) + ON CONFLICT (detection_key) DO NOTHING""", + json.dumps( + tuple( + {**detection.model_dump(mode="json"), "detection_key": _detection_key(detection)} + for detection in detections + ) + ), + ) + + def _alert(row: PrismaRecord) -> Alert: values: Final = row.dict() return Alert.model_validate( @@ -26,7 +60,7 @@ class AlertRepository: from prisma import Json from prisma.errors import UniqueViolationError - key: Final = hashlib.sha256(json.dumps(detection.identity()).encode()).hexdigest() + key: Final = _detection_key(detection) try: row: Final = await self._table.create( data={ diff --git a/litellm/repositories/performance_alert_repository.py b/litellm/repositories/performance_alert_repository.py new file mode 100644 index 00000000000..6686d158ff7 --- /dev/null +++ b/litellm/repositories/performance_alert_repository.py @@ -0,0 +1,263 @@ +import uuid +from collections.abc import AsyncGenerator, Mapping, Sequence +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from dataclasses import dataclass +from datetime import date, timedelta +from itertools import chain +from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # Prisma generates transaction types + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from typing_extensions import LiteralString + +from litellm.models.performance_alert import ( + BASELINE_DAYS, + Claim, + PerformanceScope, + ResourceUnreadableError, + ResourceWindow, + SweepOwnershipLostError, + SweepState, + advance, + detect, +) +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.repositories.alert_repository import record_alerts + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +KEY_DATE_INDEX: Final = "LiteLLM_DailyUserSpend_api_key_date_idx" +_STATEMENT_TIMEOUT: Final = "2s" +_TRANSACTION_TIMEOUT: Final = timedelta(seconds=5) + + +class _Transaction(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... + async def execute_raw(self, query: str, *args: object) -> int: ... + + +class _Database(Protocol): + def tx(self, *, timeout: timedelta) -> AbstractAsyncContextManager[_Transaction]: ... + + +class _StateRow(BaseModel): + model_config = ConfigDict(frozen=True) + + day: date | None + scope: PerformanceScope | None + cursor: str + wrapped: bool + checked: int + skipped: int + completed: bool + start_scope: PerformanceScope | None + start_after: str + + +class _IdRow(BaseModel): + model_config = ConfigDict(frozen=True) + + resource_id: str + + +_STATE_ROWS: Final = TypeAdapter(tuple[_StateRow, ...]) +_WINDOWS: Final = TypeAdapter(tuple[ResourceWindow, ...]) +_ID_ROWS: Final = TypeAdapter(tuple[_IdRow, ...]) + + +@dataclass(frozen=True, slots=True) +class _Source: + table: LiteralString + resource_id: LiteralString + requests: LiteralString + filter: LiteralString + exists: LiteralString + + +_SOURCES: Final[Mapping[PerformanceScope, _Source]] = { + "gateway": _Source('"LiteLLM_DailyModelUsage"', "'gateway'", "request_count", "", "true"), + "model": _Source('"LiteLLM_DailyModelUsage"', "model_group", "request_count", "", "true"), + "team": _Source( + '"LiteLLM_DailyTeamSpend"', + "team_id", + "api_requests", + "", + 'EXISTS (SELECT 1 FROM "LiteLLM_TeamTable" t WHERE t.team_id = page.resource_id AND NOT t.blocked)', + ), + "user": _Source( + '"LiteLLM_DailyUserSpend"', + "user_id", + "api_requests", + "", + 'EXISTS (SELECT 1 FROM "LiteLLM_UserTable" u WHERE u.user_id = page.resource_id)', + ), + "key": _Source( + '"LiteLLM_DailyUserSpend"', + "api_key", + "api_requests", + " AND api_key ~ '^[0-9a-f]{64}$'", + 'EXISTS (SELECT 1 FROM "LiteLLM_VerificationToken" k WHERE k.token = page.resource_id' + " AND k.blocked IS NOT TRUE AND (k.expires IS NULL OR k.expires > now()))", + ), +} + + +def _bounds(source: _Source, wrapped: bool) -> LiteralString: + upper: Final = f" AND {source.resource_id} <= $5" if wrapped else "" + return ( + f" AND {source.resource_id} > $4{upper} AND {source.resource_id} <> ''" + f" AND octet_length({source.resource_id}) <= 1024{source.filter}" + ) + + +def _page_sql(scope: PerformanceScope, wrapped: bool) -> LiteralString: + source: Final = _SOURCES[scope] + where: Final = "" if scope == "gateway" else _bounds(source, wrapped) + group: Final = ( + " HAVING COUNT(*) > 0" + if scope == "gateway" + else f" GROUP BY {source.resource_id} ORDER BY {source.resource_id}" + ) + return f"""WITH page AS ( + SELECT {source.resource_id} AS resource_id, + COALESCE(SUM(spend) FILTER (WHERE date = $1), 0)::float8 AS spend, + COALESCE(SUM(spend) FILTER (WHERE date < $1), 0)::float8 AS base_spend, + COALESCE(SUM({source.requests}) FILTER (WHERE date = $1), 0)::bigint AS requests, + COALESCE(SUM({source.requests}) FILTER (WHERE date < $1), 0)::bigint AS base_requests, + COALESCE(SUM(failed_requests) FILTER (WHERE date = $1), 0)::bigint AS failed, + COALESCE(SUM(failed_requests) FILTER (WHERE date < $1), 0)::bigint AS base_failed, + COALESCE(SUM(total_response_time_ms) FILTER (WHERE date = $1), 0)::bigint AS response_ms, + COALESCE(SUM(timed_requests) FILTER (WHERE date = $1), 0)::bigint AS timed, + COALESCE(SUM(total_response_time_ms) FILTER (WHERE date < $1), 0)::bigint AS base_response_ms, + COALESCE(SUM(timed_requests) FILTER (WHERE date < $1), 0)::bigint AS base_timed, + COUNT(DISTINCT date) FILTER (WHERE date < $1 AND {source.requests} > 0) AS active_days, + COUNT(DISTINCT date) FILTER (WHERE date < $1 AND timed_requests > 0) AS timed_days + FROM {source.table} + WHERE date BETWEEN $2 AND $1{where}{group} + LIMIT $3::int +) +SELECT page.*, {source.exists} AS exists FROM page ORDER BY resource_id""" + + +def _next_id_sql(scope: PerformanceScope, wrapped: bool) -> LiteralString: + source: Final = _SOURCES[scope] + return f"""SELECT {source.resource_id} AS resource_id FROM {source.table} +WHERE date BETWEEN $2 AND $1{_bounds(source, wrapped)} +ORDER BY {source.resource_id} LIMIT $3::int""" + + +_LIMITS: Final = f"""SELECT set_config('statement_timeout', '{_STATEMENT_TIMEOUT}', true), + set_config('lock_timeout', '100ms', true), set_config('work_mem', '1MB', true), + set_config('max_parallel_workers_per_gather', '0', true)""" +_CLAIM: Final = """INSERT INTO "LiteLLM_PerformanceAlertState" (id, owner, next_run_at, updated_at) +VALUES (1, $1, now() + $2::interval, now() AT TIME ZONE 'UTC') +ON CONFLICT (id) DO UPDATE SET owner = EXCLUDED.owner, next_run_at = EXCLUDED.next_run_at, + updated_at = now() AT TIME ZONE 'UTC' +WHERE "LiteLLM_PerformanceAlertState".next_run_at <= now() + interval '30 seconds' +RETURNING day, scope, resume_after AS cursor, wrapped, checked, skipped, completed_at IS NOT NULL AS completed, + start_scope, start_after""" +_SAVE: Final = """UPDATE "LiteLLM_PerformanceAlertState" +SET day = $2, scope = $3, resume_after = $4, wrapped = $5, checked = $6, skipped = $7, + start_scope = $9, start_after = $10, completed_at = CASE WHEN $8 THEN now() AT TIME ZONE 'UTC' END, updated_at = now() AT TIME ZONE 'UTC' +WHERE id = 1 AND owner = $1""" +_INDEX_READY: Final = """SELECT 1 AS resource_id FROM pg_index WHERE indexrelid = to_regclass($1) AND indisvalid""" + + +class PerformanceAlertRepository: + def __init__(self, prisma_client: "PrismaClient") -> None: + self._database: Final = cast( # cast-ok: generated Prisma transactions expose the bounded SQL surface + _Database, prisma_client.writer_db + ) + + @asynccontextmanager + async def _transaction(self) -> AsyncGenerator[_Transaction]: + async with self._database.tx(timeout=_TRANSACTION_TIMEOUT) as transaction: + await transaction.query_raw(_LIMITS) + yield transaction + + async def claim(self, interval: timedelta) -> Claim | None: + """Take this deployment's next run, at most one per `interval` across every worker and pod. A worker whose + timer fires up to 30 seconds early still takes it, so the claim's own delay never pushes a run back a whole + interval.""" + owner: Final = uuid.uuid4().hex + async with self._transaction() as transaction: + rows: Final = _STATE_ROWS.validate_python( + await transaction.query_raw(_CLAIM, owner, f"{int(interval.total_seconds())} seconds") + ) + if not rows: + return None + row: Final = rows[0] + if row.day is None or row.scope is None: + return Claim(owner, None) + return Claim(owner, SweepState.model_validate(row.model_dump())) + + async def key_index_ready(self) -> bool: + async with self._transaction() as transaction: + return bool(await transaction.query_raw(_INDEX_READY, f'"{KEY_DATE_INDEX}"')) + + async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState: + """Evaluate the next `limit` resources and save their alerts together with the new position.""" + from prisma.errors import RawQueryError + + try: + return await self._evaluate_page(owner, state, limit) + except ValidationError as exc: + raise ResourceUnreadableError() from exc + except RawQueryError as exc: + if PrismaDBExceptionHandler.is_statement_timeout_error(exc): + raise ResourceUnreadableError() from exc + raise + + async def _evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState: + async with self._transaction() as transaction: + windows: Final = _WINDOWS.validate_python( + await transaction.query_raw(_page_sql(state.scope, state.wrapped), *self._arguments(state, limit)) + ) + await record_alerts( + transaction, tuple(chain.from_iterable(detect(state.scope, window, state.day) for window in windows)) + ) + advanced: Final = advance(state, tuple(window.resource_id for window in windows), limit) + await self._save(transaction, owner, advanced) + return advanced + + async def skip_next(self, owner: str, state: SweepState) -> SweepState: + """Step over the next resource without evaluating it, for one that cannot be read within the limits.""" + async with self._transaction() as transaction: + rows: Final = ( + (_IdRow(resource_id="gateway"),) + if state.scope == "gateway" + else _ID_ROWS.validate_python( + await transaction.query_raw(_next_id_sql(state.scope, state.wrapped), *self._arguments(state, 1)) + ) + ) + advanced: Final = advance(state, tuple(row.resource_id for row in rows), 1, skipped=len(rows)) + await self._save(transaction, owner, advanced) + return advanced + + @staticmethod + def _arguments(state: SweepState, limit: int) -> tuple[str | int, ...]: + start: Final = state.day - timedelta(days=BASELINE_DAYS) + window: Final = (state.day.isoformat(), start.isoformat(), limit) + if state.scope == "gateway": + return window + if state.wrapped: + return (*window, state.cursor, state.start) + return (*window, state.cursor) + + @staticmethod + async def _save(transaction: _Transaction, owner: str, state: SweepState) -> None: + updated: Final = await transaction.execute_raw( + _SAVE, + owner, + state.day.isoformat(), + state.scope, + state.cursor, + state.wrapped, + state.checked, + state.skipped, + state.completed, + state.start_scope, + state.start_after, + ) + if updated != 1: + raise SweepOwnershipLostError() diff --git a/schema.prisma b/schema.prisma index c02b3d19fa9..70d0536cbed 100644 --- a/schema.prisma +++ b/schema.prisma @@ -897,6 +897,7 @@ model LiteLLM_DailyUserSpend { @@index([date]) @@index([user_id, date]) @@index([api_key]) + @@index([api_key, date]) @@index([model]) @@index([mcp_namespaced_tool_name]) @@index([endpoint]) @@ -1383,6 +1384,8 @@ model LiteLLM_DailyModelUsage { request_count BigInt @default(0) successful_requests BigInt @default(0) failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) created_at DateTime @default(now()) updated_at DateTime @updatedAt @@ -2224,3 +2227,19 @@ model LiteLLM_Alert { @@index([status, id]) } + +model LiteLLM_PerformanceAlertState { + id Int @id + owner String @client.VarChar(32) + next_run_at DateTime @client.Timestamptz(3) + day String? @client.VarChar(10) + scope String? @client.VarChar(16) + resume_after String @default("") @client.VarChar(1024) + wrapped Boolean @default(false) + checked Int @default(0) + skipped Int @default(0) + start_scope String? @client.VarChar(16) + start_after String @default("") @client.VarChar(1024) + completed_at DateTime? + updated_at DateTime @default(now()) @updatedAt +} diff --git a/tests/integration/database/test_performance_alerts.py b/tests/integration/database/test_performance_alerts.py new file mode 100644 index 00000000000..735f135065c --- /dev/null +++ b/tests/integration/database/test_performance_alerts.py @@ -0,0 +1,217 @@ +import asyncio +import hashlib +import os +import uuid +from collections.abc import AsyncIterator +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from typing import Final + +import psycopg +import pytest +import pytest_asyncio +from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes +from psycopg import sql + +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.spend_tracking.performance_alerts import run_performance_alerts +from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.alert_repository import AlertRepository +from litellm.repositories.performance_alert_repository import PerformanceAlertRepository +from tests.integration._support.database import read_rows, write_rows + +DAY: Final = date(2026, 1, 8) +NOW: Final = datetime(2026, 1, 9, 2, 0, tzinfo=timezone.utc) + + +@dataclass(frozen=True, slots=True) +class Day: + """One day of usage for a resource: spend, requests, failed requests, total response ms and timed requests.""" + + spend: float = 1.0 + requests: int = 1000 + failed: int = 10 + response_ms: int = 100_000 + timed: int = 1000 + + +STEADY: Final = Day() + + +_SPEND_TABLE_INSERT: Final = sql.SQL( + "INSERT INTO {table} (id, {column}, date, spend, api_requests, successful_requests, failed_requests," + " total_response_time_ms, timed_requests, api_key, model, custom_llm_provider, mcp_namespaced_tool_name," + " endpoint, updated_at) VALUES (gen_random_uuid()::text, %s, %s, %s, %s, %s, %s, %s, %s, 'k', 'm', 'p', ''," + " '', now())" +) +_KEY_INSERT: Final = ( + 'INSERT INTO "LiteLLM_DailyUserSpend" (id, api_key, date, spend, api_requests, successful_requests,' + " failed_requests, total_response_time_ms, timed_requests, user_id, model, custom_llm_provider," + " mcp_namespaced_tool_name, endpoint, updated_at) VALUES (gen_random_uuid()::text, %s, %s, %s, %s, %s, %s, %s," + " %s, '', 'm', 'p', '', '', now())" +) +_MODEL_INSERT: Final = ( + 'INSERT INTO "LiteLLM_DailyModelUsage" (model_group, date, spend, request_count, successful_requests,' + " failed_requests, total_response_time_ms, timed_requests, model, custom_llm_provider, task_type, updated_at)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'm', 'p', 'uncategorized', now())" +) + + +def _seed(scope: str, resource: str, baseline: Day, evaluated: Day | None) -> None: + days: Final = tuple((DAY - timedelta(days=offset), baseline) for offset in range(1, 8)) + ( + ((DAY, evaluated),) if evaluated is not None else () + ) + rows: Final = tuple( + (resource, day.isoformat(), usage.spend, usage.requests, usage.requests - usage.failed, usage.failed) + + (usage.response_ms, usage.timed) + for day, usage in days + ) + query: Final = { + "team": _SPEND_TABLE_INSERT.format( + table=sql.Identifier("LiteLLM_DailyTeamSpend"), + column=sql.Identifier("team_id"), + ), + "user": _SPEND_TABLE_INSERT.format( + table=sql.Identifier("LiteLLM_DailyUserSpend"), + column=sql.Identifier("user_id"), + ), + "key": sql.SQL(_KEY_INSERT), + "model": sql.SQL(_MODEL_INSERT), + }[scope] + with psycopg.connect(os.environ["DATABASE_URL"]) as connection, connection.cursor() as cursor: + cursor.executemany(query, rows) + + +@dataclass(frozen=True, slots=True) +class Fixture: + prefix: str + repository: PerformanceAlertRepository + alerts: AlertRepository + + def key(self, name: str) -> str: + return hashlib.sha256(f"{self.prefix}-{name}".encode()).hexdigest() + + async def sweep(self) -> None: + for _ in range(20): + await run_performance_alerts(self.repository, interval=timedelta(0), now=lambda: NOW) + state = read_rows( + 'SELECT completed_at IS NOT NULL AS completed FROM "LiteLLM_PerformanceAlertState" WHERE id = 1', () + ) + if state and state[0]["completed"] is True: + return + pytest.fail("the sweep did not complete") + + def saved(self) -> dict[tuple[str, str], tuple[str, str]]: + rows: Final = read_rows( + 'SELECT resource_type, resource_id, type, status FROM "LiteLLM_Alert" WHERE resource_id LIKE %s' + " OR resource_id = %s", + (f"{self.prefix}%", self.key("slow")), + ) + return { + (str(row["resource_id"]), str(row["type"])): (str(row["resource_type"]), str(row["status"])) for row in rows + } + + +@pytest_asyncio.fixture(loop_scope="function") +async def fixture() -> AsyncIterator[Fixture]: + prefix: Final = f"perf-{uuid.uuid4().hex[:12]}" + client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) + await client.connect() + write_rows('DELETE FROM "LiteLLM_PerformanceAlertState"', ()) + built: Final = Fixture(prefix, PerformanceAlertRepository(client), AlertRepository(client)) + try: + yield built + finally: + try: + pattern = f"{prefix}%" + write_rows('DELETE FROM "LiteLLM_DailyTeamSpend" WHERE team_id LIKE %s', (pattern,)) + write_rows( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id LIKE %s OR api_key = %s', + (pattern, built.key("slow")), + ) + write_rows('DELETE FROM "LiteLLM_DailyModelUsage" WHERE model_group LIKE %s', (pattern,)) + write_rows('DELETE FROM "LiteLLM_UserTable" WHERE user_id LIKE %s', (pattern,)) + write_rows( + 'DELETE FROM "LiteLLM_Alert" WHERE resource_id LIKE %s OR resource_id = %s' + " OR (resource_type = 'gateway' AND window_start = %s::timestamp)", + (pattern, built.key("slow"), DAY.isoformat()), + ) + write_rows('DELETE FROM "LiteLLM_PerformanceAlertState"', ()) + finally: + await client.disconnect() + + +@pytest.mark.asyncio +async def test_a_full_sweep_saves_one_alert_per_real_change_and_nothing_for_normal_or_new_usage( + fixture: Fixture, +) -> None: + assert ensure_request_log_indexes(os.environ["DATABASE_URL"], "public") + p: Final = fixture.prefix + _seed("team", f"{p}-team-spike", Day(spend=10.0), Day(spend=100.0)) + _seed("team", f"{p}-team-steady", STEADY, STEADY) + _seed("team", f"{p}-team-new", Day(requests=0, failed=0, timed=0, spend=0), Day(spend=900.0)) + write_rows('INSERT INTO "LiteLLM_UserTable" (user_id) VALUES (%s)', (f"{p}-user-stopped",)) + _seed("user", f"{p}-user-stopped", STEADY, None) + _seed("user", f"{p}-user-deleted", STEADY, None) + _seed("key", fixture.key("slow"), STEADY, Day(response_ms=1_000_000)) + _seed("model", f"{p}-model", STEADY, Day(failed=100)) + + await fixture.sweep() + + (age,) = read_rows( + "SELECT max(abs(extract(epoch FROM created_at - (now() AT TIME ZONE 'UTC'))))::int AS seconds" + ' FROM "LiteLLM_Alert" WHERE resource_id LIKE %s', + (f"{p}%",), + ) + assert int(str(age["seconds"])) < 300 + assert fixture.saved() == { + (f"{p}-team-spike", "spend"): ("team", "open"), + (f"{p}-user-stopped", "traffic"): ("user", "open"), + (fixture.key("slow"), "latency"): ("key", "open"), + (f"{p}-model", "reliability"): ("model", "open"), + } + + +@pytest.mark.asyncio +async def test_a_replayed_day_keeps_one_alert_and_its_dismissal(fixture: Fixture) -> None: + team: Final = f"{fixture.prefix}-team-spike" + _seed("team", team, Day(spend=10.0), Day(spend=100.0)) + await fixture.sweep() + (saved,) = read_rows('SELECT id FROM "LiteLLM_Alert" WHERE resource_id = %s', (team,)) + await fixture.alerts.set_status(int(str(saved["id"])), "dismissed", "admin") + + write_rows('DELETE FROM "LiteLLM_PerformanceAlertState"', ()) + await fixture.sweep() + + assert fixture.saved() == {(team, "spend"): ("team", "dismissed")} + + +@pytest.mark.asyncio +async def test_a_resource_that_cannot_be_read_is_skipped_without_blocking_the_rest(fixture: Fixture) -> None: + p: Final = fixture.prefix + _seed("team", f"{p}-team-a-broken", STEADY, Day(requests=-5, failed=0)) + _seed("team", f"{p}-team-b-spike", Day(spend=10.0), Day(spend=100.0)) + + await fixture.sweep() + + (state,) = read_rows('SELECT skipped FROM "LiteLLM_PerformanceAlertState" WHERE id = 1', ()) + assert int(str(state["skipped"])) >= 1 + assert fixture.saved() == {(f"{p}-team-b-spike", "spend"): ("team", "open")} + + +@pytest.mark.asyncio +async def test_only_one_of_many_concurrent_workers_gets_each_run(fixture: Fixture) -> None: + claims: Final = await asyncio.gather(*(fixture.repository.claim(timedelta(minutes=5)) for _ in range(8))) + assert sum(claim is not None for claim in claims) == 1 + + +@pytest.mark.parametrize(("seconds_until_due", "claimed"), ((20, True), (60, False))) +@pytest.mark.asyncio +async def test_a_timer_firing_just_before_the_run_is_due_still_takes_it( + fixture: Fixture, seconds_until_due: int, claimed: bool +) -> None: + write_rows( + 'INSERT INTO "LiteLLM_PerformanceAlertState" (id, owner, next_run_at) VALUES (1, %s, now() + %s::interval)', + ("earlier-worker", f"{seconds_until_due} seconds"), + ) + assert (await fixture.repository.claim(timedelta(minutes=5)) is not None) is claimed diff --git a/tests/unit/accounting/test_projection.py b/tests/unit/accounting/test_projection.py index e618c706fac..090825dca75 100644 --- a/tests/unit/accounting/test_projection.py +++ b/tests/unit/accounting/test_projection.py @@ -277,6 +277,8 @@ async def test_entity_projection_matches_real_request_queues(monkeypatch: pytest "request_count": int(model.counts_as_request), "successful_requests": int(model.successful), "failed_requests": int(not model.successful), + "total_response_time_ms": model.duration_ms or 0, + "timed_requests": int(model.duration_ms is not None), } diff --git a/tests/unit/models/test_performance_alert.py b/tests/unit/models/test_performance_alert.py new file mode 100644 index 00000000000..84b40d37c1c --- /dev/null +++ b/tests/unit/models/test_performance_alert.py @@ -0,0 +1,166 @@ +from datetime import date, datetime, timedelta, timezone +from typing import Final + +import pytest + +from litellm.models.alert import AlertDetection, MetricEvidence +from litellm.models.performance_alert import ( + ResourceWindow, + SweepState, + advance, + detect, + new_sweep, + next_scope, + target_day, +) + +DAY: Final = date(2026, 1, 8) +STEADY: Final = { + "resource_id": "team-a", + "spend": 10.0, + "base_spend": 70.0, + "requests": 1000, + "base_requests": 7000, + "failed": 10, + "base_failed": 70, + "response_ms": 100_000, + "timed": 1000, + "base_response_ms": 700_000, + "base_timed": 7000, + "active_days": 7, + "timed_days": 7, + "exists": True, +} + + +def _window(**changes: object) -> ResourceWindow: + return ResourceWindow.model_validate({**STEADY, **changes}) + + +def _types(**changes: object) -> tuple[str, ...]: + return tuple(alert.type for alert in detect("team", _window(**changes), DAY)) + + +def test_steady_usage_raises_nothing() -> None: + assert _types() == () + + +@pytest.mark.parametrize( + ("changes", "expected"), + ( + ({"spend": 60.0}, ("spend",)), + ({"spend": 59.9}, ()), + ({"base_spend": 280.0, "spend": 120.0}, ("spend",)), + ({"base_spend": 280.0, "spend": 119.9}, ()), + ({"spend": 5000.0, "active_days": 3}, ()), + ({"spend": 5000.0, "base_spend": 0.0}, ()), + ({"requests": 1100, "failed": 0, "response_ms": 110_000, "timed": 1100, "base_requests": 700}, ("traffic",)), + ({"requests": 1099, "failed": 0, "response_ms": 109_900, "timed": 1099, "base_requests": 700}, ()), + ({"requests": 50_000, "failed": 0, "active_days": 3}, ()), + ({"requests": 99, "failed": 0, "timed": 99, "response_ms": 9900, "base_requests": 3500}, ("traffic",)), + ({"requests": 100, "failed": 0, "timed": 100, "response_ms": 10_000, "base_requests": 3500}, ()), + ({"requests": 0, "failed": 0, "timed": 0, "response_ms": 0, "base_requests": 3500, "active_days": 6}, ()), + ({"requests": 0, "failed": 0, "timed": 0, "response_ms": 0, "base_requests": 3500, "exists": False}, ()), + ({"requests": 0, "failed": 0, "timed": 0, "response_ms": 0, "base_requests": 3493}, ()), + ({"response_ms": 600_000}, ("latency",)), + ({"response_ms": 599_000}, ()), + ({"response_ms": 60_000_000, "timed": 99}, ()), + ({"response_ms": 600_000, "timed_days": 3}, ()), + ({"failed": 40}, ("reliability",)), + ({"failed": 25}, ()), + ({"requests": 499, "failed": 400, "timed": 499, "response_ms": 49_900}, ()), + ({"base_failed": 0, "failed": 21}, ("reliability",)), + ), +) +def test_each_rule_fires_only_past_both_its_multiplier_and_its_floor( + changes: dict[str, object], expected: tuple[str, ...] +) -> None: + assert _types(**changes) == expected + + +def test_alert_saves_the_comparison_that_triggered_it() -> None: + alerts: Final = detect("team", _window(spend=60.0), DAY) + assert alerts == ( + AlertDetection.model_validate( + { + "type": "spend", + "resource_type": "team", + "resource_id": "team-a", + "window_start": "2026-01-08T00:00:00Z", + "window_end": "2026-01-09T00:00:00Z", + "evidence": {"value": 60.0, "baseline": 10.0, "unit": "usd", "sample_size": 1000}, + } + ), + ) + + +def test_latency_and_failure_rate_are_weighted_by_requests_not_averaged_per_day() -> None: + alerts: Final = detect("model", _window(response_ms=600_000, failed=40), DAY) + assert {alert.type: alert.evidence for alert in alerts} == { + "latency": MetricEvidence(value=600.0, baseline=100.0, unit="milliseconds", sample_size=1000), + "reliability": MetricEvidence(value=0.04, baseline=0.01, unit="ratio", sample_size=1000), + } + + +def test_gateway_alerts_use_the_gateway_resource_id() -> None: + alerts: Final = detect("gateway", _window(resource_id="anything", spend=60.0), DAY) + assert tuple((alert.resource_type, alert.resource_id) for alert in alerts) == (("gateway", "gateway"),) + + +@pytest.mark.parametrize( + ("now", "expected"), + ( + (datetime(2026, 1, 10, 0, 59, tzinfo=timezone.utc), date(2026, 1, 8)), + (datetime(2026, 1, 10, 1, 0, tzinfo=timezone.utc), date(2026, 1, 9)), + (datetime(2026, 1, 10, 9, 0, tzinfo=timezone(timedelta(hours=8))), date(2026, 1, 9)), + ), +) +def test_the_evaluated_day_changes_at_one_am_utc(now: datetime, expected: date) -> None: + assert target_day(now) == expected + + +def test_gateway_moves_to_models_starting_from_the_beginning() -> None: + assert advance(new_sweep(DAY), ("gateway",), 25) == SweepState(day=DAY, scope="model", checked=1) + + +def test_a_full_page_continues_from_its_last_resource() -> None: + state: Final = SweepState(day=DAY, scope="team") + assert advance(state, ("t1", "t2"), 2) == state.model_copy(update={"cursor": "t2", "checked": 2}) + + +def test_a_scope_without_a_carried_start_moves_on_after_one_pass() -> None: + finished: Final = advance(SweepState(day=DAY, scope="team", cursor="t9"), ("t10",), 25) + assert (finished.scope, finished.cursor, finished.wrapped) == ("user", "", False) + + +def test_an_unfinished_day_restarts_its_scope_where_it_stopped_then_wraps() -> None: + previous: Final = SweepState(day=DAY - timedelta(days=1), scope="team", cursor="t5", checked=40) + today: Final = new_sweep(DAY, previous) + assert (today.scope, today.start_scope, today.start_after, today.checked) == ("gateway", "team", "t5", 0) + at_team: Final = next_scope(next_scope(today)) + assert (at_team.scope, at_team.cursor, at_team.wrapped) == ("team", "t5", False) + after_tail: Final = advance(at_team, ("t6", "t7"), 25) + assert (after_tail.scope, after_tail.cursor, after_tail.wrapped) == ("team", "", True) + after_head: Final = advance(after_tail, ("t1", "t5"), 25) + assert (after_head.scope, after_head.cursor, after_head.wrapped) == ("user", "", False) + + +@pytest.mark.parametrize( + "previous", + ( + SweepState(day=DAY - timedelta(days=1), scope="team", cursor="t5", completed=True), + SweepState(day=DAY - timedelta(days=1), scope="gateway"), + ), +) +def test_a_finished_or_barely_started_day_carries_nothing(previous: SweepState) -> None: + assert new_sweep(DAY, previous) == SweepState(day=DAY) + + +def test_the_sweep_completes_after_keys() -> None: + finished: Final = next_scope(SweepState(day=DAY, scope="key", cursor="k", wrapped=True, checked=9)) + assert finished.completed and finished.checked == 9 + + +def test_a_skipped_resource_counts_as_skipped_not_checked() -> None: + state: Final = SweepState(day=DAY, scope="user", cursor="u1", checked=4) + assert advance(state, ("u2",), 1, skipped=1) == state.model_copy(update={"cursor": "u2", "skipped": 1}) diff --git a/tests/unit/proxy/db/test_model_usage_rollup.py b/tests/unit/proxy/db/test_model_usage_rollup.py index 1e79d06370b..2657d3fd2e3 100644 --- a/tests/unit/proxy/db/test_model_usage_rollup.py +++ b/tests/unit/proxy/db/test_model_usage_rollup.py @@ -51,11 +51,14 @@ def _payload(**overrides: Any) -> dict[str, Any]: "request_tags": "[]", "custom_llm_provider": "openai", "status": "success", + "request_duration_ms": None, **overrides, } -def _transaction(model: str, spend: float, successful: bool = True) -> ModelUsageTransaction: +def _transaction( + model: str, spend: float, successful: bool = True, duration_ms: int | None = None +) -> ModelUsageTransaction: return ModelUsageTransaction( key=ModelUsageKey( date="2026-09-28", model_group=model, model=model, custom_llm_provider="openai", task_type="debugging" @@ -64,6 +67,7 @@ def _transaction(model: str, spend: float, successful: bool = True) -> ModelUsag prompt_tokens=10, completion_tokens=5, successful=successful, + duration_ms=duration_ms, ) @@ -128,10 +132,10 @@ async def test_flush_aggregates_each_rollup_row_into_one_upsert() -> None: await flush_model_usage_transactions( prisma_client=prisma, transactions=[ - _transaction("gpt-5", 0.5), + _transaction("gpt-5", 0.5, duration_ms=200), _transaction("claude", 1.0), _transaction("gpt-5", 0.25, successful=False), - _transaction("gpt-5", 0.25), + _transaction("gpt-5", 0.25, duration_ms=400), ], ) @@ -154,8 +158,39 @@ async def test_flush_aggregates_each_rollup_row_into_one_upsert() -> None: "request_count": {"increment": 3}, "successful_requests": {"increment": 2}, "failed_requests": {"increment": 1}, + "total_response_time_ms": {"increment": 600}, + "timed_requests": {"increment": 2}, } assert upserts["claude"]["create"]["request_count"] == 1 + assert (upserts["claude"]["create"]["total_response_time_ms"], upserts["claude"]["create"]["timed_requests"]) == ( + 0, + 0, + ) + + +@pytest.mark.parametrize( + ("overrides", "expected"), + ( + ({"request_duration_ms": 250}, 250), + ({"request_duration_ms": 0}, 0), + ({"request_duration_ms": 250, "status": "failure"}, None), + ({"request_duration_ms": None}, None), + ({"request_duration_ms": -1}, None), + ( + { + "request_duration_ms": 250, + "metadata": json.dumps({INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}), + }, + None, + ), + ), +) +def test_only_successful_caller_requests_with_a_duration_count_toward_response_time( + overrides: dict[str, Any], expected: int | None +) -> None: + transaction = build_model_usage_transaction(_payload(**overrides)) + assert transaction is not None + assert transaction.duration_ms == expected @pytest.mark.asyncio @@ -166,7 +201,10 @@ async def test_a_background_response_billing_row_adds_spend_but_no_request() -> prisma = _prisma(MagicMock(return_value=batcher)) create = build_model_usage_transaction(_payload(spend=0.0)) billing = build_model_usage_transaction( - _payload(spend=0.5, metadata=json.dumps({INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN})) + _payload( + spend=0.5, + metadata=json.dumps({INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}), + ) ) assert create is not None and billing is not None diff --git a/tests/unit/proxy/management_endpoints/test_model_insights_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_insights_endpoints.py index 40b0cbe11c5..ac6714bc5df 100644 --- a/tests/unit/proxy/management_endpoints/test_model_insights_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_insights_endpoints.py @@ -253,6 +253,7 @@ async def test_model_insights_reads_back_what_the_rollup_wrote() -> None: "request_tags": '["task:debugging"]', "custom_llm_provider": "openai", "status": "success", + "request_duration_ms": None, } transactions = ( diff --git a/tests/unit/proxy/spend_tracking/test_performance_alerts.py b/tests/unit/proxy/spend_tracking/test_performance_alerts.py new file mode 100644 index 00000000000..02308ec6979 --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_performance_alerts.py @@ -0,0 +1,170 @@ +from dataclasses import dataclass, field +from datetime import date, datetime, timedelta, timezone +from typing import Final + +import pytest + +from litellm.models.performance_alert import ( + Claim, + ResourceUnreadableError, + SweepOwnershipLostError, + SweepState, + advance, + new_sweep, +) +from litellm.proxy.spend_tracking.performance_alerts import PAGE_SIZE, RUN_BUDGET_SECONDS, run_performance_alerts + +NOW: Final = datetime(2026, 1, 9, 2, 0, tzinfo=timezone.utc) +DAY: Final = date(2026, 1, 8) + + +@dataclass +class Store: + """Each scope holds `per_scope` resources. Resources named in `unreadable` fail any page that includes them.""" + + state: SweepState | None = None + claimable: bool = True + index_ready: bool = True + per_scope: int = 3 + unreadable: frozenset[str] = frozenset() + pages: list[tuple[str, int]] = field(default_factory=list) + saved: list[SweepState] = field(default_factory=list) + + async def claim(self, interval: timedelta) -> Claim | None: + return Claim("owner", self.state) if self.claimable else None + + async def key_index_ready(self) -> bool: + return self.index_ready + + def _keep(self, state: SweepState) -> None: + self.saved.append(state) + self.state = state + + def _next(self, state: SweepState, limit: int) -> tuple[str, ...]: + if state.scope == "gateway": + return ("gateway",) + names: Final = tuple(f"{state.scope}-{index}" for index in range(self.per_scope)) + after: Final = tuple(name for name in names if name > state.cursor) + return tuple(name for name in after if not state.wrapped or name <= state.start)[:limit] + + async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState: + self.pages.append((state.scope, limit)) + ids: Final = self._next(state, limit) + if self.unreadable.intersection(ids): + raise ResourceUnreadableError() + advanced: Final = advance(state, ids, limit) + self._keep(advanced) + return advanced + + async def skip_next(self, owner: str, state: SweepState) -> SweepState: + ids: Final = self._next(state, 1) + advanced: Final = advance(state, ids, 1, skipped=len(ids)) + self._keep(advanced) + return advanced + + +async def _no_sleep(seconds: float) -> None: + return None + + +async def _run(store: Store, clock: list[float] | None = None) -> None: + ticks: Final = clock if clock is not None else [0.0] + await run_performance_alerts(store, now=lambda: NOW, clock=lambda: ticks[0], sleep=_no_sleep) + + +@pytest.mark.asyncio +async def test_a_fresh_sweep_checks_every_scope_and_completes() -> None: + store: Final = Store() + await _run(store) + assert store.state is not None and store.state.completed + assert (store.state.day, store.state.checked, store.state.skipped) == (DAY, 1 + 3 * 4, 0) + + +@pytest.mark.asyncio +async def test_no_work_happens_without_the_claim() -> None: + store: Final = Store(claimable=False) + await _run(store) + assert store.pages == [] and store.saved == [] + + +@pytest.mark.asyncio +async def test_a_completed_day_does_no_more_work() -> None: + store: Final = Store(state=new_sweep(DAY).model_copy(update={"completed": True})) + await _run(store) + assert store.pages == [] + + +@pytest.mark.asyncio +async def test_an_unfinished_previous_day_resumes_its_scope_where_it_stopped_and_still_checks_everything() -> None: + store: Final = Store(state=SweepState(day=DAY - timedelta(days=1), scope="user", cursor="user-1")) + await _run(store) + assert store.saved[0].day == DAY and store.pages[0] == ("gateway", PAGE_SIZE) + users: Final = tuple(state for state in store.saved if state.scope == "user") + assert (users[0].cursor, users[0].wrapped) == ("user-1", False) + assert any(state.wrapped for state in users) + assert store.state is not None and store.state.completed and store.state.checked == 1 + 3 * 4 + + +@pytest.mark.asyncio +async def test_the_run_resumes_the_saved_position_for_the_same_day() -> None: + store: Final = Store(state=SweepState(day=DAY, scope="key", cursor="key-1", checked=10)) + await _run(store) + assert store.pages[0] == ("key", PAGE_SIZE) + assert store.state is not None and store.state.checked == 11 + + +@pytest.mark.asyncio +async def test_an_unreadable_resource_is_retried_alone_then_skipped() -> None: + store: Final = Store(unreadable=frozenset({"team-1"})) + await _run(store) + assert ("team", 1) in store.pages + assert store.state is not None and store.state.completed + assert (store.state.checked, store.state.skipped) == (12, 1) + + +@pytest.mark.asyncio +async def test_keys_wait_for_their_index_without_finishing_the_day_then_run_once_it_exists() -> None: + store: Final = Store(index_ready=False) + await _run(store) + assert all(scope != "key" for scope, _ in store.pages) + assert store.state is not None and not store.state.completed and store.state.scope == "key" + store.index_ready = True + await _run(store) + assert store.state is not None and store.state.completed and store.state.checked == 1 + 3 * 4 + + +@pytest.mark.asyncio +async def test_the_run_stops_at_its_time_budget_and_keeps_its_position() -> None: + clock: Final = [0.0] + store: Final = Store(per_scope=10_000) + + async def tick(seconds: float) -> None: + clock[0] += RUN_BUDGET_SECONDS * 0.4 + + await run_performance_alerts(store, now=lambda: NOW, clock=lambda: clock[0], sleep=tick) + assert len(store.pages) == 3 + assert store.state is not None and not store.state.completed + + +@pytest.mark.asyncio +async def test_losing_the_claim_mid_run_stops_quietly() -> None: + class TakenOver(Store): + async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState: + raise SweepOwnershipLostError() + + store: Final = TakenOver() + await _run(store) + assert store.saved == [] + + +@pytest.mark.asyncio +async def test_a_database_wide_failure_ends_the_run_without_skipping_anything() -> None: + class Unavailable(Store): + async def evaluate_page(self, owner: str, state: SweepState, limit: int) -> SweepState: + from prisma.errors import RawQueryError + + raise RawQueryError({"user_facing_error": {"message": "FATAL: sorry, too many clients already"}}) + + store: Final = Unavailable() + await _run(store) + assert store.saved == [] diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 3d4d8f74c10..5a6c9e13ab6 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1664,6 +1664,7 @@ class MockPrisma: self.database_url = database_url self.proxy_logging_obj = proxy_logging_obj self.http_client = http_client + self.writer_db = MagicMock() async def connect(self): pass diff --git a/tests/unit/repositories/test_performance_alert_repository.py b/tests/unit/repositories/test_performance_alert_repository.py new file mode 100644 index 00000000000..2ea3bc98e0f --- /dev/null +++ b/tests/unit/repositories/test_performance_alert_repository.py @@ -0,0 +1,159 @@ +from collections.abc import Mapping, Sequence +from datetime import date, timedelta +from typing import Final +from unittest.mock import ANY, AsyncMock, MagicMock + +import pytest +from pydantic import TypeAdapter + +from litellm.models.performance_alert import ResourceUnreadableError, SweepOwnershipLostError, SweepState +from litellm.repositories.performance_alert_repository import PerformanceAlertRepository + + +def _raw_query_error(message: str, code: str = "") -> Exception: + from prisma.errors import RawQueryError + + return RawQueryError({"user_facing_error": {"is_panic": False, "message": message, "meta": {"code": code}}}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("message", "code", "retryable"), + ( + ("ERROR: canceling statement due to statement timeout", "57014", True), + ("Query canceled", "57014", True), + ("ERROR: canceling statement due to statement timeout", "", True), + ("FATAL: sorry, too many clients already", "53300", False), + ("FATAL: terminating connection due to administrator command", "57P01", False), + ("ERROR: deadlock detected", "40P01", False), + ), +) +async def test_only_a_statement_timeout_marks_a_resource_unreadable(message: str, code: str, retryable: bool) -> None: + error: Final = _raw_query_error(message, code) + repository, _query, execute = _repository(error) + with pytest.raises(ResourceUnreadableError if retryable else type(error)): + await repository.evaluate_page("worker-a", SweepState(day=date(2026, 1, 8), scope="model"), 25) + execute.assert_not_awaited() + + +def _repository( + rows: Sequence[Mapping[str, object]] | Exception, +) -> tuple[PerformanceAlertRepository, AsyncMock, AsyncMock]: + query: Final = AsyncMock(side_effect=((), rows)) + execute: Final = AsyncMock(return_value=1) + transaction: Final = MagicMock(query_raw=query, execute_raw=execute) + context: Final = MagicMock(__aenter__=AsyncMock(return_value=transaction), __aexit__=AsyncMock(return_value=False)) + client: Final = MagicMock(writer_db=MagicMock(tx=MagicMock(return_value=context))) + return PerformanceAlertRepository(client), query, execute + + +@pytest.mark.asyncio +@pytest.mark.parametrize("day", (None, date(2026, 1, 8))) +async def test_claim_returns_the_saved_position_or_a_fresh_sweep(day: date | None) -> None: + repository, query, _execute = _repository( + ( + { + "day": day, + "scope": "team", + "cursor": "team-a", + "wrapped": True, + "checked": 12, + "skipped": 2, + "completed": False, + "start_scope": "team", + "start_after": "team-z", + }, + ) + ) + claim: Final = await repository.claim(timedelta(minutes=5)) + assert claim is not None + assert claim.state == ( + None + if day is None + else SweepState( + day=day, + scope="team", + cursor="team-a", + wrapped=True, + checked=12, + skipped=2, + start_scope="team", + start_after="team-z", + ) + ) + query.assert_awaited_with(ANY, claim.owner, "300 seconds") + + +@pytest.mark.asyncio +async def test_a_claim_taken_by_another_worker_returns_no_work() -> None: + repository, _query, execute = _repository(()) + assert await repository.claim(timedelta(minutes=5)) is None + execute.assert_not_awaited() + + +@pytest.fixture +def spend_window() -> Mapping[str, object]: + return { + "resource_id": "model-a", + "spend": 100.0, + "base_spend": 70.0, + "requests": 100, + "base_requests": 700, + "failed": 0, + "base_failed": 0, + "response_ms": 0, + "timed": 0, + "base_response_ms": 0, + "base_timed": 0, + "active_days": 7, + "timed_days": 0, + "exists": True, + } + + +@pytest.mark.asyncio +async def test_a_page_saves_the_detected_evidence_and_its_resume_position(spend_window: Mapping[str, object]) -> None: + repository, _query, execute = _repository((spend_window,)) + state: Final = SweepState(day=date(2026, 1, 8), scope="model") + result: Final = await repository.evaluate_page("worker-a", state, 1) + assert result == state.model_copy(update={"cursor": "model-a", "checked": 1}) + payload: Final = TypeAdapter(str).validate_python(execute.await_args_list[0].args[1]) + records: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_json(payload) + assert len(records) == 1 + assert records[0]["resource_id"] == "model-a" + assert records[0]["type"] == "spend" + assert records[0]["evidence"] == {"value": 100.0, "baseline": 10.0, "unit": "usd", "sample_size": 100} + execute.assert_awaited_with(ANY, "worker-a", "2026-01-08", "model", "model-a", False, 1, 0, False, None, "") + + +@pytest.mark.asyncio +async def test_lost_ownership_aborts_the_page_transaction(spend_window: Mapping[str, object]) -> None: + repository, _query, execute = _repository((spend_window,)) + execute.side_effect = (1, 0) + with pytest.raises(SweepOwnershipLostError): + await repository.evaluate_page("previous-worker", SweepState(day=date(2026, 1, 8), scope="model"), 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ("timeout", "invalid_data", "connection_lost")) +async def test_unreadable_data_is_retryable_but_database_failures_propagate(failure: str) -> None: + error: Final = _raw_query_error("query failed", "57014" if failure == "timeout" else "57P01") + repository, _query, execute = _repository(({},) if failure == "invalid_data" else error) + with pytest.raises(ResourceUnreadableError if failure != "connection_lost" else type(error)): + await repository.evaluate_page("worker-a", SweepState(day=date(2026, 1, 8), scope="model"), 25) + execute.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("wrapped", (False, True)) +async def test_skipping_an_unreadable_resource_saves_progress_without_counting_it_as_checked(wrapped: bool) -> None: + repository, query, execute = _repository(({"resource_id": "team-b"},)) + state: Final = SweepState( + day=date(2026, 1, 8), scope="team", cursor="team-a", wrapped=wrapped, start_scope="team", start_after="team-z" + ) + result: Final = await repository.skip_next("worker-a", state) + assert result == state.model_copy(update={"cursor": "team-b", "skipped": 1}) + query.assert_awaited_with(ANY, "2026-01-08", "2026-01-01", 1, "team-a", *(("team-z",) if wrapped else ())) + execute.assert_awaited_once_with( + ANY, "worker-a", "2026-01-08", "team", "team-b", wrapped, 0, 1, False, "team", "team-z" + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3d680bf333d..5ded88be8be 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -31505,6 +31505,12 @@ export interface components { * @description If True (default), a locally-managed user's password must contain an uppercase letter. */ password_policy_require_uppercase?: boolean | null; + /** + * Performance Alerts Enabled + * @description Check yesterday's usage for the gateway, models, teams, users and keys against the week before, and save unusual spend, traffic, response time and failure rate changes as alerts. Off by default. + * @default false + */ + performance_alerts_enabled: boolean; /** * Plugins * @description external services registered as embeddable UI plugins From 16a36c66b48971c8b365d8e92e6a0a5cc1a23091 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 12 Nov 2026 10:23:03 +0800 Subject: [PATCH 156/189] feat(ui): show performance alerts in the Alerts page (#51104) * feat(ui): restore performance alerts on updated detector * fix(ui): keep team key detail tabs independent * fix(ui): preserve team tabs while permissions load * feat(ui): show concise alerts in the notification bell * refactor(ui): remove replaced deprecation bell item * fix(ui): pin notification resource links * fix(ui): show deprecation failures in notifications * fix(ui): reset activity state when changing resources * fix(ui): reset key detail state from every entry point --- litellm/models/alert.py | 35 ++- litellm/proxy/alerting_endpoints/alerts.py | 30 ++- litellm/repositories/alert_repository.py | 157 +++++++++++- tests/integration/database/test_alerts.py | 176 +++++++++++++- .../proxy/alerting_endpoints/test_alerts.py | 97 +++++++- ui/litellm-dashboard/eslint-suppressions.json | 2 +- .../agent_info.integration.test.tsx | 3 +- .../agents/_components/agent_info.test.tsx | 62 ++++- .../agents/_components/agent_info.tsx | 19 +- .../src/app/(dashboard)/alerts/page.tsx | 20 +- .../CostOptimizationView.activity.test.tsx | 3 +- .../_components/CostOptimizationView.test.tsx | 3 +- .../useDailyActivityRange.test.tsx | 42 +++- .../_components/useDailyActivityRange.ts | 26 +- .../detailNavigation.test.ts | 23 ++ .../models-and-endpoints/detailNavigation.ts | 11 +- .../view_users.integration.test.tsx | 54 +++- .../users/_components/view_users.tsx | 8 +- .../user_info_view.integration.test.tsx | 6 +- .../view_users/user_info_view.test.tsx | 3 +- .../_components/view_users/user_info_view.tsx | 9 +- .../src/components/Alerts/AlertActions.tsx | 89 +++++++ .../src/components/Alerts/AlertChart.tsx | 123 ++++++++++ .../src/components/Alerts/AlertDetail.tsx | 83 +++++++ .../Alerts/Alerts.integration.test.tsx | 152 ++++++++++++ .../src/components/Alerts/Alerts.tsx | 230 ++++++++++++++++++ .../Alerts/alertPresentation.test.ts | 85 +++++++ .../components/Alerts/alertPresentation.ts | 166 +++++++++++++ .../src/components/Alerts/useAlerts.ts | 44 ++++ .../DashboardHeader.integration.test.tsx | 111 ++++++++- .../DeprecationAlertItem.tsx | 37 --- .../ModelAlertsPage.integration.test.tsx | 16 +- .../deprecationAlerts.test.ts | 11 + .../deprecationAlerts.ts | 19 +- .../components/Navbar/DashboardResources.tsx | 2 +- .../NotificationsBell.test.tsx | 72 +++++- .../NotificationsBell/NotificationsBell.tsx | 119 ++++++--- .../src/components/Teams.test.tsx | 12 +- ui/litellm-dashboard/src/components/Teams.tsx | 5 + .../ResourceActivity.integration.test.tsx | 36 ++- .../EntityUsage/TopKeyView.test.tsx | 51 +++- .../components/EntityUsage/TopKeyView.tsx | 10 +- .../VirtualKeysPage/VirtualKeysTable.test.tsx | 22 +- .../VirtualKeysPage/VirtualKeysTable.tsx | 14 +- .../logs/request/RequestLogsPanel.test.tsx | 38 +++ .../logs/request/RequestLogsPanel.tsx | 17 +- .../model_info_view.integration.test.tsx | 8 +- .../team/TeamInfo.integration.test.tsx | 69 +++++- .../src/components/team/TeamInfo.test.tsx | 5 +- .../src/components/team/TeamInfo.tsx | 17 +- .../team/TeamVirtualKeysTable.test.tsx | 15 +- .../components/team/TeamVirtualKeysTable.tsx | 18 +- .../KeyInfoView.handleKeyUpdate.test.tsx | 4 +- .../key_info_view.budget_display.test.tsx | 5 +- .../templates/key_info_view.test.tsx | 2 +- .../components/templates/key_info_view.tsx | 10 +- ui/litellm-dashboard/src/hooks/useUrlTab.ts | 15 +- ui/litellm-dashboard/src/i18n/locales/en.json | 90 ++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 113 ++++++++- 59 files changed, 2487 insertions(+), 237 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/Alerts/AlertActions.tsx create mode 100644 ui/litellm-dashboard/src/components/Alerts/AlertChart.tsx create mode 100644 ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx create mode 100644 ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Alerts/Alerts.tsx create mode 100644 ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts create mode 100644 ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts create mode 100644 ui/litellm-dashboard/src/components/Alerts/useAlerts.ts delete mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx diff --git a/litellm/models/alert.py b/litellm/models/alert.py index c9a26c6aff9..f96e042ff6b 100644 --- a/litellm/models/alert.py +++ b/litellm/models/alert.py @@ -1,10 +1,12 @@ -from datetime import date, datetime, timezone +from datetime import date as Date +from datetime import datetime, timezone from typing import Annotated, Final, Literal, Protocol, TypeAlias from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, field_validator, model_validator from typing_extensions import Self AlertType: TypeAlias = Literal["spend", "traffic", "latency", "reliability", "model_deprecation"] +AlertFilter: TypeAlias = AlertType | Literal["performance"] AlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] AlertResourceType: TypeAlias = Literal["gateway", "key", "team", "user", "model"] MAX_ALERT_ID: Final = 2**63 - 1 @@ -33,7 +35,7 @@ class MetricEvidence(BaseModel): class DeprecationEvidence(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - deprecation_date: date = Field(description="Date the affected model retires.") + deprecation_date: Date = Field(description="Date the affected model retires.") replacement_model: str | None = Field( default=None, min_length=1, max_length=1024, description="Suggested replacement." ) @@ -106,6 +108,22 @@ class Alert(AlertDetection): updated_by: str | None = Field(default=None, max_length=1024, description="User who last changed the status.") +class AlertHistoryPoint(BaseModel): + model_config = ConfigDict(frozen=True) + + date: Date = Field(description="UTC day represented by this value.") + value: float = Field(allow_inf_nan=False, description="Recorded daily value in the alert metric's unit.") + + +class AlertView(Alert): + resource_name: str | None = Field(default=None, max_length=1024, description="Current resource display name.") + history: tuple[AlertHistoryPoint, ...] | None = Field( + default=None, + max_length=8, + description="Current Usage values for the seven baseline days and observed day. Null if unavailable.", + ) + + class AlertUpdate(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -121,10 +139,21 @@ class AlertPage(BaseModel): ) +class AlertViewPage(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[AlertView, ...] = Field(description="Alerts ordered newest first, with current Usage history.") + next_cursor: AlertID | None = Field(default=None, description="Cursor for the next page, or null at the end.") + + class AlertStore(Protocol): async def record(self, detection: AlertDetection) -> Alert: ... - async def list(self, status: AlertStatus, limit: int, cursor: int | None) -> AlertPage: ... + async def list( + self, status: AlertStatus, limit: int, cursor: int | None, alert_type: AlertFilter | None = None + ) -> AlertPage: ... + + async def enrich(self, alerts: tuple[Alert, ...]) -> tuple[AlertView, ...]: ... async def get(self, alert_id: int) -> Alert | None: ... diff --git a/litellm/proxy/alerting_endpoints/alerts.py b/litellm/proxy/alerting_endpoints/alerts.py index fa586ca7a25..3adcab90ec8 100644 --- a/litellm/proxy/alerting_endpoints/alerts.py +++ b/litellm/proxy/alerting_endpoints/alerts.py @@ -2,7 +2,16 @@ from typing import Annotated, Final, TypeAlias from fastapi import APIRouter, Depends, HTTPException, Path, Query -from litellm.models.alert import Alert, AlertID, AlertPage, AlertStatus, AlertStore, AlertUpdate +from litellm.models.alert import ( + Alert, + AlertFilter, + AlertID, + AlertStatus, + AlertStore, + AlertUpdate, + AlertView, + AlertViewPage, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.deps.repositories import AlertRepositoryDep @@ -29,24 +38,29 @@ def _found(alert: Alert | None) -> Alert: return alert -@router.get("/alerts", response_model=AlertPage, summary="List saved alerts") +@router.get("/alerts", response_model=AlertViewPage, summary="List saved alerts") async def list_alerts( user: Auth, repository: AlertRepositoryDep, status: Annotated[AlertStatus, Query(description="Status to list.")] = "open", limit: Annotated[int, Query(ge=1, le=100, description="Maximum alerts per page.")] = 25, cursor: Annotated[AlertID | None, Query(description="next_cursor from the preceding page.")] = None, -) -> AlertPage: - """Read saved alerts, newest first. Gateway admins and view-only admins can read. No usage queries run.""" + alert_type: Annotated[ + AlertFilter | None, Query(alias="type", description="Alert category, or all performance alerts.") + ] = None, +) -> AlertViewPage: + """Read saved alerts with current resource names and eight days of Usage history. Requires a gateway admin.""" store: Final = _authorized_store(user, repository, write=False) - return await store.list(status, limit, int(cursor) if cursor is not None else None) + page: Final = await store.list(status, limit, int(cursor) if cursor is not None else None, alert_type) + return AlertViewPage(data=await store.enrich(page.data), next_cursor=page.next_cursor) -@router.get("/alerts/{alert_id}", response_model=Alert, summary="Read an alert") -async def get_alert(alert_id: AlertPath, user: Auth, repository: AlertRepositoryDep) -> Alert: +@router.get("/alerts/{alert_id}", response_model=AlertView, summary="Read an alert") +async def get_alert(alert_id: AlertPath, user: Auth, repository: AlertRepositoryDep) -> AlertView: """Return the saved evidence and status. Requires a gateway admin or view-only admin.""" store: Final = _authorized_store(user, repository, write=False) - return _found(await store.get(int(alert_id))) + alert: Final = _found(await store.get(int(alert_id))) + return (await store.enrich((alert,)))[0] @router.patch("/alerts/{alert_id}", response_model=Alert, summary="Change an alert's status") diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py index 299b1a67bdc..82048d53d83 100644 --- a/litellm/repositories/alert_repository.py +++ b/litellm/repositories/alert_repository.py @@ -1,6 +1,9 @@ import hashlib import json -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from contextlib import AbstractAsyncContextManager +from datetime import date as Date +from datetime import timedelta from typing import ( TYPE_CHECKING, Final, @@ -8,13 +11,103 @@ from typing import ( cast, # noqa: TID251 # Prisma exposes generated table actions dynamically ) -from litellm.models.alert import Alert, AlertDetection, AlertPage, AlertStatus +from pydantic import BaseModel, ConfigDict, TypeAdapter +from typing_extensions import LiteralString + +from litellm._logging import verbose_proxy_logger +from litellm.models.alert import ( + Alert, + AlertDetection, + AlertFilter, + AlertHistoryPoint, + AlertPage, + AlertResourceType, + AlertStatus, + AlertView, +) from litellm.repositories.prisma_protocols import PrismaRecord, TableActions if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient +class _HistoryTransaction(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... + + +class _HistoryDatabase(Protocol): + def tx(self, *, timeout: timedelta) -> AbstractAsyncContextManager[_HistoryTransaction]: ... + + +class _HistoryRow(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str + resource_name: str | None + date: Date | None + spend: float + requests: int + failed: int + response_ms: int + timed: int + + def point(self, kind: str) -> AlertHistoryPoint | None: + if self.date is None or (kind == "latency" and not self.timed) or (kind == "reliability" and not self.requests): + return None + value: Final = { + "spend": self.spend, + "traffic": float(self.requests), + "latency": self.response_ms / self.timed if self.timed else 0, + "reliability": min(1.0, self.failed / self.requests) if self.requests else 0, + }[kind] + return AlertHistoryPoint(date=self.date, value=value) + + +_HISTORY_ROWS: Final = TypeAdapter(tuple[_HistoryRow, ...]) +_HISTORY_SOURCES: Final[ + Mapping[AlertResourceType, tuple[LiteralString, LiteralString, LiteralString, LiteralString, LiteralString]] +] = { + "gateway": ('"LiteLLM_DailyModelUsage"', "'gateway'", "request_count", "", "'Gateway'"), + "model": ('"LiteLLM_DailyModelUsage"', "d.model_group", "request_count", "", "r.resource_id"), + "team": ( + '"LiteLLM_DailyTeamSpend"', + "d.team_id", + "api_requests", + 'LEFT JOIN "LiteLLM_TeamTable" m ON m.team_id = r.resource_id', + "m.team_alias", + ), + "user": ( + '"LiteLLM_DailyUserSpend"', + "d.user_id", + "api_requests", + 'LEFT JOIN "LiteLLM_UserTable" m ON m.user_id = r.resource_id', + "m.user_alias", + ), + "key": ( + '"LiteLLM_DailyUserSpend"', + "d.api_key", + "api_requests", + 'LEFT JOIN "LiteLLM_VerificationToken" m ON m.token = r.resource_id', + "m.key_alias", + ), +} + + +def _history_sql(scope: AlertResourceType) -> LiteralString: + table, resource, requests, metadata, name = _HISTORY_SOURCES[scope] + return f"""SELECT r.id, LEFT({name}, 1024) AS resource_name, d.date, + COALESCE(SUM(d.spend), 0)::float8 AS spend, + COALESCE(SUM(d.{requests}), 0)::bigint AS requests, + COALESCE(SUM(d.failed_requests), 0)::bigint AS failed, + COALESCE(SUM(d.total_response_time_ms), 0)::bigint AS response_ms, + COALESCE(SUM(d.timed_requests), 0)::bigint AS timed + FROM requested r + LEFT JOIN {table} d ON {resource} = r.resource_id AND d.date BETWEEN r.start AND r.day + {metadata} + WHERE r.resource_type = '{scope}' + GROUP BY r.id, LEFT({name}, 1024), d.date""" + + class AlertTransaction(Protocol): async def execute_raw(self, query: str, *args: object) -> int: ... @@ -52,6 +145,9 @@ def _alert(row: PrismaRecord) -> Alert: class AlertRepository: def __init__(self, prisma_client: "PrismaClient") -> None: + self._database: Final = cast( # cast-ok: Prisma transaction exposes bounded raw query methods + _HistoryDatabase, prisma_client.writer_db + ) self._table = cast( # cast-ok: the generated Prisma client delegates typed table actions dynamically TableActions[PrismaRecord], prisma_client.writer_db.litellm_alert ) @@ -76,10 +172,17 @@ class AlertRepository: raise return _alert(existing) - async def list(self, status: AlertStatus, limit: int, cursor: int | None) -> AlertPage: + async def list( + self, status: AlertStatus, limit: int, cursor: int | None, alert_type: AlertFilter | None = None + ) -> AlertPage: size: Final = max(1, min(limit, 100)) + category: Final = {"not": "model_deprecation"} if alert_type == "performance" else alert_type rows: Final = await self._table.find_many( - where={"status": status, **({"id": {"lt": cursor}} if cursor is not None else {})}, + where={ + "status": status, + **({"id": {"lt": cursor}} if cursor is not None else {}), + **({"type": category} if category is not None else {}), + }, order={"id": "desc"}, take=size + 1, ) @@ -95,3 +198,49 @@ class AlertRepository: async def set_status(self, alert_id: int, status: AlertStatus, actor: str) -> Alert | None: row: Final = await self._table.update(where={"id": alert_id}, data={"status": status, "updated_by": actor}) return None if row is None else _alert(row) + + async def enrich(self, alerts: tuple[Alert, ...]) -> tuple[AlertView, ...]: + from prisma.errors import PrismaError + + requested: Final = tuple( + { + "id": alert.id, + "resource_id": alert.resource_id, + "resource_type": alert.resource_type, + "day": alert.window_start.date().isoformat(), + "start": (alert.window_start.date() - timedelta(days=7)).isoformat(), + } + for alert in alerts + if alert.type != "model_deprecation" + ) + if not requested: + return tuple(AlertView.model_validate(alert.model_dump()) for alert in alerts) + scopes: Final = frozenset[AlertResourceType](alert.resource_type for alert in alerts) + histories: Final = " UNION ALL ".join(_history_sql(scope) for scope in scopes) + query: Final = f"""WITH requested AS ( + SELECT * FROM jsonb_to_recordset($1::jsonb) + AS r(id text, resource_id text, resource_type text, day text, start text) + ) {histories} ORDER BY id, date""" + try: + async with self._database.tx(timeout=timedelta(seconds=3)) as transaction: + await transaction.query_raw( + "SELECT set_config('statement_timeout', '2s', true), set_config('work_mem', '1MB', true), " + "set_config('max_parallel_workers_per_gather', '0', true)" + ) + rows: Final = _HISTORY_ROWS.validate_python(await transaction.query_raw(query, json.dumps(requested))) + return tuple(_view(alert, rows) for alert in alerts) + except (PrismaError, TimeoutError, ValueError) as error: + verbose_proxy_logger.warning("Alert Usage history unavailable (%s)", type(error).__name__) + return tuple(AlertView.model_validate(alert.model_dump()) for alert in alerts) + + +def _view(alert: Alert, rows: tuple[_HistoryRow, ...]) -> AlertView: + history: Final = tuple(row for row in rows if row.id == alert.id) + points: Final = tuple(point for row in history if (point := row.point(alert.type)) is not None) + return AlertView.model_validate( + { + **alert.model_dump(), + "resource_name": history[0].resource_name if history else None, + "history": points, + } + ) diff --git a/tests/integration/database/test_alerts.py b/tests/integration/database/test_alerts.py index b3054edba02..dc62886b75b 100644 --- a/tests/integration/database/test_alerts.py +++ b/tests/integration/database/test_alerts.py @@ -8,8 +8,17 @@ from typing import Final import pytest import pytest_asyncio +from psycopg import sql -from litellm.models.alert import Alert, AlertDetection, AlertPage, AlertType +from litellm.models.alert import ( + Alert, + AlertDetection, + AlertPage, + AlertResourceType, + AlertType, + AlertView, + AlertViewPage, +) from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.alert_repository import AlertRepository @@ -17,6 +26,18 @@ from tests.integration._support.client import Gateway from tests.integration._support.database import read_rows, write_rows +def saved_alert(content: bytes) -> Alert: + view: Final = AlertView.model_validate_json(content) + return Alert.model_validate(view.model_dump(exclude={"resource_name", "history"})) + + +def saved_page(content: bytes) -> AlertPage: + page: Final = AlertViewPage.model_validate_json(content) + return AlertPage( + data=tuple(saved_alert(alert.model_dump_json().encode()) for alert in page.data), next_cursor=page.next_cursor + ) + + @dataclass(frozen=True, slots=True) class Alerts: repository: AlertRepository @@ -71,13 +92,13 @@ async def test_each_alert_type_round_trips_through_api_and_status_changes( saved: Final = await alerts.repository.record(detection) response: Final = gateway.request("GET", f"/alerts/{saved.id}") assert response.status_code == 200, response.text - assert Alert.model_validate_json(response.content) == saved + assert saved_alert(response.content) == saved assert saved.status == "open" assert saved.model_dump(include=set(AlertDetection.model_fields)) == detection.model_dump() for status in ("resolved", "dismissed", "open"): update: Final = gateway.request("PATCH", f"/alerts/{saved.id}", {"status": status}) assert update.status_code == 200, update.text - changed: Final = Alert.model_validate_json(update.content) + changed: Final = saved_alert(update.content) assert changed.status == status assert changed.updated_by assert changed.evidence == saved.evidence @@ -100,13 +121,13 @@ async def test_gateway_alert_round_trips_without_colliding_with_resource_alerts( assert scoped.id != saved.id detail: Final = gateway.request("GET", f"/alerts/{saved.id}") assert detail.status_code == 200, detail.text - assert Alert.model_validate_json(detail.content) == saved + assert saved_alert(detail.content) == saved listed: Final = gateway.request("GET", "/alerts", params={"limit": "2"}) assert listed.status_code == 200, listed.text - assert AlertPage.model_validate_json(listed.content).data == (scoped, saved) + assert saved_page(listed.content).data == (scoped, saved) update: Final = gateway.request("PATCH", f"/alerts/{saved.id}", {"status": "dismissed"}) assert update.status_code == 200, update.text - dismissed: Final = Alert.model_validate_json(update.content) + dismissed: Final = saved_alert(update.content) assert dismissed.status == "dismissed" assert await alerts.repository.record(detection) == dismissed assert await alerts.repository.get(int(scoped.id)) == scoped @@ -143,19 +164,19 @@ async def test_keyset_pages_survive_status_changes_and_do_not_include_newer_aler newest: Final = await alerts.repository.record(alerts.detection(day=2)) first: Final = gateway.request("GET", "/alerts", params={"limit": "2"}) assert first.status_code == 200, first.text - page: Final = AlertPage.model_validate_json(first.content) + page: Final = saved_page(first.content) assert tuple(alert.id for alert in page.data) == (newest.id, middle.id) assert page.next_cursor == middle.id await alerts.repository.set_status(int(middle.id), "resolved", "reviewer") late: Final = await alerts.repository.record(alerts.detection(day=3)) second: Final = gateway.request("GET", "/alerts", params={"limit": "2", "cursor": middle.id}) assert second.status_code == 200, second.text - remainder: Final = AlertPage.model_validate_json(second.content) + remainder: Final = saved_page(second.content) assert oldest.id in tuple(alert.id for alert in remainder.data) assert not {newest.id, middle.id, late.id}.intersection(alert.id for alert in remainder.data) resolved: Final = gateway.request("GET", "/alerts", params={"status": "resolved", "limit": "1"}) assert resolved.status_code == 200, resolved.text - assert AlertPage.model_validate_json(resolved.content).data[0].id == middle.id + assert saved_page(resolved.content).data[0].id == middle.id @pytest.mark.asyncio @@ -212,3 +233,140 @@ async def test_windows_round_trip_at_database_precision_and_timezone(alerts: Ale } ) assert await alerts.repository.record(shifted) == saved + + +@pytest.mark.parametrize("scope", ("model", "key", "user", "team", "gateway")) +@pytest.mark.asyncio +async def test_daily_history_matches_scope_and_weighted_metrics( + alerts: Alerts, gateway: Gateway, scope: AlertResourceType +) -> None: + resource: Final = {"key": uuid.uuid4().hex * 2, "gateway": "gateway"}.get(scope, alerts.resource_id) + model_source: Final = scope in ("model", "gateway") + table, column = { + "model": ("LiteLLM_DailyModelUsage", "model_group"), + "gateway": ("LiteLLM_DailyModelUsage", "model_group"), + "team": ("LiteLLM_DailyTeamSpend", "team_id"), + "user": ("LiteLLM_DailyUserSpend", "user_id"), + "key": ("LiteLLM_DailyUserSpend", "api_key"), + }[scope] + marker: Final = alerts.resource_id + request_column: Final = "request_count" if model_source else "api_requests" + insert: Final = sql.SQL("""INSERT INTO {} ({}, date, model, {}, {}, spend, {}, failed_requests, + total_response_time_ms, timed_requests, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())""").format( + sql.Identifier(table), + sql.Identifier(column), + sql.Identifier("custom_llm_provider" if model_source else "id"), + sql.Identifier("task_type" if model_source else ("user_id" if scope == "key" else "api_key")), + sql.Identifier(request_column), + ) + try: + for day in ("2025-12-24", "2025-12-25", "2026-01-01", "2026-01-02"): + for deployment, count in (("a", 2), ("b", 8)): + write_rows( + insert, + ( + resource, + day, + marker + deployment, + "example" if model_source else uuid.uuid4().hex, + "chat" if model_source else "", + float(count), + count, + 1, + count * count * 100, + count, + ), + ) + for kind, expected in (("spend", 10), ("traffic", 10), ("latency", 680), ("reliability", 0.2)): + detection: Final = AlertDetection.model_validate( + { + **alerts.detection(kind).model_dump(), + "resource_type": scope, + "resource_id": resource, + } + ) + saved: Final = await alerts.repository.record(detection) + response: Final = gateway.request("GET", f"/alerts/{saved.id}") + assert response.status_code == 200, response.text + view: Final = AlertView.model_validate_json(response.content) + assert view.history is not None + assert tuple(point.date.isoformat() for point in view.history) == ("2025-12-25", "2026-01-01") + assert tuple(point.value for point in view.history) == pytest.approx((expected, expected)) + assert view.evidence == saved.evidence + assert view.resource_name == ( + resource if model_source and scope != "gateway" else "Gateway" if scope == "gateway" else None + ) + listed: Final = gateway.request("GET", "/alerts", params={"type": kind, "limit": "1"}) + assert listed.status_code == 200, listed.text + assert AlertViewPage.model_validate_json(listed.content).data[0] == view + finally: + write_rows( + sql.SQL("DELETE FROM {} WHERE model IN (%s, %s)").format(sql.Identifier(table)), + (marker + "a", marker + "b"), + ) + write_rows('DELETE FROM "LiteLLM_Alert" WHERE resource_id=%s', (resource,)) + + +@pytest.mark.asyncio +async def test_type_filter_applies_before_cursor_limit(alerts: Alerts, gateway: Gateway) -> None: + oldest: Final = await alerts.repository.record(alerts.detection("traffic")) + await alerts.repository.record(alerts.detection("spend")) + newest: Final = await alerts.repository.record(alerts.detection("traffic", day=1)) + await alerts.repository.record(alerts.detection("model_deprecation")) + page: Final = gateway.request("GET", "/alerts", params={"type": "traffic", "limit": "1"}) + assert page.status_code == 200, page.text + first: Final = AlertViewPage.model_validate_json(page.content) + assert first.data[0].id == newest.id + assert first.next_cursor == newest.id + following: Final = gateway.request("GET", "/alerts", params={"type": "traffic", "limit": "1", "cursor": newest.id}) + assert following.status_code == 200, following.text + assert AlertViewPage.model_validate_json(following.content).data[0].id == oldest.id + performance: Final = gateway.request("GET", "/alerts", params={"type": "performance"}) + assert performance.status_code == 200, performance.text + assert all( + alert.type != "model_deprecation" for alert in AlertViewPage.model_validate_json(performance.content).data + ) + + +@pytest.mark.asyncio +async def test_deleted_resource_preserves_history_and_history_timeout_preserves_alert( + alerts: Alerts, gateway: Gateway +) -> None: + import psycopg + + team: Final = alerts.resource_id + write_rows('INSERT INTO "LiteLLM_TeamTable" (team_id, team_alias) VALUES (%s, %s)', (team, "Example team")) + write_rows( + """INSERT INTO "LiteLLM_DailyTeamSpend" (id, team_id, api_key, date, spend, updated_at) + VALUES (%s, %s, '', '2026-01-01', 20, NOW())""", + (team, team), + ) + try: + saved: Final = await alerts.repository.record( + AlertDetection.model_validate( + { + **alerts.detection().model_dump(), + "resource_type": "team", + } + ) + ) + first: Final = gateway.request("GET", f"/alerts/{saved.id}") + assert first.status_code == 200, first.text + assert AlertView.model_validate_json(first.content).resource_name == "Example team" + write_rows('DELETE FROM "LiteLLM_TeamTable" WHERE team_id=%s', (team,)) + deleted: Final = gateway.request("GET", f"/alerts/{saved.id}") + assert deleted.status_code == 200, deleted.text + view: Final = AlertView.model_validate_json(deleted.content) + assert view.resource_name is None + assert view.history and view.history[0].value == 20 + with psycopg.connect(os.environ["DATABASE_URL"]) as connection: + connection.execute('LOCK TABLE "LiteLLM_DailyTeamSpend" IN ACCESS EXCLUSIVE MODE') + unavailable: Final = gateway.request("GET", f"/alerts/{saved.id}") + assert unavailable.status_code == 200, unavailable.text + fallback: Final = AlertView.model_validate_json(unavailable.content) + assert fallback.history is None + assert fallback.evidence == saved.evidence + finally: + write_rows('DELETE FROM "LiteLLM_TeamTable" WHERE team_id=%s', (team,)) + write_rows('DELETE FROM "LiteLLM_DailyTeamSpend" WHERE team_id=%s', (team,)) diff --git a/tests/unit/proxy/alerting_endpoints/test_alerts.py b/tests/unit/proxy/alerting_endpoints/test_alerts.py index 15dfad975fc..bb651ef3807 100644 --- a/tests/unit/proxy/alerting_endpoints/test_alerts.py +++ b/tests/unit/proxy/alerting_endpoints/test_alerts.py @@ -43,11 +43,17 @@ def table() -> MagicMock: @pytest.fixture -def app(table: MagicMock) -> FastAPI: +def history_query() -> AsyncMock: + return AsyncMock(return_value=[]) + + +@pytest.fixture +def app(table: MagicMock, history_query: AsyncMock) -> FastAPI: app: Final = FastAPI() app.include_router(router) prisma: Final = MagicMock() prisma.writer_db.litellm_alert = table + prisma.writer_db.tx.return_value.__aenter__.return_value.query_raw = history_query app.dependency_overrides[get_database] = lambda: Database(prisma, native_database()) app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN @@ -79,7 +85,8 @@ def test_permissions_precede_database_access( @pytest.mark.parametrize( - "query", ("limit=0", "limit=101", "cursor=0", "cursor=9223372036854775808", "cursor=oops", "status=invalid") + "query", + ("limit=0", "limit=101", "cursor=0", "cursor=9223372036854775808", "cursor=oops", "status=invalid", "type=invalid"), ) def test_invalid_page_never_reads_database(client: TestClient, table: MagicMock, query: str) -> None: assert client.get(f"/alerts?{query}").status_code == 422 @@ -134,7 +141,7 @@ def test_openapi_keeps_ids_and_cursors_as_strings(client: TestClient) -> None: assert {s["type"] for s in cursor["schema"]["anyOf"]} == {"string", "null"} schemas: Final = spec["components"]["schemas"] assert schemas["Alert"]["properties"]["id"]["type"] == "string" - assert {s["type"] for s in schemas["AlertPage"]["properties"]["next_cursor"]["anyOf"]} == {"string", "null"} + assert {s["type"] for s in schemas["AlertViewPage"]["properties"]["next_cursor"]["anyOf"]} == {"string", "null"} def test_viewer_reads_saved_alert_with_string_id(app: FastAPI, table: MagicMock) -> None: @@ -145,7 +152,10 @@ def test_viewer_reads_saved_alert_with_string_id(app: FastAPI, table: MagicMock) response: Final = client.get("/alerts?status=open&limit=1") assert response.status_code == 200, response.text expected: Final = StoredAlert().model_dump(mode="json", exclude={"detection_key"}) - assert response.json() == {"data": [{**expected, "id": "5"}], "next_cursor": None} + assert response.json() == { + "data": [{**expected, "id": "5", "resource_name": None, "history": []}], + "next_cursor": None, + } @pytest.mark.parametrize("method", ("GET", "PATCH")) @@ -159,3 +169,82 @@ def test_missing_alert_returns_not_found(client: TestClient, method: str) -> Non def test_status_update_rejects_other_changes(client: TestClient, table: MagicMock, body: object) -> None: assert client.patch("/alerts/5", json=body).status_code == 422 table.update.assert_not_awaited() + + +def test_empty_alert_page_does_not_read_usage(client: TestClient, table: MagicMock, history_query: AsyncMock) -> None: + table.find_many.return_value = [] + response: Final = client.get("/alerts") + assert response.status_code == 200 + assert response.json() == {"data": [], "next_cursor": None} + history_query.assert_not_awaited() + + +@pytest.mark.parametrize( + "kind,unit,expected", + ( + ("spend", "usd", 12), + ("traffic", "requests", 20), + ("latency", "milliseconds", 250), + ("reliability", "ratio", 0.25), + ), +) +def test_detail_returns_usage_history_without_replacing_saved_evidence( + client: TestClient, table: MagicMock, history_query: AsyncMock, kind: str, unit: str, expected: float +) -> None: + evidence: Final = {"value": 0.5, "baseline": 0.1, "unit": unit, "sample_size": 50} + table.find_unique.return_value = StoredAlert(type=kind, evidence=evidence) + history_query.return_value = [ + { + "id": "5", + "resource_name": "Research", + "date": "2026-01-08", + "spend": 12, + "requests": 20, + "failed": 5, + "response_ms": 3750, + "timed": 15, + } + ] + response: Final = client.get("/alerts/5") + assert response.status_code == 200 + assert response.json()["resource_name"] == "Research" + assert response.json()["history"] == [{"date": "2026-01-08", "value": expected}] + assert response.json()["evidence"] == evidence + + +@pytest.mark.parametrize("failure", (TimeoutError("private row value"), ValueError("private row value"))) +def test_history_failure_keeps_alert_readable_and_logs_only_error_type( + client: TestClient, table: MagicMock, history_query: AsyncMock, caplog: pytest.LogCaptureFixture, failure: Exception +) -> None: + table.find_unique.return_value = StoredAlert() + history_query.side_effect = failure + response: Final = client.get("/alerts/5") + assert response.status_code == 200 + assert response.json()["evidence"] == StoredAlert().evidence + assert response.json()["history"] is None + assert f"Alert Usage history unavailable ({type(failure).__name__})" in caplog.text + assert "private row value" not in caplog.text + + +@pytest.mark.parametrize("kind,unit", (("latency", "milliseconds"), ("reliability", "ratio"))) +def test_no_samples_leave_a_gap_in_history( + client: TestClient, table: MagicMock, history_query: AsyncMock, kind: str, unit: str +) -> None: + table.find_unique.return_value = StoredAlert( + type=kind, evidence={"value": 0.5, "baseline": 0.1, "unit": unit, "sample_size": 50} + ) + history_query.return_value = [ + { + "id": "5", + "resource_name": "Research", + "date": "2026-01-08", + "spend": 0, + "requests": 0, + "failed": 0, + "response_ms": 0, + "timed": 0, + } + ] + response: Final = client.get("/alerts/5") + assert response.status_code == 200 + assert response.json()["history"] == [] diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index e9bbeb94355..6755900f445 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1740,7 +1740,7 @@ "count": 5 }, "no-restricted-syntax": { - "count": 132 + "count": 131 }, "prefer-const": { "count": 26 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx index e08cab776c4..2a0069f20f1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx @@ -1,6 +1,7 @@ import React from "react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { fireEvent, screen, waitFor } from "@testing-library/react"; +import { renderWithProviders as render } from "../../../../../tests/test-utils"; import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; import { describe, it, expect, vi, beforeEach } from "vitest"; import AgentInfoView from "./agent_info"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx index 4eb3534ebe3..3f62ca35085 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx @@ -1,5 +1,10 @@ import React from "react"; -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { fireEvent, screen, waitFor } from "@testing-library/react"; +import { renderWithProviders as render } from "../../../../../tests/test-utils"; +import { parseAsString, useQueryStates } from "nuqs"; +import type { UrlUpdateEvent } from "nuqs/adapters/testing"; +import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; +import type { KeyResponse } from "@/components/key_team_helpers/key_list"; import { describe, it, expect, vi, beforeEach } from "vitest"; import AgentInfoView from "./agent_info"; import AgentFormFields from "./agent_form_fields"; @@ -14,7 +19,30 @@ vi.mock("@/components/networking", () => ({ })); vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ - useKeys: () => ({ data: { keys: [] }, isLoading: false, refetch: vi.fn() }), + useKeys: vi.fn(() => ({ data: { keys: [] }, isLoading: false, refetch: vi.fn() })), +})); + +vi.mock("@/components/templates/key_info_view", () => ({ + default: function KeyInfoViewMock({ onClose, onDelete }: { onClose: () => void; onDelete: () => void }) { + const [, setActivity] = useQueryStates({ + detail_tab: parseAsString, + activity_from: parseAsString, + activity_to: parseAsString, + }); + return ( +
+ + + +
+ ); + }, })); vi.mock("./AgentIdentityDetails", () => ({ @@ -82,6 +110,11 @@ const agent = { describe("AgentInfoView settings", () => { beforeEach(() => { vi.restoreAllMocks(); + vi.mocked(useKeys).mockReturnValue({ + data: { keys: [] }, + isLoading: false, + refetch: vi.fn(), + } as unknown as ReturnType); vi.mocked(AgentFormFields) .mockReset() .mockImplementation(() =>
); @@ -91,6 +124,31 @@ describe("AgentInfoView settings", () => { vi.mocked(networking.triggerAgentKillSwitchCall).mockReset(); }); + it.each(["Back to agent", "Delete key"])("clears key tabs and dates when opening a key and on %s", async (action) => { + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + const refetch = vi.fn(); + vi.mocked(useKeys).mockReturnValue({ + data: { keys: [{ token: "agent-key-1", key_alias: "Agent key" } as KeyResponse] }, + isLoading: false, + refetch, + } as unknown as ReturnType); + render(, { + searchParams: "?agent=agent-1&detail_tab=activity&activity_from=2026-10-01&activity_to=2026-10-02", + onUrlUpdate, + }); + + fireEvent.click(await screen.findByRole("button", { name: "agent-key-1..." })); + expect(await screen.findByRole("button", { name: "Back to agent" })).toBeInTheDocument(); + await waitFor(() => expect(onUrlUpdate.mock.lastCall?.[0].searchParams.toString()).toBe("agent=agent-1")); + + fireEvent.click(screen.getByRole("button", { name: "Select activity" })); + await waitFor(() => expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("activity_from")).toBe("2026-11-01")); + fireEvent.click(screen.getByRole("button", { name: action })); + await waitFor(() => expect(onUrlUpdate.mock.lastCall?.[0].searchParams.toString()).toBe("agent=agent-1")); + expect(await screen.findByRole("button", { name: "agent-key-1..." })).toBeInTheDocument(); + if (action === "Delete key") expect(refetch).toHaveBeenCalledOnce(); + }); + it("submits the edited agent when Save Changes is pressed", async () => { render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx index 554116716ea..253d31ceaf4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx @@ -51,6 +51,7 @@ import AgentCostView from "./agent_cost_view"; import { detectAgentType, parseDynamicAgentForForm } from "./agent_type_utils"; import AgentCardDiscovery, { DiscoveredAgentCardSelection } from "./agent_card_discovery"; import { buildDiscoveryRequest, overlayDiscoveredCardParams } from "./agent_discovery_utils"; +import { useResetDetailState } from "@/hooks/useUrlTab"; interface AgentInfoViewProps { agentId: string; @@ -81,8 +82,13 @@ const DetailItem: React.FC<{ label: React.ReactNode; children: React.ReactNode } const AgentInfoView: React.FC = ({ agentId, onClose, accessToken, isAdmin }) => { const { t } = useTranslation(); + const resetDetailState = useResetDetailState(); const [agent, setAgent] = useState(null); const [selectedKey, setSelectedKey] = useState(null); + const closeKey = () => { + resetDetailState(); + setSelectedKey(null); + }; const { data: keysData, isLoading: keysLoading, refetch: refetchAgentKeys } = useKeys(1, 100, { agentID: agentId }); const agentKeys = keysData?.keys ?? []; const [isLoading, setIsLoading] = useState(true); @@ -310,9 +316,9 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT setSelectedKey(null)} + onClose={closeKey} onDelete={() => { - setSelectedKey(null); + closeKey(); refetchAgentKeys(); }} teams={null} @@ -424,7 +430,14 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT {formatDate(agent.updated_at)} - + { + resetDetailState(); + setSelectedKey(key); + }} + /> {agent.object_permission && (agent.object_permission.mcp_servers?.length || diff --git a/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx index fe4692192d8..9506d21d6f4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx @@ -1,21 +1,5 @@ -"use client"; - -import { useTranslation } from "react-i18next"; -import { Page, PageTabs, PageTabsList, PageTabsTrigger, PageTabsContent } from "@/components/shared/Page"; -import ModelDeprecationsPanel from "@/components/ModelDeprecationAlerts/ModelDeprecationsPanel"; +import Alerts from "@/components/Alerts/Alerts"; export default function AlertsPage() { - const { t } = useTranslation(); - return ( - - - - {t("alerts.page.tabs.modelDeprecations")} - - - - - - - ); + return ; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx index 204c4b1a409..a9aaf8dd848 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx @@ -1,5 +1,6 @@ +import { renderWithProviders as render } from "../../../../../tests/test-utils"; import React from "react"; -import { fireEvent, render, waitFor, screen } from "@testing-library/react"; +import { fireEvent, waitFor, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx index a2f0ca5edc9..39b7428ea76 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx @@ -1,5 +1,6 @@ +import { renderWithProviders as render } from "../../../../../tests/test-utils"; import React from "react"; -import { fireEvent, render, screen } from "@testing-library/react"; +import { fireEvent, screen } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx index 90d6694ccba..3877216a424 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx @@ -1,9 +1,13 @@ -import { renderHook } from "@testing-library/react"; +import { NuqsTestingAdapter } from "nuqs/adapters/testing"; +import { act, renderHook as testingRenderHook, waitFor } from "@testing-library/react"; +import { type PropsWithChildren } from "react"; import { describe, expect, it, vi } from "vitest"; import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; const mockUseAggregatedDailyActivity = vi.fn(); +const renderHook: typeof testingRenderHook = (render, options) => + testingRenderHook(render, { ...options, wrapper: NuqsTestingAdapter }); vi.mock("@/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity", () => ({ useAggregatedDailyActivity: (options: unknown) => { @@ -87,3 +91,39 @@ describe("useDailyActivityRange", () => { }); }); }); + +it("opens the alert's complete UTC date window", () => { + const { result } = testingRenderHook(() => useActivityDateRange(), { + wrapper: ({ children }) => ( + {children} + ), + }); + expect(result.current.dateValue.from?.getDate()).toBe(1); + expect(result.current.dateValue.to?.getDate()).toBe(8); + expect(result.current.dateValue.from?.getFullYear()).toBe(2026); +}); + +it("updates an already mounted Activity view when another alert changes the URL dates", async () => { + let searchParams = "activity_from=2026-01-01&activity_to=2026-01-08"; + const wrapper = ({ children }: PropsWithChildren) => ( + + {children} + + ); + const { result, rerender } = testingRenderHook(() => useActivityDateRange(), { wrapper }); + searchParams = "activity_from=2026-02-01&activity_to=2026-02-08"; + rerender(); + await waitFor(() => expect(result.current.dateValue.from).toEqual(new Date(2026, 1, 1, 12))); + expect(result.current.dateValue.to).toEqual(new Date(2026, 1, 8, 12)); +}); + +it("updates the dates when the user changes the date picker", async () => { + const wrapper = ({ children }: PropsWithChildren) => ( + + {children} + + ); + const { result } = testingRenderHook(() => useActivityDateRange(), { wrapper }); + await act(async () => result.current.onDateChange({ from: new Date(2026, 2, 1, 12), to: new Date(2026, 2, 8, 12) })); + expect(result.current.dateValue.from).toEqual(new Date(2026, 2, 1, 12)); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts index 1af009397b1..eec33e54fc4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts @@ -1,4 +1,5 @@ -import { useMemo, useState } from "react"; +import { useMemo } from "react"; +import { createParser, useQueryStates } from "nuqs"; import { dailyActivityAggregatedCall } from "@/components/networking"; import { @@ -12,6 +13,15 @@ import { spendScopeUserId } from "@/utils/roles"; import { useAggregatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity"; const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; +const activityDate = createParser({ + parse: (value) => { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return null; + const date = new Date(`${value}T12:00:00`); + return Number.isFinite(date.getTime()) && date.toLocaleDateString("en-CA") === value ? date : null; + }, + serialize: (value) => value.toLocaleDateString("en-CA"), +}); +const activityDates = { activity_from: activityDate, activity_to: activityDate }; export interface DateRange { from?: Date; @@ -41,8 +51,18 @@ export type ActivityDateRange = Pick { const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []); const initialTo = useMemo(() => new Date(), []); - const [dateValue, setDateValue] = useState({ from: initialFrom, to: initialTo }); - return { dateValue, onDateChange: setDateValue }; + const [params, setParams] = useQueryStates(activityDates); + const dateValue = useMemo( + () => + params.activity_from && params.activity_to && params.activity_from <= params.activity_to + ? { from: params.activity_from, to: params.activity_to } + : { from: initialFrom, to: initialTo }, + [params.activity_from, params.activity_to, initialFrom, initialTo], + ); + const onDateChange = (value: DateRange) => { + void setParams({ activity_from: value.from ?? null, activity_to: value.to ?? null }); + }; + return { dateValue, onDateChange }; }; export interface ScopedActivityInput { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/detailNavigation.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/detailNavigation.test.ts index c4ea206022c..5c28fed2723 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/detailNavigation.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/detailNavigation.test.ts @@ -46,6 +46,29 @@ describe("useModelDetailRouting", () => { expect(event?.searchParams.has("team")).toBe(false); }); + it.each(["openModel", "openTeam", "close"] as const)( + "%s clears the previous resource's tab and dates", + async (action) => { + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + const { result } = renderHook(() => useModelDetailRouting(), { + wrapper: withNuqsTestingAdapter({ + searchParams: + "?model=old&detail_tab=activity&key_tab=activity&activity_from=2026-01-01&activity_to=2026-01-08&tab=all-models", + onUrlUpdate, + }), + }); + await act(async () => { + result.current[action]("next"); + }); + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + const identity = action === "close" ? {} : { [action === "openModel" ? "model" : "team"]: "next" }; + expect(Object.fromEntries(onUrlUpdate.mock.calls.at(-1)![0].searchParams)).toEqual({ + ...identity, + tab: "all-models", + }); + }, + ); + it("reads modelId and teamId from the query string", () => { const { result } = renderHook(() => useModelDetailRouting(), { wrapper: withNuqsTestingAdapter({ searchParams: "?model=xyz" }), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/detailNavigation.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/detailNavigation.ts index e83a81a53cf..2c70f184b63 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/detailNavigation.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/detailNavigation.ts @@ -1,5 +1,6 @@ import { parseAsString, useQueryState, useQueryStates } from "nuqs"; import { useCallback } from "react"; +import { useResetDetailState } from "@/hooks/useUrlTab"; export interface ModelDetailRouting { modelId: string | null; @@ -10,6 +11,7 @@ export interface ModelDetailRouting { } export function useModelDetailRouting(): ModelDetailRouting { + const resetDetailState = useResetDetailState(); const [{ model, team }, setParams] = useQueryStates( { model: parseAsString, team: parseAsString }, { history: "push" }, @@ -17,21 +19,24 @@ export function useModelDetailRouting(): ModelDetailRouting { const openModel = useCallback( (id: string) => { + resetDetailState(); void setParams({ model: id, team: null }); }, - [setParams], + [setParams, resetDetailState], ); const openTeam = useCallback( (id: string) => { + resetDetailState(); void setParams({ model: null, team: id }); }, - [setParams], + [setParams, resetDetailState], ); const close = useCallback(() => { + resetDetailState(); void setParams({ model: null, team: null }); - }, [setParams]); + }, [setParams, resetDetailState]); return { modelId: model, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.integration.test.tsx index d728f6473f5..e1166f94672 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.integration.test.tsx @@ -3,6 +3,7 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { fireEvent, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import React from "react"; +import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders } from "../../../../../tests/test-utils"; @@ -33,8 +34,21 @@ vi.mock("@/components/networking", () => ({ // The detail view has its own test; stub it so this file covers the parent's swap. vi.mock("./view_users/user_info_view", () => ({ - default: function UserInfoViewMock({ userId, startInEditMode }: { userId: string; startInEditMode?: boolean }) { - return
{`detail:${userId}:${String(Boolean(startInEditMode))}`}
; + default: function UserInfoViewMock({ + userId, + startInEditMode, + onClose, + }: { + userId: string; + startInEditMode?: boolean; + onClose: () => void; + }) { + return ( +
+ {`detail:${userId}:${String(Boolean(startInEditMode))}`} + +
+ ); }, })); @@ -85,11 +99,15 @@ const defaultProps = { teams: [], }; -const renderDashboard = (overrides: Partial = {}) => +const renderDashboard = ( + overrides: Partial = {}, + options?: { searchParams?: string; onUrlUpdate?: OnUrlUpdateFunction }, +) => renderWithProviders( , + options, ); describe("ViewUserDashboard", () => { @@ -196,7 +214,11 @@ describe("ViewUserDashboard", () => { it("should swap to the detail view when the identity cell is clicked", async () => { const user = userEvent.setup(); - renderDashboard(); + const onUrlUpdate = vi.fn(); + renderDashboard( + {}, + { searchParams: "?detail_tab=activity&activity_from=2026-01-01&activity_to=2026-01-08", onUrlUpdate }, + ); await waitFor(() => { expect(screen.getByText("test@example.com")).toBeInTheDocument(); @@ -206,11 +228,17 @@ describe("ViewUserDashboard", () => { expect(await screen.findByTestId("user-info-view")).toHaveTextContent("detail:user-1:false"); expect(screen.queryByText("test@example.com")).not.toBeInTheDocument(); + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + expect(Object.fromEntries(onUrlUpdate.mock.calls.at(-1)![0].searchParams)).toEqual({ user: "user-1" }); }); it("should open the detail view in edit mode from the row actions menu", async () => { const user = userEvent.setup(); - renderDashboard(); + const onUrlUpdate = vi.fn(); + renderDashboard( + {}, + { searchParams: "?detail_tab=activity&activity_from=2026-01-01&activity_to=2026-01-08", onUrlUpdate }, + ); await waitFor(() => { expect(screen.getByText("test@example.com")).toBeInTheDocument(); @@ -220,6 +248,22 @@ describe("ViewUserDashboard", () => { await user.click(await screen.findByTestId("user-action-edit")); expect(await screen.findByTestId("user-info-view")).toHaveTextContent("detail:user-1:true"); + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + expect(Object.fromEntries(onUrlUpdate.mock.calls.at(-1)![0].searchParams)).toEqual({ user: "user-1" }); + }); + + it("closing a user clears its tab and dates while preserving list filters", async () => { + const onUrlUpdate = vi.fn(); + renderDashboard( + {}, + { + searchParams: "?user=user-1&detail_tab=activity&activity_from=2026-01-01&activity_to=2026-01-08&tab=users", + onUrlUpdate, + }, + ); + fireEvent.click(await screen.findByRole("button", { name: "Close user" })); + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + expect(Object.fromEntries(onUrlUpdate.mock.calls.at(-1)![0].searchParams)).toEqual({ tab: "users" }); }); describe("bulk edit selection", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx index 610933e7fa0..1e40bbaa594 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx @@ -1,4 +1,5 @@ import { parseAsString, useQueryState } from "nuqs"; +import { useResetDetailState } from "@/hooks/useUrlTab"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; @@ -73,6 +74,7 @@ const ViewUserDashboard: React.FC = ({ const [selectionMode, setSelectionMode] = useState(false); const [isBulkEditModalVisible, setIsBulkEditModalVisible] = useState(false); + const resetDetailState = useResetDetailState(); const [selectedUserId, setSelectedUserId] = useQueryState("user", parseAsString.withOptions({ history: "push" })); const [openInEditMode, setOpenInEditMode] = useState(false); @@ -140,16 +142,18 @@ const ViewUserDashboard: React.FC = ({ const handleUserClick = useCallback( (userId: string, openInEdit: boolean = false) => { + resetDetailState(); void setSelectedUserId(userId); setOpenInEditMode(openInEdit); }, - [setSelectedUserId], + [setSelectedUserId, resetDetailState], ); const handleCloseUserInfo = useCallback(() => { + resetDetailState(); void setSelectedUserId(null); setOpenInEditMode(false); - }, [setSelectedUserId]); + }, [setSelectedUserId, resetDetailState]); const handleDelete = useCallback((user: UserInfo) => { setUserToDelete(user); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx index e78e4abe42c..5e2ca319091 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx @@ -418,7 +418,11 @@ describe("UserInfoView auto-router usage", () => { Promise.resolve({ ...MOCK_USER_DATA_NO_TEAMS, user_id: userId }), ); mockFetch.mockReset().mockResolvedValue(Response.json(routerUsageResponse(42))); - vi.stubGlobal("fetch", mockFetch); + vi.stubGlobal("fetch", (input: RequestInfo | URL, init?: RequestInit) => { + const url = input instanceof Request ? input.url : String(input); + if (url.includes("/telemetry/")) return Promise.resolve(Response.json({ enabled: false })); + return mockFetch(input, init); + }); }); afterEach(() => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.test.tsx index 3a29578c5c2..44d964bf13a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.test.tsx @@ -1,4 +1,5 @@ -import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import { renderWithProviders as render } from "../../../../../../tests/test-utils"; +import { fireEvent, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi, beforeEach } from "vitest"; import UserInfoView from "./user_info_view"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx index ca2c4264213..78c1875bb1d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx @@ -1,3 +1,4 @@ +import { useUrlTab } from "@/hooks/useUrlTab"; import ResourceActivity from "@/components/UsagePage/ResourceActivity"; import { MemberRoleSelect } from "./MemberRoleSelect"; import React, { useState } from "react"; @@ -99,7 +100,11 @@ export default function UserInfoView({ const [isInvitationLinkModalVisible, setIsInvitationLinkModalVisible] = useState(false); const [invitationLinkData, setInvitationLinkData] = useState(null); const [baseUrl, setBaseUrl] = useState(null); - const [activeTab, setActiveTab] = useState(initialTab === 1 ? "details" : "overview"); + const [activeTab, setActiveTab] = useUrlTab( + ["overview", "details", "activity", "savings", "auto-router-usage"], + initialTab === 1 ? "details" : "overview", + "detail_tab", + ); const hiddenUsageTab = (activeTab === "savings" || activeTab === "activity") && !canViewUserUsage; const hiddenRouterTab = activeTab === "auto-router-usage" && !canViewAutoRouterUsage; const [copiedStates, setCopiedStates] = useState>({}); @@ -484,7 +489,7 @@ export default function UserInfoView({ setActiveTab(String(v))} + onValueChange={(v: unknown) => setActiveTab(String(v) as typeof activeTab)} className="gap-0" > diff --git a/ui/litellm-dashboard/src/components/Alerts/AlertActions.tsx b/ui/litellm-dashboard/src/components/Alerts/AlertActions.tsx new file mode 100644 index 00000000000..562cc733a7b --- /dev/null +++ b/ui/litellm-dashboard/src/components/Alerts/AlertActions.tsx @@ -0,0 +1,89 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { Check, RotateCcw, X } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { backgroundFetchClient } from "@/lib/http/api"; +import { AlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; +import { AlertItem, presentAlert } from "./alertPresentation"; + +export default function AlertActions({ item, readOnly }: { item: AlertItem; readOnly: boolean }) { + const { t } = useTranslation(); + const client = useQueryClient(); + const mutation = useMutation({ + mutationFn: async (status: AlertStatus) => { + if (item.source === "performance") { + const result = await backgroundFetchClient.PATCH("/alerts/{alert_id}", { + params: { path: { alert_id: item.alert.id } }, + body: { status }, + }); + if (!result.data) throw new Error("Could not save alert status"); + return; + } + const alert = item.alert; + const result = await backgroundFetchClient.PUT("/alerting/model-deprecations/state", { + body: { + model: alert.usageModel, + deprecation_date: alert.model.deprecation_date, + status, + last_seen_date: alert.lastUsedDate, + last_seen_requests: alert.lastDayRequests, + }, + }); + if (!result.data) throw new Error("Could not save alert status"); + }, + onSuccess: () => + Promise.all( + (item.source === "performance" ? ["alerts", "alert"] : ["deprecationAlerts"]).map((key) => + client.invalidateQueries({ queryKey: [key] }), + ), + ), + }); + const status = presentAlert(item).status; + if (readOnly) return {t(`alerts.status.${status}`)}; + return ( +
+
+ {status === "open" ? ( + <> + + + + ) : ( + <> + {t(`alerts.status.${status}`)} + + + )} +
+ {mutation.isError && ( +

+ {t("alerts.unified.saveError")} +

+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/Alerts/AlertChart.tsx b/ui/litellm-dashboard/src/components/Alerts/AlertChart.tsx new file mode 100644 index 00000000000..602f0231bf7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Alerts/AlertChart.tsx @@ -0,0 +1,123 @@ +import { Bar, BarChart, CartesianGrid, Cell, ReferenceLine, XAxis, YAxis } from "recharts"; +import { useTranslation } from "react-i18next"; +import { ChartContainer, ChartTooltip, ChartTooltipContent } from "@/components/ui/chart"; +import { AlertPresentation, dateLabel, metricValue } from "./alertPresentation"; + +export default function AlertChart({ alert, compact = false }: { alert: AlertPresentation; compact?: boolean }) { + const { t } = useTranslation(); + if (!alert.history?.length) + return compact ? null : ( +

{t("alerts.unified.chart.unavailable")}

+ ); + const end = Date.parse(alert.observed); + const start = + alert.baseline === undefined + ? Math.min(...alert.history.map((point) => Date.parse(point.date))) + : end - 7 * 86400000; + const values = new Map(alert.history.map((point) => [point.date, point.value])); + const data = Array.from({ length: Math.min(31, Math.round((end - start) / 86400000) + 1) }, (_, index) => { + const date = new Date(start + index * 86400000).toISOString().slice(0, 10); + return { date, value: values.get(date) ?? null }; + }); + return ( +
+ {!compact && ( +
+

{alert.chartTitle}

+ + {dateLabel(data[0].date)} – {dateLabel(alert.observed)} UTC + +
+ )} + + + {!compact && } + + new Date(`${date}T00:00:00Z`).toLocaleDateString(undefined, { + month: "short", + day: "numeric", + timeZone: "UTC", + }) + } + /> + metricValue(value, alert.unit)} + /> + dateLabel(String(value))} + formatter={(value) => metricValue(Number(value), alert.unit)} + /> + } + /> + {!compact && alert.baseline !== undefined && ( + + )} + + {data.map((point) => ( + + ))} + + + + {!compact && ( + <> +
+ {alert.baseline !== undefined && + t("alerts.unified.chart.baseline", { value: metricValue(alert.baseline, alert.unit) })} + {t("alerts.unified.chart.missingDays")} + {alert.baseline !== undefined && t("alerts.unified.chart.currentUsage")} +
+ + + + + + + + + {data.map((point) => ( + + + + + ))} + +
{t("alerts.unified.chart.date")}{t("alerts.unified.chart.value")}
{point.date} + {point.value === null + ? t("alerts.migrationDetail.notRecorded") + : metricValue(point.value, alert.unit)} +
+ + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx b/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx new file mode 100644 index 00000000000..733641fdb17 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx @@ -0,0 +1,83 @@ +import { ArrowLeft, KeyRound, Network, Server, User, Users } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import { buttonVariants } from "@/components/ui/button"; +import RecommendedReplacement from "@/components/ModelDeprecationAlerts/RecommendedReplacement"; +import { migrationHref } from "@/components/ModelDeprecationAlerts/migrationRoute"; +import AlertActions from "./AlertActions"; +import AlertChart from "./AlertChart"; +import { AlertItem, dateLabel, presentAlert } from "./alertPresentation"; + +export default function AlertDetail({ + item, + readOnly, + onBack, +}: { + item: AlertItem; + readOnly: boolean; + onBack: () => void; +}) { + const { t } = useTranslation(); + const alert = presentAlert(item); + const Icon = { key: KeyRound, team: Users, user: User, model: Network, gateway: Server }[alert.resourceType]; + return ( + + ); +} diff --git a/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx b/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx new file mode 100644 index 00000000000..2b7c7c0d93b --- /dev/null +++ b/ui/litellm-dashboard/src/components/Alerts/Alerts.integration.test.tsx @@ -0,0 +1,152 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders, screen, testQueryClient, within } from "../../../tests/test-utils"; +import Alerts from "./Alerts"; +import type { SavedAlert } from "./alertPresentation"; + +vi.unmock("@/app/(dashboard)/hooks/useAuthorized"); +let role = "proxy_admin"; +const alert: SavedAlert = { + id: "9007199254740993", + type: "spend", + resource_type: "key", + resource_id: "a".repeat(64), + resource_name: "Research agent", + status: "open", + window_start: "2026-01-08T00:00:00Z", + window_end: "2026-01-09T00:00:00Z", + created_at: "2026-01-09T03:00:00Z", + updated_at: "2026-01-09T03:00:00Z", + evidence: { value: 20, baseline: 2, unit: "usd", sample_size: 100 }, + history: [ + { date: "2026-01-01", value: 2 }, + { date: "2026-01-08", value: 21 }, + ], +}; +const calls: { path: string; query: URLSearchParams; method: string; body: unknown }[] = []; +let status = "open"; +let rejectWrite = false; +let rejectHistory = false; +function response(data: unknown, code = 200) { + return new Response(JSON.stringify(data), { status: code, headers: { "Content-Type": "application/json" } }); +} + +describe("Alerts", () => { + beforeEach(() => { + testQueryClient.clear(); + calls.length = 0; + status = "open"; + rejectWrite = false; + rejectHistory = false; + role = "proxy_admin"; + document.cookie = `token=${btoa(JSON.stringify({ alg: "none" }))}.${btoa(JSON.stringify({ key: "example-session", exp: 4102444800 }))}.example; path=/`; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + const body = request.method === "PATCH" ? await request.clone().json() : null; + const call = { path: url.pathname, query: url.searchParams, method: request.method, body }; + calls.push(call); + if (url.pathname === "/litellm/.well-known/litellm-ui-config") return response({}); + if (url.pathname === "/management/v1/users/me") return response({ user_id: "example-admin", user_role: role }); + if (url.pathname === "/model/deprecations") return response({ deprecated: [], imminent: [] }); + if (request.method === "PATCH") { + if (rejectWrite) return response({ detail: "Unavailable" }, 503); + status = body.status; + return response({ ...alert, status }); + } + if (url.pathname === `/alerts/${alert.id}`) + return response({ ...alert, status, history: rejectHistory ? null : alert.history }); + if (url.pathname === "/alerts") { + if (url.searchParams.has("cursor")) + return response({ data: [{ ...alert, id: "7", resource_name: "Second key" }], next_cursor: null }); + return response({ + data: url.searchParams.get("status") === status ? [{ ...alert, status }] : [], + next_cursor: "8", + }); + } + throw new Error(`Unexpected endpoint ${url.pathname}`); + }); + }); + afterEach(() => { + vi.restoreAllMocks(); + document.cookie = "token=; Max-Age=0; path=/"; + }); + + it("shows saved evidence, daily gaps, and a scoped Activity link", async () => { + renderWithProviders(, { searchParams: { alert: alert.id } }); + expect( + await screen.findByRole("heading", { name: "Spend on Research agent rose to 10× its baseline" }), + ).toBeInTheDocument(); + expect(screen.getByText(/\$20.00 on/)).toBeInTheDocument(); + const chart = screen.getByRole("table", { name: "Daily spend values" }); + expect(within(chart).getByRole("row", { name: "2026-01-08 $21.00" })).toBeInTheDocument(); + expect(within(chart).getByRole("row", { name: "2026-01-02 Not recorded" })).toBeInTheDocument(); + expect(screen.getByRole("link", { name: "View key activity" })).toHaveAttribute( + "href", + `/ui/api-keys?key=${alert.resource_id}&detail_tab=activity&activity_from=2026-01-01&activity_to=2026-01-08`, + ); + }); + + it("keeps saved evidence visible when daily history cannot be loaded", async () => { + rejectHistory = true; + renderWithProviders(, { searchParams: { alert: alert.id } }); + expect(await screen.findByText(/Daily history is unavailable/)).toBeInTheDocument(); + expect(screen.getByText(/\$20.00 on/)).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Mark resolved" })).toBeEnabled(); + }); + + it("saves status and reloads the detail, retaining exact large IDs", async () => { + const user = userEvent.setup(); + renderWithProviders(, { searchParams: { alert: alert.id } }); + await user.click(await screen.findByRole("button", { name: "Mark resolved" })); + expect(await screen.findByRole("button", { name: "Reopen" })).toBeInTheDocument(); + expect(calls.find((call) => call.method === "PATCH")).toMatchObject({ + path: `/alerts/${alert.id}`, + body: { status: "resolved" }, + }); + await user.click(screen.getByRole("button", { name: "Reopen" })); + expect(await screen.findByRole("button", { name: "Dismiss" })).toBeInTheDocument(); + }); + + it("retains the open alert when a status write fails", async () => { + rejectWrite = true; + const user = userEvent.setup(); + renderWithProviders(, { searchParams: { alert: alert.id } }); + await user.click(await screen.findByRole("button", { name: "Dismiss" })); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not save status"); + expect(screen.getByRole("button", { name: "Mark resolved" })).toBeEnabled(); + }); + + it("uses the selected type and cursor when loading more results", async () => { + const user = userEvent.setup(); + renderWithProviders(, { searchParams: { type: "spend" } }); + await user.click(await screen.findByRole("button", { name: "Load more" })); + expect(await screen.findByRole("button", { name: /Spend on Second key/ })).toBeInTheDocument(); + expect( + calls.filter((call) => call.path === "/alerts").map((call) => [call.query.get("type"), call.query.get("cursor")]), + ).toEqual([ + ["spend", null], + ["spend", "8"], + ]); + await user.click(screen.getByRole("tab", { name: "resolved" })); + expect(await screen.findByText("No resolved alerts.")).toBeInTheDocument(); + }); + + it("keeps deprecation email setup accessible from the inbox", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(await screen.findByRole("button", { name: "Set up email reminders" })); + expect(screen.getByRole("link", { name: "Open Email Settings" })).toHaveAttribute( + "href", + "/ui/logging-and-alerts?tab=email-alerts", + ); + }); + + it("lets viewers read the evidence without write controls", async () => { + role = "proxy_admin_viewer"; + renderWithProviders(, { searchParams: { alert: alert.id } }); + expect(await screen.findByRole("heading", { name: /Spend on Research agent/ })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Dismiss" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Mark resolved" })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx b/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx new file mode 100644 index 00000000000..2f9d502741b --- /dev/null +++ b/ui/litellm-dashboard/src/components/Alerts/Alerts.tsx @@ -0,0 +1,230 @@ +"use client"; + +import { parseAsBoolean, parseAsString, parseAsStringEnum, parseAsStringLiteral, useQueryStates } from "nuqs"; +import { RefreshCw } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { Page, PageTabs, PageTabsList, PageTabsTrigger } from "@/components/shared/Page"; +import { Button } from "@/components/ui/button"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { isProxyAdminTierRole } from "@/utils/roles"; +import { useDeprecationAlerts } from "@/components/ModelDeprecationAlerts/useDeprecationAlerts"; +import ModelDeprecationsPanel from "@/components/ModelDeprecationAlerts/ModelDeprecationsPanel"; +import DeprecationEmailSetup from "@/components/ModelDeprecationAlerts/DeprecationEmailSetup"; +import { AlertStatus, effectiveAlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; +import { useAlert, useAlerts } from "./useAlerts"; +import { ALERT_TYPES, AlertFilter, AlertItem, TYPE_LABEL_KEYS, presentAlert } from "./alertPresentation"; +import AlertActions from "./AlertActions"; +import AlertChart from "./AlertChart"; +import AlertDetail from "./AlertDetail"; + +const parsers = { + alert: parseAsString, + model: parseAsString, + migration: parseAsBoolean.withDefault(false), + state: parseAsStringEnum(Object.values(AlertStatus)).withDefault(AlertStatus.Open), + type: parseAsStringLiteral(ALERT_TYPES).withDefault("all"), +}; +function LoadError({ children, retry }: { children: React.ReactNode; retry: () => void }) { + const { t } = useTranslation(); + return ( +
+

{children}

+ +
+ ); +} +export default function Alerts() { + const { t } = useTranslation(); + const auth = useAuthorized(); + const [params, setParams] = useQueryStates(parsers); + const allowed = Boolean(auth.isAuthorized && isProxyAdminTierRole(auth.userRole)); + if (auth.isLoading) + return ( + +

{t("alerts.unified.loading")}

+ + ); + if (!allowed) + return ( + +

{t("alerts.unified.adminRequired")}

+ + ); + return ( + + {params.alert || params.model ? ( + void setParams({ alert: null, model: null, migration: null }, { history: "push" })} + /> + ) : ( + + )} + + ); +} + +function SelectedAlert({ auth, onBack }: { auth: ReturnType; onBack: () => void }) { + const { t } = useTranslation(); + const [params] = useQueryStates(parsers); + const scope = JSON.stringify([auth.userId, auth.userRole]); + const detail = useAlert(params.alert, scope, true); + const deprecations = useDeprecationAlerts(params.model ? auth.accessToken : null); + const selectedDeprecation = deprecations.data?.find((alert) => alert.usageModel === params.model); + const selected: AlertItem | undefined = params.alert + ? detail.data && { source: "performance", alert: detail.data } + : selectedDeprecation && { source: "deprecation", alert: selectedDeprecation }; + if (params.migration && params.model) return ; + const query = params.alert ? detail : deprecations; + if (selected && !query.isError) + return ; + return ( + <> + + {query.isError ? ( + void query.refetch()}>{t("alerts.unified.loadError")} + ) : ( +

{query.isPending ? t("alerts.panel.loadingAlert") : t("alerts.unified.unavailable")}

+ )} + + ); +} + +function AlertsInbox({ auth }: { auth: ReturnType }) { + const { t } = useTranslation(); + const [params, setParams] = useQueryStates(parsers); + const scope = JSON.stringify([auth.userId, auth.userRole]); + const showDeprecations = params.type === "all" || params.type === "model-deprecations"; + const showPerformance = params.type !== "model-deprecations"; + const list = useAlerts(scope, showPerformance, params.state, params.type); + const deprecations = useDeprecationAlerts(showDeprecations ? auth.accessToken : null); + const retirementItems: AlertItem[] = (deprecations.data ?? []) + .filter((alert) => effectiveAlertStatus(alert) === params.state) + .map((alert) => ({ source: "deprecation", alert })); + const performanceItems: AlertItem[] = (list.data?.pages.flatMap((page) => page.data) ?? []).map((alert) => ({ + source: "performance", + alert, + })); + const items = [...retirementItems, ...performanceItems]; + const loading = list.isLoading || deprecations.isLoading; + const hasError = (showPerformance && list.isError) || (showDeprecations && deprecations.isError); + return ( + <> +

{t("alerts.unified.title")}

+ void setParams({ state: value as AlertStatus })} + className="flex-none" + > + + {Object.values(AlertStatus).map((status) => ( + + {t(`alerts.status.${status}`)} + + ))} + + +
+ + +
+ {showDeprecations && deprecations.isError && ( + void deprecations.refetch()}>{t("alerts.unified.deprecationsError")} + )} + {showPerformance && list.isError && ( + void list.refetch()}>{t("alerts.unified.performanceError")} + )} + +
    + {items.map((item) => { + const alert = presentAlert(item); + return ( +
  • +
    + +

    {alert.description}

    + +
    +
    + +
    +
    • + ); + })} +
+ {showPerformance && list.hasNextPage && ( + + )} + {showDeprecations && } + + ); +} + +function InboxNotice({ + loading, + error, + count, + status, +}: { + loading: boolean; + error: boolean; + count: number; + status: AlertStatus; +}) { + const { t } = useTranslation(); + if (count > 0 || error) return null; + return ( +

+ {loading ? t("alerts.unified.loading") : t(`alerts.unified.empty.${status}`)} +

+ ); +} diff --git a/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts b/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts new file mode 100644 index 00000000000..aa92ca8fc20 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { activityLink, presentAlert, type SavedAlert } from "./alertPresentation"; + +const saved: SavedAlert = { + id: "1", + type: "spend", + resource_type: "team", + resource_id: "example-team", + status: "open", + window_start: "2026-01-08T00:00:00Z", + window_end: "2026-01-09T00:00:00Z", + created_at: "2026-01-09T03:00:00Z", + updated_at: "2026-01-09T03:00:00Z", + evidence: { value: 20, baseline: 2, unit: "usd", sample_size: 100 }, +}; + +describe("alert presentation", () => { + it.each( + ( + [ + ["spend", "usd", 20, 2, "Spend", "$20.00"], + ["traffic", "requests", 1000, 100, "Traffic", "1,000"], + ["latency", "milliseconds", 4000, 1000, "Mean response time", "4 s"], + ["reliability", "ratio", 0.2, 0.01, "Error rate", "20%"], + ] as const + ).map((row) => [row] as const), + )("labels %s with the recorded units", ([type, unit, value, baseline, label, formatted]) => { + const alert = { ...saved, type, evidence: { value, baseline, unit, sample_size: 100 } }; + const presentation = presentAlert({ source: "performance", alert }); + expect(presentation.title).toContain(`${label} on example-team`); + expect(presentation.description).toContain(formatted); + expect(presentation.baseline).toBe(baseline); + }); + it("links to the resource with an inclusive eight-day window", () => { + expect(activityLink(saved)).toBe( + "/ui/teams?team=example-team&detail_tab=activity&activity_from=2026-01-01&activity_to=2026-01-08", + ); + expect(activityLink({ ...saved, resource_type: "user", resource_id: "reader@example.com" })).toContain( + "user=reader%40example.com&detail_tab=activity", + ); + expect(activityLink({ ...saved, resource_type: "gateway", resource_id: "gateway" })).toBe( + "/ui/usage?from=2026-01-01&to=2026-01-08", + ); + }); + it("links model groups to routing instead of an unrelated deployment's Activity", () => { + const presentation = presentAlert({ + source: "performance", + alert: { ...saved, resource_type: "model", resource_id: "research/model" }, + }); + expect(presentation.href).toBe("/ui/models-and-endpoints?model_group=research%2Fmodel"); + expect(presentation.linkLabel).toBe("View model routing"); + }); + it("keeps the saved comparison when current Usage has changed", () => { + const presentation = presentAlert({ + source: "performance", + alert: { ...saved, resource_name: "Research", history: [{ date: "2026-01-08", value: 40 }] }, + }); + expect(presentation.description).toContain("$20.00"); + expect(presentation.history?.[0].value).toBe(40); + expect(presentation.title).toContain("Research"); + }); + it("shows retirement and last-used dates without inventing a detection date", () => { + const presentation = presentAlert({ + source: "deprecation", + alert: { + id: "retirement", + usageModel: "example/retiring-model", + requests: 20, + lastDayRequests: 20, + lastUsedDate: "2026-01-08", + model: { + model_name: "example-group", + deprecation_date: "2026-01-31", + days_until_deprecation: 23, + status: "imminent", + }, + }, + }); + expect(presentation.dates).toEqual([ + ["Retires", "2026-01-31"], + ["Last used", "2026-01-08"], + ]); + expect(presentation.status).toBe("open"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts b/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts new file mode 100644 index 00000000000..79a5d84a2a4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts @@ -0,0 +1,166 @@ +import type { components } from "@/lib/http/schema"; +import i18n from "@/i18n/i18n"; +import type { DeprecationAlert } from "@/components/ModelDeprecationAlerts/deprecationAlerts"; +import { effectiveAlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; +import { keyDetailHref, teamDetailHref, userDetailHref, modelGroupHref } from "@/utils/entityLinks"; +import { uiHref } from "@/utils/uiHref"; + +export type SavedAlert = components["schemas"]["AlertView"]; +export type AlertItem = + | { source: "performance"; alert: SavedAlert } + | { source: "deprecation"; alert: DeprecationAlert }; +export type MetricUnit = "usd" | "requests" | "milliseconds" | "ratio"; +export type HistoryPoint = { date: string; value: number | null }; +export const ALERT_TYPES = [ + "all", + "performance", + "model-deprecations", + "spend", + "traffic", + "latency", + "reliability", +] as const; +export type AlertFilter = (typeof ALERT_TYPES)[number]; +export const TYPE_LABEL_KEYS = { + all: "alerts.unified.types.all", + performance: "alerts.unified.types.performance", + "model-deprecations": "alerts.page.tabs.modelDeprecations", + spend: "alerts.unified.types.spend", + traffic: "alerts.unified.types.traffic", + latency: "alerts.unified.types.latency", + reliability: "alerts.unified.types.reliability", +} as const; +const dateFormat = { year: "numeric", month: "short", day: "numeric", timeZone: "UTC" } as const; +export const dateLabel = (value: string) => + new Date(value.length === 10 ? `${value}T00:00:00Z` : value).toLocaleDateString(undefined, dateFormat); +export function metricValue(value: number, unit: MetricUnit): string { + if (unit === "usd") return new Intl.NumberFormat(undefined, { style: "currency", currency: "USD" }).format(value); + if (unit === "ratio") return `${(value * 100).toLocaleString(undefined, { maximumFractionDigits: 2 })}%`; + if (unit === "milliseconds") return `${(value / 1000).toLocaleString(undefined, { maximumFractionDigits: 2 })} s`; + return value.toLocaleString(undefined, { maximumFractionDigits: 0 }); +} + +export function activityLink(alert: SavedAlert): string | undefined { + const href = (() => { + switch (alert.resource_type) { + case "key": + return keyDetailHref(alert.resource_id); + case "team": + return teamDetailHref(alert.resource_id); + case "user": + return userDetailHref(alert.resource_id); + case "model": + return modelGroupHref(alert.resource_id); + case "gateway": + return uiHref("usage"); + } + })(); + if (!href || alert.resource_type === "model") return href; + const from = new Date(Date.parse(alert.window_start) - 7 * 86400000).toISOString().slice(0, 10); + const to = alert.window_start.slice(0, 10); + const query = + alert.resource_type === "gateway" + ? `from=${from}&to=${to}` + : `detail_tab=activity&activity_from=${from}&activity_to=${to}`; + return `${href}${href.includes("?") ? "&" : "?"}${query}`; +} + +export function presentAlert(item: AlertItem) { + if (item.source === "deprecation") { + const alert = item.alert; + return { + id: alert.id, + title: i18n.t( + alert.model.status === "deprecated" ? "alerts.unified.retiredTitle" : "alerts.unified.retiresTitle", + { + name: alert.usageModel, + date: dateLabel(alert.model.deprecation_date), + }, + ), + description: i18n.t("alerts.unified.deprecationDescription", { + requests: alert.requests.toLocaleString(), + date: dateLabel(alert.lastUsedDate), + }), + status: effectiveAlertStatus(alert), + resourceType: "model" as const, + resourceName: alert.usageModel, + href: modelGroupHref(alert.model.model_name), + linkLabel: i18n.t("alerts.unified.links.model"), + unit: "requests" as MetricUnit, + history: alert.history ?? null, + baseline: undefined, + observed: alert.lastUsedDate, + dates: [ + [ + i18n.t(alert.model.status === "deprecated" ? "alerts.unified.retired" : "alerts.unified.retires"), + alert.model.deprecation_date, + ], + [i18n.t("alerts.unified.lastUsed"), alert.lastUsedDate], + ], + chartTitle: i18n.t("alerts.usageChart.title"), + }; + } + const alert = item.alert; + const name = + alert.resource_name || + (alert.resource_type === "key" + ? i18n.t("alerts.unified.keyFallback", { id: alert.resource_id.slice(0, 8) }) + : alert.resource_id); + if (!("value" in alert.evidence)) { + return { + id: alert.id, + title: i18n.t("alerts.unified.retiresTitle", { name, date: dateLabel(alert.evidence.deprecation_date) }), + description: i18n.t("alerts.unified.deprecation"), + status: alert.status, + resourceType: alert.resource_type, + resourceName: name, + href: modelGroupHref(alert.resource_id), + linkLabel: i18n.t("alerts.unified.links.model"), + unit: "requests" as MetricUnit, + history: null, + baseline: undefined, + observed: alert.window_start.slice(0, 10), + dates: [[i18n.t("alerts.unified.retires"), alert.evidence.deprecation_date]], + chartTitle: i18n.t("alerts.usageChart.title"), + }; + } + const evidence = alert.evidence; + const observed = alert.window_start.slice(0, 10); + const label = i18n.t(`alerts.unified.metrics.${alert.type}`); + const direction = evidence.value < evidence.baseline ? "fell" : "rose"; + return { + id: alert.id, + title: + evidence.baseline > 0 + ? i18n.t(`alerts.unified.comparison.${direction}`, { + label, + name, + ratio: (evidence.value / evidence.baseline).toLocaleString(undefined, { maximumFractionDigits: 2 }), + }) + : i18n.t(`alerts.unified.value.${direction}`, { + label, + name, + value: metricValue(evidence.value, evidence.unit), + }), + description: i18n.t("alerts.unified.description", { + value: metricValue(evidence.value, evidence.unit), + date: dateLabel(observed), + baseline: metricValue(evidence.baseline, evidence.unit), + }), + status: alert.status, + resourceType: alert.resource_type, + resourceName: name, + href: activityLink(alert), + linkLabel: i18n.t(`alerts.unified.links.${alert.resource_type}`), + unit: evidence.unit, + history: alert.history, + baseline: evidence.baseline, + observed, + dates: [ + [i18n.t("alerts.unified.observed"), observed], + [i18n.t("alerts.unified.firstDetected"), alert.created_at], + ], + chartTitle: i18n.t(`alerts.unified.chart.titles.${evidence.unit}`), + }; +} +export type AlertPresentation = ReturnType; diff --git a/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts b/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts new file mode 100644 index 00000000000..70813ef2e93 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts @@ -0,0 +1,44 @@ +import type { components } from "@/lib/http/schema"; +import { useInfiniteQuery, useQuery } from "@tanstack/react-query"; +import { backgroundFetchClient } from "@/lib/http/api"; +import { getProxyBaseUrl } from "@/components/networking"; +import { AlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; +import type { AlertFilter } from "./alertPresentation"; + +export function useAlerts(scope: string, enabled: boolean, status: AlertStatus, filter: AlertFilter) { + const options = { + queryKey: ["alerts", getProxyBaseUrl(), scope, status, filter], + enabled: enabled && filter !== "model-deprecations", + initialPageParam: undefined as string | undefined, + queryFn: async ({ pageParam, signal }: { pageParam: string | undefined; signal: AbortSignal }) => { + const type = filter === "all" || filter === "model-deprecations" ? "performance" : filter; + const { data } = await backgroundFetchClient.GET("/alerts", { + params: { query: { status, type, limit: 25, cursor: pageParam } }, + signal, + }); + if (!data) throw new Error("Could not load alerts"); + return data; + }, + getNextPageParam: (page: components["schemas"]["AlertViewPage"]) => page.next_cursor ?? undefined, + staleTime: 60_000, + retry: false, + }; + return useInfiniteQuery(options); +} +export function useAlert(id: string | null, scope: string, enabled: boolean) { + const options = { + queryKey: ["alert", getProxyBaseUrl(), scope, id], + enabled: enabled && id !== null, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const { data } = await backgroundFetchClient.GET("/alerts/{alert_id}", { + params: { path: { alert_id: id! } }, + signal, + }); + if (!data) throw new Error("Could not load alert"); + return data; + }, + staleTime: 60_000, + retry: false, + }; + return useQuery(options); +} diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx index 993277dc972..61fcfc308ad 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.integration.test.tsx @@ -12,12 +12,48 @@ vi.mock("next/navigation", () => ({ useSearchParams: () => new URLSearchParams(), })); +let role = "proxy_admin"; +const alertRequests: string[] = []; +const performanceAlert = { + id: "9007199254740993", + type: "spend", + resource_type: "team", + resource_id: "example-team", + resource_name: "Research", + status: "open", + window_start: "2026-11-10T00:00:00Z", + window_end: "2026-11-11T00:00:00Z", + created_at: "2026-11-11T03:00:00Z", + updated_at: "2026-11-11T03:00:00Z", + evidence: { value: 1200, baseline: 200, unit: "usd", sample_size: 1000 }, +}; +let openAlerts = [performanceAlert]; +let rejectAlerts = false; +let rejectDeprecations = false; + beforeEach(() => { + role = "proxy_admin"; + alertRequests.length = 0; + openAlerts = [performanceAlert]; + rejectAlerts = false; + rejectDeprecations = false; localStorage.clear(); vi.stubGlobal( "fetch", vi.fn(async (input: RequestInfo | URL) => { - const url = String(input); + const url = input instanceof Request ? input.url : String(input); + if (url.endsWith("/management/v1/users/me")) { + return Response.json({ user_id: "example-admin", user_role: role }); + } + if (url.endsWith("/model/deprecations?warn_within_days=30")) { + if (rejectDeprecations) return Response.json({ detail: "Unavailable" }, { status: 503 }); + return Response.json({ deprecated: [], imminent: [] }); + } + if (url.includes("/alerts?")) { + alertRequests.push(url); + if (rejectAlerts) return Response.json({ detail: "Unavailable" }, { status: 503 }); + return Response.json({ data: openAlerts, next_cursor: "next-page" }); + } if (url.endsWith("/get/ui_settings")) return Response.json({ values: {} }); if (url.endsWith("/litellm/.well-known/litellm-ui-config")) { const config = { @@ -56,19 +92,19 @@ afterEach(() => { localStorage.clear(); }); -function renderHeader() { +function renderHeader(accessToken: string | null = null) { const user = userEvent.setup(); const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); render( - + , ); - return { user }; + return { user, client }; } describe("dashboard header controls", () => { @@ -84,7 +120,7 @@ describe("dashboard header controls", () => { it("offers docs, community and latest posts under the bell", async () => { const { user } = renderHeader(); await user.click(screen.getByRole("button", { name: /Notifications/ })); - const tools = await screen.findByRole("dialog", { name: "What's new" }); + const tools = await screen.findByRole("dialog", { name: "Notifications" }); expect(await within(tools).findByRole("link", { name: /Gateway update/ })).toHaveAttribute( "href", "https://example.com/update", @@ -102,7 +138,7 @@ describe("dashboard header controls", () => { const { user } = renderHeader(); expect(screen.queryByRole("link", { name: "Join Slack" })).not.toBeInTheDocument(); await user.click(screen.getByRole("button", { name: /Notifications/ })); - const tools = await screen.findByRole("dialog", { name: "What's new" }); + const tools = await screen.findByRole("dialog", { name: "Notifications" }); expect(within(tools).getByRole("link", { name: "Docs", exact: true })).toBeVisible(); expect(within(tools).queryByRole("link", { name: "Blog", exact: true })).not.toBeInTheDocument(); expect(within(tools).queryByRole("link", { name: "Discord", exact: true })).not.toBeInTheDocument(); @@ -118,3 +154,66 @@ describe("dashboard header controls", () => { expect(localStorage.getItem("litellm_selected_worker_id")).toBeNull(); }); }); + +describe("header performance notifications", () => { + it.each(["proxy_admin", "proxy_admin_viewer"])( + "loads open alerts for %s and refreshes after resolution", + async (userRole) => { + role = userRole; + const { user, client } = renderHeader("example-session"); + await user.click(await screen.findByRole("button", { name: "Notifications, alerts need attention" })); + expect(screen.getByRole("link", { name: /Spend on Research/ })).toHaveAttribute( + "href", + `/ui/alerts?alert=${performanceAlert.id}`, + ); + expect(alertRequests).toHaveLength(1); + const query = new URL(alertRequests[0]).searchParams; + expect(query.get("status")).toBe("open"); + expect(query.get("type")).toBe("performance"); + openAlerts = []; + await client.invalidateQueries({ queryKey: ["alerts"] }); + expect(await screen.findByText("No open alerts.")).toBeVisible(); + expect(screen.queryByRole("link", { name: /Spend on Research/ })).not.toBeInTheDocument(); + client.clear(); + }, + ); + + it("hides cached performance alerts when refreshing fails", async () => { + const { user, client } = renderHeader("example-session"); + await user.click(await screen.findByRole("button", { name: "Notifications, alerts need attention" })); + expect(screen.getByRole("link", { name: /Spend on Research/ })).toHaveTextContent("$1,200.00 vs. $200.00 usual"); + rejectAlerts = true; + await client.invalidateQueries({ queryKey: ["alerts"] }); + expect(await screen.findByText("Could not load performance alerts.")).toBeVisible(); + expect(screen.queryByRole("link", { name: /Spend on Research/ })).not.toBeInTheDocument(); + expect(screen.queryByText("No open alerts.")).not.toBeInTheDocument(); + client.clear(); + }); + + it.each([false, true])("reports deprecation failures with performance failure %s", async (performanceFails) => { + openAlerts = []; + rejectAlerts = performanceFails; + const { user, client } = renderHeader("example-session"); + await user.click(screen.getByRole("button", { name: /Notifications/ })); + if (performanceFails) { + expect(await screen.findByText("Could not load performance alerts.")).toBeVisible(); + } else { + expect(await screen.findByText("No open alerts.")).toBeVisible(); + } + rejectDeprecations = true; + await client.invalidateQueries({ queryKey: ["deprecationAlerts"] }); + expect(await screen.findByText("Could not load model deprecations.")).toBeVisible(); + expect(screen.queryByText("No open alerts.")).not.toBeInTheDocument(); + client.clear(); + }); + + it.each(["internal_user", "org_admin"])("does not request performance alerts for %s", async (userRole) => { + role = userRole; + const { user, client } = renderHeader("example-session"); + await user.click(screen.getByRole("button", { name: /Notifications/ })); + expect(await screen.findByRole("link", { name: /Gateway update/ })).toBeVisible(); + expect(alertRequests).toHaveLength(0); + expect(screen.queryByRole("link", { name: /Spend on Research/ })).not.toBeInTheDocument(); + client.clear(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx deleted file mode 100644 index bc7e8bc9ff3..00000000000 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx +++ /dev/null @@ -1,37 +0,0 @@ -"use client"; - -import { ArrowUpRight } from "lucide-react"; -import { useTranslation } from "react-i18next"; -import { DeprecationModel, DeprecationTiming } from "./DeprecationModel"; -import { buttonVariants } from "@/components/ui/button"; -import { cn } from "@/lib/cva.config"; -import { migrationHref } from "@/components/ModelDeprecationAlerts/migrationRoute"; -import RecommendedReplacement from "./RecommendedReplacement"; -import type { DeprecationAlert } from "./deprecationAlerts"; - -export default function DeprecationAlertItem({ alert }: { alert: DeprecationAlert }) { - const { t } = useTranslation(); - const retired = alert.model.status === "deprecated"; - const title = retired - ? t("alerts.alertItem.deprecatedTitle", { model: alert.usageModel }) - : t("alerts.alertItem.retiresTitle", { model: alert.usageModel, days: alert.model.days_until_deprecation }); - return ( - - ); -} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx index f8698556d49..4dd7fa21864 100644 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx @@ -1,7 +1,7 @@ import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; import userEvent from "@testing-library/user-event"; import { renderWithProviders, screen, testQueryClient, waitFor } from "../../../tests/test-utils"; -import ModelAlertsPage from "@/app/(dashboard)/alerts/page"; +import ModelDeprecationsPanel from "./ModelDeprecationsPanel"; vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ useTeams: () => ({ data: [{ team_id: "support", team_alias: "Support" }] }), @@ -69,19 +69,9 @@ describe("model migration impact", () => { }); }); afterEach(() => vi.restoreAllMocks()); - it("should render the inbox with last seen and the recommendation supplied by the API", () => { - renderWithProviders(); - expect(screen.getByRole("tab", { name: "Model deprecations", exact: true })).toBeInTheDocument(); - expect(screen.getByText("replacement-model")).toBeInTheDocument(); - expect(screen.getByText("2026-11-04")).toBeInTheDocument(); - expect(screen.getByRole("link", { name: "Review impact →" })).toHaveAttribute( - "href", - "/ui/alerts?type=model-deprecations&model=openai%2Fgpt-4-turbo", - ); - }); it("should page through all affected keys instead of showing a top-ten sample", async () => { const user = userEvent.setup(); - renderWithProviders(, { searchParams: { model: "openai/gpt-4-turbo" } }); + renderWithProviders(, { searchParams: { model: "openai/gpt-4-turbo" } }); expect(await screen.findByRole("link", { name: "support-key-0" })).toBeInTheDocument(); expect(screen.getByText(/Showing 1-50 of 120/)).toBeInTheDocument(); await user.click(screen.getByRole("button", { name: "Go to next page", exact: true })); @@ -97,7 +87,7 @@ describe("model migration impact", () => { }); it("should resolve the whole model using its global latest-day snapshot even in a team-filtered view", async () => { const user = userEvent.setup(); - renderWithProviders(, { + renderWithProviders(, { searchParams: { model: "openai/gpt-4-turbo", team: "support" }, }); expect(await screen.findByRole("link", { name: "support-key-0" })).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.test.ts b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.test.ts index f38eee7fccf..cf3d81fce9a 100644 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.test.ts +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.test.ts @@ -76,3 +76,14 @@ describe("deprecationUsageRange", () => { }); }); }); + +it("retains daily totals, explicit zeroes and gaps for the alert chart", () => { + const alerts = usedDeprecationAlerts( + [model], + [day("2026-11-04", 12), day("2026-11-04", 3), day("2026-11-02", 0), day("2026-11-01", 8, "example-other")], + ); + expect(alerts[0].history).toEqual([ + { date: "2026-11-02", value: 0 }, + { date: "2026-11-04", value: 15 }, + ]); +}); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.ts b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.ts index d22d940ab16..67d2809d02a 100644 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.ts +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.ts @@ -10,6 +10,7 @@ export interface DeprecationAlert { requests: number; lastUsedDate: string; lastDayRequests: number; + history?: { date: string; value: number }[]; state?: components["schemas"]["DeprecationAlertState"]; } @@ -52,7 +53,23 @@ export function usedDeprecationAlerts( } } if (requests > 0) { - const alert = { id, model, usageModel, requests, lastUsedDate, lastDayRequests }; + const history = Array.from(new Set(days.map((day) => day.date))) + .sort() + .flatMap((date) => { + const recorded = days.filter((day) => day.date === date && day.breakdown?.models?.[usageModel]); + return recorded.length + ? [ + { + date, + value: recorded.reduce( + (sum, day) => sum + (day.breakdown?.models?.[usageModel]?.metrics.api_requests ?? 0), + 0, + ), + }, + ] + : []; + }); + const alert = { id, model, usageModel, requests, lastUsedDate, lastDayRequests, history }; alerts.push(alert); } } diff --git a/ui/litellm-dashboard/src/components/Navbar/DashboardResources.tsx b/ui/litellm-dashboard/src/components/Navbar/DashboardResources.tsx index a19eaab6b79..d1784e3bb80 100644 --- a/ui/litellm-dashboard/src/components/Navbar/DashboardResources.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/DashboardResources.tsx @@ -120,7 +120,7 @@ export function DashboardResources({ children }: { children?: ReactNode }) { ))} )} -
+

, ) -> (Option, CacheOptions) where + P::HostCall: From, P::Error: From + std::error::Error, { let service = match self.backend { @@ -51,15 +41,21 @@ fn selected_cache<'py>( kwargs: &Bound<'py, PyDict>, call_type: &str, ) -> PyResult>> { + if kwargs + .get_item("caching")? + .is_some_and(|value| value.is(pyo3::types::PyBool::new(py, false))) + { + return Ok(None); + } + configured_cache(py, call_type) +} + +fn configured_cache<'py>(py: Python<'py>, call_type: &str) -> PyResult>> { let configured = py .import(CACHE_HOST_MODULE)? .getattr("configured_cache")? .call0()?; - if configured.is_none() - || kwargs - .get_item("caching")? - .is_some_and(|value| value.is(pyo3::types::PyBool::new(py, false))) - { + if configured.is_none() { return Ok(None); } let supported = configured.getattr("supported_call_types")?; @@ -90,9 +86,14 @@ pub(crate) fn configure( python: &mut python::PythonCache, py: Python<'_>, arguments: &Bound<'_, PyDict>, - call_type: &str, + function: &Bound<'_, PyAny>, ) -> PyResult { - let selected = selected_cache(py, arguments, call_type)?; + let call_type: String = function.getattr("__name__")?.extract()?; + let selected = if python.asynchronous() { + configured_cache(py, &call_type)? + } else { + None + }; let Some(cache) = selected else { return Ok(Selection { backend: Backend::Disabled, @@ -102,6 +103,9 @@ pub(crate) fn configure( let enabled = cache .call_method("should_use_cache", (), Some(arguments))? .extract::()?; + let skips_read = arguments + .get_item("caching")? + .is_some_and(|value| !value.is_none() && !value.is(pyo3::types::PyBool::new(py, true))); let controls = arguments .get_item("cache")? .filter(|value| !value.is_none()); @@ -117,7 +121,7 @@ pub(crate) fn configure( }; let options = CacheOptions { policy: CachePolicy { - no_cache: boolean("no-cache")?, + no_cache: boolean("no-cache")? || skips_read, no_store: boolean("no-store")?, ..CachePolicy::default() }, @@ -129,7 +133,7 @@ pub(crate) fn configure( .map(|value| value.extract()) .transpose()? .unwrap_or_default(); - python.bind(cache, arguments); + python.bind(cache, function, arguments); Ok(Selection { backend: if enabled { Backend::Python { namespace } diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index db5696d52ce..29c26ed30a3 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -3,7 +3,7 @@ use std::error::Error as StdError; use litellm_error::Report; use litellm_inference::Error; use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, + exceptions::{PyRuntimeError, PyTypeError, PyValueError}, prelude::*, }; use reqwest::header::HeaderMap; @@ -36,6 +36,37 @@ const MESSAGES_MISSING_CREDENTIAL: &str = "messages_missing_credential"; /// Python refuses the request only once its logger knows the resolved model and provider (the /// provider config's request transform), so the failure logs them. const MESSAGES_LATE_REJECTION: &str = "messages_late_rejection"; +/// Python's request transform raises a builtin exception (`TypeError`) that its handler does not +/// map, so the caller gets the native exception as it is. +const MESSAGES_PYTHON_RAISED: &str = "messages_python_raised"; +/// The text of a refusal Python's provider config raises from its request transform, which its +/// HTTP handler maps as a provider error. +const MESSAGES_PROVIDER_REJECTION: &str = "messages_provider_rejection"; + +fn provider_rejection(rejection: &(dyn StdError + 'static)) -> Option { + match rejection.downcast_ref::() { + Some(litellm_llms::Error::MissingField("max_tokens")) => { + return Some("max_tokens is required for Anthropic /v1/messages API".to_owned()); + } + Some(_) => return None, + None => {} + } + match rejection.downcast_ref::() { + Some( + detail @ litellm_llms::ErrorDetail::InvalidChoice { + field: "reasoning effort", + .. + }, + ) => Some(format!("litellm.BadRequestError: {detail}")), + Some( + detail @ (litellm_llms::ErrorDetail::InvalidReasoningEffort { .. } + | litellm_llms::ErrorDetail::UnsupportedValue { + field: "effort", .. + }), + ) => Some(detail.to_string()), + _ => None, + } +} pub(crate) fn upstream_error( py: Python<'_>, @@ -90,6 +121,9 @@ pub(crate) fn route_failure( if let Some(raised) = crate::secrets::python_error(py, &error) { return Ok(raised); } + if let Some(raised) = crate::routes::messages::workload_identity::python_error(py, &error) { + return Ok(raised); + } if let Some((status, body)) = error.upstream_response() { let headers = error .upstream_headers() @@ -104,6 +138,18 @@ pub(crate) fn route_failure( return public(upstream_error(py, status, body, &headers)?); } let native = match (marker, error.rejected_request()) { + (RequestMarker::Messages, Some(rejection)) + if matches!( + rejection.downcast_ref::(), + Some(litellm_llms::ErrorDetail::UnorderableTypes { .. }) + ) => + { + let native = PyTypeError::new_err(rejection.to_string()); + let value = native.value(py); + value.setattr(MESSAGES_LATE_REJECTION, true)?; + value.setattr(MESSAGES_PYTHON_RAISED, true)?; + native + } (RequestMarker::Messages, Some(rejection)) => { let native = PyValueError::new_err(Report::new(rejection).to_string()); let value = native.value(py); @@ -123,6 +169,9 @@ pub(crate) fn route_failure( ) { value.setattr(MESSAGES_LATE_REJECTION, true)?; } + if let Some(text) = provider_rejection(rejection) { + value.setattr(MESSAGES_PROVIDER_REJECTION, text)?; + } native } (RequestMarker::Messages, None) => { @@ -324,6 +373,72 @@ mod tests { }); } + #[rstest::rstest] + #[case::missing_max_tokens( + litellm_llms::Error::MissingField("max_tokens").into(), + Some("max_tokens is required for Anthropic /v1/messages API") + )] + #[case::unmapped_effort( + Error::invalid_request(litellm_llms::ErrorDetail::InvalidChoice { + field: "reasoning effort", + actual: "'ultra'".into(), + choices: vec!["low", "none"], + }), + Some("litellm.BadRequestError: Unmapped reasoning effort: 'ultra'. Must be one of: 'low', 'none'.") + )] + #[case::other_missing_field(litellm_llms::Error::MissingField("model").into(), None)] + #[case::unsupported_param( + Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedParameter { + model: "claude".into(), + param: "top_k".into(), + value: "5".into(), + hint: String::new(), + }), + None + )] + fn provider_config_rejections_carry_python_text( + #[case] error: Error, + #[case] expected: Option<&str>, + ) { + Python::initialize(); + Python::attach(|py| { + let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap(); + let text = native + .value(py) + .getattr_opt(MESSAGES_PROVIDER_REJECTION) + .unwrap() + .map(|value| value.extract::().unwrap()); + assert_eq!(text.as_deref(), expected); + }); + } + + #[test] + fn an_unorderable_comparison_raises_python_type_error_as_it_is() { + Python::initialize(); + Python::attach(|py| { + let error = Error::invalid_request(litellm_llms::ErrorDetail::UnorderableTypes { + op: "<=", + left: "str", + right: "int", + }); + let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap(); + assert!(native.is_instance_of::(py)); + let value = native.value(py); + assert_eq!( + value.str().unwrap().to_string(), + "'<=' not supported between instances of 'str' and 'int'" + ); + assert!(value.getattr_opt(MESSAGES_PYTHON_RAISED).unwrap().is_some()); + assert!( + value + .getattr_opt(MESSAGES_LATE_REJECTION) + .unwrap() + .is_some() + ); + assert!(value.getattr_opt(MESSAGES_REQUEST_ERROR).unwrap().is_none()); + }); + } + #[rstest::rstest] #[case::rejected_request( Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedParameter { diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index 57c58c3f7cd..e7cc8e35172 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -101,15 +101,22 @@ pub(crate) fn call_config( kwargs: &Bound<'_, PyDict>, asynchronous: bool, ) -> PyResult { - let settings = HttpSettings::from_layers([ + Ok(resolved(py, call_settings(py, kwargs, asynchronous)?)) +} + +fn call_settings( + py: Python<'_>, + kwargs: &Bound<'_, PyDict>, + asynchronous: bool, +) -> PyResult { + Ok(HttpSettings::from_layers([ for_call(call_ssl_verify(kwargs)?, asynchronous), HttpSettingsLayer::from_environment(&ProcessEnvironment), match PythonSettings::Http.read_or_unset(py)? { Some(snapshot) => configured(&snapshot)?, None => HttpSettingsLayer::default(), }, - ]); - Ok(resolved(py, settings)) + ])) } pub(crate) fn settings_config( @@ -132,10 +139,31 @@ pub(crate) fn settings_config( Ok(resolved(py, settings)) } +#[pyfunction] +#[pyo3(signature = (ssl_security_level=None, ssl_ecdh_curve=None))] +pub(crate) fn unsupported_tls_settings( + ssl_security_level: Option, + ssl_ecdh_curve: Option, +) -> Vec { + let settings = HttpSettings::from_layers([HttpSettingsLayer { + ssl_security_level, + ssl_ecdh_curve, + ..HttpSettingsLayer::default() + }]); + Resolution::from(&settings) + .unsupported + .iter() + .map(ToString::to_string) + .collect() +} + +fn resolution(settings: HttpSettings) -> Resolution { + Resolution::from(&settings.without_missing_files(&|path: &Path| path.exists())) +} + /// The client config `settings` resolve to, warning once per process about each setting the client cannot apply fn resolved(py: Python<'_>, settings: HttpSettings) -> HttpClientConfig { - let resolution = - Resolution::from(&settings.without_missing_files(&|path: &Path| path.exists())); + let resolution = resolution(settings); for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) { crate::logger::capture(py).scope(|| litellm_tracing::warn!("{unsupported}")); } @@ -151,6 +179,15 @@ pub(crate) fn provider_client( Ok(pool().client(&config, ClientVariant::Provider)) } +pub(crate) fn messages_client( + py: Python<'_>, + kwargs: &Bound<'_, PyDict>, + asynchronous: bool, +) -> PyResult> { + let config = resolution(call_settings(py, kwargs, asynchronous)?).config; + Ok(pool().client(&config, ClientVariant::Undecoded)) +} + pub(crate) fn host_client(py: Python<'_>, variant: ClientVariant) -> PyResult { let config = call_config(py, &PyDict::new(py), true)?; pool().client(&config, variant).map_err(client_error) diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 7edf13035c1..45ea95560e1 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -35,6 +35,8 @@ mod _native { #[pymodule_export] use crate::errors::{RustBridgeDeclined, RustUpstreamError}; #[pymodule_export] + use crate::http::unsupported_tls_settings; + #[pymodule_export] use crate::logger::NativeDiagnosticProcessor; #[pymodule_export] use crate::model_cost::{ @@ -191,6 +193,7 @@ mod tests { "gil_stats", "process_state_started", "reserve_process_for_forking", + "unsupported_tls_settings", ]; #[cfg(feature = "huggingface")] expected.push("HuggingFaceEncoding"); diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index dec8ff0b0ce..927ed741669 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -4,7 +4,7 @@ use std::{ }; use litellm_auth::InputSource; -use litellm_host_python::{from_py, from_py_argument, to_py}; +use litellm_host_python::{from_py, from_py_argument, json_to_py, to_py}; use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Map, Value}; @@ -143,6 +143,19 @@ pub(crate) fn public_response( .map(Bound::unbind) } +pub(crate) fn public_json_response( + py: Python<'_>, + module: &str, + response: &(impl Serialize + ?Sized), +) -> PyResult> { + let factory = py.import(module)?.getattr("response")?; + let document = + serde_json::to_value(response).map_err(|error| PyValueError::new_err(error.to_string()))?; + factory + .call1((json_to_py(py, &document)?,)) + .map(Bound::unbind) +} + struct RequestFieldSources<'py> { body: Option>, credentials: Option>, diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/agentic.rs b/litellm-rust/crates/python-bridge/src/routes/messages/agentic.rs new file mode 100644 index 00000000000..f6652d82050 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/agentic.rs @@ -0,0 +1,172 @@ +use std::sync::{Arc, Mutex, PoisonError}; + +use litellm_host::{ + hooks::CallHooks, + interceptors::{RequestContext, WireRequest}, + lifecycle::Timing, +}; +use litellm_host_python::{HookStep, PythonOwned, PythonRuntime, json_to_py, to_py}; +use pyo3::{ + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::PyDict, +}; + +use super::host::ROUTE_HOST_MODULE; + +/// What the provider was sent, which Python's agentic hooks read back as the handler's own +/// model, provider and request params. +struct Sent { + request: Py, + hooked: bool, + body: Py, + model: String, + provider: String, + requested: Py, +} + +/// Python's Messages handler hands the provider's answer to the callbacks overriding +/// `async_should_run_agentic_loop` before `@client` sees it, and wraps a stream so they run +/// once it ends. This hook asks the route host to do both at the same points. +pub(super) type HookedRequest = Arc>>>; + +pub(super) struct AgenticLoop { + asynchronous: bool, + request: Option>, + hooked: HookedRequest, + decided: bool, + sent: Option, +} + +impl AgenticLoop { + pub(super) fn new(asynchronous: bool, hooked: HookedRequest) -> Self { + Self { + asynchronous, + request: None, + hooked, + decided: false, + sent: None, + } + } + + fn hooked_request(&self, py: Python<'_>) -> Option> { + self.hooked + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .map(|request| request.clone_ref(py)) + } + + fn host_call<'py>( + &self, + py: Python<'py>, + name: &str, + subject: &Bound<'py, PyAny>, + sent: &Sent, + ) -> PyResult> { + py.import(ROUTE_HOST_MODULE)?.getattr(name)?.call1(( + subject, + sent.request.bind(py), + sent.body.bind(py), + &sent.model, + &sent.provider, + sent.requested.bind(py), + sent.hooked, + )) + } + + fn resume_response( + &mut self, + _: Python<'_>, + result: PyResult>, + ) -> PyResult>> { + result.map(HookStep::Ready) + } +} + +impl CallHooks for AgenticLoop { + fn arguments_prepared(&mut self, py: Python<'_>, arguments: &Py) -> PyResult<()> { + if self.asynchronous { + self.request = Some(arguments.clone_ref(py)); + } + Ok(()) + } + + fn before_provider_request( + &mut self, + py: Python<'_>, + wire: Box, + context: &RequestContext, + ) -> PyResult>> { + if self.decided { + return Ok(HookStep::Ready(wire)); + } + let Some(caller) = &self.request else { + return Ok(HookStep::Ready(wire)); + }; + self.decided = true; + let hooked = self.hooked_request(py); + let hooked_flag = hooked.is_some(); + let request = hooked.unwrap_or_else(|| caller.clone_ref(py)); + let module = py.import(ROUTE_HOST_MODULE)?; + if module + .getattr("runs_agentic_hooks")? + .call1((request.bind(py),))? + .extract::()? + { + module + .getattr("record_agentic_loop")? + .call1((request.bind(py), &context.custom_llm_provider))?; + self.sent = Some(Sent { + request, + hooked: hooked_flag, + body: to_py(py, &wire.body)?, + model: context.model.clone(), + provider: context.custom_llm_provider.clone(), + requested: json_to_py(py, &context.optional_params)?.unbind(), + }); + } + Ok(HookStep::Ready(wire)) + } + + fn transform_response( + &mut self, + py: Python<'_>, + response: Py, + _: Timing, + ) -> PyResult>> { + let Some(sent) = &self.sent else { + return Ok(HookStep::Ready(response)); + }; + let awaitable = self.host_call(py, "agentic_response", response.bind(py), sent)?; + Ok(HookStep::Await(awaitable.unbind(), Self::resume_response)) + } + + fn on_stream_open(&mut self, py: Python<'_>, head: &Py) -> PyResult<()> { + if let Some(sent) = &self.sent { + self.host_call(py, "agentic_stream", head.bind(py), sent)?; + } + Ok(()) + } +} + +impl PythonOwned for AgenticLoop { + fn close(&mut self, _: Python<'_>) { + self.request = None; + self.sent = None; + self.hooked + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.request)?; + if let Some(sent) = &self.sent { + visit.call(&sent.request)?; + visit.call(&sent.body)?; + visit.call(&sent.requested)?; + } + Ok(()) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/answer.rs b/litellm-rust/crates/python-bridge/src/routes/messages/answer.rs new file mode 100644 index 00000000000..80cf2002287 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/answer.rs @@ -0,0 +1,29 @@ +use futures_util::future::BoxFuture; +use litellm_host::machine::HostServices; +use litellm_inference_messages::{Error, HostAnswer}; + +use super::host::{MessagesHostCall, MessagesHosted}; + +pub(super) struct PythonAnswer(HostServices); + +impl PythonAnswer { + pub(super) fn new(services: HostServices) -> Self { + Self(services) + } +} + +impl HostAnswer for PythonAnswer { + fn answer(&self, stream: bool) -> BoxFuture<'_, Result<(), Error>> { + Box::pin( + self.0 + .call(move |reply| MessagesHostCall::Answer { stream, reply }), + ) + } + + fn intercept(&self, stream: bool) -> BoxFuture<'_, Result> { + Box::pin( + self.0 + .call(move |reply| MessagesHostCall::Intercept { stream, reply }), + ) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index c975ac339c2..67a96cac32d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -1,34 +1,121 @@ -use crate::cache::{CacheCall, Cached, PythonCache, Selection}; +use crate::cache::{CacheCall, PythonCache, Selection}; use litellm_host_python::{PythonHostCalls, PythonOwned}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + use bytes::Bytes; -use litellm_host_python::{InvokeError, PythonBinding, from_py, to_py}; +use litellm_host::protocol::Reply; +use litellm_host_python::{ + InvokeError, PythonBinding, effective_py_args, from_py, json_from_py, to_py, +}; use litellm_inference_messages::{ - Error, MessagesCall, MessagesSettings, MessagesShaping, litellm_params, messages_body, + DecodingError, Error, MessagesCall, MessagesSettings, MessagesShaping, UnencodableHeader, + litellm_params, messages_body, route::{Messages, MessagesStreamHead}, }; use litellm_llms::base_llm::messages::context::MessagesModelCapabilities; use litellm_llms_types::headers::ProviderSpecificHeaders; use litellm_router_types::LitellmParams; use pyo3::{ - exceptions::PyException, + exceptions::{PyException, PyUnicodeEncodeError}, gc::{PyTraverseError, PyVisit}, prelude::*, - types::{PyBytes, PyDict}, + sync::PyOnceLock, + types::{PyBytes, PyDict, PyList, PyMapping, PyString, PyTuple}, }; use serde_json::{Map, Value}; +use super::agentic::HookedRequest; +use super::safeguards::{SafeguardsCall, SafeguardsHost}; +use super::transport::{HostedTransport, TransportCall, python_failure}; use crate::{ errors::{RequestMarker, route_failure}, marshal::{ - ArgumentError, optional_timeout, project_optional_fields, public_response, + ArgumentError, optional_timeout, project_optional_fields, public_json_response, python_timeout_seconds, }, python_settings::PythonSettings, routes::RequestView, }; -const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages"; +type MergedHeaders = (Option>, Option>); + +fn ascii_headers<'py>( + headers: &Bound<'py, PyAny>, + unencodable: &mut Option>, +) -> PyResult> { + let Ok(headers) = headers.cast::() else { + return Ok(headers.clone()); + }; + let py = headers.py(); + let kept = PyDict::new(py); + for (name, value) in headers.iter() { + if value.is_instance_of::() + && let Err(error) = value.call_method1("encode", ("ascii",)) + { + if !error.is_instance_of::(py) { + return Err(error); + } + if unencodable.is_none() { + let sent = value.call_method1("encode", ("utf-8", "surrogatepass"))?; + *unencodable = Some(sent.cast::()?.as_bytes().to_vec()); + } + continue; + } + kept.set_item(name, value)?; + } + Ok(kept.into_any()) +} + +const PROVIDER_SPECIFIC_HEADERS_MODULE: &str = + "litellm.litellm_core_utils.get_provider_specific_headers"; + +fn ascii_scoped_headers<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let ascii_entry = |entry: Bound<'py, PyAny>| -> PyResult> { + let Ok(entry) = entry.cast::() else { + return Ok(entry); + }; + let Some(headers) = entry.get_item("extra_headers")? else { + return Ok(entry.clone().into_any()); + }; + let kept = entry.copy()?; + kept.set_item("extra_headers", ascii_headers(&headers, &mut None)?)?; + Ok(kept.into_any()) + }; + if let Ok(entries) = value.cast::() { + return Ok(PyList::new( + value.py(), + entries + .iter() + .map(ascii_entry) + .collect::>>()?, + )? + .into_any()); + } + if let Ok(entries) = value.cast::() { + return Ok(PyTuple::new( + value.py(), + entries + .iter() + .map(ascii_entry) + .collect::>>()?, + )? + .into_any()); + } + ascii_entry(value.clone()) +} + +fn json_messages(messages: &Bound<'_, PyAny>) -> PyResult { + if messages.is_instance_of::() || messages.is_instance_of::() { + return json_from_py(messages); + } + json_from_py(&messages.py().get_type::().call1((messages,))?) +} + +pub(super) const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages"; const MODEL_CAPABILITIES_MODULE: &str = "litellm.rust_bridge.host.model_capabilities"; fn merge_headers( @@ -41,6 +128,19 @@ fn merge_headers( (!merged.is_empty()).then_some(merged) } +const FEDERATING_PROVIDER: &str = "anthropic"; + +fn workload_identity_fields(py: Python<'_>) -> PyResult<&'static [String]> { + static FIELDS: PyOnceLock> = PyOnceLock::new(); + FIELDS + .get_or_try_init(py, || { + py.import(ROUTE_HOST_MODULE)? + .getattr("WORKLOAD_IDENTITY_FIELDS")? + .extract() + }) + .map(Vec::as_slice) +} + fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult>> { PythonSettings::ProviderDefaults .read_or_unset(py)? @@ -50,11 +150,17 @@ fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult( +fn project_litellm_params<'a, 'py>( + names: impl IntoIterator, argument: impl Fn(&str) -> PyResult>>, global: impl Fn(&str) -> PyResult>>, ) -> PyResult> { - let fields = project_optional_fields(LitellmParams::fields(), &argument)?; + let fields = project_optional_fields( + LitellmParams::fields() + .map(|name| -> &'a str { name }) + .chain(names), + &argument, + )?; let globals = LitellmParams::specs() .filter(|spec| !spec.wire.iter().any(|name| fields.contains_key(*name))) .filter_map(|spec| spec.module_global); @@ -62,36 +168,279 @@ fn project_litellm_params<'py>( Ok(litellm_params(fields.into_iter().chain(folded).collect())) } +pub(super) struct MessagesHosted; + +impl litellm_host::protocol::Protocol for MessagesHosted { + type Request = (MessagesCall, Selection, bool); + type Response = ::Response; + type Error = Error; + type HostCall = MessagesHostCall; + type Chunk = Bytes; + type StreamHead = MessagesStreamHead; +} + +pub(super) enum MessagesHostCall { + Answer { stream: bool, reply: Reply<()> }, + Intercept { stream: bool, reply: Reply }, + Cache(CacheCall), + PreRequest(Reply>), + Safeguards(SafeguardsCall), + Transport(TransportCall), +} + +impl From for MessagesHostCall { + fn from(call: CacheCall) -> Self { + Self::Cache(call) + } +} + +impl From for MessagesHostCall { + fn from(call: TransportCall) -> Self { + Self::Transport(call) + } +} + +enum Answered { + Stream, + Whole { verbatim: bool }, +} + +#[derive(serde::Deserialize)] +struct HostSettings { + #[serde(flatten)] + settings: MessagesSettings, + model: String, + stream_cost_model: Option, + extra_headers: Option>, +} + /// The Python side of the Messages route: projects the prepared arguments and builds the /// public response, chunks and exceptions. pub(super) struct MessagesPythonHost { view: RequestView, cache: PythonCache, + transport: HostedTransport, + safeguards: SafeguardsHost, + classifies: Arc, + awaiting_safeguards: bool, + stream_cost_model: Option, + asynchronous: bool, + pre_request: Option>>, + hooked: bool, + hooked_request: HookedRequest, + requested_stream: bool, + arguments: Option>, + interceptor: Option>, + pending_answer: Option>, + pending_intercept: Option>, + pending_short_circuit: Option<(bool, Reply)>, + answer: Option>, + answered: Option, } impl MessagesPythonHost { - pub(super) fn new(view: RequestView, asynchronous: bool) -> Self { + pub(super) fn new( + view: RequestView, + asynchronous: bool, + hooked_request: HookedRequest, + classifies: Arc, + ) -> Self { Self { view, cache: PythonCache::new(asynchronous), + transport: HostedTransport::default(), + safeguards: SafeguardsHost::default(), + classifies, + awaiting_safeguards: false, + stream_cost_model: None, + asynchronous, + pre_request: None, + hooked: false, + hooked_request, + requested_stream: false, + arguments: None, + interceptor: None, + pending_answer: None, + pending_intercept: None, + pending_short_circuit: None, + answer: None, + answered: None, } } + fn interceptor(&self, py: Python<'_>) -> PyResult>> { + if !self.asynchronous { + return Ok(None); + } + let interceptor = py + .import(ROUTE_HOST_MODULE)? + .getattr("interceptor")? + .call1((self.view.request(py)?,))?; + Ok((!interceptor.is_none()).then(|| interceptor.unbind())) + } + + fn may_intercept(&self, py: Python<'_>) -> PyResult { + if !self.asynchronous { + return Ok(false); + } + let module = py.import(ROUTE_HOST_MODULE)?; + Ok(module.getattr("has_interceptors")?.call0()?.extract()? + || module.getattr("short_circuits")?.call0()?.extract()?) + } + + fn begin_intercept( + &mut self, + py: Python<'_>, + stream: bool, + reply: Reply, + ) -> PyResult>> { + let module = py.import(ROUTE_HOST_MODULE)?; + if !module + .getattr("short_circuits")? + .call0()? + .extract::()? + { + return self.begin_interceptor(py, stream, reply); + } + let pending = module.getattr("short_circuit")?.call1(( + self.view.request(py)?, + stream, + self.hooked, + ))?; + self.pending_short_circuit = Some((stream, reply)); + Ok(Some(pending.unbind())) + } + + fn resume_short_circuit( + &mut self, + py: Python<'_>, + stream: bool, + reply: Reply, + answer: Py, + ) -> PyResult>> { + if answer.is_none(py) { + return self.begin_interceptor(py, stream, reply); + } + self.answer = Some(answer); + self.answered = Some(match stream { + true => Answered::Stream, + false => Answered::Whole { verbatim: false }, + }); + reply.send(true); + Ok(None) + } + + fn begin_interceptor( + &mut self, + py: Python<'_>, + stream: bool, + reply: Reply, + ) -> PyResult>> { + self.interceptor = self.interceptor(py)?; + if self.interceptor.is_none() { + reply.send(false); + return Ok(None); + } + let pending = self.start_answer(py, stream)?; + self.pending_intercept = Some(reply); + Ok(Some(pending)) + } + + fn begin_answer( + &mut self, + py: Python<'_>, + stream: bool, + reply: Reply<()>, + ) -> PyResult>> { + let pending = self.start_answer(py, stream)?; + self.pending_answer = Some(reply); + Ok(Some(pending)) + } + + fn start_answer(&mut self, py: Python<'_>, stream: bool) -> PyResult> { + let module = py.import(ROUTE_HOST_MODULE)?; + let request = self.view.request(py)?; + let pending = match &self.interceptor { + Some(interceptor) => { + module + .getattr("answer")? + .call1((interceptor, request, stream, self.hooked))? + } + None => module + .getattr("gateway")? + .call1((request, self.asynchronous, self.hooked))?, + }; + if pending.is_none() { + return Err(litellm_host_python::Error::MissingState.into()); + } + self.answered = Some(match (stream, &self.interceptor) { + (true, _) => Answered::Stream, + (false, Some(_)) => Answered::Whole { verbatim: false }, + (false, None) => Answered::Whole { verbatim: true }, + }); + Ok(pending.unbind()) + } + + fn host_settings( + &self, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + ) -> PyResult { + from_py(&py.import(ROUTE_HOST_MODULE)?.getattr("settings")?.call1(( + arguments, + self.view.request(py)?, + self.asynchronous, + ))?) + } + + fn body_params( + &self, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + ) -> PyResult> { + let request = effective_py_args(self.view.request(py)?, arguments)?; + let params = py + .import(ROUTE_HOST_MODULE)? + .getattr("body_params")? + .call1((request, self.asynchronous, self.hooked))? + .cast_into::()?; + params + .items()? + .iter() + .map(|item| { + let (name, value): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item.extract()?; + let name: String = name.extract()?; + let value = match name.as_str() { + "messages" => json_messages(&value)?, + _ => json_from_py(&value)?, + }; + Ok((name, value)) + }) + .collect() + } + fn projection( &self, py: Python<'_>, arguments: &Bound<'_, PyDict>, + settings: HostSettings, ) -> PyResult> { let argument = |name: &str| self.view.argument(py, arguments, name); let string = |name: &str| -> PyResult> { argument(name)?.map(|value| value.extract()).transpose() }; - let model = string("model")?.ok_or(ArgumentError::Required("model"))?; + let model = settings.model; let messages = argument("messages")?.ok_or(ArgumentError::Required("messages"))?; - let fields = self.view.body_params(py, arguments, ROUTE_HOST_MODULE)?; + let mut fields = self.body_params(py, arguments)?; + let prepared = fields.remove("messages"); + let presanitized = prepared.is_some(); + let messages = match prepared { + Some(messages) => messages, + None => json_messages(&messages)?, + }; let body = [ ("model".to_string(), Value::String(model.clone())), - ("messages".to_string(), from_py(&messages)?), + ("messages".to_string(), messages), ] .into_iter() .chain(fields) @@ -112,10 +461,35 @@ impl MessagesPythonHost { .call1((&model, api_base.as_deref()))? .extract()?, }; - let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?; - let extra_headers = self.merged_headers(py, arguments)?; - let provider_specific_header = self.provider_specific_header(py, arguments)?; - let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?; + let (extra_headers, mut unencodable_request_header) = match settings.extra_headers { + Some(headers) => (Some(headers), None), + None => self.merged_headers(py, arguments)?, + }; + let provider_specific_header = self.provider_specific_header( + py, + arguments, + custom_llm_provider.as_deref(), + &mut unencodable_request_header, + )?; + let shaping = MessagesShaping { + capabilities: self.capabilities(py, &model, custom_llm_provider.as_deref())?, + settings: MessagesSettings { + whole_sse_frames: settings.stream_cost_model.is_some(), + presanitized, + intercepted: self.may_intercept(py)?, + unencodable_request_header, + ..settings.settings + }, + }; + let federation_fields: &[String] = match custom_llm_provider.as_deref() { + Some(FEDERATING_PROVIDER) => workload_identity_fields(py)?, + _ => &[], + }; + let litellm_params = project_litellm_params( + federation_fields.iter().map(String::as_str), + argument, + |name| module_global(py, name), + )?; Ok(messages_body(body).and_then(|body| { Ok(MessagesCall { body, @@ -135,49 +509,55 @@ impl MessagesPythonHost { &self, py: Python<'_>, arguments: &Bound<'_, PyDict>, - ) -> PyResult>> { - let mapping = |name: &str| -> PyResult>> { + ) -> PyResult { + let mut unencodable = None; + let mut mapping = |name: &str| -> PyResult>> { self.view .argument(py, arguments, name)? - .map(|value| from_py(&value)) + .map(|value| from_py(&ascii_headers(&value, &mut unencodable)?)) .transpose() }; - Ok(merge_headers( - mapping("headers")?, - mapping("extra_headers")?, - )) + let headers = mapping("headers")?; + let extra_headers = mapping("extra_headers")?; + Ok((merge_headers(headers, extra_headers), unencodable)) } fn provider_specific_header( &self, py: Python<'_>, arguments: &Bound<'_, PyDict>, + provider: Option<&str>, + unencodable: &mut Option>, ) -> PyResult> { - self.view + let Some(value) = self + .view .argument(py, arguments, "provider_specific_header")? - .map(|value| from_py(&value)) - .transpose() + else { + return Ok(None); + }; + if unencodable.is_none() + && let Some(provider) = provider + && let Ok(scoped) = py + .import(PROVIDER_SPECIFIC_HEADERS_MODULE)? + .getattr("ProviderSpecificHeaderUtils")? + .call_method1("get_provider_specific_headers", (&value, provider, true)) + { + ascii_headers(&scoped, unencodable)?; + } + from_py(&ascii_scoped_headers(&value)?).map(Some) } - fn shaping( + fn capabilities( &self, py: Python<'_>, model: &str, custom_llm_provider: Option<&str>, - arguments: &Bound<'_, PyDict>, - ) -> PyResult { - let module = py.import(ROUTE_HOST_MODULE)?; - let capabilities: MessagesModelCapabilities = from_py( + ) -> PyResult { + from_py( &py.import(MODEL_CAPABILITIES_MODULE)? .getattr("anthropic_model_capabilities")? .call1((model, custom_llm_provider))?, - )?; - let settings: MessagesSettings = - from_py(&module.getattr("settings")?.call1((arguments,))?)?; - Ok(MessagesShaping { - capabilities, - settings, - }) + ) } fn provider(&self, py: Python<'_>) -> String { @@ -190,6 +570,14 @@ impl MessagesPythonHost { .unwrap_or_else(|| "anthropic".into()) } + fn map_transport_failure(&self, py: Python<'_>, error: PyErr) -> PyResult { + let mapped = py + .import(ROUTE_HOST_MODULE)? + .getattr("map_transport_failure")? + .call1((error.value(py), self.view.request(py)?, self.provider(py)))?; + Ok(PyErr::from_value(mapped)) + } + fn map_failure(&self, py: Python<'_>, error: PyErr) -> PyErr { if !error.is_instance_of::(py) { return error; @@ -197,7 +585,14 @@ impl MessagesPythonHost { let mapped = py .import(ROUTE_HOST_MODULE) .and_then(|module| module.getattr("map_failure")) - .and_then(|map| map.call1((error.value(py), self.view.request(py)?, self.provider(py)))) + .and_then(|map| { + map.call1(( + error.value(py), + self.view.request(py)?, + self.provider(py), + self.asynchronous, + )) + }) .and_then(|mapped| { mapped .extract::>() @@ -210,25 +605,109 @@ impl MessagesPythonHost { } } +impl MessagesPythonHost { + fn decode_call( + &mut self, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + ) -> Result> { + let settings = self + .host_settings(py, arguments) + .map_err(InvokeError::Python)?; + self.stream_cost_model = settings.stream_cost_model.clone(); + self.projection(py, arguments, settings) + .map_err(|error| InvokeError::Python(self.map_failure(py, error)))? + .map_err(InvokeError::Native) + } + + fn begin_pre_request( + &mut self, + py: Python<'_>, + reply: Reply>, + ) -> PyResult> { + let hooks = py + .import(ROUTE_HOST_MODULE)? + .getattr("pre_request_hooks")? + .call1((self.view.request(py)?,))?; + self.pre_request = Some(reply); + Ok(hooks.unbind()) + } + + fn resume_pre_request( + &mut self, + py: Python<'_>, + reply: Reply>, + result: PyResult>, + ) -> Result<(), InvokeError> { + let hooked = result?; + let arguments = hooked + .bind(py) + .cast::() + .map_err(|error| InvokeError::Python(error.into()))?; + self.view + .resolve(py, arguments) + .map_err(InvokeError::Python)?; + self.hooked = true; + let effective = self + .view + .request(py) + .and_then(|request| effective_py_args(request, arguments)) + .map_err(InvokeError::Python)?; + self.safeguards + .rebind(py, &effective) + .map_err(InvokeError::Python)?; + *self + .hooked_request + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(arguments.clone().unbind()); + match self.decode_call(py, arguments) { + Ok(call) => reply.send(Ok(call)), + Err(InvokeError::Native(error)) => reply.send(Err(error)), + Err(error) => return Err(error), + } + Ok(()) + } +} + impl PythonBinding for MessagesPythonHost { - type Protocol = Cached; + type Protocol = MessagesHosted; type Failure = PyErr; fn decode_request( &mut self, py: Python<'_>, arguments: &Bound<'_, PyDict>, - ) -> Result<(MessagesCall, Selection), InvokeError> { + ) -> Result<(MessagesCall, Selection, bool), InvokeError> { self.view .resolve(py, arguments) .map_err(InvokeError::Python)?; - let selection = - crate::cache::configure(&mut self.cache, py, arguments, "anthropic_messages") - .map_err(InvokeError::Python)?; - self.projection(py, arguments) - .map_err(|error| InvokeError::Python(self.map_failure(py, error)))? - .map_err(InvokeError::Native) - .map(|request| (request, selection)) + let function = py + .import(ROUTE_HOST_MODULE) + .and_then(|module| module.call_method0("cached_function")) + .map_err(InvokeError::Python)?; + let selection = crate::cache::configure(&mut self.cache, py, arguments, &function) + .map_err(InvokeError::Python)?; + let effective = self + .view + .request(py) + .and_then(|request| effective_py_args(request, arguments)) + .map_err(InvokeError::Python)?; + let classifies = self + .safeguards + .configure(py, &effective) + .map_err(InvokeError::Python)?; + self.classifies.store(classifies, Ordering::Release); + self.requested_stream = arguments + .get_item("stream") + .map_err(InvokeError::Python)? + .is_some_and(|stream| stream.is(pyo3::types::PyBool::new(py, true))); + self.arguments = Some(arguments.clone().unbind()); + let request = self.decode_call(py, arguments)?; + let hosted = self + .transport + .select(py, self.view.request(py).map_err(InvokeError::Python)?) + .map_err(InvokeError::Python)?; + Ok((request, selection, hosted)) } fn encode_response( @@ -236,11 +715,23 @@ impl PythonBinding for MessagesPythonHost { py: Python<'_>, response: litellm_inference_messages::MessagesBody, ) -> PyResult> { - public_response(py, ROUTE_HOST_MODULE, &response) + if let Some(answer) = self.answer.take() { + return Ok(answer); + } + let public = public_json_response(py, ROUTE_HOST_MODULE, &response)?; + self.safeguards.published(py, public) } - fn deliver_response(&mut self, py: Python<'_>, response: &Bound<'_, PyAny>) -> PyResult<()> { - self.cache.store_response(py, response) + fn deliver_response(&mut self, py: Python<'_>, response: Py) -> PyResult> { + match self.answered.take() { + Some(Answered::Stream) => return self.cache.wrap_stream(py, response), + Some(Answered::Whole { verbatim }) => { + self.cache.store_response(py, response.bind(py), verbatim)?; + } + None if self.requested_stream => {} + None => self.cache.store_response(py, response.bind(py), false)?, + } + Ok(response) } fn encode_stream_head( @@ -248,17 +739,63 @@ impl PythonBinding for MessagesPythonHost { py: Python<'_>, head: MessagesStreamHead, ) -> PyResult> { - py.import(ROUTE_HOST_MODULE)? + let module = py.import(ROUTE_HOST_MODULE)?; + let head = module .getattr("stream_hidden_params")? - .call1((to_py(py, &head.headers)?,)) - .map(Bound::unbind) + .call1((to_py(py, &head.headers)?,))?; + if let Some(arguments) = self + .arguments + .as_ref() + .filter(|_| self.asynchronous && !self.requested_stream) + { + module + .getattr("unrequested_stream")? + .call1((&head, arguments))?; + } + Ok(head.unbind()) } fn encode_chunk(&mut self, py: Python<'_>, chunk: Bytes) -> PyResult> { - Ok(PyBytes::new(py, &chunk).into_any().unbind()) + let frames = PyBytes::new(py, &chunk).into_any(); + let Some(model) = &self.stream_cost_model else { + return Ok(frames.unbind()); + }; + py.import(ROUTE_HOST_MODULE)? + .getattr("inject_stream_cost")? + .call1((frames, model, self.view.request(py)?)) + .map(Bound::unbind) } fn map_error(&self, py: Python<'_>, error: Error) -> PyResult { + if let Some((raised, raw)) = python_failure(py, &error) { + return if raw { + Ok(raised) + } else { + self.map_transport_failure(py, raised) + }; + } + if let Some(unencodable) = litellm_error::find::(&error) { + let (value, received) = match unencodable { + UnencodableHeader::Sent(value) => (value, false), + UnencodableHeader::Received(value) => (value, true), + }; + let raised = py + .import(ROUTE_HOST_MODULE)? + .getattr("unencodable_header")? + .call1((PyBytes::new(py, value), received))?; + return self.map_transport_failure(py, PyErr::from_value(raised)); + } + if let Some(undecodable) = litellm_error::find::(&error) { + let raised = py + .import(ROUTE_HOST_MODULE)? + .getattr("decoding_failure")? + .call1(( + undecodable.to_string(), + self.view.request(py)?, + self.provider(py), + ))?; + return Ok(PyErr::from_value(raised)); + } route_failure(py, error, RequestMarker::Messages, |native| { Ok(self.map_failure(py, native)) }) @@ -269,24 +806,35 @@ impl PythonBinding for MessagesPythonHost { } } -impl PythonHostCalls> for MessagesPythonHost { +impl PythonHostCalls for MessagesPythonHost { fn handle_host_call( &mut self, py: Python<'_>, - op: CacheCall, + op: MessagesHostCall, ) -> Result<(), InvokeError> { - self.cache - .begin(py, op) - .map(|_| ()) - .map_err(InvokeError::Python) + self.begin_host_call(py, op).map(|_| ()) } fn begin_host_call( &mut self, py: Python<'_>, - op: CacheCall, + op: MessagesHostCall, ) -> Result>, InvokeError> { - self.cache.begin(py, op).map_err(InvokeError::Python) + self.awaiting_safeguards = matches!(op, MessagesHostCall::Safeguards(_)); + match op { + MessagesHostCall::Cache(op) => self.cache.begin(py, op), + MessagesHostCall::Transport(op) => { + let request = self.view.request(py).map_err(InvokeError::Python)?; + self.transport.begin(py, op, request) + } + MessagesHostCall::PreRequest(reply) => self.begin_pre_request(py, reply).map(Some), + MessagesHostCall::Safeguards(op) => self.safeguards.begin(py, op).map(Some), + MessagesHostCall::Answer { stream, reply } => self.begin_answer(py, stream, reply), + MessagesHostCall::Intercept { stream, reply } => { + self.begin_intercept(py, stream, reply) + } + } + .map_err(InvokeError::Python) } fn resume_host_call( @@ -294,6 +842,37 @@ impl PythonHostCalls> for MessagesPythonHost { py: Python<'_>, result: PyResult>, ) -> Result>, InvokeError> { + if let Some(reply) = self.pre_request.take() { + return self.resume_pre_request(py, reply, result).map(|()| None); + } + if let Some((stream, reply)) = self.pending_short_circuit.take() { + return self + .resume_short_circuit(py, stream, reply, result?) + .map_err(InvokeError::Python); + } + if let Some(reply) = self.pending_answer.take() { + self.answer = Some(result?); + reply.send(()); + return Ok(None); + } + if let Some(reply) = self.pending_intercept.take() { + self.answer = Some(result?); + reply.send(true); + return Ok(None); + } + if std::mem::take(&mut self.awaiting_safeguards) { + return self + .safeguards + .resume(py, result) + .map(|()| None) + .map_err(InvokeError::Python); + } + if self.transport.pending() { + return self + .transport + .resume(py, result) + .map_err(InvokeError::Python); + } self.cache.resume(py, result).map_err(InvokeError::Python) } } @@ -302,10 +881,25 @@ impl PythonOwned for MessagesPythonHost { fn close(&mut self, _: Python<'_>) { self.view.close(); self.cache.close(); + self.transport.close(); + self.pre_request = None; + self.safeguards.close(); + self.interceptor = None; + self.pending_answer = None; + self.pending_intercept = None; + self.pending_short_circuit = None; + self.answer = None; + self.answered = None; + self.arguments = None; } fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { self.view.traverse(visit)?; - self.cache.traverse(visit) + self.cache.traverse(visit)?; + self.transport.traverse(visit)?; + self.safeguards.traverse(visit)?; + visit.call(&self.interceptor)?; + visit.call(&self.arguments)?; + visit.call(&self.answer) } } @@ -416,7 +1010,40 @@ mod tests { assert_projection(kwargs, globals, expected); } + #[rstest] + #[case::named_fields_are_projected( + json!({"model": "m", "anthropic_federation_rule_id": "fdrl_1", "anthropic_issuer_ttl_seconds": 60}), + Ok(json!({"model": "m", "anthropic_federation_rule_id": "fdrl_1", "anthropic_issuer_ttl_seconds": 60})), + )] + #[case::unnamed_fields_are_not( + json!({"model": "m", "anthropic_unrelated": "x"}), + Ok(json!({"model": "m"})), + )] + fn workload_identity_fields_are_projected_by_the_names_the_host_lists( + #[case] kwargs: Value, + #[case] expected: Result, + ) { + assert_projection_of( + &[ + "anthropic_federation_rule_id", + "anthropic_issuer_ttl_seconds", + ], + kwargs, + json!({}), + expected, + ); + } + fn assert_projection(kwargs: Value, globals: Value, expected: Result) { + assert_projection_of(&[], kwargs, globals, expected); + } + + fn assert_projection_of( + names: &[&str], + kwargs: Value, + globals: Value, + expected: Result, + ) { Python::initialize(); Python::attach(|py| { let dict = |value: &Value| { @@ -430,6 +1057,7 @@ mod tests { let globals = dict(&globals); let bound = PyDict::new(py); let projected = project_litellm_params( + names.iter().copied(), |name| present(&kwargs, &bound, name), |name| present(&globals, &bound, name), ) diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index a9ae01bd7ee..9a1de064051 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -1,7 +1,15 @@ +mod agentic; +mod answer; mod host; +mod modify_params; +mod pre_request; +mod safeguards; +mod transport; +pub(crate) mod workload_identity; use host::MessagesPythonHost; use litellm_callbacks_legacy_python::LoggingOperation; +use modify_params::ModifyParams; use pyo3::prelude::*; use super::NativeCall; @@ -9,23 +17,61 @@ use super::NativeCall; fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult> { let (arguments, hooks) = crate::routes::call_hooks(py, LoggingOperation::Messages, &call, asynchronous)?; - crate::routes::run_public_call( + let hooked_request = agentic::HookedRequest::default(); + let hooks = litellm_host_python::HookChain::new() + .with(agentic::AgenticLoop::new( + asynchronous, + std::sync::Arc::clone(&hooked_request), + )) + .with(hooks); + let modify_params = ModifyParams::new(&call.args, &call.kwargs)?; + let classifies = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let classified = std::sync::Arc::clone(&classifies); + crate::routes::run_public_call_after_policy( py, arguments, move |py, arguments, request| { + let hooked = pre_request::runs_pre_request_hooks(py)?; let route = litellm_inference_messages::MessagesRoute::new( - crate::http::provider_client(py, arguments, asynchronous)? + crate::http::messages_client(py, arguments, asynchronous)? .map_err(crate::http::client_error)?, crate::http::resources().auth.clone(), crate::secrets::source(py)?, - ); + ) + .with_workload_identity(std::sync::Arc::new( + workload_identity::PythonWorkloadIdentity::new( + litellm_host_python::PythonContext::capture(py)?, + ), + )); Ok(litellm_host::call::hosted_call( request, None, - move |(call, selection): (_, crate::cache::Selection), + move |(call, selection, hosted): (_, crate::cache::Selection, bool), services, interceptors, observers| async move { + let route = if hosted { + route.with_transport(std::sync::Arc::new(transport::PythonTransport::new( + services.clone(), + ))) + } else { + route + }; + let route = match hooked { + true => route.with_pre_request_hooks(std::sync::Arc::new( + pre_request::HostPreRequestHooks::new(services.clone()), + )), + false => route, + }; + let route = match classified.load(std::sync::atomic::Ordering::Acquire) { + true => route.with_safeguards(std::sync::Arc::new( + safeguards::PythonSafeguards::new(services.clone()), + )), + false => route, + }; + let route = route.with_host_answer(std::sync::Arc::new( + answer::PythonAnswer::new(services.clone()), + )); let (cache, options) = selection.attach(services); let route = match cache { Some(cache) => route.with_cache(cache), @@ -44,24 +90,32 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR }, )) }, - MessagesPythonHost::new(call.view()?, asynchronous), + MessagesPythonHost::new(call.view()?, asynchronous, hooked_request, classifies), hooks, + modify_params, asynchronous, ) } -/// Python's sync Messages handler raises for every native provider before it sends anything -/// or opens a logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies -/// that until both sides serve sync calls. +/// Python's sync Messages handler answers a mock without logging, hands proxy MCP tools to the +/// gateway, and raises for every other native provider call before it sends anything or opens a +/// logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies that until +/// both sides serve sync calls. const SYNC_NOT_IMPLEMENTED: &str = "anthropic_messages_handler is not implemented for sync calls"; #[pyfunction] #[pyo3(signature = (call))] -pub(crate) fn messages(call: NativeCall<'_>) -> PyResult> { - drop(call); - Err(pyo3::exceptions::PyValueError::new_err( - SYNC_NOT_IMPLEMENTED, - )) +pub(crate) fn messages(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { + let answer = py + .import(host::ROUTE_HOST_MODULE)? + .getattr("sync_answer")? + .call1((call.resolved()?,))?; + if answer.is_none() { + return Err(pyo3::exceptions::PyValueError::new_err( + SYNC_NOT_IMPLEMENTED, + )); + } + Ok(answer.unbind()) } #[pyfunction] diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/modify_params.rs b/litellm-rust/crates/python-bridge/src/routes/messages/modify_params.rs new file mode 100644 index 00000000000..c6f846478ad --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/modify_params.rs @@ -0,0 +1,152 @@ +use litellm_host::hooks::CallHooks; +use litellm_host_python::{PythonOwned, PythonRuntime}; +use pyo3::{ + exceptions::PyException, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::{PyDict, PyTuple}, +}; + +use super::host::ROUTE_HOST_MODULE; + +pub(super) struct ModifyParams { + call: Option<(Py, Py)>, +} + +impl ModifyParams { + pub(super) fn new(args: &Bound<'_, PyTuple>, kwargs: &Bound<'_, PyDict>) -> PyResult { + let model = match args.get_item(0) { + Ok(model) => model, + Err(_) => kwargs + .get_item("model")? + .unwrap_or_else(|| args.py().None().into_bound(args.py())), + }; + Ok(Self { + call: Some((model.unbind(), args.clone().unbind())), + }) + } +} + +impl CallHooks for ModifyParams { + fn arguments_prepared(&mut self, py: Python<'_>, arguments: &Py) -> PyResult<()> { + let Some((model, args)) = &self.call else { + return Ok(()); + }; + let clamped = py + .import(ROUTE_HOST_MODULE) + .and_then(|module| module.getattr("modify_max_tokens")) + .and_then(|clamp| clamp.call1((model, args, arguments))); + match clamped { + Err(error) if !error.is_instance_of::(py) => Err(error), + _ => Ok(()), + } + } +} + +impl PythonOwned for ModifyParams { + fn close(&mut self, _: Python<'_>) { + self.call = None; + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + if let Some((model, args)) = &self.call { + visit.call(model)?; + visit.call(args)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use pyo3::{ + prelude::*, + types::{PyDict, PyTuple}, + }; + use rstest::rstest; + + use super::{ModifyParams, ROUTE_HOST_MODULE}; + use litellm_host::hooks::CallHooks; + + #[rstest] + #[case::keyword_model( + "()", + "{'model': 'anthropic/claude-sonnet-4-5'}", + "'anthropic/claude-sonnet-4-5'" + )] + #[case::first_positional_wins("(64, [])", "{'model': 'anthropic/claude-sonnet-4-5'}", "64")] + #[case::no_model("()", "{}", "None")] + fn reads_the_model_the_client_wrapper_reads( + #[case] args: &str, + #[case] kwargs: &str, + #[case] expected: &str, + ) { + Python::initialize(); + Python::attach(|py| { + let eval = |source: &str| { + py.eval(&std::ffi::CString::new(source).unwrap(), None, None) + .unwrap() + }; + let args = eval(args).cast_into::().unwrap(); + let kwargs = eval(kwargs).cast_into::().unwrap(); + let hook = ModifyParams::new(&args, &kwargs).unwrap(); + let (model, captured) = hook.call.as_ref().unwrap(); + assert!(model.bind(py).eq(eval(expected)).unwrap()); + assert!(captured.bind(py).is(&args)); + }); + } + + #[rstest] + #[case::exception_leaves_max_tokens("ValueError", true)] + #[case::base_exception_propagates("KeyboardInterrupt", false)] + fn a_failing_clamp_behaves_like_the_client_wrappers_swallowed_try( + #[case] raised: &str, + #[case] swallowed: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("name", ROUTE_HOST_MODULE).unwrap(); + locals.set_item("raised", raised).unwrap(); + py.run( + c" +import builtins, sys, types +for parent in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host', 'litellm.rust_bridge.host.routes'): + sys.modules.setdefault(parent, types.ModuleType(parent)) +saved = sys.modules.get(name) +stub = types.ModuleType(name) +def modify_max_tokens(model, args, kwargs): + kwargs['max_tokens'] = 1 + raise getattr(builtins, raised)() +stub.modify_max_tokens = modify_max_tokens +sys.modules[name] = stub +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let args = PyTuple::empty(py); + let kwargs = PyDict::new(py); + kwargs + .set_item("model", "anthropic/claude-sonnet-4-5") + .unwrap(); + let mut hook = ModifyParams::new(&args, &kwargs).unwrap(); + let result = hook.arguments_prepared(py, &kwargs.clone().unbind()); + py.run( + c" +if saved is None: + sys.modules.pop(name, None) +else: + sys.modules[name] = saved +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + assert_eq!(result.is_ok(), swallowed); + if let Err(error) = result { + assert!(error.is_instance_of::(py)); + } + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/pre_request.rs b/litellm-rust/crates/python-bridge/src/routes/messages/pre_request.rs new file mode 100644 index 00000000000..0553fc0fa7f --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/pre_request.rs @@ -0,0 +1,35 @@ +use futures_util::future::BoxFuture; +use litellm_host::machine::HostServices; +use litellm_inference_messages::{Error, MessagesCall, PreRequestHooks}; +use pyo3::prelude::*; + +use super::host::{MessagesHostCall, MessagesHosted, ROUTE_HOST_MODULE}; + +pub(super) fn runs_pre_request_hooks(py: Python<'_>) -> PyResult { + py.import(ROUTE_HOST_MODULE)? + .getattr("runs_pre_request_hooks")? + .call0()? + .extract() +} + +pub(super) struct HostPreRequestHooks { + services: HostServices, +} + +impl HostPreRequestHooks { + pub(super) fn new(services: HostServices) -> Self { + Self { services } + } +} + +impl PreRequestHooks for HostPreRequestHooks { + fn run(&self, call: MessagesCall) -> BoxFuture<'_, Result> { + drop(call); + Box::pin(async move { + self.services + .call(MessagesHostCall::PreRequest) + .await + .and_then(|hooked| hooked) + }) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/safeguards.rs b/litellm-rust/crates/python-bridge/src/routes/messages/safeguards.rs new file mode 100644 index 00000000000..af478521cb3 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/safeguards.rs @@ -0,0 +1,183 @@ +use bytes::Bytes; +use futures_util::future::BoxFuture; +use litellm_host::{machine::HostServices, protocol::Reply}; +use litellm_host_python::{from_py, to_py}; +use litellm_inference_messages::{Error, MessagesBody, Safeguards}; +use pyo3::{ + exceptions::PyRuntimeError, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::{PyBytes, PyDict}, +}; +use serde_json::Value; + +use super::host::{MessagesHostCall, MessagesHosted, ROUTE_HOST_MODULE}; + +pub(super) enum SafeguardsCall { + Results { + response: MessagesBody, + reply: Reply>, + }, + OpenStream { + reply: Reply, + }, + Frames { + chunk: Bytes, + reply: Reply>, + }, + Flush { + reply: Reply>, + }, +} + +pub(super) struct PythonSafeguards { + services: HostServices, +} + +impl PythonSafeguards { + pub(super) fn new(services: HostServices) -> Self { + Self { services } + } + + async fn ask( + &self, + call: impl FnOnce(Reply) -> SafeguardsCall + Send, + ) -> Result { + self.services + .call(|reply| MessagesHostCall::Safeguards(call(reply))) + .await + } +} + +impl Safeguards for PythonSafeguards { + fn results<'a>( + &'a self, + response: &'a MessagesBody, + ) -> BoxFuture<'a, Result, Error>> { + let response = response.clone(); + Box::pin(self.ask(|reply| SafeguardsCall::Results { response, reply })) + } + + fn open_stream(&self) -> BoxFuture<'_, Result> { + Box::pin(self.ask(|reply| SafeguardsCall::OpenStream { reply })) + } + + fn frames(&self, chunk: Bytes) -> BoxFuture<'_, Result, Error>> { + Box::pin(self.ask(|reply| SafeguardsCall::Frames { chunk, reply })) + } + + fn flush(&self) -> BoxFuture<'_, Result, Error>> { + Box::pin(self.ask(|reply| SafeguardsCall::Flush { reply })) + } +} + +enum Pending { + Results(Reply>), + OpenStream(Reply), + Frames(Reply>), +} + +#[derive(Default)] +pub(super) struct SafeguardsHost { + classifier: Option>, + pending: Option, +} + +impl SafeguardsHost { + pub(super) fn configure( + &mut self, + py: Python<'_>, + request: &Bound<'_, PyDict>, + ) -> PyResult { + let classifier = py + .import(ROUTE_HOST_MODULE)? + .getattr("safeguards")? + .call1((request,))?; + self.classifier = (!classifier.is_none()).then(|| classifier.unbind()); + Ok(self.classifier.is_some()) + } + + pub(super) fn rebind(&mut self, py: Python<'_>, request: &Bound<'_, PyDict>) -> PyResult<()> { + if self.classifier.is_none() { + return Ok(()); + } + let classifier = py + .import(ROUTE_HOST_MODULE)? + .getattr("safeguards")? + .call1((request,))?; + if !classifier.is_none() { + self.classifier = Some(classifier.unbind()); + } + Ok(()) + } + + pub(super) fn begin(&mut self, py: Python<'_>, call: SafeguardsCall) -> PyResult> { + let classifier = self + .classifier + .as_ref() + .ok_or_else(|| PyRuntimeError::new_err("safeguards call without a classifier"))? + .bind(py); + let (awaitable, pending) = match call { + SafeguardsCall::Results { response, reply } => ( + classifier.call_method1("results", (to_py(py, &response)?,))?, + Pending::Results(reply), + ), + SafeguardsCall::OpenStream { reply } => ( + classifier.call_method0("open_stream")?, + Pending::OpenStream(reply), + ), + SafeguardsCall::Frames { chunk, reply } => ( + classifier.call_method1("frames", (PyBytes::new(py, &chunk),))?, + Pending::Frames(reply), + ), + SafeguardsCall::Flush { reply } => { + (classifier.call_method0("flush")?, Pending::Frames(reply)) + } + }; + self.pending = Some(pending); + Ok(awaitable.unbind()) + } + + pub(super) fn resume(&mut self, py: Python<'_>, result: PyResult>) -> PyResult<()> { + let pending = self + .pending + .take() + .ok_or_else(|| PyRuntimeError::new_err("safeguards reply without a pending call"))?; + let value = result?.into_bound(py); + match pending { + Pending::Results(reply) => reply.send(if value.is_none() { + None + } else { + Some(from_py(&value)?) + }), + Pending::OpenStream(reply) => reply.send(value.extract()?), + Pending::Frames(reply) => reply.send( + value + .extract::>>()? + .iter() + .map(|frame| Bytes::copy_from_slice(frame.as_bytes())) + .collect(), + ), + } + Ok(()) + } + + pub(super) fn published(&self, py: Python<'_>, public: Py) -> PyResult> { + match &self.classifier { + Some(classifier) => classifier + .bind(py) + .call_method1("published", (public,)) + .map(Bound::unbind), + None => Ok(public), + } + } + + pub(super) fn close(&mut self) { + self.classifier = None; + self.pending = None; + } + + pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.classifier) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/transport.rs b/litellm-rust/crates/python-bridge/src/routes/messages/transport.rs new file mode 100644 index 00000000000..12e782f61aa --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/transport.rs @@ -0,0 +1,275 @@ +use std::fmt; + +use bytes::Bytes; +use futures_util::{StreamExt, future::BoxFuture}; +use litellm_host::{ + machine::HostServices, + protocol::{Protocol, Reply}, +}; +use litellm_host_python::{json_to_py, to_py}; +use litellm_inference_messages::{ + Delivered, Error, Transport, TransportBody, TransportRequest, TransportResponse, +}; +use pyo3::{ + exceptions::{PyBaseException, PyException, PyStopAsyncIteration}, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::{PyBytes, PyDict}, +}; + +use super::host::ROUTE_HOST_MODULE; + +pub(crate) enum TransportCall { + Send { + request: TransportRequest, + reply: Reply>, + }, + Read { + reply: Reply, Error>>, + }, +} + +pub(crate) struct Sent { + status: u16, + headers: Vec<(String, String)>, + body: SentBody, +} + +enum SentBody { + Text(String), + Stream, + Rejected { text: String, error: Error }, +} + +pub(super) struct PythonTransport { + services: HostServices